jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

refactor: complete JSON format standardization updates

Browse Source 
- Updated all prompt_calls.py files (saas, bcbs_promise, clover) to use new parser pattern
- Updated prompt_templates.py with JSON format instructions
- Updated crosswalk_utils.py to handle JSON lists
- Updated json_utils.py with improved parsing
- Updated aarete_derived.py for JSON compatibility
- Updated tin_npi_funcs.py and qa_qc_utils.py for JSON parsing
- All changes align with Phase 4 completion of JSON standardization
This commit is contained in:
ppanchigar
2026-02-03 17:29:52 -06:00
parent 3c89aecfaf
commit 90b1ea283f
8 changed files with 72 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/pipelines/clients/bcbs_promise/prompts/prompt_calls.py
+15 -6
View File
@@ -129,18 +129,22 @@ def prompt_reimbursement_primary(
service_term = answer_dict["SERVICE_TERM"]
if isinstance(service_term, list):
# If list, join with comma (shouldn't happen, but defensive)
answer_dict["SERVICE_TERM"] = ", ".join(str(item) for item in service_term if item)
answer_dict["SERVICE_TERM"] = ", ".join(
str(item) for item in service_term if item
)
elif service_term is not None:
answer_dict["SERVICE_TERM"] = str(service_term)
else:
answer_dict["SERVICE_TERM"] = ""
# REIMB_TERM should be a string per row
if "REIMB_TERM" in answer_dict:
reimb_term = answer_dict["REIMB_TERM"]
if isinstance(reimb_term, list):
# If list, join with comma (shouldn't happen, but defensive)
answer_dict["REIMB_TERM"] = ", ".join(str(item) for item in reimb_term if item)
answer_dict["REIMB_TERM"] = ", ".join(
str(item) for item in reimb_term if item
)
elif reimb_term is not None:
answer_dict["REIMB_TERM"] = str(reimb_term)
else:
@@ -181,11 +185,16 @@ def prompt_methodology_breakout(
# FEE_SCHEDULE should be a string per methodology breakout result
if isinstance(methodology_breakout_answers, list):
for methodology_dict in methodology_breakout_answers:
if isinstance(methodology_dict, dict) and "FEE_SCHEDULE" in methodology_dict:
if (
isinstance(methodology_dict, dict)
and "FEE_SCHEDULE" in methodology_dict
):
fee_schedule = methodology_dict["FEE_SCHEDULE"]
if isinstance(fee_schedule, list):
# If list, take first element (shouldn't happen, but defensive)
methodology_dict["FEE_SCHEDULE"] = fee_schedule[0] if fee_schedule else ""
methodology_dict["FEE_SCHEDULE"] = (
fee_schedule[0] if fee_schedule else ""
)
elif fee_schedule is not None:
methodology_dict["FEE_SCHEDULE"] = str(fee_schedule)
else:
@@ -211,7 +220,7 @@ def prompt_fee_schedule_breakout(
# If list, take first element (shouldn't happen, but defensive)
fee_schedule = fee_schedule[0] if fee_schedule else ""
fee_schedule = str(fee_schedule) if fee_schedule else ""
prompt, _parser = prompt_templates.FEE_SCHEDULE_BREAKOUT(
reimbursement_method,
fee_schedule,
src/pipelines/clients/clover/prompts/prompt_calls.py
+15 -6
View File
@@ -136,18 +136,22 @@ def prompt_reimbursement_primary(
service_term = answer_dict["SERVICE_TERM"]
if isinstance(service_term, list):
# If list, join with comma (shouldn't happen, but defensive)
answer_dict["SERVICE_TERM"] = ", ".join(str(item) for item in service_term if item)
answer_dict["SERVICE_TERM"] = ", ".join(
str(item) for item in service_term if item
)
elif service_term is not None:
answer_dict["SERVICE_TERM"] = str(service_term)
else:
answer_dict["SERVICE_TERM"] = ""
# REIMB_TERM should be a string per row
if "REIMB_TERM" in answer_dict:
reimb_term = answer_dict["REIMB_TERM"]
if isinstance(reimb_term, list):
# If list, join with comma (shouldn't happen, but defensive)
answer_dict["REIMB_TERM"] = ", ".join(str(item) for item in reimb_term if item)
answer_dict["REIMB_TERM"] = ", ".join(
str(item) for item in reimb_term if item
)
elif reimb_term is not None:
answer_dict["REIMB_TERM"] = str(reimb_term)
else:
@@ -188,11 +192,16 @@ def prompt_methodology_breakout(
# FEE_SCHEDULE should be a string per methodology breakout result
if isinstance(methodology_breakout_answers, list):
for methodology_dict in methodology_breakout_answers:
if isinstance(methodology_dict, dict) and "FEE_SCHEDULE" in methodology_dict:
if (
isinstance(methodology_dict, dict)
and "FEE_SCHEDULE" in methodology_dict
):
fee_schedule = methodology_dict["FEE_SCHEDULE"]
if isinstance(fee_schedule, list):
# If list, take first element (shouldn't happen, but defensive)
methodology_dict["FEE_SCHEDULE"] = fee_schedule[0] if fee_schedule else ""
methodology_dict["FEE_SCHEDULE"] = (
fee_schedule[0] if fee_schedule else ""
)
elif fee_schedule is not None:
methodology_dict["FEE_SCHEDULE"] = str(fee_schedule)
else:
@@ -218,7 +227,7 @@ def prompt_fee_schedule_breakout(
# If list, take first element (shouldn't happen, but defensive)
fee_schedule = fee_schedule[0] if fee_schedule else ""
fee_schedule = str(fee_schedule) if fee_schedule else ""
prompt, _parser = prompt_templates.FEE_SCHEDULE_BREAKOUT(
reimbursement_method,
fee_schedule,
src/pipelines/shared/extraction/tin_npi_funcs.py
+5 -3
View File
@@ -294,7 +294,9 @@ def merge_provider_info_with_hybrid_smart_chunking(one_to_one_results, filename)
# Filter out records with NO_IDENTIFIERS_FOUND
prov_info_json_list = [
p for p in prov_info_json_list if "NO_IDENTIFIERS_FOUND" not in (p.get("NAME") or [])
p
for p in prov_info_json_list
if "NO_IDENTIFIERS_FOUND" not in (p.get("NAME") or [])
]
# Deduplicate provider_list by NAME, keeping records with actual TIN/NPI values
@@ -313,7 +315,7 @@ def merge_provider_info_with_hybrid_smart_chunking(one_to_one_results, filename)
# Merge other provider information into one_to_one_results
one_to_one_results["PROV_OTHER_TIN"] = other_tins if other_tins else []
one_to_one_results["PROV_OTHER_NPI"] = other_npis if other_npis else []
one_to_one_results["PROV_OTHER_NAME_FULL"] = other_names if other_names else []
one_to_one_results["PROV_OTHER_NAME_FULL"] = other_names if other_names else []
return one_to_one_results
@@ -426,7 +428,7 @@ def deduplicate_providers(
tuple(npis),
tuple(names),
)
if key not in seen:
seen.add(key)
valid_providers.append(provider)
src/pipelines/shared/postprocessing/aarete_derived.py
+27 -7
View File
@@ -50,7 +50,11 @@ def fill_na_mapping(answer_dicts):
# Process each LOB value
for lob_val in lob_items:
if isinstance(lob_val, str) and lob_val.strip() and lob_val.strip() != "N/A":
if (
isinstance(lob_val, str)
and lob_val.strip()
and lob_val.strip() != "N/A"
):
all_lob_values.add(lob_val.strip())
# Get AARETE_DERIVED_LOB from AARETE_DERIVED_PROGRAM crosswalk (always check if PROGRAM exists)
@@ -72,7 +76,9 @@ def fill_na_mapping(answer_dicts):
elif isinstance(program_filled_value_list, str):
# Handle pipe-delimited format (backward compatibility)
if "|" in program_filled_value_list:
program_lob_items = [v.strip() for v in program_filled_value_list.split("|")]
program_lob_items = [
v.strip() for v in program_filled_value_list.split("|")
]
else:
program_lob_items = [program_filled_value_list]
else:
@@ -80,7 +86,11 @@ def fill_na_mapping(answer_dicts):
# Process each program LOB value
for program_lob_val in program_lob_items:
if isinstance(program_lob_val, str) and program_lob_val.strip() and program_lob_val.strip() != "N/A":
if (
isinstance(program_lob_val, str)
and program_lob_val.strip()
and program_lob_val.strip() != "N/A"
):
program_lob_values.add(program_lob_val.strip())
# Update all_lob_values after processing all program LOB values
all_lob_values.update(program_lob_values)
@@ -104,7 +114,9 @@ def fill_na_mapping(answer_dicts):
elif isinstance(product_filled_value_list, str):
# Handle pipe-delimited format (backward compatibility)
if "|" in product_filled_value_list:
product_lob_items = [v.strip() for v in product_filled_value_list.split("|")]
product_lob_items = [
v.strip() for v in product_filled_value_list.split("|")
]
else:
product_lob_items = [product_filled_value_list]
else:
@@ -112,7 +124,11 @@ def fill_na_mapping(answer_dicts):
# Process each product LOB value
for product_lob_val in product_lob_items:
if isinstance(product_lob_val, str) and product_lob_val.strip() and product_lob_val.strip() != "N/A":
if (
isinstance(product_lob_val, str)
and product_lob_val.strip()
and product_lob_val.strip() != "N/A"
):
product_lob_values.add(product_lob_val.strip())
# Update all_lob_values after processing all product LOB values
all_lob_values.update(product_lob_values)
@@ -192,13 +208,17 @@ def get_crosswalk_fields(answer_dicts: list, constants: Constants):
# Reverse mapping returns dict[str, list[str]], so .get() returns a list
# We need to extend, not append, to avoid nested lists
reverse_mapping = crosswalk.create_reverse_mapping()
reverse_result = reverse_mapping.get(individual_from_field_value)
reverse_result = reverse_mapping.get(
individual_from_field_value
)
if reverse_result is not None:
# reverse_result is a list, extend it
to_field_answer_list.extend(reverse_result)
else:
# If not found in reverse mapping, keep original value
to_field_answer_list.append(individual_from_field_value)
to_field_answer_list.append(
individual_from_field_value
)
answer_dict[to_field_name] = to_field_answer_list
# update from field value to list if not already
if not isinstance(from_field_value, list):
src/prompts/prompt_templates.py
+1 -1
View File
@@ -890,7 +890,7 @@ Populate a JSON dictionary with the following fields:
def FEE_SCHEDULE_BREAKOUT(
methodology:str, fee_schedule:str
methodology: str, fee_schedule: str
) -> Tuple[str, Callable[[str], dict]]:
"""Returns ONLY dynamic content for fee schedule breakout.
Call FEE_SCHEDULE_BREAKOUT_INSTRUCTION() separately for the cached instruction.
src/utils/crosswalk_utils.py
+3 -1
View File
@@ -4,7 +4,9 @@ import logging
import src.utils.string_utils as string_utils
def apply_crosswalk(val: str | list, mapping: dict[str, str], default: str = "N/A") -> list:
def apply_crosswalk(
val: str | list, mapping: dict[str, str], default: str = "N/A"
) -> list:
"""Apply a crosswalk to a value
Args:
src/utils/json_utils.py
+1 -1
View File
@@ -253,4 +253,4 @@ def parse_json_dict_or_list(raw_llm_output: str) -> dict[str, Any] | list[Any]:
raise ValueError(
f"No valid JSON object found in LLM output. "
f"Output preview: {raw_llm_output[:200]}..."
)
)
src/utils/qa_qc_utils.py
+5 -3
View File
@@ -1101,7 +1101,9 @@ def format_or_preserve_tins(val: Any) -> Tuple[str, bool]:
Tuple of (formatted_tins, has_invalid)
"""
# Handle list/array values (e.g., from PROV_GROUP_TIN, PROV_OTHER_TIN)
if isinstance(val, (list, tuple)) or (hasattr(val, '__iter__') and not isinstance(val, str)):
if isinstance(val, (list, tuple)) or (
hasattr(val, "__iter__") and not isinstance(val, str)
):
# Convert list to list of strings, filtering out empty values
if len(val) == 0:
return ("", True)
@@ -1121,10 +1123,10 @@ def format_or_preserve_tins(val: Any) -> Tuple[str, bool]:
except (ValueError, TypeError):
# pd.isna() can't handle arrays, but we've already handled lists above
pass
if val is None or str(val).strip() == "":
return ("", True)
# Single value - convert to list for consistent processing
parts = [str(val).strip()]
out, any_invalid = [], False