From 90b1ea283ff38662610481ec09f9fb873db24b72 Mon Sep 17 00:00:00 2001 From: ppanchigar Date: Tue, 3 Feb 2026 17:29:52 -0600 Subject: [PATCH] refactor: complete JSON format standardization updates - Updated all prompt_calls.py files (saas, bcbs_promise, clover) to use new parser pattern - Updated prompt_templates.py with JSON format instructions - Updated crosswalk_utils.py to handle JSON lists - Updated json_utils.py with improved parsing - Updated aarete_derived.py for JSON compatibility - Updated tin_npi_funcs.py and qa_qc_utils.py for JSON parsing - All changes align with Phase 4 completion of JSON standardization --- .../bcbs_promise/prompts/prompt_calls.py | 21 ++++++++---- .../clients/clover/prompts/prompt_calls.py | 21 ++++++++---- .../shared/extraction/tin_npi_funcs.py | 8 +++-- .../shared/postprocessing/aarete_derived.py | 34 +++++++++++++++---- src/prompts/prompt_templates.py | 2 +- src/utils/crosswalk_utils.py | 4 ++- src/utils/json_utils.py | 2 +- src/utils/qa_qc_utils.py | 8 +++-- 8 files changed, 72 insertions(+), 28 deletions(-) diff --git a/src/pipelines/clients/bcbs_promise/prompts/prompt_calls.py b/src/pipelines/clients/bcbs_promise/prompts/prompt_calls.py index b74ec02..1357402 100644 --- a/src/pipelines/clients/bcbs_promise/prompts/prompt_calls.py +++ b/src/pipelines/clients/bcbs_promise/prompts/prompt_calls.py @@ -129,18 +129,22 @@ def prompt_reimbursement_primary( service_term = answer_dict["SERVICE_TERM"] if isinstance(service_term, list): # If list, join with comma (shouldn't happen, but defensive) - answer_dict["SERVICE_TERM"] = ", ".join(str(item) for item in service_term if item) + answer_dict["SERVICE_TERM"] = ", ".join( + str(item) for item in service_term if item + ) elif service_term is not None: answer_dict["SERVICE_TERM"] = str(service_term) else: answer_dict["SERVICE_TERM"] = "" - + # REIMB_TERM should be a string per row if "REIMB_TERM" in answer_dict: reimb_term = answer_dict["REIMB_TERM"] if isinstance(reimb_term, list): # If list, join with comma (shouldn't happen, but defensive) - answer_dict["REIMB_TERM"] = ", ".join(str(item) for item in reimb_term if item) + answer_dict["REIMB_TERM"] = ", ".join( + str(item) for item in reimb_term if item + ) elif reimb_term is not None: answer_dict["REIMB_TERM"] = str(reimb_term) else: @@ -181,11 +185,16 @@ def prompt_methodology_breakout( # FEE_SCHEDULE should be a string per methodology breakout result if isinstance(methodology_breakout_answers, list): for methodology_dict in methodology_breakout_answers: - if isinstance(methodology_dict, dict) and "FEE_SCHEDULE" in methodology_dict: + if ( + isinstance(methodology_dict, dict) + and "FEE_SCHEDULE" in methodology_dict + ): fee_schedule = methodology_dict["FEE_SCHEDULE"] if isinstance(fee_schedule, list): # If list, take first element (shouldn't happen, but defensive) - methodology_dict["FEE_SCHEDULE"] = fee_schedule[0] if fee_schedule else "" + methodology_dict["FEE_SCHEDULE"] = ( + fee_schedule[0] if fee_schedule else "" + ) elif fee_schedule is not None: methodology_dict["FEE_SCHEDULE"] = str(fee_schedule) else: @@ -211,7 +220,7 @@ def prompt_fee_schedule_breakout( # If list, take first element (shouldn't happen, but defensive) fee_schedule = fee_schedule[0] if fee_schedule else "" fee_schedule = str(fee_schedule) if fee_schedule else "" - + prompt, _parser = prompt_templates.FEE_SCHEDULE_BREAKOUT( reimbursement_method, fee_schedule, diff --git a/src/pipelines/clients/clover/prompts/prompt_calls.py b/src/pipelines/clients/clover/prompts/prompt_calls.py index c965ea5..01a0e44 100644 --- a/src/pipelines/clients/clover/prompts/prompt_calls.py +++ b/src/pipelines/clients/clover/prompts/prompt_calls.py @@ -136,18 +136,22 @@ def prompt_reimbursement_primary( service_term = answer_dict["SERVICE_TERM"] if isinstance(service_term, list): # If list, join with comma (shouldn't happen, but defensive) - answer_dict["SERVICE_TERM"] = ", ".join(str(item) for item in service_term if item) + answer_dict["SERVICE_TERM"] = ", ".join( + str(item) for item in service_term if item + ) elif service_term is not None: answer_dict["SERVICE_TERM"] = str(service_term) else: answer_dict["SERVICE_TERM"] = "" - + # REIMB_TERM should be a string per row if "REIMB_TERM" in answer_dict: reimb_term = answer_dict["REIMB_TERM"] if isinstance(reimb_term, list): # If list, join with comma (shouldn't happen, but defensive) - answer_dict["REIMB_TERM"] = ", ".join(str(item) for item in reimb_term if item) + answer_dict["REIMB_TERM"] = ", ".join( + str(item) for item in reimb_term if item + ) elif reimb_term is not None: answer_dict["REIMB_TERM"] = str(reimb_term) else: @@ -188,11 +192,16 @@ def prompt_methodology_breakout( # FEE_SCHEDULE should be a string per methodology breakout result if isinstance(methodology_breakout_answers, list): for methodology_dict in methodology_breakout_answers: - if isinstance(methodology_dict, dict) and "FEE_SCHEDULE" in methodology_dict: + if ( + isinstance(methodology_dict, dict) + and "FEE_SCHEDULE" in methodology_dict + ): fee_schedule = methodology_dict["FEE_SCHEDULE"] if isinstance(fee_schedule, list): # If list, take first element (shouldn't happen, but defensive) - methodology_dict["FEE_SCHEDULE"] = fee_schedule[0] if fee_schedule else "" + methodology_dict["FEE_SCHEDULE"] = ( + fee_schedule[0] if fee_schedule else "" + ) elif fee_schedule is not None: methodology_dict["FEE_SCHEDULE"] = str(fee_schedule) else: @@ -218,7 +227,7 @@ def prompt_fee_schedule_breakout( # If list, take first element (shouldn't happen, but defensive) fee_schedule = fee_schedule[0] if fee_schedule else "" fee_schedule = str(fee_schedule) if fee_schedule else "" - + prompt, _parser = prompt_templates.FEE_SCHEDULE_BREAKOUT( reimbursement_method, fee_schedule, diff --git a/src/pipelines/shared/extraction/tin_npi_funcs.py b/src/pipelines/shared/extraction/tin_npi_funcs.py index 767e0ce..132720e 100644 --- a/src/pipelines/shared/extraction/tin_npi_funcs.py +++ b/src/pipelines/shared/extraction/tin_npi_funcs.py @@ -294,7 +294,9 @@ def merge_provider_info_with_hybrid_smart_chunking(one_to_one_results, filename) # Filter out records with NO_IDENTIFIERS_FOUND prov_info_json_list = [ - p for p in prov_info_json_list if "NO_IDENTIFIERS_FOUND" not in (p.get("NAME") or []) + p + for p in prov_info_json_list + if "NO_IDENTIFIERS_FOUND" not in (p.get("NAME") or []) ] # Deduplicate provider_list by NAME, keeping records with actual TIN/NPI values @@ -313,7 +315,7 @@ def merge_provider_info_with_hybrid_smart_chunking(one_to_one_results, filename) # Merge other provider information into one_to_one_results one_to_one_results["PROV_OTHER_TIN"] = other_tins if other_tins else [] one_to_one_results["PROV_OTHER_NPI"] = other_npis if other_npis else [] - one_to_one_results["PROV_OTHER_NAME_FULL"] = other_names if other_names else [] + one_to_one_results["PROV_OTHER_NAME_FULL"] = other_names if other_names else [] return one_to_one_results @@ -426,7 +428,7 @@ def deduplicate_providers( tuple(npis), tuple(names), ) - + if key not in seen: seen.add(key) valid_providers.append(provider) diff --git a/src/pipelines/shared/postprocessing/aarete_derived.py b/src/pipelines/shared/postprocessing/aarete_derived.py index e2f326d..a4daea3 100644 --- a/src/pipelines/shared/postprocessing/aarete_derived.py +++ b/src/pipelines/shared/postprocessing/aarete_derived.py @@ -50,7 +50,11 @@ def fill_na_mapping(answer_dicts): # Process each LOB value for lob_val in lob_items: - if isinstance(lob_val, str) and lob_val.strip() and lob_val.strip() != "N/A": + if ( + isinstance(lob_val, str) + and lob_val.strip() + and lob_val.strip() != "N/A" + ): all_lob_values.add(lob_val.strip()) # Get AARETE_DERIVED_LOB from AARETE_DERIVED_PROGRAM crosswalk (always check if PROGRAM exists) @@ -72,7 +76,9 @@ def fill_na_mapping(answer_dicts): elif isinstance(program_filled_value_list, str): # Handle pipe-delimited format (backward compatibility) if "|" in program_filled_value_list: - program_lob_items = [v.strip() for v in program_filled_value_list.split("|")] + program_lob_items = [ + v.strip() for v in program_filled_value_list.split("|") + ] else: program_lob_items = [program_filled_value_list] else: @@ -80,7 +86,11 @@ def fill_na_mapping(answer_dicts): # Process each program LOB value for program_lob_val in program_lob_items: - if isinstance(program_lob_val, str) and program_lob_val.strip() and program_lob_val.strip() != "N/A": + if ( + isinstance(program_lob_val, str) + and program_lob_val.strip() + and program_lob_val.strip() != "N/A" + ): program_lob_values.add(program_lob_val.strip()) # Update all_lob_values after processing all program LOB values all_lob_values.update(program_lob_values) @@ -104,7 +114,9 @@ def fill_na_mapping(answer_dicts): elif isinstance(product_filled_value_list, str): # Handle pipe-delimited format (backward compatibility) if "|" in product_filled_value_list: - product_lob_items = [v.strip() for v in product_filled_value_list.split("|")] + product_lob_items = [ + v.strip() for v in product_filled_value_list.split("|") + ] else: product_lob_items = [product_filled_value_list] else: @@ -112,7 +124,11 @@ def fill_na_mapping(answer_dicts): # Process each product LOB value for product_lob_val in product_lob_items: - if isinstance(product_lob_val, str) and product_lob_val.strip() and product_lob_val.strip() != "N/A": + if ( + isinstance(product_lob_val, str) + and product_lob_val.strip() + and product_lob_val.strip() != "N/A" + ): product_lob_values.add(product_lob_val.strip()) # Update all_lob_values after processing all product LOB values all_lob_values.update(product_lob_values) @@ -192,13 +208,17 @@ def get_crosswalk_fields(answer_dicts: list, constants: Constants): # Reverse mapping returns dict[str, list[str]], so .get() returns a list # We need to extend, not append, to avoid nested lists reverse_mapping = crosswalk.create_reverse_mapping() - reverse_result = reverse_mapping.get(individual_from_field_value) + reverse_result = reverse_mapping.get( + individual_from_field_value + ) if reverse_result is not None: # reverse_result is a list, extend it to_field_answer_list.extend(reverse_result) else: # If not found in reverse mapping, keep original value - to_field_answer_list.append(individual_from_field_value) + to_field_answer_list.append( + individual_from_field_value + ) answer_dict[to_field_name] = to_field_answer_list # update from field value to list if not already if not isinstance(from_field_value, list): diff --git a/src/prompts/prompt_templates.py b/src/prompts/prompt_templates.py index c08c186..595a2e3 100644 --- a/src/prompts/prompt_templates.py +++ b/src/prompts/prompt_templates.py @@ -890,7 +890,7 @@ Populate a JSON dictionary with the following fields: def FEE_SCHEDULE_BREAKOUT( - methodology:str, fee_schedule:str + methodology: str, fee_schedule: str ) -> Tuple[str, Callable[[str], dict]]: """Returns ONLY dynamic content for fee schedule breakout. Call FEE_SCHEDULE_BREAKOUT_INSTRUCTION() separately for the cached instruction. diff --git a/src/utils/crosswalk_utils.py b/src/utils/crosswalk_utils.py index ea7a8f7..a4e8d97 100644 --- a/src/utils/crosswalk_utils.py +++ b/src/utils/crosswalk_utils.py @@ -4,7 +4,9 @@ import logging import src.utils.string_utils as string_utils -def apply_crosswalk(val: str | list, mapping: dict[str, str], default: str = "N/A") -> list: +def apply_crosswalk( + val: str | list, mapping: dict[str, str], default: str = "N/A" +) -> list: """Apply a crosswalk to a value Args: diff --git a/src/utils/json_utils.py b/src/utils/json_utils.py index f70f459..cef4b33 100644 --- a/src/utils/json_utils.py +++ b/src/utils/json_utils.py @@ -253,4 +253,4 @@ def parse_json_dict_or_list(raw_llm_output: str) -> dict[str, Any] | list[Any]: raise ValueError( f"No valid JSON object found in LLM output. " f"Output preview: {raw_llm_output[:200]}..." - ) \ No newline at end of file + ) diff --git a/src/utils/qa_qc_utils.py b/src/utils/qa_qc_utils.py index 69c5bda..a808afc 100644 --- a/src/utils/qa_qc_utils.py +++ b/src/utils/qa_qc_utils.py @@ -1101,7 +1101,9 @@ def format_or_preserve_tins(val: Any) -> Tuple[str, bool]: Tuple of (formatted_tins, has_invalid) """ # Handle list/array values (e.g., from PROV_GROUP_TIN, PROV_OTHER_TIN) - if isinstance(val, (list, tuple)) or (hasattr(val, '__iter__') and not isinstance(val, str)): + if isinstance(val, (list, tuple)) or ( + hasattr(val, "__iter__") and not isinstance(val, str) + ): # Convert list to list of strings, filtering out empty values if len(val) == 0: return ("", True) @@ -1121,10 +1123,10 @@ def format_or_preserve_tins(val: Any) -> Tuple[str, bool]: except (ValueError, TypeError): # pd.isna() can't handle arrays, but we've already handled lists above pass - + if val is None or str(val).strip() == "": return ("", True) - + # Single value - convert to list for consistent processing parts = [str(val).strip()] out, any_invalid = [], False