diff --git a/src/pipelines/clients/bcbs_promise/prompts/prompt_calls.py b/src/pipelines/clients/bcbs_promise/prompts/prompt_calls.py index 4d16b57..b74ec02 100644 --- a/src/pipelines/clients/bcbs_promise/prompts/prompt_calls.py +++ b/src/pipelines/clients/bcbs_promise/prompts/prompt_calls.py @@ -119,6 +119,32 @@ def prompt_reimbursement_primary( try: llm_answer_final = _parser(llm_answer_raw) + # Normalize SERVICE_TERM and REIMB_TERM to strings immediately after parsing + # These should always be strings per row, not lists + if isinstance(llm_answer_final, list): + for answer_dict in llm_answer_final: + if isinstance(answer_dict, dict): + # SERVICE_TERM should be a string per row + if "SERVICE_TERM" in answer_dict: + service_term = answer_dict["SERVICE_TERM"] + if isinstance(service_term, list): + # If list, join with comma (shouldn't happen, but defensive) + answer_dict["SERVICE_TERM"] = ", ".join(str(item) for item in service_term if item) + elif service_term is not None: + answer_dict["SERVICE_TERM"] = str(service_term) + else: + answer_dict["SERVICE_TERM"] = "" + + # REIMB_TERM should be a string per row + if "REIMB_TERM" in answer_dict: + reimb_term = answer_dict["REIMB_TERM"] + if isinstance(reimb_term, list): + # If list, join with comma (shouldn't happen, but defensive) + answer_dict["REIMB_TERM"] = ", ".join(str(item) for item in reimb_term if item) + elif reimb_term is not None: + answer_dict["REIMB_TERM"] = str(reimb_term) + else: + answer_dict["REIMB_TERM"] = "" except ValueError as e: logging.error(f"Error parsing LLM response: {e}") logging.error(f"Raw LLM output: {llm_answer_raw}") @@ -151,6 +177,19 @@ def prompt_methodology_breakout( logging.debug(f"LLM Response for {filename}: {llm_response}") try: methodology_breakout_answers = _parser(llm_response) + # Normalize single-value fields that should be strings (not lists) + # FEE_SCHEDULE should be a string per methodology breakout result + if isinstance(methodology_breakout_answers, list): + for methodology_dict in methodology_breakout_answers: + if isinstance(methodology_dict, dict) and "FEE_SCHEDULE" in methodology_dict: + fee_schedule = methodology_dict["FEE_SCHEDULE"] + if isinstance(fee_schedule, list): + # If list, take first element (shouldn't happen, but defensive) + methodology_dict["FEE_SCHEDULE"] = fee_schedule[0] if fee_schedule else "" + elif fee_schedule is not None: + methodology_dict["FEE_SCHEDULE"] = str(fee_schedule) + else: + methodology_dict["FEE_SCHEDULE"] = "" except ValueError as e: logging.error(f"Error parsing LLM response: {e}") methodology_breakout_answers = [] @@ -166,9 +205,16 @@ def prompt_fee_schedule_breakout( Call FEE_SCHEDULE_BREAKOUT prompt with cached instruction. Note: Field definitions are now included in FEE_SCHEDULE_BREAKOUT_INSTRUCTION() for caching. """ + # Normalize FEE_SCHEDULE to string before using in prompt (should already be normalized from prompt_methodology_breakout) + fee_schedule = methodology_breakout_dict.get("FEE_SCHEDULE", "") + if isinstance(fee_schedule, list): + # If list, take first element (shouldn't happen, but defensive) + fee_schedule = fee_schedule[0] if fee_schedule else "" + fee_schedule = str(fee_schedule) if fee_schedule else "" + prompt, _parser = prompt_templates.FEE_SCHEDULE_BREAKOUT( reimbursement_method, - methodology_breakout_dict.get("FEE_SCHEDULE"), + fee_schedule, ) logging.debug(f"Fee Schedule Breakout Prompt for {filename}: {prompt}") llm_answer_raw = llm_utils.invoke_claude( @@ -299,7 +345,13 @@ def prompt_lob_relationship( instruction=prompt_templates.LOB_RELATIONSHIP_INSTRUCTION(), ) llm_answer_final = _parser(llm_answer_raw) - return llm_answer_final + # LOB_RELATIONSHIP returns a list (e.g., ["Inclusive"]), but we need a string + if isinstance(llm_answer_final, list) and len(llm_answer_final) > 0: + return str(llm_answer_final[0]) + elif isinstance(llm_answer_final, str): + return llm_answer_final + else: + return "Exclusive" # Default fallback def prompt_special_case_assignment( diff --git a/src/pipelines/clients/clover/prompts/prompt_calls.py b/src/pipelines/clients/clover/prompts/prompt_calls.py index badabcd..c965ea5 100644 --- a/src/pipelines/clients/clover/prompts/prompt_calls.py +++ b/src/pipelines/clients/clover/prompts/prompt_calls.py @@ -126,6 +126,32 @@ def prompt_reimbursement_primary( try: llm_answer_final = _parser(llm_answer_raw) + # Normalize SERVICE_TERM and REIMB_TERM to strings immediately after parsing + # These should always be strings per row, not lists + if isinstance(llm_answer_final, list): + for answer_dict in llm_answer_final: + if isinstance(answer_dict, dict): + # SERVICE_TERM should be a string per row + if "SERVICE_TERM" in answer_dict: + service_term = answer_dict["SERVICE_TERM"] + if isinstance(service_term, list): + # If list, join with comma (shouldn't happen, but defensive) + answer_dict["SERVICE_TERM"] = ", ".join(str(item) for item in service_term if item) + elif service_term is not None: + answer_dict["SERVICE_TERM"] = str(service_term) + else: + answer_dict["SERVICE_TERM"] = "" + + # REIMB_TERM should be a string per row + if "REIMB_TERM" in answer_dict: + reimb_term = answer_dict["REIMB_TERM"] + if isinstance(reimb_term, list): + # If list, join with comma (shouldn't happen, but defensive) + answer_dict["REIMB_TERM"] = ", ".join(str(item) for item in reimb_term if item) + elif reimb_term is not None: + answer_dict["REIMB_TERM"] = str(reimb_term) + else: + answer_dict["REIMB_TERM"] = "" except ValueError as e: logging.error(f"Error parsing LLM response: {e}") logging.error(f"Raw LLM output: {llm_answer_raw}") @@ -158,6 +184,19 @@ def prompt_methodology_breakout( logging.debug(f"LLM Response for {filename}: {llm_response}") try: methodology_breakout_answers = _parser(llm_response) + # Normalize single-value fields that should be strings (not lists) + # FEE_SCHEDULE should be a string per methodology breakout result + if isinstance(methodology_breakout_answers, list): + for methodology_dict in methodology_breakout_answers: + if isinstance(methodology_dict, dict) and "FEE_SCHEDULE" in methodology_dict: + fee_schedule = methodology_dict["FEE_SCHEDULE"] + if isinstance(fee_schedule, list): + # If list, take first element (shouldn't happen, but defensive) + methodology_dict["FEE_SCHEDULE"] = fee_schedule[0] if fee_schedule else "" + elif fee_schedule is not None: + methodology_dict["FEE_SCHEDULE"] = str(fee_schedule) + else: + methodology_dict["FEE_SCHEDULE"] = "" except ValueError as e: logging.error(f"Error parsing LLM response: {e}") methodology_breakout_answers = [] @@ -173,9 +212,16 @@ def prompt_fee_schedule_breakout( Call FEE_SCHEDULE_BREAKOUT prompt with cached instruction. Note: Field definitions are now included in FEE_SCHEDULE_BREAKOUT_INSTRUCTION() for caching. """ + # Normalize FEE_SCHEDULE to string before using in prompt (should already be normalized from prompt_methodology_breakout) + fee_schedule = methodology_breakout_dict.get("FEE_SCHEDULE", "") + if isinstance(fee_schedule, list): + # If list, take first element (shouldn't happen, but defensive) + fee_schedule = fee_schedule[0] if fee_schedule else "" + fee_schedule = str(fee_schedule) if fee_schedule else "" + prompt, _parser = prompt_templates.FEE_SCHEDULE_BREAKOUT( reimbursement_method, - methodology_breakout_dict.get("FEE_SCHEDULE"), + fee_schedule, ) logging.debug(f"Fee Schedule Breakout Prompt for {filename}: {prompt}") llm_answer_raw = llm_utils.invoke_claude( @@ -306,7 +352,13 @@ def prompt_lob_relationship( instruction=prompt_templates.LOB_RELATIONSHIP_INSTRUCTION(), ) llm_answer_final = _parser(llm_answer_raw) - return llm_answer_final + # LOB_RELATIONSHIP returns a list (e.g., ["Inclusive"]), but we need a string + if isinstance(llm_answer_final, list) and len(llm_answer_final) > 0: + return str(llm_answer_final[0]) + elif isinstance(llm_answer_final, str): + return llm_answer_final + else: + return "Exclusive" # Default fallback def prompt_special_case_assignment( diff --git a/src/pipelines/saas/prompts/prompt_calls.py b/src/pipelines/saas/prompts/prompt_calls.py index 7a73c47..9b7b9a2 100644 --- a/src/pipelines/saas/prompts/prompt_calls.py +++ b/src/pipelines/saas/prompts/prompt_calls.py @@ -126,6 +126,11 @@ def prompt_reimbursement_primary( try: llm_answer_final = _parser(llm_answer_raw) + + # DEBUG: Print raw parsed output + print(f"\n[DEBUG REIMBURSEMENT_PRIMARY] Parsed output for {filename}:") + print(llm_answer_final) + print() except ValueError as e: logging.error(f"Error parsing LLM response: {e}") logging.error(f"Raw LLM output: {llm_answer_raw}") @@ -173,9 +178,11 @@ def prompt_fee_schedule_breakout( Call FEE_SCHEDULE_BREAKOUT prompt with cached instruction. Note: Field definitions are now included in FEE_SCHEDULE_BREAKOUT_INSTRUCTION() for caching. """ + # Normalize FEE_SCHEDULE to string before using in prompt (should already be normalized from prompt_methodology_breakout) + fee_schedule = methodology_breakout_dict.get("FEE_SCHEDULE", "") prompt, _parser = prompt_templates.FEE_SCHEDULE_BREAKOUT( reimbursement_method, - methodology_breakout_dict.get("FEE_SCHEDULE"), + fee_schedule, ) logging.debug(f"Fee Schedule Breakout Prompt for {filename}: {prompt}") llm_answer_raw = llm_utils.invoke_claude( @@ -306,7 +313,13 @@ def prompt_lob_relationship( instruction=prompt_templates.LOB_RELATIONSHIP_INSTRUCTION(), ) llm_answer_final = _parser(llm_answer_raw) - return llm_answer_final + # LOB_RELATIONSHIP returns a list (e.g., ["Inclusive"]), but we need a string + if isinstance(llm_answer_final, list) and len(llm_answer_final) > 0: + return str(llm_answer_final[0]) + elif isinstance(llm_answer_final, str): + return llm_answer_final + else: + return "Exclusive" # Default fallback def prompt_special_case_assignment( @@ -687,13 +700,17 @@ def prompt_lesser_of_distribution( return reimb_term # Return original term unchanged else: llm_answer_final = _parser(llm_answer_raw) + + # DEBUG: Print raw parsed output + print(f"\n[DEBUG prompt_lesser_of_distribution] Raw parsed output:") + print(llm_answer_final) logging.debug( f"Applied lesser-of to '{service_term}' on page {page_num}: " f"{reimb_term} → {llm_answer_final[:60]}..." ) - return llm_answer_final + return llm_answer_final[0] if isinstance(llm_answer_final, list) else llm_answer_final def prompt_lesser_of_check( diff --git a/src/pipelines/shared/extraction/dynamic_funcs.py b/src/pipelines/shared/extraction/dynamic_funcs.py index 6421934..ad51855 100644 --- a/src/pipelines/shared/extraction/dynamic_funcs.py +++ b/src/pipelines/shared/extraction/dynamic_funcs.py @@ -234,8 +234,20 @@ def add_one_to_one_field( # Only search for these fields when NO LOB has been found if field_to_add.field_name in ["PROGRAM", "PRODUCT", "NETWORK"]: - lob_values = [answer_dict.get("LOB", "") for answer_dict in answer_dicts] - unique_lobs = set([lob for lob in lob_values if not string_utils.is_empty(lob)]) + # LOB can be a string or a list (from JSON format) + # Extract all LOB values, handling both string and list formats + lob_values = [] + for answer_dict in answer_dicts: + lob_value = answer_dict.get("LOB", "") + if isinstance(lob_value, list): + # If it's a list, extract each item and ensure it's a string + for item in lob_value: + if not string_utils.is_empty(item): + lob_values.append(str(item)) + elif not string_utils.is_empty(lob_value): + # If it's a string, add it directly (already hashable) + lob_values.append(lob_value) + unique_lobs = set(lob_values) if len(unique_lobs) > 0: return one_to_one_fields @@ -283,8 +295,22 @@ def get_dynamic_one_to_one_fields( # Only search for PROGRAM, PRODUCT, NETWORK when NO LOB has been found # NOTE: Check raw LOB field, not AARETE_DERIVED_LOB, because AARETE_DERIVED_LOB # is populated later via crosswalk and may be derived from PROGRAM/PRODUCT - lob_values = [answer_dict.get("LOB", "") for answer_dict in answer_dicts] - unique_lobs = set([lob for lob in lob_values if not string_utils.is_empty(lob)]) + # LOB can be a string or a list (from JSON format) + # Extract all LOB values, handling both string and list formats + # All values must be hashable (strings) to create a set + lob_values = [] + for answer_dict in answer_dicts: + lob_value = answer_dict.get("LOB", "") + if isinstance(lob_value, list): + # If it's a list, extract each item and ensure it's a string + for item in lob_value: + if not string_utils.is_empty(item): + # Convert to string to ensure hashability (handles edge cases) + lob_values.append(str(item)) + elif not string_utils.is_empty(lob_value): + # If it's a string, add it directly (already hashable) + lob_values.append(lob_value) + unique_lobs = set(lob_values) has_lob = len(unique_lobs) > 0 # Handle ALL empty fields diff --git a/src/pipelines/shared/extraction/one_to_n_funcs.py b/src/pipelines/shared/extraction/one_to_n_funcs.py index 0ebcfa9..f087e73 100644 --- a/src/pipelines/shared/extraction/one_to_n_funcs.py +++ b/src/pipelines/shared/extraction/one_to_n_funcs.py @@ -98,6 +98,13 @@ def reimbursement_level( reimbursement_primary_answers, constants, filename ) + # ============================================================================ + # DEBUG: Print reimbursement_primary_answers after filtering + # ============================================================================ + print(f"\n[DEBUG reimbursement_level] After filtering (line 101):") + print(reimbursement_primary_answers) + # ============================================================================ + return reimbursement_primary_answers @@ -108,10 +115,11 @@ def clean_reimbursement_primary( ) -> list[dict[str, str]]: """ Cleans and processes reimbursement primary answers through several steps: - 1. Split compound reimbursements - 2. Filter out any non-reimbursement content from split answers - 3. Deduplicate against previously seen (SERVICE_TERM, REIMB_TERM) pairs - 4. Apply exhibit lesser-of statement if it exists + 1. Normalize SERVICE_TERM and REIMB_TERM to strings (they should never be lists per-row) + 2. Split compound reimbursements + 3. Filter out any non-reimbursement content from split answers + 4. Deduplicate against previously seen (SERVICE_TERM, REIMB_TERM) pairs + 5. Apply exhibit lesser-of statement if it exists Args: reimbursement_primary_answers (list[str]): List of dictionaries containing reimbursement details. @@ -122,6 +130,7 @@ def clean_reimbursement_primary( """ if reimbursement_primary_answers: # If any reimbursements found + # If needed, add Reimbursement Splitting here # If needed, add deduplication here @@ -330,9 +339,17 @@ def methodology_breakout_single_row( list[dict]: Updated list with methodology breakout details and fee schedule information. """ - reimb_term, service_term = answer_dict.get("REIMB_TERM"), answer_dict.get( - "SERVICE_TERM" - ) + # DEBUG: Check what answer_dict contains + print(f"\n[DEBUG methodology_breakout_single_row] answer_dict REIMB_TERM: {repr(answer_dict.get('REIMB_TERM'))} (type: {type(answer_dict.get('REIMB_TERM')).__name__})") + print(f"[DEBUG methodology_breakout_single_row] answer_dict SERVICE_TERM: {repr(answer_dict.get('SERVICE_TERM'))} (type: {type(answer_dict.get('SERVICE_TERM')).__name__})") + + reimb_term = answer_dict.get("REIMB_TERM", "") + service_term = answer_dict.get("SERVICE_TERM", "") + + # DEBUG: Check what we extracted + print(f"[DEBUG methodology_breakout_single_row] Extracted reimb_term: {repr(reimb_term)} (type: {type(reimb_term).__name__})") + print(f"[DEBUG methodology_breakout_single_row] Extracted service_term: {repr(service_term)} (type: {type(service_term).__name__})\n") + methodology_breakout_questions = FieldSet( config.FIELD_JSON_PATH, field_type="methodology_breakout" ) @@ -781,6 +798,31 @@ def one_to_n_cleaning( ################################ Split REIMB_DATES ################################ all_exhibit_rows = split_reimb_dates(all_exhibit_rows, filename) + ################################ Normalize Single-Value Fields to Strings ################################ + # REIMB_TERM, SERVICE_TERM, and LOB_PROGRAM_RELATIONSHIP should always be strings (not lists) + for answer_dict in all_exhibit_rows: + # LOB_PROGRAM_RELATIONSHIP should be a string (not a list) + if "LOB_PROGRAM_RELATIONSHIP" in answer_dict: + lob_program_rel = answer_dict["LOB_PROGRAM_RELATIONSHIP"] + if isinstance(lob_program_rel, list): + # If list, take first element (shouldn't happen, but defensive) + answer_dict["LOB_PROGRAM_RELATIONSHIP"] = str(lob_program_rel[0]) if lob_program_rel else "Exclusive" + elif lob_program_rel is not None: + answer_dict["LOB_PROGRAM_RELATIONSHIP"] = str(lob_program_rel) + else: + answer_dict["LOB_PROGRAM_RELATIONSHIP"] = "Exclusive" + + # LOB_PRODUCT_RELATIONSHIP should be a string (not a list) + if "LOB_PRODUCT_RELATIONSHIP" in answer_dict: + lob_product_rel = answer_dict["LOB_PRODUCT_RELATIONSHIP"] + if isinstance(lob_product_rel, list): + # If list, take first element (shouldn't happen, but defensive) + answer_dict["LOB_PRODUCT_RELATIONSHIP"] = str(lob_product_rel[0]) if lob_product_rel else "Exclusive" + elif lob_product_rel is not None: + answer_dict["LOB_PRODUCT_RELATIONSHIP"] = str(lob_product_rel) + else: + answer_dict["LOB_PRODUCT_RELATIONSHIP"] = "Exclusive" + return all_exhibit_rows diff --git a/src/pipelines/shared/postprocessing/aarete_derived.py b/src/pipelines/shared/postprocessing/aarete_derived.py index 07608b4..e2f326d 100644 --- a/src/pipelines/shared/postprocessing/aarete_derived.py +++ b/src/pipelines/shared/postprocessing/aarete_derived.py @@ -32,18 +32,26 @@ def fill_na_mapping(answer_dicts): all_lob_values = set() # Get existing AARETE_DERIVED_LOB values (if any) + # Handle both list and string formats (JSON lists or pipe-delimited strings) existing_lob_list = answer_dict.get("AARETE_DERIVED_LOB", "") if not string_utils.is_empty(existing_lob_list): - for lob_val in existing_lob_list: + # Normalize to list format + if isinstance(existing_lob_list, list): + lob_items = existing_lob_list + elif isinstance(existing_lob_list, str): + # Handle pipe-delimited format (backward compatibility) if "|" in existing_lob_list: - all_lob_values.update( - v.strip() - for v in lob_val.split("|") - if v.strip() and v.strip() != "N/A" - ) - elif lob_val.strip() and lob_val.strip() != "N/A": - all_lob_values.add(lob_val) + lob_items = [v.strip() for v in existing_lob_list.split("|")] + else: + lob_items = [existing_lob_list] + else: + lob_items = [str(existing_lob_list)] + + # Process each LOB value + for lob_val in lob_items: + if isinstance(lob_val, str) and lob_val.strip() and lob_val.strip() != "N/A": + all_lob_values.add(lob_val.strip()) # Get AARETE_DERIVED_LOB from AARETE_DERIVED_PROGRAM crosswalk (always check if PROGRAM exists) program_lob_values = set() @@ -58,16 +66,24 @@ def fill_na_mapping(answer_dicts): not string_utils.is_empty(program_filled_value_list) and "N/A" not in program_filled_value_list ): - for program_lob_val in program_filled_value_list: - if "|" in program_lob_val: - program_lob_values = set( - v.strip() - for v in program_lob_val.split("|") - if v.strip() and v.strip() != "N/A" - ) - elif program_lob_val.strip() and program_lob_val.strip() != "N/A": - program_lob_values = {program_lob_val.strip()} - all_lob_values.update(program_lob_values) + # Normalize to list format (handle both list and string) + if isinstance(program_filled_value_list, list): + program_lob_items = program_filled_value_list + elif isinstance(program_filled_value_list, str): + # Handle pipe-delimited format (backward compatibility) + if "|" in program_filled_value_list: + program_lob_items = [v.strip() for v in program_filled_value_list.split("|")] + else: + program_lob_items = [program_filled_value_list] + else: + program_lob_items = [str(program_filled_value_list)] + + # Process each program LOB value + for program_lob_val in program_lob_items: + if isinstance(program_lob_val, str) and program_lob_val.strip() and program_lob_val.strip() != "N/A": + program_lob_values.add(program_lob_val.strip()) + # Update all_lob_values after processing all program LOB values + all_lob_values.update(program_lob_values) # Get AARETE_DERIVED_LOB from PRODUCT crosswalk (always check if PRODUCT exists) product_lob_values = set() @@ -82,16 +98,24 @@ def fill_na_mapping(answer_dicts): not string_utils.is_empty(product_filled_value_list) and "N/A" not in product_filled_value_list ): - for product_lob_val in product_filled_value_list: - if "|" in product_lob_val: - product_lob_values = set( - v.strip() - for v in product_filled_value_list.split("|") - if v.strip() and v.strip() != "N/A" - ) - elif product_lob_val.strip() and product_lob_val.strip() != "N/A": - product_lob_values = {product_lob_val.strip()} - all_lob_values.update(product_lob_values) + # Normalize to list format (handle both list and string) + if isinstance(product_filled_value_list, list): + product_lob_items = product_filled_value_list + elif isinstance(product_filled_value_list, str): + # Handle pipe-delimited format (backward compatibility) + if "|" in product_filled_value_list: + product_lob_items = [v.strip() for v in product_filled_value_list.split("|")] + else: + product_lob_items = [product_filled_value_list] + else: + product_lob_items = [str(product_filled_value_list)] + + # Process each product LOB value + for product_lob_val in product_lob_items: + if isinstance(product_lob_val, str) and product_lob_val.strip() and product_lob_val.strip() != "N/A": + product_lob_values.add(product_lob_val.strip()) + # Update all_lob_values after processing all product LOB values + all_lob_values.update(product_lob_values) # Set the merged, deduplicated value if all_lob_values: @@ -165,11 +189,16 @@ def get_crosswalk_fields(answer_dicts: list, constants: Constants): if "AARETE_DERIVED" in to_field_name: to_field_answer_list.append(individual_from_field_value) else: - to_field_answer_list.append( - crosswalk.create_reverse_mapping().get( - individual_from_field_value - ) - ) + # Reverse mapping returns dict[str, list[str]], so .get() returns a list + # We need to extend, not append, to avoid nested lists + reverse_mapping = crosswalk.create_reverse_mapping() + reverse_result = reverse_mapping.get(individual_from_field_value) + if reverse_result is not None: + # reverse_result is a list, extend it + to_field_answer_list.extend(reverse_result) + else: + # If not found in reverse mapping, keep original value + to_field_answer_list.append(individual_from_field_value) answer_dict[to_field_name] = to_field_answer_list # update from field value to list if not already if not isinstance(from_field_value, list): diff --git a/src/prompts/prompt_templates.py b/src/prompts/prompt_templates.py index 102ddcd..c08c186 100644 --- a/src/prompts/prompt_templates.py +++ b/src/prompts/prompt_templates.py @@ -26,7 +26,8 @@ JSON_LIST_OF_DICTS_FORMAT_INSTRUCTIONS = """Return your final answer as a valid - Each item in the list should be a dictionary with the specified field names as keys. - If multiple items are found, include all of them as separate dictionaries in the list. - Use "N/A" for fields within each dictionary where the information doesn't apply. -- Ensure the JSON is properly formatted with square brackets, curly braces, and double quotes.""" +- Ensure the JSON is properly formatted with square brackets, curly braces, and double quotes. +- The Keys and Values must be strings, and not lists.""" # Parser helper functions for prompt templates @@ -486,7 +487,7 @@ def REIMBURSEMENT_PRIMARY(context) -> Tuple[str, Callable[[str], list]]: Call REIMBURSEMENT_PRIMARY_INSTRUCTION() separately for the cached instruction. Returns: - Tuple of (prompt_string, parser_function) where parser expects JSON list output. + Tuple of (prompt_string, parser_function) where parser expects JSON list output. Should return a list of dicts of strings """ prompt = f""" [START CONTEXT] @@ -500,7 +501,7 @@ Briefly talk through your reasoning. Then return a properly formatted JSON list def REIMBURSEMENT_PRIMARY_INSTRUCTION() -> str: - return """[OBJECTIVE] + return f"""[OBJECTIVE] Extract Reimbursement Terms from a section of a payer-provider contract The output from this prompt will be used to populate downstream fields in the system. @@ -536,6 +537,17 @@ Some types of Reimbursements are special and require additional thought. - Stop Loss and Outliers - These reimbursements usually consist of multiple rates and/or thresholds. When a Stop-Loss or Outlier is found, include the entire statement (which may be multiple sentences) as a single REIMB_TERM. - Discounts, Escalators and Premiums - These will also include multiple rates. Capture all relevant numbers +[OUTPUT FORMAT] +You MUST return your answer as a JSON list of dictionaries. Each dictionary must contain exactly two keys: "SERVICE_TERM" and "REIMB_TERM". Both values must be strings (not lists). + +Example output format: +[ + {{"SERVICE_TERM": "Service 1", "REIMB_TERM": "Reimb 1"}}, + {{"SERVICE_TERM": "Service 2", "REIMB_TERM": "Reimb 2"}} +] + +{JSON_LIST_OF_DICTS_FORMAT_INSTRUCTIONS} + [ANALYSIS CONTEXT]""" @@ -852,7 +864,7 @@ Populate a list of JSON dictionaries with the following fields: [OUTPUT FORMAT] The output should always be a JSON list/array of dictionaries, even when there is only one methodology. -For any fields that don't apply to a particular methodology, use "N/A" as the value. +{JSON_LIST_OF_DICTS_FORMAT_INSTRUCTIONS} [CONTEXT]""" @@ -874,16 +886,20 @@ Populate a JSON dictionary with the following fields: {fields_text} [OUTPUT FORMAT] -Briefly explain your reasoning, then write your JSON dictionary.""" +{JSON_DICT_FORMAT_INSTRUCTIONS}""" def FEE_SCHEDULE_BREAKOUT( - methodology, fee_schedule + methodology:str, fee_schedule:str ) -> Tuple[str, Callable[[str], dict]]: """Returns ONLY dynamic content for fee schedule breakout. Call FEE_SCHEDULE_BREAKOUT_INSTRUCTION() separately for the cached instruction. Note: Field definitions are now included in FEE_SCHEDULE_BREAKOUT_INSTRUCTION() for caching. + Args: + methodology: The reimbursement methodology string (already normalized to string in prompt_calls) + fee_schedule: The fee schedule string (already normalized to string in prompt_calls) + Returns: Tuple of (prompt_string, parser_function) where parser expects JSON dict output. """ @@ -899,7 +915,6 @@ def GROUPER_BREAKOUT_INSTRUCTION() -> str: Contains objective, extraction rules, field definitions, and output format. Field definitions are loaded from investment_prompts.json. """ - # Load fields from investment_prompts.json with resolved valid_values fields_text = _get_fields_text("grouper_breakout") return f"""[OBJECTIVE] @@ -1151,6 +1166,10 @@ def CODE_EXPLICIT(service, methodology) -> Tuple[str, Callable[[str], dict]]: Call CODE_EXPLICIT_INSTRUCTION() separately for the cached instruction. Note: Field definitions are now included in CODE_EXPLICIT_INSTRUCTION() for caching. + Args: + service: The service term (already normalized to string upstream) + methodology: The reimbursement methodology (already normalized to string upstream) + Returns: Tuple of (prompt_string, parser_function) where parser expects JSON dict output. """ @@ -1182,6 +1201,10 @@ Explain your answer in 1-2 sentences. Write your final answer in a properly-form def CODE_CATEGORY(service, choices) -> Tuple[str, Callable[[str], list]]: """Call CODE_CATEGORY_INSTRUCTION() separately for the cached instruction. + Args: + service: The service term (already normalized to string upstream) + choices: The valid descriptions to choose from + Returns: Tuple of (prompt_string, parser_function) where parser expects JSON list output. """ @@ -1255,6 +1278,10 @@ Explain your answer, ensuring each point in the instructions is addressed. Then def CODE_IMPLICIT(service, choices) -> Tuple[str, Callable[[str], list]]: """Call CODE_IMPLICIT_INSTRUCTION() separately for the cached instruction. + Args: + service: The service term (already normalized to string upstream) + choices: The valid descriptions to choose from + Returns: Tuple of (prompt_string, parser_function) where parser expects JSON list output. """ @@ -1325,6 +1352,12 @@ def FILL_BILL_TYPE( ) -> Tuple[str, Callable[[str], list]]: """Call FILL_BILL_TYPE_INSTRUCTION() separately for the cached instruction. + Args: + service: The service term (already normalized to string upstream) + choices: The valid bill type descriptions to choose from + reimb_term: The reimbursement term (already normalized to string upstream, optional) + exhibit_text: Additional exhibit context (optional) + Returns: Tuple of (prompt_string, parser_function) where parser expects JSON list output. """ @@ -1346,6 +1379,7 @@ If the Service Term and Reimbursement Term together do not provide sufficient in {exhibit_text.replace('"', "'")} """ + # service is already normalized to string upstream prompt = f"""[VALID DESCRIPTIONS] Here are the descriptions to choose from: {choices} diff --git a/src/utils/crosswalk_utils.py b/src/utils/crosswalk_utils.py index dafdd71..ea7a8f7 100644 --- a/src/utils/crosswalk_utils.py +++ b/src/utils/crosswalk_utils.py @@ -4,16 +4,16 @@ import logging import src.utils.string_utils as string_utils -def apply_crosswalk(val: str, mapping: dict[str, str], default: str = "N/A") -> str: +def apply_crosswalk(val: str | list, mapping: dict[str, str], default: str = "N/A") -> list: """Apply a crosswalk to a value Args: - val (str): Input value + val (str | list): Input value (can be string, list, or string representation of list) mapping (dict[str, str]): Mapping dictionary default (str | None, optional): Default value; if None and val is not found in the mapping, val is returned. Defaults to "". Returns: - str: Crosswalked value + list: Crosswalked value as a list """ # Handle None or blank values defensively if val is None: diff --git a/src/utils/json_utils.py b/src/utils/json_utils.py index cef4b33..f70f459 100644 --- a/src/utils/json_utils.py +++ b/src/utils/json_utils.py @@ -253,4 +253,4 @@ def parse_json_dict_or_list(raw_llm_output: str) -> dict[str, Any] | list[Any]: raise ValueError( f"No valid JSON object found in LLM output. " f"Output preview: {raw_llm_output[:200]}..." - ) + ) \ No newline at end of file diff --git a/src/utils/qa_qc_utils.py b/src/utils/qa_qc_utils.py index 6d874a4..69c5bda 100644 --- a/src/utils/qa_qc_utils.py +++ b/src/utils/qa_qc_utils.py @@ -1092,22 +1092,41 @@ def safe_token_to_str(tok: Any) -> str: def format_or_preserve_tins(val: Any) -> Tuple[str, bool]: """ - Format TINs to 9 digits, preserving pipe-delimited multiple values. + Format TINs to 9 digits, handling JSON format (direct values or list of values). Args: - val: TIN value (may be pipe-delimited) + val: TIN value (may be a string, list of strings, or None/empty) Returns: Tuple of (formatted_tins, has_invalid) """ - if pd.isna(val) or val is None or str(val).strip() == "": - return ("", True) - - parts = [ - part.strip() - for part in str(val).split("|") - if part is not None and part.strip() != "" - ] + # Handle list/array values (e.g., from PROV_GROUP_TIN, PROV_OTHER_TIN) + if isinstance(val, (list, tuple)) or (hasattr(val, '__iter__') and not isinstance(val, str)): + # Convert list to list of strings, filtering out empty values + if len(val) == 0: + return ("", True) + parts = [ + str(item).strip() + for item in val + if item is not None and str(item).strip() != "" + ] + if not parts: + return ("", True) + else: + # Handle single value (string or None) + # Check for NaN/None/empty + try: + if pd.isna(val): + return ("", True) + except (ValueError, TypeError): + # pd.isna() can't handle arrays, but we've already handled lists above + pass + + if val is None or str(val).strip() == "": + return ("", True) + + # Single value - convert to list for consistent processing + parts = [str(val).strip()] out, any_invalid = [], False for part in parts: