diff --git a/src/codes/code_funcs.py b/src/codes/code_funcs.py index c8a4a10..a64b34d 100644 --- a/src/codes/code_funcs.py +++ b/src/codes/code_funcs.py @@ -390,7 +390,11 @@ def code_implicit_rag(service, implicit_run_dict, filename, constants): if matching_codes: # Split pipe-delimited codes into individual codes for code in matching_codes: - proc_codes.extend(code) + # Handle pipe-delimited codes (e.g., "99170-99199|99200-99210") + if "|" in code: + proc_codes.extend(code.split("|")) + else: + proc_codes.append(code) proc_descs.append(description) if description in hcpcs_mapping.values(): matching_codes = [ @@ -399,7 +403,11 @@ def code_implicit_rag(service, implicit_run_dict, filename, constants): if matching_codes: # Split pipe-delimited codes into individual codes for code in matching_codes: - proc_codes.extend(code) + # Handle pipe-delimited codes (e.g., "99170-99199|99200-99210") + if "|" in code: + proc_codes.extend(code.split("|")) + else: + proc_codes.append(code) proc_descs.append(description) if description in rev_mapping.values(): matching_codes = [ @@ -408,7 +416,11 @@ def code_implicit_rag(service, implicit_run_dict, filename, constants): if matching_codes: # Split pipe-delimited codes into individual codes for code in matching_codes: - rev_codes.extend(code) + # Handle pipe-delimited codes (e.g., "99170-99199|99200-99210") + if "|" in code: + rev_codes.extend(code.split("|")) + else: + rev_codes.append(code) rev_descs.append(description) # Combine answers diff --git a/src/pipelines/clients/bcbs_promise/file_processing.py b/src/pipelines/clients/bcbs_promise/file_processing.py index a962ae5..d3c3c8c 100644 --- a/src/pipelines/clients/bcbs_promise/file_processing.py +++ b/src/pipelines/clients/bcbs_promise/file_processing.py @@ -260,6 +260,14 @@ def run_one_to_one_prompts( one_to_one_results, contract_text ) + ################## Normalize before crosswalk ################## + # Normalize before get_crosswalk_fields (which expects/returns a list) + # get_crosswalk_fields and fill_na_mapping are mechanical steps (no LLM), + # so they won't introduce single-element lists that need normalization + one_to_one_results = string_utils.normalize_one_to_one_answers_dict( + one_to_one_results + ) + ################## Crosswalk Fields ################## one_to_one_results = aarete_derived.get_crosswalk_fields( [one_to_one_results], constants diff --git a/src/pipelines/clients/bcbs_promise/prompts/prompt_calls.py b/src/pipelines/clients/bcbs_promise/prompts/prompt_calls.py index 1357402..35c1dde 100644 --- a/src/pipelines/clients/bcbs_promise/prompts/prompt_calls.py +++ b/src/pipelines/clients/bcbs_promise/prompts/prompt_calls.py @@ -31,6 +31,13 @@ def prompt_exhibit_level( llm_answer_final = _parser(llm_answer_raw) + # Normalize CLAIM_TYPE_CD if present: convert single-element lists to strings + # This field should be a single value even though it's a 1:N field + if "CLAIM_TYPE_CD" in llm_answer_final: + llm_answer_final["CLAIM_TYPE_CD"] = string_utils.normalize_one_to_one_field_value( + "CLAIM_TYPE_CD", llm_answer_final["CLAIM_TYPE_CD"] + ) + return llm_answer_final @@ -437,6 +444,11 @@ def prompt_full_context( instruction=prompt_templates.ONE_TO_ONE_MULTI_FIELD_INSTRUCTION(), ) full_context_answers_dict = _parser(claude_answer_raw) + # Normalize 1:1 fields: convert single-element lists to strings + # Preserves legitimate multi-value lists (e.g., LOB, PROGRAM when passed to 1:1) + full_context_answers_dict = string_utils.normalize_one_to_one_answers_dict( + full_context_answers_dict + ) except Exception as e: logging.error(f"Error processing high accuracy fields: {str(e)}") full_context_answers_dict = {} diff --git a/src/pipelines/clients/clover/file_processing.py b/src/pipelines/clients/clover/file_processing.py index f441d10..b9e3886 100644 --- a/src/pipelines/clients/clover/file_processing.py +++ b/src/pipelines/clients/clover/file_processing.py @@ -262,6 +262,14 @@ def run_one_to_one_prompts( one_to_one_results, contract_text ) + ################## Normalize before crosswalk ################## + # Normalize before get_crosswalk_fields (which expects/returns a list) + # get_crosswalk_fields and fill_na_mapping are mechanical steps (no LLM), + # so they won't introduce single-element lists that need normalization + one_to_one_results = string_utils.normalize_one_to_one_answers_dict( + one_to_one_results + ) + ################## Crosswalk Fields ################## one_to_one_results = aarete_derived.get_crosswalk_fields( [one_to_one_results], constants diff --git a/src/pipelines/clients/clover/prompts/prompt_calls.py b/src/pipelines/clients/clover/prompts/prompt_calls.py index 01a0e44..80c45a2 100644 --- a/src/pipelines/clients/clover/prompts/prompt_calls.py +++ b/src/pipelines/clients/clover/prompts/prompt_calls.py @@ -37,6 +37,13 @@ def prompt_exhibit_level( llm_answer_final = _parser(llm_answer_raw) + # Normalize CLAIM_TYPE_CD if present: convert single-element lists to strings + # This field should be a single value even though it's a 1:N field + if "CLAIM_TYPE_CD" in llm_answer_final: + llm_answer_final["CLAIM_TYPE_CD"] = string_utils.normalize_one_to_one_field_value( + "CLAIM_TYPE_CD", llm_answer_final["CLAIM_TYPE_CD"] + ) + return llm_answer_final @@ -463,6 +470,12 @@ def prompt_full_context( if not isinstance(full_context_answers_dict, dict): full_context_answers_dict = extract_and_parse(claude_answer_raw) + # Normalize 1:1 fields: convert single-element lists to strings + # Preserves legitimate multi-value lists (e.g., LOB, PROGRAM when passed to 1:1) + full_context_answers_dict = string_utils.normalize_one_to_one_answers_dict( + full_context_answers_dict + ) + except Exception as e: logging.error(f"Error processing high accuracy fields: {str(e)}") full_context_answers_dict = {} diff --git a/src/pipelines/saas/file_processing.py b/src/pipelines/saas/file_processing.py index 9cab5ef..2ed83aa 100644 --- a/src/pipelines/saas/file_processing.py +++ b/src/pipelines/saas/file_processing.py @@ -242,6 +242,14 @@ def run_one_to_one_prompts( one_to_one_results, contract_text ) + ################## Normalize before crosswalk ################## + # Normalize before get_crosswalk_fields (which expects/returns a list) + # get_crosswalk_fields and fill_na_mapping are mechanical steps (no LLM), + # so they won't introduce single-element lists that need normalization + one_to_one_results = string_utils.normalize_one_to_one_answers_dict( + one_to_one_results + ) + ################## Crosswalk Fields ################## one_to_one_results = aarete_derived.get_crosswalk_fields( [one_to_one_results], constants diff --git a/src/pipelines/saas/prompts/prompt_calls.py b/src/pipelines/saas/prompts/prompt_calls.py index 5e03d50..51ae68b 100644 --- a/src/pipelines/saas/prompts/prompt_calls.py +++ b/src/pipelines/saas/prompts/prompt_calls.py @@ -37,6 +37,21 @@ def prompt_exhibit_level( llm_answer_final = _parser(llm_answer_raw) + # Normalize CLAIM_TYPE_CD if present: convert single-element lists to strings + # This field should be a single value even though it's a 1:N field + if "CLAIM_TYPE_CD" in llm_answer_final: + original_value = llm_answer_final["CLAIM_TYPE_CD"] + print( + f"CLAIM_TYPE_CD before normalization: {original_value} (type: {type(original_value)})" + ) + normalized_value = string_utils.normalize_one_to_one_field_value( + "CLAIM_TYPE_CD", original_value + ) + print( + f"CLAIM_TYPE_CD after normalization: {normalized_value} (type: {type(normalized_value)})" + ) + llm_answer_final["CLAIM_TYPE_CD"] = normalized_value + return llm_answer_final @@ -294,7 +309,7 @@ def prompt_lob_relationship( """ programs = answer_dict.get(field) dynamic_primary_values = ( - f"LOB: {answer_dict.get("AARETE_DERIVED_LOB")}\nPROGRAM: {programs}" + f"LOB: {answer_dict.get('AARETE_DERIVED_LOB')}\nPROGRAM: {programs}" ) prompt, _parser = prompt_templates.LOB_RELATIONSHIP( @@ -393,6 +408,11 @@ def prompt_full_context( ) full_context_answers_dict = _parser(claude_answer_raw) + # Normalize 1:1 fields: convert single-element lists to strings + # Preserves legitimate multi-value lists (e.g., LOB, PROGRAM when passed to 1:1) + full_context_answers_dict = string_utils.normalize_one_to_one_answers_dict( + full_context_answers_dict + ) except Exception as e: logging.error(f"Error processing high accuracy fields: {str(e)}") full_context_answers_dict = {} diff --git a/src/pipelines/shared/extraction/one_to_one_funcs.py b/src/pipelines/shared/extraction/one_to_one_funcs.py index cc423ab..0b0318c 100644 --- a/src/pipelines/shared/extraction/one_to_one_funcs.py +++ b/src/pipelines/shared/extraction/one_to_one_funcs.py @@ -235,6 +235,12 @@ def run_full_context_fields( # Run Special Case Breakout on any breakout terms full_context_answers_dict = one_to_one_breakout(full_context_answers_dict, filename) + # Normalize breakout fields: prompt_special_case_breakout uses _json_dict_parser + # which may return lists, so normalize any newly added fields + full_context_answers_dict = string_utils.normalize_one_to_one_answers_dict( + full_context_answers_dict + ) + # Handle separate one-to-one fields that require individual processing separate_fields = one_to_one_fields.filter(field_type="full_context_separate") @@ -323,6 +329,13 @@ def one_to_one_breakout( breakout_answer_dict = prompt_calls.prompt_special_case_breakout( breakout_template, term_answer, filename ) + # Normalize breakout fields for 1:1: _json_dict_parser may return lists for values + # Convert single-element lists to strings, preserve multi-value lists + # Note: This normalization is only for 1:1 fields; 1:N breakouts are handled separately + if isinstance(breakout_answer_dict, dict): + breakout_answer_dict = string_utils.normalize_one_to_one_answers_dict( + breakout_answer_dict + ) full_context_answers_dict.update(breakout_answer_dict) return full_context_answers_dict diff --git a/src/pipelines/shared/extraction/tin_npi_funcs.py b/src/pipelines/shared/extraction/tin_npi_funcs.py index 132720e..8d63b47 100644 --- a/src/pipelines/shared/extraction/tin_npi_funcs.py +++ b/src/pipelines/shared/extraction/tin_npi_funcs.py @@ -228,7 +228,13 @@ def merge_provider_info_with_hybrid_smart_chunking(one_to_one_results, filename) other_npis = [] other_names = [] - provider_name = one_to_one_results.get("PROVIDER_NAME") # Returns List + provider_name = one_to_one_results.get("PROVIDER_NAME") # Can be List or string + # Normalize provider_name to list: if string, convert to list to avoid character splitting + if isinstance(provider_name, str): + provider_name = [provider_name] + elif not isinstance(provider_name, list): + provider_name = [] + provider_info = one_to_one_results[ "PROV_INFO_JSON" ] # Returns string representation of list @@ -243,6 +249,21 @@ def merge_provider_info_with_hybrid_smart_chunking(one_to_one_results, filename) tins = provider.get("TIN") or [] npis = provider.get("NPI") or [] names = provider.get("NAME") or [] + + # Normalize to lists: if string, convert to list to avoid character splitting + if isinstance(tins, str): + tins = [tins] + elif not isinstance(tins, list): + tins = [] + if isinstance(npis, str): + npis = [npis] + elif not isinstance(npis, list): + npis = [] + if isinstance(names, str): + names = [names] + elif not isinstance(names, list): + names = [] + name_matches = prompt_calls.provider_name_match_check( names, provider_name, filename ) # Returns Boolean diff --git a/src/pipelines/shared/postprocessing/aarete_derived.py b/src/pipelines/shared/postprocessing/aarete_derived.py index a4daea3..8cda91d 100644 --- a/src/pipelines/shared/postprocessing/aarete_derived.py +++ b/src/pipelines/shared/postprocessing/aarete_derived.py @@ -169,6 +169,19 @@ def fill_na_mapping(answer_dicts): def get_crosswalk_fields(answer_dicts: list, constants: Constants): + """ + Apply crosswalk mappings to derive target fields from source fields. + + Fields that should be single values (strings) even when crosswalk creates lists: + - AARETE_DERIVED_CLAIM_TYPE_CD: Claim type is a single classification value + - Other AARETE_DERIVED_* fields that represent single classifications (not multi-value like LOB) + """ + # Fields that should always be single values (strings), not lists + # These are typically classification/derived fields that represent a single value + SINGLE_VALUE_FIELDS = { + "AARETE_DERIVED_CLAIM_TYPE_CD", + # Add other single-value derived fields here as needed + } crosswalk_fields = FieldSet(file_path=config.FIELD_JSON_PATH, crosswalk=True) for to_field in crosswalk_fields.fields: @@ -219,7 +232,13 @@ def get_crosswalk_fields(answer_dicts: list, constants: Constants): to_field_answer_list.append( individual_from_field_value ) - answer_dict[to_field_name] = to_field_answer_list + + # Assign the result: convert to string if it's a single-value field with one element + if to_field_name in SINGLE_VALUE_FIELDS and len(to_field_answer_list) == 1: + answer_dict[to_field_name] = to_field_answer_list[0] + else: + answer_dict[to_field_name] = to_field_answer_list + # update from field value to list if not already if not isinstance(from_field_value, list): answer_dict[from_field_name] = from_field_value_list diff --git a/src/pipelines/shared/preprocessing/hybrid_smart_chunking_funcs.py b/src/pipelines/shared/preprocessing/hybrid_smart_chunking_funcs.py index 5942305..95a5fd4 100644 --- a/src/pipelines/shared/preprocessing/hybrid_smart_chunking_funcs.py +++ b/src/pipelines/shared/preprocessing/hybrid_smart_chunking_funcs.py @@ -283,6 +283,11 @@ def run_hybrid_smart_chunked_fields( answers_dict = string_utils.normalize_state_field( answers_dict, field_name="PAYER_STATE" ) + + # Normalize all 1:1 fields: convert single-element lists to strings + # Preserves legitimate multi-value lists (e.g., LOB, PROGRAM when passed to 1:1) + answers_dict = string_utils.normalize_one_to_one_answers_dict(answers_dict) + return answers_dict except Exception as e: diff --git a/src/utils/string_utils.py b/src/utils/string_utils.py index 8515815..6fc5f97 100644 --- a/src/utils/string_utils.py +++ b/src/utils/string_utils.py @@ -806,3 +806,94 @@ def normalize_to_json_list(val): result = json.dumps(expanded) return result return val + + +def normalize_one_to_one_field_value(field_name: str, value) -> str | list: + """ + Normalize a 1:1 field value to ensure single-value fields are strings, not lists. + + This function handles the common case where LLM returns a single-value field as + a list with one element (e.g., ['The Value']). It extracts the string from such + lists while preserving legitimate multi-value lists. + + Args: + field_name (str): Name of the field being normalized (for logging/debugging) + value: The field value to normalize. Can be: + - str: Returned as-is + - list with 1 element: Extract the string + - list with >1 elements: Returned unchanged (may be legitimate multi-value) + - None/empty: Returned as-is + + Returns: + str | list: Normalized value: + - If input is a list with exactly 1 element: returns that element as string + - If input is already a string: returns as-is + - If input is a list with >1 elements: returns list unchanged + - If input is None or empty: returns as-is + + Examples: + >>> normalize_one_to_one_field_value("CONTRACT_TITLE", ["Provider Agreement"]) + 'Provider Agreement' + >>> normalize_one_to_one_field_value("PAYER_NAME", "Aetna") + 'Aetna' + >>> normalize_one_to_one_field_value("PROVIDER_NAME", ["Provider A", "Provider B"]) + ['Provider A', 'Provider B'] # Multi-value preserved + >>> normalize_one_to_one_field_value("LOB", ["Medicare", "Medicaid"]) + ['Medicare', 'Medicaid'] # Multi-value preserved + """ + # Handle None and empty values + if value is None or (isinstance(value, str) and is_empty(value)): + return value + + # If it's already a string, return as-is + if isinstance(value, str): + return value + + # If it's a list + if isinstance(value, list): + # List with exactly 1 element: extract the string + if len(value) == 1: + return str(value[0]) if value[0] is not None else "N/A" + # List with >1 elements: preserve (may be legitimate multi-value) + # This handles cases like PROVIDER_NAME with multiple providers, + # or dynamic fields (LOB, PROGRAM, PRODUCT, NETWORK) when passed to 1:1 + elif len(value) > 1: + return value + # Empty list: return "N/A" + else: + return "N/A" + + # For any other type, convert to string + return str(value) if value is not None else "N/A" + + +def normalize_one_to_one_answers_dict(answers_dict: dict[str, str | list]) -> dict[str, str | list]: + """ + Normalize all values in a 1:1 answers dictionary. + + Applies normalization to all fields in the dictionary, converting single-element + lists to strings while preserving legitimate multi-value lists. + + Args: + answers_dict (dict): Dictionary of field names to values (may be strings or lists) + + Returns: + dict: Dictionary with normalized values (single-element lists converted to strings) + + Example: + >>> answers = { + ... "CONTRACT_TITLE": ["Provider Agreement"], + ... "PAYER_NAME": "Aetna", + ... "PROVIDER_NAME": ["Provider A", "Provider B"] + ... } + >>> normalize_one_to_one_answers_dict(answers) + { + "CONTRACT_TITLE": "Provider Agreement", + "PAYER_NAME": "Aetna", + "PROVIDER_NAME": ["Provider A", "Provider B"] + } + """ + normalized = {} + for field_name, value in answers_dict.items(): + normalized[field_name] = normalize_one_to_one_field_value(field_name, value) + return normalized