diff --git a/src/pipelines/saas/file_processing.py b/src/pipelines/saas/file_processing.py index 2ed83aa..190e62e 100644 --- a/src/pipelines/saas/file_processing.py +++ b/src/pipelines/saas/file_processing.py @@ -522,6 +522,11 @@ def run_one_to_n_prompts( ) ################## CONVERT TO DF ################## + # Debug: print CLAIM_TYPE_CD before DataFrame creation + if one_to_n_results: + for i, row in enumerate(one_to_n_results[:3]): # Check first 3 rows + if "CLAIM_TYPE_CD" in row: + print(f"CLAIM_TYPE_CD before DataFrame (row {i}): {row['CLAIM_TYPE_CD']} (type: {type(row['CLAIM_TYPE_CD'])})") one_to_n_df = pd.DataFrame(one_to_n_results) return one_to_n_df, dynamic_one_to_one_fields, first_reimbursement_page diff --git a/src/pipelines/saas/prompts/prompt_calls.py b/src/pipelines/saas/prompts/prompt_calls.py index 3cac1f9..c1e34fc 100644 --- a/src/pipelines/saas/prompts/prompt_calls.py +++ b/src/pipelines/saas/prompts/prompt_calls.py @@ -37,21 +37,6 @@ def prompt_exhibit_level( llm_answer_final = _parser(llm_answer_raw) - # Normalize CLAIM_TYPE_CD if present: convert single-element lists to strings - # This field should be a single value even though it's a 1:N field - if "CLAIM_TYPE_CD" in llm_answer_final: - original_value = llm_answer_final["CLAIM_TYPE_CD"] - print( - f"CLAIM_TYPE_CD before normalization: {original_value} (type: {type(original_value)})" - ) - normalized_value = string_utils.normalize_one_to_one_field_value( - "CLAIM_TYPE_CD", original_value - ) - print( - f"CLAIM_TYPE_CD after normalization: {normalized_value} (type: {type(normalized_value)})" - ) - llm_answer_final["CLAIM_TYPE_CD"] = normalized_value - return llm_answer_final diff --git a/src/pipelines/shared/extraction/one_to_n_funcs.py b/src/pipelines/shared/extraction/one_to_n_funcs.py index 5c1527f..02ac830 100644 --- a/src/pipelines/shared/extraction/one_to_n_funcs.py +++ b/src/pipelines/shared/extraction/one_to_n_funcs.py @@ -769,18 +769,33 @@ def get_lob_relationship(answer_dicts, exhibit_text, filename): def one_to_n_cleaning( all_exhibit_rows: list[dict], exhibit_text: str, constants: Constants, filename: str ): + # Debug: print CLAIM_TYPE_CD at start of cleaning + if all_exhibit_rows and "CLAIM_TYPE_CD" in all_exhibit_rows[0]: + print(f"CLAIM_TYPE_CD at start of one_to_n_cleaning: {all_exhibit_rows[0]['CLAIM_TYPE_CD']} (type: {type(all_exhibit_rows[0]['CLAIM_TYPE_CD'])})") ################################ Crosswalk Fields ################################ all_exhibit_rows = aarete_derived.get_crosswalk_fields(all_exhibit_rows, constants) + # Debug: print CLAIM_TYPE_CD after crosswalk + if all_exhibit_rows and "CLAIM_TYPE_CD" in all_exhibit_rows[0]: + print(f"CLAIM_TYPE_CD after get_crosswalk_fields: {all_exhibit_rows[0]['CLAIM_TYPE_CD']} (type: {type(all_exhibit_rows[0]['CLAIM_TYPE_CD'])})") ################################ Determine LOB Relationship ################################ all_exhibit_rows = get_lob_relationship(all_exhibit_rows, exhibit_text, filename) + # Debug: print CLAIM_TYPE_CD after LOB relationship + if all_exhibit_rows and "CLAIM_TYPE_CD" in all_exhibit_rows[0]: + print(f"CLAIM_TYPE_CD after get_lob_relationship: {all_exhibit_rows[0]['CLAIM_TYPE_CD']} (type: {type(all_exhibit_rows[0]['CLAIM_TYPE_CD'])})") ################################ Fill NA Mapping ################################ all_exhibit_rows = aarete_derived.fill_na_mapping(all_exhibit_rows) + # Debug: print CLAIM_TYPE_CD after fill_na_mapping + if all_exhibit_rows and "CLAIM_TYPE_CD" in all_exhibit_rows[0]: + print(f"CLAIM_TYPE_CD after fill_na_mapping: {all_exhibit_rows[0]['CLAIM_TYPE_CD']} (type: {type(all_exhibit_rows[0]['CLAIM_TYPE_CD'])})") ################################ Split REIMB_DATES ################################ all_exhibit_rows = split_reimb_dates(all_exhibit_rows, filename) + # Debug: print CLAIM_TYPE_CD after split_reimb_dates + if all_exhibit_rows and "CLAIM_TYPE_CD" in all_exhibit_rows[0]: + print(f"CLAIM_TYPE_CD after split_reimb_dates: {all_exhibit_rows[0]['CLAIM_TYPE_CD']} (type: {type(all_exhibit_rows[0]['CLAIM_TYPE_CD'])})") ################################ Normalize Single-Value Fields to Strings ################################ # REIMB_TERM, SERVICE_TERM, and LOB_PROGRAM_RELATIONSHIP should always be strings (not lists) @@ -811,6 +826,10 @@ def one_to_n_cleaning( else: answer_dict["LOB_PRODUCT_RELATIONSHIP"] = "Exclusive" + # Debug: print CLAIM_TYPE_CD at end of one_to_n_cleaning + if all_exhibit_rows and "CLAIM_TYPE_CD" in all_exhibit_rows[0]: + print(f"CLAIM_TYPE_CD at end of one_to_n_cleaning: {all_exhibit_rows[0]['CLAIM_TYPE_CD']} (type: {type(all_exhibit_rows[0]['CLAIM_TYPE_CD'])})") + return all_exhibit_rows diff --git a/src/pipelines/shared/extraction/row_funcs.py b/src/pipelines/shared/extraction/row_funcs.py index 25682f6..62d886e 100644 --- a/src/pipelines/shared/extraction/row_funcs.py +++ b/src/pipelines/shared/extraction/row_funcs.py @@ -29,6 +29,9 @@ def combine_one_to_n( if key not in combined_dict or string_utils.is_empty( combined_dict.get(key) ): + # Debug: print CLAIM_TYPE_CD when it's being added from exhibit_level + if key == "CLAIM_TYPE_CD": + print(f"CLAIM_TYPE_CD in combine_one_to_n (from exhibit_level): {value} (type: {type(value)})") combined_dict[key] = value # Add exhibit text for contextual analysis (e.g., for bill type code determination) combined_dict["EXHIBIT_TEXT"] = exhibit_text diff --git a/src/pipelines/shared/extraction/tin_npi_funcs.py b/src/pipelines/shared/extraction/tin_npi_funcs.py index 6c98c62..d16c07b 100644 --- a/src/pipelines/shared/extraction/tin_npi_funcs.py +++ b/src/pipelines/shared/extraction/tin_npi_funcs.py @@ -484,6 +484,11 @@ def get_provider_info( # Defensive programming for any json parsing errors try: providers = _parser(llm_answer_raw) + # Print raw LLM provider NAME values to debug format issues + print(f"Raw LLM provider info (parsed) on page {page_num}: {providers}") + for i, provider in enumerate(providers): + if "NAME" in provider: + print(f" Provider {i} NAME: {provider['NAME']} (type: {type(provider['NAME'])})") logging.debug(f"Parsed provider info on page {page_num}: {providers}") # Ensure we have a list of providers (not a dict or other type) if isinstance(providers, dict): diff --git a/src/pipelines/shared/postprocessing/aarete_derived.py b/src/pipelines/shared/postprocessing/aarete_derived.py index 8cda91d..585057e 100644 --- a/src/pipelines/shared/postprocessing/aarete_derived.py +++ b/src/pipelines/shared/postprocessing/aarete_derived.py @@ -239,8 +239,9 @@ def get_crosswalk_fields(answer_dicts: list, constants: Constants): else: answer_dict[to_field_name] = to_field_answer_list - # update from field value to list if not already - if not isinstance(from_field_value, list): - answer_dict[from_field_name] = from_field_value_list + # DO NOT update from_field_name - preserve its original format + # The crosswalk should only create/update the target field (to_field_name), + # not modify the source field (from_field_name) + # If from_field_name needs to be a list, that should be handled elsewhere return answer_dicts