From b64e1fe4cc3559055b75f1ba30e6c9208f4fea25 Mon Sep 17 00:00:00 2001 From: Praneel Panchigar Date: Mon, 22 Dec 2025 19:40:31 +0000 Subject: [PATCH] Merged in bugfix/derived_lob_changes (pull request #817) Bugfix/derived lob changes * Fix LOB_PROGRAM_RELATIONSHIP extraction and preserve debugging code - Add format requirements to LOB_RELATIONSHIP_INSTRUCTION for proper caching - Strengthen LOB_RELATIONSHIP prompt with explicit pipe delimiter requirements - Preserve all DEBUG blocks and print statements in dynamic_funcs.py and file_processing.py - Update crosswalk mappings and gitignore - Ensure format instructions are cached at API level to prevent missing pipe delimiters * Merge main into bugfix/dynamic_issuefixes Resolved merge conflicts in: - fieldExtraction/src/investment/dynamic_funcs.py (merged debug print statements) - fieldExtraction/src/investment/file_processing.py (merged debug statements and exhibit inheritance code) All conflicts resolved successfully. * Fix AARETE_DERIVED_LOB to merge values from all crosswalks (PROGRAM and PRODUCT) - Updated fill_na_mapping to always check both AARETE_DERIVED_PROGRAM and PRODUCT crosswalks - Merges unique values from all sources (existing, PROGRAM, PRODUCT) instead of only filling when empty - Ensures Duals is included when PRODUCT contains Molina Medicare Options Plus - Deduplicates and sorts values before pipe-delimiting - Each row processed individually for correct LOB assignment per reimbursement term * Merged main into bugfix/derived_lob_changes Approved-by: Katon Minhas --- .../mappings/crosswalk_product_lob.json | 2 +- .../src/investment/aarete_derived.py | 62 ++++++++++++++----- 2 files changed, 49 insertions(+), 15 deletions(-) diff --git a/fieldExtraction/constants/mappings/crosswalk_product_lob.json b/fieldExtraction/constants/mappings/crosswalk_product_lob.json index b5314a3..091f0be 100644 --- a/fieldExtraction/constants/mappings/crosswalk_product_lob.json +++ b/fieldExtraction/constants/mappings/crosswalk_product_lob.json @@ -27,7 +27,7 @@ }, "Molina" :{ "Molina Medicare Options" : "Medicare", - "Molina Medicare Options Plus" : "Medicare | Duals", + "Molina Medicare Options Plus" : "Duals", "Molina Health Benefit Exchange" : "Marketplace" }, "Humana" : { diff --git a/fieldExtraction/src/investment/aarete_derived.py b/fieldExtraction/src/investment/aarete_derived.py index 757049f..b1482c7 100644 --- a/fieldExtraction/src/investment/aarete_derived.py +++ b/fieldExtraction/src/investment/aarete_derived.py @@ -24,36 +24,70 @@ def fill_na_from_field(results_dict, to_field, from_field, crosswalk_path): def fill_na_mapping(answer_dicts): """ Fill in missing values in fields using mappings from other fields. + Always checks all crosswalks (PROGRAM and PRODUCT) and merges unique values. Args: - results_df (pd.DataFrame): DataFrame containing the results data + answer_dicts (list[dict]): List of answer dictionaries Returns: - pd.DataFrame: DataFrame with filled missing values + list[dict]: List of answer dictionaries with filled/merged values """ for answer_dict in answer_dicts: - # Fill AARETE_DERIVED_LOB from AARETE_DERIVED_PROGRAM - if string_utils.is_empty(answer_dict.get("AARETE_DERIVED_LOB")): - filled_value = fill_na_from_field( + # Collect all AARETE_DERIVED_LOB values from different sources + all_lob_values = set() + + # Get existing AARETE_DERIVED_LOB values (if any) + existing_lob = answer_dict.get("AARETE_DERIVED_LOB", "") + if not string_utils.is_empty(existing_lob): + if "|" in existing_lob: + all_lob_values.update(v.strip() for v in existing_lob.split("|") if v.strip() and v.strip() != "N/A") + elif existing_lob.strip() and existing_lob.strip() != "N/A": + all_lob_values.add(existing_lob.strip()) + + # Get AARETE_DERIVED_LOB from AARETE_DERIVED_PROGRAM crosswalk (always check if PROGRAM exists) + program_lob_values = set() + if not string_utils.is_empty(answer_dict.get("AARETE_DERIVED_PROGRAM")): + program_filled_value = fill_na_from_field( answer_dict, "AARETE_DERIVED_LOB", "AARETE_DERIVED_PROGRAM", "constants/mappings/crosswalk_program_lob.json", ) - - answer_dict["AARETE_DERIVED_LOB"] = filled_value - answer_dict["LOB_PROGRAM_RELATIONSHIP"] = "Exclusive" - - # Fill AARETE_DERIVED_LOB from PRODUCT - if string_utils.is_empty(answer_dict.get("AARETE_DERIVED_LOB")): - filled_value = fill_na_from_field( + if not string_utils.is_empty(program_filled_value) and program_filled_value != "N/A": + if "|" in program_filled_value: + program_lob_values = set(v.strip() for v in program_filled_value.split("|") if v.strip() and v.strip() != "N/A") + elif program_filled_value.strip() and program_filled_value.strip() != "N/A": + program_lob_values = {program_filled_value.strip()} + all_lob_values.update(program_lob_values) + + # Get AARETE_DERIVED_LOB from PRODUCT crosswalk (always check if PRODUCT exists) + product_lob_values = set() + if not string_utils.is_empty(answer_dict.get("PRODUCT")): + product_filled_value = fill_na_from_field( answer_dict, "AARETE_DERIVED_LOB", "PRODUCT", "constants/mappings/crosswalk_product_lob.json", ) - answer_dict["AARETE_DERIVED_LOB"] = filled_value - answer_dict["LOB_PRODUCT_RELATIONSHIP"] = "Exclusive" + if not string_utils.is_empty(product_filled_value) and product_filled_value != "N/A": + if "|" in product_filled_value: + product_lob_values = set(v.strip() for v in product_filled_value.split("|") if v.strip() and v.strip() != "N/A") + elif product_filled_value.strip() and product_filled_value.strip() != "N/A": + product_lob_values = {product_filled_value.strip()} + all_lob_values.update(product_lob_values) + + # Set the merged, deduplicated value + if all_lob_values: + answer_dict["AARETE_DERIVED_LOB"] = "|".join(sorted(all_lob_values)) + # Set relationship flags if values came from PROGRAM or PRODUCT + if program_lob_values: + answer_dict["LOB_PROGRAM_RELATIONSHIP"] = "Exclusive" + if product_lob_values: + answer_dict["LOB_PRODUCT_RELATIONSHIP"] = "Exclusive" + elif string_utils.is_empty(answer_dict.get("AARETE_DERIVED_LOB")): + # If still empty after all attempts, keep it as is (will be N/A or empty) + pass + return answer_dicts