Merged in bugfix/derived_lob_changes (pull request #817)

Bugfix/derived lob changes

* Fix LOB_PROGRAM_RELATIONSHIP extraction and preserve debugging code

- Add format requirements to LOB_RELATIONSHIP_INSTRUCTION for proper caching
- Strengthen LOB_RELATIONSHIP prompt with explicit pipe delimiter requirements
- Preserve all DEBUG blocks and print statements in dynamic_funcs.py and file_processing.py
- Update crosswalk mappings and gitignore
- Ensure format instructions are cached at API level to prevent missing pipe delimiters

* Merge main into bugfix/dynamic_issuefixes

Resolved merge conflicts in:
- fieldExtraction/src/investment/dynamic_funcs.py (merged debug print statements)
- fieldExtraction/src/investment/file_processing.py (merged debug statements and exhibit inheritance code)

All conflicts resolved successfully.

* Fix AARETE_DERIVED_LOB to merge values from all crosswalks (PROGRAM and PRODUCT)

- Updated fill_na_mapping to always check both AARETE_DERIVED_PROGRAM and PRODUCT crosswalks
- Merges unique values from all sources (existing, PROGRAM, PRODUCT) instead of only filling when empty
- Ensures Duals is included when PRODUCT contains Molina Medicare Options Plus
- Deduplicates and sorts values before pipe-delimiting
- Each row processed individually for correct LOB assignment per reimbursement term

* Merged main into bugfix/derived_lob_changes


Approved-by: Katon Minhas

fieldExtraction/constants/mappings/crosswalk_product_lob.json

@@ -27,7 +27,7 @@
         },
         "Molina" :{
             "Molina Medicare Options" : "Medicare",
-            "Molina Medicare Options Plus" : "Medicare | Duals",
+            "Molina Medicare Options Plus" : "Duals",
             "Molina Health Benefit Exchange" : "Marketplace"
         },
         "Humana" : {

fieldExtraction/src/investment/aarete_derived.py

@@ -24,36 +24,70 @@ def fill_na_from_field(results_dict, to_field, from_field, crosswalk_path):
 def fill_na_mapping(answer_dicts):
     """
     Fill in missing values in fields using mappings from other fields.
+    Always checks all crosswalks (PROGRAM and PRODUCT) and merges unique values.
 
     Args:
-        results_df (pd.DataFrame): DataFrame containing the results data
+        answer_dicts (list[dict]): List of answer dictionaries
 
     Returns:
-        pd.DataFrame: DataFrame with filled missing values
+        list[dict]: List of answer dictionaries with filled/merged values
     """
     for answer_dict in answer_dicts:
-        # Fill AARETE_DERIVED_LOB from AARETE_DERIVED_PROGRAM
-        if string_utils.is_empty(answer_dict.get("AARETE_DERIVED_LOB")):
-            filled_value = fill_na_from_field(
+        # Collect all AARETE_DERIVED_LOB values from different sources
+        all_lob_values = set()
+        
+        # Get existing AARETE_DERIVED_LOB values (if any)
+        existing_lob = answer_dict.get("AARETE_DERIVED_LOB", "")
+        if not string_utils.is_empty(existing_lob):
+            if "|" in existing_lob:
+                all_lob_values.update(v.strip() for v in existing_lob.split("|") if v.strip() and v.strip() != "N/A")
+            elif existing_lob.strip() and existing_lob.strip() != "N/A":
+                all_lob_values.add(existing_lob.strip())
+        
+        # Get AARETE_DERIVED_LOB from AARETE_DERIVED_PROGRAM crosswalk (always check if PROGRAM exists)
+        program_lob_values = set()
+        if not string_utils.is_empty(answer_dict.get("AARETE_DERIVED_PROGRAM")):
+            program_filled_value = fill_na_from_field(
                 answer_dict,
                 "AARETE_DERIVED_LOB",
                 "AARETE_DERIVED_PROGRAM",
                 "constants/mappings/crosswalk_program_lob.json",
             )
-
-            answer_dict["AARETE_DERIVED_LOB"] = filled_value
-            answer_dict["LOB_PROGRAM_RELATIONSHIP"] = "Exclusive"
-
-        # Fill AARETE_DERIVED_LOB from PRODUCT
-        if string_utils.is_empty(answer_dict.get("AARETE_DERIVED_LOB")):
-            filled_value = fill_na_from_field(
+            if not string_utils.is_empty(program_filled_value) and program_filled_value != "N/A":
+                if "|" in program_filled_value:
+                    program_lob_values = set(v.strip() for v in program_filled_value.split("|") if v.strip() and v.strip() != "N/A")
+                elif program_filled_value.strip() and program_filled_value.strip() != "N/A":
+                    program_lob_values = {program_filled_value.strip()}
+                all_lob_values.update(program_lob_values)
+        
+        # Get AARETE_DERIVED_LOB from PRODUCT crosswalk (always check if PRODUCT exists)
+        product_lob_values = set()
+        if not string_utils.is_empty(answer_dict.get("PRODUCT")):
+            product_filled_value = fill_na_from_field(
                 answer_dict,
                 "AARETE_DERIVED_LOB",
                 "PRODUCT",
                 "constants/mappings/crosswalk_product_lob.json",
             )
-            answer_dict["AARETE_DERIVED_LOB"] = filled_value
-            answer_dict["LOB_PRODUCT_RELATIONSHIP"] = "Exclusive"
+            if not string_utils.is_empty(product_filled_value) and product_filled_value != "N/A":
+                if "|" in product_filled_value:
+                    product_lob_values = set(v.strip() for v in product_filled_value.split("|") if v.strip() and v.strip() != "N/A")
+                elif product_filled_value.strip() and product_filled_value.strip() != "N/A":
+                    product_lob_values = {product_filled_value.strip()}
+                all_lob_values.update(product_lob_values)
+        
+        # Set the merged, deduplicated value
+        if all_lob_values:
+            answer_dict["AARETE_DERIVED_LOB"] = "|".join(sorted(all_lob_values))
+            # Set relationship flags if values came from PROGRAM or PRODUCT
+            if program_lob_values:
+                answer_dict["LOB_PROGRAM_RELATIONSHIP"] = "Exclusive"
+            if product_lob_values:
+                answer_dict["LOB_PRODUCT_RELATIONSHIP"] = "Exclusive"
+        elif string_utils.is_empty(answer_dict.get("AARETE_DERIVED_LOB")):
+            # If still empty after all attempts, keep it as is (will be N/A or empty)
+            pass
+    
     return answer_dicts