Merged in DAIP2-2162-fix-mapping-issue-program-product-not-mapping-to-lob (pull request #914)

DAIP2-2162 fix mapping issue program product not mapping to lob

* fixed empty LOB

* adding program-lob mapping when there is no client

* code cleanup

* code change refactored

* code change refactored

* additional test case removed

* pipeline error fixed


Approved-by: Katon Minhas

src/crosswalk/crosswalk_builder.py

@@ -114,8 +114,28 @@ class CrosswalkBuilder:
             for state in STATE:
                 mapping.update(data.get("state_mapping", {}).get(state, {}))
         if "client_mapping" in data:
-            for client in CLIENT:
-                mapping.update(data.get("client_mapping", {}).get(client, {}))
+            client_mapping = data.get("client_mapping", {})
+            clients = [
+                str(c).strip()
+                for c in CLIENT
+                if c is not None and str(c).strip().lower() != "none"
+            ]
+
+            if clients:
+                for client in clients:
+                    mapping.update(client_mapping.get(client, {}))
+            else:
+                # No explicit client provided: merge all client mappings with deduplication.
+                # Keep first value for duplicate keys.
+                merged_client_mapping: dict[str, str] = {}
+                for client_map in client_mapping.values():
+                    if not isinstance(client_map, dict):
+                        continue
+
+                    for key, value in client_map.items():
+                        merged_client_mapping.setdefault(key, value)
+
+                mapping.update(merged_client_mapping)
 
         self.mapping = mapping
 

src/pipelines/shared/postprocessing/aarete_derived.py

@@ -138,6 +138,8 @@ def fill_na_mapping(answer_dicts):
         # Set the merged, deduplicated value
         if all_lob_values:
             answer_dict["AARETE_DERIVED_LOB"] = sorted(all_lob_values)
+            if string_utils.is_empty(answer_dict.get("LOB")):
+                answer_dict["LOB"] = sorted(all_lob_values)
             # Set relationship flags if values came from PROGRAM or PRODUCT
             # Only set if field is empty or "N/A" (preserve LLM-determined Inclusive/Exclusive)
             if program_lob_values:

src/tests/test_crosswalk_utils.py

@@ -128,6 +128,28 @@ def test_to_csv_no_metadata(tmp_path):
     assert result_dict == {"A": "one", "B": "two"}
 
 
+def test_from_json_client_none_merges_all_client_mappings(tmp_path, monkeypatch):
+    data = {
+        "metadata": {"from_col": "from_col", "to_col": "to_col"},
+        "mapping": {},
+        "client_mapping": {
+            "ClientA": {"Connexus": "Commercial", "Synergy": "Commercial"},
+            "ClientB": {"HEALTHfirst": "Medicaid"},
+        },
+    }
+    file_path = tmp_path / "client_merge.json"
+    with open(file_path, "w") as f:
+        json.dump(data, f)
+
+    monkeypatch.setattr("src.crosswalk.crosswalk_builder.CLIENT", ["None"])
+    monkeypatch.setattr("src.crosswalk.crosswalk_builder.STATE", ["None"])
+
+    builder = CrosswalkBuilder().from_json(file_path)
+    assert builder.mapping["Connexus"] == "Commercial"
+    assert builder.mapping["Synergy"] == "Commercial"
+    assert builder.mapping["HEALTHfirst"] == "Medicaid"
+
+
 def test_apply_crosswalk():
     mapping = {"A": "1", "B": "2", "C": "3"}
     assert apply_crosswalk("A", mapping) == ["1"]

src/tests/test_postprocess.py

@@ -24,6 +24,7 @@ from src.pipelines.shared.postprocessing.postprocessing_funcs import (
     standardize_reimb_method_and_fee_schedule,
     validate_and_reformat_date,
 )
+from src.pipelines.shared.postprocessing import aarete_derived
 from src.pipelines.shared.postprocessing.postprocess import standard_postprocess
 from src.constants.constants import Constants
 
@@ -957,6 +958,25 @@ class TestPostprocessFunctions(unittest.TestCase):
         self.assertEqual(result_df.loc[0, "CONTRACT_TITLE"], "")
         self.assertEqual(result_df.loc[1, "CONTRACT_TITLE"], "Another Valid")
 
+    def test_fill_na_mapping_derives_lob_from_aarete_derived_product(self):
+        """fill_na_mapping should derive LOB from AARETE_DERIVED_PRODUCT."""
+        rows = [
+            {
+                "PRODUCT": "",
+                "AARETE_DERIVED_PRODUCT": '["Connexus", "Synergy"]',
+                "AARETE_DERIVED_LOB": "",
+                "LOB": "",
+                "AARETE_DERIVED_PROGRAM": "",
+                "LOB_PROGRAM_RELATIONSHIP": "",
+                "LOB_PRODUCT_RELATIONSHIP": "",
+            }
+        ]
+
+        result = aarete_derived.fill_na_mapping(rows)
+
+        self.assertEqual(result[0]["AARETE_DERIVED_LOB"], ["Commercial"])
+        self.assertEqual(result[0]["LOB"], ["Commercial"])
+
 
 class TestStandardizeReimbMethodAndFeeScheduleUOM(unittest.TestCase):
     """Tests for UNIT_OF_MEASURE post-processing when DEFAULT_IND='Y' and flat rate."""

src/tests/test_string_utils.py

@@ -153,6 +153,10 @@ class TestStringUtils(unittest.TestCase):
     def test_is_empty_string(self):
         self.assertTrue(string_utils.is_empty(""))
         self.assertTrue(string_utils.is_empty("N/A"))
+        self.assertTrue(string_utils.is_empty("['N/A']"))
+        self.assertTrue(string_utils.is_empty('["N/A"]'))
+        self.assertTrue(string_utils.is_empty("[]"))
+        self.assertFalse(string_utils.is_empty('["Medicaid", "N/A"]'))
         self.assertFalse(string_utils.is_empty("valid"))
 
     def test_is_empty_list(self):

src/utils/string_utils.py

@@ -463,6 +463,21 @@ def is_empty(value: str | list | pd.Series, pd_mask: bool = True) -> bool | pd.S
         if not value or value.isspace():
             return True
         lower_stripped = value.lower().strip()
+
+        # Treat serialized placeholder-only lists such as ['N/A'], ["N/A"], or [] as empty.
+        if lower_stripped.startswith("[") and lower_stripped.endswith("]"):
+            parsed_value = None
+            try:
+                parsed_value = json.loads(value)
+            except (json.JSONDecodeError, TypeError, ValueError):
+                try:
+                    parsed_value = ast.literal_eval(value)
+                except (ValueError, SyntaxError):
+                    parsed_value = None
+
+            if isinstance(parsed_value, list):
+                return not parsed_value or all(is_empty(item) for item in parsed_value)
+
         if lower_stripped in [
             "n/a",
             "na",