diff --git a/src/crosswalk/crosswalk_builder.py b/src/crosswalk/crosswalk_builder.py index 969573b..dd1c87b 100644 --- a/src/crosswalk/crosswalk_builder.py +++ b/src/crosswalk/crosswalk_builder.py @@ -114,8 +114,28 @@ class CrosswalkBuilder: for state in STATE: mapping.update(data.get("state_mapping", {}).get(state, {})) if "client_mapping" in data: - for client in CLIENT: - mapping.update(data.get("client_mapping", {}).get(client, {})) + client_mapping = data.get("client_mapping", {}) + clients = [ + str(c).strip() + for c in CLIENT + if c is not None and str(c).strip().lower() != "none" + ] + + if clients: + for client in clients: + mapping.update(client_mapping.get(client, {})) + else: + # No explicit client provided: merge all client mappings with deduplication. + # Keep first value for duplicate keys. + merged_client_mapping: dict[str, str] = {} + for client_map in client_mapping.values(): + if not isinstance(client_map, dict): + continue + + for key, value in client_map.items(): + merged_client_mapping.setdefault(key, value) + + mapping.update(merged_client_mapping) self.mapping = mapping diff --git a/src/pipelines/shared/postprocessing/aarete_derived.py b/src/pipelines/shared/postprocessing/aarete_derived.py index 4d91555..020903d 100644 --- a/src/pipelines/shared/postprocessing/aarete_derived.py +++ b/src/pipelines/shared/postprocessing/aarete_derived.py @@ -138,6 +138,8 @@ def fill_na_mapping(answer_dicts): # Set the merged, deduplicated value if all_lob_values: answer_dict["AARETE_DERIVED_LOB"] = sorted(all_lob_values) + if string_utils.is_empty(answer_dict.get("LOB")): + answer_dict["LOB"] = sorted(all_lob_values) # Set relationship flags if values came from PROGRAM or PRODUCT # Only set if field is empty or "N/A" (preserve LLM-determined Inclusive/Exclusive) if program_lob_values: diff --git a/src/tests/test_crosswalk_utils.py b/src/tests/test_crosswalk_utils.py index b6ea656..5cd1888 100644 --- a/src/tests/test_crosswalk_utils.py +++ b/src/tests/test_crosswalk_utils.py @@ -128,6 +128,28 @@ def test_to_csv_no_metadata(tmp_path): assert result_dict == {"A": "one", "B": "two"} +def test_from_json_client_none_merges_all_client_mappings(tmp_path, monkeypatch): + data = { + "metadata": {"from_col": "from_col", "to_col": "to_col"}, + "mapping": {}, + "client_mapping": { + "ClientA": {"Connexus": "Commercial", "Synergy": "Commercial"}, + "ClientB": {"HEALTHfirst": "Medicaid"}, + }, + } + file_path = tmp_path / "client_merge.json" + with open(file_path, "w") as f: + json.dump(data, f) + + monkeypatch.setattr("src.crosswalk.crosswalk_builder.CLIENT", ["None"]) + monkeypatch.setattr("src.crosswalk.crosswalk_builder.STATE", ["None"]) + + builder = CrosswalkBuilder().from_json(file_path) + assert builder.mapping["Connexus"] == "Commercial" + assert builder.mapping["Synergy"] == "Commercial" + assert builder.mapping["HEALTHfirst"] == "Medicaid" + + def test_apply_crosswalk(): mapping = {"A": "1", "B": "2", "C": "3"} assert apply_crosswalk("A", mapping) == ["1"] diff --git a/src/tests/test_postprocess.py b/src/tests/test_postprocess.py index 195fa7f..ebd227e 100644 --- a/src/tests/test_postprocess.py +++ b/src/tests/test_postprocess.py @@ -24,6 +24,7 @@ from src.pipelines.shared.postprocessing.postprocessing_funcs import ( standardize_reimb_method_and_fee_schedule, validate_and_reformat_date, ) +from src.pipelines.shared.postprocessing import aarete_derived from src.pipelines.shared.postprocessing.postprocess import standard_postprocess from src.constants.constants import Constants @@ -957,6 +958,25 @@ class TestPostprocessFunctions(unittest.TestCase): self.assertEqual(result_df.loc[0, "CONTRACT_TITLE"], "") self.assertEqual(result_df.loc[1, "CONTRACT_TITLE"], "Another Valid") + def test_fill_na_mapping_derives_lob_from_aarete_derived_product(self): + """fill_na_mapping should derive LOB from AARETE_DERIVED_PRODUCT.""" + rows = [ + { + "PRODUCT": "", + "AARETE_DERIVED_PRODUCT": '["Connexus", "Synergy"]', + "AARETE_DERIVED_LOB": "", + "LOB": "", + "AARETE_DERIVED_PROGRAM": "", + "LOB_PROGRAM_RELATIONSHIP": "", + "LOB_PRODUCT_RELATIONSHIP": "", + } + ] + + result = aarete_derived.fill_na_mapping(rows) + + self.assertEqual(result[0]["AARETE_DERIVED_LOB"], ["Commercial"]) + self.assertEqual(result[0]["LOB"], ["Commercial"]) + class TestStandardizeReimbMethodAndFeeScheduleUOM(unittest.TestCase): """Tests for UNIT_OF_MEASURE post-processing when DEFAULT_IND='Y' and flat rate.""" diff --git a/src/tests/test_string_utils.py b/src/tests/test_string_utils.py index ab3d531..b17ac09 100644 --- a/src/tests/test_string_utils.py +++ b/src/tests/test_string_utils.py @@ -153,6 +153,10 @@ class TestStringUtils(unittest.TestCase): def test_is_empty_string(self): self.assertTrue(string_utils.is_empty("")) self.assertTrue(string_utils.is_empty("N/A")) + self.assertTrue(string_utils.is_empty("['N/A']")) + self.assertTrue(string_utils.is_empty('["N/A"]')) + self.assertTrue(string_utils.is_empty("[]")) + self.assertFalse(string_utils.is_empty('["Medicaid", "N/A"]')) self.assertFalse(string_utils.is_empty("valid")) def test_is_empty_list(self): diff --git a/src/utils/string_utils.py b/src/utils/string_utils.py index 9099d5f..72a55d9 100644 --- a/src/utils/string_utils.py +++ b/src/utils/string_utils.py @@ -463,6 +463,21 @@ def is_empty(value: str | list | pd.Series, pd_mask: bool = True) -> bool | pd.S if not value or value.isspace(): return True lower_stripped = value.lower().strip() + + # Treat serialized placeholder-only lists such as ['N/A'], ["N/A"], or [] as empty. + if lower_stripped.startswith("[") and lower_stripped.endswith("]"): + parsed_value = None + try: + parsed_value = json.loads(value) + except (json.JSONDecodeError, TypeError, ValueError): + try: + parsed_value = ast.literal_eval(value) + except (ValueError, SyntaxError): + parsed_value = None + + if isinstance(parsed_value, list): + return not parsed_value or all(is_empty(item) for item in parsed_value) + if lower_stripped in [ "n/a", "na",