From 47472c0f329b860cb61aac0d4f964bd5a6c67842 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Mon, 9 Feb 2026 19:12:15 +0000 Subject: [PATCH] Merged in bugfix/fill_claim_type (pull request #874) Bugfix/fill claim type * Handle if AARETE_DERIVED_CLAIM_TYPE_CD comes in as list * add unit tests * Black format Approved-by: Siddhant Medar --- .../postprocessing/postprocessing_funcs.py | 11 +- src/tests/test_postprocess.py | 210 ++++++++++++++++++ 2 files changed, 219 insertions(+), 2 deletions(-) diff --git a/src/pipelines/shared/postprocessing/postprocessing_funcs.py b/src/pipelines/shared/postprocessing/postprocessing_funcs.py index d9da1da..65ca572 100644 --- a/src/pipelines/shared/postprocessing/postprocessing_funcs.py +++ b/src/pipelines/shared/postprocessing/postprocessing_funcs.py @@ -880,8 +880,15 @@ def fill_claim_type_from_title(df: pd.DataFrame) -> pd.DataFrame: file_mask = df["FILE_NAME"] == file_name file_rows = df.loc[file_mask, "AARETE_DERIVED_CLAIM_TYPE_CD"] - # Get non-empty values for this file - non_empty_values = [v for v in file_rows if not string_utils.is_empty(v)] + # Get non-empty values for this file, normalizing lists to strings + non_empty_values = [] + for v in file_rows: + if not string_utils.is_empty(v): + # Convert lists to their first element if present + if isinstance(v, list) and len(v) > 0: + non_empty_values.append(v[0]) + elif not isinstance(v, list): + non_empty_values.append(v) if non_empty_values: # Use mode (most common value) to fill empty rows in this file diff --git a/src/tests/test_postprocess.py b/src/tests/test_postprocess.py index dfc7d57..e5e1e29 100644 --- a/src/tests/test_postprocess.py +++ b/src/tests/test_postprocess.py @@ -5,6 +5,7 @@ import pandas as pd import pytest from src.pipelines.shared.postprocessing.postprocessing_funcs import ( deduplicate_provider_columns, + fill_claim_type_from_title, flatten_singleton_string_list, format_rate_fields_with_commas, normalize_auto_renewal_term, @@ -452,6 +453,215 @@ class TestPostprocessFunctions(unittest.TestCase): result_df7 = deduplicate_provider_columns(input_df7) pd.testing.assert_frame_equal(result_df7, input_df7) + def test_fill_claim_type_from_title_mode_fill(self): + """Test filling empty values using mode from same file.""" + input_df = pd.DataFrame( + { + "FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf", "file2.pdf"], + "AARETE_DERIVED_CLAIM_TYPE_CD": ["M", "M", "", "H"], + "CONTRACT_TITLE": ["Test", "Test", "Test", "Hospital Agreement"], + } + ) + + result_df = fill_claim_type_from_title(input_df) + + # Empty value in file1 should be filled with "M" (mode) + assert result_df.loc[2, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "M" + # file2 value should remain unchanged + assert result_df.loc[3, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "H" + + def test_fill_claim_type_from_title_list_values(self): + """Test handling list values (bug fix scenario).""" + input_df = pd.DataFrame( + { + "FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"], + "AARETE_DERIVED_CLAIM_TYPE_CD": [["M"], ["M"], ""], + "CONTRACT_TITLE": ["Test", "Test", "Test"], + } + ) + + result_df = fill_claim_type_from_title(input_df) + + # Should handle list values and fill empty row with mode + assert result_df.loc[2, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "M" + + def test_fill_claim_type_from_title_professional_keywords(self): + """Test inferring M from professional keywords in CONTRACT_TITLE.""" + input_df = pd.DataFrame( + { + "CONTRACT_TITLE": [ + "Physician Services Agreement", + "Professional Provider Agreement", + "Medical Group Contract", + "Participating Provider Agreement", + ], + "AARETE_DERIVED_CLAIM_TYPE_CD": ["", "", "", ""], + } + ) + + result_df = fill_claim_type_from_title(input_df) + + # All should be inferred as "M" + assert all(result_df["AARETE_DERIVED_CLAIM_TYPE_CD"] == "M") + + def test_fill_claim_type_from_title_ancillary_keywords(self): + """Test inferring M from ancillary keywords in CONTRACT_TITLE.""" + input_df = pd.DataFrame( + { + "CONTRACT_TITLE": [ + "Home Health Services Agreement", + "DME Provider Contract", + "Laboratory Services Agreement", + "Behavioral Health Agreement", + "Ambulatory Surgical Center", + ], + "AARETE_DERIVED_CLAIM_TYPE_CD": ["", "", "", "", ""], + } + ) + + result_df = fill_claim_type_from_title(input_df) + + # All should be inferred as "M" + assert all(result_df["AARETE_DERIVED_CLAIM_TYPE_CD"] == "M") + + def test_fill_claim_type_from_title_institutional_keywords(self): + """Test inferring H from institutional keywords in CONTRACT_TITLE.""" + input_df = pd.DataFrame( + { + "CONTRACT_TITLE": [ + "Hospital Services Agreement", + "Institutional Provider Contract", + "Facility Agreement", + "Inpatient Services", + ], + "AARETE_DERIVED_CLAIM_TYPE_CD": ["", "", "", ""], + } + ) + + result_df = fill_claim_type_from_title(input_df) + + # All should be inferred as "H" + assert all(result_df["AARETE_DERIVED_CLAIM_TYPE_CD"] == "H") + + def test_fill_claim_type_from_title_case_insensitive(self): + """Test that keyword matching is case-insensitive.""" + input_df = pd.DataFrame( + { + "CONTRACT_TITLE": [ + "physician services agreement", + "HOSPITAL AGREEMENT", + "AnCiLlArY sErViCeS", + ], + "AARETE_DERIVED_CLAIM_TYPE_CD": ["", "", ""], + } + ) + + result_df = fill_claim_type_from_title(input_df) + + assert result_df.loc[0, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "M" + assert result_df.loc[1, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "H" + assert result_df.loc[2, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "M" + + def test_fill_claim_type_from_title_preserve_existing(self): + """Test that existing non-empty values are preserved.""" + input_df = pd.DataFrame( + { + "CONTRACT_TITLE": ["Hospital Agreement", "Physician Agreement"], + "AARETE_DERIVED_CLAIM_TYPE_CD": [ + "M", + "H", + ], # Opposite of what keywords would infer + } + ) + + result_df = fill_claim_type_from_title(input_df) + + # Should preserve existing values even if they don't match keywords + assert result_df.loc[0, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "M" + assert result_df.loc[1, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "H" + + def test_fill_claim_type_from_title_no_match(self): + """Test that no value is set when no keywords match.""" + input_df = pd.DataFrame( + { + "CONTRACT_TITLE": ["Generic Contract", "Some Agreement"], + "AARETE_DERIVED_CLAIM_TYPE_CD": ["", ""], + } + ) + + result_df = fill_claim_type_from_title(input_df) + + # Should remain empty when no keywords match + assert result_df.loc[0, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "" + assert result_df.loc[1, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "" + + def test_fill_claim_type_from_title_missing_column(self): + """Test that function handles missing AARETE_DERIVED_CLAIM_TYPE_CD column.""" + input_df = pd.DataFrame( + { + "CONTRACT_TITLE": ["Physician Agreement"], + } + ) + + result_df = fill_claim_type_from_title(input_df) + + # Should create the column and infer value + assert "AARETE_DERIVED_CLAIM_TYPE_CD" in result_df.columns + assert result_df.loc[0, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "M" + + def test_fill_claim_type_from_title_missing_contract_title(self): + """Test that function handles missing CONTRACT_TITLE column.""" + input_df = pd.DataFrame( + { + "FILE_NAME": ["file1.pdf"], + "AARETE_DERIVED_CLAIM_TYPE_CD": [""], + } + ) + + result_df = fill_claim_type_from_title(input_df) + + # Should return df unchanged (can't infer without CONTRACT_TITLE) + assert result_df.loc[0, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "" + + def test_fill_claim_type_from_title_mixed_scenario(self): + """Test combined mode filling and keyword inference.""" + input_df = pd.DataFrame( + { + "FILE_NAME": ["file1.pdf", "file1.pdf", "file2.pdf", "file2.pdf"], + "CONTRACT_TITLE": [ + "Test", + "Test", + "Physician Agreement", + "Generic", + ], + "AARETE_DERIVED_CLAIM_TYPE_CD": ["H", "", "", ""], + } + ) + + result_df = fill_claim_type_from_title(input_df) + + # Row 1 should be filled with "H" (mode from file1) + assert result_df.loc[1, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "H" + # Row 2 should be inferred as "M" from keyword + assert result_df.loc[2, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "M" + # Row 3 should remain empty (no mode in file2, no keyword match) + assert result_df.loc[3, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "" + + def test_fill_claim_type_from_title_empty_list_values(self): + """Test handling empty list values.""" + input_df = pd.DataFrame( + { + "FILE_NAME": ["file1.pdf", "file1.pdf"], + "AARETE_DERIVED_CLAIM_TYPE_CD": [[], "M"], + "CONTRACT_TITLE": ["Physician Agreement", "Test"], + } + ) + + result_df = fill_claim_type_from_title(input_df) + + # Empty list should be treated as empty and filled with mode or inferred + assert result_df.loc[0, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "M" + if __name__ == "__main__": unittest.main()