diff --git a/src/codes/code_funcs.py b/src/codes/code_funcs.py index 66dc380..4b03a10 100644 --- a/src/codes/code_funcs.py +++ b/src/codes/code_funcs.py @@ -760,30 +760,6 @@ def extract_codes_from_service(answer_dict, constants: Constants): return normalize_answer_dict_codes(answer_dict) -def fill_claim_type(answer_dicts): - """ - Fills in the AARETE_DERIVED_CLAIM_TYPE_CD field in each answer_dict with the mode of the existing values. - If the field is already populated, it remains unchanged. - Args: - answer_dicts (list[dict]): List of dictionaries containing 1:1 and 1:N fields (after merge process) - Returns: - list[dict]: List of dictionaries with AARETE_DERIVED_CLAIM_TYPE - """ - claim_types = [ - d.get("AARETE_DERIVED_CLAIM_TYPE_CD") - for d in answer_dicts - if d.get("AARETE_DERIVED_CLAIM_TYPE_CD") - ] - if not claim_types: - return answer_dicts - - mode_claim_type = max(set(claim_types), key=claim_types.count) - for answer_dict in answer_dicts: - if not answer_dict.get("AARETE_DERIVED_CLAIM_TYPE_CD"): - answer_dict["AARETE_DERIVED_CLAIM_TYPE_CD"] = mode_claim_type - return answer_dicts - - def code_breakout(merged_results: pd.DataFrame, constants: Constants): """ Processes a list of answer dictionaries to extract and fill in code-related information. @@ -796,8 +772,7 @@ def code_breakout(merged_results: pd.DataFrame, constants: Constants): """ import concurrent.futures - # Fill empty AARETE_DERIVED_CLAIM_TYPE_CD with mode - answer_dicts = fill_claim_type(merged_results.to_dict(orient="records")) + answer_dicts = merged_results.to_dict(orient="records") # Parallelize code extraction def process_single_code(answer_dict): diff --git a/src/pipelines/shared/postprocessing/postprocess.py b/src/pipelines/shared/postprocessing/postprocess.py index b0252f0..0fa87a5 100644 --- a/src/pipelines/shared/postprocessing/postprocess.py +++ b/src/pipelines/shared/postprocessing/postprocess.py @@ -82,8 +82,6 @@ def postprocess(df, constants: Constants): df = postprocessing_funcs.update_grouper_base_rate_and_grouper_pct_rate(df) - df = postprocessing_funcs.fill_empty_dynamic(df) - df = postprocessing_funcs.attach_sid_column(df) # Standardize output column order - this should ALWAYS be the final postprocessing step diff --git a/src/pipelines/shared/postprocessing/postprocessing_funcs.py b/src/pipelines/shared/postprocessing/postprocessing_funcs.py index 5867ba8..61107b9 100644 --- a/src/pipelines/shared/postprocessing/postprocessing_funcs.py +++ b/src/pipelines/shared/postprocessing/postprocessing_funcs.py @@ -764,116 +764,6 @@ def update_grouper_base_rate_and_grouper_pct_rate(df): return df -def fill_empty_dynamic(df): - """ - For specified columns, fills NA values with the common value from the same EXHIBIT_PAGE group. - - If all rows with the same EXHIBIT_PAGE value have only one unique non-NA value for a column, - this function fills any NA values in that column with that unique value. - - Args: - df (pd.DataFrame): DataFrame with at least an EXHIBIT_PAGE column - - Returns: - pd.DataFrame: DataFrame with filled values - """ - if "EXHIBIT_PAGE" not in df.columns or df.empty: - return df - - # List of columns to check for filling - columns_to_fill = [ - "AARETE_DERIVED_LOB", - "AARETE_DERIVED_PROGRAM", - "AARETE_DERIVED_PRODUCT", - "AARETE_DERIVED_NETWORK", - ] - - # Only process columns that actually exist in the dataframe - columns_to_fill = [col for col in columns_to_fill if col in df.columns] - - # Make a copy to avoid SettingWithCopyWarning - result_df = df.copy() - - # Check if AARETE_DERIVED_LOB column exists - has_lob_column = "AARETE_DERIVED_LOB" in result_df.columns - - # First, handle AARETE_DERIVED_LOB by grouping on FILE_NAME and EXHIBIT_PAGE only - if has_lob_column: - for (file_name, exhibit_page), group in result_df.groupby( - ["FILE_NAME", "EXHIBIT_PAGE"] - ): - # Get non-NA LOB values - column = "AARETE_DERIVED_LOB" - non_na_values = [ - val for val in group[column].unique() if not string_utils.is_empty(val) - ] - - # If there's exactly one unique non-NA value, fill NA values with it - if len(non_na_values) == 1: - fill_value = non_na_values[0] - # Only apply to rows with this FILE_NAME and EXHIBIT_PAGE combination - mask = ( - (result_df["FILE_NAME"] == file_name) - & (result_df["EXHIBIT_PAGE"] == exhibit_page) - & result_df[column].apply(string_utils.is_empty) - ) - result_df.loc[mask, column] = fill_value - - # Then, handle other columns by grouping on FILE_NAME, EXHIBIT_PAGE, and AARETE_DERIVED_LOB (if it exists) - if has_lob_column: - for (file_name, exhibit_page, lob), group in result_df.groupby( - ["FILE_NAME", "EXHIBIT_PAGE", "AARETE_DERIVED_LOB"] - ): - for column in columns_to_fill: - # Skip AARETE_DERIVED_LOB as we already processed it - if column == "AARETE_DERIVED_LOB": - continue - - # Get non-NA values - non_na_values = [ - val - for val in group[column].unique() - if not string_utils.is_empty(val) - ] - - # If there's exactly one unique non-NA value, fill NA values with it - if len(non_na_values) == 1: - fill_value = non_na_values[0] - # Only apply to rows with this FILE_NAME, EXHIBIT_PAGE, and LOB combination - mask = ( - (result_df["FILE_NAME"] == file_name) - & (result_df["EXHIBIT_PAGE"] == exhibit_page) - & (result_df["AARETE_DERIVED_LOB"] == lob) - & result_df[column].apply(string_utils.is_empty) - ) - result_df.loc[mask, column] = fill_value - else: - # No LOB column, just group by FILE_NAME and EXHIBIT_PAGE - for (file_name, exhibit_page), group in result_df.groupby( - ["FILE_NAME", "EXHIBIT_PAGE"] - ): - for column in columns_to_fill: - # Get non-NA values - non_na_values = [ - val - for val in group[column].unique() - if not string_utils.is_empty(val) - ] - - # If there's exactly one unique non-NA value, fill NA values with it - if len(non_na_values) == 1: - fill_value = non_na_values[0] - # Only apply to rows with this FILE_NAME and EXHIBIT_PAGE combination - mask = ( - (result_df["FILE_NAME"] == file_name) - & (result_df["EXHIBIT_PAGE"] == exhibit_page) - & result_df[column].apply(string_utils.is_empty) - ) - result_df.loc[mask, column] = fill_value - - return result_df - - def attach_sid_column(df: pd.DataFrame) -> pd.DataFrame: """ Ensures the given AARETE_DERIVED_SID columns exist in the DataFrame, fills missing values, diff --git a/src/tests/test_code_funcs.py b/src/tests/test_code_funcs.py index 5802c2c..0551b1b 100644 --- a/src/tests/test_code_funcs.py +++ b/src/tests/test_code_funcs.py @@ -399,23 +399,6 @@ class TestCodeFuncs(unittest.TestCase): ) self.assertEqual(result["CODE_METHODOLOGY"], "Generic - No Match") - def test_fill_claim_type(self): - """Test the fill_claim_type function for determining claim types.""" - # Test with existing claim types - answer_dicts = [ - {"AARETE_DERIVED_CLAIM_TYPE_CD": "H"}, - {"AARETE_DERIVED_CLAIM_TYPE_CD": "H"}, - {"AARETE_DERIVED_CLAIM_TYPE_CD": "M"}, - {}, - ] - result = code_funcs.fill_claim_type(answer_dicts) - self.assertEqual(result[3]["AARETE_DERIVED_CLAIM_TYPE_CD"], "H") - - # Test with all empty claim types - answer_dicts = [{}, {}] - result = code_funcs.fill_claim_type(answer_dicts) - self.assertEqual(result, answer_dicts) - @patch("src.codes.code_funcs.fill_claim_type") @patch("src.codes.code_funcs.extract_codes_from_service") def test_code_breakout(self, mock_extract_codes, mock_fill_claim_type): diff --git a/src/tests/test_postprocess.py b/src/tests/test_postprocess.py index 339db12..ee7300f 100644 --- a/src/tests/test_postprocess.py +++ b/src/tests/test_postprocess.py @@ -15,7 +15,6 @@ from src.pipelines.shared.postprocessing.postprocessing_funcs import ( remove_redundant_reimb_info, rename_columns, validate_and_reformat_date, - fill_empty_dynamic, ) @@ -433,167 +432,6 @@ class TestPostprocessFunctions(unittest.TestCase): result_df7 = deduplicate_provider_columns(input_df7) pd.testing.assert_frame_equal(result_df7, input_df7) - def test_fill_empty_dynamic(self): - """Tests the fill_empty_dynamic function that fills NA values with common values from the same EXHIBIT_PAGE group. - - Tests: - 1. Basic filling - fills NA values with the common value for the same EXHIBIT_PAGE - 2. Multiple columns - correctly fills multiple columns independently - 3. Multiple file/page combinations - respects FILE_NAME and EXHIBIT_PAGE boundaries - 4. No common value - doesn't fill when multiple non-NA values exist - 5. All NA values - doesn't fill when all values are NA - 6. Missing columns - returns unchanged DataFrame when key columns are missing - """ - # Test case 1: Basic filling for a single column - input_df1 = pd.DataFrame( - { - "FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"], - "EXHIBIT_PAGE": ["1.0", "1.0", "1.0"], - "AARETE_DERIVED_LOB": ["Commercial", "", None], - } - ) - - expected_df1 = pd.DataFrame( - { - "FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"], - "EXHIBIT_PAGE": ["1.0", "1.0", "1.0"], - "AARETE_DERIVED_LOB": ["Commercial", "Commercial", "Commercial"], - } - ) - - result_df1 = fill_empty_dynamic(input_df1) - pd.testing.assert_frame_equal(result_df1, expected_df1) - - # Test case 2: Multiple columns - input_df2 = pd.DataFrame( - { - "FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"], - "EXHIBIT_PAGE": ["1.0", "1.0", "1.0"], - "AARETE_DERIVED_LOB": ["Commercial", "", None], - "AARETE_DERIVED_PRODUCT": ["Product A", None, ""], - } - ) - - expected_df2 = pd.DataFrame( - { - "FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"], - "EXHIBIT_PAGE": ["1.0", "1.0", "1.0"], - "AARETE_DERIVED_LOB": ["Commercial", "Commercial", "Commercial"], - "AARETE_DERIVED_PRODUCT": ["Product A", "Product A", "Product A"], - } - ) - - result_df2 = fill_empty_dynamic(input_df2) - pd.testing.assert_frame_equal(result_df2, expected_df2) - - # Test case 3: Multiple file/page combinations - input_df3 = pd.DataFrame( - { - "FILE_NAME": [ - "file1.pdf", - "file1.pdf", - "file1.pdf", - "file1.pdf", - "file2.pdf", - "file2.pdf", - ], - "EXHIBIT_PAGE": ["1.0", "1.0", "2.0", "2.0", "1.0", "1.0"], - "AARETE_DERIVED_LOB": [ - "Commercial", - "", - "Medicare", - None, - "Medicaid", - "", - ], - } - ) - - expected_df3 = pd.DataFrame( - { - "FILE_NAME": [ - "file1.pdf", - "file1.pdf", - "file1.pdf", - "file1.pdf", - "file2.pdf", - "file2.pdf", - ], - "EXHIBIT_PAGE": ["1.0", "1.0", "2.0", "2.0", "1.0", "1.0"], - "AARETE_DERIVED_LOB": [ - "Commercial", - "Commercial", - "Medicare", - "Medicare", - "Medicaid", - "Medicaid", - ], - } - ) - - result_df3 = fill_empty_dynamic(input_df3) - pd.testing.assert_frame_equal(result_df3, expected_df3) - - # Test case 4: No common value (multiple non-NA values exist) - input_df4 = pd.DataFrame( - { - "FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"], - "EXHIBIT_PAGE": ["1.0", "1.0", "1.0"], - "AARETE_DERIVED_LOB": ["Commercial", "Medicare", None], - } - ) - - expected_df4 = pd.DataFrame( - { - "FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"], - "EXHIBIT_PAGE": ["1.0", "1.0", "1.0"], - "AARETE_DERIVED_LOB": [ - "Commercial", - "Medicare", - None, - ], # Should remain unchanged - } - ) - - result_df4 = fill_empty_dynamic(input_df4) - pd.testing.assert_frame_equal(result_df4, expected_df4) - - # Test case 5: All NA values - input_df5 = pd.DataFrame( - { - "FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"], - "EXHIBIT_PAGE": ["1.0", "1.0", "1.0"], - "AARETE_DERIVED_LOB": [None, "", None], - } - ) - - expected_df5 = pd.DataFrame( - { - "FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"], - "EXHIBIT_PAGE": ["1.0", "1.0", "1.0"], - "AARETE_DERIVED_LOB": [None, "", None], # Should remain unchanged - } - ) - - result_df5 = fill_empty_dynamic(input_df5) - pd.testing.assert_frame_equal(result_df5, expected_df5) - - # Test case 6: Missing EXHIBIT_PAGE column - input_df6 = pd.DataFrame( - { - "FILE_NAME": ["file1.pdf", "file1.pdf"], - "AARETE_DERIVED_LOB": ["Commercial", None], - } - ) - - result_df6 = fill_empty_dynamic(input_df6) - pd.testing.assert_frame_equal(result_df6, input_df6) # Should remain unchanged - - # Test case 7: Empty DataFrame - empty_df = pd.DataFrame() - result_empty = fill_empty_dynamic(empty_df) - pd.testing.assert_frame_equal(result_empty, empty_df) # Should remain unchanged - if __name__ == "__main__": unittest.main()