Merged in bugfix/mi-dynamic (pull request #726)

Bugfix/mi dynamic * Refactor - split out one_to_n_cleaning * Rework postprocess * Update postprocessing - fill dynamic values if there is only 1 non-empty value * Update to use string_utils * Group by AARETE_DERIVED_LOB as well * Update crosswalk mapping to handle | lists * Update constants - run with MI flag Approved-by: Siddhant Medar
2025-10-07 15:06:10 +00:00
parent 354bb1c9ce
commit a910f27cd2
7 changed files with 292 additions and 105 deletions
@@ -144,7 +144,7 @@
        },
        "MI" : {
            "Healthy Michigan Plan" : "HMIP",
-            "MIChild" : "CHIP"
+            "MI Child" : "CHIP"
        },
        "MN" : {
            "Minnesota Medical Assistance" : "MNMA",
@@ -59,7 +59,6 @@ def fill_na_mapping(answer_dicts):

 def get_crosswalk_fields(answer_dicts: list, constants: Constants):
    crosswalk_fields = FieldSet(file_path=config.FIELD_JSON_PATH, crosswalk=True)
-
    for to_field in crosswalk_fields.fields:
        to_field_name, from_field_name = to_field.field_name, to_field.base_field
        # Find crosswalk
@@ -73,16 +72,18 @@ def get_crosswalk_fields(answer_dicts: list, constants: Constants):
                if not string_utils.is_empty(
                    from_field_value
                ) and string_utils.is_empty(to_field_value):
-                    if from_field_value in crosswalk.mapping.keys():
-                        answer_dict[to_field_name] = crosswalk.mapping.get(
-                            from_field_value
-                        )
-                    elif from_field_value in crosswalk.mapping.values():
-                        if "AARETE_DERIVED" in to_field_name:
-                            answer_dict[to_field_name] = from_field_value
-                        else:
-                            answer_dict[to_field_name] = (
-                                crosswalk.create_reverse_mapping().get(from_field_value)
-                            )
+                    from_field_value_list = from_field_value.split("|")
+                    to_field_answer_list = []
+                    for individual_from_field_value in from_field_value_list:
+                        if individual_from_field_value in crosswalk.mapping.keys():
+                            to_field_answer_list.append(crosswalk.mapping.get(
+                                individual_from_field_value
+                            ))
+                        elif from_field_value in crosswalk.mapping.values():
+                            if "AARETE_DERIVED" in to_field_name:
+                                to_field_answer_list.append(individual_from_field_value)
+                            else:
+                                to_field_answer_list.append(crosswalk.create_reverse_mapping().get(individual_from_field_value))
+                    answer_dict[to_field_name] = "|".join(to_field_answer_list)

    return answer_dicts
@@ -218,24 +218,8 @@ def run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, constants)
            filename,
        )  # returns list of dicts

-        ################################ Crosswalk Fields ################################
-        all_exhibit_rows = aarete_derived.get_crosswalk_fields(
-            all_exhibit_rows, constants
-        )
-
-        ################################ Determine LOB Relationship ################################
-        all_exhibit_rows = one_to_n_funcs.get_lob_relationship(
-            all_exhibit_rows, exhibit_text, filename
-        )
-
-        ################################ Fill NA Mapping ################################
-        all_exhibit_rows = aarete_derived.fill_na_mapping(all_exhibit_rows)
-
-        ################################ Update LOB for Duals ################################
-        all_exhibit_rows = postprocessing_funcs.update_lob_for_duals(all_exhibit_rows)
-
-        ################################ Split REIMB_DATES ################################
-        all_exhibit_rows = one_to_n_funcs.split_reimb_dates(all_exhibit_rows, filename)
+        ################################ Mapping and Cleaning ###############################
+        all_exhibit_rows = one_to_n_funcs.one_to_n_cleaning(all_exhibit_rows, exhibit_text, constants, filename)

        ################################ Add to Total ###############################
        one_to_n_results += all_exhibit_rows
@@ -1,11 +1,8 @@
 import logging

-import src.investment.dynamic_funcs as dynamic_funcs
-import src.investment.lesser_of_funcs as lesser_of_funcs
-import src.investment.prompt_calls as prompt_calls
+from src.investment import dynamic_funcs, lesser_of_funcs, prompt_calls, aarete_derived, postprocessing_funcs
+from src.utils import llm_utils, string_utils
 import src.prompts.prompt_templates as prompt_templates
-import src.utils.llm_utils as llm_utils
-import src.utils.string_utils as string_utils
 from constants.constants import Constants
 from src import config
 from src.prompts.fieldset import FieldSet
@@ -200,7 +197,7 @@ def clean_reimbursement_primary(
            filtered_answers, seen_pairs, exhibit_page, filename
        )

-        # Step 4: Apply exhibit lesser-of statement and rate_escalator statement if it exists
+        # Step 4: Apply exhibit lesser-of statement if it exists
        if not string_utils.is_empty(exhibit_lesser_of) and deduplicated_answers:
            deduplicated_answers = lesser_of_funcs.apply_exhibit_lesser_of(
                deduplicated_answers, exhibit_lesser_of, filename
@@ -818,3 +815,28 @@ def get_lob_relationship(answer_dicts, exhibit_text, filename):
            )

    return answer_dicts
+
+def one_to_n_cleaning(all_exhibit_rows: list[dict], exhibit_text: str, constants: Constants, filename: str):
+
+    ################################ Crosswalk Fields ################################
+    all_exhibit_rows = aarete_derived.get_crosswalk_fields(
+        all_exhibit_rows, constants
+    )
+
+    ################################ Determine LOB Relationship ################################
+    all_exhibit_rows = get_lob_relationship(
+        all_exhibit_rows, exhibit_text, filename
+    )
+
+    ################################ Fill NA Mapping ################################
+    all_exhibit_rows = aarete_derived.fill_na_mapping(all_exhibit_rows)
+
+    ################################ Update LOB for Duals ################################
+    all_exhibit_rows = postprocessing_funcs.update_lob_for_duals(all_exhibit_rows)
+
+    ################################ Split REIMB_DATES ################################
+    all_exhibit_rows = split_reimb_dates(all_exhibit_rows, filename)
+
+    return all_exhibit_rows
+
+
@@ -5,86 +5,91 @@ from src.investment import postprocessing_funcs


 def postprocess(df, constants: Constants):
-    if df.shape[0] > 0:
+    if df.shape[0] == 0:
+        return df

-        df["CLIENT_NAME"] = config.CLIENT_NAME

-        # Rename columns
-        df = postprocessing_funcs.rename_columns(df)
+    df["CLIENT_NAME"] = config.CLIENT_NAME

-        # Sort to ensure consistent ordering
-        sort_columns = ["FILE_NAME"]
-        if "EXHIBIT_PAGE" in df.columns:
-            sort_columns.append("EXHIBIT_PAGE")
-        if "REIMB_LESSER_OF_ID" in df.columns:
-            sort_columns.append("REIMB_LESSER_OF_ID")
-        df = df.sort_values(by=sort_columns).reset_index(drop=True)
+    # Rename columns
+    df = postprocessing_funcs.rename_columns(df)

-        # Postprocess date fields
-        df = postprocessing_funcs.date_postprocess(df, config.FIELD_JSON_PATH)
+    # Sort to ensure consistent ordering
+    sort_columns = ["FILE_NAME"]
+    if "EXHIBIT_PAGE" in df.columns:
+        sort_columns.append("EXHIBIT_PAGE")
+    if "REIMB_LESSER_OF_ID" in df.columns:
+        sort_columns.append("REIMB_LESSER_OF_ID")
+    df = df.sort_values(by=sort_columns).reset_index(drop=True)

-        # Format rate fields
-        if "REIMB_FEE_RATE" in df.columns:
-            df["REIMB_FEE_RATE"] = df["REIMB_FEE_RATE"].apply(
-                postprocessing_funcs.format_rate_fields_with_commas
-            )
-        if "REIMB_PCT_RATE" in df.columns:
-            df["REIMB_PCT_RATE"] = df["REIMB_PCT_RATE"].apply(
-                postprocessing_funcs.format_rate_fields_with_commas
-            )
+    # Postprocess date fields
+    df = postprocessing_funcs.date_postprocess(df, config.FIELD_JSON_PATH)

-        for col in df.columns:
-            if "_IND" in col:
-                df[col] = df[col].apply(postprocessing_funcs.normalize_indicator_field)
-            # Normalize _IND fields and clean up TIN/NPI fields
-            if "TIN" in col or "NPI" in col:
-                df[col] = df[col].apply(postprocessing_funcs.remove_hyphens)
-            # Apply the flatten_singleton_string_list function to the 'CPT' column
-            if "_CD" in col and "CPT" not in col:
-                df[col] = df[col].apply(
-                    postprocessing_funcs.flatten_singleton_string_list
-                )
-            # Convert DATE to YYYY/MM/DD format
-            if "_DT" in col or "DATE" in col:
-                df[col] = df[col].apply(postprocessing_funcs.validate_and_reformat_date)
-            if "CPT" in col:
-                df[col] = df[col].apply(postprocessing_funcs.normalize_cpt_fields)
-
-        # Deduplicate provider fields - remove GROUP values from OTHER fields and deduplicate OTHER lists
-        df = postprocessing_funcs.deduplicate_provider_columns(df)
-
-        # Normalize the 'AUTO_RENEWAL_TERM' column
-        df = postprocessing_funcs.auto_renewal(df)
-
-        # Check conditions for TERMINATION_DT, AUTO_RENEWAL_IND, and AARETE_DERIVED_TERMINATION_DT
-        df = postprocessing_funcs.update_termination_date_for_conditions(df)
-
-        # Standardize the 'AARETE_DERIVED_REIMB_METHOD' column and updates the 'AARETE_DERIVED_FEE_SCHEDULE' column based on specific values.
-        df = postprocessing_funcs.standardize_reimb_method_and_fee_schedule(
-            df, constants.VALID_UNIT_OF_MEASURE
+    # Format rate fields
+    if "REIMB_FEE_RATE" in df.columns:
+        df["REIMB_FEE_RATE"] = df["REIMB_FEE_RATE"].apply(
+            postprocessing_funcs.format_rate_fields_with_commas
+        )
+    if "REIMB_PCT_RATE" in df.columns:
+        df["REIMB_PCT_RATE"] = df["REIMB_PCT_RATE"].apply(
+            postprocessing_funcs.format_rate_fields_with_commas
        )

-        # update reimb_pct_rate
-        df = postprocessing_funcs.fill_empty_reimb_pct_rate(df)
+    for col in df.columns:
+        if "_IND" in col:
+            df[col] = df[col].apply(postprocessing_funcs.normalize_indicator_field)
+        # Normalize _IND fields and clean up TIN/NPI fields
+        if "TIN" in col or "NPI" in col:
+            df[col] = df[col].apply(postprocessing_funcs.remove_hyphens)
+        # Apply the flatten_singleton_string_list function to the 'CPT' column
+        if "_CD" in col and "CPT" not in col:
+            df[col] = df[col].apply(
+                postprocessing_funcs.flatten_singleton_string_list
+            )
+        # Convert DATE to YYYY/MM/DD format
+        if "_DT" in col or "DATE" in col:
+            df[col] = df[col].apply(postprocessing_funcs.validate_and_reformat_date)
+        if "CPT" in col:
+            df[col] = df[col].apply(postprocessing_funcs.normalize_cpt_fields)

-        df = postprocessing_funcs.remove_redundant_reimb_info(df)
+    # Deduplicate provider fields - remove GROUP values from OTHER fields and deduplicate OTHER lists
+    df = postprocessing_funcs.deduplicate_provider_columns(df)

-        df = postprocessing_funcs.add_aarete_derived_amendment_num(df)
+    # Normalize the 'AUTO_RENEWAL_TERM' column
+    df = postprocessing_funcs.auto_renewal(df)

-        # Process PATIENT_AGE_RANGE into PATIENT_AGE_MIN and PATIENT_AGE_MAX
-        df = postprocessing_funcs.process_patient_age_range(df)
+    # Check conditions for TERMINATION_DT, AUTO_RENEWAL_IND, and AARETE_DERIVED_TERMINATION_DT
+    df = postprocessing_funcs.update_termination_date_for_conditions(df)

-        # Add AARETE_DERIVED_PRODUCT
-        df = postprocessing_funcs.add_aarete_derived_product(df)
+    # Standardize the 'AARETE_DERIVED_REIMB_METHOD' column and updates the 'AARETE_DERIVED_FEE_SCHEDULE' column based on specific values.
+    df = postprocessing_funcs.standardize_reimb_method_and_fee_schedule(
+        df, constants.VALID_UNIT_OF_MEASURE
+    )

-        # Add GREATER_OF_IND
-        df["GREATER_OF_IND"] = "N"
+    # update reimb_pct_rate
+    df = postprocessing_funcs.fill_empty_reimb_pct_rate(df)

-        df = postprocessing_funcs.update_grouper_base_rate_and_grouper_pct_rate(df)
+    # Remove REIMB_ field values if their 1:1 counterparts are identical
+    df = postprocessing_funcs.remove_redundant_reimb_info(df)

-        # Standardize output column order - this should ALWAYS be the final postprocessing step
-        df = postprocessing_funcs.reorder_columns(df, COLUMN_ORDER)
+    # Derive int from contract text
+    df = postprocessing_funcs.add_aarete_derived_amendment_num(df)
+
+    # Process PATIENT_AGE_RANGE into PATIENT_AGE_MIN and PATIENT_AGE_MAX
+    df = postprocessing_funcs.process_patient_age_range(df)
+
+    # Add AARETE_DERIVED_PRODUCT
+    df = postprocessing_funcs.add_aarete_derived_product(df)
+
+    # Add GREATER_OF_IND
+    df["GREATER_OF_IND"] = "N"
+
+    df = postprocessing_funcs.update_grouper_base_rate_and_grouper_pct_rate(df)
+
+    df = postprocessing_funcs.fill_empty_dynamic(df)
+
+    # Standardize output column order - this should ALWAYS be the final postprocessing step
+    df = postprocessing_funcs.reorder_columns(df, COLUMN_ORDER)
+
+    return df

-        return df
-    else:
-        return df
@@ -795,3 +795,70 @@ def update_grouper_base_rate_and_grouper_pct_rate(df):
    ]

    return df
+
+def fill_empty_dynamic(df):
+    """
+    For specified columns, fills NA values with the common value from the same EXHIBIT_PAGE group.
+    
+    If all rows with the same EXHIBIT_PAGE value have only one unique non-NA value for a column,
+    this function fills any NA values in that column with that unique value.
+    
+    Args:
+        df (pd.DataFrame): DataFrame with at least an EXHIBIT_PAGE column
+        
+    Returns:
+        pd.DataFrame: DataFrame with filled values
+    """
+    if "EXHIBIT_PAGE" not in df.columns or df.empty:
+        return df
+    
+    # List of columns to check for filling
+    columns_to_fill = [
+        "AARETE_DERIVED_LOB",
+        "AARETE_DERIVED_PROGRAM",
+        "AARETE_DERIVED_PRODUCT",
+        "AARETE_DERIVED_NETWORK"
+    ]
+    
+    # Only process columns that actually exist in the dataframe
+    columns_to_fill = [col for col in columns_to_fill if col in df.columns]
+    
+    # Make a copy to avoid SettingWithCopyWarning
+    result_df = df.copy()
+    
+    # First, handle AARETE_DERIVED_LOB by grouping on FILE_NAME and EXHIBIT_PAGE only
+    for (file_name, exhibit_page), group in result_df.groupby(["FILE_NAME", "EXHIBIT_PAGE"]):
+        # Get non-NA LOB values
+        column = "AARETE_DERIVED_LOB"
+        non_na_values = [val for val in group[column].unique() if not string_utils.is_empty(val)]
+        
+        # If there's exactly one unique non-NA value, fill NA values with it
+        if len(non_na_values) == 1:
+            fill_value = non_na_values[0]
+            # Only apply to rows with this FILE_NAME and EXHIBIT_PAGE combination
+            mask = (result_df["FILE_NAME"] == file_name) & \
+                   (result_df["EXHIBIT_PAGE"] == exhibit_page) & \
+                   result_df[column].apply(string_utils.is_empty)
+            result_df.loc[mask, column] = fill_value
+    
+    # Then, handle other columns by grouping on FILE_NAME, EXHIBIT_PAGE, and AARETE_DERIVED_LOB
+    for (file_name, exhibit_page, lob), group in result_df.groupby(["FILE_NAME", "EXHIBIT_PAGE", "AARETE_DERIVED_LOB"]):
+        for column in columns_to_fill:
+            # Skip AARETE_DERIVED_LOB as we already processed it
+            if column == "AARETE_DERIVED_LOB":
+                continue
+                
+            # Get non-NA values
+            non_na_values = [val for val in group[column].unique() if not string_utils.is_empty(val)]
+            
+            # If there's exactly one unique non-NA value, fill NA values with it
+            if len(non_na_values) == 1:
+                fill_value = non_na_values[0]
+                # Only apply to rows with this FILE_NAME, EXHIBIT_PAGE, and LOB combination
+                mask = (result_df["FILE_NAME"] == file_name) & \
+                       (result_df["EXHIBIT_PAGE"] == exhibit_page) & \
+                       (result_df["AARETE_DERIVED_LOB"] == lob) & \
+                       result_df[column].apply(string_utils.is_empty)
+                result_df.loc[mask, column] = fill_value
+                
+    return result_df
@@ -8,7 +8,8 @@ from src.investment.postprocessing_funcs import (
    flatten_singleton_string_list, format_rate_fields_with_commas,
    normalize_auto_renewal_term, normalize_cpt_fields,
    normalize_indicator_field, process_patient_age_range, remove_hyphens,
-    remove_redundant_reimb_info, rename_columns, validate_and_reformat_date)
+    remove_redundant_reimb_info, rename_columns, validate_and_reformat_date,
+    fill_empty_dynamic)


 class TestPostprocessFunctions(unittest.TestCase):
@@ -486,6 +487,113 @@ class TestPostprocessFunctions(unittest.TestCase):

        result_df7 = deduplicate_provider_columns(input_df7)
        pd.testing.assert_frame_equal(result_df7, input_df7)
+    
+    def test_fill_empty_dynamic(self):
+        """Tests the fill_empty_dynamic function that fills NA values with common values from the same EXHIBIT_PAGE group.
+        
+        Tests:
+        1. Basic filling - fills NA values with the common value for the same EXHIBIT_PAGE
+        2. Multiple columns - correctly fills multiple columns independently
+        3. Multiple file/page combinations - respects FILE_NAME and EXHIBIT_PAGE boundaries
+        4. No common value - doesn't fill when multiple non-NA values exist
+        5. All NA values - doesn't fill when all values are NA
+        6. Missing columns - returns unchanged DataFrame when key columns are missing
+        """
+        # Test case 1: Basic filling for a single column
+        input_df1 = pd.DataFrame({
+            "FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"],
+            "EXHIBIT_PAGE": ["1.0", "1.0", "1.0"],
+            "AARETE_DERIVED_LOB": ["Commercial", "", None]
+        })
+        
+        expected_df1 = pd.DataFrame({
+            "FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"],
+            "EXHIBIT_PAGE": ["1.0", "1.0", "1.0"],
+            "AARETE_DERIVED_LOB": ["Commercial", "Commercial", "Commercial"]
+        })
+        
+        result_df1 = fill_empty_dynamic(input_df1)
+        pd.testing.assert_frame_equal(result_df1, expected_df1)
+        
+        # Test case 2: Multiple columns
+        input_df2 = pd.DataFrame({
+            "FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"],
+            "EXHIBIT_PAGE": ["1.0", "1.0", "1.0"],
+            "AARETE_DERIVED_LOB": ["Commercial", "", None],
+            "AARETE_DERIVED_PRODUCT": ["Product A", None, ""]
+        })
+        
+        expected_df2 = pd.DataFrame({
+            "FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"],
+            "EXHIBIT_PAGE": ["1.0", "1.0", "1.0"],
+            "AARETE_DERIVED_LOB": ["Commercial", "Commercial", "Commercial"],
+            "AARETE_DERIVED_PRODUCT": ["Product A", "Product A", "Product A"]
+        })
+        
+        result_df2 = fill_empty_dynamic(input_df2)
+        pd.testing.assert_frame_equal(result_df2, expected_df2)
+        
+        # Test case 3: Multiple file/page combinations
+        input_df3 = pd.DataFrame({
+            "FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf", "file1.pdf", "file2.pdf", "file2.pdf"],
+            "EXHIBIT_PAGE": ["1.0", "1.0", "2.0", "2.0", "1.0", "1.0"],
+            "AARETE_DERIVED_LOB": ["Commercial", "", "Medicare", None, "Medicaid", ""]
+        })
+        
+        expected_df3 = pd.DataFrame({
+            "FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf", "file1.pdf", "file2.pdf", "file2.pdf"],
+            "EXHIBIT_PAGE": ["1.0", "1.0", "2.0", "2.0", "1.0", "1.0"],
+            "AARETE_DERIVED_LOB": ["Commercial", "Commercial", "Medicare", "Medicare", "Medicaid", "Medicaid"]
+        })
+        
+        result_df3 = fill_empty_dynamic(input_df3)
+        pd.testing.assert_frame_equal(result_df3, expected_df3)
+        
+        # Test case 4: No common value (multiple non-NA values exist)
+        input_df4 = pd.DataFrame({
+            "FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"],
+            "EXHIBIT_PAGE": ["1.0", "1.0", "1.0"],
+            "AARETE_DERIVED_LOB": ["Commercial", "Medicare", None]
+        })
+        
+        expected_df4 = pd.DataFrame({
+            "FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"],
+            "EXHIBIT_PAGE": ["1.0", "1.0", "1.0"],
+            "AARETE_DERIVED_LOB": ["Commercial", "Medicare", None]  # Should remain unchanged
+        })
+        
+        result_df4 = fill_empty_dynamic(input_df4)
+        pd.testing.assert_frame_equal(result_df4, expected_df4)
+        
+        # Test case 5: All NA values
+        input_df5 = pd.DataFrame({
+            "FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"],
+            "EXHIBIT_PAGE": ["1.0", "1.0", "1.0"],
+            "AARETE_DERIVED_LOB": [None, "", None]
+        })
+        
+        expected_df5 = pd.DataFrame({
+            "FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"],
+            "EXHIBIT_PAGE": ["1.0", "1.0", "1.0"],
+            "AARETE_DERIVED_LOB": [None, "", None]  # Should remain unchanged
+        })
+        
+        result_df5 = fill_empty_dynamic(input_df5)
+        pd.testing.assert_frame_equal(result_df5, expected_df5)
+        
+        # Test case 6: Missing EXHIBIT_PAGE column
+        input_df6 = pd.DataFrame({
+            "FILE_NAME": ["file1.pdf", "file1.pdf"],
+            "AARETE_DERIVED_LOB": ["Commercial", None]
+        })
+        
+        result_df6 = fill_empty_dynamic(input_df6)
+        pd.testing.assert_frame_equal(result_df6, input_df6)  # Should remain unchanged
+        
+        # Test case 7: Empty DataFrame
+        empty_df = pd.DataFrame()
+        result_empty = fill_empty_dynamic(empty_df)
+        pd.testing.assert_frame_equal(result_empty, empty_df)  # Should remain unchanged


 if __name__ == "__main__":