jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in bugfix/mi-dynamic (pull request #726)

Browse Source 
Bugfix/mi dynamic

* Refactor - split out one_to_n_cleaning

* Rework postprocess

* Update postprocessing - fill dynamic values if there is only 1 non-empty value

* Update to use string_utils

* Group by AARETE_DERIVED_LOB as well

* Update crosswalk mapping to handle | lists

* Update constants - run with MI flag


Approved-by: Siddhant Medar
This commit is contained in:
Katon Minhas
2025-10-07 15:06:10 +00:00
parent 354bb1c9ce
commit a910f27cd2
7 changed files with 292 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fieldExtraction/constants/mappings/crosswalk_program.json
+1 -1
View File
@@ -144,7 +144,7 @@
}, },
"MI" : { "MI" : {
"Healthy Michigan Plan" : "HMIP", "Healthy Michigan Plan" : "HMIP",
"MIChild" : "CHIP" "MI Child" : "CHIP"
}, },
"MN" : { "MN" : {
"Minnesota Medical Assistance" : "MNMA", "Minnesota Medical Assistance" : "MNMA",
fieldExtraction/src/investment/aarete_derived.py
+13 -12
View File
@@ -59,7 +59,6 @@ def fill_na_mapping(answer_dicts):
def get_crosswalk_fields(answer_dicts: list, constants: Constants): def get_crosswalk_fields(answer_dicts: list, constants: Constants):
crosswalk_fields = FieldSet(file_path=config.FIELD_JSON_PATH, crosswalk=True) crosswalk_fields = FieldSet(file_path=config.FIELD_JSON_PATH, crosswalk=True)
for to_field in crosswalk_fields.fields: for to_field in crosswalk_fields.fields:
to_field_name, from_field_name = to_field.field_name, to_field.base_field to_field_name, from_field_name = to_field.field_name, to_field.base_field
# Find crosswalk # Find crosswalk
@@ -73,16 +72,18 @@ def get_crosswalk_fields(answer_dicts: list, constants: Constants):
if not string_utils.is_empty( if not string_utils.is_empty(
from_field_value from_field_value
) and string_utils.is_empty(to_field_value): ) and string_utils.is_empty(to_field_value):
if from_field_value in crosswalk.mapping.keys(): from_field_value_list = from_field_value.split("|")
answer_dict[to_field_name] = crosswalk.mapping.get( to_field_answer_list = []
from_field_value for individual_from_field_value in from_field_value_list:
) if individual_from_field_value in crosswalk.mapping.keys():
elif from_field_value in crosswalk.mapping.values(): to_field_answer_list.append(crosswalk.mapping.get(
if "AARETE_DERIVED" in to_field_name: individual_from_field_value
answer_dict[to_field_name] = from_field_value ))
else: elif from_field_value in crosswalk.mapping.values():
answer_dict[to_field_name] = ( if "AARETE_DERIVED" in to_field_name:
crosswalk.create_reverse_mapping().get(from_field_value) to_field_answer_list.append(individual_from_field_value)
) else:
to_field_answer_list.append(crosswalk.create_reverse_mapping().get(individual_from_field_value))
answer_dict[to_field_name] = "|".join(to_field_answer_list)
return answer_dicts return answer_dicts
fieldExtraction/src/investment/file_processing.py
+2 -18
View File
@@ -218,24 +218,8 @@ def run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, constants)
filename, filename,
) # returns list of dicts ) # returns list of dicts
################################ Crosswalk Fields ################################ ################################ Mapping and Cleaning ###############################
all_exhibit_rows = aarete_derived.get_crosswalk_fields( all_exhibit_rows = one_to_n_funcs.one_to_n_cleaning(all_exhibit_rows, exhibit_text, constants, filename)
all_exhibit_rows, constants
)
################################ Determine LOB Relationship ################################
all_exhibit_rows = one_to_n_funcs.get_lob_relationship(
all_exhibit_rows, exhibit_text, filename
)
################################ Fill NA Mapping ################################
all_exhibit_rows = aarete_derived.fill_na_mapping(all_exhibit_rows)
################################ Update LOB for Duals ################################
all_exhibit_rows = postprocessing_funcs.update_lob_for_duals(all_exhibit_rows)
################################ Split REIMB_DATES ################################
all_exhibit_rows = one_to_n_funcs.split_reimb_dates(all_exhibit_rows, filename)
################################ Add to Total ############################### ################################ Add to Total ###############################
one_to_n_results += all_exhibit_rows one_to_n_results += all_exhibit_rows
fieldExtraction/src/investment/one_to_n_funcs.py
+28 -6
View File
@@ -1,11 +1,8 @@
import logging import logging
import src.investment.dynamic_funcs as dynamic_funcs from src.investment import dynamic_funcs, lesser_of_funcs, prompt_calls, aarete_derived, postprocessing_funcs
import src.investment.lesser_of_funcs as lesser_of_funcs from src.utils import llm_utils, string_utils
import src.investment.prompt_calls as prompt_calls
import src.prompts.prompt_templates as prompt_templates import src.prompts.prompt_templates as prompt_templates
import src.utils.llm_utils as llm_utils
import src.utils.string_utils as string_utils
from constants.constants import Constants from constants.constants import Constants
from src import config from src import config
from src.prompts.fieldset import FieldSet from src.prompts.fieldset import FieldSet
@@ -200,7 +197,7 @@ def clean_reimbursement_primary(
filtered_answers, seen_pairs, exhibit_page, filename filtered_answers, seen_pairs, exhibit_page, filename
) )
# Step 4: Apply exhibit lesser-of statement and rate_escalator statement if it exists # Step 4: Apply exhibit lesser-of statement if it exists
if not string_utils.is_empty(exhibit_lesser_of) and deduplicated_answers: if not string_utils.is_empty(exhibit_lesser_of) and deduplicated_answers:
deduplicated_answers = lesser_of_funcs.apply_exhibit_lesser_of( deduplicated_answers = lesser_of_funcs.apply_exhibit_lesser_of(
deduplicated_answers, exhibit_lesser_of, filename deduplicated_answers, exhibit_lesser_of, filename
@@ -818,3 +815,28 @@ def get_lob_relationship(answer_dicts, exhibit_text, filename):
) )
return answer_dicts return answer_dicts
def one_to_n_cleaning(all_exhibit_rows: list[dict], exhibit_text: str, constants: Constants, filename: str):
################################ Crosswalk Fields ################################
all_exhibit_rows = aarete_derived.get_crosswalk_fields(
all_exhibit_rows, constants
)
################################ Determine LOB Relationship ################################
all_exhibit_rows = get_lob_relationship(
all_exhibit_rows, exhibit_text, filename
)
################################ Fill NA Mapping ################################
all_exhibit_rows = aarete_derived.fill_na_mapping(all_exhibit_rows)
################################ Update LOB for Duals ################################
all_exhibit_rows = postprocessing_funcs.update_lob_for_duals(all_exhibit_rows)
################################ Split REIMB_DATES ################################
all_exhibit_rows = split_reimb_dates(all_exhibit_rows, filename)
return all_exhibit_rows
fieldExtraction/src/investment/postprocess.py
+72 -67
View File
@@ -5,86 +5,91 @@ from src.investment import postprocessing_funcs
def postprocess(df, constants: Constants): def postprocess(df, constants: Constants):
if df.shape[0] > 0: if df.shape[0] == 0:
return df
df["CLIENT_NAME"] = config.CLIENT_NAME
# Rename columns df["CLIENT_NAME"] = config.CLIENT_NAME
df = postprocessing_funcs.rename_columns(df)
# Sort to ensure consistent ordering # Rename columns
sort_columns = ["FILE_NAME"] df = postprocessing_funcs.rename_columns(df)
if "EXHIBIT_PAGE" in df.columns:
sort_columns.append("EXHIBIT_PAGE")
if "REIMB_LESSER_OF_ID" in df.columns:
sort_columns.append("REIMB_LESSER_OF_ID")
df = df.sort_values(by=sort_columns).reset_index(drop=True)
# Postprocess date fields # Sort to ensure consistent ordering
df = postprocessing_funcs.date_postprocess(df, config.FIELD_JSON_PATH) sort_columns = ["FILE_NAME"]
if "EXHIBIT_PAGE" in df.columns:
sort_columns.append("EXHIBIT_PAGE")
if "REIMB_LESSER_OF_ID" in df.columns:
sort_columns.append("REIMB_LESSER_OF_ID")
df = df.sort_values(by=sort_columns).reset_index(drop=True)
# Format rate fields # Postprocess date fields
if "REIMB_FEE_RATE" in df.columns: df = postprocessing_funcs.date_postprocess(df, config.FIELD_JSON_PATH)
df["REIMB_FEE_RATE"] = df["REIMB_FEE_RATE"].apply(
postprocessing_funcs.format_rate_fields_with_commas
)
if "REIMB_PCT_RATE" in df.columns:
df["REIMB_PCT_RATE"] = df["REIMB_PCT_RATE"].apply(
postprocessing_funcs.format_rate_fields_with_commas
)
for col in df.columns: # Format rate fields
if "_IND" in col: if "REIMB_FEE_RATE" in df.columns:
df[col] = df[col].apply(postprocessing_funcs.normalize_indicator_field) df["REIMB_FEE_RATE"] = df["REIMB_FEE_RATE"].apply(
# Normalize _IND fields and clean up TIN/NPI fields postprocessing_funcs.format_rate_fields_with_commas
if "TIN" in col or "NPI" in col: )
df[col] = df[col].apply(postprocessing_funcs.remove_hyphens) if "REIMB_PCT_RATE" in df.columns:
# Apply the flatten_singleton_string_list function to the 'CPT' column df["REIMB_PCT_RATE"] = df["REIMB_PCT_RATE"].apply(
if "_CD" in col and "CPT" not in col: postprocessing_funcs.format_rate_fields_with_commas
df[col] = df[col].apply(
postprocessing_funcs.flatten_singleton_string_list
)
# Convert DATE to YYYY/MM/DD format
if "_DT" in col or "DATE" in col:
df[col] = df[col].apply(postprocessing_funcs.validate_and_reformat_date)
if "CPT" in col:
df[col] = df[col].apply(postprocessing_funcs.normalize_cpt_fields)
# Deduplicate provider fields - remove GROUP values from OTHER fields and deduplicate OTHER lists
df = postprocessing_funcs.deduplicate_provider_columns(df)
# Normalize the 'AUTO_RENEWAL_TERM' column
df = postprocessing_funcs.auto_renewal(df)
# Check conditions for TERMINATION_DT, AUTO_RENEWAL_IND, and AARETE_DERIVED_TERMINATION_DT
df = postprocessing_funcs.update_termination_date_for_conditions(df)
# Standardize the 'AARETE_DERIVED_REIMB_METHOD' column and updates the 'AARETE_DERIVED_FEE_SCHEDULE' column based on specific values.
df = postprocessing_funcs.standardize_reimb_method_and_fee_schedule(
df, constants.VALID_UNIT_OF_MEASURE
) )
# update reimb_pct_rate for col in df.columns:
df = postprocessing_funcs.fill_empty_reimb_pct_rate(df) if "_IND" in col:
df[col] = df[col].apply(postprocessing_funcs.normalize_indicator_field)
# Normalize _IND fields and clean up TIN/NPI fields
if "TIN" in col or "NPI" in col:
df[col] = df[col].apply(postprocessing_funcs.remove_hyphens)
# Apply the flatten_singleton_string_list function to the 'CPT' column
if "_CD" in col and "CPT" not in col:
df[col] = df[col].apply(
postprocessing_funcs.flatten_singleton_string_list
)
# Convert DATE to YYYY/MM/DD format
if "_DT" in col or "DATE" in col:
df[col] = df[col].apply(postprocessing_funcs.validate_and_reformat_date)
if "CPT" in col:
df[col] = df[col].apply(postprocessing_funcs.normalize_cpt_fields)
df = postprocessing_funcs.remove_redundant_reimb_info(df) # Deduplicate provider fields - remove GROUP values from OTHER fields and deduplicate OTHER lists
df = postprocessing_funcs.deduplicate_provider_columns(df)
df = postprocessing_funcs.add_aarete_derived_amendment_num(df) # Normalize the 'AUTO_RENEWAL_TERM' column
df = postprocessing_funcs.auto_renewal(df)
# Process PATIENT_AGE_RANGE into PATIENT_AGE_MIN and PATIENT_AGE_MAX # Check conditions for TERMINATION_DT, AUTO_RENEWAL_IND, and AARETE_DERIVED_TERMINATION_DT
df = postprocessing_funcs.process_patient_age_range(df) df = postprocessing_funcs.update_termination_date_for_conditions(df)
# Add AARETE_DERIVED_PRODUCT # Standardize the 'AARETE_DERIVED_REIMB_METHOD' column and updates the 'AARETE_DERIVED_FEE_SCHEDULE' column based on specific values.
df = postprocessing_funcs.add_aarete_derived_product(df) df = postprocessing_funcs.standardize_reimb_method_and_fee_schedule(
df, constants.VALID_UNIT_OF_MEASURE
)
# Add GREATER_OF_IND # update reimb_pct_rate
df["GREATER_OF_IND"] = "N" df = postprocessing_funcs.fill_empty_reimb_pct_rate(df)
df = postprocessing_funcs.update_grouper_base_rate_and_grouper_pct_rate(df) # Remove REIMB_ field values if their 1:1 counterparts are identical
df = postprocessing_funcs.remove_redundant_reimb_info(df)
# Standardize output column order - this should ALWAYS be the final postprocessing step # Derive int from contract text
df = postprocessing_funcs.reorder_columns(df, COLUMN_ORDER) df = postprocessing_funcs.add_aarete_derived_amendment_num(df)
# Process PATIENT_AGE_RANGE into PATIENT_AGE_MIN and PATIENT_AGE_MAX
df = postprocessing_funcs.process_patient_age_range(df)
# Add AARETE_DERIVED_PRODUCT
df = postprocessing_funcs.add_aarete_derived_product(df)
# Add GREATER_OF_IND
df["GREATER_OF_IND"] = "N"
df = postprocessing_funcs.update_grouper_base_rate_and_grouper_pct_rate(df)
df = postprocessing_funcs.fill_empty_dynamic(df)
# Standardize output column order - this should ALWAYS be the final postprocessing step
df = postprocessing_funcs.reorder_columns(df, COLUMN_ORDER)
return df
return df
else:
return df
fieldExtraction/src/investment/postprocessing_funcs.py
+67
View File
@@ -795,3 +795,70 @@ def update_grouper_base_rate_and_grouper_pct_rate(df):
] ]
return df return df
def fill_empty_dynamic(df):
"""
For specified columns, fills NA values with the common value from the same EXHIBIT_PAGE group.
If all rows with the same EXHIBIT_PAGE value have only one unique non-NA value for a column,
this function fills any NA values in that column with that unique value.
Args:
df (pd.DataFrame): DataFrame with at least an EXHIBIT_PAGE column
Returns:
pd.DataFrame: DataFrame with filled values
"""
if "EXHIBIT_PAGE" not in df.columns or df.empty:
return df
# List of columns to check for filling
columns_to_fill = [
"AARETE_DERIVED_LOB",
"AARETE_DERIVED_PROGRAM",
"AARETE_DERIVED_PRODUCT",
"AARETE_DERIVED_NETWORK"
]
# Only process columns that actually exist in the dataframe
columns_to_fill = [col for col in columns_to_fill if col in df.columns]
# Make a copy to avoid SettingWithCopyWarning
result_df = df.copy()
# First, handle AARETE_DERIVED_LOB by grouping on FILE_NAME and EXHIBIT_PAGE only
for (file_name, exhibit_page), group in result_df.groupby(["FILE_NAME", "EXHIBIT_PAGE"]):
# Get non-NA LOB values
column = "AARETE_DERIVED_LOB"
non_na_values = [val for val in group[column].unique() if not string_utils.is_empty(val)]
# If there's exactly one unique non-NA value, fill NA values with it
if len(non_na_values) == 1:
fill_value = non_na_values[0]
# Only apply to rows with this FILE_NAME and EXHIBIT_PAGE combination
mask = (result_df["FILE_NAME"] == file_name) & \
(result_df["EXHIBIT_PAGE"] == exhibit_page) & \
result_df[column].apply(string_utils.is_empty)
result_df.loc[mask, column] = fill_value
# Then, handle other columns by grouping on FILE_NAME, EXHIBIT_PAGE, and AARETE_DERIVED_LOB
for (file_name, exhibit_page, lob), group in result_df.groupby(["FILE_NAME", "EXHIBIT_PAGE", "AARETE_DERIVED_LOB"]):
for column in columns_to_fill:
# Skip AARETE_DERIVED_LOB as we already processed it
if column == "AARETE_DERIVED_LOB":
continue
# Get non-NA values
non_na_values = [val for val in group[column].unique() if not string_utils.is_empty(val)]
# If there's exactly one unique non-NA value, fill NA values with it
if len(non_na_values) == 1:
fill_value = non_na_values[0]
# Only apply to rows with this FILE_NAME, EXHIBIT_PAGE, and LOB combination
mask = (result_df["FILE_NAME"] == file_name) & \
(result_df["EXHIBIT_PAGE"] == exhibit_page) & \
(result_df["AARETE_DERIVED_LOB"] == lob) & \
result_df[column].apply(string_utils.is_empty)
result_df.loc[mask, column] = fill_value
return result_df
fieldExtraction/tests/test_investment_postprocess.py → fieldExtraction/tests/test_postprocess.py
+109 -1
View File
@@ -8,7 +8,8 @@ from src.investment.postprocessing_funcs import (
flatten_singleton_string_list, format_rate_fields_with_commas, flatten_singleton_string_list, format_rate_fields_with_commas,
normalize_auto_renewal_term, normalize_cpt_fields, normalize_auto_renewal_term, normalize_cpt_fields,
normalize_indicator_field, process_patient_age_range, remove_hyphens, normalize_indicator_field, process_patient_age_range, remove_hyphens,
remove_redundant_reimb_info, rename_columns, validate_and_reformat_date) remove_redundant_reimb_info, rename_columns, validate_and_reformat_date,
fill_empty_dynamic)
class TestPostprocessFunctions(unittest.TestCase): class TestPostprocessFunctions(unittest.TestCase):
@@ -486,6 +487,113 @@ class TestPostprocessFunctions(unittest.TestCase):
result_df7 = deduplicate_provider_columns(input_df7) result_df7 = deduplicate_provider_columns(input_df7)
pd.testing.assert_frame_equal(result_df7, input_df7) pd.testing.assert_frame_equal(result_df7, input_df7)
def test_fill_empty_dynamic(self):
"""Tests the fill_empty_dynamic function that fills NA values with common values from the same EXHIBIT_PAGE group.
Tests:
1. Basic filling - fills NA values with the common value for the same EXHIBIT_PAGE
2. Multiple columns - correctly fills multiple columns independently
3. Multiple file/page combinations - respects FILE_NAME and EXHIBIT_PAGE boundaries
4. No common value - doesn't fill when multiple non-NA values exist
5. All NA values - doesn't fill when all values are NA
6. Missing columns - returns unchanged DataFrame when key columns are missing
"""
# Test case 1: Basic filling for a single column
input_df1 = pd.DataFrame({
"FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"],
"EXHIBIT_PAGE": ["1.0", "1.0", "1.0"],
"AARETE_DERIVED_LOB": ["Commercial", "", None]
})
expected_df1 = pd.DataFrame({
"FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"],
"EXHIBIT_PAGE": ["1.0", "1.0", "1.0"],
"AARETE_DERIVED_LOB": ["Commercial", "Commercial", "Commercial"]
})
result_df1 = fill_empty_dynamic(input_df1)
pd.testing.assert_frame_equal(result_df1, expected_df1)
# Test case 2: Multiple columns
input_df2 = pd.DataFrame({
"FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"],
"EXHIBIT_PAGE": ["1.0", "1.0", "1.0"],
"AARETE_DERIVED_LOB": ["Commercial", "", None],
"AARETE_DERIVED_PRODUCT": ["Product A", None, ""]
})
expected_df2 = pd.DataFrame({
"FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"],
"EXHIBIT_PAGE": ["1.0", "1.0", "1.0"],
"AARETE_DERIVED_LOB": ["Commercial", "Commercial", "Commercial"],
"AARETE_DERIVED_PRODUCT": ["Product A", "Product A", "Product A"]
})
result_df2 = fill_empty_dynamic(input_df2)
pd.testing.assert_frame_equal(result_df2, expected_df2)
# Test case 3: Multiple file/page combinations
input_df3 = pd.DataFrame({
"FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf", "file1.pdf", "file2.pdf", "file2.pdf"],
"EXHIBIT_PAGE": ["1.0", "1.0", "2.0", "2.0", "1.0", "1.0"],
"AARETE_DERIVED_LOB": ["Commercial", "", "Medicare", None, "Medicaid", ""]
})
expected_df3 = pd.DataFrame({
"FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf", "file1.pdf", "file2.pdf", "file2.pdf"],
"EXHIBIT_PAGE": ["1.0", "1.0", "2.0", "2.0", "1.0", "1.0"],
"AARETE_DERIVED_LOB": ["Commercial", "Commercial", "Medicare", "Medicare", "Medicaid", "Medicaid"]
})
result_df3 = fill_empty_dynamic(input_df3)
pd.testing.assert_frame_equal(result_df3, expected_df3)
# Test case 4: No common value (multiple non-NA values exist)
input_df4 = pd.DataFrame({
"FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"],
"EXHIBIT_PAGE": ["1.0", "1.0", "1.0"],
"AARETE_DERIVED_LOB": ["Commercial", "Medicare", None]
})
expected_df4 = pd.DataFrame({
"FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"],
"EXHIBIT_PAGE": ["1.0", "1.0", "1.0"],
"AARETE_DERIVED_LOB": ["Commercial", "Medicare", None] # Should remain unchanged
})
result_df4 = fill_empty_dynamic(input_df4)
pd.testing.assert_frame_equal(result_df4, expected_df4)
# Test case 5: All NA values
input_df5 = pd.DataFrame({
"FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"],
"EXHIBIT_PAGE": ["1.0", "1.0", "1.0"],
"AARETE_DERIVED_LOB": [None, "", None]
})
expected_df5 = pd.DataFrame({
"FILE_NAME": ["file1.pdf", "file1.pdf", "file1.pdf"],
"EXHIBIT_PAGE": ["1.0", "1.0", "1.0"],
"AARETE_DERIVED_LOB": [None, "", None] # Should remain unchanged
})
result_df5 = fill_empty_dynamic(input_df5)
pd.testing.assert_frame_equal(result_df5, expected_df5)
# Test case 6: Missing EXHIBIT_PAGE column
input_df6 = pd.DataFrame({
"FILE_NAME": ["file1.pdf", "file1.pdf"],
"AARETE_DERIVED_LOB": ["Commercial", None]
})
result_df6 = fill_empty_dynamic(input_df6)
pd.testing.assert_frame_equal(result_df6, input_df6) # Should remain unchanged
# Test case 7: Empty DataFrame
empty_df = pd.DataFrame()
result_empty = fill_empty_dynamic(empty_df)
pd.testing.assert_frame_equal(result_empty, empty_df) # Should remain unchanged
if __name__ == "__main__": if __name__ == "__main__":