From b2a8e77cca0ad0863213c0c3257486da483bf97f Mon Sep 17 00:00:00 2001 From: Alex Galarce Date: Wed, 4 Dec 2024 22:14:34 +0000 Subject: [PATCH] Merged in chore/health_plan_state (pull request #304) refactored code to add recent health plan state hotfixes into the existing postprocessing pipeline * refactored code to add recent health plan state hotfixes into the existing postprocessing pipeline * Merged main into chore/health_plan_state * Merged main into chore/health_plan_state * add debug print statement * debug print statements * more debug print statements * print df columns for debug * move health plan state hotfix functionality to postprocessing_funcs.py * add better error printing * add error message for parent agreement code * remove dots in state abbreviations * remove unused import * also look for state abbreviations * Merged main into chore/health_plan_state * remove print statements * Merged main into chore/health_plan_state Approved-by: Katon Minhas --- fieldExtraction/src/file_processing.py | 2 +- fieldExtraction/src/hotfix_helper_funcs.py | 11 +++++-- fieldExtraction/src/postprocess.py | 4 ++- fieldExtraction/src/postprocessing_funcs.py | 34 +++++++++++++++++++-- 4 files changed, 44 insertions(+), 7 deletions(-) diff --git a/fieldExtraction/src/file_processing.py b/fieldExtraction/src/file_processing.py index 4318b6b..40f90ae 100644 --- a/fieldExtraction/src/file_processing.py +++ b/fieldExtraction/src/file_processing.py @@ -277,7 +277,7 @@ def run_ac_prompts(file_object): ################## POSTPROCESS ################## ac_df = pd.DataFrame([ac_answers_dict]) - ac_df = postprocess.ac_postprocess(ac_df, filename, num_pages) + ac_df = postprocess.ac_postprocess(ac_df, text_dict, filename, num_pages) ################## WRITE TO OUTPUT ################## ac_df.to_csv(os.path.join(output_dir, config.AC_RESULTS_NAME), index=False) diff --git a/fieldExtraction/src/hotfix_helper_funcs.py b/fieldExtraction/src/hotfix_helper_funcs.py index c619b98..b1877a8 100644 --- a/fieldExtraction/src/hotfix_helper_funcs.py +++ b/fieldExtraction/src/hotfix_helper_funcs.py @@ -211,7 +211,6 @@ def irs_hotfix(filename, text_dict: dict[str, str]): def state_check(states: str | list[str], text_dict: dict[str, str]) -> str: - if not isinstance(states, str) and not isinstance(states,list): states = 'N/A' @@ -227,7 +226,7 @@ def state_check(states: str | list[str], text_dict: dict[str, str]) -> str: result_states = [] for state in states: - state = state.strip() + state = state.strip().replace(".", "") # Remove "." in cases like "N.Y." # fix 1 if state == "Hlorida": @@ -277,9 +276,12 @@ def get_health_plan_state(text_dict: dict[str, str]) -> str: Returns: str: A single state extracted from the text, `multiple`, or `N/A` """ + state_abbreviations = list(STATE_MAP.keys()) state_names = list(STATE_MAP.values()) + + state_names_and_abbreviations = state_abbreviations + state_names - pattern = r'\b(?:' + '|'.join(re.escape(state) for state in state_names) + r')\b' + pattern = r'\b(?:' + '|'.join(re.escape(state) for state in state_names_and_abbreviations) + r')\b' states_set = set() @@ -289,6 +291,9 @@ def get_health_plan_state(text_dict: dict[str, str]) -> str: states_set.update(state_matches) states_list = list(states_set) + # Expand any abbreviations. If the state is already its full name, we're just gonna + # default to that. E.g., "CT" -> "Connecticut", "Connecticut" -> "Connecticut" + states_list = [STATE_MAP.get(state, state) for state in states_list] if len(states_list) > 1: state = "Multiple" diff --git a/fieldExtraction/src/postprocess.py b/fieldExtraction/src/postprocess.py index d81f11d..5be758a 100644 --- a/fieldExtraction/src/postprocess.py +++ b/fieldExtraction/src/postprocess.py @@ -55,7 +55,7 @@ def b_postprocess(filename, df, pages): return df -def ac_postprocess(df, filename, pages): +def ac_postprocess(df, text_dict, filename, pages): if df.shape[0] > 0: @@ -83,6 +83,8 @@ def ac_postprocess(df, filename, pages): df = postprocessing_funcs.clean_health_plan_state(df) + df = postprocessing_funcs.clean_health_plan_state_from_hotfix(df,filename,text_dict) + df = postprocessing_funcs.clean_notice_provider_name_and_address(df) df = postprocessing_funcs.clean_policies_and_procedures(df) diff --git a/fieldExtraction/src/postprocessing_funcs.py b/fieldExtraction/src/postprocessing_funcs.py index 4a5d8ee..4b81054 100644 --- a/fieldExtraction/src/postprocessing_funcs.py +++ b/fieldExtraction/src/postprocessing_funcs.py @@ -9,6 +9,7 @@ import valid from valid import DERIVED_INDICATOR_FIELDS import claude_funcs from utils import is_empty +from hotfix_helper_funcs import state_check import logging @@ -142,10 +143,10 @@ def get_parent_agreement_code(filename): filename = re.sub(r"\([^)]*\)", "", filename) match = re.search(r"([^\sa-zA-Z]+)(?=\.\w+$|$)", filename) - end = [i for i in match.group(1).split("_") if i] + end = [i for i in match.group(1).split("_") if i] # an error is happening here. This function needs a docstring. return end[0] except Exception as e: - print(f"Error: {e}") + print(f"Error in `get_parent_agreement_code`: {e}; returning `N/A`") return "N/A" @@ -399,6 +400,35 @@ def clean_health_plan_state(final_df): final_df["HEALTH_PLAN_STATE"] = final_df["HEALTH_PLAN_STATE"].str.title() return final_df +def clean_health_plan_state_from_hotfix( + df: pd.DataFrame, + contract_name: str, + text_dict: dict[str, str], +) -> pd.DataFrame: + + if not isinstance(df, pd.DataFrame): + # raise TypeError(f"Expected a dataframe, got {type(df).__name__}") + return df + + if contract_name is None: + # raise ValueError( + # f"Expected 'contract_name' parameter, got {contract_name}" + # ) + return df + + if "HEALTH_PLAN_STATE" not in df.columns: + # raise KeyError( + # f"Column 'Health Plan State' not found in DataFrame. Available columns are: {list(df.columns)}" + # ) + return df + + health_plan_state = df["HEALTH_PLAN_STATE"].unique().tolist()[0] + + answer = state_check(health_plan_state, text_dict) + + df["HEALTH_PLAN_STATE"] = answer + + return df def clean_notice_provider_name_and_address(final_df): if "NOTICE_PROVIDER_NAME" in final_df and "NOTICE_PROVIDER_ADDRESS" in final_df: