From 0320d6633a56474c496f4fada0c2ff66b3f990bc Mon Sep 17 00:00:00 2001 From: Alex Galarce Date: Wed, 3 Sep 2025 20:29:43 +0000 Subject: [PATCH] Merged in bugfix/lob-fixes (pull request #692) Bugfix/lob fixes * Refactor update_lob_for_duals and update_grouper_base_rate_and_grouper_pct_rate functions for improved clarity and logic; add handling for 'neither' and 'n/a' cases. * Change logging level from info to debug for various functions to reduce log verbosity. Approved-by: Katon Minhas --- .../src/investment/one_to_n_funcs.py | 18 +++---- .../src/investment/postprocessing_funcs.py | 51 +++++++++++-------- .../src/prompts/prompt_templates.py | 17 +++---- 3 files changed, 44 insertions(+), 42 deletions(-) diff --git a/fieldExtraction/src/investment/one_to_n_funcs.py b/fieldExtraction/src/investment/one_to_n_funcs.py index 14428e7..ee3448e 100644 --- a/fieldExtraction/src/investment/one_to_n_funcs.py +++ b/fieldExtraction/src/investment/one_to_n_funcs.py @@ -822,7 +822,7 @@ def deduplicate_with_accumulator( ) if duplicates_found > 0: - logging.info( + logging.debug( f"Removed {duplicates_found} duplicate pairs from exhibit {exhibit_page} (base page {base_page}) in {filename}" ) @@ -864,7 +864,7 @@ def filter_services_without_reimbursements( filtered_count = len(answers) - len(valid_reimbursements) if filtered_count > 0: - logging.info( + logging.debug( f"Filtered {filtered_count} services without clear reimbursement methodologies in {filename}." ) @@ -939,13 +939,13 @@ def split_compound_reimbursements( split_result = split_compound_reimbursement_llm(answer_dict, filename) if split_result and len(split_result) > 1: - logging.info( + logging.debug( f"Split compound reimbursement in {filename}: " f"{answer_dict['SERVICE_TERM']} - {answer_dict['REIMB_TERM']}" f" into {len(split_result)} parts." ) - logging.info(f"Split results: {split_result}") + logging.debug(f"Split results: {split_result}") split_answers.extend(split_result) else: # If LLM didn't split, keep the original @@ -1176,7 +1176,7 @@ def split_reimb_dates(one_to_n_results: list, filename: str) -> list: """ if not one_to_n_results or not isinstance(one_to_n_results, list): - logging.info("No valid data to process") + logging.debug("`split_reimb_dates`: No valid data to process") return one_to_n_results # Filter records that have non-empty REIMB_DATES @@ -1195,7 +1195,7 @@ def split_reimb_dates(one_to_n_results: list, filename: str) -> list: records_to_process.append((i, record)) if not records_to_process: - logging.info("No valid date ranges to process") + logging.debug("`split_reimb_dates`: No valid date ranges to process") # Still remove the REIMB_DATES field from all records for record in one_to_n_results: if "REIMB_DATES" in record: @@ -1261,17 +1261,17 @@ def process_special_cases( case = case.upper() if case == "TRIGGER_CAP": - logging.info(f"Running TRIGGER_CAP breakout for {service} - {methodology}") + logging.debug(f"Running TRIGGER_CAP breakout for {service} - {methodology}") trigger_cap_results = run_trigger_cap_breakout( service, methodology, filename ) special_case_results.update(trigger_cap_results) elif case == "OUTLIER": - logging.info(f"Running OUTLIERS breakout for {service} - {methodology}") + logging.debug(f"Running OUTLIERS breakout for {service} - {methodology}") outlier_results = run_outliers_breakout(service, methodology, filename) special_case_results.update(outlier_results) elif case == "GROUPER": - logging.info(f"Running GROUPER breakout for {service} - {methodology}") + logging.debug(f"Running GROUPER breakout for {service} - {methodology}") grouper_results = run_grouper_breakout(service, methodology, filename) special_case_results.update(grouper_results) else: # Other/future special cases can go here diff --git a/fieldExtraction/src/investment/postprocessing_funcs.py b/fieldExtraction/src/investment/postprocessing_funcs.py index e4759d9..c2b2f72 100644 --- a/fieldExtraction/src/investment/postprocessing_funcs.py +++ b/fieldExtraction/src/investment/postprocessing_funcs.py @@ -598,7 +598,13 @@ def update_lob_for_duals(answer_dicts): claude_answer_extracted = string_utils.extract_text_from_delimiters( claude_answer_raw, Delimiter.PIPE ) - answer_dict["AARETE_DERIVED_LOB"] = claude_answer_extracted + if ( + claude_answer_extracted.lower() == "neither" + or claude_answer_extracted.lower() == "n/a" + ): # If neither, take original AARETE_DERIVED_LOB + pass + else: + answer_dict["AARETE_DERIVED_LOB"] = claude_answer_extracted final_answer_dicts.append(answer_dict) return final_answer_dicts @@ -743,50 +749,51 @@ def deduplicate_provider_columns(df: pd.DataFrame) -> pd.DataFrame: ) return df + def update_grouper_base_rate_and_grouper_pct_rate(df): """ Update GROUPER_PCT_RATE and GROUPER_BASE_RATE columns based on grouper conditions. - + Creates new columns and populates them when: - - AARETE_DERIVED_REIMB_METHOD is 'Grouper', OR + - AARETE_DERIVED_REIMB_METHOD is 'Grouper', OR - GROUPER_TYPE is not empty (using string_utils.is_empty) AND not empty array/empty array string, OR - GROUPER_CD is not empty (using string_utils.is_empty) AND not empty array/empty array string - + Args: df: DataFrame to update - + Returns: DataFrame with updated GROUPER_PCT_RATE and GROUPER_BASE_RATE columns """ # Required columns for the operation required_cols = [ - "AARETE_DERIVED_REIMB_METHOD", - "REIMB_PCT_RATE", - "REIMB_CONVERSION_FACTOR", - "GROUPER_TYPE" - "GROUPER_CD" + "AARETE_DERIVED_REIMB_METHOD", + "REIMB_PCT_RATE", + "REIMB_CONVERSION_FACTOR", + "GROUPER_TYPE" "GROUPER_CD", ] - + # Check if all required columns exist if not all(col in df.columns for col in required_cols): # Initialize empty columns and return if required columns are missing df["GROUPER_PCT_RATE"] = "" df["GROUPER_BASE_RATE"] = "" return df - + # Create grouper condition mask - grouper_mask = ( - (df["AARETE_DERIVED_REIMB_METHOD"] == "Grouper") & ( - (~string_utils.is_empty(df["GROUPER_TYPE"]) & df["GROUPER_TYPE"].ne("[]")) | - (~string_utils.is_empty(df["GROUPER_CD"]) & df["GROUPER_CD"].ne("[]")) - )) - + grouper_mask = (df["AARETE_DERIVED_REIMB_METHOD"] == "Grouper") & ( + (~string_utils.is_empty(df["GROUPER_TYPE"]) & df["GROUPER_TYPE"].ne("[]")) + | (~string_utils.is_empty(df["GROUPER_CD"]) & df["GROUPER_CD"].ne("[]")) + ) + # Initialize new columns with empty strings df["GROUPER_PCT_RATE"] = "" df["GROUPER_BASE_RATE"] = "" - + # Update values where grouper condition is met df.loc[grouper_mask, "GROUPER_PCT_RATE"] = df.loc[grouper_mask, "REIMB_PCT_RATE"] - df.loc[grouper_mask, "GROUPER_BASE_RATE"] = df.loc[grouper_mask, "REIMB_CONVERSION_FACTOR"] - - return df \ No newline at end of file + df.loc[grouper_mask, "GROUPER_BASE_RATE"] = df.loc[ + grouper_mask, "REIMB_CONVERSION_FACTOR" + ] + + return df diff --git a/fieldExtraction/src/prompts/prompt_templates.py b/fieldExtraction/src/prompts/prompt_templates.py index 416c08b..ee3d3b4 100644 --- a/fieldExtraction/src/prompts/prompt_templates.py +++ b/fieldExtraction/src/prompts/prompt_templates.py @@ -219,6 +219,7 @@ Here is the text to analyze: Explain each answer before putting the final answer in the JSON dictionary format. """ + def DYNAMIC_PRIMARY_HEADER(context, field_name, field_prompt): return f"""Extract attributes from the provided text, paying close attention to detail. @@ -238,6 +239,7 @@ Here is the text to analyze: Briefly explain your answer before putting the final answer in |pipes|. """ + def DYNAMIC_PRIMARY_TEXT(context, field_name, field_prompt): return f"""Extract attributes from the provided text, paying close attention to detail. @@ -770,6 +772,7 @@ def DUAL_LOB_CHECK(service_term): If the Service is only for Medicaid, return 'Medicaid' If the Service is only for Medicare, return 'Medicare' If the Service is for both Medicare and Medicaid, return 'Duals' +If the service is for neither Medicare nor Medicaid, return 'neither' If it is unclear, write 'N/A'. Here is the Service: @@ -1118,9 +1121,8 @@ Return your answers in valid JSON format using the exact field names from the qu **Important:** Base your answers ONLY on the outlier terms provided. Do not make assumptions or add information not present in the contract language. """ -def GROUPER_BREAKOUT( - service, methodology, questions -): + +def GROUPER_BREAKOUT(service, methodology, questions): """ Use this prompt for grouper fields that are derived directly from the combination of Service and Methodology @@ -1151,6 +1153,7 @@ Return your answers in valid JSON format using the exact field names from the qu }} """ + def LOB_RELATIONSHIP(exhibit_text, dynamic_primary_values): return f"""You will be presented with an exhibit from a Payer-Provider contract. @@ -1185,11 +1188,3 @@ def FULL_CONTEXT_ADDITIONAL_INSTRUCTIONS(allow_na): return " Only provide an answer if that answer clearly applies to the ENTIRE contract. Otherwise, answer N/A." else: return " Do NOT leave this field N/A. Choose the most appropriate answer based on the context." - - - - - - - -