From ddefa2ff30fb246d379274a10f84346e2bbc970d Mon Sep 17 00:00:00 2001 From: Praneel Panchigar Date: Fri, 20 Feb 2026 21:19:34 +0000 Subject: [PATCH] Merged in bugfix/default_ind_postprocess (pull request #887) Bugfix/default ind postprocess * Add logic to standardize UNIT_OF_MEASURE for flat-rate reimbursement methods - Implemented functionality in `standardize_reimb_method_and_fee_schedule` to set UNIT_OF_MEASURE to blank for rows where DEFAULT_IND is 'Y' and AARETE_DERIVED_REIMB_METHOD is 'flat rate'. - Added unit tests to verify behavior for various scenarios, including case insensitivity and non-default conditions. - Ensured that UNIT_OF_MEASURE remains unchanged for non-flat rate methods. * Enhance child rank handling and ensure column consistency in parent-child mapping - Added initialization for the `child_rank` column in both parents and children DataFrames to prevent KeyError during concatenation when no children exist. - Updated `cols_to_keep` in `parent_child_mapping` to filter out columns not present in `pc_df`, ensuring robustness in data processing. * Ran Black * made a small change in code_last_check, fixed so it returns string and not single char * Made changes to make sure that default_ind postprocess only happens to the cc output and not dashboard * Merged DEV into bugfix/default_ind_postprocess * Restore deleted AARETE_DERIVED_PAYER_NAME functions Functions were removed during previous commit. Restored from DEV to fix AttributeError in prompt_calls.py. * Simplify code_last_check return logic and add error logging - Simplified return to single line with fallback - Added error logging for failed LLM response parsing Approved-by: Siddhant Medar --- src/codes/code_funcs.py | 8 +- src/parent_child/pipeline.py | 6 ++ .../shared/postprocessing/postprocess.py | 3 + .../postprocessing/postprocessing_funcs.py | 21 +++++ src/prompts/prompt_templates.py | 15 ++-- src/tests/test_postprocess.py | 78 +++++++++++++++++++ 6 files changed, 123 insertions(+), 8 deletions(-) diff --git a/src/codes/code_funcs.py b/src/codes/code_funcs.py index 58d889f..e87bd33 100644 --- a/src/codes/code_funcs.py +++ b/src/codes/code_funcs.py @@ -489,11 +489,13 @@ def code_last_check(service, filename): ) try: llm_answer_final = _parser(llm_answer_raw) - return llm_answer_final[0] - except: + return llm_answer_final if llm_answer_final else "Generic" + except Exception: + logging.error( + "Failed to parse LLM response in code_last_check: %s", llm_answer_raw + ) return "Generic" - def fill_bill_type( service, answer_dict, diff --git a/src/parent_child/pipeline.py b/src/parent_child/pipeline.py index ba471eb..1b92bd1 100644 --- a/src/parent_child/pipeline.py +++ b/src/parent_child/pipeline.py @@ -743,6 +743,11 @@ def assign_child_ranks(df, grouper_col="grouping_key"): parents = df[df["parent"]].copy() children = df[~df["parent"]].copy() + # Ensure child_rank column exists so concat result has it when there are no + # children (otherwise result["child_rank"].astype(str) raises KeyError). + parents["child_rank"] = "" + children["child_rank"] = "" + # For each child, find their actual parent's identity # Then group children by the parent they're actually assigned to children["_parent_identity"] = None @@ -1096,6 +1101,7 @@ def parent_child_mapping( "parent_child_flag", "parent_name", ] + cols_to_keep = [col for col in cols_to_keep if col in pc_df.columns] # Ensure consistent types for merge - convert both to string to avoid datetime/object mismatch pc_df["fixed_effective_date"] = pc_df["fixed_effective_date"].astype(str) diff --git a/src/pipelines/shared/postprocessing/postprocess.py b/src/pipelines/shared/postprocessing/postprocess.py index 598fed6..3d0d5ac 100644 --- a/src/pipelines/shared/postprocessing/postprocess.py +++ b/src/pipelines/shared/postprocessing/postprocess.py @@ -150,6 +150,9 @@ def contract_config_postprocess(df, constants: Constants): if cc_df.shape[0] == 0: return cc_df + # CC only: blank UNIT_OF_MEASURE for default flat-rate rows (not for dashboard). + cc_df = postprocessing_funcs.blank_uom_for_default_flat_rate_cc(cc_df) + # Contract Config specific postprocessing steps: # Apply json formatting cc_df["PAYER_STATE"] = cc_df["PAYER_STATE"].apply( diff --git a/src/pipelines/shared/postprocessing/postprocessing_funcs.py b/src/pipelines/shared/postprocessing/postprocessing_funcs.py index b10cfc7..5fd9bb6 100644 --- a/src/pipelines/shared/postprocessing/postprocessing_funcs.py +++ b/src/pipelines/shared/postprocessing/postprocessing_funcs.py @@ -613,6 +613,27 @@ def standardize_reimb_method_and_fee_schedule( return df +def blank_uom_for_default_flat_rate_cc(df: pd.DataFrame) -> pd.DataFrame: + """ + CC output only: blank UNIT_OF_MEASURE when DEFAULT_IND='Y' and method is flat rate. + + Ensures default flat-rate reimbursements are not shown as per-unit in CC output. + Dashboard output is not modified (call this only in contract_config_postprocess). + """ + if not all( + c in df.columns + for c in ("DEFAULT_IND", "AARETE_DERIVED_REIMB_METHOD", "UNIT_OF_MEASURE") + ): + return df + default_ind_y = df["DEFAULT_IND"].astype(str).str.strip().str.upper() == "Y" + flat_rate_method = ( + df["AARETE_DERIVED_REIMB_METHOD"].astype(str).str.strip().str.lower() + == "flat rate" + ) + df.loc[default_ind_y & flat_rate_method, "UNIT_OF_MEASURE"] = "" + return df + + def process_patient_age_range(df): """ Process the 'PATIENT_AGE_RANGE' column into 'PATIENT_AGE_MIN' and 'PATIENT_AGE_MAX'. diff --git a/src/prompts/prompt_templates.py b/src/prompts/prompt_templates.py index 2e528e0..98d3bf4 100644 --- a/src/prompts/prompt_templates.py +++ b/src/prompts/prompt_templates.py @@ -1286,12 +1286,14 @@ Extract explicit procedure, revenue, diagnosis, and other healthcare codes from [INSTRUCTIONS] - For each field, return a list of all codes EXPLICITLY written for that field. The code itself must be written, not language that describes the code. +- FIELD ASSIGNMENT: Assign each code to exactly one field that matches its format. Do not put diagnosis codes (ICD-10: letter followed by digits, e.g. Z00.00, A01.1) in PROCEDURE_CD. Do not put procedure codes (CPT 5 digits or HCPCS 1 letter + 4 digits) in DIAG_CD. Revenue codes (3-4 digits, may end in X) belong only in REVENUE_CD. When in doubt, use format: PROCEDURE_CD = CPT/HCPCS only; DIAG_CD = ICD-10 only. - Note that codes may be present, but not explicitly labeled as codes. For example, you may simply see "J1098", which is a PROCEDURE_CD. You may see "155" which is a REVENUE_CD, etc. - If the text gives a range of codes, without listing each code in the range individually, return the range in the format 'LowestCode-HighestCode'. - If the text describes an entire Code Category (not one specific code) based on a start letter, return the Category in the form "Category: X". e.g. "A-Codes" --> "Category: A", "K-Codes" --> "Category K", etc. - If the text explicitly says that there is no published or established rate, write "NOT_ESTABLISHED" for the PROCEDURE_CD value. - If any of the codes are not found, do not write a list for that field. Simply populate the field with an empty list []. - If a standalone 3-digit code appears without explicit labels (e.g., "DRG," "revenue," "Rev code"), analyze the surrounding context for clues. Look for any direct or indirect references—no matter how subtle—that may suggest a connection to either a Revenue Code or a Grouper Code. If any contextual clues imply relevance to either classification, categorize the code accordingly. +- Return only valid codes that match the stated format (e.g. CPT exactly 5 digits, HCPCS 1 letter + 4 digits); do not invent or guess codes. [FIELDS] Populate a JSON dictionary with the following fields: @@ -1456,16 +1458,19 @@ Examples: ["Specific"] or ["Generic"] {JSON_LIST_FORMAT_INSTRUCTIONS}""" -def CODE_LAST_CHECK(service) -> Tuple[str, Callable[[str], list]]: +def CODE_LAST_CHECK(service) -> Tuple[str, Callable[[str], Any]]: """Call CODE_LAST_CHECK_INSTRUCTION() separately for the cached instruction. - Returns: - Tuple of (prompt_string, parser_function) where parser expects JSON list output. + Uses format-aware list parser with expected_format='str' so single-item list + (e.g. ['Specific'] or ['Generic']) is normalized to a string. Parser returns str. """ prompt = f"""[SERVICE TO ANALYZE] {service}""" - return (prompt, _json_list_parser) + return ( + prompt, + _create_json_list_parser(field_name="CODE_LAST_CHECK", expected_format="str"), + ) def FILL_BILL_TYPE_INSTRUCTION() -> str: @@ -2419,7 +2424,7 @@ def AARETE_DERIVED_PAYER_NAME_INSTRUCTION(): {{"AARETE_DERIVED_PAYER_NAME":"AvMed Health Plans"}} [OUTPUT FORMAT] - Return ONLY the JSON dictionary with no explanatory text. Use "AARETE_DERIVED_PAYER_NAME" as the key. + Return ONLY the JSON dictionary with no explanatory text. Use "AARETE_DERIVED_PAYER_NAME" as the key. {JSON_DICT_FORMAT_INSTRUCTIONS} """ diff --git a/src/tests/test_postprocess.py b/src/tests/test_postprocess.py index 687c256..97e3a87 100644 --- a/src/tests/test_postprocess.py +++ b/src/tests/test_postprocess.py @@ -7,6 +7,7 @@ from unittest.mock import MagicMock, patch import pandas as pd import pytest from src.pipelines.shared.postprocessing.postprocessing_funcs import ( + blank_uom_for_default_flat_rate_cc, clean_na_values, deduplicate_provider_columns, fill_claim_type_from_title, @@ -21,6 +22,7 @@ from src.pipelines.shared.postprocessing.postprocessing_funcs import ( remove_hyphens, remove_redundant_reimb_info, rename_columns, + standardize_reimb_method_and_fee_schedule, validate_and_reformat_date, ) from src.pipelines.shared.postprocessing.postprocess import standard_postprocess @@ -920,6 +922,82 @@ class TestPostprocessFunctions(unittest.TestCase): self.assertEqual(result_df.loc[1, "CONTRACT_TITLE"], "Another Valid") +class TestStandardizeReimbMethodAndFeeScheduleUOM(unittest.TestCase): + """Tests for UNIT_OF_MEASURE post-processing when DEFAULT_IND='Y' and flat rate.""" + + def test_cc_default_flat_rate_uom_blank(self): + """CC only: DEFAULT_IND='Y' and Flat Rate -> UNIT_OF_MEASURE blank.""" + df = pd.DataFrame( + { + "DEFAULT_IND": ["Y"], + "AARETE_DERIVED_REIMB_METHOD": ["Flat Rate"], + "UNIT_OF_MEASURE": ["Per Unit"], + } + ) + result = blank_uom_for_default_flat_rate_cc(df) + self.assertEqual(result.loc[0, "UNIT_OF_MEASURE"], "") + + def test_cc_default_flat_rate_case_insensitive(self): + """CC only: DEFAULT_IND='Y' and 'flat rate' (lowercase) -> UNIT_OF_MEASURE blank.""" + df = pd.DataFrame( + { + "DEFAULT_IND": ["Y"], + "AARETE_DERIVED_REIMB_METHOD": ["flat rate"], + "UNIT_OF_MEASURE": ["Per Visit"], + } + ) + result = blank_uom_for_default_flat_rate_cc(df) + self.assertEqual(result.loc[0, "UNIT_OF_MEASURE"], "") + + def test_standardize_does_not_blank_default_flat_rate(self): + """Shared path: default flat rate keeps UOM (blanking is CC-only).""" + constants = Constants() + df = pd.DataFrame( + { + "DEFAULT_IND": ["Y"], + "AARETE_DERIVED_REIMB_METHOD": ["Flat Rate"], + "UNIT_OF_MEASURE": ["Per Unit"], + "REIMB_TERM": [""], + } + ) + result = standardize_reimb_method_and_fee_schedule( + df, constants.VALID_UNIT_OF_MEASURE + ) + self.assertEqual(result.loc[0, "UNIT_OF_MEASURE"], "Per Unit") + + def test_non_default_flat_rate_uom_per_unit(self): + """DEFAULT_IND='N', flat rate, empty UOM -> UNIT_OF_MEASURE becomes Per Unit.""" + constants = Constants() + df = pd.DataFrame( + { + "DEFAULT_IND": ["N"], + "AARETE_DERIVED_REIMB_METHOD": ["Flat Rate"], + "UNIT_OF_MEASURE": [""], + "REIMB_TERM": [""], + } + ) + result = standardize_reimb_method_and_fee_schedule( + df, constants.VALID_UNIT_OF_MEASURE + ) + self.assertEqual(result.loc[0, "UNIT_OF_MEASURE"], "Per Unit") + + def test_default_non_flat_rate_uom_unchanged(self): + """DEFAULT_IND='Y' and Fee Schedule -> UNIT_OF_MEASURE unchanged.""" + constants = Constants() + df = pd.DataFrame( + { + "DEFAULT_IND": ["Y"], + "AARETE_DERIVED_REIMB_METHOD": ["Fee Schedule"], + "UNIT_OF_MEASURE": ["Per Visit"], + "REIMB_TERM": [""], + } + ) + result = standardize_reimb_method_and_fee_schedule( + df, constants.VALID_UNIT_OF_MEASURE + ) + self.assertEqual(result.loc[0, "UNIT_OF_MEASURE"], "Per Visit") + + class TestOutputFileStructure(unittest.TestCase): """Test that the new output file structure is generated correctly."""