Merged in bugfix/default_ind_postprocess (pull request #887)

Bugfix/default ind postprocess

* Add logic to standardize UNIT_OF_MEASURE for flat-rate reimbursement methods

- Implemented functionality in `standardize_reimb_method_and_fee_schedule` to set UNIT_OF_MEASURE to blank for rows where DEFAULT_IND is 'Y' and AARETE_DERIVED_REIMB_METHOD is 'flat rate'.
- Added unit tests to verify behavior for various scenarios, including case insensitivity and non-default conditions.
- Ensured that UNIT_OF_MEASURE remains unchanged for non-flat rate methods.

* Enhance child rank handling and ensure column consistency in parent-child mapping

- Added initialization for the `child_rank` column in both parents and children DataFrames to prevent KeyError during concatenation when no children exist.
- Updated `cols_to_keep` in `parent_child_mapping` to filter out columns not present in `pc_df`, ensuring robustness in data processing.

* Ran Black

* made a small change in code_last_check, fixed so it returns string and not single char

* Made changes to make sure that default_ind postprocess only happens to the cc output and not dashboard

* Merged DEV into bugfix/default_ind_postprocess

* Restore deleted AARETE_DERIVED_PAYER_NAME functions

Functions were removed during previous commit. Restored from DEV to fix
AttributeError in prompt_calls.py.

* Simplify code_last_check return logic and add error logging

- Simplified return to single line with fallback
- Added error logging for failed LLM response parsing


Approved-by: Siddhant Medar

src/codes/code_funcs.py

@@ -489,11 +489,13 @@ def code_last_check(service, filename):
     )
     try:
         llm_answer_final = _parser(llm_answer_raw)
-        return llm_answer_final[0]
-    except:
+        return llm_answer_final if llm_answer_final else "Generic"
+    except Exception:
+        logging.error(
+            "Failed to parse LLM response in code_last_check: %s", llm_answer_raw
+        )
         return "Generic"
 
-
 def fill_bill_type(
     service,
     answer_dict,

src/parent_child/pipeline.py

@@ -743,6 +743,11 @@ def assign_child_ranks(df, grouper_col="grouping_key"):
     parents = df[df["parent"]].copy()
     children = df[~df["parent"]].copy()
 
+    # Ensure child_rank column exists so concat result has it when there are no
+    # children (otherwise result["child_rank"].astype(str) raises KeyError).
+    parents["child_rank"] = ""
+    children["child_rank"] = ""
+
     # For each child, find their actual parent's identity
     # Then group children by the parent they're actually assigned to
     children["_parent_identity"] = None
@@ -1096,6 +1101,7 @@ def parent_child_mapping(
         "parent_child_flag",
         "parent_name",
     ]
+    cols_to_keep = [col for col in cols_to_keep if col in pc_df.columns]
 
     # Ensure consistent types for merge - convert both to string to avoid datetime/object mismatch
     pc_df["fixed_effective_date"] = pc_df["fixed_effective_date"].astype(str)

src/pipelines/shared/postprocessing/postprocess.py

@@ -150,6 +150,9 @@ def contract_config_postprocess(df, constants: Constants):
     if cc_df.shape[0] == 0:
         return cc_df
 
+    # CC only: blank UNIT_OF_MEASURE for default flat-rate rows (not for dashboard).
+    cc_df = postprocessing_funcs.blank_uom_for_default_flat_rate_cc(cc_df)
+
     # Contract Config specific postprocessing steps:
     # Apply json formatting
     cc_df["PAYER_STATE"] = cc_df["PAYER_STATE"].apply(

src/pipelines/shared/postprocessing/postprocessing_funcs.py

@@ -613,6 +613,27 @@ def standardize_reimb_method_and_fee_schedule(
     return df
 
 
+def blank_uom_for_default_flat_rate_cc(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    CC output only: blank UNIT_OF_MEASURE when DEFAULT_IND='Y' and method is flat rate.
+
+    Ensures default flat-rate reimbursements are not shown as per-unit in CC output.
+    Dashboard output is not modified (call this only in contract_config_postprocess).
+    """
+    if not all(
+        c in df.columns
+        for c in ("DEFAULT_IND", "AARETE_DERIVED_REIMB_METHOD", "UNIT_OF_MEASURE")
+    ):
+        return df
+    default_ind_y = df["DEFAULT_IND"].astype(str).str.strip().str.upper() == "Y"
+    flat_rate_method = (
+        df["AARETE_DERIVED_REIMB_METHOD"].astype(str).str.strip().str.lower()
+        == "flat rate"
+    )
+    df.loc[default_ind_y & flat_rate_method, "UNIT_OF_MEASURE"] = ""
+    return df
+
+
 def process_patient_age_range(df):
     """
     Process the 'PATIENT_AGE_RANGE' column into 'PATIENT_AGE_MIN' and 'PATIENT_AGE_MAX'.

src/prompts/prompt_templates.py

@@ -1286,12 +1286,14 @@ Extract explicit procedure, revenue, diagnosis, and other healthcare codes from
 
 [INSTRUCTIONS]
 - For each field, return a list of all codes EXPLICITLY written for that field. The code itself must be written, not language that describes the code.
+- FIELD ASSIGNMENT: Assign each code to exactly one field that matches its format. Do not put diagnosis codes (ICD-10: letter followed by digits, e.g. Z00.00, A01.1) in PROCEDURE_CD. Do not put procedure codes (CPT 5 digits or HCPCS 1 letter + 4 digits) in DIAG_CD. Revenue codes (3-4 digits, may end in X) belong only in REVENUE_CD. When in doubt, use format: PROCEDURE_CD = CPT/HCPCS only; DIAG_CD = ICD-10 only.
 - Note that codes may be present, but not explicitly labeled as codes. For example, you may simply see "J1098", which is a PROCEDURE_CD. You may see "155" which is a REVENUE_CD, etc.
 - If the text gives a range of codes, without listing each code in the range individually, return the range in the format 'LowestCode-HighestCode'.
 - If the text describes an entire Code Category (not one specific code) based on a start letter, return the Category in the form "Category: X". e.g. "A-Codes" --> "Category: A", "K-Codes" --> "Category K", etc.
 - If the text explicitly says that there is no published or established rate, write "NOT_ESTABLISHED" for the PROCEDURE_CD value.
 - If any of the codes are not found, do not write a list for that field. Simply populate the field with an empty list [].
 - If a standalone 3-digit code appears without explicit labels (e.g., "DRG," "revenue," "Rev code"), analyze the surrounding context for clues. Look for any direct or indirect references—no matter how subtle—that may suggest a connection to either a Revenue Code or a Grouper Code. If any contextual clues imply relevance to either classification, categorize the code accordingly.
+- Return only valid codes that match the stated format (e.g. CPT exactly 5 digits, HCPCS 1 letter + 4 digits); do not invent or guess codes.
 
 [FIELDS]
 Populate a JSON dictionary with the following fields:
@@ -1456,16 +1458,19 @@ Examples: ["Specific"] or ["Generic"]
 {JSON_LIST_FORMAT_INSTRUCTIONS}"""
 
 
-def CODE_LAST_CHECK(service) -> Tuple[str, Callable[[str], list]]:
+def CODE_LAST_CHECK(service) -> Tuple[str, Callable[[str], Any]]:
     """Call CODE_LAST_CHECK_INSTRUCTION() separately for the cached instruction.
 
-    Returns:
-        Tuple of (prompt_string, parser_function) where parser expects JSON list output.
+    Uses format-aware list parser with expected_format='str' so single-item list
+    (e.g. ['Specific'] or ['Generic']) is normalized to a string. Parser returns str.
     """
     prompt = f"""[SERVICE TO ANALYZE]
 {service}"""
 
-    return (prompt, _json_list_parser)
+    return (
+        prompt,
+        _create_json_list_parser(field_name="CODE_LAST_CHECK", expected_format="str"),
+    )
 
 
 def FILL_BILL_TYPE_INSTRUCTION() -> str:
@@ -2419,7 +2424,7 @@ def AARETE_DERIVED_PAYER_NAME_INSTRUCTION():
     {{"AARETE_DERIVED_PAYER_NAME":"AvMed Health Plans"}}
 
     [OUTPUT FORMAT]
-    Return ONLY the JSON dictionary with no explanatory text. Use "AARETE_DERIVED_PAYER_NAME" as the key. 
+    Return ONLY the JSON dictionary with no explanatory text. Use "AARETE_DERIVED_PAYER_NAME" as the key.
     {JSON_DICT_FORMAT_INSTRUCTIONS}
     """
 

src/tests/test_postprocess.py

@@ -7,6 +7,7 @@ from unittest.mock import MagicMock, patch
 import pandas as pd
 import pytest
 from src.pipelines.shared.postprocessing.postprocessing_funcs import (
+    blank_uom_for_default_flat_rate_cc,
     clean_na_values,
     deduplicate_provider_columns,
     fill_claim_type_from_title,
@@ -21,6 +22,7 @@ from src.pipelines.shared.postprocessing.postprocessing_funcs import (
     remove_hyphens,
     remove_redundant_reimb_info,
     rename_columns,
+    standardize_reimb_method_and_fee_schedule,
     validate_and_reformat_date,
 )
 from src.pipelines.shared.postprocessing.postprocess import standard_postprocess
@@ -920,6 +922,82 @@ class TestPostprocessFunctions(unittest.TestCase):
         self.assertEqual(result_df.loc[1, "CONTRACT_TITLE"], "Another Valid")
 
 
+class TestStandardizeReimbMethodAndFeeScheduleUOM(unittest.TestCase):
+    """Tests for UNIT_OF_MEASURE post-processing when DEFAULT_IND='Y' and flat rate."""
+
+    def test_cc_default_flat_rate_uom_blank(self):
+        """CC only: DEFAULT_IND='Y' and Flat Rate -> UNIT_OF_MEASURE blank."""
+        df = pd.DataFrame(
+            {
+                "DEFAULT_IND": ["Y"],
+                "AARETE_DERIVED_REIMB_METHOD": ["Flat Rate"],
+                "UNIT_OF_MEASURE": ["Per Unit"],
+            }
+        )
+        result = blank_uom_for_default_flat_rate_cc(df)
+        self.assertEqual(result.loc[0, "UNIT_OF_MEASURE"], "")
+
+    def test_cc_default_flat_rate_case_insensitive(self):
+        """CC only: DEFAULT_IND='Y' and 'flat rate' (lowercase) -> UNIT_OF_MEASURE blank."""
+        df = pd.DataFrame(
+            {
+                "DEFAULT_IND": ["Y"],
+                "AARETE_DERIVED_REIMB_METHOD": ["flat rate"],
+                "UNIT_OF_MEASURE": ["Per Visit"],
+            }
+        )
+        result = blank_uom_for_default_flat_rate_cc(df)
+        self.assertEqual(result.loc[0, "UNIT_OF_MEASURE"], "")
+
+    def test_standardize_does_not_blank_default_flat_rate(self):
+        """Shared path: default flat rate keeps UOM (blanking is CC-only)."""
+        constants = Constants()
+        df = pd.DataFrame(
+            {
+                "DEFAULT_IND": ["Y"],
+                "AARETE_DERIVED_REIMB_METHOD": ["Flat Rate"],
+                "UNIT_OF_MEASURE": ["Per Unit"],
+                "REIMB_TERM": [""],
+            }
+        )
+        result = standardize_reimb_method_and_fee_schedule(
+            df, constants.VALID_UNIT_OF_MEASURE
+        )
+        self.assertEqual(result.loc[0, "UNIT_OF_MEASURE"], "Per Unit")
+
+    def test_non_default_flat_rate_uom_per_unit(self):
+        """DEFAULT_IND='N', flat rate, empty UOM -> UNIT_OF_MEASURE becomes Per Unit."""
+        constants = Constants()
+        df = pd.DataFrame(
+            {
+                "DEFAULT_IND": ["N"],
+                "AARETE_DERIVED_REIMB_METHOD": ["Flat Rate"],
+                "UNIT_OF_MEASURE": [""],
+                "REIMB_TERM": [""],
+            }
+        )
+        result = standardize_reimb_method_and_fee_schedule(
+            df, constants.VALID_UNIT_OF_MEASURE
+        )
+        self.assertEqual(result.loc[0, "UNIT_OF_MEASURE"], "Per Unit")
+
+    def test_default_non_flat_rate_uom_unchanged(self):
+        """DEFAULT_IND='Y' and Fee Schedule -> UNIT_OF_MEASURE unchanged."""
+        constants = Constants()
+        df = pd.DataFrame(
+            {
+                "DEFAULT_IND": ["Y"],
+                "AARETE_DERIVED_REIMB_METHOD": ["Fee Schedule"],
+                "UNIT_OF_MEASURE": ["Per Visit"],
+                "REIMB_TERM": [""],
+            }
+        )
+        result = standardize_reimb_method_and_fee_schedule(
+            df, constants.VALID_UNIT_OF_MEASURE
+        )
+        self.assertEqual(result.loc[0, "UNIT_OF_MEASURE"], "Per Visit")
+
+
 class TestOutputFileStructure(unittest.TestCase):
     """Test that the new output file structure is generated correctly."""