From 04a190d5b7436bb1ab41183000c0e332704bf67f Mon Sep 17 00:00:00 2001 From: Karan Desai Date: Mon, 5 Jan 2026 18:34:18 +0000 Subject: [PATCH] Merged in bugfix/hmk-issues (pull request #824) Bugfix/hmk issues * fix for missing AARETE_DERIVED_LOB * fix for AARETE_DERIVED_SID * bugfix in universal_json_load * bugfix for normalize_state_field * reimb eff date assignmnet prompt update * prompt update for missing reimb terms * Merged main into bugfix/hmk-issues * test case for universal_json_loads * reveresed the prompt update for missing reimb * prompt fix for reimb dates Approved-by: Katon Minhas --- .../src/investment/postprocessing_funcs.py | 84 ++++++++++++------- .../src/prompts/prompt_templates.py | 8 ++ fieldExtraction/src/utils/string_utils.py | 20 ++++- fieldExtraction/tests/test_string_utils.py | 27 ++++++ 4 files changed, 105 insertions(+), 34 deletions(-) diff --git a/fieldExtraction/src/investment/postprocessing_funcs.py b/fieldExtraction/src/investment/postprocessing_funcs.py index 19aaa1c..745c9b6 100644 --- a/fieldExtraction/src/investment/postprocessing_funcs.py +++ b/fieldExtraction/src/investment/postprocessing_funcs.py @@ -823,40 +823,60 @@ def fill_empty_dynamic(df): # Make a copy to avoid SettingWithCopyWarning result_df = df.copy() + # Check if AARETE_DERIVED_LOB column exists + has_lob_column = "AARETE_DERIVED_LOB" in result_df.columns + # First, handle AARETE_DERIVED_LOB by grouping on FILE_NAME and EXHIBIT_PAGE only - for (file_name, exhibit_page), group in result_df.groupby(["FILE_NAME", "EXHIBIT_PAGE"]): - # Get non-NA LOB values - column = "AARETE_DERIVED_LOB" - non_na_values = [val for val in group[column].unique() if not string_utils.is_empty(val)] - - # If there's exactly one unique non-NA value, fill NA values with it - if len(non_na_values) == 1: - fill_value = non_na_values[0] - # Only apply to rows with this FILE_NAME and EXHIBIT_PAGE combination - mask = (result_df["FILE_NAME"] == file_name) & \ - (result_df["EXHIBIT_PAGE"] == exhibit_page) & \ - result_df[column].apply(string_utils.is_empty) - result_df.loc[mask, column] = fill_value - - # Then, handle other columns by grouping on FILE_NAME, EXHIBIT_PAGE, and AARETE_DERIVED_LOB - for (file_name, exhibit_page, lob), group in result_df.groupby(["FILE_NAME", "EXHIBIT_PAGE", "AARETE_DERIVED_LOB"]): - for column in columns_to_fill: - # Skip AARETE_DERIVED_LOB as we already processed it - if column == "AARETE_DERIVED_LOB": - continue - - # Get non-NA values + if has_lob_column: + for (file_name, exhibit_page), group in result_df.groupby(["FILE_NAME", "EXHIBIT_PAGE"]): + # Get non-NA LOB values + column = "AARETE_DERIVED_LOB" non_na_values = [val for val in group[column].unique() if not string_utils.is_empty(val)] - + # If there's exactly one unique non-NA value, fill NA values with it if len(non_na_values) == 1: fill_value = non_na_values[0] - # Only apply to rows with this FILE_NAME, EXHIBIT_PAGE, and LOB combination + # Only apply to rows with this FILE_NAME and EXHIBIT_PAGE combination mask = (result_df["FILE_NAME"] == file_name) & \ (result_df["EXHIBIT_PAGE"] == exhibit_page) & \ - (result_df["AARETE_DERIVED_LOB"] == lob) & \ result_df[column].apply(string_utils.is_empty) result_df.loc[mask, column] = fill_value + + # Then, handle other columns by grouping on FILE_NAME, EXHIBIT_PAGE, and AARETE_DERIVED_LOB (if it exists) + if has_lob_column: + for (file_name, exhibit_page, lob), group in result_df.groupby(["FILE_NAME", "EXHIBIT_PAGE", "AARETE_DERIVED_LOB"]): + for column in columns_to_fill: + # Skip AARETE_DERIVED_LOB as we already processed it + if column == "AARETE_DERIVED_LOB": + continue + + # Get non-NA values + non_na_values = [val for val in group[column].unique() if not string_utils.is_empty(val)] + + # If there's exactly one unique non-NA value, fill NA values with it + if len(non_na_values) == 1: + fill_value = non_na_values[0] + # Only apply to rows with this FILE_NAME, EXHIBIT_PAGE, and LOB combination + mask = (result_df["FILE_NAME"] == file_name) & \ + (result_df["EXHIBIT_PAGE"] == exhibit_page) & \ + (result_df["AARETE_DERIVED_LOB"] == lob) & \ + result_df[column].apply(string_utils.is_empty) + result_df.loc[mask, column] = fill_value + else: + # No LOB column, just group by FILE_NAME and EXHIBIT_PAGE + for (file_name, exhibit_page), group in result_df.groupby(["FILE_NAME", "EXHIBIT_PAGE"]): + for column in columns_to_fill: + # Get non-NA values + non_na_values = [val for val in group[column].unique() if not string_utils.is_empty(val)] + + # If there's exactly one unique non-NA value, fill NA values with it + if len(non_na_values) == 1: + fill_value = non_na_values[0] + # Only apply to rows with this FILE_NAME and EXHIBIT_PAGE combination + mask = (result_df["FILE_NAME"] == file_name) & \ + (result_df["EXHIBIT_PAGE"] == exhibit_page) & \ + result_df[column].apply(string_utils.is_empty) + result_df.loc[mask, column] = fill_value return result_df @@ -895,17 +915,21 @@ def attach_sid_column(df: pd.DataFrame) -> pd.DataFrame: # Make a copy to avoid mutating the original DataFrame df = df.copy() - # Ensure all SID columns exist; create missing ones filled with 'xx' for col in sid_cols: if col not in df.columns: df[col] = '' # Create the concatenated SID column (replace blanks/NaN only here) - df['AARETE_DERIVED_SID'] = ( - df.assign(**{col: df[col].fillna('').astype(str) for col in sid_cols}) - .apply(lambda row: '__'.join(val if val.strip() else 'xx' for val in row[sid_cols]), axis=1) - ) + if df.empty: + df['AARETE_DERIVED_SID'] = pd.Series(dtype=str) + else: + sid_df = df[sid_cols].fillna('').astype(str) + df['AARETE_DERIVED_SID'] = sid_df.apply( + lambda row: '__'.join(val if val.strip() else 'xx' for val in row), + axis=1, + result_type='reduce' + ) return df diff --git a/fieldExtraction/src/prompts/prompt_templates.py b/fieldExtraction/src/prompts/prompt_templates.py index 55e07b8..db903cf 100644 --- a/fieldExtraction/src/prompts/prompt_templates.py +++ b/fieldExtraction/src/prompts/prompt_templates.py @@ -285,6 +285,13 @@ CRITICAL RULES FOR DATE ASSIGNMENT: 6. **NO DATES IN SECTION**: If no specific date range appears near this reimbursement term (not in the same paragraph or table section), return "N/A" +7. **DATE FORMAT PRIORITY**: + Always prioritize regular date formats over CY/FY formats. + - Regular formats: "July 1, 2018", "January 1, 2018 through December 31, 2018", etc. + - CY/FY formats (lower priority): "FY2018", "CY2019", "CY2018-CY2019", etc. + + If a regular date exists in the section, use it. If ONLY CY/FY format is available, return "N/A". + [REIMB_DATES FIELD DEFINITION] {field_prompt} @@ -1242,6 +1249,7 @@ Answer YES only if the exhibit constraint adds NO additional limitation beyond w Briefly explain your reasoning, then return YES or NO in |pipes|.""" + def CARVEOUT_CHECK(service_term: str, reimb_term: str, carveout_definitions, special_case_definitions): return f""" [OBJECTIVE] diff --git a/fieldExtraction/src/utils/string_utils.py b/fieldExtraction/src/utils/string_utils.py index d091e31..b9e309c 100644 --- a/fieldExtraction/src/utils/string_utils.py +++ b/fieldExtraction/src/utils/string_utils.py @@ -197,8 +197,19 @@ def universal_json_load(s: str): if s[i] in '{[': start = i stack = [s[i]] - for j in range(i+1, len(s)): - if s[j] in '{[': + j = i + 1 + while j < len(s): + # Skip over string literals to avoid counting brackets inside strings + if s[j] == '"': + j += 1 + while j < len(s): + if s[j] == '\\' and j + 1 < len(s): + j += 2 # Skip escaped character + continue + if s[j] == '"': + break + j += 1 + elif s[j] in '{[': stack.append(s[j]) elif s[j] in '}]': if not stack: @@ -218,6 +229,7 @@ def universal_json_load(s: str): pass i = j # Move past this top-level object break + j += 1 else: pass i += 1 @@ -675,8 +687,8 @@ def parse_raw_state_value(raw_value): except (ValueError, SyntaxError): pass # Not valid Python literal, continue with string parsing - # Regular string splitting on delimiters - states = re.split(r'[|,;\s]+', stripped) + # Regular string splitting on delimiters (not whitespace - state names can have spaces like "West Virginia") + states = re.split(r'[|,;]+', stripped) return [s.strip() for s in states if s.strip()] if isinstance(raw_value, list): diff --git a/fieldExtraction/tests/test_string_utils.py b/fieldExtraction/tests/test_string_utils.py index 11eb5a3..029edc7 100644 --- a/fieldExtraction/tests/test_string_utils.py +++ b/fieldExtraction/tests/test_string_utils.py @@ -179,6 +179,7 @@ class TestStringUtils(unittest.TestCase): self.assertEqual(string_utils.flatten_to_strings("single"), ["single"]) self.assertEqual(string_utils.flatten_to_strings([1, [2, 3], "text"]), ["1", "2", "3", "text"]) self.assertEqual(string_utils.flatten_to_strings([" text ", [" nested "]]), ["text", "nested"]) + def test_universal_json_load_multiple_invalid(self): # Multiple invalid JSON objects s = '{broken: true} some text {not: valid} more text' @@ -200,6 +201,32 @@ class TestStringUtils(unittest.TestCase): s = '{"simple": true} middle {"outer": {"inner": [1, 2]}} end' self.assertEqual(string_utils.universal_json_load(s), {"outer": {"inner": [1, 2]}}) + def test_universal_json_load_brackets_in_strings(self): + # Brackets inside string values should be ignored during bracket matching + s = '{"text": "array [1] and {2}"}' + self.assertEqual(string_utils.universal_json_load(s), {"text": "array [1] and {2}"}) + # Double brackets in strings (the original bug case) + s = '{"field": "Exhibit [[ to Attachment ]"}' + self.assertEqual(string_utils.universal_json_load(s), {"field": "Exhibit [[ to Attachment ]"}) + # Unbalanced brackets in strings + s = '{"text": "missing close [ bracket"}' + self.assertEqual(string_utils.universal_json_load(s), {"text": "missing close [ bracket"}) + # Curly braces in strings + s = '{"text": "object {key: value} here"}' + self.assertEqual(string_utils.universal_json_load(s), {"text": "object {key: value} here"}) + # Escaped quotes should not break string detection + s = '{"text": "say \\"hello\\""}' + self.assertEqual(string_utils.universal_json_load(s), {"text": 'say "hello"'}) + # Escaped quotes with brackets + s = '{"text": "say \\"hello [world]\\"" }' + self.assertEqual(string_utils.universal_json_load(s), {"text": 'say "hello [world]"'}) + # Real-world LLM response case with markdown and brackets in JSON strings + raw_input = 'Here is the analysis:\n\n```json\n[{"SERVICE_TERM": "Inpatient services", "REIMB_TERM": "100% of rates in Exhibit [[ to Attachment ]"}, {"SERVICE_TERM": "Outpatient services", "REIMB_TERM": "63% of rates in Exhibit [[ to Attachment ]"}]\n```' + result = string_utils.universal_json_load(raw_input) + self.assertEqual(len(result), 2) + self.assertEqual(result[0]["SERVICE_TERM"], "Inpatient services") + self.assertIn("Exhibit [[", result[0]["REIMB_TERM"]) + if __name__ == "__main__": unittest.main()