Merged in bugfix/hmk-issues (pull request #824)

Bugfix/hmk issues

* fix for missing AARETE_DERIVED_LOB

* fix for AARETE_DERIVED_SID

* bugfix in universal_json_load

* bugfix for normalize_state_field

*  reimb eff date assignmnet prompt update

* prompt update for missing reimb terms

* Merged main into bugfix/hmk-issues

* test case for universal_json_loads

* reveresed the prompt update for missing reimb

* prompt fix for reimb dates


Approved-by: Katon Minhas

fieldExtraction/src/investment/postprocessing_funcs.py

@@ -823,40 +823,60 @@ def fill_empty_dynamic(df):
     # Make a copy to avoid SettingWithCopyWarning
     result_df = df.copy()
     
+    # Check if AARETE_DERIVED_LOB column exists
+    has_lob_column = "AARETE_DERIVED_LOB" in result_df.columns
+
     # First, handle AARETE_DERIVED_LOB by grouping on FILE_NAME and EXHIBIT_PAGE only
-    for (file_name, exhibit_page), group in result_df.groupby(["FILE_NAME", "EXHIBIT_PAGE"]):
+    if has_lob_column:
-        # Get non-NA LOB values
+        for (file_name, exhibit_page), group in result_df.groupby(["FILE_NAME", "EXHIBIT_PAGE"]):
-        column = "AARETE_DERIVED_LOB"
+            # Get non-NA LOB values
-        non_na_values = [val for val in group[column].unique() if not string_utils.is_empty(val)]
+            column = "AARETE_DERIVED_LOB"
-        
-        # If there's exactly one unique non-NA value, fill NA values with it
-        if len(non_na_values) == 1:
-            fill_value = non_na_values[0]
-            # Only apply to rows with this FILE_NAME and EXHIBIT_PAGE combination
-            mask = (result_df["FILE_NAME"] == file_name) & \
-                   (result_df["EXHIBIT_PAGE"] == exhibit_page) & \
-                   result_df[column].apply(string_utils.is_empty)
-            result_df.loc[mask, column] = fill_value
-    
-    # Then, handle other columns by grouping on FILE_NAME, EXHIBIT_PAGE, and AARETE_DERIVED_LOB
-    for (file_name, exhibit_page, lob), group in result_df.groupby(["FILE_NAME", "EXHIBIT_PAGE", "AARETE_DERIVED_LOB"]):
-        for column in columns_to_fill:
-            # Skip AARETE_DERIVED_LOB as we already processed it
-            if column == "AARETE_DERIVED_LOB":
-                continue
-                
-            # Get non-NA values
             non_na_values = [val for val in group[column].unique() if not string_utils.is_empty(val)]
-            
+
             # If there's exactly one unique non-NA value, fill NA values with it
             if len(non_na_values) == 1:
                 fill_value = non_na_values[0]
-                # Only apply to rows with this FILE_NAME, EXHIBIT_PAGE, and LOB combination
+                # Only apply to rows with this FILE_NAME and EXHIBIT_PAGE combination
                 mask = (result_df["FILE_NAME"] == file_name) & \
                        (result_df["EXHIBIT_PAGE"] == exhibit_page) & \
-                       (result_df["AARETE_DERIVED_LOB"] == lob) & \
                        result_df[column].apply(string_utils.is_empty)
                 result_df.loc[mask, column] = fill_value
+
+    # Then, handle other columns by grouping on FILE_NAME, EXHIBIT_PAGE, and AARETE_DERIVED_LOB (if it exists)
+    if has_lob_column:
+        for (file_name, exhibit_page, lob), group in result_df.groupby(["FILE_NAME", "EXHIBIT_PAGE", "AARETE_DERIVED_LOB"]):
+            for column in columns_to_fill:
+                # Skip AARETE_DERIVED_LOB as we already processed it
+                if column == "AARETE_DERIVED_LOB":
+                    continue
+
+                # Get non-NA values
+                non_na_values = [val for val in group[column].unique() if not string_utils.is_empty(val)]
+
+                # If there's exactly one unique non-NA value, fill NA values with it
+                if len(non_na_values) == 1:
+                    fill_value = non_na_values[0]
+                    # Only apply to rows with this FILE_NAME, EXHIBIT_PAGE, and LOB combination
+                    mask = (result_df["FILE_NAME"] == file_name) & \
+                           (result_df["EXHIBIT_PAGE"] == exhibit_page) & \
+                           (result_df["AARETE_DERIVED_LOB"] == lob) & \
+                           result_df[column].apply(string_utils.is_empty)
+                    result_df.loc[mask, column] = fill_value
+    else:
+        # No LOB column, just group by FILE_NAME and EXHIBIT_PAGE
+        for (file_name, exhibit_page), group in result_df.groupby(["FILE_NAME", "EXHIBIT_PAGE"]):
+            for column in columns_to_fill:
+                # Get non-NA values
+                non_na_values = [val for val in group[column].unique() if not string_utils.is_empty(val)]
+
+                # If there's exactly one unique non-NA value, fill NA values with it
+                if len(non_na_values) == 1:
+                    fill_value = non_na_values[0]
+                    # Only apply to rows with this FILE_NAME and EXHIBIT_PAGE combination
+                    mask = (result_df["FILE_NAME"] == file_name) & \
+                           (result_df["EXHIBIT_PAGE"] == exhibit_page) & \
+                           result_df[column].apply(string_utils.is_empty)
+                    result_df.loc[mask, column] = fill_value
                 
     return result_df
 
@@ -895,17 +915,21 @@ def attach_sid_column(df: pd.DataFrame) -> pd.DataFrame:
 
     # Make a copy to avoid mutating the original DataFrame
     df = df.copy()
-
     # Ensure all SID columns exist; create missing ones filled with 'xx'
     for col in sid_cols:
         if col not in df.columns:
             df[col] = ''
 
     # Create the concatenated SID column (replace blanks/NaN only here)
-    df['AARETE_DERIVED_SID'] = (
+    if df.empty:
-        df.assign(**{col: df[col].fillna('').astype(str) for col in sid_cols})
+        df['AARETE_DERIVED_SID'] = pd.Series(dtype=str)
-          .apply(lambda row: '__'.join(val if val.strip() else 'xx' for val in row[sid_cols]), axis=1)
+    else:
-    )
+        sid_df = df[sid_cols].fillna('').astype(str)
+        df['AARETE_DERIVED_SID'] = sid_df.apply(
+            lambda row: '__'.join(val if val.strip() else 'xx' for val in row),
+            axis=1,
+            result_type='reduce'
+        )
 
     return df
 

fieldExtraction/src/prompts/prompt_templates.py

@@ -285,6 +285,13 @@ CRITICAL RULES FOR DATE ASSIGNMENT:
 6. **NO DATES IN SECTION**:
    If no specific date range appears near this reimbursement term (not in the same paragraph or table section), return "N/A"
 
+7. **DATE FORMAT PRIORITY**:
+   Always prioritize regular date formats over CY/FY formats.
+   - Regular formats: "July 1, 2018", "January 1, 2018 through December 31, 2018", etc.
+   - CY/FY formats (lower priority): "FY2018", "CY2019", "CY2018-CY2019", etc.
+
+   If a regular date exists in the section, use it. If ONLY CY/FY format is available, return "N/A".
+
 [REIMB_DATES FIELD DEFINITION]
 {field_prompt}
 
@@ -1242,6 +1249,7 @@ Answer YES only if the exhibit constraint adds NO additional limitation beyond w
 Briefly explain your reasoning, then return YES or NO in |pipes|."""
 
 
+
 def CARVEOUT_CHECK(service_term: str, reimb_term: str, carveout_definitions, special_case_definitions):
     return f"""
 [OBJECTIVE]

fieldExtraction/src/utils/string_utils.py

@@ -197,8 +197,19 @@ def universal_json_load(s: str):
         if s[i] in '{[':
             start = i
             stack = [s[i]]
-            for j in range(i+1, len(s)):
+            j = i + 1
-                if s[j] in '{[':
+            while j < len(s):
+                # Skip over string literals to avoid counting brackets inside strings
+                if s[j] == '"':
+                    j += 1
+                    while j < len(s):
+                        if s[j] == '\\' and j + 1 < len(s):
+                            j += 2  # Skip escaped character
+                            continue
+                        if s[j] == '"':
+                            break
+                        j += 1
+                elif s[j] in '{[':
                     stack.append(s[j])
                 elif s[j] in '}]':
                     if not stack:
@@ -218,6 +229,7 @@ def universal_json_load(s: str):
                             pass
                         i = j  # Move past this top-level object
                         break
+                j += 1
             else:
                 pass
         i += 1
@@ -675,8 +687,8 @@ def parse_raw_state_value(raw_value):
             except (ValueError, SyntaxError):
                 pass  # Not valid Python literal, continue with string parsing
         
-        # Regular string splitting on delimiters
+        # Regular string splitting on delimiters (not whitespace - state names can have spaces like "West Virginia")
-        states = re.split(r'[|,;\s]+', stripped)
+        states = re.split(r'[|,;]+', stripped)
         return [s.strip() for s in states if s.strip()]
     
     if isinstance(raw_value, list):

fieldExtraction/tests/test_string_utils.py

@@ -179,6 +179,7 @@ class TestStringUtils(unittest.TestCase):
         self.assertEqual(string_utils.flatten_to_strings("single"), ["single"])
         self.assertEqual(string_utils.flatten_to_strings([1, [2, 3], "text"]), ["1", "2", "3", "text"])
         self.assertEqual(string_utils.flatten_to_strings([" text ", ["  nested  "]]), ["text", "nested"])
+
     def test_universal_json_load_multiple_invalid(self):
         # Multiple invalid JSON objects
         s = '{broken: true} some text {not: valid} more text'
@@ -200,6 +201,32 @@ class TestStringUtils(unittest.TestCase):
         s = '{"simple": true} middle {"outer": {"inner": [1, 2]}} end'
         self.assertEqual(string_utils.universal_json_load(s), {"outer": {"inner": [1, 2]}})
 
+    def test_universal_json_load_brackets_in_strings(self):
+        # Brackets inside string values should be ignored during bracket matching
+        s = '{"text": "array [1] and {2}"}'
+        self.assertEqual(string_utils.universal_json_load(s), {"text": "array [1] and {2}"})
+        # Double brackets in strings (the original bug case)
+        s = '{"field": "Exhibit [[ to Attachment ]"}'
+        self.assertEqual(string_utils.universal_json_load(s), {"field": "Exhibit [[ to Attachment ]"})
+        # Unbalanced brackets in strings
+        s = '{"text": "missing close [ bracket"}'
+        self.assertEqual(string_utils.universal_json_load(s), {"text": "missing close [ bracket"})
+        # Curly braces in strings
+        s = '{"text": "object {key: value} here"}'
+        self.assertEqual(string_utils.universal_json_load(s), {"text": "object {key: value} here"})
+        # Escaped quotes should not break string detection
+        s = '{"text": "say \\"hello\\""}'
+        self.assertEqual(string_utils.universal_json_load(s), {"text": 'say "hello"'})
+        # Escaped quotes with brackets
+        s = '{"text": "say \\"hello [world]\\"" }'
+        self.assertEqual(string_utils.universal_json_load(s), {"text": 'say "hello [world]"'})
+        # Real-world LLM response case with markdown and brackets in JSON strings
+        raw_input = 'Here is the analysis:\n\n```json\n[{"SERVICE_TERM": "Inpatient services", "REIMB_TERM": "100% of rates in Exhibit [[ to Attachment ]"}, {"SERVICE_TERM": "Outpatient services", "REIMB_TERM": "63% of rates in Exhibit [[ to Attachment ]"}]\n```'
+        result = string_utils.universal_json_load(raw_input)
+        self.assertEqual(len(result), 2)
+        self.assertEqual(result[0]["SERVICE_TERM"], "Inpatient services")
+        self.assertIn("Exhibit [[", result[0]["REIMB_TERM"])
+
 if __name__ == "__main__":
     unittest.main()