jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in bugfix/hmk-issues (pull request #824)

Browse Source 
Bugfix/hmk issues

* fix for missing AARETE_DERIVED_LOB

* fix for AARETE_DERIVED_SID

* bugfix in universal_json_load

* bugfix for normalize_state_field

*  reimb eff date assignmnet prompt update

* prompt update for missing reimb terms

* Merged main into bugfix/hmk-issues

* test case for universal_json_loads

* reveresed the prompt update for missing reimb

* prompt fix for reimb dates


Approved-by: Katon Minhas
This commit is contained in:
Karan Desai
2026-01-05 18:34:18 +00:00
committed by Katon Minhas
parent b0eed711fc
commit 04a190d5b7
4 changed files with 105 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fieldExtraction/src/investment/postprocessing_funcs.py
+54 -30
View File
@@ -823,40 +823,60 @@ def fill_empty_dynamic(df):
# Make a copy to avoid SettingWithCopyWarning
result_df = df.copy()
# Check if AARETE_DERIVED_LOB column exists
has_lob_column = "AARETE_DERIVED_LOB" in result_df.columns
# First, handle AARETE_DERIVED_LOB by grouping on FILE_NAME and EXHIBIT_PAGE only
for (file_name, exhibit_page), group in result_df.groupby(["FILE_NAME", "EXHIBIT_PAGE"]):
# Get non-NA LOB values
column = "AARETE_DERIVED_LOB"
non_na_values = [val for val in group[column].unique() if not string_utils.is_empty(val)]
# If there's exactly one unique non-NA value, fill NA values with it
if len(non_na_values) == 1:
fill_value = non_na_values[0]
# Only apply to rows with this FILE_NAME and EXHIBIT_PAGE combination
mask = (result_df["FILE_NAME"] == file_name) & \
(result_df["EXHIBIT_PAGE"] == exhibit_page) & \
result_df[column].apply(string_utils.is_empty)
result_df.loc[mask, column] = fill_value
# Then, handle other columns by grouping on FILE_NAME, EXHIBIT_PAGE, and AARETE_DERIVED_LOB
for (file_name, exhibit_page, lob), group in result_df.groupby(["FILE_NAME", "EXHIBIT_PAGE", "AARETE_DERIVED_LOB"]):
for column in columns_to_fill:
# Skip AARETE_DERIVED_LOB as we already processed it
if column == "AARETE_DERIVED_LOB":
continue
# Get non-NA values
if has_lob_column:
for (file_name, exhibit_page), group in result_df.groupby(["FILE_NAME", "EXHIBIT_PAGE"]):
# Get non-NA LOB values
column = "AARETE_DERIVED_LOB"
non_na_values = [val for val in group[column].unique() if not string_utils.is_empty(val)]
# If there's exactly one unique non-NA value, fill NA values with it
if len(non_na_values) == 1:
fill_value = non_na_values[0]
# Only apply to rows with this FILE_NAME, EXHIBIT_PAGE, and LOB combination
# Only apply to rows with this FILE_NAME and EXHIBIT_PAGE combination
mask = (result_df["FILE_NAME"] == file_name) & \
(result_df["EXHIBIT_PAGE"] == exhibit_page) & \
(result_df["AARETE_DERIVED_LOB"] == lob) & \
result_df[column].apply(string_utils.is_empty)
result_df.loc[mask, column] = fill_value
# Then, handle other columns by grouping on FILE_NAME, EXHIBIT_PAGE, and AARETE_DERIVED_LOB (if it exists)
if has_lob_column:
for (file_name, exhibit_page, lob), group in result_df.groupby(["FILE_NAME", "EXHIBIT_PAGE", "AARETE_DERIVED_LOB"]):
for column in columns_to_fill:
# Skip AARETE_DERIVED_LOB as we already processed it
if column == "AARETE_DERIVED_LOB":
continue
# Get non-NA values
non_na_values = [val for val in group[column].unique() if not string_utils.is_empty(val)]
# If there's exactly one unique non-NA value, fill NA values with it
if len(non_na_values) == 1:
fill_value = non_na_values[0]
# Only apply to rows with this FILE_NAME, EXHIBIT_PAGE, and LOB combination
mask = (result_df["FILE_NAME"] == file_name) & \
(result_df["EXHIBIT_PAGE"] == exhibit_page) & \
(result_df["AARETE_DERIVED_LOB"] == lob) & \
result_df[column].apply(string_utils.is_empty)
result_df.loc[mask, column] = fill_value
else:
# No LOB column, just group by FILE_NAME and EXHIBIT_PAGE
for (file_name, exhibit_page), group in result_df.groupby(["FILE_NAME", "EXHIBIT_PAGE"]):
for column in columns_to_fill:
# Get non-NA values
non_na_values = [val for val in group[column].unique() if not string_utils.is_empty(val)]
# If there's exactly one unique non-NA value, fill NA values with it
if len(non_na_values) == 1:
fill_value = non_na_values[0]
# Only apply to rows with this FILE_NAME and EXHIBIT_PAGE combination
mask = (result_df["FILE_NAME"] == file_name) & \
(result_df["EXHIBIT_PAGE"] == exhibit_page) & \
result_df[column].apply(string_utils.is_empty)
result_df.loc[mask, column] = fill_value
return result_df
@@ -895,17 +915,21 @@ def attach_sid_column(df: pd.DataFrame) -> pd.DataFrame:
# Make a copy to avoid mutating the original DataFrame
df = df.copy()
# Ensure all SID columns exist; create missing ones filled with 'xx'
for col in sid_cols:
if col not in df.columns:
df[col] = ''
# Create the concatenated SID column (replace blanks/NaN only here)
df['AARETE_DERIVED_SID'] = (
df.assign(**{col: df[col].fillna('').astype(str) for col in sid_cols})
.apply(lambda row: '__'.join(val if val.strip() else 'xx' for val in row[sid_cols]), axis=1)
)
if df.empty:
df['AARETE_DERIVED_SID'] = pd.Series(dtype=str)
else:
sid_df = df[sid_cols].fillna('').astype(str)
df['AARETE_DERIVED_SID'] = sid_df.apply(
lambda row: '__'.join(val if val.strip() else 'xx' for val in row),
axis=1,
result_type='reduce'
)
return df
fieldExtraction/src/prompts/prompt_templates.py
+8
View File
@@ -285,6 +285,13 @@ CRITICAL RULES FOR DATE ASSIGNMENT:
6. **NO DATES IN SECTION**:
If no specific date range appears near this reimbursement term (not in the same paragraph or table section), return "N/A"
7. **DATE FORMAT PRIORITY**:
Always prioritize regular date formats over CY/FY formats.
- Regular formats: "July 1, 2018", "January 1, 2018 through December 31, 2018", etc.
- CY/FY formats (lower priority): "FY2018", "CY2019", "CY2018-CY2019", etc.
If a regular date exists in the section, use it. If ONLY CY/FY format is available, return "N/A".
[REIMB_DATES FIELD DEFINITION]
{field_prompt}
@@ -1242,6 +1249,7 @@ Answer YES only if the exhibit constraint adds NO additional limitation beyond w
Briefly explain your reasoning, then return YES or NO in |pipes|."""
def CARVEOUT_CHECK(service_term: str, reimb_term: str, carveout_definitions, special_case_definitions):
return f"""
[OBJECTIVE]
fieldExtraction/src/utils/string_utils.py
+16 -4
View File
@@ -197,8 +197,19 @@ def universal_json_load(s: str):
if s[i] in '{[':
start = i
stack = [s[i]]
for j in range(i+1, len(s)):
if s[j] in '{[':
j = i + 1
while j < len(s):
# Skip over string literals to avoid counting brackets inside strings
if s[j] == '"':
j += 1
while j < len(s):
if s[j] == '\\' and j + 1 < len(s):
j += 2 # Skip escaped character
continue
if s[j] == '"':
break
j += 1
elif s[j] in '{[':
stack.append(s[j])
elif s[j] in '}]':
if not stack:
@@ -218,6 +229,7 @@ def universal_json_load(s: str):
pass
i = j # Move past this top-level object
break
j += 1
else:
pass
i += 1
@@ -675,8 +687,8 @@ def parse_raw_state_value(raw_value):
except (ValueError, SyntaxError):
pass # Not valid Python literal, continue with string parsing
# Regular string splitting on delimiters
states = re.split(r'[|,;\s]+', stripped)
# Regular string splitting on delimiters (not whitespace - state names can have spaces like "West Virginia")
states = re.split(r'[|,;]+', stripped)
return [s.strip() for s in states if s.strip()]
if isinstance(raw_value, list):
fieldExtraction/tests/test_string_utils.py
+27
View File
@@ -179,6 +179,7 @@ class TestStringUtils(unittest.TestCase):
self.assertEqual(string_utils.flatten_to_strings("single"), ["single"])
self.assertEqual(string_utils.flatten_to_strings([1, [2, 3], "text"]), ["1", "2", "3", "text"])
self.assertEqual(string_utils.flatten_to_strings([" text ", [" nested "]]), ["text", "nested"])
def test_universal_json_load_multiple_invalid(self):
# Multiple invalid JSON objects
s = '{broken: true} some text {not: valid} more text'
@@ -200,6 +201,32 @@ class TestStringUtils(unittest.TestCase):
s = '{"simple": true} middle {"outer": {"inner": [1, 2]}} end'
self.assertEqual(string_utils.universal_json_load(s), {"outer": {"inner": [1, 2]}})
def test_universal_json_load_brackets_in_strings(self):
# Brackets inside string values should be ignored during bracket matching
s = '{"text": "array [1] and {2}"}'
self.assertEqual(string_utils.universal_json_load(s), {"text": "array [1] and {2}"})
# Double brackets in strings (the original bug case)
s = '{"field": "Exhibit [[ to Attachment ]"}'
self.assertEqual(string_utils.universal_json_load(s), {"field": "Exhibit [[ to Attachment ]"})
# Unbalanced brackets in strings
s = '{"text": "missing close [ bracket"}'
self.assertEqual(string_utils.universal_json_load(s), {"text": "missing close [ bracket"})
# Curly braces in strings
s = '{"text": "object {key: value} here"}'
self.assertEqual(string_utils.universal_json_load(s), {"text": "object {key: value} here"})
# Escaped quotes should not break string detection
s = '{"text": "say \\"hello\\""}'
self.assertEqual(string_utils.universal_json_load(s), {"text": 'say "hello"'})
# Escaped quotes with brackets
s = '{"text": "say \\"hello [world]\\"" }'
self.assertEqual(string_utils.universal_json_load(s), {"text": 'say "hello [world]"'})
# Real-world LLM response case with markdown and brackets in JSON strings
raw_input = 'Here is the analysis:\n\n```json\n[{"SERVICE_TERM": "Inpatient services", "REIMB_TERM": "100% of rates in Exhibit [[ to Attachment ]"}, {"SERVICE_TERM": "Outpatient services", "REIMB_TERM": "63% of rates in Exhibit [[ to Attachment ]"}]\n```'
result = string_utils.universal_json_load(raw_input)
self.assertEqual(len(result), 2)
self.assertEqual(result[0]["SERVICE_TERM"], "Inpatient services")
self.assertIn("Exhibit [[", result[0]["REIMB_TERM"])
if __name__ == "__main__":
unittest.main()