jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/further-phase-1-and-2-fixes (pull request #622)

Browse Source 
Table Continuation Fix

* Add EXHIBIT_HEADER_MARKERS for improved document header recognition in table handling

* Refactor EXHIBIT_HEADER_MARKERS definition for improved clarity and organization

* remove unused page span tracking code; add unit tests for new functionality

* remove debugging logs from combine_continuous_tables function

* Refactor logging setup and update test file configuration in investment field testing notebook

* Merge remote-tracking branch 'origin/main' into feature/further-phase-1-and-2-fixes


Approved-by: Katon Minhas
This commit is contained in:
Alex Galarce
2025-07-17 19:39:33 +00:00
parent 33bd9a975a
commit c094fa35b0
5 changed files with 238 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fieldExtraction/scripts/investment_field_testing/testing.ipynb
+21 -6
View File
@@ -21,11 +21,13 @@
"outputs": [],
"source": [
"import logging\n",
"import sys\n",
"\n",
"logging.basicConfig(\n",
" level=logging.DEBUG,\n",
" format='%(asctime)s.%(msecs)03d - %(levelname)s - %(message)s',\n",
" datefmt='%Y-%m-%d %H:%M:%S',\n",
" stream=sys.stdout, # Output to standard output (for consistent notebook formatting)\n",
" force=True # Overwrite any previous logging configuration\n",
")\n",
"\n",
@@ -63,11 +65,8 @@
"test_params['local_input_dir'] = TEST_DATA_INPUT_DIR\n",
"test_params['max_workers'] = 1\n",
"test_params['input_files'] = [ # you can specify specific files to run here\n",
" # \"00-125434-Central MS Diagnostic, LLC-ICMProviderAgreement_78285.txt\",\n",
" \"02-0677066-Eyemasters-ICMProviderAgreement_64220_2.txt\",\n",
" # \"CustomFac_AMD - AMD - UNIVERSITY MEDICAL CENTER OF EL PASO - 74-6000756 MU.txt\", # short contract for testing\n",
" # \"20-2401676-Community Hospital of LaGrange County, Inc dba Parkview LaGrange Hospital-ICMProviderAgreement_89356_1.txt\",\n",
" ]"
" \"test_file.txt\", # replace with your test file(s)\n",
"]"
]
},
{
@@ -87,11 +86,27 @@
"source": [
"df_out, df_error = investment_main(testing=True, test_params=test_params)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df_out"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": ".venv",
"display_name": "doczy-smart-chunking-py3.12",
"language": "python",
"name": "python3"
},
fieldExtraction/src/constants/investment_values.py
+10 -1
View File
@@ -1,6 +1,16 @@
from crosswalk.crosswalk_utils import CrosswalkBuilder
EXHIBIT_HEADER_MARKERS = [
"EXHIBIT",
"ATTACHMENT",
"ARTICLE",
"AMENDMENT",
"SCHEDULE",
"ADDENDUM",
"APPENDIX",
]
# Dynamic Primary
VALID_LOBS = list(CrosswalkBuilder().from_json(path="crosswalk/mappings/crosswalk_lob.json").mapping.keys())
VALID_PROGRAMS = list(CrosswalkBuilder().from_json(path="crosswalk/mappings/crosswalk_program.json").mapping.keys())
@@ -13,7 +23,6 @@ VALID_BILL_TYPE = list(set(CrosswalkBuilder().from_json(path="crosswalk/mappings
VALID_PLACE_OF_SERVICE = list(CrosswalkBuilder().from_json(path="crosswalk/mappings/crosswalk_place_of_service.json").mapping.keys())
VALID_CLAIM_TYPE = list(set(CrosswalkBuilder().from_json(path="crosswalk/mappings/crosswalk_claim_type.json").mapping.keys()))
# Provider Type
VALID_PROV_TYPE = [
# Professional
fieldExtraction/src/investment/table_funcs.py
+6 -27
View File
@@ -12,6 +12,7 @@ import src.utils.llm_utils as llm_utils
import src.utils.string_utils as string_utils
from src import config
from src.prompts import preprocessing_prompts
from src.constants.investment_values import EXHIBIT_HEADER_MARKERS
# Table regex patterns
START_PAGE_PATTERN: Pattern[str] = re.compile(
@@ -160,7 +161,6 @@ class Table:
end_index (int): Ending position of the table in the page text.
post_table_text (str): Text appearing after the table on the page.
is_continuation (bool): Whether this table continues from a previous page.
original_pages (List[str]): Pages this table spans (for tracking combinations).
Key Methods:
combine(other): Combine another table into this one with column alignment.
@@ -178,9 +178,6 @@ class Table:
end_index: int,
post_table_text: str = "",
is_continuation: bool = False,
original_pages: Union[
List[str], None
] = None, # Track which pages this table spans
):
self.page_num = page_num
self.metadata = metadata
@@ -189,9 +186,6 @@ class Table:
self.end_index = end_index
self.is_continuation = is_continuation
self.post_table_text: str = post_table_text # Text after the table in the page
self.original_pages = original_pages or [
page_num
] # if `original_pages` is None, initialize with current page
# Handle a few different input types
if isinstance(table_data, str):
@@ -242,25 +236,6 @@ class Table:
pd.DataFrame()
) # Default to empty DataFrame if no valid format found
def get_page_span(self) -> int:
"""
Get the number of pages this table spans.
Returns:
int: The number of pages this table spans.
"""
return len(set(self.original_pages))
def add_page_to_span(self, page_num: str):
"""
Add a page number to the original pages this table spans.
Args:
page_num (str): The page number to add.
"""
if page_num not in self.original_pages:
self.original_pages.append(page_num)
def to_list_format(self) -> List[List[str]]:
"""
Convert the DataFrame back to list-of-rows format for serialization, with headers
@@ -446,10 +421,15 @@ def get_table_info(text_dict: Dict[str, str]) -> Dict[str, List[Table]]:
header = page_text[header_start:list_start].strip()
table_data = page_text[list_start:end_index].strip()
# Check if this page starts a new exhibit from the text
exhibit_pattern = "|".join(EXHIBIT_HEADER_MARKERS)
starts_new_exhibit = re.search(rf'\b({exhibit_pattern})\b', page_text[:100], re.IGNORECASE)
is_continuation = (
header == "" # No header text
and len(page_text[header_start:list_start].strip())
== 0 # Nothing between marker and `[`
and not starts_new_exhibit # NOT starting a new exhibit
)
elif dict_start != -1:
# Old dict format
@@ -692,7 +672,6 @@ def split_table_by_rows(table: Table, row_limit: int) -> List[Table]:
end_index=table.end_index, # Keep original end index
post_table_text="", # Only last split gets post-table text
is_continuation=False,
original_pages=table.original_pages.copy(),
)
split_tables.append(split_table)
fieldExtraction/src/prompts/investment_prompts.py
+2 -7
View File
@@ -8,6 +8,7 @@ import src.utils.llm_utils as llm_utils
import src.utils.string_utils as string_utils
from src.config import MODEL_ID_CLAUDE35_SONNET
from src.constants.investment_values import (
EXHIBIT_HEADER_MARKERS,
VALID_AARETE_DERIVED_FEE_SCHEDULE,
VALID_AARETE_DERIVED_FEE_SCHEDULE_VERSION, VALID_BILL_TYPE,
VALID_CARVEOUTS, VALID_CLAIM_TYPE, VALID_LOBS, VALID_NETWORKS,
@@ -260,13 +261,7 @@ def EXHIBIT_HEADER(context):
return f"""Analyze the following contract excerpt and extract the full header found.
Capture the complete hierarchy of document identifiers present in the text. The following keywords and phrases should be considered as exhibit/attachment headers:
* EXHIBIT
* ATTACHMENT
* ARTICLE
* AMENDMENT
* SCHEDULE
* ADDENDUM
* APPENDIX
* {'\n* '.join(EXHIBIT_HEADER_MARKERS)}
Include the following when present:
* Full header names and numbers/letters (e.g., "Attachment C: Commercial-Exchange")
fieldExtraction/tests/table_funcs_test.py
+199 -1
View File
@@ -9,7 +9,7 @@ from src.investment.table_funcs import (Table, combine_continuous_tables,
convert_str_to_dataframe,
get_table_info, recreate_page,
split_large_table_with_headers,
split_tables_by_rows)
split_tables_by_rows, START_MARKER, END_MARKER)
logging.basicConfig(level=logging.INFO)
@@ -855,5 +855,203 @@ class TestTableFunctions(unittest.TestCase):
self.assertEqual(table1.table.iloc[2, 2], "") # Padded
def test_exhibit_boundary_detection_prevents_combination(self):
"""Test that exhibit boundaries prevent incorrect table combination."""
text_dict = {
"1": "EXHIBIT A-1\nCommercial rates\n-------Table Start--------\n\n[['Service', 'Rate'], ['A', '1%']]\n-------Table End--------",
"2": "EXHIBIT B-1\nMedicare rates\n-------Table Start--------\n\n[['Service', 'Rate'], ['B', '2%']]\n-------Table End--------",
}
table_dict = get_table_info(text_dict)
# Both should be detected as non-continuation tables due to exhibit boundaries
self.assertFalse(table_dict["1"][0].is_continuation)
self.assertFalse(table_dict["2"][0].is_continuation)
# Combination should not remove any pages
result = combine_continuous_tables(text_dict, table_dict)
self.assertIn("1", result)
self.assertIn("2", result)
self.assertEqual(len(result), 2)
def test_exhibit_boundary_case_insensitive(self):
"""Test that exhibit detection works with different cases."""
test_cases = [
"EXHIBIT A-1",
"Exhibit A-1",
"exhibit a-1",
"ATTACHMENT B",
"Schedule C",
"ADDENDUM FOR MEDICARE"
]
for exhibit_text in test_cases:
with self.subTest(exhibit=exhibit_text):
text_dict = {
"1": f"{exhibit_text}\nRate information\n-------Table Start--------\n\n[['A', 'B'], ['1%', '2']]\n-------Table End--------"
}
table_dict = get_table_info(text_dict)
# Should not be detected as continuation due to exhibit marker
self.assertFalse(table_dict["1"][0].is_continuation)
def test_exhibit_boundary_detection_beyond_100_chars(self):
"""Test that exhibit markers beyond 100 characters don't affect continuation detection."""
# Create a page with exhibit marker after position 100
long_prefix = "A" * 95 # 95 characters
text_dict = {
"1": f"{long_prefix}EXHIBIT A-1\nRate info\n-------Table Start--------\n\n[['A', 'B'], ['1%', '2']]\n-------Table End--------"
}
table_dict = get_table_info(text_dict)
# Should be treated as continuation since exhibit marker is beyond position 100
self.assertTrue(table_dict["1"][0].is_continuation)
def test_table_init_with_empty_list(self):
"""Test Table initialization with empty list."""
table = Table(
page_num="1",
metadata="meta",
header="Empty Table",
table_data=[],
start_index=0,
end_index=100,
)
self.assertTrue(table.table.empty)
self.assertEqual(table.table.shape, (0, 0))
def test_table_init_with_only_headers(self):
"""Test Table initialization with only header row (no data)."""
table = Table(
page_num="1",
metadata="meta",
header="Header Only",
table_data=[["Column A", "Column B"]],
start_index=0,
end_index=100,
)
self.assertEqual(table.table.shape, (0, 2)) # 0 rows, 2 columns
self.assertEqual(list(table.table.columns), ["Column A", "Column B"])
def test_continuation_table_with_generic_column_names_combination(self):
"""Test that continuation tables with generic column names combine correctly."""
main_table = Table(
page_num="1",
metadata="meta",
header="Main Table",
table_data=[["Service", "Rate"], ["A", "1%"], ["B", "2%"]],
start_index=0,
end_index=100,
)
# Continuation table with generic column names
continuation_table = Table(
page_num="2",
metadata="",
header="",
table_data=[["C", "3%"], ["D", "4%"]],
start_index=0,
end_index=100,
is_continuation=True,
)
main_table.combine(continuation_table)
# Should have 4 data rows total
self.assertEqual(main_table.table.shape, (4, 2))
# Column names should remain from main table
self.assertEqual(list(main_table.table.columns), ["Service", "Rate"])
# Data should be properly combined
self.assertEqual(main_table.table.iloc[0, 0], "A") # Original
self.assertEqual(main_table.table.iloc[2, 0], "C") # Continuation
self.assertEqual(main_table.table.iloc[3, 1], "4%") # Last continuation row
def test_recreate_page_with_multiple_tables_and_post_text(self):
"""Test page recreation with multiple tables and post-table text."""
tables = [
Table(
page_num="1",
metadata="Page header",
header="First Table",
table_data=[["A", "B"], ["1%", "2"]],
start_index=0,
end_index=100,
),
Table(
page_num="1",
metadata="",
header="Second Table",
table_data=[["X", "Y"], ["3%", "4"]],
start_index=200,
end_index=300,
post_table_text="Important footnote\nSignature required",
),
]
recreated = recreate_page(tables)
# Should contain both tables
self.assertIn("First Table", recreated)
self.assertIn("Second Table", recreated)
# Should contain post-table text from last table
self.assertIn("Important footnote", recreated)
self.assertIn("Signature required", recreated)
# Should have proper structure
self.assertIn(START_MARKER, recreated)
self.assertIn(END_MARKER, recreated)
def test_split_tables_by_rows_preserves_metadata_and_post_text(self):
"""Test that table splitting preserves all text components."""
table_data = [['Code', 'Rate']] + [[f"T{i:04d}", f"${i}.00"] for i in range(12)]
text_dict = {
"5": f"EXHIBIT F\nRate Schedule\n-------Table Start--------\nLarge Table\n{table_data}\n-------Table End--------\nEffective 2024\nContact: admin@health.net"
}
result = split_tables_by_rows(text_dict, row_limit=5)
# Should create multiple pages
expected_pages = ["5.0", "5.1", "5.2"]
for page in expected_pages:
self.assertIn(page, result)
# All pages should have metadata and header
for page in expected_pages:
self.assertIn("EXHIBIT F", result[page])
self.assertIn("Large Table", result[page])
# Only last page should have post-table text
self.assertIn("Effective 2024", result["5.2"])
self.assertIn("Contact: admin@health.net", result["5.2"])
# Earlier pages should not have post-table text
self.assertNotIn("Effective 2024", result["5.0"])
self.assertNotIn("Effective 2024", result["5.1"])
def test_convert_str_to_dataframe_edge_cases(self):
"""Test edge cases in string to DataFrame conversion."""
edge_cases = [
("[]", True), # Empty list
("[['A']]", False), # Single cell
("[['A', 'B'], []]", False), # Empty data row
("[[], ['1', '2']]", False), # Empty header row
("[['']]", False), # Single empty string
]
for test_input, is_continuation in edge_cases:
with self.subTest(input=test_input, continuation=is_continuation):
df = convert_str_to_dataframe(test_input, is_continuation=is_continuation)
self.assertIsInstance(df, pd.DataFrame)
# Should not raise exceptions
if __name__ == "__main__":
unittest.main()