diff --git a/fieldExtraction/scripts/investment_field_testing/testing.ipynb b/fieldExtraction/scripts/investment_field_testing/testing.ipynb index 0969935..8100bc0 100644 --- a/fieldExtraction/scripts/investment_field_testing/testing.ipynb +++ b/fieldExtraction/scripts/investment_field_testing/testing.ipynb @@ -21,11 +21,13 @@ "outputs": [], "source": [ "import logging\n", + "import sys\n", "\n", "logging.basicConfig(\n", " level=logging.DEBUG,\n", " format='%(asctime)s.%(msecs)03d - %(levelname)s - %(message)s',\n", " datefmt='%Y-%m-%d %H:%M:%S',\n", + " stream=sys.stdout, # Output to standard output (for consistent notebook formatting)\n", " force=True # Overwrite any previous logging configuration\n", ")\n", "\n", @@ -63,11 +65,8 @@ "test_params['local_input_dir'] = TEST_DATA_INPUT_DIR\n", "test_params['max_workers'] = 1\n", "test_params['input_files'] = [ # you can specify specific files to run here\n", - " # \"00-125434-Central MS Diagnostic, LLC-ICMProviderAgreement_78285.txt\",\n", - " \"02-0677066-Eyemasters-ICMProviderAgreement_64220_2.txt\",\n", - " # \"CustomFac_AMD - AMD - UNIVERSITY MEDICAL CENTER OF EL PASO - 74-6000756 MU.txt\", # short contract for testing\n", - " # \"20-2401676-Community Hospital of LaGrange County, Inc dba Parkview LaGrange Hospital-ICMProviderAgreement_89356_1.txt\",\n", - " ]" + " \"test_file.txt\", # replace with your test file(s)\n", + "]" ] }, { @@ -87,11 +86,27 @@ "source": [ "df_out, df_error = investment_main(testing=True, test_params=test_params)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_out" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "doczy-smart-chunking-py3.12", "language": "python", "name": "python3" }, diff --git a/fieldExtraction/src/constants/investment_values.py b/fieldExtraction/src/constants/investment_values.py index bf3c653..44041e5 100644 --- a/fieldExtraction/src/constants/investment_values.py +++ b/fieldExtraction/src/constants/investment_values.py @@ -1,6 +1,16 @@ from crosswalk.crosswalk_utils import CrosswalkBuilder +EXHIBIT_HEADER_MARKERS = [ + "EXHIBIT", + "ATTACHMENT", + "ARTICLE", + "AMENDMENT", + "SCHEDULE", + "ADDENDUM", + "APPENDIX", +] + # Dynamic Primary VALID_LOBS = list(CrosswalkBuilder().from_json(path="crosswalk/mappings/crosswalk_lob.json").mapping.keys()) VALID_PROGRAMS = list(CrosswalkBuilder().from_json(path="crosswalk/mappings/crosswalk_program.json").mapping.keys()) @@ -13,7 +23,6 @@ VALID_BILL_TYPE = list(set(CrosswalkBuilder().from_json(path="crosswalk/mappings VALID_PLACE_OF_SERVICE = list(CrosswalkBuilder().from_json(path="crosswalk/mappings/crosswalk_place_of_service.json").mapping.keys()) VALID_CLAIM_TYPE = list(set(CrosswalkBuilder().from_json(path="crosswalk/mappings/crosswalk_claim_type.json").mapping.keys())) - # Provider Type VALID_PROV_TYPE = [ # Professional diff --git a/fieldExtraction/src/investment/table_funcs.py b/fieldExtraction/src/investment/table_funcs.py index e7baf6b..0780fec 100644 --- a/fieldExtraction/src/investment/table_funcs.py +++ b/fieldExtraction/src/investment/table_funcs.py @@ -12,6 +12,7 @@ import src.utils.llm_utils as llm_utils import src.utils.string_utils as string_utils from src import config from src.prompts import preprocessing_prompts +from src.constants.investment_values import EXHIBIT_HEADER_MARKERS # Table regex patterns START_PAGE_PATTERN: Pattern[str] = re.compile( @@ -160,7 +161,6 @@ class Table: end_index (int): Ending position of the table in the page text. post_table_text (str): Text appearing after the table on the page. is_continuation (bool): Whether this table continues from a previous page. - original_pages (List[str]): Pages this table spans (for tracking combinations). Key Methods: combine(other): Combine another table into this one with column alignment. @@ -178,9 +178,6 @@ class Table: end_index: int, post_table_text: str = "", is_continuation: bool = False, - original_pages: Union[ - List[str], None - ] = None, # Track which pages this table spans ): self.page_num = page_num self.metadata = metadata @@ -189,9 +186,6 @@ class Table: self.end_index = end_index self.is_continuation = is_continuation self.post_table_text: str = post_table_text # Text after the table in the page - self.original_pages = original_pages or [ - page_num - ] # if `original_pages` is None, initialize with current page # Handle a few different input types if isinstance(table_data, str): @@ -242,25 +236,6 @@ class Table: pd.DataFrame() ) # Default to empty DataFrame if no valid format found - def get_page_span(self) -> int: - """ - Get the number of pages this table spans. - - Returns: - int: The number of pages this table spans. - """ - return len(set(self.original_pages)) - - def add_page_to_span(self, page_num: str): - """ - Add a page number to the original pages this table spans. - - Args: - page_num (str): The page number to add. - """ - if page_num not in self.original_pages: - self.original_pages.append(page_num) - def to_list_format(self) -> List[List[str]]: """ Convert the DataFrame back to list-of-rows format for serialization, with headers @@ -446,10 +421,15 @@ def get_table_info(text_dict: Dict[str, str]) -> Dict[str, List[Table]]: header = page_text[header_start:list_start].strip() table_data = page_text[list_start:end_index].strip() + # Check if this page starts a new exhibit from the text + exhibit_pattern = "|".join(EXHIBIT_HEADER_MARKERS) + starts_new_exhibit = re.search(rf'\b({exhibit_pattern})\b', page_text[:100], re.IGNORECASE) + is_continuation = ( header == "" # No header text and len(page_text[header_start:list_start].strip()) == 0 # Nothing between marker and `[` + and not starts_new_exhibit # NOT starting a new exhibit ) elif dict_start != -1: # Old dict format @@ -692,7 +672,6 @@ def split_table_by_rows(table: Table, row_limit: int) -> List[Table]: end_index=table.end_index, # Keep original end index post_table_text="", # Only last split gets post-table text is_continuation=False, - original_pages=table.original_pages.copy(), ) split_tables.append(split_table) diff --git a/fieldExtraction/src/prompts/investment_prompts.py b/fieldExtraction/src/prompts/investment_prompts.py index 5271dea..9fcd51e 100644 --- a/fieldExtraction/src/prompts/investment_prompts.py +++ b/fieldExtraction/src/prompts/investment_prompts.py @@ -8,6 +8,7 @@ import src.utils.llm_utils as llm_utils import src.utils.string_utils as string_utils from src.config import MODEL_ID_CLAUDE35_SONNET from src.constants.investment_values import ( + EXHIBIT_HEADER_MARKERS, VALID_AARETE_DERIVED_FEE_SCHEDULE, VALID_AARETE_DERIVED_FEE_SCHEDULE_VERSION, VALID_BILL_TYPE, VALID_CARVEOUTS, VALID_CLAIM_TYPE, VALID_LOBS, VALID_NETWORKS, @@ -260,13 +261,7 @@ def EXHIBIT_HEADER(context): return f"""Analyze the following contract excerpt and extract the full header found. Capture the complete hierarchy of document identifiers present in the text. The following keywords and phrases should be considered as exhibit/attachment headers: -* EXHIBIT -* ATTACHMENT -* ARTICLE -* AMENDMENT -* SCHEDULE -* ADDENDUM -* APPENDIX +* {'\n* '.join(EXHIBIT_HEADER_MARKERS)} Include the following when present: * Full header names and numbers/letters (e.g., "Attachment C: Commercial-Exchange") diff --git a/fieldExtraction/tests/table_funcs_test.py b/fieldExtraction/tests/table_funcs_test.py index dfe7ddf..aa91690 100644 --- a/fieldExtraction/tests/table_funcs_test.py +++ b/fieldExtraction/tests/table_funcs_test.py @@ -9,7 +9,7 @@ from src.investment.table_funcs import (Table, combine_continuous_tables, convert_str_to_dataframe, get_table_info, recreate_page, split_large_table_with_headers, - split_tables_by_rows) + split_tables_by_rows, START_MARKER, END_MARKER) logging.basicConfig(level=logging.INFO) @@ -855,5 +855,203 @@ class TestTableFunctions(unittest.TestCase): self.assertEqual(table1.table.iloc[2, 2], "") # Padded + def test_exhibit_boundary_detection_prevents_combination(self): + """Test that exhibit boundaries prevent incorrect table combination.""" + text_dict = { + "1": "EXHIBIT A-1\nCommercial rates\n-------Table Start--------\n\n[['Service', 'Rate'], ['A', '1%']]\n-------Table End--------", + "2": "EXHIBIT B-1\nMedicare rates\n-------Table Start--------\n\n[['Service', 'Rate'], ['B', '2%']]\n-------Table End--------", + } + + table_dict = get_table_info(text_dict) + + # Both should be detected as non-continuation tables due to exhibit boundaries + self.assertFalse(table_dict["1"][0].is_continuation) + self.assertFalse(table_dict["2"][0].is_continuation) + + # Combination should not remove any pages + result = combine_continuous_tables(text_dict, table_dict) + self.assertIn("1", result) + self.assertIn("2", result) + self.assertEqual(len(result), 2) + + def test_exhibit_boundary_case_insensitive(self): + """Test that exhibit detection works with different cases.""" + test_cases = [ + "EXHIBIT A-1", + "Exhibit A-1", + "exhibit a-1", + "ATTACHMENT B", + "Schedule C", + "ADDENDUM FOR MEDICARE" + ] + + for exhibit_text in test_cases: + with self.subTest(exhibit=exhibit_text): + text_dict = { + "1": f"{exhibit_text}\nRate information\n-------Table Start--------\n\n[['A', 'B'], ['1%', '2']]\n-------Table End--------" + } + + table_dict = get_table_info(text_dict) + + # Should not be detected as continuation due to exhibit marker + self.assertFalse(table_dict["1"][0].is_continuation) + + def test_exhibit_boundary_detection_beyond_100_chars(self): + """Test that exhibit markers beyond 100 characters don't affect continuation detection.""" + # Create a page with exhibit marker after position 100 + long_prefix = "A" * 95 # 95 characters + text_dict = { + "1": f"{long_prefix}EXHIBIT A-1\nRate info\n-------Table Start--------\n\n[['A', 'B'], ['1%', '2']]\n-------Table End--------" + } + + table_dict = get_table_info(text_dict) + + # Should be treated as continuation since exhibit marker is beyond position 100 + self.assertTrue(table_dict["1"][0].is_continuation) + + def test_table_init_with_empty_list(self): + """Test Table initialization with empty list.""" + table = Table( + page_num="1", + metadata="meta", + header="Empty Table", + table_data=[], + start_index=0, + end_index=100, + ) + + self.assertTrue(table.table.empty) + self.assertEqual(table.table.shape, (0, 0)) + + def test_table_init_with_only_headers(self): + """Test Table initialization with only header row (no data).""" + table = Table( + page_num="1", + metadata="meta", + header="Header Only", + table_data=[["Column A", "Column B"]], + start_index=0, + end_index=100, + ) + + self.assertEqual(table.table.shape, (0, 2)) # 0 rows, 2 columns + self.assertEqual(list(table.table.columns), ["Column A", "Column B"]) + + def test_continuation_table_with_generic_column_names_combination(self): + """Test that continuation tables with generic column names combine correctly.""" + main_table = Table( + page_num="1", + metadata="meta", + header="Main Table", + table_data=[["Service", "Rate"], ["A", "1%"], ["B", "2%"]], + start_index=0, + end_index=100, + ) + + # Continuation table with generic column names + continuation_table = Table( + page_num="2", + metadata="", + header="", + table_data=[["C", "3%"], ["D", "4%"]], + start_index=0, + end_index=100, + is_continuation=True, + ) + + main_table.combine(continuation_table) + + # Should have 4 data rows total + self.assertEqual(main_table.table.shape, (4, 2)) + + # Column names should remain from main table + self.assertEqual(list(main_table.table.columns), ["Service", "Rate"]) + + # Data should be properly combined + self.assertEqual(main_table.table.iloc[0, 0], "A") # Original + self.assertEqual(main_table.table.iloc[2, 0], "C") # Continuation + self.assertEqual(main_table.table.iloc[3, 1], "4%") # Last continuation row + + def test_recreate_page_with_multiple_tables_and_post_text(self): + """Test page recreation with multiple tables and post-table text.""" + tables = [ + Table( + page_num="1", + metadata="Page header", + header="First Table", + table_data=[["A", "B"], ["1%", "2"]], + start_index=0, + end_index=100, + ), + Table( + page_num="1", + metadata="", + header="Second Table", + table_data=[["X", "Y"], ["3%", "4"]], + start_index=200, + end_index=300, + post_table_text="Important footnote\nSignature required", + ), + ] + + recreated = recreate_page(tables) + + # Should contain both tables + self.assertIn("First Table", recreated) + self.assertIn("Second Table", recreated) + + # Should contain post-table text from last table + self.assertIn("Important footnote", recreated) + self.assertIn("Signature required", recreated) + + # Should have proper structure + self.assertIn(START_MARKER, recreated) + self.assertIn(END_MARKER, recreated) + + def test_split_tables_by_rows_preserves_metadata_and_post_text(self): + """Test that table splitting preserves all text components.""" + table_data = [['Code', 'Rate']] + [[f"T{i:04d}", f"${i}.00"] for i in range(12)] + + text_dict = { + "5": f"EXHIBIT F\nRate Schedule\n-------Table Start--------\nLarge Table\n{table_data}\n-------Table End--------\nEffective 2024\nContact: admin@health.net" + } + + result = split_tables_by_rows(text_dict, row_limit=5) + + # Should create multiple pages + expected_pages = ["5.0", "5.1", "5.2"] + for page in expected_pages: + self.assertIn(page, result) + + # All pages should have metadata and header + for page in expected_pages: + self.assertIn("EXHIBIT F", result[page]) + self.assertIn("Large Table", result[page]) + + # Only last page should have post-table text + self.assertIn("Effective 2024", result["5.2"]) + self.assertIn("Contact: admin@health.net", result["5.2"]) + + # Earlier pages should not have post-table text + self.assertNotIn("Effective 2024", result["5.0"]) + self.assertNotIn("Effective 2024", result["5.1"]) + + def test_convert_str_to_dataframe_edge_cases(self): + """Test edge cases in string to DataFrame conversion.""" + edge_cases = [ + ("[]", True), # Empty list + ("[['A']]", False), # Single cell + ("[['A', 'B'], []]", False), # Empty data row + ("[[], ['1', '2']]", False), # Empty header row + ("[['']]", False), # Single empty string + ] + + for test_input, is_continuation in edge_cases: + with self.subTest(input=test_input, continuation=is_continuation): + df = convert_str_to_dataframe(test_input, is_continuation=is_continuation) + self.assertIsInstance(df, pd.DataFrame) + # Should not raise exceptions + + if __name__ == "__main__": unittest.main()