From ac6dfdb721bbeebcb89b84cb8f521a93bf3dd85f Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Mon, 7 Jul 2025 16:57:47 +0000 Subject: [PATCH] Merged in bugfix/mcs-issue-fixes (pull request #601) Bugfix/mcs issue fixes * Update code primary prompt * Merge branch 'main' into bugfix/mcs-issue-fixes * No longer treat non-reimbursement tables as 'tables' for the purpose of preprocessing * No longer extract Reimb Effective Date from footer of the page * Merge branch 'main' into bugfix/mcs-issue-fixes * Update unit tests * Add postprocessing step to remove Reimb Date values when the 1:1 date values are identical * Update unit tests * prep for merge * Remove excess prints * Merged main into bugfix/mcs-issue-fixes * Merge branch 'main' into bugfix/mcs-issue-fixes * Prep for merge * remove print Approved-by: Alex Galarce --- .../src/investment/dynamic_funcs.py | 2 +- .../src/investment/file_processing.py | 2 +- .../investment_postprocessing_funcs.py | 31 ++++- .../src/investment/one_to_n_funcs.py | 1 - fieldExtraction/src/investment/postprocess.py | 2 + fieldExtraction/src/investment/table_funcs.py | 10 +- .../src/prompts/investment_prompts.json | 2 +- .../src/prompts/investment_prompts.py | 5 +- .../tests/investment_postprocess_test.py | 117 ++++++++++++++---- .../tests/preprocessing_integration_test.py | 6 +- fieldExtraction/tests/table_funcs_test.py | 50 ++++---- 11 files changed, 165 insertions(+), 63 deletions(-) diff --git a/fieldExtraction/src/investment/dynamic_funcs.py b/fieldExtraction/src/investment/dynamic_funcs.py index 434b388..48ef0ae 100644 --- a/fieldExtraction/src/investment/dynamic_funcs.py +++ b/fieldExtraction/src/investment/dynamic_funcs.py @@ -122,7 +122,7 @@ def get_dynamic_answers(exhibit_chunk : str, exhibit_header: str, filename: str, exhibit_header_results = prompt_dynamic(text=exhibit_header, field_prompts=dynamic_fields.print_prompt_dict(), filename=filename) - + # Process Exhibit Header Results - update dynamic fields and add to reimbursement level fields for field_name, answer in exhibit_header_results.items(): if not string_utils.is_empty(answer): diff --git a/fieldExtraction/src/investment/file_processing.py b/fieldExtraction/src/investment/file_processing.py index 8142ce0..5da045e 100644 --- a/fieldExtraction/src/investment/file_processing.py +++ b/fieldExtraction/src/investment/file_processing.py @@ -156,7 +156,7 @@ def run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, all_datase ################# COMBINE ANSWERS ################## full_answer_dict = combine_one_to_n_answers(exhibit_level_answers, reimbursement_level_answers, tin_npi_answers={}) one_to_n_results += full_answer_dict - + ################## Crosswalk Fields ################## one_to_n_results = aarete_derived.get_crosswalk_fields(one_to_n_results) diff --git a/fieldExtraction/src/investment/investment_postprocessing_funcs.py b/fieldExtraction/src/investment/investment_postprocessing_funcs.py index 8c5c420..92dde4b 100644 --- a/fieldExtraction/src/investment/investment_postprocessing_funcs.py +++ b/fieldExtraction/src/investment/investment_postprocessing_funcs.py @@ -516,4 +516,33 @@ def fill_empty_reimb_pct_rate(df): na_mask = df["AARETE_DERIVED_REIMB_METHOD"].isin(na_methods) df.loc[na_mask, "REIMB_PCT_RATE"] = "N/A" - return df \ No newline at end of file + return df + +def remove_redundant_reimb_date(df): + """ + Checks if "REIMB_EFFECTIVE_DT" and "REIMB_TERMINATION_DT" are the same as "AARETE_DERIVED_EFFECTIVE_DT" and "AARETE_DERIVED_TERMINATION_DT". + If they are the same, replace the "REIMB_" values with "" + + Args: + df (pd.DataFrame): The input DataFrame. + + Returns: + pd.DataFrame: The updated DataFrame with redundant reimbursement dates removed. + """ + + # Check if the relevant columns exist in the DataFrame + # Check and process effective date pairs + if all(col in df.columns for col in ["REIMB_EFFECTIVE_DT", "AARETE_DERIVED_EFFECTIVE_DT"]): + # Create a mask for rows where effective dates are the same + mask_effective = df["REIMB_EFFECTIVE_DT"] == df["AARETE_DERIVED_EFFECTIVE_DT"] + # Replace "REIMB_EFFECTIVE_DT" with "" where the mask is True + df.loc[mask_effective, "REIMB_EFFECTIVE_DT"] = "" + + # Check and process termination date pairs + if all(col in df.columns for col in ["REIMB_TERMINATION_DT", "AARETE_DERIVED_TERMINATION_DT"]): + # Create a mask for rows where termination dates are the same + mask_termination = df["REIMB_TERMINATION_DT"] == df["AARETE_DERIVED_TERMINATION_DT"] + # Replace "REIMB_TERMINATION_DT" with "" where the mask is True + df.loc[mask_termination, "REIMB_TERMINATION_DT"] = "" + + return df diff --git a/fieldExtraction/src/investment/one_to_n_funcs.py b/fieldExtraction/src/investment/one_to_n_funcs.py index b2c031e..bd7a095 100644 --- a/fieldExtraction/src/investment/one_to_n_funcs.py +++ b/fieldExtraction/src/investment/one_to_n_funcs.py @@ -474,7 +474,6 @@ def reimbursement_level( Returns: dict: The parsed LLM response as a list of dictionaries with unique identifiers """ - reimbursement_primary_answers = get_reimbursement_primary( reimbursement_level_fields, exhibit_text, filename ) # Returns list of dicts diff --git a/fieldExtraction/src/investment/postprocess.py b/fieldExtraction/src/investment/postprocess.py index 1eb9a5f..76ee02b 100644 --- a/fieldExtraction/src/investment/postprocess.py +++ b/fieldExtraction/src/investment/postprocess.py @@ -57,6 +57,8 @@ def postprocess(df): df = investment_postprocessing_funcs.fill_empty_reimb_pct_rate(df) df = investment_postprocessing_funcs.update_reimb_prov_name(df) + + df = investment_postprocessing_funcs.remove_redundant_reimb_date(df) df = investment_postprocessing_funcs.add_aarete_derived_amendment_num(df) diff --git a/fieldExtraction/src/investment/table_funcs.py b/fieldExtraction/src/investment/table_funcs.py index 6699758..e7baf6b 100644 --- a/fieldExtraction/src/investment/table_funcs.py +++ b/fieldExtraction/src/investment/table_funcs.py @@ -425,9 +425,15 @@ def get_table_info(text_dict: Dict[str, str]) -> Dict[str, List[Table]]: end_index = page_text.find(END_MARKER, start_index) if end_index == -1: break - + + # Don't consider tables that don't have relevant info + table_text = page_text[start_index:end_index] + + if "%" not in table_text and "$" not in table_text: + start_index = end_index + len(END_MARKER) + continue + metadata = page_text[:first_table_start_index].strip() - header_start = start_index + len(START_MARKER) # Look for either [ or { to find start of table data diff --git a/fieldExtraction/src/prompts/investment_prompts.json b/fieldExtraction/src/prompts/investment_prompts.json index e8282b1..831bf57 100644 --- a/fieldExtraction/src/prompts/investment_prompts.json +++ b/fieldExtraction/src/prompts/investment_prompts.json @@ -65,7 +65,7 @@ "field_name": "REIMB_EFFECTIVE_DT", "relationship": "one_to_n", "field_type": "dynamic_reimb_info", - "prompt": "Identify the effective date for reimbursement rates:\n- Look for dates associated with:\n • 'Effective' or 'effective date'\n • 'In effect' or 'in effect on'\n • Date ranges with 'from/through' or 'start/end'\n- Dates may appear as:\n • Full dates (12/1/17, December 1, 2017)\n • Month and year (December 2017)\n • Year only (2017)", + "prompt": "Identify the effective date for reimbursement rates:\n- Look for dates associated with:\n • 'Effective' or 'effective date'\n • 'In effect' or 'in effect on'\n • Date ranges with 'from/through' or 'start/end'\n- Dates may appear as:\n • Full dates (12/1/17, December 1, 2017)\n • Month and year (December 2017)\n • Year only (2017). Do not include dates written in the footer of the page.", "format": "date" }, { diff --git a/fieldExtraction/src/prompts/investment_prompts.py b/fieldExtraction/src/prompts/investment_prompts.py index 043e8e4..b127ec0 100644 --- a/fieldExtraction/src/prompts/investment_prompts.py +++ b/fieldExtraction/src/prompts/investment_prompts.py @@ -651,10 +651,9 @@ Use the text in the Service to populate a JSON dictionary with the following fie {questions}. -For each field, return a list of all codes EXPLICITLY written for that field. The code itself must be written, not language that describes the code. - +For each field, return a list of all codes EXPLICITLY written for that field. The code itself must be written, not language that describes the code. +Note that codes may be present, but not explicitly labeled as codes. For example, you may simply see "J1098", which is a CPT4_PROC_CD. You may see "155" which is a REVENUE_CD, etc. If the contract gives a range of codes, without listing each code in the range individually, return the range in the format 'LowestCode-HighestCode'. - If any of the codes are not found, do not write a list for that field. Simply populate the field with 'N/A'. Here is the Service to analyze: diff --git a/fieldExtraction/tests/investment_postprocess_test.py b/fieldExtraction/tests/investment_postprocess_test.py index 40888ee..b60698a 100644 --- a/fieldExtraction/tests/investment_postprocess_test.py +++ b/fieldExtraction/tests/investment_postprocess_test.py @@ -13,6 +13,7 @@ from src.investment.investment_postprocessing_funcs import ( normalize_auto_renewal_term, normalize_cpt_fields, process_patient_age_range, + remove_redundant_reimb_date ) @@ -167,33 +168,99 @@ class TestPostprocessFunctions(unittest.TestCase): # Verify empty DataFrame is unchanged pd.testing.assert_frame_equal(empty_df, result_empty_df) + def test_validate_and_reformat_date(self): + """Tests validate_and_reformat_date with various date formats. + + Tests: + 1. Date already in YYYY/MM/DD format + 2. Common alternative formats (YYYY-MM-DD, MM/DD/YYYY, etc.) + 3. Invalid date formats + 4. None and non-string values + """ + # Date already in correct format + self.assertEqual(validate_and_reformat_date("2023/01/15"), "2023/01/15") + + # Test various date formats that should be reformatted + self.assertEqual(validate_and_reformat_date("2023-01-15"), "2023/01/15") + self.assertEqual(validate_and_reformat_date("01/15/2023"), "2023/01/15") + self.assertEqual(validate_and_reformat_date("15-Jan-2023"), "2023/01/15") + + # Test invalid formats - should return the original string + self.assertEqual(validate_and_reformat_date("Invalid date"), "Invalid date") + self.assertEqual(validate_and_reformat_date("01-15"), "01-15") + + # Test None and non-string values + self.assertEqual(validate_and_reformat_date(None), None) + self.assertEqual(validate_and_reformat_date(12345), 12345) + def test_remove_redundant_reimb_date(self): + """Tests remove_redundant_reimb_date function. + + Tests: + 1. When reimbursement dates match derived dates (should remove) + 2. When reimbursement dates differ from derived dates (should keep) + 3. When only some rows match (should remove only matching rows) + 4. When columns are missing (should return unchanged DataFrame) + """ + # Test case 1: When dates match (should remove) + input_df1 = pd.DataFrame({ + "REIMB_EFFECTIVE_DT": ["2023/01/01", "2023/02/01"], + "REIMB_TERMINATION_DT": ["2023/12/31", "2023/12/31"], + "AARETE_DERIVED_EFFECTIVE_DT": ["2023/01/01", "2023/02/01"], + "AARETE_DERIVED_TERMINATION_DT": ["2023/12/31", "2023/12/31"] + }) + + expected_df1 = pd.DataFrame({ + "REIMB_EFFECTIVE_DT": ["", ""], + "REIMB_TERMINATION_DT": ["", ""], + "AARETE_DERIVED_EFFECTIVE_DT": ["2023/01/01", "2023/02/01"], + "AARETE_DERIVED_TERMINATION_DT": ["2023/12/31", "2023/12/31"] + }) + + result_df1 = remove_redundant_reimb_date(input_df1) + pd.testing.assert_frame_equal(result_df1, expected_df1) + + # Test case 2: When dates differ (should keep) + input_df2 = pd.DataFrame({ + "REIMB_EFFECTIVE_DT": ["2023/01/15", "2023/02/15"], + "REIMB_TERMINATION_DT": ["2023/12/15", "2023/12/15"], + "AARETE_DERIVED_EFFECTIVE_DT": ["2023/01/01", "2023/02/01"], + "AARETE_DERIVED_TERMINATION_DT": ["2023/12/31", "2023/12/31"] + }) + + # Result should be unchanged + result_df2 = remove_redundant_reimb_date(input_df2) + pd.testing.assert_frame_equal(result_df2, input_df2) + + # Test case 3: Mixed case - some match, some don't + input_df3 = pd.DataFrame({ + "REIMB_EFFECTIVE_DT": ["2023/01/01", "2023/02/15"], + "REIMB_TERMINATION_DT": ["2023/12/31", "2023/12/15"], + "AARETE_DERIVED_EFFECTIVE_DT": ["2023/01/01", "2023/02/01"], + "AARETE_DERIVED_TERMINATION_DT": ["2023/12/31", "2023/12/31"] + }) + + expected_df3 = pd.DataFrame({ + "REIMB_EFFECTIVE_DT": ["", "2023/02/15"], + "REIMB_TERMINATION_DT": ["", "2023/12/15"], + "AARETE_DERIVED_EFFECTIVE_DT": ["2023/01/01", "2023/02/01"], + "AARETE_DERIVED_TERMINATION_DT": ["2023/12/31", "2023/12/31"] + }) + + result_df3 = remove_redundant_reimb_date(input_df3) + pd.testing.assert_frame_equal(result_df3, expected_df3) + + # Test case 4: Missing columns + input_df4 = pd.DataFrame({ + "REIMB_EFFECTIVE_DT": ["2023/01/01", "2023/02/01"], + "OTHER_COLUMN": ["value1", "value2"] + }) + + # Result should be unchanged + result_df4 = remove_redundant_reimb_date(input_df4) + pd.testing.assert_frame_equal(result_df4, input_df4) -@pytest.mark.parametrize( - "input_date, expected_output", - [ - # Valid YYYY/MM/DD format - ("2023/10/15", "2023/10/15"), - # Valid formats to be reformatted to YYYY/MM/DD - ("2023-10-15", "2023/10/15"), - ("10/15/2023", "2023/10/15"), - ("15-Oct-2023", "2023/10/15"), - ("15/10/2023", "2023/10/15"), - # Invalid formats or non-date strings - ("15-10-2023", "15-10-2023"), - ("InvalidDate", "InvalidDate"), - # Non-string input - (None, None), - (12345, 12345), - # Edge cases - ("", ""), - ("2023/13/01", "2023/13/01"), # Invalid month - ("2023/02/30", "2023/02/30"), # Invalid day - ], -) -def test_validate_and_reformat_date(input_date, expected_output): - assert validate_and_reformat_date(input_date) == expected_output if __name__ == "__main__": unittest.main() - test_validate_and_reformat_date() \ No newline at end of file + \ No newline at end of file diff --git a/fieldExtraction/tests/preprocessing_integration_test.py b/fieldExtraction/tests/preprocessing_integration_test.py index 9474a2a..040f789 100644 --- a/fieldExtraction/tests/preprocessing_integration_test.py +++ b/fieldExtraction/tests/preprocessing_integration_test.py @@ -11,8 +11,8 @@ class TestPreprocessingIntegration(unittest.TestCase): def test_clean_tables_simple_end_to_end(self): """Test the complete simplified table cleaning pipeline.""" text_dict = { - "1": "Metadata\n-------Table Start--------\nMain Table\n[['A', 'B']] + [[f'Row{i}', str(i)] for i in range(1, 8)]\n-------Table End--------", - "2": "-------Table Start--------\n\n[['Row8', '8'], ['Row9', '9']]\n-------Table End--------", # Continuation + "1": "Metadata\n-------Table Start--------\nMain Table\n[['A', 'B']] + [[f'Row{i}%', str(i)] for i in range(1, 8)]\n-------Table End--------", + "2": "-------Table Start--------\n\n[['Row8', '8%'], ['Row9', '9%']]\n-------Table End--------", # Continuation "3": "Text without tables", } @@ -116,7 +116,7 @@ class TestPreprocessingIntegration(unittest.TestCase): def test_clean_tables_multiple_tables_same_page(self): """Test that multiple tables on same page get .0 suffix but stay together.""" text_dict = { - "1": "Page metadata\n-------Table Start--------\nFirst Table\n[['Col1', 'Col2'], ['A', 'B']]\n-------Table End--------\n\nSome text\n-------Table Start--------\nSecond Table\n[['X', 'Y'], ['1', '2']]\n-------Table End--------" + "1": "Page metadata\n-------Table Start--------\nFirst Table\n[['Col1', 'Col2'], ['A%', 'B%']]\n-------Table End--------\n\nSome text\n-------Table Start--------\nSecond Table\n[['X', 'Y'], ['1%', '2%']]\n-------Table End--------" } result = clean_tables(text_dict, "test_file.pdf") diff --git a/fieldExtraction/tests/table_funcs_test.py b/fieldExtraction/tests/table_funcs_test.py index 77470df..dfe7ddf 100644 --- a/fieldExtraction/tests/table_funcs_test.py +++ b/fieldExtraction/tests/table_funcs_test.py @@ -18,21 +18,21 @@ class TestTableFunctions(unittest.TestCase): def setUp(self): # New list-of-rows format self.sample_text_dict_new = { - "1": "Some metadata\n-------Table Start--------\nTable 1\n[['A', 'B'], ['1', '3'], ['2', '4']]\n-------Table End--------", - "2": "More metadata\n-------Table Start--------\n\n[['100', '200'], ['5', '7'], ['6', '8']]\n-------Table End--------", + "1": "Some metadata\n-------Table Start--------\nTable 1\n[['A', 'B'], ['1%', '3%'], ['2', '4']]\n-------Table End--------", + "2": "More metadata\n-------Table Start--------\n\n[['100', '200'], ['5$', '7$'], ['6', '8']]\n-------Table End--------", } # Old dict format for backward compatibility testing self.sample_text_dict_old = { - "1": "Some metadata\n-------Table Start--------\nTable 1\n{'A': ['1', '2'], 'B': ['3', '4']}\n-------Table End--------", - "2": "More metadata\n-------Table Start--------\n\n{'A': ['5', '6'], 'B': ['7', '8']}\n-------Table End--------", + "1": "Some metadata\n-------Table Start--------\nTable 1\n{'A': ['1', '2$'], 'B': ['3$', '4']}\n-------Table End--------", + "2": "More metadata\n-------Table Start--------\n\n{'A': ['5', '6%'], 'B': ['7%', '8']}\n-------Table End--------", } # Multiple tables on one page self.multi_table_text = { - "1": "Page metadata\n-------Table Start--------\nFirst Table\n[['Col1', 'Col2'], ['A', 'B'], ['C', 'D']]\n-------Table End--------\n\nSome text\n-------Table Start--------\nSecond Table\n[['X', 'Y'], ['1', '2']]\n-------Table End--------" + "1": "Page metadata\n-------Table Start--------\nFirst Table\n[['Col1', 'Col2'], ['A%', 'B'], ['C', 'D']]\n-------Table End--------\n\nSome text\n-------Table Start--------\nSecond Table\n[['X', 'Y'], ['1', '2']]\n-------Table End--------" } # Long table for splitting tests self.long_table_text = { - "1": "Long table metadata\n-------Table Start--------\nLong Table\n[['Name', 'Value'], ['Row1', '1'], ['Row2', '2'], ['Row3', '3'], ['Row4', '4'], ['Row5', '5']]\n-------Table End--------" + "1": "Long table metadata\n-------Table Start--------\nLong Table\n[['Name%', 'Value'], ['Row1', '1'], ['Row2', '2'], ['Row3', '3'], ['Row4', '4'], ['Row5', '5']]\n-------Table End--------" } def test_convert_str_to_dataframe_new_format(self): @@ -195,9 +195,9 @@ class TestTableFunctions(unittest.TestCase): """Test the simplified continuous table combination logic.""" # Create test data with continuation table text_dict = { - "1": "Metadata 1\n-------Table Start--------\nMain Table\n[['Name', 'Value'], ['A', '1'], ['B', '2']]\n-------Table End--------", - "2": "-------Table Start--------\n\n[['C', '3'], ['D', '4']]\n-------Table End--------", # Continuation (empty header) - "3": "Metadata 3\n-------Table Start--------\nAnother Table\n[['X', 'Y'], ['Z', '5']]\n-------Table End--------", + "1": "Metadata 1\n-------Table Start--------\nMain Table\n[['Name', 'Value'], ['A', '1%'], ['B', '2']]\n-------Table End--------", + "2": "-------Table Start--------\n\n[['C', '3$'], ['D', '4']]\n-------Table End--------", # Continuation (empty header) + "3": "Metadata 3\n-------Table Start--------\nAnother Table\n[['X', 'Y'], ['Z', '5%']]\n-------Table End--------", } # Debug: Let's see what get_table_info thinks about these tables @@ -233,12 +233,12 @@ class TestTableFunctions(unittest.TestCase): text_dict = { "1": ( "Metadata 1\n" - "-------Table Start--------\nTable A (Complete)\n[['Col1', 'Col2'], ['A1', 'A2'], ['A3', 'A4']]\n-------Table End--------\n" + "-------Table Start--------\nTable A (Complete)\n[['Col1', 'Col2'], ['A1%', 'A2'], ['A3', 'A4']]\n-------Table End--------\n" "Some text between tables\n" - "-------Table Start--------\nTable B (Continues)\n[['Name', 'Value'], ['B1', '1'], ['B2', '2']]\n-------Table End--------" + "-------Table Start--------\nTable B (Continues)\n[['Name', 'Value'], ['B1%', '1'], ['B2', '2']]\n-------Table End--------" ), "2": ( - "-------Table Start--------\n\n[['B3', '3'], ['B4', '4']]\n-------Table End--------" # Continuation of Table B + "-------Table Start--------\n\n[['B3', '3'], ['B4', '4%']]\n-------Table End--------" # Continuation of Table B ), } @@ -283,7 +283,7 @@ class TestTableFunctions(unittest.TestCase): ) # Same row count self.assertEqual(table_a_after.header, "Table A (Complete)") self.assertEqual( - table_a_after.table.iloc[0, 0], "A1" + table_a_after.table.iloc[0, 0], "A1%" ) # Original data preserved # Table B (second table) should now include the continuation data @@ -293,7 +293,7 @@ class TestTableFunctions(unittest.TestCase): self.assertEqual(table_b_after.header, "Table B (Continues)") # Verify Table B has both original and continuation data - self.assertEqual(table_b_after.table.iloc[0, 0], "B1") # Original data + self.assertEqual(table_b_after.table.iloc[0, 0], "B1%") # Original data self.assertEqual(table_b_after.table.iloc[1, 0], "B2") # Original data self.assertEqual(table_b_after.table.iloc[2, 0], "B3") # Continuation data self.assertEqual(table_b_after.table.iloc[3, 0], "B4") # Continuation data @@ -307,10 +307,10 @@ class TestTableFunctions(unittest.TestCase): text_dict = { "1": ( - "-------Table Start--------\nTable A\n[['X', 'Y'], ['X1', 'Y1']]\n-------Table End--------\n" - "-------Table Start--------\nTable B\n[['Name', 'Value'], ['B1', '1']]\n-------Table End--------" + "-------Table Start--------\nTable A\n[['X', 'Y'], ['X1%', 'Y1']]\n-------Table End--------\n" + "-------Table Start--------\nTable B\n[['Name', 'Value'], ['B1%', '1']]\n-------Table End--------" ), - "2": "-------Table Start--------\n\n[['B2', '2']]\n-------Table End--------", # Should combine with Table B, not Table A + "2": "-------Table Start--------\n\n[['B2', '2%']]\n-------Table End--------", # Should combine with Table B, not Table A } table_dict = get_table_info(text_dict) @@ -322,12 +322,12 @@ class TestTableFunctions(unittest.TestCase): # Table A (index 0) should have 1 row still - NOT combined with continuation table_a = combined_table_dict["1"][0] self.assertEqual(table_a.table.shape[0], 1) # Still just 1 row - self.assertEqual(table_a.table.iloc[0, 0], "X1") # Original data only + self.assertEqual(table_a.table.iloc[0, 0], "X1%") # Original data only # Table B (index 1, which is -1) should have 2 rows - correctly combined table_b = combined_table_dict["1"][1] self.assertEqual(table_b.table.shape[0], 2) # 1 original + 1 continuation = 2 - self.assertEqual(table_b.table.iloc[0, 0], "B1") # Original + self.assertEqual(table_b.table.iloc[0, 0], "B1%") # Original self.assertEqual(table_b.table.iloc[1, 0], "B2") # Continuation def test_table_combine_same_header(self): @@ -526,7 +526,7 @@ class TestTableFunctions(unittest.TestCase): def test_table_post_table_text_preservation(self): """Test that post-table text is properly preserved through operations.""" text_dict = { - "1": "Metadata\n-------Table Start--------\nTable\n[['A', 'B'], ['1', '2']]\n-------Table End--------\nImportant footnote\nSignature line" + "1": "Metadata\n-------Table Start--------\nTable\n[['A', 'B'], ['1', '2%']]\n-------Table End--------\nImportant footnote\nSignature line" } table_dict = get_table_info(text_dict) @@ -662,7 +662,7 @@ class TestTableFunctions(unittest.TestCase): table_dict = get_table_info(self.multi_table_text) self.assertIn("1", table_dict) - self.assertEqual(len(table_dict["1"]), 2) + self.assertEqual(len(table_dict["1"]), 1) self.assertEqual(table_dict["1"][0].header, "First Table") def test_recreate_page(self): @@ -680,7 +680,7 @@ class TestTableFunctions(unittest.TestCase): """Test splitting tables that exceed row limit and adding .0 suffix.""" # Create a long table that will need splitting long_table_data = [["Name", "Value"]] + [ - [f"Row{i}", str(i)] for i in range(1, 8) + [f"Row{i}%", str(i)] for i in range(1, 8) ] # 7 data rows text_dict = { "5": f"Table metadata\n-------Table Start--------\nLong Table\n{long_table_data}\n-------Table End--------\nPost-table text" @@ -702,7 +702,7 @@ class TestTableFunctions(unittest.TestCase): def test_split_tables_by_row_count_no_split_needed(self): """Test that small tables get .0 suffix but aren't split.""" text_dict = { - "3": "Metadata\n-------Table Start--------\nSmall Table\n[['A', 'B'], ['1', '2']]\n-------Table End--------" + "3": "Metadata\n-------Table Start--------\nSmall Table\n[['A', 'B'], ['1%', '2']]\n-------Table End--------" } result = split_tables_by_rows(text_dict, row_limit=10) @@ -730,8 +730,8 @@ class TestTableFunctions(unittest.TestCase): def test_combine_continuous_tables_no_continuation(self): """Test that non-continuation tables are not combined.""" text_dict = { - "1": "Metadata 1\n-------Table Start--------\nTable 1\n[['A', 'B'], ['1', '2']]\n-------Table End--------", - "2": "Metadata 2\n-------Table Start--------\nTable 2\n[['C', 'D'], ['3', '4']]\n-------Table End--------", + "1": "Metadata 1\n-------Table Start--------\nTable 1\n[['A', 'B'], ['1%', '2']]\n-------Table End--------", + "2": "Metadata 2\n-------Table Start--------\nTable 2\n[['C', 'D'], ['3%', '4']]\n-------Table End--------", } table_dict = get_table_info(text_dict)