Merged in bugfix/mcs-issue-fixes (pull request #601)
Bugfix/mcs issue fixes * Update code primary prompt * Merge branch 'main' into bugfix/mcs-issue-fixes * No longer treat non-reimbursement tables as 'tables' for the purpose of preprocessing * No longer extract Reimb Effective Date from footer of the page * Merge branch 'main' into bugfix/mcs-issue-fixes * Update unit tests * Add postprocessing step to remove Reimb Date values when the 1:1 date values are identical * Update unit tests * prep for merge * Remove excess prints * Merged main into bugfix/mcs-issue-fixes * Merge branch 'main' into bugfix/mcs-issue-fixes * Prep for merge * remove print Approved-by: Alex Galarce
This commit is contained in:
Katon Minhas
@@ -122,7 +122,7 @@ def get_dynamic_answers(exhibit_chunk : str, exhibit_header: str, filename: str,
|
||||
exhibit_header_results = prompt_dynamic(text=exhibit_header,
|
||||
field_prompts=dynamic_fields.print_prompt_dict(),
|
||||
filename=filename)
|
||||
|
||||
|
||||
# Process Exhibit Header Results - update dynamic fields and add to reimbursement level fields
|
||||
for field_name, answer in exhibit_header_results.items():
|
||||
if not string_utils.is_empty(answer):
|
||||
|
||||
@@ -156,7 +156,7 @@ def run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, all_datase
|
||||
################# COMBINE ANSWERS ##################
|
||||
full_answer_dict = combine_one_to_n_answers(exhibit_level_answers, reimbursement_level_answers, tin_npi_answers={})
|
||||
one_to_n_results += full_answer_dict
|
||||
|
||||
|
||||
################## Crosswalk Fields ##################
|
||||
one_to_n_results = aarete_derived.get_crosswalk_fields(one_to_n_results)
|
||||
|
||||
|
||||
@@ -516,4 +516,33 @@ def fill_empty_reimb_pct_rate(df):
|
||||
na_mask = df["AARETE_DERIVED_REIMB_METHOD"].isin(na_methods)
|
||||
df.loc[na_mask, "REIMB_PCT_RATE"] = "N/A"
|
||||
|
||||
return df
|
||||
return df
|
||||
|
||||
def remove_redundant_reimb_date(df):
|
||||
"""
|
||||
Checks if "REIMB_EFFECTIVE_DT" and "REIMB_TERMINATION_DT" are the same as "AARETE_DERIVED_EFFECTIVE_DT" and "AARETE_DERIVED_TERMINATION_DT".
|
||||
If they are the same, replace the "REIMB_" values with ""
|
||||
|
||||
Args:
|
||||
df (pd.DataFrame): The input DataFrame.
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: The updated DataFrame with redundant reimbursement dates removed.
|
||||
"""
|
||||
|
||||
# Check if the relevant columns exist in the DataFrame
|
||||
# Check and process effective date pairs
|
||||
if all(col in df.columns for col in ["REIMB_EFFECTIVE_DT", "AARETE_DERIVED_EFFECTIVE_DT"]):
|
||||
# Create a mask for rows where effective dates are the same
|
||||
mask_effective = df["REIMB_EFFECTIVE_DT"] == df["AARETE_DERIVED_EFFECTIVE_DT"]
|
||||
# Replace "REIMB_EFFECTIVE_DT" with "" where the mask is True
|
||||
df.loc[mask_effective, "REIMB_EFFECTIVE_DT"] = ""
|
||||
|
||||
# Check and process termination date pairs
|
||||
if all(col in df.columns for col in ["REIMB_TERMINATION_DT", "AARETE_DERIVED_TERMINATION_DT"]):
|
||||
# Create a mask for rows where termination dates are the same
|
||||
mask_termination = df["REIMB_TERMINATION_DT"] == df["AARETE_DERIVED_TERMINATION_DT"]
|
||||
# Replace "REIMB_TERMINATION_DT" with "" where the mask is True
|
||||
df.loc[mask_termination, "REIMB_TERMINATION_DT"] = ""
|
||||
|
||||
return df
|
||||
|
||||
@@ -474,7 +474,6 @@ def reimbursement_level(
|
||||
Returns:
|
||||
dict: The parsed LLM response as a list of dictionaries with unique identifiers
|
||||
"""
|
||||
|
||||
reimbursement_primary_answers = get_reimbursement_primary(
|
||||
reimbursement_level_fields, exhibit_text, filename
|
||||
) # Returns list of dicts
|
||||
|
||||
@@ -57,6 +57,8 @@ def postprocess(df):
|
||||
df = investment_postprocessing_funcs.fill_empty_reimb_pct_rate(df)
|
||||
|
||||
df = investment_postprocessing_funcs.update_reimb_prov_name(df)
|
||||
|
||||
df = investment_postprocessing_funcs.remove_redundant_reimb_date(df)
|
||||
|
||||
df = investment_postprocessing_funcs.add_aarete_derived_amendment_num(df)
|
||||
|
||||
|
||||
@@ -425,9 +425,15 @@ def get_table_info(text_dict: Dict[str, str]) -> Dict[str, List[Table]]:
|
||||
end_index = page_text.find(END_MARKER, start_index)
|
||||
if end_index == -1:
|
||||
break
|
||||
|
||||
|
||||
# Don't consider tables that don't have relevant info
|
||||
table_text = page_text[start_index:end_index]
|
||||
|
||||
if "%" not in table_text and "$" not in table_text:
|
||||
start_index = end_index + len(END_MARKER)
|
||||
continue
|
||||
|
||||
metadata = page_text[:first_table_start_index].strip()
|
||||
|
||||
header_start = start_index + len(START_MARKER)
|
||||
|
||||
# Look for either [ or { to find start of table data
|
||||
|
||||
@@ -65,7 +65,7 @@
|
||||
"field_name": "REIMB_EFFECTIVE_DT",
|
||||
"relationship": "one_to_n",
|
||||
"field_type": "dynamic_reimb_info",
|
||||
"prompt": "Identify the effective date for reimbursement rates:\n- Look for dates associated with:\n • 'Effective' or 'effective date'\n • 'In effect' or 'in effect on'\n • Date ranges with 'from/through' or 'start/end'\n- Dates may appear as:\n • Full dates (12/1/17, December 1, 2017)\n • Month and year (December 2017)\n • Year only (2017)",
|
||||
"prompt": "Identify the effective date for reimbursement rates:\n- Look for dates associated with:\n • 'Effective' or 'effective date'\n • 'In effect' or 'in effect on'\n • Date ranges with 'from/through' or 'start/end'\n- Dates may appear as:\n • Full dates (12/1/17, December 1, 2017)\n • Month and year (December 2017)\n • Year only (2017). Do not include dates written in the footer of the page.",
|
||||
"format": "date"
|
||||
},
|
||||
{
|
||||
|
||||
@@ -651,10 +651,9 @@ Use the text in the Service to populate a JSON dictionary with the following fie
|
||||
|
||||
{questions}.
|
||||
|
||||
For each field, return a list of all codes EXPLICITLY written for that field. The code itself must be written, not language that describes the code.
|
||||
|
||||
For each field, return a list of all codes EXPLICITLY written for that field. The code itself must be written, not language that describes the code.
|
||||
Note that codes may be present, but not explicitly labeled as codes. For example, you may simply see "J1098", which is a CPT4_PROC_CD. You may see "155" which is a REVENUE_CD, etc.
|
||||
If the contract gives a range of codes, without listing each code in the range individually, return the range in the format 'LowestCode-HighestCode'.
|
||||
|
||||
If any of the codes are not found, do not write a list for that field. Simply populate the field with 'N/A'.
|
||||
|
||||
Here is the Service to analyze:
|
||||
|
||||
@@ -13,6 +13,7 @@ from src.investment.investment_postprocessing_funcs import (
|
||||
normalize_auto_renewal_term,
|
||||
normalize_cpt_fields,
|
||||
process_patient_age_range,
|
||||
remove_redundant_reimb_date
|
||||
)
|
||||
|
||||
|
||||
@@ -167,33 +168,99 @@ class TestPostprocessFunctions(unittest.TestCase):
|
||||
# Verify empty DataFrame is unchanged
|
||||
pd.testing.assert_frame_equal(empty_df, result_empty_df)
|
||||
|
||||
def test_validate_and_reformat_date(self):
|
||||
"""Tests validate_and_reformat_date with various date formats.
|
||||
|
||||
Tests:
|
||||
1. Date already in YYYY/MM/DD format
|
||||
2. Common alternative formats (YYYY-MM-DD, MM/DD/YYYY, etc.)
|
||||
3. Invalid date formats
|
||||
4. None and non-string values
|
||||
"""
|
||||
# Date already in correct format
|
||||
self.assertEqual(validate_and_reformat_date("2023/01/15"), "2023/01/15")
|
||||
|
||||
# Test various date formats that should be reformatted
|
||||
self.assertEqual(validate_and_reformat_date("2023-01-15"), "2023/01/15")
|
||||
self.assertEqual(validate_and_reformat_date("01/15/2023"), "2023/01/15")
|
||||
self.assertEqual(validate_and_reformat_date("15-Jan-2023"), "2023/01/15")
|
||||
|
||||
# Test invalid formats - should return the original string
|
||||
self.assertEqual(validate_and_reformat_date("Invalid date"), "Invalid date")
|
||||
self.assertEqual(validate_and_reformat_date("01-15"), "01-15")
|
||||
|
||||
# Test None and non-string values
|
||||
self.assertEqual(validate_and_reformat_date(None), None)
|
||||
self.assertEqual(validate_and_reformat_date(12345), 12345)
|
||||
|
||||
def test_remove_redundant_reimb_date(self):
|
||||
"""Tests remove_redundant_reimb_date function.
|
||||
|
||||
Tests:
|
||||
1. When reimbursement dates match derived dates (should remove)
|
||||
2. When reimbursement dates differ from derived dates (should keep)
|
||||
3. When only some rows match (should remove only matching rows)
|
||||
4. When columns are missing (should return unchanged DataFrame)
|
||||
"""
|
||||
# Test case 1: When dates match (should remove)
|
||||
input_df1 = pd.DataFrame({
|
||||
"REIMB_EFFECTIVE_DT": ["2023/01/01", "2023/02/01"],
|
||||
"REIMB_TERMINATION_DT": ["2023/12/31", "2023/12/31"],
|
||||
"AARETE_DERIVED_EFFECTIVE_DT": ["2023/01/01", "2023/02/01"],
|
||||
"AARETE_DERIVED_TERMINATION_DT": ["2023/12/31", "2023/12/31"]
|
||||
})
|
||||
|
||||
expected_df1 = pd.DataFrame({
|
||||
"REIMB_EFFECTIVE_DT": ["", ""],
|
||||
"REIMB_TERMINATION_DT": ["", ""],
|
||||
"AARETE_DERIVED_EFFECTIVE_DT": ["2023/01/01", "2023/02/01"],
|
||||
"AARETE_DERIVED_TERMINATION_DT": ["2023/12/31", "2023/12/31"]
|
||||
})
|
||||
|
||||
result_df1 = remove_redundant_reimb_date(input_df1)
|
||||
pd.testing.assert_frame_equal(result_df1, expected_df1)
|
||||
|
||||
# Test case 2: When dates differ (should keep)
|
||||
input_df2 = pd.DataFrame({
|
||||
"REIMB_EFFECTIVE_DT": ["2023/01/15", "2023/02/15"],
|
||||
"REIMB_TERMINATION_DT": ["2023/12/15", "2023/12/15"],
|
||||
"AARETE_DERIVED_EFFECTIVE_DT": ["2023/01/01", "2023/02/01"],
|
||||
"AARETE_DERIVED_TERMINATION_DT": ["2023/12/31", "2023/12/31"]
|
||||
})
|
||||
|
||||
# Result should be unchanged
|
||||
result_df2 = remove_redundant_reimb_date(input_df2)
|
||||
pd.testing.assert_frame_equal(result_df2, input_df2)
|
||||
|
||||
# Test case 3: Mixed case - some match, some don't
|
||||
input_df3 = pd.DataFrame({
|
||||
"REIMB_EFFECTIVE_DT": ["2023/01/01", "2023/02/15"],
|
||||
"REIMB_TERMINATION_DT": ["2023/12/31", "2023/12/15"],
|
||||
"AARETE_DERIVED_EFFECTIVE_DT": ["2023/01/01", "2023/02/01"],
|
||||
"AARETE_DERIVED_TERMINATION_DT": ["2023/12/31", "2023/12/31"]
|
||||
})
|
||||
|
||||
expected_df3 = pd.DataFrame({
|
||||
"REIMB_EFFECTIVE_DT": ["", "2023/02/15"],
|
||||
"REIMB_TERMINATION_DT": ["", "2023/12/15"],
|
||||
"AARETE_DERIVED_EFFECTIVE_DT": ["2023/01/01", "2023/02/01"],
|
||||
"AARETE_DERIVED_TERMINATION_DT": ["2023/12/31", "2023/12/31"]
|
||||
})
|
||||
|
||||
result_df3 = remove_redundant_reimb_date(input_df3)
|
||||
pd.testing.assert_frame_equal(result_df3, expected_df3)
|
||||
|
||||
# Test case 4: Missing columns
|
||||
input_df4 = pd.DataFrame({
|
||||
"REIMB_EFFECTIVE_DT": ["2023/01/01", "2023/02/01"],
|
||||
"OTHER_COLUMN": ["value1", "value2"]
|
||||
})
|
||||
|
||||
# Result should be unchanged
|
||||
result_df4 = remove_redundant_reimb_date(input_df4)
|
||||
pd.testing.assert_frame_equal(result_df4, input_df4)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"input_date, expected_output",
|
||||
[
|
||||
# Valid YYYY/MM/DD format
|
||||
("2023/10/15", "2023/10/15"),
|
||||
# Valid formats to be reformatted to YYYY/MM/DD
|
||||
("2023-10-15", "2023/10/15"),
|
||||
("10/15/2023", "2023/10/15"),
|
||||
("15-Oct-2023", "2023/10/15"),
|
||||
("15/10/2023", "2023/10/15"),
|
||||
# Invalid formats or non-date strings
|
||||
("15-10-2023", "15-10-2023"),
|
||||
("InvalidDate", "InvalidDate"),
|
||||
# Non-string input
|
||||
(None, None),
|
||||
(12345, 12345),
|
||||
# Edge cases
|
||||
("", ""),
|
||||
("2023/13/01", "2023/13/01"), # Invalid month
|
||||
("2023/02/30", "2023/02/30"), # Invalid day
|
||||
],
|
||||
)
|
||||
def test_validate_and_reformat_date(input_date, expected_output):
|
||||
assert validate_and_reformat_date(input_date) == expected_output
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
test_validate_and_reformat_date()
|
||||
|
||||
@@ -11,8 +11,8 @@ class TestPreprocessingIntegration(unittest.TestCase):
|
||||
def test_clean_tables_simple_end_to_end(self):
|
||||
"""Test the complete simplified table cleaning pipeline."""
|
||||
text_dict = {
|
||||
"1": "Metadata\n-------Table Start--------\nMain Table\n[['A', 'B']] + [[f'Row{i}', str(i)] for i in range(1, 8)]\n-------Table End--------",
|
||||
"2": "-------Table Start--------\n\n[['Row8', '8'], ['Row9', '9']]\n-------Table End--------", # Continuation
|
||||
"1": "Metadata\n-------Table Start--------\nMain Table\n[['A', 'B']] + [[f'Row{i}%', str(i)] for i in range(1, 8)]\n-------Table End--------",
|
||||
"2": "-------Table Start--------\n\n[['Row8', '8%'], ['Row9', '9%']]\n-------Table End--------", # Continuation
|
||||
"3": "Text without tables",
|
||||
}
|
||||
|
||||
@@ -116,7 +116,7 @@ class TestPreprocessingIntegration(unittest.TestCase):
|
||||
def test_clean_tables_multiple_tables_same_page(self):
|
||||
"""Test that multiple tables on same page get .0 suffix but stay together."""
|
||||
text_dict = {
|
||||
"1": "Page metadata\n-------Table Start--------\nFirst Table\n[['Col1', 'Col2'], ['A', 'B']]\n-------Table End--------\n\nSome text\n-------Table Start--------\nSecond Table\n[['X', 'Y'], ['1', '2']]\n-------Table End--------"
|
||||
"1": "Page metadata\n-------Table Start--------\nFirst Table\n[['Col1', 'Col2'], ['A%', 'B%']]\n-------Table End--------\n\nSome text\n-------Table Start--------\nSecond Table\n[['X', 'Y'], ['1%', '2%']]\n-------Table End--------"
|
||||
}
|
||||
|
||||
result = clean_tables(text_dict, "test_file.pdf")
|
||||
|
||||
@@ -18,21 +18,21 @@ class TestTableFunctions(unittest.TestCase):
|
||||
def setUp(self):
|
||||
# New list-of-rows format
|
||||
self.sample_text_dict_new = {
|
||||
"1": "Some metadata\n-------Table Start--------\nTable 1\n[['A', 'B'], ['1', '3'], ['2', '4']]\n-------Table End--------",
|
||||
"2": "More metadata\n-------Table Start--------\n\n[['100', '200'], ['5', '7'], ['6', '8']]\n-------Table End--------",
|
||||
"1": "Some metadata\n-------Table Start--------\nTable 1\n[['A', 'B'], ['1%', '3%'], ['2', '4']]\n-------Table End--------",
|
||||
"2": "More metadata\n-------Table Start--------\n\n[['100', '200'], ['5$', '7$'], ['6', '8']]\n-------Table End--------",
|
||||
}
|
||||
# Old dict format for backward compatibility testing
|
||||
self.sample_text_dict_old = {
|
||||
"1": "Some metadata\n-------Table Start--------\nTable 1\n{'A': ['1', '2'], 'B': ['3', '4']}\n-------Table End--------",
|
||||
"2": "More metadata\n-------Table Start--------\n\n{'A': ['5', '6'], 'B': ['7', '8']}\n-------Table End--------",
|
||||
"1": "Some metadata\n-------Table Start--------\nTable 1\n{'A': ['1', '2$'], 'B': ['3$', '4']}\n-------Table End--------",
|
||||
"2": "More metadata\n-------Table Start--------\n\n{'A': ['5', '6%'], 'B': ['7%', '8']}\n-------Table End--------",
|
||||
}
|
||||
# Multiple tables on one page
|
||||
self.multi_table_text = {
|
||||
"1": "Page metadata\n-------Table Start--------\nFirst Table\n[['Col1', 'Col2'], ['A', 'B'], ['C', 'D']]\n-------Table End--------\n\nSome text\n-------Table Start--------\nSecond Table\n[['X', 'Y'], ['1', '2']]\n-------Table End--------"
|
||||
"1": "Page metadata\n-------Table Start--------\nFirst Table\n[['Col1', 'Col2'], ['A%', 'B'], ['C', 'D']]\n-------Table End--------\n\nSome text\n-------Table Start--------\nSecond Table\n[['X', 'Y'], ['1', '2']]\n-------Table End--------"
|
||||
}
|
||||
# Long table for splitting tests
|
||||
self.long_table_text = {
|
||||
"1": "Long table metadata\n-------Table Start--------\nLong Table\n[['Name', 'Value'], ['Row1', '1'], ['Row2', '2'], ['Row3', '3'], ['Row4', '4'], ['Row5', '5']]\n-------Table End--------"
|
||||
"1": "Long table metadata\n-------Table Start--------\nLong Table\n[['Name%', 'Value'], ['Row1', '1'], ['Row2', '2'], ['Row3', '3'], ['Row4', '4'], ['Row5', '5']]\n-------Table End--------"
|
||||
}
|
||||
|
||||
def test_convert_str_to_dataframe_new_format(self):
|
||||
@@ -195,9 +195,9 @@ class TestTableFunctions(unittest.TestCase):
|
||||
"""Test the simplified continuous table combination logic."""
|
||||
# Create test data with continuation table
|
||||
text_dict = {
|
||||
"1": "Metadata 1\n-------Table Start--------\nMain Table\n[['Name', 'Value'], ['A', '1'], ['B', '2']]\n-------Table End--------",
|
||||
"2": "-------Table Start--------\n\n[['C', '3'], ['D', '4']]\n-------Table End--------", # Continuation (empty header)
|
||||
"3": "Metadata 3\n-------Table Start--------\nAnother Table\n[['X', 'Y'], ['Z', '5']]\n-------Table End--------",
|
||||
"1": "Metadata 1\n-------Table Start--------\nMain Table\n[['Name', 'Value'], ['A', '1%'], ['B', '2']]\n-------Table End--------",
|
||||
"2": "-------Table Start--------\n\n[['C', '3$'], ['D', '4']]\n-------Table End--------", # Continuation (empty header)
|
||||
"3": "Metadata 3\n-------Table Start--------\nAnother Table\n[['X', 'Y'], ['Z', '5%']]\n-------Table End--------",
|
||||
}
|
||||
|
||||
# Debug: Let's see what get_table_info thinks about these tables
|
||||
@@ -233,12 +233,12 @@ class TestTableFunctions(unittest.TestCase):
|
||||
text_dict = {
|
||||
"1": (
|
||||
"Metadata 1\n"
|
||||
"-------Table Start--------\nTable A (Complete)\n[['Col1', 'Col2'], ['A1', 'A2'], ['A3', 'A4']]\n-------Table End--------\n"
|
||||
"-------Table Start--------\nTable A (Complete)\n[['Col1', 'Col2'], ['A1%', 'A2'], ['A3', 'A4']]\n-------Table End--------\n"
|
||||
"Some text between tables\n"
|
||||
"-------Table Start--------\nTable B (Continues)\n[['Name', 'Value'], ['B1', '1'], ['B2', '2']]\n-------Table End--------"
|
||||
"-------Table Start--------\nTable B (Continues)\n[['Name', 'Value'], ['B1%', '1'], ['B2', '2']]\n-------Table End--------"
|
||||
),
|
||||
"2": (
|
||||
"-------Table Start--------\n\n[['B3', '3'], ['B4', '4']]\n-------Table End--------" # Continuation of Table B
|
||||
"-------Table Start--------\n\n[['B3', '3'], ['B4', '4%']]\n-------Table End--------" # Continuation of Table B
|
||||
),
|
||||
}
|
||||
|
||||
@@ -283,7 +283,7 @@ class TestTableFunctions(unittest.TestCase):
|
||||
) # Same row count
|
||||
self.assertEqual(table_a_after.header, "Table A (Complete)")
|
||||
self.assertEqual(
|
||||
table_a_after.table.iloc[0, 0], "A1"
|
||||
table_a_after.table.iloc[0, 0], "A1%"
|
||||
) # Original data preserved
|
||||
|
||||
# Table B (second table) should now include the continuation data
|
||||
@@ -293,7 +293,7 @@ class TestTableFunctions(unittest.TestCase):
|
||||
self.assertEqual(table_b_after.header, "Table B (Continues)")
|
||||
|
||||
# Verify Table B has both original and continuation data
|
||||
self.assertEqual(table_b_after.table.iloc[0, 0], "B1") # Original data
|
||||
self.assertEqual(table_b_after.table.iloc[0, 0], "B1%") # Original data
|
||||
self.assertEqual(table_b_after.table.iloc[1, 0], "B2") # Original data
|
||||
self.assertEqual(table_b_after.table.iloc[2, 0], "B3") # Continuation data
|
||||
self.assertEqual(table_b_after.table.iloc[3, 0], "B4") # Continuation data
|
||||
@@ -307,10 +307,10 @@ class TestTableFunctions(unittest.TestCase):
|
||||
|
||||
text_dict = {
|
||||
"1": (
|
||||
"-------Table Start--------\nTable A\n[['X', 'Y'], ['X1', 'Y1']]\n-------Table End--------\n"
|
||||
"-------Table Start--------\nTable B\n[['Name', 'Value'], ['B1', '1']]\n-------Table End--------"
|
||||
"-------Table Start--------\nTable A\n[['X', 'Y'], ['X1%', 'Y1']]\n-------Table End--------\n"
|
||||
"-------Table Start--------\nTable B\n[['Name', 'Value'], ['B1%', '1']]\n-------Table End--------"
|
||||
),
|
||||
"2": "-------Table Start--------\n\n[['B2', '2']]\n-------Table End--------", # Should combine with Table B, not Table A
|
||||
"2": "-------Table Start--------\n\n[['B2', '2%']]\n-------Table End--------", # Should combine with Table B, not Table A
|
||||
}
|
||||
|
||||
table_dict = get_table_info(text_dict)
|
||||
@@ -322,12 +322,12 @@ class TestTableFunctions(unittest.TestCase):
|
||||
# Table A (index 0) should have 1 row still - NOT combined with continuation
|
||||
table_a = combined_table_dict["1"][0]
|
||||
self.assertEqual(table_a.table.shape[0], 1) # Still just 1 row
|
||||
self.assertEqual(table_a.table.iloc[0, 0], "X1") # Original data only
|
||||
self.assertEqual(table_a.table.iloc[0, 0], "X1%") # Original data only
|
||||
|
||||
# Table B (index 1, which is -1) should have 2 rows - correctly combined
|
||||
table_b = combined_table_dict["1"][1]
|
||||
self.assertEqual(table_b.table.shape[0], 2) # 1 original + 1 continuation = 2
|
||||
self.assertEqual(table_b.table.iloc[0, 0], "B1") # Original
|
||||
self.assertEqual(table_b.table.iloc[0, 0], "B1%") # Original
|
||||
self.assertEqual(table_b.table.iloc[1, 0], "B2") # Continuation
|
||||
|
||||
def test_table_combine_same_header(self):
|
||||
@@ -526,7 +526,7 @@ class TestTableFunctions(unittest.TestCase):
|
||||
def test_table_post_table_text_preservation(self):
|
||||
"""Test that post-table text is properly preserved through operations."""
|
||||
text_dict = {
|
||||
"1": "Metadata\n-------Table Start--------\nTable\n[['A', 'B'], ['1', '2']]\n-------Table End--------\nImportant footnote\nSignature line"
|
||||
"1": "Metadata\n-------Table Start--------\nTable\n[['A', 'B'], ['1', '2%']]\n-------Table End--------\nImportant footnote\nSignature line"
|
||||
}
|
||||
|
||||
table_dict = get_table_info(text_dict)
|
||||
@@ -662,7 +662,7 @@ class TestTableFunctions(unittest.TestCase):
|
||||
table_dict = get_table_info(self.multi_table_text)
|
||||
|
||||
self.assertIn("1", table_dict)
|
||||
self.assertEqual(len(table_dict["1"]), 2)
|
||||
self.assertEqual(len(table_dict["1"]), 1)
|
||||
self.assertEqual(table_dict["1"][0].header, "First Table")
|
||||
|
||||
def test_recreate_page(self):
|
||||
@@ -680,7 +680,7 @@ class TestTableFunctions(unittest.TestCase):
|
||||
"""Test splitting tables that exceed row limit and adding .0 suffix."""
|
||||
# Create a long table that will need splitting
|
||||
long_table_data = [["Name", "Value"]] + [
|
||||
[f"Row{i}", str(i)] for i in range(1, 8)
|
||||
[f"Row{i}%", str(i)] for i in range(1, 8)
|
||||
] # 7 data rows
|
||||
text_dict = {
|
||||
"5": f"Table metadata\n-------Table Start--------\nLong Table\n{long_table_data}\n-------Table End--------\nPost-table text"
|
||||
@@ -702,7 +702,7 @@ class TestTableFunctions(unittest.TestCase):
|
||||
def test_split_tables_by_row_count_no_split_needed(self):
|
||||
"""Test that small tables get .0 suffix but aren't split."""
|
||||
text_dict = {
|
||||
"3": "Metadata\n-------Table Start--------\nSmall Table\n[['A', 'B'], ['1', '2']]\n-------Table End--------"
|
||||
"3": "Metadata\n-------Table Start--------\nSmall Table\n[['A', 'B'], ['1%', '2']]\n-------Table End--------"
|
||||
}
|
||||
|
||||
result = split_tables_by_rows(text_dict, row_limit=10)
|
||||
@@ -730,8 +730,8 @@ class TestTableFunctions(unittest.TestCase):
|
||||
def test_combine_continuous_tables_no_continuation(self):
|
||||
"""Test that non-continuation tables are not combined."""
|
||||
text_dict = {
|
||||
"1": "Metadata 1\n-------Table Start--------\nTable 1\n[['A', 'B'], ['1', '2']]\n-------Table End--------",
|
||||
"2": "Metadata 2\n-------Table Start--------\nTable 2\n[['C', 'D'], ['3', '4']]\n-------Table End--------",
|
||||
"1": "Metadata 1\n-------Table Start--------\nTable 1\n[['A', 'B'], ['1%', '2']]\n-------Table End--------",
|
||||
"2": "Metadata 2\n-------Table Start--------\nTable 2\n[['C', 'D'], ['3%', '4']]\n-------Table End--------",
|
||||
}
|
||||
|
||||
table_dict = get_table_info(text_dict)
|
||||
|
||||
Reference in New Issue
Block a user