diff --git a/src/pipelines/shared/preprocessing/preprocess.py b/src/pipelines/shared/preprocessing/preprocess.py index 89b8637..e0322e7 100644 --- a/src/pipelines/shared/preprocessing/preprocess.py +++ b/src/pipelines/shared/preprocessing/preprocess.py @@ -314,25 +314,48 @@ def clean_header_footer( # 4. Clean the document cleaned_dict = {} + deleted_lines = {} first_page_num = list(text_dict.keys())[0] for page_num, lines in pages_lines.items(): + removed = [] + # Remove header (except for first page) and footer (all pages, except "signature") if header_marker and page_num != first_page_num: - lines = [line for line in lines if not line.startswith(header_marker)] + filtered = [line for line in lines if not line.startswith(header_marker)] + removed += [line for line in lines if line.startswith(header_marker)] + lines = filtered if footer_marker: - lines = [ + filtered = [ line for line in lines if not line.endswith(footer_marker) or line.strip().lower() == "signature" ] + removed += [ + line + for line in lines + if line.endswith(footer_marker) and line.strip().lower() != "signature" + ] + lines = filtered # Remove common lines (skip page 1 entirely) if page_num == first_page_num: cleaned_lines = lines else: cleaned_lines = [line for line in lines if line.strip() not in common_lines] + removed += [line for line in lines if line.strip() in common_lines] + + filtered = [ + line + for line in cleaned_lines + if "docusign envelope id:" not in line.lower() + ] + removed += [ + line for line in cleaned_lines if "docusign envelope id:" in line.lower() + ] + cleaned_lines = filtered cleaned_dict[page_num] = "\n".join(cleaned_lines).strip() + deleted_lines[page_num] = removed - return cleaned_dict, removal_metadata + return cleaned_dict, deleted_lines diff --git a/src/tests/conftest.py b/src/tests/conftest.py new file mode 100644 index 0000000..dc909b7 --- /dev/null +++ b/src/tests/conftest.py @@ -0,0 +1,12 @@ +import sys + + +def pytest_configure(config): + """Prevent src/config.py from calling head_bucket during test collection. + + config.py reads sys.argv at import time via get_arg_value(). Adding + write_to_s3=False here ensures WRITE_TO_S3 is False before any src module + is imported, which skips the S3 bucket existence check entirely. + """ + if not any(arg.startswith("write_to_s3=") for arg in sys.argv): + sys.argv.append("write_to_s3=False") diff --git a/src/tests/test_clean_header_footer.py b/src/tests/test_clean_header_footer.py new file mode 100644 index 0000000..220c42b --- /dev/null +++ b/src/tests/test_clean_header_footer.py @@ -0,0 +1,138 @@ +import pytest +from src.pipelines.shared.preprocessing.preprocess import clean_header_footer + + +class TestDocuSignRemoval: + """DocuSign envelope ID line removal — new in bugfix/filter-docusign-lines.""" + + def test_removed_from_all_pages(self): + docusign = "DocuSign Envelope ID: ABCD-1234-EFGH-5678" + text_dict = { + "1": f"{docusign}\nContract title\nPage one body.", + "2": f"{docusign}\nContinuation here.\nMore body.", + } + cleaned, _ = clean_header_footer(text_dict) + assert "DocuSign Envelope ID:" not in cleaned["1"] + assert "DocuSign Envelope ID:" not in cleaned["2"] + + def test_case_insensitive(self): + text_dict = { + "1": "docusign envelope id: abcd-1234\nBody text on page one.", + "2": "DOCUSIGN ENVELOPE ID: EFGH-5678\nBody text on page two.", + } + cleaned, _ = clean_header_footer(text_dict) + assert "docusign envelope id:" not in cleaned["1"].lower() + assert "docusign envelope id:" not in cleaned["2"].lower() + + def test_body_text_preserved(self): + body = "Important contract clause here." + text_dict = { + "1": f"DocuSign Envelope ID: XXXX-0000\n{body}", + "2": "DocuSign Envelope ID: XXXX-0000\nOther unique body text.", + } + cleaned, _ = clean_header_footer(text_dict) + assert body in cleaned["1"] + + def test_appears_in_deleted_lines(self): + docusign = "DocuSign Envelope ID: ABCD-1234-EFGH-5678" + text_dict = { + "1": f"{docusign}\nPage one body text.", + "2": f"{docusign}\nPage two body text.", + } + _, deleted = clean_header_footer(text_dict) + assert docusign in deleted["1"] + assert docusign in deleted["2"] + + def test_removed_from_first_page(self): + """DocuSign filter applies to page 1 unlike common-line removal.""" + docusign = "DocuSign Envelope ID: FIRST-PAGE-TEST" + text_dict = { + "1": f"{docusign}\nPage one body only.", + "2": f"{docusign}\nPage two body only.", + } + cleaned, deleted = clean_header_footer(text_dict) + assert "DocuSign Envelope ID:" not in cleaned["1"] + assert docusign in deleted["1"] + + +class TestDeletedLinesReturnValue: + """Second return value is now deleted_lines per page, not removal_metadata.""" + + def test_returns_dict_keyed_by_page(self): + text_dict = { + "1": "Header line\nBody one.", + "2": "Header line\nBody two.", + } + _, deleted = clean_header_footer(text_dict) + assert isinstance(deleted, dict) + assert set(deleted.keys()) == set(text_dict.keys()) + + def test_not_removal_metadata_format(self): + """Old removal_metadata keys must not appear in the new return value.""" + text_dict = { + "1": "Some content here.\nMore text.", + "2": "Different content.\nOther text.", + } + _, deleted = clean_header_footer(text_dict) + assert "common_lines" not in deleted + assert "header_markers" not in deleted + assert "footer_markers" not in deleted + + def test_empty_list_when_nothing_removed(self): + text_dict = { + "1": "Unique content on page one.", + "2": "Completely different text here.", + } + _, deleted = clean_header_footer(text_dict) + assert deleted["1"] == [] + assert deleted["2"] == [] + + def test_empty_dict_returns_empty_deleted(self): + _, deleted = clean_header_footer({}) + assert deleted == {} + + def test_header_lines_tracked_non_first_pages(self): + """Removed header lines appear in deleted_lines for pages 2+, not page 1.""" + text_dict = { + "1": "ACME CORP\nXenophon rode his horse bravely.\nUNIQUE FOOTER ONE", + "2": "ACME CORP\nQuantum particles behave strangely.\nUNIQUE FOOTER TWO", + } + _, deleted = clean_header_footer(text_dict) + # Page 1 is exempt from header removal + assert not any("ACME CORP" in line for line in deleted["1"]) + # Page 2 header was removed — must be tracked + assert any("ACME CORP" in line for line in deleted["2"]) + + def test_footer_lines_tracked_all_pages(self): + """Footer removal applies to all pages including page 1.""" + text_dict = { + "1": "ACME CORP\nXenophon rode bravely.\nCONFIDENTIAL", + "2": "ACME CORP\nQuantum behaves strangely.\nCONFIDENTIAL", + } + _, deleted = clean_header_footer(text_dict) + assert any("CONFIDENTIAL" in line for line in deleted["1"]) + assert any("CONFIDENTIAL" in line for line in deleted["2"]) + + def test_common_lines_tracked_non_first_pages(self): + """Common lines removed from pages 2+ appear in deleted_lines.""" + common_line = "Confidential and Proprietary" + text_dict = { + "1": f"Title Page\n{common_line}\nIntro body.", + "2": f"Section One content here.\n{common_line}\nMore detail.", + "3": f"Section Two extra content.\n{common_line}\nFinal words.", + } + _, deleted = clean_header_footer(text_dict) + # Page 1 is exempt — common line not tracked as removed + assert not any(common_line in line for line in deleted["1"]) + assert any(common_line in line for line in deleted["2"]) + assert any(common_line in line for line in deleted["3"]) + + def test_deleted_lines_are_list_of_strings(self): + docusign = "DocuSign Envelope ID: ABCD-1234" + text_dict = { + "1": f"{docusign}\nBody text here.", + "2": f"{docusign}\nMore body text.", + } + _, deleted = clean_header_footer(text_dict) + assert isinstance(deleted["1"], list) + assert all(isinstance(line, str) for line in deleted["1"])