Merged in bugfix/filter-docusign-lines (pull request #983)

remove docusign lines

* remove docusign lines

* add unit tests for clean_header_footer docusign/deleted_lines changes

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* black forrmatting


Approved-by: Katon Minhas

src/pipelines/shared/preprocessing/preprocess.py

@@ -314,25 +314,48 @@ def clean_header_footer(
 
     # 4. Clean the document
     cleaned_dict = {}
+    deleted_lines = {}
     first_page_num = list(text_dict.keys())[0]
     for page_num, lines in pages_lines.items():
+        removed = []
+
         # Remove header (except for first page) and footer (all pages, except "signature")
         if header_marker and page_num != first_page_num:
-            lines = [line for line in lines if not line.startswith(header_marker)]
+            filtered = [line for line in lines if not line.startswith(header_marker)]
+            removed += [line for line in lines if line.startswith(header_marker)]
+            lines = filtered
         if footer_marker:
-            lines = [
+            filtered = [
                 line
                 for line in lines
                 if not line.endswith(footer_marker)
                 or line.strip().lower() == "signature"
             ]
+            removed += [
+                line
+                for line in lines
+                if line.endswith(footer_marker) and line.strip().lower() != "signature"
+            ]
+            lines = filtered
 
         # Remove common lines (skip page 1 entirely)
         if page_num == first_page_num:
             cleaned_lines = lines
         else:
             cleaned_lines = [line for line in lines if line.strip() not in common_lines]
+            removed += [line for line in lines if line.strip() in common_lines]
+
+        filtered = [
+            line
+            for line in cleaned_lines
+            if "docusign envelope id:" not in line.lower()
+        ]
+        removed += [
+            line for line in cleaned_lines if "docusign envelope id:" in line.lower()
+        ]
+        cleaned_lines = filtered
 
         cleaned_dict[page_num] = "\n".join(cleaned_lines).strip()
+        deleted_lines[page_num] = removed
 
-    return cleaned_dict, removal_metadata
+    return cleaned_dict, deleted_lines

src/tests/conftest.py

@@ -0,0 +1,12 @@
+import sys
+
+
+def pytest_configure(config):
+    """Prevent src/config.py from calling head_bucket during test collection.
+
+    config.py reads sys.argv at import time via get_arg_value(). Adding
+    write_to_s3=False here ensures WRITE_TO_S3 is False before any src module
+    is imported, which skips the S3 bucket existence check entirely.
+    """
+    if not any(arg.startswith("write_to_s3=") for arg in sys.argv):
+        sys.argv.append("write_to_s3=False")

src/tests/test_clean_header_footer.py

@@ -0,0 +1,138 @@
+import pytest
+from src.pipelines.shared.preprocessing.preprocess import clean_header_footer
+
+
+class TestDocuSignRemoval:
+    """DocuSign envelope ID line removal — new in bugfix/filter-docusign-lines."""
+
+    def test_removed_from_all_pages(self):
+        docusign = "DocuSign Envelope ID: ABCD-1234-EFGH-5678"
+        text_dict = {
+            "1": f"{docusign}\nContract title\nPage one body.",
+            "2": f"{docusign}\nContinuation here.\nMore body.",
+        }
+        cleaned, _ = clean_header_footer(text_dict)
+        assert "DocuSign Envelope ID:" not in cleaned["1"]
+        assert "DocuSign Envelope ID:" not in cleaned["2"]
+
+    def test_case_insensitive(self):
+        text_dict = {
+            "1": "docusign envelope id: abcd-1234\nBody text on page one.",
+            "2": "DOCUSIGN ENVELOPE ID: EFGH-5678\nBody text on page two.",
+        }
+        cleaned, _ = clean_header_footer(text_dict)
+        assert "docusign envelope id:" not in cleaned["1"].lower()
+        assert "docusign envelope id:" not in cleaned["2"].lower()
+
+    def test_body_text_preserved(self):
+        body = "Important contract clause here."
+        text_dict = {
+            "1": f"DocuSign Envelope ID: XXXX-0000\n{body}",
+            "2": "DocuSign Envelope ID: XXXX-0000\nOther unique body text.",
+        }
+        cleaned, _ = clean_header_footer(text_dict)
+        assert body in cleaned["1"]
+
+    def test_appears_in_deleted_lines(self):
+        docusign = "DocuSign Envelope ID: ABCD-1234-EFGH-5678"
+        text_dict = {
+            "1": f"{docusign}\nPage one body text.",
+            "2": f"{docusign}\nPage two body text.",
+        }
+        _, deleted = clean_header_footer(text_dict)
+        assert docusign in deleted["1"]
+        assert docusign in deleted["2"]
+
+    def test_removed_from_first_page(self):
+        """DocuSign filter applies to page 1 unlike common-line removal."""
+        docusign = "DocuSign Envelope ID: FIRST-PAGE-TEST"
+        text_dict = {
+            "1": f"{docusign}\nPage one body only.",
+            "2": f"{docusign}\nPage two body only.",
+        }
+        cleaned, deleted = clean_header_footer(text_dict)
+        assert "DocuSign Envelope ID:" not in cleaned["1"]
+        assert docusign in deleted["1"]
+
+
+class TestDeletedLinesReturnValue:
+    """Second return value is now deleted_lines per page, not removal_metadata."""
+
+    def test_returns_dict_keyed_by_page(self):
+        text_dict = {
+            "1": "Header line\nBody one.",
+            "2": "Header line\nBody two.",
+        }
+        _, deleted = clean_header_footer(text_dict)
+        assert isinstance(deleted, dict)
+        assert set(deleted.keys()) == set(text_dict.keys())
+
+    def test_not_removal_metadata_format(self):
+        """Old removal_metadata keys must not appear in the new return value."""
+        text_dict = {
+            "1": "Some content here.\nMore text.",
+            "2": "Different content.\nOther text.",
+        }
+        _, deleted = clean_header_footer(text_dict)
+        assert "common_lines" not in deleted
+        assert "header_markers" not in deleted
+        assert "footer_markers" not in deleted
+
+    def test_empty_list_when_nothing_removed(self):
+        text_dict = {
+            "1": "Unique content on page one.",
+            "2": "Completely different text here.",
+        }
+        _, deleted = clean_header_footer(text_dict)
+        assert deleted["1"] == []
+        assert deleted["2"] == []
+
+    def test_empty_dict_returns_empty_deleted(self):
+        _, deleted = clean_header_footer({})
+        assert deleted == {}
+
+    def test_header_lines_tracked_non_first_pages(self):
+        """Removed header lines appear in deleted_lines for pages 2+, not page 1."""
+        text_dict = {
+            "1": "ACME CORP\nXenophon rode his horse bravely.\nUNIQUE FOOTER ONE",
+            "2": "ACME CORP\nQuantum particles behave strangely.\nUNIQUE FOOTER TWO",
+        }
+        _, deleted = clean_header_footer(text_dict)
+        # Page 1 is exempt from header removal
+        assert not any("ACME CORP" in line for line in deleted["1"])
+        # Page 2 header was removed — must be tracked
+        assert any("ACME CORP" in line for line in deleted["2"])
+
+    def test_footer_lines_tracked_all_pages(self):
+        """Footer removal applies to all pages including page 1."""
+        text_dict = {
+            "1": "ACME CORP\nXenophon rode bravely.\nCONFIDENTIAL",
+            "2": "ACME CORP\nQuantum behaves strangely.\nCONFIDENTIAL",
+        }
+        _, deleted = clean_header_footer(text_dict)
+        assert any("CONFIDENTIAL" in line for line in deleted["1"])
+        assert any("CONFIDENTIAL" in line for line in deleted["2"])
+
+    def test_common_lines_tracked_non_first_pages(self):
+        """Common lines removed from pages 2+ appear in deleted_lines."""
+        common_line = "Confidential and Proprietary"
+        text_dict = {
+            "1": f"Title Page\n{common_line}\nIntro body.",
+            "2": f"Section One content here.\n{common_line}\nMore detail.",
+            "3": f"Section Two extra content.\n{common_line}\nFinal words.",
+        }
+        _, deleted = clean_header_footer(text_dict)
+        # Page 1 is exempt — common line not tracked as removed
+        assert not any(common_line in line for line in deleted["1"])
+        assert any(common_line in line for line in deleted["2"])
+        assert any(common_line in line for line in deleted["3"])
+
+    def test_deleted_lines_are_list_of_strings(self):
+        docusign = "DocuSign Envelope ID: ABCD-1234"
+        text_dict = {
+            "1": f"{docusign}\nBody text here.",
+            "2": f"{docusign}\nMore body text.",
+        }
+        _, deleted = clean_header_footer(text_dict)
+        assert isinstance(deleted["1"], list)
+        assert all(isinstance(line, str) for line in deleted["1"])