""" Unit tests for src.investment.hybrid_smart_chunking_preprocessing module Tests contract text preprocessing and section splitting functionality including: - Section identification based on various heading patterns - Text preservation verification - Section metadata extraction """ import pytest from src.investment.hybrid_smart_chunking_preprocessing import ( split_contract_text_into_sections, verify_preservation, get_section_info, ) class TestSplitContractTextIntoSections: """Test split_contract_text_into_sections function""" def test_simple_text_no_headings(self): """Test text with no recognizable headings""" text = "This is a simple contract text.\nWith no headings at all." sections, original, line_ending, verification = split_contract_text_into_sections(text) assert "Preamble" in sections assert sections["Preamble"] == text assert original == text assert line_ending == "\n" assert verification[0] is True def test_arabic_numbered_headings(self): """Test single-digit arabic numbered headings (1. Title)""" text = "Preamble text\n1 Definitions\nDefinition content\n2 Terms\nTerms content" sections, original, line_ending, verification = split_contract_text_into_sections(text) assert "Preamble" in sections assert "1 Definitions" in sections assert "2 Terms" in sections assert verification[0] is True def test_sub_arabic_numbered_headings(self): """Test multi-level arabic numbered headings (1.1, 1.2, etc.)""" text = "1.1 First Section\nContent A\n1.2 Second Section\nContent B" sections, original, line_ending, verification = split_contract_text_into_sections(text) assert "1.1 First Section" in sections assert "1.2 Second Section" in sections assert verification[0] is True def test_roman_numeral_headings(self): """Test roman numeral headings (I. Title, II. Title)""" text = "I Introduction\nIntro content\nII Background\nBackground content" sections, original, line_ending, verification = split_contract_text_into_sections(text) assert "I Introduction" in sections assert "II Background" in sections assert verification[0] is True def test_article_headings(self): """Test ARTICLE headings (ARTICLE I Title)""" text = "ARTICLE I Definitions\nDef content\nARTICLE II Terms\nTerms content" sections, original, line_ending, verification = split_contract_text_into_sections(text) assert "ARTICLE I Definitions" in sections assert "ARTICLE II Terms" in sections assert verification[0] is True def test_exhibit_headings(self): """Test EXHIBIT headings (EXHIBIT A, EXHIBIT B Title)""" text = "Main text\nEXHIBIT A\nExhibit A content\nEXHIBIT B Schedule\nExhibit B content" sections, original, line_ending, verification = split_contract_text_into_sections(text) assert "EXHIBIT A" in sections assert "EXHIBIT B Schedule" in sections assert verification[0] is True def test_attachment_headings(self): """Test Attachment headings""" text = "Contract text\nAttachment 1\nAttachment content" sections, original, line_ending, verification = split_contract_text_into_sections(text) assert "Attachment 1" in sections assert verification[0] is True def test_recitals_and_agreement_keywords(self): """Test RECITALS and AGREEMENT keyword headings""" text = "RECITALS:\nRecital text\nAGREEMENT\nAgreement text" sections, original, line_ending, verification = split_contract_text_into_sections(text) assert "RECITALS" in sections assert "AGREEMENT" in sections assert verification[0] is True def test_in_witness_whereof_heading(self): """Test IN WITNESS WHEREOF heading""" text = "Contract body\nIN WITNESS WHEREOF\nSignature section" sections, original, line_ending, verification = split_contract_text_into_sections(text) assert "IN WITNESS WHEREOF" in sections assert verification[0] is True def test_signatory_headings(self): """Test signatory block headings (PROVIDER:, ICM:)""" text = "Agreement text\nPROVIDER:\nBy: John Doe\nICM:\nBy: Jane Smith" sections, original, line_ending, verification = split_contract_text_into_sections(text) assert "PROVIDER" in sections or "ICM" in sections assert verification[0] is True def test_multi_line_headings(self): """Test headings that span multiple lines""" text = "1\nDefinitions and Terms\nContent here\n2\nOther Section\nMore content" sections, original, line_ending, verification = split_contract_text_into_sections(text) # Should combine multi-line headings assert any("Definitions" in key for key in sections.keys()) assert verification[0] is True def test_duplicate_section_titles(self): """Test handling of duplicate section titles""" text = "1 Terms\nFirst terms\n1 Terms\nSecond terms" sections, original, line_ending, verification = split_contract_text_into_sections(text) # Should have unique keys (second occurrence gets suffix) assert "1 Terms" in sections assert "1 Terms (2)" in sections assert verification[0] is True def test_windows_line_endings(self): """Test text with Windows line endings (\\r\\n)""" text = "Section 1\r\nContent A\r\nSection 2\r\nContent B" sections, original, line_ending, verification = split_contract_text_into_sections(text) assert line_ending == "\r\n" assert verification[0] is True def test_unix_line_endings(self): """Test text with Unix line endings (\\n)""" text = "Section 1\nContent A\nSection 2\nContent B" sections, original, line_ending, verification = split_contract_text_into_sections(text) assert line_ending == "\n" assert verification[0] is True def test_no_line_endings(self): """Test text with no line endings""" text = "Single line of text with no breaks" sections, original, line_ending, verification = split_contract_text_into_sections(text) assert line_ending == "" assert "Preamble" in sections assert verification[0] is True def test_empty_text(self): """Test empty text input""" text = "" sections, original, line_ending, verification = split_contract_text_into_sections(text) # Should handle empty text gracefully assert isinstance(sections, dict) assert original == "" def test_mixed_heading_types(self): """Test document with multiple types of headings""" text = """RECITALS: Whereas clause 1 Definitions Term definitions ARTICLE I Scope Scope content EXHIBIT A Exhibit content""" sections, original, line_ending, verification = split_contract_text_into_sections(text) assert "RECITALS" in sections assert "1 Definitions" in sections assert "ARTICLE I Scope" in sections # EXHIBIT A might combine with next line, so check for both possibilities assert "EXHIBIT A" in sections or "EXHIBIT A Exhibit content" in sections assert verification[0] is True class TestVerifyPreservation: """Test verify_preservation function""" def test_perfect_preservation(self): """Test when content is perfectly preserved""" original = "Line 1\nLine 2\nLine 3" sections = {"Section1": "Line 1\nLine 2", "Section2": "Line 3"} line_ending = "\n" is_preserved, message = verify_preservation(original, sections, line_ending) assert is_preserved is True assert "Perfect preservation" in message def test_content_mismatch(self): """Test when content differs""" original = "Line 1\nLine 2\nLine 3" sections = {"Section1": "Line 1\nLine X"} # Different content line_ending = "\n" is_preserved, message = verify_preservation(original, sections, line_ending) assert is_preserved is False assert "First difference at position" in message or "Length mismatch" in message def test_length_mismatch(self): """Test when reconstructed content has different length""" original = "Short text" sections = {"Section1": "Much longer text than original"} line_ending = "\n" is_preserved, message = verify_preservation(original, sections, line_ending) assert is_preserved is False def test_empty_sections(self): """Test with empty sections dict""" original = "Some content" sections = {} line_ending = "\n" is_preserved, message = verify_preservation(original, sections, line_ending) assert is_preserved is False def test_windows_line_endings_preservation(self): """Test preservation with Windows line endings""" original = "Line 1\r\nLine 2\r\nLine 3" sections = {"Section1": "Line 1\r\nLine 2", "Section2": "Line 3"} line_ending = "\r\n" is_preserved, message = verify_preservation(original, sections, line_ending) assert is_preserved is True class TestGetSectionInfo: """Test get_section_info function""" def test_basic_section_info(self): """Test extracting basic section information""" sections = { "Section 1": "This is section one content", "Section 2": "This is section two\nWith multiple lines" } info = get_section_info(sections) assert info["total_sections"] == 2 assert "Section 1" in info["section_details"] assert "Section 2" in info["section_details"] assert info["section_details"]["Section 1"]["character_count"] == 27 assert info["section_details"]["Section 2"]["line_count"] == 2 def test_empty_sections(self): """Test with empty sections dict""" sections = {} info = get_section_info(sections) assert info["total_sections"] == 0 assert info["section_details"] == {} assert info["total_characters"] == 0 def test_section_with_empty_content(self): """Test section with empty string content""" sections = {"Empty Section": ""} info = get_section_info(sections) assert info["total_sections"] == 1 assert info["section_details"]["Empty Section"]["character_count"] == 0 def test_total_character_count(self): """Test total character count calculation""" sections = { "Section 1": "12345", # 5 chars "Section 2": "67890", # 5 chars "Section 3": "ABCDE" # 5 chars } info = get_section_info(sections) assert info["total_characters"] == 15 def test_line_count_calculation(self): """Test line count is calculated correctly""" sections = { "Single Line": "One line only", "Multi Line": "Line 1\nLine 2\nLine 3" } info = get_section_info(sections) assert info["section_details"]["Single Line"]["line_count"] == 1 assert info["section_details"]["Multi Line"]["line_count"] == 3 class TestIntegration: """Integration tests for the full preprocessing workflow""" def test_realistic_contract_structure(self): """Test with a realistic contract structure""" contract_text = """PROVIDER AGREEMENT RECITALS: This Agreement is made between the parties. AGREEMENT 1 Definitions For purposes of this Agreement, the following terms shall have the meanings set forth below. 1.1 Provider Provider means the healthcare provider. 1.2 Services Services means healthcare services. 2 Terms and Conditions The parties agree to the following terms. EXHIBIT A Payment Schedule Payment details here. IN WITNESS WHEREOF The parties have executed this Agreement. PROVIDER: By: _______________ Name: John Doe """ sections, original, line_ending, verification = split_contract_text_into_sections(contract_text) # Verify major sections are identified assert "RECITALS" in sections assert "AGREEMENT" in sections assert any("Definitions" in key for key in sections.keys()) assert "EXHIBIT A Payment Schedule" in sections assert "IN WITNESS WHEREOF" in sections # Verify preservation assert verification[0] is True # Verify section info info = get_section_info(sections) assert info["total_sections"] > 0 # Total characters should match (allow for slight differences in section joining) assert abs(info["total_characters"] - len(contract_text)) < 20