Siddhant Medar
55643b5a04
Feature/saas hybrid smart chunking * WIP: Initial work on SaaS HSC integration * clean up hsc funcs script * fix merging dict issue * Merge remote-tracking branch 'origin/main' into feature/saas-hybrid-smart-chunking * added changes as per feedback comments * add tests and remove old smart chunking funcs * add more tests Approved-by: Katon Minhas
354 lines
13 KiB
Python
354 lines
13 KiB
Python
"""
|
|
Unit tests for src.investment.hybrid_smart_chunking_preprocessing module
|
|
|
|
Tests contract text preprocessing and section splitting functionality including:
|
|
- Section identification based on various heading patterns
|
|
- Text preservation verification
|
|
- Section metadata extraction
|
|
"""
|
|
|
|
import pytest
|
|
from src.investment.hybrid_smart_chunking_preprocessing import (
|
|
split_contract_text_into_sections,
|
|
verify_preservation,
|
|
get_section_info,
|
|
)
|
|
|
|
|
|
class TestSplitContractTextIntoSections:
|
|
"""Test split_contract_text_into_sections function"""
|
|
|
|
def test_simple_text_no_headings(self):
|
|
"""Test text with no recognizable headings"""
|
|
text = "This is a simple contract text.\nWith no headings at all."
|
|
sections, original, line_ending, verification = split_contract_text_into_sections(text)
|
|
|
|
assert "Preamble" in sections
|
|
assert sections["Preamble"] == text
|
|
assert original == text
|
|
assert line_ending == "\n"
|
|
assert verification[0] is True
|
|
|
|
def test_arabic_numbered_headings(self):
|
|
"""Test single-digit arabic numbered headings (1. Title)"""
|
|
text = "Preamble text\n1 Definitions\nDefinition content\n2 Terms\nTerms content"
|
|
sections, original, line_ending, verification = split_contract_text_into_sections(text)
|
|
|
|
assert "Preamble" in sections
|
|
assert "1 Definitions" in sections
|
|
assert "2 Terms" in sections
|
|
assert verification[0] is True
|
|
|
|
def test_sub_arabic_numbered_headings(self):
|
|
"""Test multi-level arabic numbered headings (1.1, 1.2, etc.)"""
|
|
text = "1.1 First Section\nContent A\n1.2 Second Section\nContent B"
|
|
sections, original, line_ending, verification = split_contract_text_into_sections(text)
|
|
|
|
assert "1.1 First Section" in sections
|
|
assert "1.2 Second Section" in sections
|
|
assert verification[0] is True
|
|
|
|
def test_roman_numeral_headings(self):
|
|
"""Test roman numeral headings (I. Title, II. Title)"""
|
|
text = "I Introduction\nIntro content\nII Background\nBackground content"
|
|
sections, original, line_ending, verification = split_contract_text_into_sections(text)
|
|
|
|
assert "I Introduction" in sections
|
|
assert "II Background" in sections
|
|
assert verification[0] is True
|
|
|
|
def test_article_headings(self):
|
|
"""Test ARTICLE headings (ARTICLE I Title)"""
|
|
text = "ARTICLE I Definitions\nDef content\nARTICLE II Terms\nTerms content"
|
|
sections, original, line_ending, verification = split_contract_text_into_sections(text)
|
|
|
|
assert "ARTICLE I Definitions" in sections
|
|
assert "ARTICLE II Terms" in sections
|
|
assert verification[0] is True
|
|
|
|
def test_exhibit_headings(self):
|
|
"""Test EXHIBIT headings (EXHIBIT A, EXHIBIT B Title)"""
|
|
text = "Main text\nEXHIBIT A\nExhibit A content\nEXHIBIT B Schedule\nExhibit B content"
|
|
sections, original, line_ending, verification = split_contract_text_into_sections(text)
|
|
|
|
assert "EXHIBIT A" in sections
|
|
assert "EXHIBIT B Schedule" in sections
|
|
assert verification[0] is True
|
|
|
|
def test_attachment_headings(self):
|
|
"""Test Attachment headings"""
|
|
text = "Contract text\nAttachment 1\nAttachment content"
|
|
sections, original, line_ending, verification = split_contract_text_into_sections(text)
|
|
|
|
assert "Attachment 1" in sections
|
|
assert verification[0] is True
|
|
|
|
def test_recitals_and_agreement_keywords(self):
|
|
"""Test RECITALS and AGREEMENT keyword headings"""
|
|
text = "RECITALS:\nRecital text\nAGREEMENT\nAgreement text"
|
|
sections, original, line_ending, verification = split_contract_text_into_sections(text)
|
|
|
|
assert "RECITALS" in sections
|
|
assert "AGREEMENT" in sections
|
|
assert verification[0] is True
|
|
|
|
def test_in_witness_whereof_heading(self):
|
|
"""Test IN WITNESS WHEREOF heading"""
|
|
text = "Contract body\nIN WITNESS WHEREOF\nSignature section"
|
|
sections, original, line_ending, verification = split_contract_text_into_sections(text)
|
|
|
|
assert "IN WITNESS WHEREOF" in sections
|
|
assert verification[0] is True
|
|
|
|
def test_signatory_headings(self):
|
|
"""Test signatory block headings (PROVIDER:, ICM:)"""
|
|
text = "Agreement text\nPROVIDER:\nBy: John Doe\nICM:\nBy: Jane Smith"
|
|
sections, original, line_ending, verification = split_contract_text_into_sections(text)
|
|
|
|
assert "PROVIDER" in sections or "ICM" in sections
|
|
assert verification[0] is True
|
|
|
|
def test_multi_line_headings(self):
|
|
"""Test headings that span multiple lines"""
|
|
text = "1\nDefinitions and Terms\nContent here\n2\nOther Section\nMore content"
|
|
sections, original, line_ending, verification = split_contract_text_into_sections(text)
|
|
|
|
# Should combine multi-line headings
|
|
assert any("Definitions" in key for key in sections.keys())
|
|
assert verification[0] is True
|
|
|
|
def test_duplicate_section_titles(self):
|
|
"""Test handling of duplicate section titles"""
|
|
text = "1 Terms\nFirst terms\n1 Terms\nSecond terms"
|
|
sections, original, line_ending, verification = split_contract_text_into_sections(text)
|
|
|
|
# Should have unique keys (second occurrence gets suffix)
|
|
assert "1 Terms" in sections
|
|
assert "1 Terms (2)" in sections
|
|
assert verification[0] is True
|
|
|
|
def test_windows_line_endings(self):
|
|
"""Test text with Windows line endings (\\r\\n)"""
|
|
text = "Section 1\r\nContent A\r\nSection 2\r\nContent B"
|
|
sections, original, line_ending, verification = split_contract_text_into_sections(text)
|
|
|
|
assert line_ending == "\r\n"
|
|
assert verification[0] is True
|
|
|
|
def test_unix_line_endings(self):
|
|
"""Test text with Unix line endings (\\n)"""
|
|
text = "Section 1\nContent A\nSection 2\nContent B"
|
|
sections, original, line_ending, verification = split_contract_text_into_sections(text)
|
|
|
|
assert line_ending == "\n"
|
|
assert verification[0] is True
|
|
|
|
def test_no_line_endings(self):
|
|
"""Test text with no line endings"""
|
|
text = "Single line of text with no breaks"
|
|
sections, original, line_ending, verification = split_contract_text_into_sections(text)
|
|
|
|
assert line_ending == ""
|
|
assert "Preamble" in sections
|
|
assert verification[0] is True
|
|
|
|
def test_empty_text(self):
|
|
"""Test empty text input"""
|
|
text = ""
|
|
sections, original, line_ending, verification = split_contract_text_into_sections(text)
|
|
|
|
# Should handle empty text gracefully
|
|
assert isinstance(sections, dict)
|
|
assert original == ""
|
|
|
|
def test_mixed_heading_types(self):
|
|
"""Test document with multiple types of headings"""
|
|
text = """RECITALS:
|
|
Whereas clause
|
|
1 Definitions
|
|
Term definitions
|
|
ARTICLE I Scope
|
|
Scope content
|
|
EXHIBIT A
|
|
Exhibit content"""
|
|
sections, original, line_ending, verification = split_contract_text_into_sections(text)
|
|
|
|
assert "RECITALS" in sections
|
|
assert "1 Definitions" in sections
|
|
assert "ARTICLE I Scope" in sections
|
|
# EXHIBIT A might combine with next line, so check for both possibilities
|
|
assert "EXHIBIT A" in sections or "EXHIBIT A Exhibit content" in sections
|
|
assert verification[0] is True
|
|
|
|
|
|
class TestVerifyPreservation:
|
|
"""Test verify_preservation function"""
|
|
|
|
def test_perfect_preservation(self):
|
|
"""Test when content is perfectly preserved"""
|
|
original = "Line 1\nLine 2\nLine 3"
|
|
sections = {"Section1": "Line 1\nLine 2", "Section2": "Line 3"}
|
|
line_ending = "\n"
|
|
|
|
is_preserved, message = verify_preservation(original, sections, line_ending)
|
|
|
|
assert is_preserved is True
|
|
assert "Perfect preservation" in message
|
|
|
|
def test_content_mismatch(self):
|
|
"""Test when content differs"""
|
|
original = "Line 1\nLine 2\nLine 3"
|
|
sections = {"Section1": "Line 1\nLine X"} # Different content
|
|
line_ending = "\n"
|
|
|
|
is_preserved, message = verify_preservation(original, sections, line_ending)
|
|
|
|
assert is_preserved is False
|
|
assert "First difference at position" in message or "Length mismatch" in message
|
|
|
|
def test_length_mismatch(self):
|
|
"""Test when reconstructed content has different length"""
|
|
original = "Short text"
|
|
sections = {"Section1": "Much longer text than original"}
|
|
line_ending = "\n"
|
|
|
|
is_preserved, message = verify_preservation(original, sections, line_ending)
|
|
|
|
assert is_preserved is False
|
|
|
|
def test_empty_sections(self):
|
|
"""Test with empty sections dict"""
|
|
original = "Some content"
|
|
sections = {}
|
|
line_ending = "\n"
|
|
|
|
is_preserved, message = verify_preservation(original, sections, line_ending)
|
|
|
|
assert is_preserved is False
|
|
|
|
def test_windows_line_endings_preservation(self):
|
|
"""Test preservation with Windows line endings"""
|
|
original = "Line 1\r\nLine 2\r\nLine 3"
|
|
sections = {"Section1": "Line 1\r\nLine 2", "Section2": "Line 3"}
|
|
line_ending = "\r\n"
|
|
|
|
is_preserved, message = verify_preservation(original, sections, line_ending)
|
|
|
|
assert is_preserved is True
|
|
|
|
|
|
class TestGetSectionInfo:
|
|
"""Test get_section_info function"""
|
|
|
|
def test_basic_section_info(self):
|
|
"""Test extracting basic section information"""
|
|
sections = {
|
|
"Section 1": "This is section one content",
|
|
"Section 2": "This is section two\nWith multiple lines"
|
|
}
|
|
|
|
info = get_section_info(sections)
|
|
|
|
assert info["total_sections"] == 2
|
|
assert "Section 1" in info["section_details"]
|
|
assert "Section 2" in info["section_details"]
|
|
assert info["section_details"]["Section 1"]["character_count"] == 27
|
|
assert info["section_details"]["Section 2"]["line_count"] == 2
|
|
|
|
def test_empty_sections(self):
|
|
"""Test with empty sections dict"""
|
|
sections = {}
|
|
|
|
info = get_section_info(sections)
|
|
|
|
assert info["total_sections"] == 0
|
|
assert info["section_details"] == {}
|
|
assert info["total_characters"] == 0
|
|
|
|
def test_section_with_empty_content(self):
|
|
"""Test section with empty string content"""
|
|
sections = {"Empty Section": ""}
|
|
|
|
info = get_section_info(sections)
|
|
|
|
assert info["total_sections"] == 1
|
|
assert info["section_details"]["Empty Section"]["character_count"] == 0
|
|
|
|
def test_total_character_count(self):
|
|
"""Test total character count calculation"""
|
|
sections = {
|
|
"Section 1": "12345", # 5 chars
|
|
"Section 2": "67890", # 5 chars
|
|
"Section 3": "ABCDE" # 5 chars
|
|
}
|
|
|
|
info = get_section_info(sections)
|
|
|
|
assert info["total_characters"] == 15
|
|
|
|
def test_line_count_calculation(self):
|
|
"""Test line count is calculated correctly"""
|
|
sections = {
|
|
"Single Line": "One line only",
|
|
"Multi Line": "Line 1\nLine 2\nLine 3"
|
|
}
|
|
|
|
info = get_section_info(sections)
|
|
|
|
assert info["section_details"]["Single Line"]["line_count"] == 1
|
|
assert info["section_details"]["Multi Line"]["line_count"] == 3
|
|
|
|
|
|
class TestIntegration:
|
|
"""Integration tests for the full preprocessing workflow"""
|
|
|
|
def test_realistic_contract_structure(self):
|
|
"""Test with a realistic contract structure"""
|
|
contract_text = """PROVIDER AGREEMENT
|
|
|
|
RECITALS:
|
|
This Agreement is made between the parties.
|
|
|
|
AGREEMENT
|
|
|
|
1 Definitions
|
|
For purposes of this Agreement, the following terms shall have the meanings set forth below.
|
|
|
|
1.1 Provider
|
|
Provider means the healthcare provider.
|
|
|
|
1.2 Services
|
|
Services means healthcare services.
|
|
|
|
2 Terms and Conditions
|
|
The parties agree to the following terms.
|
|
|
|
EXHIBIT A Payment Schedule
|
|
Payment details here.
|
|
|
|
IN WITNESS WHEREOF
|
|
The parties have executed this Agreement.
|
|
|
|
PROVIDER:
|
|
By: _______________
|
|
Name: John Doe
|
|
"""
|
|
|
|
sections, original, line_ending, verification = split_contract_text_into_sections(contract_text)
|
|
|
|
# Verify major sections are identified
|
|
assert "RECITALS" in sections
|
|
assert "AGREEMENT" in sections
|
|
assert any("Definitions" in key for key in sections.keys())
|
|
assert "EXHIBIT A Payment Schedule" in sections
|
|
assert "IN WITNESS WHEREOF" in sections
|
|
|
|
# Verify preservation
|
|
assert verification[0] is True
|
|
|
|
# Verify section info
|
|
info = get_section_info(sections)
|
|
assert info["total_sections"] > 0
|
|
# Total characters should match (allow for slight differences in section joining)
|
|
assert abs(info["total_characters"] - len(contract_text)) < 20
|