From 73020bdc104d6c1be8109e1e369d641830cc9a6f Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Thu, 14 Nov 2024 21:16:43 +0000 Subject: [PATCH] Merged in bugfix/full_context_check (pull request #270) Update for full-context AC chunks to return exact contract_text instead of concatenated text_dict pages * Update for full-context AC chunks to return exact contract_text instead of concatenated text_dict pages * Updated test cases * Updated case 1 * Merged main into bugfix/full_context_check Approved-by: Alex Galarce --- fieldExtraction/src/preprocess.py | 2 +- fieldExtraction/src/preprocessing_funcs.py | 16 +++++++++--- fieldExtraction/tests/test_chunking.py | 29 +++++++++++++--------- 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/fieldExtraction/src/preprocess.py b/fieldExtraction/src/preprocess.py index c9dfe48..012e5d2 100644 --- a/fieldExtraction/src/preprocess.py +++ b/fieldExtraction/src/preprocess.py @@ -33,6 +33,6 @@ def preprocess(contract_text, filename, fields=config.FIELDS): ac_chunks, full_context = "", "" if fields == "ac": exhibit_pages = [] - ac_chunks = preprocessing_funcs.smart_chunk_ac(text_dict) + ac_chunks = preprocessing_funcs.smart_chunk_ac(text_dict, contract_text) return text_dict, exhibit_pages, num_pages, ac_chunks diff --git a/fieldExtraction/src/preprocessing_funcs.py b/fieldExtraction/src/preprocessing_funcs.py index 70dc79e..7d1ddb0 100644 --- a/fieldExtraction/src/preprocessing_funcs.py +++ b/fieldExtraction/src/preprocessing_funcs.py @@ -188,7 +188,9 @@ def chunk_consecutive(text_dict, exhibit_pages): def smart_chunk_ac( - text_dict: dict, keyword_mappings: dict = keywords.GROUPED_KEYWORD_MAPPINGS + text_dict: dict, + contract_text: str, + keyword_mappings: dict = keywords.GROUPED_KEYWORD_MAPPINGS ) -> dict: """Performs a keyword search on textract outputs to retrieve relevant chunks for a given field. @@ -248,9 +250,15 @@ def smart_chunk_ac( ) if field_group == "initial_page_group": keyword_page_dict["initial_pages"] = [1, 2] - chunks[field_group] = "\n".join( - [text_dict[str(page_num)] for page_num in page_list] - ) + + # chunks[field_group] = concatenated page texts + if len(page_list) == len(text_dict.keys()): + chunks[field_group] = contract_text + else: + chunks[field_group] = "\n".join( + [text_dict[str(page_num)] for page_num in page_list] + ) + chunks[field_group + "_pages"] = keyword_page_dict chunks[field_group + "_methodology"] = key_dict["methodology"] chunks[field_group + "_case"] = key_dict["case_sensitive"] diff --git a/fieldExtraction/tests/test_chunking.py b/fieldExtraction/tests/test_chunking.py index c1bf12c..d2577d8 100644 --- a/fieldExtraction/tests/test_chunking.py +++ b/fieldExtraction/tests/test_chunking.py @@ -8,6 +8,8 @@ test_chunk_dict_term = { "4": "NA", } +test_full_text_term = '\n'.join([v for v in test_chunk_dict_term.values()]) + test_late_paid_claims = { "1": "NA", "2": "LATE PAYMENT", # try lowercase @@ -18,6 +20,8 @@ test_late_paid_claims = { "12": "NA", } +test_full_text_late_paid_claims = '\n'.join([v for v in test_late_paid_claims.values()]) + test_keyword_mappings = { "or_1": { "methodology": "or", @@ -93,61 +97,61 @@ case_5 = { def test_smart_chunk_term(): - assert smart_chunk_ac(text_dict=test_chunk_dict_term)["term_group"] == "\n".join( + assert smart_chunk_ac(text_dict=test_chunk_dict_term, contract_text=test_full_text_term)["term_group"] == "\n".join( ["NA", "Termination", "NA"] ) def test_smart_chunk_late_paid(): - assert smart_chunk_ac(text_dict=test_late_paid_claims)[ + assert smart_chunk_ac(text_dict=test_late_paid_claims, contract_text=test_full_text_late_paid_claims)[ "LATE_PAID_CLAIMS_LANGUAGE" ] == "\n".join(["NA", "LATE PAID", "NA"]) def test_or_1(): - assert smart_chunk_ac(text_dict=case_1, keyword_mappings=test_keyword_mappings)[ + assert smart_chunk_ac(text_dict=case_1, contract_text='\n'.join([v for v in case_1.values()]), keyword_mappings=test_keyword_mappings)[ "or_1" ] == "\n".join(["NA", "kw1", "NA", "NA", "kw3"]) - assert smart_chunk_ac(text_dict=case_2, keyword_mappings=test_keyword_mappings)[ + assert smart_chunk_ac(text_dict=case_2, contract_text='\n'.join([v for v in case_2.values()]), keyword_mappings=test_keyword_mappings)[ "or_1" ] == "\n".join(["NA", "kw2"]) def test_or_2(): - assert smart_chunk_ac(text_dict=case_1, keyword_mappings=test_keyword_mappings)[ + assert smart_chunk_ac(text_dict=case_1, contract_text='\n'.join([v for v in case_1.values()]), keyword_mappings=test_keyword_mappings)[ "or_2" ] == "\n".join([]) - assert smart_chunk_ac(text_dict=case_2, keyword_mappings=test_keyword_mappings)[ + assert smart_chunk_ac(text_dict=case_2, contract_text='\n'.join([v for v in case_2.values()]), keyword_mappings=test_keyword_mappings)[ "or_2" ] == "\n".join(["kw4"]) def test_case_sensitive_or_1(): - assert smart_chunk_ac(text_dict=case_1, keyword_mappings=test_keyword_mappings)[ + assert smart_chunk_ac(text_dict=case_1, contract_text='\n'.join([v for v in case_1.values()]), keyword_mappings=test_keyword_mappings)[ "cs_or_1" ] == "\n".join(["NA", "kw3"]) - assert smart_chunk_ac(text_dict=case_3, keyword_mappings=test_keyword_mappings)[ + assert smart_chunk_ac(text_dict=case_3, contract_text='\n'.join([v for v in case_3.values()]), keyword_mappings=test_keyword_mappings)[ "cs_or_1" ] == "\n".join(["NA", "KW1", "NA", "NA", "kw3"]) def test_hierarchy_1(): - assert smart_chunk_ac(text_dict=case_1, keyword_mappings=test_keyword_mappings)[ + assert smart_chunk_ac(text_dict=case_1, contract_text='\n'.join([v for v in case_1.values()]), keyword_mappings=test_keyword_mappings)[ "hierarchy_1" ] == "\n".join(["NA", "h6", "NA"]) - assert smart_chunk_ac(text_dict=case_2, keyword_mappings=test_keyword_mappings)[ + assert smart_chunk_ac(text_dict=case_2, contract_text='\n'.join([v for v in case_2.values()]), keyword_mappings=test_keyword_mappings)[ "hierarchy_1" ] == "\n".join(["NA", "h5", "NA"]) def test_and(): - assert smart_chunk_ac(text_dict=case_4, keyword_mappings=test_keyword_mappings)[ + assert smart_chunk_ac(text_dict=case_4, contract_text='\n'.join([v for v in case_4.values()]), keyword_mappings=test_keyword_mappings)[ "and" ] == "\n".join(["NA", "KW1KW2", "NA"]) def test_regex(): - assert smart_chunk_ac(text_dict=case_5, keyword_mappings=test_keyword_mappings)[ + assert smart_chunk_ac(text_dict=case_5, contract_text='\n'.join([v for v in case_5.values()]), keyword_mappings=test_keyword_mappings)[ "regex" ] == "\n".join(["NA", "12-3456789", "987654321", "Next"]) @@ -156,6 +160,7 @@ def test_undefined_methodology(): with pytest.raises(ValueError): smart_chunk_ac( text_dict=case_1, + contract_text='\n'.join([v for v in case_1.values()]), keyword_mappings={ "e1": { "methodology": "Not a real methodology",