jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in bugfix/full_context_check (pull request #270)

Browse Source 
Update for full-context AC chunks to return exact contract_text instead of concatenated text_dict pages

* Update for full-context AC chunks to return exact contract_text instead of concatenated text_dict pages

* Updated test cases

* Updated case 1

* Merged main into bugfix/full_context_check


Approved-by: Alex Galarce
This commit is contained in:
Katon Minhas
2024-11-14 21:16:43 +00:00
committed by Alex Galarce
parent 5928ee0116
commit 73020bdc10
3 changed files with 30 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fieldExtraction/src/preprocess.py
+1 -1
View File
@@ -33,6 +33,6 @@ def preprocess(contract_text, filename, fields=config.FIELDS):
ac_chunks, full_context = "", ""
if fields == "ac":
exhibit_pages = []
ac_chunks = preprocessing_funcs.smart_chunk_ac(text_dict)
ac_chunks = preprocessing_funcs.smart_chunk_ac(text_dict, contract_text)
return text_dict, exhibit_pages, num_pages, ac_chunks
fieldExtraction/src/preprocessing_funcs.py
+12 -4
View File
@@ -188,7 +188,9 @@ def chunk_consecutive(text_dict, exhibit_pages):
def smart_chunk_ac(
text_dict: dict, keyword_mappings: dict = keywords.GROUPED_KEYWORD_MAPPINGS
text_dict: dict,
contract_text: str,
keyword_mappings: dict = keywords.GROUPED_KEYWORD_MAPPINGS
) -> dict:
"""Performs a keyword search on textract outputs to retrieve relevant chunks for a given field.
@@ -248,9 +250,15 @@ def smart_chunk_ac(
)
if field_group == "initial_page_group":
keyword_page_dict["initial_pages"] = [1, 2]
chunks[field_group] = "\n".join(
[text_dict[str(page_num)] for page_num in page_list]
)
# chunks[field_group] = concatenated page texts
if len(page_list) == len(text_dict.keys()):
chunks[field_group] = contract_text
else:
chunks[field_group] = "\n".join(
[text_dict[str(page_num)] for page_num in page_list]
)
chunks[field_group + "_pages"] = keyword_page_dict
chunks[field_group + "_methodology"] = key_dict["methodology"]
chunks[field_group + "_case"] = key_dict["case_sensitive"]
fieldExtraction/tests/test_chunking.py
+17 -12
View File
@@ -8,6 +8,8 @@ test_chunk_dict_term = {
"4": "NA",
}
test_full_text_term = '\n'.join([v for v in test_chunk_dict_term.values()])
test_late_paid_claims = {
"1": "NA",
"2": "LATE PAYMENT", # try lowercase
@@ -18,6 +20,8 @@ test_late_paid_claims = {
"12": "NA",
}
test_full_text_late_paid_claims = '\n'.join([v for v in test_late_paid_claims.values()])
test_keyword_mappings = {
"or_1": {
"methodology": "or",
@@ -93,61 +97,61 @@ case_5 = {
def test_smart_chunk_term():
assert smart_chunk_ac(text_dict=test_chunk_dict_term)["term_group"] == "\n".join(
assert smart_chunk_ac(text_dict=test_chunk_dict_term, contract_text=test_full_text_term)["term_group"] == "\n".join(
["NA", "Termination", "NA"]
)
def test_smart_chunk_late_paid():
assert smart_chunk_ac(text_dict=test_late_paid_claims)[
assert smart_chunk_ac(text_dict=test_late_paid_claims, contract_text=test_full_text_late_paid_claims)[
"LATE_PAID_CLAIMS_LANGUAGE"
] == "\n".join(["NA", "LATE PAID", "NA"])
def test_or_1():
assert smart_chunk_ac(text_dict=case_1, keyword_mappings=test_keyword_mappings)[
assert smart_chunk_ac(text_dict=case_1, contract_text='\n'.join([v for v in case_1.values()]), keyword_mappings=test_keyword_mappings)[
"or_1"
] == "\n".join(["NA", "kw1", "NA", "NA", "kw3"])
assert smart_chunk_ac(text_dict=case_2, keyword_mappings=test_keyword_mappings)[
assert smart_chunk_ac(text_dict=case_2, contract_text='\n'.join([v for v in case_2.values()]), keyword_mappings=test_keyword_mappings)[
"or_1"
] == "\n".join(["NA", "kw2"])
def test_or_2():
assert smart_chunk_ac(text_dict=case_1, keyword_mappings=test_keyword_mappings)[
assert smart_chunk_ac(text_dict=case_1, contract_text='\n'.join([v for v in case_1.values()]), keyword_mappings=test_keyword_mappings)[
"or_2"
] == "\n".join([])
assert smart_chunk_ac(text_dict=case_2, keyword_mappings=test_keyword_mappings)[
assert smart_chunk_ac(text_dict=case_2, contract_text='\n'.join([v for v in case_2.values()]), keyword_mappings=test_keyword_mappings)[
"or_2"
] == "\n".join(["kw4"])
def test_case_sensitive_or_1():
assert smart_chunk_ac(text_dict=case_1, keyword_mappings=test_keyword_mappings)[
assert smart_chunk_ac(text_dict=case_1, contract_text='\n'.join([v for v in case_1.values()]), keyword_mappings=test_keyword_mappings)[
"cs_or_1"
] == "\n".join(["NA", "kw3"])
assert smart_chunk_ac(text_dict=case_3, keyword_mappings=test_keyword_mappings)[
assert smart_chunk_ac(text_dict=case_3, contract_text='\n'.join([v for v in case_3.values()]), keyword_mappings=test_keyword_mappings)[
"cs_or_1"
] == "\n".join(["NA", "KW1", "NA", "NA", "kw3"])
def test_hierarchy_1():
assert smart_chunk_ac(text_dict=case_1, keyword_mappings=test_keyword_mappings)[
assert smart_chunk_ac(text_dict=case_1, contract_text='\n'.join([v for v in case_1.values()]), keyword_mappings=test_keyword_mappings)[
"hierarchy_1"
] == "\n".join(["NA", "h6", "NA"])
assert smart_chunk_ac(text_dict=case_2, keyword_mappings=test_keyword_mappings)[
assert smart_chunk_ac(text_dict=case_2, contract_text='\n'.join([v for v in case_2.values()]), keyword_mappings=test_keyword_mappings)[
"hierarchy_1"
] == "\n".join(["NA", "h5", "NA"])
def test_and():
assert smart_chunk_ac(text_dict=case_4, keyword_mappings=test_keyword_mappings)[
assert smart_chunk_ac(text_dict=case_4, contract_text='\n'.join([v for v in case_4.values()]), keyword_mappings=test_keyword_mappings)[
"and"
] == "\n".join(["NA", "KW1KW2", "NA"])
def test_regex():
assert smart_chunk_ac(text_dict=case_5, keyword_mappings=test_keyword_mappings)[
assert smart_chunk_ac(text_dict=case_5, contract_text='\n'.join([v for v in case_5.values()]), keyword_mappings=test_keyword_mappings)[
"regex"
] == "\n".join(["NA", "12-3456789", "987654321", "Next"])
@@ -156,6 +160,7 @@ def test_undefined_methodology():
with pytest.raises(ValueError):
smart_chunk_ac(
text_dict=case_1,
contract_text='\n'.join([v for v in case_1.values()]),
keyword_mappings={
"e1": {
"methodology": "Not a real methodology",