jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/topSheetConditional (pull request #281)

Browse Source 
Initial top sheet conditional commit

* Initial top sheet conditional commit

* Merged main into feature/topSheetConditional

* preprocessing_funcs.py edited online with Bitbucket
* Updated quick review docstring


Approved-by: Alex Galarce
This commit is contained in:
Katon Minhas
2024-11-22 15:30:01 +00:00
parent 27d81ff1d3
commit cb18f90a85
3 changed files with 30 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fieldExtraction/src/file_processing.py
+4 -3
View File
@@ -32,7 +32,9 @@ def run_ac_prompts(file_object):
print(f"Processing AC for {filename}...")
################## PREPROCESS - SMART CHUNK ##################
text_dict, exhibit_pages, num_pages, ac_chunks = preprocess.preprocess(
# Top Sheet Addition
text_dict, top_sheet_dict, exhibit_pages, num_pages, ac_chunks = preprocess.preprocess(
contract_text, filename, fields="ac"
)
print(f"AC Preprocessing Complete - {filename}")
@@ -54,7 +56,6 @@ def run_ac_prompts(file_object):
no_keyword_matches = []
for field_group in field_groups:
keyword_dict = keywords.GROUPED_KEYWORD_MAPPINGS[field_group]["keywords"]
fields = keywords.GROUPED_KEYWORD_MAPPINGS[field_group]["fields"]
fields.sort()
@@ -287,7 +288,7 @@ def run_b_prompts(file_object):
print(f"Processing B for {filename}...")
################## PREPROCESS ##################
text_dict, exhibit_pages, num_pages, ac_chunks = preprocess.preprocess(
text_dict, top_sheet_dict, exhibit_pages, num_pages, ac_chunks = preprocess.preprocess(
contract_text, filename, fields="b"
)
print(f"B Preprocessing Complete - {filename}")
fieldExtraction/src/preprocess.py
+2 -2
View File
@@ -17,7 +17,7 @@ def preprocess(contract_text, filename, fields=config.FIELDS):
# text_dict = {k: v for k, v in text_dict.items() if k.isdigit() and 42 <= int(k) <= 42}
num_pages = len(text_dict.keys())
text_dict = preprocessing_funcs.filter_quick_review(text_dict)
text_dict, top_sheet_dict = preprocessing_funcs.filter_quick_review(text_dict)
if fields == "b":
exhibit_pages = top_down_funcs.get_exhibit_pages(
@@ -36,4 +36,4 @@ def preprocess(contract_text, filename, fields=config.FIELDS):
exhibit_pages = []
ac_chunks = preprocessing_funcs.smart_chunk_ac(text_dict, contract_text, keywords.GROUPED_KEYWORD_MAPPINGS)
return text_dict, exhibit_pages, num_pages, ac_chunks
return text_dict, top_sheet_dict, exhibit_pages, num_pages, ac_chunks
fieldExtraction/src/preprocessing_funcs.py
+24
View File
@@ -226,10 +226,34 @@ def smart_chunk_ac(
def filter_quick_review(text_dict):
"""
Cover Sheets (aka Top Sheets or Quick Review pages) are pages stapled to the front of the contract that contain manually written summaries of the contract's contents.
They are NOT legally binding documents, and because they are often manually filled out and hand-written, are more prone to have erroneous information than the rest of the contract.
For Doczy.AI, we should not pull any information from top sheets, except as a very last resort.
filter_quick_review() splits the input dictionary into two dictionaries, one containing the contract pages, the other containing the top sheet pages.
The function checks each page's text for the keywords "QUICK REVIEW", "TOP SHEET", and "COVER SHEET". It categorizes the text into two separate dictionaries: one for texts that do not contain any of these keywords, and another for texts that do.
Parameters:
text_dict (dict): A dictionary where the key is the page number and the value is the text of that page.
Returns:
tuple of two dicts:
- The first dictionary contains pages that do not have the specified keywords.
- The second dictionary includes pages that contain any of the specified keywords.
"""
return {
page_num: page_text
for page_num, page_text in text_dict.items()
if "QUICK REVIEW" not in page_text.upper()
and "TOP SHEET" not in page_text.upper()
and "COVER SHEET" not in page_text.upper()
}, {
page_num: page_text
for page_num, page_text in text_dict.items()
if "QUICK REVIEW" in page_text.upper()
or "TOP SHEET" in page_text.upper()
or "COVER SHEET" in page_text.upper()
}