From cb18f90a8577e26f7f70eca8f09da0b251c6f90a Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Fri, 22 Nov 2024 15:30:01 +0000 Subject: [PATCH] Merged in feature/topSheetConditional (pull request #281) Initial top sheet conditional commit * Initial top sheet conditional commit * Merged main into feature/topSheetConditional * preprocessing_funcs.py edited online with Bitbucket * Updated quick review docstring Approved-by: Alex Galarce --- fieldExtraction/src/file_processing.py | 7 ++++--- fieldExtraction/src/preprocess.py | 4 ++-- fieldExtraction/src/preprocessing_funcs.py | 24 ++++++++++++++++++++++ 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/fieldExtraction/src/file_processing.py b/fieldExtraction/src/file_processing.py index accffcf..9215840 100644 --- a/fieldExtraction/src/file_processing.py +++ b/fieldExtraction/src/file_processing.py @@ -32,7 +32,9 @@ def run_ac_prompts(file_object): print(f"Processing AC for {filename}...") ################## PREPROCESS - SMART CHUNK ################## - text_dict, exhibit_pages, num_pages, ac_chunks = preprocess.preprocess( + + # Top Sheet Addition + text_dict, top_sheet_dict, exhibit_pages, num_pages, ac_chunks = preprocess.preprocess( contract_text, filename, fields="ac" ) print(f"AC Preprocessing Complete - {filename}") @@ -54,7 +56,6 @@ def run_ac_prompts(file_object): no_keyword_matches = [] for field_group in field_groups: - keyword_dict = keywords.GROUPED_KEYWORD_MAPPINGS[field_group]["keywords"] fields = keywords.GROUPED_KEYWORD_MAPPINGS[field_group]["fields"] fields.sort() @@ -287,7 +288,7 @@ def run_b_prompts(file_object): print(f"Processing B for {filename}...") ################## PREPROCESS ################## - text_dict, exhibit_pages, num_pages, ac_chunks = preprocess.preprocess( + text_dict, top_sheet_dict, exhibit_pages, num_pages, ac_chunks = preprocess.preprocess( contract_text, filename, fields="b" ) print(f"B Preprocessing Complete - {filename}") diff --git a/fieldExtraction/src/preprocess.py b/fieldExtraction/src/preprocess.py index 4d5b639..91199d7 100644 --- a/fieldExtraction/src/preprocess.py +++ b/fieldExtraction/src/preprocess.py @@ -17,7 +17,7 @@ def preprocess(contract_text, filename, fields=config.FIELDS): # text_dict = {k: v for k, v in text_dict.items() if k.isdigit() and 42 <= int(k) <= 42} num_pages = len(text_dict.keys()) - text_dict = preprocessing_funcs.filter_quick_review(text_dict) + text_dict, top_sheet_dict = preprocessing_funcs.filter_quick_review(text_dict) if fields == "b": exhibit_pages = top_down_funcs.get_exhibit_pages( @@ -36,4 +36,4 @@ def preprocess(contract_text, filename, fields=config.FIELDS): exhibit_pages = [] ac_chunks = preprocessing_funcs.smart_chunk_ac(text_dict, contract_text, keywords.GROUPED_KEYWORD_MAPPINGS) - return text_dict, exhibit_pages, num_pages, ac_chunks + return text_dict, top_sheet_dict, exhibit_pages, num_pages, ac_chunks diff --git a/fieldExtraction/src/preprocessing_funcs.py b/fieldExtraction/src/preprocessing_funcs.py index 729eb73..78de8b5 100644 --- a/fieldExtraction/src/preprocessing_funcs.py +++ b/fieldExtraction/src/preprocessing_funcs.py @@ -226,10 +226,34 @@ def smart_chunk_ac( def filter_quick_review(text_dict): + """ + Cover Sheets (aka Top Sheets or Quick Review pages) are pages stapled to the front of the contract that contain manually written summaries of the contract's contents. + They are NOT legally binding documents, and because they are often manually filled out and hand-written, are more prone to have erroneous information than the rest of the contract. + For Doczy.AI, we should not pull any information from top sheets, except as a very last resort. + + filter_quick_review() splits the input dictionary into two dictionaries, one containing the contract pages, the other containing the top sheet pages. + + The function checks each page's text for the keywords "QUICK REVIEW", "TOP SHEET", and "COVER SHEET". It categorizes the text into two separate dictionaries: one for texts that do not contain any of these keywords, and another for texts that do. + + Parameters: + text_dict (dict): A dictionary where the key is the page number and the value is the text of that page. + + Returns: + tuple of two dicts: + - The first dictionary contains pages that do not have the specified keywords. + - The second dictionary includes pages that contain any of the specified keywords. + """ return { page_num: page_text for page_num, page_text in text_dict.items() if "QUICK REVIEW" not in page_text.upper() and "TOP SHEET" not in page_text.upper() and "COVER SHEET" not in page_text.upper() + }, { + page_num: page_text + for page_num, page_text in text_dict.items() + if "QUICK REVIEW" in page_text.upper() + or "TOP SHEET" in page_text.upper() + or "COVER SHEET" in page_text.upper() } +