From dfff68e9c8a51ba93333c5d76915a19e0f061daa Mon Sep 17 00:00:00 2001 From: Karan Desai Date: Wed, 13 Nov 2024 21:33:17 +0000 Subject: [PATCH] Merged in bugfix/clean_full_context_in_smart_chunking (pull request #269) Bugfix/clean full context in smart chunking * move full context bugfix to clean branch * took out commented code * black formating * Add explanatory comments to file_processing.py Approved-by: Alex Galarce --- fieldExtraction/src/file_processing.py | 243 +++++++++++---------- fieldExtraction/src/keywords.py | 77 ++----- fieldExtraction/src/preprocessing_funcs.py | 61 ------ 3 files changed, 152 insertions(+), 229 deletions(-) diff --git a/fieldExtraction/src/file_processing.py b/fieldExtraction/src/file_processing.py index cb78647..cb7a0c3 100644 --- a/fieldExtraction/src/file_processing.py +++ b/fieldExtraction/src/file_processing.py @@ -47,11 +47,13 @@ def run_ac_prompts(file_object): ## Check `ac_funcs.create_prompt()` vs. `prompts.AC_*_template()` all_fields = set(prompts.AC_DICT.keys()) - field_groups = [ field_group for field_group in keywords.GROUPED_KEYWORD_MAPPINGS.keys() ] + # list to append the fields with no matching keywords + no_keyword_matches = [] + for field_group in field_groups: keyword_dict = keywords.GROUPED_KEYWORD_MAPPINGS[field_group]["keywords"] fields = keywords.GROUPED_KEYWORD_MAPPINGS[field_group]["fields"] @@ -60,90 +62,151 @@ def run_ac_prompts(file_object): questions[field] = prompts.AC_DICT[field] # Use chunk if available and different from full context, otherwise use full context + ############# Important Note ############# + # the above logic is changed + # New Logic - chunk if available and different from full context, otherwise add the field to full context list if ( field_group in ac_chunks and ac_chunks[field_group].strip() and ac_chunks[field_group] != contract_text ): + context = ac_chunks[field_group] context = context[0 : min(100000, len(context)) - 1] context_type = "Smart Chunking" - else: - context = contract_text[0 : min(100000, len(contract_text) - 1)] - context_type = "Full Context" - - if len(fields) == 1: - question = questions[ - fields[0] - ] # Take the one question for the single field included - prompt = prompts.AC_SINGLE_FIELD_TEMPLATE(context, question) - elif len(fields) > 1: - prompt = prompts.AC_MULTI_FIELD_TEMPLATE(context, questions) - else: - raise ValueError("Field group has no defined fields") - - # TODO: Make sure prompt looks good for groups - - try: - # field_group_answer_raw = {"test field": "test value"} ## uncomment this and comment below line to test keyword search - field_group_answer_raw = claude_funcs.invoke_claude( - prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 8192 - ) if len(fields) == 1: - field_group_answer_dict = {field: field_group_answer_raw} + question = questions[ + fields[0] + ] # Take the one question for the single field included + prompt = prompts.AC_SINGLE_FIELD_TEMPLATE(context, question) elif len(fields) > 1: - field_group_answer_dict = json.loads(field_group_answer_raw) + prompt = prompts.AC_MULTI_FIELD_TEMPLATE(context, questions) + else: + raise ValueError("Field group has no defined fields") - # TODO: Validate field names coming out of Claude, check against fields in group, make sure they're all there, make sure there aren't any extras - for field in fields: - field_answer = field_group_answer_dict[field] - ac_answers_dict[field] = field_answer + try: + field_group_answer_raw = claude_funcs.invoke_claude( + prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 8192 + ) - log_data.append( - { - "Field": field, - "Prompt": f"Question: {questions}\n\nContext: {context}", # Include full context - "Context Type": context_type, - "Response": field_answer, - } + if len(fields) == 1: + field_group_answer_dict = {field: field_group_answer_raw} + elif len(fields) > 1: + field_group_answer_dict = json.loads(field_group_answer_raw) + + # TODO: Validate field names coming out of Claude, check against fields in group, make sure they're all there, make sure there aren't any extras + for field in fields: + field_answer = field_group_answer_dict[field] + ac_answers_dict[field] = field_answer + + log_data.append( + { + "Field": field, + "Prompt": f"Question: {questions}\n\nContext: {context}", # Include full context + "Context Type": context_type, + "Response": field_answer, + } + ) + + chunk_data.append( + { + "Contract name": filename, + "Field group": field_group, + "Field list": fields, + "Methodology": ac_chunks[field_group + "_methodology"], + "Case sensitivity": ac_chunks[field_group + "_case"], + "Page list": ac_chunks[field_group + "_pages"], + "Chunk size": len(context), + "% Reduction": f"{100*(1-len(context)/len(contract_text)):0.2f}%", + } + ) + except Exception as e: + print(f"Error processing field {field}: {type(e)}, {str(e)}") + ac_answers_dict[field] = f"Error: {str(e)}" + log_data.append( + { + "Field": field, + "Prompt": f"Question: {questions}\n\nContext: {context}", # Include full context + "Context Type": context_type, + "Response": f"Error: {str(e)}", + } + ) + chunk_data.append( + { + "Contract name": filename, + "Field group": field_group, + "Field list": fields, + "Methodology": ac_chunks[field_group + "_methodology"], + "Case sensitivity": ac_chunks[field_group + "_case"], + "Page list": ac_chunks[field_group + "_pages"], + "Chunk size": len(context), + "% Reduction": f"{100*(1-len(context)/len(contract_text)):0.2f}%", + } + ) + else: + no_keyword_matches.append(field) + + ################## RUN FULL CONTEXT PROMPTS ################## + # Process 'full_context' fields together + + smartly_chunked = [] + for field_groups in keywords.GROUPED_KEYWORD_MAPPINGS.keys(): + for fields in keywords.GROUPED_KEYWORD_MAPPINGS[field_groups]["fields"]: + smartly_chunked.append(fields) + + # Instantiate full context fields from those that weren't smart chunked + full_context_fields = [ + field for field in prompts.AC_DICT if field not in smartly_chunked + ] + + # Add fields with no keyword matches to full context list + if len(no_keyword_matches) > 0: + full_context_fields.extend(no_keyword_matches) + + # Filter out fields that already have answers + # e.g., some have already been filled out by regex chunking + full_context_fields = [ + field for field in full_context_fields if field not in ac_answers_dict.keys() + ] + + full_context_questions = { + field: prompts.AC_DICT.get(field) for field in full_context_fields + } + if full_context_questions: + full_context_prompt = ac_funcs.create_prompt( + contract_text[0 : min(100000, len(contract_text) - 1)], + question=full_context_questions, + ) + try: + full_context_answers = ac_funcs.get_ac_answer( + full_context_prompt, + filename, + fields=list(full_context_questions.keys()), ) + ac_answers_dict.update(full_context_answers) - chunk_data.append( - { - "Contract name": filename, - "Field group": field_group, - "Field list": fields, - "Methodology": ac_chunks[field_group + "_methodology"], - "Case sensitivity": ac_chunks[field_group + "_case"], - "Page list": ac_chunks[field_group + "_pages"], - "Chunk size": len(context), - "% Reduction": f"{100*(1-len(context)/len(contract_text)):0.2f}%", - } - ) + for field, answer in full_context_answers.items(): + log_data.append( + { + "Field": field, + "Prompt": f"Question: {full_context_questions[field]}\n\nContext: {contract_text}", # Include full context + "Context Type": "Full Context (High Accuracy)", + "Response": answer, + } + ) except Exception as e: - print(f"Error processing field {field}: {type(e)}, {str(e)}") - ac_answers_dict[field] = f"Error: {str(e)}" - log_data.append( - { - "Field": field, - "Prompt": f"Question: {questions}\n\nContext: {context}", # Include full context - "Context Type": context_type, - "Response": f"Error: {str(e)}", - } - ) - chunk_data.append( - { - "Contract name": filename, - "Field group": field_group, - "Field list": fields, - "Methodology": ac_chunks[field_group + "_methodology"], - "Case sensitivity": ac_chunks[field_group + "_case"], - "Page list": ac_chunks[field_group + "_pages"], - "Chunk size": len(context), - "% Reduction": f"{100*(1-len(context)/len(contract_text)):0.2f}%", - } - ) + print(f"Error processing high accuracy fields: {str(e)}") + for field in full_context_questions.keys(): + ac_answers_dict[field] = f"Error: {str(e)}" + log_data.append( + { + "Field": field, + "Prompt": f"Question: {full_context_questions[field]}\n\nContext: {contract_text}", # Include full context + "Context Type": "Full Context (High Accuracy)", + "Response": f"Error: {str(e)}", + } + ) ################## AC CONDITIONAL PROMPTS ################## # Non-Renewal - Days @@ -168,48 +231,6 @@ def run_ac_prompts(file_object): ) ac_answers_dict["CONTRACT_EFFECTIVE_DT"] = date_answer - ################## RUN FULL CONTEXT PROMPTS ################## - # Process 'full_context' fields together - - smartly_chunked = [] - for field_groups in keywords.GROUPED_KEYWORD_MAPPINGS.keys(): - for fields in keywords.GROUPED_KEYWORD_MAPPINGS[field_groups]["fields"]: - smartly_chunked.append(fields) - full_context_fields = [ - field for field in prompts.AC_DICT if field not in smartly_chunked - ] - full_context_fields = [ - field for field in full_context_fields if field not in ac_answers_dict.keys() - ] - - full_context_questions = { - field: prompts.AC_DICT.get(field) for field in full_context_fields - } - if full_context_questions: - full_context_prompt = ac_funcs.create_prompt(contract_text[0:min(100000, len(contract_text)-1)], question=full_context_questions) - try: - full_context_answers = ac_funcs.get_ac_answer(full_context_prompt, filename, fields=list(full_context_questions.keys())) - ac_answers_dict.update(full_context_answers) - - for field, answer in full_context_answers.items(): - log_data.append({ - 'Field': field, - 'Prompt': f"Question: {full_context_questions[field]}\n\nContext: {contract_text}", # Include full context - 'Context Type': 'Full Context (High Accuracy)', - 'Response': answer - }) - except Exception as e: - print(f"Error processing high accuracy fields: {str(e)}") - for field in full_context_questions.keys(): - ac_answers_dict[field] = f"Error: {str(e)}" - log_data.append({ - 'Field': field, - 'Prompt': f"Question: {full_context_questions[field]}\n\nContext: {contract_text}", # Include full context - 'Context Type': 'Full Context (High Accuracy)', - 'Response': f"Error: {str(e)}" - }) - - ################## ENSURE ALL FIELDS ARE PRESENT ################## for field in all_fields: if field not in ac_answers_dict: diff --git a/fieldExtraction/src/keywords.py b/fieldExtraction/src/keywords.py index 8aef71b..de08b50 100644 --- a/fieldExtraction/src/keywords.py +++ b/fieldExtraction/src/keywords.py @@ -15,47 +15,11 @@ GROUPED_KEYWORD_MAPPINGS = { "fields": [ "TERM_CLAUSE", "CONTRACT_AUTO_RENEWAL_IND", - "CONTRACT_TERMINATION_DT", "TERMINATION_UPON_NOTICE", "TERMINATION_UPON_CAUSE", ], "methodology": "or", - "keywords": [ - "TERM AND TERMINATION", - "Term", - "Renew", - "upon notice", - "With Cause", - ], - "case_sensitive": True, - }, - "initial_page_group": { - "fields": ["CONTRACT_TITLE", "PAYER_NAME", "HEALTH_PLAN_STATE"], - "methodology": "or", - "keywords": ["Health Plan Name", "Health Maintenance Organization"], - "case_sensitive": False, - }, - "section_5_6_group": { - "fields": [ - "INSURANCE_REQUIREMENT", - "INDEMNIFICATION", - "ARBITRATION_AND_DISPUTES", - ], - "methodology": "or", - "keywords": [ - "INSURANCE AND INDEMNIFICATION", - "5.1", - "Insurance", - "Insurance Requirement", - "5.2", - "Indemnification", - "Demnification", - "6.1", - "6.2", - "Arbitration", - "Arbitrate", - "Dispute", - ], + "keywords": ["Term", "Clause", "Renew", "upon notice", "With Cause"], "case_sensitive": False, }, # IRS group fields not smart chunked currently. @@ -141,26 +105,25 @@ GROUPED_KEYWORD_MAPPINGS = { ], # TODO Switch methodology to regex for date formats + 'or' methodology is case-insensitive -remove duplicates "case_sensitive": False, }, - # "TERMINATION_UPON_NOTICE": { - # "fields": ["TERMINATION_UPON_NOTICE"], - # "methodology": "or", - # "keywords": ["notice", "upon notice", "termination"], - # "case_sensitive": False, - # }, - # "PROV_GROUP_TIN": { - # "fields": ["PROV_GROUP_TIN"], - # "methodology": "or", - # "keywords": [ - # "Tax Identification Number", - # "TIN", - # "Tax", - # "ID", - # "Tax ID Number", - # "Taxpayer ID", - # ], - # "case_sensitive": False, - # }, - # TODO Add regex to match ##-####### or ######### + "TERMINATION_UPON_NOTICE": { + "fields": ["TERMINATION_UPON_NOTICE"], + "methodology": "or", + "keywords": ["notice", "upon notice", "termination"], + "case_sensitive": False, + }, + "PROV_GROUP_TIN": { + "fields": ["PROV_GROUP_TIN"], + "methodology": "or", + "keywords": [ + "Tax Identification Number", + "TIN", + "Tax", + "ID", + "Tax ID Number", + "Taxpayer ID", + ], + "case_sensitive": False, + }, # TODO Add regex to match ##-####### or ######### # "group_name": { # "fields" : [], # 'methodology' : "", diff --git a/fieldExtraction/src/preprocessing_funcs.py b/fieldExtraction/src/preprocessing_funcs.py index 1fb5b8f..70dc79e 100644 --- a/fieldExtraction/src/preprocessing_funcs.py +++ b/fieldExtraction/src/preprocessing_funcs.py @@ -265,64 +265,3 @@ def filter_quick_review(text_dict): and "TOP SHEET" not in page_text.upper() and "COVER SHEET" not in page_text.upper() } - - -# Regex TIN Logic -# Regex search for all TINs (##-####### format) -# If exactly one TIN is found: that is the TIN -# If more than one TIN is found: chunk + prompt to find which of them is the correct TIN - -# If zero TINs are found, regex search for all TINs (######### format) -# Then repeat steps 2 and 3 with the new pattern -# If still zero TINs are found, check for a TIN in the filename -# If still none, return empty - - -# def tin_regex(filename, text_dict: dict[str, str]): -# pattern_1 = r'\b\d{2}-?\d{7}\b' -# matches = [] -# page_list = [] - -# for page in text_dict.keys(): -# if len(re.findall((pattern_1, text_dict[page]))) != 0: -# matches.extend(re.findall((pattern_1, text_dict[page]))) -# page_list.append(page) - -# if len(matches) == 1: -# return matches[0] - -# elif len(matches)>1: -# chunks = {} -# for pages in page_list: -# chunks[pages] = text_dict[pages] -# return chunks - -# elif len(matches) == 0: -# pattern_2 = r'\b\d{9}\b' -# matches = [] -# page_list = [] - -# for page in text_dict.keys(): -# if len(re.findall((pattern_2, text_dict[page]))) != 0: -# matches.extend(re.findall((pattern_2, text_dict[page]))) -# page_list.append(page) - - -# if len(matches) == 1: -# return matches[0] - -# elif len(matches)>1: -# chunks = {} -# for pages in page_list: -# chunks[pages] = text_dict[pages] -# return chunks - -# # find regex in file name -# tin = re.findall(pattern_1, filename) -# if len(tin) == 1: -# return tin[0] -# if len(tin) == 0: -# tin = re.findall(pattern_2, filename) -# if len(tin) == 1: -# return tin[0] -# return None