From b4de4048ca8427e384716d07bd32a5ddb5bcf07c Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Wed, 27 Aug 2025 17:42:35 +0000 Subject: [PATCH] Merged in bugfix/reimb-primary-issues (pull request #682) Bugfix/reimb primary issues * Merge + test * Extract (and remove) headers and footers * Simplify prompt * Restore * Merged main into bugfix/reimb-primary-issues * Successful E2E test * Cleanup * Return proper * Merged main into bugfix/reimb-primary-issues Approved-by: Alex Galarce --- .../src/investment/file_processing.py | 1 + fieldExtraction/src/investment/preprocess.py | 77 +++++++++++++++++++ fieldExtraction/src/investment/row_funcs.py | 2 +- .../src/prompts/investment_prompts.json | 2 +- 4 files changed, 80 insertions(+), 2 deletions(-) diff --git a/fieldExtraction/src/investment/file_processing.py b/fieldExtraction/src/investment/file_processing.py index e0cad4a..1e028e7 100644 --- a/fieldExtraction/src/investment/file_processing.py +++ b/fieldExtraction/src/investment/file_processing.py @@ -28,6 +28,7 @@ def process_file(file_object, constants: Constants, run_timestamp): ################## PREPROCESS ################## contract_text = preprocess.clean_text(contract_text) text_dict, top_sheet_dict = preprocess.split_text(contract_text) + text_dict, header, footer = preprocess.find_headers_and_footers(text_dict) text_dict = preprocess.clean_tables( text_dict, constants.EXHIBIT_HEADER_MARKERS, filename ) diff --git a/fieldExtraction/src/investment/preprocess.py b/fieldExtraction/src/investment/preprocess.py index 2b16a94..91b7409 100644 --- a/fieldExtraction/src/investment/preprocess.py +++ b/fieldExtraction/src/investment/preprocess.py @@ -131,3 +131,80 @@ def clean_tables( ) return text_dict_final + + + +def find_headers_and_footers(text_dict: dict[str, str]) -> tuple[dict, str, str]: + """ + Returns a tuple of two lists: (header_markers, footer_markers). + + header_markers: The longest substring that is found on EVERY page of the text_dict at the START of the page. + footer_markers: The longest substring that is found on EVERY page of the text_dict at the END of the page. + + Args: + text_dict (dict): A dictionary where keys are page numbers and values are the text of those pages. + Returns: + tuple: A tuple containing two strings: + - header_marker: A string representing the longest common substring found at the start of each page. Can be empty string + - footer_marker: A string representing the longest common substring found at the foot of each page. Can be empty string + """ + + if not text_dict: + return {}, "", "" + + # Get the pages as a list + pages = list(text_dict.values()) + + # If there's only one page, return empty strings and original dict + if len(pages) <= 1: + return text_dict.copy(), "", "" + + # Split pages into words + pages_words = [page.split() for page in pages] + + # Find header (common prefix words) + header_words = [] + min_word_count = min(len(page_words) for page_words in pages_words) + + for i in range(min_word_count): + word = pages_words[0][i] + if all(page_words[i] == word for page_words in pages_words[1:]): + header_words.append(word) + else: + break + + # Find footer (common suffix words) + footer_words = [] + for i in range(1, min_word_count + 1): + word = pages_words[0][-i] + if all(page_words[-i] == word for page_words in pages_words[1:]): + footer_words.insert(0, word) # Insert at beginning to maintain order + else: + break + + # Convert back to strings with proper spacing + header_marker = " ".join(header_words) + footer_marker = " ".join(footer_words) + + # Create a modified text_dict with headers and footers removed + modified_text_dict = {} + + first_page = True + for page_num, page_text in text_dict.items(): + modified_text = page_text + + if not first_page: # Keep global header for the first page that it occurs + # Remove header if it exists + if header_marker: + modified_text = modified_text[len(header_marker):].lstrip() + + # Remove footer if it exists + if footer_marker: + modified_text = modified_text[:-len(footer_marker)].rstrip() + + modified_text_dict[page_num] = modified_text + else: + first_page = False + modified_text_dict[page_num] = page_text + + return modified_text_dict, header_marker, footer_marker diff --git a/fieldExtraction/src/investment/row_funcs.py b/fieldExtraction/src/investment/row_funcs.py index b8cc514..37a3a00 100644 --- a/fieldExtraction/src/investment/row_funcs.py +++ b/fieldExtraction/src/investment/row_funcs.py @@ -123,5 +123,5 @@ def inject_global_lesser_of_rows( logging.debug( f"No new rows created for global lesser of injection for {filename}." ) - + return one_to_n_results diff --git a/fieldExtraction/src/prompts/investment_prompts.json b/fieldExtraction/src/prompts/investment_prompts.json index 8e200fc..40c1d18 100644 --- a/fieldExtraction/src/prompts/investment_prompts.json +++ b/fieldExtraction/src/prompts/investment_prompts.json @@ -21,7 +21,7 @@ "field_name": "GLOBAL_LESSER_OF_STATEMENT", "relationship": "one_to_one", "field_type": "full_context_separate", - "prompt": "Find ONLY contract-wide global constraints that apply to ALL services regardless of product line.\n\nA global constraint must:\n- Appear OUTSIDE of product-specific sections\n- Use universal language like 'All services', 'Except as otherwise provided', 'Notwithstanding any other provision'\n- Apply to the ENTIRE contract, not just specific products\n\nDO NOT extract:\n- Constraints within Medicaid/Medicare/Commercial/Marketplace sections\n- Repeated similar language across different products\n- Product-specific methodologies\n\nValid global examples:\n✅ 'Except as otherwise provided in this Agreement, all services shall be reimbursed at the lesser of...'\n✅ 'Notwithstanding any specific methodology in this contract, payment shall not exceed...'\n\nInvalid examples:\n❌ Section 1.1 Medicaid constraints\n❌ Section 2.1 Medicare constraints \n❌ Any constraint that repeats for different products\n\nIf this contract only has product-specific constraints (even if they're similar), return 'N/A'.\nIf you find a true contract-wide global constraint, extract it exactly.\n\nMost contracts will return 'N/A' - only extract if genuinely global." + "prompt": "Find 'Lesser of' language that applies to ALL Services in the contract. Be careful not to include 'Lesser of' phrasing that only applies to the specific Exhibit. If you find a true contract-wide global constraint, extract it exactly. Most contracts will return 'N/A' - only extract if genuinely global." }, { "field_name": "EXHIBIT_LESSER_OF_STATEMENT",