Merged in bugfix/reimb-primary-issues (pull request #682)

Bugfix/reimb primary issues

* Merge + test

* Extract (and remove) headers and footers

* Simplify prompt

* Restore

* Merged main into bugfix/reimb-primary-issues

* Successful E2E test

* Cleanup

* Return proper

* Merged main into bugfix/reimb-primary-issues


Approved-by: Alex Galarce

fieldExtraction/src/investment/file_processing.py

@@ -28,6 +28,7 @@ def process_file(file_object, constants: Constants, run_timestamp):
     ################## PREPROCESS ##################
     contract_text = preprocess.clean_text(contract_text)
     text_dict, top_sheet_dict = preprocess.split_text(contract_text)
+    text_dict, header, footer = preprocess.find_headers_and_footers(text_dict)
     text_dict = preprocess.clean_tables(
         text_dict, constants.EXHIBIT_HEADER_MARKERS, filename
     )

fieldExtraction/src/investment/preprocess.py

@@ -131,3 +131,80 @@ def clean_tables(
     )
 
     return text_dict_final
+
+
+
+def find_headers_and_footers(text_dict: dict[str, str]) -> tuple[dict, str, str]:
+    """
+    Returns a tuple of two lists: (header_markers, footer_markers).
+
+    header_markers: The longest substring that is found on EVERY page of the text_dict at the START of the page.
+    footer_markers: The longest substring that is found on EVERY page of the text_dict at the END of the page.
+
+    Args:
+        text_dict (dict): A dictionary where keys are page numbers and values are the text of those pages.
+    Returns:
+        tuple: A tuple containing two strings:
+            - header_marker: A string representing the longest common substring found at the start of each page. Can be empty string
+            - footer_marker: A string representing the longest common substring found at the foot of each page. Can be empty string
+    """
+   
+    if not text_dict:
+        return {}, "", ""
+    
+    # Get the pages as a list
+    pages = list(text_dict.values())
+    
+    # If there's only one page, return empty strings and original dict
+    if len(pages) <= 1:
+        return  text_dict.copy(), "", ""
+    
+    # Split pages into words
+    pages_words = [page.split() for page in pages]
+    
+    # Find header (common prefix words)
+    header_words = []
+    min_word_count = min(len(page_words) for page_words in pages_words)
+    
+    for i in range(min_word_count):
+        word = pages_words[0][i]
+        if all(page_words[i] == word for page_words in pages_words[1:]):
+            header_words.append(word)
+        else:
+            break
+    
+    # Find footer (common suffix words)
+    footer_words = []
+    for i in range(1, min_word_count + 1):
+        word = pages_words[0][-i]
+        if all(page_words[-i] == word for page_words in pages_words[1:]):
+            footer_words.insert(0, word)  # Insert at beginning to maintain order
+        else:
+            break
+    
+    # Convert back to strings with proper spacing
+    header_marker = " ".join(header_words)
+    footer_marker = " ".join(footer_words)
+    
+    # Create a modified text_dict with headers and footers removed
+    modified_text_dict = {}
+    
+    first_page = True
+    for page_num, page_text in text_dict.items():
+        modified_text = page_text
+
+        if not first_page: # Keep global header for the first page that it occurs
+            # Remove header if it exists
+            if header_marker:
+                modified_text = modified_text[len(header_marker):].lstrip()
+                
+            # Remove footer if it exists
+            if footer_marker:
+                modified_text = modified_text[:-len(footer_marker)].rstrip()
+                
+            modified_text_dict[page_num] = modified_text
+        else:
+            first_page = False
+            modified_text_dict[page_num] = page_text
+    
+    return modified_text_dict, header_marker, footer_marker

fieldExtraction/src/investment/row_funcs.py

@@ -123,5 +123,5 @@ def inject_global_lesser_of_rows(
         logging.debug(
             f"No new rows created for global lesser of injection for {filename}."
         )
-
+        
     return one_to_n_results

fieldExtraction/src/prompts/investment_prompts.json

@@ -21,7 +21,7 @@
         "field_name": "GLOBAL_LESSER_OF_STATEMENT",
         "relationship": "one_to_one",
         "field_type": "full_context_separate",
-        "prompt": "Find ONLY contract-wide global constraints that apply to ALL services regardless of product line.\n\nA global constraint must:\n- Appear OUTSIDE of product-specific sections\n- Use universal language like 'All services', 'Except as otherwise provided', 'Notwithstanding any other provision'\n- Apply to the ENTIRE contract, not just specific products\n\nDO NOT extract:\n- Constraints within Medicaid/Medicare/Commercial/Marketplace sections\n- Repeated similar language across different products\n- Product-specific methodologies\n\nValid global examples:\n✅ 'Except as otherwise provided in this Agreement, all services shall be reimbursed at the lesser of...'\n✅ 'Notwithstanding any specific methodology in this contract, payment shall not exceed...'\n\nInvalid examples:\n❌ Section 1.1 Medicaid constraints\n❌ Section 2.1 Medicare constraints  \n❌ Any constraint that repeats for different products\n\nIf this contract only has product-specific constraints (even if they're similar), return 'N/A'.\nIf you find a true contract-wide global constraint, extract it exactly.\n\nMost contracts will return 'N/A' - only extract if genuinely global."
+        "prompt": "Find 'Lesser of' language that applies to ALL Services in the contract. Be careful not to include 'Lesser of' phrasing that only applies to the specific Exhibit. If you find a true contract-wide global constraint, extract it exactly. Most contracts will return 'N/A' - only extract if genuinely global."
     },
     {
         "field_name": "EXHIBIT_LESSER_OF_STATEMENT",