Merged in feature/claude4-duplicate-and-missing-rows (pull request #636)

Feature/claude4 duplicate and missing rows

* Enhance crosswalk functionality with flatten_to_strings helper and improve value handling in apply_crosswalk

* Improve apply_crosswalk function to return original value as fallback when no mapping is found

* Enhance apply_crosswalk function to handle empty values and improve defensive programming tests

* Enhance reimbursement method prompt to include charge master limitations and improve mapping rules

* Merge remote-tracking branch 'origin/main' into feature/claude4-duplicate-and-missing-rows

* Refine CHECK_REDUNDANT_LESSER prompt for clarity and detail in evaluating redundancy of exhibit constraints

* Enable logging of LLM response for redundancy check in is_exhibit_lesser_of_redundant function

* Merge remote-tracking branch 'origin/main' into feature/claude4-duplicate-and-missing-rows

* rework prompt for exhibit-wide lesser of

* Refine reimbursement methodology guidelines by clarifying invalid entries and examples for risk-sharing

* Enhance exhibit linking logic to treat pages with substantial tables as separate exhibits and update function signature to include text_dict.


Approved-by: Katon Minhas

fieldExtraction/src/preprocessing_funcs.py

@@ -6,6 +6,7 @@ import src.utils.llm_utils as llm_utils
 import src.utils.string_utils as string_utils
 from src import config, keywords
 from src.regex.regex_patterns import PIPE_PATTERN
+from src.investment import table_funcs
 
 
 from src import config, keywords
@@ -259,7 +260,7 @@ def get_exhibit_pages(text_dict: dict[str, str], filename: str) -> tuple[list[st
     return exhibit_pages, exhibit_header_dict
 
 
-def link_exhibit_pages(all_exhibit_headers, filename):
+def link_exhibit_pages(all_exhibit_headers: dict[str, str], text_dict: dict[str, str], filename: str):
     """
     Filters exhibit pages to include only those with meaningfully different headers.
     This function processes exhibit headers sequentially and uses literal comparison and 
@@ -270,6 +271,7 @@ def link_exhibit_pages(all_exhibit_headers, filename):
     This function will link contiguous header pages together, so that the final list of exhibit pages will only contain pages that are meaningfully different from the previous page.  E.g. "1" with header {"Exhibit 1"} and "2" with header {"Continued exhibit 1"} will be linked together, but "3" with header {"Exhibit 2"} will not be linked to either of them.
     Args:
         all_exhibit_headers (dict[str, str]): A dictionary mapping page numbers (as strings) to their exhibit headers.  Expected to be in chronological order
+        text_dict (dict[str, str]): A dictionary mapping page numbers to their text content.
         filename (str): The name of the file being processed, used for logging and LLM invocation.
     Returns:
         tuple[list[str], dict[str, str]]: A tuple containing:
@@ -296,12 +298,20 @@ def link_exhibit_pages(all_exhibit_headers, filename):
             final_exhibit_headers[page_num] = page_header
             first_exhibit = False
         else:
-            # If literally identical, skip prompt
-            if page_header.upper().strip() == previous_header.upper().strip():
+            # Check if this page has a substantial table
+            has_table = has_substantial_table(text_dict.get(page_num, ""))
+
+            # If literally identical, skip prompt (unless it has a table)
+            if page_header.upper().strip() == previous_header.upper().strip() and not has_table:
                 exhibit_is_different = "N"
             elif string_utils.is_empty(previous_header):
                 exhibit_is_different = "Y"
                 previous_header = page_header
+            elif has_table:
+                # Always treat pages with substantial tables as different exhibits
+                logging.debug(f"Page {page_num} has substantial table, treating as separate exhibit")
+                exhibit_is_different = "Y"
+                previous_header = page_header
             else:
                 prompt = investment_prompts.EXHIBIT_LINKAGE(page_header, previous_header)
                 claude_answer_raw = llm_utils.invoke_claude(prompt, "sonnet_latest", filename)
@@ -316,6 +326,33 @@ def link_exhibit_pages(all_exhibit_headers, filename):
     return final_exhibit_pages, final_exhibit_headers
 
 
+def has_substantial_table(page_text: str) -> bool:
+    """Check if a page contains a substantial table worth preserving as a separate chunk.
+
+    Args:
+        page_text (str): The text content of the page to check.
+
+    Returns:
+        bool: True if the page contains a substantial table, False otherwise.
+    """
+    if not page_text or table_funcs.START_MARKER not in page_text:
+        return False
+    
+    # Count table rows to determine if it's substantial
+    table_start = page_text.find(table_funcs.START_MARKER)
+    table_end = page_text.find(table_funcs.END_MARKER)
+
+    if table_start == -1 or table_end == -1:
+        return False
+    
+    table_content = page_text[table_start:table_end]
+
+    # Count rows (each row is a list in the table representation)
+    row_count = table_content.count("'], ['") # Rough count of table rows
+
+    return row_count >= 5 # Right now, using "5 rows" as "substantial".  Alternative: use config.TABLE_ROW_LIMIT
+
+
 def chunk_by_exhibit(text_dict: dict, 
                      exhibit_pages: list
                      ) -> dict: