Merged in feature/tin-npi-prompt-formatting (pull request #490)

Collect TIN/NPI/Name page-by-page and run a second pass for IS_GROUP

* configure first pass: no providers marked IS_GROUP.  Allows simplification of prompt

* Add group provider identification logic in second pass

* group identification: don't take 20% of pages.  Take first 3 and last 3

* Refine group provider identification logic to use pipes for JSON formatting and improve error handling

* fix mypy error for `identify_group_provider`

* Merge remote-tracking branch 'origin/main' into feature/tin-npi-prompt-formatting

* Add "PROV_INFO_JSON" to investment column order

* Refactor group provider identification to use signature page extraction and fix JSON format

* move signature page extraction to `string_utils.py` and add unit tests for it

* Update identify_group_provider to return provider name along with TIN and NPI in JSON format

* Update identify_group_provider to always use group-provided name when available

* move out GROUP_TIN_NPI_TEMPLATE to `investment_prompts.py`

* Remove deprecated PROV_GROUP_TIN_CHECK and PROV_GROUP_NPI_CHECK prompts

* Move out provider info fields from `one_to_one_funcs` to `tin_npi_funcs`

* isort

* remove debug print statements

* Remove debug print statements from provider info functions

* add note to regex_utils.py about where functions are used

* Add page_key_sort function and corresponding tests for sorting page keys

* Refactor identify_group_provider to sort pages and avoid duplicates in extracted sections

* Remove debug print statements from identify_group_provider function

* Merge remote-tracking branch 'origin/main' into feature/tin-npi-prompt-formatting

* extract_signature_page function: prioritize Textract marker for signatures and fallback to keyword search

* fix mypy error

* Update test cases to reflect changes in signature extraction logic


Approved-by: Katon Minhas

fieldExtraction/src/utils/string_utils.py

@@ -1,16 +1,15 @@
-from datetime import datetime
 import json
 import re
+import traceback
 import warnings
+from datetime import datetime
 
 import numpy as np
 import pandas as pd
-import traceback
-
-from src.prompts import preprocessing_prompts
 import src.utils.llm_utils as llm_utils
 from src import config
 from src.enums.delimiters import Delimiter
+from src.prompts import preprocessing_prompts
 from src.regex.regex_patterns import (BACKTICK_PATTERN, PIPE_PATTERN,
                                       TRIPLE_BACKTICK_PATTERN)
 
@@ -72,6 +71,35 @@ def extract_text_from_delimiters(
 
     return matches[match_index]
 
+def page_key_sort(page_key: str) -> tuple[int, int|str]:
+    """Sorts page keys, prioritizing numeric keys first.
+
+    Args:
+        page_key (str): The page key to sort.
+
+    Returns:
+        tuple[int, str]: A tuple where the first element is 0 for numeric keys and 1 for non-numeric keys, and the second element is the page key.
+    
+    Example:
+        >>> page_key_sort("1")
+        (0, 1)
+        >>> page_key_sort("A")
+        (1, "A")
+    
+    
+    Example Usage with mixed types:
+        >>> L = ["10", "2", "A", "1"]
+        >>> sort(L, key=page_key_sort)
+        >>> print(L)
+        ['1', '2', '10', 'A']
+    
+    """
+    try:
+        return (0, int(page_key))  # Numeric pages first, in numerical order
+    except ValueError:
+        return (1, str(page_key))  # Non-numeric pages after, in alphabetical order
+
+
 def json_parsing_search(response_text: str, field_list: list[str]) -> dict[str, str]:
     """
     Parses a JSON-like string to extract values for fields.
@@ -355,3 +383,35 @@ def get_exhibit_chunk(text_dict: dict,
 
 def datetime_str():
     return datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")
+
+
+def extract_signature_page(text_dict: dict, filename: str) -> dict:
+    """Extracts the signature page(s) from a contract text dictionary and returns it as a dictionary.
+    First tries to match the textract marker "this page has N signature(s)", then falls back to looking 
+    for the word "signature" if no matches are found.  "this page has N signature(s)" is Textract's marker for signatures
+
+    Args:
+        text_dict (dict): A dictionary containing text data, organized by pages.
+        filename (str): The name of the file being processed.
+
+    Returns:
+        dict: A dictionary containing the extracted signature page(s).
+    """
+
+    signature_pages = {}
+    # This regex pattern matches the Textract marker for signatures
+    # It will look for both numeric and a few textual representations of numbers (one-ten)
+    textract_pattern = re.compile(r"this page has (?:\d+|one|two|three|four|five|six|seven|eight|nine|ten) signature", re.IGNORECASE)
+        
+    # First pass: Try to match the full textract pattern
+    for page_num, page_text in text_dict.items():
+        if textract_pattern.search(page_text):
+            signature_pages[page_num] = page_text
+    
+    # If no pages found with textract pattern, fall back to "signature" keyword
+    if not signature_pages:
+        for page_num, page_text in text_dict.items():
+            if "signature" in page_text.lower():
+                signature_pages[page_num] = page_text
+
+    return signature_pages