Merged in feature/tin-npi-prompt-formatting (pull request #490)

Collect TIN/NPI/Name page-by-page and run a second pass for IS_GROUP

* configure first pass: no providers marked IS_GROUP.  Allows simplification of prompt

* Add group provider identification logic in second pass

* group identification: don't take 20% of pages.  Take first 3 and last 3

* Refine group provider identification logic to use pipes for JSON formatting and improve error handling

* fix mypy error for `identify_group_provider`

* Merge remote-tracking branch 'origin/main' into feature/tin-npi-prompt-formatting

* Add "PROV_INFO_JSON" to investment column order

* Refactor group provider identification to use signature page extraction and fix JSON format

* move signature page extraction to `string_utils.py` and add unit tests for it

* Update identify_group_provider to return provider name along with TIN and NPI in JSON format

* Update identify_group_provider to always use group-provided name when available

* move out GROUP_TIN_NPI_TEMPLATE to `investment_prompts.py`

* Remove deprecated PROV_GROUP_TIN_CHECK and PROV_GROUP_NPI_CHECK prompts

* Move out provider info fields from `one_to_one_funcs` to `tin_npi_funcs`

* isort

* remove debug print statements

* Remove debug print statements from provider info functions

* add note to regex_utils.py about where functions are used

* Add page_key_sort function and corresponding tests for sorting page keys

* Refactor identify_group_provider to sort pages and avoid duplicates in extracted sections

* Remove debug print statements from identify_group_provider function

* Merge remote-tracking branch 'origin/main' into feature/tin-npi-prompt-formatting

* extract_signature_page function: prioritize Textract marker for signatures and fallback to keyword search

* fix mypy error

* Update test cases to reflect changes in signature extraction logic


Approved-by: Katon Minhas

fieldExtraction/tests/string_utils_test.py

@@ -1,6 +1,6 @@
 import pytest
 import src.utils as utils
-from src.utils.string_utils import extract_text_from_delimiters, json_parsing_search, secondary_string_to_dict, primary_string_to_dict, contains_reimbursement, is_empty, count_reimbursements_in_exhibit
+from src.utils.string_utils import extract_text_from_delimiters, json_parsing_search, secondary_string_to_dict, primary_string_to_dict, contains_reimbursement, is_empty, count_reimbursements_in_exhibit, extract_signature_page, page_key_sort
 import src.utils.llm_utils as llm_utils
 import numpy as np
 import pandas as pd
@@ -195,6 +195,69 @@ class TestStringUtils:
         result = is_empty(value)
         assert result == expected
 
+    @pytest.mark.parametrize("text_dict, filename, expected", [
+            (
+                {"1": "This page has 2 signatures.", "2": "None here.", "3": "Signature of the party (but ignored)"},
+                "test_file.txt",
+                {"1": "This page has 2 signatures."}
+            ),
+            (
+                {"1": "No relevant content here.", "2": "Nor over here."},
+                "test_file.txt",
+                {}
+            ),
+            (
+                {"1": "This page has a signature.", "2": "Another signature is found here."},
+                "test_file.txt",
+                {"1": "This page has a signature.", "2": "Another signature is found here."}
+            ),
+            (
+                {"1": "This page has 1 signature.", "2": "No relevant content.", "3": "Signature block here.  But ignored bc of the first textract-like page"},
+                "test_file.txt",
+                {"1": "This page has 1 signature."}
+            ),
+            (
+                {"1": "No signed names at all.", "2": "Just some random text."},
+                "test_file.txt",
+                {}
+            ),
+            (
+                {"1": "This page has a signature.", "2": "Signature is mentioned here too."},
+                "test_file.txt",
+                {"1": "This page has a signature.", "2": "Signature is mentioned here too."}
+            ),
+            (
+                {},
+                "test_file.txt",
+                {}
+            ),
+        ])
+    def test_extract_signature_page(self, text_dict, filename, expected):
+        result = extract_signature_page(text_dict, filename)
+        assert result == expected
+
+    @pytest.mark.parametrize("page_key, expected", [
+    ("1", (0, 1)),  # Numeric key
+    ("10", (0, 10)),  # Larger numeric key
+    ("A", (1, "A")),  # Alphabetic key
+    ("Z", (1, "Z")),  # Another alphabetic key
+    ("1A", (1, "1A")),  # Alphanumeric key
+    ("", (1, "")),  # Empty string
+    ("001", (0, 1)),  # Numeric key with leading zeros
+    ("B2", (1, "B2")),  # Alphanumeric key with letter first
+    ("-5", (0, -5)),  # Negative numeric
+    (" ", (1, " ")),  # Space character
+    ])
+    def test_page_key_sort(self, page_key, expected):
+        result = page_key_sort(page_key)
+        assert result == expected
+
+    def test_page_key_sort_with_sorting(self):
+        keys = ["10", "IV", "2", "A", "1", "B", "Z"]
+        expected_sorted_keys = ["1", "2", "10", "A", "B", "IV", "Z"]
+        sorted_keys = sorted(keys, key=page_key_sort)
+        assert sorted_keys == expected_sorted_keys
+
 class TestCountReimbursementsInExhibit:
     @pytest.mark.parametrize("exhibit_text, expected_count", [
         ("This exhibit includes a 10% reimbursement and a $100 reimbursement.", 2),