Merged in bugfix/provider_info (pull request #710)

Bugfix/provider info to refactor/one-to-n

* Applied suggestion to fieldExtraction/src/prompts/investment_prompts.json
* update unit test

* Update Carveout IND and Default IND logic

* Split out exhibit-level lesser

* Bugfix

* update lesser of exhibit prompt

* Bugfix

* exhibit lesser of varible name bugfix

* Update exhibit lesser of passing

* Update Exhibit-level prompt template

* Shorten full_context prompt length if the questions are long

* Special case assignment index

* Resolve unit tests

* updated UC IND prompt

* Remove print

* update one-to-n unit test

* added dot based date example

* updated date fix prompt

* Merged in bugfix/json_error_and_unit_of_measure (pull request #708)

Bugfix/json error and unit of measure

* json parser fixes and prompt changes

* universal json load fixed

* json error fixed

* json error fixed 2

* Merged refactor/one-to-n into bugfix/json_error_and_unit_of_measure


Approved-by: Katon Minhas

* added new tin format

* updated signature page prompt

* updated tin search and tin duplication

* Merge remote-tracking branch 'origin/main' into bugfix/provider_info

* extended duplicate logic to npi and names

* Merged main into bugfix/provider_info


Approved-by: Katon Minhas

fieldExtraction/src/investment/tin_npi_funcs.py

@@ -231,6 +231,19 @@ def merge_provider_info(one_to_one_results: dict, provider_info: list[dict]) ->
     group_npis = [v for v in group_npis if v != "UNKNOWN"]
     group_names = [v for v in group_names if v != "UNKNOWN"]
 
+    # Deduplicate while preserving order and remove any "UNKNOWN" entries and overlaps between group and other
+    group_tins = list(dict.fromkeys("|".join(group_tins).split("|")))
+    other_tins = list(dict.fromkeys("|".join(other_tins).split("|")))
+    other_tins = [tin for tin in other_tins if tin not in group_tins and tin != "UNKNOWN"] # remove any group TINs from other TINs
+
+    group_npis = list(dict.fromkeys("|".join(group_npis).split("|")))
+    other_npis = list(dict.fromkeys("|".join(other_npis).split("|")))
+    other_npis = [npi for npi in other_npis if npi not in group_npis and npi != "UNKNOWN"] # remove any group NPIs from other NPIs
+
+    group_names = list(dict.fromkeys("|".join(group_names).split("|")))
+    other_names = list(dict.fromkeys("|".join(other_names).split("|")))
+    other_names = [name for name in other_names if name not in group_names and name != "UNKNOWN"] # remove any group Names from other Names
+
     # Merge group provider information into one_to_one_results
     one_to_one_results["PROV_GROUP_TIN"] = "|".join(group_tins) if group_tins else ""
     one_to_one_results["PROV_GROUP_NPI"] = "|".join(group_npis) if group_npis else ""
@@ -662,11 +675,8 @@ def run_provider_info_fields(
         relevant_pages = [
             page_num
             for page_num, page_text in text_dict.items()
-            if any(
-                tin in page_text or (tin[:2] + "-" + tin[2:]) in page_text
-                for tin in all_tins
-            )
-            or any(npi in page_text for npi in all_npis)
+            if any(tin.replace("-", "") in page_text.replace("-", "") for tin in all_tins)
+            or any(npi.replace("-", "") in page_text.replace("-", "") for npi in all_npis)
         ]
 
         # Stopgap for when the TIN is written in the header or footer of every page (or nearly every page)

fieldExtraction/src/prompts/investment_prompts.json

@@ -239,7 +239,7 @@
         "field_name": "TIN",
         "relationship": "one_to_one",
         "field_type": "provider_info",
-        "prompt": "What is the Tax Identification Number (TIN)? A TIN is typically a 9-digit number that may be formatted as XXX-XX-XXXX or XXXXXXXXX. Look for it near labels like 'Tax ID', 'EIN', 'Employer ID', or 'TIN'. It may also unlabeled, in the header or footer of the page."
+        "prompt": "What is the Tax Identification Number (TIN)? A TIN is typically a 9-digit number that may be formatted as XXX-XX-XXXX or XXXXXXXXX or XX-XXXXXXX or XX-XXXX-XXX. Look for it near labels like 'Tax ID', 'EIN', 'Employer ID', or 'TIN'. It may also unlabeled, in the header or footer of the page."
     },
     {
         "field_name": "NPI",
@@ -257,7 +257,7 @@
         "field_name": "ON_SIGNATURE_PAGE",
         "relationship": "one_to_one",
         "field_type": "provider_info",
-        "prompt": "Is this TIN, NPI, or Name found on the Signature page of the contract? Signature pages are those which contain a signature section. Write 'Y' or 'N'."
+        "prompt": "Is this TIN, NPI, or Name found on the same page as the signature section of the contract? A signature page contains signature blocks, signature lines, or signing elements AND may also contain other contract information like TIN, NPI, or provider names. Look for signature indicators such as: 'IN WITNESS WHEREOF', 'executed this agreement', 'undersigned', 'By:', signature lines, date fields for signing, or phrases indicating contract execution. If the text contains BOTH the TIN/NPI/Name information AND signature elements (even if they appear in different sections of the same page), respond 'Y'. Only respond 'N' if the TIN/NPI/Name appears on a completely separate page from any signature elements. Write 'Y' or 'N'."
     },
     {
         "field_name": "REIMB_PROV_TIN",

fieldExtraction/src/prompts/prompt_templates.py

@@ -665,6 +665,7 @@ Briefly explain your reasoning. After your explanation, include the heading "FIN
 [EXTRACTION AND FORMATTING INSTRUCTIONS]
 - First, analyze the text thoroughly and identify all provider entities
 - Return TIN and NPIs that are found on the page even if they aren't explicitly labeled as such.
+- Once TIN, NPI or NAME are identified, the check if the same provider is on the signature page. If so, set "ON_SIGNATURE_PAGE" to "Y", otherwise "N"
 - In your response, include ONLY ONE SINGLE properly formatted JSON array, after your explanation
 - Each provider must be an object within this array
 - All provider objects must have these exact keys: "TIN", "NPI", "NAME", "ON_SIGNATURE_PAGE"