From 814d3e0636f2fc0cd5c4639d3cecf0bb36620ba7 Mon Sep 17 00:00:00 2001 From: VenkataKrishna Reddy Avula Date: Thu, 25 Sep 2025 16:48:23 +0000 Subject: [PATCH] Merged in bugfix/provider_info (pull request #710) Bugfix/provider info to refactor/one-to-n * Applied suggestion to fieldExtraction/src/prompts/investment_prompts.json * update unit test * Update Carveout IND and Default IND logic * Split out exhibit-level lesser * Bugfix * update lesser of exhibit prompt * Bugfix * exhibit lesser of varible name bugfix * Update exhibit lesser of passing * Update Exhibit-level prompt template * Shorten full_context prompt length if the questions are long * Special case assignment index * Resolve unit tests * updated UC IND prompt * Remove print * update one-to-n unit test * added dot based date example * updated date fix prompt * Merged in bugfix/json_error_and_unit_of_measure (pull request #708) Bugfix/json error and unit of measure * json parser fixes and prompt changes * universal json load fixed * json error fixed * json error fixed 2 * Merged refactor/one-to-n into bugfix/json_error_and_unit_of_measure Approved-by: Katon Minhas * added new tin format * updated signature page prompt * updated tin search and tin duplication * Merge remote-tracking branch 'origin/main' into bugfix/provider_info * extended duplicate logic to npi and names * Merged main into bugfix/provider_info Approved-by: Katon Minhas --- fieldExtraction/constants/regex_patterns.py | 2 +- .../src/investment/tin_npi_funcs.py | 20 ++++++++++++++----- .../src/prompts/investment_prompts.json | 4 ++-- .../src/prompts/prompt_templates.py | 1 + 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/fieldExtraction/constants/regex_patterns.py b/fieldExtraction/constants/regex_patterns.py index a8f541c..b8633b9 100644 --- a/fieldExtraction/constants/regex_patterns.py +++ b/fieldExtraction/constants/regex_patterns.py @@ -4,7 +4,7 @@ BACKTICK_PATTERN = r"`([^`]+)`" TRIPLE_BACKTICK_PATTERN = r"```([^`]+)```" # TIN and NPI Regex Patterns -TIN_PATTERN = r"\b\d{9}\b|\b\d{2}[-.]\d{7}\b|\b\d{3}[-.]\d{2}[-.]\d{4}\b" +TIN_PATTERN = r"\b\d{9}\b|\b\d{2}[-.]\d{7}\b|\b\d{3}[-.]\d{2}[-.]\d{4}\b|\b\d{2}-\d{4}-\d{3}\b" NPI_PATTERN = r"\b\d{10}\b" # OCR-Correctable Patterns diff --git a/fieldExtraction/src/investment/tin_npi_funcs.py b/fieldExtraction/src/investment/tin_npi_funcs.py index 15aa72b..ec0a31a 100644 --- a/fieldExtraction/src/investment/tin_npi_funcs.py +++ b/fieldExtraction/src/investment/tin_npi_funcs.py @@ -231,6 +231,19 @@ def merge_provider_info(one_to_one_results: dict, provider_info: list[dict]) -> group_npis = [v for v in group_npis if v != "UNKNOWN"] group_names = [v for v in group_names if v != "UNKNOWN"] + # Deduplicate while preserving order and remove any "UNKNOWN" entries and overlaps between group and other + group_tins = list(dict.fromkeys("|".join(group_tins).split("|"))) + other_tins = list(dict.fromkeys("|".join(other_tins).split("|"))) + other_tins = [tin for tin in other_tins if tin not in group_tins and tin != "UNKNOWN"] # remove any group TINs from other TINs + + group_npis = list(dict.fromkeys("|".join(group_npis).split("|"))) + other_npis = list(dict.fromkeys("|".join(other_npis).split("|"))) + other_npis = [npi for npi in other_npis if npi not in group_npis and npi != "UNKNOWN"] # remove any group NPIs from other NPIs + + group_names = list(dict.fromkeys("|".join(group_names).split("|"))) + other_names = list(dict.fromkeys("|".join(other_names).split("|"))) + other_names = [name for name in other_names if name not in group_names and name != "UNKNOWN"] # remove any group Names from other Names + # Merge group provider information into one_to_one_results one_to_one_results["PROV_GROUP_TIN"] = "|".join(group_tins) if group_tins else "" one_to_one_results["PROV_GROUP_NPI"] = "|".join(group_npis) if group_npis else "" @@ -662,11 +675,8 @@ def run_provider_info_fields( relevant_pages = [ page_num for page_num, page_text in text_dict.items() - if any( - tin in page_text or (tin[:2] + "-" + tin[2:]) in page_text - for tin in all_tins - ) - or any(npi in page_text for npi in all_npis) + if any(tin.replace("-", "") in page_text.replace("-", "") for tin in all_tins) + or any(npi.replace("-", "") in page_text.replace("-", "") for npi in all_npis) ] # Stopgap for when the TIN is written in the header or footer of every page (or nearly every page) diff --git a/fieldExtraction/src/prompts/investment_prompts.json b/fieldExtraction/src/prompts/investment_prompts.json index dd713f2..930e5e3 100644 --- a/fieldExtraction/src/prompts/investment_prompts.json +++ b/fieldExtraction/src/prompts/investment_prompts.json @@ -239,7 +239,7 @@ "field_name": "TIN", "relationship": "one_to_one", "field_type": "provider_info", - "prompt": "What is the Tax Identification Number (TIN)? A TIN is typically a 9-digit number that may be formatted as XXX-XX-XXXX or XXXXXXXXX. Look for it near labels like 'Tax ID', 'EIN', 'Employer ID', or 'TIN'. It may also unlabeled, in the header or footer of the page." + "prompt": "What is the Tax Identification Number (TIN)? A TIN is typically a 9-digit number that may be formatted as XXX-XX-XXXX or XXXXXXXXX or XX-XXXXXXX or XX-XXXX-XXX. Look for it near labels like 'Tax ID', 'EIN', 'Employer ID', or 'TIN'. It may also unlabeled, in the header or footer of the page." }, { "field_name": "NPI", @@ -257,7 +257,7 @@ "field_name": "ON_SIGNATURE_PAGE", "relationship": "one_to_one", "field_type": "provider_info", - "prompt": "Is this TIN, NPI, or Name found on the Signature page of the contract? Signature pages are those which contain a signature section. Write 'Y' or 'N'." + "prompt": "Is this TIN, NPI, or Name found on the same page as the signature section of the contract? A signature page contains signature blocks, signature lines, or signing elements AND may also contain other contract information like TIN, NPI, or provider names. Look for signature indicators such as: 'IN WITNESS WHEREOF', 'executed this agreement', 'undersigned', 'By:', signature lines, date fields for signing, or phrases indicating contract execution. If the text contains BOTH the TIN/NPI/Name information AND signature elements (even if they appear in different sections of the same page), respond 'Y'. Only respond 'N' if the TIN/NPI/Name appears on a completely separate page from any signature elements. Write 'Y' or 'N'." }, { "field_name": "REIMB_PROV_TIN", diff --git a/fieldExtraction/src/prompts/prompt_templates.py b/fieldExtraction/src/prompts/prompt_templates.py index 204e90b..59cc30a 100644 --- a/fieldExtraction/src/prompts/prompt_templates.py +++ b/fieldExtraction/src/prompts/prompt_templates.py @@ -665,6 +665,7 @@ Briefly explain your reasoning. After your explanation, include the heading "FIN [EXTRACTION AND FORMATTING INSTRUCTIONS] - First, analyze the text thoroughly and identify all provider entities - Return TIN and NPIs that are found on the page even if they aren't explicitly labeled as such. +- Once TIN, NPI or NAME are identified, the check if the same provider is on the signature page. If so, set "ON_SIGNATURE_PAGE" to "Y", otherwise "N" - In your response, include ONLY ONE SINGLE properly formatted JSON array, after your explanation - Each provider must be an object within this array - All provider objects must have these exact keys: "TIN", "NPI", "NAME", "ON_SIGNATURE_PAGE"