From 2482adb465627f1d5d4fd6592d3d056ea806995e Mon Sep 17 00:00:00 2001 From: VenkataKrishna Reddy Avula Date: Wed, 26 Nov 2025 17:21:44 +0000 Subject: [PATCH] Merged in optimization/tin_npi (pull request #790) Optimization/tin npi * updated signature page and added for tin npi * updated signature page * removed print statements * removed print statement * updated test for extract signature page * updated data type Approved-by: Katon Minhas --- .../src/investment/tin_npi_funcs.py | 39 +++++++++++-------- fieldExtraction/src/utils/string_utils.py | 32 ++++++++------- fieldExtraction/tests/test_string_utils.py | 5 ++- 3 files changed, 44 insertions(+), 32 deletions(-) diff --git a/fieldExtraction/src/investment/tin_npi_funcs.py b/fieldExtraction/src/investment/tin_npi_funcs.py index 2cf74d7..1050fa0 100644 --- a/fieldExtraction/src/investment/tin_npi_funcs.py +++ b/fieldExtraction/src/investment/tin_npi_funcs.py @@ -668,14 +668,10 @@ def run_provider_info_fields( # If there are no identifiers, return early with default values if not all_tins and not all_npis: - deduplicated_provider_info = [ - { - "TIN": "UNKNOWN", - "NPI": "UNKNOWN", - "NAME": "NO_IDENTIFIERS_FOUND", - "IS_GROUP": "N", - } - ] + # if there are no TINs or NPIs, we need to extract signature pages + relevant_pages = list(string_utils.extract_signature_page( + text_dict, filename + ).keys()) else: # Identify relevant pages containing either NPIs or TINs (with or without hyphen formatting, e.g., 68-0640053) relevant_pages = [ @@ -699,16 +695,25 @@ def run_provider_info_fields( ): relevant_pages = [relevant_pages[0]] - # Process each page separately and combine results - combined_provider_info = [] - for page_num in relevant_pages: - page_provider_info = get_provider_info(text_dict, page_num, filename) - combined_provider_info.extend(page_provider_info) + if not relevant_pages: + deduplicated_provider_info = [ + { + "TIN": "UNKNOWN", + "NPI": "UNKNOWN", + "NAME": "NO_IDENTIFIERS_FOUND", + "IS_GROUP": "N", + } + ] + # Process each page separately and combine results + combined_provider_info = [] + for page_num in relevant_pages: + page_provider_info = get_provider_info(text_dict, page_num, filename) + combined_provider_info.extend(page_provider_info) - # Clean and standardize results - cleaned_provider_info = clean_provider_info(combined_provider_info) - # Deduplicate providers based on TIN and NPI - deduplicated_provider_info = deduplicate_providers(cleaned_provider_info) + # Clean and standardize results + cleaned_provider_info = clean_provider_info(combined_provider_info) + # Deduplicate providers based on TIN and NPI + deduplicated_provider_info = deduplicate_providers(cleaned_provider_info) # Create columns one_to_one_results = {} diff --git a/fieldExtraction/src/utils/string_utils.py b/fieldExtraction/src/utils/string_utils.py index 302767b..755ecd8 100644 --- a/fieldExtraction/src/utils/string_utils.py +++ b/fieldExtraction/src/utils/string_utils.py @@ -379,6 +379,12 @@ def extract_signature_page(text_dict: dict, filename: str) -> dict: re.IGNORECASE, ) + keywords = [ + "signature and information", + "signature authorization", + "in witness whereof", + ] + # First pass: Try to match the full textract pattern for page_num, page_text in text_dict.items(): if textract_pattern.search(page_text): @@ -386,22 +392,20 @@ def extract_signature_page(text_dict: dict, filename: str) -> dict: # If no pages found with textract pattern, try with keyword if not signature_pages: - page_items = list(text_dict.items()) - for i, (page_num, page_text) in enumerate(page_items): + # Iterate through each page in the text dictionary and check for keywords + for page_num, page_text in text_dict.items(): lower_text = page_text.lower() - if ( - "signature page follow" in lower_text - or "signature authorization" in lower_text - ): - # curr_page = int(float(page_num)) - if page_num not in signature_pages: - signature_pages[page_num] = page_text # current page - if i + 1 < len(page_items): - next_page_num, next_page_text = page_items[i + 1] - # next_page = int(float(next_page_num)) - if next_page_num not in signature_pages: - signature_pages[next_page_num] = next_page_text # next page + if any(keyword in lower_text for keyword in keywords): + signature_pages[page_num] = page_text + # If no signature pages found, try checking for the word "signature page follows" and return the next page only if it exists + if not signature_pages: + for page_num, page_text in text_dict.items(): + if "signature page follows" in page_text.lower(): + next_page_num = str(int(page_num) + 1) + if next_page_num in text_dict: + signature_pages[next_page_num] = text_dict[next_page_num] + return signature_pages diff --git a/fieldExtraction/tests/test_string_utils.py b/fieldExtraction/tests/test_string_utils.py index a37ca19..11eb5a3 100644 --- a/fieldExtraction/tests/test_string_utils.py +++ b/fieldExtraction/tests/test_string_utils.py @@ -152,9 +152,12 @@ class TestStringUtils(unittest.TestCase): result = string_utils.extract_signature_page(text_dict, "test.pdf") self.assertIn("2", result) self.assertEqual(result["2"], "This page has 2 signatures at the bottom") - text_dict = {"1": "Contract page 1", "2": "Signature page follows this section", "3": "Final signature authorization page"} + text_dict = {"1": "Contract page 1", "2": "In Witness Whereof", "3": "Final page"} result = string_utils.extract_signature_page(text_dict, "test.pdf") self.assertIn("2", result) + self.assertEqual(result["2"], "In Witness Whereof") + text_dict = {"1": "Contract page 1", "2": "Signature page follows this section", "3": "signature page"} + result = string_utils.extract_signature_page(text_dict, "test.pdf") self.assertIn("3", result) def test_extract_effective_date_pages(self):