From 66f429d0dbe3f465f5fa6bcff52a8072fe990c31 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Fri, 31 Jan 2025 22:40:17 +0000 Subject: [PATCH] Merged in field/tin-npi-initialization (pull request #374) Initialize 1:1 TIN and NPI Fields (Using CNC methodology) * 1:1 TIN and NPI fields added (using CNC methodology) * Merge branch 'main' into field/tin-npi-initialization * Merged main into field/tin-npi-initialization * Merged main into field/tin-npi-initialization * Merged main into field/tin-npi-initialization Approved-by: Alex Galarce --- fieldExtraction/src/client/file_processing.py | 6 +++--- fieldExtraction/src/investment/file_processing.py | 3 +-- fieldExtraction/src/investment/one_to_one_funcs.py | 14 ++++++++++++-- fieldExtraction/src/regex/regex_utils.py | 5 ++--- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/fieldExtraction/src/client/file_processing.py b/fieldExtraction/src/client/file_processing.py index 37c1222..2024535 100644 --- a/fieldExtraction/src/client/file_processing.py +++ b/fieldExtraction/src/client/file_processing.py @@ -9,7 +9,7 @@ import src.client.postprocess as postprocess import src.client.preprocess as preprocess import src.client.smart_chunking_funcs as smart_chunking_funcs from src import config -from src.regex.regex_utils import irs_hotfix, npi_hotfix +from src.regex.regex_utils import get_tin_fields, get_npi_fields def run_ac_prompts(file_object): @@ -33,10 +33,10 @@ def run_ac_prompts(file_object): ################## RUN REGEX FUNCTIONALITY ################## # no prompting for these fields - irs_answers = irs_hotfix(filename, text_dict, top_sheet_dict) + irs_answers = get_tin_fields(filename, text_dict, top_sheet_dict) ac_answers_dict.update(irs_answers) - npi_answers = npi_hotfix(text_dict, top_sheet_dict) + npi_answers = get_npi_fields(text_dict, top_sheet_dict) ac_answers_dict.update(npi_answers) ################## RUN SMART CHUNKED PROMPTS ################## diff --git a/fieldExtraction/src/investment/file_processing.py b/fieldExtraction/src/investment/file_processing.py index 6a4c55d..1bc7702 100644 --- a/fieldExtraction/src/investment/file_processing.py +++ b/fieldExtraction/src/investment/file_processing.py @@ -11,7 +11,6 @@ import src.investment.aarete_derived as aarete_derived import src.investment.smart_chunking_funcs as smart_chunking_funcs from src.prompts.investment_prompts import Field, FieldSet from src import config -from src.regex.regex_utils import irs_hotfix, npi_hotfix import src.utils.string_utils as string_utils import src.utils.io_utils as io_utils from src.config import FIELD_JSON_PATH @@ -65,7 +64,7 @@ def run_one_to_one_prompts(filename, contract_text, text_dict, top_sheet_dict): one_to_one_fields = FieldSet(relationship="one_to_one", file_path=config.FIELD_JSON_PATH) ################## RUN REGEX ################## - regex_answers_dict = one_to_one_funcs.run_regex_fields(one_to_one_fields, contract_text, filename) + regex_answers_dict = one_to_one_funcs.run_regex_fields(one_to_one_fields, text_dict, top_sheet_dict, filename) ################## RUN SMART CHUNKED PROMPTS ################## smart_chunked_answers_dict = one_to_one_funcs.run_smart_chunked_fields( diff --git a/fieldExtraction/src/investment/one_to_one_funcs.py b/fieldExtraction/src/investment/one_to_one_funcs.py index 75e6e39..426778c 100644 --- a/fieldExtraction/src/investment/one_to_one_funcs.py +++ b/fieldExtraction/src/investment/one_to_one_funcs.py @@ -6,15 +6,25 @@ import src.utils.llm_utils as llm_utils import src.utils.string_utils as string_utils from src.enums.delimiters import Delimiter from src import config, keywords, postprocessing_funcs +from src.regex.regex_utils import get_tin_fields, get_npi_fields MAX_CONTEXT_LENGTH = 100000 def run_regex_fields(one_to_one_fields: FieldSet, - contract_text: str, + text_dict: str, + top_sheet_dict: str, filename: str): # Fill NPI/TIN regex here regex_fields = one_to_one_fields.filter(field_type="regex") - return {field.field_name : "Placeholder" for field in regex_fields.fields} + + # One to One TIN Fields + tin_answers = get_tin_fields(filename, text_dict, top_sheet_dict) + tin_answers = {k:v for k, v in tin_answers.items() if k in ['PROV_GROUP_TIN', 'PROV_TIN_OTHER']} + + # One to One NPI Fields + npi_answers = get_npi_fields(text_dict, top_sheet_dict) + + return {**tin_answers, **npi_answers} def run_smart_chunked_fields(one_to_one_fields: FieldSet, contract_text: str, diff --git a/fieldExtraction/src/regex/regex_utils.py b/fieldExtraction/src/regex/regex_utils.py index cdbada8..e64b1d1 100644 --- a/fieldExtraction/src/regex/regex_utils.py +++ b/fieldExtraction/src/regex/regex_utils.py @@ -107,7 +107,7 @@ def npi_pattern_match(text_dict: dict[str, str]) -> dict[str, str]: return NPI_dict -def npi_hotfix( +def get_npi_fields( text_dict: dict[str, str], top_sheet_dict: dict[str, str] ) -> dict[str, str]: """Checks for NPI 10-digit matches in the text_dict first. If N/A checks in TOP SHEET. @@ -267,7 +267,7 @@ def irs_pattern_match(filename: str, text_dict: dict[str, str]): return TIN_dict -def irs_hotfix( +def get_tin_fields( filename: str, text_dict: dict[str, str], top_sheet_dict: dict[str, str] ) -> dict[str, str]: """Checks for IRS pattern matches in the text_dict first. If N/A checks in TOP SHEET. @@ -289,4 +289,3 @@ def irs_hotfix( ): irs_answers = irs_pattern_match(filename, top_sheet_dict) return irs_answers -