diff --git a/fieldExtraction/src/investment/file_processing.py b/fieldExtraction/src/investment/file_processing.py index aa7aa63..846bf30 100644 --- a/fieldExtraction/src/investment/file_processing.py +++ b/fieldExtraction/src/investment/file_processing.py @@ -121,18 +121,17 @@ def run_one_to_one_prompts( relationship="one_to_one", file_path=config.FIELD_JSON_PATH ).combine(dynamic_one_to_one_fields) - - ################## RUN PROVIDER INFO ################## - one_to_one_results, one_to_one_fields = tin_npi_funcs.run_provider_info_fields( - contract_text, one_to_one_fields, text_dict, filename - ) - ################## RUN HYBRID SMART CHUNKED PROMPTS ################## # RAG function loads retrieval questions internally and matches with investment_prompts.json hybrid_smart_chunked_answers_dict = hybrid_smart_chunking_funcs.run_hybrid_smart_chunked_fields( one_to_one_fields, constants, contract_text, filename, text_dict ) + ################## RUN PROVIDER INFO ################## + payer_name = hybrid_smart_chunked_answers_dict.get('PAYER_NAME') + one_to_one_results = tin_npi_funcs.run_provider_info_fields( + contract_text, text_dict, filename, payer_name + ) ################## RUN FULL CONTEXT PROMPTS ################## full_context_answers_dict = one_to_one_funcs.run_full_context_fields( diff --git a/fieldExtraction/src/investment/tin_npi_funcs.py b/fieldExtraction/src/investment/tin_npi_funcs.py index 6cd1502..10c4bab 100644 --- a/fieldExtraction/src/investment/tin_npi_funcs.py +++ b/fieldExtraction/src/investment/tin_npi_funcs.py @@ -273,6 +273,9 @@ def merge_provider_info_with_hybrid_smart_chunking(one_to_one_results, filename) name for name in other_names if name not in group_names and name != "UNKNOWN" ] # remove any group Names from other Names + # Filter out records with NO_IDENTIFIERS_FOUND + provider_list = [p for p in provider_list if p.get('NAME') != 'NO_IDENTIFIERS_FOUND'] + # Reconstruct PROV_INFO_JSON and PROV_INFO_JSON_FORMATTED with IS_GROUP flags one_to_one_results["PROV_INFO_JSON"] = json.dumps(provider_list) one_to_one_results["PROV_INFO_JSON_FORMATTED"] = "\n".join( @@ -412,7 +415,8 @@ def deduplicate_providers( def get_provider_info( text_dict: dict, page_num: str, - filename: str + filename: str, + payer_name:str ) -> list: """ Extracts provider information from a specified page in a document using a language model. @@ -436,7 +440,7 @@ def get_provider_info( file_path=config.FIELD_JSON_PATH, field_type="provider_info" ) instruction, prompt = prompt_templates.TIN_NPI_TEMPLATE( - chunk, provider_fields.print_prompt_dict() + chunk, provider_fields.print_prompt_dict(), payer_name ) claude_answer_raw = llm_utils.invoke_claude( prompt, "sonnet_latest", filename, max_tokens=8192, cache=True, instruction=instruction @@ -534,10 +538,9 @@ def get_provider_info( def run_provider_info_fields( contract_text: str, - one_to_one_fields: FieldSet, text_dict: dict, filename: str, - # payer_name: str # This is not being used right now, but it might be useful for filtering providers later + payer_name: str ): """ Extracts provider information from a contract text using regex-based identifier extraction, @@ -632,7 +635,7 @@ def run_provider_info_fields( # Process each page separately and combine results combined_provider_info = [] for page_num in relevant_pages: - page_provider_info = get_provider_info(text_dict, page_num, filename) + page_provider_info = get_provider_info(text_dict, page_num, filename, payer_name) combined_provider_info.extend(page_provider_info) # Clean and standardize results @@ -657,4 +660,4 @@ def run_provider_info_fields( filename_tin = get_all_matches(filename, regex_patterns.FILE_NAME_TIN_PATTERN) one_to_one_results["FILENAME_TIN"] = filename_tin[0] if filename_tin else "N/A" - return one_to_one_results, one_to_one_fields + return one_to_one_results diff --git a/fieldExtraction/src/prompts/prompt_templates.py b/fieldExtraction/src/prompts/prompt_templates.py index 6becae5..b3fd069 100644 --- a/fieldExtraction/src/prompts/prompt_templates.py +++ b/fieldExtraction/src/prompts/prompt_templates.py @@ -1,6 +1,7 @@ import src.config as config from src.prompts.fieldset import FieldSet +from src.utils import string_utils PIPE_FORMAT_INSTRUCTIONS = "Briefly explain your answer, then enclose your final answer in |pipes|. If the requested information doesn't apply to this context, return |N/A|. If the information should exist but cannot be clearly identified, return |UNKNOWN|." @@ -862,13 +863,17 @@ def ONE_TO_ONE_SINGLE_FIELD_INSTRUCTION() -> str: ) -def TIN_NPI_TEMPLATE(context, questions): +def TIN_NPI_TEMPLATE(context, questions, payer_name): + payer_instruction = "" + if not string_utils.is_empty(payer_name): + payer_instruction = f'\n\nIMPORTANT: "{payer_name}" is the PAYER (insurance company) in this contract. Do NOT extract TIN or NPI information for "{payer_name}" - only extract information for healthcare providers who deliver services.' + instruction = f""" Analyze the following healthcare payer-provider contract text and extract provider entities - these are healthcare providers who deliver services, NOT payers/insurance companies who reimburse for services. IMPORTANT DISTINCTION: - Providers: Physicians, physician groups, hospitals, clinics who DELIVER healthcare -- Payers: Insurance companies, health plans who PAY for healthcare services +- Payers: Insurance companies, health plans who PAY for healthcare services{payer_instruction} CRITICAL: If a provider has multiple TINs or NPIs, include ALL of them separated by pipes (|) in the same provider record. Do NOT create separate records for the same provider entity. diff --git a/fieldExtraction/tests/test_tin_npi_funcs.py b/fieldExtraction/tests/test_tin_npi_funcs.py index 12555e8..0717898 100644 --- a/fieldExtraction/tests/test_tin_npi_funcs.py +++ b/fieldExtraction/tests/test_tin_npi_funcs.py @@ -118,7 +118,7 @@ class TestTinNpiFuncs: ] ) - result = tin_npi_funcs.get_provider_info(sample_text_dict, "1", "test.pdf") + result = tin_npi_funcs.get_provider_info(sample_text_dict, "1", "test.pdf", payer_name="Test Payer") assert len(result) == 1 @patch("src.utils.llm_utils.invoke_claude") @@ -134,11 +134,10 @@ class TestTinNpiFuncs: ] ) - one_to_one_fields = FieldSet(relationship="one_to_one") contract_text = "Contract with TIN 12-3456789" - results, fields = tin_npi_funcs.run_provider_info_fields( - contract_text, one_to_one_fields, sample_text_dict, "test.pdf" + results = tin_npi_funcs.run_provider_info_fields( + contract_text, sample_text_dict, "test.pdf", "Test Payer" ) assert "PROV_INFO_JSON" in results @@ -274,12 +273,11 @@ class TestTinNpiFuncs: ] ) - one_to_one_fields = FieldSet(relationship="one_to_one") # Include both exact and OCR-correctable TINs contract_text = "Contract with TIN 12-3456789 and corrupted TIN 98765432O" - results, fields = tin_npi_funcs.run_provider_info_fields( - contract_text, one_to_one_fields, sample_text_dict, "test.pdf" + results = tin_npi_funcs.run_provider_info_fields( + contract_text, sample_text_dict, "test.pdf", "Test Pyer" ) # Check that quality tracking fields are present @@ -333,12 +331,11 @@ class TestTinNpiFuncs: ] ) - one_to_one_fields = FieldSet(relationship="one_to_one") # Only OCR-corrupted identifiers, no clean ones contract_text = "Contract with corrupted TIN 12345678O and NPI 123456789O" - results, fields = tin_npi_funcs.run_provider_info_fields( - contract_text, one_to_one_fields, sample_text_dict, "test.pdf" + results = tin_npi_funcs.run_provider_info_fields( + contract_text, sample_text_dict, "test.pdf", "Test Payer" ) # Should find OCR-corrected values