diff --git a/src/pipelines/saas/test.py b/src/pipelines/saas/test.py index 70f7a50..3b7f57e 100644 --- a/src/pipelines/saas/test.py +++ b/src/pipelines/saas/test.py @@ -1,14 +1,12 @@ import src.pipelines.shared.extraction.tin_npi_funcs as tin_npi_funcs -prov_info_json = [{'TIN': '971098746', 'NPI': "N/A", 'NAME': 'Midwest Medical Equipment Solutions, Inc.'}] -print("Before Clean: ", prov_info_json) +one_to_one_results = {'PROV_INFO_JSON': [{'TIN': '971098746', 'NPI': 'UNKNOWN', 'NAME': 'Midwest Medical Equipment Solutions, Inc'}], 'PROV_INFO_JSON_FORMATTED': '{"TIN": "971098746", "NPI": "UNKNOWN", "NAME": "Midwest Medical Equipment Solutions, Inc"}', 'TIN_EXTRACTION_QUALITY': '1 exact, 0 OCR-corrected', 'NPI_EXTRACTION_QUALITY': '0 exact, 0 OCR-corrected', 'FILENAME_TIN': 'N/A', 'PAYER_STATE': ['UNKNOWN'], 'TERMINATION_DT': 'N/A', 'AUTO_RENEWAL_TERM': 'N/A', 'CONTRACT_AMENDMENT_NUM': 'UNKNOWN', 'CONTRACT_TITLE': 'MEDICARE PRODUCT ATTACHMENT COMPENSATION SCHEDULE EXHIBIT ANCILLARY SERVICES DURABLE MEDICAL EQUIPMENT', 'PAYER_NAME': 'IlliniCare Health', 'PROVIDER_NAME': 'Midwest Medical Equipment Solutions, Inc.', 'NUM_AVAILABLE_SIGNATORY_LINE_COUNT': '0', 'PROVIDER_STATE': ['IL'], 'AUTO_RENEWAL_IND': 'N', 'EFFECTIVE_DT': 'January 1, 2018', 'AARETE_DERIVED_EFFECTIVE_DT': '2018/01/01'} + +one_to_one_results = tin_npi_funcs.merge_provider_info_with_hybrid_smart_chunking(one_to_one_results, filename="") + +print("\nFinal: ", one_to_one_results) + -# Clean and standardize results -prov_info_json_clean = tin_npi_funcs.clean_provider_info(prov_info_json) -print("After Clean: ", prov_info_json_clean) -# Deduplicate providers based on TIN and NPI -prov_info_json_clean = tin_npi_funcs.deduplicate_providers(prov_info_json_clean) -print("After dedup: ", prov_info_json_clean) \ No newline at end of file diff --git a/src/pipelines/shared/extraction/tin_npi_funcs.py b/src/pipelines/shared/extraction/tin_npi_funcs.py index 0dba81f..cce1f2e 100644 --- a/src/pipelines/shared/extraction/tin_npi_funcs.py +++ b/src/pipelines/shared/extraction/tin_npi_funcs.py @@ -282,22 +282,24 @@ def merge_provider_info_with_hybrid_smart_chunking(one_to_one_results: dict, fil elif not isinstance(provider_name, list): provider_name = [] - provider_info = one_to_one_results[ + prov_info_json = one_to_one_results[ "PROV_INFO_JSON" - ] # Returns string representation of list + ] # Returns list prov_info_json_list = ( - json.loads(provider_info) if isinstance(provider_info, str) else provider_info + json.loads(prov_info_json) if isinstance(prov_info_json, str) else prov_info_json ) # Track if we found any name matches found_match = False for provider_dict in prov_info_json_list: + print("Provider dict: ", provider_dict) tins, npis, names = normalize_to_lists(provider_dict) name_matches = prompt_calls.provider_name_match_check( names, provider_name, filename ) # Returns Boolean + print("Name matches: ", name_matches) if name_matches: found_match = True @@ -336,10 +338,13 @@ def merge_provider_info_with_hybrid_smart_chunking(one_to_one_results: dict, fil for p in prov_info_json_list if "NO_IDENTIFIERS_FOUND" not in (p.get("NAME", [])) ] + print("Before dedup: ", prov_info_json_list) # Deduplicate provider_list by NAME, keeping records with actual TIN/NPI values prov_info_json_list = deduplicate_providers_by_name(prov_info_json_list) + print("After dedup: ", prov_info_json_list) + # Reconstruct PROV_INFO_JSON and PROV_INFO_JSON_FORMATTED with IS_GROUP flags one_to_one_results["PROV_INFO_JSON"] = prov_info_json_list one_to_one_results["PROV_INFO_JSON_FORMATTED"] = "\n".join( @@ -396,7 +401,7 @@ def reimbursement_tin_npi( answer_dict = {} # Extract all providers from the exhibit chunk - providers = get_provider_info( + providers = prompt_calls.prompt_provider_info( text_dict=exhibit_dict, page_num=exhibit_page, filename=filename ) @@ -452,9 +457,9 @@ def deduplicate_providers( names = provider.get("NAME") # # Normalize missing values - # tins = tins if isinstance(tins, list) else [] - # npis = npis if isinstance(npis, list) else [] - # names = names if isinstance(names, list) else [] + tins = tins if isinstance(tins, list) else [] + npis = npis if isinstance(npis, list) else [] + names = names if isinstance(names, list) else [] # Skip providers where all critical identification fields are unknown if ( (not tins or tins == "UNKNOWN") @@ -561,7 +566,7 @@ def get_prov_info_json(relevant_pages, text_dict, filename, payer_name): # Process each page separately and combine results prov_info_json = [] for page_num in relevant_pages: - page_provider_info = get_provider_info( + page_provider_info = prompt_calls.prompt_provider_info( text_dict, page_num, filename, payer_name ) prov_info_json.extend(page_provider_info) @@ -640,13 +645,11 @@ def run_provider_info_fields( """ all_tins, all_npis, tin_matches, npi_matches = get_all_tins_and_npis(contract_text) - print("All Tins: ", all_tins) exact_tins, exact_npis, ocr_corrected_tins, ocr_corrected_npis = get_tin_extraction_quality(tin_matches, npi_matches, filename) relevant_pages = get_relevant_pages(all_tins, all_npis, text_dict, filename) - print("Relevant Pages: ", relevant_pages) - + prov_info_json_clean = get_prov_info_json(relevant_pages, text_dict, filename, payer_name) # Create columns @@ -693,10 +696,6 @@ def deduplicate_providers_by_name(provider_list: list[dict]) -> list[dict]: for provider in sorted_providers: names = provider.get("NAME", []) - # Skip invalid NAME values - if not isinstance(names, list) or not names or names == ["UNKNOWN"]: - continue - # Check if any name was already seen is_duplicate = False for name in names: