update tin_npi_funcs dedups
This commit is contained in:
Katon Minhas
@@ -1,14 +1,12 @@
|
||||
|
||||
import src.pipelines.shared.extraction.tin_npi_funcs as tin_npi_funcs
|
||||
|
||||
prov_info_json = [{'TIN': '971098746', 'NPI': "N/A", 'NAME': 'Midwest Medical Equipment Solutions, Inc.'}]
|
||||
|
||||
print("Before Clean: ", prov_info_json)
|
||||
one_to_one_results = {'PROV_INFO_JSON': [{'TIN': '971098746', 'NPI': 'UNKNOWN', 'NAME': 'Midwest Medical Equipment Solutions, Inc'}], 'PROV_INFO_JSON_FORMATTED': '{"TIN": "971098746", "NPI": "UNKNOWN", "NAME": "Midwest Medical Equipment Solutions, Inc"}', 'TIN_EXTRACTION_QUALITY': '1 exact, 0 OCR-corrected', 'NPI_EXTRACTION_QUALITY': '0 exact, 0 OCR-corrected', 'FILENAME_TIN': 'N/A', 'PAYER_STATE': ['UNKNOWN'], 'TERMINATION_DT': 'N/A', 'AUTO_RENEWAL_TERM': 'N/A', 'CONTRACT_AMENDMENT_NUM': 'UNKNOWN', 'CONTRACT_TITLE': 'MEDICARE PRODUCT ATTACHMENT COMPENSATION SCHEDULE EXHIBIT ANCILLARY SERVICES DURABLE MEDICAL EQUIPMENT', 'PAYER_NAME': 'IlliniCare Health', 'PROVIDER_NAME': 'Midwest Medical Equipment Solutions, Inc.', 'NUM_AVAILABLE_SIGNATORY_LINE_COUNT': '0', 'PROVIDER_STATE': ['IL'], 'AUTO_RENEWAL_IND': 'N', 'EFFECTIVE_DT': 'January 1, 2018', 'AARETE_DERIVED_EFFECTIVE_DT': '2018/01/01'}
|
||||
|
||||
one_to_one_results = tin_npi_funcs.merge_provider_info_with_hybrid_smart_chunking(one_to_one_results, filename="")
|
||||
|
||||
print("\nFinal: ", one_to_one_results)
|
||||
|
||||
|
||||
# Clean and standardize results
|
||||
prov_info_json_clean = tin_npi_funcs.clean_provider_info(prov_info_json)
|
||||
print("After Clean: ", prov_info_json_clean)
|
||||
|
||||
# Deduplicate providers based on TIN and NPI
|
||||
prov_info_json_clean = tin_npi_funcs.deduplicate_providers(prov_info_json_clean)
|
||||
print("After dedup: ", prov_info_json_clean)
|
||||
@@ -282,22 +282,24 @@ def merge_provider_info_with_hybrid_smart_chunking(one_to_one_results: dict, fil
|
||||
elif not isinstance(provider_name, list):
|
||||
provider_name = []
|
||||
|
||||
provider_info = one_to_one_results[
|
||||
prov_info_json = one_to_one_results[
|
||||
"PROV_INFO_JSON"
|
||||
] # Returns string representation of list
|
||||
] # Returns list
|
||||
prov_info_json_list = (
|
||||
json.loads(provider_info) if isinstance(provider_info, str) else provider_info
|
||||
json.loads(prov_info_json) if isinstance(prov_info_json, str) else prov_info_json
|
||||
)
|
||||
# Track if we found any name matches
|
||||
found_match = False
|
||||
|
||||
for provider_dict in prov_info_json_list:
|
||||
print("Provider dict: ", provider_dict)
|
||||
|
||||
tins, npis, names = normalize_to_lists(provider_dict)
|
||||
|
||||
name_matches = prompt_calls.provider_name_match_check(
|
||||
names, provider_name, filename
|
||||
) # Returns Boolean
|
||||
print("Name matches: ", name_matches)
|
||||
|
||||
if name_matches:
|
||||
found_match = True
|
||||
@@ -336,10 +338,13 @@ def merge_provider_info_with_hybrid_smart_chunking(one_to_one_results: dict, fil
|
||||
for p in prov_info_json_list
|
||||
if "NO_IDENTIFIERS_FOUND" not in (p.get("NAME", []))
|
||||
]
|
||||
print("Before dedup: ", prov_info_json_list)
|
||||
|
||||
# Deduplicate provider_list by NAME, keeping records with actual TIN/NPI values
|
||||
prov_info_json_list = deduplicate_providers_by_name(prov_info_json_list)
|
||||
|
||||
print("After dedup: ", prov_info_json_list)
|
||||
|
||||
# Reconstruct PROV_INFO_JSON and PROV_INFO_JSON_FORMATTED with IS_GROUP flags
|
||||
one_to_one_results["PROV_INFO_JSON"] = prov_info_json_list
|
||||
one_to_one_results["PROV_INFO_JSON_FORMATTED"] = "\n".join(
|
||||
@@ -396,7 +401,7 @@ def reimbursement_tin_npi(
|
||||
answer_dict = {}
|
||||
|
||||
# Extract all providers from the exhibit chunk
|
||||
providers = get_provider_info(
|
||||
providers = prompt_calls.prompt_provider_info(
|
||||
text_dict=exhibit_dict, page_num=exhibit_page, filename=filename
|
||||
)
|
||||
|
||||
@@ -452,9 +457,9 @@ def deduplicate_providers(
|
||||
names = provider.get("NAME")
|
||||
|
||||
# # Normalize missing values
|
||||
# tins = tins if isinstance(tins, list) else []
|
||||
# npis = npis if isinstance(npis, list) else []
|
||||
# names = names if isinstance(names, list) else []
|
||||
tins = tins if isinstance(tins, list) else []
|
||||
npis = npis if isinstance(npis, list) else []
|
||||
names = names if isinstance(names, list) else []
|
||||
# Skip providers where all critical identification fields are unknown
|
||||
if (
|
||||
(not tins or tins == "UNKNOWN")
|
||||
@@ -561,7 +566,7 @@ def get_prov_info_json(relevant_pages, text_dict, filename, payer_name):
|
||||
# Process each page separately and combine results
|
||||
prov_info_json = []
|
||||
for page_num in relevant_pages:
|
||||
page_provider_info = get_provider_info(
|
||||
page_provider_info = prompt_calls.prompt_provider_info(
|
||||
text_dict, page_num, filename, payer_name
|
||||
)
|
||||
prov_info_json.extend(page_provider_info)
|
||||
@@ -640,13 +645,11 @@ def run_provider_info_fields(
|
||||
"""
|
||||
|
||||
all_tins, all_npis, tin_matches, npi_matches = get_all_tins_and_npis(contract_text)
|
||||
print("All Tins: ", all_tins)
|
||||
|
||||
exact_tins, exact_npis, ocr_corrected_tins, ocr_corrected_npis = get_tin_extraction_quality(tin_matches, npi_matches, filename)
|
||||
|
||||
relevant_pages = get_relevant_pages(all_tins, all_npis, text_dict, filename)
|
||||
print("Relevant Pages: ", relevant_pages)
|
||||
|
||||
|
||||
prov_info_json_clean = get_prov_info_json(relevant_pages, text_dict, filename, payer_name)
|
||||
|
||||
# Create columns
|
||||
@@ -693,10 +696,6 @@ def deduplicate_providers_by_name(provider_list: list[dict]) -> list[dict]:
|
||||
for provider in sorted_providers:
|
||||
names = provider.get("NAME", [])
|
||||
|
||||
# Skip invalid NAME values
|
||||
if not isinstance(names, list) or not names or names == ["UNKNOWN"]:
|
||||
continue
|
||||
|
||||
# Check if any name was already seen
|
||||
is_duplicate = False
|
||||
for name in names:
|
||||
|
||||
Reference in New Issue
Block a user