diff --git a/src/pipelines/saas/file_processing.py b/src/pipelines/saas/file_processing.py index 2ed83aa..03c8cba 100644 --- a/src/pipelines/saas/file_processing.py +++ b/src/pipelines/saas/file_processing.py @@ -258,6 +258,7 @@ def run_one_to_one_prompts( ################## Fill NA Mapping ################## one_to_one_results = aarete_derived.fill_na_mapping(one_to_one_results) + print("After all one_to_one: ", one_to_one_results) return one_to_one_results[0] diff --git a/src/pipelines/shared/extraction/tin_npi_funcs.py b/src/pipelines/shared/extraction/tin_npi_funcs.py index 4d07a31..4e704b4 100644 --- a/src/pipelines/shared/extraction/tin_npi_funcs.py +++ b/src/pipelines/shared/extraction/tin_npi_funcs.py @@ -157,7 +157,7 @@ def chunk_on_matches(matches: list[str], text_dict: dict) -> str: return "\n".join(valid_pages) -def clean_provider_info(provider_info: list[dict]) -> list[dict]: +def clean_provider_info(prov_info_json: list[dict]) -> list[dict]: """Cleans provider information by standardizing Name/TIN/NPI formats. Args: @@ -167,8 +167,8 @@ def clean_provider_info(provider_info: list[dict]) -> list[dict]: Returns: list: Cleaned provider information """ - cleaned_providers = [] - for provider in provider_info: + prov_info_json_clean = [] + for provider in prov_info_json: cleaned_provider = {} # ---- Clean TIN (list, digits only, 9 chars) ---- @@ -180,7 +180,7 @@ def clean_provider_info(provider_info: list[dict]) -> list[dict]: tin_str = str(tin).replace("-", "").replace(".", "") if tin_str.isdigit() and len(tin_str) == 9: cleaned_tins.append(tin_str) - cleaned_provider["TIN"] = cleaned_tins if cleaned_tins else ["UNKNOWN"] + cleaned_provider["TIN"] = cleaned_tins if cleaned_tins else "UNKNOWN" # ---- Clean NPI (list, digits only, 10 chars) ---- cleaned_npis = [] @@ -191,7 +191,7 @@ def clean_provider_info(provider_info: list[dict]) -> list[dict]: npi_str = str(npi).replace("-", "").replace(".", "") if npi_str.isdigit() and len(npi_str) == 10: cleaned_npis.append(npi_str) - cleaned_provider["NPI"] = cleaned_npis if cleaned_npis else ["UNKNOWN"] + cleaned_provider["NPI"] = cleaned_npis if cleaned_npis else "UNKNOWN" # ---- Clean NAME (list) ---- cleaned_names = [] @@ -202,20 +202,66 @@ def clean_provider_info(provider_info: list[dict]) -> list[dict]: name_str = " ".join(str(name).split()).strip().rstrip(".") if name_str: cleaned_names.append(name_str) - cleaned_provider["NAME"] = cleaned_names if cleaned_names else ["UNKNOWN"] + cleaned_provider["NAME"] = cleaned_names if cleaned_names else "UNKNOWN" - cleaned_providers.append(cleaned_provider) - return cleaned_providers + prov_info_json_clean.append(cleaned_provider) + return prov_info_json_clean -def merge_provider_info_with_hybrid_smart_chunking(one_to_one_results, filename): +def normalize_to_lists(provider: dict): + + tins = provider.get("TIN") + npis = provider.get("NPI") + names = provider.get("NAME") + + # Normalize to lists: if string, convert to list to avoid character splitting + if isinstance(tins, str): + tins = [tins] + elif not isinstance(tins, list): + tins = [] + if isinstance(npis, str): + npis = [npis] + elif not isinstance(npis, list): + npis = [] + if isinstance(names, str): + names = [names] + elif not isinstance(names, list): + names = [] + + return tins, npis, names + +def deunknown_and_deduplicate(group_tins, group_npis, group_names, other_tins, other_npis, other_names): + + # De-Unknown GROUP fields + group_tins = [v for v in group_tins if v != "UNKNOWN"] + group_npis = [v for v in group_npis if v != "UNKNOWN"] + group_names = [v for v in group_names if v != "UNKNOWN"] + + # Deduplicate while preserving order and remove any "UNKNOWN" entries and overlaps between group and other + group_tins = list(dict.fromkeys(group_tins)) + other_tins = list(dict.fromkeys(other_tins)) + other_tins = [v for v in other_tins if v not in group_tins and v != "UNKNOWN"] + + group_npis = list(dict.fromkeys(group_npis)) + other_npis = list(dict.fromkeys(other_npis)) + other_npis = [v for v in other_npis if v not in group_npis and v != "UNKNOWN"] + + group_names = list(dict.fromkeys(group_names)) + other_names = list(dict.fromkeys(other_names)) + other_names = [v for v in other_names if v not in group_names and v != "UNKNOWN"] + + return group_tins, group_npis, group_names, other_tins, other_npis, other_names + + + +def merge_provider_info_with_hybrid_smart_chunking(one_to_one_results: dict, filename: str): """ Merges provider_info into one_to_one_results. Args: one_to_one_results (dict): Dictionary of one-to-one field results. provider_info (list[dict]): List of dictionaries containing provider information. - Each dictionary is keyed by TIN, NPI, NAME, IS_GROUP. + Each dictionary is keyed by TIN, NPI, NAME Returns: dict: Updated one_to_one_results with merged values from provider_info. @@ -244,33 +290,17 @@ def merge_provider_info_with_hybrid_smart_chunking(one_to_one_results, filename) # Track if we found any name matches found_match = False - for provider in prov_info_json_list: - # Ensure list format (FORMAT FIX ONLY) - tins = provider.get("TIN") or [] - npis = provider.get("NPI") or [] - names = provider.get("NAME") or [] - - # Normalize to lists: if string, convert to list to avoid character splitting - if isinstance(tins, str): - tins = [tins] - elif not isinstance(tins, list): - tins = [] - if isinstance(npis, str): - npis = [npis] - elif not isinstance(npis, list): - npis = [] - if isinstance(names, str): - names = [names] - elif not isinstance(names, list): - names = [] - + for provider_dict in prov_info_json_list: + + tins, npis, names = normalize_to_lists(provider_dict) + name_matches = prompt_calls.provider_name_match_check( names, provider_name, filename ) # Returns Boolean if name_matches: found_match = True - provider["IS_GROUP"] = "Y" + provider_dict["IS_GROUP"] = "Y" for tin in tins: group_tins.add(tin) for npi in npis: @@ -278,7 +308,7 @@ def merge_provider_info_with_hybrid_smart_chunking(one_to_one_results, filename) for name in names: group_names.add(name) else: - provider["IS_GROUP"] = "N" + provider_dict["IS_GROUP"] = "N" other_tins.extend(tins) other_npis.extend(npis) other_names.extend(names) @@ -295,36 +325,22 @@ def merge_provider_info_with_hybrid_smart_chunking(one_to_one_results, filename) for name in provider_name: group_names.add(name) - # De-Unknown GROUP fields - group_tins = [v for v in group_tins if v != "UNKNOWN"] - group_npis = [v for v in group_npis if v != "UNKNOWN"] - group_names = [v for v in group_names if v != "UNKNOWN"] - - # Deduplicate while preserving order and remove any "UNKNOWN" entries and overlaps between group and other - group_tins = list(dict.fromkeys(group_tins)) - other_tins = list(dict.fromkeys(other_tins)) - other_tins = [v for v in other_tins if v not in group_tins and v != "UNKNOWN"] - - group_npis = list(dict.fromkeys(group_npis)) - other_npis = list(dict.fromkeys(other_npis)) - other_npis = [v for v in other_npis if v not in group_npis and v != "UNKNOWN"] - - group_names = list(dict.fromkeys(group_names)) - other_names = list(dict.fromkeys(other_names)) - other_names = [v for v in other_names if v not in group_names and v != "UNKNOWN"] + group_tins, group_npis, group_names, other_tins, other_npis, other_names = deunknown_and_deduplicate( + group_tins, group_npis, group_names, other_tins, other_npis, other_names + ) # Filter out records with NO_IDENTIFIERS_FOUND prov_info_json_list = [ p for p in prov_info_json_list - if "NO_IDENTIFIERS_FOUND" not in (p.get("NAME") or []) + if "NO_IDENTIFIERS_FOUND" not in (p.get("NAME", [])) ] # Deduplicate provider_list by NAME, keeping records with actual TIN/NPI values prov_info_json_list = deduplicate_providers_by_name(prov_info_json_list) # Reconstruct PROV_INFO_JSON and PROV_INFO_JSON_FORMATTED with IS_GROUP flags - one_to_one_results["PROV_INFO_JSON"] = json.dumps(prov_info_json_list) + one_to_one_results["PROV_INFO_JSON"] = prov_info_json_list one_to_one_results["PROV_INFO_JSON_FORMATTED"] = "\n".join( [json.dumps(provider) for provider in prov_info_json_list] ) @@ -337,6 +353,7 @@ def merge_provider_info_with_hybrid_smart_chunking(one_to_one_results, filename) one_to_one_results["PROV_OTHER_TIN"] = other_tins if other_tins else [] one_to_one_results["PROV_OTHER_NPI"] = other_npis if other_npis else [] one_to_one_results["PROV_OTHER_NAME_FULL"] = other_names if other_names else [] + return one_to_one_results @@ -599,9 +616,9 @@ def get_prov_info_json(relevant_pages, text_dict, filename, payer_name): if not relevant_pages: prov_info_json_clean = [ { - "TIN": ["UNKNOWN"], - "NPI": ["UNKNOWN"], - "NAME": ["NO_IDENTIFIERS_FOUND"], + "TIN": "UNKNOWN", + "NPI": "UNKNOWN", + "NAME": "NO_IDENTIFIERS_FOUND", } ] else: @@ -615,8 +632,10 @@ def get_prov_info_json(relevant_pages, text_dict, filename, payer_name): # Clean and standardize results prov_info_json_clean = clean_provider_info(prov_info_json) + # Deduplicate providers based on TIN and NPI prov_info_json_clean = deduplicate_providers(prov_info_json_clean) + return prov_info_json_clean def get_all_tins_and_npis(contract_text): @@ -690,7 +709,7 @@ def run_provider_info_fields( # Create columns one_to_one_results = {} - one_to_one_results["PROV_INFO_JSON"] = json.dumps(prov_info_json_clean) + one_to_one_results["PROV_INFO_JSON"] = prov_info_json_clean one_to_one_results["PROV_INFO_JSON_FORMATTED"] = "\n".join( [json.dumps(provider) for provider in prov_info_json_clean] ) diff --git a/src/pipelines/shared/postprocessing/postprocessing_funcs.py b/src/pipelines/shared/postprocessing/postprocessing_funcs.py index 31480b5..c117867 100644 --- a/src/pipelines/shared/postprocessing/postprocessing_funcs.py +++ b/src/pipelines/shared/postprocessing/postprocessing_funcs.py @@ -677,6 +677,10 @@ def deduplicate_provider_columns(df: pd.DataFrame) -> pd.DataFrame: df.at[index, other_col] = [] continue + # Normalize other_vals to a list if it's a string + if isinstance(other_vals, str): + other_vals = [other_vals] + # Normalize group values (list or empty) group_vals = group_vals or [] diff --git a/src/utils/string_utils.py b/src/utils/string_utils.py index 6fc5f97..6cfca22 100644 --- a/src/utils/string_utils.py +++ b/src/utils/string_utils.py @@ -893,7 +893,17 @@ def normalize_one_to_one_answers_dict(answers_dict: dict[str, str | list]) -> di "PROVIDER_NAME": ["Provider A", "Provider B"] } """ + DO_NOT_NORMALIZE = [ + "PROV_INFO_JSON", + "PROV_INFO_JSON_FORMATTED", + "PAYER_STATE", + "PROVIDER_STATE" + ] normalized = {} for field_name, value in answers_dict.items(): - normalized[field_name] = normalize_one_to_one_field_value(field_name, value) + if field_name not in DO_NOT_NORMALIZE: + normalized[field_name] = normalize_one_to_one_field_value(field_name, value) + else: + normalized[field_name] = value + return normalized