From 18c094568b54e2fad4874dc935b3710dabc856c9 Mon Sep 17 00:00:00 2001 From: Venkat Date: Tue, 3 Feb 2026 14:01:17 +0000 Subject: [PATCH] fixed clean provider info --- .../shared/extraction/tin_npi_funcs.py | 56 ++++++++++--------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/src/pipelines/shared/extraction/tin_npi_funcs.py b/src/pipelines/shared/extraction/tin_npi_funcs.py index 1fc9f27..b965070 100644 --- a/src/pipelines/shared/extraction/tin_npi_funcs.py +++ b/src/pipelines/shared/extraction/tin_npi_funcs.py @@ -171,34 +171,38 @@ def clean_provider_info(provider_info: list[dict]) -> list[dict]: for provider in provider_info: cleaned_provider = {} - # Clean TIN - remove hyphens and dots, ensure it's 9 digits - if "TIN" in provider and provider["TIN"]: - cleaned_provider["TIN"] = provider["TIN"].replace("-", "").replace(".", "") - # Validate that TIN only contains digits or "|" - if not all(c.isdigit() or c == "|" for c in cleaned_provider["TIN"]): - cleaned_provider["TIN"] = "UNKNOWN" - else: - cleaned_provider["TIN"] = "UNKNOWN" + # ---- Clean TIN (list, digits only, 9 chars) ---- + cleaned_tins = [] + if isinstance(provider.get("TIN"), list): + for tin in provider["TIN"]: + if not tin: + continue + tin_str = str(tin).replace("-", "").replace(".", "") + if tin_str.isdigit() and len(tin_str) == 9: + cleaned_tins.append(tin_str) + cleaned_provider["TIN"] = cleaned_tins if cleaned_tins else ["UNKNOWN"] - # Clean NPI - remove hyphens and dots, ensure it's 10 digits - if "NPI" in provider and provider["NPI"]: - cleaned_provider["NPI"] = provider["NPI"].replace("-", "").replace(".", "") - # Validate that NPI only contains digits or "|" - if not all(c.isdigit() or c == "|" for c in cleaned_provider["NPI"]): - cleaned_provider["NPI"] = "UNKNOWN" - else: - cleaned_provider["NPI"] = "UNKNOWN" + # ---- Clean NPI (list, digits only, 10 chars) ---- + cleaned_npis = [] + if isinstance(provider.get("NPI"), list): + for npi in provider["NPI"]: + if not npi: + continue + npi_str = str(npi).replace("-", "").replace(".", "") + if npi_str.isdigit() and len(npi_str) == 10: + cleaned_npis.append(npi_str) + cleaned_provider["NPI"] = cleaned_npis if cleaned_npis else ["UNKNOWN"] - # Clean Name - remove extra spaces and ensure it's not empty - if "NAME" in provider and provider["NAME"]: - cleaned_provider["NAME"] = ( - " ".join(provider["NAME"].split()).strip().rstrip(".") - ) - # If name is empty after cleaning, set to "UNKNOWN" - if not cleaned_provider["NAME"]: - cleaned_provider["NAME"] = "UNKNOWN" - else: - cleaned_provider["NAME"] = "UNKNOWN" + # ---- Clean NAME (list) ---- + cleaned_names = [] + if isinstance(provider.get("NAME"), list): + for name in provider["NAME"]: + if not name: + continue + name_str = " ".join(str(name).split()).strip().rstrip(".") + if name_str: + cleaned_names.append(name_str) + cleaned_provider["NAME"] = cleaned_names if cleaned_names else ["UNKNOWN"] cleaned_providers.append(cleaned_provider) return cleaned_providers