Resolve PROV_INFO issue

src/pipelines/saas/file_processing.py

@@ -258,6 +258,7 @@ def run_one_to_one_prompts(
     ################## Fill NA Mapping ##################
     one_to_one_results = aarete_derived.fill_na_mapping(one_to_one_results)
 
+    print("After all one_to_one: ", one_to_one_results)
     return one_to_one_results[0]
 
 

src/pipelines/shared/extraction/tin_npi_funcs.py

@@ -157,7 +157,7 @@ def chunk_on_matches(matches: list[str], text_dict: dict) -> str:
     return "\n".join(valid_pages)
 
 
-def clean_provider_info(provider_info: list[dict]) -> list[dict]:
+def clean_provider_info(prov_info_json: list[dict]) -> list[dict]:
     """Cleans provider information by standardizing Name/TIN/NPI formats.
 
     Args:
@@ -167,8 +167,8 @@ def clean_provider_info(provider_info: list[dict]) -> list[dict]:
     Returns:
         list: Cleaned provider information
     """
-    cleaned_providers = []
-    for provider in provider_info:
+    prov_info_json_clean = []
+    for provider in prov_info_json:
         cleaned_provider = {}
 
         # ---- Clean TIN (list, digits only, 9 chars) ----
@@ -180,7 +180,7 @@ def clean_provider_info(provider_info: list[dict]) -> list[dict]:
                 tin_str = str(tin).replace("-", "").replace(".", "")
                 if tin_str.isdigit() and len(tin_str) == 9:
                     cleaned_tins.append(tin_str)
-        cleaned_provider["TIN"] = cleaned_tins if cleaned_tins else ["UNKNOWN"]
+        cleaned_provider["TIN"] = cleaned_tins if cleaned_tins else "UNKNOWN"
 
         # ---- Clean NPI (list, digits only, 10 chars) ----
         cleaned_npis = []
@@ -191,7 +191,7 @@ def clean_provider_info(provider_info: list[dict]) -> list[dict]:
                 npi_str = str(npi).replace("-", "").replace(".", "")
                 if npi_str.isdigit() and len(npi_str) == 10:
                     cleaned_npis.append(npi_str)
-        cleaned_provider["NPI"] = cleaned_npis if cleaned_npis else ["UNKNOWN"]
+        cleaned_provider["NPI"] = cleaned_npis if cleaned_npis else "UNKNOWN"
 
         # ---- Clean NAME (list) ----
         cleaned_names = []
@@ -202,20 +202,66 @@ def clean_provider_info(provider_info: list[dict]) -> list[dict]:
                 name_str = " ".join(str(name).split()).strip().rstrip(".")
                 if name_str:
                     cleaned_names.append(name_str)
-        cleaned_provider["NAME"] = cleaned_names if cleaned_names else ["UNKNOWN"]
+        cleaned_provider["NAME"] = cleaned_names if cleaned_names else "UNKNOWN"
 
-        cleaned_providers.append(cleaned_provider)
-    return cleaned_providers
+        prov_info_json_clean.append(cleaned_provider)
+    return prov_info_json_clean
 
 
-def merge_provider_info_with_hybrid_smart_chunking(one_to_one_results, filename):
+def normalize_to_lists(provider: dict):
+
+    tins = provider.get("TIN")
+    npis = provider.get("NPI")
+    names = provider.get("NAME")
+    
+    # Normalize to lists: if string, convert to list to avoid character splitting
+    if isinstance(tins, str):
+        tins = [tins]
+    elif not isinstance(tins, list):
+        tins = []
+    if isinstance(npis, str):
+        npis = [npis]
+    elif not isinstance(npis, list):
+        npis = []
+    if isinstance(names, str):
+        names = [names]
+    elif not isinstance(names, list):
+        names = []
+    
+    return tins, npis, names
+
+def deunknown_and_deduplicate(group_tins, group_npis, group_names, other_tins, other_npis, other_names):
+
+    # De-Unknown GROUP fields
+    group_tins = [v for v in group_tins if v != "UNKNOWN"]
+    group_npis = [v for v in group_npis if v != "UNKNOWN"]
+    group_names = [v for v in group_names if v != "UNKNOWN"]
+
+    # Deduplicate while preserving order and remove any "UNKNOWN" entries and overlaps between group and other
+    group_tins = list(dict.fromkeys(group_tins))
+    other_tins = list(dict.fromkeys(other_tins))
+    other_tins = [v for v in other_tins if v not in group_tins and v != "UNKNOWN"]
+
+    group_npis = list(dict.fromkeys(group_npis))
+    other_npis = list(dict.fromkeys(other_npis))
+    other_npis = [v for v in other_npis if v not in group_npis and v != "UNKNOWN"]
+
+    group_names = list(dict.fromkeys(group_names))
+    other_names = list(dict.fromkeys(other_names))
+    other_names = [v for v in other_names if v not in group_names and v != "UNKNOWN"]
+
+    return group_tins, group_npis, group_names, other_tins, other_npis, other_names
+
+
+
+def merge_provider_info_with_hybrid_smart_chunking(one_to_one_results: dict, filename: str):
     """
     Merges provider_info into one_to_one_results.
 
     Args:
         one_to_one_results (dict): Dictionary of one-to-one field results.
         provider_info (list[dict]): List of dictionaries containing provider information.
-            Each dictionary is keyed by TIN, NPI, NAME, IS_GROUP.
+            Each dictionary is keyed by TIN, NPI, NAME
 
     Returns:
         dict: Updated one_to_one_results with merged values from provider_info.
@@ -244,33 +290,17 @@ def merge_provider_info_with_hybrid_smart_chunking(one_to_one_results, filename)
     # Track if we found any name matches
     found_match = False
 
-    for provider in prov_info_json_list:
-        # Ensure list format (FORMAT FIX ONLY)
-        tins = provider.get("TIN") or []
-        npis = provider.get("NPI") or []
-        names = provider.get("NAME") or []
-        
-        # Normalize to lists: if string, convert to list to avoid character splitting
-        if isinstance(tins, str):
-            tins = [tins]
-        elif not isinstance(tins, list):
-            tins = []
-        if isinstance(npis, str):
-            npis = [npis]
-        elif not isinstance(npis, list):
-            npis = []
-        if isinstance(names, str):
-            names = [names]
-        elif not isinstance(names, list):
-            names = []
-        
+    for provider_dict in prov_info_json_list:
+
+        tins, npis, names = normalize_to_lists(provider_dict)
+
         name_matches = prompt_calls.provider_name_match_check(
             names, provider_name, filename
         )  # Returns Boolean
 
         if name_matches:
             found_match = True
-            provider["IS_GROUP"] = "Y"
+            provider_dict["IS_GROUP"] = "Y"
             for tin in tins:
                 group_tins.add(tin)
             for npi in npis:
@@ -278,7 +308,7 @@ def merge_provider_info_with_hybrid_smart_chunking(one_to_one_results, filename)
             for name in names:
                 group_names.add(name)
         else:
-            provider["IS_GROUP"] = "N"
+            provider_dict["IS_GROUP"] = "N"
             other_tins.extend(tins)
             other_npis.extend(npis)
             other_names.extend(names)
@@ -295,36 +325,22 @@ def merge_provider_info_with_hybrid_smart_chunking(one_to_one_results, filename)
         for name in provider_name:
             group_names.add(name)
 
-    # De-Unknown GROUP fields
-    group_tins = [v for v in group_tins if v != "UNKNOWN"]
-    group_npis = [v for v in group_npis if v != "UNKNOWN"]
-    group_names = [v for v in group_names if v != "UNKNOWN"]
-
-    # Deduplicate while preserving order and remove any "UNKNOWN" entries and overlaps between group and other
-    group_tins = list(dict.fromkeys(group_tins))
-    other_tins = list(dict.fromkeys(other_tins))
-    other_tins = [v for v in other_tins if v not in group_tins and v != "UNKNOWN"]
-
-    group_npis = list(dict.fromkeys(group_npis))
-    other_npis = list(dict.fromkeys(other_npis))
-    other_npis = [v for v in other_npis if v not in group_npis and v != "UNKNOWN"]
-
-    group_names = list(dict.fromkeys(group_names))
-    other_names = list(dict.fromkeys(other_names))
-    other_names = [v for v in other_names if v not in group_names and v != "UNKNOWN"]
+    group_tins, group_npis, group_names, other_tins, other_npis, other_names = deunknown_and_deduplicate(
+        group_tins, group_npis, group_names, other_tins, other_npis, other_names
+    )
 
     # Filter out records with NO_IDENTIFIERS_FOUND
     prov_info_json_list = [
         p
         for p in prov_info_json_list
-        if "NO_IDENTIFIERS_FOUND" not in (p.get("NAME") or [])
+        if "NO_IDENTIFIERS_FOUND" not in (p.get("NAME", []))
     ]
 
     # Deduplicate provider_list by NAME, keeping records with actual TIN/NPI values
     prov_info_json_list = deduplicate_providers_by_name(prov_info_json_list)
 
     # Reconstruct PROV_INFO_JSON and PROV_INFO_JSON_FORMATTED with IS_GROUP flags
-    one_to_one_results["PROV_INFO_JSON"] = json.dumps(prov_info_json_list)
+    one_to_one_results["PROV_INFO_JSON"] = prov_info_json_list
     one_to_one_results["PROV_INFO_JSON_FORMATTED"] = "\n".join(
         [json.dumps(provider) for provider in prov_info_json_list]
     )
@@ -337,6 +353,7 @@ def merge_provider_info_with_hybrid_smart_chunking(one_to_one_results, filename)
     one_to_one_results["PROV_OTHER_TIN"] = other_tins if other_tins else []
     one_to_one_results["PROV_OTHER_NPI"] = other_npis if other_npis else []
     one_to_one_results["PROV_OTHER_NAME_FULL"] = other_names if other_names else []
+
     return one_to_one_results
 
 
@@ -599,9 +616,9 @@ def get_prov_info_json(relevant_pages, text_dict, filename, payer_name):
     if not relevant_pages:
         prov_info_json_clean = [
             {
-                "TIN": ["UNKNOWN"],
-                "NPI": ["UNKNOWN"],
-                "NAME": ["NO_IDENTIFIERS_FOUND"],
+                "TIN": "UNKNOWN",
+                "NPI": "UNKNOWN",
+                "NAME": "NO_IDENTIFIERS_FOUND",
             }
         ]
     else:
@@ -615,8 +632,10 @@ def get_prov_info_json(relevant_pages, text_dict, filename, payer_name):
 
         # Clean and standardize results
         prov_info_json_clean = clean_provider_info(prov_info_json)
+
         # Deduplicate providers based on TIN and NPI
         prov_info_json_clean = deduplicate_providers(prov_info_json_clean)
+    
     return prov_info_json_clean
 
 def get_all_tins_and_npis(contract_text):
@@ -690,7 +709,7 @@ def run_provider_info_fields(
 
     # Create columns
     one_to_one_results = {}
-    one_to_one_results["PROV_INFO_JSON"] = json.dumps(prov_info_json_clean)
+    one_to_one_results["PROV_INFO_JSON"] = prov_info_json_clean
     one_to_one_results["PROV_INFO_JSON_FORMATTED"] = "\n".join(
         [json.dumps(provider) for provider in prov_info_json_clean]
     )

src/pipelines/shared/postprocessing/postprocessing_funcs.py

@@ -677,6 +677,10 @@ def deduplicate_provider_columns(df: pd.DataFrame) -> pd.DataFrame:
                 df.at[index, other_col] = []
                 continue
 
+            # Normalize other_vals to a list if it's a string
+            if isinstance(other_vals, str):
+                other_vals = [other_vals]
+
             # Normalize group values (list or empty)
             group_vals = group_vals or []
 

src/utils/string_utils.py

@@ -893,7 +893,17 @@ def normalize_one_to_one_answers_dict(answers_dict: dict[str, str | list]) -> di
             "PROVIDER_NAME": ["Provider A", "Provider B"]
         }
     """
+    DO_NOT_NORMALIZE = [
+        "PROV_INFO_JSON",
+        "PROV_INFO_JSON_FORMATTED",
+        "PAYER_STATE",
+        "PROVIDER_STATE"
+    ]
     normalized = {}
     for field_name, value in answers_dict.items():
-        normalized[field_name] = normalize_one_to_one_field_value(field_name, value)
+        if field_name not in DO_NOT_NORMALIZE:
+            normalized[field_name] = normalize_one_to_one_field_value(field_name, value)
+        else:
+            normalized[field_name] = value
+    
     return normalized