Merged in bugfix/hmk-1-to-1 (pull request #820)

fix for redundantly populated  provider twice

* fix for redundantly populated  provider twice

* Merged main into bugfix/hmk-1-to-1


Approved-by: Katon Minhas

fieldExtraction/src/investment/tin_npi_funcs.py

@@ -275,6 +275,9 @@ def merge_provider_info_with_hybrid_smart_chunking(one_to_one_results, filename)
 
     # Filter out records with NO_IDENTIFIERS_FOUND
     provider_list = [p for p in provider_list if p.get('NAME') != 'NO_IDENTIFIERS_FOUND']
+    
+    # Deduplicate provider_list by NAME, keeping records with actual TIN/NPI values
+    provider_list = deduplicate_providers_by_name(provider_list)
 
     # Reconstruct PROV_INFO_JSON and PROV_INFO_JSON_FORMATTED with IS_GROUP flags
     one_to_one_results["PROV_INFO_JSON"] = json.dumps(provider_list)
@@ -661,3 +664,24 @@ def run_provider_info_fields(
     one_to_one_results["FILENAME_TIN"] = filename_tin[0] if filename_tin else "N/A"
 
     return one_to_one_results
+
+
+def deduplicate_providers_by_name(provider_list: list[dict]) -> list[dict]:
+    """
+    Deduplicates a list of provider dictionaries by NAME, preferring records with actual TIN/NPI values.
+
+    Args:
+        provider_list (list[dict]): List of provider dictionaries with NAME, TIN, NPI keys.
+
+    Returns:
+        list[dict]: Deduplicated list with one entry per unique NAME, preferring non-UNKNOWN values.
+    """
+    seen_names = set()
+    deduplicated_list = []
+    # Sort so providers with actual TIN/NPI values come first
+    for provider in sorted(provider_list, key=lambda p: (p.get('TIN') == 'UNKNOWN', p.get('NPI') == 'UNKNOWN')):
+        name = provider.get('NAME')
+        if name not in seen_names:
+            seen_names.add(name)
+            deduplicated_list.append(provider)
+    return deduplicated_list