Merged in DAIP2-2121-prov-info-fields-issue-fixes (pull request #908)

empty prov_info_json fixed

* empty prov_info_json fixed

* Merged DEV into DAIP2-2121-prov-info-fields-issue-fixes

* format of PROV_OTHER_TIN fixed

* Revert "format of PROV_OTHER_TIN fixed"

This reverts commit 580847de03265520b1b4fad473bbf38ef1573753.

* formatting and deduplication

* pipeline error fixed

* test cases added

* pipeline error fixed


Approved-by: Katon Minhas

src/pipelines/shared/extraction/row_funcs.py

@@ -85,11 +85,18 @@ def merge_one_to_one_into_one_to_n(
     for k, v in one_to_one_results.items():
         if k not in one_to_n_results.columns:
             # If k is not a column in one_to_n_results, create it and populate all rows with v
-            one_to_n_results[k] = str(v)
+            # Don't convert lists to strings - keep them as lists for proper JSON serialization
+            value_to_use = v if isinstance(v, list) else str(v)
+            # Assign the same value to all rows by creating a list with length matching the DataFrame
+            one_to_n_results[k] = [value_to_use] * len(one_to_n_results)
         else:
             # If k is a column in one_to_n_results, replace empty values with v
+            # For lists, keep as list; for others, convert to string
+            value_to_use = v if isinstance(v, list) else str(v)
             one_to_n_results[k] = one_to_n_results[k].apply(
-                lambda row_val: str(v) if string_utils.is_empty(row_val) else row_val
+                lambda row_val: (
+                    value_to_use if string_utils.is_empty(row_val) else row_val
+                )
             )
 
     return one_to_n_results

src/pipelines/shared/extraction/tin_npi_funcs.py

@@ -33,6 +33,9 @@ def run_provider_info_fields(text_dict: dict, filename: str, payer_name: str):
 
     # Prompt to get prov_info_json
     prov_info_json = get_prov_info_json(relevant_pages, text_dict, filename, payer_name)
+
+    # Deduplicate providers based on TIN, NPI, and NAME fields
+    prov_info_json = deduplicate_providers(prov_info_json)
     prov_info_results["PROV_INFO_JSON"] = prov_info_json
 
     # Add FILENAME_TIN
@@ -243,7 +246,7 @@ def deduplicate_providers(
     provider is kept. If all three fields are unknown, the provider is skipped.
 
     Args:
-        provider_info (list[dict]): A list of provider information dictionaries to be deduplicated.
+        provider_info_json (list[dict]): A list of provider information dictionaries to be deduplicated.
 
     Returns:
         list[dict]: A list of deduplicated provider information dictionaries.
@@ -265,7 +268,10 @@ def deduplicate_providers(
             and string_utils.is_empty(name)
         ):
             continue
-        key = (tin, npi, name)
+
+        # Normalize the name for case-insensitive comparison
+        normalized_name = name.casefold() if isinstance(name, str) else name
+        key = (tin, npi, normalized_name)
 
         if key not in seen:
             seen.add(key)
@@ -305,16 +311,30 @@ def add_filename_tin(one_to_one_results: dict, filename: str) -> dict:
         filename_tin_list if filename_tin_list else "N/A"
     )
 
+    prov_info_json = one_to_one_results.get("PROV_INFO_JSON", [])
+
     # Add to PROV_INFO_JSON
     for filename_tin in filename_tin_list:
-        one_to_one_results["PROV_INFO_JSON"].append(
-            {
-                "TIN": "".join(c for c in str(filename_tin) if c.isdigit()),
-                "NPI": "N/A",
-                "NAME": "N/A",
-                "FROM_FILENAME": "Y",
-            }
-        )
+        clean_tin = "".join(c for c in str(filename_tin) if c.isdigit())
+
+        # MERGE LOGIC: If there's exactly 1 extracted provider (with TIN as N/A) and 1 filename TIN, merge them.
+        if (
+            len(prov_info_json) == 1
+            and len(filename_tin_list) == 1
+            and string_utils.is_empty(prov_info_json[0].get("TIN"))
+        ):
+            prov_info_json[0]["TIN"] = clean_tin
+            prov_info_json[0]["FROM_FILENAME"] = "Y"
+        else:
+            prov_info_json.append(
+                {
+                    "TIN": clean_tin,
+                    "NPI": "N/A",
+                    "NAME": "N/A",
+                    "FROM_FILENAME": "Y",
+                }
+            )
+    one_to_one_results["PROV_INFO_JSON"] = prov_info_json
 
     return one_to_one_results
 

src/pipelines/shared/postprocessing/postprocess.py

@@ -160,6 +160,27 @@ def contract_config_postprocess(df, constants: Constants):
 
     # Contract Config specific postprocessing steps:
     # Apply json formatting
+    cc_df["FILENAME_TIN"] = cc_df["FILENAME_TIN"].apply(
+        postprocessing_funcs.format_as_json_list
+    )
+    cc_df["PROV_GROUP_TIN"] = cc_df["PROV_GROUP_TIN"].apply(
+        postprocessing_funcs.format_as_json_list
+    )
+    cc_df["PROV_GROUP_NPI"] = cc_df["PROV_GROUP_NPI"].apply(
+        postprocessing_funcs.format_as_json_list
+    )
+    cc_df["PROV_GROUP_NAME_FULL"] = cc_df["PROV_GROUP_NAME_FULL"].apply(
+        postprocessing_funcs.format_as_json_list
+    )
+    cc_df["PROV_OTHER_TIN"] = cc_df["PROV_OTHER_TIN"].apply(
+        postprocessing_funcs.format_as_json_list
+    )
+    cc_df["PROV_OTHER_NPI"] = cc_df["PROV_OTHER_NPI"].apply(
+        postprocessing_funcs.format_as_json_list
+    )
+    cc_df["PROV_OTHER_NAME_FULL"] = cc_df["PROV_OTHER_NAME_FULL"].apply(
+        postprocessing_funcs.format_as_json_list
+    )
     cc_df["PAYER_STATE"] = cc_df["PAYER_STATE"].apply(
         postprocessing_funcs.format_as_json_list
     )

src/tests/test_tin_npi_funcs.py

@@ -255,3 +255,84 @@ class TestTinNpiFuncs:
         results = tin_npi_funcs.run_provider_info_fields(
             sample_text_dict, "test.pdf", "Test Payer"
         )
+
+    def test_deduplicate_providers_case_insensitive_names(self):
+        """Test that provider deduplication is case-insensitive for names."""
+        provider_info_json = [
+            {"TIN": "123456789", "NPI": "1234567890", "NAME": "Test Provider"},
+            {"TIN": "123456789", "NPI": "1234567890", "NAME": "test provider"},
+            {"TIN": "123456789", "NPI": "1234567890", "NAME": "TEST PROVIDER"},
+        ]
+
+        result = tin_npi_funcs.deduplicate_providers(provider_info_json)
+
+        # Should only have one provider since all fields match (case-insensitive for name)
+        assert len(result) == 1
+        assert result[0]["NAME"] == "Test Provider"
+
+    def test_deduplicate_providers_different_cases(self):
+        """Test deduplication with different TINs and case variations in names."""
+        provider_info_json = [
+            {"TIN": "123456789", "NPI": "N/A", "NAME": "Provider One"},
+            {"TIN": "123456789", "NPI": "N/A", "NAME": "provider one"},
+            {"TIN": "987654321", "NPI": "N/A", "NAME": "provider one"},
+            {"TIN": "111222333", "NPI": "N/A", "NAME": "Provider Two"},
+        ]
+
+        result = tin_npi_funcs.deduplicate_providers(provider_info_json)
+
+        # Should have 3 providers:
+        # - Row 1 and 2 are duplicates (same TIN, same name case-insensitively) -> keep Row 1
+        # - Row 3 is unique (different TIN)
+        # - Row 4 is unique
+        assert len(result) == 3
+
+    def test_add_filename_tin_merge_single_provider_single_tin(self):
+        """Test merge logic when there's 1 provider with N/A TIN and 1 filename TIN."""
+        one_to_one_results = {
+            "PROV_INFO_JSON": [
+                {"TIN": "N/A", "NPI": "1234567890", "NAME": "Test Provider"}
+            ]
+        }
+        filename = "contract-123456789-test.txt"
+
+        result = tin_npi_funcs.add_filename_tin(one_to_one_results, filename)
+
+        # Should merge the filename TIN into the existing provider
+        assert len(result["PROV_INFO_JSON"]) == 1
+        assert result["PROV_INFO_JSON"][0]["TIN"] == "123456789"
+        assert result["PROV_INFO_JSON"][0]["FROM_FILENAME"] == "Y"
+        assert result["PROV_INFO_JSON"][0]["NPI"] == "1234567890"
+
+    def test_add_filename_tin_no_merge_multiple_providers(self):
+        """Test that merge doesn't happen with multiple providers."""
+        one_to_one_results = {
+            "PROV_INFO_JSON": [
+                {"TIN": "N/A", "NPI": "1234567890", "NAME": "Provider 1"},
+                {"TIN": "987654321", "NPI": "N/A", "NAME": "Provider 2"},
+            ]
+        }
+        filename = "contract-123456789-test.txt"
+
+        result = tin_npi_funcs.add_filename_tin(one_to_one_results, filename)
+
+        # Should add as separate provider, not merge
+        assert len(result["PROV_INFO_JSON"]) == 3
+        assert result["PROV_INFO_JSON"][2]["TIN"] == "123456789"
+        assert result["PROV_INFO_JSON"][2]["FROM_FILENAME"] == "Y"
+
+    def test_add_filename_tin_no_merge_multiple_filename_tins(self):
+        """Test that merge doesn't happen with multiple filename TINs."""
+        one_to_one_results = {
+            "PROV_INFO_JSON": [
+                {"TIN": "N/A", "NPI": "1234567890", "NAME": "Test Provider"}
+            ]
+        }
+        filename = "contract-123456789-987654321.txt"
+
+        result = tin_npi_funcs.add_filename_tin(one_to_one_results, filename)
+
+        # Should add both TINs as separate providers, not merge
+        assert len(result["PROV_INFO_JSON"]) == 3
+        # Sort both lists for comparison since order may not be deterministic
+        assert sorted(result["FILENAME_TIN"]) == sorted(["123456789", "987654321"])

src/utils/json_utils.py

@@ -15,7 +15,6 @@ import ast
 import json
 import logging
 import re
-import ast
 from typing import Any
 import math
 
@@ -331,6 +330,12 @@ def format_prov_info_json(val: Any) -> str:
                     return serialize_prov_info_json(parsed)
             except json.JSONDecodeError:
                 pass
+            try:
+                parsed = ast.literal_eval(text)
+                if isinstance(parsed, list):
+                    return serialize_prov_info_json(parsed)
+            except (ValueError, SyntaxError):
+                pass
         return "[]"
     if isinstance(val, list):
         return serialize_prov_info_json(val)