From dc19570ef13bd5c4c29c70bc98de4224b0377d19 Mon Sep 17 00:00:00 2001 From: Mayank Aamseek Date: Fri, 13 Mar 2026 14:52:33 +0000 Subject: [PATCH] Merged in DAIP2-2121-prov-info-fields-issue-fixes (pull request #908) empty prov_info_json fixed * empty prov_info_json fixed * Merged DEV into DAIP2-2121-prov-info-fields-issue-fixes * format of PROV_OTHER_TIN fixed * Revert "format of PROV_OTHER_TIN fixed" This reverts commit 580847de03265520b1b4fad473bbf38ef1573753. * formatting and deduplication * pipeline error fixed * test cases added * pipeline error fixed Approved-by: Katon Minhas --- src/pipelines/shared/extraction/row_funcs.py | 11 ++- .../shared/extraction/tin_npi_funcs.py | 40 ++++++--- .../shared/postprocessing/postprocess.py | 21 +++++ src/tests/test_tin_npi_funcs.py | 81 +++++++++++++++++++ src/utils/json_utils.py | 7 +- 5 files changed, 147 insertions(+), 13 deletions(-) diff --git a/src/pipelines/shared/extraction/row_funcs.py b/src/pipelines/shared/extraction/row_funcs.py index 25682f6..ac3f689 100644 --- a/src/pipelines/shared/extraction/row_funcs.py +++ b/src/pipelines/shared/extraction/row_funcs.py @@ -85,11 +85,18 @@ def merge_one_to_one_into_one_to_n( for k, v in one_to_one_results.items(): if k not in one_to_n_results.columns: # If k is not a column in one_to_n_results, create it and populate all rows with v - one_to_n_results[k] = str(v) + # Don't convert lists to strings - keep them as lists for proper JSON serialization + value_to_use = v if isinstance(v, list) else str(v) + # Assign the same value to all rows by creating a list with length matching the DataFrame + one_to_n_results[k] = [value_to_use] * len(one_to_n_results) else: # If k is a column in one_to_n_results, replace empty values with v + # For lists, keep as list; for others, convert to string + value_to_use = v if isinstance(v, list) else str(v) one_to_n_results[k] = one_to_n_results[k].apply( - lambda row_val: str(v) if string_utils.is_empty(row_val) else row_val + lambda row_val: ( + value_to_use if string_utils.is_empty(row_val) else row_val + ) ) return one_to_n_results diff --git a/src/pipelines/shared/extraction/tin_npi_funcs.py b/src/pipelines/shared/extraction/tin_npi_funcs.py index 18966fe..5cba825 100644 --- a/src/pipelines/shared/extraction/tin_npi_funcs.py +++ b/src/pipelines/shared/extraction/tin_npi_funcs.py @@ -33,6 +33,9 @@ def run_provider_info_fields(text_dict: dict, filename: str, payer_name: str): # Prompt to get prov_info_json prov_info_json = get_prov_info_json(relevant_pages, text_dict, filename, payer_name) + + # Deduplicate providers based on TIN, NPI, and NAME fields + prov_info_json = deduplicate_providers(prov_info_json) prov_info_results["PROV_INFO_JSON"] = prov_info_json # Add FILENAME_TIN @@ -243,7 +246,7 @@ def deduplicate_providers( provider is kept. If all three fields are unknown, the provider is skipped. Args: - provider_info (list[dict]): A list of provider information dictionaries to be deduplicated. + provider_info_json (list[dict]): A list of provider information dictionaries to be deduplicated. Returns: list[dict]: A list of deduplicated provider information dictionaries. @@ -265,7 +268,10 @@ def deduplicate_providers( and string_utils.is_empty(name) ): continue - key = (tin, npi, name) + + # Normalize the name for case-insensitive comparison + normalized_name = name.casefold() if isinstance(name, str) else name + key = (tin, npi, normalized_name) if key not in seen: seen.add(key) @@ -305,16 +311,30 @@ def add_filename_tin(one_to_one_results: dict, filename: str) -> dict: filename_tin_list if filename_tin_list else "N/A" ) + prov_info_json = one_to_one_results.get("PROV_INFO_JSON", []) + # Add to PROV_INFO_JSON for filename_tin in filename_tin_list: - one_to_one_results["PROV_INFO_JSON"].append( - { - "TIN": "".join(c for c in str(filename_tin) if c.isdigit()), - "NPI": "N/A", - "NAME": "N/A", - "FROM_FILENAME": "Y", - } - ) + clean_tin = "".join(c for c in str(filename_tin) if c.isdigit()) + + # MERGE LOGIC: If there's exactly 1 extracted provider (with TIN as N/A) and 1 filename TIN, merge them. + if ( + len(prov_info_json) == 1 + and len(filename_tin_list) == 1 + and string_utils.is_empty(prov_info_json[0].get("TIN")) + ): + prov_info_json[0]["TIN"] = clean_tin + prov_info_json[0]["FROM_FILENAME"] = "Y" + else: + prov_info_json.append( + { + "TIN": clean_tin, + "NPI": "N/A", + "NAME": "N/A", + "FROM_FILENAME": "Y", + } + ) + one_to_one_results["PROV_INFO_JSON"] = prov_info_json return one_to_one_results diff --git a/src/pipelines/shared/postprocessing/postprocess.py b/src/pipelines/shared/postprocessing/postprocess.py index b2dc19a..8ec3637 100644 --- a/src/pipelines/shared/postprocessing/postprocess.py +++ b/src/pipelines/shared/postprocessing/postprocess.py @@ -160,6 +160,27 @@ def contract_config_postprocess(df, constants: Constants): # Contract Config specific postprocessing steps: # Apply json formatting + cc_df["FILENAME_TIN"] = cc_df["FILENAME_TIN"].apply( + postprocessing_funcs.format_as_json_list + ) + cc_df["PROV_GROUP_TIN"] = cc_df["PROV_GROUP_TIN"].apply( + postprocessing_funcs.format_as_json_list + ) + cc_df["PROV_GROUP_NPI"] = cc_df["PROV_GROUP_NPI"].apply( + postprocessing_funcs.format_as_json_list + ) + cc_df["PROV_GROUP_NAME_FULL"] = cc_df["PROV_GROUP_NAME_FULL"].apply( + postprocessing_funcs.format_as_json_list + ) + cc_df["PROV_OTHER_TIN"] = cc_df["PROV_OTHER_TIN"].apply( + postprocessing_funcs.format_as_json_list + ) + cc_df["PROV_OTHER_NPI"] = cc_df["PROV_OTHER_NPI"].apply( + postprocessing_funcs.format_as_json_list + ) + cc_df["PROV_OTHER_NAME_FULL"] = cc_df["PROV_OTHER_NAME_FULL"].apply( + postprocessing_funcs.format_as_json_list + ) cc_df["PAYER_STATE"] = cc_df["PAYER_STATE"].apply( postprocessing_funcs.format_as_json_list ) diff --git a/src/tests/test_tin_npi_funcs.py b/src/tests/test_tin_npi_funcs.py index 559bfb4..6e1bb51 100644 --- a/src/tests/test_tin_npi_funcs.py +++ b/src/tests/test_tin_npi_funcs.py @@ -255,3 +255,84 @@ class TestTinNpiFuncs: results = tin_npi_funcs.run_provider_info_fields( sample_text_dict, "test.pdf", "Test Payer" ) + + def test_deduplicate_providers_case_insensitive_names(self): + """Test that provider deduplication is case-insensitive for names.""" + provider_info_json = [ + {"TIN": "123456789", "NPI": "1234567890", "NAME": "Test Provider"}, + {"TIN": "123456789", "NPI": "1234567890", "NAME": "test provider"}, + {"TIN": "123456789", "NPI": "1234567890", "NAME": "TEST PROVIDER"}, + ] + + result = tin_npi_funcs.deduplicate_providers(provider_info_json) + + # Should only have one provider since all fields match (case-insensitive for name) + assert len(result) == 1 + assert result[0]["NAME"] == "Test Provider" + + def test_deduplicate_providers_different_cases(self): + """Test deduplication with different TINs and case variations in names.""" + provider_info_json = [ + {"TIN": "123456789", "NPI": "N/A", "NAME": "Provider One"}, + {"TIN": "123456789", "NPI": "N/A", "NAME": "provider one"}, + {"TIN": "987654321", "NPI": "N/A", "NAME": "provider one"}, + {"TIN": "111222333", "NPI": "N/A", "NAME": "Provider Two"}, + ] + + result = tin_npi_funcs.deduplicate_providers(provider_info_json) + + # Should have 3 providers: + # - Row 1 and 2 are duplicates (same TIN, same name case-insensitively) -> keep Row 1 + # - Row 3 is unique (different TIN) + # - Row 4 is unique + assert len(result) == 3 + + def test_add_filename_tin_merge_single_provider_single_tin(self): + """Test merge logic when there's 1 provider with N/A TIN and 1 filename TIN.""" + one_to_one_results = { + "PROV_INFO_JSON": [ + {"TIN": "N/A", "NPI": "1234567890", "NAME": "Test Provider"} + ] + } + filename = "contract-123456789-test.txt" + + result = tin_npi_funcs.add_filename_tin(one_to_one_results, filename) + + # Should merge the filename TIN into the existing provider + assert len(result["PROV_INFO_JSON"]) == 1 + assert result["PROV_INFO_JSON"][0]["TIN"] == "123456789" + assert result["PROV_INFO_JSON"][0]["FROM_FILENAME"] == "Y" + assert result["PROV_INFO_JSON"][0]["NPI"] == "1234567890" + + def test_add_filename_tin_no_merge_multiple_providers(self): + """Test that merge doesn't happen with multiple providers.""" + one_to_one_results = { + "PROV_INFO_JSON": [ + {"TIN": "N/A", "NPI": "1234567890", "NAME": "Provider 1"}, + {"TIN": "987654321", "NPI": "N/A", "NAME": "Provider 2"}, + ] + } + filename = "contract-123456789-test.txt" + + result = tin_npi_funcs.add_filename_tin(one_to_one_results, filename) + + # Should add as separate provider, not merge + assert len(result["PROV_INFO_JSON"]) == 3 + assert result["PROV_INFO_JSON"][2]["TIN"] == "123456789" + assert result["PROV_INFO_JSON"][2]["FROM_FILENAME"] == "Y" + + def test_add_filename_tin_no_merge_multiple_filename_tins(self): + """Test that merge doesn't happen with multiple filename TINs.""" + one_to_one_results = { + "PROV_INFO_JSON": [ + {"TIN": "N/A", "NPI": "1234567890", "NAME": "Test Provider"} + ] + } + filename = "contract-123456789-987654321.txt" + + result = tin_npi_funcs.add_filename_tin(one_to_one_results, filename) + + # Should add both TINs as separate providers, not merge + assert len(result["PROV_INFO_JSON"]) == 3 + # Sort both lists for comparison since order may not be deterministic + assert sorted(result["FILENAME_TIN"]) == sorted(["123456789", "987654321"]) diff --git a/src/utils/json_utils.py b/src/utils/json_utils.py index e140e9a..31176fc 100644 --- a/src/utils/json_utils.py +++ b/src/utils/json_utils.py @@ -15,7 +15,6 @@ import ast import json import logging import re -import ast from typing import Any import math @@ -331,6 +330,12 @@ def format_prov_info_json(val: Any) -> str: return serialize_prov_info_json(parsed) except json.JSONDecodeError: pass + try: + parsed = ast.literal_eval(text) + if isinstance(parsed, list): + return serialize_prov_info_json(parsed) + except (ValueError, SyntaxError): + pass return "[]" if isinstance(val, list): return serialize_prov_info_json(val)