updated postprocessing funcs

src/pipelines/shared/postprocessing/postprocessing_funcs.py

@@ -69,18 +69,20 @@ def format_rate_fields_with_commas(value: str) -> str:
         return ""
 
 
-def remove_hyphens(value: str) -> str:
+def remove_hyphens(value):
     """
-    Remove hyphens from the input string.
+    Remove hyphens from the input value.
 
     Args:
-        value (str): The input string.
+        value : The input might be list or string
     Returns:
-        str: The cleaned string with hyphens removed.
+        str: The cleaned value with hyphens removed.
     """
-    if not isinstance(value, str):
+    if isinstance(value, str):
-        return ""
+        return value.replace("-", "")
-    return value.replace("-", "")
+    elif isinstance(value, list):
+        return [v.replace("-", "") if isinstance(v, str) else v for v in value]
+    return value
 
 
 def flatten_singleton_string_list(list_str: str) -> str:
@@ -638,8 +640,8 @@ def deduplicate_provider_columns(df: pd.DataFrame) -> pd.DataFrame:
     This function removes PROV_GROUP_* values from the PROV_OTHER_* columns if they are already present.
     In addition, it deduplicates PROV_OTHER_* values.
 
-    E.g. If PROV_GROUP_TIN is "123456789" and PROV_OTHER_TIN is "123456789 | 987654321 | 987654321",
+    E.g. If PROV_GROUP_TIN is ["123456789"] and PROV_OTHER_TIN is ["123456789", "987654321", "987654321"],
-    then PROV_OTHER_TIN will be updated to "987654321".
+    then PROV_OTHER_TIN will be updated to ["987654321"].
 
 
     Args:
@@ -656,62 +658,43 @@ def deduplicate_provider_columns(df: pd.DataFrame) -> pd.DataFrame:
         "PROV_OTHER_NPI",
         "PROV_OTHER_NAME_FULL",
     ]
-    other_columns = ["PROV_OTHER_TIN", "PROV_OTHER_NPI", "PROV_OTHER_NAME_FULL"]
 
-    # Check if all provider columns exist in the DataFrame
+    if not all(col in df.columns for col in provider_columns):
-    if all(col in df.columns for col in provider_columns):
+        return df
-        # Process each row
-        for index, row in df.iterrows():
-            # Get GROUP values for this row
-            group_tin = str(row.get("PROV_GROUP_TIN", "")).strip()
-            group_npi = str(row.get("PROV_GROUP_NPI", "")).strip()
-            group_name = str(row.get("PROV_GROUP_NAME_FULL", "")).strip()
 
-            # Simple mapping of OTHER columns to their corresponding GROUP values
+    for index, row in df.iterrows():
-            column_mappings = {
+        group_values = {
-                "PROV_OTHER_TIN": group_tin,
+            "PROV_OTHER_TIN": row.get("PROV_GROUP_TIN"),
-                "PROV_OTHER_NPI": group_npi,
+            "PROV_OTHER_NPI": row.get("PROV_GROUP_NPI"),
-                "PROV_OTHER_NAME_FULL": group_name,
+            "PROV_OTHER_NAME_FULL": row.get("PROV_GROUP_NAME_FULL"),
-            }
+        }
 
-            # Process each OTHER column
+        for other_col, group_vals in group_values.items():
-            for other_col, group_val in column_mappings.items():
+            other_vals = row.get(other_col)
-                other_value = str(row.get(other_col, "")).strip()
 
-                if (
+            if not other_vals:
-                    other_value
+                # None, NaN, or empty list
-                    and not string_utils.is_empty(other_value)
+                df.at[index, other_col] = []
-                    and other_value != "UNKNOWN"
+                continue
-                ):
-                    # Split by pipe and strip whitespace
-                    other_list = [
-                        v.strip() for v in other_value.split("|") if v.strip()
-                    ]
 
-                    # Remove corresponding GROUP value if present
+            # Normalize group values (list or empty)
-                    if (
+            group_vals = group_vals or []
-                        group_val
-                        and not string_utils.is_empty(group_val)
-                        and group_val != "UNKNOWN"
-                    ):
-                        other_list = [v for v in other_list if v != group_val]
 
-                    # Remove duplicates while preserving order
+            # Remove GROUP values
-                    seen = set()
+            filtered = [
-                    deduped_list = []
+                v for v in other_vals if v not in group_vals and v and v != "UNKNOWN"
-                    for item in other_list:
+            ]
-                        if (
+
-                            item not in seen
+            # Deduplicate while preserving order
-                            and not string_utils.is_empty(item)
+            seen = set()
-                            and item != "UNKNOWN"
+            deduped = []
-                        ):
+            for v in filtered:
-                            seen.add(item)
+                if v not in seen:
-                            deduped_list.append(item)
+                    seen.add(v)
+                    deduped.append(v)
+
+            df.at[index, other_col] = deduped
 
-                    # Update the DataFrame
-                    df.at[index, other_col] = (
-                        " | ".join(deduped_list) if deduped_list else ""
-                    )
     return df