Merged in feature/prov-info-testbed-to-json (pull request #514)

Add function to group provider information into JSON array format post-hoc from the test bed

* Add function to group provider information into JSON array format

* Add example notebook


Approved-by: Katon Minhas

fieldExtraction/src/testbed/testbed_utils.py

@@ -1,6 +1,7 @@
 
 import pandas as pd
 import src.utils.string_utils as string_utils
+import itertools
 
 
 def convert_to_float(value):
@@ -27,6 +28,100 @@ def testbed_preprocess(df):
     
     return df
 
+def group_providers_into_json_array(df):
+    """Groups provider information from a DataFrame into a JSON array format and appends it as a new column.
+
+    This function processes a DataFrame containing provider information, groups the data by the "FILE_NAME" column, 
+    and creates a JSON array for each group. The JSON array includes details about the group provider and other 
+    associated providers. The resulting JSON array is added as a new column, "PROV_INFO_JSON", to the DataFrame.
+
+    Args:
+        df (pandas.DataFrame): The input DataFrame containing the following columns:
+            - "FILE_NAME": Identifier for grouping rows.
+            - "PROV_GROUP_TIN": Tax Identification Number (TIN) of the group provider.
+            - "PROV_GROUP_NPI": National Provider Identifier (NPI) of the group provider.
+            - "PROV_GROUP_NAME_FULL": Full name of the group provider.
+            - "PROV_OTHER_TIN": Pipe-separated TINs of other providers.
+            - "PROV_OTHER_NPI": Pipe-separated NPIs of other providers.
+            - "PROV_OTHER_NAME_FULL": Pipe-separated full names of other providers.
+
+    Returns:
+        pandas.DataFrame: A modified DataFrame with an additional column "PROV_INFO_JSON", 
+        which contains a JSON array for each "FILE_NAME" group. Each JSON array includes:
+            - "TIN": TIN of the provider (or "N/A" if not available).
+            - "NPI": NPI of the provider (or "N/A" if not available).
+            - "NAME": Full name of the provider (or "N/A" if not available).
+            - "IS_GROUP": Boolean indicating whether the provider is the group provider (True) 
+              or an associated provider (False).
+
+    Notes:
+        - Missing or NaN values in the input DataFrame are replaced with "N/A" in the JSON output.
+        - If the lengths of "PROV_OTHER_TIN", "PROV_OTHER_NPI", and "PROV_OTHER_NAME_FULL" are not equal, 
+          missing values are filled with "N/A" using itertools.zip_longest.
+        - If the OTHER columns are misaligned with one another, the function will NOT 
+          be able to correct them. It will simply fill in the gaps with "N/A".
+    """
+
+    df = df.copy()
+    # Create a dictionary to hold provider data in JSON array format for each file
+    group_providers_dict = {}
+
+    for group, grouped_df in df.groupby("FILE_NAME"):
+        json_dict_list = []
+
+        # Get the first row of the grouped DataFrame
+        first_row = grouped_df.iloc[0]
+
+
+        # Extract TIN/NPI/NAME for group provider.
+        group_tin = str(first_row["PROV_GROUP_TIN"]).replace("nan", "N/A")
+        group_npi = str(first_row["PROV_GROUP_NPI"]).replace("nan", "N/A")
+        group_name_full = str(first_row["PROV_GROUP_NAME_FULL"])
+
+        # Append the group provider's TIN/NPI/NAME to the list.
+        json_dict_list.append({
+            "TIN": group_tin,
+            "NPI": group_npi,
+            "NAME": group_name_full, "IS_GROUP": True})
+
+        # Create lists of TIN/NPI/NAME for other providers.
+        # Split on pipe, then strip whitespace.
+        other_tins = str(first_row["PROV_OTHER_TIN"]).split("|")
+        other_npis = str(first_row["PROV_OTHER_NPI"]).split("|")
+        other_names = str(first_row["PROV_OTHER_NAME_FULL"]).split("|")
+        other_tins = [tin.strip().replace("nan", "N/A") for tin in other_tins]
+        other_npis = [npi.strip().replace("nan", "N/A") for npi in other_npis]
+        other_names = [name.strip().replace("nan", "N/A") for name in other_names]
+
+        # Append each other provider's TIN/NPI/NAME to the list.
+        # Use zip to iterate over the three lists in parallel.
+        # IF the lengths of the lists are not equal, use itertools.zip_longest to fill in the gaps with N/A.
+        for tin, npi, name in itertools.zip_longest(other_tins, other_npis, other_names, fillvalue="N/A"):
+            json_dict_list.append({
+                "TIN": tin,
+                "NPI": npi,
+                "NAME": name,
+                "IS_GROUP": False
+            })
+
+        group_providers_dict[group] = json_dict_list
+
+    # Add JSON data back to original DataFrame
+    df["PROV_INFO_JSON"] = df["FILE_NAME"].map(group_providers_dict)
+
+
+    ## Reorder columns to place PROV_INFO_JSON right after PROV_OTHER_NAME_FULL
+    cols = list(df.columns)
+    # Find the position of PROV_OTHER_NAME_FULL
+    prov_other_name_full_pos = cols.index("PROV_OTHER_NAME_FULL")
+    # Remove PROV_INFO_JSON from its current position
+    cols.remove("PROV_INFO_JSON")
+    # Insert it after PROV_OTHER_NAME_FULL
+    cols.insert(prov_other_name_full_pos + 1, "PROV_INFO_JSON")
+    # Reorder the DataFrame
+    df = df[cols]
+
+    return df
 
 def calculate_field_metrics(testbed_df, results_df, field):
     """Calculate precision, recall, and accuracy for a single field.