import pandas as pd import src.utils.string_utils as string_utils import itertools from collections import Counter # Imports for the testbed postprocessing functions from src.investment import investment_postprocessing_funcs from src.constants.investment_columns import COLUMN_ORDER def convert_to_float(value): try: return f"{float(value):.2f}" if isinstance(value, (int, float)) or value.replace('.', '', 1).isdigit() else value except: return value def normalize_tin_npi(value): """Normalize TIN/NPI values by removing decimal points and trailing zeros.""" if string_utils.is_empty(value): return "" # Skip "UNKNOWN" and non-numeric values if value == "UNKNOWN" or not value.replace('.', '', 1).isdigit(): return value # Convert to integer-like string by removing decimal part try: # Remove decimal and any trailing zeros return str(int(float(value))) except: return value def testbed_preprocess(df): # Apply numeric conversion to non-CPT/diag/rev fields only for col in df.columns: if "_CD" not in col and "CPT" not in col and "DIAG" not in col and "REV" not in col: df[col] = df[col].apply(convert_to_float) else: # For CPT/diag/rev fields, apply string conversion AND convert | delimiters # to commas df[col] = df[col].astype(str).str.replace("|", ",", regex=False) df = df.applymap(lambda x: str(x).strip().upper()) df = df.replace("UNKNOWN", "").fillna("") df = df.replace("NAN", "") df = df.replace("N/N", "") # Normalize TIN and NPI columns tin_columns = [col for col in df.columns if 'TIN' in col] npi_columns = [col for col in df.columns if 'NPI' in col] for col in tin_columns + npi_columns: df[col] = df[col].str.replace("-", "", regex=False) df[col] = df[col].apply(normalize_tin_npi) df[col] = df[col].str.replace(" ", "", regex=False) df["AUTO_RENEWAL_TERM"] = df['AUTO_RENEWAL_TERM'].str.replace("ONE YEAR", "1 YEAR", regex=False) # Ensure that all empty cells are filled with empty strings df = df.fillna("") return df def calculate_field_metrics(testbed_df, results_df, field): """Calculate precision, recall, and accuracy for a single field. Args: testbed_df: DataFrame containing ground truth results_df: DataFrame containing predictions field: Name of field to evaluate """ # Get unique combinations of FILE_NAME and field value results_unique = results_df[['FILE_NAME', field]].drop_duplicates() testbed_unique = testbed_df[['FILE_NAME', field]].drop_duplicates() # Merge on FILE_NAME to compare values comparison = pd.merge( testbed_unique, results_unique, on='FILE_NAME', suffixes=('_truth', '_pred') ) # Calculate metrics true_positives = sum((comparison[f'{field}_truth'] == comparison[f'{field}_pred']) & (comparison[f'{field}_truth'] != '')) truth_populated = sum(comparison[f'{field}_truth'] != '') pred_populated = sum(comparison[f'{field}_pred'] != '') total_files = len(testbed_unique) precision = true_positives / pred_populated if pred_populated > 0 else 0 recall = true_positives / truth_populated if truth_populated > 0 else 0 accuracy = true_positives / total_files if total_files > 0 else 0 return precision, recall, accuracy def calculate_field_metrics_for_blanks(testbed_df, results_df, field): """Calculate metrics specially handling completely blank fields. Args: testbed_df (_type_): DataFrame containing ground truth results_df (_type_): DataFrame containing predictions field (_type_): Name of field to evaluate Returns: tuple: (precision, recall, accuracy) """ # Check if field is completely blank in both datasets testbed_all_blank = string_utils.is_empty(testbed_df[field], pd_mask=True).all() results_all_blank = string_utils.is_empty(results_df[field], pd_mask=True).all() if testbed_all_blank and results_all_blank: # Both are all blank - perfect agreement return 1.0, 1.0, 1.0 # Regular calculation for non-blank fields return calculate_field_metrics(testbed_df, results_df, field) def evaluate_provider_fields_separately(testbed_df, results_df, verbose=False): """Evaluates TIN, NPI, and NAME fields separately for each provider.""" # Initialize metrics for each field type metrics = { "TIN": {'tp' : 0, 'fp': 0, 'fn': 0, 'accuracy_file': []}, "NPI": {'tp' : 0, 'fp': 0, 'fn': 0, 'accuracy_file': []}, "NAME": {'tp' : 0, 'fp': 0, 'fn': 0, 'accuracy_file': []} } if verbose: # instantiate provider_metrics.txt if verbose csv_output_dicts = [] with open("provider_metrics.txt", "w") as f: f.write("Provider Metrics\n") for file in testbed_df['FILE_NAME'].unique(): # Get rows for this file testbed_file = testbed_df[testbed_df['FILE_NAME'] == file] results_file = results_df[results_df['FILE_NAME'] == file] # Extract all TINs, NPIs, and NAMEs for ground truth truth_tins = [] truth_npis = [] truth_names = [] # Add group provider info (with splitting) for col_type, truth_list in [ ("PROV_GROUP_TIN", truth_tins), ("PROV_GROUP_NPI", truth_npis), ("PROV_GROUP_NAME_FULL", truth_names) ]: value = testbed_file[col_type].iloc[0] values = [] pipe_split = value.split("|") for item in pipe_split: if col_type in ["PROV_GROUP_NPI", "PROV_GROUP_TIN"] and "," in item: # further split by comma, only if col_type is PROV_GROUP_NPI or PROV_GROUP_TIN for comma_item in item.split(","): comma_item = comma_item.strip() if comma_item: values.append(comma_item) elif item.strip(): values.append(item.strip()) # Now append all values to the list for v in values: if v and v != "UNKNOWN": truth_list.append(v) # Add other provider info for col_type, truth_list in [ ("PROV_OTHER_TIN", truth_tins), ("PROV_OTHER_NPI", truth_npis), ("PROV_OTHER_NAME_FULL", truth_names) ]: if not string_utils.is_empty(testbed_file[col_type].iloc[0]): values = testbed_file[col_type].iloc[0].split("|") if len(values) == 1: # possibly delimited by comma values = testbed_file[col_type].iloc[0].split(",") for value in values: value = value.strip() if value: truth_list.append(value) # Similarly, extract all TINs, NPIs, and NAMEs for predictions pred_tins = [] pred_npis = [] pred_names = [] # Add group provider info value = results_file['PROV_GROUP_TIN'].iloc[0] pred_tins.append(value) value = results_file['PROV_GROUP_NPI'].iloc[0] pred_npis.append(value) value = results_file['PROV_GROUP_NAME_FULL'].iloc[0] pred_names.append(value) # Add other provider info for col_type, pred_list in [ ("PROV_OTHER_TIN", pred_tins), ("PROV_OTHER_NPI", pred_npis), ("PROV_OTHER_NAME_FULL", pred_names) ]: values = results_file[col_type].iloc[0].split("|") if len(values) == 1: # possibly delimited by comma values = results_file[col_type].iloc[0].split(",") for value in values: value = value.strip() pred_list.append(value) file_metrics = {} # Store file-specific metrics for verbose output # Calculate metrics for each field type and print verbose output if requested for field_type, truth_list, pred_list in [ ("TIN", truth_tins, pred_tins), ("NPI", truth_npis, pred_npis), ("NAME", truth_names, pred_names) ]: # Count occurrences in lists, excluding empty strings and "UNKNOWN" truth_counter = Counter([x for x in truth_list if x and x != "UNKNOWN"]) pred_counter = Counter([x for x in pred_list if x and x != "UNKNOWN"]) # Filter lists to remove empty strings and "UNKNOWN" filtered_truth_list = [x for x in truth_list if x and x != "UNKNOWN"] filtered_pred_list = [x for x in pred_list if x and x != "UNKNOWN"] # Keep counters for display/debugging purposes truth_counter = Counter(filtered_truth_list) pred_counter = Counter(filtered_pred_list) # Convert to sets for set operations truth_set = set(filtered_truth_list) pred_set = set(filtered_pred_list) # Compute metrics using set operations file_tp = len(truth_set.intersection(pred_set)) # True Positives file_fp = len(pred_set - truth_set) # False Positives file_fn = len(truth_set - pred_set) # False Negatives # Calculate accuracy for this file file_accuracy = file_tp / (file_tp + file_fp + file_fn) if (file_tp + file_fp + file_fn) > 0 else 0 # Update overall metrics metrics[field_type]['tp'] += file_tp metrics[field_type]['fp'] += file_fp metrics[field_type]['fn'] += file_fn metrics[field_type]['accuracy_file'].append(file_accuracy) # Store file-specific metrics for verbose output file_metrics[field_type] = { "truth_counter": dict(truth_counter), "pred_counter": dict(pred_counter), "file_tp": file_tp, "file_fp": file_fp, "file_fn": file_fn, "file_accuracy": file_accuracy } if verbose: # Write verbose output to file # Initialize file if it doesn't exist with open("provider_metrics.txt", "a") as f: f.write(f"File: {file}\n") f.write(f"Total TINs: {len(set([tin for tin in truth_tins if tin and tin != 'UNKNOWN']))}, Predicted TINs: {len(set([tin for tin in pred_tins if tin and tin != 'UNKNOWN']))}\n") f.write(f"Total NPIs: {len(set([npi for npi in truth_npis if npi and npi != 'UNKNOWN']))}, Predicted NPIs: {len(set([npi for npi in pred_npis if npi and npi != 'UNKNOWN']))}\n") f.write(f"Total Names: {len(set([name for name in truth_names if name and name != 'UNKNOWN']))}, Predicted Names: {len(set([name for name in pred_names if name and name != 'UNKNOWN']))}\n") f.write("-" * 50 + "\n") for field_type in metrics: csv_output_dict = {} csv_output_dict["Filename"] = file csv_output_dict["Test Bed TINs"] = len(set([tin for tin in truth_tins if tin and tin != 'UNKNOWN'])) csv_output_dict["Test Bed NPIs"] = len(set([npi for npi in truth_npis if npi and npi != 'UNKNOWN'])) csv_output_dict["Test Bed Names"] = len(set([name for name in truth_names if name and name != 'UNKNOWN'])) csv_output_dict["Predicted TINs"] = len(set([tin for tin in pred_tins if tin and tin != 'UNKNOWN'])) csv_output_dict["Predicted NPIs"] = len(set([npi for npi in pred_npis if npi and npi != 'UNKNOWN'])) csv_output_dict["Predicted Names"] = len(set([name for name in pred_names if name and name != 'UNKNOWN'])) print_lines = [ f"Field Type: {field_type}", f"Ground Truth: {dict(file_metrics[field_type]['truth_counter'])}", f"Predictions: {dict(file_metrics[field_type]['pred_counter'])}", f"True Positives: {file_metrics[field_type]['file_tp']}", f"False Positives: {file_metrics[field_type]['file_fp']}", f"False Negatives: {file_metrics[field_type]['file_fn']}", f"File Accuracy: {file_metrics[field_type]['file_accuracy']}", f"Precision: {file_metrics[field_type]['file_tp'] / (file_metrics[field_type]['file_tp'] + file_metrics[field_type]['file_fp']) if (file_metrics[field_type]['file_tp'] + file_metrics[field_type]['file_fp']) > 0 else 0}", f"Recall: {file_metrics[field_type]['file_tp'] / (file_metrics[field_type]['file_tp'] + file_metrics[field_type]['file_fn']) if (file_metrics[field_type]['file_tp'] + file_metrics[field_type]['file_fn']) > 0 else 0}", "-" * 50 ] csv_output_dict["field_type"] = field_type csv_output_dict["Ground Truth"] = dict(file_metrics[field_type]['truth_counter']).copy() csv_output_dict["Predictions"] = dict(file_metrics[field_type]['pred_counter']).copy() csv_output_dict["True Positives"] = file_metrics[field_type]['file_tp'] csv_output_dict["False Positives"] = file_metrics[field_type]['file_fp'] csv_output_dict["False Negatives"] = file_metrics[field_type]['file_fn'] csv_output_dict["File Accuracy"] = file_metrics[field_type]['file_accuracy'] csv_output_dict["Precision"] = file_metrics[field_type]['file_tp'] / (file_metrics[field_type]['file_tp'] + file_metrics[field_type]['file_fp']) if (file_metrics[field_type]['file_tp'] + file_metrics[field_type]['file_fp']) > 0 else 0 csv_output_dict["Recall"] = file_metrics[field_type]['file_tp'] / (file_metrics[field_type]['file_tp'] + file_metrics[field_type]['file_fn']) if (file_metrics[field_type]['file_tp'] + file_metrics[field_type]['file_fn']) > 0 else 0 csv_output_dicts.append(csv_output_dict) with open("provider_metrics.txt", "a") as f: f.write("\n".join(print_lines) + "\n") # Convert the list of dictionaries to a DataFrame csv_output_df = pd.DataFrame(csv_output_dicts) # Save the DataFrame to a CSV file csv_output_df.to_csv("provider_metrics.csv", index=False) # Calculate accuracy, precision, recall, and F1 for each field type results = {} for field_type, field_metrics in metrics.items(): precision = field_metrics['tp'] / (field_metrics['tp'] + field_metrics['fp']) if (field_metrics['tp'] + field_metrics['fp']) > 0 else 0 recall = field_metrics['tp'] / (field_metrics['tp'] + field_metrics['fn']) if (field_metrics['tp'] + field_metrics['fn']) > 0 else 0 f1 = (2 * precision * recall) / (precision + recall) if (precision + recall) > 0 else 0 results[field_type] = { "precision": precision, "recall": recall, "f1": f1, "tp": field_metrics['tp'], "fp": field_metrics['fp'], "fn": field_metrics['fn'], "average_accuracy": sum(field_metrics['accuracy_file']) / len(field_metrics['accuracy_file']) if field_metrics['accuracy_file'] else 0 } if verbose: print("Verbose output written to provider_metrics.txt") return results def match_rows(fields, labels_df, predictions_df, N): confusion_matrix = {field_name: {"tp": 0, "fp": 0} for field_name in fields} # Check for completely blank fields and handle them separately for field in fields: labels_all_blank = string_utils.is_empty(labels_df[field], pd_mask=True).all() preds_all_blank = string_utils.is_empty(predictions_df[field], pd_mask=True).all() if labels_all_blank and preds_all_blank: # Perfect agreement on blank field confusion_matrix[field]["tp"] = min(len(labels_df), len(predictions_df)) confusion_matrix[field]["fp"] = 0 confusion_matrix[field]["fn"] = 0 # Skip this field in the regular matching process fields = [f for f in fields if f != field] # Only proceed with matching if there are fields less to process if not fields: return confusion_matrix # Track which label rows have been matched matched_label_indices = set() available_predictions = set(predictions_df.index) for i in range(N): num_columns_to_match = N - i # Only consider unmatched label rows for label_idx, label_row in labels_df.iterrows(): if label_idx in matched_label_indices: continue # Skip already matched label rows best_match_idx = None max_matches = 0 best_matches = {} # Find the best match among available predictions for prediction_idx in available_predictions: prediction_row = predictions_df.loc[prediction_idx] # Count matching fields for this pair field_counter_dict = {field_name: {"tp": 0} for field_name in fields} num_matches = 0 for field in fields: if label_row[field].replace(" ", "") == prediction_row[field].replace(" ", "") or (string_utils.is_empty(label_row[field]) and string_utils.is_empty(prediction_row[field])): num_matches += 1 field_counter_dict[field]["tp"] = 1 if num_matches > max_matches: max_matches = num_matches best_match_idx = prediction_idx best_matches = field_counter_dict # If we found a good enough match if max_matches >= num_columns_to_match and best_match_idx is not None: # Mark this label row as matched matched_label_indices.add(label_idx) # Remove the matched prediction row from available ones available_predictions.remove(best_match_idx) # Update the confusion matrix with the best match for field in fields: confusion_matrix[field]["tp"] += best_matches[field]["tp"] # Calculate FP and FN based on the number of matched fields for field in fields: # FN = number of label rows where this field wasn't matched confusion_matrix[field]['fn'] = len(labels_df) - confusion_matrix[field]['tp'] # FP = number of prediction rows where this field wasn't matched confusion_matrix[field]['fp'] = len(predictions_df) - confusion_matrix[field]['tp'] return confusion_matrix def calculate_precision_recall(accuracies): """ FOR ONE-TO-N FIELDS Calculate precision and recall for each file and overall metrics for all fields. Args: accuracies (list): A list of confusion matrices for each file. Returns: tuple: (precision_df, recall_df, overall_metrics_df) - precision_df: Precision for each field per file. - recall_df: Recall for each field per file. - overall_metrics_df: Overall precision, recall, and F1 score for all fields. """ precision_dicts, recall_dicts = [], [] overall_metrics = {field: {"tp": 0, "fp": 0, "fn": 0} for field in accuracies[0] if field != "Filename"} for confusion_matrix in accuracies: filename = confusion_matrix["Filename"] precision_dict, recall_dict = {"Filename": filename}, {"Filename": filename} for field in confusion_matrix: if field == "Filename": continue tp = confusion_matrix[field]["tp"] fp = confusion_matrix[field]["fp"] fn = confusion_matrix[field]["fn"] # Update overall metrics overall_metrics[field]["tp"] += tp overall_metrics[field]["fp"] += fp overall_metrics[field]["fn"] += fn # Calculate precision and recall precision = tp / (tp + fp) if (tp + fp) > 0 else 0 recall = tp / (tp + fn) if (tp + fn) > 0 else 0 precision_dict[field] = precision recall_dict[field] = recall precision_dicts.append(precision_dict) recall_dicts.append(recall_dict) # Calculate overall precision, recall, and F1 score for all fields overall_metrics_list = [] for field, metrics in overall_metrics.items(): tp = metrics["tp"] fp = metrics["fp"] fn = metrics["fn"] precision = tp / (tp + fp) if (tp + fp) > 0 else 0 recall = tp / (tp + fn) if (tp + fn) > 0 else 0 f1 = (2 * precision * recall) / (precision + recall) if (precision + recall) > 0 else 0 overall_metrics_list.append({ "Field": field, "tp": tp, "fp": fp, "fn": fn, "prec": precision, "rec": recall, "f1": f1 }) # Convert overall metrics to a DataFrame overall_metrics_df = pd.DataFrame(overall_metrics_list).set_index("Field") # Convert precision and recall dictionaries to DataFrames precision_df = pd.DataFrame(precision_dicts) recall_df = pd.DataFrame(recall_dicts) return precision_df, recall_df, overall_metrics_df ########## TESTBED POSTPROCESSING CODE ########## # These functions are to take MCS's test bed that they provided and apply # (most of) the same postprocessing steps that we apply to the results of the model. def group_providers_into_json_array(df): """Groups provider information from a DataFrame into a JSON array format and appends it as a new column. This function processes a DataFrame containing provider information, groups the data by the "FILE_NAME" column, and creates a JSON array for each group. The JSON array includes details about the group provider and other associated providers. The resulting JSON array is added as a new column, "PROV_INFO_JSON", to the DataFrame. Args: df (pandas.DataFrame): The input DataFrame containing the following columns: - "FILE_NAME": Identifier for grouping rows. - "PROV_GROUP_TIN": Tax Identification Number (TIN) of the group provider. - "PROV_GROUP_NPI": National Provider Identifier (NPI) of the group provider. - "PROV_GROUP_NAME_FULL": Full name of the group provider. - "PROV_OTHER_TIN": Pipe-separated TINs of other providers. - "PROV_OTHER_NPI": Pipe-separated NPIs of other providers. - "PROV_OTHER_NAME_FULL": Pipe-separated full names of other providers. Returns: pandas.DataFrame: A modified DataFrame with an additional column "PROV_INFO_JSON", which contains a JSON array for each "FILE_NAME" group. Each JSON array includes: - "TIN": TIN of the provider (or "N/A" if not available). - "NPI": NPI of the provider (or "N/A" if not available). - "NAME": Full name of the provider (or "N/A" if not available). - "IS_GROUP": Boolean indicating whether the provider is the group provider (True) or an associated provider (False). Notes: - Missing or NaN values in the input DataFrame are replaced with "N/A" in the JSON output. - If the lengths of "PROV_OTHER_TIN", "PROV_OTHER_NPI", and "PROV_OTHER_NAME_FULL" are not equal, missing values are filled with "N/A" using itertools.zip_longest. - If the OTHER columns are misaligned with one another, the function will NOT be able to correct them. It will simply fill in the gaps with "N/A". """ df = df.copy() # Create a dictionary to hold provider data in JSON array format for each file group_providers_dict = {} for group, grouped_df in df.groupby("FILE_NAME"): json_dict_list = [] # Get the first row of the grouped DataFrame first_row = grouped_df.iloc[0] # Extract TIN/NPI/NAME for group provider. group_tin = str(first_row["PROV_GROUP_TIN"]).replace("nan", "N/A") group_npi = str(first_row["PROV_GROUP_NPI"]).replace("nan", "N/A") group_name_full = str(first_row["PROV_GROUP_NAME_FULL"]) # Append the group provider's TIN/NPI/NAME to the list. json_dict_list.append({ "TIN": group_tin, "NPI": group_npi, "NAME": group_name_full, "IS_GROUP": True}) # Create lists of TIN/NPI/NAME for other providers. # Split on pipe, then strip whitespace. other_tins = str(first_row["PROV_OTHER_TIN"]).split("|") other_npis = str(first_row["PROV_OTHER_NPI"]).split("|") other_names = str(first_row["PROV_OTHER_NAME_FULL"]).split("|") other_tins = [tin.strip().replace("nan", "N/A") for tin in other_tins] other_npis = [npi.strip().replace("nan", "N/A") for npi in other_npis] other_names = [name.strip().replace("nan", "N/A") for name in other_names] # Append each other provider's TIN/NPI/NAME to the list. # Use zip to iterate over the three lists in parallel. # IF the lengths of the lists are not equal, use itertools.zip_longest to fill in the gaps with N/A. for tin, npi, name in itertools.zip_longest(other_tins, other_npis, other_names, fillvalue="N/A"): json_dict_list.append({ "TIN": tin, "NPI": npi, "NAME": name, "IS_GROUP": False }) group_providers_dict[group] = json_dict_list # Add JSON data back to original DataFrame df["PROV_INFO_JSON"] = df["FILE_NAME"].map(group_providers_dict) ## Reorder columns to place PROV_INFO_JSON right after PROV_OTHER_NAME_FULL cols = list(df.columns) # Find the position of PROV_OTHER_NAME_FULL prov_other_name_full_pos = cols.index("PROV_OTHER_NAME_FULL") # Remove PROV_INFO_JSON from its current position cols.remove("PROV_INFO_JSON") # Insert it after PROV_OTHER_NAME_FULL cols.insert(prov_other_name_full_pos + 1, "PROV_INFO_JSON") # Reorder the DataFrame df = df[cols] return df def testbed_postprocess(df): """Postprocess the testbed DataFrame. Args: df (pandas.DataFrame): The input DataFrame containing the testbed data. Returns: pandas.DataFrame: The postprocessed DataFrame. """ # Apply preprocessing steps df = testbed_preprocess(df) # Group providers into JSON array format df = group_providers_into_json_array(df) # Add CLIENT_NAME column df['CLIENT_NAME'] = "AARETE_TESTBED" # Reformat rate fields if "REIMB_FEE_RATE" in df.columns: # Format rate fields df['REIMB_FEE_RATE'] = df['REIMB_FEE_RATE'].apply(investment_postprocessing_funcs.format_rate_fields_with_commas) if "REIMB_PCT_RATE" in df.columns: df['REIMB_PCT_RATE'] = df['REIMB_PCT_RATE'].apply(investment_postprocessing_funcs.format_rate_fields_with_commas) for col in df.columns: if "_IND" in col: df[col] = df[col].apply(investment_postprocessing_funcs.normalize_indicator_field) # Normalize _IND fields and clean up TIN/NPI fields if "TIN" in col or "NPI" in col: df[col] = df[col].apply(investment_postprocessing_funcs.remove_hyphens) # Apply the flatten_singleton_string_list function if "_CD" in col and "CPT" not in col: df[col] = df[col].apply(investment_postprocessing_funcs.flatten_singleton_string_list) # Convert DATE to YYYY/MM/DD format if "_DT" in col or "DATE" in col: df[col] = df[col].apply(investment_postprocessing_funcs.validate_and_reformat_date) # Normalize CPT fields if "CPT" in col: df[col] = df[col].apply(investment_postprocessing_funcs.normalize_cpt_fields) df = investment_postprocessing_funcs.update_reimb_prov_name(df) # Add reimb_id df = investment_postprocessing_funcs.generate_reimb_ids(df) # Standardize output column order - this should ALWAYS be the final postprocessing step df = investment_postprocessing_funcs.reorder_columns(df, COLUMN_ORDER) return df