diff --git a/fieldExtraction/src/testbed/test.py b/fieldExtraction/src/testbed/test.py index 8c00554..4de5f43 100644 --- a/fieldExtraction/src/testbed/test.py +++ b/fieldExtraction/src/testbed/test.py @@ -58,7 +58,7 @@ one_to_one_columns = ['FILE_NAME'] + [f for f in one_to_one_fields if f in testb testbed_one_to_one = testbed[one_to_one_columns].drop_duplicates() results_one_to_one = results[one_to_one_columns].drop_duplicates() -metrics_df = pd.DataFrame(columns=['Field', 'Precision', 'Recall', 'Accuracy']) +metrics_df = pd.DataFrame(columns=['Field', 'Precision', 'Recall', 'Accuracy', 'FN_list']) comparison_df = pd.DataFrame(index=testbed_one_to_one['FILE_NAME'].unique()) for field in one_to_one_fields: @@ -81,10 +81,10 @@ for field in one_to_one_fields: raise # Calculate metrics - precision, recall, accuracy = testbed_utils.calculate_field_metrics(merged, field) - metrics_df.loc[len(metrics_df)] = [field, precision, recall, accuracy] + precision, recall, accuracy, FN_list = testbed_utils.calculate_field_metrics(merged, field) + metrics_df.loc[len(metrics_df)] = [field, precision, recall, accuracy, FN_list] else: - metrics_df.loc[len(metrics_df)] = [field, None, None, None] + metrics_df.loc[len(metrics_df)] = [field, None, None, None, None] comparison_df[field] = None # Display results with clean formatting diff --git a/fieldExtraction/src/testbed/testbed_utils.py b/fieldExtraction/src/testbed/testbed_utils.py index 6d3de27..8436eea 100644 --- a/fieldExtraction/src/testbed/testbed_utils.py +++ b/fieldExtraction/src/testbed/testbed_utils.py @@ -167,6 +167,10 @@ def calculate_field_metrics(merged, field): for pred, truth in zip(merged[f'{field}_pred'], merged[f'{field}_truth']) ) + FN_list = [ + truth for pred, truth in zip(merged[f'{field}_pred'], merged[f'{field}_truth']) + if is_negative(pred) and is_positive(truth) + ] # print(field) # print(f"TP: {TP}, FP: {FP}, TN: {TN}, FN: {FN}") @@ -175,7 +179,7 @@ def calculate_field_metrics(merged, field): recall = TP / (TP + FN) if (TP + FN) > 0 else 0 accuracy = (TP + TN) / (TP + FP + TN + FN) - return precision, recall, accuracy + return precision, recall, accuracy, FN_list def evaluate_provider_fields_separately(testbed_df, results_df, verbose=False): @@ -426,7 +430,7 @@ def match_rows(fields, labels_df, predictions_df, N): return False - confusion_matrix = {field_name: {"tp": 0, "fp": 0} for field_name in fields} + confusion_matrix = {field_name: {"tp": 0, "fp": 0, "unmatched_labels": []} for field_name in fields} # Check for completely blank fields and handle them separately for field in fields: @@ -438,6 +442,7 @@ def match_rows(fields, labels_df, predictions_df, N): confusion_matrix[field]["tp"] = min(len(labels_df), len(predictions_df)) confusion_matrix[field]["fp"] = 0 confusion_matrix[field]["fn"] = 0 + confusion_matrix[field]["unmatched_labels"] = [] # Skip this field in the regular matching process fields = [f for f in fields if f != field] @@ -488,6 +493,8 @@ def match_rows(fields, labels_df, predictions_df, N): # Update the confusion matrix with the best match for field in fields: confusion_matrix[field]["tp"] += best_matches[field]["tp"] + if best_matches[field]["tp"] == 0: + confusion_matrix[field]["unmatched_labels"].append((label_idx, label_row[field])) # Calculate FP and FN based on the number of matched fields for field in fields: @@ -495,6 +502,11 @@ def match_rows(fields, labels_df, predictions_df, N): confusion_matrix[field]['fn'] = len(labels_df) - confusion_matrix[field]['tp'] # FP = number of prediction rows where this field wasn't matched confusion_matrix[field]['fp'] = len(predictions_df) - confusion_matrix[field]['tp'] + # add unmatched labels to the confusion matrix for cases where whole rows were unmatched + if list(set(labels_df.index) - set(matched_label_indices)): + confusion_matrix[field]["unmatched_labels"].append(("|", "|")) + for idx in list(set(labels_df.index) - set(matched_label_indices)): + confusion_matrix[field]["unmatched_labels"].append((idx, labels_df.loc[idx, field])) return confusion_matrix @@ -679,6 +691,7 @@ def calculate_precision_recall(accuracies): for confusion_matrix in accuracies: filename = confusion_matrix["Filename"] precision_dict, recall_dict = {"Filename": filename}, {"Filename": filename} + recall_dict_value = {} for field in confusion_matrix: if field == "Filename": @@ -699,9 +712,10 @@ def calculate_precision_recall(accuracies): precision_dict[field] = precision recall_dict[field] = recall + recall_dict_value[field+"_missed"] = confusion_matrix[field]["unmatched_labels"] precision_dicts.append(precision_dict) - recall_dicts.append(recall_dict) + recall_dicts.append(recall_dict | recall_dict_value) # Calculate overall precision, recall, and F1 score for all fields overall_metrics_list = []