Merged in feature/test_files (pull request #578)

Feature/test files

* printing incorrect recall values

* values responsible for low recall added in spreadsheet

* changing match list to unmatch list for breakouts

* recall values added for 1-to-N

* print statements removed

* removed print statements

* Merged main into feature/test_files

* output formatting fix

* Merged main into feature/test_files


Approved-by: Katon Minhas

fieldExtraction/src/testbed/test.py

@@ -58,7 +58,7 @@ one_to_one_columns = ['FILE_NAME'] + [f for f in one_to_one_fields if f in testb
 testbed_one_to_one = testbed[one_to_one_columns].drop_duplicates()
 results_one_to_one = results[one_to_one_columns].drop_duplicates()
 
-metrics_df = pd.DataFrame(columns=['Field', 'Precision', 'Recall', 'Accuracy'])
+metrics_df = pd.DataFrame(columns=['Field', 'Precision', 'Recall', 'Accuracy', 'FN_list'])
 comparison_df = pd.DataFrame(index=testbed_one_to_one['FILE_NAME'].unique())
 
 for field in one_to_one_fields:
@@ -81,10 +81,10 @@ for field in one_to_one_fields:
             raise
         
         # Calculate metrics
-        precision, recall, accuracy = testbed_utils.calculate_field_metrics(merged, field)
-        metrics_df.loc[len(metrics_df)] = [field, precision, recall, accuracy]
+        precision, recall, accuracy, FN_list = testbed_utils.calculate_field_metrics(merged, field)
+        metrics_df.loc[len(metrics_df)] = [field, precision, recall, accuracy, FN_list]
     else:
-        metrics_df.loc[len(metrics_df)] = [field, None, None, None]
+        metrics_df.loc[len(metrics_df)] = [field, None, None, None, None]
         comparison_df[field] = None
 
 # Display results with clean formatting

fieldExtraction/src/testbed/testbed_utils.py

@@ -167,6 +167,10 @@ def calculate_field_metrics(merged, field):
         for pred, truth in zip(merged[f'{field}_pred'], merged[f'{field}_truth'])
     )
 
+    FN_list = [
+        truth for pred, truth in zip(merged[f'{field}_pred'], merged[f'{field}_truth'])
+        if is_negative(pred) and is_positive(truth)
+    ]
     # print(field)
     # print(f"TP: {TP}, FP: {FP}, TN: {TN}, FN: {FN}")
 
@@ -175,7 +179,7 @@ def calculate_field_metrics(merged, field):
     recall = TP / (TP + FN) if (TP + FN) > 0 else 0
     accuracy = (TP + TN) / (TP + FP + TN + FN)
 
-    return precision, recall, accuracy
+    return precision, recall, accuracy, FN_list
 
 
 def evaluate_provider_fields_separately(testbed_df, results_df, verbose=False):
@@ -426,7 +430,7 @@ def match_rows(fields, labels_df, predictions_df, N):
                 
         return False
 
-    confusion_matrix = {field_name: {"tp": 0, "fp": 0} for field_name in fields}
+    confusion_matrix = {field_name: {"tp": 0, "fp": 0, "unmatched_labels": []} for field_name in fields}
 
     # Check for completely blank fields and handle them separately
     for field in fields:
@@ -438,6 +442,7 @@ def match_rows(fields, labels_df, predictions_df, N):
             confusion_matrix[field]["tp"] = min(len(labels_df), len(predictions_df))
             confusion_matrix[field]["fp"] = 0
             confusion_matrix[field]["fn"] = 0
+            confusion_matrix[field]["unmatched_labels"] = []
             # Skip this field in the regular matching process
             fields = [f for f in fields if f != field]
         
@@ -488,6 +493,8 @@ def match_rows(fields, labels_df, predictions_df, N):
                 # Update the confusion matrix with the best match
                 for field in fields:
                     confusion_matrix[field]["tp"] += best_matches[field]["tp"]
+                    if best_matches[field]["tp"] == 0:
+                        confusion_matrix[field]["unmatched_labels"].append((label_idx, label_row[field]))
     
     # Calculate FP and FN based on the number of matched fields
     for field in fields:
@@ -495,6 +502,11 @@ def match_rows(fields, labels_df, predictions_df, N):
         confusion_matrix[field]['fn'] = len(labels_df) - confusion_matrix[field]['tp']
         # FP = number of prediction rows where this field wasn't matched
         confusion_matrix[field]['fp'] = len(predictions_df) - confusion_matrix[field]['tp']
+        # add unmatched labels to the confusion matrix for cases where whole rows were unmatched
+        if list(set(labels_df.index) - set(matched_label_indices)):
+            confusion_matrix[field]["unmatched_labels"].append(("|", "|"))
+            for idx in list(set(labels_df.index) - set(matched_label_indices)):
+                confusion_matrix[field]["unmatched_labels"].append((idx, labels_df.loc[idx, field]))
     
     return confusion_matrix
 
@@ -679,6 +691,7 @@ def calculate_precision_recall(accuracies):
     for confusion_matrix in accuracies:
         filename = confusion_matrix["Filename"]
         precision_dict, recall_dict = {"Filename": filename}, {"Filename": filename}
+        recall_dict_value = {}
 
         for field in confusion_matrix:
             if field == "Filename":
@@ -699,9 +712,10 @@ def calculate_precision_recall(accuracies):
 
             precision_dict[field] = precision
             recall_dict[field] = recall
+            recall_dict_value[field+"_missed"] = confusion_matrix[field]["unmatched_labels"]
 
         precision_dicts.append(precision_dict)
-        recall_dicts.append(recall_dict)
+        recall_dicts.append(recall_dict | recall_dict_value)
 
     # Calculate overall precision, recall, and F1 score for all fields
     overall_metrics_list = []