diff --git a/fieldExtraction/src/testbed/testbed_utils.py b/fieldExtraction/src/testbed/testbed_utils.py index 81645bf..6d3de27 100644 --- a/fieldExtraction/src/testbed/testbed_utils.py +++ b/fieldExtraction/src/testbed/testbed_utils.py @@ -2,6 +2,7 @@ import pandas as pd import src.utils.string_utils as string_utils import itertools +import re from collections import Counter from rapidfuzz import fuzz @@ -497,7 +498,7 @@ def match_rows(fields, labels_df, predictions_df, N): return confusion_matrix -def compare_term_extraction_results(file_name: str, predictions_df: pd.DataFrame, gt_df: pd.DataFrame, comparison_columns: list[str] = ['SERVICE_TERM', 'REIMB_TERM']): +def compare_term_extraction_results(file_name: str, predictions_df: pd.DataFrame, gt_df: pd.DataFrame, comparison_columns: list[str] = ['SERVICE_TERM', 'REIMB_TERM'], text_threshold=65): """Compare term extraction results between predictions and ground truth. This function was originally written to evaluate the results of the Methodology Primary process, which extracts `SERVICE_TERM` and `REIMB_TERM` from the text. @@ -510,6 +511,7 @@ def compare_term_extraction_results(file_name: str, predictions_df: pd.DataFrame predictions_df (pd.DataFrame): The dataframe containing the predictions. gt_df (pd.DataFrame): The dataframe containing the ground truth labels. comparison_columns (list, optional): Columns to compare. Defaults to ['SERVICE_TERM', 'REIMB_TERM']. + text_threshold (int, optional): The minimum similarity score (0-100) for text parts to be considered matching. Defaults to 65 (especially for service term which can be short). Returns: dict: A dictionary containing comparison metrics and sets including: @@ -521,38 +523,143 @@ def compare_term_extraction_results(file_name: str, predictions_df: pd.DataFrame - false_positives: Set of pairs in predictions but not in ground truth - error: Error message if columns are missing (only included if error occurs) """ + # Step 1: Filter dataframes to only rows for the current file pred_rows = predictions_df[predictions_df['FILE_NAME'] == file_name].copy() gt_rows = gt_df[gt_df['FILE_NAME'] == file_name].copy() - # Check if required columns exist + # Step 2: Validate that the comparison columns exist in both dataframes for col in comparison_columns: if col not in pred_rows.columns or col not in gt_rows.columns: return {"error": f"Column {col} not found in one of the dataframes."} - # Apply str.lower() to the relevant columns + # Step 3: normalize text by converting to lowercase for case-insensitive comparison pred_rows[comparison_columns] = pred_rows[comparison_columns].apply(lambda x: x.str.lower()) gt_rows[comparison_columns] = gt_rows[comparison_columns].apply(lambda x: x.str.lower()) - # Get unique pairs - pred_unique_set = set(map(tuple, pred_rows[comparison_columns].drop_duplicates().values)) - gt_unique_set = set(map(tuple, gt_rows[comparison_columns].drop_duplicates().values)) + # Step 4: extract unique tuples of the comparison columns from both dataframes + # This removes duplicates and converts to lists for indexed access + pred_tuples = list(set(map(tuple, pred_rows[comparison_columns].drop_duplicates().values))) + gt_tuples = list(set(map(tuple, gt_rows[comparison_columns].drop_duplicates().values))) - common_pairs = pred_unique_set.intersection(gt_unique_set) - false_negatives = gt_unique_set - pred_unique_set - false_positives = pred_unique_set - gt_unique_set + # Step 5: Initialize tracking sets for the matching algorithm + matched_pred = set() # Indices of prediction tuples that have been matched + matched_gt = set() # Indices of ground truth tuples that have been matched + common_pairs = set() # Indices that were successfully matched - # Return structured data + # Step 6: Main matching loop - find best fuzzy match for each prediction + for i, pred_tuple in enumerate(pred_tuples): + # Initialize variables to track best match for this prediction + best_match= None # The GT tuple that matches best + best_score = 0 # The similarity score of the best match + best_gt_idx = None # The index of the best matching GT tuple + + # Step 7: Compare this prediction against all unmatched ground truth tuples + for j, gt_tuple in enumerate(gt_tuples): + if j in matched_gt: # Skip already matched ground truth tuples + continue + + # Step 8: Check if all columns in the tuple pair match using fuzzy logic + total_score = 0 # Sum of similarity scores across all columns in the tuple + all_match = True # Flag to track if all columns pass the threshold + + # Compare each column value in the tuple pair + for pred_val, gt_val in zip(pred_tuple, gt_tuple): + # Use fuzzy matching that treats numbers exactly but text fuzzily + is_match, score = fuzzy_match_except_numbers(pred_val, gt_val, text_threshold=text_threshold) + if not is_match: + all_match = False # If any column fails, the whole tuple fails + break + total_score += score # Accumulates scores for successful matches + + # Step 9: If all columns match, check if this is the best match so far + if all_match: + avg_score = total_score / len(pred_tuple) # Calculate average similarity + if avg_score > best_score: + best_score = avg_score + best_match = gt_tuple + best_gt_idx = j + + # Step 10: if we found a valid match, record it and prevent re-matching + if best_match is not None: + matched_pred.add(i) + matched_gt.add(best_gt_idx) + common_pairs.add(pred_tuple) + + # Step 11: calculate false negatives (ground truth tuples not matched) and false positives (predicted tuples not matched) + false_negatives = set() + false_positives = set() + + for j, gt_tuple in enumerate(gt_tuples): + if j not in matched_gt: + false_negatives.add(gt_tuple) + + for i, pred_tuple in enumerate(pred_tuples): + if i not in matched_pred: + false_positives.add(pred_tuple) + + # Step 12: Return the results as a dictionary return { "file_name": file_name, - "pred_count": len(pred_unique_set), - "testbed_count": len(gt_unique_set), + "pred_count": len(pred_tuples), + "testbed_count": len(gt_tuples), + "matched_count": len(common_pairs), + "false_negatives_count": len(false_negatives), + "false_positives_count": len(false_positives), + + # Convert tuples to readable strings for Excel + "common_pairs_readable": "\n".join([" | ".join(map(str, pair)) for pair in common_pairs]), + "false_negatives_readable": "\n".join([" | ".join(map(str, pair)) for pair in false_negatives]), + "false_positives_readable": "\n".join([" | ".join(map(str, pair)) for pair in false_positives]), + + # Keep original tuples for programmatic use if needed "common_pairs": common_pairs, "false_negatives": false_negatives, "false_positives": false_positives, - "all_predicted_pairs": pred_unique_set, - "all_testbed_pairs": gt_unique_set, + "all_predicted_pairs": set(pred_tuples), + "all_testbed_pairs": set(gt_tuples), } +def fuzzy_match_except_numbers(str1, str2, text_threshold=80): + """ + Compare two strings with fuzzy matching for text but exact matching for numbers. + + Parameters: + - str1, str2: Strings to compare + - text_threshold: Minimum similarity score (0-100) for text parts to be considered matching + + Returns: + - Boolean: True if strings match according to criteria, False otherwise + - Float: Overall similarity score + """ + if not str1 or not str2: + return str1 == str2, 0 if str1 != str2 else 100 + + # Extract numbers separately + numbers1 = re.findall(r'-?\d+\.?\d*', str1) # Handles decimals and negatives + numbers2 = re.findall(r'-?\d+\.?\d*', str2) + + # If numbers don't match exactly, return False + if numbers1 != numbers2: + return False, 0 + + # Remove numbers and clean text for fuzzy comparison + text1 = re.sub(r'-?\d+\.?\d*', '', str1).strip() + text2 = re.sub(r'-?\d+\.?\d*', '', str2).strip() + + # Remove extra punctuation and normalize whitespace + text1 = re.sub(r'[^\w\s]', ' ', text1) + text2 = re.sub(r'[^\w\s]', ' ', text2) + text1 = ' '.join(text1.split()) # Normalize whitespace + text2 = ' '.join(text2.split()) + + # If no text remains after cleaning, check if both are empty + if not text1 and not text2: + return True, 100.0 + + # Use fuzzy matching for text parts + similarity = fuzz.ratio(text1.lower(), text2.lower()) + return similarity >= text_threshold, similarity + def calculate_precision_recall(accuracies): """ FOR ONE-TO-N FIELDS Calculate precision and recall for each file and overall metrics for all fields.