Merged in feature/fuzzy-match-no-numbers (pull request #549)

Feature/fuzzy match no numbers

* Add fuzzy matching function for string comparison and enhance term extraction result evaluation

* Enhance term extraction results evaluation by adding counts and readable strings for matched, false negatives, and false positives

* Adjust text similarity threshold to 65 and improve readability of comparison results in term extraction evaluation

* Merged main into feature/fuzzy-match-no-numbers


Approved-by: Katon Minhas

fieldExtraction/src/testbed/testbed_utils.py

@@ -2,6 +2,7 @@
 import pandas as pd
 import src.utils.string_utils as string_utils
 import itertools
+import re
 
 from collections import Counter
 from rapidfuzz import fuzz
@@ -497,7 +498,7 @@ def match_rows(fields, labels_df, predictions_df, N):
     
     return confusion_matrix
 
-def compare_term_extraction_results(file_name: str, predictions_df: pd.DataFrame, gt_df: pd.DataFrame, comparison_columns: list[str] = ['SERVICE_TERM', 'REIMB_TERM']):
+def compare_term_extraction_results(file_name: str, predictions_df: pd.DataFrame, gt_df: pd.DataFrame, comparison_columns: list[str] = ['SERVICE_TERM', 'REIMB_TERM'], text_threshold=65):
     """Compare term extraction results between predictions and ground truth.  This function
     was originally written to evaluate the results of the Methodology Primary process,
     which extracts `SERVICE_TERM` and `REIMB_TERM` from the text. 
@@ -510,6 +511,7 @@ def compare_term_extraction_results(file_name: str, predictions_df: pd.DataFrame
         predictions_df (pd.DataFrame): The dataframe containing the predictions.
         gt_df (pd.DataFrame): The dataframe containing the ground truth labels.
         comparison_columns (list, optional): Columns to compare. Defaults to ['SERVICE_TERM', 'REIMB_TERM'].
+        text_threshold (int, optional): The minimum similarity score (0-100) for text parts to be considered matching. Defaults to 65 (especially for service term which can be short).
 
     Returns:
         dict: A dictionary containing comparison metrics and sets including:
@@ -521,38 +523,143 @@ def compare_term_extraction_results(file_name: str, predictions_df: pd.DataFrame
             - false_positives: Set of pairs in predictions but not in ground truth
             - error: Error message if columns are missing (only included if error occurs)
     """
+    # Step 1: Filter dataframes to only rows for the current file
     pred_rows = predictions_df[predictions_df['FILE_NAME'] == file_name].copy()
     gt_rows = gt_df[gt_df['FILE_NAME'] == file_name].copy()
 
-    # Check if required columns exist
+    # Step 2: Validate that the comparison columns exist in both dataframes
     for col in comparison_columns:
         if col not in pred_rows.columns or col not in gt_rows.columns:
             return {"error": f"Column {col} not found in one of the dataframes."}
     
-    # Apply str.lower() to the relevant columns
+    # Step 3: normalize text by converting to lowercase for case-insensitive comparison
     pred_rows[comparison_columns] = pred_rows[comparison_columns].apply(lambda x: x.str.lower())
     gt_rows[comparison_columns] = gt_rows[comparison_columns].apply(lambda x: x.str.lower())
 
-    # Get unique pairs
-    pred_unique_set = set(map(tuple, pred_rows[comparison_columns].drop_duplicates().values))
-    gt_unique_set = set(map(tuple, gt_rows[comparison_columns].drop_duplicates().values))
+    # Step 4: extract unique tuples of the comparison columns from both dataframes
+    # This removes duplicates and converts to lists for indexed access
+    pred_tuples = list(set(map(tuple, pred_rows[comparison_columns].drop_duplicates().values)))
+    gt_tuples = list(set(map(tuple, gt_rows[comparison_columns].drop_duplicates().values)))
 
-    common_pairs = pred_unique_set.intersection(gt_unique_set)
-    false_negatives = gt_unique_set - pred_unique_set
-    false_positives = pred_unique_set - gt_unique_set
+    # Step 5: Initialize tracking sets for the matching algorithm
+    matched_pred = set() # Indices of prediction tuples that have been matched
+    matched_gt = set() # Indices of ground truth tuples that have been matched
+    common_pairs = set() # Indices that were successfully matched
 
-    # Return structured data
+    # Step 6: Main matching loop - find best fuzzy match for each prediction
+    for i, pred_tuple in enumerate(pred_tuples):
+        # Initialize variables to track best match for this prediction
+        best_match= None # The GT tuple that matches best
+        best_score = 0 # The similarity score of the best match
+        best_gt_idx = None # The index of the best matching GT tuple
+        
+        # Step 7: Compare this prediction against all unmatched ground truth tuples
+        for j, gt_tuple in enumerate(gt_tuples):
+            if j in matched_gt: # Skip already matched ground truth tuples
+                continue
+
+            # Step 8: Check if all columns in the tuple pair match using fuzzy logic
+            total_score = 0 # Sum of similarity scores across all columns in the tuple
+            all_match = True # Flag to track if all columns pass the threshold
+
+            # Compare each column value in the tuple pair
+            for pred_val, gt_val in zip(pred_tuple, gt_tuple):
+                # Use fuzzy matching that treats numbers exactly but text fuzzily
+                is_match, score = fuzzy_match_except_numbers(pred_val, gt_val, text_threshold=text_threshold)
+                if not is_match:
+                    all_match = False # If any column fails, the whole tuple fails
+                    break
+                total_score += score # Accumulates scores for successful matches
+
+            # Step 9: If all columns match, check if this is the best match so far
+            if all_match:
+                avg_score = total_score / len(pred_tuple) # Calculate average similarity
+                if avg_score > best_score:
+                    best_score = avg_score
+                    best_match = gt_tuple
+                    best_gt_idx = j
+        
+        # Step 10: if we found a valid match, record it and prevent re-matching
+        if best_match is not None:
+            matched_pred.add(i)
+            matched_gt.add(best_gt_idx)
+            common_pairs.add(pred_tuple)
+
+    # Step 11: calculate false negatives (ground truth tuples not matched) and false positives (predicted tuples not matched)
+    false_negatives = set()
+    false_positives = set()
+
+    for j, gt_tuple in enumerate(gt_tuples):
+        if j not in matched_gt:
+            false_negatives.add(gt_tuple)
+
+    for i, pred_tuple in enumerate(pred_tuples):
+        if i not in matched_pred:
+            false_positives.add(pred_tuple)
+
+    # Step 12: Return the results as a dictionary
     return {
         "file_name": file_name,
-        "pred_count": len(pred_unique_set),
-        "testbed_count": len(gt_unique_set),
+        "pred_count": len(pred_tuples),
+        "testbed_count": len(gt_tuples),
+        "matched_count": len(common_pairs),
+        "false_negatives_count": len(false_negatives),
+        "false_positives_count": len(false_positives),
+
+        # Convert tuples to readable strings for Excel
+        "common_pairs_readable": "\n".join([" | ".join(map(str, pair)) for pair in common_pairs]),
+        "false_negatives_readable": "\n".join([" | ".join(map(str, pair)) for pair in false_negatives]),
+        "false_positives_readable": "\n".join([" | ".join(map(str, pair)) for pair in false_positives]),
+
+        # Keep original tuples for programmatic use if needed
         "common_pairs": common_pairs,
         "false_negatives": false_negatives,
         "false_positives": false_positives,
-        "all_predicted_pairs": pred_unique_set,
-        "all_testbed_pairs": gt_unique_set,
+        "all_predicted_pairs": set(pred_tuples),
+        "all_testbed_pairs": set(gt_tuples),
     }
 
+def fuzzy_match_except_numbers(str1, str2, text_threshold=80):
+    """
+    Compare two strings with fuzzy matching for text but exact matching for numbers.
+    
+    Parameters:
+    - str1, str2: Strings to compare
+    - text_threshold: Minimum similarity score (0-100) for text parts to be considered matching
+    
+    Returns:
+    - Boolean: True if strings match according to criteria, False otherwise
+    - Float: Overall similarity score
+    """
+    if not str1 or not str2:
+        return str1 == str2, 0 if str1 != str2 else 100
+    
+    # Extract numbers separately
+    numbers1 = re.findall(r'-?\d+\.?\d*', str1)  # Handles decimals and negatives
+    numbers2 = re.findall(r'-?\d+\.?\d*', str2)
+    
+    # If numbers don't match exactly, return False
+    if numbers1 != numbers2:
+        return False, 0
+    
+    # Remove numbers and clean text for fuzzy comparison
+    text1 = re.sub(r'-?\d+\.?\d*', '', str1).strip()
+    text2 = re.sub(r'-?\d+\.?\d*', '', str2).strip()
+
+    # Remove extra punctuation and normalize whitespace
+    text1 = re.sub(r'[^\w\s]', ' ', text1)
+    text2 = re.sub(r'[^\w\s]', ' ', text2)
+    text1 = ' '.join(text1.split())  # Normalize whitespace
+    text2 = ' '.join(text2.split())
+    
+    # If no text remains after cleaning, check if both are empty
+    if not text1 and not text2:
+        return True, 100.0
+    
+    # Use fuzzy matching for text parts
+    similarity = fuzz.ratio(text1.lower(), text2.lower())
+    return similarity >= text_threshold, similarity
+
 def calculate_precision_recall(accuracies):
     """ FOR ONE-TO-N FIELDS
     Calculate precision and recall for each file and overall metrics for all fields.