From 2abf33b20db30a4ad793f7a4d0596141ae581ede Mon Sep 17 00:00:00 2001 From: Alex Galarce Date: Mon, 30 Jun 2025 20:39:14 +0000 Subject: [PATCH] Merged in bugfix/fix_nonunique_one_to_ones_and_strip_columns (pull request #595) Fix data quality issues in testbed preprocessing and enhance error reporting * Fix data quality issues in testbed preprocessing and enhance error reporting * Merge remote-tracking branch 'origin/main' into bugfix/fix_nonunique_one_to_ones_and_strip_columns Approved-by: Siddhant Medar --- fieldExtraction/src/testbed/testbed_utils.py | 32 +++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/fieldExtraction/src/testbed/testbed_utils.py b/fieldExtraction/src/testbed/testbed_utils.py index f22c919..abff719 100644 --- a/fieldExtraction/src/testbed/testbed_utils.py +++ b/fieldExtraction/src/testbed/testbed_utils.py @@ -48,7 +48,9 @@ def normalize_tin_npi(value): return value def testbed_preprocess(df): - + # Fix column names with leading/trailing spaces + df.columns = df.columns.str.strip() + # Capitalize and remove ".TXT" suffixes from FILE_NAME df['FILE_NAME'] = df['FILE_NAME'].str.upper().str.replace(".TXT", "", regex=False) @@ -1161,9 +1163,31 @@ def get_one_to_one_analysis(testbed, results): # Add to comparison DataFrame try: comparison_df[field] = merged.set_index('FILE_NAME')[field] - except: - print(merged['FILE_NAME'].value_counts()) - raise + except Exception as e: + print(f"\n❌ ERROR: Field '{field}' has inconsistent values for the same FILE_NAME", + f"This indicates a data quality issue in your testbed - one-to-one fields should have", + f"the same value for all rows of the same file.\n") + + # Check for problematic files in the original data before any deduplication + problem_files = [] + for file_name in testbed_one_to_one['FILE_NAME'].unique(): + file_data = testbed_one_to_one[testbed_one_to_one['FILE_NAME'] == file_name] + unique_values = file_data[field].unique() + if len(unique_values) > 1: + problem_files.append((file_name, unique_values.tolist())) + + if problem_files: + print("Problematic file(s) and ALL their conflicting values:") + for file_name, values in problem_files: + print(f"\nFile: {file_name}") + print(f" All '{field}' values found: {values}") + print(f" (Should all be the same value)") + + print(f"\n💡 Fix: Ensure all rows for the same file have identical '{field}' values in your testbed.") + print(f"Original pandas error: {e}") + + raise ValueError(f"Data quality error: Field '{field}' has inconsistent values for FILE_NAME. " + f"Please fix the testbed data (or the results data) before proceeding.") # Calculate metrics precision, recall, accuracy, FN_list = calculate_field_metrics(merged, field)