jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in bugfix/fix_nonunique_one_to_ones_and_strip_columns (pull request #595)

Browse Source 
Fix data quality issues in testbed preprocessing and enhance error reporting

* Fix data quality issues in testbed preprocessing and enhance error reporting

* Merge remote-tracking branch 'origin/main' into bugfix/fix_nonunique_one_to_ones_and_strip_columns


Approved-by: Siddhant Medar
This commit is contained in:
Alex Galarce
2025-06-30 20:39:14 +00:00
parent 737050b75c
commit 2abf33b20d
1 changed files with 28 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fieldExtraction/src/testbed/testbed_utils.py
+28 -4
View File
@@ -48,7 +48,9 @@ def normalize_tin_npi(value):
return value
def testbed_preprocess(df):
# Fix column names with leading/trailing spaces
df.columns = df.columns.str.strip()
# Capitalize and remove ".TXT" suffixes from FILE_NAME
df['FILE_NAME'] = df['FILE_NAME'].str.upper().str.replace(".TXT", "", regex=False)
@@ -1161,9 +1163,31 @@ def get_one_to_one_analysis(testbed, results):
# Add to comparison DataFrame
try:
comparison_df[field] = merged.set_index('FILE_NAME')[field]
except:
print(merged['FILE_NAME'].value_counts())
raise
except Exception as e:
print(f"\n❌ ERROR: Field '{field}' has inconsistent values for the same FILE_NAME",
f"This indicates a data quality issue in your testbed - one-to-one fields should have",
f"the same value for all rows of the same file.\n")
# Check for problematic files in the original data before any deduplication
problem_files = []
for file_name in testbed_one_to_one['FILE_NAME'].unique():
file_data = testbed_one_to_one[testbed_one_to_one['FILE_NAME'] == file_name]
unique_values = file_data[field].unique()
if len(unique_values) > 1:
problem_files.append((file_name, unique_values.tolist()))
if problem_files:
print("Problematic file(s) and ALL their conflicting values:")
for file_name, values in problem_files:
print(f"\nFile: {file_name}")
print(f" All '{field}' values found: {values}")
print(f" (Should all be the same value)")
print(f"\n💡 Fix: Ensure all rows for the same file have identical '{field}' values in your testbed.")
print(f"Original pandas error: {e}")
raise ValueError(f"Data quality error: Field '{field}' has inconsistent values for FILE_NAME. "
f"Please fix the testbed data (or the results data) before proceeding.")
# Calculate metrics
precision, recall, accuracy, FN_list = calculate_field_metrics(merged, field)