Merged in feature/testbed-updates (pull request #589)
Feature/testbed updates * Begin refactor * Update print * finalize testbed script Approved-by: Alex Galarce
This commit is contained in:
Katon Minhas
@@ -19,376 +19,45 @@ results = pd.read_csv("Doczy-Results.csv") # This is the doczy output
|
||||
testbed = testbed_utils.testbed_preprocess(testbed)
|
||||
results = testbed_utils.testbed_preprocess(results)
|
||||
|
||||
# Ensure FILE_NAME values match between testbed and results
|
||||
common_file_names = set(testbed['FILE_NAME']).intersection(set(results['FILE_NAME']))
|
||||
print(f"common_file_names: {len(common_file_names)} items")
|
||||
if "86-0898663-Founder Health, LLC dba Preferred Homecare-ICMProviderAgreement_120661".upper() in common_file_names:
|
||||
print("found 86-0898663-Founder Health, LLC dba Preferred Homecare-ICMProviderAgreement_120661 in both testbed and results; removing it from both datasets")
|
||||
common_file_names.remove("86-0898663-Founder Health, LLC dba Preferred Homecare-ICMProviderAgreement_120661".upper())
|
||||
print(f"common_file_names now has {len(common_file_names)} items")
|
||||
testbed = testbed[testbed['FILE_NAME'].isin(common_file_names)]
|
||||
results = results[results['FILE_NAME'].isin(common_file_names)]
|
||||
# Filter out filenames we don't want to include
|
||||
testbed, results = testbed_utils.filter_file_names(testbed, results)
|
||||
|
||||
########################## Analyze Testbed ##########################
|
||||
print("*************** Test Bed Stats ****************")
|
||||
print(f"{len(testbed['FILE_NAME'].unique())} contracts")
|
||||
print(f"{len(testbed.columns)} fields")
|
||||
|
||||
print(f"{(testbed == '').all().sum()} fields not populated:") # Count of completely empty fields
|
||||
print(f"{testbed.columns[(testbed == '').all()].tolist()}") # List of completely empty fields
|
||||
testbed_utils.get_df_stats(testbed)
|
||||
|
||||
########################## Row Comparison ##########################
|
||||
row_comparison = pd.DataFrame(index=testbed['FILE_NAME'].unique())
|
||||
row_comparison['testbed'] = testbed.groupby('FILE_NAME').size()
|
||||
row_comparison['results'] = results.groupby('FILE_NAME').size()
|
||||
|
||||
print("\n*************** Row Count Comparison ****************")
|
||||
print(f"Files with different row counts: {(row_comparison['testbed'] != row_comparison['results']).sum()}")
|
||||
row_comparison = testbed_utils.get_row_comparison(testbed, results)
|
||||
|
||||
########################## Page-Level Row Count Comparison ##########################
|
||||
print("\n*************** Page-Level Row Count Comparison ****************")
|
||||
def get_consecutive_page_groups(pages) -> list:
|
||||
"""Group consecutive page numbers together.
|
||||
e.g., [4, 5, 6, 8, 9] -> [[4, 5, 6], [8, 9]]
|
||||
|
||||
"""
|
||||
|
||||
if not pages:
|
||||
return []
|
||||
|
||||
# Convert to sorted list of integers
|
||||
page_nums = sorted([int(p) for p in pages if str(p).isdigit()])
|
||||
|
||||
if not page_nums:
|
||||
return []
|
||||
|
||||
groups = []
|
||||
current_group = [page_nums[0]]
|
||||
|
||||
for i in range(1, len(page_nums)):
|
||||
if page_nums[i] == page_nums[i - 1] + 1: # consecutive pages
|
||||
current_group.append(page_nums[i])
|
||||
else:
|
||||
groups.append(current_group)
|
||||
current_group = [page_nums[i]]
|
||||
groups.append(current_group) # Append the last group
|
||||
return groups
|
||||
|
||||
def group_name(group):
|
||||
"""Convert a group of consecutive pages to a readable name.
|
||||
E.g., [4, 5, 6] -> "4-6", [8, 9] -> "8-9", [13] -> "13"
|
||||
"""
|
||||
if len(group) == 1:
|
||||
return str(group[0])
|
||||
else:
|
||||
return f"{group[0]}-{group[-1]}"
|
||||
|
||||
def normalize_page(page_str):
|
||||
"""Normalize page numbers to integer strings for comparison (e.g., "4.0" -> "4", "4" -> "4", "13.0.0" -> "13")"""
|
||||
if pd.isna(page_str) or page_str == "":
|
||||
return None
|
||||
try:
|
||||
# Split on first period
|
||||
return str(page_str).strip().split(".")[0]
|
||||
except (ValueError, TypeError):
|
||||
return str(page_str).strip()
|
||||
|
||||
if "EXHIBIT_PAGE" in testbed.columns and "EXHIBIT_PAGE" in results.columns:
|
||||
page_mismatches = []
|
||||
for file in testbed['FILE_NAME'].unique():
|
||||
testbed_file = testbed[testbed['FILE_NAME'] == file].copy()
|
||||
results_file = results[results['FILE_NAME'] == file].copy()
|
||||
|
||||
# Normalize page numbers
|
||||
testbed_file['EXHIBIT_PAGE_NORM'] = testbed_file['EXHIBIT_PAGE'].apply(normalize_page)
|
||||
results_file['EXHIBIT_PAGE_NORM'] = results_file['EXHIBIT_PAGE'].apply(normalize_page)
|
||||
|
||||
# Get unique pages from each dataset (excluding None/empty)
|
||||
testbed_pages = set(testbed_file['EXHIBIT_PAGE_NORM'].dropna())
|
||||
results_pages = set(results_file['EXHIBIT_PAGE_NORM'].dropna())
|
||||
testbed_pages.discard("") # Remove empty strings
|
||||
results_pages.discard("") # Remove empty strings
|
||||
|
||||
# Group consecutive pages
|
||||
testbed_groups = get_consecutive_page_groups(testbed_pages)
|
||||
results_groups = get_consecutive_page_groups(results_pages)
|
||||
|
||||
# Count rows for each group
|
||||
testbed_group_counts = {}
|
||||
|
||||
for group in testbed_groups:
|
||||
group_key = group_name(group)
|
||||
count = sum(testbed_file['EXHIBIT_PAGE_NORM'].isin([str(p) for p in group]))
|
||||
testbed_group_counts[group_key] = count
|
||||
|
||||
results_group_counts = {}
|
||||
|
||||
for group in results_groups:
|
||||
group_key = group_name(group)
|
||||
count = sum(results_file['EXHIBIT_PAGE_NORM'].isin([str(p) for p in group]))
|
||||
results_group_counts[group_key] = count
|
||||
|
||||
# Compare group counts
|
||||
all_groups = set(testbed_group_counts.keys()) | set(results_group_counts.keys())
|
||||
|
||||
group_comparison = {}
|
||||
has_mismatch = False
|
||||
|
||||
for group_key in all_groups:
|
||||
testbed_count = testbed_group_counts.get(group_key, 0)
|
||||
results_count = results_group_counts.get(group_key, 0)
|
||||
|
||||
group_comparison[group_key] = {
|
||||
'testbed_count': testbed_count,
|
||||
'results_count': results_count,
|
||||
'difference': results_count - testbed_count
|
||||
}
|
||||
|
||||
if testbed_count != results_count:
|
||||
has_mismatch = True
|
||||
|
||||
if has_mismatch:
|
||||
page_mismatches.append({
|
||||
'FILE_NAME': file,
|
||||
'group_comparison': group_comparison,
|
||||
'total_testbed_rows': len(testbed_file),
|
||||
'total_results_rows': len(results_file)
|
||||
})
|
||||
|
||||
|
||||
if page_mismatches:
|
||||
# Create detailed comparison DataFrame
|
||||
detailed_page_comparison = []
|
||||
for file_data in page_mismatches:
|
||||
file_name = file_data['FILE_NAME']
|
||||
for group_key, counts in file_data['group_comparison'].items():
|
||||
detailed_page_comparison.append({
|
||||
'FILE_NAME': file_name,
|
||||
'PAGE_GROUP': group_key,
|
||||
'TESTBED_COUNT': counts['testbed_count'],
|
||||
'RESULTS_COUNT': counts['results_count'],
|
||||
'DIFFERENCE': counts['difference']
|
||||
})
|
||||
page_comparison_df = pd.DataFrame(detailed_page_comparison)
|
||||
|
||||
# Sort by file name and first page number in group
|
||||
def sort_key(group_str):
|
||||
"""Extract the first page number from a group string for sorting."""
|
||||
parts = group_str.split('-')
|
||||
return int(parts[0]) if parts else float('inf')
|
||||
page_comparison_df['SORT_KEY'] = page_comparison_df['PAGE_GROUP'].apply(sort_key)
|
||||
page_comparison_df = page_comparison_df.sort_values(by=['FILE_NAME', 'SORT_KEY']).drop(columns=['SORT_KEY'])
|
||||
|
||||
# Summary statistics
|
||||
total_missing_rows = page_comparison_df[page_comparison_df['DIFFERENCE'] < 0]['DIFFERENCE'].sum()
|
||||
total_extra_rows = page_comparison_df[page_comparison_df['DIFFERENCE'] > 0]['DIFFERENCE'].sum()
|
||||
pages_with_missing = (page_comparison_df['DIFFERENCE'] < 0).sum()
|
||||
pages_with_extra = (page_comparison_df['DIFFERENCE'] > 0).sum()
|
||||
|
||||
print(f"Pages with missing rows in results: {pages_with_missing}")
|
||||
print(f"Pages with extra rows in results: {pages_with_extra}")
|
||||
print(f"Total missing rows: {abs(total_missing_rows)}")
|
||||
print(f"Total extra rows: {total_extra_rows}")
|
||||
|
||||
|
||||
else:
|
||||
print("All files have matching row counts for consecutive page groups!")
|
||||
page_comparison_df = pd.DataFrame()
|
||||
else:
|
||||
print("EXHIBIT_PAGE column not found in either testbed or results. Skipping page-level row count comparison.")
|
||||
page_comparison_df = pd.DataFrame()
|
||||
page_comparison_df = testbed_utils.get_page_comparison_df(testbed, results)
|
||||
|
||||
########################## One-to-One Analysis ##########################
|
||||
print("\n*************** One-to-One Stats ****************")
|
||||
one_to_one_fields = FieldSet(file_path=config.FIELD_JSON_PATH).filter(relationship="one_to_one").list_fields()
|
||||
one_to_one_fields.extend(['PROV_GROUP_TIN',
|
||||
'PROV_GROUP_NPI',
|
||||
'PROV_GROUP_NAME_FULL'])
|
||||
one_to_one_fields = [f for f in one_to_one_fields if f not in {'TIN', 'NPI', 'NAME', "CLIENT_NAME"}]
|
||||
|
||||
# Create filtered dataframes with only one-to-one fields
|
||||
one_to_one_columns = ['FILE_NAME'] + [f for f in one_to_one_fields if f in testbed.columns and f in results.columns]
|
||||
testbed_one_to_one = testbed[one_to_one_columns].drop_duplicates()
|
||||
results_one_to_one = results[one_to_one_columns].drop_duplicates()
|
||||
|
||||
metrics_df = pd.DataFrame(columns=['Field', 'Precision', 'Recall', 'Accuracy', 'FN_list'])
|
||||
comparison_df = pd.DataFrame(index=testbed_one_to_one['FILE_NAME'].unique())
|
||||
|
||||
for field in one_to_one_fields:
|
||||
if field in testbed.columns and field in results.columns and field != 'FILE_NAME':
|
||||
# Merge and compare values (no need to drop_duplicates here since we already did)
|
||||
merged = pd.merge(testbed_one_to_one[['FILE_NAME', field]],
|
||||
results_one_to_one[['FILE_NAME', field]],
|
||||
on='FILE_NAME',
|
||||
suffixes=('_truth', '_pred'))
|
||||
# Compare values
|
||||
merged[field] = (merged[f"{field}_truth"] == merged[f"{field}_pred"]) & (merged[f"{field}_truth"] != '')
|
||||
# Drop duplicate rows
|
||||
merged = merged.drop_duplicates(subset=['FILE_NAME', field])
|
||||
|
||||
# Add to comparison DataFrame
|
||||
try:
|
||||
comparison_df[field] = merged.set_index('FILE_NAME')[field]
|
||||
except:
|
||||
print(merged['FILE_NAME'].value_counts())
|
||||
raise
|
||||
|
||||
# Calculate metrics
|
||||
precision, recall, accuracy, FN_list = testbed_utils.calculate_field_metrics(merged, field)
|
||||
metrics_df.loc[len(metrics_df)] = [field, precision, recall, accuracy, FN_list]
|
||||
else:
|
||||
metrics_df.loc[len(metrics_df)] = [field, None, None, None, None]
|
||||
comparison_df[field] = None
|
||||
|
||||
# Display results with clean formatting
|
||||
print(metrics_df.to_string(
|
||||
index=False,
|
||||
float_format=lambda x: '{:.2f}'.format(x) if pd.notnull(x) else 'Not found',
|
||||
justify='left',
|
||||
col_space={'Field': 40, 'Precision': 15, 'Recall': 15, 'Accuracy': 15}
|
||||
))
|
||||
metrics_df, comparison_df = testbed_utils.get_one_to_one_analysis(testbed, results)
|
||||
|
||||
########################## One-to-N Analysis ##########################
|
||||
print("\n*************** One-to-N Stats ****************")
|
||||
|
||||
one_to_n_fields = {"methodology_breakout" : [field for field in FieldSet(file_path=config.FIELD_JSON_PATH).filter(field_type="methodology_breakout").list_fields() if field != "GREATER_OF_IND"],
|
||||
"fee_schedule_breakout" : ["AARETE_DERIVED_FEE_SCHEDULE", "AARETE_DERIVED_FEE_SCHEDULE_VERSION"],
|
||||
"dynamic_primary" : ["AARETE_DERIVED_LOB", "AARETE_DERIVED_PROGRAM", "AARETE_DERIVED_NETWORK", "AARETE_DERIVED_PRODUCT"],
|
||||
"reimb_prov_info" : ["REIMB_PROV_TIN", "REIMB_PROV_NPI", "REIMB_PROV_NAME"]
|
||||
}
|
||||
one_to_n_metrics = {}
|
||||
for field_type, fields in one_to_n_fields.items():
|
||||
N = len(fields)
|
||||
accuracies = []
|
||||
|
||||
for file in testbed['FILE_NAME'].unique():
|
||||
testbed_file = testbed[testbed['FILE_NAME'] == file]
|
||||
results_file = results[results['FILE_NAME'] == file]
|
||||
|
||||
# Ensure there are rows in both dataframes
|
||||
if testbed_file.shape[0] == 0 or results_file.shape[0] == 0:
|
||||
print(f"No rows in either labels or predictions for file: {file}")
|
||||
continue
|
||||
|
||||
confusion_matrix = testbed_utils.match_rows(fields, testbed_file, results_file, N)
|
||||
|
||||
confusion_matrix["Filename"] = file
|
||||
accuracies.append(confusion_matrix)
|
||||
|
||||
precision_df, recall_df, overall = testbed_utils.calculate_precision_recall(accuracies)
|
||||
one_to_n_metrics[field_type] = {'precision': precision_df, 'recall': recall_df, 'overall': overall}
|
||||
|
||||
print(f"\nOverall Metrics: {field_type}")
|
||||
print(overall.to_string(
|
||||
index=True,
|
||||
float_format=lambda x: '{:.2f}'.format(x) if pd.notnull(x) else 'Not found',
|
||||
justify='left',
|
||||
col_space={'tp': 15, 'fp': 15, 'fn': 15, 'prec': 15, 'rec': 15, 'f1': 15}
|
||||
))
|
||||
one_to_n_metrics = testbed_utils.get_one_to_n_analysis(testbed, results)
|
||||
|
||||
########################## Reimbursement Primary ##########################
|
||||
# Special handling for reimbursement primary
|
||||
common_files = set(testbed['FILE_NAME']).intersection(set(results['FILE_NAME']))
|
||||
reimb_primary_metric_rows = []
|
||||
for file in common_files:
|
||||
result = testbed_utils.compare_term_extraction_results(file, predictions_df=results, gt_df=testbed)
|
||||
reimb_primary_metric_rows.append(result.copy())
|
||||
|
||||
reimb_primary_df = pd.DataFrame(reimb_primary_metric_rows)
|
||||
|
||||
reimb_primary_df = testbed_utils.get_reimb_primary(testbed, results)
|
||||
|
||||
########################## Order-Independent Provider Analysis ##########################
|
||||
print("\n*************** Order-Independent Provider Analysis ****************")
|
||||
provider_results = testbed_utils.evaluate_provider_fields_separately(
|
||||
testbed,
|
||||
results, verbose=True)
|
||||
|
||||
# Display the results
|
||||
provider_metrics_df = pd.DataFrame([
|
||||
{
|
||||
'Field': f"PROV_OTHER_{field_type}",
|
||||
'Precision': metrics['precision'],
|
||||
'Recall': metrics['recall'],
|
||||
'F1': metrics['f1'],
|
||||
'TP': metrics['tp'],
|
||||
'FP': metrics['fp'],
|
||||
'FN': metrics['fn'],
|
||||
'average_accuracy': metrics['average_accuracy'],
|
||||
}
|
||||
for field_type, metrics in provider_results.items()
|
||||
])
|
||||
|
||||
print(provider_metrics_df.to_string(
|
||||
index=False,
|
||||
float_format=lambda x: '{:.2f}'.format(x) if pd.notnull(x) else 'Not found',
|
||||
justify='left',
|
||||
col_space={'Field': 15, 'Precision': 12, 'Recall': 12, 'F1': 12, 'TP': 8, 'FP': 8, 'FN': 8}
|
||||
))
|
||||
provider_results, provider_metrics_df = testbed_utils.evaluate_provider_fields_separately(testbed, results, verbose=True)
|
||||
|
||||
########################## Export to Excel ##########################
|
||||
print("\nExporting results to Excel...")
|
||||
|
||||
# Create Excel writer object
|
||||
output_file = f"{config.TODAY}-testbed-metrics.xlsx"
|
||||
with pd.ExcelWriter(output_file, engine='openpyxl') as writer:
|
||||
# Test Bed Stats
|
||||
stats_df = pd.DataFrame({
|
||||
'Metric': ['Total Contracts', 'Total Fields', 'Empty Fields'],
|
||||
'Value': [
|
||||
len(testbed['FILE_NAME'].unique()),
|
||||
len(testbed.columns),
|
||||
(testbed == '').all().sum()
|
||||
]
|
||||
})
|
||||
empty_fields_df = pd.DataFrame({'Empty Fields': testbed.columns[(testbed == '').all()].tolist()})
|
||||
|
||||
stats_df.to_excel(writer, sheet_name='Testbed Stats', index=False)
|
||||
row_comparison.to_excel(writer, sheet_name='Row Counts')
|
||||
empty_fields_df.to_excel(writer, sheet_name='Empty Fields', index=False)
|
||||
|
||||
# Add page-level row count comparison if it exists
|
||||
if not page_comparison_df.empty:
|
||||
page_comparison_df.to_excel(writer, sheet_name='Page-Level Row Counts', index=False)
|
||||
else:
|
||||
print("No page-level row count mismatches found; skipping export.")
|
||||
|
||||
# One-to-One Results
|
||||
metrics_df.to_excel(writer, sheet_name='One-to-One Metrics', index=False)
|
||||
comparison_df.to_excel(writer, sheet_name='One-to-One Details') # Add new sheet
|
||||
|
||||
# One-to-N Results
|
||||
for field_type, metrics in one_to_n_metrics.items():
|
||||
precision_df = metrics['precision']
|
||||
recall_df = metrics['recall']
|
||||
overall = metrics['overall']
|
||||
|
||||
# Add the field type to the sheet names
|
||||
precision_df.to_excel(writer, sheet_name=f'{field_type} Precision', index=False)
|
||||
recall_df.to_excel(writer, sheet_name=f'{field_type} Recall', index=False)
|
||||
overall.to_excel(writer, sheet_name=f'{field_type} Overall')
|
||||
|
||||
# Reimbursement Primary Results
|
||||
reimb_primary_df.to_excel(writer, sheet_name='Reimbursement Primary', index=False)
|
||||
|
||||
# Order-independent provider analysis
|
||||
provider_metrics_df.to_excel(writer, sheet_name='Provider Fields', index=False)
|
||||
|
||||
# Auto-adjust column widths
|
||||
for sheet_name in writer.sheets:
|
||||
worksheet = writer.sheets[sheet_name]
|
||||
for column in worksheet.columns:
|
||||
max_length = 0
|
||||
column = [cell for cell in column]
|
||||
for cell in column:
|
||||
try:
|
||||
if len(str(cell.value)) > max_length:
|
||||
max_length = len(str(cell.value))
|
||||
except:
|
||||
pass
|
||||
adjusted_width = (max_length + 2)
|
||||
worksheet.column_dimensions[column[0].column_letter].width = adjusted_width
|
||||
|
||||
print(f"Results exported to {output_file}")
|
||||
testbed_utils.export_to_excel(output_file,
|
||||
testbed,
|
||||
row_comparison,
|
||||
page_comparison_df,
|
||||
metrics_df,
|
||||
comparison_df,
|
||||
one_to_n_metrics,
|
||||
reimb_primary_df,
|
||||
provider_metrics_df)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -10,7 +10,20 @@ from rapidfuzz import fuzz
|
||||
# Imports for the testbed postprocessing functions
|
||||
from src.investment import investment_postprocessing_funcs
|
||||
from src.constants.investment_columns import COLUMN_ORDER
|
||||
from src.prompts.investment_prompts import FieldSet
|
||||
import src.config as config
|
||||
|
||||
def filter_file_names(testbed, results):
|
||||
# Ensure FILE_NAME values match between testbed and results
|
||||
common_file_names = set(testbed['FILE_NAME']).intersection(set(results['FILE_NAME']))
|
||||
print(f"common_file_names: {len(common_file_names)} items")
|
||||
if "86-0898663-Founder Health, LLC dba Preferred Homecare-ICMProviderAgreement_120661".upper() in common_file_names:
|
||||
print("found 86-0898663-Founder Health, LLC dba Preferred Homecare-ICMProviderAgreement_120661 in both testbed and results; removing it from both datasets")
|
||||
common_file_names.remove("86-0898663-Founder Health, LLC dba Preferred Homecare-ICMProviderAgreement_120661".upper())
|
||||
print(f"common_file_names now has {len(common_file_names)} items")
|
||||
testbed = testbed[testbed['FILE_NAME'].isin(common_file_names)]
|
||||
results = results[results['FILE_NAME'].isin(common_file_names)]
|
||||
return testbed, results
|
||||
|
||||
def convert_to_float(value):
|
||||
try:
|
||||
@@ -196,6 +209,7 @@ def calculate_field_metrics(merged, field):
|
||||
|
||||
def evaluate_provider_fields_separately(testbed_df, results_df, verbose=False):
|
||||
"""Evaluates TIN, NPI, and NAME fields separately for each provider."""
|
||||
print("\n*************** Order-Independent Provider Analysis ****************")
|
||||
# Initialize metrics for each field type
|
||||
metrics = {
|
||||
"TIN": {'tp' : 0, 'fp': 0, 'fn': 0, 'accuracy_file': []},
|
||||
@@ -380,8 +394,6 @@ def evaluate_provider_fields_separately(testbed_df, results_df, verbose=False):
|
||||
# Save the DataFrame to a CSV file
|
||||
csv_output_df.to_csv("provider_metrics.csv", index=False)
|
||||
|
||||
|
||||
|
||||
# Calculate accuracy, precision, recall, and F1 for each field type
|
||||
results = {}
|
||||
for field_type, field_metrics in metrics.items():
|
||||
@@ -389,9 +401,6 @@ def evaluate_provider_fields_separately(testbed_df, results_df, verbose=False):
|
||||
recall = field_metrics['tp'] / (field_metrics['tp'] + field_metrics['fn']) if (field_metrics['tp'] + field_metrics['fn']) > 0 else 0
|
||||
f1 = (2 * precision * recall) / (precision + recall) if (precision + recall) > 0 else 0
|
||||
|
||||
|
||||
|
||||
|
||||
results[field_type] = {
|
||||
"precision": precision,
|
||||
"recall": recall,
|
||||
@@ -404,7 +413,30 @@ def evaluate_provider_fields_separately(testbed_df, results_df, verbose=False):
|
||||
|
||||
if verbose:
|
||||
print("Verbose output written to provider_metrics.txt")
|
||||
return results
|
||||
|
||||
# Display the results
|
||||
provider_metrics_df = pd.DataFrame([
|
||||
{
|
||||
'Field': f"PROV_OTHER_{field_type}",
|
||||
'Precision': metrics['precision'],
|
||||
'Recall': metrics['recall'],
|
||||
'F1': metrics['f1'],
|
||||
'TP': metrics['tp'],
|
||||
'FP': metrics['fp'],
|
||||
'FN': metrics['fn'],
|
||||
'average_accuracy': metrics['average_accuracy'],
|
||||
}
|
||||
for field_type, metrics in results.items()
|
||||
])
|
||||
|
||||
print(provider_metrics_df.to_string(
|
||||
index=False,
|
||||
float_format=lambda x: '{:.2f}'.format(x) if pd.notnull(x) else 'Not found',
|
||||
justify='left',
|
||||
col_space={'Field': 15, 'Precision': 12, 'Recall': 12, 'F1': 12, 'TP': 8, 'FP': 8, 'FN': 8}
|
||||
))
|
||||
|
||||
return results, provider_metrics_df
|
||||
|
||||
|
||||
def match_rows(fields, labels_df, predictions_df, N):
|
||||
@@ -911,4 +943,362 @@ def testbed_postprocess(df):
|
||||
# Standardize output column order - this should ALWAYS be the final postprocessing step
|
||||
df = investment_postprocessing_funcs.reorder_columns(df, COLUMN_ORDER)
|
||||
|
||||
return df
|
||||
return df
|
||||
|
||||
def get_df_stats(df):
|
||||
print("*************** Test Bed Stats ****************")
|
||||
print(f"{len(df['FILE_NAME'].unique())} contracts")
|
||||
print(f"{len(df.columns)} fields")
|
||||
|
||||
print(f"{(df == '').all().sum()} fields not populated:") # Count of completely empty fields
|
||||
print(f"{df.columns[(df == '').all()].tolist()}") # List of completely empty fields
|
||||
|
||||
def get_row_comparison(testbed, results):
|
||||
row_comparison = pd.DataFrame(index=testbed['FILE_NAME'].unique())
|
||||
row_comparison['testbed'] = testbed.groupby('FILE_NAME').size()
|
||||
row_comparison['results'] = results.groupby('FILE_NAME').size()
|
||||
|
||||
print("\n*************** Row Count Comparison ****************")
|
||||
print(f"Files with different row counts: {(row_comparison['testbed'] != row_comparison['results']).sum()}")
|
||||
|
||||
return row_comparison
|
||||
|
||||
def get_consecutive_page_groups(pages) -> list:
|
||||
"""Group consecutive page numbers together.
|
||||
e.g., [4, 5, 6, 8, 9] -> [[4, 5, 6], [8, 9]]
|
||||
|
||||
"""
|
||||
|
||||
if not pages:
|
||||
return []
|
||||
|
||||
# Convert to sorted list of integers
|
||||
page_nums = sorted([int(p) for p in pages if str(p).isdigit()])
|
||||
|
||||
if not page_nums:
|
||||
return []
|
||||
|
||||
groups = []
|
||||
current_group = [page_nums[0]]
|
||||
|
||||
for i in range(1, len(page_nums)):
|
||||
if page_nums[i] == page_nums[i - 1] + 1: # consecutive pages
|
||||
current_group.append(page_nums[i])
|
||||
else:
|
||||
groups.append(current_group)
|
||||
current_group = [page_nums[i]]
|
||||
groups.append(current_group) # Append the last group
|
||||
return groups
|
||||
|
||||
def group_name(group):
|
||||
"""Convert a group of consecutive pages to a readable name.
|
||||
E.g., [4, 5, 6] -> "4-6", [8, 9] -> "8-9", [13] -> "13"
|
||||
"""
|
||||
if len(group) == 1:
|
||||
return str(group[0])
|
||||
else:
|
||||
return f"{group[0]}-{group[-1]}"
|
||||
|
||||
def normalize_page(page_str):
|
||||
"""Normalize page numbers to integer strings for comparison (e.g., "4.0" -> "4", "4" -> "4", "13.0.0" -> "13")"""
|
||||
if pd.isna(page_str) or page_str == "":
|
||||
return None
|
||||
try:
|
||||
# Split on first period
|
||||
return str(page_str).strip().split(".")[0]
|
||||
except (ValueError, TypeError):
|
||||
return str(page_str).strip()
|
||||
|
||||
def get_page_comparison_df(testbed, results):
|
||||
|
||||
print("\n*************** Page-Level Row Count Comparison ****************")
|
||||
if "EXHIBIT_PAGE" in testbed.columns and "EXHIBIT_PAGE" in results.columns:
|
||||
page_mismatches = []
|
||||
for file in testbed['FILE_NAME'].unique():
|
||||
testbed_file = testbed[testbed['FILE_NAME'] == file].copy()
|
||||
results_file = results[results['FILE_NAME'] == file].copy()
|
||||
|
||||
# Normalize page numbers
|
||||
testbed_file['EXHIBIT_PAGE_NORM'] = testbed_file['EXHIBIT_PAGE'].apply(normalize_page)
|
||||
results_file['EXHIBIT_PAGE_NORM'] = results_file['EXHIBIT_PAGE'].apply(normalize_page)
|
||||
|
||||
# Get unique pages from each dataset (excluding None/empty)
|
||||
testbed_pages = set(testbed_file['EXHIBIT_PAGE_NORM'].dropna())
|
||||
results_pages = set(results_file['EXHIBIT_PAGE_NORM'].dropna())
|
||||
testbed_pages.discard("") # Remove empty strings
|
||||
results_pages.discard("") # Remove empty strings
|
||||
|
||||
# Group consecutive pages
|
||||
testbed_groups = get_consecutive_page_groups(testbed_pages)
|
||||
results_groups = get_consecutive_page_groups(results_pages)
|
||||
|
||||
# Count rows for each group
|
||||
testbed_group_counts = {}
|
||||
|
||||
for group in testbed_groups:
|
||||
group_key = group_name(group)
|
||||
count = sum(testbed_file['EXHIBIT_PAGE_NORM'].isin([str(p) for p in group]))
|
||||
testbed_group_counts[group_key] = count
|
||||
|
||||
results_group_counts = {}
|
||||
|
||||
for group in results_groups:
|
||||
group_key = group_name(group)
|
||||
count = sum(results_file['EXHIBIT_PAGE_NORM'].isin([str(p) for p in group]))
|
||||
results_group_counts[group_key] = count
|
||||
|
||||
# Compare group counts
|
||||
all_groups = set(testbed_group_counts.keys()) | set(results_group_counts.keys())
|
||||
|
||||
group_comparison = {}
|
||||
has_mismatch = False
|
||||
|
||||
for group_key in all_groups:
|
||||
testbed_count = testbed_group_counts.get(group_key, 0)
|
||||
results_count = results_group_counts.get(group_key, 0)
|
||||
|
||||
group_comparison[group_key] = {
|
||||
'testbed_count': testbed_count,
|
||||
'results_count': results_count,
|
||||
'difference': results_count - testbed_count
|
||||
}
|
||||
|
||||
if testbed_count != results_count:
|
||||
has_mismatch = True
|
||||
|
||||
if has_mismatch:
|
||||
page_mismatches.append({
|
||||
'FILE_NAME': file,
|
||||
'group_comparison': group_comparison,
|
||||
'total_testbed_rows': len(testbed_file),
|
||||
'total_results_rows': len(results_file)
|
||||
})
|
||||
|
||||
|
||||
if page_mismatches:
|
||||
# Create detailed comparison DataFrame
|
||||
detailed_page_comparison = []
|
||||
for file_data in page_mismatches:
|
||||
file_name = file_data['FILE_NAME']
|
||||
for group_key, counts in file_data['group_comparison'].items():
|
||||
detailed_page_comparison.append({
|
||||
'FILE_NAME': file_name,
|
||||
'PAGE_GROUP': group_key,
|
||||
'TESTBED_COUNT': counts['testbed_count'],
|
||||
'RESULTS_COUNT': counts['results_count'],
|
||||
'DIFFERENCE': counts['difference']
|
||||
})
|
||||
page_comparison_df = pd.DataFrame(detailed_page_comparison)
|
||||
|
||||
# Sort by file name and first page number in group
|
||||
def sort_key(group_str):
|
||||
"""Extract the first page number from a group string for sorting."""
|
||||
parts = group_str.split('-')
|
||||
return int(parts[0]) if parts else float('inf')
|
||||
page_comparison_df['SORT_KEY'] = page_comparison_df['PAGE_GROUP'].apply(sort_key)
|
||||
page_comparison_df = page_comparison_df.sort_values(by=['FILE_NAME', 'SORT_KEY']).drop(columns=['SORT_KEY'])
|
||||
|
||||
# Summary statistics
|
||||
total_missing_rows = page_comparison_df[page_comparison_df['DIFFERENCE'] < 0]['DIFFERENCE'].sum()
|
||||
total_extra_rows = page_comparison_df[page_comparison_df['DIFFERENCE'] > 0]['DIFFERENCE'].sum()
|
||||
pages_with_missing = (page_comparison_df['DIFFERENCE'] < 0).sum()
|
||||
pages_with_extra = (page_comparison_df['DIFFERENCE'] > 0).sum()
|
||||
|
||||
print(f"Pages with missing rows in results: {pages_with_missing}")
|
||||
print(f"Pages with extra rows in results: {pages_with_extra}")
|
||||
print(f"Total missing rows: {abs(total_missing_rows)}")
|
||||
print(f"Total extra rows: {total_extra_rows}")
|
||||
|
||||
|
||||
else:
|
||||
print("All files have matching row counts for consecutive page groups!")
|
||||
page_comparison_df = pd.DataFrame()
|
||||
else:
|
||||
print("EXHIBIT_PAGE column not found in either testbed or results. Skipping page-level row count comparison.")
|
||||
page_comparison_df = pd.DataFrame()
|
||||
|
||||
return page_comparison_df
|
||||
|
||||
|
||||
def get_one_to_one_analysis(testbed, results):
|
||||
print("\n*************** One-to-One Stats ****************")
|
||||
one_to_one_fields = FieldSet(file_path=config.FIELD_JSON_PATH).filter(relationship="one_to_one").list_fields()
|
||||
one_to_one_fields.extend(['PROV_GROUP_TIN',
|
||||
'PROV_GROUP_NPI',
|
||||
'PROV_GROUP_NAME_FULL'])
|
||||
one_to_one_fields = [f for f in one_to_one_fields if f not in {'TIN', 'NPI', 'NAME', "CLIENT_NAME"}]
|
||||
|
||||
# Create filtered dataframes with only one-to-one fields
|
||||
one_to_one_columns = ['FILE_NAME'] + [f for f in one_to_one_fields if f in testbed.columns and f in results.columns]
|
||||
testbed_one_to_one = testbed[one_to_one_columns].drop_duplicates()
|
||||
results_one_to_one = results[one_to_one_columns].drop_duplicates()
|
||||
|
||||
metrics_df = pd.DataFrame(columns=['Field', 'Precision', 'Recall', 'Accuracy', 'FN_list'])
|
||||
comparison_df = pd.DataFrame(index=testbed_one_to_one['FILE_NAME'].unique())
|
||||
|
||||
for field in one_to_one_fields:
|
||||
if field in testbed.columns and field in results.columns and field != 'FILE_NAME':
|
||||
# Merge and compare values (no need to drop_duplicates here since we already did)
|
||||
merged = pd.merge(testbed_one_to_one[['FILE_NAME', field]],
|
||||
results_one_to_one[['FILE_NAME', field]],
|
||||
on='FILE_NAME',
|
||||
suffixes=('_truth', '_pred'))
|
||||
# Compare values
|
||||
merged[field] = (merged[f"{field}_truth"] == merged[f"{field}_pred"]) & (merged[f"{field}_truth"] != '')
|
||||
# Drop duplicate rows
|
||||
merged = merged.drop_duplicates(subset=['FILE_NAME', field])
|
||||
|
||||
# Add to comparison DataFrame
|
||||
try:
|
||||
comparison_df[field] = merged.set_index('FILE_NAME')[field]
|
||||
except:
|
||||
print(merged['FILE_NAME'].value_counts())
|
||||
raise
|
||||
|
||||
# Calculate metrics
|
||||
precision, recall, accuracy, FN_list = calculate_field_metrics(merged, field)
|
||||
metrics_df.loc[len(metrics_df)] = [field, precision, recall, accuracy, FN_list]
|
||||
else:
|
||||
metrics_df.loc[len(metrics_df)] = [field, None, None, None, None]
|
||||
comparison_df[field] = None
|
||||
|
||||
# Display results with clean formatting
|
||||
print(metrics_df[["Field", "Precision", "Recall", "Accuracy"]].to_string(
|
||||
index=False,
|
||||
float_format=lambda x: '{:.2f}'.format(x) if pd.notnull(x) else 'Not found',
|
||||
justify='left',
|
||||
col_space={'Field': 40, 'Precision': 15, 'Recall': 15, 'Accuracy': 15}
|
||||
))
|
||||
|
||||
return metrics_df, comparison_df
|
||||
|
||||
|
||||
|
||||
def get_one_to_n_analysis(testbed, results):
|
||||
print("\n*************** One-to-N Stats ****************")
|
||||
|
||||
one_to_n_fields = {"methodology_breakout" : [field for field in FieldSet(file_path=config.FIELD_JSON_PATH).filter(field_type="methodology_breakout").list_fields() if field != "GREATER_OF_IND"],
|
||||
"fee_schedule_breakout" : ["AARETE_DERIVED_FEE_SCHEDULE", "AARETE_DERIVED_FEE_SCHEDULE_VERSION"],
|
||||
"dynamic_primary" : ["AARETE_DERIVED_LOB", "AARETE_DERIVED_PROGRAM", "AARETE_DERIVED_NETWORK", "AARETE_DERIVED_PRODUCT"],
|
||||
"reimb_prov_info" : ["REIMB_PROV_TIN", "REIMB_PROV_NPI", "REIMB_PROV_NAME"]
|
||||
}
|
||||
one_to_n_metrics = {}
|
||||
for field_type, fields in one_to_n_fields.items():
|
||||
N = len(fields)
|
||||
accuracies = []
|
||||
|
||||
for file in testbed['FILE_NAME'].unique():
|
||||
testbed_file = testbed[testbed['FILE_NAME'] == file]
|
||||
results_file = results[results['FILE_NAME'] == file]
|
||||
|
||||
# Ensure there are rows in both dataframes
|
||||
if testbed_file.shape[0] == 0 or results_file.shape[0] == 0:
|
||||
print(f"No rows in either labels or predictions for file: {file}")
|
||||
continue
|
||||
|
||||
confusion_matrix = match_rows(fields, testbed_file, results_file, N)
|
||||
|
||||
confusion_matrix["Filename"] = file
|
||||
accuracies.append(confusion_matrix)
|
||||
|
||||
precision_df, recall_df, overall = calculate_precision_recall(accuracies)
|
||||
one_to_n_metrics[field_type] = {'precision': precision_df, 'recall': recall_df, 'overall': overall}
|
||||
|
||||
print(f"\nOverall Metrics: {field_type}")
|
||||
print(overall.to_string(
|
||||
index=True,
|
||||
float_format=lambda x: '{:.2f}'.format(x) if pd.notnull(x) else 'Not found',
|
||||
justify='left',
|
||||
col_space={'tp': 15, 'fp': 15, 'fn': 15, 'prec': 15, 'rec': 15, 'f1': 15}
|
||||
))
|
||||
|
||||
return one_to_n_metrics
|
||||
|
||||
|
||||
def get_reimb_primary(testbed, results):
|
||||
common_files = set(testbed['FILE_NAME']).intersection(set(results['FILE_NAME']))
|
||||
reimb_primary_metric_rows = []
|
||||
for file in common_files:
|
||||
result = compare_term_extraction_results(file, predictions_df=results, gt_df=testbed)
|
||||
reimb_primary_metric_rows.append(result.copy())
|
||||
|
||||
reimb_primary_df = pd.DataFrame(reimb_primary_metric_rows)
|
||||
return reimb_primary_df
|
||||
|
||||
|
||||
def export_to_excel(output_file,
|
||||
testbed,
|
||||
row_comparison,
|
||||
page_comparison_df,
|
||||
metrics_df,
|
||||
comparison_df,
|
||||
one_to_n_metrics,
|
||||
reimb_primary_df,
|
||||
provider_metrics_df):
|
||||
with pd.ExcelWriter(output_file, engine='openpyxl') as writer:
|
||||
# Test Bed Stats
|
||||
stats_df = pd.DataFrame({
|
||||
'Metric': ['Total Contracts', 'Total Fields', 'Empty Fields'],
|
||||
'Value': [
|
||||
len(testbed['FILE_NAME'].unique()),
|
||||
len(testbed.columns),
|
||||
(testbed == '').all().sum()
|
||||
]
|
||||
})
|
||||
empty_fields_df = pd.DataFrame({'Empty Fields': testbed.columns[(testbed == '').all()].tolist()})
|
||||
|
||||
stats_df.to_excel(writer, sheet_name='Testbed Stats', index=False)
|
||||
row_comparison.to_excel(writer, sheet_name='Row Counts')
|
||||
empty_fields_df.to_excel(writer, sheet_name='Empty Fields', index=False)
|
||||
|
||||
# Add page-level row count comparison if it exists
|
||||
if not page_comparison_df.empty:
|
||||
page_comparison_df.to_excel(writer, sheet_name='Page-Level Row Counts', index=False)
|
||||
else:
|
||||
print("No page-level row count mismatches found; skipping export.")
|
||||
|
||||
# One-to-One Results
|
||||
metrics_df.to_excel(writer, sheet_name='One-to-One Metrics', index=False)
|
||||
comparison_df.to_excel(writer, sheet_name='One-to-One Details') # Add new sheet
|
||||
|
||||
# One-to-N Results
|
||||
for field_type, metrics in one_to_n_metrics.items():
|
||||
precision_df = metrics['precision']
|
||||
recall_df = metrics['recall']
|
||||
overall = metrics['overall']
|
||||
|
||||
# Add the field type to the sheet names
|
||||
precision_df.to_excel(writer, sheet_name=f'{field_type} Precision', index=False)
|
||||
recall_df.to_excel(writer, sheet_name=f'{field_type} Recall', index=False)
|
||||
overall.to_excel(writer, sheet_name=f'{field_type} Overall')
|
||||
|
||||
# Reimbursement Primary Results
|
||||
reimb_primary_df.to_excel(writer, sheet_name='Reimbursement Primary', index=False)
|
||||
|
||||
# Order-independent provider analysis
|
||||
provider_metrics_df.to_excel(writer, sheet_name='Provider Fields', index=False)
|
||||
|
||||
# Auto-adjust column widths
|
||||
for sheet_name in writer.sheets:
|
||||
worksheet = writer.sheets[sheet_name]
|
||||
for column in worksheet.columns:
|
||||
max_length = 0
|
||||
column = [cell for cell in column]
|
||||
for cell in column:
|
||||
try:
|
||||
if len(str(cell.value)) > max_length:
|
||||
max_length = len(str(cell.value))
|
||||
except:
|
||||
pass
|
||||
adjusted_width = (max_length + 2)
|
||||
worksheet.column_dimensions[column[0].column_letter].width = adjusted_width
|
||||
|
||||
print(f"Results exported to {output_file}")
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user