From 6d27b17819c1b258e25c6ef2f00e41cfa9de69b7 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Wed, 25 Jun 2025 15:57:24 +0000 Subject: [PATCH] Merged in feature/testbed-updates (pull request #589) Feature/testbed updates * Begin refactor * Update print * finalize testbed script Approved-by: Alex Galarce --- fieldExtraction/src/testbed/test.py | 369 +---------------- fieldExtraction/src/testbed/testbed_utils.py | 404 ++++++++++++++++++- 2 files changed, 416 insertions(+), 357 deletions(-) diff --git a/fieldExtraction/src/testbed/test.py b/fieldExtraction/src/testbed/test.py index 5791a1a..241d02e 100644 --- a/fieldExtraction/src/testbed/test.py +++ b/fieldExtraction/src/testbed/test.py @@ -19,376 +19,45 @@ results = pd.read_csv("Doczy-Results.csv") # This is the doczy output testbed = testbed_utils.testbed_preprocess(testbed) results = testbed_utils.testbed_preprocess(results) -# Ensure FILE_NAME values match between testbed and results -common_file_names = set(testbed['FILE_NAME']).intersection(set(results['FILE_NAME'])) -print(f"common_file_names: {len(common_file_names)} items") -if "86-0898663-Founder Health, LLC dba Preferred Homecare-ICMProviderAgreement_120661".upper() in common_file_names: - print("found 86-0898663-Founder Health, LLC dba Preferred Homecare-ICMProviderAgreement_120661 in both testbed and results; removing it from both datasets") - common_file_names.remove("86-0898663-Founder Health, LLC dba Preferred Homecare-ICMProviderAgreement_120661".upper()) - print(f"common_file_names now has {len(common_file_names)} items") -testbed = testbed[testbed['FILE_NAME'].isin(common_file_names)] -results = results[results['FILE_NAME'].isin(common_file_names)] +# Filter out filenames we don't want to include +testbed, results = testbed_utils.filter_file_names(testbed, results) ########################## Analyze Testbed ########################## -print("*************** Test Bed Stats ****************") -print(f"{len(testbed['FILE_NAME'].unique())} contracts") -print(f"{len(testbed.columns)} fields") - -print(f"{(testbed == '').all().sum()} fields not populated:") # Count of completely empty fields -print(f"{testbed.columns[(testbed == '').all()].tolist()}") # List of completely empty fields +testbed_utils.get_df_stats(testbed) ########################## Row Comparison ########################## -row_comparison = pd.DataFrame(index=testbed['FILE_NAME'].unique()) -row_comparison['testbed'] = testbed.groupby('FILE_NAME').size() -row_comparison['results'] = results.groupby('FILE_NAME').size() - -print("\n*************** Row Count Comparison ****************") -print(f"Files with different row counts: {(row_comparison['testbed'] != row_comparison['results']).sum()}") +row_comparison = testbed_utils.get_row_comparison(testbed, results) ########################## Page-Level Row Count Comparison ########################## -print("\n*************** Page-Level Row Count Comparison ****************") -def get_consecutive_page_groups(pages) -> list: - """Group consecutive page numbers together. - e.g., [4, 5, 6, 8, 9] -> [[4, 5, 6], [8, 9]] - - """ - - if not pages: - return [] - - # Convert to sorted list of integers - page_nums = sorted([int(p) for p in pages if str(p).isdigit()]) - - if not page_nums: - return [] - - groups = [] - current_group = [page_nums[0]] - - for i in range(1, len(page_nums)): - if page_nums[i] == page_nums[i - 1] + 1: # consecutive pages - current_group.append(page_nums[i]) - else: - groups.append(current_group) - current_group = [page_nums[i]] - groups.append(current_group) # Append the last group - return groups - -def group_name(group): - """Convert a group of consecutive pages to a readable name. - E.g., [4, 5, 6] -> "4-6", [8, 9] -> "8-9", [13] -> "13" - """ - if len(group) == 1: - return str(group[0]) - else: - return f"{group[0]}-{group[-1]}" - -def normalize_page(page_str): - """Normalize page numbers to integer strings for comparison (e.g., "4.0" -> "4", "4" -> "4", "13.0.0" -> "13")""" - if pd.isna(page_str) or page_str == "": - return None - try: - # Split on first period - return str(page_str).strip().split(".")[0] - except (ValueError, TypeError): - return str(page_str).strip() - -if "EXHIBIT_PAGE" in testbed.columns and "EXHIBIT_PAGE" in results.columns: - page_mismatches = [] - for file in testbed['FILE_NAME'].unique(): - testbed_file = testbed[testbed['FILE_NAME'] == file].copy() - results_file = results[results['FILE_NAME'] == file].copy() - - # Normalize page numbers - testbed_file['EXHIBIT_PAGE_NORM'] = testbed_file['EXHIBIT_PAGE'].apply(normalize_page) - results_file['EXHIBIT_PAGE_NORM'] = results_file['EXHIBIT_PAGE'].apply(normalize_page) - - # Get unique pages from each dataset (excluding None/empty) - testbed_pages = set(testbed_file['EXHIBIT_PAGE_NORM'].dropna()) - results_pages = set(results_file['EXHIBIT_PAGE_NORM'].dropna()) - testbed_pages.discard("") # Remove empty strings - results_pages.discard("") # Remove empty strings - - # Group consecutive pages - testbed_groups = get_consecutive_page_groups(testbed_pages) - results_groups = get_consecutive_page_groups(results_pages) - - # Count rows for each group - testbed_group_counts = {} - - for group in testbed_groups: - group_key = group_name(group) - count = sum(testbed_file['EXHIBIT_PAGE_NORM'].isin([str(p) for p in group])) - testbed_group_counts[group_key] = count - - results_group_counts = {} - - for group in results_groups: - group_key = group_name(group) - count = sum(results_file['EXHIBIT_PAGE_NORM'].isin([str(p) for p in group])) - results_group_counts[group_key] = count - - # Compare group counts - all_groups = set(testbed_group_counts.keys()) | set(results_group_counts.keys()) - - group_comparison = {} - has_mismatch = False - - for group_key in all_groups: - testbed_count = testbed_group_counts.get(group_key, 0) - results_count = results_group_counts.get(group_key, 0) - - group_comparison[group_key] = { - 'testbed_count': testbed_count, - 'results_count': results_count, - 'difference': results_count - testbed_count - } - - if testbed_count != results_count: - has_mismatch = True - - if has_mismatch: - page_mismatches.append({ - 'FILE_NAME': file, - 'group_comparison': group_comparison, - 'total_testbed_rows': len(testbed_file), - 'total_results_rows': len(results_file) - }) - - - if page_mismatches: - # Create detailed comparison DataFrame - detailed_page_comparison = [] - for file_data in page_mismatches: - file_name = file_data['FILE_NAME'] - for group_key, counts in file_data['group_comparison'].items(): - detailed_page_comparison.append({ - 'FILE_NAME': file_name, - 'PAGE_GROUP': group_key, - 'TESTBED_COUNT': counts['testbed_count'], - 'RESULTS_COUNT': counts['results_count'], - 'DIFFERENCE': counts['difference'] - }) - page_comparison_df = pd.DataFrame(detailed_page_comparison) - - # Sort by file name and first page number in group - def sort_key(group_str): - """Extract the first page number from a group string for sorting.""" - parts = group_str.split('-') - return int(parts[0]) if parts else float('inf') - page_comparison_df['SORT_KEY'] = page_comparison_df['PAGE_GROUP'].apply(sort_key) - page_comparison_df = page_comparison_df.sort_values(by=['FILE_NAME', 'SORT_KEY']).drop(columns=['SORT_KEY']) - - # Summary statistics - total_missing_rows = page_comparison_df[page_comparison_df['DIFFERENCE'] < 0]['DIFFERENCE'].sum() - total_extra_rows = page_comparison_df[page_comparison_df['DIFFERENCE'] > 0]['DIFFERENCE'].sum() - pages_with_missing = (page_comparison_df['DIFFERENCE'] < 0).sum() - pages_with_extra = (page_comparison_df['DIFFERENCE'] > 0).sum() - - print(f"Pages with missing rows in results: {pages_with_missing}") - print(f"Pages with extra rows in results: {pages_with_extra}") - print(f"Total missing rows: {abs(total_missing_rows)}") - print(f"Total extra rows: {total_extra_rows}") - - - else: - print("All files have matching row counts for consecutive page groups!") - page_comparison_df = pd.DataFrame() -else: - print("EXHIBIT_PAGE column not found in either testbed or results. Skipping page-level row count comparison.") - page_comparison_df = pd.DataFrame() +page_comparison_df = testbed_utils.get_page_comparison_df(testbed, results) ########################## One-to-One Analysis ########################## -print("\n*************** One-to-One Stats ****************") -one_to_one_fields = FieldSet(file_path=config.FIELD_JSON_PATH).filter(relationship="one_to_one").list_fields() -one_to_one_fields.extend(['PROV_GROUP_TIN', - 'PROV_GROUP_NPI', - 'PROV_GROUP_NAME_FULL']) -one_to_one_fields = [f for f in one_to_one_fields if f not in {'TIN', 'NPI', 'NAME', "CLIENT_NAME"}] - -# Create filtered dataframes with only one-to-one fields -one_to_one_columns = ['FILE_NAME'] + [f for f in one_to_one_fields if f in testbed.columns and f in results.columns] -testbed_one_to_one = testbed[one_to_one_columns].drop_duplicates() -results_one_to_one = results[one_to_one_columns].drop_duplicates() - -metrics_df = pd.DataFrame(columns=['Field', 'Precision', 'Recall', 'Accuracy', 'FN_list']) -comparison_df = pd.DataFrame(index=testbed_one_to_one['FILE_NAME'].unique()) - -for field in one_to_one_fields: - if field in testbed.columns and field in results.columns and field != 'FILE_NAME': - # Merge and compare values (no need to drop_duplicates here since we already did) - merged = pd.merge(testbed_one_to_one[['FILE_NAME', field]], - results_one_to_one[['FILE_NAME', field]], - on='FILE_NAME', - suffixes=('_truth', '_pred')) - # Compare values - merged[field] = (merged[f"{field}_truth"] == merged[f"{field}_pred"]) & (merged[f"{field}_truth"] != '') - # Drop duplicate rows - merged = merged.drop_duplicates(subset=['FILE_NAME', field]) - - # Add to comparison DataFrame - try: - comparison_df[field] = merged.set_index('FILE_NAME')[field] - except: - print(merged['FILE_NAME'].value_counts()) - raise - - # Calculate metrics - precision, recall, accuracy, FN_list = testbed_utils.calculate_field_metrics(merged, field) - metrics_df.loc[len(metrics_df)] = [field, precision, recall, accuracy, FN_list] - else: - metrics_df.loc[len(metrics_df)] = [field, None, None, None, None] - comparison_df[field] = None - -# Display results with clean formatting -print(metrics_df.to_string( - index=False, - float_format=lambda x: '{:.2f}'.format(x) if pd.notnull(x) else 'Not found', - justify='left', - col_space={'Field': 40, 'Precision': 15, 'Recall': 15, 'Accuracy': 15} -)) +metrics_df, comparison_df = testbed_utils.get_one_to_one_analysis(testbed, results) ########################## One-to-N Analysis ########################## -print("\n*************** One-to-N Stats ****************") - -one_to_n_fields = {"methodology_breakout" : [field for field in FieldSet(file_path=config.FIELD_JSON_PATH).filter(field_type="methodology_breakout").list_fields() if field != "GREATER_OF_IND"], - "fee_schedule_breakout" : ["AARETE_DERIVED_FEE_SCHEDULE", "AARETE_DERIVED_FEE_SCHEDULE_VERSION"], - "dynamic_primary" : ["AARETE_DERIVED_LOB", "AARETE_DERIVED_PROGRAM", "AARETE_DERIVED_NETWORK", "AARETE_DERIVED_PRODUCT"], - "reimb_prov_info" : ["REIMB_PROV_TIN", "REIMB_PROV_NPI", "REIMB_PROV_NAME"] -} -one_to_n_metrics = {} -for field_type, fields in one_to_n_fields.items(): - N = len(fields) - accuracies = [] - - for file in testbed['FILE_NAME'].unique(): - testbed_file = testbed[testbed['FILE_NAME'] == file] - results_file = results[results['FILE_NAME'] == file] - - # Ensure there are rows in both dataframes - if testbed_file.shape[0] == 0 or results_file.shape[0] == 0: - print(f"No rows in either labels or predictions for file: {file}") - continue - - confusion_matrix = testbed_utils.match_rows(fields, testbed_file, results_file, N) - - confusion_matrix["Filename"] = file - accuracies.append(confusion_matrix) - - precision_df, recall_df, overall = testbed_utils.calculate_precision_recall(accuracies) - one_to_n_metrics[field_type] = {'precision': precision_df, 'recall': recall_df, 'overall': overall} - - print(f"\nOverall Metrics: {field_type}") - print(overall.to_string( - index=True, - float_format=lambda x: '{:.2f}'.format(x) if pd.notnull(x) else 'Not found', - justify='left', - col_space={'tp': 15, 'fp': 15, 'fn': 15, 'prec': 15, 'rec': 15, 'f1': 15} - )) +one_to_n_metrics = testbed_utils.get_one_to_n_analysis(testbed, results) ########################## Reimbursement Primary ########################## -# Special handling for reimbursement primary -common_files = set(testbed['FILE_NAME']).intersection(set(results['FILE_NAME'])) -reimb_primary_metric_rows = [] -for file in common_files: - result = testbed_utils.compare_term_extraction_results(file, predictions_df=results, gt_df=testbed) - reimb_primary_metric_rows.append(result.copy()) - -reimb_primary_df = pd.DataFrame(reimb_primary_metric_rows) - +reimb_primary_df = testbed_utils.get_reimb_primary(testbed, results) ########################## Order-Independent Provider Analysis ########################## -print("\n*************** Order-Independent Provider Analysis ****************") -provider_results = testbed_utils.evaluate_provider_fields_separately( - testbed, - results, verbose=True) - -# Display the results -provider_metrics_df = pd.DataFrame([ - { - 'Field': f"PROV_OTHER_{field_type}", - 'Precision': metrics['precision'], - 'Recall': metrics['recall'], - 'F1': metrics['f1'], - 'TP': metrics['tp'], - 'FP': metrics['fp'], - 'FN': metrics['fn'], - 'average_accuracy': metrics['average_accuracy'], - } - for field_type, metrics in provider_results.items() -]) - -print(provider_metrics_df.to_string( - index=False, - float_format=lambda x: '{:.2f}'.format(x) if pd.notnull(x) else 'Not found', - justify='left', - col_space={'Field': 15, 'Precision': 12, 'Recall': 12, 'F1': 12, 'TP': 8, 'FP': 8, 'FN': 8} -)) +provider_results, provider_metrics_df = testbed_utils.evaluate_provider_fields_separately(testbed, results, verbose=True) ########################## Export to Excel ########################## print("\nExporting results to Excel...") # Create Excel writer object output_file = f"{config.TODAY}-testbed-metrics.xlsx" -with pd.ExcelWriter(output_file, engine='openpyxl') as writer: - # Test Bed Stats - stats_df = pd.DataFrame({ - 'Metric': ['Total Contracts', 'Total Fields', 'Empty Fields'], - 'Value': [ - len(testbed['FILE_NAME'].unique()), - len(testbed.columns), - (testbed == '').all().sum() - ] - }) - empty_fields_df = pd.DataFrame({'Empty Fields': testbed.columns[(testbed == '').all()].tolist()}) - - stats_df.to_excel(writer, sheet_name='Testbed Stats', index=False) - row_comparison.to_excel(writer, sheet_name='Row Counts') - empty_fields_df.to_excel(writer, sheet_name='Empty Fields', index=False) - - # Add page-level row count comparison if it exists - if not page_comparison_df.empty: - page_comparison_df.to_excel(writer, sheet_name='Page-Level Row Counts', index=False) - else: - print("No page-level row count mismatches found; skipping export.") - - # One-to-One Results - metrics_df.to_excel(writer, sheet_name='One-to-One Metrics', index=False) - comparison_df.to_excel(writer, sheet_name='One-to-One Details') # Add new sheet - - # One-to-N Results - for field_type, metrics in one_to_n_metrics.items(): - precision_df = metrics['precision'] - recall_df = metrics['recall'] - overall = metrics['overall'] - - # Add the field type to the sheet names - precision_df.to_excel(writer, sheet_name=f'{field_type} Precision', index=False) - recall_df.to_excel(writer, sheet_name=f'{field_type} Recall', index=False) - overall.to_excel(writer, sheet_name=f'{field_type} Overall') - - # Reimbursement Primary Results - reimb_primary_df.to_excel(writer, sheet_name='Reimbursement Primary', index=False) - - # Order-independent provider analysis - provider_metrics_df.to_excel(writer, sheet_name='Provider Fields', index=False) - - # Auto-adjust column widths - for sheet_name in writer.sheets: - worksheet = writer.sheets[sheet_name] - for column in worksheet.columns: - max_length = 0 - column = [cell for cell in column] - for cell in column: - try: - if len(str(cell.value)) > max_length: - max_length = len(str(cell.value)) - except: - pass - adjusted_width = (max_length + 2) - worksheet.column_dimensions[column[0].column_letter].width = adjusted_width - -print(f"Results exported to {output_file}") +testbed_utils.export_to_excel(output_file, + testbed, + row_comparison, + page_comparison_df, + metrics_df, + comparison_df, + one_to_n_metrics, + reimb_primary_df, + provider_metrics_df) + diff --git a/fieldExtraction/src/testbed/testbed_utils.py b/fieldExtraction/src/testbed/testbed_utils.py index 4ac8796..7f416b6 100644 --- a/fieldExtraction/src/testbed/testbed_utils.py +++ b/fieldExtraction/src/testbed/testbed_utils.py @@ -10,7 +10,20 @@ from rapidfuzz import fuzz # Imports for the testbed postprocessing functions from src.investment import investment_postprocessing_funcs from src.constants.investment_columns import COLUMN_ORDER +from src.prompts.investment_prompts import FieldSet +import src.config as config +def filter_file_names(testbed, results): + # Ensure FILE_NAME values match between testbed and results + common_file_names = set(testbed['FILE_NAME']).intersection(set(results['FILE_NAME'])) + print(f"common_file_names: {len(common_file_names)} items") + if "86-0898663-Founder Health, LLC dba Preferred Homecare-ICMProviderAgreement_120661".upper() in common_file_names: + print("found 86-0898663-Founder Health, LLC dba Preferred Homecare-ICMProviderAgreement_120661 in both testbed and results; removing it from both datasets") + common_file_names.remove("86-0898663-Founder Health, LLC dba Preferred Homecare-ICMProviderAgreement_120661".upper()) + print(f"common_file_names now has {len(common_file_names)} items") + testbed = testbed[testbed['FILE_NAME'].isin(common_file_names)] + results = results[results['FILE_NAME'].isin(common_file_names)] + return testbed, results def convert_to_float(value): try: @@ -196,6 +209,7 @@ def calculate_field_metrics(merged, field): def evaluate_provider_fields_separately(testbed_df, results_df, verbose=False): """Evaluates TIN, NPI, and NAME fields separately for each provider.""" + print("\n*************** Order-Independent Provider Analysis ****************") # Initialize metrics for each field type metrics = { "TIN": {'tp' : 0, 'fp': 0, 'fn': 0, 'accuracy_file': []}, @@ -380,8 +394,6 @@ def evaluate_provider_fields_separately(testbed_df, results_df, verbose=False): # Save the DataFrame to a CSV file csv_output_df.to_csv("provider_metrics.csv", index=False) - - # Calculate accuracy, precision, recall, and F1 for each field type results = {} for field_type, field_metrics in metrics.items(): @@ -389,9 +401,6 @@ def evaluate_provider_fields_separately(testbed_df, results_df, verbose=False): recall = field_metrics['tp'] / (field_metrics['tp'] + field_metrics['fn']) if (field_metrics['tp'] + field_metrics['fn']) > 0 else 0 f1 = (2 * precision * recall) / (precision + recall) if (precision + recall) > 0 else 0 - - - results[field_type] = { "precision": precision, "recall": recall, @@ -404,7 +413,30 @@ def evaluate_provider_fields_separately(testbed_df, results_df, verbose=False): if verbose: print("Verbose output written to provider_metrics.txt") - return results + + # Display the results + provider_metrics_df = pd.DataFrame([ + { + 'Field': f"PROV_OTHER_{field_type}", + 'Precision': metrics['precision'], + 'Recall': metrics['recall'], + 'F1': metrics['f1'], + 'TP': metrics['tp'], + 'FP': metrics['fp'], + 'FN': metrics['fn'], + 'average_accuracy': metrics['average_accuracy'], + } + for field_type, metrics in results.items() + ]) + + print(provider_metrics_df.to_string( + index=False, + float_format=lambda x: '{:.2f}'.format(x) if pd.notnull(x) else 'Not found', + justify='left', + col_space={'Field': 15, 'Precision': 12, 'Recall': 12, 'F1': 12, 'TP': 8, 'FP': 8, 'FN': 8} + )) + + return results, provider_metrics_df def match_rows(fields, labels_df, predictions_df, N): @@ -911,4 +943,362 @@ def testbed_postprocess(df): # Standardize output column order - this should ALWAYS be the final postprocessing step df = investment_postprocessing_funcs.reorder_columns(df, COLUMN_ORDER) - return df \ No newline at end of file + return df + +def get_df_stats(df): + print("*************** Test Bed Stats ****************") + print(f"{len(df['FILE_NAME'].unique())} contracts") + print(f"{len(df.columns)} fields") + + print(f"{(df == '').all().sum()} fields not populated:") # Count of completely empty fields + print(f"{df.columns[(df == '').all()].tolist()}") # List of completely empty fields + +def get_row_comparison(testbed, results): + row_comparison = pd.DataFrame(index=testbed['FILE_NAME'].unique()) + row_comparison['testbed'] = testbed.groupby('FILE_NAME').size() + row_comparison['results'] = results.groupby('FILE_NAME').size() + + print("\n*************** Row Count Comparison ****************") + print(f"Files with different row counts: {(row_comparison['testbed'] != row_comparison['results']).sum()}") + + return row_comparison + +def get_consecutive_page_groups(pages) -> list: + """Group consecutive page numbers together. + e.g., [4, 5, 6, 8, 9] -> [[4, 5, 6], [8, 9]] + + """ + + if not pages: + return [] + + # Convert to sorted list of integers + page_nums = sorted([int(p) for p in pages if str(p).isdigit()]) + + if not page_nums: + return [] + + groups = [] + current_group = [page_nums[0]] + + for i in range(1, len(page_nums)): + if page_nums[i] == page_nums[i - 1] + 1: # consecutive pages + current_group.append(page_nums[i]) + else: + groups.append(current_group) + current_group = [page_nums[i]] + groups.append(current_group) # Append the last group + return groups + +def group_name(group): + """Convert a group of consecutive pages to a readable name. + E.g., [4, 5, 6] -> "4-6", [8, 9] -> "8-9", [13] -> "13" + """ + if len(group) == 1: + return str(group[0]) + else: + return f"{group[0]}-{group[-1]}" + +def normalize_page(page_str): + """Normalize page numbers to integer strings for comparison (e.g., "4.0" -> "4", "4" -> "4", "13.0.0" -> "13")""" + if pd.isna(page_str) or page_str == "": + return None + try: + # Split on first period + return str(page_str).strip().split(".")[0] + except (ValueError, TypeError): + return str(page_str).strip() + +def get_page_comparison_df(testbed, results): + + print("\n*************** Page-Level Row Count Comparison ****************") + if "EXHIBIT_PAGE" in testbed.columns and "EXHIBIT_PAGE" in results.columns: + page_mismatches = [] + for file in testbed['FILE_NAME'].unique(): + testbed_file = testbed[testbed['FILE_NAME'] == file].copy() + results_file = results[results['FILE_NAME'] == file].copy() + + # Normalize page numbers + testbed_file['EXHIBIT_PAGE_NORM'] = testbed_file['EXHIBIT_PAGE'].apply(normalize_page) + results_file['EXHIBIT_PAGE_NORM'] = results_file['EXHIBIT_PAGE'].apply(normalize_page) + + # Get unique pages from each dataset (excluding None/empty) + testbed_pages = set(testbed_file['EXHIBIT_PAGE_NORM'].dropna()) + results_pages = set(results_file['EXHIBIT_PAGE_NORM'].dropna()) + testbed_pages.discard("") # Remove empty strings + results_pages.discard("") # Remove empty strings + + # Group consecutive pages + testbed_groups = get_consecutive_page_groups(testbed_pages) + results_groups = get_consecutive_page_groups(results_pages) + + # Count rows for each group + testbed_group_counts = {} + + for group in testbed_groups: + group_key = group_name(group) + count = sum(testbed_file['EXHIBIT_PAGE_NORM'].isin([str(p) for p in group])) + testbed_group_counts[group_key] = count + + results_group_counts = {} + + for group in results_groups: + group_key = group_name(group) + count = sum(results_file['EXHIBIT_PAGE_NORM'].isin([str(p) for p in group])) + results_group_counts[group_key] = count + + # Compare group counts + all_groups = set(testbed_group_counts.keys()) | set(results_group_counts.keys()) + + group_comparison = {} + has_mismatch = False + + for group_key in all_groups: + testbed_count = testbed_group_counts.get(group_key, 0) + results_count = results_group_counts.get(group_key, 0) + + group_comparison[group_key] = { + 'testbed_count': testbed_count, + 'results_count': results_count, + 'difference': results_count - testbed_count + } + + if testbed_count != results_count: + has_mismatch = True + + if has_mismatch: + page_mismatches.append({ + 'FILE_NAME': file, + 'group_comparison': group_comparison, + 'total_testbed_rows': len(testbed_file), + 'total_results_rows': len(results_file) + }) + + + if page_mismatches: + # Create detailed comparison DataFrame + detailed_page_comparison = [] + for file_data in page_mismatches: + file_name = file_data['FILE_NAME'] + for group_key, counts in file_data['group_comparison'].items(): + detailed_page_comparison.append({ + 'FILE_NAME': file_name, + 'PAGE_GROUP': group_key, + 'TESTBED_COUNT': counts['testbed_count'], + 'RESULTS_COUNT': counts['results_count'], + 'DIFFERENCE': counts['difference'] + }) + page_comparison_df = pd.DataFrame(detailed_page_comparison) + + # Sort by file name and first page number in group + def sort_key(group_str): + """Extract the first page number from a group string for sorting.""" + parts = group_str.split('-') + return int(parts[0]) if parts else float('inf') + page_comparison_df['SORT_KEY'] = page_comparison_df['PAGE_GROUP'].apply(sort_key) + page_comparison_df = page_comparison_df.sort_values(by=['FILE_NAME', 'SORT_KEY']).drop(columns=['SORT_KEY']) + + # Summary statistics + total_missing_rows = page_comparison_df[page_comparison_df['DIFFERENCE'] < 0]['DIFFERENCE'].sum() + total_extra_rows = page_comparison_df[page_comparison_df['DIFFERENCE'] > 0]['DIFFERENCE'].sum() + pages_with_missing = (page_comparison_df['DIFFERENCE'] < 0).sum() + pages_with_extra = (page_comparison_df['DIFFERENCE'] > 0).sum() + + print(f"Pages with missing rows in results: {pages_with_missing}") + print(f"Pages with extra rows in results: {pages_with_extra}") + print(f"Total missing rows: {abs(total_missing_rows)}") + print(f"Total extra rows: {total_extra_rows}") + + + else: + print("All files have matching row counts for consecutive page groups!") + page_comparison_df = pd.DataFrame() + else: + print("EXHIBIT_PAGE column not found in either testbed or results. Skipping page-level row count comparison.") + page_comparison_df = pd.DataFrame() + + return page_comparison_df + + +def get_one_to_one_analysis(testbed, results): + print("\n*************** One-to-One Stats ****************") + one_to_one_fields = FieldSet(file_path=config.FIELD_JSON_PATH).filter(relationship="one_to_one").list_fields() + one_to_one_fields.extend(['PROV_GROUP_TIN', + 'PROV_GROUP_NPI', + 'PROV_GROUP_NAME_FULL']) + one_to_one_fields = [f for f in one_to_one_fields if f not in {'TIN', 'NPI', 'NAME', "CLIENT_NAME"}] + + # Create filtered dataframes with only one-to-one fields + one_to_one_columns = ['FILE_NAME'] + [f for f in one_to_one_fields if f in testbed.columns and f in results.columns] + testbed_one_to_one = testbed[one_to_one_columns].drop_duplicates() + results_one_to_one = results[one_to_one_columns].drop_duplicates() + + metrics_df = pd.DataFrame(columns=['Field', 'Precision', 'Recall', 'Accuracy', 'FN_list']) + comparison_df = pd.DataFrame(index=testbed_one_to_one['FILE_NAME'].unique()) + + for field in one_to_one_fields: + if field in testbed.columns and field in results.columns and field != 'FILE_NAME': + # Merge and compare values (no need to drop_duplicates here since we already did) + merged = pd.merge(testbed_one_to_one[['FILE_NAME', field]], + results_one_to_one[['FILE_NAME', field]], + on='FILE_NAME', + suffixes=('_truth', '_pred')) + # Compare values + merged[field] = (merged[f"{field}_truth"] == merged[f"{field}_pred"]) & (merged[f"{field}_truth"] != '') + # Drop duplicate rows + merged = merged.drop_duplicates(subset=['FILE_NAME', field]) + + # Add to comparison DataFrame + try: + comparison_df[field] = merged.set_index('FILE_NAME')[field] + except: + print(merged['FILE_NAME'].value_counts()) + raise + + # Calculate metrics + precision, recall, accuracy, FN_list = calculate_field_metrics(merged, field) + metrics_df.loc[len(metrics_df)] = [field, precision, recall, accuracy, FN_list] + else: + metrics_df.loc[len(metrics_df)] = [field, None, None, None, None] + comparison_df[field] = None + + # Display results with clean formatting + print(metrics_df[["Field", "Precision", "Recall", "Accuracy"]].to_string( + index=False, + float_format=lambda x: '{:.2f}'.format(x) if pd.notnull(x) else 'Not found', + justify='left', + col_space={'Field': 40, 'Precision': 15, 'Recall': 15, 'Accuracy': 15} + )) + + return metrics_df, comparison_df + + + +def get_one_to_n_analysis(testbed, results): + print("\n*************** One-to-N Stats ****************") + + one_to_n_fields = {"methodology_breakout" : [field for field in FieldSet(file_path=config.FIELD_JSON_PATH).filter(field_type="methodology_breakout").list_fields() if field != "GREATER_OF_IND"], + "fee_schedule_breakout" : ["AARETE_DERIVED_FEE_SCHEDULE", "AARETE_DERIVED_FEE_SCHEDULE_VERSION"], + "dynamic_primary" : ["AARETE_DERIVED_LOB", "AARETE_DERIVED_PROGRAM", "AARETE_DERIVED_NETWORK", "AARETE_DERIVED_PRODUCT"], + "reimb_prov_info" : ["REIMB_PROV_TIN", "REIMB_PROV_NPI", "REIMB_PROV_NAME"] + } + one_to_n_metrics = {} + for field_type, fields in one_to_n_fields.items(): + N = len(fields) + accuracies = [] + + for file in testbed['FILE_NAME'].unique(): + testbed_file = testbed[testbed['FILE_NAME'] == file] + results_file = results[results['FILE_NAME'] == file] + + # Ensure there are rows in both dataframes + if testbed_file.shape[0] == 0 or results_file.shape[0] == 0: + print(f"No rows in either labels or predictions for file: {file}") + continue + + confusion_matrix = match_rows(fields, testbed_file, results_file, N) + + confusion_matrix["Filename"] = file + accuracies.append(confusion_matrix) + + precision_df, recall_df, overall = calculate_precision_recall(accuracies) + one_to_n_metrics[field_type] = {'precision': precision_df, 'recall': recall_df, 'overall': overall} + + print(f"\nOverall Metrics: {field_type}") + print(overall.to_string( + index=True, + float_format=lambda x: '{:.2f}'.format(x) if pd.notnull(x) else 'Not found', + justify='left', + col_space={'tp': 15, 'fp': 15, 'fn': 15, 'prec': 15, 'rec': 15, 'f1': 15} + )) + + return one_to_n_metrics + + +def get_reimb_primary(testbed, results): + common_files = set(testbed['FILE_NAME']).intersection(set(results['FILE_NAME'])) + reimb_primary_metric_rows = [] + for file in common_files: + result = compare_term_extraction_results(file, predictions_df=results, gt_df=testbed) + reimb_primary_metric_rows.append(result.copy()) + + reimb_primary_df = pd.DataFrame(reimb_primary_metric_rows) + return reimb_primary_df + + +def export_to_excel(output_file, + testbed, + row_comparison, + page_comparison_df, + metrics_df, + comparison_df, + one_to_n_metrics, + reimb_primary_df, + provider_metrics_df): + with pd.ExcelWriter(output_file, engine='openpyxl') as writer: + # Test Bed Stats + stats_df = pd.DataFrame({ + 'Metric': ['Total Contracts', 'Total Fields', 'Empty Fields'], + 'Value': [ + len(testbed['FILE_NAME'].unique()), + len(testbed.columns), + (testbed == '').all().sum() + ] + }) + empty_fields_df = pd.DataFrame({'Empty Fields': testbed.columns[(testbed == '').all()].tolist()}) + + stats_df.to_excel(writer, sheet_name='Testbed Stats', index=False) + row_comparison.to_excel(writer, sheet_name='Row Counts') + empty_fields_df.to_excel(writer, sheet_name='Empty Fields', index=False) + + # Add page-level row count comparison if it exists + if not page_comparison_df.empty: + page_comparison_df.to_excel(writer, sheet_name='Page-Level Row Counts', index=False) + else: + print("No page-level row count mismatches found; skipping export.") + + # One-to-One Results + metrics_df.to_excel(writer, sheet_name='One-to-One Metrics', index=False) + comparison_df.to_excel(writer, sheet_name='One-to-One Details') # Add new sheet + + # One-to-N Results + for field_type, metrics in one_to_n_metrics.items(): + precision_df = metrics['precision'] + recall_df = metrics['recall'] + overall = metrics['overall'] + + # Add the field type to the sheet names + precision_df.to_excel(writer, sheet_name=f'{field_type} Precision', index=False) + recall_df.to_excel(writer, sheet_name=f'{field_type} Recall', index=False) + overall.to_excel(writer, sheet_name=f'{field_type} Overall') + + # Reimbursement Primary Results + reimb_primary_df.to_excel(writer, sheet_name='Reimbursement Primary', index=False) + + # Order-independent provider analysis + provider_metrics_df.to_excel(writer, sheet_name='Provider Fields', index=False) + + # Auto-adjust column widths + for sheet_name in writer.sheets: + worksheet = writer.sheets[sheet_name] + for column in worksheet.columns: + max_length = 0 + column = [cell for cell in column] + for cell in column: + try: + if len(str(cell.value)) > max_length: + max_length = len(str(cell.value)) + except: + pass + adjusted_width = (max_length + 2) + worksheet.column_dimensions[column[0].column_letter].width = adjusted_width + + print(f"Results exported to {output_file}") + + + + + + + +