diff --git a/fieldExtraction/src/testbed/testbed_metrics.py b/fieldExtraction/src/testbed/testbed_metrics.py index 0e8dfa7..2e43762 100644 --- a/fieldExtraction/src/testbed/testbed_metrics.py +++ b/fieldExtraction/src/testbed/testbed_metrics.py @@ -12,11 +12,9 @@ warnings.filterwarnings("ignore") testbed = pd.read_excel("Doczy-Testbed.xlsx") # This is the testbed results = pd.read_csv("Doczy-Results.csv") # This is the doczy output -# Preprocess testbed and results data +########################## Preprocess ########################## testbed = testbed_utils.testbed_preprocess(testbed) results = testbed_utils.testbed_preprocess(results) - -# Filter out filenames we don't want to include testbed, results = testbed_utils.filter_file_names(testbed, results) ########################## Analyze Testbed ########################## @@ -25,9 +23,6 @@ testbed_utils.get_df_stats(testbed) ########################## Row Comparison ########################## row_comparison = testbed_utils.get_row_comparison(testbed, results) -########################## Page-Level Row Count Comparison ########################## -page_comparison_df = testbed_utils.get_page_comparison_df(testbed, results) - ########################## One-to-One Analysis ########################## metrics_df, comparison_df = testbed_utils.get_one_to_one_analysis(testbed, results) @@ -51,7 +46,6 @@ testbed_utils.export_to_excel( output_file, testbed, row_comparison, - page_comparison_df, metrics_df, comparison_df, one_to_n_metrics, diff --git a/fieldExtraction/src/testbed/testbed_utils.py b/fieldExtraction/src/testbed/testbed_utils.py index 1020284..adf75e2 100644 --- a/fieldExtraction/src/testbed/testbed_utils.py +++ b/fieldExtraction/src/testbed/testbed_utils.py @@ -17,18 +17,15 @@ def filter_file_names(testbed, results): common_file_names = set(testbed["FILE_NAME"]).intersection( set(results["FILE_NAME"]) ) - print(f"common_file_names: {len(common_file_names)} items") - if ( - "86-0898663-Founder Health, LLC dba Preferred Homecare-ICMProviderAgreement_120661".upper() - in common_file_names - ): - print( - "found 86-0898663-Founder Health, LLC dba Preferred Homecare-ICMProviderAgreement_120661 in both testbed and results; removing it from both datasets" - ) - common_file_names.remove( - "86-0898663-Founder Health, LLC dba Preferred Homecare-ICMProviderAgreement_120661".upper() - ) - print(f"common_file_names now has {len(common_file_names)} items") + + files_to_remove = [ + "86-0898663-Founder Health, LLC dba Preferred Homecare-ICMProviderAgreement_120661" + ] + + for filename in files_to_remove: + if filename in common_file_names: + common_file_names.remove(filename.upper()) + testbed = testbed[testbed["FILE_NAME"].isin(common_file_names)] results = results[results["FILE_NAME"].isin(common_file_names)] return testbed, results @@ -103,18 +100,6 @@ def testbed_preprocess(df): # Ensure that all empty cells are filled with empty strings df = df.fillna("") - # # Create multiple rows for multiple values stored in columns AARETE_DERIVED_LOB, AARETE_DERIVED_PROGRAM, AARETE_DERIVED_NETWORK - # # to be implemented later - # for col in ['AARETE_DERIVED_LOB', 'AARETE_DERIVED_PROGRAM', 'AARETE_DERIVED_NETWORK']: - # if col in df.columns: - # # Split the column by "|" and explode into multiple rows - # df[col] = df[col].str.split("|") - # df = df.explode(col) - # # Remove any leading/trailing whitespace from exploded values - # df[col] = df[col].str.strip() - - # df = df.reset_index(drop=True) - return df @@ -1186,141 +1171,6 @@ def normalize_page(page_str): except (ValueError, TypeError): return str(page_str).strip() - -def get_page_comparison_df(testbed, results): - - print("\n*************** Page-Level Row Count Comparison ****************") - if "EXHIBIT_PAGE" in testbed.columns and "EXHIBIT_PAGE" in results.columns: - page_mismatches = [] - for file in testbed["FILE_NAME"].unique(): - testbed_file = testbed[testbed["FILE_NAME"] == file].copy() - results_file = results[results["FILE_NAME"] == file].copy() - - # Normalize page numbers - testbed_file["EXHIBIT_PAGE_NORM"] = testbed_file["EXHIBIT_PAGE"].apply( - normalize_page - ) - results_file["EXHIBIT_PAGE_NORM"] = results_file["EXHIBIT_PAGE"].apply( - normalize_page - ) - - # Get unique pages from each dataset (excluding None/empty) - testbed_pages = set(testbed_file["EXHIBIT_PAGE_NORM"].dropna()) - results_pages = set(results_file["EXHIBIT_PAGE_NORM"].dropna()) - testbed_pages.discard("") # Remove empty strings - results_pages.discard("") # Remove empty strings - - # Group consecutive pages - testbed_groups = get_consecutive_page_groups(testbed_pages) - results_groups = get_consecutive_page_groups(results_pages) - - # Count rows for each group - testbed_group_counts = {} - - for group in testbed_groups: - group_key = group_name(group) - count = sum( - testbed_file["EXHIBIT_PAGE_NORM"].isin([str(p) for p in group]) - ) - testbed_group_counts[group_key] = count - - results_group_counts = {} - - for group in results_groups: - group_key = group_name(group) - count = sum( - results_file["EXHIBIT_PAGE_NORM"].isin([str(p) for p in group]) - ) - results_group_counts[group_key] = count - - # Compare group counts - all_groups = set(testbed_group_counts.keys()) | set( - results_group_counts.keys() - ) - - group_comparison = {} - has_mismatch = False - - for group_key in all_groups: - testbed_count = testbed_group_counts.get(group_key, 0) - results_count = results_group_counts.get(group_key, 0) - - group_comparison[group_key] = { - "testbed_count": testbed_count, - "results_count": results_count, - "difference": results_count - testbed_count, - } - - if testbed_count != results_count: - has_mismatch = True - - if has_mismatch: - page_mismatches.append( - { - "FILE_NAME": file, - "group_comparison": group_comparison, - "total_testbed_rows": len(testbed_file), - "total_results_rows": len(results_file), - } - ) - - if page_mismatches: - # Create detailed comparison DataFrame - detailed_page_comparison = [] - for file_data in page_mismatches: - file_name = file_data["FILE_NAME"] - for group_key, counts in file_data["group_comparison"].items(): - detailed_page_comparison.append( - { - "FILE_NAME": file_name, - "PAGE_GROUP": group_key, - "TESTBED_COUNT": counts["testbed_count"], - "RESULTS_COUNT": counts["results_count"], - "DIFFERENCE": counts["difference"], - } - ) - page_comparison_df = pd.DataFrame(detailed_page_comparison) - - # Sort by file name and first page number in group - def sort_key(group_str): - """Extract the first page number from a group string for sorting.""" - parts = group_str.split("-") - return int(parts[0]) if parts else float("inf") - - page_comparison_df["SORT_KEY"] = page_comparison_df["PAGE_GROUP"].apply( - sort_key - ) - page_comparison_df = page_comparison_df.sort_values( - by=["FILE_NAME", "SORT_KEY"] - ).drop(columns=["SORT_KEY"]) - - # Summary statistics - total_missing_rows = page_comparison_df[ - page_comparison_df["DIFFERENCE"] < 0 - ]["DIFFERENCE"].sum() - total_extra_rows = page_comparison_df[page_comparison_df["DIFFERENCE"] > 0][ - "DIFFERENCE" - ].sum() - pages_with_missing = (page_comparison_df["DIFFERENCE"] < 0).sum() - pages_with_extra = (page_comparison_df["DIFFERENCE"] > 0).sum() - - print(f"Pages with missing rows in results: {pages_with_missing}") - print(f"Pages with extra rows in results: {pages_with_extra}") - print(f"Total missing rows: {abs(total_missing_rows)}") - print(f"Total extra rows: {total_extra_rows}") - - else: - print("All files have matching row counts for consecutive page groups!") - page_comparison_df = pd.DataFrame() - else: - print( - "EXHIBIT_PAGE column not found in either testbed or results. Skipping page-level row count comparison." - ) - page_comparison_df = pd.DataFrame() - - return page_comparison_df - - def get_one_to_one_analysis(testbed, results): print("\n*************** One-to-One Stats ****************") one_to_one_fields = ( @@ -1349,9 +1199,7 @@ def get_one_to_one_analysis(testbed, results): ) comparison_df = pd.DataFrame(index=testbed["FILE_NAME"].unique()) - # for field in one_to_one_fields: - for field in ["PROV_GROUP_TIN"]: - + for field in one_to_one_fields: if ( field in testbed.columns and field in results.columns @@ -1548,7 +1396,6 @@ def export_to_excel( output_file, testbed, row_comparison, - page_comparison_df, metrics_df, comparison_df, one_to_n_metrics, @@ -1575,14 +1422,6 @@ def export_to_excel( row_comparison.to_excel(writer, sheet_name="Row Counts") empty_fields_df.to_excel(writer, sheet_name="Empty Fields", index=False) - # Add page-level row count comparison if it exists - if not page_comparison_df.empty: - page_comparison_df.to_excel( - writer, sheet_name="Page-Level Row Counts", index=False - ) - else: - print("No page-level row count mismatches found; skipping export.") - # One-to-One Results metrics_df.to_excel(writer, sheet_name="One-to-One Metrics", index=False) comparison_df.to_excel(writer, sheet_name="One-to-One Details") # Add new sheet