From 27e09b5fbc1e44f730804b2b1a322937f91fbb81 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Tue, 24 Jun 2025 17:42:45 +0000 Subject: [PATCH] Merged in feature/row-count-by-pages (pull request #585) Feature/row count by pages * Add page-level row count comparison to analyze discrepancies between testbed and results * Fix key naming in page mismatch comparison for consistency * Merged main into feature/row-count-by-pages * Implement page-level grouping and comparison for row counts in testbed and results * Fix row count difference calculation in page comparison Approved-by: Alex Galarce --- fieldExtraction/src/testbed/test.py | 159 ++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) diff --git a/fieldExtraction/src/testbed/test.py b/fieldExtraction/src/testbed/test.py index 4de5f43..f5aca07 100644 --- a/fieldExtraction/src/testbed/test.py +++ b/fieldExtraction/src/testbed/test.py @@ -45,6 +45,159 @@ row_comparison['results'] = results.groupby('FILE_NAME').size() print("\n*************** Row Count Comparison ****************") print(f"Files with different row counts: {(row_comparison['testbed'] != row_comparison['results']).sum()}") +########################## Page-Level Row Count Comparison ########################## +print("\n*************** Page-Level Row Count Comparison ****************") +def get_consecutive_page_groups(pages) -> list: + """Group consecutive page numbers together. + e.g., [4, 5, 6, 8, 9] -> [[4, 5, 6], [8, 9]] + + """ + + if not pages: + return [] + + # Convert to sorted list of integers + page_nums = sorted([int(p) for p in pages if str(p).isdigit()]) + + if not page_nums: + return [] + + groups = [] + current_group = [page_nums[0]] + + for i in range(1, len(page_nums)): + if page_nums[i] == page_nums[i - 1] + 1: # consecutive pages + current_group.append(page_nums[i]) + else: + groups.append(current_group) + current_group = [page_nums[i]] + groups.append(current_group) # Append the last group + return groups + +def group_name(group): + """Convert a group of consecutive pages to a readable name. + E.g., [4, 5, 6] -> "4-6", [8, 9] -> "8-9", [13] -> "13" + """ + if len(group) == 1: + return str(group[0]) + else: + return f"{group[0]}-{group[-1]}" + +def normalize_page(page_str): + """Normalize page numbers to integer strings for comparison (e.g., "4.0" -> "4", "4" -> "4", "13.0.0" -> "13")""" + if pd.isna(page_str) or page_str == "": + return None + try: + # Split on first period + return str(page_str).strip().split(".")[0] + except (ValueError, TypeError): + return str(page_str).strip() + +if "EXHIBIT_PAGE" in testbed.columns and "EXHIBIT_PAGE" in results.columns: + page_mismatches = [] + for file in testbed['FILE_NAME'].unique(): + testbed_file = testbed[testbed['FILE_NAME'] == file].copy() + results_file = results[results['FILE_NAME'] == file].copy() + + # Normalize page numbers + testbed_file['EXHIBIT_PAGE_NORM'] = testbed_file['EXHIBIT_PAGE'].apply(normalize_page) + results_file['EXHIBIT_PAGE_NORM'] = results_file['EXHIBIT_PAGE'].apply(normalize_page) + + # Get unique pages from each dataset (excluding None/empty) + testbed_pages = set(testbed_file['EXHIBIT_PAGE_NORM'].dropna()) + results_pages = set(results_file['EXHIBIT_PAGE_NORM'].dropna()) + testbed_pages.discard("") # Remove empty strings + results_pages.discard("") # Remove empty strings + + # Group consecutive pages + testbed_groups = get_consecutive_page_groups(testbed_pages) + results_groups = get_consecutive_page_groups(results_pages) + + # Count rows for each group + testbed_group_counts = {} + + for group in testbed_groups: + group_key = group_name(group) + count = sum(results_file['EXHIBIT_PAGE_NORM'].isin([str(p) for p in group])) + testbed_group_counts[group_key] = count + + results_group_counts = {} + + for group in results_groups: + group_key = group_name(group) + count = sum(testbed_file['EXHIBIT_PAGE_NORM'].isin([str(p) for p in group])) + results_group_counts[group_key] = count + + # Compare group counts + all_groups = set(testbed_group_counts.keys()) | set(results_group_counts.keys()) + + group_comparison = {} + has_mismatch = False + + for group_key in all_groups: + testbed_count = testbed_group_counts.get(group_key, 0) + results_count = results_group_counts.get(group_key, 0) + + group_comparison[group_key] = { + 'testbed_count': testbed_count, + 'results_count': results_count, + 'difference': results_count - testbed_count + } + + if testbed_count != results_count: + has_mismatch = True + + if has_mismatch: + page_mismatches.append({ + 'FILE_NAME': file, + 'group_comparison': group_comparison, + 'total_testbed_rows': len(testbed_file), + 'total_results_rows': len(results_file) + }) + + + if page_mismatches: + # Create detailed comparison DataFrame + detailed_page_comparison = [] + for file_data in page_mismatches: + file_name = file_data['FILE_NAME'] + for group_key, counts in file_data['group_comparison'].items(): + detailed_page_comparison.append({ + 'FILE_NAME': file_name, + 'PAGE_GROUP': group_key, + 'TESTBED_COUNT': counts['testbed_count'], + 'RESULTS_COUNT': counts['results_count'], + 'DIFFERENCE': counts['difference'] + }) + page_comparison_df = pd.DataFrame(detailed_page_comparison) + + # Sort by file name and first page number in group + def sort_key(group_str): + """Extract the first page number from a group string for sorting.""" + parts = group_str.split('-') + return int(parts[0]) if parts else float('inf') + page_comparison_df['SORT_KEY'] = page_comparison_df['PAGE_GROUP'].apply(sort_key) + page_comparison_df = page_comparison_df.sort_values(by=['FILE_NAME', 'SORT_KEY']).drop(columns=['SORT_KEY']) + + # Summary statistics + total_missing_rows = page_comparison_df[page_comparison_df['DIFFERENCE'] < 0]['DIFFERENCE'].sum() + total_extra_rows = page_comparison_df[page_comparison_df['DIFFERENCE'] > 0]['DIFFERENCE'].sum() + pages_with_missing = (page_comparison_df['DIFFERENCE'] < 0).sum() + pages_with_extra = (page_comparison_df['DIFFERENCE'] > 0).sum() + + print(f"Pages with missing rows in results: {pages_with_missing}") + print(f"Pages with extra rows in results: {pages_with_extra}") + print(f"Total missing rows: {abs(total_missing_rows)}") + print(f"Total extra rows: {total_extra_rows}") + + + else: + print("All files have matching row counts for consecutive page groups!") + page_comparison_df = pd.DataFrame() +else: + print("EXHIBIT_PAGE column not found in either testbed or results. Skipping page-level row count comparison.") + page_comparison_df = pd.DataFrame() + ########################## One-to-One Analysis ########################## print("\n*************** One-to-One Stats ****************") one_to_one_fields = FieldSet(file_path=config.FIELD_JSON_PATH).filter(relationship="one_to_one").list_fields() @@ -192,6 +345,12 @@ with pd.ExcelWriter(output_file, engine='openpyxl') as writer: stats_df.to_excel(writer, sheet_name='Testbed Stats', index=False) row_comparison.to_excel(writer, sheet_name='Row Counts') empty_fields_df.to_excel(writer, sheet_name='Empty Fields', index=False) + + # Add page-level row count comparison if it exists + if not page_comparison_df.empty: + page_comparison_df.to_excel(writer, sheet_name='Page-Level Row Counts', index=False) + else: + print("No page-level row count mismatches found; skipping export.") # One-to-One Results metrics_df.to_excel(writer, sheet_name='One-to-One Metrics', index=False)