Merged in feature/row-count-by-pages (pull request #585)
Feature/row count by pages * Add page-level row count comparison to analyze discrepancies between testbed and results * Fix key naming in page mismatch comparison for consistency * Merged main into feature/row-count-by-pages * Implement page-level grouping and comparison for row counts in testbed and results * Fix row count difference calculation in page comparison Approved-by: Alex Galarce
This commit is contained in:
Katon Minhas
committed by
Alex Galarce
Alex Galarce
parent
29612ed042
commit
27e09b5fbc
@@ -45,6 +45,159 @@ row_comparison['results'] = results.groupby('FILE_NAME').size()
|
||||
print("\n*************** Row Count Comparison ****************")
|
||||
print(f"Files with different row counts: {(row_comparison['testbed'] != row_comparison['results']).sum()}")
|
||||
|
||||
########################## Page-Level Row Count Comparison ##########################
|
||||
print("\n*************** Page-Level Row Count Comparison ****************")
|
||||
def get_consecutive_page_groups(pages) -> list:
|
||||
"""Group consecutive page numbers together.
|
||||
e.g., [4, 5, 6, 8, 9] -> [[4, 5, 6], [8, 9]]
|
||||
|
||||
"""
|
||||
|
||||
if not pages:
|
||||
return []
|
||||
|
||||
# Convert to sorted list of integers
|
||||
page_nums = sorted([int(p) for p in pages if str(p).isdigit()])
|
||||
|
||||
if not page_nums:
|
||||
return []
|
||||
|
||||
groups = []
|
||||
current_group = [page_nums[0]]
|
||||
|
||||
for i in range(1, len(page_nums)):
|
||||
if page_nums[i] == page_nums[i - 1] + 1: # consecutive pages
|
||||
current_group.append(page_nums[i])
|
||||
else:
|
||||
groups.append(current_group)
|
||||
current_group = [page_nums[i]]
|
||||
groups.append(current_group) # Append the last group
|
||||
return groups
|
||||
|
||||
def group_name(group):
|
||||
"""Convert a group of consecutive pages to a readable name.
|
||||
E.g., [4, 5, 6] -> "4-6", [8, 9] -> "8-9", [13] -> "13"
|
||||
"""
|
||||
if len(group) == 1:
|
||||
return str(group[0])
|
||||
else:
|
||||
return f"{group[0]}-{group[-1]}"
|
||||
|
||||
def normalize_page(page_str):
|
||||
"""Normalize page numbers to integer strings for comparison (e.g., "4.0" -> "4", "4" -> "4", "13.0.0" -> "13")"""
|
||||
if pd.isna(page_str) or page_str == "":
|
||||
return None
|
||||
try:
|
||||
# Split on first period
|
||||
return str(page_str).strip().split(".")[0]
|
||||
except (ValueError, TypeError):
|
||||
return str(page_str).strip()
|
||||
|
||||
if "EXHIBIT_PAGE" in testbed.columns and "EXHIBIT_PAGE" in results.columns:
|
||||
page_mismatches = []
|
||||
for file in testbed['FILE_NAME'].unique():
|
||||
testbed_file = testbed[testbed['FILE_NAME'] == file].copy()
|
||||
results_file = results[results['FILE_NAME'] == file].copy()
|
||||
|
||||
# Normalize page numbers
|
||||
testbed_file['EXHIBIT_PAGE_NORM'] = testbed_file['EXHIBIT_PAGE'].apply(normalize_page)
|
||||
results_file['EXHIBIT_PAGE_NORM'] = results_file['EXHIBIT_PAGE'].apply(normalize_page)
|
||||
|
||||
# Get unique pages from each dataset (excluding None/empty)
|
||||
testbed_pages = set(testbed_file['EXHIBIT_PAGE_NORM'].dropna())
|
||||
results_pages = set(results_file['EXHIBIT_PAGE_NORM'].dropna())
|
||||
testbed_pages.discard("") # Remove empty strings
|
||||
results_pages.discard("") # Remove empty strings
|
||||
|
||||
# Group consecutive pages
|
||||
testbed_groups = get_consecutive_page_groups(testbed_pages)
|
||||
results_groups = get_consecutive_page_groups(results_pages)
|
||||
|
||||
# Count rows for each group
|
||||
testbed_group_counts = {}
|
||||
|
||||
for group in testbed_groups:
|
||||
group_key = group_name(group)
|
||||
count = sum(results_file['EXHIBIT_PAGE_NORM'].isin([str(p) for p in group]))
|
||||
testbed_group_counts[group_key] = count
|
||||
|
||||
results_group_counts = {}
|
||||
|
||||
for group in results_groups:
|
||||
group_key = group_name(group)
|
||||
count = sum(testbed_file['EXHIBIT_PAGE_NORM'].isin([str(p) for p in group]))
|
||||
results_group_counts[group_key] = count
|
||||
|
||||
# Compare group counts
|
||||
all_groups = set(testbed_group_counts.keys()) | set(results_group_counts.keys())
|
||||
|
||||
group_comparison = {}
|
||||
has_mismatch = False
|
||||
|
||||
for group_key in all_groups:
|
||||
testbed_count = testbed_group_counts.get(group_key, 0)
|
||||
results_count = results_group_counts.get(group_key, 0)
|
||||
|
||||
group_comparison[group_key] = {
|
||||
'testbed_count': testbed_count,
|
||||
'results_count': results_count,
|
||||
'difference': results_count - testbed_count
|
||||
}
|
||||
|
||||
if testbed_count != results_count:
|
||||
has_mismatch = True
|
||||
|
||||
if has_mismatch:
|
||||
page_mismatches.append({
|
||||
'FILE_NAME': file,
|
||||
'group_comparison': group_comparison,
|
||||
'total_testbed_rows': len(testbed_file),
|
||||
'total_results_rows': len(results_file)
|
||||
})
|
||||
|
||||
|
||||
if page_mismatches:
|
||||
# Create detailed comparison DataFrame
|
||||
detailed_page_comparison = []
|
||||
for file_data in page_mismatches:
|
||||
file_name = file_data['FILE_NAME']
|
||||
for group_key, counts in file_data['group_comparison'].items():
|
||||
detailed_page_comparison.append({
|
||||
'FILE_NAME': file_name,
|
||||
'PAGE_GROUP': group_key,
|
||||
'TESTBED_COUNT': counts['testbed_count'],
|
||||
'RESULTS_COUNT': counts['results_count'],
|
||||
'DIFFERENCE': counts['difference']
|
||||
})
|
||||
page_comparison_df = pd.DataFrame(detailed_page_comparison)
|
||||
|
||||
# Sort by file name and first page number in group
|
||||
def sort_key(group_str):
|
||||
"""Extract the first page number from a group string for sorting."""
|
||||
parts = group_str.split('-')
|
||||
return int(parts[0]) if parts else float('inf')
|
||||
page_comparison_df['SORT_KEY'] = page_comparison_df['PAGE_GROUP'].apply(sort_key)
|
||||
page_comparison_df = page_comparison_df.sort_values(by=['FILE_NAME', 'SORT_KEY']).drop(columns=['SORT_KEY'])
|
||||
|
||||
# Summary statistics
|
||||
total_missing_rows = page_comparison_df[page_comparison_df['DIFFERENCE'] < 0]['DIFFERENCE'].sum()
|
||||
total_extra_rows = page_comparison_df[page_comparison_df['DIFFERENCE'] > 0]['DIFFERENCE'].sum()
|
||||
pages_with_missing = (page_comparison_df['DIFFERENCE'] < 0).sum()
|
||||
pages_with_extra = (page_comparison_df['DIFFERENCE'] > 0).sum()
|
||||
|
||||
print(f"Pages with missing rows in results: {pages_with_missing}")
|
||||
print(f"Pages with extra rows in results: {pages_with_extra}")
|
||||
print(f"Total missing rows: {abs(total_missing_rows)}")
|
||||
print(f"Total extra rows: {total_extra_rows}")
|
||||
|
||||
|
||||
else:
|
||||
print("All files have matching row counts for consecutive page groups!")
|
||||
page_comparison_df = pd.DataFrame()
|
||||
else:
|
||||
print("EXHIBIT_PAGE column not found in either testbed or results. Skipping page-level row count comparison.")
|
||||
page_comparison_df = pd.DataFrame()
|
||||
|
||||
########################## One-to-One Analysis ##########################
|
||||
print("\n*************** One-to-One Stats ****************")
|
||||
one_to_one_fields = FieldSet(file_path=config.FIELD_JSON_PATH).filter(relationship="one_to_one").list_fields()
|
||||
@@ -192,6 +345,12 @@ with pd.ExcelWriter(output_file, engine='openpyxl') as writer:
|
||||
stats_df.to_excel(writer, sheet_name='Testbed Stats', index=False)
|
||||
row_comparison.to_excel(writer, sheet_name='Row Counts')
|
||||
empty_fields_df.to_excel(writer, sheet_name='Empty Fields', index=False)
|
||||
|
||||
# Add page-level row count comparison if it exists
|
||||
if not page_comparison_df.empty:
|
||||
page_comparison_df.to_excel(writer, sheet_name='Page-Level Row Counts', index=False)
|
||||
else:
|
||||
print("No page-level row count mismatches found; skipping export.")
|
||||
|
||||
# One-to-One Results
|
||||
metrics_df.to_excel(writer, sheet_name='One-to-One Metrics', index=False)
|
||||
|
||||
Reference in New Issue
Block a user