jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/update-testbed-metrics (pull request #750)

Browse Source 
Feature/update testbed metrics

* Clean up filename filter function

* Strip out unused tests

* Strip out page-level comparison

* Add 1:1 fields back to analysis:

* Run through

* Merged main into feature/update-testbed-metrics

* Merged main into feature/update-testbed-metrics

* Merged main into feature/update-testbed-metrics

* Merged main into feature/update-testbed-metrics

* Merged main into feature/update-testbed-metrics


Approved-by: Faizan Mohiuddin
This commit is contained in:
Katon Minhas
2025-10-31 19:08:10 +00:00
parent 3361c689fb
commit 07dc47e9d1
2 changed files with 11 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fieldExtraction/src/testbed/testbed_metrics.py
+1 -7
View File
@@ -12,11 +12,9 @@ warnings.filterwarnings("ignore")
testbed = pd.read_excel("Doczy-Testbed.xlsx") # This is the testbed
results = pd.read_csv("Doczy-Results.csv") # This is the doczy output
# Preprocess testbed and results data
########################## Preprocess ##########################
testbed = testbed_utils.testbed_preprocess(testbed)
results = testbed_utils.testbed_preprocess(results)
# Filter out filenames we don't want to include
testbed, results = testbed_utils.filter_file_names(testbed, results)
########################## Analyze Testbed ##########################
@@ -25,9 +23,6 @@ testbed_utils.get_df_stats(testbed)
########################## Row Comparison ##########################
row_comparison = testbed_utils.get_row_comparison(testbed, results)
########################## Page-Level Row Count Comparison ##########################
page_comparison_df = testbed_utils.get_page_comparison_df(testbed, results)
########################## One-to-One Analysis ##########################
metrics_df, comparison_df = testbed_utils.get_one_to_one_analysis(testbed, results)
@@ -51,7 +46,6 @@ testbed_utils.export_to_excel(
output_file,
testbed,
row_comparison,
page_comparison_df,
metrics_df,
comparison_df,
one_to_n_metrics,
fieldExtraction/src/testbed/testbed_utils.py
+10 -171
View File
@@ -17,18 +17,15 @@ def filter_file_names(testbed, results):
common_file_names = set(testbed["FILE_NAME"]).intersection(
set(results["FILE_NAME"])
)
print(f"common_file_names: {len(common_file_names)} items")
if (
"86-0898663-Founder Health, LLC dba Preferred Homecare-ICMProviderAgreement_120661".upper()
in common_file_names
):
print(
"found 86-0898663-Founder Health, LLC dba Preferred Homecare-ICMProviderAgreement_120661 in both testbed and results; removing it from both datasets"
)
common_file_names.remove(
"86-0898663-Founder Health, LLC dba Preferred Homecare-ICMProviderAgreement_120661".upper()
)
print(f"common_file_names now has {len(common_file_names)} items")
files_to_remove = [
"86-0898663-Founder Health, LLC dba Preferred Homecare-ICMProviderAgreement_120661"
]
for filename in files_to_remove:
if filename in common_file_names:
common_file_names.remove(filename.upper())
testbed = testbed[testbed["FILE_NAME"].isin(common_file_names)]
results = results[results["FILE_NAME"].isin(common_file_names)]
return testbed, results
@@ -103,18 +100,6 @@ def testbed_preprocess(df):
# Ensure that all empty cells are filled with empty strings
df = df.fillna("")
# # Create multiple rows for multiple values stored in columns AARETE_DERIVED_LOB, AARETE_DERIVED_PROGRAM, AARETE_DERIVED_NETWORK
# # to be implemented later
# for col in ['AARETE_DERIVED_LOB', 'AARETE_DERIVED_PROGRAM', 'AARETE_DERIVED_NETWORK']:
# if col in df.columns:
# # Split the column by "|" and explode into multiple rows
# df[col] = df[col].str.split("|")
# df = df.explode(col)
# # Remove any leading/trailing whitespace from exploded values
# df[col] = df[col].str.strip()
# df = df.reset_index(drop=True)
return df
@@ -1186,141 +1171,6 @@ def normalize_page(page_str):
except (ValueError, TypeError):
return str(page_str).strip()
def get_page_comparison_df(testbed, results):
print("\n*************** Page-Level Row Count Comparison ****************")
if "EXHIBIT_PAGE" in testbed.columns and "EXHIBIT_PAGE" in results.columns:
page_mismatches = []
for file in testbed["FILE_NAME"].unique():
testbed_file = testbed[testbed["FILE_NAME"] == file].copy()
results_file = results[results["FILE_NAME"] == file].copy()
# Normalize page numbers
testbed_file["EXHIBIT_PAGE_NORM"] = testbed_file["EXHIBIT_PAGE"].apply(
normalize_page
)
results_file["EXHIBIT_PAGE_NORM"] = results_file["EXHIBIT_PAGE"].apply(
normalize_page
)
# Get unique pages from each dataset (excluding None/empty)
testbed_pages = set(testbed_file["EXHIBIT_PAGE_NORM"].dropna())
results_pages = set(results_file["EXHIBIT_PAGE_NORM"].dropna())
testbed_pages.discard("") # Remove empty strings
results_pages.discard("") # Remove empty strings
# Group consecutive pages
testbed_groups = get_consecutive_page_groups(testbed_pages)
results_groups = get_consecutive_page_groups(results_pages)
# Count rows for each group
testbed_group_counts = {}
for group in testbed_groups:
group_key = group_name(group)
count = sum(
testbed_file["EXHIBIT_PAGE_NORM"].isin([str(p) for p in group])
)
testbed_group_counts[group_key] = count
results_group_counts = {}
for group in results_groups:
group_key = group_name(group)
count = sum(
results_file["EXHIBIT_PAGE_NORM"].isin([str(p) for p in group])
)
results_group_counts[group_key] = count
# Compare group counts
all_groups = set(testbed_group_counts.keys()) | set(
results_group_counts.keys()
)
group_comparison = {}
has_mismatch = False
for group_key in all_groups:
testbed_count = testbed_group_counts.get(group_key, 0)
results_count = results_group_counts.get(group_key, 0)
group_comparison[group_key] = {
"testbed_count": testbed_count,
"results_count": results_count,
"difference": results_count - testbed_count,
}
if testbed_count != results_count:
has_mismatch = True
if has_mismatch:
page_mismatches.append(
{
"FILE_NAME": file,
"group_comparison": group_comparison,
"total_testbed_rows": len(testbed_file),
"total_results_rows": len(results_file),
}
)
if page_mismatches:
# Create detailed comparison DataFrame
detailed_page_comparison = []
for file_data in page_mismatches:
file_name = file_data["FILE_NAME"]
for group_key, counts in file_data["group_comparison"].items():
detailed_page_comparison.append(
{
"FILE_NAME": file_name,
"PAGE_GROUP": group_key,
"TESTBED_COUNT": counts["testbed_count"],
"RESULTS_COUNT": counts["results_count"],
"DIFFERENCE": counts["difference"],
}
)
page_comparison_df = pd.DataFrame(detailed_page_comparison)
# Sort by file name and first page number in group
def sort_key(group_str):
"""Extract the first page number from a group string for sorting."""
parts = group_str.split("-")
return int(parts[0]) if parts else float("inf")
page_comparison_df["SORT_KEY"] = page_comparison_df["PAGE_GROUP"].apply(
sort_key
)
page_comparison_df = page_comparison_df.sort_values(
by=["FILE_NAME", "SORT_KEY"]
).drop(columns=["SORT_KEY"])
# Summary statistics
total_missing_rows = page_comparison_df[
page_comparison_df["DIFFERENCE"] < 0
]["DIFFERENCE"].sum()
total_extra_rows = page_comparison_df[page_comparison_df["DIFFERENCE"] > 0][
"DIFFERENCE"
].sum()
pages_with_missing = (page_comparison_df["DIFFERENCE"] < 0).sum()
pages_with_extra = (page_comparison_df["DIFFERENCE"] > 0).sum()
print(f"Pages with missing rows in results: {pages_with_missing}")
print(f"Pages with extra rows in results: {pages_with_extra}")
print(f"Total missing rows: {abs(total_missing_rows)}")
print(f"Total extra rows: {total_extra_rows}")
else:
print("All files have matching row counts for consecutive page groups!")
page_comparison_df = pd.DataFrame()
else:
print(
"EXHIBIT_PAGE column not found in either testbed or results. Skipping page-level row count comparison."
)
page_comparison_df = pd.DataFrame()
return page_comparison_df
def get_one_to_one_analysis(testbed, results):
print("\n*************** One-to-One Stats ****************")
one_to_one_fields = (
@@ -1349,9 +1199,7 @@ def get_one_to_one_analysis(testbed, results):
)
comparison_df = pd.DataFrame(index=testbed["FILE_NAME"].unique())
# for field in one_to_one_fields:
for field in ["PROV_GROUP_TIN"]:
for field in one_to_one_fields:
if (
field in testbed.columns
and field in results.columns
@@ -1548,7 +1396,6 @@ def export_to_excel(
output_file,
testbed,
row_comparison,
page_comparison_df,
metrics_df,
comparison_df,
one_to_n_metrics,
@@ -1575,14 +1422,6 @@ def export_to_excel(
row_comparison.to_excel(writer, sheet_name="Row Counts")
empty_fields_df.to_excel(writer, sheet_name="Empty Fields", index=False)
# Add page-level row count comparison if it exists
if not page_comparison_df.empty:
page_comparison_df.to_excel(
writer, sheet_name="Page-Level Row Counts", index=False
)
else:
print("No page-level row count mismatches found; skipping export.")
# One-to-One Results
metrics_df.to_excel(writer, sheet_name="One-to-One Metrics", index=False)
comparison_df.to_excel(writer, sheet_name="One-to-One Details") # Add new sheet