Merged in feature/lesser-table-caching-refactor-hybrid (pull request #847)
Feature/lesser table caching refactor hybrid * chore: Remove unused duplicate main.py from shared pipeline * fix: Correct crosswalk paths in aarete_derived.py * chore: Remove unused documentation files from fieldExtraction * docs: Add documentation files to documentation folder * docs: Update README with uv setup, expanded project structure, and branching conventions * docs: Add uv installation steps with Ubuntu/WSL emphasis * Enable prompt caching for all remaining LLM calls - Add _INSTRUCTION() functions for: EXHIBIT_HEADER, EXHIBIT_LINKAGE, EXHIBIT_TITLE_MATCH, DATE_FIX, DERIVED_TERM_DATE, CHECK_PROVIDER_NAME_MATCH, SPECIAL_CASE_ASSIGNMENT - Update all invoke_claude() calls in saas and clover pipelines to use cache=True with corresponding _INSTRUCTION() functions - Add new instructions to get_cacheable_instructions() for cache warming - Update tests for new instruction functions Functions now using caching: - prompt_exhibit_level - prompt_exhibit_lesser (EXHIBIT_LEVEL_LESSER_OF) - prompt_fee_schedule_breakout - prompt_grouper_breakout - prompt_special_case_assignment - prompt_exhibit_linkage - prompt_exhibit_header - prompt_smart_chunked (ONE_TO_ONE templates) - prompt_date_fix - prompt_derived_term_date - prompt_exhibit_title_match - provider_name_match_check 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Reorder * feat: Add bcbs_promise client pipeline with OFFSET_TERM extraction - Add new bcbs_promise client with HSC-based OFFSET_TERM field extraction - Extract full paragraph text of offset/recoupment provisions from contracts - Derive OFFSET_INDICATOR (Y/N) from OFFSET_TERM presence - Fix reorder_columns to preserve extra columns not in COLUMN_ORDER - Update QC/QA output path to outputs/qc_qa/ * fix: Update dev deps and test assertions for QC/QA output path - Add pytest/pytest-mock to dev dependencies for mypy type checking - Update test assertions to expect outputs/qc_qa instead of qa_qc_output * style: Apply black formatting to prompt_templates.py * Merge main, move scripts * Archive some scripts * update py version * remove .py version file * Remove ASCII characters * Restore testbed code * restore tracking * Update testbed metrics * Enable prompt caching for CODE_LAST_CHECK, FILL_BILL_TYPE, DUAL_LOB_CHECK, and GROUPER_BREAKOUT - Add CODE_LAST_CHECK_INSTRUCTION() for service specificity classification - Add FILL_BILL_TYPE_INSTRUCTION() for bill type code determination - Add DUAL_LOB_CHECK_INSTRUCTION() for Medicare/Medicaid classification - Update code_funcs.py to use caching for CODE_LAST_CHECK, FILL_BILL_TYPE, GROUPER_BREAKOUT - Update postprocessing_funcs.py to use caching for DUAL_LOB_CHECK - Add new instructions to get_cacheable_instructions() for cache warming - Add unit tests for new instruction functions 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Fix postprocessing_funcs to remove invalid columns * Merge branch 'main' into feature/lesser-table-caching-refactor-hybrid * Revert prompt caching changes from aed1b73c * update formatting * Update imports Approved-by: Sha Brown Approved-by: Praneel Panchigar
This commit is contained in:
Katon Minhas
@@ -0,0 +1,193 @@
|
||||
import src.config as config
|
||||
from datetime import datetime
|
||||
import pandas as pd
|
||||
import src.utils.string_utils as string_utils
|
||||
import io
|
||||
|
||||
|
||||
def find_inv_test_results():
|
||||
"""
|
||||
Efficiently find the 30 most recent directories containing 'inv-test'
|
||||
and list their RESULTS.csv files.
|
||||
|
||||
Strategy: Only list top-level directories first, filter for 'inv-test',
|
||||
then search only those directories for RESULTS.csv files.
|
||||
"""
|
||||
s3_client = config.S3_CLIENT
|
||||
bucket_name = config.S3_OUTPUT_BUCKET
|
||||
|
||||
print(f"\nSearching for 'inv-test' directories in bucket: {bucket_name}")
|
||||
print("=" * 80)
|
||||
|
||||
# Step 1: Get only top-level directories (much faster than scanning all files)
|
||||
print("Step 1: Listing top-level directories...")
|
||||
response = s3_client.list_objects_v2(Bucket=bucket_name, Delimiter="/")
|
||||
|
||||
if "CommonPrefixes" not in response:
|
||||
print("No directories found in bucket")
|
||||
return []
|
||||
|
||||
all_top_level_dirs = [
|
||||
prefix["Prefix"].rstrip("/") for prefix in response["CommonPrefixes"]
|
||||
]
|
||||
print(f"Found {len(all_top_level_dirs)} total top-level directories")
|
||||
|
||||
# Step 2: Filter for 'inv-test' directories only
|
||||
inv_test_dirs = [d for d in all_top_level_dirs if "inv-test" in d]
|
||||
print(f"Found {len(inv_test_dirs)} directories containing 'inv-test'")
|
||||
|
||||
if not inv_test_dirs:
|
||||
print("No 'inv-test' directories found")
|
||||
return []
|
||||
|
||||
# Step 3: Get timestamps for inv-test directories only (not all files in bucket)
|
||||
print("\nStep 2: Getting timestamps for inv-test directories...")
|
||||
dir_timestamps = []
|
||||
|
||||
for dir_name in inv_test_dirs:
|
||||
# Get just the first file in each directory to get a timestamp
|
||||
response = s3_client.list_objects_v2(
|
||||
Bucket=bucket_name, Prefix=f"{dir_name}/", MaxKeys=1
|
||||
)
|
||||
|
||||
if "Contents" in response and len(response["Contents"]) > 0:
|
||||
timestamp = response["Contents"][0]["LastModified"]
|
||||
dir_timestamps.append((dir_name, timestamp))
|
||||
|
||||
# Sort by timestamp (most recent first)
|
||||
dir_timestamps.sort(key=lambda x: x[1], reverse=True)
|
||||
|
||||
# Get the 30 most recent
|
||||
recent_inv_test_dirs = dir_timestamps[:30]
|
||||
|
||||
print(
|
||||
f"\nStep 3: Searching 30 most recent 'inv-test' directories for RESULTS.csv files:"
|
||||
)
|
||||
print("=" * 80)
|
||||
|
||||
total_results_files = 0
|
||||
all_results_files = {} # Dictionary to store directory -> list of file paths
|
||||
|
||||
for i, (dir_name, timestamp) in enumerate(recent_inv_test_dirs, 1):
|
||||
print(f"\n[{i}/30] Directory: {dir_name}")
|
||||
print(f" Last Modified: {timestamp.strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
|
||||
# Search only this directory for RESULTS.csv files
|
||||
results_files = []
|
||||
paginator = s3_client.get_paginator("list_objects_v2")
|
||||
|
||||
for page in paginator.paginate(Bucket=bucket_name, Prefix=f"{dir_name}/"):
|
||||
if "Contents" not in page:
|
||||
continue
|
||||
|
||||
for obj in page["Contents"]:
|
||||
key = obj["Key"]
|
||||
# Ignore files in subdirectories named "individual"
|
||||
if "/individual/" in key:
|
||||
continue
|
||||
if key.endswith("-RESULTS.csv"):
|
||||
results_files.append(key)
|
||||
|
||||
if results_files:
|
||||
print(f" Found {len(results_files)} RESULTS.csv file(s):")
|
||||
for file_key in results_files:
|
||||
print(f" ✓ {file_key}")
|
||||
total_results_files += 1
|
||||
all_results_files[dir_name] = results_files
|
||||
else:
|
||||
print(f" ✗ No -RESULTS.csv files found")
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
print(f"Summary:")
|
||||
print(f" - Searched 30 most recent 'inv-test' directories")
|
||||
print(f" - Total -RESULTS.csv files found: {total_results_files}")
|
||||
print("=" * 80)
|
||||
|
||||
# Step 4: Generate missingness report
|
||||
if total_results_files > 0:
|
||||
print(f"\nStep 4: Generating missingness report...")
|
||||
generate_missingness_report(s3_client, bucket_name, all_results_files)
|
||||
|
||||
return recent_inv_test_dirs
|
||||
|
||||
|
||||
def generate_missingness_report(s3_client, bucket_name, results_files_dict):
|
||||
"""
|
||||
Generate a missingness report for all RESULTS.csv files.
|
||||
|
||||
Args:
|
||||
s3_client: S3 client object
|
||||
bucket_name: S3 bucket name
|
||||
results_files_dict: Dictionary mapping directory names to lists of result file paths
|
||||
"""
|
||||
print("=" * 80)
|
||||
print("Generating Missingness Report")
|
||||
print("=" * 80)
|
||||
|
||||
missingness_data = []
|
||||
|
||||
for dir_name, file_paths in results_files_dict.items():
|
||||
for file_path in file_paths:
|
||||
print(f"\nProcessing: {file_path}")
|
||||
|
||||
try:
|
||||
# Read CSV from S3
|
||||
response = s3_client.get_object(Bucket=bucket_name, Key=file_path)
|
||||
csv_content = response["Body"].read().decode("utf-8")
|
||||
df = pd.read_csv(io.StringIO(csv_content))
|
||||
|
||||
if df.empty:
|
||||
print(f" ⚠ File is empty, skipping")
|
||||
continue
|
||||
|
||||
# Calculate missingness for each column
|
||||
row_data = {"csv_file": file_path}
|
||||
|
||||
for column in df.columns:
|
||||
empty_mask = string_utils.is_empty(df[column], pd_mask=True)
|
||||
missingness_pct = (empty_mask.sum() / len(df)) * 100
|
||||
row_data[column] = round(missingness_pct, 2)
|
||||
|
||||
missingness_data.append(row_data)
|
||||
print(f" ✓ Processed {len(df)} rows, {len(df.columns)} columns")
|
||||
|
||||
except Exception as e:
|
||||
print(f" ✗ Error processing file: {e}")
|
||||
continue
|
||||
|
||||
if not missingness_data:
|
||||
print("\nNo valid data to create report")
|
||||
return
|
||||
|
||||
# Create missingness DataFrame
|
||||
missingness_df = pd.DataFrame(missingness_data)
|
||||
|
||||
# Set csv_file as index for better readability
|
||||
missingness_df.set_index("csv_file", inplace=True)
|
||||
|
||||
# Save report
|
||||
output_filename = (
|
||||
f"missingness_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv"
|
||||
)
|
||||
missingness_df.to_csv(output_filename)
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
print(f"Missingness Report Summary:")
|
||||
print(f" - Total CSV files processed: {len(missingness_data)}")
|
||||
print(f" - Total columns analyzed: {len(missingness_df.columns)}")
|
||||
print(f" - Report saved to: {output_filename}")
|
||||
print("=" * 80)
|
||||
|
||||
# Display sample of the report
|
||||
print("\nSample of Missingness Report (first 5 columns):")
|
||||
print("-" * 80)
|
||||
if len(missingness_df.columns) > 5:
|
||||
print(missingness_df.iloc[:, :5].to_string())
|
||||
print(f"\n... and {len(missingness_df.columns) - 5} more columns")
|
||||
else:
|
||||
print(missingness_df.to_string())
|
||||
print("-" * 80)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
find_inv_test_results()
|
||||
Reference in New Issue
Block a user