Katon Minhas
afb6d5185d
Feature/lesser table caching refactor hybrid * chore: Remove unused duplicate main.py from shared pipeline * fix: Correct crosswalk paths in aarete_derived.py * chore: Remove unused documentation files from fieldExtraction * docs: Add documentation files to documentation folder * docs: Update README with uv setup, expanded project structure, and branching conventions * docs: Add uv installation steps with Ubuntu/WSL emphasis * Enable prompt caching for all remaining LLM calls - Add _INSTRUCTION() functions for: EXHIBIT_HEADER, EXHIBIT_LINKAGE, EXHIBIT_TITLE_MATCH, DATE_FIX, DERIVED_TERM_DATE, CHECK_PROVIDER_NAME_MATCH, SPECIAL_CASE_ASSIGNMENT - Update all invoke_claude() calls in saas and clover pipelines to use cache=True with corresponding _INSTRUCTION() functions - Add new instructions to get_cacheable_instructions() for cache warming - Update tests for new instruction functions Functions now using caching: - prompt_exhibit_level - prompt_exhibit_lesser (EXHIBIT_LEVEL_LESSER_OF) - prompt_fee_schedule_breakout - prompt_grouper_breakout - prompt_special_case_assignment - prompt_exhibit_linkage - prompt_exhibit_header - prompt_smart_chunked (ONE_TO_ONE templates) - prompt_date_fix - prompt_derived_term_date - prompt_exhibit_title_match - provider_name_match_check 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Reorder * feat: Add bcbs_promise client pipeline with OFFSET_TERM extraction - Add new bcbs_promise client with HSC-based OFFSET_TERM field extraction - Extract full paragraph text of offset/recoupment provisions from contracts - Derive OFFSET_INDICATOR (Y/N) from OFFSET_TERM presence - Fix reorder_columns to preserve extra columns not in COLUMN_ORDER - Update QC/QA output path to outputs/qc_qa/ * fix: Update dev deps and test assertions for QC/QA output path - Add pytest/pytest-mock to dev dependencies for mypy type checking - Update test assertions to expect outputs/qc_qa instead of qa_qc_output * style: Apply black formatting to prompt_templates.py * Merge main, move scripts * Archive some scripts * update py version * remove .py version file * Remove ASCII characters * Restore testbed code * restore tracking * Update testbed metrics * Enable prompt caching for CODE_LAST_CHECK, FILL_BILL_TYPE, DUAL_LOB_CHECK, and GROUPER_BREAKOUT - Add CODE_LAST_CHECK_INSTRUCTION() for service specificity classification - Add FILL_BILL_TYPE_INSTRUCTION() for bill type code determination - Add DUAL_LOB_CHECK_INSTRUCTION() for Medicare/Medicaid classification - Update code_funcs.py to use caching for CODE_LAST_CHECK, FILL_BILL_TYPE, GROUPER_BREAKOUT - Update postprocessing_funcs.py to use caching for DUAL_LOB_CHECK - Add new instructions to get_cacheable_instructions() for cache warming - Add unit tests for new instruction functions 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Fix postprocessing_funcs to remove invalid columns * Merge branch 'main' into feature/lesser-table-caching-refactor-hybrid * Revert prompt caching changes from aed1b73c * update formatting * Update imports Approved-by: Sha Brown Approved-by: Praneel Panchigar
65 lines
2.4 KiB
Python
65 lines
2.4 KiB
Python
import csv
|
|
import boto3
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from botocore.exceptions import ClientError
|
|
|
|
"""
|
|
This script is used to search for files in an S3 bucket and copy them to a new location.
|
|
This is used when CNC team comes back with a list of missing files and we need to search for them in the S3 bucket.
|
|
|
|
We usually run this script to first just search for the files without copying, then we analyze the results and if needed, we run the script again with copy=True.
|
|
|
|
"""
|
|
|
|
|
|
|
|
s3 = boto3.Session(profile_name='temp_cred').client('s3')
|
|
|
|
def list_all_files(bucket_name):
|
|
all_keys = []
|
|
try:
|
|
paginator = s3.get_paginator('list_objects_v2')
|
|
for page in paginator.paginate(Bucket=bucket_name):
|
|
if 'Contents' in page:
|
|
all_keys.extend([item['Key'] for item in page['Contents']])
|
|
except ClientError as e:
|
|
print(f"Error listing files in bucket {bucket_name}: {e}")
|
|
return all_keys
|
|
|
|
def check_file_in_bucket(filename, all_keys):
|
|
locations = [key for key in all_keys if (key.endswith(filename))]
|
|
if copy:
|
|
for l in locations:
|
|
s3.copy_object(
|
|
CopySource={'Bucket': bucket_name, 'Key': l},
|
|
Bucket=bucket_name,
|
|
Key=destination_prefix + "/" + filename
|
|
)
|
|
exists = bool(locations)
|
|
return filename, exists, locations
|
|
|
|
def process_csv(input_csv, output_csv, bucket_name):
|
|
|
|
with open(input_csv, mode='r', newline='', encoding='utf8') as infile, open(output_csv, mode='w', newline='', encoding='utf8') as outfile:
|
|
reader = csv.reader(infile)
|
|
writer = csv.writer(outfile)
|
|
writer.writerow(['File Name', 'exists', 'locations']) # Write header
|
|
|
|
with ThreadPoolExecutor() as executor:
|
|
futures = [executor.submit(check_file_in_bucket, row[0] + file_extension, all_keys) for row in reader]
|
|
|
|
for future in futures:
|
|
filename, exists, locations = future.result()
|
|
writer.writerow([filename, exists, locations])
|
|
|
|
input_csv = 'batch11/batch11_missing_files.csv'
|
|
output_csv = 'batch11/batch11_missing_files_movement_confirmation.csv'
|
|
bucket_name = 'centene-national-contracting-files'
|
|
destination_prefix = 'batch_11_priority_files/diff_files_v2'
|
|
copy = True
|
|
file_extension = '.txt'
|
|
|
|
all_keys = list_all_files(bucket_name)
|
|
process_csv(input_csv, output_csv, bucket_name)
|
|
print("CSV processing completed.")
|