Katon Minhas
afb6d5185d
Feature/lesser table caching refactor hybrid * chore: Remove unused duplicate main.py from shared pipeline * fix: Correct crosswalk paths in aarete_derived.py * chore: Remove unused documentation files from fieldExtraction * docs: Add documentation files to documentation folder * docs: Update README with uv setup, expanded project structure, and branching conventions * docs: Add uv installation steps with Ubuntu/WSL emphasis * Enable prompt caching for all remaining LLM calls - Add _INSTRUCTION() functions for: EXHIBIT_HEADER, EXHIBIT_LINKAGE, EXHIBIT_TITLE_MATCH, DATE_FIX, DERIVED_TERM_DATE, CHECK_PROVIDER_NAME_MATCH, SPECIAL_CASE_ASSIGNMENT - Update all invoke_claude() calls in saas and clover pipelines to use cache=True with corresponding _INSTRUCTION() functions - Add new instructions to get_cacheable_instructions() for cache warming - Update tests for new instruction functions Functions now using caching: - prompt_exhibit_level - prompt_exhibit_lesser (EXHIBIT_LEVEL_LESSER_OF) - prompt_fee_schedule_breakout - prompt_grouper_breakout - prompt_special_case_assignment - prompt_exhibit_linkage - prompt_exhibit_header - prompt_smart_chunked (ONE_TO_ONE templates) - prompt_date_fix - prompt_derived_term_date - prompt_exhibit_title_match - provider_name_match_check 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Reorder * feat: Add bcbs_promise client pipeline with OFFSET_TERM extraction - Add new bcbs_promise client with HSC-based OFFSET_TERM field extraction - Extract full paragraph text of offset/recoupment provisions from contracts - Derive OFFSET_INDICATOR (Y/N) from OFFSET_TERM presence - Fix reorder_columns to preserve extra columns not in COLUMN_ORDER - Update QC/QA output path to outputs/qc_qa/ * fix: Update dev deps and test assertions for QC/QA output path - Add pytest/pytest-mock to dev dependencies for mypy type checking - Update test assertions to expect outputs/qc_qa instead of qa_qc_output * style: Apply black formatting to prompt_templates.py * Merge main, move scripts * Archive some scripts * update py version * remove .py version file * Remove ASCII characters * Restore testbed code * restore tracking * Update testbed metrics * Enable prompt caching for CODE_LAST_CHECK, FILL_BILL_TYPE, DUAL_LOB_CHECK, and GROUPER_BREAKOUT - Add CODE_LAST_CHECK_INSTRUCTION() for service specificity classification - Add FILL_BILL_TYPE_INSTRUCTION() for bill type code determination - Add DUAL_LOB_CHECK_INSTRUCTION() for Medicare/Medicaid classification - Update code_funcs.py to use caching for CODE_LAST_CHECK, FILL_BILL_TYPE, GROUPER_BREAKOUT - Update postprocessing_funcs.py to use caching for DUAL_LOB_CHECK - Add new instructions to get_cacheable_instructions() for cache warming - Add unit tests for new instruction functions 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Fix postprocessing_funcs to remove invalid columns * Merge branch 'main' into feature/lesser-table-caching-refactor-hybrid * Revert prompt caching changes from aed1b73c * update formatting * Update imports Approved-by: Sha Brown Approved-by: Praneel Panchigar
78 lines
3.3 KiB
Python
78 lines
3.3 KiB
Python
import boto3
|
|
import pandas as pd
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
|
"""
|
|
This script is used to search for files in a specific S3 bucket with a specific prefix and copy them to another S3 bucket with a different prefix.
|
|
|
|
This is similar to the script s3_search_and_copy.py, but this script is designed to be used with a CSV file that contains a list of file names to search for in the prefix to limit the scope.
|
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
s3_client = boto3.Session(profile_name='temp_cred').client('s3')
|
|
|
|
def get_filenames_from_s3(bucket, prefix):
|
|
paginator = s3_client.get_paginator('list_objects_v2')
|
|
page_iterator = paginator.paginate(Bucket=bucket, Prefix=prefix)
|
|
|
|
filenames = set()
|
|
for page in page_iterator:
|
|
if 'Contents' in page:
|
|
for obj in page['Contents']:
|
|
filenames.add((obj['Key'].split('/')[-1])[:-4])
|
|
|
|
return filenames
|
|
|
|
def copy_file_if_exists(file_name, source_bucket, source_prefix, destination_bucket, destination_prefix, existing_files):
|
|
if file_name in existing_files:
|
|
source_key = f"{source_prefix}/{file_name}".strip('/') + ".pdf"
|
|
destination_key = f"{destination_prefix}/{file_name}".strip('/') + ".pdf"
|
|
|
|
# s3_client.copy_object(
|
|
# CopySource={'Bucket': source_bucket, 'Key': source_key},
|
|
# Bucket=destination_bucket,
|
|
# Key=destination_key
|
|
# )
|
|
|
|
response = s3_client.get_object(Bucket=source_bucket, Key=source_key)
|
|
file_content = response['Body'].read()
|
|
# print(f"Downloaded {source_key} from s3://{source_bucket}")
|
|
s3_client.put_object(Bucket=destination_bucket, Key=destination_key, Body=file_content)
|
|
print(f"Copied: {file_name} to {destination_prefix}")
|
|
else:
|
|
print(f"File not found: {file_name} in {source_prefix}")
|
|
rerun = pd.concat([rerun, df[df['File name Without Extension'] == file_name]])
|
|
|
|
def copy_files_multithreaded(file_list, source_bucket, source_prefix, destination_bucket, destination_prefix, existing_files, max_workers=20):
|
|
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
|
futures = [
|
|
executor.submit(copy_file_if_exists, file_name, source_bucket, source_prefix, destination_bucket, destination_prefix, existing_files)
|
|
for file_name in file_list
|
|
]
|
|
|
|
for future in as_completed(futures):
|
|
try:
|
|
future.result()
|
|
except Exception as e:
|
|
print(f"Error copying file: {e}")
|
|
|
|
csv_path = 'new_duplicates_in_batch3.csv'
|
|
file_name_column = 'File Name Without Extension'
|
|
source_bucket = 'centene-national-contracting-files'
|
|
search_prefix = 'batch_3_priority_files/txt_files'
|
|
source_prefix = 'batch_3_priority_files/pdf_files'
|
|
destination_bucket = 'centene-national-contracting-files'
|
|
destination_prefix = 'batch_3_priority_files/re_run_files'
|
|
|
|
df = pd.read_csv(csv_path)
|
|
rerun = pd.DataFrame(columns=df.columns)
|
|
file_names = df[file_name_column].dropna().tolist()
|
|
print(f"Loaded {len(file_names)} file names from CSV.")
|
|
existing_files = get_filenames_from_s3(source_bucket, search_prefix)
|
|
print(f"Found {len(existing_files)} files in S3 source folder '{source_prefix}'.")
|
|
copy_files_multithreaded(file_names, source_bucket, source_prefix, destination_bucket, destination_prefix, existing_files)
|
|
rerun.reset_index(drop=True, inplace=True)
|
|
rerun.to_csv('batch3_rerun.csv') |