Katon Minhas
afb6d5185d
Feature/lesser table caching refactor hybrid * chore: Remove unused duplicate main.py from shared pipeline * fix: Correct crosswalk paths in aarete_derived.py * chore: Remove unused documentation files from fieldExtraction * docs: Add documentation files to documentation folder * docs: Update README with uv setup, expanded project structure, and branching conventions * docs: Add uv installation steps with Ubuntu/WSL emphasis * Enable prompt caching for all remaining LLM calls - Add _INSTRUCTION() functions for: EXHIBIT_HEADER, EXHIBIT_LINKAGE, EXHIBIT_TITLE_MATCH, DATE_FIX, DERIVED_TERM_DATE, CHECK_PROVIDER_NAME_MATCH, SPECIAL_CASE_ASSIGNMENT - Update all invoke_claude() calls in saas and clover pipelines to use cache=True with corresponding _INSTRUCTION() functions - Add new instructions to get_cacheable_instructions() for cache warming - Update tests for new instruction functions Functions now using caching: - prompt_exhibit_level - prompt_exhibit_lesser (EXHIBIT_LEVEL_LESSER_OF) - prompt_fee_schedule_breakout - prompt_grouper_breakout - prompt_special_case_assignment - prompt_exhibit_linkage - prompt_exhibit_header - prompt_smart_chunked (ONE_TO_ONE templates) - prompt_date_fix - prompt_derived_term_date - prompt_exhibit_title_match - provider_name_match_check 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Reorder * feat: Add bcbs_promise client pipeline with OFFSET_TERM extraction - Add new bcbs_promise client with HSC-based OFFSET_TERM field extraction - Extract full paragraph text of offset/recoupment provisions from contracts - Derive OFFSET_INDICATOR (Y/N) from OFFSET_TERM presence - Fix reorder_columns to preserve extra columns not in COLUMN_ORDER - Update QC/QA output path to outputs/qc_qa/ * fix: Update dev deps and test assertions for QC/QA output path - Add pytest/pytest-mock to dev dependencies for mypy type checking - Update test assertions to expect outputs/qc_qa instead of qa_qc_output * style: Apply black formatting to prompt_templates.py * Merge main, move scripts * Archive some scripts * update py version * remove .py version file * Remove ASCII characters * Restore testbed code * restore tracking * Update testbed metrics * Enable prompt caching for CODE_LAST_CHECK, FILL_BILL_TYPE, DUAL_LOB_CHECK, and GROUPER_BREAKOUT - Add CODE_LAST_CHECK_INSTRUCTION() for service specificity classification - Add FILL_BILL_TYPE_INSTRUCTION() for bill type code determination - Add DUAL_LOB_CHECK_INSTRUCTION() for Medicare/Medicaid classification - Update code_funcs.py to use caching for CODE_LAST_CHECK, FILL_BILL_TYPE, GROUPER_BREAKOUT - Update postprocessing_funcs.py to use caching for DUAL_LOB_CHECK - Add new instructions to get_cacheable_instructions() for cache warming - Add unit tests for new instruction functions 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Fix postprocessing_funcs to remove invalid columns * Merge branch 'main' into feature/lesser-table-caching-refactor-hybrid * Revert prompt caching changes from aed1b73c * update formatting * Update imports Approved-by: Sha Brown Approved-by: Praneel Panchigar
70 lines
2.9 KiB
Python
70 lines
2.9 KiB
Python
import boto3
|
|
import os
|
|
import pandas as pd
|
|
from datetime import datetime
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from io import StringIO
|
|
|
|
PROFILE_NAME = 'temp_cred'
|
|
SOURCE_BUCKET = 'doczyai-use2-u-cn1-s3-textract-processing-001'
|
|
DESTINATION_BUCKET = 'healthnet-json-files'
|
|
ERROR_LOG_FILE = "error_log_files.txt"
|
|
MAX_WORKERS = 200
|
|
CSV_FILE_PATH = "processed_batch_ids/large-pdf-files-processed-3.csv"
|
|
DESTINATION_PATHS = {
|
|
"text": 'new-batch6/large-pdf-txt-files/',
|
|
"stuck": 'new-batch6/stuck-files/',
|
|
"pdf": 'new-batch6/large-pdf-processed-files/',
|
|
"invalid": 'new-batch6/invalid-files/',
|
|
"unprocessed": 'new-batch6/unprocessed-files/'
|
|
}
|
|
|
|
session = boto3.Session(profile_name=PROFILE_NAME)
|
|
s3_client = session.client('s3')
|
|
|
|
def log_error_to_file(key, error_message):
|
|
with open(ERROR_LOG_FILE, 'a') as f:
|
|
f.write(f"Error with file: {key} - {error_message}\n")
|
|
|
|
def copy_single_file(key, destination_prefix):
|
|
destination_prefix = DESTINATION_PATHS[destination_prefix]
|
|
try:
|
|
copy_source = {'Bucket': SOURCE_BUCKET, 'Key': key}
|
|
destination_key = os.path.join(destination_prefix, os.path.basename(key))
|
|
s3_client.copy(copy_source, DESTINATION_BUCKET, destination_key)
|
|
print(f"Copied {key} to {destination_key}")
|
|
except Exception as e:
|
|
print(f"Error copying {key} to {destination_key}")
|
|
log_error_to_file(key, str(e))
|
|
|
|
def main():
|
|
response = s3_client.get_object(Bucket=DESTINATION_BUCKET, Key=CSV_FILE_PATH)
|
|
csv_content = response['Body'].read().decode('utf-8')
|
|
batch_ids_df = pd.read_csv(StringIO(csv_content))
|
|
batch_ids = batch_ids_df['batch_id'].unique()
|
|
contract_names = batch_ids_df['contract_name']
|
|
|
|
print(f"Total number of batches: {len(batch_ids)}")
|
|
file_keys = []
|
|
for batch_id, contract_name in zip(batch_ids_df['batch_id'], contract_names):
|
|
contract_name = contract_name[:-4]
|
|
txt_file_key = f'contract-text-file/{batch_id}/{contract_name}.txt'
|
|
stuck_file_key = f'all-pdfs/{batch_id}/{contract_name}.pdf'
|
|
pdf_file_key = f'textract-receiver-processed-pdfs/{batch_id}/{contract_name}.pdf'
|
|
invalid_file_key = f'invalid-pdfs/{batch_id}/{contract_name}.pdf'
|
|
unprocessed_file_key = f'textract-sender-staging-pdfs/{batch_id}/{contract_name}.pdf'
|
|
file_keys.append((txt_file_key, 'text'))
|
|
file_keys.append((stuck_file_key, 'stuck'))
|
|
file_keys.append((pdf_file_key, 'pdf'))
|
|
file_keys.append((invalid_file_key, 'invalid'))
|
|
file_keys.append((unprocessed_file_key, 'unprocessed'))
|
|
|
|
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
|
|
futures = [executor.submit(copy_single_file, key, dest) for key, dest in file_keys]
|
|
for future in as_completed(futures):
|
|
future.result()
|
|
|
|
print("Copy operation completed.")
|
|
|
|
if __name__ == '__main__':
|
|
main() |