Katon Minhas
afb6d5185d
Feature/lesser table caching refactor hybrid * chore: Remove unused duplicate main.py from shared pipeline * fix: Correct crosswalk paths in aarete_derived.py * chore: Remove unused documentation files from fieldExtraction * docs: Add documentation files to documentation folder * docs: Update README with uv setup, expanded project structure, and branching conventions * docs: Add uv installation steps with Ubuntu/WSL emphasis * Enable prompt caching for all remaining LLM calls - Add _INSTRUCTION() functions for: EXHIBIT_HEADER, EXHIBIT_LINKAGE, EXHIBIT_TITLE_MATCH, DATE_FIX, DERIVED_TERM_DATE, CHECK_PROVIDER_NAME_MATCH, SPECIAL_CASE_ASSIGNMENT - Update all invoke_claude() calls in saas and clover pipelines to use cache=True with corresponding _INSTRUCTION() functions - Add new instructions to get_cacheable_instructions() for cache warming - Update tests for new instruction functions Functions now using caching: - prompt_exhibit_level - prompt_exhibit_lesser (EXHIBIT_LEVEL_LESSER_OF) - prompt_fee_schedule_breakout - prompt_grouper_breakout - prompt_special_case_assignment - prompt_exhibit_linkage - prompt_exhibit_header - prompt_smart_chunked (ONE_TO_ONE templates) - prompt_date_fix - prompt_derived_term_date - prompt_exhibit_title_match - provider_name_match_check 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Reorder * feat: Add bcbs_promise client pipeline with OFFSET_TERM extraction - Add new bcbs_promise client with HSC-based OFFSET_TERM field extraction - Extract full paragraph text of offset/recoupment provisions from contracts - Derive OFFSET_INDICATOR (Y/N) from OFFSET_TERM presence - Fix reorder_columns to preserve extra columns not in COLUMN_ORDER - Update QC/QA output path to outputs/qc_qa/ * fix: Update dev deps and test assertions for QC/QA output path - Add pytest/pytest-mock to dev dependencies for mypy type checking - Update test assertions to expect outputs/qc_qa instead of qa_qc_output * style: Apply black formatting to prompt_templates.py * Merge main, move scripts * Archive some scripts * update py version * remove .py version file * Remove ASCII characters * Restore testbed code * restore tracking * Update testbed metrics * Enable prompt caching for CODE_LAST_CHECK, FILL_BILL_TYPE, DUAL_LOB_CHECK, and GROUPER_BREAKOUT - Add CODE_LAST_CHECK_INSTRUCTION() for service specificity classification - Add FILL_BILL_TYPE_INSTRUCTION() for bill type code determination - Add DUAL_LOB_CHECK_INSTRUCTION() for Medicare/Medicaid classification - Update code_funcs.py to use caching for CODE_LAST_CHECK, FILL_BILL_TYPE, GROUPER_BREAKOUT - Update postprocessing_funcs.py to use caching for DUAL_LOB_CHECK - Add new instructions to get_cacheable_instructions() for cache warming - Add unit tests for new instruction functions 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Fix postprocessing_funcs to remove invalid columns * Merge branch 'main' into feature/lesser-table-caching-refactor-hybrid * Revert prompt caching changes from aed1b73c * update formatting * Update imports Approved-by: Sha Brown Approved-by: Praneel Panchigar
73 lines
2.8 KiB
Python
73 lines
2.8 KiB
Python
import os
|
|
import pandas as pd
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from collections import defaultdict
|
|
import csv
|
|
|
|
"""
|
|
This script searches for files in a given directory based on a list of filenames in a CSV file.
|
|
The CSV file should have a column named 'filename' containing the filenames to search for.
|
|
|
|
This is maily used to find files that are not present in the s3 or cannot be located.
|
|
In that case, we use the list of missing files (in the CSV) to search for them in the T drive.
|
|
|
|
This script may not be needed as all CNC batches have been staged for execution.
|
|
|
|
"""
|
|
|
|
def build_file_cache(base_directory):
|
|
file_cache = defaultdict(list)
|
|
|
|
for root, dirs, files in os.walk(base_directory):
|
|
for file in files:
|
|
file_cache[file].append(os.path.join(root, file))
|
|
|
|
return file_cache
|
|
|
|
def find_files(file_cache, filenames):
|
|
found_files = {}
|
|
for filename in filenames:
|
|
if filename in file_cache:
|
|
found_files[filename] = file_cache[filename][0]
|
|
else:
|
|
found_files[filename] = "Not Found"
|
|
return found_files
|
|
|
|
def parallel_search(base_directory, filenames, max_workers=50):
|
|
print("Building file cache...")
|
|
file_cache = build_file_cache(base_directory)
|
|
with open('all_file_paths.csv', mode='w', newline='', encoding='utf-8') as csv_file:
|
|
writer = csv.writer(csv_file)
|
|
for key, values in file_cache.items():
|
|
row = [key] + values if isinstance(values, list) else [key, values]
|
|
writer.writerow(row)
|
|
print(f"Dictionary has been successfully written to 'all_file_paths.csv'.")
|
|
print(f"File cache built with {len(file_cache)} unique files.")
|
|
found_files = {}
|
|
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
|
futures = {executor.submit(find_files, file_cache, chunk): chunk
|
|
for chunk in chunked(filenames, len(filenames) // max_workers)}
|
|
|
|
for future in as_completed(futures):
|
|
found_files.update(future.result())
|
|
|
|
return found_files
|
|
|
|
def chunked(iterable, n):
|
|
for i in range(0, len(iterable), n):
|
|
yield iterable[i:i + n]
|
|
|
|
def search_files_from_csv(csv_file, base_directory, output_csv):
|
|
df = pd.read_csv(csv_file)
|
|
df['filename'] += ".Pdf"
|
|
filenames = df['filename'].tolist()
|
|
print(f"Searching for {len(filenames)} files in '{base_directory}'...")
|
|
file_paths = parallel_search(base_directory, filenames)
|
|
df['file_path'] = df['filename'].apply(lambda x: file_paths.get(x, "Not Found"))
|
|
df.to_csv(output_csv, index=False)
|
|
print(f"Results saved to '{output_csv}'.")
|
|
|
|
input_csv = "missing_files_renaming.csv"
|
|
base_dir = "T:/AArete Client Work/Doczy-Production/Restricted/2024-06-28-pdf/"
|
|
output_csv = "missing_file_paths_2.csv"
|
|
search_files_from_csv(input_csv, base_dir, output_csv) |