Katon Minhas
afb6d5185d
Feature/lesser table caching refactor hybrid * chore: Remove unused duplicate main.py from shared pipeline * fix: Correct crosswalk paths in aarete_derived.py * chore: Remove unused documentation files from fieldExtraction * docs: Add documentation files to documentation folder * docs: Update README with uv setup, expanded project structure, and branching conventions * docs: Add uv installation steps with Ubuntu/WSL emphasis * Enable prompt caching for all remaining LLM calls - Add _INSTRUCTION() functions for: EXHIBIT_HEADER, EXHIBIT_LINKAGE, EXHIBIT_TITLE_MATCH, DATE_FIX, DERIVED_TERM_DATE, CHECK_PROVIDER_NAME_MATCH, SPECIAL_CASE_ASSIGNMENT - Update all invoke_claude() calls in saas and clover pipelines to use cache=True with corresponding _INSTRUCTION() functions - Add new instructions to get_cacheable_instructions() for cache warming - Update tests for new instruction functions Functions now using caching: - prompt_exhibit_level - prompt_exhibit_lesser (EXHIBIT_LEVEL_LESSER_OF) - prompt_fee_schedule_breakout - prompt_grouper_breakout - prompt_special_case_assignment - prompt_exhibit_linkage - prompt_exhibit_header - prompt_smart_chunked (ONE_TO_ONE templates) - prompt_date_fix - prompt_derived_term_date - prompt_exhibit_title_match - provider_name_match_check 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Reorder * feat: Add bcbs_promise client pipeline with OFFSET_TERM extraction - Add new bcbs_promise client with HSC-based OFFSET_TERM field extraction - Extract full paragraph text of offset/recoupment provisions from contracts - Derive OFFSET_INDICATOR (Y/N) from OFFSET_TERM presence - Fix reorder_columns to preserve extra columns not in COLUMN_ORDER - Update QC/QA output path to outputs/qc_qa/ * fix: Update dev deps and test assertions for QC/QA output path - Add pytest/pytest-mock to dev dependencies for mypy type checking - Update test assertions to expect outputs/qc_qa instead of qa_qc_output * style: Apply black formatting to prompt_templates.py * Merge main, move scripts * Archive some scripts * update py version * remove .py version file * Remove ASCII characters * Restore testbed code * restore tracking * Update testbed metrics * Enable prompt caching for CODE_LAST_CHECK, FILL_BILL_TYPE, DUAL_LOB_CHECK, and GROUPER_BREAKOUT - Add CODE_LAST_CHECK_INSTRUCTION() for service specificity classification - Add FILL_BILL_TYPE_INSTRUCTION() for bill type code determination - Add DUAL_LOB_CHECK_INSTRUCTION() for Medicare/Medicaid classification - Update code_funcs.py to use caching for CODE_LAST_CHECK, FILL_BILL_TYPE, GROUPER_BREAKOUT - Update postprocessing_funcs.py to use caching for DUAL_LOB_CHECK - Add new instructions to get_cacheable_instructions() for cache warming - Add unit tests for new instruction functions 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Fix postprocessing_funcs to remove invalid columns * Merge branch 'main' into feature/lesser-table-caching-refactor-hybrid * Revert prompt caching changes from aed1b73c * update formatting * Update imports Approved-by: Sha Brown Approved-by: Praneel Panchigar
82 lines
2.3 KiB
Python
82 lines
2.3 KiB
Python
import os
|
|
import pickle
|
|
|
|
import faiss
|
|
import numpy as np
|
|
import src.config as config
|
|
from src.utils.string_utils import datetime_str
|
|
|
|
|
|
def create_faiss_index(
|
|
choices,
|
|
model,
|
|
save_path="faiss_index.bin",
|
|
embedding_path="embeddings.npy",
|
|
choices_path="choices.pkl",
|
|
):
|
|
embeddings = model.encode(choices, normalize_embeddings=True).astype("float32")
|
|
|
|
# Create FAISS index
|
|
index = faiss.IndexFlatIP(embeddings.shape[1])
|
|
index.add(embeddings)
|
|
|
|
# Save index
|
|
faiss.write_index(index, save_path)
|
|
|
|
# Save embeddings and choices for future use
|
|
np.save(embedding_path, embeddings)
|
|
with open(choices_path, "wb") as f:
|
|
pickle.dump(choices, f)
|
|
|
|
return index
|
|
|
|
|
|
def load_faiss_index(
|
|
index_path="faiss_index.bin",
|
|
embedding_path="embeddings.npy",
|
|
choices_path="choices.pkl",
|
|
):
|
|
index = faiss.read_index(index_path)
|
|
embeddings = np.load(embedding_path)
|
|
with open(choices_path, "rb") as f:
|
|
choices = pickle.load(f)
|
|
return index, embeddings, choices
|
|
|
|
|
|
def load_embeddings():
|
|
"""loads all embedding files required for processing indirect codes (e.g. proc codes)"""
|
|
|
|
# list all embeddings present in s3
|
|
s3_client = config.S3_CLIENT
|
|
response = s3_client.list_objects_v2(Bucket="doczy-investment", Prefix="embeddings")
|
|
file_list = [
|
|
obj["Key"]
|
|
for obj in response.get("Contents", [])
|
|
if not obj["Key"].endswith("/")
|
|
]
|
|
|
|
# list embeddings already present locally
|
|
if not os.path.exists("embeddings"):
|
|
os.makedirs("embeddings")
|
|
local_files = []
|
|
for root, dirs, files in os.walk("embeddings"):
|
|
for name in files:
|
|
local_files.append(os.path.join(root, name))
|
|
|
|
# download embeddings not present locally
|
|
for file in file_list:
|
|
local_path = file
|
|
|
|
# create directory structure for this file
|
|
directory = os.path.dirname(local_path)
|
|
if not os.path.exists(directory):
|
|
os.makedirs(directory)
|
|
|
|
# check if file already exists
|
|
if not os.path.exists(local_path):
|
|
try:
|
|
print(f"{datetime_str()} Downloading {file} to {local_path}...")
|
|
s3_client.download_file("doczy-investment", file, local_path)
|
|
except Exception as e:
|
|
print(f"{datetime_str()} Error downloading {file}: {e}")
|