Merged in feature/lesser-table-caching-refactor-hybrid (pull request #847)
Feature/lesser table caching refactor hybrid * chore: Remove unused duplicate main.py from shared pipeline * fix: Correct crosswalk paths in aarete_derived.py * chore: Remove unused documentation files from fieldExtraction * docs: Add documentation files to documentation folder * docs: Update README with uv setup, expanded project structure, and branching conventions * docs: Add uv installation steps with Ubuntu/WSL emphasis * Enable prompt caching for all remaining LLM calls - Add _INSTRUCTION() functions for: EXHIBIT_HEADER, EXHIBIT_LINKAGE, EXHIBIT_TITLE_MATCH, DATE_FIX, DERIVED_TERM_DATE, CHECK_PROVIDER_NAME_MATCH, SPECIAL_CASE_ASSIGNMENT - Update all invoke_claude() calls in saas and clover pipelines to use cache=True with corresponding _INSTRUCTION() functions - Add new instructions to get_cacheable_instructions() for cache warming - Update tests for new instruction functions Functions now using caching: - prompt_exhibit_level - prompt_exhibit_lesser (EXHIBIT_LEVEL_LESSER_OF) - prompt_fee_schedule_breakout - prompt_grouper_breakout - prompt_special_case_assignment - prompt_exhibit_linkage - prompt_exhibit_header - prompt_smart_chunked (ONE_TO_ONE templates) - prompt_date_fix - prompt_derived_term_date - prompt_exhibit_title_match - provider_name_match_check 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Reorder * feat: Add bcbs_promise client pipeline with OFFSET_TERM extraction - Add new bcbs_promise client with HSC-based OFFSET_TERM field extraction - Extract full paragraph text of offset/recoupment provisions from contracts - Derive OFFSET_INDICATOR (Y/N) from OFFSET_TERM presence - Fix reorder_columns to preserve extra columns not in COLUMN_ORDER - Update QC/QA output path to outputs/qc_qa/ * fix: Update dev deps and test assertions for QC/QA output path - Add pytest/pytest-mock to dev dependencies for mypy type checking - Update test assertions to expect outputs/qc_qa instead of qa_qc_output * style: Apply black formatting to prompt_templates.py * Merge main, move scripts * Archive some scripts * update py version * remove .py version file * Remove ASCII characters * Restore testbed code * restore tracking * Update testbed metrics * Enable prompt caching for CODE_LAST_CHECK, FILL_BILL_TYPE, DUAL_LOB_CHECK, and GROUPER_BREAKOUT - Add CODE_LAST_CHECK_INSTRUCTION() for service specificity classification - Add FILL_BILL_TYPE_INSTRUCTION() for bill type code determination - Add DUAL_LOB_CHECK_INSTRUCTION() for Medicare/Medicaid classification - Update code_funcs.py to use caching for CODE_LAST_CHECK, FILL_BILL_TYPE, GROUPER_BREAKOUT - Update postprocessing_funcs.py to use caching for DUAL_LOB_CHECK - Add new instructions to get_cacheable_instructions() for cache warming - Add unit tests for new instruction functions 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Fix postprocessing_funcs to remove invalid columns * Merge branch 'main' into feature/lesser-table-caching-refactor-hybrid * Revert prompt caching changes from aed1b73c * update formatting * Update imports Approved-by: Sha Brown Approved-by: Praneel Panchigar
This commit is contained in:
Katon Minhas
@@ -0,0 +1,49 @@
|
||||
import pandas as pd
|
||||
|
||||
import os
|
||||
|
||||
|
||||
def read_and_compare_csvs(csv_file1, csv_file2):
|
||||
# Read CSV files into pandas DataFrames
|
||||
df1 = pd.read_csv(csv_file1)
|
||||
df2 = pd.read_csv(csv_file2)
|
||||
|
||||
df2 = df2[df1.columns]
|
||||
|
||||
# Check for any differences
|
||||
diff = (df1 != df2) & df1.notnull() & df2.notnull()
|
||||
|
||||
# Create a writer for Excel
|
||||
writer = pd.ExcelWriter("data/differences.xlsx", engine="xlsxwriter")
|
||||
df2.to_excel(writer, sheet_name="Differences", index=False)
|
||||
|
||||
# Get the xlsxwriter workbook and worksheet objects
|
||||
workbook = writer.book
|
||||
worksheet = writer.sheets["Differences"]
|
||||
|
||||
# Define a format for changed cells
|
||||
highlight_fmt = workbook.add_format({"font_color": "red", "bg_color": "yellow"})
|
||||
|
||||
# Apply formatting to the changed cells
|
||||
for col in df1.columns:
|
||||
col_idx = (
|
||||
df1.columns.get_loc(col) + 1
|
||||
) # Excel index starts from 1, and there is an index column
|
||||
for row in diff.index[diff[col]]:
|
||||
value = df2.at[row, col]
|
||||
if pd.isna(value):
|
||||
value = (
|
||||
"NaN" # Change this if you want a different representation for NaN
|
||||
)
|
||||
worksheet.write(row + 1, col_idx, value, highlight_fmt)
|
||||
|
||||
# Close the Pandas Excel writer and output the Excel file
|
||||
writer.close()
|
||||
|
||||
|
||||
# Example usage
|
||||
base_path = "output/57-2883440-Cordia Anderson-Hopkins LCSW-ICMProviderAgreement_221673"
|
||||
read_and_compare_csvs(
|
||||
os.path.join(base_path, "combined_results_post_processed.csv"),
|
||||
os.path.join(base_path, "test_3.5.csv"),
|
||||
)
|
||||
@@ -0,0 +1,31 @@
|
||||
import src.utils.embedding_utils as embedding_utils
|
||||
from src.utils.crosswalk_utils import CrosswalkBuilder
|
||||
import src.codes.code_funcs as code_funcs
|
||||
|
||||
import os
|
||||
import pandas as pd
|
||||
|
||||
from sentence_transformers import SentenceTransformer
|
||||
|
||||
# Load model
|
||||
roberta_model = SentenceTransformer("all-roberta-large-v1")
|
||||
|
||||
mapping_dir = "crosswalk/mapping_csvs/proc_cd"
|
||||
pkl_dir = "embeddings"
|
||||
for filename in os.listdir(mapping_dir):
|
||||
if filename.endswith(".csv") and "level" in filename:
|
||||
print(f"Creating embeddings for {filename}")
|
||||
stripped_filename = filename.replace(".csv", "")
|
||||
mapping_df = pd.read_csv(os.path.join(mapping_dir, filename))
|
||||
proc_crosswalk = CrosswalkBuilder().from_df(
|
||||
mapping_df, from_col="Code", to_col="Description"
|
||||
)
|
||||
proc_choices = [x for x in proc_crosswalk.mapping.values()]
|
||||
os.makedirs(os.path.join(pkl_dir, stripped_filename), exist_ok=True)
|
||||
proc_index = embedding_utils.create_faiss_index(
|
||||
choices=proc_choices,
|
||||
model=roberta_model,
|
||||
save_path=os.path.join(pkl_dir, stripped_filename, "faiss_index.bin"),
|
||||
embedding_path=os.path.join(pkl_dir, stripped_filename, "embeddings.npy"),
|
||||
choices_path=os.path.join(pkl_dir, stripped_filename, "choices.pkl"),
|
||||
)
|
||||
@@ -0,0 +1,98 @@
|
||||
import concurrent.futures
|
||||
import traceback
|
||||
import re
|
||||
import os
|
||||
import pandas as pd
|
||||
|
||||
import utils
|
||||
import config
|
||||
import preprocessing_funcs
|
||||
|
||||
COMPLEX_LIST = []
|
||||
|
||||
|
||||
def count_reimbursements(page_text: str) -> int:
|
||||
"""Count number of reimbursement items found on a page text, through use of a
|
||||
regex that catches either '$' or '%'.
|
||||
|
||||
Args:
|
||||
page_text (str): Input page text, as a string
|
||||
|
||||
Returns:
|
||||
int: Number of reimbursement items
|
||||
"""
|
||||
pattern = r"[%$]"
|
||||
matches = re.findall(pattern, page_text)
|
||||
return len(matches)
|
||||
|
||||
|
||||
def detect_complex(filename, contract_text):
|
||||
print(f"Processing {filename}")
|
||||
contract_text = preprocessing_funcs.clean_newlines(contract_text)
|
||||
contract_text = preprocessing_funcs.clean_law_symbols(contract_text)
|
||||
text_dict = preprocessing_funcs.split_text(contract_text)
|
||||
rate_dict = {}
|
||||
table_dict = {}
|
||||
for page_num, page_text in text_dict.items():
|
||||
rate_dict[page_num] = count_reimbursements(page_text)
|
||||
table_dict[page_num] = "-------Table Start--------" in page_text
|
||||
|
||||
page_count, consecutive_page_count, consecutive_rate_count = 0, 0, 0
|
||||
|
||||
for page_num in table_dict.keys():
|
||||
if table_dict[page_num] and rate_dict[page_num] > 0:
|
||||
consecutive_rate_count += rate_dict[page_num]
|
||||
consecutive_page_count += 1
|
||||
else:
|
||||
if consecutive_rate_count >= config.RATE_THRESHOLD:
|
||||
page_count += consecutive_page_count
|
||||
consecutive_page_count, consecutive_rate_count = 0, 0
|
||||
|
||||
COMPLEX_LIST.append(
|
||||
{"Filename": filename, "COMPLEX_FLAG": page_count >= config.TABLE_THRESHOLD}
|
||||
)
|
||||
|
||||
|
||||
def process_item(contract):
|
||||
data = s3_client.get_object(Bucket=config.BUCKET, Key=contract)
|
||||
contents = data["Body"].read()
|
||||
contract_text = contents.decode("utf-8")
|
||||
|
||||
detect_complex(contract, contract_text)
|
||||
|
||||
|
||||
s3_client = config.S3_CLIENT
|
||||
|
||||
file_list = []
|
||||
paginator = s3_client.get_paginator("list_objects_v2")
|
||||
pages = paginator.paginate(Bucket=config.BUCKET, Prefix=config.PREFIX)
|
||||
for page in pages:
|
||||
for obj in page["Contents"]:
|
||||
if not obj["Key"].endswith("/"):
|
||||
file_list.append(obj["Key"])
|
||||
|
||||
contract_list = sorted(file_list)
|
||||
files = {}
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(
|
||||
max_workers=config.COMPLEX_MAX_WORKERS
|
||||
) as executor:
|
||||
# Process each item individually
|
||||
futures = [executor.submit(process_item, contract) for contract in contract_list]
|
||||
|
||||
# Wait for all futures to complete
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
# Retrieve any exceptions raised by the thread
|
||||
try:
|
||||
future.result()
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
|
||||
complex_df = pd.DataFrame(COMPLEX_LIST)
|
||||
# complex_df.to_csv(os.path.join(config.COMPLEX_OUTPUT_PATH, config.COMPLEX_OUTPUT_FILENAME))
|
||||
|
||||
complex_df.to_csv(
|
||||
os.path.join(
|
||||
config.COMPLEX_OUTPUT_PATH, "TX_priority_tin_complex_categorization.csv"
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,31 @@
|
||||
import utils
|
||||
import config
|
||||
|
||||
import pandas as pd
|
||||
|
||||
print(f"Input Dir: {config.LOCAL_PATH}")
|
||||
print(f"S3 Bucket: {config.S3_BUCKET}")
|
||||
print(f"S3 Prefix: {config.S3_PREFIX}")
|
||||
|
||||
input_dict = (
|
||||
utils.read_input()
|
||||
) # keys are contract names, values are full contract text
|
||||
total_files = len(input_dict)
|
||||
|
||||
print(f"Total Input Files : {len(input_dict)}")
|
||||
|
||||
# Filter B
|
||||
input_dict_b = {
|
||||
k: v for k, v in input_dict.items() if utils.contains_reimbursement(str(v))
|
||||
} # Filter out non-contracts
|
||||
print(
|
||||
f"B Input Files after reimbursement filter: {len(input_dict_b)} | {total_files-len(input_dict_b)} files removed"
|
||||
)
|
||||
|
||||
# Filter already processed
|
||||
if config.FILTER_ALREADY_PROCESSED:
|
||||
input_dict_ac, input_dict_b = utils.filter_already_processed(
|
||||
input_dict, input_dict_b
|
||||
)
|
||||
print(f"AC Input Files left to be processed : {len(input_dict_ac)}")
|
||||
print(f"B Input Files left to be processed : {len(input_dict_b)}")
|
||||
@@ -0,0 +1,154 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Manual Test Bed\n",
|
||||
"Runs contracts through from `input_data`\n",
|
||||
"\n",
|
||||
"To make this work: in `config.py`, change READ_MODE and RUN_MODE to 'local'.\n",
|
||||
"Create an .env in the base directory of your project according to these instructions:\n",
|
||||
"https://aarete.atlassian.net/wiki/spaces/DoczyAI/pages/1283751954/Using+.env+for+AWS+keys\n",
|
||||
"\n",
|
||||
"Feel free to comment out parts of `file_processing.py` that do not apply to the fields you are optimizing. For instance, if you're optimizing a smart chunked field, comment out the `one_to_n_fields` block and feed just `one_to_n_results` into `postprocess.postprocess`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"#### BASE IMPORTS ####\n",
|
||||
"\n",
|
||||
"import logging\n",
|
||||
"import sys\n",
|
||||
"import os\n",
|
||||
"import pandas as pd"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"#### LOGGING ####\n",
|
||||
"\n",
|
||||
"os.chdir(\"../../\") # go to FieldExtraction folder\n",
|
||||
"\n",
|
||||
"import src.config as config\n",
|
||||
"# For this testing notebook, we want per-file logging\n",
|
||||
"config.ENABLE_PER_FILE_LOGGING = True\n",
|
||||
"config.PER_FILE_LOG_LEVEL = \"DEBUG\"\n",
|
||||
"\n",
|
||||
"import src.utils.logging_utils as logging_utils\n",
|
||||
"\n",
|
||||
"# Clear any existing handlers first\n",
|
||||
"logging.getLogger().handlers.clear()\n",
|
||||
"\n",
|
||||
"# Set up per-file logging (this includes console output)\n",
|
||||
"# logging_utils.setup_per_file_logging()\n",
|
||||
"per_file_handler = logging_utils.PerFileHandler()\n",
|
||||
"per_file_handler.setLevel(logging.DEBUG)\n",
|
||||
"logging.getLogger().addHandler(per_file_handler)\n",
|
||||
"\n",
|
||||
"# Add a single console handler for notebook monitoring\n",
|
||||
"console_handler = logging.StreamHandler(sys.stdout)\n",
|
||||
"console_handler.setLevel(logging.INFO) # Only show INFO and above in notebook\n",
|
||||
"console_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')\n",
|
||||
"console_handler.setFormatter(console_formatter)\n",
|
||||
"logging.getLogger().addHandler(console_handler)\n",
|
||||
"\n",
|
||||
"# Set level for the root logger\n",
|
||||
"logging.getLogger().setLevel(logging.DEBUG)\n",
|
||||
"\n",
|
||||
"# Suppress noisy third-party loggers\n",
|
||||
"for logger_name in ['botocore', 'boto3', 'faiss', 'urllib3', 'sentence_transformers', 's3transfer']:\n",
|
||||
" logging.getLogger(logger_name).setLevel(logging.WARNING)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"#### TEST CONFIG ####\n",
|
||||
"\n",
|
||||
"TEST_DATA_INPUT_DIR = \"scripts/investment_field_testing/input_data\"\n",
|
||||
"\n",
|
||||
"# Also change config.RUN_MODE and config.READ_MODE to 'local'\n",
|
||||
"test_params = {}\n",
|
||||
"test_params['local_input_dir'] = TEST_DATA_INPUT_DIR\n",
|
||||
"test_params['max_workers'] = 1\n",
|
||||
"test_params['input_files'] = [ # you can specify specific files to run here\n",
|
||||
" \"Akron General Health System_Eleventh Amendment_20171001.txt\", # replace with your test file(s)\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"# Specify state(s) for testing, if needed\n",
|
||||
"# Specify them here instead of in `test_params` bc `config` is initialized \n",
|
||||
"# before `test_params` is read in main.py, which means the state-level constants\n",
|
||||
"# won't be set correctly if we put it in `test_params`.\n",
|
||||
"config.STATE = [\"OH\"] # for single states\n",
|
||||
"# config.STATE = [\"OH\", \"CA\"] # for multi-state testing"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": "#### MAIN IMPORTS ####\n\nfrom src.pipelines.saas.main import main as investment_main"
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"#### TEST EXECUTION ####\n",
|
||||
"\n",
|
||||
"df_out, df_error = investment_main(testing=True, test_params=test_params)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"df_out"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "doczy-field-extraction-py3.12 (3.12.7)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.12.7"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
import os
|
||||
import re
|
||||
import csv
|
||||
|
||||
|
||||
def split_text(text):
|
||||
text_list = text.split("Start of Page No. = ")
|
||||
text_dict = {page.split()[0]: page for page in text_list[1:]}
|
||||
return text_dict
|
||||
|
||||
|
||||
def contains_reimbursement(text, page):
|
||||
if isinstance(text, dict):
|
||||
return page.isdigit() and (
|
||||
"%" in text[page] or "$" in text[page] or "percent " in text[page].lower()
|
||||
)
|
||||
elif isinstance(text, str):
|
||||
return "%" in text or "$" in text or "percent " in text.lower()
|
||||
else:
|
||||
print("contains_reimbursement - Invalid data type")
|
||||
return False
|
||||
|
||||
|
||||
def search_files(directory):
|
||||
pattern = re.compile(r"'',\s*'',")
|
||||
results = []
|
||||
total_files = 0
|
||||
files_with_pattern = 0
|
||||
|
||||
for root, dirs, files in os.walk(directory):
|
||||
for file in files:
|
||||
if file.endswith(".txt"):
|
||||
total_files += 1
|
||||
file_path = os.path.join(root, file)
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
pages = split_text(content)
|
||||
|
||||
reimbursement_pages = [
|
||||
page
|
||||
for page in pages
|
||||
if contains_reimbursement(pages, page)
|
||||
]
|
||||
table_pages = [
|
||||
page
|
||||
for page in reimbursement_pages
|
||||
if "-------Table Start--------" in pages[page]
|
||||
]
|
||||
|
||||
matches = []
|
||||
for page in table_pages:
|
||||
matches.extend(pattern.findall(pages[page]))
|
||||
|
||||
if matches:
|
||||
files_with_pattern += 1
|
||||
results.append(
|
||||
(
|
||||
file_path,
|
||||
len(matches),
|
||||
True,
|
||||
len(reimbursement_pages),
|
||||
len(table_pages),
|
||||
)
|
||||
)
|
||||
else:
|
||||
results.append(
|
||||
(
|
||||
file_path,
|
||||
0,
|
||||
False,
|
||||
len(reimbursement_pages),
|
||||
len(table_pages),
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"Error reading {file_path}: {str(e)}")
|
||||
results.append((file_path, 0, False, 0, 0))
|
||||
|
||||
return results, total_files, files_with_pattern
|
||||
|
||||
|
||||
def write_to_csv(results, total_files, files_with_pattern, output_file):
|
||||
with open(output_file, "w", newline="", encoding="utf-8") as csvfile:
|
||||
writer = csv.writer(csvfile)
|
||||
writer.writerow(
|
||||
[
|
||||
"File Name",
|
||||
"Match Count",
|
||||
"Contains Pattern",
|
||||
"Reimbursement Pages",
|
||||
"Table Pages",
|
||||
"Percentage of Total",
|
||||
]
|
||||
)
|
||||
|
||||
for (
|
||||
file_path,
|
||||
match_count,
|
||||
contains_pattern,
|
||||
reimbursement_pages,
|
||||
table_pages,
|
||||
) in results:
|
||||
percentage = (match_count / total_files) * 100 if total_files > 0 else 0
|
||||
writer.writerow(
|
||||
[
|
||||
file_path,
|
||||
match_count,
|
||||
contains_pattern,
|
||||
reimbursement_pages,
|
||||
table_pages,
|
||||
f"{percentage:.2f}%",
|
||||
]
|
||||
)
|
||||
|
||||
percentage_with_pattern = (
|
||||
(files_with_pattern / total_files) * 100 if total_files > 0 else 0
|
||||
)
|
||||
writer.writerow(["", "", "", "", "", ""])
|
||||
writer.writerow(["Summary", "", "", "", "", ""])
|
||||
writer.writerow(["Total Files", total_files, "", "", "", ""])
|
||||
writer.writerow(
|
||||
[
|
||||
"Files with Pattern",
|
||||
files_with_pattern,
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
f"{percentage_with_pattern:.2f}%",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
facility_dir = "batch1_priority"
|
||||
|
||||
if os.path.isdir(facility_dir):
|
||||
results, total_files, files_with_pattern = search_files(facility_dir)
|
||||
output_file = os.path.join(current_dir, "contract_analysis_results.csv")
|
||||
write_to_csv(results, total_files, files_with_pattern, output_file)
|
||||
print(f"Results have been written to {output_file}")
|
||||
else:
|
||||
print(f"Error: The directory {facility_dir} does not exist.")
|
||||
@@ -0,0 +1,193 @@
|
||||
import src.config as config
|
||||
from datetime import datetime
|
||||
import pandas as pd
|
||||
import src.utils.string_utils as string_utils
|
||||
import io
|
||||
|
||||
|
||||
def find_inv_test_results():
|
||||
"""
|
||||
Efficiently find the 30 most recent directories containing 'inv-test'
|
||||
and list their RESULTS.csv files.
|
||||
|
||||
Strategy: Only list top-level directories first, filter for 'inv-test',
|
||||
then search only those directories for RESULTS.csv files.
|
||||
"""
|
||||
s3_client = config.S3_CLIENT
|
||||
bucket_name = config.S3_OUTPUT_BUCKET
|
||||
|
||||
print(f"\nSearching for 'inv-test' directories in bucket: {bucket_name}")
|
||||
print("=" * 80)
|
||||
|
||||
# Step 1: Get only top-level directories (much faster than scanning all files)
|
||||
print("Step 1: Listing top-level directories...")
|
||||
response = s3_client.list_objects_v2(Bucket=bucket_name, Delimiter="/")
|
||||
|
||||
if "CommonPrefixes" not in response:
|
||||
print("No directories found in bucket")
|
||||
return []
|
||||
|
||||
all_top_level_dirs = [
|
||||
prefix["Prefix"].rstrip("/") for prefix in response["CommonPrefixes"]
|
||||
]
|
||||
print(f"Found {len(all_top_level_dirs)} total top-level directories")
|
||||
|
||||
# Step 2: Filter for 'inv-test' directories only
|
||||
inv_test_dirs = [d for d in all_top_level_dirs if "inv-test" in d]
|
||||
print(f"Found {len(inv_test_dirs)} directories containing 'inv-test'")
|
||||
|
||||
if not inv_test_dirs:
|
||||
print("No 'inv-test' directories found")
|
||||
return []
|
||||
|
||||
# Step 3: Get timestamps for inv-test directories only (not all files in bucket)
|
||||
print("\nStep 2: Getting timestamps for inv-test directories...")
|
||||
dir_timestamps = []
|
||||
|
||||
for dir_name in inv_test_dirs:
|
||||
# Get just the first file in each directory to get a timestamp
|
||||
response = s3_client.list_objects_v2(
|
||||
Bucket=bucket_name, Prefix=f"{dir_name}/", MaxKeys=1
|
||||
)
|
||||
|
||||
if "Contents" in response and len(response["Contents"]) > 0:
|
||||
timestamp = response["Contents"][0]["LastModified"]
|
||||
dir_timestamps.append((dir_name, timestamp))
|
||||
|
||||
# Sort by timestamp (most recent first)
|
||||
dir_timestamps.sort(key=lambda x: x[1], reverse=True)
|
||||
|
||||
# Get the 30 most recent
|
||||
recent_inv_test_dirs = dir_timestamps[:30]
|
||||
|
||||
print(
|
||||
f"\nStep 3: Searching 30 most recent 'inv-test' directories for RESULTS.csv files:"
|
||||
)
|
||||
print("=" * 80)
|
||||
|
||||
total_results_files = 0
|
||||
all_results_files = {} # Dictionary to store directory -> list of file paths
|
||||
|
||||
for i, (dir_name, timestamp) in enumerate(recent_inv_test_dirs, 1):
|
||||
print(f"\n[{i}/30] Directory: {dir_name}")
|
||||
print(f" Last Modified: {timestamp.strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
|
||||
# Search only this directory for RESULTS.csv files
|
||||
results_files = []
|
||||
paginator = s3_client.get_paginator("list_objects_v2")
|
||||
|
||||
for page in paginator.paginate(Bucket=bucket_name, Prefix=f"{dir_name}/"):
|
||||
if "Contents" not in page:
|
||||
continue
|
||||
|
||||
for obj in page["Contents"]:
|
||||
key = obj["Key"]
|
||||
# Ignore files in subdirectories named "individual"
|
||||
if "/individual/" in key:
|
||||
continue
|
||||
if key.endswith("-RESULTS.csv"):
|
||||
results_files.append(key)
|
||||
|
||||
if results_files:
|
||||
print(f" Found {len(results_files)} RESULTS.csv file(s):")
|
||||
for file_key in results_files:
|
||||
print(f" ✓ {file_key}")
|
||||
total_results_files += 1
|
||||
all_results_files[dir_name] = results_files
|
||||
else:
|
||||
print(f" ✗ No -RESULTS.csv files found")
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
print(f"Summary:")
|
||||
print(f" - Searched 30 most recent 'inv-test' directories")
|
||||
print(f" - Total -RESULTS.csv files found: {total_results_files}")
|
||||
print("=" * 80)
|
||||
|
||||
# Step 4: Generate missingness report
|
||||
if total_results_files > 0:
|
||||
print(f"\nStep 4: Generating missingness report...")
|
||||
generate_missingness_report(s3_client, bucket_name, all_results_files)
|
||||
|
||||
return recent_inv_test_dirs
|
||||
|
||||
|
||||
def generate_missingness_report(s3_client, bucket_name, results_files_dict):
|
||||
"""
|
||||
Generate a missingness report for all RESULTS.csv files.
|
||||
|
||||
Args:
|
||||
s3_client: S3 client object
|
||||
bucket_name: S3 bucket name
|
||||
results_files_dict: Dictionary mapping directory names to lists of result file paths
|
||||
"""
|
||||
print("=" * 80)
|
||||
print("Generating Missingness Report")
|
||||
print("=" * 80)
|
||||
|
||||
missingness_data = []
|
||||
|
||||
for dir_name, file_paths in results_files_dict.items():
|
||||
for file_path in file_paths:
|
||||
print(f"\nProcessing: {file_path}")
|
||||
|
||||
try:
|
||||
# Read CSV from S3
|
||||
response = s3_client.get_object(Bucket=bucket_name, Key=file_path)
|
||||
csv_content = response["Body"].read().decode("utf-8")
|
||||
df = pd.read_csv(io.StringIO(csv_content))
|
||||
|
||||
if df.empty:
|
||||
print(f" ⚠ File is empty, skipping")
|
||||
continue
|
||||
|
||||
# Calculate missingness for each column
|
||||
row_data = {"csv_file": file_path}
|
||||
|
||||
for column in df.columns:
|
||||
empty_mask = string_utils.is_empty(df[column], pd_mask=True)
|
||||
missingness_pct = (empty_mask.sum() / len(df)) * 100
|
||||
row_data[column] = round(missingness_pct, 2)
|
||||
|
||||
missingness_data.append(row_data)
|
||||
print(f" ✓ Processed {len(df)} rows, {len(df.columns)} columns")
|
||||
|
||||
except Exception as e:
|
||||
print(f" ✗ Error processing file: {e}")
|
||||
continue
|
||||
|
||||
if not missingness_data:
|
||||
print("\nNo valid data to create report")
|
||||
return
|
||||
|
||||
# Create missingness DataFrame
|
||||
missingness_df = pd.DataFrame(missingness_data)
|
||||
|
||||
# Set csv_file as index for better readability
|
||||
missingness_df.set_index("csv_file", inplace=True)
|
||||
|
||||
# Save report
|
||||
output_filename = (
|
||||
f"missingness_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv"
|
||||
)
|
||||
missingness_df.to_csv(output_filename)
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
print(f"Missingness Report Summary:")
|
||||
print(f" - Total CSV files processed: {len(missingness_data)}")
|
||||
print(f" - Total columns analyzed: {len(missingness_df.columns)}")
|
||||
print(f" - Report saved to: {output_filename}")
|
||||
print("=" * 80)
|
||||
|
||||
# Display sample of the report
|
||||
print("\nSample of Missingness Report (first 5 columns):")
|
||||
print("-" * 80)
|
||||
if len(missingness_df.columns) > 5:
|
||||
print(missingness_df.iloc[:, :5].to_string())
|
||||
print(f"\n... and {len(missingness_df.columns) - 5} more columns")
|
||||
else:
|
||||
print(missingness_df.to_string())
|
||||
print("-" * 80)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
find_inv_test_results()
|
||||
@@ -0,0 +1,25 @@
|
||||
import pandas as pd
|
||||
from constants.constants import Constants
|
||||
from src.investment.postprocess import postprocess
|
||||
|
||||
# File paths
|
||||
input_excel_path = "input.xlsx"
|
||||
output_csv_path = "output.csv"
|
||||
|
||||
# Read Excel file into DataFrame
|
||||
print(f"Reading Excel file: {input_excel_path}")
|
||||
df = pd.read_excel(input_excel_path, dtype=str)
|
||||
print(f"Loaded {len(df)} rows from Excel")
|
||||
|
||||
# Initialize Constants
|
||||
constants = Constants()
|
||||
|
||||
# Process the DataFrame
|
||||
print("Processing data through postprocess()...")
|
||||
df_processed = postprocess(df, constants)
|
||||
print(f"Processing complete. Output has {len(df_processed)} rows")
|
||||
|
||||
# Write to CSV
|
||||
print(f"Writing CSV file: {output_csv_path}")
|
||||
df_processed.to_csv(output_csv_path, index=False)
|
||||
print("Done!")
|
||||
@@ -0,0 +1,101 @@
|
||||
import table_funcs
|
||||
import config
|
||||
import utils
|
||||
import preprocessing_funcs
|
||||
import concurrent.futures
|
||||
import pandas as pd
|
||||
import os
|
||||
|
||||
input_dict = utils.read_input()
|
||||
table_analysis_dicts = []
|
||||
|
||||
|
||||
def table_analysis(file_object):
|
||||
filename, contract_text = file_object
|
||||
text_dict = preprocessing_funcs.split_text(contract_text)
|
||||
file_dict = {"Filename": filename, "Filename_pdf": filename.replace(".txt", ".pdf")}
|
||||
|
||||
contains_reimbursement = utils.contains_reimbursement(contract_text)
|
||||
file_dict["Contains Reimbursement"] = contains_reimbursement
|
||||
|
||||
# Determine if the file should be present in the output
|
||||
file_dict["Present in Output"] = contains_reimbursement
|
||||
|
||||
stats_dict = table_funcs.get_table_stats(text_dict)
|
||||
file_dict.update(stats_dict)
|
||||
table_analysis_dicts.append(file_dict)
|
||||
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor:
|
||||
futures = [executor.submit(table_analysis, item) for item in input_dict.items()]
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
try:
|
||||
future.result()
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
|
||||
# Create the main table analysis DataFrame
|
||||
table_analysis_df = pd.DataFrame(table_analysis_dicts)
|
||||
table_analysis_df.to_csv(
|
||||
os.path.join(config.REPORTING_OUTPUT_DIRECTORY, config.TABLE_ANALYSIS_NAME)
|
||||
)
|
||||
|
||||
# Define the columns we want in our report
|
||||
report_columns = [
|
||||
"Filename",
|
||||
"Contains Reimbursement",
|
||||
"Present in Output",
|
||||
"Table Count",
|
||||
"Table Page Count",
|
||||
"Rate Count",
|
||||
"Num Rate Pages",
|
||||
"Num >=10 Rates",
|
||||
]
|
||||
|
||||
# Create the new report DataFrame with available columns
|
||||
report_df = table_analysis_df[
|
||||
[col for col in report_columns if col in table_analysis_df.columns]
|
||||
]
|
||||
|
||||
|
||||
# Function to safely create summary columns
|
||||
def create_summary_column(df, source_col, target_col):
|
||||
if source_col in df.columns:
|
||||
df[target_col] = df[source_col].apply(
|
||||
lambda x: ", ".join(map(str, eval(x))) if x != "[]" else "None"
|
||||
)
|
||||
else:
|
||||
df[target_col] = "N/A"
|
||||
|
||||
|
||||
# Add summary columns if the data is available
|
||||
create_summary_column(report_df, "Table Pages", "Table Pages Summary")
|
||||
create_summary_column(report_df, "Rate Pages", "Rate Pages Summary")
|
||||
create_summary_column(report_df, ">=10 Rate Pages", ">=10 Rate Pages Summary")
|
||||
|
||||
# Define the final column order
|
||||
final_columns = [
|
||||
"Filename",
|
||||
"Contains Reimbursement",
|
||||
"Present in Output",
|
||||
"Table Count",
|
||||
"Table Page Count",
|
||||
"Table Pages Summary",
|
||||
"Rate Count",
|
||||
"Num Rate Pages",
|
||||
"Rate Pages Summary",
|
||||
"Num >=10 Rates",
|
||||
">=10 Rate Pages Summary",
|
||||
]
|
||||
|
||||
# Reorder columns, including only those that exist
|
||||
report_df = report_df[[col for col in final_columns if col in report_df.columns]]
|
||||
|
||||
# Save the new report to a CSV file
|
||||
report_output_path = os.path.join(
|
||||
config.REPORTING_OUTPUT_DIRECTORY, "enhanced_reimbursement_report.csv"
|
||||
)
|
||||
report_df.to_csv(report_output_path, index=False)
|
||||
|
||||
print(f"Enhanced reimbursement report saved to: {report_output_path}")
|
||||
print("Columns in the report:", ", ".join(report_df.columns))
|
||||
@@ -0,0 +1,111 @@
|
||||
import pandas as pd
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
import re
|
||||
from openpyxl import Workbook
|
||||
from openpyxl.utils.dataframe import dataframe_to_rows
|
||||
import argparse
|
||||
import boto3
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
sys.path.append("src")
|
||||
import utils
|
||||
import config
|
||||
from qa_qc_helpers import perform_qc_qa, save_qcqa_wb
|
||||
from consolidate_output import consolidate_output
|
||||
|
||||
|
||||
file_name_column = config.FILE_NAME_COLUMN
|
||||
|
||||
"""python scripts/qa_qc.py input_dir=______ output_dir=_______ ac_df=________ b_df=______ read_mode=____ df_read_mode=________
|
||||
You should have both input and output (input is local or s3, ouput is just an empty folder) and then one of ac_df or b_df or both
|
||||
The final report will be saved in the parent directory of when you are running under reports/
|
||||
Run this from the base folder of the repo."""
|
||||
|
||||
|
||||
def main():
|
||||
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
parent_dir = os.path.dirname(script_dir)
|
||||
|
||||
reports_dir = os.path.join(parent_dir, "reports")
|
||||
os.makedirs(reports_dir, exist_ok=True)
|
||||
|
||||
if config.AC_DF:
|
||||
print(f"AC DataFrame: {config.AC_DF}")
|
||||
if config.B_DF:
|
||||
print(f"B DataFrame: {config.B_DF}")
|
||||
if config.LOCAL_PATH:
|
||||
print(f"Input Dir: {config.LOCAL_PATH}")
|
||||
if (config.S3_BUCKET) and (config.S3_PREFIX):
|
||||
print(f"S3 Bucket: {config.S3_BUCKET}")
|
||||
print(f"S3 Prefix: {config.S3_PREFIX}")
|
||||
print(f"Output Dir: {config.OUTPUT_DIRECTORY}")
|
||||
print(f"Reports Dir: {reports_dir}")
|
||||
|
||||
if not config.LOCAL_PATH or (config.S3_BUCKET and config.S3_PREFIX):
|
||||
print("Needs an Input Directory!")
|
||||
if not config.AC_DF and not config.B_DF:
|
||||
print("Needs at least one of AC Outputs and B Outputs!")
|
||||
return
|
||||
|
||||
input_dict = utils.read_input()
|
||||
ac_df, b_df = utils.read_input_csv()
|
||||
total_files = len(input_dict)
|
||||
print(f"Total Input Files: {total_files}")
|
||||
|
||||
all_input_filenames = (
|
||||
[utils.remove_txt_extension(f.lower()) for f in input_dict.keys()]
|
||||
if input_dict
|
||||
else []
|
||||
)
|
||||
|
||||
if ac_df is not None:
|
||||
ac_df = utils.remove_unnamed_columns(ac_df)
|
||||
ac_df[file_name_column] = ac_df[file_name_column].apply(
|
||||
lambda x: utils.remove_txt_extension(str(x)).lower()
|
||||
)
|
||||
ac_filenames = set(ac_df[file_name_column])
|
||||
|
||||
if b_df is not None:
|
||||
b_df = utils.remove_unnamed_columns(b_df)
|
||||
b_filenames = set(b_df[file_name_column])
|
||||
b_df[file_name_column] = b_df[file_name_column].apply(
|
||||
lambda x: utils.remove_txt_extension(str(x)).lower()
|
||||
)
|
||||
|
||||
if ac_df is not None and b_df is not None:
|
||||
unmatched_b_filenames = b_filenames - ac_filenames
|
||||
unmatched_b_df = pd.DataFrame({file_name_column: list(unmatched_b_filenames)})
|
||||
unmatched_b_output_path = os.path.join(
|
||||
config.OUTPUT_DIRECTORY, "unmatched_B_filenames.csv"
|
||||
)
|
||||
unmatched_b_df.to_csv(unmatched_b_output_path, index=False)
|
||||
print(f"Unmatched B filenames saved to '{unmatched_b_output_path}'")
|
||||
|
||||
# Perform an inner merge to keep only matching rows for the main output
|
||||
ac_df, b_df, merged_df = consolidate_output(timestamp, ac_df, b_df)
|
||||
merged_df = utils.remove_unnamed_columns(merged_df)
|
||||
ac_rows_before = len(ac_df)
|
||||
b_rows_before = len(b_df)
|
||||
merged_rows = len(merged_df)
|
||||
unmatched_b_count = len(unmatched_b_filenames)
|
||||
|
||||
print(f"Rows in AC DataFrame before merge: {ac_rows_before}")
|
||||
print(f"Rows in B DataFrame before merge: {b_rows_before}")
|
||||
print(f"Rows in merged DataFrame: {merged_rows}")
|
||||
print(f"Unique filenames in B not found in AC: {unmatched_b_count}")
|
||||
|
||||
merged_output_path = os.path.join(
|
||||
config.OUTPUT_DIRECTORY, "DOCZYAI_merged_results.csv"
|
||||
)
|
||||
merged_df.to_csv(merged_output_path, index=False)
|
||||
print(f"Merged data saved to '{merged_output_path}'")
|
||||
|
||||
wb = perform_qc_qa(ac_df, b_df, merged_df, input_dict)
|
||||
save_qcqa_wb(wb, timestamp)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,38 @@
|
||||
import table_funcs
|
||||
import config
|
||||
import utils
|
||||
import preprocessing_funcs
|
||||
|
||||
import concurrent.futures
|
||||
import pandas as pd
|
||||
import os
|
||||
|
||||
input_dict = utils.read_input()
|
||||
table_analysis_dicts = []
|
||||
|
||||
|
||||
def table_analysis(file_object):
|
||||
filename, contract_text = file_object
|
||||
text_dict = preprocessing_funcs.split_text(contract_text)
|
||||
file_dict = {"Filename": filename, "Filename_pdf": filename.replace(".txt", ".pdf")}
|
||||
if utils.contains_reimbursement(contract_text):
|
||||
file_dict["Contains Reimbursement"] = True
|
||||
else:
|
||||
file_dict["Contains Reimbursement"] = False
|
||||
stats_dict = table_funcs.get_table_stats(text_dict)
|
||||
file_dict.update(stats_dict)
|
||||
table_analysis_dicts.append(file_dict)
|
||||
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor:
|
||||
futures = [executor.submit(table_analysis, item) for item in input_dict.items()]
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
try:
|
||||
future.result()
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
|
||||
table_analysis_df = pd.DataFrame(table_analysis_dicts)
|
||||
table_analysis_df.to_csv(
|
||||
os.path.join(config.REPORTING_OUTPUT_DIRECTORY, config.TABLE_ANALYSIS_NAME)
|
||||
)
|
||||
@@ -0,0 +1,223 @@
|
||||
"""
|
||||
This script was written to extract .txt files from pdf for adhoc client runs.
|
||||
Ensure you have permissions for API gateway, S3, Lambda function and SQS to execute this (DEVELOPER & above roles in DEV & UAT for Doczy should suffice)
|
||||
If you have Analyst / Test roles and need the permissions to be elevated or policies to be updated then please ask Sannan Iqbal to make the changes.
|
||||
|
||||
The flow to execute the pipeline is:
|
||||
Create batch using api req
|
||||
Add files to the newly created batch along with the batch id as the tag
|
||||
Construct payload to trigger pipeline
|
||||
Post request to trigger pipeline with payload
|
||||
The output text file can be found in the client_bucket/contract_text_file/batch_123456
|
||||
And the final LLM parsed outputs can be found in client_bucket/final_output/batch_123456
|
||||
"""
|
||||
|
||||
import os
|
||||
import boto3
|
||||
import requests
|
||||
import json
|
||||
import boto3
|
||||
from datetime import datetime
|
||||
import os
|
||||
from botocore.auth import SigV4Auth
|
||||
from botocore.awsrequest import AWSRequest
|
||||
from botocore.credentials import get_credentials
|
||||
from botocore.session import Session
|
||||
import time
|
||||
|
||||
|
||||
# Function to upload files to S3
|
||||
def upload_files_to_s3(directory, bucket, batch_id):
|
||||
contract_list = []
|
||||
for filename in os.listdir(directory):
|
||||
if filename.endswith(".pdf"):
|
||||
file_path = os.path.join(directory, filename)
|
||||
s3_key = f"contracts-landing-zone/{batch_id}/{filename}"
|
||||
|
||||
# Upload file to S3
|
||||
s3_client.upload_file(file_path, bucket, s3_key)
|
||||
print(f"Uploaded {filename} to S3 bucket\n")
|
||||
|
||||
# Add tags to the uploaded file
|
||||
s3_client.put_object_tagging(
|
||||
Bucket=bucket,
|
||||
Key=s3_key,
|
||||
Tagging={"TagSet": [{"Key": "BatchId", "Value": batch_id}]},
|
||||
)
|
||||
print(f"Added tags to {filename}\n")
|
||||
# Add file details to contract list
|
||||
# For now we can leave it as is since only A, C have been operationalized
|
||||
contract_list.append(
|
||||
{
|
||||
"contract_name": filename,
|
||||
"groups": [
|
||||
"A", # This needs to be dynamic in UI 1 based on what group has been selected
|
||||
"C",
|
||||
],
|
||||
"contract_source_path": s3_key,
|
||||
}
|
||||
)
|
||||
return contract_list
|
||||
|
||||
|
||||
def create_batch(client_bucket, create_batch_url):
|
||||
myobj = {"client-bucket-name": client_bucket}
|
||||
|
||||
# Call create batch API endpoint
|
||||
response = requests.post(create_batch_url, json=myobj)
|
||||
if response.status_code >= 200 and response.status_code < 300:
|
||||
try:
|
||||
new_batch_id = json.loads(json.loads(response.text)["body"])["batch_id"]
|
||||
landing_zone = json.loads(json.loads(response.text)["body"])["landing_zone"]
|
||||
except:
|
||||
print(myobj)
|
||||
print(response.text)
|
||||
new_batch_id = "failed_cases"
|
||||
landing_zone = "contracts_landing_zone"
|
||||
else:
|
||||
print(response.text)
|
||||
new_batch_id = "failed_cases"
|
||||
landing_zone = "contracts_landing_zone"
|
||||
return new_batch_id, landing_zone
|
||||
|
||||
|
||||
def list_filtered_files(bucket_name, prefix, start_date, end_date, profile_name):
|
||||
"""
|
||||
List all files in an S3 bucket filtered by date range.
|
||||
|
||||
Parameters:
|
||||
- bucket_name: str, the name of the S3 bucket
|
||||
- prefix: str, the prefix (folder) in the S3 bucket
|
||||
- start_date: str, the start date in ISO 8601 format
|
||||
- end_date: str, the end date in ISO 8601 format
|
||||
- profile_name: str, the AWS profile name
|
||||
|
||||
Returns:
|
||||
- list of str: the keys of the filtered files
|
||||
"""
|
||||
# Parse the dates
|
||||
start_date = datetime.fromisoformat(start_date.replace("Z", "+00:00"))
|
||||
end_date = datetime.fromisoformat(end_date.replace("Z", "+00:00"))
|
||||
|
||||
# Initialize a session using the specified profile
|
||||
session = boto3.Session(profile_name=profile_name)
|
||||
s3_client = session.client("s3")
|
||||
|
||||
# List objects in the bucket with the specified prefix
|
||||
paginator = s3_client.get_paginator("list_objects_v2")
|
||||
page_iterator = paginator.paginate(Bucket=bucket_name, Prefix=prefix)
|
||||
|
||||
# Filter files by date
|
||||
filtered_files = []
|
||||
for page in page_iterator:
|
||||
if "Contents" in page:
|
||||
for obj in page["Contents"]:
|
||||
last_modified = obj["LastModified"]
|
||||
if start_date <= last_modified <= end_date:
|
||||
filtered_files.append(obj["Key"])
|
||||
|
||||
return filtered_files
|
||||
|
||||
|
||||
def download_files(bucket_name, file_keys, profile_name, local_directory):
|
||||
"""
|
||||
Download files from an S3 bucket.
|
||||
|
||||
Parameters:
|
||||
- bucket_name: str, the name of the S3 bucket
|
||||
- file_keys: list of str, the keys of the files to download
|
||||
- profile_name: str, the AWS profile name
|
||||
- local_directory: str, the local directory to download files to
|
||||
"""
|
||||
# Initialize a session using the specified profile
|
||||
session = boto3.Session(profile_name=profile_name)
|
||||
s3_client = session.client("s3")
|
||||
|
||||
# Ensure the local directory exists
|
||||
if not os.path.exists(local_directory):
|
||||
os.makedirs(local_directory)
|
||||
|
||||
# Download each file
|
||||
for key in file_keys:
|
||||
file_name = os.path.basename(key) # Get only the file name from the key
|
||||
local_file_path = os.path.join(local_directory, file_name)
|
||||
print(f"Downloading {key} to {local_file_path}")
|
||||
s3_client.download_file(bucket_name, key, local_file_path)
|
||||
|
||||
print("Download completed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Define your variables
|
||||
s3_bucket = "doczyai-use2-u-cn1-s3-textract-processing-001"
|
||||
# batch_id = 'batch_100524101551'
|
||||
client_name = "Priority Health"
|
||||
username = "ADHOC USER"
|
||||
|
||||
# These endpoints are in UAT, please change them to DEV if there are access issues with UAT
|
||||
api_endpoint = (
|
||||
"https://29gm8cek03.execute-api.us-east-2.amazonaws.com/dev/trigger-pipeline"
|
||||
)
|
||||
create_batch_url = (
|
||||
"https://29gm8cek03.execute-api.us-east-2.amazonaws.com/dev/create-batch"
|
||||
)
|
||||
|
||||
session = boto3.Session(
|
||||
profile_name="temp_cred"
|
||||
) # Change the profile name to the one you have in your .aws/credentials file
|
||||
s3_client = session.client("s3")
|
||||
|
||||
# Use current directory as PDF directory
|
||||
pdf_directory = "C:\\Doczy\\Priority Health\\For_Textract\\For_Textract\\UAT- test East Paris Surgical Center copy"
|
||||
|
||||
# Create batch
|
||||
batch_id, landing_zone = create_batch(s3_bucket, create_batch_url)
|
||||
print(f"Batch ID: {batch_id}")
|
||||
print(f"Landing Zone: {landing_zone}")
|
||||
# batch_id = 'batch_110624213433'
|
||||
# landing_zone = 'contracts_landing_zone/batch_110624213433/'
|
||||
|
||||
if batch_id == "failed_cases":
|
||||
print("Batch creation failed. Exiting...")
|
||||
exit()
|
||||
# Upload files and get contract list
|
||||
contract_list = upload_files_to_s3(pdf_directory, s3_bucket, batch_id)
|
||||
|
||||
# Create JSON object
|
||||
data = {
|
||||
"s3_bucket": s3_bucket,
|
||||
"batch_id": batch_id,
|
||||
"client_name": client_name,
|
||||
"username": username,
|
||||
"contract_list": contract_list,
|
||||
}
|
||||
|
||||
# Make POST request to API
|
||||
response = requests.post(api_endpoint, json=data)
|
||||
|
||||
# Print response
|
||||
print(response.status_code)
|
||||
print(response.json())
|
||||
|
||||
##################
|
||||
# Get text files
|
||||
|
||||
prefix = f"contract-text-file/{batch_id}/" # if you have a specific prefix (folder) in your bucket
|
||||
|
||||
# These dates are to filter the contracts in case there are older contracts in the same batch
|
||||
start_date = "2024-06-04T00:00:00Z" # ISO 8601 format
|
||||
end_date = "2024-06-14T23:59:59Z" # ISO 8601 format
|
||||
profile_name = "temp_cred"
|
||||
local_directory = "C:\\Doczy\\Priority Health\\For_Textract\\For_Textract\\text otuput" # local directory to save files
|
||||
|
||||
# List filtered files
|
||||
time.sleep(
|
||||
30
|
||||
) # Waiting for the text files to be generated, this may take longer and files may not be available after 30 seconds sometimes
|
||||
filtered_files = list_filtered_files(
|
||||
s3_bucket, prefix, start_date, end_date, profile_name
|
||||
)
|
||||
print(f"Filtered files: {len(filtered_files)}")
|
||||
|
||||
# Download files
|
||||
download_files(s3_bucket, filtered_files, profile_name, local_directory)
|
||||
@@ -0,0 +1,178 @@
|
||||
import json
|
||||
import boto3
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from datetime import datetime
|
||||
import os
|
||||
import anthropic
|
||||
import re
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
import config
|
||||
import utils
|
||||
|
||||
# Global configuration variables
|
||||
S3_REGION = "us-east-2"
|
||||
BEDROCK_REGION = "us-east-1"
|
||||
LLM_SELECTED = "Claude Instant"
|
||||
bedrock_runtime = config.BEDROCK_RUNTIME
|
||||
|
||||
|
||||
def find_tin_numbers(text):
|
||||
# Regular expression to find sets of 9 digits with an optional dash after the first two digits
|
||||
pattern = r"\b\d{2}-?\d{7}\b"
|
||||
|
||||
# Find all matches in the provided text
|
||||
matches = re.findall(pattern, text)
|
||||
|
||||
return matches
|
||||
|
||||
|
||||
def invoke_llm(context, llm_selected=LLM_SELECTED):
|
||||
prompt_data = f"""
|
||||
|
||||
Human: Use the following pieces of context to provide a concise answer to the questions at the end. You must answer in python list format.
|
||||
|
||||
{context}
|
||||
|
||||
Question: What is the Group provider's taxpayer identification number stated in the contract? This is a 9 digit long number typically with a hyphen after the first 2 digits. Return only this 9 digit number with a hyphen after the first 2 digits. Do not elaborate or add context.
|
||||
|
||||
Assistant: Answer in list format: ["""
|
||||
|
||||
if llm_selected == "Claude 3 - Sonnet":
|
||||
model_id = "anthropic.claude-3-5-sonnet-20240620-v1:0"
|
||||
body = json.dumps(
|
||||
{
|
||||
"anthropic_version": "bedrock-2023-05-31",
|
||||
"max_tokens": 4096,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": anthropic.HUMAN_PROMPT
|
||||
+ prompt_data
|
||||
+ anthropic.AI_PROMPT,
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
"temperature": 0.0,
|
||||
}
|
||||
)
|
||||
elif llm_selected in ["Claude Instant"]:
|
||||
# Note: Claude 2 support has been removed - using Claude 3 Haiku instead
|
||||
model_id = config.MODEL_ID_CLAUDE3_HAIKU
|
||||
body = json.dumps(
|
||||
{
|
||||
"prompt": anthropic.HUMAN_PROMPT + prompt_data + anthropic.AI_PROMPT,
|
||||
"max_tokens_to_sample": 2048,
|
||||
"temperature": 0.0,
|
||||
"top_p": 1,
|
||||
"top_k": 250,
|
||||
"stop_sequences": [anthropic.HUMAN_PROMPT],
|
||||
}
|
||||
)
|
||||
else:
|
||||
raise ValueError("Unsupported LLM selected")
|
||||
|
||||
try:
|
||||
response = bedrock_runtime.invoke_model(
|
||||
body=body,
|
||||
modelId=model_id,
|
||||
accept="application/json",
|
||||
contentType="application/json",
|
||||
)
|
||||
response_body = json.loads(response.get("body").read())
|
||||
if llm_selected == "Claude 3 - Sonnet":
|
||||
response_text = response_body["content"][0]["text"]
|
||||
elif llm_selected in ["Claude Instant"]:
|
||||
response_text = response_body["completion"]
|
||||
else:
|
||||
print("Unsupported LLM selected")
|
||||
except Exception as e:
|
||||
print(f"Error invoking LLM: {e}")
|
||||
response_text = "failed"
|
||||
return response_text.strip()
|
||||
|
||||
|
||||
def main(filename, contract_text):
|
||||
print(f"Processing contract: {filename}")
|
||||
try:
|
||||
df = pd.DataFrame(columns=["Contract Name", "Raw value", "Final value"])
|
||||
|
||||
tin_number = find_tin_numbers(contract_text)
|
||||
final_tin = list(set(tin_number))
|
||||
|
||||
if len(final_tin) != 1:
|
||||
final_tin = find_tin_numbers(
|
||||
invoke_llm(contract_text, llm_selected=LLM_SELECTED)
|
||||
)
|
||||
|
||||
df["Raw value"] = [tin_number]
|
||||
df["Final value"] = [final_tin]
|
||||
df["Contract Name"] = [filename]
|
||||
|
||||
df.to_csv(f"tin_output/{filename}.csv", index=False)
|
||||
except Exception as e:
|
||||
print(f"Error processing contract: {e}")
|
||||
|
||||
|
||||
# bucket = "centene-national-contracting-files"
|
||||
# contract_path = "no_tins_text_files"
|
||||
|
||||
# s3_client = boto3.Session(aws_access_key_id=config.S3_ACCESS_KEY_ID,
|
||||
# aws_secret_access_key=config.S3_SECRET_ACCESS_KEY,
|
||||
# aws_session_token=config.S3_SESSION_TOKEN, region_name="us-east-2").client('s3')
|
||||
|
||||
# objects = s3_client.list_objects_v2(Bucket=bucket, Prefix=contract_path)
|
||||
file_list = []
|
||||
|
||||
OUTPUT_DIRECTORY = r"tin_output/no_tins_text_files"
|
||||
NUM_WORKERS = 10
|
||||
BATCH_SIZE = 10
|
||||
|
||||
|
||||
def process_file(item):
|
||||
filename, contract_text = item
|
||||
if filename.endswith(".txt") and not os.path.exists(
|
||||
os.path.join(OUTPUT_DIRECTORY, (filename[:-3] + "csv"))
|
||||
):
|
||||
main(filename, contract_text)
|
||||
|
||||
|
||||
def process_directory(input_dict):
|
||||
# for item in input_dict.items():
|
||||
with ThreadPoolExecutor(max_workers=NUM_WORKERS) as executor:
|
||||
futures = {executor.submit(process_file, item) for item in input_dict.items()}
|
||||
for future in as_completed(futures):
|
||||
future.result() # This will raise an exception if the callable raised one
|
||||
|
||||
|
||||
input_dict = utils.read_input(path="data/cnc/no_tins")
|
||||
process_directory(input_dict)
|
||||
|
||||
individual_path = r"tin_output/no_tins_text_files"
|
||||
|
||||
|
||||
def individual_to_dict(filename):
|
||||
df = pd.read_csv(os.path.join(individual_path, filename))
|
||||
df = df.dropna(axis=1, how="all")
|
||||
column_names = ["Filename", "Answer", "Final Answer"]
|
||||
new_column_names = column_names[: len(df.columns)] + list(
|
||||
df.columns[len(column_names) :]
|
||||
)
|
||||
df.columns = new_column_names
|
||||
df_dict = {"Filename": filename}
|
||||
df_dict["Answer"] = df["Answer"].iloc[0]
|
||||
df_dict["Final Answer"] = df["Final Answer"].iloc[0]
|
||||
return df_dict
|
||||
|
||||
|
||||
all_dicts = []
|
||||
for filename in os.listdir(individual_path):
|
||||
df_dict = individual_to_dict(filename)
|
||||
all_dicts.append(df_dict)
|
||||
final_df = pd.DataFrame(all_dicts)
|
||||
final_df.to_csv("results.csv", index=False)
|
||||
Reference in New Issue
Block a user