Merged in feature/historical-cost-analysis (pull request #901)
Feature/historical cost analysis * Initial commit * Finalize cost_analysis * Black format * Merged DEV into feature/historical-cost-analysis * Refactor * Black * Merged DEV into feature/historical-cost-analysis Approved-by: Siddhant Medar
This commit is contained in:
Katon Minhas
@@ -0,0 +1,173 @@
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from src.utils import io_utils
|
||||
|
||||
|
||||
def _build_summary_row(df: pd.DataFrame, usage_key: str, file_count: int) -> dict:
|
||||
"""Build one summary row for a usage file."""
|
||||
if "file_name" not in df.columns or "total_cost" not in df.columns:
|
||||
raise ValueError(
|
||||
f"Required columns missing in {usage_key}. Expected file_name and total_cost."
|
||||
)
|
||||
|
||||
usage_df = df.copy()
|
||||
usage_df["file_name"] = usage_df["file_name"].astype(str)
|
||||
usage_df["total_cost"] = pd.to_numeric(
|
||||
usage_df["total_cost"], errors="coerce"
|
||||
).fillna(0.0)
|
||||
|
||||
is_cache = usage_df["file_name"].str.contains("cache_warming_", na=False)
|
||||
caching_cost = usage_df.loc[is_cache, "total_cost"].sum()
|
||||
|
||||
non_cache_df = usage_df.loc[~is_cache].copy()
|
||||
if non_cache_df.empty:
|
||||
adjusted_total_cost = 0.0
|
||||
else:
|
||||
by_file = non_cache_df.groupby("file_name", dropna=False)["total_cost"].sum()
|
||||
outlier_file = by_file.idxmax()
|
||||
adjusted_total_cost = non_cache_df.loc[
|
||||
non_cache_df["file_name"] != outlier_file, "total_cost"
|
||||
].sum()
|
||||
|
||||
return {
|
||||
"Batch": os.path.basename(usage_key),
|
||||
"caching_cost": round(float(caching_cost), 6),
|
||||
"total_cost": round(float(adjusted_total_cost), 6),
|
||||
"file_count": int(file_count),
|
||||
"cost_per_contract": (
|
||||
round(float(adjusted_total_cost) / float(file_count), 6)
|
||||
if file_count
|
||||
else 0.0
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _build_detail_row(df: pd.DataFrame, usage_key: str) -> dict:
|
||||
"""Build one detail row with one column per file_name containing summed total_cost."""
|
||||
if "file_name" not in df.columns or "total_cost" not in df.columns:
|
||||
raise ValueError(
|
||||
f"Required columns missing in {usage_key}. Expected file_name and total_cost."
|
||||
)
|
||||
|
||||
usage_df = df.copy()
|
||||
usage_df["file_name"] = usage_df["file_name"].astype(str)
|
||||
usage_df["total_cost"] = pd.to_numeric(
|
||||
usage_df["total_cost"], errors="coerce"
|
||||
).fillna(0.0)
|
||||
|
||||
by_file = usage_df.groupby("file_name", dropna=False)["total_cost"].sum()
|
||||
row = {"Batch": os.path.basename(usage_key)}
|
||||
for file_name, cost in by_file.items():
|
||||
row[file_name] = round(float(cost), 6)
|
||||
|
||||
return row
|
||||
|
||||
|
||||
def run_historical_cost_analysis(
|
||||
bucket: str = "doczy-output",
|
||||
output_path: str = "historical_cost_summary.xlsx",
|
||||
) -> pd.DataFrame:
|
||||
"""Generate historical cost analysis workbook across inv-test batches."""
|
||||
rows = []
|
||||
detail_rows = []
|
||||
batch_folders = io_utils.extract_matching_top_level_s3_prefixes(
|
||||
match_substring="inv-test-", bucket=bucket
|
||||
)
|
||||
logging.info(
|
||||
f"Found {len(batch_folders)} top-level batches containing 'inv-test-'."
|
||||
)
|
||||
|
||||
for batch_folder in batch_folders:
|
||||
latest_run = io_utils.get_latest_run_folder(
|
||||
batch_folder=batch_folder, bucket=bucket
|
||||
)
|
||||
if not latest_run:
|
||||
logging.info(f"Skipping {batch_folder}: no run_ folders found.")
|
||||
continue
|
||||
|
||||
usage_key = io_utils.find_usage_key(
|
||||
batch_folder=batch_folder, run_folder=latest_run, bucket=bucket
|
||||
)
|
||||
if not usage_key:
|
||||
logging.info(
|
||||
f"Skipping {batch_folder}/{latest_run}: no -USAGE.csv in tracking/."
|
||||
)
|
||||
continue
|
||||
|
||||
usage_s3_path = f"s3://{bucket}/{usage_key}"
|
||||
usage_df = io_utils.read_s3_csv(usage_s3_path)
|
||||
if usage_df is None:
|
||||
logging.warning(f"Skipping {usage_s3_path}: unable to read CSV.")
|
||||
continue
|
||||
|
||||
file_count = io_utils.count_s3_files_under_prefix(
|
||||
prefix=f"{batch_folder}/{latest_run}/individual/",
|
||||
bucket=bucket,
|
||||
include_folders=False,
|
||||
)
|
||||
rows.append(_build_summary_row(usage_df, usage_key, file_count))
|
||||
detail_rows.append(_build_detail_row(usage_df, usage_key))
|
||||
|
||||
summary_df = pd.DataFrame(
|
||||
rows,
|
||||
columns=[
|
||||
"Batch",
|
||||
"caching_cost",
|
||||
"total_cost",
|
||||
"file_count",
|
||||
"cost_per_contract",
|
||||
],
|
||||
)
|
||||
if summary_df.empty:
|
||||
summary_df = pd.DataFrame(
|
||||
columns=["caching_cost", "total_cost", "file_count", "cost_per_contract"]
|
||||
)
|
||||
summary_df.index.name = "Batch"
|
||||
else:
|
||||
summary_df = summary_df.set_index("Batch")
|
||||
summary_df.index.name = "Batch"
|
||||
|
||||
detail_df = pd.DataFrame(detail_rows)
|
||||
if detail_df.empty:
|
||||
detail_df = pd.DataFrame()
|
||||
detail_df.index.name = "Batch"
|
||||
else:
|
||||
detail_df = detail_df.fillna(0.0).set_index("Batch")
|
||||
detail_df.index.name = "Batch"
|
||||
|
||||
if not output_path.lower().endswith(".xlsx"):
|
||||
output_path = f"{output_path}.xlsx"
|
||||
|
||||
with pd.ExcelWriter(output_path) as writer:
|
||||
summary_df.to_excel(writer, sheet_name="Summary", index=True)
|
||||
detail_df.to_excel(writer, sheet_name="Detail", index=True)
|
||||
|
||||
logging.info(
|
||||
f"Wrote workbook with {len(summary_df)} summary rows and {len(detail_df)} detail rows to {output_path}"
|
||||
)
|
||||
return summary_df
|
||||
|
||||
|
||||
def _parse_args():
|
||||
parser = argparse.ArgumentParser(description="Historical usage cost analysis")
|
||||
parser.add_argument(
|
||||
"--bucket",
|
||||
default="doczy-output",
|
||||
help="S3 bucket containing inv-test batches (default: doczy-output)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default="historical_cost_summary.xlsx",
|
||||
help="Local Excel output path (default: historical_cost_summary.xlsx)",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
|
||||
args = _parse_args()
|
||||
run_historical_cost_analysis(bucket=args.bucket, output_path=args.output)
|
||||
@@ -1,162 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Manual Test Bed\n",
|
||||
"Runs contracts through from `input_data`\n",
|
||||
"\n",
|
||||
"To make this work: in `config.py`, change READ_MODE and RUN_MODE to 'local'.\n",
|
||||
"Create an .env in the base directory of your project according to these instructions:\n",
|
||||
"https://aarete.atlassian.net/wiki/spaces/DoczyAI/pages/1283751954/Using+.env+for+AWS+keys\n",
|
||||
"\n",
|
||||
"Feel free to comment out parts of `file_processing.py` that do not apply to the fields you are optimizing. For instance, if you're optimizing a smart chunked field, comment out the `one_to_n_fields` block and feed just `one_to_n_results` into `postprocess.postprocess`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"#### BASE IMPORTS ####\n",
|
||||
"\n",
|
||||
"import logging\n",
|
||||
"import sys\n",
|
||||
"import os\n",
|
||||
"import pandas as pd"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"#### LOGGING ####\n",
|
||||
"\n",
|
||||
"os.chdir(\"../../\") # go to FieldExtraction folder\n",
|
||||
"\n",
|
||||
"import src.config as config\n",
|
||||
"\n",
|
||||
"# For this testing notebook, we want per-file logging\n",
|
||||
"config.ENABLE_PER_FILE_LOGGING = True\n",
|
||||
"config.PER_FILE_LOG_LEVEL = \"DEBUG\"\n",
|
||||
"\n",
|
||||
"import src.utils.logging_utils as logging_utils\n",
|
||||
"\n",
|
||||
"# Clear any existing handlers first\n",
|
||||
"logging.getLogger().handlers.clear()\n",
|
||||
"\n",
|
||||
"# Set up per-file logging (this includes console output)\n",
|
||||
"# logging_utils.setup_per_file_logging()\n",
|
||||
"per_file_handler = logging_utils.PerFileHandler()\n",
|
||||
"per_file_handler.setLevel(logging.DEBUG)\n",
|
||||
"logging.getLogger().addHandler(per_file_handler)\n",
|
||||
"\n",
|
||||
"# Add a single console handler for notebook monitoring\n",
|
||||
"console_handler = logging.StreamHandler(sys.stdout)\n",
|
||||
"console_handler.setLevel(logging.INFO) # Only show INFO and above in notebook\n",
|
||||
"console_formatter = logging.Formatter(\"%(asctime)s - %(levelname)s - %(message)s\")\n",
|
||||
"console_handler.setFormatter(console_formatter)\n",
|
||||
"logging.getLogger().addHandler(console_handler)\n",
|
||||
"\n",
|
||||
"# Set level for the root logger\n",
|
||||
"logging.getLogger().setLevel(logging.DEBUG)\n",
|
||||
"\n",
|
||||
"# Suppress noisy third-party loggers\n",
|
||||
"for logger_name in [\n",
|
||||
" \"botocore\",\n",
|
||||
" \"boto3\",\n",
|
||||
" \"faiss\",\n",
|
||||
" \"urllib3\",\n",
|
||||
" \"sentence_transformers\",\n",
|
||||
" \"s3transfer\",\n",
|
||||
"]:\n",
|
||||
" logging.getLogger(logger_name).setLevel(logging.WARNING)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"#### TEST CONFIG ####\n",
|
||||
"\n",
|
||||
"TEST_DATA_INPUT_DIR = \"scripts/investment_field_testing/input_data\"\n",
|
||||
"\n",
|
||||
"# Also change config.RUN_MODE and config.READ_MODE to 'local'\n",
|
||||
"test_params = {}\n",
|
||||
"test_params[\"local_input_dir\"] = TEST_DATA_INPUT_DIR\n",
|
||||
"test_params[\"max_workers\"] = 1\n",
|
||||
"test_params[\"input_files\"] = [ # you can specify specific files to run here\n",
|
||||
" \"Akron General Health System_Eleventh Amendment_20171001.txt\", # replace with your test file(s)\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"# Specify state(s) for testing, if needed\n",
|
||||
"# Specify them here instead of in `test_params` bc `config` is initialized\n",
|
||||
"# before `test_params` is read in main.py, which means the state-level constants\n",
|
||||
"# won't be set correctly if we put it in `test_params`.\n",
|
||||
"config.STATE = [\"OH\"] # for single states\n",
|
||||
"# config.STATE = [\"OH\", \"CA\"] # for multi-state testing"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": "#### MAIN IMPORTS ####\n\nfrom src.pipelines.saas.main import main as investment_main"
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"#### TEST EXECUTION ####\n",
|
||||
"\n",
|
||||
"df_out, df_error = investment_main(testing=True, test_params=test_params)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"df_out"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "doczy-field-extraction-py3.12 (3.12.7)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.12.7"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from io import StringIO
|
||||
import json
|
||||
@@ -17,6 +18,8 @@ from typing import Optional
|
||||
# Thread-safe lock for JSON file operations
|
||||
json_write_lock = Lock()
|
||||
|
||||
RUN_FOLDER_PATTERN = re.compile(r"^run_(\d{8})_(\d{2})-(\d{2})")
|
||||
|
||||
|
||||
def read_local(file_path) -> str | pd.DataFrame | None:
|
||||
"""Reads a local (.txt, .csv, or .xlsx) file and returns its contents.
|
||||
@@ -75,6 +78,133 @@ def list_s3_files(prefix=config.S3_PREFIX, bucket=config.S3_BUCKET): # io_utils
|
||||
return [item["Key"] for item in response.get("Contents", [])]
|
||||
|
||||
|
||||
def list_s3_files_paginated(
|
||||
prefix=config.S3_PREFIX, bucket=config.S3_BUCKET
|
||||
) -> list[str]:
|
||||
"""List all S3 objects under a prefix using pagination."""
|
||||
all_keys = []
|
||||
continuation_token = None
|
||||
|
||||
while True:
|
||||
kwargs = {"Bucket": bucket, "Prefix": prefix}
|
||||
if continuation_token:
|
||||
kwargs["ContinuationToken"] = continuation_token
|
||||
|
||||
response = config.S3_CLIENT.list_objects_v2(**kwargs)
|
||||
all_keys.extend([item["Key"] for item in response.get("Contents", [])])
|
||||
|
||||
if response.get("IsTruncated"):
|
||||
continuation_token = response.get("NextContinuationToken")
|
||||
else:
|
||||
break
|
||||
|
||||
return all_keys
|
||||
|
||||
|
||||
def list_s3_child_prefixes(
|
||||
prefix=config.S3_PREFIX, bucket=config.S3_BUCKET
|
||||
) -> list[str]:
|
||||
"""List immediate child prefixes under an S3 prefix using Delimiter='/' pagination."""
|
||||
child_prefixes = []
|
||||
continuation_token = None
|
||||
|
||||
while True:
|
||||
kwargs = {"Bucket": bucket, "Prefix": prefix, "Delimiter": "/"}
|
||||
if continuation_token:
|
||||
kwargs["ContinuationToken"] = continuation_token
|
||||
|
||||
response = config.S3_CLIENT.list_objects_v2(**kwargs)
|
||||
child_prefixes.extend([p["Prefix"] for p in response.get("CommonPrefixes", [])])
|
||||
|
||||
if response.get("IsTruncated"):
|
||||
continuation_token = response.get("NextContinuationToken")
|
||||
else:
|
||||
break
|
||||
|
||||
return child_prefixes
|
||||
|
||||
|
||||
def extract_matching_top_level_s3_prefixes(
|
||||
match_substring: str, bucket=config.S3_BUCKET
|
||||
) -> list[str]:
|
||||
"""Return top-level S3 prefixes that contain the provided substring."""
|
||||
top_level_prefixes = list_s3_child_prefixes(prefix="", bucket=bucket)
|
||||
matching_prefixes = {
|
||||
prefix.rstrip("/")
|
||||
for prefix in top_level_prefixes
|
||||
if match_substring in prefix.rstrip("/")
|
||||
}
|
||||
return sorted(matching_prefixes)
|
||||
|
||||
|
||||
def get_latest_run_folder(batch_folder: str, bucket=config.S3_BUCKET) -> str | None:
|
||||
"""Find latest run_YYYYMMDD_HH-MM... folder under a batch folder."""
|
||||
prefix = f"{batch_folder}/"
|
||||
run_prefixes = list_s3_child_prefixes(prefix=prefix, bucket=bucket)
|
||||
run_candidates = {
|
||||
rp[len(prefix) :].rstrip("/")
|
||||
for rp in run_prefixes
|
||||
if rp[len(prefix) :].startswith("run_")
|
||||
}
|
||||
|
||||
if not run_candidates:
|
||||
return None
|
||||
|
||||
def sort_key(run_folder: str):
|
||||
match = RUN_FOLDER_PATTERN.match(run_folder)
|
||||
if not match:
|
||||
return ("", "", "", run_folder)
|
||||
date_part, hour_part, minute_part = match.groups()
|
||||
return (date_part, hour_part, minute_part, run_folder)
|
||||
|
||||
return sorted(run_candidates, key=sort_key, reverse=True)[0]
|
||||
|
||||
|
||||
def find_usage_key(
|
||||
batch_folder: str, run_folder: str, bucket=config.S3_BUCKET
|
||||
) -> str | None:
|
||||
"""Find the latest -USAGE.csv key in tracking/ for a batch run."""
|
||||
tracking_prefix = f"{batch_folder}/{run_folder}/tracking/"
|
||||
tracking_keys = list_s3_files_paginated(prefix=tracking_prefix, bucket=bucket)
|
||||
usage_keys = sorted([key for key in tracking_keys if key.endswith("-USAGE.csv")])
|
||||
if not usage_keys:
|
||||
return None
|
||||
return usage_keys[-1]
|
||||
|
||||
|
||||
def count_s3_files_under_prefix(
|
||||
prefix: str, bucket=config.S3_BUCKET, include_folders: bool = False
|
||||
) -> int:
|
||||
"""Count S3 objects under a prefix, optionally excluding folder placeholders."""
|
||||
count = 0
|
||||
continuation_token = None
|
||||
|
||||
while True:
|
||||
kwargs = {"Bucket": bucket, "Prefix": prefix}
|
||||
if continuation_token:
|
||||
kwargs["ContinuationToken"] = continuation_token
|
||||
|
||||
response = config.S3_CLIENT.list_objects_v2(**kwargs)
|
||||
|
||||
if include_folders:
|
||||
count += len(response.get("Contents", []))
|
||||
else:
|
||||
count += len(
|
||||
[
|
||||
item["Key"]
|
||||
for item in response.get("Contents", [])
|
||||
if not item["Key"].endswith("/")
|
||||
]
|
||||
)
|
||||
|
||||
if response.get("IsTruncated"):
|
||||
continuation_token = response.get("NextContinuationToken")
|
||||
else:
|
||||
break
|
||||
|
||||
return count
|
||||
|
||||
|
||||
def read_s3_pdf(pdf_path):
|
||||
"""
|
||||
Downloads a PDF file from an S3 bucket and returns a temporary file path.
|
||||
|
||||
Reference in New Issue
Block a user