Merged in feature/new-model-testing (pull request #741)
Feature/new model testing * rename and set up * Setup * Get dynamic primary crosswalked answers * Generate stats for one file * Update dynamic primary evaluation * Multithreading * Set up reimbursement primary evaluation * UPdate output report generation * Update TN calculation * update poetry lock * reorder reimb primary * evaluate 3 models * Configure 4.5 * switch to debug logging * update type checking * Fix mypy * Resolve mypy * Merged main into feature/new-model-testing Approved-by: Karan Desai
This commit is contained in:
Katon Minhas
Generated
+175
-459
File diff suppressed because it is too large
Load Diff
@@ -30,6 +30,7 @@ word2number = "^1.1"
|
||||
orjson = "^3.10.16"
|
||||
pymupdf = "^1.25.5"
|
||||
pillow = "^11.2.1"
|
||||
xlsxwriter = "^3.2.9"
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
black = "^24.10.0"
|
||||
|
||||
@@ -230,11 +230,15 @@ MODEL_ID_CLAUDE3_HAIKU = "anthropic.claude-3-haiku-20240307-v1:0"
|
||||
MODEL_ID_CLAUDE35_SONNET = "anthropic.claude-3-5-sonnet-20240620-v1:0"
|
||||
MODEL_ID_CLAUDE2 = "anthropic.claude-instant-v1"
|
||||
MODEL_ID_CLAUDE35_SONNET_V2 = "anthropic.claude-3-5-sonnet-20241022-v2:0"
|
||||
MODEL_ID_CLAUDE37_SONNET = "anthropic.claude-3-7-sonnet-20250219-v1:0"
|
||||
MODEL_ID_CLAUDE37_SONNET = "us.anthropic.claude-3-7-sonnet-20250219-v1:0"
|
||||
MODEL_ID_CLAUDE4_SONNET = "us.anthropic.claude-sonnet-4-20250514-v1:0"
|
||||
MODEL_ID_CLAUDE45_SONNET = "us.anthropic.claude-sonnet-4-5-20250929-v1:0"
|
||||
|
||||
# Model aliases for easy upgrades
|
||||
MODEL_ALIASES = {
|
||||
"sonnet_37" : MODEL_ID_CLAUDE37_SONNET,
|
||||
"sonnet_4" : MODEL_ID_CLAUDE4_SONNET,
|
||||
"sonnet_45": MODEL_ID_CLAUDE45_SONNET,
|
||||
"sonnet_latest": MODEL_ID_CLAUDE4_SONNET,
|
||||
"haiku_latest": MODEL_ID_CLAUDE3_HAIKU,
|
||||
"legacy_sonnet": MODEL_ID_CLAUDE35_SONNET,
|
||||
|
||||
@@ -39,7 +39,6 @@ def process_file(file_object, constants: Constants, run_timestamp):
|
||||
|
||||
# Set default values
|
||||
dynamic_one_to_one_fields = FieldSet()
|
||||
|
||||
process_one_to_n = config.FIELDS in ['all', 'one_to_n']
|
||||
process_one_to_one = config.FIELDS in ['all', 'one_to_one']
|
||||
|
||||
|
||||
@@ -100,7 +100,7 @@ def main(testing=False, test_params={}):
|
||||
|
||||
# Warm prompt caches before parallel processing
|
||||
# This ensures the cache is registered before multiple threads try to use it
|
||||
logging.info("Warming prompt caches before parallel processing...")
|
||||
logging.debug("Warming prompt caches before parallel processing...")
|
||||
try:
|
||||
cacheable_instructions = prompt_templates.get_cacheable_instructions(constants)
|
||||
for cache_key, instruction in cacheable_instructions.items():
|
||||
@@ -109,7 +109,7 @@ def main(testing=False, test_params={}):
|
||||
cache_key=cache_key,
|
||||
model_id="sonnet_latest"
|
||||
)
|
||||
logging.info(f"Successfully warmed {len(cacheable_instructions)} prompt caches")
|
||||
logging.debug(f"Successfully warmed {len(cacheable_instructions)} prompt caches")
|
||||
except Exception as e:
|
||||
logging.warning(f"Error warming caches (will proceed anyway): {str(e)}")
|
||||
|
||||
|
||||
@@ -186,7 +186,7 @@ def date_postprocess(df, field_json_path):
|
||||
)
|
||||
else:
|
||||
missing = [col for col in required_cols if col not in df.columns]
|
||||
logging.info(f"Skipping AARETE_DERIVED_TERMINATION_DT creation. Missing: {missing}")
|
||||
logging.debug(f"Skipping AARETE_DERIVED_TERMINATION_DT creation. Missing: {missing}")
|
||||
df["AARETE_DERIVED_TERMINATION_DT"] = pd.NA
|
||||
|
||||
return df
|
||||
|
||||
@@ -90,7 +90,6 @@ class Field:
|
||||
- Subsequent calls with same file_path: Uses cached data (no file I/O)
|
||||
- Thread-safe: Multiple threads can safely call this simultaneously
|
||||
|
||||
|
||||
:param file_path: Path to the JSON file.
|
||||
:param field_name: The field name to look for in the JSON data.
|
||||
:return: A Field object if found, otherwise raises a ValueError.
|
||||
@@ -210,10 +209,15 @@ class FieldSet:
|
||||
FieldSet(file_path="investment_prompts.json", relationship="one_to_one")
|
||||
FieldSet(file_path="investment_prompts.json", relationship="one_to_n", field_type="exhibit_level")
|
||||
FieldSet(file_path="investment_prompts.json", field_type="reimbursement_level")
|
||||
|
||||
# Now supports lists for any attribute:
|
||||
FieldSet(file_path="investment_prompts.json", field_name=["LOB", "PROGRAM"])
|
||||
FieldSet(file_path="investment_prompts.json", field_type=["exhibit_level", "reimbursement_level"])
|
||||
|
||||
:param file_path: Path to a JSON file to load fields from (default: None).
|
||||
:param fields: List of Field objects to initialize the FieldSet instance with (default: None).
|
||||
:param filters: Key-value pairs for filtering fields based on their attributes.
|
||||
Each value can be a single value or a list of values.
|
||||
"""
|
||||
self.fields = fields if fields is not None else []
|
||||
if file_path:
|
||||
@@ -228,6 +232,11 @@ class FieldSet:
|
||||
- Each FieldSet can still apply different filters to the same cached data
|
||||
- Thread-safe: Multiple threads can safely call this simultaneously
|
||||
|
||||
FILTERING BEHAVIOR:
|
||||
- Single values: Field attribute must match exactly
|
||||
- Lists: Field attribute must match any value in the list
|
||||
- True: Field attribute must exist (not None)
|
||||
|
||||
PERFORMANCE IMPACT:
|
||||
- Before caching: 1000+ file reads for the same JSON → "too many open files"
|
||||
- After caching: 1 file read per unique path → fast and efficient
|
||||
@@ -243,21 +252,33 @@ class FieldSet:
|
||||
# CACHE HIT: Use cached data
|
||||
data = _field_data_cache[file_path]
|
||||
|
||||
# Apply filters to cached data - this allows different FieldSets
|
||||
# to get different subsets from the same cached JSON
|
||||
self.fields = [
|
||||
Field(field_dict)
|
||||
for field_dict in data
|
||||
if all(
|
||||
(
|
||||
value is True and field_dict.get(attr) is not None
|
||||
) # Check for "not None" values
|
||||
or (
|
||||
value is not True and field_dict.get(attr) == value
|
||||
) # Check for exact match
|
||||
for attr, value in filters.items()
|
||||
)
|
||||
]
|
||||
# Apply filters to cached data with enhanced list support
|
||||
self.fields = []
|
||||
for field_dict in data:
|
||||
# Check if this field_dict matches all filters
|
||||
matches_all_filters = True
|
||||
|
||||
for attr, filter_value in filters.items():
|
||||
field_value = field_dict.get(attr)
|
||||
|
||||
# Handle special case for True filter (check if attribute exists)
|
||||
if filter_value is True:
|
||||
if field_value is None:
|
||||
matches_all_filters = False
|
||||
break
|
||||
# Handle list-based filters (match any value in the list)
|
||||
elif isinstance(filter_value, list):
|
||||
if field_value not in filter_value:
|
||||
matches_all_filters = False
|
||||
break
|
||||
# Handle single value filters (exact match)
|
||||
elif field_value != filter_value:
|
||||
matches_all_filters = False
|
||||
break
|
||||
|
||||
# If all filters matched, add this field
|
||||
if matches_all_filters:
|
||||
self.fields.append(Field(field_dict))
|
||||
|
||||
def add_field(self, field):
|
||||
"""Adds a new field manually to the FieldSet instance.
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
import pandas as pd
|
||||
import concurrent.futures
|
||||
import threading
|
||||
from src.utils import io_utils, llm_utils
|
||||
from src.investment import preprocess
|
||||
from constants.constants import Constants
|
||||
from src.testbed import model_evaluation_utils
|
||||
from src import config
|
||||
from src.prompts.fieldset import FieldSet
|
||||
|
||||
prompt = "This is a test prompt. Write 'test', nothing else."
|
||||
|
||||
def process_file(filename, contract_text, model_id_dict, testbed_df, dynamic_primary_fields, reimbursement_primary_fields, constants):
|
||||
"""Process a single file and return its statistics."""
|
||||
print(f"Processing: {filename}")
|
||||
|
||||
testbed_df = testbed_df[testbed_df["FILE_NAME"] == filename]
|
||||
|
||||
# Preprocess the contract
|
||||
contract_text = preprocess.clean_text(contract_text)
|
||||
text_dict, top_sheet_dict = preprocess.split_text(contract_text)
|
||||
text_dict, header, footer = preprocess.find_headers_and_footers(text_dict)
|
||||
text_dict = preprocess.clean_tables(
|
||||
text_dict, constants.EXHIBIT_HEADER_MARKERS, filename
|
||||
)
|
||||
exhibit_dict, all_exhibit_headers = preprocess.one_to_n_exhibit_chunking(
|
||||
text_dict, constants.EXHIBIT_HEADER_MARKERS, filename
|
||||
)
|
||||
|
||||
# Evaluate Dynamic Primary
|
||||
dynamic_primary_stats_dicts = model_evaluation_utils.evaluate_dynamic_primary(
|
||||
exhibit_dict,
|
||||
model_id_dict,
|
||||
testbed_df,
|
||||
dynamic_primary_fields,
|
||||
constants,
|
||||
filename
|
||||
) # list of dicts
|
||||
|
||||
# Evaluate Reimbursement Primary
|
||||
reimbursement_primary_stats_dict = model_evaluation_utils.evaluate_reimbursement_primary(
|
||||
exhibit_dict,
|
||||
model_id_dict,
|
||||
testbed_df,
|
||||
reimbursement_primary_fields,
|
||||
constants,
|
||||
filename
|
||||
) # dict
|
||||
|
||||
return dynamic_primary_stats_dicts, reimbursement_primary_stats_dict
|
||||
|
||||
|
||||
# Initialize variables
|
||||
dynamic_primary_fields = FieldSet(
|
||||
config.FIELD_JSON_PATH, field_name=["LOB", "PROGRAM", "NETWORK"]
|
||||
)
|
||||
reimbursement_primary_fields = FieldSet(
|
||||
config.FIELD_JSON_PATH, field_type="reimbursement_level"
|
||||
)
|
||||
|
||||
model_id_dict = {
|
||||
"SONNET_37" : "sonnet_37",
|
||||
"SONNET_4" : "sonnet_4",
|
||||
"SONNET_45" : "sonnet_45"
|
||||
}
|
||||
input_dict = io_utils.read_input()
|
||||
testbed_df = pd.read_excel("Doczy-Testbed.xlsx")
|
||||
testbed_files = list(testbed_df["FILE_NAME"])
|
||||
constants = Constants()
|
||||
|
||||
# Initialize dynamic primary stats for each model
|
||||
dynamic_primary = [{"Model": model_name, "TP": 0, "TN": 0, "FP": 0, "FN": 0} for model_name in model_id_dict.keys()]
|
||||
|
||||
# Initialize list to collect all reimbursement primary stats
|
||||
reimbursement_primary_stats_list = []
|
||||
|
||||
# Lock for thread-safe updates to stats
|
||||
stats_lock = threading.Lock()
|
||||
|
||||
# Function to update global stats with results from one file
|
||||
def update_stats(results):
|
||||
dynamic_primary_stats_dicts, reimbursement_primary_stats_dict = results
|
||||
|
||||
with stats_lock:
|
||||
# Update dynamic primary stats
|
||||
for file_stats_dict in dynamic_primary_stats_dicts:
|
||||
model_name = file_stats_dict["Model"]
|
||||
for model_stats in dynamic_primary:
|
||||
if model_stats["Model"] == model_name:
|
||||
for metric in ["TP", "TN", "FP", "FN"]:
|
||||
model_stats[metric] += file_stats_dict.get(metric, 0)
|
||||
break
|
||||
|
||||
# Add reimbursement primary stats to the list
|
||||
if reimbursement_primary_stats_dict:
|
||||
reimbursement_primary_stats_list.append(reimbursement_primary_stats_dict)
|
||||
|
||||
# Process files in parallel
|
||||
filtered_input = {filename: text for filename, text in input_dict.items() if filename in testbed_files}
|
||||
|
||||
# Use ThreadPoolExecutor for parallel processing
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor:
|
||||
# Submit all file processing tasks
|
||||
future_to_file = {
|
||||
executor.submit(
|
||||
process_file,
|
||||
filename,
|
||||
contract_text,
|
||||
model_id_dict,
|
||||
testbed_df,
|
||||
dynamic_primary_fields,
|
||||
reimbursement_primary_fields,
|
||||
constants
|
||||
): filename for filename, contract_text in filtered_input.items()
|
||||
}
|
||||
|
||||
# Process results as they complete
|
||||
for future in concurrent.futures.as_completed(future_to_file):
|
||||
filename = future_to_file[future]
|
||||
try:
|
||||
results = future.result()
|
||||
update_stats(results)
|
||||
print(f"Completed processing: {filename}")
|
||||
except Exception as e:
|
||||
print(f"Error processing {filename}: {e}")
|
||||
|
||||
# Generate Excel report with two sheets
|
||||
def generate_excel_report(output_path="model_evaluation_results.xlsx"):
|
||||
# Create Excel writer
|
||||
with pd.ExcelWriter(output_path, engine='xlsxwriter') as writer:
|
||||
|
||||
# Sheet 1: Dynamic Primary Stats
|
||||
dynamic_primary_df = pd.DataFrame(dynamic_primary)
|
||||
|
||||
# Calculate additional metrics for each model
|
||||
for idx, row in dynamic_primary_df.iterrows():
|
||||
tp = row["TP"]
|
||||
tn = 0
|
||||
fp = row["FP"]
|
||||
fn = row["FN"]
|
||||
|
||||
# Add calculated metrics
|
||||
dynamic_primary_df.loc[idx, "Accuracy"] = (tp + tn) / (tp + tn + fp + fn) if (tp + tn + fp + fn) > 0 else 0
|
||||
dynamic_primary_df.loc[idx, "Precision"] = tp / (tp + fp) if (tp + fp) > 0 else 0
|
||||
dynamic_primary_df.loc[idx, "Recall"] = tp / (tp + fn) if (tp + fn) > 0 else 0
|
||||
precision = dynamic_primary_df.loc[idx, "Precision"]
|
||||
recall = dynamic_primary_df.loc[idx, "Recall"]
|
||||
dynamic_primary_df.loc[idx, "F1"] = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0
|
||||
|
||||
# Write to Excel
|
||||
dynamic_primary_df.to_excel(writer, sheet_name='Dynamic Primary Stats', index=False)
|
||||
|
||||
# Sheet 2: Reimbursement Primary Stats
|
||||
if reimbursement_primary_stats_list:
|
||||
# Combine all reimbursement stats into a DataFrame
|
||||
reimbursement_df = pd.DataFrame(reimbursement_primary_stats_list)
|
||||
|
||||
# Write individual file stats
|
||||
reimbursement_df.to_excel(writer, sheet_name='Reimbursement Stats', index=False)
|
||||
|
||||
# Get a reference to the worksheet to add summary statistics
|
||||
worksheet = writer.sheets['Reimbursement Stats']
|
||||
|
||||
# Add a summary section
|
||||
summary_row = len(reimbursement_df) + 3 # Leave a gap of 2 rows
|
||||
|
||||
# Add headers for the summary section
|
||||
worksheet.write(summary_row, 0, "Summary Statistics")
|
||||
worksheet.write(summary_row + 1, 0, "Model")
|
||||
worksheet.write(summary_row + 1, 1, "Average Row Count Diff")
|
||||
worksheet.write(summary_row + 1, 2, "Average Row Count Pct Diff")
|
||||
|
||||
# Add summary data
|
||||
row_offset = summary_row + 2
|
||||
|
||||
# Add model stats - only proceed if we have testbed data
|
||||
if 'Testbed' in reimbursement_df.columns:
|
||||
for model_name in model_id_dict.keys():
|
||||
if model_name in reimbursement_df.columns:
|
||||
# Calculate the absolute difference between model and testbed row counts
|
||||
reimbursement_df['Abs_Diff'] = abs(reimbursement_df[model_name] - reimbursement_df['Testbed'])
|
||||
|
||||
# Calculate the percentage difference
|
||||
reimbursement_df['Pct_Diff'] = reimbursement_df['Abs_Diff'] / reimbursement_df['Testbed'] * 100
|
||||
reimbursement_df['Pct_Diff'] = reimbursement_df['Pct_Diff'].fillna(0) # Handle division by zero
|
||||
|
||||
# Calculate averages
|
||||
avg_abs_diff = reimbursement_df['Abs_Diff'].mean()
|
||||
avg_pct_diff = reimbursement_df['Pct_Diff'].mean()
|
||||
|
||||
# Write to summary
|
||||
worksheet.write(row_offset, 0, model_name)
|
||||
worksheet.write(row_offset, 1, avg_abs_diff)
|
||||
worksheet.write(row_offset, 2, f"{avg_pct_diff:.2f}%")
|
||||
|
||||
row_offset += 1
|
||||
|
||||
# Remove the temporary columns from the DataFrame before writing
|
||||
if 'Abs_Diff' in reimbursement_df.columns:
|
||||
reimbursement_df = reimbursement_df.drop(['Abs_Diff', 'Pct_Diff'], axis=1)
|
||||
# We need to write the dataframe again since we modified it
|
||||
reimbursement_df.to_excel(writer, sheet_name='Reimbursement Stats', index=False)
|
||||
|
||||
# Call the report generation function after all processing is complete
|
||||
generate_excel_report()
|
||||
@@ -0,0 +1,183 @@
|
||||
|
||||
import pandas as pd
|
||||
from src.prompts.fieldset import FieldSet
|
||||
from src.utils import llm_utils, string_utils
|
||||
from src.prompts import prompt_templates
|
||||
from constants.constants import Constants
|
||||
from src.investment import aarete_derived, one_to_n_funcs
|
||||
from src import config
|
||||
|
||||
|
||||
def prompt_dynamic_primary(exhibit_dict: dict[str, str], model_id_dict: dict[str, str], dynamic_primary_fields: FieldSet, constants: Constants, filename: str):
|
||||
model_answer_dicts = {}
|
||||
for model_name, model_id in model_id_dict.items():
|
||||
answer_dicts = []
|
||||
for exhibit_page, exhibit_text in exhibit_dict.items():
|
||||
answer_dict = {}
|
||||
for field in dynamic_primary_fields.fields:
|
||||
prompt = prompt_templates.DYNAMIC_PRIMARY_TEXT(exhibit_text, field.field_name, field.get_prompt(constants))
|
||||
llm_answer_raw = llm_utils.invoke_claude(
|
||||
prompt=prompt,
|
||||
model_id=model_id,
|
||||
filename=filename
|
||||
)
|
||||
llm_answer_final = string_utils.extract_text_from_delimiters(
|
||||
llm_answer_raw, string_utils.Delimiter.PIPE
|
||||
)
|
||||
answer_dict[field.field_name] = llm_answer_final
|
||||
answer_dicts.append(answer_dict)
|
||||
|
||||
answer_dicts = aarete_derived.get_crosswalk_fields(answer_dicts, constants)
|
||||
model_answer_dicts[model_name] = answer_dicts
|
||||
return model_answer_dicts
|
||||
|
||||
def consolidate_dynamic_primary(model_answer_dicts: dict):
|
||||
consolidated_answer_dicts = {}
|
||||
for model_name, answer_dicts in model_answer_dicts.items():
|
||||
unique_values = {}
|
||||
for answer_dict in answer_dicts:
|
||||
for field_name, value in answer_dict.items():
|
||||
if "AARETE_DERIVED" not in field_name:
|
||||
continue
|
||||
if field_name not in unique_values:
|
||||
unique_values[field_name] = set()
|
||||
if not string_utils.is_empty(value):
|
||||
# Split pipe-separated values and add each individually
|
||||
if isinstance(value, str) and "|" in value:
|
||||
for split_value in value.split("|"):
|
||||
split_value = split_value.strip() # Remove any extra whitespace
|
||||
if not string_utils.is_empty(split_value):
|
||||
unique_values[field_name].add(split_value)
|
||||
else:
|
||||
unique_values[field_name].add(value)
|
||||
consolidated_answer_dicts[model_name] = {k: list(v) for k, v in unique_values.items()}
|
||||
|
||||
return consolidated_answer_dicts
|
||||
|
||||
def generate_dynamic_primary_stats(consolidated_answer_dicts: dict[str, dict], testbed_df: pd.DataFrame, filename: str):
|
||||
"""
|
||||
Generate confusion matrix statistics for dynamic primary field extraction.
|
||||
|
||||
Args:
|
||||
consolidated_answer_dicts: Dictionary with model names as keys and dictionaries of field values as values
|
||||
testbed_df: DataFrame containing the ground truth data
|
||||
filename: Name of the file being evaluated
|
||||
|
||||
Returns:
|
||||
List of dictionaries with confusion matrix statistics for each model
|
||||
"""
|
||||
|
||||
# Extract all unique ground truth values from testbed_df
|
||||
ground_truth_values = set()
|
||||
aarete_cols = ["AARETE_DERIVED_LOB", "AARETE_DERIVED_PROGRAM", "AARETE_DERIVED_NETWORK"]
|
||||
|
||||
# Only include columns that actually exist in the DataFrame
|
||||
aarete_cols = [col for col in aarete_cols if col in testbed_df.columns]
|
||||
|
||||
# Collect unique values from each column
|
||||
for col in aarete_cols:
|
||||
# Handle potential pipe-separated values
|
||||
for value in testbed_df[col].dropna():
|
||||
if isinstance(value, str):
|
||||
if "|" in value:
|
||||
for split_value in value.split("|"):
|
||||
split_value = split_value.strip()
|
||||
if not string_utils.is_empty(split_value):
|
||||
ground_truth_values.add(split_value)
|
||||
else:
|
||||
if not string_utils.is_empty(value):
|
||||
ground_truth_values.add(value)
|
||||
|
||||
stats_dicts = []
|
||||
for model_name, field_dict in consolidated_answer_dicts.items():
|
||||
# Collect all unique values predicted by this model
|
||||
model_values = set()
|
||||
for field_name, values_list in field_dict.items():
|
||||
for value in values_list:
|
||||
if not string_utils.is_empty(value):
|
||||
model_values.add(value)
|
||||
TP = len([v for v in model_values if v in ground_truth_values])
|
||||
FP = len([v for v in model_values if v not in ground_truth_values])
|
||||
FN = len([v for v in ground_truth_values if v not in model_values])
|
||||
stats_dicts.append({"Model" : model_name, "TP" : TP, "FP" : FP, "FN" : FN})
|
||||
|
||||
return stats_dicts
|
||||
|
||||
def evaluate_dynamic_primary(exhibit_dict: dict[str, str], model_id_dict: dict[str, str], testbed_df: pd.DataFrame, dynamic_primary_fields: FieldSet, constants: Constants, filename: str):
|
||||
|
||||
## Prompt for model_answer_dicts
|
||||
dynamic_answer_dicts = prompt_dynamic_primary(
|
||||
exhibit_dict,
|
||||
model_id_dict,
|
||||
dynamic_primary_fields,
|
||||
constants,
|
||||
filename
|
||||
)
|
||||
|
||||
# Consolidate model_answer_dicts
|
||||
consolidated_answer_dicts = consolidate_dynamic_primary(dynamic_answer_dicts)
|
||||
|
||||
# Generate stats
|
||||
stats_dicts = generate_dynamic_primary_stats(consolidated_answer_dicts, testbed_df, filename)
|
||||
|
||||
return stats_dicts
|
||||
|
||||
def evaluate_reimbursement_primary(exhibit_dict: dict[str, str], model_id_dict: dict[str, str], testbed_df: pd.DataFrame, reimbursement_primary_fields: FieldSet, constants: Constants, filename: str) -> dict[str, object]:
|
||||
# Create two separate dictionaries and merge them at the end
|
||||
metadata: dict[str, str] = {"FILE_NAME": filename}
|
||||
|
||||
# Create a numeric counts dictionary with explicit types
|
||||
counts: dict[str, int] = {}
|
||||
num_lesser = len(testbed_df[testbed_df['LESSER_OF_IND'] == 'Y'])
|
||||
counts["Testbed"] = int(num_lesser/2) + (len(testbed_df)-num_lesser)
|
||||
|
||||
# Initialize model counts
|
||||
for model_name in model_id_dict:
|
||||
counts[model_name] = 0
|
||||
|
||||
# Process each model
|
||||
for model_name, model_id in model_id_dict.items():
|
||||
seen_pairs = set()
|
||||
for exhibit_page, exhibit_text in exhibit_dict.items():
|
||||
field_prompts = reimbursement_primary_fields.print_prompt_dict(constants)
|
||||
instruction, prompt = prompt_templates.REIMBURSEMENT_PRIMARY(exhibit_text, field_prompts)
|
||||
llm_answer_raw = llm_utils.invoke_claude(
|
||||
prompt,
|
||||
model_id,
|
||||
filename,
|
||||
max_tokens=25000,
|
||||
cache=True,
|
||||
instruction=instruction,
|
||||
usage_label="REIMBURSEMENT_PRIMARY",
|
||||
)
|
||||
# Check for the special "no results" case
|
||||
if "NO_REIMBURSEMENT_TERMS_FOUND" in llm_answer_raw:
|
||||
continue
|
||||
|
||||
try:
|
||||
llm_answer_final = string_utils.universal_json_load(llm_answer_raw)
|
||||
|
||||
reimbursement_primary_answers = one_to_n_funcs.clean_reimbursement_primary(
|
||||
llm_answer_final,
|
||||
"",
|
||||
seen_pairs,
|
||||
exhibit_page,
|
||||
constants,
|
||||
filename
|
||||
)
|
||||
counts[model_name] += len(reimbursement_primary_answers)
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
# Merge the dictionaries before returning
|
||||
result: dict[str, object] = {}
|
||||
result.update(metadata)
|
||||
result.update(counts)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -194,6 +194,18 @@ def invoke_claude(
|
||||
)
|
||||
input_cost_per_1k, output_cost_per_1k = 0.003, 0.015
|
||||
cache_creation_cost_per_1k, cache_read_cost_per_1k = 0.00375, 0.0003
|
||||
elif resolved_model_id == config.MODEL_ID_CLAUDE45_SONNET:
|
||||
response = local_claude_3_and_up(
|
||||
prompt,
|
||||
resolved_model_id,
|
||||
filename,
|
||||
max_tokens,
|
||||
cache=cache,
|
||||
instruction=instruction,
|
||||
usage_label=usage_label,
|
||||
)
|
||||
input_cost_per_1k, output_cost_per_1k = 0.003, 0.015
|
||||
cache_creation_cost_per_1k, cache_read_cost_per_1k = 0.00375, 0.0003
|
||||
else:
|
||||
raise ValueError(f"Unsupported model_id: {model_id}")
|
||||
elif config.RUN_MODE == "ec2":
|
||||
@@ -247,6 +259,16 @@ def invoke_claude(
|
||||
instruction=instruction,
|
||||
usage_label=usage_label,
|
||||
)
|
||||
elif resolved_model_id == config.MODEL_ID_CLAUDE45_SONNET:
|
||||
response = ec2_claude_3_and_up(
|
||||
prompt,
|
||||
resolved_model_id,
|
||||
filename,
|
||||
max_tokens,
|
||||
cache=cache,
|
||||
instruction=instruction,
|
||||
usage_label=usage_label,
|
||||
)
|
||||
input_cost_per_1k, output_cost_per_1k = 0.003, 0.015
|
||||
cache_creation_cost_per_1k, cache_read_cost_per_1k = 0.00375, 0.0003
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user