Merged in bugfix/effective_date_fix (pull request #504)
DRAFT: Bugfix/effective date fix * Merged in bugfix/extra-fields (pull request #500) Do not run Exhibit level prompt if there are no exhibit level fields * Do not run Exhibit level prompt if there are no exhibit level fields * Update dynamic test * Remove test from one-to-n test module * Update test * Fix unit test Approved-by: Alex Galarce * Merged in bugfix/tin-npi-prompt-fixes (pull request #499) Bugfix/tin npi prompt fixes * Change delimiter for other provider information from comma to pipe * filter out invalid providers * Update extraction and formatting instructions for provider entities in TIN_NPI_TEMPLATE * Merged main into bugfix/tin-npi-small-bugfixes * Refactor deduplication logic to use string_utils for checking empty provider fields * Merge branch 'bugfix/tin-npi-small-bugfixes' of https://bitbucket.org/aarete/doczy.ai into bugfix/tin-npi-small-bugfixes * Update deduplication logic to exclude providers with 'UNKNOWN' TIN, NPI, or NAME * Merge remote-tracking branch 'origin/main' into bugfix/tin-npi-small-bugfixes * Fix deduplication logic to not drop subsequent names * Fix formatting of provider names in deduplication logic * add payer name filtering * reorder execution order to allow to find payer_name first * Add handling for previously unfound group providers in identify_group_provider function * Merge remote-trackin… * Merged in feature/update-testbed-script (pull request #501) Feature/update testbed script * Update preprocessing * Update preprocessing * remove quit * Update date for filename * Merged main into feature/update-testbed-script Approved-by: Alex Galarce * Merged in bugfix/smart-chunks (pull request #503) missing of text in chunks fixed * missing of text in chunks fixed Approved-by: Katon Minhas * Merged in feature/rework-reimb-tin-npi (pull request #502) Feature/rework reimb tin npi * use LLM for reimb_[tin/npi/name] * Merge remote-tracking branch 'origin/main' into feature/rework-reimb-tin-npi * Merge remote-tracking branch 'origin/main' into feature/rework-reimb-tin-npi * deduplicate tin/name/npi before adding to prompt * Pull out prompt and llm_response in methodology_breakout_single_row for easier debugging * Refactor reimbursement_tin_npi to handle multiple providers and update valid TIN, NPI, and NAME fields accordingly * Reorder REIMB_PROV_NAME field in investment_prompts.json * Update REIMB_PROV_NAME prompt to specify valid values for provider names * Remove valid_values constraint from prov_name to raise accuracy * Remove debugging print statements from reimbursement_tin_npi and related functions * Remove debugging print statements from run_provider_info_fields function * Merge remote-tracking branch 'origin/main' into feature/rework-reimb-tin-npi * Remove debugging print statem… * Merged in feature/preformat-single-quotes-input (pull request #506) Replace double quotes with single quotes in context strings for parsability * Replace double quotes with single quotes in context strings for parsability * Merged main into feature/preformat-single-quotes-input Approved-by: Katon Minhas * updated sig pages fnxn * Merge branch 'main' into bugfix/effective_date_fix * poetry add pymupdf * created vision funcs * Merged main into bugfix/effective_date_fix * fixed pipeline issue * updated string_utils_test * updated test values * updated extract signature page fxn * updated test values * updated test values * Merged main into bugfix/effective_date_fix * Prompt template in all-caps * remove debugging prints * Merged main into bugfix/effective_date_fix * Change vision to False by default * Change vision to False by default * fix tests * fix tests Approved-by: Katon Minhas
This commit is contained in:
VenkataKrishna Reddy Avula
committed by
Katon Minhas
Katon Minhas
parent
806bd9b3bb
commit
0d6bdf9539
Generated
+141
-390
File diff suppressed because it is too large
Load Diff
@@ -29,6 +29,8 @@ sentence-transformers = "^3.4.1"
|
||||
faiss-cpu = "^1.10.0"
|
||||
word2number = "^1.1"
|
||||
orjson = "^3.10.16"
|
||||
pymupdf = "^1.25.5"
|
||||
pillow = "^11.2.1"
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
black = "^24.10.0"
|
||||
|
||||
@@ -233,13 +233,13 @@ if WRITE_TO_S3:
|
||||
MODEL_ID_CLAUDE3_HAIKU = "anthropic.claude-3-haiku-20240307-v1:0"
|
||||
MODEL_ID_CLAUDE35_SONNET = "anthropic.claude-3-5-sonnet-20240620-v1:0"
|
||||
MODEL_ID_CLAUDE2 = "anthropic.claude-instant-v1"
|
||||
MODEL_ID_CLAUDE35_SONNET_V2 = "anthropic.claude-3-5-sonnet-20241022-v2:0"
|
||||
MODEL_ID_CLAUDE37_SONNET = "anthropic.claude-3-7-sonnet-20250219-v1:0"
|
||||
|
||||
######################################## CLAUDE AND PROMPT TRACKING ########################################
|
||||
|
||||
ENABLE_ANSWER_TRACKING = False
|
||||
ANSWER_TRACKING_DICT = {}
|
||||
|
||||
|
||||
######################################## POSTPROCESSING SETTINGS ########################################
|
||||
FUZZY_MATCH_THRESHOLD = 0.8
|
||||
FILE_NAME_COLUMN = 'Contract Name'
|
||||
@@ -303,3 +303,10 @@ dev_role_name = "doczy-dev-bedrock-lambda-role"
|
||||
|
||||
prod_account_id = "211125720533"
|
||||
prod_role_name = "doczy-prod-bedrock-lambda-role"
|
||||
|
||||
############## S3 BUCKETS FOR PDF FILES ##############
|
||||
DOCZY_PDF_FILES_BUCKET_NAME = "doczy-pdf-files" # sample bucket name added. we can either use this or create a new one
|
||||
DOCZY_PDF_FILES_BUCKET_S3_URL = f"https://{DOCZY_PDF_FILES_BUCKET_NAME}.s3.us-east-2.amazonaws.com/" # sample s3 url added. we can either use this or create a new one
|
||||
|
||||
############## VISION API SETTINGS ##############
|
||||
ENABLE_VISION = get_arg_value("enable_vision", "False") == "True" # False by default
|
||||
@@ -103,7 +103,7 @@ def run_one_to_one_prompts(filename, contract_text, text_dict, top_sheet_dict, d
|
||||
|
||||
################## RUN SMART CHUNKED PROMPTS ##################
|
||||
smart_chunked_answers_dict = one_to_one_funcs.run_smart_chunked_fields(
|
||||
one_to_one_fields, contract_text, filename
|
||||
one_to_one_fields, contract_text, filename,text_dict
|
||||
)
|
||||
|
||||
################## RUN FULL CONTEXT PROMPTS ##################
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from rapidfuzz import fuzz
|
||||
|
||||
import os
|
||||
import src.prompts.investment_prompts as prompts
|
||||
import src.utils.llm_utils as llm_utils
|
||||
import src.utils.string_utils as string_utils
|
||||
@@ -13,6 +13,9 @@ from src.investment.tin_npi_funcs import (clean_provider_info,
|
||||
get_provider_info,
|
||||
identify_group_provider)
|
||||
from src.prompts.investment_prompts import Field, FieldSet
|
||||
from src.investment import investment_postprocessing_funcs
|
||||
from src import postprocessing_funcs as generic_postprocessing_funcs
|
||||
from src.investment.vision_funcs import get_image_array_based_answer
|
||||
|
||||
MAX_CONTEXT_LENGTH = 100000
|
||||
|
||||
@@ -115,7 +118,8 @@ def filter_payer_from_providers(providers: list[dict], payer_name: str, threshol
|
||||
|
||||
def run_smart_chunked_fields(one_to_one_fields: FieldSet,
|
||||
contract_text: str,
|
||||
filename: str) -> dict[str, str]:
|
||||
filename: str,
|
||||
text_dict: dict)-> dict[str, str]:
|
||||
"""Process fields that require smart chunking and return the extracted answers.
|
||||
|
||||
Args:
|
||||
@@ -209,14 +213,54 @@ def run_smart_chunked_fields(one_to_one_fields: FieldSet,
|
||||
except Exception as e:
|
||||
print(f"Error processing field {field}: {type(e)}, {str(e)}")
|
||||
answers_dict[field.field_name] = f"Error: {str(e)}"
|
||||
if "AARETE_DERIVED_EFFECTIVE_DT" in answers_dict:
|
||||
|
||||
answers_dict = check_and_update_effective_date(answers_dict, text_dict, filename)
|
||||
|
||||
# even after running the vision api, if the effective date is still N/A, we need to run the prompt again with full context
|
||||
if "AARETE_DERIVED_EFFECTIVE_DT" in answers_dict and string_utils.is_empty(answers_dict.get("AARETE_DERIVED_EFFECTIVE_DT")):
|
||||
# if the effective date is N/A, we need to run the prompt again with full context
|
||||
if answers_dict['AARETE_DERIVED_EFFECTIVE_DT'] == "N/A":
|
||||
smart_chunked_fields.update_field_attr( field_name = 'AARETE_DERIVED_EFFECTIVE_DT',
|
||||
attr = 'field_type',
|
||||
value = "full_context")
|
||||
smart_chunked_fields.update_field_attr( field_name = 'AARETE_DERIVED_EFFECTIVE_DT',
|
||||
attr = 'field_type',
|
||||
value = "full_context")
|
||||
return answers_dict
|
||||
|
||||
def check_and_update_effective_date(answers_dict, text_dict, filename):
|
||||
"""
|
||||
Checks and updates the effective date in the answers dictionary. If the effective date is empty,
|
||||
it attempts to extract the effective date using vision-based LLM or full context processing.
|
||||
|
||||
Args:
|
||||
answers_dict (dict): Dictionary containing extracted answers.
|
||||
text_dict (dict): Dictionary containing text data, typically organized by pages or sections.
|
||||
filename (str): The name of the file being processed.
|
||||
|
||||
Returns:
|
||||
dict: Updated answers dictionary with the effective date.
|
||||
"""
|
||||
# print(f"Effective date for {filename} from smart chunking: {answers_dict.get('AARETE_DERIVED_EFFECTIVE_DT')}")
|
||||
|
||||
# Postprocess the effective date
|
||||
answers_dict["AARETE_DERIVED_EFFECTIVE_DT"] = generic_postprocessing_funcs.date_postprocessing(
|
||||
answers_dict.get("AARETE_DERIVED_EFFECTIVE_DT", "N/A")
|
||||
)
|
||||
answers_dict["AARETE_DERIVED_EFFECTIVE_DT"] = investment_postprocessing_funcs.validate_and_reformat_date(
|
||||
answers_dict["AARETE_DERIVED_EFFECTIVE_DT"]
|
||||
)
|
||||
|
||||
# If the effective date is empty, Get the effective date using the vision-based LLM
|
||||
if config.ENABLE_VISION and string_utils.is_empty(answers_dict.get("AARETE_DERIVED_EFFECTIVE_DT")):
|
||||
# if answers_dict.get("AARETE_DERIVED_EFFECTIVE_DT") in ["N/A", "UNKNOWN"]:
|
||||
effective_date_page_numbers = list(string_utils.extract_effective_date_pages(text_dict, filename).keys())
|
||||
|
||||
pdf_base_path = "/root/doczy.ai/fieldExtraction/src/investment/pdf_files" # local path
|
||||
pdf_path = os.path.join(pdf_base_path, filename.replace("txt", "pdf"))
|
||||
# pdf_path = os.path.join(DOCZY_PDF_FILES_BUCKET_S3_URL, filename.replace("txt", "pdf")) # s3 path
|
||||
image_answer_dict = get_image_array_based_answer(
|
||||
pdf_path, effective_date_page_numbers, filename, fields=["AARETE_DERIVED_EFFECTIVE_DT"]
|
||||
)
|
||||
answers_dict["AARETE_DERIVED_EFFECTIVE_DT"] = image_answer_dict.get("AARETE_DERIVED_EFFECTIVE_DT", "N/A")
|
||||
|
||||
return answers_dict
|
||||
|
||||
def run_full_context_fields(
|
||||
one_to_one_fields: FieldSet,
|
||||
@@ -240,4 +284,4 @@ def run_full_context_fields(
|
||||
full_context_answers_dict = {}
|
||||
for field in full_context_fields.list_fields():
|
||||
full_context_answers_dict[field] = f"Error: {str(e)}"
|
||||
return full_context_answers_dict
|
||||
return full_context_answers_dict
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
import fitz # PyMuPDF
|
||||
import base64
|
||||
from typing import List
|
||||
from io import BytesIO
|
||||
from PIL import Image # Pillow library
|
||||
from src.config import DOCZY_PDF_FILES_BUCKET_S3_URL
|
||||
from src.utils import io_utils
|
||||
import src.prompts.investment_prompts as prompts
|
||||
import src.utils.llm_utils as llm_utils
|
||||
import src.utils.string_utils as string_utils
|
||||
|
||||
def get_image_array_based_answer(
|
||||
pdf_path: str,
|
||||
effective_date_page_numbers: List[int],
|
||||
filename: str,
|
||||
fields: List[str] = ["AARETE_DERIVED_EFFECTIVE_DT"]
|
||||
) -> dict:
|
||||
"""
|
||||
Processes specific pages of a PDF, converts them to a base64-encoded image,
|
||||
and retrieves answers using the Claude 3.5 Sonnet model.
|
||||
|
||||
Args:
|
||||
pdf_path (str): Path to the PDF file.
|
||||
effective_date_page_numbers (List[int]): List of page numbers to process and extract data from.
|
||||
filename (str): Name of the file being operated upon, used for logging or tracking purposes.
|
||||
fields (List[str]): List of fields to extract from the response. Defaults to ["AARETE_DERIVED_EFFECTIVE_DT"].
|
||||
|
||||
Returns:
|
||||
dict: Extracted answers from the response in JSON format. Example:
|
||||
{"AARETE_DERIVED_EFFECTIVE_DT": "YYYY/MM/DD"}
|
||||
"""
|
||||
# Convert specified PDF pages to a base64-encoded image
|
||||
base64_images = process_pdf_pages_to_base64_images(pdf_path, filename, effective_date_page_numbers)
|
||||
# print("No of base64 images:", len(base64_images)) # Log a snippet of the base64 data for debugging
|
||||
|
||||
# Prepare the prompt for the LLM
|
||||
extended_prompt = prompts.effective_date_fix_prompt()
|
||||
|
||||
# Invoke the LLM to get the response
|
||||
response_dict = llm_utils.get_image_array_answer(extended_prompt, filename, base64_images)
|
||||
return response_dict
|
||||
|
||||
|
||||
def process_pdf_pages_to_base64_images(pdf_path: str, filename: str, page_numbers: List[int]) -> List[str]:
|
||||
"""
|
||||
Convert multiple specific pages of a PDF into a list of base64-encoded JPEG images.
|
||||
|
||||
Inputs:
|
||||
pdf_path (str): Path to the PDF file.
|
||||
page_numbers (List[int]): List of 1-based page numbers to process.
|
||||
|
||||
Output:
|
||||
List[str]: List of base64-encoded JPEG images, one for each page.
|
||||
|
||||
Functionality:
|
||||
- Renders each specified PDF page into a high-resolution image.
|
||||
- Converts each rendered image into a base64-encoded JPEG.
|
||||
- Returns the list of base64-encoded images.
|
||||
"""
|
||||
|
||||
|
||||
page_numbers = sorted({int(float(x)) for x in page_numbers}) # Ensure unique and sorted page numbers
|
||||
if not page_numbers:
|
||||
raise ValueError("No valid page numbers provided")
|
||||
try:
|
||||
# Check if the PDF path is an S3 URL
|
||||
if pdf_path.startswith("s3://"):
|
||||
temp_pdf_path = io_utils.read_s3_pdf(pdf_path)
|
||||
else:
|
||||
temp_pdf_path = pdf_path
|
||||
with fitz.open(temp_pdf_path) as pdf_document:
|
||||
zoom = 2 # Resolution multiplier
|
||||
mat = fitz.Matrix(zoom, zoom)
|
||||
base64_images = []
|
||||
|
||||
for page_num in page_numbers:
|
||||
page_num = int(page_num) # Ensure page number is an integer
|
||||
page_idx = page_num - 1
|
||||
if page_idx < 0 or page_idx >= len(pdf_document):
|
||||
raise ValueError(f"Page {page_num} is out of range")
|
||||
# Get the page and convert to image
|
||||
page = pdf_document[page_idx]
|
||||
pix = page.get_pixmap(matrix=mat)
|
||||
img_bytes = pix.tobytes("jpeg")
|
||||
# Save the image to a file (optional)
|
||||
# with open(f"page_{page_num}_{filename}.jpg", "wb") as img_file:
|
||||
# img_file.write(img_bytes) # saving for debugging purposes
|
||||
# Resize the image to ensure it is less than 1 megapixel
|
||||
img = Image.open(BytesIO(img_bytes))
|
||||
max_pixels = 1_000_000 # 1 megapixel
|
||||
if img.width * img.height > max_pixels:
|
||||
scale_factor = (max_pixels / (img.width * img.height)) ** 0.5
|
||||
new_width = int(img.width * scale_factor)
|
||||
new_height = int(img.height * scale_factor)
|
||||
img = img.resize((new_width, new_height), Image.Resampling.LANCZOS)
|
||||
# Save resized image to bytes
|
||||
output_buffer = BytesIO()
|
||||
img.save(output_buffer, format="JPEG")
|
||||
img_bytes = output_buffer.getvalue()
|
||||
# Convert to base64 JPEG
|
||||
base64_image = base64.b64encode(img_bytes).decode("utf-8")
|
||||
base64_images.append(base64_image)
|
||||
|
||||
if not base64_images:
|
||||
raise ValueError("No valid pages provided")
|
||||
|
||||
return base64_images
|
||||
|
||||
except (ValueError, fitz.FileDataError) as e:
|
||||
print(f"Error processing PDF pages: {str(e)}")
|
||||
raise
|
||||
@@ -1183,7 +1183,7 @@ def date_postprocessing(date: str) -> str:
|
||||
if is_empty(date):
|
||||
return "N/A"
|
||||
|
||||
prompt = investment_prompts.date_fix_prompt(date)
|
||||
prompt = investment_prompts.DATE_FIX_PROMPT(date)
|
||||
response = llm_utils.invoke_claude(prompt, config.MODEL_ID_CLAUDE35_SONNET, "date_fix")
|
||||
return extract_text_from_delimiters(response, Delimiter.PIPE)
|
||||
|
||||
|
||||
@@ -51,6 +51,9 @@ def split_text(text: str) -> dict[str, str]:
|
||||
|
||||
|
||||
def clean_law_symbols(contract_text):
|
||||
if not contract_text:
|
||||
return contract_text
|
||||
# First correction: Replace '$$' with '$'
|
||||
contract_text = contract_text.replace("$$", "$")
|
||||
|
||||
contract_text = re.sub(r"(U\.?S\.?C\.?) \$", r"\1§", contract_text) # replaces $ with § when it follows abbreviations like "U.S.C." (United States Code).
|
||||
|
||||
@@ -504,7 +504,7 @@ def ONE_TO_ONE_SINGLE_FIELD_TEMPLATE(context, question):
|
||||
{PIPE_FORMAT_INSTRUCTIONS}
|
||||
"""
|
||||
|
||||
def date_fix_prompt(context:str) -> str:
|
||||
def DATE_FIX_PROMPT(context:str) -> str:
|
||||
return f"""Given a date string, convert it to YYYY/MM/DD format following these rules:
|
||||
Input: Text potentially containing a date
|
||||
Output: Date in YYYY/MM/DD format, or input if it's a duration, or "N/A" if ambiguous/incomplete
|
||||
@@ -546,7 +546,7 @@ Please analyze this date:
|
||||
{PIPE_FORMAT_INSTRUCTIONS}
|
||||
"""
|
||||
|
||||
def derived_term_date_prompt(effective_date:str=None, termination_information:str=None) -> str:
|
||||
def DERIVED_TERM_DATE_PROMPT(effective_date:str=None, termination_information:str=None) -> str:
|
||||
"""
|
||||
Generates a prompt to extract the termination date from a legal contract.
|
||||
|
||||
@@ -561,7 +561,7 @@ def derived_term_date_prompt(effective_date:str=None, termination_information:st
|
||||
|
||||
@cache
|
||||
def invoke_derived_term_date(effective_date:str=None, termination_information:str=None) -> str:
|
||||
prompt = derived_term_date_prompt(effective_date, termination_information)
|
||||
prompt = DERIVED_TERM_DATE_PROMPT(effective_date, termination_information)
|
||||
response = llm_utils.invoke_claude(prompt, MODEL_ID_CLAUDE35_SONNET, "derive_term_date")
|
||||
return extract_text_from_delimiters(response, Delimiter.PIPE)
|
||||
|
||||
@@ -730,3 +730,13 @@ def GROUP_TIN_NPI_TEMPLATE(key_sections: str, provider_list: str) -> str:
|
||||
|
||||
Feel free to justify your answer, but enclose your final answer in |pipes|.
|
||||
"""
|
||||
|
||||
def EFFECTIVE_DATE_FIX_PROMPT() -> str:
|
||||
return """Extract the effective date of the contract or contract effective date or Effective Date of Agreement or Effective Date of Amendment or the amendment effective date from the given image data. ALSO LOOK FOR dates in phrases that semantically indicate an effective date (may or may not be explictly labelled as effective date), even if they do not exactly match the phrases above.\nExamples include but ARE NOT LIMITED TO:\nPhrases indicating when an contract or amendment begins or takes effect, Phrases specifying the start date of the contract or amendment, Phrases defining when the contract or amendment becomes operational, or Any phrase indicating contract or amendment creation or execution date.\nIt is possible that the contract will specify that the effective date is derived from elsewhere in the contract. Give the date in a JSON FORMAT with AARETE_DERIVED_EFFECTIVE_DATE as the key and effective date value in YYYY/MM/DD format as the value.
|
||||
Rules:
|
||||
1. Add 2000 to 2-digit years 0-49, 1900 to 50-99
|
||||
2. Return N/A if: missing/ambiguous month or year components, or invalid dates. If just the day is missing, assume it's the first of the month.
|
||||
3. For ambiguous input cases, assume American format (MM/DD/YYYY) and apply YYYY/MM/DD formatting
|
||||
4. Any day values with spaces or leading zeros (e.g., '0 1' or '01') are interpreted as single-digit days
|
||||
|
||||
Return N/A only for date if NO valid effective date is found. give the answer in the format: {{\"AARETE_DERIVED_EFFECTIVE_DT\": \"YYYY/MM/DD\"}}. nothing else."""
|
||||
@@ -10,6 +10,7 @@ from src import config
|
||||
from src.investment.code_funcs import crosswalk_levels, get_mappings
|
||||
import src.constants.investment_values as investment_values
|
||||
from src.utils.string_utils import datetime_str
|
||||
import tempfile
|
||||
|
||||
|
||||
def read_local(file_path): # io_utils.py
|
||||
@@ -50,6 +51,32 @@ def list_s3_files(prefix=config.S3_PREFIX, bucket=config.S3_BUCKET): # io_utils.
|
||||
response = config.S3_CLIENT.list_objects_v2(Bucket=bucket, Prefix=prefix)
|
||||
return [item["Key"] for item in response.get("Contents", [])]
|
||||
|
||||
def read_s3_pdf(pdf_path):
|
||||
"""
|
||||
Downloads a PDF file from an S3 bucket and returns a temporary file path.
|
||||
|
||||
Args:
|
||||
pdf_path (str): The S3 URL of the PDF file (e.g., s3://bucket-name/path/to/file.pdf).
|
||||
|
||||
Returns:
|
||||
str: The local temporary file path of the downloaded PDF.
|
||||
"""
|
||||
|
||||
s3_client = config.S3_CLIENT
|
||||
s3_parts = pdf_path[5:].split("/", 1)
|
||||
bucket = s3_parts[0]
|
||||
prefix = s3_parts[1] if len(s3_parts) > 1 else ""
|
||||
|
||||
try:
|
||||
response = s3_client.get_object(Bucket=bucket, Key=prefix)
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as temp_pdf:
|
||||
temp_pdf.write(response["Body"].read())
|
||||
temp_pdf_path = temp_pdf.name
|
||||
return temp_pdf_path
|
||||
except ClientError as e:
|
||||
print(f"Error fetching PDF from S3: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def read_s3():
|
||||
s3_client = config.S3_CLIENT
|
||||
|
||||
@@ -15,7 +15,7 @@ from collections import OrderedDict
|
||||
import hashlib
|
||||
import inspect
|
||||
import csv
|
||||
|
||||
import base64
|
||||
|
||||
def update_stats(filename, tokens_sent, tokens_received, elapsed_time, model_type):
|
||||
# Update per-file statistics
|
||||
@@ -164,6 +164,54 @@ def invoke_claude(prompt, model_id, filename, max_tokens=4096):
|
||||
|
||||
return response
|
||||
|
||||
def get_cache_key_with_image_array(prompt, model_id, image_base64_list):
|
||||
"""Generates a unique hash key for caching with a list of base64 images."""
|
||||
combined_images = ":".join(image_base64_list) # Combine all base64 images into a single string
|
||||
return hashlib.sha256(f"{model_id}:{prompt}:{combined_images}".encode()).hexdigest()
|
||||
|
||||
def invoke_claude_with_image_array(prompt, model_id, filename, base64_images,max_tokens=4096):
|
||||
cache_key = get_cache_key_with_image_array(prompt, model_id, base64_images)
|
||||
# print(f"Cache key for {filename}: {cache_key}") # printing for debugging
|
||||
# Check cache first - use get() method from LRUCache
|
||||
if cache_key in claude_cache:
|
||||
claude_cache.move_to_end(cache_key) # Mark as recently used
|
||||
return claude_cache[cache_key]
|
||||
|
||||
# Select execution mode
|
||||
if config.RUN_MODE == "local":
|
||||
if model_id == config.MODEL_ID_CLAUDE35_SONNET:
|
||||
response = local_claude_3_with_image_array(prompt, model_id, filename, max_tokens,base64_images)
|
||||
print(f"Eff Dt Response from local Claude 3 for {filename}", response)
|
||||
input_cost_per_1k, output_cost_per_1k = 0.003, 0.015
|
||||
elif config.RUN_MODE == "ec2":
|
||||
if model_id == config.MODEL_ID_CLAUDE2:
|
||||
response = ec2_claude2(prompt, model_id, filename, max_tokens)
|
||||
input_cost_per_1k, output_cost_per_1k = 0.008, 0.024
|
||||
|
||||
else:
|
||||
print("Usage: python local_main.py <-local> OR python main.py <ec2>")
|
||||
raise
|
||||
|
||||
# Store response in cache with LRU eviction
|
||||
if len(claude_cache) >= CACHE_LIMIT:
|
||||
claude_cache.popitem(last=False) # Remove the oldest (least recently used) item
|
||||
claude_cache[cache_key] = response # Insert new response
|
||||
claude_cache.move_to_end(cache_key) # Mark as recently used
|
||||
|
||||
# Cost logging
|
||||
current_frame = inspect.currentframe()
|
||||
caller_frame = current_frame.f_back
|
||||
caller_name = caller_frame.f_code.co_name
|
||||
del current_frame, caller_frame # Avoid reference cycles
|
||||
|
||||
cost_log_entry = create_cost_log_entry(
|
||||
filename, caller_name, prompt, response, input_cost_per_1k, output_cost_per_1k
|
||||
)
|
||||
with open(config.COST_LOG_NAME, 'a', newline="", encoding='utf-8') as f:
|
||||
writer = csv.DictWriter(f, fieldnames=config.COST_LOG_FIELDS)
|
||||
writer.writerow(cost_log_entry)
|
||||
|
||||
return response
|
||||
|
||||
# Claude calls
|
||||
def local_claude_2(prompt, model_id, filename, max_tokens):
|
||||
@@ -247,6 +295,94 @@ def local_claude_3(
|
||||
raise Exception("Max retries exceeded")
|
||||
|
||||
|
||||
def local_claude_3_with_image_array(
|
||||
prompt, model_id, filename, max_tokens, base64_images, max_retries=5, initial_delay=1, image_media_type="image/png"
|
||||
):
|
||||
"""
|
||||
Processes a prompt along with an image in base64 format using the Claude 3 model via AWS Bedrock.
|
||||
|
||||
Args:
|
||||
prompt (str): Input prompt to LLM.
|
||||
model_id (str): Model ID to use.
|
||||
filename (str): Name of the file being operated upon.
|
||||
max_tokens (int): Maximum tokens to sample.
|
||||
image_base64 (str): Base64 encoded image data.
|
||||
max_retries (int): Maximum number of retries for the request.
|
||||
initial_delay (int): Initial delay for exponential backoff.
|
||||
image_media_type (str): Media type of the image (default: "image/jpeg").
|
||||
|
||||
Returns:
|
||||
str: Response text from the model.
|
||||
"""
|
||||
if config.ENABLE_ANSWER_TRACKING and prompt in config.ANSWER_TRACKING_DICT.keys():
|
||||
return config.ANSWER_TRACKING_DICT[prompt]
|
||||
|
||||
# Create the request body with the new message format
|
||||
body = json.dumps(
|
||||
{
|
||||
"anthropic_version": "bedrock-2023-05-31",
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": 0.0,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Images:"
|
||||
},
|
||||
*[
|
||||
{
|
||||
"type": "image",
|
||||
"source": {
|
||||
"type": "base64",
|
||||
"media_type": image_media_type,
|
||||
"data": image_base64,
|
||||
},
|
||||
}
|
||||
for image_base64 in base64_images
|
||||
],
|
||||
{
|
||||
"type": "text",
|
||||
"text": prompt
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
response = config.BEDROCK_RUNTIME.invoke_model(
|
||||
body=body,
|
||||
modelId=model_id,
|
||||
accept="application/json",
|
||||
contentType="application/json",
|
||||
)
|
||||
response_body = json.loads(response.get("body").read())
|
||||
response_text = response_body["content"][0]["text"]
|
||||
|
||||
if config.ENABLE_ANSWER_TRACKING:
|
||||
config.ANSWER_TRACKING_DICT[prompt] = response_text
|
||||
|
||||
return response_text
|
||||
|
||||
except ClientError as e:
|
||||
if e.response["Error"]["Code"] in [
|
||||
"ThrottlingException",
|
||||
"TooManyRequestsException",
|
||||
]:
|
||||
if attempt < max_retries - 1:
|
||||
delay = (2**attempt + random.uniform(0, 1)) * initial_delay
|
||||
time.sleep(delay)
|
||||
else:
|
||||
raise
|
||||
else:
|
||||
raise
|
||||
|
||||
raise Exception("Max retries exceeded")
|
||||
|
||||
def ec2_claude2(prompt, model_id, filename, max_tokens):
|
||||
start_time = time.time()
|
||||
body = json.dumps(
|
||||
@@ -452,3 +588,8 @@ def get_ac_answer(prompt, filename, fields):
|
||||
# response_dict = dict_operations.secondary_string_to_dict(response_text, filename)
|
||||
response_dict = string_utils.json_parsing_search(response_text, fields)
|
||||
return response_dict
|
||||
|
||||
def get_image_array_answer(prompt,filename,base64_images):
|
||||
response_text = invoke_claude_with_image_array(prompt,config.MODEL_ID_CLAUDE35_SONNET,filename,base64_images)
|
||||
response_dict = json.loads(response_text)
|
||||
return response_dict
|
||||
|
||||
@@ -384,7 +384,6 @@ def get_exhibit_chunk(text_dict: dict,
|
||||
def datetime_str():
|
||||
return datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")
|
||||
|
||||
|
||||
def extract_signature_page(text_dict: dict, filename: str) -> dict:
|
||||
"""Extracts the signature page(s) from a contract text dictionary and returns it as a dictionary.
|
||||
First tries to match the textract marker "this page has N signature(s)", then falls back to looking
|
||||
@@ -408,10 +407,57 @@ def extract_signature_page(text_dict: dict, filename: str) -> dict:
|
||||
if textract_pattern.search(page_text):
|
||||
signature_pages[page_num] = page_text
|
||||
|
||||
# If no pages found with textract pattern, fall back to "signature" keyword
|
||||
# If no pages found with textract pattern, try with keyword
|
||||
if not signature_pages:
|
||||
for page_num, page_text in text_dict.items():
|
||||
if "signature" in page_text.lower():
|
||||
signature_pages[page_num] = page_text
|
||||
page_items = list(text_dict.items())
|
||||
for i, (page_num, page_text) in enumerate(page_items):
|
||||
lower_text = page_text.lower()
|
||||
if "signature page follow" in lower_text or "signature authorization" in lower_text:
|
||||
# curr_page = int(float(page_num))
|
||||
if page_num not in signature_pages:
|
||||
signature_pages[page_num] = page_text # current page
|
||||
if i + 1 < len(page_items):
|
||||
next_page_num, next_page_text = page_items[i + 1]
|
||||
# next_page = int(float(next_page_num))
|
||||
if next_page_num not in signature_pages:
|
||||
signature_pages[next_page_num] = next_page_text # next page
|
||||
|
||||
|
||||
return signature_pages
|
||||
|
||||
|
||||
def extract_effective_date_pages(text_dict: dict, filename: str) -> dict:
|
||||
"""
|
||||
Extracts the effective date page(s) from a contract text dictionary and returns it as a dictionary.
|
||||
|
||||
This function identifies pages that either contain the Textract marker
|
||||
"this page has N signature(s)" or the keyword 'effective date:'.
|
||||
Additionally, it ensures that the first page (page 1) is always included
|
||||
in the result, as it often contains effective date.
|
||||
|
||||
Args:
|
||||
text_dict (dict): A dictionary containing text data, organized by pages.
|
||||
filename (str): The name of the file being processed.
|
||||
|
||||
Returns:
|
||||
dict: A dictionary containing the extracted effective date page(s),
|
||||
where keys are page numbers and values are the corresponding page text.
|
||||
"""
|
||||
|
||||
effective_date_pages = {}
|
||||
|
||||
# Regex pattern to match the Textract marker for signature pages which often has effective dates
|
||||
# This pattern accounts for both numeric and textual representations of numbers (one-ten).
|
||||
textract_pattern = re.compile(r"this page has (?:\d+|one|two|three|four|five|six|seven|eight|nine|ten) signature", re.IGNORECASE)
|
||||
|
||||
# Iterate through each page in the text dictionary.
|
||||
for page_num, page_text in text_dict.items():
|
||||
# page_num = int(float(page_num))
|
||||
# Check if the page contains the Textract marker or the keyword 'effective date:'.
|
||||
if textract_pattern.search(page_text) or 'effective date:' in page_text.lower():
|
||||
effective_date_pages[page_num] = page_text
|
||||
# Always include the first page (page 1) as it often contains effective date.
|
||||
elif page_num == 1:
|
||||
effective_date_pages[page_num] = page_text
|
||||
|
||||
return effective_date_pages
|
||||
|
||||
@@ -196,42 +196,42 @@ class TestStringUtils:
|
||||
assert result == expected
|
||||
|
||||
@pytest.mark.parametrize("text_dict, filename, expected", [
|
||||
(
|
||||
{"1": "This page has 2 signatures.", "2": "None here.", "3": "Signature of the party (but ignored)"},
|
||||
"test_file.txt",
|
||||
{"1": "This page has 2 signatures."}
|
||||
),
|
||||
(
|
||||
{"1": "No relevant content here.", "2": "Nor over here."},
|
||||
"test_file.txt",
|
||||
{}
|
||||
),
|
||||
(
|
||||
{"1": "This page has a signature.", "2": "Another signature is found here."},
|
||||
"test_file.txt",
|
||||
{"1": "This page has a signature.", "2": "Another signature is found here."}
|
||||
),
|
||||
(
|
||||
{"1": "This page has 1 signature.", "2": "No relevant content.", "3": "Signature block here. But ignored bc of the first textract-like page"},
|
||||
"test_file.txt",
|
||||
{"1": "This page has 1 signature."}
|
||||
),
|
||||
(
|
||||
{"1": "No signed names at all.", "2": "Just some random text."},
|
||||
"test_file.txt",
|
||||
{}
|
||||
),
|
||||
(
|
||||
{"1": "This page has a signature.", "2": "Signature is mentioned here too."},
|
||||
"test_file.txt",
|
||||
{"1": "This page has a signature.", "2": "Signature is mentioned here too."}
|
||||
),
|
||||
(
|
||||
{},
|
||||
"test_file.txt",
|
||||
{}
|
||||
),
|
||||
])
|
||||
(
|
||||
{"1": "This page has 2 signatures.", "2": "None here.", "3": "Signature of the party (but ignored)"},
|
||||
"test_file.txt",
|
||||
{"1": "This page has 2 signatures."}
|
||||
),
|
||||
(
|
||||
{"1": "No relevant content here.", "2": "Nor over here."},
|
||||
"test_file.txt",
|
||||
{}
|
||||
),
|
||||
(
|
||||
{"1": "This page has a signature.", "2": "signature page follow.","3": "Signature of the party (but ignored)"},
|
||||
"test_file.txt",
|
||||
{"2": "signature page follow.","3": "Signature of the party (but ignored)"}
|
||||
),
|
||||
(
|
||||
{"1": "This page has 1 signature.", "2": "No relevant content.", "3": "Signature block here. But ignored bc of the first textract-like page"},
|
||||
"test_file.txt",
|
||||
{"1": "This page has 1 signature."}
|
||||
),
|
||||
(
|
||||
{"1": "No signed names at all.", "2": "Just some random text."},
|
||||
"test_file.txt",
|
||||
{}
|
||||
),
|
||||
(
|
||||
{"1": "some random text", "2": "Signature authorization", "3": "Signature of the party"},
|
||||
"test_file.txt",
|
||||
{"2": "Signature authorization", "3": "Signature of the party"}
|
||||
),
|
||||
(
|
||||
{},
|
||||
"test_file.txt",
|
||||
{}
|
||||
),
|
||||
])
|
||||
def test_extract_signature_page(self, text_dict, filename, expected):
|
||||
result = extract_signature_page(text_dict, filename)
|
||||
assert result == expected
|
||||
|
||||
@@ -1,27 +1,27 @@
|
||||
import pytest
|
||||
from src.prompts.investment_prompts import derived_term_date_prompt
|
||||
from src.prompts.investment_prompts import DERIVED_TERM_DATE_PROMPT
|
||||
from src.prompts.investment_prompts import PIPE_FORMAT_INSTRUCTIONS
|
||||
|
||||
def test_derived_term_date_prompt_with_all_params():
|
||||
effective_date = "January 1, 2024"
|
||||
termination_information = "The contract terminates after 3 years."
|
||||
expected_output = f"The following are excerpts from a legal contract related to effective and termination dates. Determine the initial termination date of the contract, excluding any renewals or extensions.\nIf termination information comes in as a duration, derive the termination date exclusive of the final day. For example, if effective date is 2023/02/01 and termination information is 'two years', termination date would be 2025/01/31 and not 2025/02/01. If effective date is 2023/02/01 and termination information is 'year-to-year', termination date would be 2024/01/31.\nIf the termination information comes in as a date, return that date. For example, if effective date is 2023/02/01 and termination information is 2024/07/01, return 2024/07/01.\nEffective Date: January 1, 2024\nTermination Information: The contract terminates after 3 years.\nOnly provide the first termination date, ignoring any automatic or optional renewals. Provide the date in YYYY/MM/DD format only. \n{PIPE_FORMAT_INSTRUCTIONS}"
|
||||
assert derived_term_date_prompt(effective_date, termination_information) == expected_output
|
||||
assert DERIVED_TERM_DATE_PROMPT(effective_date, termination_information) == expected_output
|
||||
|
||||
def test_derived_term_date_prompt_with_only_effective_date():
|
||||
effective_date = "January 1, 2024"
|
||||
termination_information = None
|
||||
expected_output = f"The following are excerpts from a legal contract related to effective and termination dates. Determine the initial termination date of the contract, excluding any renewals or extensions.\nIf termination information comes in as a duration, derive the termination date exclusive of the final day. For example, if effective date is 2023/02/01 and termination information is 'two years', termination date would be 2025/01/31 and not 2025/02/01. If effective date is 2023/02/01 and termination information is 'year-to-year', termination date would be 2024/01/31.\nIf the termination information comes in as a date, return that date. For example, if effective date is 2023/02/01 and termination information is 2024/07/01, return 2024/07/01.\nEffective Date: January 1, 2024\nTermination Information: None\nOnly provide the first termination date, ignoring any automatic or optional renewals. Provide the date in YYYY/MM/DD format only. \n{PIPE_FORMAT_INSTRUCTIONS}"
|
||||
assert derived_term_date_prompt(effective_date, termination_information) == expected_output
|
||||
assert DERIVED_TERM_DATE_PROMPT(effective_date, termination_information) == expected_output
|
||||
|
||||
def test_derived_term_date_prompt_with_only_termination_information():
|
||||
effective_date = None
|
||||
termination_information = "The contract terminates after 3 years."
|
||||
expected_output = f"The following are excerpts from a legal contract related to effective and termination dates. Determine the initial termination date of the contract, excluding any renewals or extensions.\nIf termination information comes in as a duration, derive the termination date exclusive of the final day. For example, if effective date is 2023/02/01 and termination information is 'two years', termination date would be 2025/01/31 and not 2025/02/01. If effective date is 2023/02/01 and termination information is 'year-to-year', termination date would be 2024/01/31.\nIf the termination information comes in as a date, return that date. For example, if effective date is 2023/02/01 and termination information is 2024/07/01, return 2024/07/01.\nEffective Date: None\nTermination Information: The contract terminates after 3 years.\nOnly provide the first termination date, ignoring any automatic or optional renewals. Provide the date in YYYY/MM/DD format only. \n{PIPE_FORMAT_INSTRUCTIONS}"
|
||||
assert derived_term_date_prompt(effective_date, termination_information) == expected_output
|
||||
assert DERIVED_TERM_DATE_PROMPT(effective_date, termination_information) == expected_output
|
||||
|
||||
def test_derived_term_date_prompt_with_no_params():
|
||||
effective_date = None
|
||||
termination_information = None
|
||||
expected_output = f"The following are excerpts from a legal contract related to effective and termination dates. Determine the initial termination date of the contract, excluding any renewals or extensions.\nIf termination information comes in as a duration, derive the termination date exclusive of the final day. For example, if effective date is 2023/02/01 and termination information is 'two years', termination date would be 2025/01/31 and not 2025/02/01. If effective date is 2023/02/01 and termination information is 'year-to-year', termination date would be 2024/01/31.\nIf the termination information comes in as a date, return that date. For example, if effective date is 2023/02/01 and termination information is 2024/07/01, return 2024/07/01.\nEffective Date: None\nTermination Information: None\nOnly provide the first termination date, ignoring any automatic or optional renewals. Provide the date in YYYY/MM/DD format only. \n{PIPE_FORMAT_INSTRUCTIONS}"
|
||||
assert derived_term_date_prompt(effective_date, termination_information) == expected_output
|
||||
assert DERIVED_TERM_DATE_PROMPT(effective_date, termination_information) == expected_output
|
||||
Generated
+117
-1
@@ -740,6 +740,105 @@ files = [
|
||||
{file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pillow"
|
||||
version = "11.2.1"
|
||||
description = "Python Imaging Library (Fork)"
|
||||
optional = false
|
||||
python-versions = ">=3.9"
|
||||
files = [
|
||||
{file = "pillow-11.2.1-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:d57a75d53922fc20c165016a20d9c44f73305e67c351bbc60d1adaf662e74047"},
|
||||
{file = "pillow-11.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:127bf6ac4a5b58b3d32fc8289656f77f80567d65660bc46f72c0d77e6600cc95"},
|
||||
{file = "pillow-11.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4ba4be812c7a40280629e55ae0b14a0aafa150dd6451297562e1764808bbe61"},
|
||||
{file = "pillow-11.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8bd62331e5032bc396a93609982a9ab6b411c05078a52f5fe3cc59234a3abd1"},
|
||||
{file = "pillow-11.2.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:562d11134c97a62fe3af29581f083033179f7ff435f78392565a1ad2d1c2c45c"},
|
||||
{file = "pillow-11.2.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:c97209e85b5be259994eb5b69ff50c5d20cca0f458ef9abd835e262d9d88b39d"},
|
||||
{file = "pillow-11.2.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0c3e6d0f59171dfa2e25d7116217543310908dfa2770aa64b8f87605f8cacc97"},
|
||||
{file = "pillow-11.2.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:cc1c3bc53befb6096b84165956e886b1729634a799e9d6329a0c512ab651e579"},
|
||||
{file = "pillow-11.2.1-cp310-cp310-win32.whl", hash = "sha256:312c77b7f07ab2139924d2639860e084ec2a13e72af54d4f08ac843a5fc9c79d"},
|
||||
{file = "pillow-11.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:9bc7ae48b8057a611e5fe9f853baa88093b9a76303937449397899385da06fad"},
|
||||
{file = "pillow-11.2.1-cp310-cp310-win_arm64.whl", hash = "sha256:2728567e249cdd939f6cc3d1f049595c66e4187f3c34078cbc0a7d21c47482d2"},
|
||||
{file = "pillow-11.2.1-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:35ca289f712ccfc699508c4658a1d14652e8033e9b69839edf83cbdd0ba39e70"},
|
||||
{file = "pillow-11.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0409af9f829f87a2dfb7e259f78f317a5351f2045158be321fd135973fff7bf"},
|
||||
{file = "pillow-11.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4e5c5edee874dce4f653dbe59db7c73a600119fbea8d31f53423586ee2aafd7"},
|
||||
{file = "pillow-11.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b93a07e76d13bff9444f1a029e0af2964e654bfc2e2c2d46bfd080df5ad5f3d8"},
|
||||
{file = "pillow-11.2.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:e6def7eed9e7fa90fde255afaf08060dc4b343bbe524a8f69bdd2a2f0018f600"},
|
||||
{file = "pillow-11.2.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:8f4f3724c068be008c08257207210c138d5f3731af6c155a81c2b09a9eb3a788"},
|
||||
{file = "pillow-11.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a0a6709b47019dff32e678bc12c63008311b82b9327613f534e496dacaefb71e"},
|
||||
{file = "pillow-11.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f6b0c664ccb879109ee3ca702a9272d877f4fcd21e5eb63c26422fd6e415365e"},
|
||||
{file = "pillow-11.2.1-cp311-cp311-win32.whl", hash = "sha256:cc5d875d56e49f112b6def6813c4e3d3036d269c008bf8aef72cd08d20ca6df6"},
|
||||
{file = "pillow-11.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:0f5c7eda47bf8e3c8a283762cab94e496ba977a420868cb819159980b6709193"},
|
||||
{file = "pillow-11.2.1-cp311-cp311-win_arm64.whl", hash = "sha256:4d375eb838755f2528ac8cbc926c3e31cc49ca4ad0cf79cff48b20e30634a4a7"},
|
||||
{file = "pillow-11.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:78afba22027b4accef10dbd5eed84425930ba41b3ea0a86fa8d20baaf19d807f"},
|
||||
{file = "pillow-11.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:78092232a4ab376a35d68c4e6d5e00dfd73454bd12b230420025fbe178ee3b0b"},
|
||||
{file = "pillow-11.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25a5f306095c6780c52e6bbb6109624b95c5b18e40aab1c3041da3e9e0cd3e2d"},
|
||||
{file = "pillow-11.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c7b29dbd4281923a2bfe562acb734cee96bbb129e96e6972d315ed9f232bef4"},
|
||||
{file = "pillow-11.2.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:3e645b020f3209a0181a418bffe7b4a93171eef6c4ef6cc20980b30bebf17b7d"},
|
||||
{file = "pillow-11.2.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b2dbea1012ccb784a65349f57bbc93730b96e85b42e9bf7b01ef40443db720b4"},
|
||||
{file = "pillow-11.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:da3104c57bbd72948d75f6a9389e6727d2ab6333c3617f0a89d72d4940aa0443"},
|
||||
{file = "pillow-11.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:598174aef4589af795f66f9caab87ba4ff860ce08cd5bb447c6fc553ffee603c"},
|
||||
{file = "pillow-11.2.1-cp312-cp312-win32.whl", hash = "sha256:1d535df14716e7f8776b9e7fee118576d65572b4aad3ed639be9e4fa88a1cad3"},
|
||||
{file = "pillow-11.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:14e33b28bf17c7a38eede290f77db7c664e4eb01f7869e37fa98a5aa95978941"},
|
||||
{file = "pillow-11.2.1-cp312-cp312-win_arm64.whl", hash = "sha256:21e1470ac9e5739ff880c211fc3af01e3ae505859392bf65458c224d0bf283eb"},
|
||||
{file = "pillow-11.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fdec757fea0b793056419bca3e9932eb2b0ceec90ef4813ea4c1e072c389eb28"},
|
||||
{file = "pillow-11.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b0e130705d568e2f43a17bcbe74d90958e8a16263868a12c3e0d9c8162690830"},
|
||||
{file = "pillow-11.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bdb5e09068332578214cadd9c05e3d64d99e0e87591be22a324bdbc18925be0"},
|
||||
{file = "pillow-11.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d189ba1bebfbc0c0e529159631ec72bb9e9bc041f01ec6d3233d6d82eb823bc1"},
|
||||
{file = "pillow-11.2.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:191955c55d8a712fab8934a42bfefbf99dd0b5875078240943f913bb66d46d9f"},
|
||||
{file = "pillow-11.2.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:ad275964d52e2243430472fc5d2c2334b4fc3ff9c16cb0a19254e25efa03a155"},
|
||||
{file = "pillow-11.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:750f96efe0597382660d8b53e90dd1dd44568a8edb51cb7f9d5d918b80d4de14"},
|
||||
{file = "pillow-11.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fe15238d3798788d00716637b3d4e7bb6bde18b26e5d08335a96e88564a36b6b"},
|
||||
{file = "pillow-11.2.1-cp313-cp313-win32.whl", hash = "sha256:3fe735ced9a607fee4f481423a9c36701a39719252a9bb251679635f99d0f7d2"},
|
||||
{file = "pillow-11.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:74ee3d7ecb3f3c05459ba95eed5efa28d6092d751ce9bf20e3e253a4e497e691"},
|
||||
{file = "pillow-11.2.1-cp313-cp313-win_arm64.whl", hash = "sha256:5119225c622403afb4b44bad4c1ca6c1f98eed79db8d3bc6e4e160fc6339d66c"},
|
||||
{file = "pillow-11.2.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:8ce2e8411c7aaef53e6bb29fe98f28cd4fbd9a1d9be2eeea434331aac0536b22"},
|
||||
{file = "pillow-11.2.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:9ee66787e095127116d91dea2143db65c7bb1e232f617aa5957c0d9d2a3f23a7"},
|
||||
{file = "pillow-11.2.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9622e3b6c1d8b551b6e6f21873bdcc55762b4b2126633014cea1803368a9aa16"},
|
||||
{file = "pillow-11.2.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63b5dff3a68f371ea06025a1a6966c9a1e1ee452fc8020c2cd0ea41b83e9037b"},
|
||||
{file = "pillow-11.2.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:31df6e2d3d8fc99f993fd253e97fae451a8db2e7207acf97859732273e108406"},
|
||||
{file = "pillow-11.2.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:062b7a42d672c45a70fa1f8b43d1d38ff76b63421cbbe7f88146b39e8a558d91"},
|
||||
{file = "pillow-11.2.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4eb92eca2711ef8be42fd3f67533765d9fd043b8c80db204f16c8ea62ee1a751"},
|
||||
{file = "pillow-11.2.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f91ebf30830a48c825590aede79376cb40f110b387c17ee9bd59932c961044f9"},
|
||||
{file = "pillow-11.2.1-cp313-cp313t-win32.whl", hash = "sha256:e0b55f27f584ed623221cfe995c912c61606be8513bfa0e07d2c674b4516d9dd"},
|
||||
{file = "pillow-11.2.1-cp313-cp313t-win_amd64.whl", hash = "sha256:36d6b82164c39ce5482f649b437382c0fb2395eabc1e2b1702a6deb8ad647d6e"},
|
||||
{file = "pillow-11.2.1-cp313-cp313t-win_arm64.whl", hash = "sha256:225c832a13326e34f212d2072982bb1adb210e0cc0b153e688743018c94a2681"},
|
||||
{file = "pillow-11.2.1-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:7491cf8a79b8eb867d419648fff2f83cb0b3891c8b36da92cc7f1931d46108c8"},
|
||||
{file = "pillow-11.2.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8b02d8f9cb83c52578a0b4beadba92e37d83a4ef11570a8688bbf43f4ca50909"},
|
||||
{file = "pillow-11.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:014ca0050c85003620526b0ac1ac53f56fc93af128f7546623cc8e31875ab928"},
|
||||
{file = "pillow-11.2.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3692b68c87096ac6308296d96354eddd25f98740c9d2ab54e1549d6c8aea9d79"},
|
||||
{file = "pillow-11.2.1-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:f781dcb0bc9929adc77bad571b8621ecb1e4cdef86e940fe2e5b5ee24fd33b35"},
|
||||
{file = "pillow-11.2.1-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:2b490402c96f907a166615e9a5afacf2519e28295f157ec3a2bb9bd57de638cb"},
|
||||
{file = "pillow-11.2.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:dd6b20b93b3ccc9c1b597999209e4bc5cf2853f9ee66e3fc9a400a78733ffc9a"},
|
||||
{file = "pillow-11.2.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:4b835d89c08a6c2ee7781b8dd0a30209a8012b5f09c0a665b65b0eb3560b6f36"},
|
||||
{file = "pillow-11.2.1-cp39-cp39-win32.whl", hash = "sha256:b10428b3416d4f9c61f94b494681280be7686bda15898a3a9e08eb66a6d92d67"},
|
||||
{file = "pillow-11.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:6ebce70c3f486acf7591a3d73431fa504a4e18a9b97ff27f5f47b7368e4b9dd1"},
|
||||
{file = "pillow-11.2.1-cp39-cp39-win_arm64.whl", hash = "sha256:c27476257b2fdcd7872d54cfd119b3a9ce4610fb85c8e32b70b42e3680a29a1e"},
|
||||
{file = "pillow-11.2.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:9b7b0d4fd2635f54ad82785d56bc0d94f147096493a79985d0ab57aedd563156"},
|
||||
{file = "pillow-11.2.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:aa442755e31c64037aa7c1cb186e0b369f8416c567381852c63444dd666fb772"},
|
||||
{file = "pillow-11.2.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f0d3348c95b766f54b76116d53d4cb171b52992a1027e7ca50c81b43b9d9e363"},
|
||||
{file = "pillow-11.2.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85d27ea4c889342f7e35f6d56e7e1cb345632ad592e8c51b693d7b7556043ce0"},
|
||||
{file = "pillow-11.2.1-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:bf2c33d6791c598142f00c9c4c7d47f6476731c31081331664eb26d6ab583e01"},
|
||||
{file = "pillow-11.2.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e616e7154c37669fc1dfc14584f11e284e05d1c650e1c0f972f281c4ccc53193"},
|
||||
{file = "pillow-11.2.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:39ad2e0f424394e3aebc40168845fee52df1394a4673a6ee512d840d14ab3013"},
|
||||
{file = "pillow-11.2.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:80f1df8dbe9572b4b7abdfa17eb5d78dd620b1d55d9e25f834efdbee872d3aed"},
|
||||
{file = "pillow-11.2.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:ea926cfbc3957090becbcbbb65ad177161a2ff2ad578b5a6ec9bb1e1cd78753c"},
|
||||
{file = "pillow-11.2.1-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:738db0e0941ca0376804d4de6a782c005245264edaa253ffce24e5a15cbdc7bd"},
|
||||
{file = "pillow-11.2.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9db98ab6565c69082ec9b0d4e40dd9f6181dab0dd236d26f7a50b8b9bfbd5076"},
|
||||
{file = "pillow-11.2.1-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:036e53f4170e270ddb8797d4c590e6dd14d28e15c7da375c18978045f7e6c37b"},
|
||||
{file = "pillow-11.2.1-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:14f73f7c291279bd65fda51ee87affd7c1e097709f7fdd0188957a16c264601f"},
|
||||
{file = "pillow-11.2.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:208653868d5c9ecc2b327f9b9ef34e0e42a4cdd172c2988fd81d62d2bc9bc044"},
|
||||
{file = "pillow-11.2.1.tar.gz", hash = "sha256:a64dd61998416367b7ef979b73d3a85853ba9bec4c2925f74e588879a58716b6"},
|
||||
]
|
||||
|
||||
[package.extras]
|
||||
docs = ["furo", "olefile", "sphinx (>=8.2)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinxext-opengraph"]
|
||||
fpx = ["olefile"]
|
||||
mic = ["olefile"]
|
||||
test-arrow = ["pyarrow"]
|
||||
tests = ["check-manifest", "coverage (>=7.4.2)", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout", "trove-classifiers (>=2024.10.12)"]
|
||||
typing = ["typing-extensions"]
|
||||
xmp = ["defusedxml"]
|
||||
|
||||
[[package]]
|
||||
name = "platformdirs"
|
||||
version = "4.3.6"
|
||||
@@ -895,6 +994,23 @@ files = [
|
||||
[package.dependencies]
|
||||
typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0"
|
||||
|
||||
[[package]]
|
||||
name = "pymupdf"
|
||||
version = "1.25.5"
|
||||
description = "A high performance Python library for data extraction, analysis, conversion & manipulation of PDF (and other) documents."
|
||||
optional = false
|
||||
python-versions = ">=3.9"
|
||||
files = [
|
||||
{file = "pymupdf-1.25.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:cde4e1c9cfb09c0e1e9c2b7f4b787dd6bb34a32cfe141a4675e24af7c0c25dd3"},
|
||||
{file = "pymupdf-1.25.5-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:5a35e2725fae0ab57f058dff77615c15eb5961eac50ba04f41ebc792cd8facad"},
|
||||
{file = "pymupdf-1.25.5-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d94b800e9501929c42283d39bc241001dd87fdeea297b5cb40d5b5714534452f"},
|
||||
{file = "pymupdf-1.25.5-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ee22155d3a634642d76553204867d862ae1bdd9f7cf70c0797d8127ebee6bed5"},
|
||||
{file = "pymupdf-1.25.5-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6ed7fc25271004d6d3279c20a80cb2bb4cda3efa9f9088dcc07cd790eca0bc63"},
|
||||
{file = "pymupdf-1.25.5-cp39-abi3-win32.whl", hash = "sha256:65e18ddb37fe8ec4edcdbebe9be3a8486b6a2f42609d0a142677e42f3a0614f8"},
|
||||
{file = "pymupdf-1.25.5-cp39-abi3-win_amd64.whl", hash = "sha256:7f44bc3d03ea45b2f68c96464f96105e8c7908896f2fb5e8c04f1fb8dae7981e"},
|
||||
{file = "pymupdf-1.25.5.tar.gz", hash = "sha256:5f96311cacd13254c905f6654a004a0a2025b71cabc04fda667f5472f72c15a0"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pytest"
|
||||
version = "8.3.3"
|
||||
@@ -1241,4 +1357,4 @@ zstd = ["zstandard (>=0.18.0)"]
|
||||
[metadata]
|
||||
lock-version = "2.0"
|
||||
python-versions = "^3.12"
|
||||
content-hash = "b437b66e0a7d68067a1615d3adbcd90a7ae55d4230f6ed639de8b1b143dcb92d"
|
||||
content-hash = "11f240679ad504f8cfc3a866067918339f2dd77f3764b1bbf77f992dac4e7bb3"
|
||||
|
||||
@@ -11,6 +11,8 @@ python = "^3.12"
|
||||
pandas = "^2.2.3"
|
||||
boto3 = "^1.35.40"
|
||||
anthropic = "^0.36.0"
|
||||
pymupdf = "^1.25.5"
|
||||
pillow = "^11.2.1"
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
pytest = "^8.3.3"
|
||||
|
||||
Reference in New Issue
Block a user