From 0c07167094ff2907a891ea30714f5496a082f2a9 Mon Sep 17 00:00:00 2001 From: Alex Galarce Date: Tue, 10 Jun 2025 18:12:56 +0000 Subject: [PATCH] Merged in hotfix/exhibit-headers-bug (pull request #567) switch to `preprocessing_funcs.get_exhibit_headers` * switch to `preprocessing_funcs.get_exhibit_headers` Approved-by: Katon Minhas --- fieldExtraction/src/investment/preprocess.py | 23 +------------------- fieldExtraction/src/preprocessing_funcs.py | 22 ++++++++++++++----- 2 files changed, 18 insertions(+), 27 deletions(-) diff --git a/fieldExtraction/src/investment/preprocess.py b/fieldExtraction/src/investment/preprocess.py index f88acb1..bdcc622 100644 --- a/fieldExtraction/src/investment/preprocess.py +++ b/fieldExtraction/src/investment/preprocess.py @@ -45,27 +45,6 @@ def split_text(contract_text): return text_dict, top_sheet_dict -def get_exhibit_header(exhibit_chunk, filename): - """ - Extracts the exhibit header using an LLM prompt. - - Args: - exhibit_chunk (str): A chunk of exhibit text to analyze. - filename (str): The name of the file being processed. - - Returns: - str: The extracted exhibit header. - """ - prompt = investment_prompts.EXHIBIT_HEADER(exhibit_chunk[0:400]) - llm_answer_raw = llm_utils.invoke_claude( - prompt, config.MODEL_ID_CLAUDE35_SONNET, filename - ) - llm_answer_final = string_utils.extract_text_from_delimiters( - llm_answer_raw, Delimiter.PIPE - ) - return llm_answer_final - - def one_to_n_exhibit_chunking(text_dict, filename) -> tuple[dict, dict]: """ Process the text dictionary to identify and chunk exhibits. @@ -84,7 +63,7 @@ def one_to_n_exhibit_chunking(text_dict, filename) -> tuple[dict, dict]: exhibit_pages = preprocessing_funcs.get_exhibit_pages(text_dict, filename) exhibit_chunk_mapping = preprocessing_funcs.chunk_by_exhibit(text_dict, exhibit_pages) exhibit_dict = preprocessing_funcs.get_exhibit_dict(text_dict, exhibit_chunk_mapping) - all_exhibit_headers = {page: get_exhibit_header(exhibit_dict[page], filename) for page in exhibit_pages} + all_exhibit_headers = preprocessing_funcs.get_exhibit_headers(exhibit_dict=exhibit_dict, filename=filename) return exhibit_dict, all_exhibit_headers def one_to_one_smart_chunking(text_dict, contract_text, one_to_one_fields): diff --git a/fieldExtraction/src/preprocessing_funcs.py b/fieldExtraction/src/preprocessing_funcs.py index a13c253..89ac431 100644 --- a/fieldExtraction/src/preprocessing_funcs.py +++ b/fieldExtraction/src/preprocessing_funcs.py @@ -427,16 +427,28 @@ def get_exhibit_dict(text_dict: dict, exhibit_dict[chunk_key] = "\n".join(text_dict[bp] for bp in base_pages) return exhibit_dict -def get_exhibit_headers(exhibit_dict, filename): +def get_exhibit_headers(exhibit_dict: dict[str, str], filename: str) -> dict[str, str]: """ - Extracts the exhibit header using an LLM prompt. + Extracts each exhibit header using an LLM prompt. + + This function processes exhibit chunks to extract their headers, optimizing LLM calls + by reusing headers for chunks that share the same base page number (e.g., "5.0", "5.1", "5.2" + all share base page "5"). Args: - exhibit_chunk (str): A chunk of exhibit text to analyze. - filename (str): The name of the file being processed. + exhibit_dict (dict[str, str]): A dictionary where keys are exhibit page numbers + (e.g., "1.0", "5.1") and values are the concatenated + text of those exhibit chunks. + filename (str): The name of the file being processed (used for LLM tracking). Returns: - str: The extracted exhibit header. + dict[str, str]: A dictionary where keys are exhibit page numbers and values are + the extracted exhibit headers. + + Notes: + - Uses only the first 400 characters of each chunk for header extraction + - Caches headers by base page number to avoid redundant LLM calls + - Base page number is extracted by splitting on '.' and taking the first part """ exhibit_header_dict = {} base_pages = {}