diff --git a/fieldExtraction/src/investment/file_processing.py b/fieldExtraction/src/investment/file_processing.py index 0e9d447..93d3371 100644 --- a/fieldExtraction/src/investment/file_processing.py +++ b/fieldExtraction/src/investment/file_processing.py @@ -118,12 +118,12 @@ def process_file(file_object, all_dataset, run_timestamp): contract_text = preprocess.clean_text(contract_text) text_dict, top_sheet_dict = preprocess.split_text(contract_text) text_dict = preprocess.clean_tables(text_dict, filename) - exhibit_dict, all_exhibit_headers, reimbursement_exhibits = preprocess.one_to_n_exhibit_chunking(text_dict, filename) + exhibit_dict, all_exhibit_headers = preprocess.one_to_n_exhibit_chunking(text_dict, filename) print(f"{datetime_str()} Preprocessing Complete - {filename}") ################## ONE TO N ################## if string_utils.contains_reimbursement(contract_text): - one_to_n_results, dynamic_one_to_one_fields = run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, reimbursement_exhibits, all_dataset) # Return df + one_to_n_results, dynamic_one_to_one_fields = run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, all_dataset) # Return df one_to_n_results['FILE_NAME'] = filename one_to_n_results = investment_postprocessing_funcs.generate_reimb_ids(one_to_n_results) # Add reimb_id print(f"{datetime_str()} One to N Complete - {filename}") @@ -200,11 +200,11 @@ def run_one_to_one_prompts(filename, contract_text, text_dict, top_sheet_dict, d return one_to_one_results[0] -def run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, reimbursement_exhibits, all_dataset): +def run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, all_dataset): ################## RUN PROMPTS ################## one_to_n_results = [] - for exhibit_page in reimbursement_exhibits: + for exhibit_page in exhibit_dict.keys(): exhibit_text = exhibit_dict[exhibit_page] exhibit_header = all_exhibit_headers[exhibit_page] diff --git a/fieldExtraction/src/investment/preprocess.py b/fieldExtraction/src/investment/preprocess.py index d1032bb..364ba6a 100644 --- a/fieldExtraction/src/investment/preprocess.py +++ b/fieldExtraction/src/investment/preprocess.py @@ -6,7 +6,6 @@ import src.prompts.investment_prompts as investment_prompts from src.utils import llm_utils, string_utils from src.enums.delimiters import Delimiter import logging -import yaml def clean_text(contract_text): @@ -66,49 +65,8 @@ def get_exhibit_header(exhibit_chunk, filename): ) return llm_answer_final -def identify_reimbursement_exhibits(exhibit_headers: dict, filename: str) -> list[str]: - """Use LLM to identify which exhibits likely contain reimbursement terms. - Args: - exhibit_headers (dict): Dictionary of exhibit headers to analyze, keyed by - string page number, valued by the exhibit header text. - filename (str): The name of the file being processed. - - Returns: - list[str]: List of page numbers (as strings) that likely contain reimbursement terms. - """ - prompt = investment_prompts.IDENTIFY_REIMBURSEMENT_EXHIBITS_PROMPT(exhibit_headers, yaml_output=True) # Trying YAML output - logging.debug(f"""Identifying reimbursement exhibits for {filename} with prompt: -{prompt}""") - llm_answer_raw = llm_utils.invoke_claude( - prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 2000 - ) - yaml_content = string_utils.extract_text_from_delimiters( - llm_answer_raw, Delimiter.TRIPLE_BACKTICK) - - # Check if the first line is just `yaml` and if so remove it - lines = yaml_content.strip().split('\n') - if lines[0].strip().lower() == 'yaml': - yaml_content = '\n'.join(lines[1:]) - - logging.debug(f"""LLM raw output for {filename}: -{llm_answer_raw}""") - # print("Extracted YAML content:\n", yaml_content) # Debugging line - logging.debug(f"""YAML content for {filename}: -{yaml_content}""") - try: - # Parse the YAML output into a dictionary - exhibits_dict = yaml.safe_load(yaml_content) - # Return both EXPLICIT and IMPLICIT exhibits (exclude only 'NO') - return [str(page) for page, value in exhibits_dict.items() - if value in ['EXPLICIT', 'IMPLICIT']] - except yaml.YAMLError as e: - logging.error(f"Error parsing YAML: {e}") - logging.error(f"Raw LLM output: {llm_answer_raw}") - return list(exhibit_headers.keys()) # Return all keys of the exhibit headers if parsing fails - - -def one_to_n_exhibit_chunking(text_dict, filename): +def one_to_n_exhibit_chunking(text_dict, filename) -> tuple[dict, dict]: """ Process the text dictionary to identify and chunk exhibits. This function extracts exhibit pages, maps them to their respective chunks, @@ -121,15 +79,13 @@ def one_to_n_exhibit_chunking(text_dict, filename): tuple: A tuple containing: - dict: A dictionary where keys are exhibit page numbers and values are the corresponding exhibit text chunks. - dict: A dictionary where keys are exhibit page numbers and values are the exhibit headers. - - list: A list of exhibit page numbers that likely contain reimbursement terms. """ exhibit_pages = preprocessing_funcs.get_exhibit_pages(text_dict, filename) exhibit_chunk_mapping = preprocessing_funcs.chunk_by_exhibit(text_dict, exhibit_pages) exhibit_dict = {page : string_utils.get_exhibit_chunk(text_dict, exhibit_chunk_mapping, page) for page in exhibit_pages} all_exhibit_headers = {page: get_exhibit_header(exhibit_dict[page], filename) for page in exhibit_pages} - reimbursement_exhibits = identify_reimbursement_exhibits(all_exhibit_headers, filename) - return exhibit_dict, all_exhibit_headers, reimbursement_exhibits + return exhibit_dict, all_exhibit_headers def one_to_one_smart_chunking(text_dict, contract_text, one_to_one_fields): """ diff --git a/fieldExtraction/src/prompts/investment_prompts.py b/fieldExtraction/src/prompts/investment_prompts.py index 0c22d2d..37fb28b 100644 --- a/fieldExtraction/src/prompts/investment_prompts.py +++ b/fieldExtraction/src/prompts/investment_prompts.py @@ -307,31 +307,6 @@ Before returning any output, ensure that all instructions for inclusion were fol Enclose your final answer in |pipes|. """ -def IDENTIFY_REIMBURSEMENT_EXHIBITS_PROMPT(exhibit_headers: dict, yaml_output: bool = False): - formatted_headers = [f"Page {page}: {header}" for page, header in exhibit_headers.items()] - return f"""Review these exhibit headers and identify which ones are likely to contain reimbursement methodologies, payment terms, or compensation schedules. - -Consider both explicit and implicit indicators of reimbursement information: - -EXPLICIT INDICATORS: -- Exhibits that have "Compensation Schedule", "Reimbursement Schedule", "Rates", or similar in the title. The title must somehow explicitly indicate that the Exhibit contains payment terms. - -IMPLICIT INDICATORS: -- Sections titled "COVERED SERVICES" for specific insurance products typically include payment terms because they define what services are reimbursable and how -- Attachments specifying plan types (Medicare, Medicaid, Commercial, Marketplace) usually contain reimbursement details -- References to specific programs (Medicare Advantage, CHIP) that have distinct payment methodologies -- Attachments that appear to define specific services or service categories - -For each header, return 'EXPLICIT' if there is an EXPLICIT INDICATOR, 'IMPLICIT' if there is an IMPLICIT INDICATOR, or 'NO' if the header indicates that the exhibit is unlikely to contain reimbursement information. - -Headers to review: -{"\n".join(formatted_headers)} - -Return your answer as a {"JSON object" if not yaml_output else "YAML list"} where keys are PAGE NUMBERS and values are 'EXPLICIT', 'IMPLICIT', or 'NO'. -Feel free to explain your reasoning, but place your final {"JSON" if not yaml_output else "YAML"} output between triple backticks. -If you're uncertain about a header, err on the side of inclusion ('IMPLICIT'). -""" - def EXHIBIT_LEVEL(context, fields): return f"""Extract attributes from a section of a contract.