jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/remove-exhibit-filtering (pull request #565)

Browse Source 
strip out reimbursement exhibit identification

* strip out reimbursement exhibit identification


Approved-by: Katon Minhas
This commit is contained in:
Alex Galarce
2025-06-09 21:35:20 +00:00
parent 0d3a110e26
commit 32d431ec4e
3 changed files with 6 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fieldExtraction/src/investment/file_processing.py
+4 -4
View File
@@ -118,12 +118,12 @@ def process_file(file_object, all_dataset, run_timestamp):
contract_text = preprocess.clean_text(contract_text)
text_dict, top_sheet_dict = preprocess.split_text(contract_text)
text_dict = preprocess.clean_tables(text_dict, filename)
exhibit_dict, all_exhibit_headers, reimbursement_exhibits = preprocess.one_to_n_exhibit_chunking(text_dict, filename)
exhibit_dict, all_exhibit_headers = preprocess.one_to_n_exhibit_chunking(text_dict, filename)
print(f"{datetime_str()} Preprocessing Complete - {filename}")
################## ONE TO N ##################
if string_utils.contains_reimbursement(contract_text):
one_to_n_results, dynamic_one_to_one_fields = run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, reimbursement_exhibits, all_dataset) # Return df
one_to_n_results, dynamic_one_to_one_fields = run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, all_dataset) # Return df
one_to_n_results['FILE_NAME'] = filename
one_to_n_results = investment_postprocessing_funcs.generate_reimb_ids(one_to_n_results) # Add reimb_id
print(f"{datetime_str()} One to N Complete - {filename}")
@@ -200,11 +200,11 @@ def run_one_to_one_prompts(filename, contract_text, text_dict, top_sheet_dict, d
return one_to_one_results[0]
def run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, reimbursement_exhibits, all_dataset):
def run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, all_dataset):
################## RUN PROMPTS ##################
one_to_n_results = []
for exhibit_page in reimbursement_exhibits:
for exhibit_page in exhibit_dict.keys():
exhibit_text = exhibit_dict[exhibit_page]
exhibit_header = all_exhibit_headers[exhibit_page]
fieldExtraction/src/investment/preprocess.py
+2 -46
View File
@@ -6,7 +6,6 @@ import src.prompts.investment_prompts as investment_prompts
from src.utils import llm_utils, string_utils
from src.enums.delimiters import Delimiter
import logging
import yaml
def clean_text(contract_text):
@@ -66,49 +65,8 @@ def get_exhibit_header(exhibit_chunk, filename):
)
return llm_answer_final
def identify_reimbursement_exhibits(exhibit_headers: dict, filename: str) -> list[str]:
"""Use LLM to identify which exhibits likely contain reimbursement terms.
Args:
exhibit_headers (dict): Dictionary of exhibit headers to analyze, keyed by
string page number, valued by the exhibit header text.
filename (str): The name of the file being processed.
Returns:
list[str]: List of page numbers (as strings) that likely contain reimbursement terms.
"""
prompt = investment_prompts.IDENTIFY_REIMBURSEMENT_EXHIBITS_PROMPT(exhibit_headers, yaml_output=True) # Trying YAML output
logging.debug(f"""Identifying reimbursement exhibits for {filename} with prompt:
{prompt}""")
llm_answer_raw = llm_utils.invoke_claude(
prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 2000
)
yaml_content = string_utils.extract_text_from_delimiters(
llm_answer_raw, Delimiter.TRIPLE_BACKTICK)
# Check if the first line is just `yaml` and if so remove it
lines = yaml_content.strip().split('\n')
if lines[0].strip().lower() == 'yaml':
yaml_content = '\n'.join(lines[1:])
logging.debug(f"""LLM raw output for {filename}:
{llm_answer_raw}""")
# print("Extracted YAML content:\n", yaml_content) # Debugging line
logging.debug(f"""YAML content for {filename}:
{yaml_content}""")
try:
# Parse the YAML output into a dictionary
exhibits_dict = yaml.safe_load(yaml_content)
# Return both EXPLICIT and IMPLICIT exhibits (exclude only 'NO')
return [str(page) for page, value in exhibits_dict.items()
if value in ['EXPLICIT', 'IMPLICIT']]
except yaml.YAMLError as e:
logging.error(f"Error parsing YAML: {e}")
logging.error(f"Raw LLM output: {llm_answer_raw}")
return list(exhibit_headers.keys()) # Return all keys of the exhibit headers if parsing fails
def one_to_n_exhibit_chunking(text_dict, filename):
def one_to_n_exhibit_chunking(text_dict, filename) -> tuple[dict, dict]:
"""
Process the text dictionary to identify and chunk exhibits.
This function extracts exhibit pages, maps them to their respective chunks,
@@ -121,15 +79,13 @@ def one_to_n_exhibit_chunking(text_dict, filename):
tuple: A tuple containing:
- dict: A dictionary where keys are exhibit page numbers and values are the corresponding exhibit text chunks.
- dict: A dictionary where keys are exhibit page numbers and values are the exhibit headers.
- list: A list of exhibit page numbers that likely contain reimbursement terms.
"""
exhibit_pages = preprocessing_funcs.get_exhibit_pages(text_dict, filename)
exhibit_chunk_mapping = preprocessing_funcs.chunk_by_exhibit(text_dict, exhibit_pages)
exhibit_dict = {page : string_utils.get_exhibit_chunk(text_dict, exhibit_chunk_mapping, page) for page in exhibit_pages}
all_exhibit_headers = {page: get_exhibit_header(exhibit_dict[page], filename) for page in exhibit_pages}
reimbursement_exhibits = identify_reimbursement_exhibits(all_exhibit_headers, filename)
return exhibit_dict, all_exhibit_headers, reimbursement_exhibits
return exhibit_dict, all_exhibit_headers
def one_to_one_smart_chunking(text_dict, contract_text, one_to_one_fields):
"""
fieldExtraction/src/prompts/investment_prompts.py
-25
View File
@@ -307,31 +307,6 @@ Before returning any output, ensure that all instructions for inclusion were fol
Enclose your final answer in |pipes|.
"""
def IDENTIFY_REIMBURSEMENT_EXHIBITS_PROMPT(exhibit_headers: dict, yaml_output: bool = False):
formatted_headers = [f"Page {page}: {header}" for page, header in exhibit_headers.items()]
return f"""Review these exhibit headers and identify which ones are likely to contain reimbursement methodologies, payment terms, or compensation schedules.
Consider both explicit and implicit indicators of reimbursement information:
EXPLICIT INDICATORS:
- Exhibits that have "Compensation Schedule", "Reimbursement Schedule", "Rates", or similar in the title. The title must somehow explicitly indicate that the Exhibit contains payment terms.
IMPLICIT INDICATORS:
- Sections titled "COVERED SERVICES" for specific insurance products typically include payment terms because they define what services are reimbursable and how
- Attachments specifying plan types (Medicare, Medicaid, Commercial, Marketplace) usually contain reimbursement details
- References to specific programs (Medicare Advantage, CHIP) that have distinct payment methodologies
- Attachments that appear to define specific services or service categories
For each header, return 'EXPLICIT' if there is an EXPLICIT INDICATOR, 'IMPLICIT' if there is an IMPLICIT INDICATOR, or 'NO' if the header indicates that the exhibit is unlikely to contain reimbursement information.
Headers to review:
{"\n".join(formatted_headers)}
Return your answer as a {"JSON object" if not yaml_output else "YAML list"} where keys are PAGE NUMBERS and values are 'EXPLICIT', 'IMPLICIT', or 'NO'.
Feel free to explain your reasoning, but place your final {"JSON" if not yaml_output else "YAML"} output between triple backticks.
If you're uncertain about a header, err on the side of inclusion ('IMPLICIT').
"""
def EXHIBIT_LEVEL(context, fields):
return f"""Extract attributes from a section of a contract.