doczyai-pipelines/fieldExtraction/src/file_processing.py

import os
import pandas as pd
import time
import csv
import json

import config
import keywords
import preprocess
import preprocessing_funcs
import table_funcs
import bottom_up_funcs
import top_down_funcs
import conditional_funcs
import postprocess
import merge_funcs
import utils
import ac_funcs
import prompts
import valid
import claude_funcs
from ac_regex_chunking import npi_hotfix, irs_hotfix

def run_ac_prompts(file_object):

    ac_answers_dict = {}
    log_data = []
    chunk_data = []

    ################## INITIATE PROCESSING ##################
    filename, contract_text = file_object
    print(f"Processing AC for {filename}...")

    ################## PREPROCESS - SMART CHUNK ##################

    # Top Sheet Addition
    text_dict, top_sheet_dict, exhibit_pages, num_pages, ac_chunks = preprocess.preprocess(
        contract_text, filename, fields="ac"
    )
    print(f"AC Preprocessing Complete - {filename}")

    ################## RUN REGEX FUNCTIONALITY ##################
    # no prompting for these fields
    
    irs_answers = irs_hotfix(filename, text_dict, top_sheet_dict)
    ac_answers_dict.update(irs_answers)

    npi_answers = npi_hotfix(text_dict, top_sheet_dict)
    ac_answers_dict.update(npi_answers)

    ################## RUN CHUNKED PROMPTS ##################

    ## Check `ac_funcs.create_prompt()` vs. `prompts.AC_*_template()`
    all_fields = set(prompts.AC_DICT.keys())
    field_groups = [
        field_group for field_group in keywords.GROUPED_KEYWORD_MAPPINGS.keys()
    ]

    # list to append the fields with no matching keywords
    no_keyword_matches = []

    for field_group in field_groups:
        keyword_dict = keywords.GROUPED_KEYWORD_MAPPINGS[field_group]["keywords"]
        fields = keywords.GROUPED_KEYWORD_MAPPINGS[field_group]["fields"]
        fields.sort()
        questions = {}
        for field in fields:
            questions[field] = prompts.AC_DICT[field]

        # Use chunk if available and different from full context, otherwise use full context
        ############# Important Note #############
        # the above logic is changed
        # New Logic - chunk if available and different from full context, otherwise add the field to full context list
        if (
            field_group in ac_chunks
            and ac_chunks[field_group].strip()
            and ac_chunks[field_group] != contract_text
        ):

            context = ac_chunks[field_group]
            context = context[0 : min(100000, len(context)) - 1]
            context_type = "Smart Chunking"

            if len(fields) == 1:
                question = questions[
                    fields[0]
                ]  # Take the one question for the single field included
                prompt = prompts.AC_SINGLE_FIELD_TEMPLATE(context, question)
            elif len(fields) > 1:
                prompt = prompts.AC_MULTI_FIELD_TEMPLATE(context, questions)
            else:
                raise ValueError("Field group has no defined fields")

            try:
                if field_group == "AGREEMENT_NAME":

                    # context, prompt and answer based on first 5 pages
                    context_agreement = "\n".join(
                            [text_dict[str(page_num)] for page_num in text_dict.keys() if str(page_num) in ['1', '2', '3', '4', '5']]
                        )
                    prompt_agreement = prompts.AC_SINGLE_FIELD_TEMPLATE(context_agreement, question)
                    field_group_answer_raw = claude_funcs.invoke_claude(
                        prompt_agreement, config.MODEL_ID_CLAUDE35_SONNET, filename, 8192
                    )

                    # if answer is N/A use the smart-chunked context
                    if field_group_answer_raw == "N/A":
                        field_group_answer_raw = claude_funcs.invoke_claude(
                            prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 8192
                        )
                else:
                    field_group_answer_raw = claude_funcs.invoke_claude(
                        prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 8192
                    )

                if len(fields) == 1:
                    field_group_answer_dict = {field: field_group_answer_raw}
                elif len(fields) > 1:
                    field_group_answer_raw = field_group_answer_raw.replace("_DATE", "_DT")
                    field_group_answer_dict = ac_funcs.json_parsing_search(field_group_answer_raw, fields)

                # TODO: Validate field names coming out of Claude, check against fields in group, make sure they're all there, make sure there aren't any extras
                for field in fields:
                    field_answer = field_group_answer_dict[field]
                    ac_answers_dict[field] = field_answer


                log_data.append(
                    {
                        "Field": field,
                        "Prompt": f"Question: {questions}\n\nContext: {context}",  # Include full context
                        "Context Type": context_type,
                        "Response": field_answer,
                    }
                )

                chunk_data.append(
                    {
                        "Contract name": filename,
                        "Field group": field_group,
                        "Field list": fields,
                        "Methodology": ac_chunks[field_group + "_methodology"],
                        "Case sensitivity": ac_chunks[field_group + "_case"],
                        "Page list": ac_chunks[field_group + "_pages"],
                        "Chunk size": len(context),
                        "% Reduction": f"{100*(1-len(context)/len(contract_text)):0.2f}%",
                    }
                )
            except Exception as e:
                print(f"Error processing field {field}: {type(e)}, {str(e)}")
                ac_answers_dict[field] = f"Error: {str(e)}"
                log_data.append(
                    {
                        "Field": field,
                        "Prompt": f"Question: {questions}\n\nContext: {context}",  # Include full context
                        "Context Type": context_type,
                        "Response": f"Error: {str(e)}",
                    }
                )
                chunk_data.append(
                    {
                        "Contract name": filename,
                        "Field group": field_group,
                        "Field list": fields,
                        "Methodology": ac_chunks[field_group + "_methodology"],
                        "Case sensitivity": ac_chunks[field_group + "_case"],
                        "Page list": ac_chunks[field_group + "_pages"],
                        "Chunk size": len(context),
                        "% Reduction": f"{100*(1-len(context)/len(contract_text)):0.2f}%",
                    }
                )
        else:
            no_keyword_matches.extend(fields)

    ################## RUN FULL CONTEXT PROMPTS ##################
    # Process 'full_context' fields together

    smartly_chunked = []
    for field_groups in keywords.GROUPED_KEYWORD_MAPPINGS.keys():
        for fields in keywords.GROUPED_KEYWORD_MAPPINGS[field_groups]["fields"]:
            smartly_chunked.append(fields)
    

    # Instantiate full context fields from those that weren't smart chunked
    full_context_fields = [
        field for field in prompts.AC_DICT if field not in smartly_chunked
    ]
    
    # Add fields with no keyword matches to full context list
    if len(no_keyword_matches) > 0:
        full_context_fields.extend(no_keyword_matches)
        
    # Filter out fields that already have answers
    # e.g., some have already been filled out by regex chunking
    full_context_fields = [
        field for field in full_context_fields if field not in ac_answers_dict.keys()
    ]

    full_context_questions = {
        field: prompts.AC_DICT.get(field) for field in full_context_fields
    }

    if full_context_questions:
        full_context_prompt = prompts.AC_MULTI_FIELD_TEMPLATE(
            context=contract_text[0 : min(100000, len(contract_text) - 1)],
            questions=full_context_questions
        )

        try:
            full_context_answers = ac_funcs.get_ac_answer(
                full_context_prompt,
                filename,
                fields=list(full_context_questions.keys()),
            )

            ac_answers_dict.update(full_context_answers)

            for field, answer in full_context_answers.items():
                log_data.append(
                    {
                        "Field": field,
                        "Prompt": f"Question: {full_context_questions[field]}\n\nContext: {contract_text}",  # Include full context
                        "Context Type": "Full Context (High Accuracy)",
                        "Response": answer,
                    }
                )
        except Exception as e:
            print(f"Error processing high accuracy fields: {str(e)}")
            for field in full_context_questions.keys():
                ac_answers_dict[field] = f"Error: {str(e)}"
                log_data.append(
                    {
                        "Field": field,
                        "Prompt": f"Question: {full_context_questions[field]}\n\nContext: {contract_text}",  # Include full context
                        "Context Type": "Full Context (High Accuracy)",
                        "Response": f"Error: {str(e)}",
                    }
                )

    # Ensure all AC fields have a key, populated with N/A if not from prompt
    for key in prompts.AC_DICT.keys():
        if key not in ac_answers_dict:
            ac_answers_dict[key] = "N/A"

    ################## AC CONDITIONAL PROMPTS ##################
    # Non-Renewal - Days
    if ac_answers_dict["NON_RENEWAL_LANGUAGE"] != 'N/A':
        nrd_prompt = prompts.AC_SINGLE_FIELD_TEMPLATE(
            ac_answers_dict["NON_RENEWAL_LANGUAGE"], prompts.AC_DICT["NON_RENEWAL_DAYS"]
        )
        nrd_answer = claude_funcs.invoke_claude(
            nrd_prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 8192
        )
        ac_answers_dict["NON_RENEWAL_DAYS"] = nrd_answer
    else:
        ac_answers_dict["NON_RENEWAL_DAYS"] = 'N/A'

    
    # Clean Up Contract Effective Date
    if (
        utils.is_empty(ac_answers_dict["CONTRACT_EFFECTIVE_DT"])
        and "meridian" in ac_answers_dict["PAYER_NAME"].lower()
    ):
        date_prompt = prompts.AC_EFFECTIVE_DATE_CLEANUP(
            contract_text[0 : min(100000, len(contract_text)) - 1]
        )
        date_answer = claude_funcs.invoke_claude(
            date_prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 124
        )
        ac_answers_dict["CONTRACT_EFFECTIVE_DT"] = date_answer

    ################## CREATE OUTPUT DIRECTORIES ##################
    base_filename = os.path.splitext(filename)[0].strip()
    output_dir = os.path.join(config.OUTPUT_DIRECTORY, base_filename)
    os.makedirs(output_dir, exist_ok=True)

    ################## POSTPROCESS ##################

    ac_df = pd.DataFrame([ac_answers_dict])
    ac_df = postprocess.ac_postprocess(ac_df, text_dict, filename, num_pages)

    ################## WRITE TO OUTPUT ##################
    ac_df.to_csv(os.path.join(output_dir, config.AC_RESULTS_NAME), index=False)

    chunk_file_path = "chunk_log.csv"
    with open(chunk_file_path, "a", newline="", encoding="utf-8") as chunk_file:
        fieldnames = [
            "Contract name",
            "Field group",
            "Field list",
            "Methodology",
            "Case sensitivity",
            "Page list",
            "Chunk size",
            "% Reduction",
        ]
        writer = csv.DictWriter(chunk_file, fieldnames=fieldnames)
        writer.writeheader()
        writer.writerows(chunk_data)

    print(f"Chunk log written to {chunk_file_path}")

    return ac_answers_dict


def run_b_prompts(file_object):
    ################## INITIATE PROCESSING ##################
    filename, contract_text = file_object
    print(f"Processing B for {filename}...")

    ################## PREPROCESS ##################
    text_dict, top_sheet_dict, exhibit_pages, num_pages, ac_chunks = preprocess.preprocess(
        contract_text, filename, fields="b"
    )
    print(f"B Preprocessing Complete - {filename}")

    ################## RUN BOTTOM UP PROMPTS ##################
    bu_results = bottom_up_funcs.run_bottom_up(
        filename, text_dict
    )  # Returns list of dictionaries
    print(f"B Bottom Up Complete - {filename}")

    ################## RUN TOP DOWN PROMPTS ##################
    combined_results = top_down_funcs.run_top_down(
        filename, text_dict, bu_results, exhibit_pages
    )  # Returns list of dictionaries
    print(f"B Top Down Complete - {filename}")

    ################## RUN CONDITIONAL PROMPTS ##################
    conditional_results = conditional_funcs.run_conditional(
        combined_results, text_dict, filename
    )  # List of dictionaries
    print(f"B Conditional Prompts Complete - {filename}")

    ################## CREATE OUTPUT DIRECTORIES ##################
    base_filename = os.path.splitext(filename)[0].strip()
    output_dir = os.path.join(config.OUTPUT_DIRECTORY, base_filename)
    os.makedirs(output_dir, exist_ok=True)

    ################## WRITE UNPROCESSED OUTPUT ##################
    combined_df = pd.DataFrame(conditional_results)
    combined_df.to_csv(
        os.path.join(output_dir, config.UNPROCESSED_RESULTS_NAME), index=False
    )

    ################## RUN POSTPROCESSING ##################
    final_df = postprocess.b_postprocess(filename, combined_df, num_pages)
    print(f"B Postprocessing Complete - {filename} ")

    ################## WRITE FINAL ##################
    final_df.to_csv(os.path.join(output_dir, config.B_RESULTS_NAME), index=False)
    print(f"B Output Complete - {filename} ")