doczyai-pipelines/fieldExtraction/scripts/adhoc/cnc_ac1_rerun.py

import concurrent.futures
import traceback
import csv
import time
import sys
import random

random.seed(42)
import pandas as pd
import os
import re

import utils
import config
import claude_funcs
import file_processing
import tracking
import consolidate_output
import top_down_funcs
import valid
import ac_funcs
import preprocess
import prompts
import postprocess
import postprocessing_funcs
import preprocessing_funcs

individual_path = "output_individual/cnc_3_ac1"
clean_path = "reference_files/CNC-Batch 3 Output_File1.xlsx"


AC_DICT = {
    "CONTRACT_EFFECTIVE_DT": "What is the contract effective date mentioned in any of the following locations: the signatory section or the preamble of the agreement, or the start of the amendment ? Return this date converted to YYYY-MM-DD format.",
    "PROV_GROUP_NPI": "What is the group provider's national provider identifier number mentioned in the contract? This may be found on the signature page or on the roster. It consists of 10 digits. Only return these 10 digits.",
    "PROV_GROUP_TIN": "What is the Group provider's taxpayer identification number stated in the contract? This is a 9 digit long number typically with a hyphen after the first 2 digits. Return only this 9 digit number with a hyphen after the first 2 digits. Do not elaborate or add context.",
}

combined_mapping = {
    "Contract_Name": "Filename",
    "Agreement_Name_Contract_Title": "CONTRACT_TITLE",
    "Payer_Name": "PAYER_NAME",
    "Health_Plan_State": "HEALTH_PLAN_STATE",
    "Affiliate_Y_N": "AFFILIATION_CLAUSE_IND",
    "Credentialing_Application_Indicator": "CREDENTIALING_APP_IND",
    "Term_Clause": "TERM_CLAUSE",
    "Contract_Auto_Renewal_Indicator": "CONTRACT_AUTO_RENEWAL_IND",
    "Termination_Date": "CONTRACT_TERMINATION_DT",
    "Termination_Upon_Notice_Days": "TERMINATION_UPON_NOTICE",
    "Termination_With_Cause_Days": "TERMINATION_UPON_CAUSE",
    "Amend_Contract_Upon_notice_Flag_Y_N": "AMEND_CONTRACT_NOTICE_IND",
    "Timeframe_to_Object_Days": "TIME_TO_OBJECT",
    "Assignments_Clause_Y_N": "ASSIGNMENTS_CLAUSE_IND",
    "Contract_Effective_Date": "CONTRACT_EFFECTIVE_DT",
    "IRS_Num": "PROV_GROUP_TIN",
    "PROV_GROUP_NAME": "PROV_GROUP_NAME",
    "NPI_10_digits": "PROV_GROUP_NPI",
    "NPI_NAME": "NPI_NAME",
    "PROV_GROUP_TIN_SIGNATORY": "PROV_GROUP_TIN_SIGNATORY",
    "PROV_TIN_OTHER": "PROV_TIN_OTHER",
    "PROV_NPI_OTHER": "PROV_NPI_OTHER",
    "Notice_to_Provider_Name": "NOTICE_PROVIDER_NAME",
    "Notice_to_Provider_Address": "NOTICE_PROVIDER_ADDRESS",
    "Sequestration_Language": "SEQUESTRATION_LANGUAGE",
    "Sequestration_Reductions_included_Medicare_only_Y_N": "SEQUESTRATION_REDUCTIONS_IND",
    "Parent_Agreement_Code": "Parent Agreement Code",
    "Pages": "Pages",
    "Page_num": "page_num",
    "Attachment_Exhibit": "EXHIBIT",
    "Line_of_Business": "CONTRACT_LOB",
    "Provider_Type": "PROV_TYPE",
    "Provider_Type_Level_2": "PROV_TYPE_LEVEL_2",
    "IP_OP": "IP_OP",
    "Service_Type": "FULL_SERVICE",
    "Plan_Type": "CONTRACT_PROGRAM",
    "Lesser_of_Logic_Language_included_Y_N": "LESSER",
    "Lesser_of_Rate": "LESSER_RATE",
    "Reimb_Methodology": "FULL_METHODOLOGY",
    "Reimb_Methodology_short": "SHORT_METHODOLOGY",
    "If_rate_is_percent_of_Payer_or_MCR_STANDARD": "RATE_STANDARD",
    "If_rate_is_percent_of_Payer_or_MCR_STANDARD_Short": "RATE_SHORT",
    "Flat_Fee": "FLAT_FEE_STANDARD",
    "Default_Term": "DEFAULT_TERM",
    "Default_Rate": "DEFAULT_RATE",
    "Inclusion_of_essential_RBRVS_Fee_Source_Language_Y_N": 'Inclusion of essential RBRVS "Fee Source" Language (Y/N)',
    "CDM_Language_included_Y_N": "CDM_IND",
    "CHARGEMASTER": "CHARGEMASTER",
    "IP_DSH_IME_UC_included_Y_N": "IP - DSH/IME/UC, included (Y/N)",
    "IP_Stoploss_Catastrophic_Threshold": "IP - Stoploss Catastrophic Threshold",
    "EXCLUSIONS": "EXCLUSIONS",
    "Not_to_Exceed": "NOT_TO_EXCEED",
    "Escalator_or_COLA_Y_N": "RATE_ESCALATOR_IND",
    "Escalator_I_Eff_Date": "RATE_ESCALATOR_DT",
}

import ac_funcs
import config
import postprocess
import tracking


def ac_postprocess(final_df):
    final_df = postprocessing_funcs.clean_ac_fields(final_df)
    final_df = final_df.rename(columns=valid.AC_MAPPING)
    final_df = final_df[[col for col in valid.ABC_COLUMNS if col in final_df.columns]]
    return final_df


def run_ac_prompts(file_object):
    ################## INITIATE PROCESSING ##################
    filename, contract_text = file_object
    print(f"Processing AC for {filename}...")
    # print(f"Total contract word count: {len(contract_text.split())}")

    ################## PREPROCESS ##################
    text_dict, exhibit_pages, num_pages, ac_chunks = preprocess.preprocess(
        contract_text, filename, fields="ac"
    )
    # print(f"AC Preprocessing Complete - {filename}")

    ################## RUN FULL CONTEXT PROMPTS ##################
    ac_dict = {}
    log_data = []

    # Process 'full_context' fields together
    full_context_fields = [field for field in AC_DICT]
    full_context_questions = AC_DICT

    if full_context_questions:
        full_context_prompt = ac_funcs.create_prompt(
            contract_text[0 : min(len(contract_text), 600000)],
            question=full_context_questions,
        )
        # print(f"High accuracy prompt word count: {len(high_accuracy_prompt.split())}")
        try:
            full_context_answers = ac_funcs.get_ac_answer(
                full_context_prompt,
                filename,
                fields=list(full_context_questions.keys()),
            )
            ac_dict.update(full_context_answers)

            for field, answer in full_context_answers.items():
                log_data.append(
                    {
                        "Field": field,
                        "Prompt": f"Question: {full_context_questions[field]}\n\nContext: {contract_text}",  # Include full context
                        "Context Type": "Full Context (High Accuracy)",
                        "Response": answer,
                    }
                )
        except Exception as e:
            # print(f"Error processing high accuracy fields: {str(e)}")
            for field in full_context_questions.keys():
                ac_dict[field] = f"Error: {str(e)}"
                log_data.append(
                    {
                        "Field": field,
                        "Prompt": f"Question: {full_context_questions[field]}\n\nContext: {contract_text}",  # Include full context
                        "Context Type": "Full Context (High Accuracy)",
                        "Response": f"Error: {str(e)}",
                    }
                )

    ################## CREATE OUTPUT DIRECTORIES ##################
    base_filename = os.path.splitext(filename)[0].strip()
    output_dir = os.path.join(individual_path, base_filename)
    os.makedirs(output_dir, exist_ok=True)

    ################## POSTPROCESS ##################
    ac_dict["Contract Name"] = filename
    ac_df = pd.DataFrame([ac_dict])
    ac_df = ac_postprocess(ac_df)

    ################## WRITE TO OUTPUT ##################
    ac_df.to_csv(f"{output_dir}/ac_output.csv", index=False)
    print(f"AC Output written: {filename}")


def process_ac_adhoc(item):
    filename, contract_text = item
    try:
        results = run_ac_prompts(item)
        if results is not None:
            tracking.update_results_csv(item, results)
            return item, results
    except Exception as e:
        print(f"Error processing item {filename}: {e}")
        traceback.print_exc()


########################## Process Starts Here ##########################

# Clean
abc = pd.read_excel(clean_path)
abc.rename(columns=combined_mapping, inplace=True)
abc_names = list(abc["Contract Name"].unique())
abc_dicts = abc.to_dict(orient="records")

# Input
input_dict = utils.read_input("data_cnc/batch3A")

# Current Output
output_dirs = list(os.listdir(individual_path))

# Run only files that are not in abc, are in input, and are not in output
input_dict = {
    k: v
    for k, v in input_dict.items()
    if k.split(".txt")[0] not in abc_names and k.split(".txt")[0] not in output_dirs
}
print("Input dict to run: ", len(input_dict))


with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor:
    if "a" in config.FIELDS and "c" in config.FIELDS:
        futures = [
            executor.submit(process_ac_adhoc, item) for item in input_dict.items()
        ]

    for future in concurrent.futures.as_completed(futures):
        try:
            result = future.result()
        except Exception as e:
            print(f"Error in future: {e}")
            traceback.print_exc()

# for item in input_dict.items():
#     process_ac_adhoc(item)