Michael McGuinness
592 lines
29 KiB
Python
592 lines
29 KiB
Python
import concurrent.futures
|
||
import traceback
|
||
import csv
|
||
import time
|
||
import sys
|
||
import random
|
||
|
||
random.seed(42)
|
||
import pandas as pd
|
||
import os
|
||
import re
|
||
|
||
import utils
|
||
import config
|
||
import claude_funcs
|
||
import file_processing
|
||
import tracking
|
||
import consolidate_output
|
||
import top_down_funcs
|
||
import valid
|
||
import ac_funcs
|
||
import preprocess
|
||
import prompts
|
||
import postprocess
|
||
import postprocessing_funcs
|
||
import preprocessing_funcs
|
||
|
||
input_path = "data_cnc/batch3"
|
||
ac2_output_path = "output_individual/cnc_3_ac2"
|
||
b_output_path = "output_individual/cnc_3_b"
|
||
clean_path = "reference_files/CNC-Batch 3 Output_File1.xlsx"
|
||
run_path = "reference_files/Batch 3_091124.xlsx"
|
||
|
||
|
||
# Ad-Hoc runs of sections of the script
|
||
"""
|
||
Batch 1 and 2
|
||
Fields to Run:
|
||
AC
|
||
- All Chunked Fields
|
||
- All Part 2 Fields
|
||
- Postprocess AC
|
||
|
||
B
|
||
- All B Postprocessing
|
||
|
||
750 additional
|
||
- Run all ABC
|
||
"""
|
||
dup_mapping = {
|
||
"Payer Name": "PAYER_NAME",
|
||
"IRS Name": "IRS_Name",
|
||
"NPI NAME": "NPI_NAME",
|
||
"Sequestration Reductions, included [Medicare only] (Y/N)": "SEQUESTRATION_REDUCTIONS_IND",
|
||
"Page_num": "page_num",
|
||
"Lesser of Logic Language, included (Y/N)": "LESSER",
|
||
"Reimb. Methodology_short": "SHORT_METHODOLOGY",
|
||
"If rate is % of Payer or MCR [STANDARD]": "RATE_STANDARD",
|
||
"If rate is % of Payer or MCR [STANDARD]_Short": "RATE_SHORT",
|
||
"Flat Fee": "FLAT_FEE_STANDARD",
|
||
"CDM Language, included (Y/N)": "CDM_IND",
|
||
"CHARGEMASTER": "CONTRACT_CHARGEMASTER_PROTECTION_LANGUAGE",
|
||
"EXCLUSIONS": "Exclusions",
|
||
}
|
||
|
||
combined_mapping = {
|
||
"Contract_Name": "Filename",
|
||
"Agreement_Name_Contract_Title": "CONTRACT_TITLE",
|
||
"Payer_Name": "PAYER_NAME",
|
||
"Health_Plan_State": "HEALTH_PLAN_STATE",
|
||
"Affiliate_Y_N": "AFFILIATION_CLAUSE_IND",
|
||
"Credentialing_Application_Indicator": "CREDENTIALING_APP_IND",
|
||
"Term_Clause": "TERM_CLAUSE",
|
||
"Contract_Auto_Renewal_Indicator": "CONTRACT_AUTO_RENEWAL_IND",
|
||
"Termination_Date": "CONTRACT_TERMINATION_DT",
|
||
"Termination_Upon_Notice_Days": "TERMINATION_UPON_NOTICE",
|
||
"Termination_With_Cause_Days": "TERMINATION_UPON_CAUSE",
|
||
"Amend_Contract_Upon_notice_Flag_Y_N": "AMEND_CONTRACT_NOTICE_IND",
|
||
"Timeframe_to_Object_Days": "TIME_TO_OBJECT",
|
||
"Assignments_Clause_Y_N": "ASSIGNMENTS_CLAUSE_IND",
|
||
"Contract_Effective_Date": "CONTRACT_EFFECTIVE_DT",
|
||
"IRS_Num": "PROV_GROUP_TIN",
|
||
"PROV_GROUP_NAME": "PROV_GROUP_NAME",
|
||
"NPI_10_digits": "PROV_GROUP_NPI",
|
||
"NPI_NAME": "NPI_NAME",
|
||
"PROV_GROUP_TIN_SIGNATORY": "PROV_GROUP_TIN_SIGNATORY",
|
||
"PROV_TIN_OTHER": "PROV_TIN_OTHER",
|
||
"PROV_NPI_OTHER": "PROV_NPI_OTHER",
|
||
"Notice_to_Provider_Name": "NOTICE_PROVIDER_NAME",
|
||
"Notice_to_Provider_Address": "NOTICE_PROVIDER_ADDRESS",
|
||
"Sequestration_Language": "SEQUESTRATION_LANGUAGE",
|
||
"Sequestration Reductions, included [Medicare only] (Y/N)": "SEQUESTRATION_REDUCTIONS_IND",
|
||
"Parent_Agreement_Code": "Parent Agreement Code",
|
||
"Pages": "Pages",
|
||
"Page_num": "page_num",
|
||
"Attachment_Exhibit": "EXHIBIT",
|
||
"Line_of_Business": "CONTRACT_LOB",
|
||
"Provider_Type": "PROV_TYPE",
|
||
"Provider_Type_Level_2": "PROV_TYPE_LEVEL_2",
|
||
"IP_OP": "IP_OP",
|
||
"Service_Type": "FULL_SERVICE",
|
||
"Plan_Type": "CONTRACT_PROGRAM",
|
||
"Lesser_of_Logic_Language_included_Y_N": "LESSER",
|
||
"Lesser_of_Rate": "LESSER_RATE",
|
||
"Reimb_Methodology": "FULL_METHODOLOGY",
|
||
"Reimb_Methodology_short": "SHORT_METHODOLOGY",
|
||
"If rate is % of Payer or MCR [STANDARD]": "RATE_STANDARD",
|
||
"If rate is % of Payer or MCR [STANDARD]_Short": "RATE_SHORT",
|
||
"Flat Fee": "FLAT_FEE_STANDARD",
|
||
"Default_Term": "DEFAULT_TERM",
|
||
"Default_Rate": "DEFAULT_RATE",
|
||
"IP - DSH/IME/UC, included (Y/N)": 'Inclusion of essential RBRVS "Fee Source" Language (Y/N)',
|
||
"CDM_Language_included_Y_N": "CDM_IND",
|
||
"CHARGEMASTER": "CHARGEMASTER",
|
||
"IP_DSH_IME_UC_included_Y_N": "IP - DSH/IME/UC, included (Y/N)",
|
||
"IP_Stoploss_Catastrophic_Threshold": "IP - Stoploss Catastrophic Threshold",
|
||
"EXCLUSIONS": "EXCLUSIONS",
|
||
"Not_to_Exceed": "NOT_TO_EXCEED",
|
||
"Escalator_or_COLA_Y_N": "RATE_ESCALATOR_IND",
|
||
"Escalator_I_Eff_Date": "RATE_ESCALATOR_DT",
|
||
}
|
||
|
||
|
||
AC_DICT = {
|
||
"ACCESS_TO_MEDICAL_RECORDS": "Access clause contains information regarding access to medical records. It may be found in section 4.2 of the contract. Extract the Access clause present in the contract. If it is not present, answer N/A.",
|
||
"CARVEOUT_VENDORS": "Carve-Out Vendors clause may be found in section 2.9 of the contract. Extract the Carve-Out Vendors clause present in the contract. If it is not present, answer N/A.",
|
||
"CLAIMS_EDITING_LANGUAGE": "Claims Editing Language is typically found in the claims section in the beginning parts of the contract. Extract Claims Editing Language present in the contract. If it is not present, answer N/A.",
|
||
"CLEAN_CLAIM": 'Extract the "Clean Claim" subsection present in definition section of the contract. If it is not present, answer N/A.',
|
||
"CONFLICTS_BETWEEN_CERTAIN_DOCUMENTS_LANGUAGE": "Extract the Conflicts Between Certain Documents clause present in the contract. If it is not present, answer N/A.",
|
||
"COST_SETTLEMENT_LANGUAGE": "Cost settlement language may be present in disputes and arbitration subsection of the contract. It usually contains keywords like settlementor cost settlement. Extract the cost settlement language present in the contract. If it is not present, answer N/A.",
|
||
"DEEMER_AMENDMENT": "Deemer Amendment clause may be found in section 8.7.2 of the contract. Extract the Deemer Amendment clause present in the contract. If it is not present, answer N/A.",
|
||
"DELEGATED_TERMS": "Extract Delegated Terms present in the contract. If it is not present, answer N/A.",
|
||
"ECM": 'What is the "ECM" number mentioned in the contract? Only Answer the ECM number, do not provide any context. Do not provide ICM number. If ECM number is not present, answer N/A.',
|
||
"EXCLUSIVITY_REQUIREMENT_LANGUAGE": "Extract Exclusivity requirement clause present in the contract. If it is not present, answer N/A.",
|
||
"GUARANTEE_OF_PROVIDER_YIELD_LANGUAGE": "Extract Guarantee of Provider Yield Language present in the contract. If it is not present, answer N/A.",
|
||
"HCBS_SERVICES": 'Extract HCBS services clause present either in the compensation or exhibit section of the contract. It may contain keywords like "for covered HCBS services, plan shall pay". Do not include definitions of HCBS. If it is not present, answer N/A.',
|
||
"INDEMNIFICATION": "Indemnification clause may be found in section 5.2 of the contract. Extract the Indemnification clause present in the contract. If it is not present, answer N/A.",
|
||
"INDEPENDENT_REVIEW_LANGUAGE": "Extract Independent Review language present in the contract. If it is not present, answer N/A.",
|
||
"INVOICE_PRICING_LANGUAGE": 'Extract table with heading "INVOICED SERVICES" if it is present in compensation schedule of the contract. If it is not present, answer N/A',
|
||
"LATE_PAID_CLAIMS_LANGUAGE": "Extract Late Paid Claims language present in the contract. This will be a sentence or paragraph describing late paid claims, interest payments or similar. Here is an example of a correct response: 'Any Clean Claim, as defined in 42 C.F.R. § 422.500, shall be paid within thirty (30) days of receipt by Plan at such address as may be designated by Plan, and Plan shall pay interest on any Clean Claim not paid within thirty (30) days of such receipt by Plan at the rate of interest required by law, or as otherwise set forth in the Provider Manual.'. Here is an example of an incorrect response: 'Any Clean Claim, as defined in 42 C.F.R. 422.500, shall be paid within thirty (30) days of receipt by Health Plan, Payor or (if Provider contracts with Downstream Entities) Provider, as applicable, as designated by Provider or such Downstream Entity, as applicable.'. If no valid answer is present, answer N/A.",
|
||
"MEMBER_CONFINEMENT_DAYS_LANGUAGE": "Extract Member Confinement Days Language present in the contract. If it is not present, answer N/A.",
|
||
"NATIONAL_AGREEMENT_IND": "Are more than one states covered in the contract? Answer with a Yes or No, do not provide any other context.",
|
||
"NETWORK_ACCESS_FEES_LANGUAGE": 'Extract "Network Access Fee" verbiage present in the contract. If it is not present, answer N/A.',
|
||
"NONSTANDARD_APPEALS_PROCESS_LANGUAGE": "Extract Nonstandard Appeals Process language present in the contract. If it is not present, answer N/A.",
|
||
"PAYMENT_IN_ADVANCE_OF_CLAIMS_SUBMISSION_LANGUAGE": "Extract Payment in Advance of Claims Submission language present in the contract. If it is not present, answer N/A.",
|
||
"PAYOR": '"Payor" is usually found in section 1.12 of the contract. Extract the "Payor" subsection present in definition section of the contract. If it is not present, answer N/A.',
|
||
"PMPM": "What is the PMPM rate mentioned in the contract? Only return the rate. do not provide any context. If it is not present, answer N/A.",
|
||
"PREAUTHORIZATION": "Preauthorization clause may be found in section 2.7 of the contract. Extract the Preauthorization clause present in the contract. If it is not present, answer N/A.",
|
||
"PRODUCT_REMOVAL": "Product removal may be found in the products attachment, stating a product is being removed. Extract product removal language present in the contract. If it is not present, answer N/A.",
|
||
"PROVIDER_BASED_BILLING_EXCLUSION_LANGUAGE": "Provider-based Billing Exclusion is often listed under the ‘Additional Provisions’ section. Extract the Provider-based Billing Exclusion language mentioned in the contract. If it is not present, answer N/A.",
|
||
"RECOVERY_RIGHTS": "Recovery Rights clause may be found in section 3.5 of the contract. Extract the entire Recovery Rights sentence or paragraph present in the contract. If it is not present, answer N/A.",
|
||
"TEMPLATE": "Is this contract in standard Centene template containing definition, terms clauses and other standard sections. Answer with a Yes or No, do not provide any other context.",
|
||
"ARBITRATION_AND_DISPUTES": "Dispute Resolution clause is usually found in section 6.1 of the contract. Arbitration clause is usually found in section 6.2 of the contract. Extract both the Dispute Resolution and the arbitration clause present in the contract. In case one of them is present, extract that one. If none of them are present, answer N/A.",
|
||
"DISPARAGEMENT_PROHIBITION_IND": "Is Disparagement Prohibition clause present in the contract? It may be found in section 2.1 of the contract. Answer with a Yes or No, do not provide any other context.",
|
||
"DISPARAGEMENT_PROHIBITION_LANGUAGE": "Disparagement Prohibition clause may be found in section 2.1 of the contract. Extract the Disparagement Prohibition clause present in the contract. If it is not present, answer N/A.",
|
||
"ELIGIBILITY_VERIFICATION": "Eligibility Verification or Determination clause may be found in section 2.6 of the contract. Extract the Eligibility Verification or Determination clause present in the contract. If it is not present, answer N/A.",
|
||
"INSURANCE_REQUIREMENT": "Extract Insurance requirement mentioned in the contract. If it is not present, answer N/A.",
|
||
"PARTICIPATION_IN_PRODUCTS": "Participation in Products clause may be found in section 2.2.2 of the contract. Extract the Participation in Products clause present in the contract. If it is not present, answer N/A.",
|
||
"POLICIES_AND_PROCEDURES": "Extract the entire Policies and Procedures sentence or paragraph present in the contract. Do not extract language about conflicts and construction. Do not extract language about PP (Preferred Provider) nor PPG (Preferred Provider Group). If it is not present, answer N/A.",
|
||
"REGULATORY_REQUIREMENTS": 'Extract the entire Regulatory Requirements paragraph present in the contract. It usually contains the keyword "Regulatory Requirements". If it is not present, answer N/A.',
|
||
"RELATIONSHIP_OF_PARTIES_LANGUAGE": "Extract the entire Relationship of Parties passage present in the contract. If it is not present, answer N/A.",
|
||
"TERM_CLAUSE": """Extract the subsection or paragraph labeled Term. It may also be labeled Termination. If there are no such sections, extract the paragraph the most closely resembles this label. If there is still no correct answer, simply return 'N/A'.""",
|
||
"LATE_PAID_CLAIMS_LANGUAGE": "Extract Late Paid Claims language present in the contract. This will be a sentence or paragraph describing late paid claims, interest payments or similar. Here is an example of a correct response: 'Any Clean Claim, as defined in 42 C.F.R. § 422.500, shall be paid within thirty (30) days of receipt by Plan at such address as may be designated by Plan, and Plan shall pay interest on any Clean Claim not paid within thirty (30) days of such receipt by Plan at the rate of interest required by law, or as otherwise set forth in the Provider Manual.'. Here is an example of an incorrect response: 'Any Clean Claim, as defined in 42 C.F.R. 422.500, shall be paid within thirty (30) days of receipt by Health Plan, Payor or (if Provider contracts with Downstream Entities) Provider, as applicable, as designated by Provider or such Downstream Entity, as applicable.'. If no valid answer is present, answer N/A.",
|
||
"NON_RENEWAL_LANGUAGE": "Extract the section of the contract that describes non-renewal obligations, especially the number of days notice that must be given. Write the full relevant sentence or paragraph. Ensure that the exact term 'non-renewal' is present in the answer. Do NOT return an answer that does not have 'non-renewal'.",
|
||
"HCBS_SERVICES": 'Extract HCBS services clause present either in the compensation or exhibit section of the contract. It may contain keywords like "for covered HCBS services, plan shall pay". Do not include definitions of HCBS. If it is not present, answer N/A.',
|
||
}
|
||
|
||
KEYWORD_MAPPINGS = {
|
||
"TERM_CLAUSE": {
|
||
"methodology": "hierarchy",
|
||
"keywords": ["Term", "term", "duration", "agreement period"],
|
||
},
|
||
"LATE_PAID_CLAIMS_LANGUAGE": {
|
||
"methodology": "hierarchy",
|
||
"keywords": [
|
||
"late paid",
|
||
"late payment",
|
||
"interest",
|
||
"late fee",
|
||
"interest shall be paid",
|
||
"interest payment",
|
||
],
|
||
},
|
||
"NON_RENEWAL_LANGUAGE": {
|
||
"methodology": "or",
|
||
"keywords": ["not to renew", "non-renewal", "non renewal", "nonrenewal"],
|
||
},
|
||
# "NON_RENEWAL_DAYS": {'methodology': 'or', 'keywords': ["not to renew", "non-renewal", "non renewal", "nonrenewal"]},
|
||
"HCBS_SERVICES": {
|
||
"methodology": "hierarchy",
|
||
"keywords": ["%", "Plan shall pay", "hcbs"],
|
||
},
|
||
"POLICIES_AND_PROCEDURES": {
|
||
"methodology": "or",
|
||
"keywords": ["2.4", "policies", "procedures"],
|
||
},
|
||
"REGULATORY_REQUIREMENTS": {
|
||
"methodology": "or",
|
||
"keywords": ["8.3", "6.4", "regulatory requirements", "regulatory"],
|
||
},
|
||
"RECOVERY_RIGHTS": {"methodology": "or", "keywords": ["3.5", "recovery rights"]},
|
||
"RELATIONSHIP_OF_PARTIES_LANGUAGE": {
|
||
"methodology": "or",
|
||
"keywords": ["8.1", "relationship of parties"],
|
||
},
|
||
}
|
||
|
||
|
||
def run_ac2_prompts(file_object):
|
||
################## INITIATE PROCESSING ##################
|
||
filename, contract_text = file_object
|
||
print(f"Processing AC for {filename}...")
|
||
# print(f"Total contract word count: {len(contract_text.split())}")
|
||
|
||
################## PREPROCESS ##################
|
||
text_dict, exhibit_pages, num_pages, ac_chunks = preprocess.preprocess(
|
||
contract_text, filename, fields="ac"
|
||
)
|
||
# print(f"AC Preprocessing Complete - {filename}")
|
||
|
||
################## RUN FULL CONTEXT PROMPTS ##################
|
||
ac_dict = {}
|
||
log_data = []
|
||
|
||
# Process 'full_context' fields together
|
||
full_context_fields = [field for field in AC_DICT if field not in KEYWORD_MAPPINGS]
|
||
full_context_questions = {
|
||
field: prompts.AC_DICT.get(field) for field in full_context_fields
|
||
}
|
||
|
||
if full_context_questions:
|
||
full_context_prompt = ac_funcs.create_prompt(
|
||
contract_text[0 : min(len(contract_text), 600000)],
|
||
question=full_context_questions,
|
||
)
|
||
# print(f"High accuracy prompt word count: {len(high_accuracy_prompt.split())}")
|
||
try:
|
||
full_context_answers = ac_funcs.get_ac_answer(
|
||
full_context_prompt,
|
||
filename,
|
||
fields=list(full_context_questions.keys()),
|
||
)
|
||
ac_dict.update(full_context_answers)
|
||
|
||
for field, answer in full_context_answers.items():
|
||
log_data.append(
|
||
{
|
||
"Field": field,
|
||
"Prompt": f"Question: {full_context_questions[field]}\n\nContext: {contract_text}", # Include full context
|
||
"Context Type": "Full Context (High Accuracy)",
|
||
"Response": answer,
|
||
}
|
||
)
|
||
except Exception as e:
|
||
# print(f"Error processing high accuracy fields: {str(e)}")
|
||
for field in full_context_questions.keys():
|
||
ac_dict[field] = f"Error: {str(e)}"
|
||
log_data.append(
|
||
{
|
||
"Field": field,
|
||
"Prompt": f"Question: {full_context_questions[field]}\n\nContext: {contract_text}", # Include full context
|
||
"Context Type": "Full Context (High Accuracy)",
|
||
"Response": f"Error: {str(e)}",
|
||
}
|
||
)
|
||
|
||
################## RUN CHUNKED PROMPTS ##################
|
||
all_fields = set(prompts.AC_DICT.keys())
|
||
chunked_fields = [field for field in KEYWORD_MAPPINGS]
|
||
|
||
for field in chunked_fields:
|
||
if field in prompts.AC_DICT:
|
||
question = prompts.AC_DICT[field]
|
||
else:
|
||
# print(f"Field {field} not found in prompts. Skipping...")
|
||
continue
|
||
|
||
# Use chunk if available and different from full context, otherwise use full context
|
||
if (
|
||
field in ac_chunks
|
||
and ac_chunks[field].strip()
|
||
and ac_chunks[field] != contract_text
|
||
):
|
||
context = ac_chunks[field]
|
||
context_type = "Smart Chunking"
|
||
else:
|
||
context = contract_text
|
||
context_type = "Full Context"
|
||
|
||
prompt = prompts.AC_SINGLE_FIELD_TEMPLATE(context, question)
|
||
|
||
# print(f"Field {field} prompt word count: {len(prompt.split())} ({context_type})")
|
||
|
||
try:
|
||
field_answer = claude_funcs.invoke_claude(
|
||
prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 8192
|
||
)
|
||
print(field, field_answer)
|
||
ac_dict[field] = field_answer
|
||
|
||
log_data.append(
|
||
{
|
||
"Field": field,
|
||
"Prompt": f"Question: {question}\n\nContext: {context}", # Include full context
|
||
"Context Type": context_type,
|
||
"Response": field_answer,
|
||
}
|
||
)
|
||
except Exception as e:
|
||
# print(f"Error processing field {field}: {str(e)}")
|
||
ac_dict[field] = f"Error: {str(e)}"
|
||
log_data.append(
|
||
{
|
||
"Field": field,
|
||
"Prompt": f"Question: {question}\n\nContext: {context}", # Include full context
|
||
"Context Type": context_type,
|
||
"Response": f"Error: {str(e)}",
|
||
}
|
||
)
|
||
|
||
################## AC CONDITIONAL PROMPTS ##################
|
||
## Non-Renewal - Days
|
||
nrd_prompt = prompts.AC_SINGLE_FIELD_TEMPLATE(
|
||
ac_dict["NON_RENEWAL_LANGUAGE"], prompts.AC_DICT["NON_RENEWAL_DAYS"]
|
||
)
|
||
nrd_answer = claude_funcs.invoke_claude(
|
||
nrd_prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 8192
|
||
)
|
||
ac_dict["NON_RENEWAL_DAYS"] = nrd_answer
|
||
|
||
################## ENSURE ALL FIELDS ARE PRESENT ##################
|
||
ac_dict = {k: v for k, v in ac_dict.items() if k in valid.AC_MAPPING}
|
||
for field in all_fields:
|
||
if field not in ac_dict:
|
||
# print(f"Field not found in results. {field}")
|
||
# ac_dict[field] = "N/A"
|
||
log_data.append(
|
||
{
|
||
"Field": field,
|
||
"Prompt": "N/A",
|
||
"Context Type": "N/A",
|
||
"Response": "N/A",
|
||
}
|
||
)
|
||
|
||
################## CREATE OUTPUT DIRECTORIES ##################
|
||
base_filename = os.path.splitext(filename)[0].strip()
|
||
output_dir = os.path.join(ac2_output_path, base_filename)
|
||
os.makedirs(output_dir, exist_ok=True)
|
||
|
||
################## POSTPROCESS ##################
|
||
ac_dict["Contract Name"] = filename
|
||
ac_df = pd.DataFrame([ac_dict])
|
||
ac_df = postprocess.ac_postprocess(ac_df)
|
||
|
||
################## WRITE TO OUTPUT ##################
|
||
ac_df.to_csv(f"{output_dir}/ac_output.csv", index=False)
|
||
print(f"AC Output written: {filename}")
|
||
|
||
################## WRITE LOG TO CSV ##################
|
||
# log_file_path = os.path.join(output_dir, f"{base_filename}_prompt_log.csv")
|
||
# with open(log_file_path, 'w', newline='', encoding='utf-8') as log_file:
|
||
# fieldnames = ['Field', 'Prompt', 'Context Type', 'Response']
|
||
# writer = csv.DictWriter(log_file, fieldnames=fieldnames)
|
||
# writer.writeheader()
|
||
# writer.writerows(log_data)
|
||
# print(f"Prompt log written to {log_file_path}")
|
||
return ac_dict
|
||
|
||
|
||
def process_ac2_adhoc(item):
|
||
filename, contract_text = item
|
||
try:
|
||
results = run_ac2_prompts(item)
|
||
if results is not None:
|
||
tracking.update_results_csv(item, results)
|
||
return item, results
|
||
except Exception as e:
|
||
print(f"Error processing item {filename}: {e}")
|
||
traceback.print_exc()
|
||
|
||
|
||
def preprocess_ad_hoc(contract_text, filename, fields=config.FIELDS):
|
||
|
||
contract_text = preprocessing_funcs.clean_newlines(contract_text)
|
||
contract_text = preprocessing_funcs.clean_law_symbols(contract_text)
|
||
|
||
text_dict = preprocessing_funcs.split_text(
|
||
contract_text
|
||
) # return a dictionary with keys - page_num (str), values as the page_text
|
||
|
||
num_pages = len(text_dict.keys()) - 1
|
||
text_dict = preprocessing_funcs.filter_quick_review(text_dict)
|
||
|
||
if fields == "b":
|
||
exhibit_pages = top_down_funcs.get_exhibit_pages(
|
||
text_dict, filename
|
||
) # All pages with exhibit headers
|
||
# text_dict = table_funcs.align_and_format_tables(text_dict, filename)
|
||
text_dict = preprocessing_funcs.chunk_consecutive(text_dict, exhibit_pages)
|
||
ac_chunks = ""
|
||
if fields == "ac":
|
||
exhibit_pages = []
|
||
ac_chunks = preprocessing_funcs.smart_chunk_ac(text_dict)
|
||
|
||
return text_dict, exhibit_pages, num_pages, ac_chunks
|
||
|
||
|
||
def get_add_on(final_dicts, text_dict):
|
||
add_on_dict = {}
|
||
for td_page in list(set([d["page_num"] for d in final_dicts])):
|
||
td_page = str(td_page).split(".")[0]
|
||
print(f"TD Page : {td_page}")
|
||
# TD page is the original result top down page, it is not necessarily in the new text_dict.
|
||
# Find add_on_page = the key of the new text_dict that points to the correct page of the old text dict
|
||
|
||
if str(td_page) in list(text_dict.keys()):
|
||
add_on_page = td_page
|
||
else:
|
||
add_on_page = [
|
||
str(page) for page in text_dict.keys() if float(page) > float(td_page)
|
||
]
|
||
print(add_on_page)
|
||
|
||
if not add_on_page:
|
||
add_on_dict[td_page] = {
|
||
"ADD_ON_REIMBURSEMENT_LANGUAGE": "N/A",
|
||
"ADD_ON_REIMBURSEMENT_IND": "N",
|
||
}
|
||
continue
|
||
elif not isinstance(add_on_page, str):
|
||
add_on_page = add_on_page[0]
|
||
|
||
ad = {}
|
||
|
||
add_on_prompt = f"## Page Start ## {text_dict[str(add_on_page)]} ## Page End ## . Extract the Add On Reimbursement language present in the page above. If there is no Add On language present, answer N/A. Write only the answer, with no additional commentary or explanation."
|
||
add_on_answer = claude_funcs.invoke_claude(
|
||
add_on_prompt, config.MODEL_ID_CLAUDE35_SONNET, "", 256
|
||
)
|
||
|
||
ad["ADD_ON_REIMBURSEMENT_LANGUAGE"] = add_on_answer
|
||
|
||
if "N/A" in ad["ADD_ON_REIMBURSEMENT_LANGUAGE"]:
|
||
ad["ADD_ON_REIMBURSEMENT_IND"] = "N"
|
||
else:
|
||
ad["ADD_ON_REIMBURSEMENT_IND"] = "Y"
|
||
add_on_dict[str(td_page)] = ad
|
||
|
||
return add_on_dict
|
||
|
||
|
||
def process_b_adhoc(item):
|
||
filename, contract_text = item
|
||
print(f"Processing {filename}")
|
||
|
||
merged_dicts = [d for d in abc_dicts if d["Filename"] + ".txt" == filename]
|
||
print(len(merged_dicts))
|
||
|
||
if len(merged_dicts) > 0:
|
||
|
||
text_dict, exhibit_pages, num_pages, ac_chunks = preprocess_ad_hoc(
|
||
contract_text, filename
|
||
)
|
||
print(f"Preprocessing {filename}")
|
||
|
||
# Exclusions
|
||
exclusion_dict = top_down_funcs.get_td_dict(
|
||
text_dict, filename, prompts.TOP_DOWN_EXCLUSIONS, valid.EXCLUSION_TERMS
|
||
)
|
||
final_dicts = top_down_funcs.add_exclusions(merged_dicts, exclusion_dict)
|
||
print(f"Exclusions - {filename}")
|
||
|
||
# Medically Necessary
|
||
medically_necessary_dict = top_down_funcs.get_td_dict(
|
||
text_dict,
|
||
filename,
|
||
prompts.TOP_DOWN_MEDICALLY_NECESSARY,
|
||
valid.VALID_MEDICALLY_NECESSARY,
|
||
)
|
||
medically_necessary_dict = postprocessing_funcs.filter_dict(
|
||
medically_necessary_dict,
|
||
pattern=re.compile(
|
||
"|".join(
|
||
re.escape(keyword) for keyword in valid.INVALID_MEDICALLY_NECESSARY
|
||
),
|
||
re.IGNORECASE,
|
||
),
|
||
)
|
||
final_dicts = top_down_funcs.add_medically_necessary(
|
||
final_dicts, medically_necessary_dict
|
||
)
|
||
print(f"Medically Necessary - {filename}")
|
||
|
||
# Add On
|
||
add_on_dict = get_add_on(final_dicts, text_dict)
|
||
print(f"Add On Results: {add_on_dict}")
|
||
print(f"Add On - {filename}")
|
||
|
||
for td_page in add_on_dict.keys():
|
||
for d in final_dicts:
|
||
if str(d["page_num"]) == str(td_page):
|
||
print("Add On Match")
|
||
d.update(add_on_dict[str(td_page)])
|
||
|
||
# print(len(final_dicts))
|
||
|
||
output_df = pd.DataFrame(final_dicts)
|
||
print(f"B finished -{filename} - output {output_df.shape}")
|
||
print(output_df.columns)
|
||
time.sleep(3)
|
||
base_filename = os.path.splitext(filename)[0].strip()
|
||
output_dir = os.path.join(b_output_path, base_filename)
|
||
os.makedirs(output_dir, exist_ok=True)
|
||
output_df.to_csv(f"{output_dir}/b_output.csv")
|
||
|
||
|
||
########################## Process Starts Here ##########################
|
||
|
||
# Run List
|
||
run_list = pd.read_excel(run_path)
|
||
run_list = run_list[run_list["Batch"] == "3A"]["Filename"].unique().tolist()
|
||
print(f"Run List: {len(run_list)} files")
|
||
|
||
# Input Dict
|
||
input_dict_3a = utils.read_input(local_path=input_path + "A")
|
||
input_dict_3b = utils.read_input(local_path=input_path + "B")
|
||
|
||
input_dict = {**input_dict_3a, **input_dict_3b}
|
||
print(f"Input Dict: {len(input_dict)} files")
|
||
|
||
# Filter Input Dict
|
||
input_dict = {
|
||
filename: contract_text
|
||
for filename, contract_text in input_dict.items()
|
||
if filename.split(".txt")[0] in run_list
|
||
}
|
||
print(f"Input Dict In Run List: {len(input_dict)} files")
|
||
|
||
# Left to run
|
||
if "a" in config.FIELDS:
|
||
input_dict_ac2 = {
|
||
filename: contract_text
|
||
for filename, contract_text in input_dict.items()
|
||
if filename.split(".txt")[0] not in os.listdir(ac2_output_path)
|
||
}
|
||
print(f"Input Dict Left to Run (AC2): {len(input_dict_ac2)} files")
|
||
|
||
# ABC Dicts
|
||
abc_df = pd.read_excel(clean_path)
|
||
abc_df.rename(columns={v: k for k, v in valid.B_MAPPING.items()}, inplace=True)
|
||
abc_df.rename(columns={v: k for k, v in valid.AC_MAPPING.items()}, inplace=True)
|
||
abc_df.rename(columns=combined_mapping, inplace=True)
|
||
abc_df.drop(["State from Ben", "Unnamed: 54"], axis=1, inplace=True)
|
||
abc_names = list(abc_df["Filename"].unique())
|
||
abc_dicts = abc_df.to_dict(orient="records")
|
||
print(f"Clean abc : {len(abc_dicts)}")
|
||
|
||
# Left to run
|
||
if "b" in config.FIELDS:
|
||
input_dict_b = {
|
||
filename: contract_text
|
||
for filename, contract_text in input_dict.items()
|
||
if filename.split(".txt")[0] in abc_df["Filename"].unique()
|
||
}
|
||
input_dict_b = {
|
||
filename: contract_text
|
||
for filename, contract_text in input_dict_b.items()
|
||
if filename.split(".txt")[0] not in os.listdir(b_output_path)
|
||
}
|
||
print(f"Input Dict Left to Run (B): {len(input_dict_b)} files")
|
||
|
||
with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor:
|
||
if "a" in config.FIELDS and "c" in config.FIELDS:
|
||
futures = [
|
||
executor.submit(process_ac2_adhoc, item) for item in input_dict_ac2.items()
|
||
]
|
||
if "b" in config.FIELDS:
|
||
futures = [
|
||
executor.submit(process_b_adhoc, item) for item in input_dict_b.items()
|
||
]
|
||
|
||
for future in concurrent.futures.as_completed(futures):
|
||
try:
|
||
result = future.result()
|
||
except Exception as e:
|
||
print(f"Error in future: {e}")
|
||
traceback.print_exc()
|
||
|
||
|
||
# for item in input_dict_b.items():
|
||
# process_b_adhoc(item)
|