import concurrent.futures import traceback import csv import time import sys import random random.seed(42) import pandas as pd import os import re import utils import config import claude_funcs import file_processing import tracking import consolidate_output import top_down_funcs import valid import ac_funcs import preprocess import prompts import postprocess import postprocessing_funcs import preprocessing_funcs input_path = "data_cnc/batch3" ac2_output_path = "output_individual/cnc_3_ac2" b_output_path = "output_individual/cnc_3_b" clean_path = "reference_files/CNC-Batch 3 Output_File1.xlsx" run_path = "reference_files/Batch 3_091124.xlsx" # Ad-Hoc runs of sections of the script """ Batch 1 and 2 Fields to Run: AC - All Chunked Fields - All Part 2 Fields - Postprocess AC B - All B Postprocessing 750 additional - Run all ABC """ dup_mapping = { "Payer Name": "PAYER_NAME", "IRS Name": "IRS_Name", "NPI NAME": "NPI_NAME", "Sequestration Reductions, included [Medicare only] (Y/N)": "SEQUESTRATION_REDUCTIONS_IND", "Page_num": "page_num", "Lesser of Logic Language, included (Y/N)": "LESSER", "Reimb. Methodology_short": "SHORT_METHODOLOGY", "If rate is % of Payer or MCR [STANDARD]": "RATE_STANDARD", "If rate is % of Payer or MCR [STANDARD]_Short": "RATE_SHORT", "Flat Fee": "FLAT_FEE_STANDARD", "CDM Language, included (Y/N)": "CDM_IND", "CHARGEMASTER": "CONTRACT_CHARGEMASTER_PROTECTION_LANGUAGE", "EXCLUSIONS": "Exclusions", } combined_mapping = { "Contract_Name": "Filename", "Agreement_Name_Contract_Title": "CONTRACT_TITLE", "Payer_Name": "PAYER_NAME", "Health_Plan_State": "HEALTH_PLAN_STATE", "Affiliate_Y_N": "AFFILIATION_CLAUSE_IND", "Credentialing_Application_Indicator": "CREDENTIALING_APP_IND", "Term_Clause": "TERM_CLAUSE", "Contract_Auto_Renewal_Indicator": "CONTRACT_AUTO_RENEWAL_IND", "Termination_Date": "CONTRACT_TERMINATION_DT", "Termination_Upon_Notice_Days": "TERMINATION_UPON_NOTICE", "Termination_With_Cause_Days": "TERMINATION_UPON_CAUSE", "Amend_Contract_Upon_notice_Flag_Y_N": "AMEND_CONTRACT_NOTICE_IND", "Timeframe_to_Object_Days": "TIME_TO_OBJECT", "Assignments_Clause_Y_N": "ASSIGNMENTS_CLAUSE_IND", "Contract_Effective_Date": "CONTRACT_EFFECTIVE_DT", "IRS_Num": "PROV_GROUP_TIN", "PROV_GROUP_NAME": "PROV_GROUP_NAME", "NPI_10_digits": "PROV_GROUP_NPI", "NPI_NAME": "NPI_NAME", "PROV_GROUP_TIN_SIGNATORY": "PROV_GROUP_TIN_SIGNATORY", "PROV_TIN_OTHER": "PROV_TIN_OTHER", "PROV_NPI_OTHER": "PROV_NPI_OTHER", "Notice_to_Provider_Name": "NOTICE_PROVIDER_NAME", "Notice_to_Provider_Address": "NOTICE_PROVIDER_ADDRESS", "Sequestration_Language": "SEQUESTRATION_LANGUAGE", "Sequestration Reductions, included [Medicare only] (Y/N)": "SEQUESTRATION_REDUCTIONS_IND", "Parent_Agreement_Code": "Parent Agreement Code", "Pages": "Pages", "Page_num": "page_num", "Attachment_Exhibit": "EXHIBIT", "Line_of_Business": "CONTRACT_LOB", "Provider_Type": "PROV_TYPE", "Provider_Type_Level_2": "PROV_TYPE_LEVEL_2", "IP_OP": "IP_OP", "Service_Type": "FULL_SERVICE", "Plan_Type": "CONTRACT_PROGRAM", "Lesser_of_Logic_Language_included_Y_N": "LESSER", "Lesser_of_Rate": "LESSER_RATE", "Reimb_Methodology": "FULL_METHODOLOGY", "Reimb_Methodology_short": "SHORT_METHODOLOGY", "If rate is % of Payer or MCR [STANDARD]": "RATE_STANDARD", "If rate is % of Payer or MCR [STANDARD]_Short": "RATE_SHORT", "Flat Fee": "FLAT_FEE_STANDARD", "Default_Term": "DEFAULT_TERM", "Default_Rate": "DEFAULT_RATE", "IP - DSH/IME/UC, included (Y/N)": 'Inclusion of essential RBRVS "Fee Source" Language (Y/N)', "CDM_Language_included_Y_N": "CDM_IND", "CHARGEMASTER": "CHARGEMASTER", "IP_DSH_IME_UC_included_Y_N": "IP - DSH/IME/UC, included (Y/N)", "IP_Stoploss_Catastrophic_Threshold": "IP - Stoploss Catastrophic Threshold", "EXCLUSIONS": "EXCLUSIONS", "Not_to_Exceed": "NOT_TO_EXCEED", "Escalator_or_COLA_Y_N": "RATE_ESCALATOR_IND", "Escalator_I_Eff_Date": "RATE_ESCALATOR_DT", } AC_DICT = { "ACCESS_TO_MEDICAL_RECORDS": "Access clause contains information regarding access to medical records. It may be found in section 4.2 of the contract. Extract the Access clause present in the contract. If it is not present, answer N/A.", "CARVEOUT_VENDORS": "Carve-Out Vendors clause may be found in section 2.9 of the contract. Extract the Carve-Out Vendors clause present in the contract. If it is not present, answer N/A.", "CLAIMS_EDITING_LANGUAGE": "Claims Editing Language is typically found in the claims section in the beginning parts of the contract. Extract Claims Editing Language present in the contract. If it is not present, answer N/A.", "CLEAN_CLAIM": 'Extract the "Clean Claim" subsection present in definition section of the contract. If it is not present, answer N/A.', "CONFLICTS_BETWEEN_CERTAIN_DOCUMENTS_LANGUAGE": "Extract the Conflicts Between Certain Documents clause present in the contract. If it is not present, answer N/A.", "COST_SETTLEMENT_LANGUAGE": "Cost settlement language may be present in disputes and arbitration subsection of the contract. It usually contains keywords like settlementor cost settlement. Extract the cost settlement language present in the contract. If it is not present, answer N/A.", "DEEMER_AMENDMENT": "Deemer Amendment clause may be found in section 8.7.2 of the contract. Extract the Deemer Amendment clause present in the contract. If it is not present, answer N/A.", "DELEGATED_TERMS": "Extract Delegated Terms present in the contract. If it is not present, answer N/A.", "ECM": 'What is the "ECM" number mentioned in the contract? Only Answer the ECM number, do not provide any context. Do not provide ICM number. If ECM number is not present, answer N/A.', "EXCLUSIVITY_REQUIREMENT_LANGUAGE": "Extract Exclusivity requirement clause present in the contract. If it is not present, answer N/A.", "GUARANTEE_OF_PROVIDER_YIELD_LANGUAGE": "Extract Guarantee of Provider Yield Language present in the contract. If it is not present, answer N/A.", "HCBS_SERVICES": 'Extract HCBS services clause present either in the compensation or exhibit section of the contract. It may contain keywords like "for covered HCBS services, plan shall pay". Do not include definitions of HCBS. If it is not present, answer N/A.', "INDEMNIFICATION": "Indemnification clause may be found in section 5.2 of the contract. Extract the Indemnification clause present in the contract. If it is not present, answer N/A.", "INDEPENDENT_REVIEW_LANGUAGE": "Extract Independent Review language present in the contract. If it is not present, answer N/A.", "INVOICE_PRICING_LANGUAGE": 'Extract table with heading "INVOICED SERVICES" if it is present in compensation schedule of the contract. If it is not present, answer N/A', "LATE_PAID_CLAIMS_LANGUAGE": "Extract Late Paid Claims language present in the contract. This will be a sentence or paragraph describing late paid claims, interest payments or similar. Here is an example of a correct response: 'Any Clean Claim, as defined in 42 C.F.R. § 422.500, shall be paid within thirty (30) days of receipt by Plan at such address as may be designated by Plan, and Plan shall pay interest on any Clean Claim not paid within thirty (30) days of such receipt by Plan at the rate of interest required by law, or as otherwise set forth in the Provider Manual.'. Here is an example of an incorrect response: 'Any Clean Claim, as defined in 42 C.F.R. 422.500, shall be paid within thirty (30) days of receipt by Health Plan, Payor or (if Provider contracts with Downstream Entities) Provider, as applicable, as designated by Provider or such Downstream Entity, as applicable.'. If no valid answer is present, answer N/A.", "MEMBER_CONFINEMENT_DAYS_LANGUAGE": "Extract Member Confinement Days Language present in the contract. If it is not present, answer N/A.", "NATIONAL_AGREEMENT_IND": "Are more than one states covered in the contract? Answer with a Yes or No, do not provide any other context.", "NETWORK_ACCESS_FEES_LANGUAGE": 'Extract "Network Access Fee" verbiage present in the contract. If it is not present, answer N/A.', "NONSTANDARD_APPEALS_PROCESS_LANGUAGE": "Extract Nonstandard Appeals Process language present in the contract. If it is not present, answer N/A.", "PAYMENT_IN_ADVANCE_OF_CLAIMS_SUBMISSION_LANGUAGE": "Extract Payment in Advance of Claims Submission language present in the contract. If it is not present, answer N/A.", "PAYOR": '"Payor" is usually found in section 1.12 of the contract. Extract the "Payor" subsection present in definition section of the contract. If it is not present, answer N/A.', "PMPM": "What is the PMPM rate mentioned in the contract? Only return the rate. do not provide any context. If it is not present, answer N/A.", "PREAUTHORIZATION": "Preauthorization clause may be found in section 2.7 of the contract. Extract the Preauthorization clause present in the contract. If it is not present, answer N/A.", "PRODUCT_REMOVAL": "Product removal may be found in the products attachment, stating a product is being removed. Extract product removal language present in the contract. If it is not present, answer N/A.", "PROVIDER_BASED_BILLING_EXCLUSION_LANGUAGE": "Provider-based Billing Exclusion is often listed under the ‘Additional Provisions’ section. Extract the Provider-based Billing Exclusion language mentioned in the contract. If it is not present, answer N/A.", "RECOVERY_RIGHTS": "Recovery Rights clause may be found in section 3.5 of the contract. Extract the entire Recovery Rights sentence or paragraph present in the contract. If it is not present, answer N/A.", "TEMPLATE": "Is this contract in standard Centene template containing definition, terms clauses and other standard sections. Answer with a Yes or No, do not provide any other context.", "ARBITRATION_AND_DISPUTES": "Dispute Resolution clause is usually found in section 6.1 of the contract. Arbitration clause is usually found in section 6.2 of the contract. Extract both the Dispute Resolution and the arbitration clause present in the contract. In case one of them is present, extract that one. If none of them are present, answer N/A.", "DISPARAGEMENT_PROHIBITION_IND": "Is Disparagement Prohibition clause present in the contract? It may be found in section 2.1 of the contract. Answer with a Yes or No, do not provide any other context.", "DISPARAGEMENT_PROHIBITION_LANGUAGE": "Disparagement Prohibition clause may be found in section 2.1 of the contract. Extract the Disparagement Prohibition clause present in the contract. If it is not present, answer N/A.", "ELIGIBILITY_VERIFICATION": "Eligibility Verification or Determination clause may be found in section 2.6 of the contract. Extract the Eligibility Verification or Determination clause present in the contract. If it is not present, answer N/A.", "INSURANCE_REQUIREMENT": "Extract Insurance requirement mentioned in the contract. If it is not present, answer N/A.", "PARTICIPATION_IN_PRODUCTS": "Participation in Products clause may be found in section 2.2.2 of the contract. Extract the Participation in Products clause present in the contract. If it is not present, answer N/A.", "POLICIES_AND_PROCEDURES": "Extract the entire Policies and Procedures sentence or paragraph present in the contract. Do not extract language about conflicts and construction. Do not extract language about PP (Preferred Provider) nor PPG (Preferred Provider Group). If it is not present, answer N/A.", "REGULATORY_REQUIREMENTS": 'Extract the entire Regulatory Requirements paragraph present in the contract. It usually contains the keyword "Regulatory Requirements". If it is not present, answer N/A.', "RELATIONSHIP_OF_PARTIES_LANGUAGE": "Extract the entire Relationship of Parties passage present in the contract. If it is not present, answer N/A.", "TERM_CLAUSE": """Extract the subsection or paragraph labeled Term. It may also be labeled Termination. If there are no such sections, extract the paragraph the most closely resembles this label. If there is still no correct answer, simply return 'N/A'.""", "LATE_PAID_CLAIMS_LANGUAGE": "Extract Late Paid Claims language present in the contract. This will be a sentence or paragraph describing late paid claims, interest payments or similar. Here is an example of a correct response: 'Any Clean Claim, as defined in 42 C.F.R. § 422.500, shall be paid within thirty (30) days of receipt by Plan at such address as may be designated by Plan, and Plan shall pay interest on any Clean Claim not paid within thirty (30) days of such receipt by Plan at the rate of interest required by law, or as otherwise set forth in the Provider Manual.'. Here is an example of an incorrect response: 'Any Clean Claim, as defined in 42 C.F.R. 422.500, shall be paid within thirty (30) days of receipt by Health Plan, Payor or (if Provider contracts with Downstream Entities) Provider, as applicable, as designated by Provider or such Downstream Entity, as applicable.'. If no valid answer is present, answer N/A.", "NON_RENEWAL_LANGUAGE": "Extract the section of the contract that describes non-renewal obligations, especially the number of days notice that must be given. Write the full relevant sentence or paragraph. Ensure that the exact term 'non-renewal' is present in the answer. Do NOT return an answer that does not have 'non-renewal'.", "HCBS_SERVICES": 'Extract HCBS services clause present either in the compensation or exhibit section of the contract. It may contain keywords like "for covered HCBS services, plan shall pay". Do not include definitions of HCBS. If it is not present, answer N/A.', } KEYWORD_MAPPINGS = { "TERM_CLAUSE": { "methodology": "hierarchy", "keywords": ["Term", "term", "duration", "agreement period"], }, "LATE_PAID_CLAIMS_LANGUAGE": { "methodology": "hierarchy", "keywords": [ "late paid", "late payment", "interest", "late fee", "interest shall be paid", "interest payment", ], }, "NON_RENEWAL_LANGUAGE": { "methodology": "or", "keywords": ["not to renew", "non-renewal", "non renewal", "nonrenewal"], }, # "NON_RENEWAL_DAYS": {'methodology': 'or', 'keywords': ["not to renew", "non-renewal", "non renewal", "nonrenewal"]}, "HCBS_SERVICES": { "methodology": "hierarchy", "keywords": ["%", "Plan shall pay", "hcbs"], }, "POLICIES_AND_PROCEDURES": { "methodology": "or", "keywords": ["2.4", "policies", "procedures"], }, "REGULATORY_REQUIREMENTS": { "methodology": "or", "keywords": ["8.3", "6.4", "regulatory requirements", "regulatory"], }, "RECOVERY_RIGHTS": {"methodology": "or", "keywords": ["3.5", "recovery rights"]}, "RELATIONSHIP_OF_PARTIES_LANGUAGE": { "methodology": "or", "keywords": ["8.1", "relationship of parties"], }, } def run_ac2_prompts(file_object): ################## INITIATE PROCESSING ################## filename, contract_text = file_object print(f"Processing AC for {filename}...") # print(f"Total contract word count: {len(contract_text.split())}") ################## PREPROCESS ################## text_dict, exhibit_pages, num_pages, ac_chunks = preprocess.preprocess( contract_text, filename, fields="ac" ) # print(f"AC Preprocessing Complete - {filename}") ################## RUN FULL CONTEXT PROMPTS ################## ac_dict = {} log_data = [] # Process 'full_context' fields together full_context_fields = [field for field in AC_DICT if field not in KEYWORD_MAPPINGS] full_context_questions = { field: prompts.AC_DICT.get(field) for field in full_context_fields } if full_context_questions: full_context_prompt = ac_funcs.create_prompt( contract_text[0 : min(len(contract_text), 600000)], question=full_context_questions, ) # print(f"High accuracy prompt word count: {len(high_accuracy_prompt.split())}") try: full_context_answers = ac_funcs.get_ac_answer( full_context_prompt, filename, fields=list(full_context_questions.keys()), ) ac_dict.update(full_context_answers) for field, answer in full_context_answers.items(): log_data.append( { "Field": field, "Prompt": f"Question: {full_context_questions[field]}\n\nContext: {contract_text}", # Include full context "Context Type": "Full Context (High Accuracy)", "Response": answer, } ) except Exception as e: # print(f"Error processing high accuracy fields: {str(e)}") for field in full_context_questions.keys(): ac_dict[field] = f"Error: {str(e)}" log_data.append( { "Field": field, "Prompt": f"Question: {full_context_questions[field]}\n\nContext: {contract_text}", # Include full context "Context Type": "Full Context (High Accuracy)", "Response": f"Error: {str(e)}", } ) ################## RUN CHUNKED PROMPTS ################## all_fields = set(prompts.AC_DICT.keys()) chunked_fields = [field for field in KEYWORD_MAPPINGS] for field in chunked_fields: if field in prompts.AC_DICT: question = prompts.AC_DICT[field] else: # print(f"Field {field} not found in prompts. Skipping...") continue # Use chunk if available and different from full context, otherwise use full context if ( field in ac_chunks and ac_chunks[field].strip() and ac_chunks[field] != contract_text ): context = ac_chunks[field] context_type = "Smart Chunking" else: context = contract_text context_type = "Full Context" prompt = prompts.AC_SINGLE_FIELD_TEMPLATE(context, question) # print(f"Field {field} prompt word count: {len(prompt.split())} ({context_type})") try: field_answer = claude_funcs.invoke_claude( prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 8192 ) print(field, field_answer) ac_dict[field] = field_answer log_data.append( { "Field": field, "Prompt": f"Question: {question}\n\nContext: {context}", # Include full context "Context Type": context_type, "Response": field_answer, } ) except Exception as e: # print(f"Error processing field {field}: {str(e)}") ac_dict[field] = f"Error: {str(e)}" log_data.append( { "Field": field, "Prompt": f"Question: {question}\n\nContext: {context}", # Include full context "Context Type": context_type, "Response": f"Error: {str(e)}", } ) ################## AC CONDITIONAL PROMPTS ################## ## Non-Renewal - Days nrd_prompt = prompts.AC_SINGLE_FIELD_TEMPLATE( ac_dict["NON_RENEWAL_LANGUAGE"], prompts.AC_DICT["NON_RENEWAL_DAYS"] ) nrd_answer = claude_funcs.invoke_claude( nrd_prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 8192 ) ac_dict["NON_RENEWAL_DAYS"] = nrd_answer ################## ENSURE ALL FIELDS ARE PRESENT ################## ac_dict = {k: v for k, v in ac_dict.items() if k in valid.AC_MAPPING} for field in all_fields: if field not in ac_dict: # print(f"Field not found in results. {field}") # ac_dict[field] = "N/A" log_data.append( { "Field": field, "Prompt": "N/A", "Context Type": "N/A", "Response": "N/A", } ) ################## CREATE OUTPUT DIRECTORIES ################## base_filename = os.path.splitext(filename)[0].strip() output_dir = os.path.join(ac2_output_path, base_filename) os.makedirs(output_dir, exist_ok=True) ################## POSTPROCESS ################## ac_dict["Contract Name"] = filename ac_df = pd.DataFrame([ac_dict]) ac_df = postprocess.ac_postprocess(ac_df) ################## WRITE TO OUTPUT ################## ac_df.to_csv(f"{output_dir}/ac_output.csv", index=False) print(f"AC Output written: {filename}") ################## WRITE LOG TO CSV ################## # log_file_path = os.path.join(output_dir, f"{base_filename}_prompt_log.csv") # with open(log_file_path, 'w', newline='', encoding='utf-8') as log_file: # fieldnames = ['Field', 'Prompt', 'Context Type', 'Response'] # writer = csv.DictWriter(log_file, fieldnames=fieldnames) # writer.writeheader() # writer.writerows(log_data) # print(f"Prompt log written to {log_file_path}") return ac_dict def process_ac2_adhoc(item): filename, contract_text = item try: results = run_ac2_prompts(item) if results is not None: tracking.update_results_csv(item, results) return item, results except Exception as e: print(f"Error processing item {filename}: {e}") traceback.print_exc() def preprocess_ad_hoc(contract_text, filename, fields=config.FIELDS): contract_text = preprocessing_funcs.clean_newlines(contract_text) contract_text = preprocessing_funcs.clean_law_symbols(contract_text) text_dict = preprocessing_funcs.split_text( contract_text ) # return a dictionary with keys - page_num (str), values as the page_text num_pages = len(text_dict.keys()) - 1 text_dict = preprocessing_funcs.filter_quick_review(text_dict) if fields == "b": exhibit_pages = top_down_funcs.get_exhibit_pages( text_dict, filename ) # All pages with exhibit headers # text_dict = table_funcs.align_and_format_tables(text_dict, filename) text_dict = preprocessing_funcs.chunk_consecutive(text_dict, exhibit_pages) ac_chunks = "" if fields == "ac": exhibit_pages = [] ac_chunks = preprocessing_funcs.smart_chunk_ac(text_dict) return text_dict, exhibit_pages, num_pages, ac_chunks def get_add_on(final_dicts, text_dict): add_on_dict = {} for td_page in list(set([d["page_num"] for d in final_dicts])): td_page = str(td_page).split(".")[0] print(f"TD Page : {td_page}") # TD page is the original result top down page, it is not necessarily in the new text_dict. # Find add_on_page = the key of the new text_dict that points to the correct page of the old text dict if str(td_page) in list(text_dict.keys()): add_on_page = td_page else: add_on_page = [ str(page) for page in text_dict.keys() if float(page) > float(td_page) ] print(add_on_page) if not add_on_page: add_on_dict[td_page] = { "ADD_ON_REIMBURSEMENT_LANGUAGE": "N/A", "ADD_ON_REIMBURSEMENT_IND": "N", } continue elif not isinstance(add_on_page, str): add_on_page = add_on_page[0] ad = {} add_on_prompt = f"## Page Start ## {text_dict[str(add_on_page)]} ## Page End ## . Extract the Add On Reimbursement language present in the page above. If there is no Add On language present, answer N/A. Write only the answer, with no additional commentary or explanation." add_on_answer = claude_funcs.invoke_claude( add_on_prompt, config.MODEL_ID_CLAUDE35_SONNET, "", 256 ) ad["ADD_ON_REIMBURSEMENT_LANGUAGE"] = add_on_answer if "N/A" in ad["ADD_ON_REIMBURSEMENT_LANGUAGE"]: ad["ADD_ON_REIMBURSEMENT_IND"] = "N" else: ad["ADD_ON_REIMBURSEMENT_IND"] = "Y" add_on_dict[str(td_page)] = ad return add_on_dict def process_b_adhoc(item): filename, contract_text = item print(f"Processing {filename}") merged_dicts = [d for d in abc_dicts if d["Filename"] + ".txt" == filename] print(len(merged_dicts)) if len(merged_dicts) > 0: text_dict, exhibit_pages, num_pages, ac_chunks = preprocess_ad_hoc( contract_text, filename ) print(f"Preprocessing {filename}") # Exclusions exclusion_dict = top_down_funcs.get_td_dict( text_dict, filename, prompts.TOP_DOWN_EXCLUSIONS, valid.EXCLUSION_TERMS ) final_dicts = top_down_funcs.add_exclusions(merged_dicts, exclusion_dict) print(f"Exclusions - {filename}") # Medically Necessary medically_necessary_dict = top_down_funcs.get_td_dict( text_dict, filename, prompts.TOP_DOWN_MEDICALLY_NECESSARY, valid.VALID_MEDICALLY_NECESSARY, ) medically_necessary_dict = postprocessing_funcs.filter_dict( medically_necessary_dict, pattern=re.compile( "|".join( re.escape(keyword) for keyword in valid.INVALID_MEDICALLY_NECESSARY ), re.IGNORECASE, ), ) final_dicts = top_down_funcs.add_medically_necessary( final_dicts, medically_necessary_dict ) print(f"Medically Necessary - {filename}") # Add On add_on_dict = get_add_on(final_dicts, text_dict) print(f"Add On Results: {add_on_dict}") print(f"Add On - {filename}") for td_page in add_on_dict.keys(): for d in final_dicts: if str(d["page_num"]) == str(td_page): print("Add On Match") d.update(add_on_dict[str(td_page)]) # print(len(final_dicts)) output_df = pd.DataFrame(final_dicts) print(f"B finished -{filename} - output {output_df.shape}") print(output_df.columns) time.sleep(3) base_filename = os.path.splitext(filename)[0].strip() output_dir = os.path.join(b_output_path, base_filename) os.makedirs(output_dir, exist_ok=True) output_df.to_csv(f"{output_dir}/b_output.csv") ########################## Process Starts Here ########################## # Run List run_list = pd.read_excel(run_path) run_list = run_list[run_list["Batch"] == "3A"]["Filename"].unique().tolist() print(f"Run List: {len(run_list)} files") # Input Dict input_dict_3a = utils.read_input(local_path=input_path + "A") input_dict_3b = utils.read_input(local_path=input_path + "B") input_dict = {**input_dict_3a, **input_dict_3b} print(f"Input Dict: {len(input_dict)} files") # Filter Input Dict input_dict = { filename: contract_text for filename, contract_text in input_dict.items() if filename.split(".txt")[0] in run_list } print(f"Input Dict In Run List: {len(input_dict)} files") # Left to run if "a" in config.FIELDS: input_dict_ac2 = { filename: contract_text for filename, contract_text in input_dict.items() if filename.split(".txt")[0] not in os.listdir(ac2_output_path) } print(f"Input Dict Left to Run (AC2): {len(input_dict_ac2)} files") # ABC Dicts abc_df = pd.read_excel(clean_path) abc_df.rename(columns={v: k for k, v in valid.B_MAPPING.items()}, inplace=True) abc_df.rename(columns={v: k for k, v in valid.AC_MAPPING.items()}, inplace=True) abc_df.rename(columns=combined_mapping, inplace=True) abc_df.drop(["State from Ben", "Unnamed: 54"], axis=1, inplace=True) abc_names = list(abc_df["Filename"].unique()) abc_dicts = abc_df.to_dict(orient="records") print(f"Clean abc : {len(abc_dicts)}") # Left to run if "b" in config.FIELDS: input_dict_b = { filename: contract_text for filename, contract_text in input_dict.items() if filename.split(".txt")[0] in abc_df["Filename"].unique() } input_dict_b = { filename: contract_text for filename, contract_text in input_dict_b.items() if filename.split(".txt")[0] not in os.listdir(b_output_path) } print(f"Input Dict Left to Run (B): {len(input_dict_b)} files") with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor: if "a" in config.FIELDS and "c" in config.FIELDS: futures = [ executor.submit(process_ac2_adhoc, item) for item in input_dict_ac2.items() ] if "b" in config.FIELDS: futures = [ executor.submit(process_b_adhoc, item) for item in input_dict_b.items() ] for future in concurrent.futures.as_completed(futures): try: result = future.result() except Exception as e: print(f"Error in future: {e}") traceback.print_exc() # for item in input_dict_b.items(): # process_b_adhoc(item)