Merged in bugfix/all_cnc_hotfixes (pull request #302)
DRAFT: Bugfix/all cnc hotfixes * Pipeline fix - health plan state * Update for prompt vs non-prompt fixes * Revert utils.find_regex_matches() * Remove prints * Non-prompt run * Health Plan State, IRS, NPI, LOB, Lesser, Prov 2, Default, Rate Standard, Contract Effective Date, Term Group * Merged in bugfix/agreement_name (pull request #294) contract title fixed * contract title fixed * Merged bugfix/all_cnc_hotfixes into bugfix/agreement_name Approved-by: Katon Minhas * All hotfixes added * Fixed utils * fix for clean_contract_effective_date * Merged in default-fix (pull request #296) moved default funcs to the top, dropped intermediate columns * moved default funcs to the top, dropped intermediate columns Approved-by: Katon Minhas * quick fix to dropping extra columns * Merged in bugfix/effective_date_meridian (pull request #297) fixed effective date for meridian contracts * fixed effective date for meridian contracts Approved-by: Katon Minhas * Default rate and Lesser fix * Merged in hotfix/irs_npi_updated (pull request #298) updated irs and npi hotfix funcs * updated irs and npi hotfix funcs * Merged bugfix/all_cnc_hotfixes into hotfix/irs_npi_updated Approved-by: Katon Minhas * non-functional commit - default only * debug commit - premerge * Batch 1 Dup File Reconciliation * Default rate fix * Full CNC 1A Run Config * Standard column refactor * Working stitching for 1A * Merged in Alex-Galarce/cnc_hotfix_effective_date_utilspy-edited-1732659983687 (pull request #301) Remove "license" from effective date smart chunking * Remove "license" from effective date smart chunking * Merged bugfix/all_cnc_hotfixes into Alex-Galarce/cnc_hotfix_effective_date_utilspy-edited-1732659983687 Approved-by: Katon Minhas * Merged in hotfix/irs_batch2_feedback (pull request #300) IRS fix for feedback on batch2 * IRS fix for feedback on batch2 * Merged bugfix/all_cnc_hotfixes into hotfix/irs_batch2_feedback Approved-by: Katon Minhas * Merged main into bugfix/all_cnc_hotfixes Approved-by: Katon Minhas
This commit is contained in:
Alex Galarce
committed by
Katon Minhas
Katon Minhas
parent
88a57f1e1f
commit
56e3fdd1f8
+17
-1
@@ -85,6 +85,21 @@ texts/
|
||||
*.json
|
||||
myenv/
|
||||
*.env
|
||||
<<<<<<< HEAD
|
||||
*.txt
|
||||
!requirements.txt
|
||||
*.pdf
|
||||
subset/
|
||||
output/
|
||||
docs/
|
||||
allfiles/
|
||||
CNC_testruns_AC/
|
||||
cnc-abc-output/
|
||||
humana-input/
|
||||
humana-output/
|
||||
output_consolidated/
|
||||
|
||||
=======
|
||||
subset/
|
||||
output/
|
||||
docs/
|
||||
@@ -94,4 +109,5 @@ docs/
|
||||
__pycache__/
|
||||
|
||||
.env
|
||||
new/
|
||||
new/
|
||||
>>>>>>> main
|
||||
|
||||
@@ -1,163 +0,0 @@
|
||||
import pandas as pd
|
||||
from difflib import SequenceMatcher
|
||||
import re
|
||||
import time
|
||||
|
||||
import prompts
|
||||
import claude_funcs
|
||||
|
||||
|
||||
def get_closest_substring_match(val, valid_values):
|
||||
"""Returns the first match from valid_values using a case-insensitive substring match."""
|
||||
if pd.isna(val):
|
||||
return None
|
||||
val = val.strip().upper()
|
||||
for valid_value in valid_values:
|
||||
if valid_value.upper() in val:
|
||||
return valid_value
|
||||
return None
|
||||
|
||||
|
||||
def get_best_carveout_from_claude(carveout_list, service):
|
||||
print("using claude to get best carveout...")
|
||||
prompt = f"Given the service '{service}', please choose the best matching carveout from the following list: {', '.join(carveout_list)}. ONLY SELECT ONE FROM THE LIST. DO NOT RETURN A SENTENCE"
|
||||
while True:
|
||||
try:
|
||||
response = claude_funcs.invoke_claude_3(prompt, max_tokens=100)
|
||||
return response.strip()
|
||||
except Exception as e:
|
||||
print(f"Error occurred: {e}. Waiting for 60 seconds before retrying...")
|
||||
time.sleep(60)
|
||||
|
||||
|
||||
def check_prov_type_similarity(prov_type, carveout):
|
||||
print("checking similarity between prov_type and carveout with claude...")
|
||||
prompt = f"Do the provider type '{prov_type}' and the carveout '{carveout}' mean the same thing or are they very similar? ONLY ANSWER WITH 'True' OR 'False'"
|
||||
while True:
|
||||
try:
|
||||
response = claude_funcs.invoke_claude_3(prompt, max_tokens=10)
|
||||
return response.strip()
|
||||
except Exception as e:
|
||||
print(f"Error occurred: {e}. Waiting for 60 seconds before retrying...")
|
||||
time.sleep(60)
|
||||
|
||||
|
||||
def label_services(filepath, carveout_list, output_filepath):
|
||||
# Read the CSV file
|
||||
df = pd.read_csv(filepath)
|
||||
|
||||
# Select the first 50 rows of the DataFrame
|
||||
df = df.head(500)
|
||||
|
||||
# Define primary service terms
|
||||
primary_terms = [
|
||||
"Covered Services",
|
||||
"Inpatient Services",
|
||||
"Outpatient Services",
|
||||
"Physician Services",
|
||||
"Inpatient",
|
||||
"Outpatient",
|
||||
"Physician",
|
||||
"Medical",
|
||||
"Surgical",
|
||||
"Diagnostic",
|
||||
"Therapeutic",
|
||||
]
|
||||
|
||||
# Initialize columns for the results
|
||||
df["carveout_matched"] = ""
|
||||
df["label"] = ""
|
||||
df["IS_CARVEOUT"] = ""
|
||||
|
||||
# Initialize a nested dictionary to track occurrences for each Filename and TD_LOB
|
||||
occurrence_tracker = {}
|
||||
|
||||
carveout_count = 0
|
||||
|
||||
# Iterate through the rows in the DataFrame
|
||||
for index, row in df.iterrows():
|
||||
service = str(row["SERVICE"])
|
||||
prov_type = str(row["PROV_TYPE"]) if pd.notna(row["PROV_TYPE"]) else ""
|
||||
td_lob = row["TD_LOB"]
|
||||
filename = row["Filename"]
|
||||
|
||||
# Initialize the nested dictionary for the filename and TD_LOB if not present
|
||||
if filename not in occurrence_tracker:
|
||||
occurrence_tracker[filename] = {}
|
||||
if td_lob not in occurrence_tracker[filename]:
|
||||
occurrence_tracker[filename][td_lob] = {term: 0 for term in primary_terms}
|
||||
|
||||
# Check if the service is a primary or similar to a primary
|
||||
primary = get_closest_substring_match(service, primary_terms)
|
||||
if primary:
|
||||
# Ensure the primary term is initialized in the occurrence tracker
|
||||
if primary not in occurrence_tracker[filename][td_lob]:
|
||||
occurrence_tracker[filename][td_lob][primary] = 0
|
||||
|
||||
# Determine the label based on the count of this primary term for the given Filename and TD_LOB
|
||||
count = occurrence_tracker[filename][td_lob][primary]
|
||||
if count == 0:
|
||||
df.at[index, "label"] = "primary"
|
||||
elif count == 1:
|
||||
df.at[index, "label"] = "secondary"
|
||||
elif count == 2:
|
||||
df.at[index, "label"] = "tertiary"
|
||||
else:
|
||||
df.at[index, "label"] = "additional"
|
||||
occurrence_tracker[filename][td_lob][primary] += 1
|
||||
df.at[index, "IS_CARVEOUT"] = "N"
|
||||
else:
|
||||
# If PROV_TYPE is blank, continue with carveout determination
|
||||
best_carveout_response = get_best_carveout_from_claude(
|
||||
carveout_list, service
|
||||
)
|
||||
matched_carveout = best_carveout_response # Use the model response directly
|
||||
|
||||
df.at[index, "carveout_matched"] = matched_carveout
|
||||
|
||||
if matched_carveout:
|
||||
# Check similarity between PROV_TYPE and carveout
|
||||
prov_type_similarity_score = SequenceMatcher(
|
||||
None, prov_type, matched_carveout
|
||||
).ratio()
|
||||
if prov_type_similarity_score < 0.8:
|
||||
prov_type_similarity = check_prov_type_similarity(
|
||||
prov_type, matched_carveout
|
||||
)
|
||||
else:
|
||||
prov_type_similarity = "True"
|
||||
|
||||
if re.search(r"\b(True|Yes)\b", prov_type_similarity, re.IGNORECASE):
|
||||
if matched_carveout not in occurrence_tracker[filename][td_lob]:
|
||||
print(
|
||||
"claude returned a match between provider type and service"
|
||||
)
|
||||
occurrence_tracker[filename][td_lob][matched_carveout] = 0
|
||||
count = occurrence_tracker[filename][td_lob][matched_carveout]
|
||||
if count == 0:
|
||||
df.at[index, "label"] = "primary"
|
||||
elif count == 1:
|
||||
df.at[index, "label"] = "secondary"
|
||||
elif count == 2:
|
||||
df.at[index, "label"] = "tertiary"
|
||||
else:
|
||||
df.at[index, "label"] = "additional"
|
||||
occurrence_tracker[filename][td_lob][matched_carveout] += 1
|
||||
df.at[index, "IS_CARVEOUT"] = "N"
|
||||
else:
|
||||
df.at[index, "IS_CARVEOUT"] = "Y"
|
||||
else:
|
||||
df.at[index, "IS_CARVEOUT"] = "Y"
|
||||
|
||||
carveout_count += 1
|
||||
|
||||
# Save the updated DataFrame to a new CSV file after every 10 carveouts
|
||||
if carveout_count % 10 == 0:
|
||||
df.to_csv(output_filepath, index=False)
|
||||
print(f"Saved progress after processing {carveout_count} carveouts.")
|
||||
|
||||
# Final save of the updated DataFrame to a new CSV file
|
||||
df.to_csv(output_filepath, index=False)
|
||||
|
||||
|
||||
label_services("service.csv", prompts.get_carveout_list(), "carveouts50.csv")
|
||||
@@ -1,228 +0,0 @@
|
||||
import concurrent.futures
|
||||
import traceback
|
||||
import csv
|
||||
import time
|
||||
import sys
|
||||
import random
|
||||
|
||||
random.seed(42)
|
||||
import pandas as pd
|
||||
import os
|
||||
import re
|
||||
|
||||
import utils
|
||||
import config
|
||||
import claude_funcs
|
||||
import file_processing
|
||||
import tracking
|
||||
import consolidate_output
|
||||
import top_down_funcs
|
||||
import valid
|
||||
import ac_funcs
|
||||
import preprocess
|
||||
import prompts
|
||||
import postprocess
|
||||
import postprocessing_funcs
|
||||
import preprocessing_funcs
|
||||
|
||||
individual_path = "output_individual/cnc_3_ac1"
|
||||
clean_path = "reference_files/CNC-Batch 3 Output_File1.xlsx"
|
||||
|
||||
|
||||
AC_DICT = {
|
||||
"CONTRACT_EFFECTIVE_DT": "What is the contract effective date mentioned in any of the following locations: the signatory section or the preamble of the agreement, or the start of the amendment ? Return this date converted to YYYY-MM-DD format.",
|
||||
"PROV_GROUP_NPI": "What is the group provider's national provider identifier number mentioned in the contract? This may be found on the signature page or on the roster. It consists of 10 digits. Only return these 10 digits.",
|
||||
"PROV_GROUP_TIN": "What is the Group provider's taxpayer identification number stated in the contract? This is a 9 digit long number typically with a hyphen after the first 2 digits. Return only this 9 digit number with a hyphen after the first 2 digits. Do not elaborate or add context.",
|
||||
}
|
||||
|
||||
combined_mapping = {
|
||||
"Contract_Name": "Filename",
|
||||
"Agreement_Name_Contract_Title": "CONTRACT_TITLE",
|
||||
"Payer_Name": "PAYER_NAME",
|
||||
"Health_Plan_State": "HEALTH_PLAN_STATE",
|
||||
"Affiliate_Y_N": "AFFILIATION_CLAUSE_IND",
|
||||
"Credentialing_Application_Indicator": "CREDENTIALING_APP_IND",
|
||||
"Term_Clause": "TERM_CLAUSE",
|
||||
"Contract_Auto_Renewal_Indicator": "CONTRACT_AUTO_RENEWAL_IND",
|
||||
"Termination_Date": "CONTRACT_TERMINATION_DT",
|
||||
"Termination_Upon_Notice_Days": "TERMINATION_UPON_NOTICE",
|
||||
"Termination_With_Cause_Days": "TERMINATION_UPON_CAUSE",
|
||||
"Amend_Contract_Upon_notice_Flag_Y_N": "AMEND_CONTRACT_NOTICE_IND",
|
||||
"Timeframe_to_Object_Days": "TIME_TO_OBJECT",
|
||||
"Assignments_Clause_Y_N": "ASSIGNMENTS_CLAUSE_IND",
|
||||
"Contract_Effective_Date": "CONTRACT_EFFECTIVE_DT",
|
||||
"IRS_Num": "PROV_GROUP_TIN",
|
||||
"PROV_GROUP_NAME": "PROV_GROUP_NAME",
|
||||
"NPI_10_digits": "PROV_GROUP_NPI",
|
||||
"NPI_NAME": "NPI_NAME",
|
||||
"PROV_GROUP_TIN_SIGNATORY": "PROV_GROUP_TIN_SIGNATORY",
|
||||
"PROV_TIN_OTHER": "PROV_TIN_OTHER",
|
||||
"PROV_NPI_OTHER": "PROV_NPI_OTHER",
|
||||
"Notice_to_Provider_Name": "NOTICE_PROVIDER_NAME",
|
||||
"Notice_to_Provider_Address": "NOTICE_PROVIDER_ADDRESS",
|
||||
"Sequestration_Language": "SEQUESTRATION_LANGUAGE",
|
||||
"Sequestration_Reductions_included_Medicare_only_Y_N": "SEQUESTRATION_REDUCTIONS_IND",
|
||||
"Parent_Agreement_Code": "Parent Agreement Code",
|
||||
"Pages": "Pages",
|
||||
"Page_num": "page_num",
|
||||
"Attachment_Exhibit": "EXHIBIT",
|
||||
"Line_of_Business": "CONTRACT_LOB",
|
||||
"Provider_Type": "PROV_TYPE",
|
||||
"Provider_Type_Level_2": "PROV_TYPE_LEVEL_2",
|
||||
"IP_OP": "IP_OP",
|
||||
"Service_Type": "FULL_SERVICE",
|
||||
"Plan_Type": "CONTRACT_PROGRAM",
|
||||
"Lesser_of_Logic_Language_included_Y_N": "LESSER",
|
||||
"Lesser_of_Rate": "LESSER_RATE",
|
||||
"Reimb_Methodology": "FULL_METHODOLOGY",
|
||||
"Reimb_Methodology_short": "SHORT_METHODOLOGY",
|
||||
"If_rate_is_percent_of_Payer_or_MCR_STANDARD": "RATE_STANDARD",
|
||||
"If_rate_is_percent_of_Payer_or_MCR_STANDARD_Short": "RATE_SHORT",
|
||||
"Flat_Fee": "FLAT_FEE_STANDARD",
|
||||
"Default_Term": "DEFAULT_TERM",
|
||||
"Default_Rate": "DEFAULT_RATE",
|
||||
"Inclusion_of_essential_RBRVS_Fee_Source_Language_Y_N": 'Inclusion of essential RBRVS "Fee Source" Language (Y/N)',
|
||||
"CDM_Language_included_Y_N": "CDM_IND",
|
||||
"CHARGEMASTER": "CHARGEMASTER",
|
||||
"IP_DSH_IME_UC_included_Y_N": "IP - DSH/IME/UC, included (Y/N)",
|
||||
"IP_Stoploss_Catastrophic_Threshold": "IP - Stoploss Catastrophic Threshold",
|
||||
"EXCLUSIONS": "EXCLUSIONS",
|
||||
"Not_to_Exceed": "NOT_TO_EXCEED",
|
||||
"Escalator_or_COLA_Y_N": "RATE_ESCALATOR_IND",
|
||||
"Escalator_I_Eff_Date": "RATE_ESCALATOR_DT",
|
||||
}
|
||||
|
||||
import ac_funcs
|
||||
import config
|
||||
import postprocess
|
||||
import tracking
|
||||
|
||||
|
||||
def ac_postprocess(final_df):
|
||||
final_df = postprocessing_funcs.clean_ac_fields(final_df)
|
||||
final_df = final_df.rename(columns=valid.AC_MAPPING)
|
||||
final_df = final_df[[col for col in valid.ABC_COLUMNS if col in final_df.columns]]
|
||||
return final_df
|
||||
|
||||
|
||||
def run_ac_prompts(file_object):
|
||||
################## INITIATE PROCESSING ##################
|
||||
filename, contract_text = file_object
|
||||
print(f"Processing AC for {filename}...")
|
||||
# print(f"Total contract word count: {len(contract_text.split())}")
|
||||
|
||||
################## PREPROCESS ##################
|
||||
text_dict, exhibit_pages, num_pages, ac_chunks = preprocess.preprocess(
|
||||
contract_text, filename, fields="ac"
|
||||
)
|
||||
# print(f"AC Preprocessing Complete - {filename}")
|
||||
|
||||
################## RUN FULL CONTEXT PROMPTS ##################
|
||||
ac_dict = {}
|
||||
log_data = []
|
||||
|
||||
# Process 'full_context' fields together
|
||||
full_context_fields = [field for field in AC_DICT]
|
||||
full_context_questions = AC_DICT
|
||||
|
||||
if full_context_questions:
|
||||
full_context_prompt = ac_funcs.create_prompt(
|
||||
contract_text[0 : min(len(contract_text), 600000)],
|
||||
question=full_context_questions,
|
||||
)
|
||||
# print(f"High accuracy prompt word count: {len(high_accuracy_prompt.split())}")
|
||||
try:
|
||||
full_context_answers = ac_funcs.get_ac_answer(
|
||||
full_context_prompt,
|
||||
filename,
|
||||
fields=list(full_context_questions.keys()),
|
||||
)
|
||||
ac_dict.update(full_context_answers)
|
||||
|
||||
for field, answer in full_context_answers.items():
|
||||
log_data.append(
|
||||
{
|
||||
"Field": field,
|
||||
"Prompt": f"Question: {full_context_questions[field]}\n\nContext: {contract_text}", # Include full context
|
||||
"Context Type": "Full Context (High Accuracy)",
|
||||
"Response": answer,
|
||||
}
|
||||
)
|
||||
except Exception as e:
|
||||
# print(f"Error processing high accuracy fields: {str(e)}")
|
||||
for field in full_context_questions.keys():
|
||||
ac_dict[field] = f"Error: {str(e)}"
|
||||
log_data.append(
|
||||
{
|
||||
"Field": field,
|
||||
"Prompt": f"Question: {full_context_questions[field]}\n\nContext: {contract_text}", # Include full context
|
||||
"Context Type": "Full Context (High Accuracy)",
|
||||
"Response": f"Error: {str(e)}",
|
||||
}
|
||||
)
|
||||
|
||||
################## CREATE OUTPUT DIRECTORIES ##################
|
||||
base_filename = os.path.splitext(filename)[0].strip()
|
||||
output_dir = os.path.join(individual_path, base_filename)
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
################## POSTPROCESS ##################
|
||||
ac_dict["Contract Name"] = filename
|
||||
ac_df = pd.DataFrame([ac_dict])
|
||||
ac_df = ac_postprocess(ac_df)
|
||||
|
||||
################## WRITE TO OUTPUT ##################
|
||||
ac_df.to_csv(f"{output_dir}/ac_output.csv", index=False)
|
||||
print(f"AC Output written: {filename}")
|
||||
|
||||
|
||||
def process_ac_adhoc(item):
|
||||
filename, contract_text = item
|
||||
try:
|
||||
results = run_ac_prompts(item)
|
||||
if results is not None:
|
||||
tracking.update_results_csv(item, results)
|
||||
return item, results
|
||||
except Exception as e:
|
||||
print(f"Error processing item {filename}: {e}")
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
########################## Process Starts Here ##########################
|
||||
|
||||
# Clean
|
||||
abc = pd.read_excel(clean_path)
|
||||
abc.rename(columns=combined_mapping, inplace=True)
|
||||
abc_names = list(abc["Contract Name"].unique())
|
||||
abc_dicts = abc.to_dict(orient="records")
|
||||
|
||||
# Input
|
||||
input_dict = utils.read_input("data_cnc/batch3A")
|
||||
|
||||
# Current Output
|
||||
output_dirs = list(os.listdir(individual_path))
|
||||
|
||||
# Run only files that are not in abc, are in input, and are not in output
|
||||
input_dict = {
|
||||
k: v
|
||||
for k, v in input_dict.items()
|
||||
if k.split(".txt")[0] not in abc_names and k.split(".txt")[0] not in output_dirs
|
||||
}
|
||||
print("Input dict to run: ", len(input_dict))
|
||||
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor:
|
||||
if "a" in config.FIELDS and "c" in config.FIELDS:
|
||||
futures = [
|
||||
executor.submit(process_ac_adhoc, item) for item in input_dict.items()
|
||||
]
|
||||
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
try:
|
||||
result = future.result()
|
||||
except Exception as e:
|
||||
print(f"Error in future: {e}")
|
||||
traceback.print_exc()
|
||||
|
||||
# for item in input_dict.items():
|
||||
# process_ac_adhoc(item)
|
||||
@@ -1,584 +0,0 @@
|
||||
import concurrent.futures
|
||||
import traceback
|
||||
import csv
|
||||
import time
|
||||
import sys
|
||||
import random
|
||||
|
||||
random.seed(42)
|
||||
import pandas as pd
|
||||
import os
|
||||
import re
|
||||
|
||||
import utils
|
||||
import config
|
||||
import claude_funcs
|
||||
import file_processing
|
||||
import tracking
|
||||
import consolidate_output
|
||||
import top_down_funcs
|
||||
import valid
|
||||
import ac_funcs
|
||||
import preprocess
|
||||
import prompts
|
||||
import postprocess
|
||||
import postprocessing_funcs
|
||||
import preprocessing_funcs
|
||||
|
||||
# Ad-Hoc runs of sections of the script
|
||||
"""
|
||||
Batch 1 and 2
|
||||
Fields to Run:
|
||||
AC
|
||||
- All Chunked Fields
|
||||
- All Part 2 Fields
|
||||
- Postprocess AC
|
||||
|
||||
B
|
||||
- All B Postprocessing
|
||||
|
||||
750 additional
|
||||
- Run all ABC
|
||||
"""
|
||||
dup_mapping = {
|
||||
"Payer Name": "PAYER_NAME",
|
||||
"IRS Name": "IRS_Name",
|
||||
"NPI NAME": "NPI_NAME",
|
||||
"Sequestration Reductions, included [Medicare only] (Y/N)": "SEQUESTRATION_REDUCTIONS_IND",
|
||||
"Page_num": "page_num",
|
||||
"Lesser of Logic Language, included (Y/N)": "LESSER",
|
||||
"Reimb. Methodology_short": "SHORT_METHODOLOGY",
|
||||
"If rate is % of Payer or MCR [STANDARD]": "RATE_STANDARD",
|
||||
"If rate is % of Payer or MCR [STANDARD]_Short": "RATE_SHORT",
|
||||
"Flat Fee": "FLAT_FEE_STANDARD",
|
||||
"CDM Language, included (Y/N)": "CDM_IND",
|
||||
"CHARGEMASTER": "CONTRACT_CHARGEMASTER_PROTECTION_LANGUAGE",
|
||||
"EXCLUSIONS": "Exclusions",
|
||||
}
|
||||
|
||||
combined_mapping = {
|
||||
"Contract_Name": "Filename",
|
||||
"Agreement_Name_Contract_Title": "CONTRACT_TITLE",
|
||||
"Payer_Name": "PAYER_NAME",
|
||||
"Health_Plan_State": "HEALTH_PLAN_STATE",
|
||||
"Affiliate_Y_N": "AFFILIATION_CLAUSE_IND",
|
||||
"Credentialing_Application_Indicator": "CREDENTIALING_APP_IND",
|
||||
"Term_Clause": "TERM_CLAUSE",
|
||||
"Contract_Auto_Renewal_Indicator": "CONTRACT_AUTO_RENEWAL_IND",
|
||||
"Termination_Date": "CONTRACT_TERMINATION_DT",
|
||||
"Termination_Upon_Notice_Days": "TERMINATION_UPON_NOTICE",
|
||||
"Termination_With_Cause_Days": "TERMINATION_UPON_CAUSE",
|
||||
"Amend_Contract_Upon_notice_Flag_Y_N": "AMEND_CONTRACT_NOTICE_IND",
|
||||
"Timeframe_to_Object_Days": "TIME_TO_OBJECT",
|
||||
"Assignments_Clause_Y_N": "ASSIGNMENTS_CLAUSE_IND",
|
||||
"Contract_Effective_Date": "CONTRACT_EFFECTIVE_DT",
|
||||
"IRS_Num": "PROV_GROUP_TIN",
|
||||
"PROV_GROUP_NAME": "PROV_GROUP_NAME",
|
||||
"NPI_10_digits": "PROV_GROUP_NPI",
|
||||
"NPI_NAME": "NPI_NAME",
|
||||
"PROV_GROUP_TIN_SIGNATORY": "PROV_GROUP_TIN_SIGNATORY",
|
||||
"PROV_TIN_OTHER": "PROV_TIN_OTHER",
|
||||
"PROV_NPI_OTHER": "PROV_NPI_OTHER",
|
||||
"Notice_to_Provider_Name": "NOTICE_PROVIDER_NAME",
|
||||
"Notice_to_Provider_Address": "NOTICE_PROVIDER_ADDRESS",
|
||||
"Sequestration_Language": "SEQUESTRATION_LANGUAGE",
|
||||
"Sequestration_Reductions_included_Medicare_only_Y_N": "SEQUESTRATION_REDUCTIONS_IND",
|
||||
"Parent_Agreement_Code": "Parent Agreement Code",
|
||||
"Pages": "Pages",
|
||||
"Page_num": "page_num",
|
||||
"Attachment_Exhibit": "EXHIBIT",
|
||||
"Line_of_Business": "CONTRACT_LOB",
|
||||
"Provider_Type": "PROV_TYPE",
|
||||
"Provider_Type_Level_2": "PROV_TYPE_LEVEL_2",
|
||||
"IP_OP": "IP_OP",
|
||||
"Service_Type": "FULL_SERVICE",
|
||||
"Plan_Type": "CONTRACT_PROGRAM",
|
||||
"Lesser_of_Logic_Language_included_Y_N": "LESSER",
|
||||
"Lesser_of_Rate": "LESSER_RATE",
|
||||
"Reimb_Methodology": "FULL_METHODOLOGY",
|
||||
"Reimb_Methodology_short": "SHORT_METHODOLOGY",
|
||||
"If_rate_is_percent_of_Payer_or_MCR_STANDARD": "RATE_STANDARD",
|
||||
"If_rate_is_percent_of_Payer_or_MCR_STANDARD_Short": "RATE_SHORT",
|
||||
"Flat_Fee": "FLAT_FEE_STANDARD",
|
||||
"Default_Term": "DEFAULT_TERM",
|
||||
"Default_Rate": "DEFAULT_RATE",
|
||||
"Inclusion_of_essential_RBRVS_Fee_Source_Language_Y_N": 'Inclusion of essential RBRVS "Fee Source" Language (Y/N)',
|
||||
"CDM_Language_included_Y_N": "CDM_IND",
|
||||
"CHARGEMASTER": "CHARGEMASTER",
|
||||
"IP_DSH_IME_UC_included_Y_N": "IP - DSH/IME/UC, included (Y/N)",
|
||||
"IP_Stoploss_Catastrophic_Threshold": "IP - Stoploss Catastrophic Threshold",
|
||||
"EXCLUSIONS": "EXCLUSIONS",
|
||||
"Not_to_Exceed": "NOT_TO_EXCEED",
|
||||
"Escalator_or_COLA_Y_N": "RATE_ESCALATOR_IND",
|
||||
"Escalator_I_Eff_Date": "RATE_ESCALATOR_DT",
|
||||
}
|
||||
|
||||
|
||||
AC_DICT = {
|
||||
"ACCESS_TO_MEDICAL_RECORDS": "Access clause contains information regarding access to medical records. It may be found in section 4.2 of the contract. Extract the Access clause present in the contract. If it is not present, answer N/A.",
|
||||
"CARVEOUT_VENDORS": "Carve-Out Vendors clause may be found in section 2.9 of the contract. Extract the Carve-Out Vendors clause present in the contract. If it is not present, answer N/A.",
|
||||
"CLAIMS_EDITING_LANGUAGE": "Claims Editing Language is typically found in the claims section in the beginning parts of the contract. Extract Claims Editing Language present in the contract. If it is not present, answer N/A.",
|
||||
"CLEAN_CLAIM": 'Extract the "Clean Claim" subsection present in definition section of the contract. If it is not present, answer N/A.',
|
||||
"CONFLICTS_BETWEEN_CERTAIN_DOCUMENTS_LANGUAGE": "Extract the Conflicts Between Certain Documents clause present in the contract. If it is not present, answer N/A.",
|
||||
"COST_SETTLEMENT_LANGUAGE": "Cost settlement language may be present in disputes and arbitration subsection of the contract. It usually contains keywords like settlementor cost settlement. Extract the cost settlement language present in the contract. If it is not present, answer N/A.",
|
||||
"DEEMER_AMENDMENT": "Deemer Amendment clause may be found in section 8.7.2 of the contract. Extract the Deemer Amendment clause present in the contract. If it is not present, answer N/A.",
|
||||
"DELEGATED_TERMS": "Extract Delegated Terms present in the contract. If it is not present, answer N/A.",
|
||||
"ECM": 'What is the "ECM" number mentioned in the contract? Only Answer the ECM number, do not provide any context. Do not provide ICM number. If ECM number is not present, answer N/A.',
|
||||
"EXCLUSIVITY_REQUIREMENT_LANGUAGE": "Extract Exclusivity requirement clause present in the contract. If it is not present, answer N/A.",
|
||||
"GUARANTEE_OF_PROVIDER_YIELD_LANGUAGE": "Extract Guarantee of Provider Yield Language present in the contract. If it is not present, answer N/A.",
|
||||
"HCBS_SERVICES": 'Extract HCBS services clause present either in the compensation or exhibit section of the contract. It may contain keywords like "for covered HCBS services, plan shall pay". Do not include definitions of HCBS. If it is not present, answer N/A.',
|
||||
"INDEMNIFICATION": "Indemnification clause may be found in section 5.2 of the contract. Extract the Indemnification clause present in the contract. If it is not present, answer N/A.",
|
||||
"INDEPENDENT_REVIEW_LANGUAGE": "Extract Independent Review language present in the contract. If it is not present, answer N/A.",
|
||||
"INVOICE_PRICING_LANGUAGE": 'Extract table with heading "INVOICED SERVICES" if it is present in compensation schedule of the contract. If it is not present, answer N/A',
|
||||
"LATE_PAID_CLAIMS_LANGUAGE": "Extract Late Paid Claims language present in the contract. This will be a sentence or paragraph describing late paid claims, interest payments or similar. Here is an example of a correct response: 'Any Clean Claim, as defined in 42 C.F.R. § 422.500, shall be paid within thirty (30) days of receipt by Plan at such address as may be designated by Plan, and Plan shall pay interest on any Clean Claim not paid within thirty (30) days of such receipt by Plan at the rate of interest required by law, or as otherwise set forth in the Provider Manual.'. Here is an example of an incorrect response: 'Any Clean Claim, as defined in 42 C.F.R. 422.500, shall be paid within thirty (30) days of receipt by Health Plan, Payor or (if Provider contracts with Downstream Entities) Provider, as applicable, as designated by Provider or such Downstream Entity, as applicable.'. If no valid answer is present, answer N/A.",
|
||||
"MEMBER_CONFINEMENT_DAYS_LANGUAGE": "Extract Member Confinement Days Language present in the contract. If it is not present, answer N/A.",
|
||||
"NATIONAL_AGREEMENT_IND": "Are more than one states covered in the contract? Answer with a Yes or No, do not provide any other context.",
|
||||
"NETWORK_ACCESS_FEES_LANGUAGE": 'Extract "Network Access Fee" verbiage present in the contract. If it is not present, answer N/A.',
|
||||
"NONSTANDARD_APPEALS_PROCESS_LANGUAGE": "Extract Nonstandard Appeals Process language present in the contract. If it is not present, answer N/A.",
|
||||
"PAYMENT_IN_ADVANCE_OF_CLAIMS_SUBMISSION_LANGUAGE": "Extract Payment in Advance of Claims Submission language present in the contract. If it is not present, answer N/A.",
|
||||
"PAYOR": '"Payor" is usually found in section 1.12 of the contract. Extract the "Payor" subsection present in definition section of the contract. If it is not present, answer N/A.',
|
||||
"PMPM": "What is the PMPM rate mentioned in the contract? Only return the rate. do not provide any context. If it is not present, answer N/A.",
|
||||
"PREAUTHORIZATION": "Preauthorization clause may be found in section 2.7 of the contract. Extract the Preauthorization clause present in the contract. If it is not present, answer N/A.",
|
||||
"PRODUCT_REMOVAL": "Product removal may be found in the products attachment, stating a product is being removed. Extract product removal language present in the contract. If it is not present, answer N/A.",
|
||||
"PROVIDER_BASED_BILLING_EXCLUSION_LANGUAGE": "Provider-based Billing Exclusion is often listed under the ‘Additional Provisions’ section. Extract the Provider-based Billing Exclusion language mentioned in the contract. If it is not present, answer N/A.",
|
||||
"RECOVERY_RIGHTS": "Recovery Rights clause may be found in section 3.5 of the contract. Extract the entire Recovery Rights sentence or paragraph present in the contract. If it is not present, answer N/A.",
|
||||
"TEMPLATE": "Is this contract in standard Centene template containing definition, terms clauses and other standard sections. Answer with a Yes or No, do not provide any other context.",
|
||||
"ARBITRATION_AND_DISPUTES": "Dispute Resolution clause is usually found in section 6.1 of the contract. Arbitration clause is usually found in section 6.2 of the contract. Extract both the Dispute Resolution and the arbitration clause present in the contract. In case one of them is present, extract that one. If none of them are present, answer N/A.",
|
||||
"DISPARAGEMENT_PROHIBITION_IND": "Is Disparagement Prohibition clause present in the contract? It may be found in section 2.1 of the contract. Answer with a Yes or No, do not provide any other context.",
|
||||
"DISPARAGEMENT_PROHIBITION_LANGUAGE": "Disparagement Prohibition clause may be found in section 2.1 of the contract. Extract the Disparagement Prohibition clause present in the contract. If it is not present, answer N/A.",
|
||||
"ELIGIBILITY_VERIFICATION": "Eligibility Verification or Determination clause may be found in section 2.6 of the contract. Extract the Eligibility Verification or Determination clause present in the contract. If it is not present, answer N/A.",
|
||||
"INSURANCE_REQUIREMENT": "Extract Insurance requirement mentioned in the contract. If it is not present, answer N/A.",
|
||||
"PARTICIPATION_IN_PRODUCTS": "Participation in Products clause may be found in section 2.2.2 of the contract. Extract the Participation in Products clause present in the contract. If it is not present, answer N/A.",
|
||||
"POLICIES_AND_PROCEDURES": "Extract the entire Policies and Procedures sentence or paragraph present in the contract. Do not extract language about conflicts and construction. Do not extract language about PP (Preferred Provider) nor PPG (Preferred Provider Group). If it is not present, answer N/A.",
|
||||
"REGULATORY_REQUIREMENTS": 'Extract the entire Regulatory Requirements paragraph present in the contract. It usually contains the keyword "Regulatory Requirements". If it is not present, answer N/A.',
|
||||
"RELATIONSHIP_OF_PARTIES_LANGUAGE": "Extract the entire Relationship of Parties passage present in the contract. If it is not present, answer N/A.",
|
||||
"TERM_CLAUSE": """Extract the subsection or paragraph labeled Term. It may also be labeled Termination. If there are no such sections, extract the paragraph the most closely resembles this label. If there is still no correct answer, simply return 'N/A'.""",
|
||||
"LATE_PAID_CLAIMS_LANGUAGE": "Extract Late Paid Claims language present in the contract. This will be a sentence or paragraph describing late paid claims, interest payments or similar. Here is an example of a correct response: 'Any Clean Claim, as defined in 42 C.F.R. § 422.500, shall be paid within thirty (30) days of receipt by Plan at such address as may be designated by Plan, and Plan shall pay interest on any Clean Claim not paid within thirty (30) days of such receipt by Plan at the rate of interest required by law, or as otherwise set forth in the Provider Manual.'. Here is an example of an incorrect response: 'Any Clean Claim, as defined in 42 C.F.R. 422.500, shall be paid within thirty (30) days of receipt by Health Plan, Payor or (if Provider contracts with Downstream Entities) Provider, as applicable, as designated by Provider or such Downstream Entity, as applicable.'. If no valid answer is present, answer N/A.",
|
||||
"NON_RENEWAL_LANGUAGE": "Extract the section of the contract that describes non-renewal obligations, especially the number of days notice that must be given. Write the full relevant sentence or paragraph. Ensure that the exact term 'non-renewal' is present in the answer. Do NOT return an answer that does not have 'non-renewal'.",
|
||||
"HCBS_SERVICES": 'Extract HCBS services clause present either in the compensation or exhibit section of the contract. It may contain keywords like "for covered HCBS services, plan shall pay". Do not include definitions of HCBS. If it is not present, answer N/A.',
|
||||
}
|
||||
|
||||
KEYWORD_MAPPINGS = {
|
||||
"TERM_CLAUSE": {
|
||||
"methodology": "hierarchy",
|
||||
"keywords": ["Term", "term", "duration", "agreement period"],
|
||||
},
|
||||
"LATE_PAID_CLAIMS_LANGUAGE": {
|
||||
"methodology": "hierarchy",
|
||||
"keywords": [
|
||||
"late paid",
|
||||
"late payment",
|
||||
"interest",
|
||||
"late fee",
|
||||
"interest shall be paid",
|
||||
"interest payment",
|
||||
],
|
||||
},
|
||||
"NON_RENEWAL_LANGUAGE": {
|
||||
"methodology": "or",
|
||||
"keywords": ["not to renew", "non-renewal", "non renewal", "nonrenewal"],
|
||||
},
|
||||
# "NON_RENEWAL_DAYS": {'methodology': 'or', 'keywords': ["not to renew", "non-renewal", "non renewal", "nonrenewal"]},
|
||||
"HCBS_SERVICES": {
|
||||
"methodology": "hierarchy",
|
||||
"keywords": ["%", "Plan shall pay", "hcbs"],
|
||||
},
|
||||
"POLICIES_AND_PROCEDURES": {
|
||||
"methodology": "or",
|
||||
"keywords": ["2.4", "policies", "procedures"],
|
||||
},
|
||||
"REGULATORY_REQUIREMENTS": {
|
||||
"methodology": "or",
|
||||
"keywords": ["8.3", "6.4", "regulatory requirements", "regulatory"],
|
||||
},
|
||||
"RECOVERY_RIGHTS": {"methodology": "or", "keywords": ["3.5", "recovery rights"]},
|
||||
"RELATIONSHIP_OF_PARTIES_LANGUAGE": {
|
||||
"methodology": "or",
|
||||
"keywords": ["8.1", "relationship of parties"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def run_ac_prompts(file_object):
|
||||
################## INITIATE PROCESSING ##################
|
||||
filename, contract_text = file_object
|
||||
print(f"Processing AC for {filename}...")
|
||||
# print(f"Total contract word count: {len(contract_text.split())}")
|
||||
|
||||
################## PREPROCESS ##################
|
||||
text_dict, exhibit_pages, num_pages, ac_chunks = preprocess.preprocess(
|
||||
contract_text, filename, fields="ac"
|
||||
)
|
||||
# print(f"AC Preprocessing Complete - {filename}")
|
||||
|
||||
################## RUN FULL CONTEXT PROMPTS ##################
|
||||
ac_dict = {}
|
||||
log_data = []
|
||||
|
||||
# Process 'full_context' fields together
|
||||
full_context_fields = [field for field in AC_DICT if field not in KEYWORD_MAPPINGS]
|
||||
full_context_questions = {
|
||||
field: prompts.AC_DICT.get(field) for field in full_context_fields
|
||||
}
|
||||
|
||||
if full_context_questions:
|
||||
full_context_prompt = ac_funcs.create_prompt(
|
||||
contract_text[0 : min(len(contract_text), 600000)],
|
||||
question=full_context_questions,
|
||||
)
|
||||
# print(f"High accuracy prompt word count: {len(high_accuracy_prompt.split())}")
|
||||
try:
|
||||
full_context_answers = ac_funcs.get_ac_answer(
|
||||
full_context_prompt,
|
||||
filename,
|
||||
fields=list(full_context_questions.keys()),
|
||||
)
|
||||
ac_dict.update(full_context_answers)
|
||||
|
||||
for field, answer in full_context_answers.items():
|
||||
log_data.append(
|
||||
{
|
||||
"Field": field,
|
||||
"Prompt": f"Question: {full_context_questions[field]}\n\nContext: {contract_text}", # Include full context
|
||||
"Context Type": "Full Context (High Accuracy)",
|
||||
"Response": answer,
|
||||
}
|
||||
)
|
||||
except Exception as e:
|
||||
# print(f"Error processing high accuracy fields: {str(e)}")
|
||||
for field in full_context_questions.keys():
|
||||
ac_dict[field] = f"Error: {str(e)}"
|
||||
log_data.append(
|
||||
{
|
||||
"Field": field,
|
||||
"Prompt": f"Question: {full_context_questions[field]}\n\nContext: {contract_text}", # Include full context
|
||||
"Context Type": "Full Context (High Accuracy)",
|
||||
"Response": f"Error: {str(e)}",
|
||||
}
|
||||
)
|
||||
|
||||
################## RUN CHUNKED PROMPTS ##################
|
||||
all_fields = set(prompts.AC_DICT.keys())
|
||||
chunked_fields = [field for field in KEYWORD_MAPPINGS]
|
||||
|
||||
for field in chunked_fields:
|
||||
if field in prompts.AC_DICT:
|
||||
question = prompts.AC_DICT[field]
|
||||
else:
|
||||
# print(f"Field {field} not found in prompts. Skipping...")
|
||||
continue
|
||||
|
||||
# Use chunk if available and different from full context, otherwise use full context
|
||||
if (
|
||||
field in ac_chunks
|
||||
and ac_chunks[field].strip()
|
||||
and ac_chunks[field] != contract_text
|
||||
):
|
||||
context = ac_chunks[field]
|
||||
context_type = "Smart Chunking"
|
||||
else:
|
||||
context = contract_text
|
||||
context_type = "Full Context"
|
||||
|
||||
prompt = prompts.AC_SINGLE_FIELD_TEMPLATE(context, question)
|
||||
|
||||
print(
|
||||
f"Field {field} prompt word count: {len(prompt.split())} ({context_type})"
|
||||
)
|
||||
|
||||
try:
|
||||
field_answer = claude_funcs.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 8192
|
||||
)
|
||||
ac_dict[field] = field_answer
|
||||
|
||||
log_data.append(
|
||||
{
|
||||
"Field": field,
|
||||
"Prompt": f"Question: {question}\n\nContext: {context}", # Include full context
|
||||
"Context Type": context_type,
|
||||
"Response": field_answer,
|
||||
}
|
||||
)
|
||||
except Exception as e:
|
||||
# print(f"Error processing field {field}: {str(e)}")
|
||||
ac_dict[field] = f"Error: {str(e)}"
|
||||
log_data.append(
|
||||
{
|
||||
"Field": field,
|
||||
"Prompt": f"Question: {question}\n\nContext: {context}", # Include full context
|
||||
"Context Type": context_type,
|
||||
"Response": f"Error: {str(e)}",
|
||||
}
|
||||
)
|
||||
|
||||
################## AC CONDITIONAL PROMPTS ##################
|
||||
## Non-Renewal - Days
|
||||
nrd_prompt = prompts.AC_SINGLE_FIELD_TEMPLATE(
|
||||
ac_dict["NON_RENEWAL_LANGUAGE"], prompts.AC_DICT["NON_RENEWAL_DAYS"]
|
||||
)
|
||||
nrd_answer = claude_funcs.invoke_claude(
|
||||
nrd_prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 8192
|
||||
)
|
||||
ac_dict["NON_RENEWAL_DAYS"] = nrd_answer
|
||||
|
||||
################## ENSURE ALL FIELDS ARE PRESENT ##################
|
||||
ac_dict = {k: v for k, v in ac_dict.items() if k in valid.AC_MAPPING}
|
||||
for field in all_fields:
|
||||
if field not in ac_dict:
|
||||
# print(f"Field not found in results. {field}")
|
||||
# ac_dict[field] = "N/A"
|
||||
log_data.append(
|
||||
{
|
||||
"Field": field,
|
||||
"Prompt": "N/A",
|
||||
"Context Type": "N/A",
|
||||
"Response": "N/A",
|
||||
}
|
||||
)
|
||||
|
||||
################## CREATE OUTPUT DIRECTORIES ##################
|
||||
base_filename = os.path.splitext(filename)[0].strip()
|
||||
output_dir = os.path.join(config.OUTPUT_DIRECTORY, base_filename)
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
################## POSTPROCESS ##################
|
||||
ac_dict["Contract Name"] = filename
|
||||
ac_df = pd.DataFrame([ac_dict])
|
||||
ac_df = postprocess.ac_postprocess(ac_df)
|
||||
|
||||
################## WRITE TO OUTPUT ##################
|
||||
ac_df.to_csv(f"{output_dir}/ac_output.csv", index=False)
|
||||
print(f"AC Output written: {filename}")
|
||||
|
||||
################## WRITE LOG TO CSV ##################
|
||||
# log_file_path = os.path.join(output_dir, f"{base_filename}_prompt_log.csv")
|
||||
# with open(log_file_path, 'w', newline='', encoding='utf-8') as log_file:
|
||||
# fieldnames = ['Field', 'Prompt', 'Context Type', 'Response']
|
||||
# writer = csv.DictWriter(log_file, fieldnames=fieldnames)
|
||||
# writer.writeheader()
|
||||
# writer.writerows(log_data)
|
||||
# print(f"Prompt log written to {log_file_path}")
|
||||
return ac_dict
|
||||
|
||||
|
||||
def process_ac_adhoc(item):
|
||||
filename, contract_text = item
|
||||
try:
|
||||
results = run_ac_prompts(item)
|
||||
if results is not None:
|
||||
tracking.update_results_csv(item, results)
|
||||
return item, results
|
||||
except Exception as e:
|
||||
print(f"Error processing item {filename}: {e}")
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
def preprocess_ad_hoc(contract_text, filename, fields=config.FIELDS):
|
||||
|
||||
contract_text = preprocessing_funcs.clean_newlines(contract_text)
|
||||
contract_text = preprocessing_funcs.clean_law_symbols(contract_text)
|
||||
|
||||
text_dict = preprocessing_funcs.split_text(
|
||||
contract_text
|
||||
) # return a dictionary with keys - page_num (str), values as the page_text
|
||||
|
||||
num_pages = len(text_dict.keys()) - 1
|
||||
text_dict = preprocessing_funcs.filter_quick_review(text_dict)
|
||||
|
||||
if fields == "b":
|
||||
exhibit_pages = top_down_funcs.get_exhibit_pages(
|
||||
text_dict, filename
|
||||
) # All pages with exhibit headers
|
||||
# text_dict = table_funcs.align_and_format_tables(text_dict, filename)
|
||||
text_dict = preprocessing_funcs.chunk_consecutive(text_dict, exhibit_pages)
|
||||
ac_chunks = ""
|
||||
if fields == "ac":
|
||||
exhibit_pages = []
|
||||
ac_chunks = preprocessing_funcs.smart_chunk_ac(text_dict)
|
||||
|
||||
return text_dict, exhibit_pages, num_pages, ac_chunks
|
||||
|
||||
|
||||
def get_add_on(final_dicts, text_dict):
|
||||
add_on_dict = {}
|
||||
for td_page in list(set([d["page_num"] for d in final_dicts])):
|
||||
td_page = str(td_page)
|
||||
# TD page is the original result top down page, it is not necessarily in the new text_dict.
|
||||
# Find add_on_page = the key of the new text_dict that points to the correct page of the old text dict
|
||||
|
||||
if td_page in text_dict.keys():
|
||||
add_on_page = td_page
|
||||
else:
|
||||
add_on_page = [
|
||||
str(page) for page in text_dict.keys() if int(page) > int(td_page)
|
||||
][0]
|
||||
|
||||
if not add_on_page:
|
||||
add_on_dict[td_page] = {
|
||||
"ADD_ON_REIMBURSEMENT_LANGUAGE": "N/A",
|
||||
"ADD_ON_REIMBURSEMENT_IND": "N",
|
||||
}
|
||||
continue
|
||||
|
||||
print(f"TD Page: {td_page} | Add On Page: {add_on_page}")
|
||||
|
||||
ad = {}
|
||||
|
||||
add_on_prompt = f"## Page Start ## {text_dict[str(add_on_page)]} ## Page End ## . Extract the Add On Reimbursement language present in the page above. If there is no Add On language present, answer N/A. Write only the answer, with no additional commentary or explanation."
|
||||
add_on_answer = claude_funcs.invoke_claude(
|
||||
add_on_prompt, config.MODEL_ID_CLAUDE35_SONNET, "", 256
|
||||
)
|
||||
|
||||
ad["ADD_ON_REIMBURSEMENT_LANGUAGE"] = add_on_answer
|
||||
|
||||
if "N/A" in ad["ADD_ON_REIMBURSEMENT_LANGUAGE"]:
|
||||
ad["ADD_ON_REIMBURSEMENT_IND"] = "N"
|
||||
else:
|
||||
ad["ADD_ON_REIMBURSEMENT_IND"] = "Y"
|
||||
add_on_dict[str(td_page)] = ad
|
||||
|
||||
return add_on_dict
|
||||
|
||||
|
||||
def process_b_adhoc(item):
|
||||
filename, contract_text = item
|
||||
print(f"Processing {filename}")
|
||||
|
||||
merged_dicts = [d for d in abc_dicts if d["Filename"] + ".txt" == filename]
|
||||
print(len(merged_dicts))
|
||||
|
||||
if len(merged_dicts) > 0:
|
||||
|
||||
text_dict, exhibit_pages, num_pages, ac_chunks = preprocess_ad_hoc(
|
||||
contract_text, filename
|
||||
)
|
||||
print(f"Preprocessing {filename}")
|
||||
|
||||
# Exclusions
|
||||
exclusion_dict = top_down_funcs.get_td_dict(
|
||||
text_dict, filename, prompts.TOP_DOWN_EXCLUSIONS, valid.EXCLUSION_TERMS
|
||||
)
|
||||
final_dicts = top_down_funcs.add_exclusions(merged_dicts, exclusion_dict)
|
||||
print(f"Exclusions - {filename}")
|
||||
|
||||
# Medically Necessary
|
||||
medically_necessary_dict = top_down_funcs.get_td_dict(
|
||||
text_dict,
|
||||
filename,
|
||||
prompts.TOP_DOWN_MEDICALLY_NECESSARY,
|
||||
valid.VALID_MEDICALLY_NECESSARY,
|
||||
)
|
||||
medically_necessary_dict = postprocessing_funcs.filter_dict(
|
||||
medically_necessary_dict,
|
||||
pattern=re.compile(
|
||||
"|".join(
|
||||
re.escape(keyword) for keyword in valid.INVALID_MEDICALLY_NECESSARY
|
||||
),
|
||||
re.IGNORECASE,
|
||||
),
|
||||
)
|
||||
final_dicts = top_down_funcs.add_medically_necessary(
|
||||
final_dicts, medically_necessary_dict
|
||||
)
|
||||
print(f"Medically Necessary - {filename}")
|
||||
|
||||
# Add On
|
||||
add_on_dict = get_add_on(final_dicts, text_dict)
|
||||
print(f"Add On Results: {add_on_dict}")
|
||||
print(f"Add On - {filename}")
|
||||
|
||||
for td_page in add_on_dict.keys():
|
||||
for d in final_dicts:
|
||||
if str(d["page_num"]) == str(td_page):
|
||||
print("Add On Match")
|
||||
d.update(add_on_dict[str(td_page)])
|
||||
|
||||
# print(len(final_dicts))
|
||||
|
||||
output_df = pd.DataFrame(final_dicts)
|
||||
print(f"B finished -{filename} - output {output_df.shape}")
|
||||
print(output_df.columns)
|
||||
|
||||
base_filename = os.path.splitext(filename)[0].strip()
|
||||
output_dir = os.path.join(config.OUTPUT_DIRECTORY, base_filename)
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
output_df.to_csv(f"{output_dir}/b_output.csv")
|
||||
|
||||
|
||||
########################## Process Starts Here ##########################
|
||||
individual_path = "output_individual/cnc_batch1_b"
|
||||
clean_path = "reference_files/CNC-Batch 1 Output_File1.xlsx"
|
||||
run_path = "reference_files/Batch 1_082024.xlsx"
|
||||
dup_path = "reference_files/Batch 1 Duplicates_10092024.xlsx"
|
||||
|
||||
# Run List
|
||||
run_list = pd.read_excel(run_path)
|
||||
run_list = list(run_list[run_list.Status == "1A"]["Filename"])
|
||||
print(f"Run List Length: {len(run_list)}")
|
||||
|
||||
# Input
|
||||
input_dict = utils.read_input()
|
||||
print(f"Input Dict Length: {len(input_dict)}")
|
||||
|
||||
# Current Output
|
||||
output_dirs = list(os.listdir(individual_path))
|
||||
print(f"Current Outputs: {len(output_dirs)}")
|
||||
|
||||
# Need to Run
|
||||
input_dict = {
|
||||
filename: page_text
|
||||
for filename, page_text in input_dict.items()
|
||||
if filename.split(".txt")[0] in run_list
|
||||
} # and filename.split('.txt')[0] not in config.OUTPUT_DIRECTORY}
|
||||
print(f"Need to Run: {len(input_dict)}")
|
||||
|
||||
# Clean ABC
|
||||
abc = pd.read_excel(clean_path)
|
||||
abc.rename(columns=combined_mapping, inplace=True)
|
||||
abc_names = list(abc["Filename"].unique())
|
||||
abc_dicts = abc.to_dict(orient="records")
|
||||
print(f"ABC Clean File Length (B): {len(abc_names)}")
|
||||
|
||||
# # ABC Dups
|
||||
abc_dups = pd.read_excel(dup_path)
|
||||
abc_dups.rename(columns={v: k for k, v in valid.B_MAPPING.items()}, inplace=True)
|
||||
abc_dups.rename(columns={v: k for k, v in valid.AC_MAPPING.items()}, inplace=True)
|
||||
abc_dups.rename(columns=dup_mapping, inplace=True)
|
||||
abc_dup_names = list(abc_dups["Filename"].unique())
|
||||
print(f"ABC Dup File Length (B): {len(abc_dup_names)}")
|
||||
|
||||
# # Concat ABC with Dups
|
||||
abc = pd.concat([abc, abc_dups], axis=0, ignore_index=True)
|
||||
abc_names = list(abc["Filename"].unique())
|
||||
abc_dicts = abc.to_dict(orient="records")
|
||||
print(f"Final ABC File Length (B+Dups): {len(abc_names)}")
|
||||
|
||||
# Run only files that are in abc, are in input, and are not in output
|
||||
input_dict = {
|
||||
k: v for k, v in input_dict.items() if k.split(".txt")[0] in abc_dup_names
|
||||
}
|
||||
input_dict = {
|
||||
k: v
|
||||
for k, v in input_dict.items()
|
||||
if k
|
||||
== "04-3122933-Dentsply IH, Inc. dba Wellspect Healthcare-ICMProviderAgreement_72694(2).txt"
|
||||
}
|
||||
print("Input dict to run: ", len(input_dict))
|
||||
|
||||
# with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor:
|
||||
# if 'a' in config.FIELDS and 'c' in config.FIELDS:
|
||||
# futures = [executor.submit(process_ac_adhoc, item) for item in input_dict.items()]
|
||||
# if 'b' in config.FIELDS:
|
||||
# futures = [executor.submit(process_b_adhoc, item) for item in input_dict.items()]
|
||||
|
||||
# for future in concurrent.futures.as_completed(futures):
|
||||
# try:
|
||||
# result = future.result()
|
||||
# except Exception as e:
|
||||
# print(f"Error in future: {e}")
|
||||
# traceback.print_exc()
|
||||
|
||||
|
||||
for item in input_dict.items():
|
||||
process_b_adhoc(item)
|
||||
@@ -1,591 +0,0 @@
|
||||
import concurrent.futures
|
||||
import traceback
|
||||
import csv
|
||||
import time
|
||||
import sys
|
||||
import random
|
||||
|
||||
random.seed(42)
|
||||
import pandas as pd
|
||||
import os
|
||||
import re
|
||||
|
||||
import utils
|
||||
import config
|
||||
import claude_funcs
|
||||
import file_processing
|
||||
import tracking
|
||||
import consolidate_output
|
||||
import top_down_funcs
|
||||
import valid
|
||||
import ac_funcs
|
||||
import preprocess
|
||||
import prompts
|
||||
import postprocess
|
||||
import postprocessing_funcs
|
||||
import preprocessing_funcs
|
||||
|
||||
input_path = "data_cnc/batch3"
|
||||
ac2_output_path = "output_individual/cnc_3_ac2"
|
||||
b_output_path = "output_individual/cnc_3_b"
|
||||
clean_path = "reference_files/CNC-Batch 3 Output_File1.xlsx"
|
||||
run_path = "reference_files/Batch 3_091124.xlsx"
|
||||
|
||||
|
||||
# Ad-Hoc runs of sections of the script
|
||||
"""
|
||||
Batch 1 and 2
|
||||
Fields to Run:
|
||||
AC
|
||||
- All Chunked Fields
|
||||
- All Part 2 Fields
|
||||
- Postprocess AC
|
||||
|
||||
B
|
||||
- All B Postprocessing
|
||||
|
||||
750 additional
|
||||
- Run all ABC
|
||||
"""
|
||||
dup_mapping = {
|
||||
"Payer Name": "PAYER_NAME",
|
||||
"IRS Name": "IRS_Name",
|
||||
"NPI NAME": "NPI_NAME",
|
||||
"Sequestration Reductions, included [Medicare only] (Y/N)": "SEQUESTRATION_REDUCTIONS_IND",
|
||||
"Page_num": "page_num",
|
||||
"Lesser of Logic Language, included (Y/N)": "LESSER",
|
||||
"Reimb. Methodology_short": "SHORT_METHODOLOGY",
|
||||
"If rate is % of Payer or MCR [STANDARD]": "RATE_STANDARD",
|
||||
"If rate is % of Payer or MCR [STANDARD]_Short": "RATE_SHORT",
|
||||
"Flat Fee": "FLAT_FEE_STANDARD",
|
||||
"CDM Language, included (Y/N)": "CDM_IND",
|
||||
"CHARGEMASTER": "CONTRACT_CHARGEMASTER_PROTECTION_LANGUAGE",
|
||||
"EXCLUSIONS": "Exclusions",
|
||||
}
|
||||
|
||||
combined_mapping = {
|
||||
"Contract_Name": "Filename",
|
||||
"Agreement_Name_Contract_Title": "CONTRACT_TITLE",
|
||||
"Payer_Name": "PAYER_NAME",
|
||||
"Health_Plan_State": "HEALTH_PLAN_STATE",
|
||||
"Affiliate_Y_N": "AFFILIATION_CLAUSE_IND",
|
||||
"Credentialing_Application_Indicator": "CREDENTIALING_APP_IND",
|
||||
"Term_Clause": "TERM_CLAUSE",
|
||||
"Contract_Auto_Renewal_Indicator": "CONTRACT_AUTO_RENEWAL_IND",
|
||||
"Termination_Date": "CONTRACT_TERMINATION_DT",
|
||||
"Termination_Upon_Notice_Days": "TERMINATION_UPON_NOTICE",
|
||||
"Termination_With_Cause_Days": "TERMINATION_UPON_CAUSE",
|
||||
"Amend_Contract_Upon_notice_Flag_Y_N": "AMEND_CONTRACT_NOTICE_IND",
|
||||
"Timeframe_to_Object_Days": "TIME_TO_OBJECT",
|
||||
"Assignments_Clause_Y_N": "ASSIGNMENTS_CLAUSE_IND",
|
||||
"Contract_Effective_Date": "CONTRACT_EFFECTIVE_DT",
|
||||
"IRS_Num": "PROV_GROUP_TIN",
|
||||
"PROV_GROUP_NAME": "PROV_GROUP_NAME",
|
||||
"NPI_10_digits": "PROV_GROUP_NPI",
|
||||
"NPI_NAME": "NPI_NAME",
|
||||
"PROV_GROUP_TIN_SIGNATORY": "PROV_GROUP_TIN_SIGNATORY",
|
||||
"PROV_TIN_OTHER": "PROV_TIN_OTHER",
|
||||
"PROV_NPI_OTHER": "PROV_NPI_OTHER",
|
||||
"Notice_to_Provider_Name": "NOTICE_PROVIDER_NAME",
|
||||
"Notice_to_Provider_Address": "NOTICE_PROVIDER_ADDRESS",
|
||||
"Sequestration_Language": "SEQUESTRATION_LANGUAGE",
|
||||
"Sequestration Reductions, included [Medicare only] (Y/N)": "SEQUESTRATION_REDUCTIONS_IND",
|
||||
"Parent_Agreement_Code": "Parent Agreement Code",
|
||||
"Pages": "Pages",
|
||||
"Page_num": "page_num",
|
||||
"Attachment_Exhibit": "EXHIBIT",
|
||||
"Line_of_Business": "CONTRACT_LOB",
|
||||
"Provider_Type": "PROV_TYPE",
|
||||
"Provider_Type_Level_2": "PROV_TYPE_LEVEL_2",
|
||||
"IP_OP": "IP_OP",
|
||||
"Service_Type": "FULL_SERVICE",
|
||||
"Plan_Type": "CONTRACT_PROGRAM",
|
||||
"Lesser_of_Logic_Language_included_Y_N": "LESSER",
|
||||
"Lesser_of_Rate": "LESSER_RATE",
|
||||
"Reimb_Methodology": "FULL_METHODOLOGY",
|
||||
"Reimb_Methodology_short": "SHORT_METHODOLOGY",
|
||||
"If rate is % of Payer or MCR [STANDARD]": "RATE_STANDARD",
|
||||
"If rate is % of Payer or MCR [STANDARD]_Short": "RATE_SHORT",
|
||||
"Flat Fee": "FLAT_FEE_STANDARD",
|
||||
"Default_Term": "DEFAULT_TERM",
|
||||
"Default_Rate": "DEFAULT_RATE",
|
||||
"IP - DSH/IME/UC, included (Y/N)": 'Inclusion of essential RBRVS "Fee Source" Language (Y/N)',
|
||||
"CDM_Language_included_Y_N": "CDM_IND",
|
||||
"CHARGEMASTER": "CHARGEMASTER",
|
||||
"IP_DSH_IME_UC_included_Y_N": "IP - DSH/IME/UC, included (Y/N)",
|
||||
"IP_Stoploss_Catastrophic_Threshold": "IP - Stoploss Catastrophic Threshold",
|
||||
"EXCLUSIONS": "EXCLUSIONS",
|
||||
"Not_to_Exceed": "NOT_TO_EXCEED",
|
||||
"Escalator_or_COLA_Y_N": "RATE_ESCALATOR_IND",
|
||||
"Escalator_I_Eff_Date": "RATE_ESCALATOR_DT",
|
||||
}
|
||||
|
||||
|
||||
AC_DICT = {
|
||||
"ACCESS_TO_MEDICAL_RECORDS": "Access clause contains information regarding access to medical records. It may be found in section 4.2 of the contract. Extract the Access clause present in the contract. If it is not present, answer N/A.",
|
||||
"CARVEOUT_VENDORS": "Carve-Out Vendors clause may be found in section 2.9 of the contract. Extract the Carve-Out Vendors clause present in the contract. If it is not present, answer N/A.",
|
||||
"CLAIMS_EDITING_LANGUAGE": "Claims Editing Language is typically found in the claims section in the beginning parts of the contract. Extract Claims Editing Language present in the contract. If it is not present, answer N/A.",
|
||||
"CLEAN_CLAIM": 'Extract the "Clean Claim" subsection present in definition section of the contract. If it is not present, answer N/A.',
|
||||
"CONFLICTS_BETWEEN_CERTAIN_DOCUMENTS_LANGUAGE": "Extract the Conflicts Between Certain Documents clause present in the contract. If it is not present, answer N/A.",
|
||||
"COST_SETTLEMENT_LANGUAGE": "Cost settlement language may be present in disputes and arbitration subsection of the contract. It usually contains keywords like settlementor cost settlement. Extract the cost settlement language present in the contract. If it is not present, answer N/A.",
|
||||
"DEEMER_AMENDMENT": "Deemer Amendment clause may be found in section 8.7.2 of the contract. Extract the Deemer Amendment clause present in the contract. If it is not present, answer N/A.",
|
||||
"DELEGATED_TERMS": "Extract Delegated Terms present in the contract. If it is not present, answer N/A.",
|
||||
"ECM": 'What is the "ECM" number mentioned in the contract? Only Answer the ECM number, do not provide any context. Do not provide ICM number. If ECM number is not present, answer N/A.',
|
||||
"EXCLUSIVITY_REQUIREMENT_LANGUAGE": "Extract Exclusivity requirement clause present in the contract. If it is not present, answer N/A.",
|
||||
"GUARANTEE_OF_PROVIDER_YIELD_LANGUAGE": "Extract Guarantee of Provider Yield Language present in the contract. If it is not present, answer N/A.",
|
||||
"HCBS_SERVICES": 'Extract HCBS services clause present either in the compensation or exhibit section of the contract. It may contain keywords like "for covered HCBS services, plan shall pay". Do not include definitions of HCBS. If it is not present, answer N/A.',
|
||||
"INDEMNIFICATION": "Indemnification clause may be found in section 5.2 of the contract. Extract the Indemnification clause present in the contract. If it is not present, answer N/A.",
|
||||
"INDEPENDENT_REVIEW_LANGUAGE": "Extract Independent Review language present in the contract. If it is not present, answer N/A.",
|
||||
"INVOICE_PRICING_LANGUAGE": 'Extract table with heading "INVOICED SERVICES" if it is present in compensation schedule of the contract. If it is not present, answer N/A',
|
||||
"LATE_PAID_CLAIMS_LANGUAGE": "Extract Late Paid Claims language present in the contract. This will be a sentence or paragraph describing late paid claims, interest payments or similar. Here is an example of a correct response: 'Any Clean Claim, as defined in 42 C.F.R. § 422.500, shall be paid within thirty (30) days of receipt by Plan at such address as may be designated by Plan, and Plan shall pay interest on any Clean Claim not paid within thirty (30) days of such receipt by Plan at the rate of interest required by law, or as otherwise set forth in the Provider Manual.'. Here is an example of an incorrect response: 'Any Clean Claim, as defined in 42 C.F.R. 422.500, shall be paid within thirty (30) days of receipt by Health Plan, Payor or (if Provider contracts with Downstream Entities) Provider, as applicable, as designated by Provider or such Downstream Entity, as applicable.'. If no valid answer is present, answer N/A.",
|
||||
"MEMBER_CONFINEMENT_DAYS_LANGUAGE": "Extract Member Confinement Days Language present in the contract. If it is not present, answer N/A.",
|
||||
"NATIONAL_AGREEMENT_IND": "Are more than one states covered in the contract? Answer with a Yes or No, do not provide any other context.",
|
||||
"NETWORK_ACCESS_FEES_LANGUAGE": 'Extract "Network Access Fee" verbiage present in the contract. If it is not present, answer N/A.',
|
||||
"NONSTANDARD_APPEALS_PROCESS_LANGUAGE": "Extract Nonstandard Appeals Process language present in the contract. If it is not present, answer N/A.",
|
||||
"PAYMENT_IN_ADVANCE_OF_CLAIMS_SUBMISSION_LANGUAGE": "Extract Payment in Advance of Claims Submission language present in the contract. If it is not present, answer N/A.",
|
||||
"PAYOR": '"Payor" is usually found in section 1.12 of the contract. Extract the "Payor" subsection present in definition section of the contract. If it is not present, answer N/A.',
|
||||
"PMPM": "What is the PMPM rate mentioned in the contract? Only return the rate. do not provide any context. If it is not present, answer N/A.",
|
||||
"PREAUTHORIZATION": "Preauthorization clause may be found in section 2.7 of the contract. Extract the Preauthorization clause present in the contract. If it is not present, answer N/A.",
|
||||
"PRODUCT_REMOVAL": "Product removal may be found in the products attachment, stating a product is being removed. Extract product removal language present in the contract. If it is not present, answer N/A.",
|
||||
"PROVIDER_BASED_BILLING_EXCLUSION_LANGUAGE": "Provider-based Billing Exclusion is often listed under the ‘Additional Provisions’ section. Extract the Provider-based Billing Exclusion language mentioned in the contract. If it is not present, answer N/A.",
|
||||
"RECOVERY_RIGHTS": "Recovery Rights clause may be found in section 3.5 of the contract. Extract the entire Recovery Rights sentence or paragraph present in the contract. If it is not present, answer N/A.",
|
||||
"TEMPLATE": "Is this contract in standard Centene template containing definition, terms clauses and other standard sections. Answer with a Yes or No, do not provide any other context.",
|
||||
"ARBITRATION_AND_DISPUTES": "Dispute Resolution clause is usually found in section 6.1 of the contract. Arbitration clause is usually found in section 6.2 of the contract. Extract both the Dispute Resolution and the arbitration clause present in the contract. In case one of them is present, extract that one. If none of them are present, answer N/A.",
|
||||
"DISPARAGEMENT_PROHIBITION_IND": "Is Disparagement Prohibition clause present in the contract? It may be found in section 2.1 of the contract. Answer with a Yes or No, do not provide any other context.",
|
||||
"DISPARAGEMENT_PROHIBITION_LANGUAGE": "Disparagement Prohibition clause may be found in section 2.1 of the contract. Extract the Disparagement Prohibition clause present in the contract. If it is not present, answer N/A.",
|
||||
"ELIGIBILITY_VERIFICATION": "Eligibility Verification or Determination clause may be found in section 2.6 of the contract. Extract the Eligibility Verification or Determination clause present in the contract. If it is not present, answer N/A.",
|
||||
"INSURANCE_REQUIREMENT": "Extract Insurance requirement mentioned in the contract. If it is not present, answer N/A.",
|
||||
"PARTICIPATION_IN_PRODUCTS": "Participation in Products clause may be found in section 2.2.2 of the contract. Extract the Participation in Products clause present in the contract. If it is not present, answer N/A.",
|
||||
"POLICIES_AND_PROCEDURES": "Extract the entire Policies and Procedures sentence or paragraph present in the contract. Do not extract language about conflicts and construction. Do not extract language about PP (Preferred Provider) nor PPG (Preferred Provider Group). If it is not present, answer N/A.",
|
||||
"REGULATORY_REQUIREMENTS": 'Extract the entire Regulatory Requirements paragraph present in the contract. It usually contains the keyword "Regulatory Requirements". If it is not present, answer N/A.',
|
||||
"RELATIONSHIP_OF_PARTIES_LANGUAGE": "Extract the entire Relationship of Parties passage present in the contract. If it is not present, answer N/A.",
|
||||
"TERM_CLAUSE": """Extract the subsection or paragraph labeled Term. It may also be labeled Termination. If there are no such sections, extract the paragraph the most closely resembles this label. If there is still no correct answer, simply return 'N/A'.""",
|
||||
"LATE_PAID_CLAIMS_LANGUAGE": "Extract Late Paid Claims language present in the contract. This will be a sentence or paragraph describing late paid claims, interest payments or similar. Here is an example of a correct response: 'Any Clean Claim, as defined in 42 C.F.R. § 422.500, shall be paid within thirty (30) days of receipt by Plan at such address as may be designated by Plan, and Plan shall pay interest on any Clean Claim not paid within thirty (30) days of such receipt by Plan at the rate of interest required by law, or as otherwise set forth in the Provider Manual.'. Here is an example of an incorrect response: 'Any Clean Claim, as defined in 42 C.F.R. 422.500, shall be paid within thirty (30) days of receipt by Health Plan, Payor or (if Provider contracts with Downstream Entities) Provider, as applicable, as designated by Provider or such Downstream Entity, as applicable.'. If no valid answer is present, answer N/A.",
|
||||
"NON_RENEWAL_LANGUAGE": "Extract the section of the contract that describes non-renewal obligations, especially the number of days notice that must be given. Write the full relevant sentence or paragraph. Ensure that the exact term 'non-renewal' is present in the answer. Do NOT return an answer that does not have 'non-renewal'.",
|
||||
"HCBS_SERVICES": 'Extract HCBS services clause present either in the compensation or exhibit section of the contract. It may contain keywords like "for covered HCBS services, plan shall pay". Do not include definitions of HCBS. If it is not present, answer N/A.',
|
||||
}
|
||||
|
||||
KEYWORD_MAPPINGS = {
|
||||
"TERM_CLAUSE": {
|
||||
"methodology": "hierarchy",
|
||||
"keywords": ["Term", "term", "duration", "agreement period"],
|
||||
},
|
||||
"LATE_PAID_CLAIMS_LANGUAGE": {
|
||||
"methodology": "hierarchy",
|
||||
"keywords": [
|
||||
"late paid",
|
||||
"late payment",
|
||||
"interest",
|
||||
"late fee",
|
||||
"interest shall be paid",
|
||||
"interest payment",
|
||||
],
|
||||
},
|
||||
"NON_RENEWAL_LANGUAGE": {
|
||||
"methodology": "or",
|
||||
"keywords": ["not to renew", "non-renewal", "non renewal", "nonrenewal"],
|
||||
},
|
||||
# "NON_RENEWAL_DAYS": {'methodology': 'or', 'keywords': ["not to renew", "non-renewal", "non renewal", "nonrenewal"]},
|
||||
"HCBS_SERVICES": {
|
||||
"methodology": "hierarchy",
|
||||
"keywords": ["%", "Plan shall pay", "hcbs"],
|
||||
},
|
||||
"POLICIES_AND_PROCEDURES": {
|
||||
"methodology": "or",
|
||||
"keywords": ["2.4", "policies", "procedures"],
|
||||
},
|
||||
"REGULATORY_REQUIREMENTS": {
|
||||
"methodology": "or",
|
||||
"keywords": ["8.3", "6.4", "regulatory requirements", "regulatory"],
|
||||
},
|
||||
"RECOVERY_RIGHTS": {"methodology": "or", "keywords": ["3.5", "recovery rights"]},
|
||||
"RELATIONSHIP_OF_PARTIES_LANGUAGE": {
|
||||
"methodology": "or",
|
||||
"keywords": ["8.1", "relationship of parties"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def run_ac2_prompts(file_object):
|
||||
################## INITIATE PROCESSING ##################
|
||||
filename, contract_text = file_object
|
||||
print(f"Processing AC for {filename}...")
|
||||
# print(f"Total contract word count: {len(contract_text.split())}")
|
||||
|
||||
################## PREPROCESS ##################
|
||||
text_dict, exhibit_pages, num_pages, ac_chunks = preprocess.preprocess(
|
||||
contract_text, filename, fields="ac"
|
||||
)
|
||||
# print(f"AC Preprocessing Complete - {filename}")
|
||||
|
||||
################## RUN FULL CONTEXT PROMPTS ##################
|
||||
ac_dict = {}
|
||||
log_data = []
|
||||
|
||||
# Process 'full_context' fields together
|
||||
full_context_fields = [field for field in AC_DICT if field not in KEYWORD_MAPPINGS]
|
||||
full_context_questions = {
|
||||
field: prompts.AC_DICT.get(field) for field in full_context_fields
|
||||
}
|
||||
|
||||
if full_context_questions:
|
||||
full_context_prompt = ac_funcs.create_prompt(
|
||||
contract_text[0 : min(len(contract_text), 600000)],
|
||||
question=full_context_questions,
|
||||
)
|
||||
# print(f"High accuracy prompt word count: {len(high_accuracy_prompt.split())}")
|
||||
try:
|
||||
full_context_answers = ac_funcs.get_ac_answer(
|
||||
full_context_prompt,
|
||||
filename,
|
||||
fields=list(full_context_questions.keys()),
|
||||
)
|
||||
ac_dict.update(full_context_answers)
|
||||
|
||||
for field, answer in full_context_answers.items():
|
||||
log_data.append(
|
||||
{
|
||||
"Field": field,
|
||||
"Prompt": f"Question: {full_context_questions[field]}\n\nContext: {contract_text}", # Include full context
|
||||
"Context Type": "Full Context (High Accuracy)",
|
||||
"Response": answer,
|
||||
}
|
||||
)
|
||||
except Exception as e:
|
||||
# print(f"Error processing high accuracy fields: {str(e)}")
|
||||
for field in full_context_questions.keys():
|
||||
ac_dict[field] = f"Error: {str(e)}"
|
||||
log_data.append(
|
||||
{
|
||||
"Field": field,
|
||||
"Prompt": f"Question: {full_context_questions[field]}\n\nContext: {contract_text}", # Include full context
|
||||
"Context Type": "Full Context (High Accuracy)",
|
||||
"Response": f"Error: {str(e)}",
|
||||
}
|
||||
)
|
||||
|
||||
################## RUN CHUNKED PROMPTS ##################
|
||||
all_fields = set(prompts.AC_DICT.keys())
|
||||
chunked_fields = [field for field in KEYWORD_MAPPINGS]
|
||||
|
||||
for field in chunked_fields:
|
||||
if field in prompts.AC_DICT:
|
||||
question = prompts.AC_DICT[field]
|
||||
else:
|
||||
# print(f"Field {field} not found in prompts. Skipping...")
|
||||
continue
|
||||
|
||||
# Use chunk if available and different from full context, otherwise use full context
|
||||
if (
|
||||
field in ac_chunks
|
||||
and ac_chunks[field].strip()
|
||||
and ac_chunks[field] != contract_text
|
||||
):
|
||||
context = ac_chunks[field]
|
||||
context_type = "Smart Chunking"
|
||||
else:
|
||||
context = contract_text
|
||||
context_type = "Full Context"
|
||||
|
||||
prompt = prompts.AC_SINGLE_FIELD_TEMPLATE(context, question)
|
||||
|
||||
# print(f"Field {field} prompt word count: {len(prompt.split())} ({context_type})")
|
||||
|
||||
try:
|
||||
field_answer = claude_funcs.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 8192
|
||||
)
|
||||
print(field, field_answer)
|
||||
ac_dict[field] = field_answer
|
||||
|
||||
log_data.append(
|
||||
{
|
||||
"Field": field,
|
||||
"Prompt": f"Question: {question}\n\nContext: {context}", # Include full context
|
||||
"Context Type": context_type,
|
||||
"Response": field_answer,
|
||||
}
|
||||
)
|
||||
except Exception as e:
|
||||
# print(f"Error processing field {field}: {str(e)}")
|
||||
ac_dict[field] = f"Error: {str(e)}"
|
||||
log_data.append(
|
||||
{
|
||||
"Field": field,
|
||||
"Prompt": f"Question: {question}\n\nContext: {context}", # Include full context
|
||||
"Context Type": context_type,
|
||||
"Response": f"Error: {str(e)}",
|
||||
}
|
||||
)
|
||||
|
||||
################## AC CONDITIONAL PROMPTS ##################
|
||||
## Non-Renewal - Days
|
||||
nrd_prompt = prompts.AC_SINGLE_FIELD_TEMPLATE(
|
||||
ac_dict["NON_RENEWAL_LANGUAGE"], prompts.AC_DICT["NON_RENEWAL_DAYS"]
|
||||
)
|
||||
nrd_answer = claude_funcs.invoke_claude(
|
||||
nrd_prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 8192
|
||||
)
|
||||
ac_dict["NON_RENEWAL_DAYS"] = nrd_answer
|
||||
|
||||
################## ENSURE ALL FIELDS ARE PRESENT ##################
|
||||
ac_dict = {k: v for k, v in ac_dict.items() if k in valid.AC_MAPPING}
|
||||
for field in all_fields:
|
||||
if field not in ac_dict:
|
||||
# print(f"Field not found in results. {field}")
|
||||
# ac_dict[field] = "N/A"
|
||||
log_data.append(
|
||||
{
|
||||
"Field": field,
|
||||
"Prompt": "N/A",
|
||||
"Context Type": "N/A",
|
||||
"Response": "N/A",
|
||||
}
|
||||
)
|
||||
|
||||
################## CREATE OUTPUT DIRECTORIES ##################
|
||||
base_filename = os.path.splitext(filename)[0].strip()
|
||||
output_dir = os.path.join(ac2_output_path, base_filename)
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
################## POSTPROCESS ##################
|
||||
ac_dict["Contract Name"] = filename
|
||||
ac_df = pd.DataFrame([ac_dict])
|
||||
ac_df = postprocess.ac_postprocess(ac_df)
|
||||
|
||||
################## WRITE TO OUTPUT ##################
|
||||
ac_df.to_csv(f"{output_dir}/ac_output.csv", index=False)
|
||||
print(f"AC Output written: {filename}")
|
||||
|
||||
################## WRITE LOG TO CSV ##################
|
||||
# log_file_path = os.path.join(output_dir, f"{base_filename}_prompt_log.csv")
|
||||
# with open(log_file_path, 'w', newline='', encoding='utf-8') as log_file:
|
||||
# fieldnames = ['Field', 'Prompt', 'Context Type', 'Response']
|
||||
# writer = csv.DictWriter(log_file, fieldnames=fieldnames)
|
||||
# writer.writeheader()
|
||||
# writer.writerows(log_data)
|
||||
# print(f"Prompt log written to {log_file_path}")
|
||||
return ac_dict
|
||||
|
||||
|
||||
def process_ac2_adhoc(item):
|
||||
filename, contract_text = item
|
||||
try:
|
||||
results = run_ac2_prompts(item)
|
||||
if results is not None:
|
||||
tracking.update_results_csv(item, results)
|
||||
return item, results
|
||||
except Exception as e:
|
||||
print(f"Error processing item {filename}: {e}")
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
def preprocess_ad_hoc(contract_text, filename, fields=config.FIELDS):
|
||||
|
||||
contract_text = preprocessing_funcs.clean_newlines(contract_text)
|
||||
contract_text = preprocessing_funcs.clean_law_symbols(contract_text)
|
||||
|
||||
text_dict = preprocessing_funcs.split_text(
|
||||
contract_text
|
||||
) # return a dictionary with keys - page_num (str), values as the page_text
|
||||
|
||||
num_pages = len(text_dict.keys()) - 1
|
||||
text_dict = preprocessing_funcs.filter_quick_review(text_dict)
|
||||
|
||||
if fields == "b":
|
||||
exhibit_pages = top_down_funcs.get_exhibit_pages(
|
||||
text_dict, filename
|
||||
) # All pages with exhibit headers
|
||||
# text_dict = table_funcs.align_and_format_tables(text_dict, filename)
|
||||
text_dict = preprocessing_funcs.chunk_consecutive(text_dict, exhibit_pages)
|
||||
ac_chunks = ""
|
||||
if fields == "ac":
|
||||
exhibit_pages = []
|
||||
ac_chunks = preprocessing_funcs.smart_chunk_ac(text_dict)
|
||||
|
||||
return text_dict, exhibit_pages, num_pages, ac_chunks
|
||||
|
||||
|
||||
def get_add_on(final_dicts, text_dict):
|
||||
add_on_dict = {}
|
||||
for td_page in list(set([d["page_num"] for d in final_dicts])):
|
||||
td_page = str(td_page).split(".")[0]
|
||||
print(f"TD Page : {td_page}")
|
||||
# TD page is the original result top down page, it is not necessarily in the new text_dict.
|
||||
# Find add_on_page = the key of the new text_dict that points to the correct page of the old text dict
|
||||
|
||||
if str(td_page) in list(text_dict.keys()):
|
||||
add_on_page = td_page
|
||||
else:
|
||||
add_on_page = [
|
||||
str(page) for page in text_dict.keys() if float(page) > float(td_page)
|
||||
]
|
||||
print(add_on_page)
|
||||
|
||||
if not add_on_page:
|
||||
add_on_dict[td_page] = {
|
||||
"ADD_ON_REIMBURSEMENT_LANGUAGE": "N/A",
|
||||
"ADD_ON_REIMBURSEMENT_IND": "N",
|
||||
}
|
||||
continue
|
||||
elif not isinstance(add_on_page, str):
|
||||
add_on_page = add_on_page[0]
|
||||
|
||||
ad = {}
|
||||
|
||||
add_on_prompt = f"## Page Start ## {text_dict[str(add_on_page)]} ## Page End ## . Extract the Add On Reimbursement language present in the page above. If there is no Add On language present, answer N/A. Write only the answer, with no additional commentary or explanation."
|
||||
add_on_answer = claude_funcs.invoke_claude(
|
||||
add_on_prompt, config.MODEL_ID_CLAUDE35_SONNET, "", 256
|
||||
)
|
||||
|
||||
ad["ADD_ON_REIMBURSEMENT_LANGUAGE"] = add_on_answer
|
||||
|
||||
if "N/A" in ad["ADD_ON_REIMBURSEMENT_LANGUAGE"]:
|
||||
ad["ADD_ON_REIMBURSEMENT_IND"] = "N"
|
||||
else:
|
||||
ad["ADD_ON_REIMBURSEMENT_IND"] = "Y"
|
||||
add_on_dict[str(td_page)] = ad
|
||||
|
||||
return add_on_dict
|
||||
|
||||
|
||||
def process_b_adhoc(item):
|
||||
filename, contract_text = item
|
||||
print(f"Processing {filename}")
|
||||
|
||||
merged_dicts = [d for d in abc_dicts if d["Filename"] + ".txt" == filename]
|
||||
print(len(merged_dicts))
|
||||
|
||||
if len(merged_dicts) > 0:
|
||||
|
||||
text_dict, exhibit_pages, num_pages, ac_chunks = preprocess_ad_hoc(
|
||||
contract_text, filename
|
||||
)
|
||||
print(f"Preprocessing {filename}")
|
||||
|
||||
# Exclusions
|
||||
exclusion_dict = top_down_funcs.get_td_dict(
|
||||
text_dict, filename, prompts.TOP_DOWN_EXCLUSIONS, valid.EXCLUSION_TERMS
|
||||
)
|
||||
final_dicts = top_down_funcs.add_exclusions(merged_dicts, exclusion_dict)
|
||||
print(f"Exclusions - {filename}")
|
||||
|
||||
# Medically Necessary
|
||||
medically_necessary_dict = top_down_funcs.get_td_dict(
|
||||
text_dict,
|
||||
filename,
|
||||
prompts.TOP_DOWN_MEDICALLY_NECESSARY,
|
||||
valid.VALID_MEDICALLY_NECESSARY,
|
||||
)
|
||||
medically_necessary_dict = postprocessing_funcs.filter_dict(
|
||||
medically_necessary_dict,
|
||||
pattern=re.compile(
|
||||
"|".join(
|
||||
re.escape(keyword) for keyword in valid.INVALID_MEDICALLY_NECESSARY
|
||||
),
|
||||
re.IGNORECASE,
|
||||
),
|
||||
)
|
||||
final_dicts = top_down_funcs.add_medically_necessary(
|
||||
final_dicts, medically_necessary_dict
|
||||
)
|
||||
print(f"Medically Necessary - {filename}")
|
||||
|
||||
# Add On
|
||||
add_on_dict = get_add_on(final_dicts, text_dict)
|
||||
print(f"Add On Results: {add_on_dict}")
|
||||
print(f"Add On - {filename}")
|
||||
|
||||
for td_page in add_on_dict.keys():
|
||||
for d in final_dicts:
|
||||
if str(d["page_num"]) == str(td_page):
|
||||
print("Add On Match")
|
||||
d.update(add_on_dict[str(td_page)])
|
||||
|
||||
# print(len(final_dicts))
|
||||
|
||||
output_df = pd.DataFrame(final_dicts)
|
||||
print(f"B finished -{filename} - output {output_df.shape}")
|
||||
print(output_df.columns)
|
||||
time.sleep(3)
|
||||
base_filename = os.path.splitext(filename)[0].strip()
|
||||
output_dir = os.path.join(b_output_path, base_filename)
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
output_df.to_csv(f"{output_dir}/b_output.csv")
|
||||
|
||||
|
||||
########################## Process Starts Here ##########################
|
||||
|
||||
# Run List
|
||||
run_list = pd.read_excel(run_path)
|
||||
run_list = run_list[run_list["Batch"] == "3A"]["Filename"].unique().tolist()
|
||||
print(f"Run List: {len(run_list)} files")
|
||||
|
||||
# Input Dict
|
||||
input_dict_3a = utils.read_input(local_path=input_path + "A")
|
||||
input_dict_3b = utils.read_input(local_path=input_path + "B")
|
||||
|
||||
input_dict = {**input_dict_3a, **input_dict_3b}
|
||||
print(f"Input Dict: {len(input_dict)} files")
|
||||
|
||||
# Filter Input Dict
|
||||
input_dict = {
|
||||
filename: contract_text
|
||||
for filename, contract_text in input_dict.items()
|
||||
if filename.split(".txt")[0] in run_list
|
||||
}
|
||||
print(f"Input Dict In Run List: {len(input_dict)} files")
|
||||
|
||||
# Left to run
|
||||
if "a" in config.FIELDS:
|
||||
input_dict_ac2 = {
|
||||
filename: contract_text
|
||||
for filename, contract_text in input_dict.items()
|
||||
if filename.split(".txt")[0] not in os.listdir(ac2_output_path)
|
||||
}
|
||||
print(f"Input Dict Left to Run (AC2): {len(input_dict_ac2)} files")
|
||||
|
||||
# ABC Dicts
|
||||
abc_df = pd.read_excel(clean_path)
|
||||
abc_df.rename(columns={v: k for k, v in valid.B_MAPPING.items()}, inplace=True)
|
||||
abc_df.rename(columns={v: k for k, v in valid.AC_MAPPING.items()}, inplace=True)
|
||||
abc_df.rename(columns=combined_mapping, inplace=True)
|
||||
abc_df.drop(["State from Ben", "Unnamed: 54"], axis=1, inplace=True)
|
||||
abc_names = list(abc_df["Filename"].unique())
|
||||
abc_dicts = abc_df.to_dict(orient="records")
|
||||
print(f"Clean abc : {len(abc_dicts)}")
|
||||
|
||||
# Left to run
|
||||
if "b" in config.FIELDS:
|
||||
input_dict_b = {
|
||||
filename: contract_text
|
||||
for filename, contract_text in input_dict.items()
|
||||
if filename.split(".txt")[0] in abc_df["Filename"].unique()
|
||||
}
|
||||
input_dict_b = {
|
||||
filename: contract_text
|
||||
for filename, contract_text in input_dict_b.items()
|
||||
if filename.split(".txt")[0] not in os.listdir(b_output_path)
|
||||
}
|
||||
print(f"Input Dict Left to Run (B): {len(input_dict_b)} files")
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor:
|
||||
if "a" in config.FIELDS and "c" in config.FIELDS:
|
||||
futures = [
|
||||
executor.submit(process_ac2_adhoc, item) for item in input_dict_ac2.items()
|
||||
]
|
||||
if "b" in config.FIELDS:
|
||||
futures = [
|
||||
executor.submit(process_b_adhoc, item) for item in input_dict_b.items()
|
||||
]
|
||||
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
try:
|
||||
result = future.result()
|
||||
except Exception as e:
|
||||
print(f"Error in future: {e}")
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
# for item in input_dict_b.items():
|
||||
# process_b_adhoc(item)
|
||||
@@ -1,87 +0,0 @@
|
||||
from utils import read_local
|
||||
import preprocess
|
||||
import preprocessing_funcs
|
||||
import keywords
|
||||
import argparse
|
||||
import os
|
||||
from keywords import KEYWORD_MAPPINGS
|
||||
from collections import defaultdict
|
||||
|
||||
# import tqdm
|
||||
|
||||
|
||||
def parse_arguments():
|
||||
parser = argparse.ArgumentParser(description="Smart Chunking Tester")
|
||||
parser.add_argument(
|
||||
"--input_dir", help="Input directory (local path or S3 URI)", default="src/ip2"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output_dir", help="Output directory (local)", default="src/output_chunks"
|
||||
)
|
||||
parser.add_argument("--keyword", help="Keyword to be used for chunking")
|
||||
parser.add_argument(
|
||||
"--case_sensitive",
|
||||
help="case sensitivity",
|
||||
action=argparse.BooleanOptionalAction,
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_arguments()
|
||||
input_dir = args.input_dir
|
||||
output_dir = args.output_dir # src/output_chunks
|
||||
case_sensitive = args.case_sensitive
|
||||
|
||||
if case_sensitive is None: # This shouldn't have to be assigned
|
||||
case_sensitive = False
|
||||
|
||||
keyword = args.keyword
|
||||
keyword = list(map(str, keyword.split(",")))
|
||||
keyword = [kw.strip() for kw in keyword]
|
||||
print("Generating Chunks for keywords --> ", keyword)
|
||||
print("Case sensitive -->", case_sensitive)
|
||||
print(type(case_sensitive))
|
||||
|
||||
kws = KEYWORD_MAPPINGS
|
||||
|
||||
retained_rates = []
|
||||
|
||||
for file in os.listdir(input_dir):
|
||||
full_path = os.path.join(input_dir, file)
|
||||
contract_text = read_local(full_path)
|
||||
contract_text = preprocessing_funcs.clean_newlines(contract_text)
|
||||
contract_text = preprocessing_funcs.clean_law_symbols(contract_text)
|
||||
text_dict = preprocessing_funcs.split_text(
|
||||
contract_text
|
||||
) # return a dictionary with keys - page_num (str), values as the page_text
|
||||
|
||||
test_chunck = preprocessing_funcs.smart_chunk_ac(
|
||||
text_dict=text_dict,
|
||||
keyword_mappings={
|
||||
"place_holder": {
|
||||
"methodology": "or",
|
||||
"keywords": keyword,
|
||||
"case_sensitive": case_sensitive,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
retained_rate = len(test_chunck["place_holder"]) / len(contract_text)
|
||||
retained_rates.append(retained_rate)
|
||||
|
||||
print(
|
||||
f"\nfrom file {file} {len(contract_text)} characters were retrieved;\n{len(test_chunck['place_holder'])} were retained by smart chunking on keywords:\n{keyword}"
|
||||
)
|
||||
print(f"Smart chunking reduced the document by {100*(1-retained_rate):0.2f}%")
|
||||
|
||||
with open(f"{output_dir}/{file}_CHUNKED.txt", "w") as f:
|
||||
f.write(test_chunck["place_holder"])
|
||||
|
||||
print("avg reduction:", 1 - (sum(retained_rates) / len(retained_rates)))
|
||||
print("max retention:", max(retained_rates))
|
||||
print("min retention:", min(retained_rates))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,41 +0,0 @@
|
||||
import argparse
|
||||
import os
|
||||
from utils import read_local
|
||||
import preprocessing_funcs
|
||||
import ac_funcs
|
||||
|
||||
|
||||
def parse_arguments():
|
||||
parser = argparse.ArgumentParser(description="Smart Chunking Tester")
|
||||
parser.add_argument(
|
||||
"--input_dir", help="Input directory (local path or S3 URI)", default="src/new/"
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_arguments()
|
||||
input_dir = args.input_dir
|
||||
|
||||
for file in os.listdir(input_dir):
|
||||
full_path = os.path.join(input_dir, file)
|
||||
print("--" * 20)
|
||||
print(file)
|
||||
contract_text = read_local(full_path)
|
||||
contract_text = preprocessing_funcs.clean_newlines(contract_text)
|
||||
contract_text = preprocessing_funcs.clean_law_symbols(contract_text)
|
||||
text_dict = preprocessing_funcs.split_text(
|
||||
contract_text
|
||||
) # return a dictionary with keys - page_num (str), values as the page_text
|
||||
|
||||
irs_answers = ac_funcs.tin_regex(filename=str(file), text_dict=text_dict)
|
||||
if irs_answers is None:
|
||||
irs_answers = {}
|
||||
irs_answers["PROV_GROUP_TIN"] = "N/A"
|
||||
irs_answers["PROV_TIN_OTHER"] = "N/A"
|
||||
|
||||
print(irs_answers)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,5 +1,26 @@
|
||||
import re
|
||||
|
||||
"""
|
||||
Note:
|
||||
The below pattern will match with the following date formats:
|
||||
Numeric Formats
|
||||
MM/DD/YYYY or MM-DD-YYYY (e.g., 11/18/2024, 11-18-2024)
|
||||
MM/DD/YY or MM-DD-YY (e.g., 11/18/24, 11-18-24)
|
||||
YYYY/MM/DD or YYYY-MM-DD (e.g., 2024/11/18, 2024-11-18)
|
||||
DD/MM/YY or DD-MM-YY (e.g., 18/11/24, 18-11-24)
|
||||
Textual Formats
|
||||
Month DD, YYYY (e.g., December 1, 2024)
|
||||
Month D, YYYY (e.g., December 1, 2024 — with a single-digit day)
|
||||
YYYY Month DD (e.g., 2024 December 1)
|
||||
Abbreviated Month Formats
|
||||
Mon DD, YYYY (e.g., Dec 1, 2024)
|
||||
DD Mon YYYY (e.g., 1 Dec 2024)
|
||||
YYYY Mon DD (e.g., 2024 Dec 1)
|
||||
|
||||
Accounts for month names in lowercase and uppercase
|
||||
"""
|
||||
|
||||
DATE_PATTERN = r'\b((0?[1-9]|1[0-2])[-/](0?[1-9]|[12][0-9]|3[01])[-/](\d{4}|\d{2})|(\d{4}[-/](0?[1-9]|1[0-2])[-/](0?[1-9]|[12][0-9]|3[01]))|((0?[1-9]|[12][0-9]|3[01])[-/](0?[1-9]|1[0-2])[-/](\d{4}|\d{2}))|(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec|january|february|march|april|may|june|july|august|september|october|november|december|jan|feb|mar|apr|jun|jul|aug|sep|oct|nov|dec)\s+\d{1,2},\s+\d{4}|(?:\d{4}\s+(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec|january|february|march|april|may|june|july|august|september|october|november|december|jan|feb|mar|apr|jun|jul|aug|sep|oct|nov|dec)\s+\d{1,2})|(?:\d{1,2}\s+(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec|january|february|march|april|may|june|july|august|september|october|november|december|jan|feb|mar|apr|jun|jul|aug|sep|oct|nov|dec)\s+\d{4}))\b'
|
||||
|
||||
def chunk_hierarchical(
|
||||
text_dict: dict[str, str], keywords: list[str], case_sensitive: bool = False
|
||||
@@ -198,3 +219,89 @@ def keyword_search(
|
||||
)
|
||||
|
||||
return page_dict_sorted
|
||||
|
||||
def chunk_with_include_exclude_keywords(
|
||||
text_dict: dict[str, str],
|
||||
include_keywords: list[str],
|
||||
exclude_keywords: list[str],
|
||||
case_sensitive: bool = False,
|
||||
apply_date_filtering: bool = True
|
||||
) -> list[str]:
|
||||
|
||||
"""Look for pages that include any of the include_keywords and exclude any pages that contain any of the exclude_keywords.
|
||||
|
||||
Args:
|
||||
text_dict (dict[str, str]): A dictionary keyed by string page number and valued
|
||||
by string page contents
|
||||
include_keywords (list[str]): A list of string keywords to include
|
||||
exclude_keywords (list[str]): A list of string keywords to exclude
|
||||
case_sensitive (bool): A flag for converting all keywords and text to lower case,
|
||||
in order to ignore case altogether.
|
||||
apply_date_filtering (bool): A flag to filter out the pages not containing a date.
|
||||
|
||||
Returns:
|
||||
list[str]: The sorted list of string page numbers that include all include_keywords and exclude any exclude_keywords
|
||||
"""
|
||||
|
||||
if not case_sensitive:
|
||||
include_keywords = [keyword.lower() for keyword in include_keywords]
|
||||
exclude_keywords = [keyword.lower() for keyword in exclude_keywords]
|
||||
text_dict = {key: value.lower() for key, value in text_dict.items()}
|
||||
|
||||
if apply_date_filtering:
|
||||
pages_containing_date = set()
|
||||
|
||||
for page_num, page_content in text_dict.items():
|
||||
dates_list = re.findall(DATE_PATTERN, page_content)
|
||||
|
||||
if len(dates_list) > 0:
|
||||
pages_containing_date.add(page_num)
|
||||
|
||||
pages_containing_date = sorted(pages_containing_date,key=int)
|
||||
|
||||
# debug lines for inclusion and exclusion keywords found on each page
|
||||
|
||||
# print(f"Pages containing date: {pages_containing_date}")
|
||||
|
||||
# for page_num, page_content in text_dict.items():
|
||||
# if page_num in pages_containing_date:
|
||||
|
||||
# inclusion_keywords = set()
|
||||
# exclusion_keywords = set()
|
||||
|
||||
# for keyword in include_keywords:
|
||||
# if keyword in page_content:
|
||||
# inclusion_keywords.add(keyword)
|
||||
|
||||
# for keyword in exclude_keywords:
|
||||
# if keyword in page_content:
|
||||
# exclusion_keywords.add(keyword)
|
||||
|
||||
# print(f"Page #: {page_num} contains inclusion keywords: {inclusion_keywords} and contain exclusion keywords: {exclusion_keywords}")
|
||||
|
||||
page_list = []
|
||||
|
||||
for page in text_dict.keys():
|
||||
if not apply_date_filtering or (apply_date_filtering and page in pages_containing_date):
|
||||
if any(keyword in text_dict[page] for keyword in include_keywords) and not any(
|
||||
keyword in text_dict[page] for keyword in exclude_keywords
|
||||
):
|
||||
# if str(int(page) - 1) in text_dict.keys():
|
||||
# page_list.append(str(int(page) - 1))
|
||||
# print(f"Taking page #: {str(int(page) - 1)} (as buffer for following page)")
|
||||
|
||||
page_list.append(page)
|
||||
# print(f"Taking page #: {page}")
|
||||
|
||||
# if str(int(page) + 1) in text_dict.keys():
|
||||
# page_list.append(str(int(page) + 1))
|
||||
# # print(f"Taking page #: {str(int(page) + 1)} (as buffer for previous page)")
|
||||
|
||||
page_list_sorted = sorted([int(page_str) for page_str in set(page_list)])
|
||||
page_list_sorted_str = list(map(str, page_list_sorted))
|
||||
|
||||
# print(", ".join([page for page in page_list_sorted_str]))
|
||||
|
||||
# print(f"Obtained total pages: {len(page_list_sorted_str)}")
|
||||
|
||||
return page_list_sorted_str
|
||||
@@ -0,0 +1,140 @@
|
||||
|
||||
import utils
|
||||
import config
|
||||
import cnc_hotfix
|
||||
import preprocessing_funcs
|
||||
import keywords
|
||||
import valid
|
||||
|
||||
import pandas as pd
|
||||
import concurrent.futures
|
||||
|
||||
ABC_PATH = '~/centene-national-contracting/doczy.ai/fieldExtraction/clean_output/CNC-1A-ABC.xlsx'
|
||||
FILEMAP_PATH = '~/centene-national-contracting/doczy.ai/fieldExtraction/reference_files/batch1_duplicates_mapping.csv'
|
||||
|
||||
def process_hotfix(abc, contract_text):
|
||||
"""
|
||||
Processes a contract by applying a series of hotfixes to the original output data.
|
||||
|
||||
Parameters:
|
||||
abc (DataFrame): The original output DataFrame for one contract. This DataFrame contains 114 columns.
|
||||
contract_text (str): The text of the contract.
|
||||
|
||||
Returns:
|
||||
DataFrame: The original abc DataFrame augmented with additional columns for each hotfix applied.
|
||||
"""
|
||||
|
||||
filename = abc['Contract Name'].unique()[0]
|
||||
|
||||
original_length = len(abc)
|
||||
|
||||
if contract_text is not None:
|
||||
# print(filename, len(contract_text))
|
||||
|
||||
# Preprocess
|
||||
contract_text = preprocessing_funcs.clean_newlines(contract_text)
|
||||
contract_text = preprocessing_funcs.clean_law_symbols(contract_text)
|
||||
|
||||
text_dict = preprocessing_funcs.split_text(
|
||||
contract_text
|
||||
) # return a dictionary with keys - page_num (str), values as the page_text
|
||||
text_dict, top_sheet_dict = preprocessing_funcs.filter_quick_review(text_dict)
|
||||
ac_chunks = preprocessing_funcs.smart_chunk_ac(text_dict, contract_text, keywords.GROUPED_KEYWORD_MAPPINGS)
|
||||
|
||||
######################################### Hot fixes ##########################################
|
||||
|
||||
######### Rule/Regex-based #########
|
||||
# Default
|
||||
abc = cnc_hotfix.check_default_term(abc)
|
||||
abc = cnc_hotfix.check_default_rate(abc)
|
||||
abc = cnc_hotfix.populate_default_term(abc, text_dict)
|
||||
|
||||
# # Health Plan State
|
||||
abc = cnc_hotfix.clean_healthplan_state(abc, filename, text_dict)
|
||||
|
||||
# # IRS
|
||||
abc = cnc_hotfix.clean_irs(abc, filename, text_dict)
|
||||
|
||||
# # NPI
|
||||
abc = cnc_hotfix.clean_npi(abc, filename, text_dict, top_sheet_dict)
|
||||
|
||||
# # LOB
|
||||
abc = cnc_hotfix.clean_lob(abc, text_dict)
|
||||
|
||||
# # Lesser
|
||||
abc = cnc_hotfix.clean_lesser(abc, text_dict)
|
||||
|
||||
# # Provider Type Level 2
|
||||
abc = cnc_hotfix.clean_provider_type_2(abc)
|
||||
|
||||
# ######## Prompt-based #########
|
||||
# Rate Standard
|
||||
# abc = cnc_hotfix.clean_rate_standard(abc)
|
||||
|
||||
# Contract Effective Date
|
||||
# abc = cnc_hotfix.clean_contract_effective_date(abc, filename, text_dict)
|
||||
|
||||
# Term Group
|
||||
# abc = cnc_hotfix.clean_term_group(abc, filename, text_dict, ac_chunks)
|
||||
|
||||
# Agreement Name
|
||||
# abc = cnc_hotfix.clean_agreement_name(abc, filename, text_dict)
|
||||
|
||||
else:
|
||||
abc["Contract Duplicate Issue"] = "No Contract Found"
|
||||
|
||||
return abc
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
# Read clean data
|
||||
print("Reading clean data...")
|
||||
abc_clean = pd.read_excel(ABC_PATH)
|
||||
unique_output_files = abc_clean['Contract Name'].unique()
|
||||
print("ABC Clean: ", abc_clean.shape)
|
||||
print(len(unique_output_files), 'files in clean output...')
|
||||
# print(list(abc_clean.columns))
|
||||
abc_clean.rename(columns=valid.HOTFIX_MAPPING, inplace=True)
|
||||
|
||||
# Read input dict
|
||||
print("Reading input .txt files...")
|
||||
input_dict = utils.read_input()
|
||||
input_dict = {filename.strip('.txt') : contract_text for filename, contract_text in input_dict.items()}
|
||||
print("Input Dict: ", len(input_dict))
|
||||
|
||||
input_dict_dups = utils.read_input('data_cnc/CNC-1-Dups')
|
||||
input_dict_dups = {filename.strip('.txt') : contract_text for filename, contract_text in input_dict_dups.items()}
|
||||
print("Input Dict Dups: ", len(input_dict_dups))
|
||||
|
||||
input_dict_full = input_dict | input_dict_dups
|
||||
# input_dict_full = input_dict
|
||||
|
||||
print(f"{len([file for file in input_dict_full.keys() if file not in unique_output_files])} files in full s3, not in output")
|
||||
print(f"{len([file for file in unique_output_files if file not in input_dict_full.keys()])} files in output, not in full s3")
|
||||
# print(f"{[file for file in unique_output_files if file not in input_dict_full.keys()]} files in output, not in full s3")
|
||||
|
||||
# Create list of tuples
|
||||
abc_clean_list = [(group, input_dict_full.get(name)) for name, group in abc_clean.groupby('Contract Name')]
|
||||
print(f"{len(abc_clean_list)} - files in input")
|
||||
print(f"{len([f for f in abc_clean_list if f[1]])} - valid files in input")
|
||||
|
||||
df_list = []
|
||||
print("Processing hotfixes...")
|
||||
|
||||
for abc_df, contract_text in abc_clean_list:
|
||||
new_df = process_hotfix(abc_df, contract_text)
|
||||
df_list.append(new_df)
|
||||
|
||||
abc_final = pd.concat(df_list, ignore_index=True)
|
||||
|
||||
print(abc_final.shape)
|
||||
print(list(abc_final.columns))
|
||||
|
||||
print("Writing to excel...")
|
||||
abc_final.to_excel(f'{config.BATCH_ID}-ABC-NonPrompt-HotfixFull.xlsx')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -1,94 +1,655 @@
|
||||
import utils
|
||||
import pandas as pd
|
||||
import itertools
|
||||
import numpy as np
|
||||
import re
|
||||
import pandas as pd
|
||||
from valid import STATE_MAP
|
||||
from prompts import BOTTOM_UP_METHODOLOGY_BREAKOUT
|
||||
from bottom_up_funcs import get_short_methodology
|
||||
import claude_funcs
|
||||
from utils import is_empty
|
||||
import dict_operations
|
||||
import config
|
||||
import warnings
|
||||
warnings.simplefilter(action='ignore', category=FutureWarning)
|
||||
|
||||
def state_check(states: str | list[str], is_dataframe: bool = False) -> str | list[str]:
|
||||
# TODO: genericize this function to take in and put out a dataframe and put it into
|
||||
# postprocessing_funcs.py
|
||||
|
||||
keywords_to_detect_invalid_state_name = [
|
||||
"suggest",
|
||||
"The Contract Text",
|
||||
"refers",
|
||||
"For ",
|
||||
"Answer",
|
||||
"Multiple",
|
||||
"Statewide",
|
||||
]
|
||||
|
||||
if not isinstance(states, str) and not isinstance(states,list):
|
||||
if is_empty(states):
|
||||
states = 'N/A'
|
||||
import ac_smart_chunking
|
||||
import claude_funcs
|
||||
import cnc_hotfix_effective_date_utils
|
||||
import config
|
||||
import utils
|
||||
import valid
|
||||
import table_funcs
|
||||
from hotfix_helper_funcs import irs_hotfix, clean_output_postprocess, state_check, check_field_for_matches, npi_hotfix, npi_post_process
|
||||
import keywords
|
||||
import prompts
|
||||
import ac_funcs
|
||||
import postprocessing_funcs
|
||||
|
||||
####### Default Term/Rate #######
|
||||
def concatenate_lists(row):
|
||||
total = []
|
||||
for col in ['Line of Business List', 'Provider Type List', 'Provider Type - Level 2 List', 'IP/OP List', 'Service Type List', 'Plan Type List']:
|
||||
value = row[col]
|
||||
if isinstance(value, list):
|
||||
total.extend([str(item) for item in value])
|
||||
else:
|
||||
total.append(str(value))
|
||||
return total
|
||||
|
||||
def check_default_term(df: pd.DataFrame):
|
||||
|
||||
df['Contains Only'] = df['Default Term'].str.lower().str.contains('only')
|
||||
if df['Contains Only'].any():
|
||||
df_only_grouped = df[['Attachment/Exhibit', 'Line of Business', 'Provider Type', 'Provider Type - Level 2', 'IP/OP', 'Service Type', 'Plan Type', 'Default Term']]
|
||||
df_transformed = df_only_grouped.groupby(['Attachment/Exhibit', 'Default Term'], as_index=False).transform(lambda x: list(set(x.dropna().to_list())))
|
||||
df_only_grouped[['Line of Business List', 'Provider Type List', 'Provider Type - Level 2 List', 'IP/OP List', 'Service Type List', 'Plan Type List']] = df_transformed
|
||||
|
||||
if (
|
||||
isinstance(states, list) and all(isinstance(state, str) for state in states)
|
||||
) or isinstance(states, str):
|
||||
|
||||
if isinstance(states, str):
|
||||
if (
|
||||
not any(
|
||||
kw.lower() in states.lower()
|
||||
for kw in keywords_to_detect_invalid_state_name
|
||||
)
|
||||
and "," in states
|
||||
):
|
||||
states = states.split(",")
|
||||
else:
|
||||
states = [states]
|
||||
df_only_grouped['All Checks'] = df_only_grouped[['Line of Business List', 'Provider Type List', 'Provider Type - Level 2 List', 'IP/OP List', 'Service Type List', 'Plan Type List']].apply(concatenate_lists, axis=1)
|
||||
df_only_grouped['All Checks'] = df_only_grouped['All Checks'].apply(lambda x: [y.replace('OP','Outpatient').replace('IP', 'Inpatient') for y in x])
|
||||
df_only_grouped = df_only_grouped[['Attachment/Exhibit', 'Line of Business', 'Provider Type', 'Provider Type - Level 2', 'IP/OP', 'Service Type', 'Plan Type','All Checks']]
|
||||
|
||||
df = df.merge(df_only_grouped, on = ['Attachment/Exhibit', 'Line of Business', 'Provider Type', 'Provider Type - Level 2', 'IP/OP', 'Service Type', 'Plan Type'], how = 'left')
|
||||
|
||||
new_states = []
|
||||
for state in states:
|
||||
state = state.strip()
|
||||
df['Default Term'].fillna('', inplace=True)
|
||||
df['All Checks'] = df['All Checks'].apply(lambda d: d if isinstance(d, list) else [])
|
||||
|
||||
# fix 1
|
||||
if state == "Hlorida":
|
||||
state = "Florida"
|
||||
df['Default Match?'] = df.apply(lambda row: any([x.lower() in row['Default Term'].lower() for x in row['All Checks']]), axis = 1)
|
||||
|
||||
# fix 2 - LLM self-talking
|
||||
if any(
|
||||
kw.lower() in state.lower()
|
||||
for kw in keywords_to_detect_invalid_state_name
|
||||
):
|
||||
state = "N/A"
|
||||
df['Default Rate Corrected Step 1'] = df['Default Rate']
|
||||
df.loc[(df['Contains Only']==True) & (df['Default Match?']==False), 'Default Rate Corrected Step 1'] = ''
|
||||
|
||||
if state in STATE_MAP.keys() or state.upper() in STATE_MAP.keys():
|
||||
new_states.append(STATE_MAP[state.upper()])
|
||||
|
||||
return df
|
||||
|
||||
elif state.strip().title() in STATE_MAP.values():
|
||||
new_states.append(state.strip().title())
|
||||
else:
|
||||
df['All Checks'] = ''
|
||||
df['Default Match?'] = True
|
||||
df['Default Rate Corrected Step 1'] = df['Default Rate']
|
||||
return df
|
||||
|
||||
else:
|
||||
new_states.append("N/A")
|
||||
def check_default_rate(df: pd.DataFrame):
|
||||
|
||||
if is_dataframe is False:
|
||||
return new_states # list
|
||||
mapping = {
|
||||
'medicare' : 'MCR',
|
||||
'medicaid' : 'MCD',
|
||||
'billed charge' : 'BC',
|
||||
'allowable charge' : 'AC',
|
||||
'allowed charge' : 'AC',
|
||||
'average wholesale price' : 'AWP'
|
||||
}
|
||||
|
||||
return ", ".join(new_states) # concatenated string
|
||||
def replace_substring(string):
|
||||
for k, v in mapping.items():
|
||||
string = re.sub(r'\b' + k + r'\b', v, string, flags=re.IGNORECASE)
|
||||
return string
|
||||
|
||||
raise TypeError(
|
||||
f"Expected a string or list of strings, got {type(states).__name__}"
|
||||
)
|
||||
def safe_extract(pattern, text):
|
||||
match = re.search(pattern, text)
|
||||
return match.group(0) if match else ''
|
||||
|
||||
df['Default Rate Corrected Step 2'] = df['Default Rate Corrected Step 1']
|
||||
df_relevant = df.loc[(df['Default Rate Corrected Step 1'].isna()==False) & (df['Default Rate Corrected Step 1'] != '') & ((df['Default Term']=='') | (df['Default Term']!=df['Default Term'])), :].copy()
|
||||
|
||||
if not df_relevant.empty:
|
||||
pct_pattern = r'\d+\% of '
|
||||
reimb_pattern = r'(medicare|allowed charge|allowable charge|billed charge|medicaid|average wholesale price)'
|
||||
df_relevant.loc[:, 'Default Rate Shortened'] = df_relevant.loc[:, 'Default Rate Corrected Step 1'].apply(lambda x:
|
||||
safe_extract(pct_pattern, x.lower()) + safe_extract(reimb_pattern, x.lower())
|
||||
)
|
||||
df_relevant.loc[:,'Default Rate Shortened'] = df_relevant.loc[:, 'Default Rate Shortened'].apply(replace_substring)
|
||||
df_grouped = df_relevant.groupby(['Contract Name', 'Attachment/Exhibit', 'Default Rate Shortened'], as_index = False)[r'If rate is % of Payor or MCR [STANDARD]'].agg(lambda x: list(set(x.dropna().to_list()))).reset_index()
|
||||
df_grouped = df_grouped.rename(columns = {r'If rate is % of Payor or MCR [STANDARD]' : 'Rate Standard List'})
|
||||
try:
|
||||
df_grouped['Remove Default Rate?'] = df_grouped.apply(lambda row: any([x.lower() in row['Default Rate Shortened'].lower() for x in row['Rate Standard List']]), axis=1)
|
||||
df = df.merge(df_grouped, how = 'left', on = ['Contract Name', 'Attachment/Exhibit'])
|
||||
df['Default Rate Corrected Step 2'] = df['Default Rate Corrected Step 1']
|
||||
df.loc[df['Remove Default Rate?'] == True, 'Default Rate Corrected Step 2'] = ''
|
||||
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
#df.drop(columns = ['Default Rate Shortened'], inplace=True)
|
||||
return df
|
||||
|
||||
def populate_default_term(df, text_dict):
|
||||
|
||||
file_df = df.loc[(df['Default Rate Corrected Step 2'].isna()==False) & (df['Default Rate Corrected Step 2'] != '') & ((df['Default Term'].isna()==True) | (df['Default Term'] == '')), :]
|
||||
|
||||
file_df = file_df.dropna(subset=['Page_Num', 'Attachment/Exhibit'])
|
||||
|
||||
if not file_df.empty:
|
||||
exhibits = file_df['Attachment/Exhibit'].unique()
|
||||
for exhibit in exhibits:
|
||||
start_page = file_df.loc[file_df['Attachment/Exhibit'] == exhibit, 'Page_Num'].min()
|
||||
|
||||
end_page = file_df.loc[file_df['Attachment/Exhibit'] == exhibit, 'Pages'].min() + 1
|
||||
|
||||
extracted_text = ''
|
||||
|
||||
for page in range(int(round(start_page)), int(round(end_page))):
|
||||
|
||||
exhibit_text = text_dict[f'{page:.0f}']
|
||||
|
||||
def_rate = file_df.loc[file_df['Page_Num'] == start_page, 'Default Rate Corrected Step 2']
|
||||
|
||||
if isinstance(def_rate, pd.Series):
|
||||
def_rate = def_rate.min()
|
||||
elif not isinstance(def_rate, str):
|
||||
def_rate = str(def_rate)
|
||||
|
||||
sentences = exhibit_text.split('.')
|
||||
|
||||
pct_pattern = r'\d+\%'
|
||||
|
||||
reimb_pattern = r'(medicare|allowed charge|allowable charge|billed charge|medicaid|average wholesale price|fee schedule)'
|
||||
|
||||
if (def_rate == def_rate) and (def_rate != '') and (re.search(pct_pattern, def_rate)) and (re.search(reimb_pattern, def_rate.lower())):
|
||||
def_pct = re.search(pct_pattern, def_rate).group(0)
|
||||
def_reimb = re.search(reimb_pattern, def_rate.lower()).group(0)
|
||||
for sentence in sentences:
|
||||
sentence = sentence.replace('\n', ' ')
|
||||
if (def_pct in sentence) and (def_reimb in sentence.lower()):
|
||||
extracted_text = sentence
|
||||
break
|
||||
|
||||
if extracted_text != '':
|
||||
break
|
||||
|
||||
file_df.loc[file_df['Attachment/Exhibit'] == exhibit, 'Default Term Corrected Step 3'] = extracted_text.strip()
|
||||
file_df = file_df[['Attachment/Exhibit', 'Default Term Corrected Step 3']]
|
||||
file_df.drop_duplicates(inplace=True)
|
||||
df = df.merge(file_df, how='left', on=['Attachment/Exhibit'])
|
||||
else:
|
||||
file_df['Default Term Corrected Step 3'] = file_df['Default Term']
|
||||
file_df = file_df[['Page_Num', 'Attachment/Exhibit', 'Default Term Corrected Step 3']]
|
||||
df = df.merge(file_df, how = 'left', on = ['Page_Num', 'Attachment/Exhibit'])
|
||||
|
||||
df['Default Term Corrected Step 3'] = df.apply(lambda x: x['Default Term'] if (x['Default Term Corrected Step 3'] == '') or (x['Default Term Corrected Step 3'] != x['Default Term Corrected Step 3']) else x['Default Term Corrected Step 3'], axis = 1)
|
||||
|
||||
for col in ['Contains Only','All Checks','Default Match?','index', 'Default Rate Corrected Step 1','Default Rate Shortened','Rate Standard List','Remove Default Rate?']:
|
||||
if col in df.columns:
|
||||
df.drop(col, axis=1, inplace=True)
|
||||
else:
|
||||
pass
|
||||
|
||||
df.rename(columns={'Default Rate Corrected Step 2' : 'Default Rate Corrected', 'Default Term Corrected Step 3' : 'Default Term Corrected'}, inplace=True)
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def apply_state_check_to_df(
|
||||
df: pd.DataFrame, is_dataframe: bool = True
|
||||
#### HEALTH PLAN STATE ####
|
||||
def clean_healthplan_state(
|
||||
df: pd.DataFrame,
|
||||
contract_name: str,
|
||||
text_dict: dict[str, str],
|
||||
) -> pd.DataFrame:
|
||||
|
||||
if not isinstance(df, pd.DataFrame):
|
||||
raise TypeError(f"Expected a dataframe, got {type(df).__name__}")
|
||||
# raise TypeError(f"Expected a dataframe, got {type(df).__name__}")
|
||||
return df
|
||||
|
||||
if is_dataframe is False:
|
||||
raise ValueError(
|
||||
f"Expected 'is_dataframe' parameter as True, got {is_dataframe}"
|
||||
)
|
||||
if contract_name is None:
|
||||
# raise ValueError(
|
||||
# f"Expected 'contract_name' parameter, got {contract_name}"
|
||||
# )
|
||||
return df
|
||||
|
||||
if "Health Plan State" not in df.columns:
|
||||
raise KeyError(
|
||||
f"Column 'Health Plan State' not found in DataFrame. Available columns are: {list(df.columns)}"
|
||||
)
|
||||
# raise KeyError(
|
||||
# f"Column 'Health Plan State' not found in DataFrame. Available columns are: {list(df.columns)}"
|
||||
# )
|
||||
return df
|
||||
|
||||
df["Health Plan State_corrected"] = df["Health Plan State"].apply(
|
||||
state_check, args=(is_dataframe,)
|
||||
)
|
||||
health_plan_state = df['Health Plan State'].unique().tolist()[0]
|
||||
|
||||
answer = state_check(health_plan_state, text_dict)
|
||||
|
||||
df["Health Plan State_corrected"] = answer
|
||||
|
||||
return df
|
||||
|
||||
#### LINE OF BUSINESS ####
|
||||
def clean_lob(df, text_dict):
|
||||
|
||||
df['Line of Business'] = df['Line of Business'].str.upper()
|
||||
incorrect_map = {"MMP PLAN": "", "COMPLETE": "", "ALL OTHER HEALTH PLANS": "", "NAN" : ""}
|
||||
df['Line of Business'] = df['Line of Business'].replace(incorrect_map)
|
||||
df['LOB_Correct'] = df['Line of Business']
|
||||
|
||||
match_pattern = r'|'.join(valid.VALID_LOBS)
|
||||
|
||||
for index, row in df.iterrows():
|
||||
# Issue 1: Missing
|
||||
if utils.is_empty(row['Line of Business']) and pd.notna(row['Page_Num']): # Check if 'Line of Business' is missing
|
||||
|
||||
# Prepare to search text
|
||||
page_text = text_dict.get(str(int(row['Page_Num'])), "")
|
||||
next_page_text = text_dict.get(str(int(row['Page_Num']) + 1), "") # Assuming page_num is a numerical index
|
||||
second_page_text = text_dict.get(str(int(row['Page_Num']) + 2), "")
|
||||
|
||||
# First check in or near exhibit
|
||||
exhibit_index = page_text.find(str(row['Attachment/Exhibit']).strip())
|
||||
if exhibit_index != -1 and pd.notna(row['Attachment/Exhibit']):
|
||||
search_area = page_text[exhibit_index:exhibit_index + 200 + len(row['Attachment/Exhibit'])]
|
||||
matches = re.findall(match_pattern, search_area, re.IGNORECASE)
|
||||
if matches:
|
||||
df.at[index, 'LOB_Correct'] = ', '.join(set([m.upper() for m in matches]))
|
||||
continue
|
||||
|
||||
# Then check on entire page
|
||||
matches = re.findall(match_pattern, page_text, re.IGNORECASE)
|
||||
if matches:
|
||||
df.at[index, 'LOB_Correct'] = ', '.join(set([m.upper() for m in matches]))
|
||||
continue
|
||||
|
||||
# Then check on next page
|
||||
matches = re.findall(match_pattern, next_page_text, re.IGNORECASE)
|
||||
if matches:
|
||||
df.at[index, 'LOB_Correct'] = ', '.join(set([m.upper() for m in matches]))
|
||||
continue
|
||||
|
||||
df.at[index, 'LOB_Correct'] = 'NO MATCH FOUND - REVIEW'
|
||||
|
||||
# Issue 2: When Payer..., Medicare and/not Medicaid
|
||||
elif pd.notna(row["Service Type"]):
|
||||
service = str(row['Service Type']).lower()
|
||||
if 'where' in service and ('payer' in service or 'payor' in service):
|
||||
# Check Exhibit first
|
||||
matches = re.findall(match_pattern, str(row['Attachment/Exhibit']), re.IGNORECASE)
|
||||
if matches:
|
||||
df.at[index, 'LOB_Correct'] = ', '.join(set([m.upper() for m in matches]))
|
||||
else:
|
||||
df.at[index, 'LOB_Correct'] = str(row['Line of Business']).upper()
|
||||
|
||||
# Non-Issue
|
||||
else:
|
||||
df.at[index, 'LOB_Correct'] = str(row['Line of Business']).upper()
|
||||
|
||||
return df
|
||||
|
||||
|
||||
#### Rate Standard ####
|
||||
def reimb_methodology_fix(rm: str):
|
||||
|
||||
d = {'FULL_METHODOLOGY' : rm}
|
||||
prompt = BOTTOM_UP_METHODOLOGY_BREAKOUT(d)
|
||||
answer = claude_funcs.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, '', max_tokens=512
|
||||
)
|
||||
answer_dict = dict_operations.secondary_string_to_dict(answer, '')
|
||||
d.update(answer_dict)
|
||||
|
||||
# Ensure all bottom up methodology keys are present
|
||||
for key in [
|
||||
"LESSER",
|
||||
"RATE_STANDARD",
|
||||
"RATE_SHORT",
|
||||
"FLAT_FEE_STANDARD",
|
||||
"LESSER_RATE",
|
||||
"NOT_TO_EXCEED",
|
||||
]:
|
||||
if key not in d.keys():
|
||||
d[key] = ""
|
||||
|
||||
# SHORT_METHODOLOGY
|
||||
if "%" in d["RATE_STANDARD"]:
|
||||
d["SHORT_METHODOLOGY"] = get_short_methodology(d["RATE_STANDARD"])
|
||||
elif "$" in d["FLAT_FEE_STANDARD"]:
|
||||
d["SHORT_METHODOLOGY"] = "Flat Fee"
|
||||
else:
|
||||
d["SHORT_METHODOLOGY"] = "N/A"
|
||||
|
||||
return d
|
||||
|
||||
def clean_rate_standard(df: pd.DataFrame) -> pd.DataFrame:
|
||||
|
||||
df_relevant = df.loc[(df['Reimb. Methodology'].isna() == False) & (df[r'If rate is % of Payor or MCR [STANDARD]'].isna() == True) & (df['Flat Fee'].isna() == True), :]
|
||||
|
||||
if not df_relevant.empty:
|
||||
rm_list = df_relevant['Reimb. Methodology'].unique()
|
||||
|
||||
dict_list = []
|
||||
|
||||
for rm in rm_list:
|
||||
|
||||
d = reimb_methodology_fix(rm)
|
||||
|
||||
dict_list.append(d)
|
||||
|
||||
results_df = pd.DataFrame.from_records(dict_list)
|
||||
|
||||
results_df = results_df.add_suffix('_fixed')
|
||||
|
||||
df = pd.merge(df, results_df, how = 'left', right_on = 'FULL_METHODOLOGY_fixed', left_on = 'Reimb. Methodology')
|
||||
|
||||
return df
|
||||
|
||||
|
||||
#### LESSER ####
|
||||
def clean_lesser(df, text_dict):
|
||||
df['Lesser_Flag'] = ""
|
||||
# For Exhibit
|
||||
all_dfs = []
|
||||
for page_num, page_df in df.groupby('Page_Num', dropna=False):
|
||||
if pd.notna(page_num) and str(int(page_num)) in text_dict.keys():
|
||||
contract_page = str(int(page_num))
|
||||
try:
|
||||
contract_text = '\n'.join([text_dict[str(int(contract_page)+i)] for i in range(2)])
|
||||
except:
|
||||
contract_text = text_dict[str(int(contract_page))]
|
||||
|
||||
output_lesser_count = (page_df['Lesser of Logic language, included (Y/N)'] == 'Y').sum()
|
||||
row_count = page_df.shape[0]
|
||||
page_lesser_count = len(re.findall(r"lesser|lessor", contract_text, re.IGNORECASE))
|
||||
# Likely Incorrect
|
||||
if output_lesser_count == 1 and row_count > 1 and page_lesser_count == 1:
|
||||
page_df.loc[:, 'Lesser_Flag'] = 'Likely Incorrect'
|
||||
# Likely Correct
|
||||
elif page_lesser_count == output_lesser_count == row_count:
|
||||
page_df.loc[:, 'Lesser_Flag'] = 'Likely Correct'
|
||||
# Other
|
||||
else:
|
||||
page_df.loc[:, 'Lesser_Flag'] = "No Flag"
|
||||
else:
|
||||
page_df.loc[:, 'Lesser_Flag'] = "No Flag"
|
||||
|
||||
all_dfs.append(page_df)
|
||||
|
||||
try:
|
||||
final_df = pd.concat(all_dfs, ignore_index=True)
|
||||
except:
|
||||
final_df = df.copy()
|
||||
|
||||
# Second Column
|
||||
final_df['Service-Methodology'] = final_df['Service Type'].astype(str) + ' - ' + final_df['Reimb. Methodology'].astype(str)
|
||||
return final_df
|
||||
|
||||
|
||||
#### PROVIDER TYPE LEVEL 2 ####
|
||||
def clean_provider_type_2(df):
|
||||
valid_lists = {
|
||||
'PROF': [
|
||||
"Physician", "Primary Care Provider", "Dual Capacity Physician",
|
||||
"Mid-Level Practicioner", "Specialist", "OB/GYN", "Behavioral Health",
|
||||
"Therapist", "Surgery", "Independent RHC", "Ambulance Services",
|
||||
"Home Health", "SNF"
|
||||
],
|
||||
'ANC': [
|
||||
"DME", "Durable Medical Equipment", "Radiology", "Lab", "Home Health",
|
||||
"Hospice", "Dialysis", "PT/OT/ST", "Urgent Care", "Home Infusion",
|
||||
"Ambulatory Surgery Center", "ASC", "Ambulance Services", "Home Health"
|
||||
],
|
||||
'FAC': [
|
||||
"Skilled Nursing Facility", "Ambulatory Surgery Center", "ASC",
|
||||
"Hospital", "Clinic", "Rural Health Clinic", "FQHC", "Long Term Care",
|
||||
"Community Mental Health Center"
|
||||
]
|
||||
}
|
||||
result_df = df.copy()
|
||||
result_df['PROV_TYPE_LEVEL_2_Corrected'] = result_df['Provider Type - Level 2']
|
||||
unique_exhibits = df['Attachment/Exhibit'].unique()
|
||||
for exhibit in unique_exhibits:
|
||||
if pd.isna(exhibit):
|
||||
continue
|
||||
match, _ = check_field_for_matches(exhibit, valid_lists)
|
||||
if match:
|
||||
exhibit_mask = (result_df['Attachment/Exhibit'] == exhibit) & (result_df['PROV_TYPE_LEVEL_2_Corrected'].isna())
|
||||
if exhibit_mask.any():
|
||||
result_df.loc[exhibit_mask, 'PROV_TYPE_LEVEL_2_Corrected'] = match
|
||||
blank_mask = result_df['PROV_TYPE_LEVEL_2_Corrected'].isna()
|
||||
for idx, row in result_df[blank_mask].iterrows():
|
||||
for field in df.columns:
|
||||
if field != 'PROV_TYPE_LEVEL_2_Corrected':
|
||||
match, _ = check_field_for_matches(row[field], valid_lists)
|
||||
if match:
|
||||
result_df.at[idx, 'PROV_TYPE_LEVEL_2_Corrected'] = match
|
||||
break
|
||||
return result_df
|
||||
|
||||
#### IRS ####
|
||||
def clean_irs(abc_df, filename, text_dict ):
|
||||
"""Gets the answers for fixed IRS, TIN Others and merges them with clean output
|
||||
Args:
|
||||
abc_df(df): Clean input which is grouped on 'Contract Name'
|
||||
filename: Name of the contract
|
||||
text_dict (dict): Dictionary keyed by string page-number and valued by actual textract output page
|
||||
Returns:
|
||||
merged_df(df): Left joined [abc_df, irs_hotfix_df] on 'Contract Name' with additional '_corrected' columns
|
||||
"""
|
||||
|
||||
# gets the irs hotfixes in
|
||||
irs_answers = irs_hotfix(filename, text_dict)
|
||||
irs_df = pd.DataFrame([irs_answers])
|
||||
|
||||
# adds corrected suffix for the hotfixes
|
||||
irs_df.columns = [col + '_corrected' for col in irs_df.columns]
|
||||
|
||||
# join on 'Contract Name'
|
||||
irs_df["Contract Name"] = str(filename)
|
||||
df_merged = pd.merge(abc_df, irs_df, on='Contract Name', indicator=True, how='left')
|
||||
df_merged = clean_output_postprocess(df_merged)
|
||||
|
||||
# overwrite the corrected columns with clean output if it already has an answer
|
||||
df_merged['PROV_GROUP_TIN_corrected'] = df_merged['dummy'].fillna(df_merged['PROV_GROUP_TIN_corrected'])
|
||||
df_merged['PROV_TIN_OTHER_corrected'] = df_merged['dummy_other'].fillna(df_merged['PROV_TIN_OTHER_corrected'])
|
||||
df_merged['PROV_TIN_GROUP_SIGNATORY_corrected'] = df_merged['dummy_other_signatory'].fillna(df_merged['PROV_TIN_GROUP_SIGNATORY_corrected'])
|
||||
df_merged = df_merged.rename(columns={'PROV_GROUP_TIN_corrected': 'IRS_corrected'})
|
||||
df_merged = df_merged.loc[:, ~df_merged.columns.str.contains('dummy')]
|
||||
|
||||
return df_merged
|
||||
|
||||
|
||||
##### CONTRACT EFFECTIVE DATE #####
|
||||
def contract_effective_date_fix(contract_name: str, text_dict: dict[str, str],is_meridian: bool=False) -> str:
|
||||
# perform smart chunking
|
||||
page_list = ac_smart_chunking.chunk_with_include_exclude_keywords(text_dict,
|
||||
include_keywords = cnc_hotfix_effective_date_utils.included_keyword,
|
||||
exclude_keywords = cnc_hotfix_effective_date_utils.excluded_keyword
|
||||
)
|
||||
|
||||
context = "\n".join(text_dict[page] for page in page_list)
|
||||
prompt = cnc_hotfix_effective_date_utils.get_effective_date_prompt(context) # get effective date - this is for both meridian and non-meridian
|
||||
|
||||
try:
|
||||
claude_answer_raw = claude_funcs.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192)
|
||||
claude_answer_extracted = re.findall(pattern=cnc_hotfix_effective_date_utils.regex_backticks, string=claude_answer_raw)[-1]
|
||||
|
||||
except Exception as e:
|
||||
raise
|
||||
|
||||
# meridian special case
|
||||
if is_meridian:
|
||||
if claude_answer_extracted == "N/A":
|
||||
# perform smart chunking with signature date pages included
|
||||
page_list = ac_smart_chunking.chunk_with_include_exclude_keywords(text_dict,
|
||||
include_keywords = cnc_hotfix_effective_date_utils.included_keyword + ["Signature"],
|
||||
exclude_keywords = cnc_hotfix_effective_date_utils.excluded_keyword
|
||||
)
|
||||
|
||||
context = "\n".join(text_dict[page] for page in page_list)
|
||||
prompt = cnc_hotfix_effective_date_utils.get_effective_date_meridian_prompt(context)
|
||||
|
||||
try:
|
||||
claude_answer_raw = claude_funcs.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192)
|
||||
claude_answer_extracted = re.findall(pattern=cnc_hotfix_effective_date_utils.regex_backticks, string=claude_answer_raw)[-1]
|
||||
except Exception as e:
|
||||
raise
|
||||
|
||||
return claude_answer_extracted
|
||||
|
||||
def clean_contract_effective_date(
|
||||
df: pd.DataFrame,
|
||||
file_name: str,
|
||||
text_dict: dict[str, str]
|
||||
) -> pd.DataFrame:
|
||||
"""Apply `contract_effective_date_fix` to a grouped input dataframe, `df`. A new column
|
||||
is added.
|
||||
|
||||
Args:
|
||||
df (pd.DataFrame): Grouped dataframe for one contract. Can have multiple rows because
|
||||
of B-fields. Contract effective date is an AC-field, so one per document.
|
||||
file_name (str): Filename of the contract
|
||||
text_dict (dict[str, str]): String-keyed dictionary valued by contract text
|
||||
|
||||
Raises:
|
||||
TypeError: If a non-dataframe is passed in as `df`
|
||||
KeyError: If 'Contract Effective Date' is not found in the input `df`
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: A grouped dataframe with an extra column,
|
||||
`Contract Effective Date_corrected`
|
||||
"""
|
||||
|
||||
if not isinstance(df, pd.DataFrame):
|
||||
raise TypeError(f"Expected a dataframe, got {type(df).__name__}")
|
||||
|
||||
if "Contract Effective Date" not in df.columns:
|
||||
raise KeyError(
|
||||
f"Column 'Contract Effective Date' not found in DataFrame. Available columns are: {list(df.columns)}"
|
||||
)
|
||||
|
||||
contract_effective_date = df['Contract Effective Date'].dropna().unique().tolist()[0] if not df['Contract Effective Date'].dropna().empty else None # previous run's contract effective date (if any)
|
||||
|
||||
payer_name = df['Payer Name'].dropna().unique().tolist()
|
||||
|
||||
if len(payer_name) == 0:
|
||||
payer_name = ""
|
||||
else:
|
||||
payer_name = payer_name[0]
|
||||
|
||||
is_meridian = any(re.search(r'\b' + keyword + r'\b', payer_name, re.IGNORECASE) for keyword in ['meridian'])
|
||||
|
||||
if utils.is_empty(contract_effective_date): # Try prompting (with smart chunking) when the contract effective date is missing
|
||||
try:
|
||||
answer = contract_effective_date_fix(file_name, text_dict,is_meridian)
|
||||
except Exception as e:
|
||||
# KATON: Do we want to give out "answer='Error'" in this case?
|
||||
print(f"Error processing {file_name}, got error: {e}")
|
||||
else:
|
||||
answer = contract_effective_date
|
||||
|
||||
if answer == "N/A":
|
||||
df["Contract Effective Date_corrected"] = answer
|
||||
else:
|
||||
df["Contract Effective Date_corrected"] = pd.to_datetime(answer, dayfirst=False).strftime('%m/%d/%Y')
|
||||
|
||||
return df
|
||||
|
||||
|
||||
#### NPI ####
|
||||
def clean_npi(abc_df, filename, text_dict, top_sheet_dict ):
|
||||
npi_answers = npi_hotfix(text_dict=text_dict)
|
||||
if npi_answers["NPI"] == "N/A" and npi_answers["NPI_other"] == "N/A":
|
||||
npi_answers = npi_hotfix(top_sheet_dict)
|
||||
|
||||
npi_df = pd.DataFrame([npi_answers])
|
||||
npi_df.columns = [col + '_corrected' for col in npi_df.columns]
|
||||
npi_df["Contract Name"] = str(filename)
|
||||
|
||||
df_merged = pd.merge(abc_df, npi_df, on='Contract Name', how='left')
|
||||
df_merged = npi_post_process(df_merged)
|
||||
|
||||
df_merged['NPI_corrected'] = df_merged['dummy'].fillna(df_merged['NPI_corrected'])
|
||||
df_merged['NPI_other_corrected'] = df_merged['dummy_other'].fillna(df_merged['NPI_other_corrected'])
|
||||
df_merged = df_merged.loc[:, ~df_merged.columns.str.contains('dummy')]
|
||||
|
||||
return df_merged
|
||||
|
||||
|
||||
#### TERM GROUP ####
|
||||
def clean_term_group(df: pd.DataFrame,
|
||||
contract_name: str,
|
||||
text_dict: dict[str, str],
|
||||
ac_chunks: dict[str, str]) -> pd.DataFrame:
|
||||
"""this function takes previously generated file as dataframe and one contract name as input. If value for term is null
|
||||
it runs the term group prompts and sets term clause_corrected, auto-renewal_corrected and termination date_columns
|
||||
"""
|
||||
df["Term Clause_corrected"] = df['Term Clause']
|
||||
df["Contract Auto-Renewal Indicator_corrected"] = df['Contract Auto-Renewal Indicator']
|
||||
df["Termination Date_corrected"] = df['Termination Date']
|
||||
|
||||
# identify contracts where term is blank
|
||||
term_df = df[(df['Term Clause'].isna())|(df['Term Clause'].str.len() < 10)]
|
||||
term_df['Filename'] = term_df['Contract Name'] + '.txt'
|
||||
contract_list = term_df['Filename'].to_list()
|
||||
contract_list = list(set(contract_list))
|
||||
|
||||
# get term group fields and corresponding prompts
|
||||
fields = keywords.GROUPED_KEYWORD_MAPPINGS['term_group']["fields"]
|
||||
fields.sort()
|
||||
questions = {}
|
||||
for field in fields:
|
||||
questions[field] = prompts.AC_DICT[field]
|
||||
|
||||
# process the contract
|
||||
if contract_name in contract_list:
|
||||
ac_answers_dict = {}
|
||||
context = ac_chunks['term_group']
|
||||
context = context[0 : min(100000, len(context)) - 1]
|
||||
prompt = prompts.AC_MULTI_FIELD_TEMPLATE(context, questions)
|
||||
field_group_answer_raw = claude_funcs.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192
|
||||
)
|
||||
field_group_answer_raw = field_group_answer_raw.replace("_DATE", "_DT")
|
||||
field_group_answer_dict = ac_funcs.json_parsing_search(field_group_answer_raw, fields)
|
||||
|
||||
for field in fields:
|
||||
field_answer = field_group_answer_dict[field]
|
||||
ac_answers_dict[field] = field_answer
|
||||
|
||||
ac_df = pd.DataFrame([ac_answers_dict])
|
||||
ac_df = postprocessing_funcs.clean_term_clause(ac_df)
|
||||
ac_df = postprocessing_funcs.clean_auto_renewal_ind(ac_df)
|
||||
|
||||
if 'TERM_CLAUSE' in ac_df.columns:
|
||||
df.loc[df['Contract Name'] == contract_name.replace('.txt',''), 'Term Clause_corrected'] = ac_df['TERM_CLAUSE'].iloc[0]
|
||||
if 'CONTRACT_AUTO_RENEWAL_IND' in ac_df.columns:
|
||||
df.loc[df['Contract Name'] == contract_name.replace('.txt',''), 'Contract Auto-Renewal Indicator_corrected'] = ac_df['CONTRACT_AUTO_RENEWAL_IND'].iloc[0]
|
||||
if 'CONTRACT_TERMINATION_DT' in ac_df.columns:
|
||||
df.loc[df['Contract Name'] == contract_name.replace('.txt',''), 'Termination Date_corrected'] = ac_df['CONTRACT_TERMINATION_DT'].iloc[0]
|
||||
|
||||
return df
|
||||
|
||||
#### Agreement Name ####
|
||||
# ensure that prompt for CONTRACT_TITLE is updated and df contains field Agreement Name (Contract Title)_corrected
|
||||
def clean_agreement_name(df: pd.DataFrame,
|
||||
contract_name: str,
|
||||
text_dict: dict[str, str]) -> pd.DataFrame:
|
||||
"""this function first runs agreement name prompt on first 5 pages.
|
||||
If agreement name is not found, it runs on the context selected based on keywords.
|
||||
"""
|
||||
|
||||
contract_title_chunk_dict = dict(itertools.islice(text_dict.items(), 5))
|
||||
question = prompts.AC_DICT["CONTRACT_TITLE"]
|
||||
|
||||
context = "\n".join(
|
||||
[contract_title_chunk_dict[str(page_num)] for page_num in contract_title_chunk_dict.keys()]
|
||||
)
|
||||
|
||||
prompt = prompts.AC_SINGLE_FIELD_TEMPLATE(context, question)
|
||||
prompt_answer_raw = claude_funcs.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192)
|
||||
|
||||
if prompt_answer_raw == "N/A":
|
||||
agreement_keywords = ["AGREEMENT", "AMENDMENT", "ADDENDUM", "PROVIDER", "CONTRACT", "MEMORANDUM", "LETTER", "REQUEST", "EFFECTIVE"]
|
||||
page_list = ac_smart_chunking.chunk_hierarchical(
|
||||
text_dict, agreement_keywords, False
|
||||
)
|
||||
context = "\n".join(
|
||||
[text_dict[str(page_num)] for page_num in page_list]
|
||||
)
|
||||
prompt = prompts.AC_SINGLE_FIELD_TEMPLATE(context, question)
|
||||
prompt_answer_raw = claude_funcs.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192)
|
||||
|
||||
df.loc[df['Contract Name'] == contract_name.replace('.txt',''), 'Agreement Name (Contract Title)_corrected'] = prompt_answer_raw
|
||||
|
||||
return df
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
included_keyword = ["Effective Date", "entered into", "shall be effective"]
|
||||
excluded_keyword = ["%", "$","Permit","Certificat","Registration","Department of Public Health"]
|
||||
regex_backticks = r"\|([^|]+)\|"
|
||||
|
||||
def get_effective_date_prompt(context):
|
||||
prompt_template = f""""
|
||||
Please analyze the following contract and extract ONLY the effective date. Follow these precise rules in order:
|
||||
|
||||
1. Look for dates that appear after phrases matching these exact patterns:
|
||||
- "Effective Date:" followed by a date
|
||||
- "FOR HEALTH PLAN USE ONLY" section containing "Effective Date:"
|
||||
- "To be completed by Health Plan only:" section containing "Effective Date:"
|
||||
|
||||
2. If multiple matches are found:
|
||||
- Prioritize dates within "FOR HEALTH PLAN USE ONLY" or "To be completed by Health Plan only" sections
|
||||
- Then prioritize dates that directly follow "Effective Date:"
|
||||
|
||||
3. Do not use the date found in the "Signature Date" section.
|
||||
- If there is nothing next to "Effective Date:", return N/A.
|
||||
- If there is only one date in the document, don't extract this date unless it is explicitly labeled "effective date". It is okay to return N/A.
|
||||
|
||||
4. If partial dates are found, return N/A.
|
||||
|
||||
5. Return the date in YYYY-MM-DD format or N/A if date not found. Enclose just your final answer in |pipes|.
|
||||
|
||||
Here is the contract to analyze:
|
||||
|
||||
## START CONTRACT TEXT ##
|
||||
|
||||
{context}
|
||||
|
||||
## END CONTRACT TEXT ##
|
||||
|
||||
Ensure that you do not use the date found in the "Signature Date" section.
|
||||
|
||||
Before returning the output, VERIFY the date format is YYYY-MM-DD. If the date is not found, return N/A.
|
||||
|
||||
Finally state "Answer: |YYYY-MM-DD| or |N/A|"
|
||||
"""
|
||||
|
||||
return prompt_template
|
||||
|
||||
def get_effective_date_meridian_prompt(context):
|
||||
prompt_template = f""""
|
||||
Please analyze the following contract and extract ONLY the Signature Date. Follow these precise rules in order:
|
||||
|
||||
1. Look for Signature Date
|
||||
|
||||
2. If multiple matches are found:
|
||||
- Prioritize the latest full date
|
||||
|
||||
3. If only partial dates are found, return N/A.
|
||||
|
||||
4. Return the date in YYYY-MM-DD format or N/A if date not found. Enclose just your final answer in |pipes|.
|
||||
|
||||
Here is the contract to analyze:
|
||||
|
||||
## START CONTRACT TEXT ##
|
||||
|
||||
{context}
|
||||
|
||||
## END CONTRACT TEXT ##
|
||||
|
||||
Before returning the output, VERIFY the date format is YYYY-MM-DD. If the date is not found, return N/A.
|
||||
|
||||
Finally state "Answer: |YYYY-MM-DD| or |N/A|"
|
||||
"""
|
||||
|
||||
return prompt_template
|
||||
@@ -164,6 +164,8 @@ TABLE_ANALYSIS_NAME = f"{BATCH_ID}-Table-Analysis.csv"
|
||||
|
||||
######################################## AWS SETTINGS #########################################
|
||||
|
||||
# RUN_MODE = "local" # for testing purposes only
|
||||
|
||||
# Bedrock
|
||||
if RUN_MODE == "local":
|
||||
AWS_ACCESS_KEY_ID = get_value_from_env("AWS_ACCESS_KEY_ID")
|
||||
|
||||
@@ -0,0 +1,457 @@
|
||||
import ast
|
||||
|
||||
from utils import find_regex_matches, add_hyphen_if_needed
|
||||
import re
|
||||
import pandas as pd
|
||||
|
||||
from valid import STATE_MAP
|
||||
from utils import is_empty
|
||||
import ac_smart_chunking
|
||||
import cnc_hotfix_effective_date_utils
|
||||
import claude_funcs
|
||||
import config
|
||||
|
||||
keywords_to_detect_invalid_state_name = [ #This is for health plan state when the LLM talks to itself
|
||||
"suggest",
|
||||
"The Contract Text",
|
||||
"refers",
|
||||
"For ",
|
||||
"Answer",
|
||||
"Multiple",
|
||||
"Statewide",
|
||||
]
|
||||
|
||||
def convert_to_list(value):
|
||||
if isinstance(value, str):
|
||||
# Remove any trailing commas or spaces
|
||||
value = value.strip().replace(',', ' ') # Replace commas with spaces
|
||||
|
||||
# Check if the string is in the form of a list and safely convert it
|
||||
if value.startswith('[') and value.endswith(']'):
|
||||
try:
|
||||
return ast.literal_eval(value) # Safely convert string to list
|
||||
except:
|
||||
return value.split() # Fallback to split by spaces if literal_eval fails
|
||||
else:
|
||||
# Split by spaces and handle both alphanumeric and numeric values
|
||||
return value.split() # Just split by spaces, treat everything as a string
|
||||
elif isinstance(value, list):
|
||||
return value # If it's already a list, return as is
|
||||
else:
|
||||
return [value] # Wrap non-string, non-list types in a list
|
||||
|
||||
|
||||
def list_to_string(lst):
|
||||
if not lst: # Check if the list is empty
|
||||
return None
|
||||
else:
|
||||
return ", ".join(lst) # Join elements with commas
|
||||
|
||||
def remove_alphabets(input_str):
|
||||
# Check if the string contains any alphabetic characters
|
||||
if any(char.isalpha() for char in input_str):
|
||||
return None # Return None if there are alphabets
|
||||
else:
|
||||
return input_str # Return the string if no alphabets are present
|
||||
|
||||
|
||||
def clean_output_postprocess(merged_df):
|
||||
merged_df['dummy'] = merged_df['IRS #']
|
||||
merged_df['dummy_other'] = merged_df['PROV_TIN_OTHER']
|
||||
merged_df['dummy_other_signatory'] = merged_df["PROV_GROUP_TIN_SIGNATORY"]
|
||||
|
||||
merged_df['dummy'] = merged_df['dummy'].astype(str)
|
||||
merged_df['dummy_other'] = merged_df['dummy_other'].astype(str)
|
||||
merged_df['dummy_other_signatory'] = merged_df['dummy_other_signatory'].astype(str)
|
||||
|
||||
merged_df['dummy'] = merged_df['dummy'].apply(convert_to_list)
|
||||
merged_df['dummy_other'] = merged_df['dummy_other'].apply(convert_to_list)
|
||||
merged_df['dummy_other_signatory'] = merged_df['dummy_other_signatory'].apply(convert_to_list)
|
||||
|
||||
merged_df['dummy'] = merged_df['dummy'].apply(lambda lst: [x for x in [remove_alphabets(x) for x in lst] if x is not None])
|
||||
merged_df['dummy_other'] = merged_df['dummy_other'].apply(lambda lst: [x for x in [remove_alphabets(x) for x in lst] if x is not None])
|
||||
merged_df['dummy_other_signatory'] = merged_df['dummy_other_signatory'].apply(lambda lst: [x for x in [remove_alphabets(x) for x in lst] if x is not None])
|
||||
|
||||
merged_df['dummy'] = merged_df['dummy'].apply(lambda lst: [add_hyphen_if_needed(x) for x in lst])
|
||||
merged_df['dummy_other'] = merged_df['dummy_other'].apply(lambda lst: [add_hyphen_if_needed(x) for x in lst])
|
||||
merged_df['dummy_other_signatory'] = merged_df['dummy_other_signatory'].apply(lambda lst: [add_hyphen_if_needed(x) for x in lst])
|
||||
|
||||
merged_df['dummy'] = merged_df['dummy'].apply(list_to_string)
|
||||
merged_df['dummy_other'] = merged_df['dummy_other'].apply(list_to_string)
|
||||
merged_df['dummy_other_signatory'] = merged_df['dummy_other_signatory'].apply(list_to_string)
|
||||
|
||||
return merged_df
|
||||
|
||||
|
||||
def check_tin_in_filename(filename):
|
||||
"""
|
||||
Checks for the pattern match in filename
|
||||
|
||||
Args:
|
||||
filename(str): Name of the file
|
||||
Returns:
|
||||
tin_from_title(str): regex match for patterns in filename
|
||||
"""
|
||||
patterns = [r"\b\d{2}-\d{7}\b", r"\b\d{9}\b"]
|
||||
for pattern in patterns:
|
||||
tin_from_title = re.findall(pattern, filename)
|
||||
if len(tin_from_title) == 1:
|
||||
tin_from_title = [add_hyphen_if_needed(s) for s in tin_from_title]
|
||||
return tin_from_title[0]
|
||||
return 'empty'
|
||||
|
||||
|
||||
def signature_page_irs(text_dict, page_list, matches, pattern, filename):
|
||||
"""
|
||||
In case of multiple matches, checks if a TIN is on the Signature page
|
||||
|
||||
Args:
|
||||
text_dict (dict): Dictionary keyed by string page-number and valued by actual textract output page
|
||||
page_list(list): List of pages where the match was found
|
||||
matches(list): List of regex match(s)
|
||||
Returns
|
||||
provider_tin(str): Provider group TIN - either from signature page or filename
|
||||
other_tins(str): Remaining TINs
|
||||
"""
|
||||
# loops through regex matches and thier pages to check for signature page
|
||||
tin_on_signature_page = []
|
||||
for tin in matches:
|
||||
for page in page_list:
|
||||
if tin in text_dict[page] and 'signature' in text_dict[page]:
|
||||
tin_on_signature_page.append(tin)
|
||||
tin_on_signature_page = list(set(tin_on_signature_page))
|
||||
|
||||
#adds hyphen if needed
|
||||
matches = [add_hyphen_if_needed(s) for s in matches]
|
||||
tin_on_signature_page = [add_hyphen_if_needed(s) for s in tin_on_signature_page]
|
||||
|
||||
#case1 - If 1 TIN found on signature page, return it, put the remaining ones in "TIN Others" have signature_tin = N/A
|
||||
if len(tin_on_signature_page) == 1:
|
||||
provider_tin = tin_on_signature_page[0]
|
||||
signature_tin = "N/A"
|
||||
matches = [item for item in matches if item not in tin_on_signature_page]
|
||||
other_tins = ", ".join(matches)
|
||||
return provider_tin, signature_tin, other_tins
|
||||
|
||||
#case2 - No TIN found on the signature page, check in filename
|
||||
elif len(tin_on_signature_page) == 0:
|
||||
provider_tin = check_tin_in_filename(filename)
|
||||
if provider_tin == 'empty':
|
||||
provider_tin = "N/A"
|
||||
signature_tin = "N/A"
|
||||
other_tins = [item for item in matches if item != provider_tin]
|
||||
other_tins = ", ".join(other_tins)
|
||||
return provider_tin, signature_tin, other_tins
|
||||
|
||||
#case3 - More than 1 TIN on signature page - check for TIN in file name, let the others go in signatory_tins
|
||||
# and finally check for TINs not on signature page but in "matches" list to add them to "tin_others"
|
||||
else:
|
||||
filename_tin_check = check_tin_in_filename(filename)
|
||||
if filename_tin_check == 'empty':
|
||||
provider_tin = "N/A"
|
||||
signature_tin = ", ".join(tin_on_signature_page)
|
||||
other_tins = "N/A"
|
||||
return provider_tin, signature_tin, other_tins
|
||||
else:
|
||||
provider_tin = filename_tin_check
|
||||
signature_tin = [item for item in tin_on_signature_page if item != provider_tin]
|
||||
signature_tin = ", ".join(signature_tin)
|
||||
|
||||
if len(matches)> len(tin_on_signature_page):
|
||||
other_tins = [item for item in matches if item not in tin_on_signature_page and item != provider_tin]
|
||||
other_tins = ", ".join(other_tins)
|
||||
return provider_tin, signature_tin, other_tins
|
||||
else:
|
||||
other_tins = "N/A"
|
||||
return provider_tin, signature_tin, other_tins
|
||||
|
||||
|
||||
def irs_hotfix(filename, text_dict: dict[str, str]):
|
||||
"""Returns dictionery with Provider and other TIN(s) based on regex matches and presence on Signatory page
|
||||
|
||||
Args:
|
||||
filename: Name of the contract
|
||||
text_dict (dict): Dictionary keyed by string page-number and valued by actual textract output page
|
||||
|
||||
Returns:
|
||||
TIN_dict(dict): A dictionery with keys 'PROV_GROUP_TIN' & 'PROV_TIN_OTHER' and values as respective TIN(s)
|
||||
"""
|
||||
TIN_dict = {}
|
||||
|
||||
patterns = [r"\b\d{2}-\d{7}\b",
|
||||
r"\b\d{9}\b"]
|
||||
for pattern in patterns:
|
||||
matches, page_list = find_regex_matches(pattern, text_dict)
|
||||
matches = list(set(matches))
|
||||
if matches:
|
||||
if len(matches) == 1:
|
||||
matches = [add_hyphen_if_needed(s) for s in matches]
|
||||
TIN_dict["PROV_GROUP_TIN"] = matches[0]
|
||||
TIN_dict["PROV_TIN_OTHER"] = "N/A"
|
||||
TIN_dict["PROV_TIN_GROUP_SIGNATORY"] = "N/A"
|
||||
return TIN_dict
|
||||
|
||||
elif len(matches) >= 1:
|
||||
irs_tin, signatory_tin, other_tins = signature_page_irs(text_dict, page_list, matches, pattern, filename)
|
||||
TIN_dict["PROV_GROUP_TIN"] = irs_tin
|
||||
TIN_dict["PROV_TIN_OTHER"] = other_tins
|
||||
TIN_dict["PROV_TIN_GROUP_SIGNATORY"] = signatory_tin
|
||||
return TIN_dict
|
||||
|
||||
# If no matches were found in the text_dict, check the filename
|
||||
for pattern in patterns:
|
||||
tin_from_title = check_tin_in_filename(filename)
|
||||
TIN_dict["PROV_TIN_OTHER"] = "N/A"
|
||||
TIN_dict["PROV_TIN_GROUP_SIGNATORY"] = "N/A"
|
||||
if tin_from_title == "empty":
|
||||
TIN_dict["PROV_GROUP_TIN"] = "N/A"
|
||||
else:
|
||||
TIN_dict["PROV_GROUP_TIN"] = tin_from_title
|
||||
return TIN_dict
|
||||
|
||||
|
||||
|
||||
def state_check(states: str | list[str], text_dict: dict[str, str]) -> str:
|
||||
|
||||
if not isinstance(states, str) and not isinstance(states,list):
|
||||
states = 'N/A'
|
||||
|
||||
if (
|
||||
isinstance(states, list) and all(isinstance(state, str) for state in states)
|
||||
) or isinstance(states, str): # If it's a list of strings or a string...
|
||||
|
||||
if isinstance(states, str): # If it's a string, turn it to a list (even a singleton)
|
||||
if "," in states:
|
||||
states = states.split(",")
|
||||
else:
|
||||
states = [states]
|
||||
|
||||
result_states = []
|
||||
for state in states:
|
||||
state = state.strip()
|
||||
|
||||
# fix 1
|
||||
if state == "Hlorida":
|
||||
state = "Florida"
|
||||
|
||||
# fix 2 - LLM self-talking
|
||||
if any(
|
||||
kw.lower() in state.lower()
|
||||
for kw in keywords_to_detect_invalid_state_name
|
||||
):
|
||||
state = "N/A"
|
||||
|
||||
if state.upper() in STATE_MAP.keys(): # Check if `state` is a state abbreviation.
|
||||
result_states.append(STATE_MAP[state.upper()].title())
|
||||
|
||||
elif state.strip().title() in STATE_MAP.values(): # Check if `state` is already an expanded name state from the validation set.
|
||||
result_states.append(state.strip().title())
|
||||
|
||||
else:
|
||||
result_states.append("N/A")
|
||||
|
||||
result_states = set(result_states)
|
||||
result_states.discard("N/A") # Discard any N/As (so if `result_states` is ['Illinois', 'N/A'], it returns 'Illinois' instead of 'Multiple')
|
||||
|
||||
if len(result_states) == 1:
|
||||
return list(result_states)[0]
|
||||
|
||||
elif len(result_states) > 1:
|
||||
return "Multiple"
|
||||
|
||||
elif len(result_states) == 0:
|
||||
state = get_health_plan_state(text_dict) # search for state(s) in contract. If it's not found, `N/A` will be returned
|
||||
return state
|
||||
|
||||
raise TypeError(
|
||||
f"Expected a string or list of strings, got {type(states).__name__}"
|
||||
)
|
||||
|
||||
|
||||
|
||||
def get_health_plan_state(text_dict: dict[str, str]) -> str:
|
||||
"""Go through a contract text and extract any mention of any state
|
||||
|
||||
Args:
|
||||
text_dict (dict[str, str]): A string-keyed dictionary valued by contract text
|
||||
|
||||
Returns:
|
||||
str: A single state extracted from the text, `multiple`, or `N/A`
|
||||
"""
|
||||
state_names = list(STATE_MAP.values())
|
||||
|
||||
pattern = r'\b(?:' + '|'.join(re.escape(state) for state in state_names) + r')\b'
|
||||
|
||||
states_set = set()
|
||||
|
||||
for page_num, page_content in text_dict.items():
|
||||
state_matches = re.findall(pattern, page_content, flags=re.IGNORECASE)
|
||||
state_matches = [state.strip().title() for state in state_matches]
|
||||
states_set.update(state_matches)
|
||||
|
||||
states_list = list(states_set)
|
||||
|
||||
if len(states_list) > 1:
|
||||
state = "Multiple"
|
||||
elif len(states_list) == 1:
|
||||
state = states_list[0]
|
||||
else:
|
||||
state = "N/A"
|
||||
|
||||
return state
|
||||
|
||||
def check_field_for_matches(text, valid_lists):
|
||||
"""
|
||||
Check a field for matches from the valid lists using case-insensitive regex
|
||||
Returns tuple of (matched_term, list_type) or (None, None)
|
||||
"""
|
||||
if pd.isna(text) or text == "":
|
||||
return None, None
|
||||
text = str(text).lower()
|
||||
for list_type, valid_list in valid_lists.items():
|
||||
for term in valid_list:
|
||||
if re.search(rf'\b{re.escape(term.lower())}\b', text):
|
||||
return term, list_type
|
||||
return None, None
|
||||
|
||||
|
||||
|
||||
##### NPI #####
|
||||
def find_10_digit_numbers(row):
|
||||
matches = re.findall(r'\b\d{10}\b', row)
|
||||
return matches
|
||||
|
||||
|
||||
def signature_page_npi(text_dict, page_list, matches):
|
||||
npi_on_signature_page = []
|
||||
for npi in matches:
|
||||
for page in page_list:
|
||||
if npi in text_dict[page] and 'signature' in text_dict[page]:
|
||||
npi_on_signature_page.append(npi)
|
||||
|
||||
if len(npi_on_signature_page) == 1:
|
||||
npi = npi_on_signature_page[0]
|
||||
npi_other = [item for item in matches if item not in npi_on_signature_page]
|
||||
npi_other = ", ".join(npi_other)
|
||||
return npi, npi_other
|
||||
elif len(npi_on_signature_page) == 0:
|
||||
npi = "N/A"
|
||||
npi_other = ", ".join(matches)
|
||||
return npi, npi_other
|
||||
|
||||
else:
|
||||
npi = ", ".join(npi_on_signature_page)
|
||||
npi_other = [item for item in matches if item not in npi_on_signature_page]
|
||||
if len(npi_other) == 0:
|
||||
npi_other = "N/A"
|
||||
return npi, npi_other
|
||||
else:
|
||||
npi_other = ", ".join(npi_other)
|
||||
return npi, npi_other
|
||||
|
||||
|
||||
def npi_hotfix(text_dict: dict[str, str]):
|
||||
"""Returns dictionery with Provider and other TIN(s) based on regex matches and presence on Signatory page
|
||||
|
||||
Args:
|
||||
filename: Name of the contract
|
||||
text_dict (dict): Dictionary keyed by string page-number and valued by actual textract output page
|
||||
|
||||
Returns:
|
||||
TIN_dict(dict): A dictionery with keys 'PROV_GROUP_TIN' & 'PROV_TIN_OTHER' and values as respective TIN(s)
|
||||
"""
|
||||
NPI_dict = {}
|
||||
|
||||
patterns = [r"\b\d{10}\b"]
|
||||
|
||||
for pattern in patterns:
|
||||
matches, page_list = find_regex_matches(pattern, text_dict)
|
||||
matches = [s for s in matches if s[0] in ('1', '2')]
|
||||
matches = list(set(matches))
|
||||
if len(matches)==1:
|
||||
NPI_dict["NPI"] = matches[0]
|
||||
NPI_dict["NPI_other"] = "N/A"
|
||||
elif len(matches) == 0:
|
||||
NPI_dict["NPI"] = "N/A"
|
||||
NPI_dict["NPI_other"] = "N/A"
|
||||
else:
|
||||
npi, npi_other = signature_page_npi(text_dict, page_list, matches)
|
||||
NPI_dict["NPI"] = npi
|
||||
NPI_dict["NPI_other"] = npi_other
|
||||
return NPI_dict
|
||||
|
||||
|
||||
##################################################################################################################################
|
||||
|
||||
# remove '-' from numbers and drop numbers with more than 2 '-' in them
|
||||
def remove_dashes(lst):
|
||||
processed_list = []
|
||||
for item in lst:
|
||||
# If the item contains more than 2 dashes, remove it
|
||||
if item.count('-') > 2:
|
||||
continue
|
||||
else:
|
||||
# Remove dashes and keep the digits only
|
||||
processed_list.append(item.replace('-', ''))
|
||||
|
||||
return processed_list
|
||||
|
||||
##################################################################################################################################
|
||||
|
||||
def filter_elements_by_length(lst):
|
||||
return [item for item in lst if 7 <= len(item) <= 10]
|
||||
|
||||
##################################################################################################################################
|
||||
# Function to filter out non-digit elements from a list
|
||||
def filter_non_digits(lst):
|
||||
return [item for item in lst if item.isdigit()] # Only keep items that are digits
|
||||
|
||||
##################################################################################################################################
|
||||
|
||||
def filter_start_with_1_or_2(lst):
|
||||
return [item for item in lst if item.startswith('1') or item.startswith('2')]
|
||||
|
||||
|
||||
##################################################################################################################################
|
||||
def convert_to_string(value):
|
||||
if pd.isna(value):
|
||||
# If the value is NaN, return it as is
|
||||
return str(value)
|
||||
elif isinstance(value, float):
|
||||
# If it's a float, convert to int and then to string to avoid decimal point
|
||||
return str(int(value)) if value == value else 'nan'
|
||||
return str(value)
|
||||
##################################################################################################################################
|
||||
|
||||
def npi_post_process(df_merged):
|
||||
df_merged['dummy'] = df_merged['NPI (10-digits)']
|
||||
df_merged['dummy_other'] = df_merged["PROV_NPI_OTHER"]
|
||||
|
||||
df_merged['dummy'] = df_merged['dummy'].apply(convert_to_string)
|
||||
df_merged['dummy_other'] = df_merged['dummy_other'].astype(str)
|
||||
|
||||
df_merged['dummy'] = df_merged['dummy'].apply(convert_to_list)
|
||||
df_merged['dummy_other'] = df_merged['dummy_other'].apply(convert_to_list)
|
||||
|
||||
df_merged['dummy'] = df_merged['dummy'].apply(remove_dashes)
|
||||
df_merged['dummy_other'] = df_merged['dummy_other'].apply(remove_dashes)
|
||||
|
||||
df_merged['dummy'] = df_merged['dummy'].apply(filter_elements_by_length)
|
||||
df_merged['dummy_other'] = df_merged['dummy_other'].apply(filter_elements_by_length)
|
||||
|
||||
df_merged['dummy'] = df_merged['dummy'].apply(filter_non_digits)
|
||||
df_merged['dummy_other'] = df_merged['dummy_other'].apply(filter_non_digits)
|
||||
|
||||
df_merged['dummy'] = df_merged['dummy'].apply(filter_start_with_1_or_2)
|
||||
df_merged['dummy_other'] = df_merged['dummy_other'].apply(filter_start_with_1_or_2)
|
||||
|
||||
df_merged['dummy'] = df_merged['dummy'].apply(list_to_string)
|
||||
df_merged['dummy_other'] = df_merged['dummy_other'].apply(list_to_string)
|
||||
|
||||
return df_merged
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
|
||||
import pandas as pd
|
||||
|
||||
|
||||
# # Nonprompt
|
||||
abc_nonprompt = pd.read_excel('hotfix_output/CNC-1A-ABC-NonPrompt-HotfixFull.xlsx')
|
||||
print("NonPrompt:", abc_nonprompt.shape)
|
||||
print(len(abc_nonprompt['Contract Name'].unique()))
|
||||
|
||||
# Agreement Name
|
||||
abc_agreement = pd.read_csv('hotfix_output/CNC-1A-AgreementName-All.csv')
|
||||
print("Agreement Name:", abc_agreement.shape)
|
||||
print(len(abc_agreement['Contract Name'].unique()))
|
||||
|
||||
# Effective Date
|
||||
abc_date = pd.read_csv('hotfix_output/CNC-1A-ContractEffectiveDate-All.csv')
|
||||
abc_date = abc_date[['Contract Name', 'Contract Effective Date_corrected']]
|
||||
abc_date.drop_duplicates(inplace=True)
|
||||
print("Effective Date:", abc_date.shape)
|
||||
print(len(abc_date['Contract Name'].unique()))
|
||||
|
||||
# Term Group
|
||||
abc_term = pd.read_csv('hotfix_output/CNC-1A-TermGroup-All.csv')
|
||||
abc_term.columns = ['Contract Name', 'Term Clause_corrected', 'Contract Auto-Renewal Indicator_corrected', 'Termination Date_corrected']
|
||||
print("Term Group:", abc_term.shape)
|
||||
print(len(abc_term['Contract Name'].unique()))
|
||||
|
||||
# Rate
|
||||
rate_full = pd.read_csv('hotfix_output/CNC-1A-Rate.csv')
|
||||
rate_468 = pd.read_excel('hotfix_output/CNC-1A-Rate-468.xlsx')
|
||||
mask = ~rate_full['Contract Name'].isin(rate_468['Contract Name'])
|
||||
rate_full_filtered = rate_full[mask]
|
||||
abc_rate = pd.concat([rate_full_filtered, rate_468], ignore_index=True)
|
||||
abc_rate = abc_rate[['Contract Name', 'Attachment/Exhibit', 'Reimb. Methodology', 'FULL_METHODOLOGY_fixed', 'LESSER_fixed', 'RATE_STANDARD_fixed', 'RATE_SHORT_fixed', 'FLAT_FEE_STANDARD_fixed', 'LESSER_RATE_fixed', 'NOT_TO_EXCEED_fixed', 'SHORT_METHODOLOGY_fixed']]
|
||||
abc_rate.drop_duplicates(inplace=True)
|
||||
print("Rate:", abc_rate.shape)
|
||||
print(len(abc_rate['Contract Name'].unique()))
|
||||
|
||||
# Merge Effective Date
|
||||
abc_final = pd.merge(abc_nonprompt, abc_date, how='left', on='Contract Name')
|
||||
print("After Date:", abc_final.shape)
|
||||
print(len(abc_final['Contract Name'].unique()))
|
||||
|
||||
# Merge Rate
|
||||
abc_final = pd.merge(abc_nonprompt, abc_rate, how='left', on=['Contract Name', 'Attachment/Exhibit', 'Reimb. Methodology'])
|
||||
print("After Rate:", abc_final.shape)
|
||||
print(len(abc_final['Contract Name'].unique()))
|
||||
|
||||
# Merge Agreement Name
|
||||
abc_final = pd.merge(abc_final, abc_agreement, how='left', on='Contract Name')
|
||||
print("After Agreement:", abc_final.shape)
|
||||
print(len(abc_final['Contract Name'].unique()))
|
||||
|
||||
# Merge Term Group
|
||||
abc_final = pd.merge(abc_final, abc_term, how='left', on='Contract Name')
|
||||
print("After Term (Final):", abc_final.shape)
|
||||
print(len(abc_final['Contract Name'].unique()))
|
||||
|
||||
print(list(abc_final.columns))
|
||||
|
||||
abc_final.to_excel('CNC-1A-Hotfix-Final.xlsx')
|
||||
@@ -74,6 +74,44 @@ def clean_law_symbols(contract_text):
|
||||
return contract_text
|
||||
|
||||
|
||||
# ORIGINAL
|
||||
def chunk_consecutive_og(text_dict, exhibit_pages):
|
||||
# If needed - extend page 1 to page 1 and 2, then cut chunking off after (edge case: compensation terms not found on first page of exhibit)
|
||||
|
||||
reimbursement_pages = [page_num for page_num in text_dict.keys() if utils.contains_reimbursement(text_dict, page_num)]
|
||||
|
||||
page_dict = {}
|
||||
current_exhibit = None
|
||||
for page_num in text_dict.keys():
|
||||
|
||||
# Page is Reimbursement AND Exhibit
|
||||
if page_num in reimbursement_pages and page_num in exhibit_pages:
|
||||
current_exhibit = page_num
|
||||
page_dict[page_num] = [page_num]
|
||||
|
||||
# Page is Reimbursement NOT Exhibit
|
||||
elif page_num in reimbursement_pages and page_num not in exhibit_pages:
|
||||
if current_exhibit:
|
||||
page_dict[current_exhibit].append(page_num)
|
||||
else:
|
||||
page_dict[page_num] = [page_num]
|
||||
|
||||
# Page is Exhibit NOT Reimbursement
|
||||
elif page_num in exhibit_pages and page_num not in reimbursement_pages:
|
||||
current_exhibit = page_num
|
||||
page_dict[page_num] = [page_num]
|
||||
|
||||
# Page is NOT Exhibit NOT Reimbursment
|
||||
elif page_num not in exhibit_pages and page_num not in reimbursement_pages:
|
||||
current_exhibit = None
|
||||
page_dict[page_num] = [page_num]
|
||||
|
||||
final_dict = {page_num : '' for page_num in page_dict.keys()}
|
||||
for page_num in page_dict.keys():
|
||||
for p in page_dict[page_num]:
|
||||
final_dict[page_num] += text_dict[p]
|
||||
return final_dict
|
||||
|
||||
|
||||
def chunk_consecutive(text_dict, exhibit_pages):
|
||||
exhibit_pages = set(str(page) for page in exhibit_pages)
|
||||
|
||||
@@ -58,7 +58,9 @@ NOT_TO_EXCEED : Only if the methodology contains the specific phrase "Not to exc
|
||||
|
||||
For the format 'X% of Y', if Y is in the below list, then apply the indicated abbreviation. If Y is not in the list, then write the full Y value:
|
||||
Medicare --> MCR
|
||||
Medicare Advantage --> MCR
|
||||
Medicaid --> MCD
|
||||
Medicare Advantage --> MA
|
||||
Allowable Charges --> AC
|
||||
Allowed Amount --> AA
|
||||
Billed Charges --> BC
|
||||
@@ -90,6 +92,15 @@ Example 5 -
|
||||
Methodology: "Multiple procedures performed during the same day will be reimbursed at 100% for the primary procedure, 50% for the second procedure, and 50% for the third procedure, subsequent procedures shall not be eligible for reimbursement."
|
||||
Correct Response: {{'LESSER' : 'N', 'RATE_STANDARD' : '100% of AC / 50% of AC / 50% of AC / 0% of AC', 'RATE_SHORT' : '1 / 0.5 / 0.5 / 0', 'FLAT_FEE_STANDARD' : 'N/A', 'LESSER_RATE' : 'N/A', 'NOT_TO_EXCEED' : 'N/A'.}}
|
||||
|
||||
Example 6 -
|
||||
Methodology: "Provider shall be entitled to the lesser of: (1) Anciliary Provider's billed charges; or (2) the amount payable by Medicare, not including Medicare coinsurance and deductibles, plus the amount payable by Medicaid as a secondary coverage based on the Medicaid fee schedule in effect on the date of service."
|
||||
|
||||
Correct Response: {{'LESSER' : 'Y', 'RATE_STANDARD' : '100% of MCR', 'RATE_SHORT' : '1', 'FLAT_FEE_STANDARD' : 'N/A', 'LESSER_RATE' : '100% of BC', 'NOT_TO_EXCEED' : 'N/A'.}}
|
||||
|
||||
Example 7 -
|
||||
Methodology: "The Compensation Schedule for the Medicare Product at any given time is the lesser of (i) the Allowable Charges for the particular Covered Service, or (ii) the appropriate amount for such Covered Service under the Company's fee schedule in effect on the date of service for the Medicare Product."
|
||||
Correct Response: {{'LESSER' : 'Y', 'RATE_STANDARD' : '100% of MCR', 'RATE_SHORT' : '1', 'FLAT_FEE_STANDARD' : 'N/A', 'LESSER_RATE' : '100% of AC', 'NOT_TO_EXCEED' : 'N/A'.}}
|
||||
|
||||
Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting.
|
||||
"""
|
||||
|
||||
@@ -387,7 +398,7 @@ If for any question, no correct answer is found, return 'N/A'.
|
||||
|
||||
AC_DICT = {
|
||||
"CONTRACT_EFFECTIVE_DT": "Extract the contract effective date. This may be mentioned in one of the following locations: the signatory section, the preamble of the agreement, or the start of the amendment. Look for phrases such as /'This amendment is effective/'. Return this date in YYYY-MM-DD format.",
|
||||
"CONTRACT_TITLE": "What is the title of the contract? It generally contains the word Agreement and is found at the very beginning of the document before a paragraph. This will be filled out if there is an amendment or numbered amendment. It typically contains the words agreement, amendment, or contract. An amendment typically has a number attached to it. The typical structure is the provider name, followed by the phrase with agreement, followed potentially by a number signifying a version of the contract.",
|
||||
"CONTRACT_TITLE": "What is the complete title of the contract or the document? It is generally written in CAPITAL LETTERS and is found at the beginning of the document before a paragraph containing provider and payer name. It typically contains the words AGREEMENT, AMENDMENT, CONTRACT, ADDENDUM, MEMORANDUM, LETTER, PROVIDER or REQUEST. In case of agreement, typical structure is the provider name, followed by the phrase with agreement, followed potentially by a number signifying a version of the contract. In case of amendment, typical structure is word amendment followed by number, followed by the phrase with agreement. Extract the complete title of the contract or document as accurately as possible. Ensure: All leading characters are included for each word (e.g. 'NETWORK' instead of 'ETWORK'). Spaces between words are preserved in phrases (e.g., 'PROVIDER COLLABORATION' instead of 'PROVIDERCOLLABORATION'). If text appears incomplete or lacks spacing, use logical patterns like capitalization or common formatting rules to infer and correct the output. Replace new lines and line breaks with space.",
|
||||
"PAYER_NAME": "What is the name of the payer that is a party to the contract as stated in the Preamble?",
|
||||
"PROV_GROUP_NAME": "What is the name of the Group provider that is a party to the contract? This is generally found near keyword provider. Only return the name of the Group provider along with dba name.",
|
||||
"PROV_GROUP_NPI": "What is the group provider's national provider identifier number mentioned in the contract? This may be found on the signature page or on the roster. It consists of 10 digits. Only return these 10 digits.",
|
||||
|
||||
@@ -5,7 +5,7 @@ import shutil
|
||||
import time
|
||||
import sys
|
||||
import numpy as np
|
||||
|
||||
from collections import defaultdict
|
||||
import config
|
||||
|
||||
|
||||
@@ -214,6 +214,19 @@ def split_and_write_csv(
|
||||
sub_df.to_csv(path, index=False)
|
||||
print(f"Written {path}")
|
||||
|
||||
def contains_lol(text, page='1'):
|
||||
if isinstance(text, dict):
|
||||
return page.isdigit() and (
|
||||
re.search('les+e*o*r of+', text[page].lower())
|
||||
)
|
||||
elif isinstance(text, str):
|
||||
return (
|
||||
re.search('les+e*o*r of+', text.lower())
|
||||
)
|
||||
else:
|
||||
print("contains_lol - Invalid data type")
|
||||
|
||||
|
||||
|
||||
def filter_already_processed(input_dict, input_dict_b):
|
||||
already_processed_ac, already_processed_b = [], []
|
||||
@@ -269,9 +282,17 @@ def find_regex_matches(
|
||||
for page, text in text_dict.items():
|
||||
found = re.findall(pattern, text)
|
||||
if found:
|
||||
matches.extend(found)
|
||||
matches.extend(found) # make it append
|
||||
page_list.append(page)
|
||||
|
||||
return matches, page_list
|
||||
|
||||
# matches = defaultdict(list)
|
||||
# for page, text in text_dict.items():
|
||||
# matches_found = re.findall(pattern, text)
|
||||
# if matches_found:
|
||||
# matches[page].extend(matches_found)
|
||||
# return list(matches.values()), list(matches.keys())
|
||||
|
||||
|
||||
def chunk_selected_pages(text_dict: dict[str, str], page_list: list) -> str:
|
||||
@@ -289,3 +310,48 @@ def chunk_selected_pages(text_dict: dict[str, str], page_list: list) -> str:
|
||||
page_list = sorted([int(page_str) for page_str in set(page_list)])
|
||||
context = "\n".join([text_dict[str(page_num)] for page_num in page_list])
|
||||
return context
|
||||
|
||||
|
||||
def add_hyphen_if_needed(input_str):
|
||||
"""Checks for specific patterns in the input string and modifies it by adding
|
||||
a '0' at the start and a hyphen at the 3rd index, or just adds a hyphen at
|
||||
the 3rd index if it's a 9-digit number.
|
||||
|
||||
If the input string is exactly 8 characters long, it adds a '0' at the start
|
||||
and inserts a hyphen at the 3rd position. If the string matches the pattern
|
||||
of a 9-digit number (exactly 9 digits), it inserts a hyphen at the 3rd index.
|
||||
If the input string matches the pattern 'x-xxxxxxx', it adds '0' at the start.
|
||||
|
||||
All conditions will return a string in the format 'xx-xxxxxxx'.
|
||||
|
||||
Args:
|
||||
- input_str (str): The input string (IRS).
|
||||
|
||||
Returns:
|
||||
- str: The modified string in the format 'xx-xxxxxxx'.
|
||||
|
||||
Example:
|
||||
- '1-2345678' → '01-2345678'
|
||||
- '12345678' → '01-2345678'
|
||||
- '123456789' → '12-3456789'
|
||||
"""
|
||||
# Check if the string matches the pattern 'x-xxxxxxx' (1 character, hyphen, 7 characters)
|
||||
if re.fullmatch(r"\d-\d{7}", input_str):
|
||||
# Add '0' at the start
|
||||
input_str = '0' + input_str
|
||||
|
||||
# Check if the string is exactly 'xxxxxxxx' (8 characters)
|
||||
elif bool(re.fullmatch(r"\b\d{8}\b", input_str)):
|
||||
# Add '0' at the start and insert a hyphen at the 3rd index
|
||||
input_str = '0' + input_str
|
||||
input_str = input_str[:2] + '-' + input_str[2:]
|
||||
|
||||
# If the string matches the pattern of a 9-digit number, add a hyphen at the 3rd index
|
||||
elif bool(re.fullmatch(r"\b\d{9}\b", input_str)):
|
||||
input_str = input_str[:2] + '-' + input_str[2:]
|
||||
|
||||
# Ensure the output is in the format 'xx-xxxxxxx'
|
||||
if not bool(re.fullmatch(r"\b\d{2}-\d{7}\b", input_str)):
|
||||
input_str = 'N/A'
|
||||
|
||||
return input_str
|
||||
@@ -10,7 +10,6 @@ VALID_LOBS = [
|
||||
"MARKETPLACE",
|
||||
"COMMERCIAL",
|
||||
"COMMERCIAL-EXCHANGE",
|
||||
"GROUP",
|
||||
"MEDICAID-MEDICARE",
|
||||
]
|
||||
|
||||
@@ -655,3 +654,133 @@ STATE_MAP = {
|
||||
"WI": "Wisconsin",
|
||||
"WY": "Wyoming",
|
||||
}
|
||||
|
||||
HOTFIX_MAPPING = {
|
||||
"Filename": "Contract Name",
|
||||
"Parent Agreement Code": "Parent Agreement Code",
|
||||
"Pages": "Pages",
|
||||
"page_num": "Page_Num",
|
||||
"Page Num" : "Page_Num",
|
||||
"EXHIBIT": "Attachment/Exhibit",
|
||||
"CONTRACT_LOB": "Line of Business",
|
||||
"PROV_TYPE": "Provider Type",
|
||||
"PROV_TYPE_LEVEL_2": "Provider Type - Level 2",
|
||||
"IP_OP": "IP/OP",
|
||||
"FULL_SERVICE": "Service Type",
|
||||
"CONTRACT_PROGRAM": "Plan Type",
|
||||
'Lesser of Logic Language, Included (Y/N)' : "Lesser of Logic language, included (Y/N)",
|
||||
"LESSER": "Lesser of Logic language, included (Y/N)",
|
||||
"LESSER_RATE": "Lesser of Rate",
|
||||
"FULL_METHODOLOGY": "Reimb. Methodology",
|
||||
"SHORT_METHODOLOGY": "Reimb. Methodology_Short",
|
||||
'If Rate is % of Payor or MCR [Standard]' : "If rate is % of Payor or MCR [STANDARD]",
|
||||
"RATE_STANDARD": "If rate is % of Payor or MCR [STANDARD]",
|
||||
"RATE_SHORT": "If rate is % of Payor or MCR [STANDARD]_Short",
|
||||
"FLAT_FEE_STANDARD": "FLAT FEE",
|
||||
"DEFAULT_TERM": "Default Term",
|
||||
"DEFAULT_RATE": "Default Rate",
|
||||
"MEDICAL_NECESSITY_LANGUAGE": "Medical Necessity Language (Language)",
|
||||
"MEDICAL_NECESSITY_LANGUAGE_IND": "Medical Necessity Language (Y/N)",
|
||||
'Inclusion of essential RBRVS "Fee Source" Language (Y/N)': 'Inclusion of essential RBRVS "Fee Source" Language (Y/N)',
|
||||
"CDM_IND": "CDM Neutralization Language, included (Y/N)",
|
||||
"CHARGEMASTER": "CONTRACT_CHARGEMASTER_PROTECTION_LANGUAGE",
|
||||
"ADD_ON_REIMBURSEMENT_LANGUAGE": "Add On Reimbursement (Language)",
|
||||
"ADD_ON_REIMBURSEMENT_IND": "Add On Reimbursement (Y/N)",
|
||||
"SINGLE_CODE_MULTIPLE_RATES_LANGUAGE": "Single Code Multiple Rates (Language)",
|
||||
"SINGLE_CODE_MULTIPLE_RATES_IND": "Single Code Multiple Rates (Y/N)",
|
||||
"IP - DSH/IME/UC, included (Y/N)": "IP - DSH/IME/UC, included (Y/N)",
|
||||
"IP - Stoploss Catastrophic Threshold": "IP - Stoploss Catastrophic Threshold",
|
||||
"EXCLUSIONS": "Exclusions",
|
||||
"NOT_TO_EXCEED": "Not to Exceed",
|
||||
"RATE_ESCALATOR_IND": "Escalator or COLA (Y/N)",
|
||||
"RATE_ESCALATOR_DT": "Escalator I, Eff. Date",
|
||||
"CONTRACT_TITLE": "Agreement_Name (Contract Title)",
|
||||
"PAYER_NAME": "PAYER NAME",
|
||||
"AFFILIATION_CLAUSE_IND": "Affiliate (Y/N)",
|
||||
"TERM_CLAUSE": "Term Clause",
|
||||
"CONTRACT_AUTO_RENEWAL_IND": "Contract Auto-Renewal Indicator",
|
||||
"CONTRACT_TERMINATION_DT": "Termination Date",
|
||||
"TERMINATION_UPON_NOTICE": "Termination Upon Notice - Days",
|
||||
"TERMINATION_UPON_CAUSE": "Termination With Cause - Days",
|
||||
"AMEND_CONTRACT_NOTICE_IND": "Amend Contract Upon notice Flag (Y/N)",
|
||||
"TIME_TO_OBJECT": "Timeframe to Object - Days",
|
||||
"ASSIGNMENTS_CLAUSE_IND": "Assignments Clause (Y/N)",
|
||||
"CONTRACT_EFFECTIVE_DT": "Contract Effective Date",
|
||||
"PROV_GROUP_TIN": "IRS #",
|
||||
"PROV_GROUP_NAME": "IRS_Name",
|
||||
"PROV_GROUP_NPI": "NPI (10-digits)",
|
||||
"PROV_GROUP_TIN_SIGNATORY": "PROV_GROUP_TIN_SIGNATORY",
|
||||
'Prov Group TIN Signatory' : "PROV_GROUP_TIN_SIGNATORY",
|
||||
"PROV_TIN_OTHER": "PROV_TIN_OTHER",
|
||||
"Prov TIN Other" : "PROV_TIN_OTHER",
|
||||
"PROV_NPI_OTHER": "PROV_NPI_OTHER",
|
||||
'Prov NPI Other' : "PROV_NPI_OTHER",
|
||||
"NOTICE_PROVIDER_NAME": "Notice to Provider Name",
|
||||
"NOTICE_PROVIDER_ADDRESS": "Notice to Provider Address",
|
||||
"HEALTH_PLAN_STATE": "Health Plan State",
|
||||
"CREDENTIALING_APP_IND": "Credentialing Application Indicator",
|
||||
"SEQUESTRATION_LANGUAGE": "Sequestration Language",
|
||||
"SEQUESTRATION_LANGUAGE_IND": "Sequestration Reductions (Y/N)",
|
||||
"NPI_NAME": "NPI Name",
|
||||
"DELEGATED_TERMS_IND": "Delegated Function Indicator",
|
||||
"DELEGATED_TERMS": "Delegated Terms",
|
||||
"ECM": "ECM",
|
||||
"NATIONAL_AGREEMENT_IND": "National Agreement Indicator",
|
||||
"COST_SETTLEMENT_LANGUAGE_IND": "Cost Settlement (Y/N)",
|
||||
"COST_SETTLEMENT_LANGUAGE": "Cost Settlement (Language)",
|
||||
"LATE_PAID_CLAIMS_LANGUAGE_IND": "Late Paid Claims (Y/N)",
|
||||
"LATE_PAID_CLAIMS_LANGUAGE": "Late Paid Claims (Language)",
|
||||
"DEEMER_AMENDMENT": "Deemer Amendment",
|
||||
"REGULATORY_REQUIREMENTS": "Regulatory Requirements",
|
||||
"RECOVERY_RIGHTS": "Recovery Rights",
|
||||
"ARBITRATION_AND_DISPUTES": "Arbitration and Disputes",
|
||||
"EXCLUSIVITY_REQUIREMENT_LANGUAGE_IND": "Exclusivity Requirement (Y/N)",
|
||||
"EXCLUSIVITY_REQUIREMENT_LANGUAGE": "Exclusivity Requirement (Language)",
|
||||
"PAYOR": "Payor",
|
||||
"PARTICIPATION_IN_PRODUCTS": "Participation in Products",
|
||||
"CLEAN_CLAIM": "Clean Claim",
|
||||
"INDEPENDENT_REVIEW_LANGUAGE_IND": "Independent Review (Y/N)",
|
||||
"INDEPENDENT_REVIEW_LANGUAGE": "Independent Review (Language)",
|
||||
"INDEMNIFICATION": "Indemnification",
|
||||
"ACCESS_TO_MEDICAL_RECORDS": "Access to Medical Records",
|
||||
"MEMBER_CONFINEMENT_DAYS_LANGUAGE_IND": "Member Confinement Days Language (Y/N)",
|
||||
"MEMBER_CONFINEMENT_DAYS_LANGUAGE": "Member Confinement Days Language (Language)",
|
||||
"NETWORK_ACCESS_FEES_LANGUAGE_IND": "Network Access Fees (Y/N)",
|
||||
"NETWORK_ACCESS_FEES_LANGUAGE": "Network Access Fees (Language)",
|
||||
"PAYMENT_IN_ADVANCE_OF_CLAIMS_SUBMISSION_LANGUAGE_IND": "Payment in Advance of Claims Submission Language (Y/N)",
|
||||
"PAYMENT_IN_ADVANCE_OF_CLAIMS_SUBMISSION_LANGUAGE": "Payment in Advance of Claims Submission Language",
|
||||
"ELIGIBILITY_VERIFICATION": "Eligibility Verification",
|
||||
"PREAUTHORIZATION": "Preauthorization",
|
||||
"POLICIES_AND_PROCEDURES": "Policies and Procedures",
|
||||
"INSURANCE_REQUIREMENT": "Insurance Requirement",
|
||||
"CARVEOUT_VENDORS": "Carve-Out Vendors",
|
||||
"CONFLICTS_BETWEEN_CERTAIN_DOCUMENTS_LANGUAGE_IND": "Conflicts Between Certain Documents (Y/N)",
|
||||
"CONFLICTS_BETWEEN_CERTAIN_DOCUMENTS_LANGUAGE": "Conflicts Between Certain Documents (Language)",
|
||||
"RELATIONSHIP_OF_PARTIES_LANGUAGE_IND": "Relationship of Parties (Y/N)",
|
||||
"RELATIONSHIP_OF_PARTIES_LANGUAGE": "Relationship of Parties (Language)",
|
||||
"NONSTANDARD_APPEALS_PROCESS_LANGUAGE_IND": "Nonstandard Appeals Process (Y/N)",
|
||||
"NONSTANDARD_APPEALS_PROCESS_LANGUAGE": "Nonstandard Appeals Process (Language)",
|
||||
"PRODUCT_REMOVAL": "Product Removal",
|
||||
"DISPARAGEMENT_PROHIBITION_LANGUAGE_IND": "Disparagement Prohibition (Y/N)",
|
||||
"DISPARAGEMENT_PROHIBITION_LANGUAGE": "Disparagement Prohibition (Language)",
|
||||
"CLAIMS_EDITING_LANGUAGE_IND": "Claims Editing Language (Y/N)",
|
||||
"CLAIMS_EDITING_LANGUAGE": "Claims Editing Language (Language)",
|
||||
"GUARANTEE_OF_PROVIDER_YIELD_LANGUAGE_IND": "Guarantee of Provider Yield (Y/N)",
|
||||
"GUARANTEE_OF_PROVIDER_YIELD_LANGUAGE": "Guarantee of Provider Yield (Language)",
|
||||
"HCBS_SERVICES": "HCBS Services",
|
||||
"ADD_ON_REIMBURSEMENT_IND": "Add On Reimbursement (Y/N)",
|
||||
"ADD_ON_REIMBURSEMENT_LANGUAGE": "Add On Reimbursement (Language)",
|
||||
"PMPM": "PMPM",
|
||||
"SINGLE_CODE_MULTIPLE_RATES_IND": "Single Code Multiple Rates (Y/N)",
|
||||
"SINGLE_CODE_MULTIPLE_RATES_LANGUAGE": "Single Code Multiple Rates (Language)",
|
||||
"INVOICE_PRICING_LANGUAGE_IND": "Invoice Pricing (Y/N)",
|
||||
"INVOICE_PRICING_LANGUAGE": "Invoice Pricing (Language)",
|
||||
"TEMPLATE": "Template",
|
||||
"PROVIDER_BASED_BILLING_EXCLUSION_LANGUAGE_IND": "Provider-based Billing Exclusion (Y/N)",
|
||||
"PROVIDER_BASED_BILLING_EXCLUSION_LANGUAGE": "Provider-based Billing Exclusion (Language)",
|
||||
"NON_RENEWAL_LANGUAGE": "Non-Renewal Language",
|
||||
"NON_RENEWAL_DAYS": "Non-Renewal - Days",
|
||||
"TIMELY_FILING": "Timely Filing",
|
||||
}
|
||||
|
||||
|
||||
|
||||
+38
-76
@@ -1,19 +1,17 @@
|
||||
|
||||
import os
|
||||
import pandas as pd
|
||||
import logging
|
||||
import concurrent.futures
|
||||
|
||||
logging.getLogger().setLevel("ERROR")
|
||||
|
||||
import warnings
|
||||
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
import postprocessing_funcs
|
||||
import valid
|
||||
import config
|
||||
|
||||
|
||||
def b_postprocess(filename, combined_df, pages):
|
||||
if combined_df.shape[0] > 0:
|
||||
# Metadata fields
|
||||
@@ -37,15 +35,12 @@ def b_postprocess(filename, combined_df, pages):
|
||||
combined_df = postprocessing_funcs.clean_lob(combined_df, filename)
|
||||
# Rename and reorder
|
||||
combined_df.rename(columns=valid.B_MAPPING, inplace=True)
|
||||
column_order = [
|
||||
col for col in valid.B_MAPPING.values() if col in combined_df.columns
|
||||
]
|
||||
column_order = [col for col in valid.B_MAPPING.values() if col in combined_df.columns]
|
||||
final_df = combined_df[column_order]
|
||||
return final_df
|
||||
else:
|
||||
return combined_df
|
||||
|
||||
|
||||
def read_individual(filepath, filename):
|
||||
df_list = []
|
||||
for file in os.listdir(filepath):
|
||||
@@ -54,98 +49,76 @@ def read_individual(filepath, filename):
|
||||
df_list.append(df)
|
||||
return pd.concat(df_list, ignore_index=True)
|
||||
|
||||
|
||||
all_column_mappings = valid.B_MAPPING.copy()
|
||||
all_column_mappings.update(valid.AC_MAPPING)
|
||||
|
||||
# For Original ABC -
|
||||
# For Duplicates -
|
||||
# For Original ABC -
|
||||
# For Duplicates -
|
||||
# For Non-B: All AC1 and AC2 Fields
|
||||
|
||||
# TERM_CLAUSE, CONTRACT_EFFECTIVE_DATE, PROV_GROUP_TIN, PROV_GROUP_NPI, DEFAULT_TERM + [ALL PART 2 FIELDS]
|
||||
|
||||
|
||||
# # ############################## READ - Clean ABC Part 1 + 3 New B Fields ##############################
|
||||
abc = read_individual("output_individual/cnc_batch1_b", "b_output.csv")
|
||||
abc = read_individual('output_individual/cnc_batch1_b', 'b_output.csv')
|
||||
|
||||
abc.drop(["Exclusions"], axis=1, inplace=True)
|
||||
abc = abc[[col for col in abc.columns if "Unnamed" not in col]]
|
||||
abc.rename(columns={v: k for k, v in all_column_mappings.items()}, inplace=True)
|
||||
abc.rename(
|
||||
columns={
|
||||
"If rate is % of Payer or MCR [STANDARD]": "RATE_STANDARD",
|
||||
"If rate is % of Payer or MCR [STANDARD]_Short": "RATE_SHORT",
|
||||
"Lesser of Logic Language, included (Y/N)": "LESSER",
|
||||
"Flat Fee": "FLAT_FEE_STANDARD",
|
||||
"Reimb. Methodology_short": "SHORT_METHODOLOGY",
|
||||
},
|
||||
inplace=True,
|
||||
)
|
||||
abc["Filename"] = abc["Filename"].str.replace(".txt", "", regex=False)
|
||||
abc.drop(['Exclusions'], axis=1, inplace=True)
|
||||
abc = abc[[col for col in abc.columns if 'Unnamed' not in col]]
|
||||
abc.rename(columns={v : k for k, v in all_column_mappings.items()}, inplace=True)
|
||||
abc.rename(columns={'If rate is % of Payer or MCR [STANDARD]' : 'RATE_STANDARD', 'If rate is % of Payer or MCR [STANDARD]_Short' : 'RATE_SHORT',
|
||||
'Lesser of Logic Language, included (Y/N)' : 'LESSER', 'Flat Fee' : 'FLAT_FEE_STANDARD', 'Reimb. Methodology_short' : 'SHORT_METHODOLOGY'}, inplace=True)
|
||||
abc['Filename'] = abc['Filename'].str.replace('.txt', '', regex=False)
|
||||
|
||||
print("Clean ABC Part 1 + 3 New B Fields: ")
|
||||
print(f"Filenames: {len(abc.Filename.unique())}", abc.Filename[0])
|
||||
print(list(abc.columns))
|
||||
print(
|
||||
f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}"
|
||||
)
|
||||
print(f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}")
|
||||
print(abc.shape)
|
||||
|
||||
# ############################## READ - AC2 For All Contracts ##############################
|
||||
ac2 = read_individual("output_individual/cnc_batch1_ac", "ac_output.csv")
|
||||
ac2 = ac2[[col for col in ac2.columns if "Unnamed" not in col]]
|
||||
ac2.rename(columns={v: k for k, v in all_column_mappings.items()}, inplace=True)
|
||||
ac2["Filename"] = ac2["Filename"].str.replace(".txt", "", regex=False)
|
||||
ac2 = read_individual('output_individual/cnc_batch1_ac', 'ac_output.csv')
|
||||
ac2 = ac2[[col for col in ac2.columns if 'Unnamed' not in col]]
|
||||
ac2.rename(columns={v : k for k, v in all_column_mappings.items()}, inplace=True)
|
||||
ac2['Filename'] = ac2['Filename'].str.replace('.txt', '', regex=False)
|
||||
print("AC2 For All Contracts: ")
|
||||
print(f"Filenames: {len(ac2.Filename.unique())}", ac2.Filename[0])
|
||||
print(list(ac2.columns))
|
||||
print(
|
||||
f"Invalid Cols: {[col for col in ac2.columns if col not in all_column_mappings.keys()]}"
|
||||
)
|
||||
print(f"Invalid Cols: {[col for col in ac2.columns if col not in all_column_mappings.keys()]}")
|
||||
print(ac2.shape)
|
||||
|
||||
# ############################## READ - New AC1 (3 fields) For All Contracts ##############################
|
||||
ac1 = read_individual("output_individual/cnc_batch1_ac1", "ac_output.csv")
|
||||
ac1 = read_individual('output_individual/cnc_batch1_ac1', 'ac_output.csv')
|
||||
# ac1.columns = ['Filename', 'Contract Effective Date', 'IRS #', 'NPI (10-digits)']
|
||||
ac1.rename(columns={v: k for k, v in all_column_mappings.items()}, inplace=True)
|
||||
ac1["Filename"] = ac1["Filename"].str.replace(".txt", "", regex=False)
|
||||
ac1.rename(columns={v : k for k, v in all_column_mappings.items()}, inplace=True)
|
||||
ac1['Filename'] = ac1['Filename'].str.replace('.txt', '', regex=False)
|
||||
print("New AC1 For All Contracts: ")
|
||||
print(f"Filenames: {len(ac1.Filename.unique())}", ac1.Filename[0])
|
||||
print(list(ac1.columns))
|
||||
print(ac1.shape)
|
||||
|
||||
# ############################## READ - Original AC1 For All Contracts ##############################
|
||||
ac = pd.read_csv("reference_files/CNC-1-AC.csv")
|
||||
ac.rename(columns={v: k for k, v in all_column_mappings.items()}, inplace=True)
|
||||
ac.rename(
|
||||
columns={
|
||||
"Evergreen, Fixed or Hard Term": "CONTRACT_AUTO_RENEWAL_IND",
|
||||
"Sequestration Reductions, included [Medicare only] (Y/N)": "SEQUESTRATION_REDUCTIONS_IND",
|
||||
},
|
||||
inplace=True,
|
||||
)
|
||||
ac = pd.read_csv('reference_files/CNC-1-AC.csv')
|
||||
ac.rename(columns={v : k for k, v in all_column_mappings.items()}, inplace=True)
|
||||
ac.rename(columns={'Evergreen, Fixed or Hard Term' : 'CONTRACT_AUTO_RENEWAL_IND',
|
||||
'Sequestration Reductions, included [Medicare only] (Y/N)' : 'SEQUESTRATION_REDUCTIONS_IND'}, inplace=True)
|
||||
print("Original AC1 For Non-B Contracts: ")
|
||||
print(f"Filenames: {len(ac.Filename.unique())}", ac.Filename[0])
|
||||
print(list(ac.columns))
|
||||
print(
|
||||
f"Invalid Cols: {[col for col in ac.columns if col not in all_column_mappings.keys()]}"
|
||||
)
|
||||
print(f"Invalid Cols: {[col for col in ac.columns if col not in all_column_mappings.keys()]}")
|
||||
print(ac.shape)
|
||||
|
||||
############################## MERGE - Original AC1 + 3 New AC1 Fields ##############################
|
||||
ac.drop(
|
||||
["CONTRACT_EFFECTIVE_DT", "PROV_GROUP_TIN", "PROV_GROUP_NPI"], axis=1, inplace=True
|
||||
)
|
||||
ac = pd.merge(ac, ac1, on="Filename", how="right") # .reset_index(drop=True)
|
||||
ac.drop(['CONTRACT_EFFECTIVE_DT', 'PROV_GROUP_TIN', 'PROV_GROUP_NPI'], axis=1, inplace=True)
|
||||
ac = pd.merge(ac, ac1, on='Filename', how='right') # .reset_index(drop=True)
|
||||
# ac.rename(columns=all_column_mappings, inplace=True)
|
||||
ac = ac[[col for col in ac.columns if "Unnamed" not in col]]
|
||||
ac = ac[[col for col in ac.columns if 'Unnamed' not in col]]
|
||||
print("Final AC1 (with 3 replaced fields)")
|
||||
print(f"Filenames: {len(ac.Filename.unique())}")
|
||||
print(list(ac.columns))
|
||||
print(ac.shape)
|
||||
|
||||
############################## FILTER - Only AC1 that are not already in Clean ABC ##############################
|
||||
ac_new_only = ac[~ac["Filename"].isin(abc.Filename.unique())]
|
||||
ac_new_only = ac[~ac['Filename'].isin(abc.Filename.unique())]
|
||||
print("AC New Only (with 3 replaced fields)")
|
||||
print(f"Filenames: {len(ac_new_only.Filename.unique())}")
|
||||
print(list(ac_new_only.columns))
|
||||
@@ -158,41 +131,30 @@ abc = pd.concat([abc, ac_new_only], axis=0)
|
||||
print("Updated ABC (With non-B AC1 Fields added)")
|
||||
print(f"Filenames: {len(abc.Filename.unique())}")
|
||||
print(list(abc.columns))
|
||||
print(
|
||||
f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}"
|
||||
)
|
||||
print(f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}")
|
||||
print(abc.shape)
|
||||
|
||||
############################## MERGE - AC2 to abc ##############################
|
||||
abc = abc[
|
||||
[
|
||||
col
|
||||
for col in abc.columns
|
||||
if col not in [c for c in ac2.columns if c != "Filename"]
|
||||
]
|
||||
]
|
||||
abc = abc[[col for col in abc.columns if col not in [c for c in ac2.columns if c != 'Filename']]]
|
||||
|
||||
# Merge ac2 to abc, keeping the columns from ac2 when there is a conflict. Except for CREDENTIALING_APP_IND
|
||||
# abc = pd.merge(abc, ac2, on='Filename', how='left')
|
||||
abc = pd.merge(abc, ac2, on="Filename", how="left", suffixes=("", "_from_ac2"))
|
||||
abc = pd.merge(abc, ac2, on='Filename', how='left', suffixes=('', '_from_ac2'))
|
||||
for column in abc.columns:
|
||||
if column.endswith("_from_ac2"):
|
||||
if column.endswith('_from_ac2'):
|
||||
orig_column = column[:-10] # Remove the '_from_ac2' suffix
|
||||
if orig_column != "CREDENTIALING_APP_IND":
|
||||
if orig_column != 'CREDENTIALING_APP_IND':
|
||||
abc[orig_column] = abc[column]
|
||||
abc.drop(column, axis=1, inplace=True)
|
||||
|
||||
print("Updated ABC (with Non-B AC1 fields, new AC2 field, new B fields)")
|
||||
print(f"Filenames: {len(abc.Filename.unique())}")
|
||||
print(list(abc.columns))
|
||||
print(
|
||||
f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}"
|
||||
)
|
||||
print(f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}")
|
||||
print(abc.shape)
|
||||
# abc.rename(columns=all_column_mappings, inplace=True)
|
||||
# abc.to_csv('output_consolidated/CNC-Batch1-ABC-BeforePostprocessing.csv')
|
||||
|
||||
|
||||
############################## POSTPROCESS - Final Postprocess Step ##############################
|
||||
def postprocess_ad_hoc(combined_df):
|
||||
# B Steps
|
||||
@@ -205,7 +167,7 @@ def postprocess_ad_hoc(combined_df):
|
||||
print(combined_df.shape)
|
||||
combined_df = postprocessing_funcs.clean_default_term(combined_df)
|
||||
print(combined_df.shape)
|
||||
|
||||
|
||||
# combined_df = postprocessing_funcs.clean_msr_lesser(combined_df) # Modify for ad hoc
|
||||
# combined_df = postprocessing_funcs.clean_prov_2(combined_df) # Modify for ad hoc
|
||||
# combined_df = postprocessing_funcs.clean_lob(combined_df, "")
|
||||
@@ -216,11 +178,11 @@ def postprocess_ad_hoc(combined_df):
|
||||
print(combined_df.shape)
|
||||
return combined_df
|
||||
|
||||
|
||||
abc_final = postprocess_ad_hoc(abc)
|
||||
abc_final.rename(columns=all_column_mappings, inplace=True)
|
||||
abc_final.to_csv("output_consolidated/CNC-1-RERUN-DRAFT4.csv")
|
||||
abc_final.to_csv('output_consolidated/CNC-1-RERUN-DRAFT4.csv')
|
||||
print("Final ABC")
|
||||
print(f"Filenames: {len(abc.Filename.unique())}")
|
||||
print(list(abc_final.columns))
|
||||
print(abc_final.shape)
|
||||
|
||||
+56
-99
@@ -1,19 +1,17 @@
|
||||
|
||||
import os
|
||||
import pandas as pd
|
||||
import logging
|
||||
import concurrent.futures
|
||||
|
||||
logging.getLogger().setLevel("ERROR")
|
||||
|
||||
import warnings
|
||||
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
import postprocessing_funcs
|
||||
import valid
|
||||
import config
|
||||
|
||||
|
||||
def b_postprocess(filename, combined_df, pages):
|
||||
if combined_df.shape[0] > 0:
|
||||
# Metadata fields
|
||||
@@ -37,15 +35,12 @@ def b_postprocess(filename, combined_df, pages):
|
||||
combined_df = postprocessing_funcs.clean_lob(combined_df, filename)
|
||||
# Rename and reorder
|
||||
combined_df.rename(columns=valid.B_MAPPING, inplace=True)
|
||||
column_order = [
|
||||
col for col in valid.B_MAPPING.values() if col in combined_df.columns
|
||||
]
|
||||
column_order = [col for col in valid.B_MAPPING.values() if col in combined_df.columns]
|
||||
final_df = combined_df[column_order]
|
||||
return final_df
|
||||
else:
|
||||
return combined_df
|
||||
|
||||
|
||||
def read_individual(filepath, filename):
|
||||
df_list = []
|
||||
for file in os.listdir(filepath):
|
||||
@@ -63,115 +58,92 @@ all_column_mappings.update(valid.AC_MAPPING)
|
||||
|
||||
|
||||
# ############################## READ - Clean AC1 + B + 3 New B ##############################
|
||||
abc = read_individual("output_individual/cnc_batch2_b", "b_output.csv")
|
||||
abc.drop(["Exclusions"], axis=1, inplace=True)
|
||||
abc = abc[[col for col in abc.columns if "Unnamed" not in col]]
|
||||
abc.rename(columns={v: k for k, v in all_column_mappings.items()}, inplace=True)
|
||||
abc.rename(
|
||||
columns={
|
||||
"If rate is % of Payer or MCR [STANDARD]": "RATE_STANDARD",
|
||||
"If rate is % of Payer or MCR [STANDARD]_Short": "RATE_SHORT",
|
||||
"Lesser of Logic Language, included (Y/N)": "LESSER",
|
||||
"Flat Fee": "FLAT_FEE_STANDARD",
|
||||
"Reimb. Methodology_short": "SHORT_METHODOLOGY",
|
||||
},
|
||||
inplace=True,
|
||||
)
|
||||
abc["Filename"] = abc["Filename"].str.replace(".txt", "", regex=False)
|
||||
abc = read_individual('output_individual/cnc_batch2_b', 'b_output.csv')
|
||||
abc.drop(['Exclusions'], axis=1, inplace=True)
|
||||
abc = abc[[col for col in abc.columns if 'Unnamed' not in col]]
|
||||
abc.rename(columns={v : k for k, v in all_column_mappings.items()}, inplace=True)
|
||||
abc.rename(columns={'If rate is % of Payer or MCR [STANDARD]' : 'RATE_STANDARD', 'If rate is % of Payer or MCR [STANDARD]_Short' : 'RATE_SHORT',
|
||||
'Lesser of Logic Language, included (Y/N)' : 'LESSER', 'Flat Fee' : 'FLAT_FEE_STANDARD', 'Reimb. Methodology_short' : 'SHORT_METHODOLOGY'}, inplace=True)
|
||||
abc['Filename'] = abc['Filename'].str.replace('.txt', '', regex=False)
|
||||
print("\n\nABC + New B: ")
|
||||
print(abc.shape)
|
||||
print(list(abc.columns))
|
||||
print(
|
||||
f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}"
|
||||
)
|
||||
print(f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}")
|
||||
|
||||
|
||||
# ############################## READ - New AC2 + Smart Chunked ##############################
|
||||
ac2 = read_individual("output_individual/cnc_batch2_ac", "ac_output.csv")
|
||||
ac2 = ac2[[col for col in ac2.columns if "Unnamed" not in col]]
|
||||
ac2.rename(columns={v: k for k, v in all_column_mappings.items()}, inplace=True)
|
||||
ac2.drop(["TERM_CLAUSE"], axis=1, inplace=True)
|
||||
ac2["Filename"] = ac2["Filename"].str.replace(".txt", "", regex=False)
|
||||
ac2 = read_individual('output_individual/cnc_batch2_ac', 'ac_output.csv')
|
||||
ac2 = ac2[[col for col in ac2.columns if 'Unnamed' not in col]]
|
||||
ac2.rename(columns={v : k for k, v in all_column_mappings.items()}, inplace=True)
|
||||
ac2.drop(['TERM_CLAUSE'], axis=1, inplace=True)
|
||||
ac2['Filename'] = ac2['Filename'].str.replace('.txt', '', regex=False)
|
||||
print("\n\nNew AC2: ")
|
||||
print(ac2.shape)
|
||||
print(list(ac2.columns))
|
||||
print(
|
||||
f"Invalid Cols: {[col for col in ac2.columns if col not in all_column_mappings.keys()]}"
|
||||
)
|
||||
print(f"Invalid Cols: {[col for col in ac2.columns if col not in all_column_mappings.keys()]}")
|
||||
|
||||
# ############################## READ - New AC1 (3 fields) ##############################
|
||||
ac1 = read_individual("output_individual/cnc_batch2_ac1", "ac_output.csv")
|
||||
ac1 = read_individual('output_individual/cnc_batch2_ac1', 'ac_output.csv')
|
||||
# ac1.columns = ['Filename', 'Contract Effective Date', 'IRS #', 'NPI (10-digits)']
|
||||
ac1.rename(columns={v: k for k, v in all_column_mappings.items()}, inplace=True)
|
||||
ac1.rename(columns={v : k for k, v in all_column_mappings.items()}, inplace=True)
|
||||
print("\n\nNew AC1: ")
|
||||
print(ac1.shape)
|
||||
print(list(ac1.columns))
|
||||
|
||||
# ############################## READ - Original AC1 ##############################
|
||||
ac = pd.read_excel("reference_files/CNC-2.0-AC.xlsx")
|
||||
ac.rename(columns={v: k for k, v in all_column_mappings.items()}, inplace=True)
|
||||
ac.rename(
|
||||
columns={
|
||||
"Evergreen, Fixed or Hard Term": "CONTRACT_AUTO_RENEWAL_IND",
|
||||
"Sequestration Reductions, included [Medicare only] (Y/N)": "SEQUESTRATION_REDUCTIONS_IND",
|
||||
},
|
||||
inplace=True,
|
||||
)
|
||||
ac["Filename"] = ac["Filename"].str.replace(".txt", "", regex=False)
|
||||
ac = ac[ac["Filename"].isin(abc["Filename"])]
|
||||
ac = pd.read_excel('reference_files/CNC-2.0-AC.xlsx')
|
||||
ac.rename(columns={v : k for k, v in all_column_mappings.items()}, inplace=True)
|
||||
ac.rename(columns={'Evergreen, Fixed or Hard Term' : 'CONTRACT_AUTO_RENEWAL_IND',
|
||||
'Sequestration Reductions, included [Medicare only] (Y/N)' : 'SEQUESTRATION_REDUCTIONS_IND'}, inplace=True)
|
||||
ac['Filename'] = ac['Filename'].str.replace('.txt', '', regex=False)
|
||||
ac = ac[ac['Filename'].isin(abc['Filename'])]
|
||||
|
||||
print("\n\nOriginal AC1: ")
|
||||
print(ac.shape)
|
||||
print(list(ac.columns))
|
||||
print(
|
||||
f"Invalid Cols: {[col for col in ac.columns if col not in all_column_mappings.keys()]}"
|
||||
)
|
||||
print(f"Invalid Cols: {[col for col in ac.columns if col not in all_column_mappings.keys()]}")
|
||||
|
||||
|
||||
############################## READ - Missing ABC ##############################
|
||||
base_path = "output_individual/cnc_batch2_b_missing"
|
||||
base_path = 'output_individual/cnc_batch2_b_missing'
|
||||
ac_frames = []
|
||||
b_frames = []
|
||||
for subdir in os.listdir(base_path):
|
||||
subdir_path = os.path.join(base_path, subdir)
|
||||
|
||||
|
||||
# Construct file paths
|
||||
ac_file_path = os.path.join(subdir_path, "test_batch-AC.csv")
|
||||
b_file_path = os.path.join(subdir_path, "test_batch-B.csv")
|
||||
|
||||
ac_file_path = os.path.join(subdir_path, 'test_batch-AC.csv')
|
||||
b_file_path = os.path.join(subdir_path, 'test_batch-B.csv')
|
||||
|
||||
# Check if files exist and then read them
|
||||
if os.path.exists(ac_file_path):
|
||||
ac_df = pd.read_csv(ac_file_path)
|
||||
ac_df.rename(columns={v: k for k, v in valid.AC_MAPPING.items()}, inplace=True)
|
||||
ac_df.rename(columns={v : k for k, v in valid.AC_MAPPING.items()}, inplace=True)
|
||||
ac_frames.append(ac_df)
|
||||
if os.path.exists(b_file_path):
|
||||
b_df = pd.read_csv(b_file_path)
|
||||
b_df.rename(columns={v: k for k, v in valid.B_MAPPING.items()}, inplace=True)
|
||||
b_df.rename(columns={v : k for k, v in valid.B_MAPPING.items()}, inplace=True)
|
||||
b_frames.append(b_df)
|
||||
|
||||
ac_missing = pd.concat(ac_frames, ignore_index=True)
|
||||
b_missing = pd.concat(b_frames, ignore_index=True)
|
||||
b_missing = b_missing[[col for col in b_missing.columns if ".1" not in col]]
|
||||
b_missing = b_missing[[col for col in b_missing.columns if '.1' not in col]]
|
||||
|
||||
abc_missing = pd.merge(ac_missing, b_missing, on="Filename", how="left")
|
||||
abc_missing.rename(columns={v: k for k, v in all_column_mappings.items()}, inplace=True)
|
||||
abc_missing["Filename"] = abc_missing["Filename"].str.replace(".txt", "", regex=False)
|
||||
abc_missing = pd.merge(ac_missing, b_missing, on='Filename', how='left')
|
||||
abc_missing.rename(columns={v : k for k, v in all_column_mappings.items()}, inplace=True)
|
||||
abc_missing['Filename'] = abc_missing['Filename'].str.replace('.txt', '', regex=False)
|
||||
|
||||
print("\n\nABC missing: ")
|
||||
print(abc_missing.shape)
|
||||
print(list(abc_missing.columns))
|
||||
print(
|
||||
f"Invalid Cols: {[col for col in abc_missing.columns if col not in all_column_mappings.keys()]}"
|
||||
)
|
||||
print(f"Invalid Cols: {[col for col in abc_missing.columns if col not in all_column_mappings.keys()]}")
|
||||
|
||||
|
||||
# ############################## MERGE - Original AC + New AC1 ##############################
|
||||
ac.drop(
|
||||
["CONTRACT_EFFECTIVE_DT", "PROV_GROUP_TIN", "PROV_GROUP_NPI"], axis=1, inplace=True
|
||||
)
|
||||
ac = pd.merge(ac, ac1, on="Filename", how="right") # .reset_index(drop=True)
|
||||
ac.drop(['CONTRACT_EFFECTIVE_DT', 'PROV_GROUP_TIN', 'PROV_GROUP_NPI'], axis=1, inplace=True)
|
||||
ac = pd.merge(ac, ac1, on='Filename', how='right') # .reset_index(drop=True)
|
||||
# ac.rename(columns=all_column_mappings, inplace=True)
|
||||
ac = ac[[col for col in ac.columns if "Unnamed" not in col]]
|
||||
ac = ac[[col for col in ac.columns if 'Unnamed' not in col]]
|
||||
print("\n\nFinal AC1 (with 3 replaced fields)")
|
||||
print(ac.shape)
|
||||
print(list(ac.columns))
|
||||
@@ -184,25 +156,15 @@ abc = pd.concat([abc, ac], axis=0)
|
||||
print("\n\nUpdated ABC (With non-B AC1 Fields added)")
|
||||
print(abc.shape)
|
||||
print(list(abc.columns))
|
||||
print(
|
||||
f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}"
|
||||
)
|
||||
print(f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}")
|
||||
|
||||
|
||||
############################## MERGE - AC2 to abc ##############################
|
||||
abc = abc[
|
||||
[
|
||||
col
|
||||
for col in abc.columns
|
||||
if col not in [c for c in ac2.columns if c != "Filename"]
|
||||
]
|
||||
]
|
||||
abc = pd.merge(abc, ac2, on="Filename", how="left")
|
||||
abc = abc[[col for col in abc.columns if col not in [c for c in ac2.columns if c != 'Filename']]]
|
||||
abc = pd.merge(abc, ac2, on='Filename', how='left')
|
||||
print("\n\nUpdated ABC (with Non-B AC1 fields, new AC2 field, new B fields)")
|
||||
print(list(abc.columns))
|
||||
print(
|
||||
f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}"
|
||||
)
|
||||
print(f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}")
|
||||
print(abc.shape)
|
||||
|
||||
print(abc.TERM_CLAUSE.value_counts())
|
||||
@@ -211,14 +173,10 @@ print(abc.TERM_CLAUSE.value_counts())
|
||||
|
||||
############################## ADD - ABC missing to abc ##############################
|
||||
abc = pd.concat([abc, abc_missing], axis=0)
|
||||
print(
|
||||
"\n\nFinal ABC (with Non-B AC1 fields, new AC2 fields, new B fields, missing ABC files)"
|
||||
)
|
||||
print("\n\nFinal ABC (with Non-B AC1 fields, new AC2 fields, new B fields, missing ABC files)")
|
||||
print(abc.shape)
|
||||
print(list(abc.columns))
|
||||
print(
|
||||
f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}"
|
||||
)
|
||||
print(f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}")
|
||||
|
||||
|
||||
############################## POSTPROCESS - Final Postprocess Step ##############################
|
||||
@@ -233,9 +191,9 @@ def postprocess_ad_hoc(combined_df):
|
||||
print(combined_df.shape)
|
||||
combined_df = postprocessing_funcs.clean_default_term(combined_df)
|
||||
print(combined_df.shape)
|
||||
|
||||
|
||||
# combined_df = postprocessing_funcs.clean_msr_lesser(combined_df) # Modify for ad hoc
|
||||
combined_df = postprocessing_funcs.clean_prov_2(combined_df) # Modify for ad hoc
|
||||
combined_df = postprocessing_funcs.clean_prov_2(combined_df) # Modify for ad hoc
|
||||
# combined_df = postprocessing_funcs.clean_lob(combined_df, "")
|
||||
|
||||
# AC Steps
|
||||
@@ -244,24 +202,23 @@ def postprocess_ad_hoc(combined_df):
|
||||
print(combined_df.shape)
|
||||
return combined_df
|
||||
|
||||
|
||||
abc_final = postprocess_ad_hoc(abc)
|
||||
abc_final.rename(columns=all_column_mappings, inplace=True)
|
||||
print("\n\nFinal ABC")
|
||||
print(abc_final.shape)
|
||||
print(f"Unique files: {abc_final['Contract Name'].unique()}")
|
||||
print(f'Unique files: {abc_final['Contract Name'].unique()}')
|
||||
print(list(abc_final.columns))
|
||||
print(
|
||||
f"Invalid Cols: {[col for col in abc_final.columns if col not in valid.ABC_COLUMNS]}"
|
||||
)
|
||||
print(f"Invalid Cols: {[col for col in abc_final.columns if col not in valid.ABC_COLUMNS]}")
|
||||
|
||||
abc_final = abc_final[[col for col in valid.ABC_COLUMNS if col in abc_final.columns]]
|
||||
print("\n\nFinal ABC")
|
||||
print(abc_final.shape)
|
||||
print(list(abc_final.columns))
|
||||
print(
|
||||
f"Invalid Cols: {[col for col in abc_final.columns if col not in valid.ABC_COLUMNS]}"
|
||||
)
|
||||
print(f"Invalid Cols: {[col for col in abc_final.columns if col not in valid.ABC_COLUMNS]}")
|
||||
|
||||
|
||||
abc_final.to_csv('output_consolidated/CNC-2-RERUN-DRAFT4.csv')
|
||||
|
||||
|
||||
|
||||
|
||||
abc_final.to_csv("output_consolidated/CNC-2-RERUN-DRAFT4.csv")
|
||||
+35
-44
@@ -1,16 +1,14 @@
|
||||
|
||||
import os
|
||||
import pandas as pd
|
||||
|
||||
pd.set_option("display.max_columns", None)
|
||||
pd.set_option("display.max_rows", None)
|
||||
pd.set_option('display.max_columns', None)
|
||||
pd.set_option('display.max_rows', None)
|
||||
import logging
|
||||
import concurrent.futures
|
||||
|
||||
logging.getLogger().setLevel("ERROR")
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import warnings
|
||||
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
import postprocessing_funcs
|
||||
@@ -18,7 +16,6 @@ import valid
|
||||
import config
|
||||
import prompts
|
||||
|
||||
|
||||
def b_postprocess(filename, combined_df, pages):
|
||||
if combined_df.shape[0] > 0:
|
||||
# Metadata fields
|
||||
@@ -42,15 +39,12 @@ def b_postprocess(filename, combined_df, pages):
|
||||
combined_df = postprocessing_funcs.clean_lob(combined_df, filename)
|
||||
# Rename and reorder
|
||||
combined_df.rename(columns=valid.B_MAPPING, inplace=True)
|
||||
column_order = [
|
||||
col for col in valid.B_MAPPING.values() if col in combined_df.columns
|
||||
]
|
||||
column_order = [col for col in valid.B_MAPPING.values() if col in combined_df.columns]
|
||||
final_df = combined_df[column_order]
|
||||
return final_df
|
||||
else:
|
||||
return combined_df
|
||||
|
||||
|
||||
def read_csv_file(full_path, filename):
|
||||
"""Helper function to read CSV file."""
|
||||
try:
|
||||
@@ -60,7 +54,6 @@ def read_csv_file(full_path, filename):
|
||||
# print(f"Failed to read {os.path.join(full_path, filename)}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def read_individual(filepath, filename):
|
||||
df_list = []
|
||||
# Use ThreadPoolExecutor to handle file reading in parallel
|
||||
@@ -71,13 +64,13 @@ def read_individual(filepath, filename):
|
||||
full_path = os.path.join(filepath, file)
|
||||
# Schedule the read_csv_file function to be executed and store the future object
|
||||
futures.append(executor.submit(read_csv_file, full_path, filename))
|
||||
|
||||
|
||||
# As each future completes, get the result (a DataFrame or None)
|
||||
for future in futures:
|
||||
result = future.result()
|
||||
if result is not None:
|
||||
df_list.append(result)
|
||||
|
||||
|
||||
# Concatenate all the DataFrames in the list
|
||||
if df_list:
|
||||
return pd.concat(df_list, ignore_index=True)
|
||||
@@ -86,18 +79,18 @@ def read_individual(filepath, filename):
|
||||
all_column_mappings = valid.B_MAPPING.copy()
|
||||
all_column_mappings.update(valid.AC_MAPPING)
|
||||
|
||||
abc_path = "output_individual/cnc_3_b"
|
||||
ac1_original_path = "output_consolidated/CNC-3.0-A-AC.csv"
|
||||
ac1_new_path = "output_individual/cnc_3_ac1"
|
||||
ac2_path = "output_individual/cnc_3_ac2"
|
||||
abc_path = 'output_individual/cnc_3_b'
|
||||
ac1_original_path = 'output_consolidated/CNC-3.0-A-AC.csv'
|
||||
ac1_new_path = 'output_individual/cnc_3_ac1'
|
||||
ac2_path = 'output_individual/cnc_3_ac2'
|
||||
|
||||
# Read New AC2
|
||||
# ac2 = read_individual(ac2_path, 'ac_output.csv')
|
||||
# ac2.rename(columns={v : k for k, v in all_column_mappings.items()}, inplace=True)
|
||||
# ac2.to_csv('reference_files/ac2_new.csv')
|
||||
ac2 = pd.read_csv("reference_files/ac2_new.csv")
|
||||
ac2 = ac2[[col for col in ac2.columns if "Unnamed" not in col]]
|
||||
ac2["Filename"] = ac2["Filename"].str.replace(".txt", "", regex=False)
|
||||
ac2 = pd.read_csv('reference_files/ac2_new.csv')
|
||||
ac2 = ac2[[col for col in ac2.columns if 'Unnamed' not in col]]
|
||||
ac2['Filename'] = ac2['Filename'].str.replace('.txt', '', regex=False)
|
||||
print("\n\nNew AC2")
|
||||
print(f"Unique Filenames: {len(ac2['Filename'].unique())}")
|
||||
print(ac2.shape)
|
||||
@@ -115,8 +108,8 @@ print(list(ac2.columns))
|
||||
# overlapping_columns = [col for col in abc.columns if col in ac2.columns and col != 'Filename']
|
||||
# abc.drop(overlapping_columns, axis=1, inplace=True)
|
||||
# abc.to_csv('reference_files/abc_original.csv')
|
||||
abc = pd.read_csv("reference_files/abc_original.csv")
|
||||
abc = abc[[col for col in abc.columns if "Unnamed" not in col]]
|
||||
abc = pd.read_csv('reference_files/abc_original.csv')
|
||||
abc = abc[[col for col in abc.columns if 'Unnamed' not in col]]
|
||||
print("\n\nOriginal A1B + 3 New B")
|
||||
print(f"Unique Filenames: {len(abc.Filename.unique())}")
|
||||
print(abc.shape)
|
||||
@@ -132,8 +125,8 @@ print(list(abc.columns))
|
||||
# overlapping_columns = [col for col in ac1.columns if col in ac2.columns and col != 'Filename']
|
||||
# ac1.drop(overlapping_columns, axis=1, inplace=True)
|
||||
# ac1.to_csv('reference_files/ac1_original.csv')
|
||||
ac1 = pd.read_csv("reference_files/ac1_original.csv")
|
||||
ac1 = ac1[[col for col in ac1.columns if "Unnamed" not in col]]
|
||||
ac1 = pd.read_csv('reference_files/ac1_original.csv')
|
||||
ac1 = ac1[[col for col in ac1.columns if 'Unnamed' not in col]]
|
||||
print("\n\nOriginal AC1")
|
||||
print(f"Unique Filenames: {len(ac1['Filename'].unique())}")
|
||||
print(ac1.shape)
|
||||
@@ -144,18 +137,16 @@ print(list(ac1.columns))
|
||||
# ac1_new.rename(columns={v : k for k, v in all_column_mappings.items()}, inplace=True)
|
||||
# ac1_new['Filename'] = ac1_new['Filename'].str.replace('.txt', '', regex=False)
|
||||
# ac1_new.to_csv('reference_files/ac1_new.csv')
|
||||
ac1_new = pd.read_csv("reference_files/ac1_new.csv")
|
||||
ac1_new = ac1_new[[col for col in ac1_new.columns if "Unnamed" not in col]]
|
||||
ac1_new = pd.read_csv('reference_files/ac1_new.csv')
|
||||
ac1_new = ac1_new[[col for col in ac1_new.columns if 'Unnamed' not in col]]
|
||||
print("\n\n3 New AC1")
|
||||
print(f"Unique Filenames: {len(ac1_new.Filename.unique())}")
|
||||
print(ac1_new.shape)
|
||||
print(list(ac1_new.columns))
|
||||
|
||||
# Merge 3 new AC1 to original AC1
|
||||
ac1.drop(
|
||||
["CONTRACT_EFFECTIVE_DT", "PROV_GROUP_TIN", "PROV_GROUP_NPI"], axis=1, inplace=True
|
||||
)
|
||||
ac1_full = pd.merge(ac1, ac1_new, on="Filename", how="left")
|
||||
ac1.drop(['CONTRACT_EFFECTIVE_DT', 'PROV_GROUP_TIN', 'PROV_GROUP_NPI'], axis=1, inplace=True)
|
||||
ac1_full = pd.merge(ac1, ac1_new, on='Filename', how='left')
|
||||
print("\n\nAC 1 Full")
|
||||
print(f"Unique Filenames: {len(ac1_full.Filename.unique())}")
|
||||
print(ac1_full.shape)
|
||||
@@ -163,13 +154,7 @@ print(list(ac1_full.columns))
|
||||
|
||||
# Append Full AC1 to Original ABC
|
||||
abc = pd.concat([abc, ac1_full], axis=0)
|
||||
abc = abc[
|
||||
[
|
||||
col
|
||||
for col in abc.columns
|
||||
if col not in [c for c in ac2.columns if c != "Filename"]
|
||||
]
|
||||
]
|
||||
abc = abc[[col for col in abc.columns if col not in [c for c in ac2.columns if c != 'Filename']]]
|
||||
print("\n\nFull ABC")
|
||||
print(f"Unique Filenames: {len(abc['Filename'].unique())}")
|
||||
print(abc.shape)
|
||||
@@ -177,18 +162,17 @@ print(list(abc.columns))
|
||||
|
||||
# Merge AC2
|
||||
# Filenames in abc not in ac2
|
||||
print(abc[~abc["Filename"].isin(ac2["Filename"])]["Filename"].unique())
|
||||
print(abc[~abc['Filename'].isin(ac2['Filename'])]['Filename'].unique())
|
||||
|
||||
# Filenames in ac2 not in abc
|
||||
print(ac2[~ac2["Filename"].isin(abc["Filename"])]["Filename"].unique())
|
||||
print(ac2[~ac2['Filename'].isin(abc['Filename'])]['Filename'].unique())
|
||||
|
||||
abc = pd.merge(abc, ac2, on="Filename", how="left")
|
||||
abc = pd.merge(abc, ac2, on='Filename', how='left')
|
||||
print("\n\nABC Full Final (before postprocessing)")
|
||||
print(f"Unique Filenames: {len(abc.Filename.unique())}")
|
||||
print(abc.shape)
|
||||
print(list(abc.columns))
|
||||
|
||||
|
||||
#### Postprocess ####
|
||||
def postprocess_ad_hoc(combined_df):
|
||||
# B Steps
|
||||
@@ -200,7 +184,7 @@ def postprocess_ad_hoc(combined_df):
|
||||
print(combined_df.shape)
|
||||
combined_df = postprocessing_funcs.clean_default_term(combined_df)
|
||||
print(combined_df.shape)
|
||||
|
||||
|
||||
# combined_df = postprocessing_funcs.clean_msr_lesser(combined_df) # Modify for ad hoc
|
||||
# combined_df = postprocessing_funcs.clean_prov_2(combined_df) # Modify for ad hoc
|
||||
# combined_df = postprocessing_funcs.clean_lob(combined_df, "")
|
||||
@@ -211,7 +195,6 @@ def postprocess_ad_hoc(combined_df):
|
||||
print(combined_df.shape)
|
||||
return combined_df
|
||||
|
||||
|
||||
abc_final = postprocess_ad_hoc(abc)
|
||||
abc_final.rename(columns=all_column_mappings, inplace=True)
|
||||
print("ABC Final (after postprocessing)")
|
||||
@@ -225,4 +208,12 @@ print(f"Unique Filenames: {len(abc_final['Contract Name'].unique())}")
|
||||
print(abc_final.shape)
|
||||
print(list(abc_final.columns))
|
||||
|
||||
abc_final.to_csv("output_consolidated/CNC-3-RERUN-DRAFT6.csv")
|
||||
abc_final.to_csv('output_consolidated/CNC-3-RERUN-DRAFT6.csv')
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+43
-53
@@ -1,7 +1,7 @@
|
||||
import pandas as pd
|
||||
|
||||
pd.set_option("display.max_columns", None)
|
||||
pd.set_option("display.max_rows", None)
|
||||
import pandas as pd
|
||||
pd.set_option('display.max_columns', None)
|
||||
pd.set_option('display.max_rows', None)
|
||||
import numpy as np
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import os
|
||||
@@ -17,17 +17,17 @@ import valid
|
||||
|
||||
|
||||
def clean_prov_2(df):
|
||||
valid_types = valid.select_valid_prov_2(df["Provider Type"])
|
||||
target_rows = df[df["Provider Type - Level 2"].apply(is_empty)]
|
||||
valid_types = valid.select_valid_prov_2(df['Provider Type'])
|
||||
target_rows = df[df['Provider Type - Level 2'].apply(is_empty)]
|
||||
|
||||
def find_exact_match(text):
|
||||
if pd.isna(text) or text == "":
|
||||
if pd.isna(text) or text == '':
|
||||
return None
|
||||
|
||||
words = re.findall(r"\b[\w/]+(?:[-\s][\w/]+)*\b", text)
|
||||
words = re.findall(r'\b[\w/]+(?:[-\s][\w/]+)*\b', text)
|
||||
for i in range(len(words)):
|
||||
for j in range(i + 1, len(words) + 1):
|
||||
phrase = " ".join(words[i:j])
|
||||
for j in range(i+1, len(words)+1):
|
||||
phrase = ' '.join(words[i:j])
|
||||
if phrase in valid_types: # Case-sensitive matching
|
||||
return phrase
|
||||
return None
|
||||
@@ -35,60 +35,55 @@ def clean_prov_2(df):
|
||||
for index, row in target_rows.iterrows():
|
||||
match = None
|
||||
|
||||
if not is_empty(row["Service Type"]):
|
||||
match = find_exact_match(str(row["Service Type"]))
|
||||
if not is_empty(row['Service Type']):
|
||||
match = find_exact_match(str(row['Service Type']))
|
||||
if match:
|
||||
df.at[index, "Provider Type - Level 2"] = match
|
||||
df.at[index, 'Provider Type - Level 2'] = match
|
||||
continue
|
||||
|
||||
if not is_empty(row["Attachment/Exhibit"]):
|
||||
match = find_exact_match(str(row["Attachment/Exhibit"]))
|
||||
if not is_empty(row['Attachment/Exhibit']):
|
||||
match = find_exact_match(str(row['Attachment/Exhibit']))
|
||||
if match:
|
||||
df.at[index, "Provider Type - Level 2"] = match
|
||||
df.at[index, 'Provider Type - Level 2'] = match
|
||||
continue
|
||||
|
||||
exhibit_rows = df[df["Attachment/Exhibit"] == row["Attachment/Exhibit"]]
|
||||
exhibit_rows = df[df['Attachment/Exhibit'] == row['Attachment/Exhibit']]
|
||||
if not exhibit_rows.empty:
|
||||
for _, exhibit_row in exhibit_rows.iterrows():
|
||||
if not is_empty(exhibit_row["Provider Type - Level 2"]):
|
||||
match = find_exact_match(
|
||||
str(exhibit_row["Provider Type - Level 2"])
|
||||
)
|
||||
if not is_empty(exhibit_row['Provider Type - Level 2']):
|
||||
match = find_exact_match(str(exhibit_row['Provider Type - Level 2']))
|
||||
if match:
|
||||
df.at[index, "Provider Type - Level 2"] = match
|
||||
df.at[index, 'Provider Type - Level 2'] = match
|
||||
break
|
||||
elif not is_empty(exhibit_row["Service Type"]):
|
||||
match = find_exact_match(str(exhibit_row["Service Type"]))
|
||||
elif not is_empty(exhibit_row['Service Type']):
|
||||
match = find_exact_match(str(exhibit_row['Service Type']))
|
||||
if match:
|
||||
df.at[index, "Provider Type - Level 2"] = match
|
||||
df.at[index, 'Provider Type - Level 2'] = match
|
||||
break
|
||||
|
||||
# Final check to ensure no invalid values were assigned
|
||||
invalid_assignments = df[
|
||||
(~df["Provider Type - Level 2"].isin(valid_types))
|
||||
& (~df["Provider Type - Level 2"].apply(is_empty))
|
||||
(~df['Provider Type - Level 2'].isin(valid_types)) &
|
||||
(~df['Provider Type - Level 2'].apply(is_empty))
|
||||
]
|
||||
if not invalid_assignments.empty:
|
||||
df.loc[invalid_assignments.index, "Provider Type - Level 2"] = ""
|
||||
df.loc[invalid_assignments.index, 'Provider Type - Level 2'] = ''
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def get_new_effective_date(unique_contract_names):
|
||||
new_dates = {}
|
||||
for contract_name in unique_contract_names:
|
||||
if contract_name + ".txt" in input_dict.keys():
|
||||
contract_text = input_dict[contract_name + ".txt"]
|
||||
|
||||
if contract_name+'.txt' in input_dict.keys():
|
||||
contract_text = input_dict[contract_name+'.txt']
|
||||
|
||||
try:
|
||||
prompt = f"""### Contract Start ### {contract_text} ### Contract End ###
|
||||
Above is a contract. What is the contract effective date mentioned in any of the following locations: the signatory section, the preamble of the agreement, or the start of the amendment? Look for phrases such as /'This amendment is effective/'.
|
||||
If there is no clear effective date, return the date from the signature page.
|
||||
Return the date converted to YYYY-MM-DD format, with no other commentary or explanation.
|
||||
"""
|
||||
date_answer = claude_funcs.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 128
|
||||
)
|
||||
date_answer = claude_funcs.invoke_claude(prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 128)
|
||||
print(date_answer)
|
||||
new_dates[contract_name] = date_answer
|
||||
except:
|
||||
@@ -97,37 +92,33 @@ def get_new_effective_date(unique_contract_names):
|
||||
If there is no clear effective date, return the date from the signature page.
|
||||
Return the date converted to YYYY-MM-DD format, with no other commentary or explanation.
|
||||
"""
|
||||
date_answer = claude_funcs.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 128
|
||||
)
|
||||
date_answer = claude_funcs.invoke_claude(prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 128)
|
||||
new_dates[contract_name] = date_answer
|
||||
return new_dates
|
||||
|
||||
|
||||
|
||||
|
||||
#################### Process Starts Here ###################
|
||||
abc = pd.read_csv("output_consolidated/CNC-3-RERUN-DRAFT6.csv")
|
||||
abc = pd.read_csv('output_consolidated/CNC-3-RERUN-DRAFT6.csv')
|
||||
# null_counts = abc.isnull().sum()
|
||||
# print(null_counts)
|
||||
# quit()
|
||||
|
||||
input_dict = utils.read_input("data_cnc/batch3A")
|
||||
input_dict = utils.read_input('data_cnc/batch3A')
|
||||
|
||||
# Clean Prov 2
|
||||
abc_grouped = abc.groupby("Contract Name")
|
||||
abc_grouped = abc.groupby('Contract Name')
|
||||
abc_clean = pd.concat([clean_prov_2(group) for name, group in abc_grouped])
|
||||
|
||||
# Effective Date
|
||||
contains_meridian = abc_clean["PAYER NAME"].str.contains(
|
||||
"meridian", case=False, na=False
|
||||
)
|
||||
contains_meridian = abc_clean['PAYER NAME'].str.contains('meridian', case=False, na=False)
|
||||
|
||||
# Use your utility function to identify empty dates
|
||||
empty_dates = abc_clean["Contract Effective Date"].apply(utils.is_empty)
|
||||
empty_dates = abc_clean['Contract Effective Date'].apply(utils.is_empty)
|
||||
|
||||
# Combine filters to find the relevant 'Contract Names'
|
||||
relevant_contracts = abc_clean[contains_meridian & empty_dates][
|
||||
"Contract Name"
|
||||
].unique()
|
||||
relevant_contracts = abc_clean[contains_meridian & empty_dates]['Contract Name'].unique()
|
||||
|
||||
# Get new effective dates for these contracts
|
||||
new_effective_dates = get_new_effective_date(relevant_contracts)
|
||||
@@ -135,13 +126,11 @@ new_effective_dates = get_new_effective_date(relevant_contracts)
|
||||
# Apply the new effective dates to the DataFrame
|
||||
for contract_name, new_date in new_effective_dates.items():
|
||||
# Find rows with this 'Contract Name' where dates need replacing
|
||||
condition = (
|
||||
(abc_clean["Contract Name"] == contract_name) & contains_meridian & empty_dates
|
||||
)
|
||||
abc_clean.loc[condition, "Contract Effective Date"] = new_date
|
||||
condition = (abc_clean['Contract Name'] == contract_name) & contains_meridian & empty_dates
|
||||
abc_clean.loc[condition, 'Contract Effective Date'] = new_date
|
||||
|
||||
|
||||
abc_clean.to_csv("output_consolidated/CNC-3-RERUN-DRAFT7.csv")
|
||||
abc_clean.to_csv('output_consolidated/CNC-3-RERUN-DRAFT7.csv')
|
||||
|
||||
print("ABC Full Final (after column renaming)")
|
||||
print(f"Unique Filenames: {len(abc_clean['Contract Name'].unique())}")
|
||||
@@ -149,6 +138,7 @@ print(abc_clean.shape)
|
||||
print(list(abc_clean.columns))
|
||||
|
||||
|
||||
|
||||
# abc = pd.read_excel('output_consolidated/CNC-3-RERUN-DRAFT6.csv')
|
||||
# print(abc.shape)
|
||||
# print(len(abc['Contract Effective Date'].unique()))
|
||||
@@ -165,4 +155,4 @@ print(list(abc_clean.columns))
|
||||
# print(len(abc_postprocessed['Contract Effective Date'].unique()))
|
||||
# print(list(abc_postprocessed['Contract Effective Date'].unique()))
|
||||
|
||||
# abc_postprocessed.to_csv('output_consolidated/CNC-1-RERUN-DRAFT7.csv')
|
||||
# abc_postprocessed.to_csv('output_consolidated/CNC-1-RERUN-DRAFT7.csv')
|
||||
+28
-26
@@ -9,7 +9,6 @@ import logging
|
||||
logging.getLogger().setLevel("ERROR")
|
||||
|
||||
import warnings
|
||||
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
"""
|
||||
@@ -17,7 +16,6 @@ This script consolidates a&c fields. Its data sources are `reference_files/CNC-
|
||||
as well as .csv files from `output_individual/cnc_batch[1/2]_ac/[name of provider]/ac_output.csv.
|
||||
"""
|
||||
|
||||
|
||||
def b_postprocess(combined_df, filename):
|
||||
if combined_df.shape[0] > 0:
|
||||
# Add Single Code, Multiple Rate
|
||||
@@ -31,7 +29,7 @@ def b_postprocess(combined_df, filename):
|
||||
|
||||
# Clean MSR
|
||||
combined_df = postprocessing_funcs.clean_msr_lesser(combined_df)
|
||||
|
||||
|
||||
# Clean Default - commented out bc DEFAULT_TERM is full of NAs
|
||||
try:
|
||||
combined_df = postprocessing_funcs.clean_default_term(combined_df)
|
||||
@@ -45,16 +43,14 @@ def b_postprocess(combined_df, filename):
|
||||
try:
|
||||
combined_df = postprocessing_funcs.clean_lesser_rate(combined_df)
|
||||
except Exception as e:
|
||||
pass
|
||||
pass
|
||||
|
||||
# Clean LOB - commented out until we're ready
|
||||
# combined_df = postprocessing_funcs.clean_lob(combined_df, filename)
|
||||
|
||||
# Rename and reorder
|
||||
combined_df.rename(columns=valid.B_MAPPING, inplace=True)
|
||||
column_order = [
|
||||
col for col in valid.B_MAPPING.values() if col in combined_df.columns
|
||||
]
|
||||
column_order = [col for col in valid.B_MAPPING.values() if col in combined_df.columns]
|
||||
final_df = combined_df[column_order]
|
||||
return final_df
|
||||
else:
|
||||
@@ -64,18 +60,16 @@ def b_postprocess(combined_df, filename):
|
||||
##### Batch 1 #####
|
||||
# New B Outputs
|
||||
print("Starting B")
|
||||
dir_batch_1_individual_b_outputs = "output_individual/cnc_batch1_b/"
|
||||
dir_batch_1_individual_b_outputs = 'output_individual/cnc_batch1_b/'
|
||||
|
||||
fn_b_batch_1_outputs_list = [
|
||||
dir_batch_1_individual_b_outputs + f + "/b_output.csv"
|
||||
for f in os.listdir(dir_batch_1_individual_b_outputs)
|
||||
]
|
||||
fn_b_batch_1_outputs_list = [dir_batch_1_individual_b_outputs + f + "/b_output.csv"
|
||||
for f in os.listdir(dir_batch_1_individual_b_outputs)]
|
||||
pre_concat_list_b_batch_1 = []
|
||||
print("starting B concatenation for batch 1")
|
||||
print(str(len(fn_b_batch_1_outputs_list)) + " B files pre-CSV reading")
|
||||
|
||||
for fn in fn_b_batch_1_outputs_list:
|
||||
# try:
|
||||
# try:
|
||||
file_b_df = pd.read_csv(fn)
|
||||
# for colname in file_b_df.columns:
|
||||
# print(colname)
|
||||
@@ -84,8 +78,8 @@ for fn in fn_b_batch_1_outputs_list:
|
||||
# print(fn)
|
||||
file_b_df = b_postprocess(file_b_df, fn)
|
||||
pre_concat_list_b_batch_1.append(file_b_df)
|
||||
# except Exception as e:
|
||||
# print(f"Appending failed on {fn} with exception {e}")
|
||||
# except Exception as e:
|
||||
# print(f"Appending failed on {fn} with exception {e}")
|
||||
|
||||
print(str(len(pre_concat_list_b_batch_1)) + " B files post-CSV reading")
|
||||
abc = pd.concat(pre_concat_list_b_batch_1, axis=1)
|
||||
@@ -93,7 +87,7 @@ abc = pd.concat(pre_concat_list_b_batch_1, axis=1)
|
||||
abc.to_csv("src/scratchfiles/b_concat_df.csv", index=False)
|
||||
|
||||
|
||||
abc.drop(["Exclusions", "Unnamed: 0"], axis=1, inplace=True)
|
||||
abc.drop(['Exclusions', 'Unnamed: 0'], axis=1, inplace=True)
|
||||
# post = b_postprocess(abc, "")
|
||||
# print(post.shape)
|
||||
# print(post.columns)
|
||||
@@ -116,20 +110,21 @@ quit()
|
||||
# print("Batch 1 clean shape: " + str(df_batch_1_clean_abc.shape))
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# New AC2 Outputs
|
||||
dir_batch_1_individual_ac2_outputs = "output_individual/cnc_batch1_ac/"
|
||||
fn_ac2_batch_1_outputs_list = [
|
||||
dir_batch_1_individual_ac2_outputs + f + "/ac_output.csv"
|
||||
for f in os.listdir(dir_batch_1_individual_ac2_outputs)
|
||||
]
|
||||
dir_batch_1_individual_ac2_outputs = 'output_individual/cnc_batch1_ac/'
|
||||
fn_ac2_batch_1_outputs_list = [dir_batch_1_individual_ac2_outputs + f + "/ac_output.csv"
|
||||
for f in os.listdir(dir_batch_1_individual_ac2_outputs)]
|
||||
pre_concat_list_batch_1 = []
|
||||
print("starting AC2 concatenation for batch 1")
|
||||
print(str(len(fn_ac2_batch_1_outputs_list)) + " files pre-CSV reading")
|
||||
for fn in fn_ac2_batch_1_outputs_list:
|
||||
try:
|
||||
pre_concat_list_batch_1.append(pd.read_csv(fn))
|
||||
except Exception as e:
|
||||
print(f"Appending failed on {fn} with exception {e}")
|
||||
try:
|
||||
pre_concat_list_batch_1.append(pd.read_csv(fn))
|
||||
except Exception as e:
|
||||
print(f"Appending failed on {fn} with exception {e}")
|
||||
print(str(len(pre_concat_list_batch_1)) + " AC2 files post-CSV reading")
|
||||
df_concat_batch_1_indiv_outputs = ac_postprocess(pd.concat(pre_concat_list_batch_1))
|
||||
|
||||
@@ -171,6 +166,12 @@ Just doing:
|
||||
# batch_1_abc.to_csv("batch_1_merged_abc.csv", index=False)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
##### Batch 2 #####
|
||||
|
||||
# fn_batch_2_clean_abc = 'reference_files/CNC-Batch 2 Output_File1.xlsx'
|
||||
@@ -184,8 +185,9 @@ Just doing:
|
||||
# print("Batch 2 clean shape: " + str(df_batch_2_clean_abc.shape))
|
||||
|
||||
|
||||
|
||||
# dir_batch_2_individual_ac2_outputs = 'output_individual/cnc_batch2_ac/'
|
||||
# fn_ac2_batch_2_outputs_list = [dir_batch_2_individual_ac2_outputs + f + "/ac_output.csv"
|
||||
# fn_ac2_batch_2_outputs_list = [dir_batch_2_individual_ac2_outputs + f + "/ac_output.csv"
|
||||
# for f in os.listdir(dir_batch_2_individual_ac2_outputs)]
|
||||
|
||||
# pre_concat_list_batch_2 = []
|
||||
+41
-48
@@ -10,7 +10,6 @@ This script consolidates a&c fields. Its data sources are `reference_files/CNC-
|
||||
as well as .csv files from `output_individual/cnc_batch[1/2]_ac/[name of provider]/ac_output.csv.
|
||||
"""
|
||||
|
||||
|
||||
def b_postprocess(combined_df, filename):
|
||||
if combined_df.shape[0] > 0:
|
||||
# Add Single Code, Multiple Rate
|
||||
@@ -21,7 +20,7 @@ def b_postprocess(combined_df, filename):
|
||||
|
||||
# Clean MSR
|
||||
combined_df = postprocessing_funcs.clean_msr_lesser(combined_df)
|
||||
|
||||
|
||||
# Clean Default
|
||||
combined_df = postprocessing_funcs.clean_default_term(combined_df)
|
||||
|
||||
@@ -36,9 +35,7 @@ def b_postprocess(combined_df, filename):
|
||||
|
||||
# Rename and reorder
|
||||
combined_df.rename(columns=valid.B_MAPPING, inplace=True)
|
||||
column_order = [
|
||||
col for col in valid.B_MAPPING.values() if col in combined_df.columns
|
||||
]
|
||||
column_order = [col for col in valid.B_MAPPING.values() if col in combined_df.columns]
|
||||
final_df = combined_df[column_order]
|
||||
return final_df
|
||||
else:
|
||||
@@ -48,64 +45,65 @@ def b_postprocess(combined_df, filename):
|
||||
##### Batch 1 #####
|
||||
# New B Outputs
|
||||
print("Starting B")
|
||||
dir_batch_1_individual_b_outputs = "output_individual/cnc_batch1_b/"
|
||||
dir_batch_1_individual_b_outputs = 'output_individual/cnc_batch1_b/'
|
||||
|
||||
fn_b_batch_1_outputs_list = [
|
||||
dir_batch_1_individual_b_outputs + f + "/b_output.csv"
|
||||
for f in os.listdir(dir_batch_1_individual_b_outputs)
|
||||
]
|
||||
fn_b_batch_1_outputs_list = [dir_batch_1_individual_b_outputs + f + "/b_output.csv"
|
||||
for f in os.listdir(dir_batch_1_individual_b_outputs)]
|
||||
pre_concat_list_b_batch_1 = []
|
||||
print("starting B concatenation for batch 1")
|
||||
print(str(len(fn_b_batch_1_outputs_list)) + " B files pre-CSV reading")
|
||||
|
||||
for fn in fn_b_batch_1_outputs_list:
|
||||
try:
|
||||
file_b_df = pd.read_csv(fn)
|
||||
file_b_df = b_postprocess(file_b_df, fn)
|
||||
pre_concat_list_b_batch_1.append(file_b_df)
|
||||
except Exception as e:
|
||||
print(f"Appending failed on {fn} with exception {e}")
|
||||
try:
|
||||
file_b_df = pd.read_csv(fn)
|
||||
file_b_df = b_postprocess(file_b_df, fn)
|
||||
pre_concat_list_b_batch_1.append(file_b_df)
|
||||
except Exception as e:
|
||||
print(f"Appending failed on {fn} with exception {e}")
|
||||
|
||||
print(str(len(pre_concat_list_b_batch_1)) + " B files post-CSV reading")
|
||||
abc = pd.concat(pre_concat_list_b_batch_1)
|
||||
|
||||
abc.drop(["Exclusions", "Unnamed: 0"], axis=1, inplace=True)
|
||||
abc.drop(['Exclusions', 'Unnamed: 0'], axis=1, inplace=True)
|
||||
print(abc.shape)
|
||||
print(abc.columns)
|
||||
|
||||
# Original AC1 Outputs - All files
|
||||
df_batch_1_ac1 = pd.read_csv("reference_files/CNC-1-AC.csv")
|
||||
df_batch_1_ac1 = pd.read_csv('reference_files/CNC-1-AC.csv')
|
||||
print(df_batch_1_ac1.shape)
|
||||
print(df_batch_1_ac1.columns)
|
||||
|
||||
# quit()
|
||||
|
||||
# Cleaned ABC original
|
||||
fn_batch_1_clean_abc = "reference_files/CNC-Batch 1 Output_File1.xlsx"
|
||||
fn_batch_1_clean_abc = 'reference_files/CNC-Batch 1 Output_File1.xlsx'
|
||||
df_batch_1_clean_abc = pd.read_excel(fn_batch_1_clean_abc)
|
||||
print("Head of df_batch_1_clean_abc['Contract_Name']:")
|
||||
print(df_batch_1_clean_abc["Contract_Name"].head())
|
||||
print(df_batch_1_clean_abc['Contract_Name'].head())
|
||||
print("Batch 1 clean shape: " + str(df_batch_1_clean_abc.shape))
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# New AC2 Outputs
|
||||
dir_batch_1_individual_ac2_outputs = "output_individual/cnc_batch1_ac/"
|
||||
fn_ac2_batch_1_outputs_list = [
|
||||
dir_batch_1_individual_ac2_outputs + f + "/ac_output.csv"
|
||||
for f in os.listdir(dir_batch_1_individual_ac2_outputs)
|
||||
]
|
||||
dir_batch_1_individual_ac2_outputs = 'output_individual/cnc_batch1_ac/'
|
||||
fn_ac2_batch_1_outputs_list = [dir_batch_1_individual_ac2_outputs + f + "/ac_output.csv"
|
||||
for f in os.listdir(dir_batch_1_individual_ac2_outputs)]
|
||||
pre_concat_list_batch_1 = []
|
||||
print("starting AC2 concatenation for batch 1")
|
||||
print(str(len(fn_ac2_batch_1_outputs_list)) + " files pre-CSV reading")
|
||||
for fn in fn_ac2_batch_1_outputs_list:
|
||||
try:
|
||||
pre_concat_list_batch_1.append(pd.read_csv(fn))
|
||||
except Exception as e:
|
||||
print(f"Appending failed on {fn} with exception {e}")
|
||||
try:
|
||||
pre_concat_list_batch_1.append(pd.read_csv(fn))
|
||||
except Exception as e:
|
||||
print(f"Appending failed on {fn} with exception {e}")
|
||||
print(str(len(pre_concat_list_batch_1)) + " AC2 files post-CSV reading")
|
||||
df_concat_batch_1_indiv_outputs = ac_postprocess(pd.concat(pre_concat_list_batch_1))
|
||||
|
||||
|
||||
|
||||
|
||||
"""
|
||||
1. Read B - create abc - remove old columns (Exclusions, Add-On, MedNes)
|
||||
2. Updae AC 1 column names - Add AC1 original to the end of abc
|
||||
@@ -122,30 +120,18 @@ Just doing:
|
||||
|
||||
|
||||
print("starting merging for batch 1")
|
||||
df_batch_1_clean_abc["Contract_Name"] = df_batch_1_clean_abc["Contract_Name"] + ".txt"
|
||||
batch_1_merged_ac = pd.merge(
|
||||
df_batch_1_clean_abc,
|
||||
df_concat_batch_1_indiv_outputs,
|
||||
how="left",
|
||||
left_on="Contract_Name",
|
||||
right_on="Contract Name",
|
||||
indicator=True,
|
||||
)
|
||||
df_batch_1_clean_abc['Contract_Name'] = df_batch_1_clean_abc['Contract_Name'] + '.txt'
|
||||
batch_1_merged_ac = pd.merge(df_batch_1_clean_abc, df_concat_batch_1_indiv_outputs, how='left', left_on='Contract_Name',
|
||||
right_on='Contract Name', indicator=True)
|
||||
|
||||
# batch_1_merged.to_csv("output_consolidated/CNC-1-AC2-Added.csv", index=False)
|
||||
|
||||
print("Batch1 AC2 merge completed. Merge status counts:")
|
||||
print(batch_1_merged["_merge"].value_counts())
|
||||
print(batch_1_merged['_merge'].value_counts())
|
||||
|
||||
|
||||
batch_1_abc = pd.merge(
|
||||
batch_1_merged_ac,
|
||||
df_concat_batch_1_indiv_b_outputs,
|
||||
how="left",
|
||||
left_on="Contract_Name",
|
||||
right_on="Contract Name",
|
||||
indicator=True,
|
||||
)
|
||||
batch_1_abc = pd.merge(batch_1_merged_ac, df_concat_batch_1_indiv_b_outputs, how='left', left_on='Contract_Name',
|
||||
right_on='Contract Name', indicator=True)
|
||||
print(batch_1_abc.shape)
|
||||
|
||||
print("Batch 1 merged and written to CSV")
|
||||
@@ -153,6 +139,12 @@ print("Batch 1 merged and written to CSV")
|
||||
batch_1_abc.to_csv("batch_1_merged_abc.csv", index=False)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
##### Batch 2 #####
|
||||
|
||||
# fn_batch_2_clean_abc = 'reference_files/CNC-Batch 2 Output_File1.xlsx'
|
||||
@@ -166,8 +158,9 @@ batch_1_abc.to_csv("batch_1_merged_abc.csv", index=False)
|
||||
# print("Batch 2 clean shape: " + str(df_batch_2_clean_abc.shape))
|
||||
|
||||
|
||||
|
||||
# dir_batch_2_individual_ac2_outputs = 'output_individual/cnc_batch2_ac/'
|
||||
# fn_ac2_batch_2_outputs_list = [dir_batch_2_individual_ac2_outputs + f + "/ac_output.csv"
|
||||
# fn_ac2_batch_2_outputs_list = [dir_batch_2_individual_ac2_outputs + f + "/ac_output.csv"
|
||||
# for f in os.listdir(dir_batch_2_individual_ac2_outputs)]
|
||||
|
||||
# pre_concat_list_batch_2 = []
|
||||
+18
-39
@@ -7,7 +7,6 @@ import sys
|
||||
import random
|
||||
import pandas as pd
|
||||
import random
|
||||
|
||||
random.seed(42)
|
||||
|
||||
import utils
|
||||
@@ -18,6 +17,7 @@ import tracking
|
||||
import consolidate_output
|
||||
|
||||
|
||||
|
||||
def process_b(item):
|
||||
filename, contract_text = item
|
||||
if utils.contains_reimbursement(contract_text, 0):
|
||||
@@ -25,80 +25,59 @@ def process_b(item):
|
||||
results = file_processing.run_b_prompts(item)
|
||||
if results is not None:
|
||||
tracking.update_results_csv(item, results)
|
||||
return item, results
|
||||
return item, results
|
||||
except Exception as e:
|
||||
print(f"Error processing item {filename}: {e}")
|
||||
traceback.print_exc()
|
||||
else:
|
||||
print(f"No reimbursement information found in {filename}. Skipping processing.")
|
||||
|
||||
|
||||
def process_ac(item):
|
||||
filename, contract_text = item
|
||||
try:
|
||||
results = file_processing.run_ac_prompts(item)
|
||||
results = file_processing.run_new_ac_prompts(item)
|
||||
if results is not None:
|
||||
tracking.update_results_csv(item, results)
|
||||
return item, results
|
||||
return item, results
|
||||
except Exception as e:
|
||||
print(f"Error processing item {filename}: {e}")
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
def main():
|
||||
if config.TEST:
|
||||
print(
|
||||
claude_funcs.invoke_claude(
|
||||
"Write 'test', nothing more.",
|
||||
model_id=config.MODEL_ID_CLAUDE2,
|
||||
filename="test",
|
||||
max_tokens=10,
|
||||
)
|
||||
)
|
||||
print(claude_funcs.invoke_claude("Write 'test', nothing more.", model_id=config.MODEL_ID_CLAUDE2, filename="test", max_tokens=10))
|
||||
else:
|
||||
input_dict = (
|
||||
utils.read_input()
|
||||
) # keys are contract names, values are full contract text
|
||||
input_dict = utils.read_input() # keys are contract names, values are full contract text
|
||||
total_files = len(input_dict)
|
||||
|
||||
print(f"Input Files : {len(input_dict)}")
|
||||
|
||||
# Filter no reimbursement
|
||||
input_dict = {k: input_dict[k] for k in list(input_dict)[:100]}
|
||||
input_dict = {
|
||||
k: v for k, v in input_dict.items() if utils.contains_reimbursement(str(v))
|
||||
} # Filter out non-contracts
|
||||
print(
|
||||
f"Input Files after reimbursement filter: {len(input_dict)} | {total_files-len(input_dict)} files removed"
|
||||
)
|
||||
|
||||
input_dict = {k : v for k, v in input_dict.items() if utils.contains_reimbursement(str(v))} # Filter out non-contracts
|
||||
print(f"Input Files after reimbursement filter: {len(input_dict)} | {total_files-len(input_dict)} files removed")
|
||||
|
||||
# Filter already processed
|
||||
if config.FILTER_ALREADY_PROCESSED:
|
||||
input_dict = utils.filter_already_processed(input_dict)
|
||||
print(f"Input Files left to be processed : {len(input_dict)}")
|
||||
|
||||
|
||||
batch_results = []
|
||||
processed_count = 0
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(
|
||||
max_workers=config.MAX_WORKERS
|
||||
) as executor:
|
||||
if "b" in config.FIELDS:
|
||||
futures = [
|
||||
executor.submit(process_b, item) for item in input_dict.items()
|
||||
]
|
||||
if "a" in config.FIELDS and "c" in config.FIELDS:
|
||||
futures = [
|
||||
executor.submit(process_ac, item) for item in input_dict.items()
|
||||
]
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor:
|
||||
if 'b' in config.FIELDS:
|
||||
futures = [executor.submit(process_b, item) for item in input_dict.items()]
|
||||
if 'a' in config.FIELDS and 'c' in config.FIELDS:
|
||||
futures = [executor.submit(process_ac, item) for item in input_dict.items()]
|
||||
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
try:
|
||||
result = future.result()
|
||||
if result:
|
||||
batch_results.append(result)
|
||||
processed_count += 1
|
||||
|
||||
|
||||
if processed_count % 5 == 0:
|
||||
tracking.write_batch_results(batch_results)
|
||||
batch_results = []
|
||||
@@ -118,4 +97,4 @@ def main():
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
main()
|
||||
+5
-5
@@ -13,10 +13,10 @@
|
||||
# N = 1
|
||||
|
||||
# # Paste contract text here
|
||||
# text = """The maximum compensation for Covered Services rendered to a Covered Person shall be the "Allowed Amount."
|
||||
# The Allowed Amount for Covered Services is the lesser of: (i) Allowable Charges; or (ii) as applicable,
|
||||
# (a) 100% of the Payor's Medicaid fee schedule in effect on the date of service,
|
||||
# or (b) for those Covered Services included in the Service Categories in Table 1 below,
|
||||
# text = """The maximum compensation for Covered Services rendered to a Covered Person shall be the "Allowed Amount."
|
||||
# The Allowed Amount for Covered Services is the lesser of: (i) Allowable Charges; or (ii) as applicable,
|
||||
# (a) 100% of the Payor's Medicaid fee schedule in effect on the date of service,
|
||||
# or (b) for those Covered Services included in the Service Categories in Table 1 below,
|
||||
# the applicable "Contracted Rate" set forth in Table 1 below. Physical, Occupational and Speech Therapy | 80% of the Medicaid Rate in effect on the date of service.
|
||||
# """
|
||||
|
||||
@@ -29,6 +29,6 @@
|
||||
# answer_dict = dict_operations.secondary_string_to_dict(answer, "")
|
||||
# print(answer_dict)
|
||||
# all_dicts.append(answer_dict)
|
||||
|
||||
|
||||
# # answer_df = pd.DataFrame(all_dicts)
|
||||
# # answer_df.to_csv('full_methodology_unit_test.csv')
|
||||
@@ -3,116 +3,116 @@ import math
|
||||
import pandas as pd
|
||||
from pandas.testing import assert_frame_equal
|
||||
|
||||
from cnc_hotfix import state_check, apply_state_check_to_df
|
||||
from hotfix_helper_funcs import state_check
|
||||
|
||||
|
||||
class TestStateCheck:
|
||||
# class TestStateCheck:
|
||||
|
||||
@pytest.mark.parametrize("input_state,expected_df_output,expected_non_df_output", [
|
||||
("Multiple", "N/A",["N/A"]),
|
||||
("Peach State", "N/A",["N/A"]),
|
||||
("Statewide", "N/A",["N/A"]),
|
||||
("Sunshine State Health Plan Suggests This Is For Florida, But Since The State Is Not Explicitly Specified, The Most Appropriate Answer Is:\n\nN/A", "N/A", ["N/A"]),
|
||||
("The Contract Text Indicates This Health Plan Is For The Sunshine State, Which Is Florida. Therefore, The Answer Is:\n\nFlorida->N/A", "N/A", ["N/A"]),
|
||||
("Sunshine State", "N/A", ["N/A"]),
|
||||
("The Health Plan Is For The Sunshine State, Which Refers To Florida. However, Since The State Is Not Explicitly Specified, The Most Accurate Answer Based Solely On The Information Provided Is:\n\nN/A", "N/A", ["N/A"]),
|
||||
])
|
||||
# @pytest.mark.parametrize("input_state,expected_df_output,expected_non_df_output", [
|
||||
# ("Multiple", "N/A",["N/A"]),
|
||||
# ("Peach State", "N/A",["N/A"]),
|
||||
# ("Statewide", "N/A",["N/A"]),
|
||||
# ("Sunshine State Health Plan Suggests This Is For Florida, But Since The State Is Not Explicitly Specified, The Most Appropriate Answer Is:\n\nN/A", "N/A", ["N/A"]),
|
||||
# ("The Contract Text Indicates This Health Plan Is For The Sunshine State, Which Is Florida. Therefore, The Answer Is:\n\nFlorida->N/A", "N/A", ["N/A"]),
|
||||
# ("Sunshine State", "N/A", ["N/A"]),
|
||||
# ("The Health Plan Is For The Sunshine State, Which Refers To Florida. However, Since The State Is Not Explicitly Specified, The Most Accurate Answer Based Solely On The Information Provided Is:\n\nN/A", "N/A", ["N/A"]),
|
||||
# ])
|
||||
|
||||
def test_invalid_state_names(self, input_state, expected_df_output, expected_non_df_output):
|
||||
"""Test invalid inputs"""
|
||||
# def test_invalid_state_names(self, input_state, expected_df_output, expected_non_df_output):
|
||||
# """Test invalid inputs"""
|
||||
|
||||
assert state_check(input_state,is_dataframe=True) == expected_df_output
|
||||
assert state_check(input_state,is_dataframe=False) == expected_non_df_output
|
||||
# assert state_check(input_state,is_dataframe=True) == expected_df_output
|
||||
# assert state_check(input_state,is_dataframe=False) == expected_non_df_output
|
||||
|
||||
def test_hlorida_correction(self):
|
||||
"""Test fix - Hlorida -> Florida"""
|
||||
assert state_check("Hlorida",is_dataframe=False) == ["Florida"]
|
||||
# def test_hlorida_correction(self):
|
||||
# """Test fix - Hlorida -> Florida"""
|
||||
# assert state_check("Hlorida",is_dataframe=False) == ["Florida"]
|
||||
|
||||
@pytest.mark.parametrize("input_state,expected_df_output,expected_non_df_output", [
|
||||
("AL", "Alabama",["Alabama"]),
|
||||
("Florida", "Florida",["Florida"]),
|
||||
("FLORIDA", "Florida",["Florida"]),
|
||||
("florida", "Florida",["Florida"]),
|
||||
])
|
||||
# @pytest.mark.parametrize("input_state,expected_df_output,expected_non_df_output", [
|
||||
# ("AL", "Alabama",["Alabama"]),
|
||||
# ("Florida", "Florida",["Florida"]),
|
||||
# ("FLORIDA", "Florida",["Florida"]),
|
||||
# ("florida", "Florida",["Florida"]),
|
||||
# ])
|
||||
|
||||
def test_valid_states(self, input_state, expected_df_output, expected_non_df_output):
|
||||
"""Test valid inputs"""
|
||||
assert state_check(input_state,is_dataframe=True) == expected_df_output
|
||||
assert state_check(input_state,is_dataframe=False) == expected_non_df_output
|
||||
# def test_valid_states(self, input_state, expected_df_output, expected_non_df_output):
|
||||
# """Test valid inputs"""
|
||||
# assert state_check(input_state,is_dataframe=True) == expected_df_output
|
||||
# assert state_check(input_state,is_dataframe=False) == expected_non_df_output
|
||||
|
||||
def test_list_inputs(self):
|
||||
"""Test a list of input"""
|
||||
input_states = ["AL", "Florida", "Hlorida", "Multiple"]
|
||||
# def test_list_inputs(self):
|
||||
# """Test a list of input"""
|
||||
# input_states = ["AL", "Florida", "Hlorida", "Multiple"]
|
||||
|
||||
expected_df_output = "Alabama, Florida, Florida, N/A"
|
||||
expected_non_df_output = ["Alabama", "Florida", "Florida", "N/A"]
|
||||
# expected_df_output = "Alabama, Florida, Florida, N/A"
|
||||
# expected_non_df_output = ["Alabama", "Florida", "Florida", "N/A"]
|
||||
|
||||
assert state_check(input_states,is_dataframe=True) == expected_df_output
|
||||
assert state_check(input_states,is_dataframe=False) == expected_non_df_output
|
||||
# assert state_check(input_states,is_dataframe=True) == expected_df_output
|
||||
# assert state_check(input_states,is_dataframe=False) == expected_non_df_output
|
||||
|
||||
@pytest.mark.parametrize("invalid_input", [
|
||||
123,
|
||||
{"state": "Florida"},
|
||||
True
|
||||
])
|
||||
# @pytest.mark.parametrize("invalid_input", [
|
||||
# 123,
|
||||
# {"state": "Florida"},
|
||||
# True
|
||||
# ])
|
||||
|
||||
def test_invalid_input_types(self, invalid_input):
|
||||
"""Test invalid input types"""
|
||||
# def test_invalid_input_types(self, invalid_input):
|
||||
# """Test invalid input types"""
|
||||
|
||||
with pytest.raises(TypeError):
|
||||
state_check(invalid_input)
|
||||
# with pytest.raises(TypeError):
|
||||
# state_check(invalid_input)
|
||||
|
||||
@pytest.mark.parametrize("empty_input,expected_df_output,expected_non_df_output", [
|
||||
("", "N/A",["N/A"]),
|
||||
([], "",[]),
|
||||
([" "], "N/A",["N/A"])
|
||||
])
|
||||
# @pytest.mark.parametrize("empty_input,expected_df_output,expected_non_df_output", [
|
||||
# ("", "N/A",["N/A"]),
|
||||
# ([], "",[]),
|
||||
# ([" "], "N/A",["N/A"])
|
||||
# ])
|
||||
|
||||
def test_empty_inputs(self, empty_input, expected_df_output, expected_non_df_output):
|
||||
"""Test empty inputs"""
|
||||
assert state_check(empty_input,is_dataframe=True) == expected_df_output
|
||||
assert state_check(empty_input,is_dataframe=False) == expected_non_df_output
|
||||
# def test_empty_inputs(self, empty_input, expected_df_output, expected_non_df_output):
|
||||
# """Test empty inputs"""
|
||||
# assert state_check(empty_input,is_dataframe=True) == expected_df_output
|
||||
# assert state_check(empty_input,is_dataframe=False) == expected_non_df_output
|
||||
|
||||
def test_multiple_invalid_states(self):
|
||||
"""Test multiple invalid inputs"""
|
||||
# def test_multiple_invalid_states(self):
|
||||
# """Test multiple invalid inputs"""
|
||||
|
||||
input_states = [
|
||||
"Multiple",
|
||||
"Peach State",
|
||||
"The Health Plan Is For The Sunshine State, Which Refers To Florida. So The Answer Is:\n\nFlorida->N/A",
|
||||
"The Health Plan Is For The Florida State, Which Refers To Florida",
|
||||
"Sunshine State Health Plan Suggests This Is For Florida, But Since The State Is Not Explicitly Specified, The Most Appropriate Answer Is:\n\nN/A->N/A",
|
||||
"Sunshine State Health Plan->N/A",
|
||||
"Sunshine State->N/A",
|
||||
"The Contract Text Indicates This Health Plan Is For Florida, As It Mentions Sunshine State Health Plan, Inc. Which Operates In Florida. However, To Strictly Follow The Instructions, I Will Provide The Answer:\n\nN/A->N/A"
|
||||
# input_states = [
|
||||
# "Multiple",
|
||||
# "Peach State",
|
||||
# "The Health Plan Is For The Sunshine State, Which Refers To Florida. So The Answer Is:\n\nFlorida->N/A",
|
||||
# "The Health Plan Is For The Florida State, Which Refers To Florida",
|
||||
# "Sunshine State Health Plan Suggests This Is For Florida, But Since The State Is Not Explicitly Specified, The Most Appropriate Answer Is:\n\nN/A->N/A",
|
||||
# "Sunshine State Health Plan->N/A",
|
||||
# "Sunshine State->N/A",
|
||||
# "The Contract Text Indicates This Health Plan Is For Florida, As It Mentions Sunshine State Health Plan, Inc. Which Operates In Florida. However, To Strictly Follow The Instructions, I Will Provide The Answer:\n\nN/A->N/A"
|
||||
|
||||
]
|
||||
# ]
|
||||
|
||||
expected_df_output = ", ".join(["N/A"]*8)
|
||||
expected_non_df_output = ["N/A"]*8
|
||||
# expected_df_output = ", ".join(["N/A"]*8)
|
||||
# expected_non_df_output = ["N/A"]*8
|
||||
|
||||
assert state_check(input_states,is_dataframe=True) == expected_df_output
|
||||
assert state_check(input_states,is_dataframe=False) == expected_non_df_output
|
||||
# assert state_check(input_states,is_dataframe=True) == expected_df_output
|
||||
# assert state_check(input_states,is_dataframe=False) == expected_non_df_output
|
||||
|
||||
def test_mixed_states(self):
|
||||
"""Test mixed input"""
|
||||
input_states = ["AL", "Multiple", "Florida", "Hlorida", "Florida State Health Plan"]
|
||||
# def test_mixed_states(self):
|
||||
# """Test mixed input"""
|
||||
# input_states = ["AL", "Multiple", "Florida", "Hlorida", "Florida State Health Plan"]
|
||||
|
||||
expected_df_output = "Alabama, N/A, Florida, Florida, N/A"
|
||||
expected_non_df_output = ["Alabama", "N/A", "Florida", "Florida", "N/A"]
|
||||
# expected_df_output = "Alabama, N/A, Florida, Florida, N/A"
|
||||
# expected_non_df_output = ["Alabama", "N/A", "Florida", "Florida", "N/A"]
|
||||
|
||||
assert state_check(input_states,is_dataframe=True) == expected_df_output
|
||||
assert state_check(input_states,is_dataframe=False) == expected_non_df_output
|
||||
# assert state_check(input_states,is_dataframe=True) == expected_df_output
|
||||
# assert state_check(input_states,is_dataframe=False) == expected_non_df_output
|
||||
|
||||
def concatenated_string_states(self):
|
||||
"""Test concatentated input"""
|
||||
input_states = ["Arizona, Louisiana","NY, NJ, Connecticut"]
|
||||
# def concatenated_string_states(self):
|
||||
# """Test concatentated input"""
|
||||
# input_states = ["Arizona, Louisiana","NY, NJ, Connecticut"]
|
||||
|
||||
expected_df_output = "Arizona, Louisiana, New York, New Jersey"
|
||||
expected_non_df_output = ["Arizona", "Louisiana", "New York", "New Jersey"]
|
||||
# expected_df_output = "Arizona, Louisiana, New York, New Jersey"
|
||||
# expected_non_df_output = ["Arizona", "Louisiana", "New York", "New Jersey"]
|
||||
|
||||
assert state_check(input_states,is_dataframe=True) == expected_df_output
|
||||
assert state_check(input_states,is_dataframe=False) == expected_non_df_output
|
||||
# assert state_check(input_states,is_dataframe=True) == expected_df_output
|
||||
# assert state_check(input_states,is_dataframe=False) == expected_non_df_output
|
||||
|
||||
# class TestApplyStateCheckToDataFrame:
|
||||
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
import pytest
|
||||
# from hotfix_helper_funcs import apply_date_formatting_fix
|
||||
|
||||
# def test_change_date_formatting():
|
||||
# assert apply_date_formatting_fix("31/12/2012") == "12/31/2012"
|
||||
# assert apply_date_formatting_fix("20/07/2025") == "07/20/2025"
|
||||
# assert apply_date_formatting_fix("13/12/1992") == "12/13/1992"
|
||||
|
||||
# def test_date_formatting_no_change():
|
||||
# assert apply_date_formatting_fix("11/21/2024") == "11/21/2024"
|
||||
# assert apply_date_formatting_fix("07/20/1992") == "07/20/1992"
|
||||
# assert apply_date_formatting_fix("01/02/2000") == "01/02/2000"
|
||||
|
||||
# def test_date_formatting_na():
|
||||
# assert apply_date_formatting_fix("N/A") == "N/A"
|
||||
|
||||
# def test_invalid_date():
|
||||
# with pytest.raises(ValueError):
|
||||
# apply_date_formatting_fix("7/20/1992")
|
||||
# with pytest.raises(ValueError):
|
||||
# apply_date_formatting_fix("some string")
|
||||
+29
-45
@@ -1,29 +1,29 @@
|
||||
import pandas as pd
|
||||
import os
|
||||
|
||||
fn_batch_1_clean_abc = "reference_files/CNC-Batch 1 Output_File1.xlsx"
|
||||
fn_batch_1_clean_abc = 'reference_files/CNC-Batch 1 Output_File1.xlsx'
|
||||
df_batch_1_clean_abc = pd.read_excel(fn_batch_1_clean_abc)
|
||||
|
||||
print("Head of df_batch_1_clean_abc['Contract_Name']:")
|
||||
print(df_batch_1_clean_abc["Contract_Name"].head())
|
||||
print(df_batch_1_clean_abc['Contract_Name'].head())
|
||||
|
||||
fn_batch_2_clean_abc = "reference_files/CNC-Batch 2 Output_File1.xlsx"
|
||||
fn_batch_2_clean_abc = 'reference_files/CNC-Batch 2 Output_File1.xlsx'
|
||||
df_batch_2_clean_abc = pd.read_excel(fn_batch_2_clean_abc)
|
||||
|
||||
df_batch_2_clean_abc["Contract_Name"] = df_batch_2_clean_abc["Contract Name"]
|
||||
df_batch_2_clean_abc['Contract_Name'] = df_batch_2_clean_abc['Contract Name']
|
||||
print("Head of df_batch_2_clean_abc['Contract_Name']:")
|
||||
print(df_batch_2_clean_abc["Contract_Name"].head())
|
||||
print(df_batch_2_clean_abc['Contract_Name'].head())
|
||||
|
||||
print("Batch 1 clean shape: " + str(df_batch_1_clean_abc.shape))
|
||||
print("Batch 2 clean shape: " + str(df_batch_2_clean_abc.shape))
|
||||
|
||||
print("batch 1 columns: ")
|
||||
for column in df_batch_1_clean_abc.columns:
|
||||
print(column)
|
||||
print(column)
|
||||
|
||||
print("\nbatch 2 columns: ")
|
||||
for column in df_batch_2_clean_abc.columns:
|
||||
print(column)
|
||||
print(column)
|
||||
|
||||
print("\n columns in batch 2 not in batch 1: ")
|
||||
b1cols = set(df_batch_1_clean_abc.columns)
|
||||
@@ -32,17 +32,13 @@ print(b2cols.difference(b1cols))
|
||||
|
||||
quit()
|
||||
|
||||
dir_batch_1_individual_ac2_outputs = "output_individual/cnc_batch1_ac/"
|
||||
fn_ac2_batch_1_outputs_list = [
|
||||
dir_batch_1_individual_ac2_outputs + f + "/ac_output.csv"
|
||||
for f in os.listdir(dir_batch_1_individual_ac2_outputs)
|
||||
]
|
||||
dir_batch_1_individual_ac2_outputs = 'output_individual/cnc_batch1_ac/'
|
||||
fn_ac2_batch_1_outputs_list = [dir_batch_1_individual_ac2_outputs + f + "/ac_output.csv"
|
||||
for f in os.listdir(dir_batch_1_individual_ac2_outputs)]
|
||||
|
||||
dir_batch_2_individual_ac2_outputs = "output_individual/cnc_batch2_ac/"
|
||||
fn_ac2_batch_2_outputs_list = [
|
||||
dir_batch_2_individual_ac2_outputs + f + "/ac_output.csv"
|
||||
for f in os.listdir(dir_batch_2_individual_ac2_outputs)
|
||||
]
|
||||
dir_batch_2_individual_ac2_outputs = 'output_individual/cnc_batch2_ac/'
|
||||
fn_ac2_batch_2_outputs_list = [dir_batch_2_individual_ac2_outputs + f + "/ac_output.csv"
|
||||
for f in os.listdir(dir_batch_2_individual_ac2_outputs)]
|
||||
|
||||
|
||||
pre_concat_list_batch_1 = []
|
||||
@@ -51,10 +47,10 @@ print("starting concatenation for batch 1")
|
||||
print(str(len(fn_ac2_batch_1_outputs_list)) + " files pre-CSV reading")
|
||||
|
||||
for fn in fn_ac2_batch_1_outputs_list:
|
||||
try:
|
||||
pre_concat_list_batch_1.append(pd.read_csv(fn))
|
||||
except Exception as e:
|
||||
print(f"Appending failed on {fn} with exception {e}")
|
||||
try:
|
||||
pre_concat_list_batch_1.append(pd.read_csv(fn))
|
||||
except Exception as e:
|
||||
print(f"Appending failed on {fn} with exception {e}")
|
||||
|
||||
print(str(len(pre_concat_list_batch_1)) + " files post-CSV reading")
|
||||
df_concat_batch_1_indiv_outputs = pd.concat(pre_concat_list_batch_1)
|
||||
@@ -64,10 +60,10 @@ print("batch 1 concatenation complete")
|
||||
print("starting concatenation for batch 2")
|
||||
print(str(len(fn_ac2_batch_2_outputs_list)) + " files pre-CSV reading")
|
||||
for fn in fn_ac2_batch_2_outputs_list:
|
||||
try:
|
||||
pre_concat_list_batch_2.append(pd.read_csv(fn))
|
||||
except Exception as e:
|
||||
print(f"Appending failed on {fn} with exception {e}")
|
||||
try:
|
||||
pre_concat_list_batch_2.append(pd.read_csv(fn))
|
||||
except Exception as e:
|
||||
print(f"Appending failed on {fn} with exception {e}")
|
||||
|
||||
print(str(len(pre_concat_list_batch_2)) + " files post-CSV reading")
|
||||
df_concat_batch_2_indiv_outputs = pd.concat(pre_concat_list_batch_2)
|
||||
@@ -76,31 +72,19 @@ print("batch 2 concatenation complete")
|
||||
|
||||
print("starting merging for batch 1")
|
||||
|
||||
df_batch_1_clean_abc["Contract_Name"] = df_batch_1_clean_abc["Contract_Name"] + ".txt"
|
||||
batch_1_merged = pd.merge(
|
||||
df_batch_1_clean_abc,
|
||||
df_concat_batch_1_indiv_outputs,
|
||||
how="left",
|
||||
left_on="Contract_Name",
|
||||
right_on="Contract Name",
|
||||
indicator=True,
|
||||
)
|
||||
df_batch_1_clean_abc['Contract_Name'] = df_batch_1_clean_abc['Contract_Name'] + '.txt'
|
||||
batch_1_merged = pd.merge(df_batch_1_clean_abc, df_concat_batch_1_indiv_outputs, how='left', left_on='Contract_Name',
|
||||
right_on='Contract Name', indicator=True)
|
||||
batch_1_merged.to_csv("batch_1_merged.csv", index=False)
|
||||
print("batch 1 merge completed. Merge status counts:")
|
||||
print(batch_1_merged["_merge"].value_counts())
|
||||
print(batch_1_merged['_merge'].value_counts())
|
||||
print("Batch 1 merged and written to CSV")
|
||||
|
||||
print("starting merging for batch 2")
|
||||
df_batch_2_clean_abc["Contract_Name"] = df_batch_2_clean_abc["Contract_Name"] + ".txt"
|
||||
batch_2_merged = pd.merge(
|
||||
df_batch_2_clean_abc,
|
||||
df_concat_batch_2_indiv_outputs,
|
||||
how="left",
|
||||
left_on="Contract_Name",
|
||||
right_on="Contract Name",
|
||||
indicator=True,
|
||||
)
|
||||
df_batch_2_clean_abc['Contract_Name'] = df_batch_2_clean_abc['Contract_Name'] + '.txt'
|
||||
batch_2_merged = pd.merge(df_batch_2_clean_abc, df_concat_batch_2_indiv_outputs, how='left', left_on='Contract_Name',
|
||||
right_on='Contract Name', indicator=True)
|
||||
batch_2_merged.to_csv("batch_2_merged.csv", index=False)
|
||||
print("batch 2 merge completed. Merge status counts:")
|
||||
print(batch_2_merged["_merge"].value_counts())
|
||||
print(batch_2_merged['_merge'].value_counts())
|
||||
print("Batch 2 merged and written to CSV")
|
||||
@@ -8,7 +8,6 @@ from openpyxl.utils.dataframe import dataframe_to_rows
|
||||
import argparse
|
||||
import boto3
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
sys.path.append("src")
|
||||
import utils
|
||||
|
||||
Generated
+927
-46
File diff suppressed because it is too large
Load Diff
+8
-7
@@ -1,19 +1,20 @@
|
||||
[tool.poetry]
|
||||
name = "pipes"
|
||||
name = "doczy-smart-chunking"
|
||||
version = "0.1.0"
|
||||
description = ""
|
||||
authors = ["Michael McGuinness <mmcguinness@aarete.com>"]
|
||||
authors = ["Alex Galarce <agalarce@aarete.com>"]
|
||||
readme = "README.md"
|
||||
package-mode = false
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = "3.12.7"
|
||||
requests = "^2.32.3"
|
||||
|
||||
python = "^3.12"
|
||||
pandas = "^2.2.3"
|
||||
boto3 = "^1.35.40"
|
||||
anthropic = "^0.36.0"
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
pytest = "^8.3.3"
|
||||
black = "^24.10.0"
|
||||
mypy = "^1.12.0"
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry-core"]
|
||||
@@ -26,4 +27,4 @@ target-version = ['py312']
|
||||
[tool.mypy]
|
||||
#disable_error_code = []
|
||||
#install_types = true
|
||||
#non_interactive = true
|
||||
#non_interactive = true
|
||||
|
||||
Reference in New Issue
Block a user