jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in hotfix/Exhibit (pull request #330)

Browse Source 
Hotfix/Exhibit

* Initial draft - Exhibit test

* Update logic

* Add Exhibit page_num column, add pipe format to prompt

* Working test

* Update LOB to check new Exhibit

* Greenlight file commit - needs minor efficiency improvement

* Reordering, intermediate file saving, Exhibit+LOB Fix

* Merged main into Exhibit branch

* removed excess print statements

* Merged main into hotfix/Exhibit

* Updated valid lists for mapping, reconfigured for rerun filter

* Extra print

* added fixes for batch 7

* added fixes for batch 7

* Merged main into hotfix/Exhibit

* apply_cnc_hotfix.py edited online with Bitbucket

Approved-by: Alex Galarce
This commit is contained in:
Katon Minhas
2024-12-16 18:03:42 +00:00
parent 365527d3d0
commit 036052e75f
5 changed files with 205 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fieldExtraction/src/apply_cnc_hotfix.py
+97 -39
View File
@@ -13,7 +13,7 @@ from rapidfuzz import fuzz, process
import concurrent.futures
clean_file_name = config.get_arg_value('clean_file', '')
ABC_PATH = f'clean_output/{clean_file_name}'
ABC_PATH = f'{clean_file_name}'
def get_highest_similarity(target_str, str_list):
if target_str in str_list:
@@ -58,54 +58,92 @@ def process_hotfix(abc, contract_text):
abc = cnc_hotfix.check_default_term(abc)
abc = cnc_hotfix.check_default_rate(abc)
abc = cnc_hotfix.populate_default_term(abc, text_dict)
abc.drop_duplicates(inplace=True)
# # Health Plan State
# Health Plan State
abc = cnc_hotfix.clean_healthplan_state(abc, filename, text_dict)
abc.drop_duplicates(inplace=True)
# # IRS
# IRS
abc = cnc_hotfix.clean_irs(abc, filename, text_dict)
abc.drop_duplicates(inplace=True)
# # NPI
# NPI
abc = cnc_hotfix.clean_npi(abc, filename, text_dict, top_sheet_dict)
abc.drop_duplicates(inplace=True)
# # LOB
abc = cnc_hotfix.clean_lob(abc, text_dict)
# Exhibit
abc = cnc_hotfix.clean_exhibit(abc, text_dict)
abc.drop_duplicates(inplace=True)
# # Lesser
# LOB
abc = cnc_hotfix.clean_lob(abc, text_dict)
abc.drop_duplicates(inplace=True)
# Lesser
abc = cnc_hotfix.clean_lesser(abc, text_dict)
abc.drop_duplicates(inplace=True)
# # Provider Type Level 2
# Provider Type Level 2
abc = cnc_hotfix.clean_provider_type_2(abc)
abc.drop_duplicates(inplace=True)
######## Prompt-based #########
# Rate Standard
abc = cnc_hotfix.clean_rate_standard(abc)
abc.drop_duplicates(inplace=True)
# Contract Effective Date
abc = cnc_hotfix.clean_contract_effective_date(abc, filename, text_dict)
abc.drop_duplicates(inplace=True)
# Term Group
abc = cnc_hotfix.clean_term_group(abc, filename, text_dict, ac_chunks)
abc.drop_duplicates(inplace=True)
# Agreement Name
abc = cnc_hotfix.clean_agreement_name(abc, filename, text_dict)
abc.drop_duplicates(inplace=True)
else:
abc["Contract Duplicate Issue"] = "No Contract Found"
################## CREATE OUTPUT DIRECTORIES ##################
base_filename = os.path.splitext(filename)[0].strip()
output_dir = os.path.join(config.OUTPUT_DIRECTORY, base_filename)
os.makedirs(output_dir, exist_ok=True)
################## WRITE UNPROCESSED OUTPUT ##################
# Reorder
abc = abc[[col for col in cnc_hotfix.HOTFIX_ORDER if col in abc.columns] + [col for col in abc.columns if col not in cnc_hotfix.HOTFIX_ORDER]]
# Write to Excel
abc.to_csv(
os.path.join(output_dir, config.B_RESULTS_NAME), index=False
)
return abc
def main():
# Create Excel
file_path = f'{config.BATCH_ID}-Hotfix-Final.xlsx'
if not os.path.exists(file_path):
already_ran = None
with pd.ExcelWriter(file_path, engine='openpyxl') as writer:
pd.DataFrame(columns=cnc_hotfix.HOTFIX_ORDER).to_excel(writer, sheet_name='Hotfix_Results', index=False)
else:
starting_df = pd.read_excel(file_path)
already_ran = set(starting_df['Contract Name'].unique())
REVERSE_MAPPING = {
'Agreement Name (Contract Title)': 'Agreement_Name (Contract Title)',
'Payer Name': 'PAYER NAME',
'NPI (10-digits': 'NPI (10-digits)',
'Prov Group TIN Signatory': 'PROV_GROUP_TIN_SIGNATORY',
'Prov TIN Other': 'PROV_TIN_OTHER',
'Prov NPI Other': 'PROV_NPI_OTHER',
'Page Num': 'Page_Num',
'Lesser of Logic Language, Included (Y/N)': 'Lesser of Logic language, included (Y/N)',
'Reimb Methodology Short': 'Reimb. Methodology_Short',
'If Rate is % of Payor or MCR [Standard]': 'If rate is % of Payor or MCR [STANDARD]',
'If Rate is % of Payor or MCR [Standard] Short': 'If rate is % of Payor or MCR [STANDARD]_Short',
'Flat Fee': 'FLAT FEE',
'CDM Neutralization Language, Included (Y/N)': 'CDM Neutralization Language, included (Y/N)',
'Contract Chargemaster Protection Language': 'CONTRACT_CHARGEMASTER_PROTECTION_LANGUAGE',
'IP - DSH/IME/UC, Included (Y/N)': 'IP - DSH/IME/UC, included (Y/N)',
'State': 'Health Plan State'
}
# Read clean data
print("Reading clean data...")
@@ -115,7 +153,21 @@ def main():
abc_clean = pd.read_csv(ABC_PATH)
elif '.xlsb' in ABC_PATH:
abc_clean = utils.read_xlsb(ABC_PATH)
# print(list(abc_clean.columns))
# Print current columns and identify invalid ones
print("Current columns:")
for col in abc_clean.columns:
print(col)
abc_clean.rename(columns=REVERSE_MAPPING, inplace=True)
print(f"Invalid columns: {[col for col in abc_clean.columns if col not in cnc_hotfix.HOTFIX_ORDER]}")
abc_clean = abc_clean.dropna(how='all')
abc_clean = abc_clean.dropna(axis=1, how='all')
unique_output_files = abc_clean['Contract Name'].unique()
@@ -124,53 +176,59 @@ def main():
# print(list(abc_clean.columns))
abc_clean.rename(columns=valid.HOTFIX_MAPPING, inplace=True)
abc_clean['Contract Name'] = abc_clean['Contract Name'].apply(lambda x: ('Filename: ' + str(x) if not str(x).startswith('Filename: ') else str(x)) + ('.txt' if not str(x).endswith('.txt') else ''))
print(list(abc_clean.columns))
unique_output_files = abc_clean['Contract Name'].unique()
already_ran = os.listdir(config.OUTPUT_DIRECTORY)
# Read input dict
print("Reading input .txt files...")
input_dict = utils.read_input()
input_dict = {'Filename: ' + filename.strip() : contract_text for filename, contract_text in input_dict.items()}
if already_ran:
input_dict = {'Filename: ' + filename.replace('.txt', '').strip() : contract_text for filename, contract_text in input_dict.items()}
if config.FILTER_ALREADY_PROCESSED:
input_dict = {filename : contract_text for filename, contract_text in input_dict.items() if filename not in already_ran}
print("Input Dict: ", len(input_dict))
# input_keys = list(input_dict.keys())
# abc_clean['Contract Name'] = abc_clean['Contract Name'].apply(
# lambda x: get_highest_similarity(x, input_keys)
# )
unique_output_files = abc_clean['Contract Name'].unique()
print(unique_output_files[0:2])
print(list(input_dict.keys())[0:2])
print(f"{len([file for file in input_dict.keys() if file not in unique_output_files])} files in full s3, not in output")
print(f"{len([file for file in unique_output_files if file not in input_dict.keys()])} files in output, not in full s3")
# Create list of tuples
abc_clean_list = [(group, input_dict.get(name)) for name, group in abc_clean.groupby('Contract Name')]
abc_clean_list = [(group, input_dict.get(name.replace('.txt', ''))) for name, group in abc_clean.groupby('Contract Name')]
print(f"{len(abc_clean_list)} - files in input")
print(f"{len([f for f in abc_clean_list if f[1]])} - valid files in input")
del abc_clean
del input_dict
def process_pair(abc_df, contract_text):
return process_hotfix(abc_df, contract_text)
abc_clean_list = [f for f in abc_clean_list if f[1]]
print(len(abc_clean_list))
df_list = []
with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor:
futures = [executor.submit(process_pair, abc_df, contract_text) for abc_df, contract_text in abc_clean_list]
# Create a list of future objects by submitting process_pair function to the executor
futures = [executor.submit(process_hotfix, abc_df, contract_text) for abc_df, contract_text in abc_clean_list]
# As each future completes, retrieve its result and add to df_list
for future in concurrent.futures.as_completed(futures):
try:
result = future.result()
if result is not None:
# Standardize columns by reindexing with all known columns, filling missing columns with NaN
standardized_result = result.reindex(columns=cnc_hotfix.HOTFIX_ORDER, fill_value=pd.NA)
# Append the standardized DataFrame to the Excel file on the same sheet
with pd.ExcelWriter(file_path, engine='openpyxl', mode='a', if_sheet_exists='overlay') as writer:
# Determine the start row for new data
start_row = writer.sheets['Hotfix_Results'].max_row if 'Hotfix_Results' in writer.sheets else 0
standardized_result.to_excel(writer, sheet_name='Hotfix_Results', startrow=start_row, header=start_row == 0, index=False)
new_df = future.result()
if new_df is not None:
df_list.append(new_df)
except Exception as e:
# Log the exception or handle it otherwise
print(f"An error occurred: {e}")
abc_final = pd.concat(df_list, ignore_index=True)
print(abc_final.shape)
print(len(abc_final['Contract Name'].unique()), ' unique contracts')
print("Writing to excel...")
abc_final.to_excel(f'output_consolidated/{config.BATCH_ID}-Hotfix-Final.xlsx')
print("Completed writing to Excel.")
print("Completed writing to Excel.")
if __name__ == "__main__":
fieldExtraction/src/cnc_hotfix.py
+91 -9
View File
@@ -28,6 +28,74 @@ import prompts
import ac_funcs
import postprocessing_funcs
####### Exhibit #######
def clean_exhibit(abc, text_dict):
abc["Attachment/Exhibit_fixed"], abc["Attachment/Exhibit_page"] = "", ""
def adhoc_exhibit_check(page):
prompt = prompts.TOP_DOWN_EXHIBIT_CHECK(page[0:100])
answer = claude_funcs.invoke_claude(
prompt, config.MODEL_ID_CLAUDE2, "", max_tokens=10
)
if "Y" in answer:
return True
else:
return False
def adhoc_exhibit_prompt(page):
prompt = f"""
### PAGE_START ### {page} ### PAGE_END ###
List any exhibit, attachment, or amendment names found on the page. Write the full name of the exhibit, including the exhibit number or letter, as well as any other subtitles describing the contents of the exhibit. Do NOT write the page number.
If no Exhibit, Attachment, or Amendment is found, write 'N/A'. Enclose only your final answer in |pipes|.
"""
answer = claude_funcs.invoke_claude(
prompt, config.MODEL_ID_CLAUDE35_SONNET, "", max_tokens=1000
)
return re.findall(pattern=cnc_hotfix_effective_date_utils.regex_backticks, string=answer)[-1]
answer_dict = {}
for index, row in abc.iterrows():
if pd.notna(row['Page_Num']):
page_num = str(int(row['Page_Num']))
original_page_num = page_num
# If we've already seen this page, get the answer previously found
if original_page_num in answer_dict.keys():
abc.loc[index, "Attachment/Exhibit_fixed"] = answer_dict[original_page_num]
abc.loc[index, "Attachment/Exhibit_page"] = original_page_num
continue
exhibit_val = str(abc.loc[index, "Attachment/Exhibit"]).strip()
normalized_exhibit_val = exhibit_val.replace(" ", "").replace("\n", "").upper()
# If the exhibit is already on that page, then it's correct
if normalized_exhibit_val in text_dict[page_num].replace(" ", "").replace("\n", "").upper():
abc.loc[index, "Attachment/Exhibit_fixed"] = exhibit_val
abc.loc[index, "Attachment/Exhibit_page"] = page_num
answer_dict[original_page_num] = exhibit_val
else:
contains_exhibit = False
while not contains_exhibit:
# Run TD Exhibit Check
if page_num in text_dict.keys():
contains_exhibit = adhoc_exhibit_check(text_dict[page_num])
if contains_exhibit:
page_exhibit = adhoc_exhibit_prompt(text_dict[page_num])
if 'N/A' not in page_exhibit:
abc.loc[index, "Attachment/Exhibit_fixed"] = page_exhibit
abc.loc[index, "Attachment/Exhibit_page"] = page_num
answer_dict[original_page_num] = page_exhibit
else:
contains_exhibit = False
page_num = str(int(page_num) - 1)
else:
page_num = str(int(page_num) - 1)
else:
contains_exhibit=True # Set to True to exit loop
return abc
####### Default Term/Rate #######
def concatenate_lists(row):
total = []
@@ -224,6 +292,18 @@ def clean_lob(df, text_dict):
match_pattern = r'|'.join(valid.VALID_LOBS)
for index, row in df.iterrows():
if 'Attachment/Exhibit_fixed' in df.columns:
exhibit_col = 'Attachment/Exhibit_fixed'
else:
exhibit_col = 'Attachment/Exhibit'
exhibit_fixed = str(row[exhibit_col]).upper()
# Issue 0: Incorrect - check exhibit fixed (if the LOB is not in the Exhibit, but another LOB is in the exhibit)
if (str(row['Line of Business']).upper() not in exhibit_fixed) and any([lob.upper() in exhibit_fixed for lob in valid.VALID_LOBS]):
df.at[index, 'LOB_Correct'] = ', '.join(set([m for m in valid.VALID_LOBS if m in exhibit_fixed]))
continue
# Issue 1: Missing
if utils.is_empty(row['Line of Business']) and pd.notna(row['Page_Num']): # Check if 'Line of Business' is missing
@@ -233,9 +313,9 @@ def clean_lob(df, text_dict):
second_page_text = text_dict.get(str(int(row['Page_Num']) + 2), "")
# First check in or near exhibit
exhibit_index = page_text.find(str(row['Attachment/Exhibit']).strip())
if exhibit_index != -1 and pd.notna(row['Attachment/Exhibit']):
search_area = page_text[exhibit_index:exhibit_index + 200 + len(row['Attachment/Exhibit'])]
exhibit_index = page_text.find(str(row[exhibit_col]).strip())
if exhibit_index != -1 and pd.notna(row[exhibit_col]):
search_area = page_text[exhibit_index:exhibit_index + 200 + len(row[exhibit_col])]
matches = re.findall(match_pattern, search_area, re.IGNORECASE)
if matches:
df.at[index, 'LOB_Correct'] = ', '.join(set([m.upper() for m in matches]))
@@ -260,7 +340,7 @@ def clean_lob(df, text_dict):
service = str(row['Service Type']).lower()
if 'where' in service and ('payer' in service or 'payor' in service):
# Check Exhibit first
matches = re.findall(match_pattern, str(row['Attachment/Exhibit']), re.IGNORECASE)
matches = re.findall(match_pattern, str(row[exhibit_col]), re.IGNORECASE)
if matches:
df.at[index, 'LOB_Correct'] = ', '.join(set([m.upper() for m in matches]))
else:
@@ -445,7 +525,7 @@ def clean_irs(abc_df, filename, text_dict ):
df_merged = df_merged.rename(columns={'PROV_GROUP_TIN_corrected': 'IRS_corrected'})
df_merged = df_merged.loc[:, ~df_merged.columns.str.contains('dummy')]
df_merged = df_merged.loc[:, ~df_merged.columns.str.contains('dummy', na=False)]
return df_merged
@@ -581,7 +661,7 @@ def clean_npi(abc_df, filename, text_dict, top_sheet_dict ):
df_merged['NPI_corrected'] = df_merged['dummy'].fillna(df_merged['NPI_corrected'])
if 'dummy_other' in df_merged.columns:
df_merged['NPI_other_corrected'] = df_merged['dummy_other'].fillna(df_merged['NPI_other_corrected'])
df_merged = df_merged.loc[:, ~df_merged.columns.str.contains('dummy')]
df_merged = df_merged.loc[:, ~df_merged.columns.str.contains('dummy', na=False)]
return df_merged
@@ -732,14 +812,14 @@ HOTFIX_ORDER = [
"Reimb. Methodology",
"Reimb. Methodology_Short",
"If rate is % of Payor or MCR [STANDARD]",
"If Rate is % of Payor or MCR [Standard] Short",
'If rate is % of Payor or MCR [STANDARD]_Short',
"FLAT FEE",
"Default Term",
"Default Rate",
'Inclusion of Essential RBRVS "Fee Source" Language (Y/N)',
"CDM Neutralization Language, Included (Y/N)",
'CDM Neutralization Language, included (Y/N)',
"CONTRACT_CHARGEMASTER_PROTECTION_LANGUAGE",
"IP - DSH/IME/UC, Included (Y/N)",
"IP - DSH/IME/UC, included (Y/N)",
"IP - Stoploss Catastrophic Threshold",
"Exclusions",
"Not to Exceed",
@@ -828,6 +908,8 @@ HOTFIX_ORDER = [
"LESSER_RATE_fixed",
"NOT_TO_EXCEED_fixed",
"SHORT_METHODOLOGY_fixed",
"Attachment/Exhibit_fixed",
"Attachment/Exhibit_page",
"_merge",
"key1_fixed",
"key2_fixed",
fieldExtraction/src/consolidate_output.py
+4 -3
View File
@@ -100,12 +100,13 @@ def consolidate_atscale() -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
except:
pass
b_final_df = pd.concat(b_dfs, ignore_index=True)
b_final_df.to_csv(
os.path.join(
final_path = os.path.join(
config.CONSOLIDATED_OUTPUT_DIRECTORY, f"{config.BATCH_ID}-B.csv"
)
b_final_df.to_csv(
final_path
)
print(f"file written to {final_path}")
abc_final_df = None
# If ABC
fieldExtraction/src/main.py
+2 -3
View File
@@ -80,9 +80,8 @@ def main():
# Read input first
input_dict = utils.read_input()
total_files = len(input_dict)
print(f"Total Input Files : {len(input_dict)}")
# Filter B
input_dict_b = {
k: v for k, v in input_dict.items()
@@ -104,7 +103,7 @@ def main():
)
else:
input_dict_ac = input_dict
files_processing_types = {}
if "b" in config.FIELDS.lower():
for filename in input_dict_b:
fieldExtraction/src/valid.py
+11 -5
View File
@@ -387,8 +387,9 @@ AC_MAPPING = {
"INVOICE_PRICING_LANGUAGE_IND": "Invoice Pricing (Y/N)",
"INVOICE_PRICING_LANGUAGE": "Invoice Pricing (Language)",
"TEMPLATE": "Template",
"PROVIDER_BASED_BILLING_EXCLUSION_LANGUAGE_IND": "Provider-based Billing Exclusion (Y/N)",
"PROVIDER_BASED_BILLING_EXCLUSION_LANGUAGE": "Provider-based Billing Exclusion (Language)",
"PROVIDER_BASED_BILLING_EXCLUSION_LANGUAGE_IND": "Provider-Based Billing Exclusion (Y/N)",
"PROVIDER_BASED_BILLING_EXCLUSION_LANGUAGE": "Provider-Based Billing Exclusion (Language)",
'Provider-based Billing Exclusion (Language)' : "Provider-Based Billing Exclusion (Language)",
"NON_RENEWAL_LANGUAGE": "Non-Renewal Language",
"NON_RENEWAL_DAYS": "Non-Renewal - Days",
"TIMELY_FILING": "Timely Filing",
@@ -672,6 +673,7 @@ HOTFIX_MAPPING = {
"LESSER": "Lesser of Logic language, included (Y/N)",
"LESSER_RATE": "Lesser of Rate",
"FULL_METHODOLOGY": "Reimb. Methodology",
"Reimb. Methodology Standard" : "Reimb. Methodology",
"SHORT_METHODOLOGY": "Reimb. Methodology_Short",
'Reimb. Methodology Short' : 'Reimb. Methodology_Short',
'If rate is % of Payor or MCR [Standard]' : 'If rate is % of Payor or MCR [STANDARD]',
@@ -679,6 +681,7 @@ HOTFIX_MAPPING = {
'If rate is % of Payor or MCR [STANDARD]' : "If rate is % of Payor or MCR [STANDARD]",
"RATE_STANDARD": "If rate is % of Payor or MCR [STANDARD]",
'If rate is % of Payor or MCR [Standard] Short' : 'If rate is % of Payor or MCR [STANDARD]_Short',
'If rate is % of Payor or MCR [STANDARD]_Short' : 'If rate is % of Payor or MCR [STANDARD]_Short',
"RATE_SHORT": "If rate is % of Payor or MCR [STANDARD]_Short",
"FLAT_FEE_STANDARD": "FLAT FEE",
'Flat Fee' : 'FLAT FEE',
@@ -686,7 +689,7 @@ HOTFIX_MAPPING = {
"DEFAULT_RATE": "Default Rate",
"MEDICAL_NECESSITY_LANGUAGE": "Medical Necessity Language (Language)",
"MEDICAL_NECESSITY_LANGUAGE_IND": "Medical Necessity Language (Y/N)",
'Inclusion of essential RBRVS "Fee Source" Language (Y/N)': 'Inclusion of essential RBRVS "Fee Source" Language (Y/N)',
'Inclusion of essential RBRVS "Fee Source" Language (Y/N)': 'Inclusion of Essential RBRVS "Fee Source" Language (Y/N)',
"CDM_IND": "CDM Neutralization Language, included (Y/N)",
"CHARGEMASTER": "CONTRACT_CHARGEMASTER_PROTECTION_LANGUAGE",
'Contract Chargemaster Protection Language' : "CONTRACT_CHARGEMASTER_PROTECTION_LANGUAGE",
@@ -711,6 +714,7 @@ HOTFIX_MAPPING = {
"TERMINATION_UPON_NOTICE": "Termination Upon Notice - Days",
"TERMINATION_UPON_CAUSE": "Termination With Cause - Days",
"AMEND_CONTRACT_NOTICE_IND": "Amend Contract Upon notice Flag (Y/N)",
'Amend Contract Upon Notice (Y/N)' : "Amend Contract Upon Notice Flag (Y/N)",
"TIME_TO_OBJECT": "Timeframe to Object - Days",
"ASSIGNMENTS_CLAUSE_IND": "Assignments Clause (Y/N)",
"CONTRACT_EFFECTIVE_DT": "Contract Effective Date",
@@ -784,8 +788,10 @@ HOTFIX_MAPPING = {
"INVOICE_PRICING_LANGUAGE_IND": "Invoice Pricing (Y/N)",
"INVOICE_PRICING_LANGUAGE": "Invoice Pricing (Language)",
"TEMPLATE": "Template",
"PROVIDER_BASED_BILLING_EXCLUSION_LANGUAGE_IND": "Provider-based Billing Exclusion (Y/N)",
"PROVIDER_BASED_BILLING_EXCLUSION_LANGUAGE": "Provider-based Billing Exclusion (Language)",
"PROVIDER_BASED_BILLING_EXCLUSION_LANGUAGE_IND": "Provider-Based Billing Exclusion (Y/N)",
'Provider-based Billing Exclusion (Y/N)' : "Provider-Based Billing Exclusion (Y/N)",
"PROVIDER_BASED_BILLING_EXCLUSION_LANGUAGE": "Provider-Based Billing Exclusion (Language)",
"Provider-based Billing Exclusion (Language)" : "Provider-Based Billing Exclusion (Language)",
"NON_RENEWAL_LANGUAGE": "Non-Renewal Language",
"NON_RENEWAL_DAYS": "Non-Renewal - Days",
"TIMELY_FILING": "Timely Filing",