Merged in refactor/hotfix_refactor (pull request #344)
Refactor/hotfix refactor * Initial hotfix refactor * Fix references * Remove test Approved-by: Alex Galarce
This commit is contained in:
Katon Minhas
@@ -0,0 +1,236 @@
|
||||
|
||||
import concurrent.futures
|
||||
import os
|
||||
|
||||
import pandas as pd
|
||||
from rapidfuzz import fuzz, process
|
||||
|
||||
import cnc_hotfix
|
||||
import config
|
||||
import io_utils
|
||||
import keywords
|
||||
import preprocessing_funcs
|
||||
import valid
|
||||
|
||||
clean_file_name = config.get_arg_value('clean_file', '')
|
||||
ABC_PATH = f'{clean_file_name}'
|
||||
|
||||
def get_highest_similarity(target_str, str_list):
|
||||
if target_str in str_list:
|
||||
return target_str
|
||||
best_match = process.extractOne(target_str, str_list)
|
||||
return best_match[0] if best_match else None
|
||||
|
||||
|
||||
def process_hotfix(abc, contract_text):
|
||||
"""
|
||||
Processes a contract by applying a series of hotfixes to the original output data.
|
||||
|
||||
Parameters:
|
||||
abc (DataFrame): The original output DataFrame for one contract. This DataFrame contains 114 columns.
|
||||
contract_text (str): The text of the contract.
|
||||
|
||||
Returns:
|
||||
DataFrame: The original abc DataFrame augmented with additional columns for each hotfix applied.
|
||||
"""
|
||||
|
||||
filename = abc['Contract Name'].unique()[0]
|
||||
|
||||
original_length = len(abc)
|
||||
|
||||
if contract_text is not None:
|
||||
# print(filename, len(contract_text))
|
||||
|
||||
# Preprocess
|
||||
contract_text = preprocessing_funcs.clean_newlines(contract_text)
|
||||
contract_text = preprocessing_funcs.clean_law_symbols(contract_text)
|
||||
|
||||
text_dict = preprocessing_funcs.split_text(
|
||||
contract_text
|
||||
) # return a dictionary with keys - page_num (str), values as the page_text
|
||||
text_dict, top_sheet_dict = preprocessing_funcs.filter_quick_review(text_dict)
|
||||
ac_chunks = preprocessing_funcs.smart_chunk_ac(text_dict, contract_text, keywords.GROUPED_KEYWORD_MAPPINGS)
|
||||
|
||||
######################################### Hot fixes ##########################################
|
||||
|
||||
######### Rule/Regex-based #########
|
||||
# Default
|
||||
abc = cnc_hotfix.check_default_term(abc)
|
||||
abc = cnc_hotfix.check_default_rate(abc)
|
||||
abc = cnc_hotfix.populate_default_term(abc, text_dict)
|
||||
abc.drop_duplicates(inplace=True)
|
||||
|
||||
# Health Plan State
|
||||
abc = cnc_hotfix.clean_healthplan_state(abc, filename, text_dict)
|
||||
abc.drop_duplicates(inplace=True)
|
||||
|
||||
# IRS
|
||||
abc = cnc_hotfix.clean_irs(abc, filename, text_dict)
|
||||
abc.drop_duplicates(inplace=True)
|
||||
|
||||
# NPI
|
||||
abc = cnc_hotfix.clean_npi(abc, filename, text_dict, top_sheet_dict)
|
||||
abc.drop_duplicates(inplace=True)
|
||||
|
||||
# Exhibit
|
||||
abc = cnc_hotfix.clean_exhibit(abc, text_dict)
|
||||
abc.drop_duplicates(inplace=True)
|
||||
|
||||
# LOB
|
||||
abc = cnc_hotfix.clean_lob(abc, text_dict)
|
||||
abc.drop_duplicates(inplace=True)
|
||||
|
||||
# Lesser
|
||||
abc = cnc_hotfix.clean_lesser(abc, text_dict)
|
||||
abc.drop_duplicates(inplace=True)
|
||||
|
||||
# Provider Type Level 2
|
||||
abc = cnc_hotfix.clean_provider_type_2(abc)
|
||||
abc.drop_duplicates(inplace=True)
|
||||
|
||||
######## Prompt-based #########
|
||||
# Rate Standard
|
||||
abc = cnc_hotfix.clean_rate_standard(abc)
|
||||
abc.drop_duplicates(inplace=True)
|
||||
|
||||
# Contract Effective Date
|
||||
abc = cnc_hotfix.clean_contract_effective_date(abc, filename, text_dict)
|
||||
abc.drop_duplicates(inplace=True)
|
||||
|
||||
# Term Group
|
||||
abc = cnc_hotfix.clean_term_group(abc, filename, text_dict, ac_chunks)
|
||||
abc.drop_duplicates(inplace=True)
|
||||
|
||||
# Agreement Name
|
||||
abc = cnc_hotfix.clean_agreement_name(abc, filename, text_dict)
|
||||
abc.drop_duplicates(inplace=True)
|
||||
|
||||
else:
|
||||
abc["Contract Duplicate Issue"] = "No Contract Found"
|
||||
|
||||
################## CREATE OUTPUT DIRECTORIES ##################
|
||||
base_filename = os.path.splitext(filename)[0].strip()
|
||||
output_dir = os.path.join(config.OUTPUT_DIRECTORY, base_filename)
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
################## WRITE UNPROCESSED OUTPUT ##################
|
||||
# Reorder
|
||||
abc = abc[[col for col in cnc_hotfix.HOTFIX_ORDER if col in abc.columns] + [col for col in abc.columns if col not in cnc_hotfix.HOTFIX_ORDER]]
|
||||
# Write to Excel
|
||||
abc.to_csv(
|
||||
os.path.join(output_dir, config.B_RESULTS_NAME), index=False
|
||||
)
|
||||
|
||||
return abc
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
REVERSE_MAPPING = {
|
||||
'Agreement Name (Contract Title)': 'Agreement_Name (Contract Title)',
|
||||
'Payer Name': 'PAYER NAME',
|
||||
'NPI (10-digits': 'NPI (10-digits)',
|
||||
'Prov Group TIN Signatory': 'PROV_GROUP_TIN_SIGNATORY',
|
||||
'Prov TIN Other': 'PROV_TIN_OTHER',
|
||||
'Prov NPI Other': 'PROV_NPI_OTHER',
|
||||
'Page Num': 'Page_Num',
|
||||
'Lesser of Logic Language, Included (Y/N)': 'Lesser of Logic language, included (Y/N)',
|
||||
'Reimb Methodology Short': 'Reimb. Methodology_Short',
|
||||
'If Rate is % of Payor or MCR [Standard]': 'If rate is % of Payor or MCR [STANDARD]',
|
||||
'If Rate is % of Payor or MCR [Standard] Short': 'If rate is % of Payor or MCR [STANDARD]_Short',
|
||||
'Flat Fee': 'FLAT FEE',
|
||||
'CDM Neutralization Language, Included (Y/N)': 'CDM Neutralization Language, included (Y/N)',
|
||||
'Contract Chargemaster Protection Language': 'CONTRACT_CHARGEMASTER_PROTECTION_LANGUAGE',
|
||||
'IP - DSH/IME/UC, Included (Y/N)': 'IP - DSH/IME/UC, included (Y/N)',
|
||||
'State': 'Health Plan State'
|
||||
}
|
||||
|
||||
# Read clean data
|
||||
print("Reading clean data...")
|
||||
if '.xlsx' in ABC_PATH:
|
||||
abc_clean = pd.read_excel(ABC_PATH)
|
||||
elif '.csv' in ABC_PATH:
|
||||
abc_clean = pd.read_csv(ABC_PATH)
|
||||
elif '.xlsb' in ABC_PATH:
|
||||
abc_clean = io_utils.read_xlsb(ABC_PATH)
|
||||
|
||||
|
||||
# Print current columns and identify invalid ones
|
||||
print("Current columns:")
|
||||
for col in abc_clean.columns:
|
||||
print(col)
|
||||
|
||||
abc_clean.rename(columns=REVERSE_MAPPING, inplace=True)
|
||||
|
||||
print(f"Invalid columns: {[col for col in abc_clean.columns if col not in cnc_hotfix.HOTFIX_ORDER]}")
|
||||
|
||||
|
||||
|
||||
abc_clean = abc_clean.dropna(how='all')
|
||||
abc_clean = abc_clean.dropna(axis=1, how='all')
|
||||
|
||||
unique_output_files = abc_clean['Contract Name'].unique()
|
||||
|
||||
print("ABC Clean: ", abc_clean.shape)
|
||||
print(len(unique_output_files), 'files in clean output...')
|
||||
# print(list(abc_clean.columns))
|
||||
abc_clean.rename(columns=valid.HOTFIX_MAPPING, inplace=True)
|
||||
abc_clean['Contract Name'] = abc_clean['Contract Name'].apply(lambda x: ('Filename: ' + str(x) if not str(x).startswith('Filename: ') else str(x)) + ('.txt' if not str(x).endswith('.txt') else ''))
|
||||
unique_output_files = abc_clean['Contract Name'].unique()
|
||||
|
||||
already_ran = os.listdir(config.OUTPUT_DIRECTORY)
|
||||
|
||||
# Read input dict
|
||||
print("Reading input .txt files...")
|
||||
input_dict = io_utils.read_input()
|
||||
input_dict = {'Filename: ' + filename.replace('.txt', '').strip() : contract_text for filename, contract_text in input_dict.items()}
|
||||
if config.FILTER_ALREADY_PROCESSED:
|
||||
input_dict = {filename : contract_text for filename, contract_text in input_dict.items() if filename not in already_ran}
|
||||
print("Input Dict: ", len(input_dict))
|
||||
|
||||
# input_keys = list(input_dict.keys())
|
||||
# abc_clean['Contract Name'] = abc_clean['Contract Name'].apply(
|
||||
# lambda x: get_highest_similarity(x, input_keys)
|
||||
# )
|
||||
|
||||
unique_output_files = abc_clean['Contract Name'].unique()
|
||||
print(f"{len([file for file in input_dict.keys() if file not in unique_output_files])} files in full s3, not in output")
|
||||
print(f"{len([file for file in unique_output_files if file not in input_dict.keys()])} files in output, not in full s3")
|
||||
|
||||
# Create list of tuples
|
||||
abc_clean_list = [(group, input_dict.get(name.replace('.txt', ''))) for name, group in abc_clean.groupby('Contract Name')]
|
||||
print(f"{len(abc_clean_list)} - files in input")
|
||||
print(f"{len([f for f in abc_clean_list if f[1]])} - valid files in input")
|
||||
del abc_clean
|
||||
del input_dict
|
||||
|
||||
abc_clean_list = [f for f in abc_clean_list if f[1]]
|
||||
print(len(abc_clean_list))
|
||||
|
||||
df_list = []
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor:
|
||||
# Create a list of future objects by submitting process_pair function to the executor
|
||||
futures = [executor.submit(process_hotfix, abc_df, contract_text) for abc_df, contract_text in abc_clean_list]
|
||||
# As each future completes, retrieve its result and add to df_list
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
try:
|
||||
new_df = future.result()
|
||||
if new_df is not None:
|
||||
df_list.append(new_df)
|
||||
except Exception as e:
|
||||
# Log the exception or handle it otherwise
|
||||
print(f"An error occurred: {e}")
|
||||
|
||||
abc_final = pd.concat(df_list, ignore_index=True)
|
||||
print(abc_final.shape)
|
||||
print(len(abc_final['Contract Name'].unique()), ' unique contracts')
|
||||
|
||||
print("Writing to excel...")
|
||||
abc_final.to_excel(f'output_consolidated/{config.BATCH_ID}-Hotfix-Final.xlsx')
|
||||
|
||||
print("Completed writing to Excel.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -0,0 +1,918 @@
|
||||
import itertools
|
||||
import re
|
||||
import warnings
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
import claude_funcs
|
||||
import config
|
||||
import postprocessing_funcs
|
||||
import string_funcs
|
||||
from one_to_n_funcs import get_short_methodology
|
||||
from prompts import BOTTOM_UP_METHODOLOGY_BREAKOUT
|
||||
from valid import STATE_MAP
|
||||
|
||||
warnings.simplefilter(action='ignore', category=FutureWarning)
|
||||
|
||||
|
||||
import claude_funcs
|
||||
import scripts.cnc_hotfixes.cnc_hotfix_effective_date_utils as cnc_hotfix_effective_date_utils
|
||||
import config
|
||||
import keywords
|
||||
import postprocessing_funcs
|
||||
import prompts
|
||||
import smart_chunking_funcs
|
||||
import string_funcs
|
||||
import valid
|
||||
from hotfix_helper_funcs import (check_field_for_matches,
|
||||
clean_output_postprocess, irs_hotfix,
|
||||
npi_hotfix, npi_post_process, state_check)
|
||||
|
||||
|
||||
####### Exhibit #######
|
||||
def clean_exhibit(abc, text_dict):
|
||||
abc["Attachment/Exhibit_fixed"], abc["Attachment/Exhibit_page"] = "", ""
|
||||
|
||||
def adhoc_exhibit_check(page):
|
||||
prompt = prompts.TOP_DOWN_EXHIBIT_CHECK(page[0:100])
|
||||
answer = claude_funcs.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE2, "", max_tokens=10
|
||||
)
|
||||
if "Y" in answer:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def adhoc_exhibit_prompt(page):
|
||||
prompt = f"""
|
||||
### PAGE_START ### {page} ### PAGE_END ###
|
||||
List any exhibit, attachment, or amendment names found on the page. Write the full name of the exhibit, including the exhibit number or letter, as well as any other subtitles describing the contents of the exhibit. Do NOT write the page number.
|
||||
If no Exhibit, Attachment, or Amendment is found, write 'N/A'. Enclose only your final answer in |pipes|.
|
||||
"""
|
||||
answer = claude_funcs.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, "", max_tokens=1000
|
||||
)
|
||||
return re.findall(pattern=cnc_hotfix_effective_date_utils.regex_backticks, string=answer)[-1]
|
||||
|
||||
answer_dict = {}
|
||||
for index, row in abc.iterrows():
|
||||
if pd.notna(row['Page_Num']):
|
||||
page_num = str(int(row['Page_Num']))
|
||||
original_page_num = page_num
|
||||
|
||||
# If we've already seen this page, get the answer previously found
|
||||
if original_page_num in answer_dict.keys():
|
||||
abc.loc[index, "Attachment/Exhibit_fixed"] = answer_dict[original_page_num]
|
||||
abc.loc[index, "Attachment/Exhibit_page"] = original_page_num
|
||||
continue
|
||||
|
||||
exhibit_val = str(abc.loc[index, "Attachment/Exhibit"]).strip()
|
||||
normalized_exhibit_val = exhibit_val.replace(" ", "").replace("\n", "").upper()
|
||||
|
||||
# If the exhibit is already on that page, then it's correct
|
||||
if normalized_exhibit_val in text_dict[page_num].replace(" ", "").replace("\n", "").upper():
|
||||
abc.loc[index, "Attachment/Exhibit_fixed"] = exhibit_val
|
||||
abc.loc[index, "Attachment/Exhibit_page"] = page_num
|
||||
answer_dict[original_page_num] = exhibit_val
|
||||
else:
|
||||
contains_exhibit = False
|
||||
while not contains_exhibit:
|
||||
# Run TD Exhibit Check
|
||||
if page_num in text_dict.keys():
|
||||
contains_exhibit = adhoc_exhibit_check(text_dict[page_num])
|
||||
if contains_exhibit:
|
||||
page_exhibit = adhoc_exhibit_prompt(text_dict[page_num])
|
||||
if 'N/A' not in page_exhibit:
|
||||
abc.loc[index, "Attachment/Exhibit_fixed"] = page_exhibit
|
||||
abc.loc[index, "Attachment/Exhibit_page"] = page_num
|
||||
answer_dict[original_page_num] = page_exhibit
|
||||
else:
|
||||
contains_exhibit = False
|
||||
page_num = str(int(page_num) - 1)
|
||||
else:
|
||||
page_num = str(int(page_num) - 1)
|
||||
else:
|
||||
contains_exhibit=True # Set to True to exit loop
|
||||
return abc
|
||||
|
||||
|
||||
####### Default Term/Rate #######
|
||||
def concatenate_lists(row):
|
||||
total = []
|
||||
for col in ['Line of Business List', 'Provider Type List', 'Provider Type - Level 2 List', 'IP/OP List', 'Service Type List', 'Plan Type List']:
|
||||
value = row[col]
|
||||
if isinstance(value, list):
|
||||
total.extend([str(item) for item in value])
|
||||
else:
|
||||
total.append(str(value))
|
||||
return total
|
||||
|
||||
def check_default_term(df: pd.DataFrame):
|
||||
|
||||
df['Contains Only'] = df['Default Term'].str.lower().str.contains('only')
|
||||
if df['Contains Only'].any():
|
||||
df_only_grouped = df[['Attachment/Exhibit', 'Line of Business', 'Provider Type', 'Provider Type - Level 2', 'IP/OP', 'Service Type', 'Plan Type', 'Default Term']]
|
||||
df_transformed = df_only_grouped.groupby(['Attachment/Exhibit', 'Default Term'], as_index=False).transform(lambda x: list(set(x.dropna().to_list())))
|
||||
df_only_grouped[['Line of Business List', 'Provider Type List', 'Provider Type - Level 2 List', 'IP/OP List', 'Service Type List', 'Plan Type List']] = df_transformed
|
||||
|
||||
|
||||
df_only_grouped['All Checks'] = df_only_grouped[['Line of Business List', 'Provider Type List', 'Provider Type - Level 2 List', 'IP/OP List', 'Service Type List', 'Plan Type List']].apply(concatenate_lists, axis=1)
|
||||
df_only_grouped['All Checks'] = df_only_grouped['All Checks'].apply(lambda x: [y.replace('OP','Outpatient').replace('IP', 'Inpatient') for y in x])
|
||||
df_only_grouped = df_only_grouped[['Attachment/Exhibit', 'Line of Business', 'Provider Type', 'Provider Type - Level 2', 'IP/OP', 'Service Type', 'Plan Type','All Checks']]
|
||||
|
||||
df = df.merge(df_only_grouped, on = ['Attachment/Exhibit', 'Line of Business', 'Provider Type', 'Provider Type - Level 2', 'IP/OP', 'Service Type', 'Plan Type'], how = 'left')
|
||||
|
||||
df['Default Term'].fillna('', inplace=True)
|
||||
df['All Checks'] = df['All Checks'].apply(lambda d: d if isinstance(d, list) else [])
|
||||
|
||||
df['Default Match?'] = df.apply(lambda row: any([x.lower() in row['Default Term'].lower() for x in row['All Checks']]), axis = 1)
|
||||
|
||||
df['Default Rate Corrected Step 1'] = df['Default Rate']
|
||||
df.loc[(df['Contains Only']==True) & (df['Default Match?']==False), 'Default Rate Corrected Step 1'] = ''
|
||||
|
||||
|
||||
return df
|
||||
|
||||
else:
|
||||
df['All Checks'] = ''
|
||||
df['Default Match?'] = True
|
||||
df['Default Rate Corrected Step 1'] = df['Default Rate']
|
||||
return df
|
||||
|
||||
def check_default_rate(df: pd.DataFrame):
|
||||
|
||||
mapping = {
|
||||
'medicare' : 'MCR',
|
||||
'medicaid' : 'MCD',
|
||||
'billed charge' : 'BC',
|
||||
'allowable charge' : 'AC',
|
||||
'allowed charge' : 'AC',
|
||||
'average wholesale price' : 'AWP'
|
||||
}
|
||||
|
||||
def replace_substring(string):
|
||||
for k, v in mapping.items():
|
||||
string = re.sub(r'\b' + k + r'\b', v, string, flags=re.IGNORECASE)
|
||||
return string
|
||||
|
||||
def safe_extract(pattern, text):
|
||||
match = re.search(pattern, text)
|
||||
return match.group(0) if match else ''
|
||||
|
||||
df['Default Rate Corrected Step 2'] = df['Default Rate Corrected Step 1']
|
||||
df_relevant = df.loc[(df['Default Rate Corrected Step 1'].isna()==False) & (df['Default Rate Corrected Step 1'] != '') & ((df['Default Term']=='') | (df['Default Term']!=df['Default Term'])), :].copy()
|
||||
|
||||
if not df_relevant.empty:
|
||||
pct_pattern = r'\d+\% of '
|
||||
reimb_pattern = r'(medicare|allowed charge|allowable charge|billed charge|medicaid|average wholesale price)'
|
||||
df_relevant.loc[:, 'Default Rate Shortened'] = df_relevant.loc[:, 'Default Rate Corrected Step 1'].apply(lambda x:
|
||||
safe_extract(pct_pattern, x.lower()) + safe_extract(reimb_pattern, x.lower())
|
||||
)
|
||||
df_relevant.loc[:,'Default Rate Shortened'] = df_relevant.loc[:, 'Default Rate Shortened'].apply(replace_substring)
|
||||
df_grouped = df_relevant.groupby(['Contract Name', 'Attachment/Exhibit', 'Default Rate Shortened'], as_index = False)[r'If rate is % of Payor or MCR [STANDARD]'].agg(lambda x: list(set(x.dropna().to_list()))).reset_index()
|
||||
df_grouped = df_grouped.rename(columns = {r'If rate is % of Payor or MCR [STANDARD]' : 'Rate Standard List'})
|
||||
try:
|
||||
df_grouped['Remove Default Rate?'] = df_grouped.apply(lambda row: any([x.lower() in row['Default Rate Shortened'].lower() for x in row['Rate Standard List']]), axis=1)
|
||||
df = df.merge(df_grouped, how = 'left', on = ['Contract Name', 'Attachment/Exhibit'])
|
||||
df['Default Rate Corrected Step 2'] = df['Default Rate Corrected Step 1']
|
||||
df.loc[df['Remove Default Rate?'] == True, 'Default Rate Corrected Step 2'] = ''
|
||||
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
#df.drop(columns = ['Default Rate Shortened'], inplace=True)
|
||||
return df
|
||||
|
||||
def populate_default_term(df, text_dict):
|
||||
|
||||
file_df = df.loc[(df['Default Rate Corrected Step 2'].isna()==False) & (df['Default Rate Corrected Step 2'] != '') & ((df['Default Term'].isna()==True) | (df['Default Term'] == '')), :]
|
||||
|
||||
file_df = file_df.dropna(subset=['Page_Num', 'Attachment/Exhibit'])
|
||||
|
||||
if not file_df.empty:
|
||||
exhibits = file_df['Attachment/Exhibit'].unique()
|
||||
for exhibit in exhibits:
|
||||
start_page = file_df.loc[file_df['Attachment/Exhibit'] == exhibit, 'Page_Num'].min()
|
||||
|
||||
end_page = file_df.loc[file_df['Attachment/Exhibit'] == exhibit, 'Pages'].min() + 1
|
||||
|
||||
extracted_text = ''
|
||||
|
||||
for page in range(int(round(start_page)), int(round(end_page))):
|
||||
str_page = f'{page:.0f}'
|
||||
if str_page in text_dict.keys():
|
||||
exhibit_text = text_dict[str_page]
|
||||
|
||||
def_rate = file_df.loc[file_df['Page_Num'] == start_page, 'Default Rate Corrected Step 2']
|
||||
|
||||
if isinstance(def_rate, pd.Series):
|
||||
def_rate = def_rate.min()
|
||||
elif not isinstance(def_rate, str):
|
||||
def_rate = str(def_rate)
|
||||
|
||||
sentences = exhibit_text.split('.')
|
||||
|
||||
pct_pattern = r'\d+\%'
|
||||
|
||||
reimb_pattern = r'(medicare|allowed charge|allowable charge|billed charge|medicaid|average wholesale price|fee schedule)'
|
||||
|
||||
if (def_rate == def_rate) and (def_rate != '') and (re.search(pct_pattern, def_rate)) and (re.search(reimb_pattern, def_rate.lower())):
|
||||
def_pct = re.search(pct_pattern, def_rate).group(0)
|
||||
def_reimb = re.search(reimb_pattern, def_rate.lower()).group(0)
|
||||
for sentence in sentences:
|
||||
sentence = sentence.replace('\n', ' ')
|
||||
if (def_pct in sentence) and (def_reimb in sentence.lower()):
|
||||
extracted_text = sentence
|
||||
break
|
||||
|
||||
if extracted_text != '':
|
||||
break
|
||||
|
||||
file_df.loc[file_df['Attachment/Exhibit'] == exhibit, 'Default Term Corrected Step 3'] = extracted_text.strip()
|
||||
file_df = file_df[['Attachment/Exhibit', 'Default Term Corrected Step 3']]
|
||||
file_df.drop_duplicates(inplace=True)
|
||||
df = df.merge(file_df, how='left', on=['Attachment/Exhibit'])
|
||||
else:
|
||||
file_df['Default Term Corrected Step 3'] = file_df['Default Term']
|
||||
file_df = file_df[['Page_Num', 'Attachment/Exhibit', 'Default Term Corrected Step 3']]
|
||||
df = df.merge(file_df, how = 'left', on = ['Page_Num', 'Attachment/Exhibit'])
|
||||
|
||||
df['Default Term Corrected Step 3'] = df.apply(lambda x: x['Default Term'] if (x['Default Term Corrected Step 3'] == '') or (x['Default Term Corrected Step 3'] != x['Default Term Corrected Step 3']) else x['Default Term Corrected Step 3'], axis = 1)
|
||||
|
||||
for col in ['Contains Only','All Checks','Default Match?','index', 'Default Rate Corrected Step 1','Default Rate Shortened','Rate Standard List','Remove Default Rate?']:
|
||||
if col in df.columns:
|
||||
df.drop(col, axis=1, inplace=True)
|
||||
else:
|
||||
pass
|
||||
|
||||
df.rename(columns={'Default Rate Corrected Step 2' : 'Default Rate Corrected', 'Default Term Corrected Step 3' : 'Default Term Corrected'}, inplace=True)
|
||||
|
||||
return df
|
||||
|
||||
|
||||
#### HEALTH PLAN STATE ####
|
||||
def clean_healthplan_state(
|
||||
df: pd.DataFrame,
|
||||
contract_name: str,
|
||||
text_dict: dict[str, str],
|
||||
) -> pd.DataFrame:
|
||||
|
||||
if not isinstance(df, pd.DataFrame):
|
||||
# raise TypeError(f"Expected a dataframe, got {type(df).__name__}")
|
||||
return df
|
||||
|
||||
if contract_name is None:
|
||||
# raise ValueError(
|
||||
# f"Expected 'contract_name' parameter, got {contract_name}"
|
||||
# )
|
||||
return df
|
||||
|
||||
if "Health Plan State" not in df.columns:
|
||||
# raise KeyError(
|
||||
# f"Column 'Health Plan State' not found in DataFrame. Available columns are: {list(df.columns)}"
|
||||
# )
|
||||
return df
|
||||
|
||||
health_plan_state = df['Health Plan State'].unique().tolist()[0]
|
||||
|
||||
answer = state_check(health_plan_state, text_dict)
|
||||
|
||||
df["Health Plan State_corrected"] = answer
|
||||
|
||||
return df
|
||||
|
||||
#### LINE OF BUSINESS ####
|
||||
def clean_lob(df, text_dict):
|
||||
|
||||
df['Line of Business'] = df['Line of Business'].str.upper()
|
||||
incorrect_map = {"MMP PLAN": "", "COMPLETE": "", "ALL OTHER HEALTH PLANS": "", "NAN" : ""}
|
||||
df['Line of Business'] = df['Line of Business'].replace(incorrect_map)
|
||||
df['LOB_Correct'] = df['Line of Business']
|
||||
|
||||
match_pattern = r'|'.join(valid.VALID_LOBS)
|
||||
|
||||
for index, row in df.iterrows():
|
||||
if 'Attachment/Exhibit_fixed' in df.columns:
|
||||
exhibit_col = 'Attachment/Exhibit_fixed'
|
||||
else:
|
||||
exhibit_col = 'Attachment/Exhibit'
|
||||
|
||||
exhibit_fixed = str(row[exhibit_col]).upper()
|
||||
|
||||
# Issue 0: Incorrect - check exhibit fixed (if the LOB is not in the Exhibit, but another LOB is in the exhibit)
|
||||
if (str(row['Line of Business']).upper() not in exhibit_fixed) and any([lob.upper() in exhibit_fixed for lob in valid.VALID_LOBS]):
|
||||
df.at[index, 'LOB_Correct'] = ', '.join(set([m for m in valid.VALID_LOBS if m in exhibit_fixed]))
|
||||
continue
|
||||
|
||||
# Issue 1: Missing
|
||||
if string_funcs.is_empty(row['Line of Business']) and pd.notna(row['Page_Num']): # Check if 'Line of Business' is missing
|
||||
|
||||
# Prepare to search text
|
||||
page_text = text_dict.get(str(int(row['Page_Num'])), "")
|
||||
next_page_text = text_dict.get(str(int(row['Page_Num']) + 1), "") # Assuming page_num is a numerical index
|
||||
second_page_text = text_dict.get(str(int(row['Page_Num']) + 2), "")
|
||||
|
||||
# First check in or near exhibit
|
||||
exhibit_index = page_text.find(str(row[exhibit_col]).strip())
|
||||
if exhibit_index != -1 and pd.notna(row[exhibit_col]):
|
||||
search_area = page_text[exhibit_index:exhibit_index + 200 + len(row[exhibit_col])]
|
||||
matches = re.findall(match_pattern, search_area, re.IGNORECASE)
|
||||
if matches:
|
||||
df.at[index, 'LOB_Correct'] = ', '.join(set([m.upper() for m in matches]))
|
||||
continue
|
||||
|
||||
# Then check on entire page
|
||||
matches = re.findall(match_pattern, page_text, re.IGNORECASE)
|
||||
if matches:
|
||||
df.at[index, 'LOB_Correct'] = ', '.join(set([m.upper() for m in matches]))
|
||||
continue
|
||||
|
||||
# Then check on next page
|
||||
matches = re.findall(match_pattern, next_page_text, re.IGNORECASE)
|
||||
if matches:
|
||||
df.at[index, 'LOB_Correct'] = ', '.join(set([m.upper() for m in matches]))
|
||||
continue
|
||||
|
||||
df.at[index, 'LOB_Correct'] = 'NO MATCH FOUND - REVIEW'
|
||||
|
||||
# Issue 2: When Payer..., Medicare and/not Medicaid
|
||||
elif pd.notna(row["Service Type"]):
|
||||
service = str(row['Service Type']).lower()
|
||||
if 'where' in service and ('payer' in service or 'payor' in service):
|
||||
# Check Exhibit first
|
||||
matches = re.findall(match_pattern, str(row[exhibit_col]), re.IGNORECASE)
|
||||
if matches:
|
||||
df.at[index, 'LOB_Correct'] = ', '.join(set([m.upper() for m in matches]))
|
||||
else:
|
||||
df.at[index, 'LOB_Correct'] = str(row['Line of Business']).upper()
|
||||
|
||||
# Non-Issue
|
||||
else:
|
||||
df.at[index, 'LOB_Correct'] = str(row['Line of Business']).upper()
|
||||
|
||||
return df
|
||||
|
||||
|
||||
#### Rate Standard ####
|
||||
def reimb_methodology_fix(rm: str):
|
||||
|
||||
d = {'FULL_METHODOLOGY' : rm}
|
||||
prompt = BOTTOM_UP_METHODOLOGY_BREAKOUT(d)
|
||||
answer = claude_funcs.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, '', max_tokens=512
|
||||
)
|
||||
answer_dict = string_funcs.secondary_string_to_dict(answer, '')
|
||||
d.update(answer_dict)
|
||||
|
||||
# Ensure all bottom up methodology keys are present
|
||||
for key in [
|
||||
"LESSER",
|
||||
"RATE_STANDARD",
|
||||
"RATE_SHORT",
|
||||
"FLAT_FEE_STANDARD",
|
||||
"LESSER_RATE",
|
||||
"NOT_TO_EXCEED",
|
||||
]:
|
||||
if key not in d.keys():
|
||||
d[key] = ""
|
||||
|
||||
# SHORT_METHODOLOGY
|
||||
if "%" in d["RATE_STANDARD"]:
|
||||
d["SHORT_METHODOLOGY"] = get_short_methodology(d["RATE_STANDARD"])
|
||||
elif "$" in d["FLAT_FEE_STANDARD"]:
|
||||
d["SHORT_METHODOLOGY"] = "Flat Fee"
|
||||
else:
|
||||
d["SHORT_METHODOLOGY"] = "N/A"
|
||||
|
||||
return d
|
||||
|
||||
def clean_rate_standard(df: pd.DataFrame) -> pd.DataFrame:
|
||||
|
||||
df_relevant = df.loc[(df['Reimb. Methodology'].isna() == False) & (df[r'If rate is % of Payor or MCR [STANDARD]'].isna() == True) & (df['FLAT FEE'].isna() == True), :]
|
||||
|
||||
if not df_relevant.empty:
|
||||
rm_list = df_relevant['Reimb. Methodology'].unique()
|
||||
|
||||
dict_list = []
|
||||
|
||||
for rm in rm_list:
|
||||
|
||||
d = reimb_methodology_fix(rm)
|
||||
|
||||
dict_list.append(d)
|
||||
|
||||
results_df = pd.DataFrame.from_records(dict_list)
|
||||
|
||||
results_df = results_df.add_suffix('_fixed')
|
||||
|
||||
df = pd.merge(df, results_df, how = 'left', right_on = 'FULL_METHODOLOGY_fixed', left_on = 'Reimb. Methodology')
|
||||
|
||||
return df
|
||||
|
||||
|
||||
#### LESSER ####
|
||||
def clean_lesser(df, text_dict):
|
||||
df['Lesser_Flag'] = ""
|
||||
# For Exhibit
|
||||
all_dfs = []
|
||||
for page_num, page_df in df.groupby('Page_Num', dropna=False):
|
||||
if pd.notna(page_num) and str(int(page_num)) in text_dict.keys():
|
||||
contract_page = str(int(page_num))
|
||||
try:
|
||||
contract_text = '\n'.join([text_dict[str(int(contract_page)+i)] for i in range(2)])
|
||||
except:
|
||||
contract_text = text_dict[str(int(contract_page))]
|
||||
|
||||
output_lesser_count = (page_df['Lesser of Logic language, included (Y/N)'] == 'Y').sum()
|
||||
row_count = page_df.shape[0]
|
||||
page_lesser_count = len(re.findall(r"lesser|lessor", contract_text, re.IGNORECASE))
|
||||
# Likely Incorrect
|
||||
if output_lesser_count == 1 and row_count > 1 and page_lesser_count == 1:
|
||||
page_df.loc[:, 'Lesser_Flag'] = 'Likely Incorrect'
|
||||
# Likely Correct
|
||||
elif page_lesser_count == output_lesser_count == row_count:
|
||||
page_df.loc[:, 'Lesser_Flag'] = 'Likely Correct'
|
||||
# Other
|
||||
else:
|
||||
page_df.loc[:, 'Lesser_Flag'] = "No Flag"
|
||||
else:
|
||||
page_df.loc[:, 'Lesser_Flag'] = "No Flag"
|
||||
|
||||
all_dfs.append(page_df)
|
||||
|
||||
try:
|
||||
final_df = pd.concat(all_dfs, ignore_index=True)
|
||||
except:
|
||||
final_df = df.copy()
|
||||
|
||||
# Second Column
|
||||
final_df['Service-Methodology'] = final_df['Service Type'].astype(str) + ' - ' + final_df['Reimb. Methodology'].astype(str)
|
||||
return final_df
|
||||
|
||||
|
||||
#### PROVIDER TYPE LEVEL 2 ####
|
||||
def clean_provider_type_2(df):
|
||||
valid_lists = {
|
||||
'PROF': [
|
||||
"Physician", "Primary Care Provider", "Dual Capacity Physician",
|
||||
"Mid-Level Practicioner", "Specialist", "OB/GYN", "Behavioral Health",
|
||||
"Therapist", "Surgery", "Independent RHC", "Ambulance Services",
|
||||
"Home Health", "SNF"
|
||||
],
|
||||
'ANC': [
|
||||
"DME", "Durable Medical Equipment", "Radiology", "Lab", "Home Health",
|
||||
"Hospice", "Dialysis", "PT/OT/ST", "Urgent Care", "Home Infusion",
|
||||
"Ambulatory Surgery Center", "ASC", "Ambulance Services", "Home Health"
|
||||
],
|
||||
'FAC': [
|
||||
"Skilled Nursing Facility", "Ambulatory Surgery Center", "ASC",
|
||||
"Hospital", "Clinic", "Rural Health Clinic", "FQHC", "Long Term Care",
|
||||
"Community Mental Health Center"
|
||||
]
|
||||
}
|
||||
result_df = df.copy()
|
||||
result_df['PROV_TYPE_LEVEL_2_Corrected'] = result_df['Provider Type - Level 2']
|
||||
unique_exhibits = df['Attachment/Exhibit'].unique()
|
||||
for exhibit in unique_exhibits:
|
||||
if pd.isna(exhibit):
|
||||
continue
|
||||
match, _ = check_field_for_matches(exhibit, valid_lists)
|
||||
if match:
|
||||
exhibit_mask = (result_df['Attachment/Exhibit'] == exhibit) & (result_df['PROV_TYPE_LEVEL_2_Corrected'].isna())
|
||||
if exhibit_mask.any():
|
||||
result_df.loc[exhibit_mask, 'PROV_TYPE_LEVEL_2_Corrected'] = match
|
||||
blank_mask = result_df['PROV_TYPE_LEVEL_2_Corrected'].isna()
|
||||
for idx, row in result_df[blank_mask].iterrows():
|
||||
for field in df.columns:
|
||||
if field != 'PROV_TYPE_LEVEL_2_Corrected':
|
||||
match, _ = check_field_for_matches(row[field], valid_lists)
|
||||
if match:
|
||||
result_df.at[idx, 'PROV_TYPE_LEVEL_2_Corrected'] = match
|
||||
break
|
||||
return result_df
|
||||
|
||||
#### IRS ####
|
||||
def clean_irs(abc_df, filename, text_dict ):
|
||||
"""Gets the answers for fixed IRS, TIN Others and merges them with clean output
|
||||
Args:
|
||||
abc_df(df): Clean input which is grouped on 'Contract Name'
|
||||
filename: Name of the contract
|
||||
text_dict (dict): Dictionary keyed by string page-number and valued by actual textract output page
|
||||
Returns:
|
||||
merged_df(df): Left joined [abc_df, irs_hotfix_df] on 'Contract Name' with additional '_corrected' columns
|
||||
"""
|
||||
|
||||
# gets the irs hotfixes in
|
||||
irs_answers = irs_hotfix(filename, text_dict)
|
||||
irs_df = pd.DataFrame([irs_answers])
|
||||
|
||||
# adds corrected suffix for the hotfixes
|
||||
irs_df.columns = [col + '_corrected' for col in irs_df.columns]
|
||||
|
||||
# join on 'Contract Name'
|
||||
irs_df["Contract Name"] = str(filename)
|
||||
df_merged = pd.merge(abc_df, irs_df, on='Contract Name', indicator=True, how='left')
|
||||
df_merged = clean_output_postprocess(df_merged)
|
||||
|
||||
# overwrite the corrected columns with clean output if it already has an answer
|
||||
df_merged['PROV_GROUP_TIN_corrected'] = df_merged['dummy'].fillna(df_merged['PROV_GROUP_TIN_corrected'])
|
||||
|
||||
if 'dummy_other' in df_merged.columns:
|
||||
df_merged['PROV_TIN_OTHER_corrected'] = df_merged['dummy_other'].fillna(df_merged['PROV_TIN_OTHER_corrected'])
|
||||
|
||||
if 'dummy_other_signatory' in df_merged.columns:
|
||||
df_merged['PROV_TIN_GROUP_SIGNATORY_corrected'] = df_merged['dummy_other_signatory'].fillna(df_merged['PROV_TIN_GROUP_SIGNATORY_corrected'])
|
||||
|
||||
df_merged = df_merged.rename(columns={'PROV_GROUP_TIN_corrected': 'IRS_corrected'})
|
||||
|
||||
df_merged = df_merged.loc[:, ~df_merged.columns.str.contains('dummy', na=False)]
|
||||
return df_merged
|
||||
|
||||
|
||||
##### CONTRACT EFFECTIVE DATE #####
|
||||
def contract_effective_date_fix(contract_name: str, text_dict: dict[str, str],is_meridian: bool=False):
|
||||
# perform smart chunking
|
||||
page_list,d = cnc_hotfix_effective_date_utils.chunk_with_include_exclude_keywords(text_dict,
|
||||
include_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["included_keywords"],
|
||||
exclude_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["excluded_keywords"],
|
||||
)
|
||||
if len(page_list) > 0:
|
||||
context = "\n".join(text_dict[page] for page in page_list)
|
||||
prompt = prompts.get_effective_date_prompt(context) # get effective date - this is for both meridian and non-meridian
|
||||
|
||||
try:
|
||||
claude_answer_raw = claude_funcs.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192)
|
||||
claude_answer_extracted = re.findall(pattern=cnc_hotfix_effective_date_utils.regex_backticks, string=claude_answer_raw)[-1]
|
||||
|
||||
except Exception as e:
|
||||
raise
|
||||
|
||||
# meridian special case
|
||||
if is_meridian:
|
||||
if claude_answer_extracted == "N/A":
|
||||
# perform smart chunking with signature date pages included
|
||||
|
||||
included_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["included_keywords"]
|
||||
|
||||
included_keywords = list(itertools.chain(included_keywords, ["Signature"]))
|
||||
|
||||
excluded_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["excluded_keywords"]
|
||||
|
||||
page_list, d = cnc_hotfix_effective_date_utils.chunk_with_include_exclude_keywords(text_dict,
|
||||
include_keywords = included_keywords,
|
||||
exclude_keywords = excluded_keywords
|
||||
)
|
||||
|
||||
context = "\n".join(text_dict[page] for page in page_list)
|
||||
prompt = cnc_hotfix_effective_date_utils.get_effective_date_meridian_prompt(context)
|
||||
|
||||
try:
|
||||
claude_answer_raw = claude_funcs.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192)
|
||||
claude_answer_extracted = re.findall(pattern=cnc_hotfix_effective_date_utils.regex_backticks, string=claude_answer_raw)[-1]
|
||||
except Exception as e:
|
||||
raise
|
||||
|
||||
return claude_answer_extracted, d, page_list, claude_answer_raw
|
||||
|
||||
else:
|
||||
return "N/A", d, page_list, ""
|
||||
|
||||
def clean_contract_effective_date(
|
||||
df: pd.DataFrame,
|
||||
file_name: str,
|
||||
text_dict: dict[str, str]
|
||||
) -> pd.DataFrame:
|
||||
"""Apply `contract_effective_date_fix` to a grouped input dataframe, `df`. A new column
|
||||
is added.
|
||||
|
||||
Args:
|
||||
df (pd.DataFrame): Grouped dataframe for one contract. Can have multiple rows because
|
||||
of B-fields. Contract effective date is an AC-field, so one per document.
|
||||
file_name (str): Filename of the contract
|
||||
text_dict (dict[str, str]): String-keyed dictionary valued by contract text
|
||||
|
||||
Raises:
|
||||
TypeError: If a non-dataframe is passed in as `df`
|
||||
KeyError: If 'Contract Effective Date' is not found in the input `df`
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: A grouped dataframe with an extra column,
|
||||
`Contract Effective Date_corrected`
|
||||
"""
|
||||
|
||||
if not isinstance(df, pd.DataFrame):
|
||||
raise TypeError(f"Expected a dataframe, got {type(df).__name__}")
|
||||
|
||||
if "Contract Effective Date" not in df.columns:
|
||||
raise KeyError(
|
||||
f"Column 'Contract Effective Date' not found in DataFrame. Available columns are: {list(df.columns)}"
|
||||
)
|
||||
|
||||
contract_effective_date = df['Contract Effective Date'].dropna().unique().tolist()[0] if not df['Contract Effective Date'].dropna().empty else None # previous run's contract effective date (if any)
|
||||
|
||||
payer_name = df["PAYER NAME"].dropna().unique().tolist()
|
||||
|
||||
if len(payer_name) == 0:
|
||||
payer_name = ""
|
||||
else:
|
||||
payer_name = payer_name[0]
|
||||
|
||||
is_meridian = any(re.search(r'\b' + keyword + r'\b', payer_name, re.IGNORECASE) for keyword in ['meridian'])
|
||||
|
||||
if string_funcs.is_empty(contract_effective_date): # Try prompting (with smart chunking) when the contract effective date is missing
|
||||
try:
|
||||
answer,d,page_list,claude_answer_raw = contract_effective_date_fix(file_name,text_dict,is_meridian)
|
||||
except Exception as e:
|
||||
# print(f"Error processing {file_name}, got error: {e}")
|
||||
answer,d,page_list,claude_answer_raw = "N/A",{},[],""
|
||||
try:
|
||||
corrected_date = postprocessing_funcs.convert_to_us_date_format(str(answer)) # apply date formatting fix
|
||||
except postprocessing_funcs.InvalidDateException as e:
|
||||
corrected_date = "N/A"
|
||||
else:
|
||||
corrected_date = contract_effective_date # don't correct format when we already have an answer
|
||||
d,page_list,claude_answer_raw = {},[],""
|
||||
|
||||
|
||||
|
||||
df["Contract Effective Date_corrected"] = corrected_date
|
||||
df['Log Information'] = str(d)
|
||||
df['Pages selected'] = str(page_list)
|
||||
df["LLM Justification"] = claude_answer_raw
|
||||
|
||||
return df
|
||||
|
||||
|
||||
#### NPI ####
|
||||
def clean_npi(abc_df, filename, text_dict, top_sheet_dict ):
|
||||
npi_answers = npi_hotfix(text_dict=text_dict)
|
||||
if npi_answers["NPI"] == "N/A" and npi_answers["NPI_other"] == "N/A":
|
||||
npi_answers = npi_hotfix(top_sheet_dict)
|
||||
|
||||
npi_df = pd.DataFrame([npi_answers])
|
||||
npi_df.columns = [col + '_corrected' for col in npi_df.columns]
|
||||
npi_df["Contract Name"] = str(filename)
|
||||
|
||||
df_merged = pd.merge(abc_df, npi_df, on='Contract Name', how='left')
|
||||
df_merged = npi_post_process(df_merged)
|
||||
|
||||
df_merged['NPI_corrected'] = df_merged['dummy'].fillna(df_merged['NPI_corrected'])
|
||||
if 'dummy_other' in df_merged.columns:
|
||||
df_merged['NPI_other_corrected'] = df_merged['dummy_other'].fillna(df_merged['NPI_other_corrected'])
|
||||
df_merged = df_merged.loc[:, ~df_merged.columns.str.contains('dummy', na=False)]
|
||||
|
||||
return df_merged
|
||||
|
||||
|
||||
#### TERM GROUP ####
|
||||
def clean_term_group(df: pd.DataFrame,
|
||||
contract_name: str,
|
||||
text_dict: dict[str, str],
|
||||
ac_chunks: dict[str, str]) -> pd.DataFrame:
|
||||
"""this function takes previously generated file as dataframe and one contract name as input. If value for term is null
|
||||
it runs the term group prompts and sets term clause_corrected, auto-renewal_corrected and termination date_columns
|
||||
"""
|
||||
df["Term Clause_corrected"] = df['Term Clause']
|
||||
df["Contract Auto-Renewal Indicator_corrected"] = df['Contract Auto-Renewal Indicator']
|
||||
df["Termination Date_corrected"] = df['Termination Date']
|
||||
|
||||
# identify contracts where term is blank
|
||||
term_df = df[(df['Term Clause'].isna())|(df['Term Clause'].str.len() < 10)]
|
||||
term_df['Filename'] = term_df['Contract Name']
|
||||
contract_list = term_df['Filename'].to_list()
|
||||
contract_list = list(set(contract_list))
|
||||
|
||||
# get term group fields and corresponding prompts
|
||||
fields = keywords.GROUPED_KEYWORD_MAPPINGS['term_group']["fields"]
|
||||
fields.sort()
|
||||
questions = {}
|
||||
for field in fields:
|
||||
questions[field] = prompts.AC_DICT[field]
|
||||
|
||||
# process the contract
|
||||
if contract_name in contract_list:
|
||||
ac_answers_dict = {}
|
||||
context = ac_chunks['term_group']
|
||||
context = context[0 : min(100000, len(context)) - 1]
|
||||
prompt = prompts.AC_MULTI_FIELD_TEMPLATE(context, questions)
|
||||
field_group_answer_raw = claude_funcs.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192
|
||||
)
|
||||
field_group_answer_raw = field_group_answer_raw.replace("_DATE", "_DT")
|
||||
field_group_answer_dict = string_funcs.json_parsing_search(field_group_answer_raw, fields)
|
||||
|
||||
for field in fields:
|
||||
if field in field_group_answer_dict:
|
||||
field_answer = field_group_answer_dict[field]
|
||||
ac_answers_dict[field] = field_answer
|
||||
else:
|
||||
ac_answers_dict[field] = field_answer
|
||||
|
||||
ac_df = pd.DataFrame([ac_answers_dict])
|
||||
ac_df = postprocessing_funcs.clean_term_clause(ac_df)
|
||||
ac_df = postprocessing_funcs.clean_auto_renewal_ind(ac_df)
|
||||
|
||||
if 'TERM_CLAUSE' in ac_df.columns:
|
||||
df.loc[df['Contract Name'] == contract_name, 'Term Clause_corrected'] = ac_df['TERM_CLAUSE'].iloc[0]
|
||||
if 'CONTRACT_AUTO_RENEWAL_IND' in ac_df.columns:
|
||||
df.loc[df['Contract Name'] == contract_name, 'Contract Auto-Renewal Indicator_corrected'] = ac_df['CONTRACT_AUTO_RENEWAL_IND'].iloc[0]
|
||||
if 'CONTRACT_TERMINATION_DT' in ac_df.columns:
|
||||
df.loc[df['Contract Name'] == contract_name, 'Termination Date_corrected'] = ac_df['CONTRACT_TERMINATION_DT'].iloc[0]
|
||||
|
||||
return df
|
||||
|
||||
#### Agreement Name ####
|
||||
# ensure that prompt for CONTRACT_TITLE is updated and df contains field Agreement Name (Contract Title)_corrected
|
||||
def clean_agreement_name(df: pd.DataFrame,
|
||||
contract_name: str,
|
||||
text_dict: dict[str, str]) -> pd.DataFrame:
|
||||
"""this function first runs agreement name prompt on first 5 pages.
|
||||
If agreement name is not found, it runs on the context selected based on keywords.
|
||||
"""
|
||||
|
||||
df["Agreement_Name (Contract Title)_corrected"] = df['Agreement_Name (Contract Title)']
|
||||
agreement_df = df[(df['Agreement_Name (Contract Title)'].isna())|(df['Agreement_Name (Contract Title)'].astype(str).str.len() < 5)]
|
||||
agreement_df['Filename'] = agreement_df['Contract Name']
|
||||
contract_list = agreement_df['Filename'].to_list()
|
||||
contract_list = list(set(contract_list))
|
||||
|
||||
# process the contract
|
||||
if contract_name in contract_list:
|
||||
contract_title_chunk_dict = dict(itertools.islice(text_dict.items(), 5))
|
||||
question = prompts.AC_DICT["CONTRACT_TITLE"]
|
||||
|
||||
context = "\n".join(
|
||||
[contract_title_chunk_dict[str(page_num)] for page_num in contract_title_chunk_dict.keys()]
|
||||
)
|
||||
|
||||
prompt = prompts.AC_SINGLE_FIELD_TEMPLATE(context, question)
|
||||
prompt_answer_raw = claude_funcs.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192)
|
||||
|
||||
if prompt_answer_raw == "N/A":
|
||||
agreement_keywords = ["AGREEMENT", "AMENDMENT", "ADDENDUM", "PROVIDER", "CONTRACT", "MEMORANDUM", "LETTER", "REQUEST", "EFFECTIVE"]
|
||||
page_list = smart_chunking_funcs.chunk_hierarchical(
|
||||
text_dict, agreement_keywords, False
|
||||
)
|
||||
context = "\n".join(
|
||||
[text_dict[str(page_num)] for page_num in page_list]
|
||||
)
|
||||
prompt = prompts.AC_SINGLE_FIELD_TEMPLATE(context, question)
|
||||
prompt_answer_raw = claude_funcs.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192)
|
||||
|
||||
df.loc[df['Contract Name'] == contract_name, 'Agreement_Name (Contract Title)_corrected'] = prompt_answer_raw
|
||||
|
||||
return df
|
||||
|
||||
|
||||
HOTFIX_ORDER = [
|
||||
"Contract Name",
|
||||
"Agreement_Name (Contract Title)",
|
||||
"PAYER NAME",
|
||||
"Health Plan State",
|
||||
"Affiliate (Y/N)",
|
||||
"Credentialing Application Indicator",
|
||||
"Term Clause",
|
||||
"Contract Auto-Renewal Indicator",
|
||||
"Termination Date",
|
||||
"Termination Upon Notice - Days",
|
||||
"Termination With Cause - Days",
|
||||
"Non-Renewal Language",
|
||||
"Non-Renewal - Days",
|
||||
"Amend Contract Upon Notice Flag (Y/N)",
|
||||
"Timeframe to Object - Days",
|
||||
"Assignments Clause (Y/N)",
|
||||
"Contract Effective Date",
|
||||
"IRS #",
|
||||
"IRS Name",
|
||||
"NPI (10-digits)",
|
||||
"NPI Name",
|
||||
"PROV_GROUP_TIN_SIGNATORY",
|
||||
"PROV_TIN_OTHER",
|
||||
"PROV_NPI_OTHER",
|
||||
"Notice to Provider Name",
|
||||
"Notice to Provider Address",
|
||||
"Sequestration Language",
|
||||
"Sequestration Reductions (Y/N)",
|
||||
"Parent Agreement Code",
|
||||
"Pages",
|
||||
"Page_Num",
|
||||
"Attachment/Exhibit",
|
||||
"Line of Business",
|
||||
"Provider Type",
|
||||
"Provider Type - Level 2",
|
||||
"IP/OP",
|
||||
"Service Type",
|
||||
"Plan Type",
|
||||
"Lesser of Logic language, included (Y/N)",
|
||||
"Lesser of Rate",
|
||||
"Reimb. Methodology",
|
||||
"Reimb. Methodology_Short",
|
||||
"If rate is % of Payor or MCR [STANDARD]",
|
||||
'If rate is % of Payor or MCR [STANDARD]_Short',
|
||||
"FLAT FEE",
|
||||
"Default Term",
|
||||
"Default Rate",
|
||||
'Inclusion of Essential RBRVS "Fee Source" Language (Y/N)',
|
||||
'CDM Neutralization Language, included (Y/N)',
|
||||
"CONTRACT_CHARGEMASTER_PROTECTION_LANGUAGE",
|
||||
"IP - DSH/IME/UC, included (Y/N)",
|
||||
"IP - Stoploss Catastrophic Threshold",
|
||||
"Exclusions",
|
||||
"Not to Exceed",
|
||||
"Escalator or COLA (Y/N)",
|
||||
"Escalator I, Eff. Date",
|
||||
"Delegated Function Indicator",
|
||||
"Delegated Terms",
|
||||
"ECM",
|
||||
"National Agreement Indicator",
|
||||
"Cost Settlement (Y/N)",
|
||||
"Cost Settlement (Language)",
|
||||
"Late Paid Claims (Y/N)",
|
||||
"Late Paid Claims (Language)",
|
||||
"Deemer Amendment",
|
||||
"Regulatory Requirements",
|
||||
"Recovery Rights",
|
||||
"Arbitration and Disputes",
|
||||
"Exclusivity Requirement (Y/N)",
|
||||
"Exclusivity Requirement (Language)",
|
||||
"Payor",
|
||||
"Participation in Products",
|
||||
"Clean Claim",
|
||||
"Independent Review (Y/N)",
|
||||
"Independent Review (Language)",
|
||||
"Indemnification",
|
||||
"Access to Medical Records",
|
||||
"Member Confinement Days Language (Y/N)",
|
||||
"Member Confinement Days Language (Language)",
|
||||
"Network Access Fees (Y/N)",
|
||||
"Network Access Fees (Language)",
|
||||
"Payment in Advance of Claims Submission Language (Y/N)",
|
||||
"Payment in Advance of Claims Submission Language",
|
||||
"Eligibility Verification",
|
||||
"Preauthorization",
|
||||
"Policies and Procedures",
|
||||
"Insurance Requirement",
|
||||
"Carve-Out Vendors",
|
||||
"Conflicts Between Certain Documents (Y/N)",
|
||||
"Conflicts Between Certain Documents (Language)",
|
||||
"Relationship of Parties (Y/N)",
|
||||
"Relationship of Parties (Language)",
|
||||
"Nonstandard Appeals Process (Y/N)",
|
||||
"Nonstandard Appeals Process (Language)",
|
||||
"Product Removal",
|
||||
"Disparagement Prohibition (Y/N)",
|
||||
"Disparagement Prohibition (Language)",
|
||||
"Claims Editing Language (Y/N)",
|
||||
"Claims Editing Language (Language)",
|
||||
"Guarantee of Provider Yield (Y/N)",
|
||||
"Guarantee of Provider Yield (Language)",
|
||||
"HCBS Services",
|
||||
"Add On Reimbursement (Y/N)",
|
||||
"Add On Reimbursement (Language)",
|
||||
"PMPM",
|
||||
"Single Code Multiple Rates (Y/N)",
|
||||
"Single Code Multiple Rates (Language)",
|
||||
"Invoice Pricing (Y/N)",
|
||||
"Invoice Pricing (Language)",
|
||||
"Medical Necessity Language (Y/N)",
|
||||
"Medical Necessity Language (Language)",
|
||||
"Template",
|
||||
"Provider-Based Billing Exclusion (Y/N)",
|
||||
"Provider-Based Billing Exclusion (Language)",
|
||||
"Agreement Name (Contract Title)_corrected",
|
||||
"Health Plan State_corrected",
|
||||
"Term Clause_corrected",
|
||||
"Contract Auto-Renewal Indicator_corrected",
|
||||
"Termination Date_corrected",
|
||||
"Contract Effective Date_corrected",
|
||||
"IRS_corrected",
|
||||
"PROV_TIN_OTHER_corrected",
|
||||
"PROV_TIN_GROUP_SIGNATORY_corrected",
|
||||
"NPI_corrected",
|
||||
"NPI_other_corrected",
|
||||
"LOB_Correct",
|
||||
"PROV_TYPE_LEVEL_2_Corrected",
|
||||
"Lesser_Flag",
|
||||
"Service-Methodology",
|
||||
"Default Term Corrected",
|
||||
"Default Rate Corrected",
|
||||
"FULL_METHODOLOGY_fixed",
|
||||
"LESSER_fixed",
|
||||
"RATE_STANDARD_fixed",
|
||||
"RATE_SHORT_fixed",
|
||||
"FLAT_FEE_STANDARD_fixed",
|
||||
"LESSER_RATE_fixed",
|
||||
"NOT_TO_EXCEED_fixed",
|
||||
"SHORT_METHODOLOGY_fixed",
|
||||
"Attachment/Exhibit_fixed",
|
||||
"Attachment/Exhibit_page",
|
||||
"_merge",
|
||||
"key1_fixed",
|
||||
"key2_fixed",
|
||||
"key3_fixed"]
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
import re
|
||||
|
||||
import smart_chunking_funcs
|
||||
|
||||
regex_backticks = r"\|([^|]+)\|"
|
||||
|
||||
|
||||
# TODO: leave this one in here; don't move it to prompts until we get confirmation that we want to handle meridian differently.
|
||||
def get_effective_date_meridian_prompt(context):
|
||||
prompt_template = f""""
|
||||
Please analyze the following contract and extract ONLY the Signature Date. Follow these precise rules in order:
|
||||
|
||||
1. Look for Signature Date
|
||||
|
||||
2. If multiple matches are found:
|
||||
- Prioritize the latest full date
|
||||
|
||||
3. If only partial dates are found, return N/A.
|
||||
|
||||
4. Return the date in YYYY-MM-DD format or N/A if date not found. Enclose just your final answer in |pipes|.
|
||||
|
||||
Here is the contract to analyze:
|
||||
|
||||
## START CONTRACT TEXT ##
|
||||
|
||||
{context}
|
||||
|
||||
## END CONTRACT TEXT ##
|
||||
|
||||
Before returning the output, VERIFY the date format is YYYY-MM-DD. If the date is not found, return N/A.
|
||||
|
||||
Finally state "Answer: |YYYY-MM-DD| or |N/A|"
|
||||
"""
|
||||
|
||||
return prompt_template
|
||||
|
||||
|
||||
def chunk_with_include_exclude_keywords(
|
||||
text_dict: dict[str, str],
|
||||
include_keywords: list[str],
|
||||
exclude_keywords: list[str],
|
||||
case_sensitive: bool = False,
|
||||
apply_date_filtering: bool = True,
|
||||
):
|
||||
"""Look for pages that include any of the include_keywords and exclude any pages that contain any of the exclude_keywords.
|
||||
|
||||
Args:
|
||||
text_dict (dict[str, str]): A dictionary keyed by string page number and valued
|
||||
by string page contents
|
||||
include_keywords (list[str]): A list of string keywords to include
|
||||
exclude_keywords (list[str]): A list of string keywords to exclude
|
||||
case_sensitive (bool): A flag for converting all keywords and text to lower case,
|
||||
in order to ignore case altogether.
|
||||
apply_date_filtering (bool): A flag to filter out the pages not containing a date.
|
||||
|
||||
Returns:
|
||||
list[str]: The sorted list of string page numbers that include all include_keywords and exclude any exclude_keywords
|
||||
"""
|
||||
|
||||
if not case_sensitive:
|
||||
include_keywords = [keyword.lower() for keyword in include_keywords]
|
||||
exclude_keywords = [keyword.lower() for keyword in exclude_keywords]
|
||||
text_dict = {key: value.lower() for key, value in text_dict.items()}
|
||||
|
||||
if apply_date_filtering:
|
||||
pages_containing_date = set()
|
||||
|
||||
for page_num, page_content in text_dict.items():
|
||||
dates_list = re.findall(smart_chunking_funcs.DATE_PATTERN, page_content)
|
||||
|
||||
if len(dates_list) > 0:
|
||||
pages_containing_date.add(page_num)
|
||||
|
||||
pages_containing_date = sorted(pages_containing_date, key=int)
|
||||
|
||||
# debug lines for inclusion and exclusion keywords found on each page
|
||||
|
||||
# print(f"Pages containing date: {pages_containing_date}")
|
||||
|
||||
d = list() # to record details
|
||||
|
||||
for page_num, page_content in text_dict.items():
|
||||
if page_num in pages_containing_date:
|
||||
inclusion_keywords = set()
|
||||
exclusion_keywords = set()
|
||||
|
||||
contains_inclusion_keyword = False
|
||||
|
||||
for keyword in include_keywords:
|
||||
contains_inclusion_keyword = True
|
||||
if keyword in page_content:
|
||||
inclusion_keywords.add(keyword)
|
||||
|
||||
for keyword in exclude_keywords:
|
||||
if keyword in page_content:
|
||||
exclusion_keywords.add(keyword)
|
||||
|
||||
# print(f"Page #: {page_num} contains inclusion keywords: {inclusion_keywords} and contain exclusion keywords: {exclusion_keywords} | Inclusion Flag: {contains_inclusion_keyword}")
|
||||
|
||||
if contains_inclusion_keyword:
|
||||
d.append(
|
||||
{
|
||||
"Page Number": page_num,
|
||||
"Inclusion Keywords": inclusion_keywords,
|
||||
"Exclusion Keywords": exclusion_keywords,
|
||||
}
|
||||
)
|
||||
|
||||
# print(f"Page #: {page_num} contains inclusion keywords: {inclusion_keywords} and contain exclusion keywords: {exclusion_keywords}")
|
||||
|
||||
page_list = []
|
||||
|
||||
for page in text_dict.keys():
|
||||
if not apply_date_filtering or (
|
||||
apply_date_filtering and page in pages_containing_date
|
||||
):
|
||||
if any(
|
||||
keyword in text_dict[page] for keyword in include_keywords
|
||||
) and not any(keyword in text_dict[page] for keyword in exclude_keywords):
|
||||
# if str(int(page) - 1) in text_dict.keys():
|
||||
# page_list.append(str(int(page) - 1))
|
||||
# print(f"Taking page #: {str(int(page) - 1)} (as buffer for following page)")
|
||||
|
||||
page_list.append(page)
|
||||
# print(f"Taking page #: {page}")
|
||||
|
||||
# if str(int(page) + 1) in text_dict.keys():
|
||||
# page_list.append(str(int(page) + 1))
|
||||
# # print(f"Taking page #: {str(int(page) + 1)} (as buffer for previous page)")
|
||||
|
||||
page_list_sorted = sorted([int(page_str) for page_str in set(page_list)])
|
||||
page_list_sorted_str = list(map(str, page_list_sorted))
|
||||
|
||||
# print(", ".join([page for page in page_list_sorted_str]))
|
||||
|
||||
# print(f"Obtained total pages: {len(page_list_sorted_str)}")
|
||||
|
||||
return page_list_sorted_str, d
|
||||
@@ -0,0 +1,346 @@
|
||||
import ast
|
||||
import re
|
||||
|
||||
import pandas as pd
|
||||
|
||||
import postprocessing_funcs
|
||||
from regex_funcs import find_regex_matches
|
||||
from valid import STATE_MAP
|
||||
|
||||
|
||||
def convert_to_list(value):
|
||||
if isinstance(value, str):
|
||||
# Remove any trailing commas or spaces
|
||||
value = value.strip().replace(',', ' ') # Replace commas with spaces
|
||||
|
||||
# Check if the string is in the form of a list and safely convert it
|
||||
if value.startswith('[') and value.endswith(']'):
|
||||
try:
|
||||
return ast.literal_eval(value) # Safely convert string to list
|
||||
except:
|
||||
return value.split() # Fallback to split by spaces if literal_eval fails
|
||||
else:
|
||||
# Split by spaces and handle both alphanumeric and numeric values
|
||||
return value.split() # Just split by spaces, treat everything as a string
|
||||
elif isinstance(value, list):
|
||||
return value # If it's already a list, return as is
|
||||
else:
|
||||
return [value] # Wrap non-string, non-list types in a list
|
||||
|
||||
|
||||
def list_to_string(lst):
|
||||
if not lst: # Check if the list is empty
|
||||
return None
|
||||
else:
|
||||
return ", ".join(lst) # Join elements with commas
|
||||
|
||||
def remove_alphabets(input_str):
|
||||
# Check if the string contains any alphabetic characters
|
||||
if any(char.isalpha() for char in input_str):
|
||||
return None # Return None if there are alphabets
|
||||
else:
|
||||
return input_str # Return the string if no alphabets are present
|
||||
|
||||
|
||||
def clean_output_postprocess(merged_df):
|
||||
merged_df['dummy'] = merged_df['IRS #']
|
||||
merged_df['dummy'] = merged_df['dummy'].astype(str)
|
||||
merged_df['dummy'] = merged_df['dummy'].apply(convert_to_list)
|
||||
merged_df['dummy'] = merged_df['dummy'].apply(lambda lst: [x for x in [remove_alphabets(x) for x in lst] if x is not None])
|
||||
merged_df['dummy'] = merged_df['dummy'].apply(lambda lst: [x for x in [postprocessing_funcs.add_hyphen_if_needed(x) for x in lst] if x is not None])
|
||||
merged_df['dummy'] = merged_df['dummy'].apply(list_to_string)
|
||||
|
||||
if 'PROV_TIN_OTHER' in merged_df.columns:
|
||||
merged_df['dummy_other'] = merged_df['PROV_TIN_OTHER']
|
||||
merged_df['dummy_other'] = merged_df['dummy_other'].astype(str)
|
||||
merged_df['dummy_other'] = merged_df['dummy_other'].apply(convert_to_list)
|
||||
merged_df['dummy_other'] = merged_df['dummy_other'].apply(lambda lst: [x for x in [remove_alphabets(x) for x in lst] if x is not None])
|
||||
merged_df['dummy_other'] = merged_df['dummy_other'].apply(lambda lst: [x for x in [postprocessing_funcs.add_hyphen_if_needed(x) for x in lst] if x is not None])
|
||||
merged_df['dummy_other'] = merged_df['dummy_other'].apply(list_to_string)
|
||||
|
||||
if 'PROV_GROUP_TIN_SIGNATORY' in merged_df.columns:
|
||||
merged_df['dummy_other_signatory'] = merged_df["PROV_GROUP_TIN_SIGNATORY"]
|
||||
merged_df['dummy_other_signatory'] = merged_df['dummy_other_signatory'].astype(str)
|
||||
merged_df['dummy_other_signatory'] = merged_df['dummy_other_signatory'].apply(convert_to_list)
|
||||
merged_df['dummy_other_signatory'] = merged_df['dummy_other_signatory'].apply(lambda lst: [x for x in [remove_alphabets(x) for x in lst] if x is not None])
|
||||
merged_df['dummy_other_signatory'] = merged_df['dummy_other_signatory'].apply(lambda lst: [x for x in [postprocessing_funcs.add_hyphen_if_needed(x) for x in lst] if x is not None])
|
||||
merged_df['dummy_other_signatory'] = merged_df['dummy_other_signatory'].apply(list_to_string)
|
||||
|
||||
return merged_df
|
||||
|
||||
|
||||
def check_tin_in_filename(filename):
|
||||
"""
|
||||
Checks for the pattern match in filename
|
||||
|
||||
Args:
|
||||
filename(str): Name of the file
|
||||
Returns:
|
||||
tin_from_title(str): regex match for patterns in filename
|
||||
"""
|
||||
patterns = [r"\b\d{2}-\d{7}\b", r"\b\d{9}\b"]
|
||||
for pattern in patterns:
|
||||
tin_from_title = re.findall(pattern, filename)
|
||||
if len(tin_from_title) == 1:
|
||||
tin_from_title = [postprocessing_funcs.add_hyphen_if_needed(s) for s in tin_from_title]
|
||||
return tin_from_title[0]
|
||||
return 'empty'
|
||||
|
||||
|
||||
def signature_page_irs(text_dict, page_list, matches, pattern, filename):
|
||||
"""
|
||||
In case of multiple matches, checks if a TIN is on the Signature page
|
||||
|
||||
Args:
|
||||
text_dict (dict): Dictionary keyed by string page-number and valued by actual textract output page
|
||||
page_list(list): List of pages where the match was found
|
||||
matches(list): List of regex match(s)
|
||||
Returns
|
||||
provider_tin(str): Provider group TIN - either from signature page or filename
|
||||
other_tins(str): Remaining TINs
|
||||
"""
|
||||
# loops through regex matches and thier pages to check for signature page
|
||||
tin_on_signature_page = []
|
||||
for tin in matches:
|
||||
for page in page_list:
|
||||
if tin in text_dict[page] and 'signature' in text_dict[page]:
|
||||
tin_on_signature_page.append(tin)
|
||||
tin_on_signature_page = list(set(tin_on_signature_page))
|
||||
|
||||
#adds hyphen if needed
|
||||
matches = [postprocessing_funcs.add_hyphen_if_needed(s) for s in matches]
|
||||
tin_on_signature_page = [postprocessing_funcs.add_hyphen_if_needed(s) for s in tin_on_signature_page]
|
||||
|
||||
#case1 - If 1 TIN found on signature page, return it, put the remaining ones in "TIN Others" have signature_tin = N/A
|
||||
if len(tin_on_signature_page) == 1:
|
||||
provider_tin = tin_on_signature_page[0]
|
||||
signature_tin = "N/A"
|
||||
matches = [item for item in matches if item not in tin_on_signature_page]
|
||||
other_tins = ", ".join(matches)
|
||||
return provider_tin, signature_tin, other_tins
|
||||
|
||||
#case2 - No TIN found on the signature page, check in filename
|
||||
elif len(tin_on_signature_page) == 0:
|
||||
provider_tin = check_tin_in_filename(filename)
|
||||
if provider_tin == 'empty':
|
||||
provider_tin = "N/A"
|
||||
signature_tin = "N/A"
|
||||
other_tins = [item for item in matches if item != provider_tin]
|
||||
other_tins = ", ".join(other_tins)
|
||||
return provider_tin, signature_tin, other_tins
|
||||
|
||||
#case3 - More than 1 TIN on signature page - check for TIN in file name, let the others go in signatory_tins
|
||||
# and finally check for TINs not on signature page but in "matches" list to add them to "tin_others"
|
||||
else:
|
||||
filename_tin_check = check_tin_in_filename(filename)
|
||||
if filename_tin_check == 'empty':
|
||||
provider_tin = "N/A"
|
||||
signature_tin = ", ".join(tin_on_signature_page)
|
||||
other_tins = "N/A"
|
||||
return provider_tin, signature_tin, other_tins
|
||||
else:
|
||||
provider_tin = filename_tin_check
|
||||
signature_tin = [item for item in tin_on_signature_page if item != provider_tin]
|
||||
signature_tin = ", ".join(signature_tin)
|
||||
|
||||
if len(matches)> len(tin_on_signature_page):
|
||||
other_tins = [item for item in matches if item not in tin_on_signature_page and item != provider_tin]
|
||||
other_tins = ", ".join(other_tins)
|
||||
return provider_tin, signature_tin, other_tins
|
||||
else:
|
||||
other_tins = "N/A"
|
||||
return provider_tin, signature_tin, other_tins
|
||||
|
||||
|
||||
def irs_hotfix(filename, text_dict: dict[str, str]):
|
||||
"""Returns dictionery with Provider and other TIN(s) based on regex matches and presence on Signatory page
|
||||
|
||||
Args:
|
||||
filename: Name of the contract
|
||||
text_dict (dict): Dictionary keyed by string page-number and valued by actual textract output page
|
||||
|
||||
Returns:
|
||||
TIN_dict(dict): A dictionery with keys 'PROV_GROUP_TIN' & 'PROV_TIN_OTHER' and values as respective TIN(s)
|
||||
"""
|
||||
TIN_dict = {}
|
||||
|
||||
patterns = [r"\b\d{2}-\d{7}\b",
|
||||
r"\b\d{9}\b"]
|
||||
for pattern in patterns:
|
||||
matches, page_list = find_regex_matches(pattern, text_dict)
|
||||
matches = list(set(matches))
|
||||
if matches:
|
||||
if len(matches) == 1:
|
||||
matches = [postprocessing_funcs.add_hyphen_if_needed(s) for s in matches]
|
||||
TIN_dict["PROV_GROUP_TIN"] = matches[0]
|
||||
TIN_dict["PROV_TIN_OTHER"] = "N/A"
|
||||
TIN_dict["PROV_TIN_GROUP_SIGNATORY"] = "N/A"
|
||||
return TIN_dict
|
||||
|
||||
elif len(matches) >= 1:
|
||||
irs_tin, signatory_tin, other_tins = signature_page_irs(text_dict, page_list, matches, pattern, filename)
|
||||
TIN_dict["PROV_GROUP_TIN"] = irs_tin
|
||||
TIN_dict["PROV_TIN_OTHER"] = other_tins
|
||||
TIN_dict["PROV_TIN_GROUP_SIGNATORY"] = signatory_tin
|
||||
return TIN_dict
|
||||
|
||||
# If no matches were found in the text_dict, check the filename
|
||||
for pattern in patterns:
|
||||
tin_from_title = check_tin_in_filename(filename)
|
||||
TIN_dict["PROV_TIN_OTHER"] = "N/A"
|
||||
TIN_dict["PROV_TIN_GROUP_SIGNATORY"] = "N/A"
|
||||
if tin_from_title == "empty":
|
||||
TIN_dict["PROV_GROUP_TIN"] = "N/A"
|
||||
else:
|
||||
TIN_dict["PROV_GROUP_TIN"] = tin_from_title
|
||||
return TIN_dict
|
||||
|
||||
def check_field_for_matches(text, valid_lists):
|
||||
"""
|
||||
Check a field for matches from the valid lists using case-insensitive regex
|
||||
Returns tuple of (matched_term, list_type) or (None, None)
|
||||
"""
|
||||
if pd.isna(text) or text == "":
|
||||
return None, None
|
||||
text = str(text).lower()
|
||||
for list_type, valid_list in valid_lists.items():
|
||||
for term in valid_list:
|
||||
if re.search(rf'\b{re.escape(term.lower())}\b', text):
|
||||
return term, list_type
|
||||
return None, None
|
||||
|
||||
##### NPI #####
|
||||
def find_10_digit_numbers(row):
|
||||
matches = re.findall(r'\b\d{10}\b', row)
|
||||
return matches
|
||||
|
||||
|
||||
def signature_page_npi(text_dict, page_list, matches):
|
||||
npi_on_signature_page = []
|
||||
for npi in matches:
|
||||
for page in page_list:
|
||||
if npi in text_dict[page] and 'signature' in text_dict[page]:
|
||||
npi_on_signature_page.append(npi)
|
||||
npi_on_signature_page = list(set(npi_on_signature_page))
|
||||
|
||||
if len(npi_on_signature_page) == 1:
|
||||
npi = npi_on_signature_page[0]
|
||||
npi_other = [item for item in matches if item not in npi_on_signature_page]
|
||||
npi_other = ", ".join(npi_other)
|
||||
return npi, npi_other
|
||||
elif len(npi_on_signature_page) == 0:
|
||||
npi = "N/A"
|
||||
npi_other = ", ".join(matches)
|
||||
return npi, npi_other
|
||||
|
||||
else:
|
||||
npi = ", ".join(npi_on_signature_page)
|
||||
npi_other = [item for item in matches if item not in npi_on_signature_page]
|
||||
if len(npi_other) == 0:
|
||||
npi_other = "N/A"
|
||||
return npi, npi_other
|
||||
else:
|
||||
npi_other = ", ".join(npi_other)
|
||||
return npi, npi_other
|
||||
|
||||
|
||||
def npi_hotfix(text_dict: dict[str, str]):
|
||||
"""Returns dictionery with Provider and other TIN(s) based on regex matches and presence on Signatory page
|
||||
|
||||
Args:
|
||||
filename: Name of the contract
|
||||
text_dict (dict): Dictionary keyed by string page-number and valued by actual textract output page
|
||||
|
||||
Returns:
|
||||
TIN_dict(dict): A dictionery with keys 'PROV_GROUP_TIN' & 'PROV_TIN_OTHER' and values as respective TIN(s)
|
||||
"""
|
||||
NPI_dict = {}
|
||||
|
||||
patterns = [r"\b\d{10}\b"]
|
||||
|
||||
for pattern in patterns:
|
||||
matches, page_list = find_regex_matches(pattern, text_dict)
|
||||
matches = [s for s in matches if s[0] in ('1', '2')]
|
||||
matches = list(set(matches))
|
||||
if len(matches)==1:
|
||||
NPI_dict["NPI"] = matches[0]
|
||||
NPI_dict["NPI_other"] = "N/A"
|
||||
elif len(matches) == 0:
|
||||
NPI_dict["NPI"] = "N/A"
|
||||
NPI_dict["NPI_other"] = "N/A"
|
||||
else:
|
||||
npi, npi_other = signature_page_npi(text_dict, page_list, matches)
|
||||
NPI_dict["NPI"] = npi
|
||||
NPI_dict["NPI_other"] = npi_other
|
||||
return NPI_dict
|
||||
|
||||
|
||||
##################################################################################################################################
|
||||
|
||||
# remove '-' from numbers and drop numbers with more than 2 '-' in them
|
||||
def remove_dashes(lst):
|
||||
processed_list = []
|
||||
for item in lst:
|
||||
# If the item contains more than 2 dashes, remove it
|
||||
if item.count('-') > 2:
|
||||
continue
|
||||
else:
|
||||
# Remove dashes and keep the digits only
|
||||
processed_list.append(item.replace('-', ''))
|
||||
|
||||
return processed_list
|
||||
|
||||
##################################################################################################################################
|
||||
|
||||
def filter_elements_by_length(lst):
|
||||
return [item for item in lst if 7 <= len(item) <= 10]
|
||||
|
||||
##################################################################################################################################
|
||||
# Function to filter out non-digit elements from a list
|
||||
def filter_non_digits(lst):
|
||||
return [item for item in lst if item.isdigit()] # Only keep items that are digits
|
||||
|
||||
##################################################################################################################################
|
||||
|
||||
def filter_start_with_1_or_2(lst):
|
||||
return [item for item in lst if item.startswith('1') or item.startswith('2')]
|
||||
|
||||
|
||||
##################################################################################################################################
|
||||
def convert_to_string(value):
|
||||
if pd.isna(value):
|
||||
# If the value is NaN, return it as is
|
||||
return str(value)
|
||||
elif isinstance(value, float):
|
||||
# If it's a float, convert to int and then to string to avoid decimal point
|
||||
return str(int(value)) if value == value else 'nan'
|
||||
return str(value)
|
||||
##################################################################################################################################
|
||||
|
||||
def npi_post_process(df_merged):
|
||||
df_merged['dummy'] = df_merged['NPI (10-digits)']
|
||||
df_merged['dummy'] = df_merged['dummy'].apply(convert_to_string)
|
||||
df_merged['dummy'] = df_merged['dummy'].apply(convert_to_list)
|
||||
df_merged['dummy'] = df_merged['dummy'].apply(remove_dashes)
|
||||
df_merged['dummy'] = df_merged['dummy'].apply(filter_elements_by_length)
|
||||
df_merged['dummy'] = df_merged['dummy'].apply(filter_non_digits)
|
||||
df_merged['dummy'] = df_merged['dummy'].apply(filter_start_with_1_or_2)
|
||||
df_merged['dummy'] = df_merged['dummy'].apply(list_to_string)
|
||||
|
||||
if 'PROV_NPI_OTHER' in df_merged.columns:
|
||||
df_merged['dummy_other'] = df_merged["PROV_NPI_OTHER"]
|
||||
df_merged['dummy_other'] = df_merged['dummy_other'].astype(str)
|
||||
df_merged['dummy_other'] = df_merged['dummy_other'].apply(convert_to_list)
|
||||
df_merged['dummy_other'] = df_merged['dummy_other'].apply(remove_dashes)
|
||||
df_merged['dummy_other'] = df_merged['dummy_other'].apply(filter_elements_by_length)
|
||||
df_merged['dummy_other'] = df_merged['dummy_other'].apply(filter_non_digits)
|
||||
df_merged['dummy_other'] = df_merged['dummy_other'].apply(filter_start_with_1_or_2)
|
||||
df_merged['dummy_other'] = df_merged['dummy_other'].apply(list_to_string)
|
||||
|
||||
return df_merged
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user