Merged in cnc_fl_main (pull request #258)

Cnc fl main

* prov type moved

* comment out invalid prov2 bc it's overly restrictive

* Update prompt template to ensure N/A rather than blanks

* Update AC multi-prompt template. Update prompts to ensure all keys are present and N/As are filled

* Update AC postprocessing

* Update clean_tin() to add N/A for values other than 10 digits

* update tin postprocessing

* Update npi postprocessing

* Add 'Filename:' to the start of all filenames

* Update group TIN/NPI postprocessing

* Update parent agreement code

* fix health plan state mapping

* Update B postprocessing

* Updated indicators, re-ordering

* Update postprocessing - all columns included and reordered

* Ensure Pages is included in AC-only output

* Remove excess print statements

* remove example_test to pass pipeline

* Fix parent agreement code for AC

* Updated postprocessing for NPI/TIN other

* Update prompts for N explicitely on AC prompt-based indicators

* Updated main and utils for more intuitive filtering

* Updated file_counting to match new main process

* update test

* Add example_test


Approved-by: Michael McGuinness
This commit is contained in:
Katon Minhas
2024-11-04 22:54:31 +00:00
parent de41567da0
commit bb279b45fd
16 changed files with 935 additions and 381 deletions
+459
View File
@@ -0,0 +1,459 @@
import pandas as pd
import numpy as np
import re
import difflib
import config
import prompts
import valid
from valid import DERIVED_INDICATOR_FIELDS
import claude_funcs
from utils import is_empty
import logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
def sanitize_value(value):
try:
if isinstance(value, list):
return ', '.join(str(v) for v in value)
elif isinstance(value, str):
value = value.strip('[]')
return ', '.join([item.strip(" '") for item in value.split(',')])
elif pd.isna(value):
return "N/A"
except:
return value
def exact_match(val, valid_values):
val = val.strip().upper()
for valid_val in valid_values:
if val == valid_val.upper():
return valid_val
return None
def get_closest_match(val, valid_values, similarity_threshold=0.7):
if pd.isna(val):
return None
val = val.strip().upper()
matches = difflib.get_close_matches(val, [v.upper() for v in valid_values], n=1, cutoff=similarity_threshold)
return matches[0] if matches else None
def correct_misplaced_values(df, columns, valid_values_dict):
for index, row in df.iterrows():
for col in columns:
if pd.notna(row[col]):
terms = row[col].split(',')
for term in terms:
term = term.strip()
for target_col, valid_values in valid_values_dict.items():
if target_col != col:
match = exact_match(term, valid_values)
if match:
if pd.isna(row[target_col]) or not row[target_col].strip():
df.at[index, target_col] = match
df.at[index, col] = None
else:
current_value = row[target_col].strip()
if get_closest_match(match, [current_value], config.FUZZY_MATCH_THRESHOLD) is None:
df.at[index, 'Corrected_' + target_col] = f"Found {term} in {col} cell"
df.at[index, col] = None
return df
def filter_service_column(answer_dicts):
filtered_list = []
for d in answer_dicts:
if 'FULL_SERVICE' not in d:
continue # Skip this dictionary if it doesn't have FULL_SERVICE
clean_dict = True
for keyword in valid.SERVICE_FILTER:
if keyword.upper() in d['FULL_SERVICE'].upper() or d['FULL_SERVICE'].upper() in keyword.upper():
clean_dict = False
break
if clean_dict:
filtered_list.append(d)
return filtered_list
def filter_methodology_column(answer_dicts):
filtered_list = []
for d in answer_dicts:
if 'FULL_METHODOLOGY' not in d:
continue # Skip this dictionary if it doesn't have FULL_METHODOLOGY
clean_dict = True
for keyword in valid.METHODOLOGY_FILTER:
if keyword.upper() in d['FULL_METHODOLOGY'].upper() or d['FULL_METHODOLOGY'].upper() in keyword.upper():
clean_dict = False
break
if clean_dict:
filtered_list.append(d)
return filtered_list
def clean_td(td):
td_clean = []
for d in td:
new_d = {}
for k, v in d.items():
if 'DATE' in k:
new_d[k] = v if isinstance(v, list) else [v]
elif k not in ['page_num', 'Filename']:
if isinstance(v, str) and ',' in v:
new_d[k] = [item.strip() for item in v.split(',')]
elif v == 'N/A':
new_d[k] = []
else:
new_d[k] = [v] if isinstance(v, str) else v
else:
new_d[k] = v
td_clean.append(new_d)
return td_clean
def get_parent_agreement_code(filename):
try:
filename = filename.split('.txt')[0]
match = re.search(r'([^\sa-zA-Z]+)(?=\.\w+$|$)', filename)
end = [i for i in match.group(1).split('_') if i]
return end[0]
except:
return 'N/A'
def consolidate_subheader(dict_list):
modified_list = []
for d in dict_list:
if 'FULL_SERVICE' in d and 'SUBHEADER' in d:
if d['SUBHEADER'] != 'N/A':
d['FULL_SERVICE'] = d['SUBHEADER'] + ' - ' + d['FULL_SERVICE']
# Remove the 'SUBHEADER' key
del d['SUBHEADER']
modified_list.append(d)
return modified_list
def clean_msr_lesser(df, ls=valid.VALID_MSR):
pattern = '|'.join(re.escape(item) for item in ls)
mask = df['FULL_SERVICE'].str.contains(pattern, case=False, na=False)
target_rows = df[mask]
# Iterate over these rows
for index, row in target_rows.iterrows():
# Get all rows with the same 'EXHIBIT' value
exhibit_rows = df[df['EXHIBIT'] == row['EXHIBIT']]
# Check the 'LESSER' values of these rows
if (exhibit_rows['LESSER'] == 'Y').any():
# If any row has 'LESSER' == 'Y', set the 'LESSER' value of the original row to 'Y'
df.at[index, 'LESSER'] = 'Y'
# Move first LESSER_RATE where LESSER==Y to MSR row
lesser_rate = exhibit_rows[exhibit_rows['LESSER'] == 'Y']['LESSER_RATE'].iloc[0]
df.at[index, 'LESSER_RATE'] = lesser_rate
return df
def clean_default_term(df, ls=valid.INVALID_DEFAULT):
# Create a case-insensitive regex pattern that matches any of the values in ls
pattern = '|'.join(re.escape(term) for term in ls)
# Find rows where the DEFAULT_TERM column contains any of the terms from ls
mask = df['DEFAULT_TERM'].str.contains(pattern, case=False, na=False)
# Replace these values with 'N/A'
df.loc[mask, 'DEFAULT_TERM'] = 'N/A'
df.loc[mask, 'DEFAULT_RATE'] = 'N/A'
return df
def clean_lesser_rate(df):
for index, row in df.iterrows():
if 'Y' in row['LESSER'] and is_empty(row['LESSER_RATE']) and not is_empty(row['RATE_STANDARD']):
df.at[index, 'LESSER_RATE'] = row['RATE_STANDARD']
df.at[index, 'RATE_STANDARD'] = 'N/A'
return df
def clean_prov_2(df):
valid_types = valid.select_valid_prov_2(df['PROV_TYPE'])
target_rows = df[df['PROV_TYPE_LEVEL_2'].apply(is_empty)]
def find_exact_match(text):
if pd.isna(text) or text == '':
return None
words = re.findall(r'\b[\w/]+(?:[-\s][\w/]+)*\b', text)
for i in range(len(words)):
for j in range(i+1, len(words)+1):
phrase = ' '.join(words[i:j])
if phrase in valid_types: # Case-sensitive matching
return phrase
return None
for index, row in target_rows.iterrows():
match = None
if not is_empty(row["FULL_SERVICE"]):
match = find_exact_match(str(row["FULL_SERVICE"]))
if match:
logging.info(f"Row {index}: Matched in FULL_SERVICE: {match}")
df.at[index, 'PROV_TYPE_LEVEL_2'] = match
continue
if not is_empty(row["EXHIBIT"]):
match = find_exact_match(str(row["EXHIBIT"]))
if match:
logging.info(f"Row {index}: Matched in EXHIBIT: {match}")
df.at[index, 'PROV_TYPE_LEVEL_2'] = match
continue
exhibit_rows = df[df['EXHIBIT'] == row['EXHIBIT']]
if not exhibit_rows.empty:
for _, exhibit_row in exhibit_rows.iterrows():
if not is_empty(exhibit_row['PROV_TYPE_LEVEL_2']):
match = find_exact_match(str(exhibit_row['PROV_TYPE_LEVEL_2']))
if match:
logging.info(f"Row {index}: Matched in other row's PROV_TYPE_LEVEL_2: {match}")
df.at[index, 'PROV_TYPE_LEVEL_2'] = match
break
elif not is_empty(exhibit_row['FULL_SERVICE']):
match = find_exact_match(str(exhibit_row['FULL_SERVICE']))
if match:
logging.info(f"Row {index}: Matched in other row's FULL_SERVICE: {match}")
df.at[index, 'PROV_TYPE_LEVEL_2'] = match
break
if match is None:
logging.warning(f"Row {index}: No valid match found")
# Final check to ensure no invalid values were assigned
invalid_assignments = df[
(~df['PROV_TYPE_LEVEL_2'].isin(valid_types)) &
(~df['PROV_TYPE_LEVEL_2'].apply(is_empty))
]
if not invalid_assignments.empty:
df.loc[invalid_assignments.index, 'PROV_TYPE_LEVEL_2'] = ''
return df
def clean_ac_fields(final_df):
# additional post-processing
final_df = final_df.fillna('')
# final_df.loc[final_df['CREDENTIALING_APP_IND'] != 'Yes', 'CREDENTIALING_APP_IND'] = "No"
# final_df.loc[final_df['AFFILIATION_CLAUSE_IND'] != 'Yes', 'AFFILIATION_CLAUSE_IND'] = "No"
# final_df.loc[final_df['ASSIGNMENTS_CLAUSE_IND'] != 'Yes', 'ASSIGNMENTS_CLAUSE_IND'] = "No"
if 'TERM_CLAUSE' in final_df:
final_df.loc[final_df['TERM_CLAUSE'].str.startswith('IL-4 Termination'), 'TERM_CLAUSE'] = "N/A"
final_df.loc[final_df['TERM_CLAUSE'].str.startswith('bonus payment shall be effective'), 'TERM_CLAUSE'] = "N/A"
final_df.loc[final_df['TERM_CLAUSE'].str.contains('does not contain'), 'TERM_CLAUSE'] = "N/A"
final_df.loc[final_df['TERM_CLAUSE'].str.contains('No term or termination'), 'TERM_CLAUSE'] = "N/A"
# if 'CONTRACT_SIGNATORY_IND' in final_df and 'PROV_PARTICIPATION_STATUS' in final_df:
# final_df.loc[final_df['CONTRACT_SIGNATORY_IND'] == 'Yes', 'PROV_PARTICIPATION_STATUS'] = "Yes"
if 'CONTRACT_AUTO_RENEWAL_IND' in final_df and 'CONTRACT_TERMINATION_DT' in final_df:
final_df.loc[final_df['CONTRACT_AUTO_RENEWAL_IND'] == 'Yes', 'CONTRACT_TERMINATION_DT'] = np.nan
# check if NPI has 10 digits
if 'PROV_GROUP_NPI' in final_df:
final_df['PROV_GROUP_NPI'] = final_df['PROV_GROUP_NPI'].map(lambda x: x if sum(c.isdigit() for c in x+' ') == 10 else '')
# NETWORK_ACCESS_FEES_IND
if 'NETWORK_ACCESS_FEES_IND' in final_df:
final_df.loc[~final_df['NETWORK_ACCESS_FEES_IND'].isin(['N/A', 'No', '', ' ']), 'NETWORK_ACCESS_FEES_IND'] = "Yes"
if 'PAYER_NAME' in final_df and 'HEALTH_PLAN_STATE' in final_df:
final_df.loc[final_df['PAYER_NAME'].str.startswith('Illini'), 'HEALTH_PLAN_STATE'] = "Illinois"
if 'HEALTH_PLAN_STATE' in final_df:
final_df['HEALTH_PLAN_STATE'] = final_df['HEALTH_PLAN_STATE'].map(valid.STATE_MAP).fillna(final_df['HEALTH_PLAN_STATE'])
if 'NOTICE_PROVIDER_NAME' in final_df and 'NOTICE_PROVIDER_ADDRESS' in final_df:
final_df.loc[final_df['NOTICE_PROVIDER_NAME'].str.contains('Superior HealthPlan', na=False), 'NOTICE_PROVIDER_NAME'] = np.nan
final_df.loc[final_df['NOTICE_PROVIDER_NAME'].str.contains('Superior HealthPlan', na=False), 'NOTICE_PROVIDER_ADDRESS'] = np.nan
final_df['temp_filename'] = final_df['Contract Name'].str[:10].str.replace('-','')
if 'PROV_GROUP_TIN' in final_df and 'Contract Name' in final_df:
final_df.loc[(final_df['PROV_GROUP_TIN'].isin([' ', '']))&(final_df['temp_filename'].str.isnumeric()), 'PROV_GROUP_TIN'] = final_df['temp_filename']
final_df.drop(columns=['temp_filename'], inplace=True)
# For POLICIES_AND_PROCEDURES, filter out anything without either "policies" or "procedures".
if 'POLICIES_AND_PROCEDURES' in final_df:
final_df.loc[~final_df['POLICIES_AND_PROCEDURES'].str.contains('policies', flags=re.IGNORECASE) &
~final_df['POLICIES_AND_PROCEDURES'].str.contains('procedures', flags=re.IGNORECASE), 'POLICIES_AND_PROCEDURES'] = "N/A"
final_df = final_df.apply(lambda x: x.map(replace_null_terms))
final_df = final_df.apply(lambda x: x.map(replace_quotes))
return final_df
def replace_quotes(value):
try:
return str(value).replace(r'\"', '"')
except:
return value
def replace_null_terms(value):
# Convert value to string and check if any term from NULL_ANSWER_TERMS is in the value
if any(term.lower() in str(value).lower() for term in valid.NULL_ANSWER_TERMS):
return 'N/A'
return value
def filter_dict(d, pattern):
final_dict = {}
for page_num, answer in d.items():
if not pattern.search(answer):
final_dict[page_num] = answer
return final_dict
def filter_add_ons(df):
pattern = r"(in no event).+(includ.?)|(forward).+(payments)"
mask = df['ADD_ON_REIMBURSEMENT_LANGUAGE'].str.contains(pattern, case=False, na=False, regex=True)
df.loc[mask, 'ADD_ON_REIMBURSEMENT_LANGUAGE'] = 'N/A'
df.loc[mask, "ADD_ON_REIMBURSEMENT_IND"] = 'N'
def check_add_on(group):
# Check if any 'SERVICE_TYPE' contains 'Add-on' or 'Add On'
if group['FULL_SERVICE'].str.contains('Add-on|Add On', regex=True, case=False, na=False).any():
group['ADD_ON_REIMBURSEMENT_LANGUAGE'] = 'N/A' # Set 'ADD_ON' to 'N/A' for the whole group
group["ADD_ON_REIMBURSEMENT_IND"] = 'N'
return group
# Group by 'Contract Name' and 'EXHIBIT', then apply the check_add_on function
df = df.groupby(['Contract Name', 'EXHIBIT']).apply(check_add_on)
# TODO : If original add on value is actually an Exclusion, and the Exclusion value for the row is invalid, then move the Add On value to the Exclusions column
return df
import pandas as pd
import re
def clean_process_AWP_FLATFEE_cols(df):
# Function to extract dollar amount after "NET INVOICE PRICE" or "FLAT FEE"
def extract_dollar_amount(text, pattern):
if pd.isna(text):
return None
match = re.search(pattern, str(text), re.IGNORECASE)
if match:
return match.group(1).replace(',', '')
return None
# Apply changes based on conditions
def apply_changes(row):
if 'FULL_METHODOLOGY' in row and isinstance(row['FULL_METHODOLOGY'], str) and 'AWP' in row['FULL_METHODOLOGY'].upper():
full_methodology = row['FULL_METHODOLOGY'].upper()
dollar_amount_found = False
# Check FLAT_FEE_STANDARD first
if 'FLAT_FEE_STANDARD' in row and row['FLAT_FEE_STANDARD'] != 'N/A' and pd.notna(row['FLAT_FEE_STANDARD']) and row['FLAT_FEE_STANDARD'] != '':
row['SHORT_METHODOLOGY'] = 'Flat Fee'
dollar_amount_found = True
else:
# Then check for FLAT FEE in FULL_METHODOLOGY
flat_fee = extract_dollar_amount(full_methodology, r'FLAT FEE\s*:\s*\$?([\d,]+(?:\.\d{2})?)')
if flat_fee:
row['FLAT_FEE_STANDARD'] = flat_fee
row['SHORT_METHODOLOGY'] = 'Flat Fee'
dollar_amount_found = True
else:
# Finally, check for NET INVOICE PRICE
net_invoice_price = extract_dollar_amount(full_methodology, r'NET INVOICE PRICE\s*:\s*\$?([\d,]+(?:\.\d{2})?)')
if net_invoice_price:
row['FLAT_FEE_STANDARD'] = net_invoice_price
row['SHORT_METHODOLOGY'] = 'Flat Fee'
dollar_amount_found = True
# Set RATE_STANDARD and RATE_SHORT to 'N/A' if a dollar amount was found
if dollar_amount_found:
row['RATE_STANDARD'] = 'N/A'
row['RATE_SHORT'] = 'N/A'
return row
# Apply the changes to the DataFrame
df = df.apply(apply_changes, axis=1)
return df
def add_scmr(df):
def count_dollar_values(s):
return str(s).count('$')
df['dollar_count'] = df['FLAT_FEE_STANDARD'].apply(count_dollar_values)
df['Single Code Multiple Rates (Language)'] = df.apply(
lambda row: row['FLAT_FEE_STANDARD'] if row['dollar_count'] > 1 else 'N/A', axis=1)
df['Single Code Multiple Rates (Y/N)'] = df['dollar_count'].apply(
lambda x: 'Y' if x > 1 else 'N')
df.drop('dollar_count', axis=1, inplace=True)
return df
def clean_lob(df, filename):
def update_contract_lob(row):
# Check if more than one valid lob is present in the 'FULL_SERVICE' column
if sum(val.lower() in row['FULL_SERVICE'].lower() for val in valid.VALID_LOBS) > 1:
prompt = prompts.LOB_SWEEPER(row['EXHIBIT'])
answer = claude_funcs.invoke_claude(prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 256)
return answer # Return the value from 'EXHIBIT' if condition is met
return row['CONTRACT_LOB']
df['CONTRACT_LOB'] = df.apply(update_contract_lob, axis=1)
return df
def derive_indicators(results):
def is_populated(value):
if not value: # Handles None and empty string
return False
# Convert to string and lowercase for comparison
str_value = str(value).lower().strip()
# Check if the value is any variation of N/A
na_values = ['n/a', 'na', 'not applicable', '']
if str_value in na_values:
return False
return True
# Process standard derived indicator fields
for field in DERIVED_INDICATOR_FIELDS:
root_field = field
indicator_field = field.rsplit('_', 1)[0] + '_IND'
if root_field in results and is_populated(results[root_field]):
results[indicator_field] = 'Y'
else:
results[indicator_field] = 'N'
# Handle special cases
special_mappings = {
'DELEGATED_TERMS': 'DELEGATED_FUNCTION_IND',
'SEQUESTRATION_REDUCTIONS': 'SEQUESTRATION_REDUCTIONS_IND'
}
for root_field, indicator_field in special_mappings.items():
if root_field in results and is_populated(results[root_field]):
results[indicator_field] = 'Y'
else:
results[indicator_field] = 'N'
return results