jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Modify prov type and lob logic

Browse Source
This commit is contained in:
Katon Minhas
2024-05-09 15:18:41 -07:00
committed by Michael McGuinness
parent 5d0e943a57
commit faec7dc540
4 changed files with 111 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/config.py
+6 -9
View File
@@ -18,20 +18,17 @@ MAX_WORKERS = 20
# Prompt Debugging
RUN_PRIMARY = True # Always True
RUN_LOB = False
RUN_LOB = True
RUN_LESSER = True
RUN_METHODOLOGY = False
RUN_FS = False
RUN_EXCEPTION = False
RUN_CODES = False
RUN_METHODOLOGY = True
RUN_FS = True
RUN_EXCEPTION = True
RUN_CODES = True
# AWS Keys
AWS_ACCESS_KEY_ID="ASIAZTMXAXNXC6PRLROX"
AWS_SECRET_ACCESS_KEY="3UNtSv9VyM2WFccTZWBeBCdxZE4nK00rY/cTSIug"
AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjELD//////////wEaCXVzLWVhc3QtMiJHMEUCIQCBH8QRxxRn9Cscy5DbAiF520ouH2jDR3DPOYVTBKlfhAIgIk2i9y4UT25ndFseQNGfPorENxsdG4FKcVNRBiEsLx8qgwMIGRAAGgw2NjAxMzEwNjg3ODIiDAWrX1j/1i3BywL6iyrgAmC0BF21WmupY/8rcaQ0B7/cm0Wg4yphqP81R6j/VYZKvIp81UdMzbMaTOsfzpRcjgF89sstUGTCIA93u2liI3gkfcEXQVA8/XxyF96gKok3OP9jBHgIbBJQDXXpVSr7TUUZUpFLbYFDISrVMJDmte9UJIh/Dwj1P4r8OSiVwXWrfvgPienLBu2xmrRyNuhijV260EoJMOGLTkWWZpaYsWua50R5PXvGKIdJizMsr4u+VA8r+XAyqp5c0FIFdcPmbiFGg1W48hjxIZ86Ec2SHIowEqkI62kcXJ7JSNnOR0s/53ok07UZQ2Khr8RYEa9k5SR2H6GESlag/sTRIRngyo/qCThJ8eKW6VQKttC1UrFURNIu4rkajuV7vOY2U5P0tqsLvUuOVspIt5ViielkvfihQ6USCV7w0+E0FJIdztviIm1VTGX5kF+O4ctt1xwblBaqc5A+rTwQoZac80jGrS8wp+vzsQY6pgHQv5Ojg/pVpKLhQ/TQiGIMWw4n88Uo9N1BucRRciTjhXmTdLN5ZyKopiAIDlnaiKqhYDRbvhBbYc63jU3Y8RzwUHjC8RY6+hAqB/LuK6LweOE3pwanfK8uDp3tXBkiyfyz/++xUtcrz2ujf8DDRsyynxkNPI6G0R8s8RwLyjnAhvKn3nxxpJbqTSUTdFLHzy/WxNHxAOpQT+MocnFVIdjJTaIHi8lW"
# File Paths
LOCAL_PATH = 'docs/test/' # Replace with local
LOCAL_PATH = 'docs/all_txt_updated/' # Replace with local
# S3 Settings
S3_CLIENT = boto3.client('s3',
src/postprocess.py
+19 -80
View File
@@ -14,87 +14,26 @@ def primary_filter(answer_dict):
filtered_dicts.append(d)
return filtered_dicts
import pandas as pd
import difflib
VALID_LOBS = ['MEDICARE', 'MEDICARE ADVANTAGE', 'MEDICAID', 'MARKETPLACE', 'COMMERCIAL', 'GROUP', 'MEDICARE-MEDICAID']
VALID_PROGRAMS = ['CHIP', 'CHIP-P', 'CHIP-PERINATE', 'STAR', 'STAR+PLUS', 'MA', 'DUAL SPECIAL NEEDS PLAN', 'DSNP', 'DUAL']
#VALID_NETWORKS = ['HMO', 'PPO', 'EPO', 'POS', 'FFS', 'PPO-POS']
def get_closest_match(val, val_list, similarity_threshold=0.6):
matches = difflib.get_close_matches(val.upper(), val_list, n=1, cutoff=similarity_threshold)
if matches:
index = val_list.index(matches[0])
return val_list[index]
else:
return None
def get_mappings(original_vals, actual_vals, similarity_threshold):
field_mapping = {}
non_field_mapping = {}
for original_val in original_vals:
val = original_val.upper()
if ',' in val:
val_list = [v.strip() for v in val.split(',')]
def create_in_out_column(df):
df['IN_OUT'] = ''
for idx, row in df.iterrows():
service = str(row['SERVICE']).upper()
prov_type = str(row['PROV_TYPE']).upper()
if ('INPATIENT' in service and 'OUTPATIENT' in service) or ('INPATIENT' in prov_type and 'OUTPATIENT' in prov_type) or ('IP ' in service and 'OP ' in service) or ('IP ' in prov_type and 'OP ' in prov_type):
df.loc[idx, 'IN_OUT'] = 'Inpatient and Outpatient'
new_service = service.replace('INPATIENT', '').replace('OUTPATIENT', '').replace('IP ', '').replace('OP ', '')
df.loc[idx, 'SERVICE'] = new_service
elif ('INPATIENT' in service) or ('INPATIENT' in prov_type) or ('IP ' in service) or ('IP ' in prov_type):
df.loc[idx, 'IN_OUT'] = 'Inpatient'
new_service = service.replace('INPATIENT', '').replace('IP ', '')
df.loc[idx, 'SERVICE'] = new_service
elif ('OUTPATIENT' in service) or ('OUTPATIENT' in prov_type) or ('OP ' in service) or ('OP ' in prov_type):
df.loc[idx, 'IN_OUT'] = 'Outpatient'
new_service = service.replace('OUTPATIENT', '').replace('OP ', '')
df.loc[idx, 'SERVICE'] = new_service
else:
val_list = [val]
map_to = []
non_field = []
for sub_val in val_list:
if sub_val in actual_vals:
map_to.append(sub_val)
else:
closest_match = get_closest_match(sub_val, actual_vals, similarity_threshold)
if closest_match:
map_to.append(closest_match)
else:
non_field.append(sub_val)
field_mapping[original_val] = ', '.join(map_to)
if non_field:
non_field_mapping[original_val] = ', '.join(non_field)
return field_mapping, non_field_mapping
def lob_product_program_clean(df):
# Ensure string type and handle NaN
df['CONTRACT_LOB'] = df['CONTRACT_LOB'].fillna('').astype(str)
df['CONTRACT_PROGRAM'] = df['CONTRACT_PROGRAM'].fillna('').astype(str)
df['PRODUCT'] = df['PRODUCT'].fillna('').astype(str)
original_lobs = df['CONTRACT_LOB'].unique()
original_programs = df['CONTRACT_PROGRAM'].unique()
# Get Mappings
lob_mapping, non_lob_mapping = get_mappings(original_lobs, VALID_LOBS, 0.5)
program_mapping, non_program_mapping = get_mappings(original_programs, VALID_PROGRAMS, 0.6)
# Apply mappings
df['CONTRACT_LOB'] = df['CONTRACT_LOB'].map(lob_mapping).fillna(df['CONTRACT_LOB'])
df['CONTRACT_PROGRAM'] = df['CONTRACT_PROGRAM'].map(program_mapping).fillna(df['CONTRACT_PROGRAM'])
# Initialize new column
df['potential_products_detected'] = ''
for index, row in df.iterrows():
lob_entries = row['CONTRACT_LOB'].split(', ')
program_entries = row['CONTRACT_PROGRAM'].split(', ')
product_entries = row['PRODUCT'].split(', ')
# Filtering valid entries
valid_lob_entries = [entry for entry in lob_entries if entry in VALID_LOBS]
valid_program_entries = [entry for entry in program_entries if entry in VALID_PROGRAMS]
# Update the DataFrame with valid entries
df.at[index, 'CONTRACT_LOB'] = ', '.join(valid_lob_entries)
df.at[index, 'CONTRACT_PROGRAM'] = ', '.join(valid_program_entries)
# Detect and handle products
detected_products = set(valid_lob_entries).intersection(product_entries) | set(valid_program_entries).intersection(product_entries)
df.at[index, 'potential_products_detected'] = ', '.join(detected_products)
# Remove detected products from LOB and Program
df.at[index, 'CONTRACT_LOB'] = ', '.join([lob for lob in valid_lob_entries if lob not in detected_products])
df.at[index, 'CONTRACT_PROGRAM'] = ', '.join([prog for prog in valid_program_entries if prog not in detected_products])
pass
return df
@@ -102,7 +41,7 @@ def bottom_up_postprocess(df):
# Remove Unnamed columns
df = df[[col for col in df.columns if 'UNNAMED' not in col.upper()]]
df = lob_product_program_clean(df)
#df = lob_product_program_clean(df)
# Reimbursement Flat Fee
df.loc[:, 'REIMBURSEMENT_FLAT_FEE'] = df['REIMBURSEMENT_FLAT_FEE'].astype(str).str.replace(r'[^\d.]', '', regex=True)
src/prompts.py
+5 -1
View File
@@ -36,7 +36,11 @@ Return a dictionary object with the following attributes:
'CONTRACT_LOB' : What Line of Business is the reimbursement schedule for? It will be Medicaid, Medicare, Marketplace, or similar. It will likely only be listed once per page, if at all.
'CONTRACT_PROGRAM' : What Program is the reimbursement schedule for? This may be something like CHIP, CHIP PERINATE, STAR, STAR PLUS, DSNP, Dual, or similar. This will likely only be listed once per page, if at all.
'PRODUCT' : What Product is the reimbursement schedule for? This the brand name of the health plan. It is NOT the same as LOB. It is NOT the name of a hospital. This will likely only be listed once per page, if at all.
'PROV_TYPE' : For what provider type or place of service is this fee schedule for? It could be Hospital, Professional, Ancillary, or similar. This will likely only be listed once per page, if at all.
'PROV_TYPE' : For what provider type or place of service is this fee schedule for? This will likely only be listed once per page, if at all.
For CONTRACT_LOB, choose ONLY from the following: 'Medicaid', 'Medicare Advantage', 'Medicare Supplement', 'Marketplace', 'Commercial', 'Group'
Some examples of PROV_TYPE are: 'Primary Care Provider', 'Specialist', 'Ancillary', 'Ambulatory Surgery Center', 'Hospital','Rehab Facility', 'Skilled Nursing Facility', 'Ambulance', 'Rural Health Clinic', 'Home Health'
This list is non-exhaustive.
If there are multiple correct answers for any of the above attributes, return only the answer corresponding to the SECOND of the two pages.
If there are multiple correct answers for the second page, return them both in a comma-separated list. However, in most instances there will be only one.
src/test.py
+81 -75
View File
@@ -4,50 +4,11 @@ import numpy as np
import difflib
import prompts
import claude_funcs
#df = pd.read_csv('output/consolidated_results_20240503_v1.csv')
df = pd.read_csv('output/consolidated_results_20240503_v1.csv')
def create_in_out_column(df):
df['IN_OUT'] = ''
for idx, row in df.iterrows():
service = str(row['SERVICE']).upper()
prov_type = str(row['PROV_TYPE']).upper()
if ('INPATIENT' in service and 'OUTPATIENT' in service) or ('INPATIENT' in prov_type and 'OUTPATIENT' in prov_type) or ('IP ' in service and 'OP ' in service) or ('IP ' in prov_type and 'OP ' in prov_type):
df.loc[idx, 'IN_OUT'] = 'Inpatient and Outpatient'
new_service = service.replace('INPATIENT', '').replace('OUTPATIENT', '').replace('IP ', '').replace('OP ', '')
df.loc[idx, 'SERVICE'] = new_service
elif ('INPATIENT' in service) or ('INPATIENT' in prov_type) or ('IP ' in service) or ('IP ' in prov_type):
df.loc[idx, 'IN_OUT'] = 'Inpatient'
new_service = service.replace('INPATIENT', '').replace('IP ', '')
df.loc[idx, 'SERVICE'] = new_service
elif ('OUTPATIENT' in service) or ('OUTPATIENT' in prov_type) or ('OP ' in service) or ('OP ' in prov_type):
df.loc[idx, 'IN_OUT'] = 'Outpatient'
new_service = service.replace('OUTPATIENT', '').replace('OP ', '')
df.loc[idx, 'SERVICE'] = new_service
else:
pass
return df
#df_new = create_in_out_column(df)
#df_new.to_csv('output/post_test.csv')
def primary_filter(answer_dict):
filtered_dicts = []
for d in answer_dict:
if 'SERVICE' in d.keys():
if 'INSURANCE' not in d['SERVICE'].upper():
filtered_dicts.append(d)
else:
filtered_dicts.append(d)
return filtered_dicts
import pandas as pd
import difflib
VALID_LOBS = ['MEDICARE', 'MEDICARE ADVANTAGE', 'MEDICAID', 'MARKETPLACE', 'COMMERCIAL', 'GROUP', 'MEDICARE-MEDICAID']
VALID_PROGRAMS = ['CHIP', 'CHIP-P', 'CHIP-PERINATE', 'STAR', 'STAR+PLUS', 'MA', 'DUAL SPECIAL NEEDS PLAN', 'DSNP', 'DUAL']
VALID_NETWORKS = ['HMO', 'PPO', 'EPO', 'POS', 'FFS', 'PPO-POS']
def get_closest_match(val, val_list, similarity_threshold=0.6):
matches = difflib.get_close_matches(val.upper(), val_list, n=1, cutoff=similarity_threshold)
@@ -55,8 +16,9 @@ def get_closest_match(val, val_list, similarity_threshold=0.6):
index = val_list.index(matches[0])
return val_list[index]
else:
return None # Claude prompt here prompt = f"{value} What value in this list is the value closest to? {list}. If none are close return None."
prompt = f"{val.upper()} is closest to which medical program mentioned in the list {val_list}. If none are close return None. If you are not sure return None. Answer in one word."
#return claude_funcs.invoke_claude_3(prompt, max_tokens=100)
return None
def get_mappings(original_vals, actual_vals, similarity_threshold):
field_mapping = {}
@@ -84,6 +46,10 @@ def get_mappings(original_vals, actual_vals, similarity_threshold):
return field_mapping, non_field_mapping
def lob_product_program_clean(df):
# 1. If something in LOB is invalid, check if it is a Program. If yes, move to Program (append if needed), if no, move to potential_products_detected
# 2. If something in Program is invalid, check if it is an LOB. If yes, move to LOB (append if needed). If no, move to potential_products_detected
# 3. If something in Product is a valid LOB, copy to LOB (append if needed). If something in Product is a valid Program, copy to Program (append if needed). Keep product the same.
# Ensure string type and handle NaN
df['CONTRACT_LOB'] = df['CONTRACT_LOB'].fillna('').astype(str)
df['CONTRACT_PROGRAM'] = df['CONTRACT_PROGRAM'].fillna('').astype(str)
@@ -93,43 +59,83 @@ def lob_product_program_clean(df):
original_programs = df['CONTRACT_PROGRAM'].unique()
# Get Mappings
lob_mapping, non_lob_mapping = get_mappings(original_lobs, VALID_LOBS, 0.5)
program_mapping, non_program_mapping = get_mappings(original_programs, VALID_PROGRAMS, 0.6)
lob_mapping, non_lob_mapping = get_mappings(original_lobs, prompts.VALID_LOBS, 0.5)
program_mapping, non_program_mapping = get_mappings(original_programs, prompts.VALID_PROGRAMS, 0.6)
# Apply mappings
df['CONTRACT_LOB'] = df['CONTRACT_LOB'].map(lob_mapping).fillna(df['CONTRACT_LOB'])
df['CONTRACT_PROGRAM'] = df['CONTRACT_PROGRAM'].map(program_mapping).fillna(df['CONTRACT_PROGRAM'])
# Get all lob mappings
all_lob_mappings = {}
for original_lob in lob_mapping:
full_lob_map = {}
full_lob_map['new_lob'] = lob_mapping[original_lob]
if original_lob in non_lob_mapping.keys():
all_non_lobs = non_lob_mapping[original_lob]
for non_lob in all_non_lobs.split(','):
non_lob = non_lob.strip()
closest_program = get_closest_match(non_lob, prompts.VALID_PROGRAMS, 0.7)
if closest_program:
full_lob_map['new_program'] = closest_program
else:
full_lob_map['potential_product'] = non_lob
all_lob_mappings[original_lob] = full_lob_map
# Initialize new column
df['potential_products_detected'] = ''
# Get all program mappings
all_program_mappings = {}
for original_program in program_mapping:
full_program_map = {}
full_program_map['new_program'] = program_mapping[original_program]
if original_program in non_program_mapping.keys():
all_non_programs = non_program_mapping[original_program]
for non_program in all_non_programs.split(','):
non_program = non_program.strip()
closest_lob = get_closest_match(non_program, prompts.VALID_LOBS, 0.7)
if closest_lob:
full_program_map['new_lob'] = closest_lob
else:
full_program_map['potential_product'] = non_program
all_program_mappings[original_program] = full_program_map
for index, row in df.iterrows():
lob_entries = [v.strip() for v in row['CONTRACT_LOB'].split(',')]
program_entries = [v.strip() for v in row['CONTRACT_PROGRAM'].split(',')]
product_entries = [v.strip() for v in row['PRODUCT'].split(',')]
df['Potential_Product'] = ""
for idx, row in df.iterrows():
original_lob = row['CONTRACT_LOB']
original_program = row['CONTRACT_PROGRAM']
original_product = row['PRODUCT']
# Filtering valid entries
valid_lob_entries = [entry for entry in lob_entries if entry.upper() in VALID_LOBS]
valid_program_entries = [entry for entry in program_entries if entry.upper() in VALID_PROGRAMS]
# Handle LOB
lob_dict = all_lob_mappings[original_lob]
new_lob = lob_dict['new_lob']
if 'new_program' in lob_dict.keys():
new_program = lob_dict['new_program']
else:
new_program = ""
if 'potential_product' in lob_dict.keys():
potential_product = lob_dict['potential_product']
else:
potential_product = ""
df.loc[idx, 'CONTRACT_LOB'] = new_lob
if original_program != new_program:
df.loc[idx, 'CONTRACT_PROGRAM'] = original_program + ' ' + new_program
if original_product != potential_product:
df.loc[idx, 'Potential_Product'] = df.loc[idx, 'Potential_Product'] + ', ' + potential_product
# Update the DataFrame with valid entries
df.at[index, 'CONTRACT_LOB'] = ', '.join(valid_lob_entries)
df.at[index, 'CONTRACT_PROGRAM'] = ', '.join(valid_program_entries)
# Detect and handle products
detected_products = set(valid_lob_entries) | set(valid_program_entries)
df.at[index, 'potential_products_detected'] = ', '.join(detected_products)
# Remove detected products from LOB and Program
df.at[index, 'CONTRACT_LOB'] = ', '.join([lob for lob in valid_lob_entries if lob not in detected_products])
df.at[index, 'CONTRACT_PROGRAM'] = ', '.join([prog for prog in valid_program_entries if prog not in detected_products])
# If something in LOB is invalid, check if it is a Program. If yes, move to Program (append if needed), if no, move to potential_products_detected
# If something in Program is invalid, check if it is an LOB. If yes, move to LOB (append if needed). If no, move to potential_products_detected
# If something in Product is a valid LOB, copy to LOB (append if needed). If something in Product is a valid Program, copy to Program (append if needed). Keep product the same.
# Handle Program
program_dict = all_program_mappings[original_program]
new_program = program_dict['new_program']
if 'new_lob' in program_dict.keys():
new_lob = program_dict['new_lob']
else:
new_lob = ""
if 'potential_product' in program_dict.keys():
potential_product = program_dict['potential_product']
else:
potential_product = ""
df.loc[idx, 'CONTRACT_PROGRAM'] = new_program
if original_lob != new_lob:
df.loc[idx, 'CONTRACT_LOB'] = original_lob + ' ' + new_lob
if original_product != potential_product:
df.loc[idx, 'Potential_Product'] = df.loc[idx, 'Potential_Product'] + ', ' + potential_product
return df
#processed_df = lob_product_program_clean(df)
#processed_df.to_csv('output/test.csv')
processed_df = lob_product_program_clean(df)
processed_df.to_csv('output/test.csv')