Merged postprocessing updates
This commit is contained in:
Katon Minhas
committed by
Michael McGuinness
Michael McGuinness
parent
38d18f97c5
commit
a5fe337309
+3
-3
@@ -35,9 +35,9 @@ RUN_EXCEPTION = True
|
||||
RUN_CODES = True
|
||||
|
||||
# AWS Keys
|
||||
AWS_ACCESS_KEY_ID="ASIAZTMXAXNXFTD5MQUK"
|
||||
AWS_SECRET_ACCESS_KEY="yGCDnqRupWsJyABleBXd3NkzySjf2tuU1L5pPsUh"
|
||||
AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEDUaCXVzLWVhc3QtMiJIMEYCIQC79KaOSV9wiHBrCPKOdoGQ10Hhl2BlX9kN2LI0AWtbzQIhAOniVNrVlB0wbuaJUVcFjuV8LRO/ZZdCk4ac9O9HftrdKowDCL7//////////wEQABoMNjYwMTMxMDY4NzgyIgz6LOHKqD7pmvQXBf8q4AJkRxmxk5UhNmr6iAx+zsccBvcFR2XBFywrgrPCHk5mUzQXfgPFBmcTW7XTB2F2Rie5y/oZGV1YxK8s2JYXd9gkbJvwUt7K1bNjQYg/IMQ1goecxH2dRHq//JYDs9TQiU01nacnI6eMPQqSCpmtc2yI45GovWwONKJviddUI5Ln/IS7Ktq4dcmy7xx/GqbgqsWb3bCFhmT/9UrOGaBSOHA0fhshKaw8ihbWgz1BtVFSt2i8FsTXRlNXn5l79iwo+vWs4CQzmHgRjw3ZIytrw2V2nppgJlw1Tb+AEwBdbC9wW9WsmDmw9QmBeyxcKAKkmPf66wWqmBqK0/2w8r4q6H7gEta/Ibc08137zd0fLWEu74iZE0I5jhLYOGwh+U0DhsR0ZDufWurcA+YIfWZAsfG0PePonv8+6drcqtNjTqKs0KIfAQ6PHAb4xuoCYEzvLeyMcY1t5sOl/oU+RWxkPrsIMMLLgbMGOqUBfx6TS5599bddNIEV2CH3AHym/18n3lyA19PHcVcfldqU7nMrYTZdgNBMy1Or1OVjcxYV988OI8ZOqWPiYzMGR6cVSzwA1ESjLUVma3vKcOTq4GbgBvRxA44PMfBm/P3ow3Vhx3ONe/oaxqgKre26JBheTN5s0Uua4lEcO4lvgB6/Kap+/aaR9vnEwLtJZ8u5d84k/Ajw1nG8bDMfLPsyWXjtUUZI"
|
||||
AWS_ACCESS_KEY_ID="ASIAZTMXAXNXG7BOCV4A"
|
||||
AWS_SECRET_ACCESS_KEY="xDx1H2SOL71C1N9jl9pv5HqCyLccRsYs8vQvPU5y"
|
||||
AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjED0aCXVzLWVhc3QtMiJIMEYCIQCJo1mxCG+aqqh1B5Jy1vvEcANPejqBUIOVnbPcUs5/+QIhAJmpSAygER0WddjIenyY8hu0V4aqiFCsdBBQbkupzvwzKowDCMf//////////wEQABoMNjYwMTMxMDY4NzgyIgxuroZvh2Y2VPEKEYgq4AK36l5Ls8q/xw+e0A1s6+TpUjvecAr9tUrwEF5MyY0ToWSyac7/N8y/g7p8Kd9lMiJqJ1C7YlePsgGQen9BBxLeLWp/yEWsKuQsyJmh2mcO9DtSFYkWeZgRmxT6KVflCySKzMDOtytwU1vzwrbjQeh8pRG7zijvO0WvGPfBykHYOp9dIGIG8kGa6IUq58EhpzKtCSahOFbaddamvzf+HabTZ45tv5RJbG/Lrw3FqEFAzrj7RkiUS3dnJoScH595PtEHZ3IHDYE2jvyeS/FqV+t454IvTV4avq9SePDtAAjw4CnQVyFR7gchPil+qIKgt2+IDHaqTAIRbR3isC/X2TTS8kw7pkm1Uqgz60DWsFrjrpWpIGqHAYmxmnbEX0rNkGzJgaX6YdDF2o7LTm6z0rBvjmY96uKk1QPs07tVJMxklx1+QFzMzpnZEuDp8KEaKW9xQg3IGP2TuSJM8ZAKDQ4uMNaug7MGOqUBRaIqNEebTiBJ9b1LvNOyonXyhqeq8GsA4y0MPcUDnk5JDFUfUt/1o6gefIV6pFsel5RIwHf49rfsrIsGQs5YzuViwP3whkloYPbg8ZXlQTJRVTXHaOqjQHVfeT26aU9S4VOFfqH8N/dhLVNUMpesutnkzrngyv0nUTubYt7LtBF5A0RvcTqMMfmU9pbjUTNk1fhsd4mOBAuHDrOTMXm+1LMvDJyc"
|
||||
|
||||
# File Paths
|
||||
LOCAL_PATH = 'data/test/' # Replace with local
|
||||
|
||||
@@ -8,6 +8,8 @@ import table_funcs
|
||||
import prompt_funcs
|
||||
import prompts
|
||||
import claude_funcs
|
||||
import postprocessingfuncs
|
||||
import postprocess
|
||||
|
||||
def clean_td(td):
|
||||
td_clean = []
|
||||
@@ -104,22 +106,22 @@ def process_file(file_object):
|
||||
|
||||
# Run Top Down
|
||||
td_results = prompt_funcs.run_top_down(filename, text_dict) # Returns list of dictionaries for each page
|
||||
print("TD Results:", td_results)
|
||||
print("Top Down complete...")
|
||||
|
||||
# Run Bottom Up
|
||||
bu_results = prompt_funcs.run_bottom_up(filename, text_dict) # Returns list of dictionaries
|
||||
print("BU Results:", bu_results)
|
||||
print("Bottom Up complete...")
|
||||
|
||||
# Combine
|
||||
combined_results = merge_results(clean_td(td_results), bu_results, text_dict)
|
||||
print("Combined Results:", combined_results)
|
||||
print("TD/BU Merge complete...")
|
||||
|
||||
# Convert to DataFrame
|
||||
combined_df = pd.DataFrame(combined_results)
|
||||
|
||||
# Post-process combined results
|
||||
# post_processed_combined_df = postprocess.postprocess_results(combined_df)
|
||||
# print("Post-processed Results:", post_processed_combined_df)
|
||||
combined_df = combined_df.applymap(postprocessingfuncs.sanitize_value)
|
||||
post_processed_combined_df = postprocess.postprocess_results(combined_df)
|
||||
|
||||
# Create directories
|
||||
base_filename = os.path.splitext(filename)[0]
|
||||
@@ -129,6 +131,5 @@ def process_file(file_object):
|
||||
# Save results
|
||||
pd.DataFrame(td_results).to_csv(os.path.join(output_dir, 'td_results.csv'), index=False)
|
||||
pd.DataFrame(bu_results).to_csv(os.path.join(output_dir, 'bu_results.csv'), index=False)
|
||||
combined_df.to_csv(os.path.join(output_dir, 'combined_results.csv'), index=False)
|
||||
# post_processed_combined_df.to_csv(os.path.join(output_dir, 'combined_results.csv'), index=False)
|
||||
|
||||
post_processed_combined_df.to_csv(os.path.join(output_dir, 'combined_results_post_processed.csv'), index=False)
|
||||
|
||||
+53
-2
@@ -1,4 +1,5 @@
|
||||
import pandas as pd
|
||||
import re
|
||||
import postprocessingfuncs
|
||||
|
||||
def sanitize_combined(df):
|
||||
@@ -7,17 +8,66 @@ def sanitize_combined(df):
|
||||
df[column] = df[column].apply(postprocessingfuncs.sanitize_value)
|
||||
return df
|
||||
|
||||
def post_process_combined(df, ):
|
||||
def extract_codes(value, pattern):
|
||||
"""
|
||||
Extract codes from a given string using a specified regex pattern.
|
||||
"""
|
||||
if pd.isna(value):
|
||||
return value
|
||||
matches = re.findall(pattern, str(value))
|
||||
if matches:
|
||||
return ', '.join(matches)
|
||||
else:
|
||||
# Check if there are any numbers in the value
|
||||
if any(char.isdigit() for char in value):
|
||||
return value
|
||||
else:
|
||||
return ""
|
||||
|
||||
def clean_code_columns(df, column_name, pattern):
|
||||
"""
|
||||
Clean the specified column in the DataFrame by extracting codes based on the provided regex pattern.
|
||||
"""
|
||||
df[column_name] = df[column_name].apply(lambda x: extract_codes(x, pattern))
|
||||
return df
|
||||
|
||||
def post_process_combined(df):
|
||||
VALID_LOBS = ['MEDICARE', 'MEDICARE ADVANTAGE', 'MEDICAID', 'MARKETPLACE', 'COMMERCIAL', 'GROUP', 'MEDICARE-MEDICAID']
|
||||
VALID_PROGRAMS = ['CHIP', 'CHIP-P', 'CHIP-PERINATE', 'STAR', 'STAR+PLUS', 'MA', 'DUAL SPECIAL NEEDS PLAN', 'DSNP', 'DUAL']
|
||||
VALID_NETWORKS = ['HMO', 'PPO', 'EPO', 'POS', 'FFS']
|
||||
|
||||
# Define valid values dictionary
|
||||
valid_values_dict = {
|
||||
'CONTRACT_LOB': VALID_LOBS,
|
||||
'CONTRACT_PROGRAM': VALID_PROGRAMS,
|
||||
'CONTRACT_NETWORK': VALID_NETWORKS
|
||||
}
|
||||
|
||||
# Sanitize the combined data
|
||||
df = sanitize_combined(df)
|
||||
|
||||
# Clean specific columns for exact matches
|
||||
postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_LOB', VALID_LOBS, 'Corrected_LOB')
|
||||
postprocessingfuncs.clean_columns_combined(df, 'PRODUCT', VALID_PROGRAMS, 'Corrected_PRODUCT')
|
||||
postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_PROGRAM', VALID_PROGRAMS, 'Corrected_PROGRAM')
|
||||
postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_NETWORK', VALID_NETWORKS, 'Corrected_NETWORK')
|
||||
|
||||
# Apply fuzzy matching to the columns
|
||||
postprocessingfuncs.clean_columns_combined_fuzzy(df, 'CONTRACT_LOB', VALID_LOBS, 0.8)
|
||||
postprocessingfuncs.clean_columns_combined_fuzzy(df, 'CONTRACT_PROGRAM', VALID_PROGRAMS, 0.8)
|
||||
postprocessingfuncs.clean_columns_combined_fuzzy(df, 'CONTRACT_NETWORK', VALID_NETWORKS, 0.8)
|
||||
|
||||
# Correct misplaced values across columns
|
||||
df = postprocessingfuncs.correct_misplaced_values(df, ['CONTRACT_LOB', 'CONTRACT_PROGRAM', 'CONTRACT_NETWORK'], valid_values_dict)
|
||||
|
||||
# Define the patterns for the different code types
|
||||
cpt_code_pattern = r'[A-Za-z0-9]{4}[A-Za-z]|[A-Za-z0-9]{3}[A-Za-z][0-9]|[A-Za-z0-9]{2}[A-Za-z][0-9]{2}|[A-Za-z0-9][A-Za-z][0-9]{3}|[A-Za-z][0-9]{4}'
|
||||
drg_code_pattern = r'\b\d{3}\b'
|
||||
rev_code_pattern = r'\b\d[A-Za-z]\d{2}|\d{2}[A-Za-z]\d|\d{3}[A-Za-z]|[A-Za-z]\d{3}|\d{4}\b'
|
||||
|
||||
# Clean the specified columns
|
||||
df = clean_code_columns(df, 'REIMBURSEMENT_PROC_CODES', cpt_code_pattern)
|
||||
df = clean_code_columns(df, 'REIMBURSEMENT_DIAG_CODES', drg_code_pattern)
|
||||
df = clean_code_columns(df, 'REIMBURSEMENT_REVENUE_CODES', rev_code_pattern)
|
||||
|
||||
clean_df = postprocessingfuncs.clean_pagenumbers(df)
|
||||
return clean_df
|
||||
@@ -25,3 +75,4 @@ def post_process_combined(df, ):
|
||||
def postprocess_results(combined_df):
|
||||
combined_df = post_process_combined(combined_df)
|
||||
return combined_df
|
||||
|
||||
|
||||
+32
-23
@@ -6,12 +6,11 @@ def sanitize_value(value):
|
||||
""" Remove brackets from list items and clean the values. """
|
||||
if pd.isna(value):
|
||||
return value
|
||||
if isinstance(value, list):
|
||||
value = ', '.join(str(v) for v in value)
|
||||
if isinstance(value, str):
|
||||
value = value.strip('[]')
|
||||
if ',' in value:
|
||||
value = ', '.join([item.strip(" '") for item in value.split(',')])
|
||||
else:
|
||||
value = value.strip(" '")
|
||||
value = ', '.join([item.strip(" '") for item in value.split(',')])
|
||||
return value
|
||||
|
||||
def exact_match(val, valid_values):
|
||||
@@ -24,11 +23,12 @@ def exact_match(val, valid_values):
|
||||
def clean_columns_combined(df, column_name, valid_values, new_column_name):
|
||||
""" Cleans a column by applying an exact match check and updates it to a new column. """
|
||||
changes = {}
|
||||
df[column_name] = df[column_name].apply(sanitize_value)
|
||||
original_values = df[column_name].unique()
|
||||
|
||||
def update_column(entry):
|
||||
if pd.notna(entry):
|
||||
terms = sanitize_value(entry).split(',')
|
||||
terms = entry.split(',')
|
||||
for term in terms:
|
||||
match = exact_match(term, valid_values)
|
||||
if match:
|
||||
@@ -39,7 +39,7 @@ def clean_columns_combined(df, column_name, valid_values, new_column_name):
|
||||
cleaned_values = df[new_column_name].unique()
|
||||
return original_values, cleaned_values, changes
|
||||
|
||||
def get_closest_match(val, valid_values, similarity_threshold=0.5):
|
||||
def get_closest_match(val, valid_values, similarity_threshold=0.7):
|
||||
if pd.isna(val):
|
||||
return None
|
||||
val = val.strip().upper()
|
||||
@@ -49,11 +49,12 @@ def get_closest_match(val, valid_values, similarity_threshold=0.5):
|
||||
def clean_columns_combined_fuzzy(df, column_name, valid_values, threshold):
|
||||
""" Applies fuzzy matching to a column in the dataframe and logs changes. """
|
||||
changes = {}
|
||||
df[column_name] = df[column_name].apply(sanitize_value)
|
||||
original_values = df[column_name].unique()
|
||||
|
||||
def log_and_clean(entry):
|
||||
if pd.notna(entry):
|
||||
words = sanitize_value(entry).split(',')
|
||||
words = entry.split(',')
|
||||
cleaned_words = []
|
||||
for word in words:
|
||||
cleaned_word = get_closest_match(word.strip(), valid_values, similarity_threshold=threshold)
|
||||
@@ -75,21 +76,29 @@ def extract_page_number(page_text):
|
||||
return page_text
|
||||
|
||||
def clean_pagenumbers(df):
|
||||
df['Page'] = df['Page'].apply(extract_page_number)
|
||||
df['page_num'] = df['page_num'].apply(extract_page_number)
|
||||
return df
|
||||
|
||||
def consolidate_csv(df, output_csv='output/consolidated.csv'):
|
||||
df['page_num'] = df['Page'].apply(lambda x: x.split()[1])
|
||||
|
||||
grouped = df.groupby(['Filename', 'Corrected_LOB']).agg({
|
||||
'PRODUCT': lambda x: ', '.join(x.dropna().unique()),
|
||||
'Corrected_NETWORK': lambda x: ', '.join(x.dropna().unique()),
|
||||
'CONTRACT_SERVICE_AREA': lambda x: ', '.join(x.dropna().unique()),
|
||||
'LOB_PRICING_TERMS_EFFECTIVE_DT': lambda x: ', '.join(x.dropna().unique()),
|
||||
'LOB_PRICING_TERMS_TERMINATION_DT': lambda x: ', '.join(x.dropna().unique()),
|
||||
'CONTRACT_MARKETPLACE_METAL_LEVEL': lambda x: ', '.join(x.dropna().unique()),
|
||||
'page_num' : lambda x : ', '.join(x.dropna().unique())
|
||||
}).reset_index()
|
||||
|
||||
grouped.to_csv(output_csv, index=False)
|
||||
print(f"Consolidated CSV has been saved to {output_csv}")
|
||||
def correct_misplaced_values(df, columns, valid_values_dict):
|
||||
"""
|
||||
Check and correct misplaced values across specified columns.
|
||||
"""
|
||||
for index, row in df.iterrows():
|
||||
for col in columns:
|
||||
if pd.notna(row[col]):
|
||||
terms = row[col].split(',')
|
||||
for term in terms:
|
||||
term = term.strip()
|
||||
for target_col, valid_values in valid_values_dict.items():
|
||||
if target_col != col:
|
||||
match = exact_match(term, valid_values)
|
||||
if match:
|
||||
if pd.isna(row[target_col]) or not row[target_col].strip():
|
||||
df.at[index, target_col] = match
|
||||
df.at[index, col] = None
|
||||
else:
|
||||
current_value = row[target_col].strip()
|
||||
if get_closest_match(match, [current_value], 0.8) is None:
|
||||
df.at[index, 'Corrected_' + target_col] = f"Found {term} in {col} cell"
|
||||
df.at[index, col] = None
|
||||
return df
|
||||
|
||||
@@ -148,6 +148,8 @@ Read and analyze the page, then populate a JSON dictionary with the following co
|
||||
|
||||
Return a value for each type of code - return N/A for any values not found. Multiple codes for each type may be found. In this case, return them all. They may also present as a range. If so, return the range.
|
||||
|
||||
Ensure you ONLY return codes if they are associated directly with the Service listed. Codes referring to other services should not be included.
|
||||
|
||||
Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting.
|
||||
"""
|
||||
|
||||
|
||||
+20
-3
@@ -1,5 +1,22 @@
|
||||
|
||||
import utils
|
||||
import preprocess
|
||||
import table_funcs
|
||||
import prompts
|
||||
import prompt_funcs
|
||||
|
||||
input_dict = utils.read_input()
|
||||
|
||||
filename = list(input_dict.keys())[0]
|
||||
contract_text = input_dict[filename]
|
||||
|
||||
|
||||
contract_text = preprocess.clean_newlines(contract_text)
|
||||
text_dict = preprocess.split_text(contract_text)
|
||||
text_dict = table_funcs.align_and_format_tables(text_dict)
|
||||
text_dict = preprocess.highlight_rates(text_dict)
|
||||
|
||||
bu_results = prompt_funcs.run_bottom_up(filename, text_dict)
|
||||
|
||||
print(bu_results)
|
||||
|
||||
#import llama_index
|
||||
import openai
|
||||
print(openai.__version__)
|
||||
Reference in New Issue
Block a user