jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Add error handling for postprocessing

Browse Source
This commit is contained in:
Katon Minhas
2024-06-07 08:48:16 -07:00
committed by Michael McGuinness
parent d160441365
commit 6889101f6b
4 changed files with 84 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/config.py
+4 -1
View File
@@ -34,13 +34,16 @@ RUN_FS = True
RUN_EXCEPTION = True
RUN_CODES = True
# Postprocessing Settings
FUZZY_MATCH_THRESHOLD = 0.8
# AWS Keys
AWS_ACCESS_KEY_ID="ASIAZTMXAXNXAI7RYNWW"
AWS_SECRET_ACCESS_KEY="fPm3Rn3cde8AOVj0hP/n3u1MwSrQU17qxDPRK7ar"
AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEGYaCXVzLWVhc3QtMiJHMEUCIEFjYLb1fysSczLo2Ae7zUOD61H0YhfYPtrzlwXGw6TxAiEAnPVt3UX0UGGxJrAS3T14mVud26zn8i2qIbgjUZAkQmIqjAMI7///////////ARAAGgw2NjAxMzEwNjg3ODIiDH2L5zRJcBcGAXDxMSrgAhoB9QUZFydQiXFS7YUNrYBwSRX3fxPM6dml/WETf9qesJ7tkuOlXJIa7ZZLjJreuMCRfS4eUis0mMDZg+8pb1lLYQAAwuCe6JHN757NH7lwOXSdLY/pzqHc+w4atcGfKNsexj6BIGCyOKbqjbA0wcMgz1y5DqEY8lBeU+E/FTMBa6r9SKYH2hyD41KZQrtUAbKeOWrw2lMjxsr4TzlkdHXw8uQJC4WY7uC6iAzZoR+WDQvfOFu1ltzeJlMiJZA74V1pStK0s/f+QtlwmT6KLu8gVafepohAFs4tuWlz5Ohmn0T/hGQURD8Em1dU22Ljj0qfcGejEUFw8HQDI5kYtwFBDU4XLnDh5Z01Dyxidted8xV3DNI6Ok79cAT950jXXSipBuHKxNv9qMAbrgrAir5GUPhs04XnocjIw7FpOoYzglURU+Y/R+wHeOtXRou4VcDLdoOtlrnrKX9/WR5eOI8woZ+MswY6pgG+tl5kEnjwJJDMCErNLFBnAmBHaC+0oBFRPOaX7ZYHY7ryIfo21kqLEkgXD54XY6xRIyfpEMjBIled+AUOJC17BEjZUq3tUUMi1FEo4KFwANQntvQVDHQ+nwmtNQ4NWeCBfBOqqEArsXjG3SROTQOBKr75NHlPFYtsTr2vK03xCe6dVF/Uq0giyXvoEMyT9eCJTFW1Bzs9HUpUhG9h0lgO2q4LKaLd"
# File Paths
LOCAL_PATH = 'data/priority_health/' # Replace with local
LOCAL_PATH = 'data/test/' # Replace with local
# S3 Settings
S3_CLIENT = boto3.client('s3',
src/file_processing.py
+21 -17
View File
@@ -93,45 +93,49 @@ def merge_results(td_results, bu_results, text_dict):
def process_file(file_object):
# Start Processing
filename, contract_text = file_object
if config.VERBOSE: print(f"Processing {filename}...")
# Create directories
base_filename = os.path.splitext(filename)[0].strip()
output_dir = os.path.join(config.OUTPUT_FOLDER, base_filename)
os.makedirs(output_dir, exist_ok=True)
#### PREPROCESS ####
contract_text = preprocess.clean_newlines(contract_text)
text_dict = preprocess.split_text(contract_text)
text_dict = table_funcs.align_and_format_tables(text_dict)
text_dict = preprocess.highlight_rates(text_dict)
# Run Top Down
#### Run Top Down ####
td_results = prompt_funcs.run_top_down(filename, text_dict) # Returns list of dictionaries for each page
print(f"Top Down Complete - {filename} ")
# Write TD results
pd.DataFrame(td_results).to_csv(os.path.join(output_dir, 'td_results.csv'), index=False)
print(f"Top Down Complete - {filename}")
# Run Bottom Up
#### Run Bottom Up ####
bu_results = prompt_funcs.run_bottom_up(filename, text_dict) # Returns list of dictionaries
# Write BU results
pd.DataFrame(bu_results).to_csv(os.path.join(output_dir, 'bu_results.csv'), index=False)
print(f"Bottom Up Complete - {filename}")
# Combine
#### Combine ####
combined_results = merge_results(clean_td(td_results), bu_results, text_dict)
print(f"TD/BU Merge Complete - {filename} ")
#### Write unprocessed output ####
# Convert to DataFrame
combined_df = pd.DataFrame(combined_results)
combined_df.to_csv(os.path.join(output_dir, 'combined_results_unprocessed.csv'), index=False)
# Post-process combined results
#### POSTPROCESSING ####
combined_df = combined_df.applymap(postprocessingfuncs.sanitize_value)
post_processed_combined_df = postprocess.postprocess_results(combined_df)
print(f"Postprocessing Complete - {filename} ")
# Create directories
base_filename = os.path.splitext(filename)[0]
output_dir = os.path.join(config.OUTPUT_FOLDER, base_filename)
os.makedirs(output_dir, exist_ok=True)
# Save results
pd.DataFrame(td_results).to_csv(os.path.join(output_dir, 'td_results.csv'), index=False)
pd.DataFrame(bu_results).to_csv(os.path.join(output_dir, 'bu_results.csv'), index=False)
combined_df.to_csv(os.path.join(output_dir, 'combined_results_unprocessed.csv'), index=False)
post_processed_combined_df.to_csv(os.path.join(output_dir, 'combined_results_post_processed.csv'), index=False)
print(f"Postprocessing Complete - {filename} ")
print(f"Output Complete - {filename} ")
src/postprocess.py
+58 -15
View File
@@ -1,4 +1,5 @@
import pandas as pd
import config
import re
import postprocessingfuncs
@@ -62,31 +63,73 @@ def post_process_combined(df):
df = sanitize_combined(df)
# Clean specific columns for exact matches
postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_LOB', VALID_LOBS, 'Corrected_LOB')
postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_PROGRAM', VALID_PROGRAMS, 'Corrected_PROGRAM')
postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_NETWORK', VALID_NETWORKS, 'Corrected_NETWORK')
for field_list in [['CONTRACT_LOB', 'Corrected_LOB', VALID_LOBS], ['CONTRACT_PROGRAM', 'Corrected_PROGRAM', VALID_PROGRAMS], ['CONTRACT_NETWORK', 'Corrected_NETWORK', VALID_NETWORKS]]:
try:
postprocessingfuncs.clean_columns_combined(df, field_list[0], field_list[2], field_list[1])
except:
pass
try:
postprocessingfuncs.clean_columns_combined_fuzzy(df, field_list[0], field_list[2], config.FUZZY_MATCH_THRESHOLD)
except:
pass
# postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_LOB', VALID_LOBS, 'Corrected_LOB')
# Apply fuzzy matching to the columns
postprocessingfuncs.clean_columns_combined_fuzzy(df, 'CONTRACT_LOB', VALID_LOBS, 0.8)
postprocessingfuncs.clean_columns_combined_fuzzy(df, 'CONTRACT_PROGRAM', VALID_PROGRAMS, 0.8)
postprocessingfuncs.clean_columns_combined_fuzzy(df, 'CONTRACT_NETWORK', VALID_NETWORKS, 0.8)
# postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_PROGRAM', VALID_PROGRAMS, 'Corrected_PROGRAM')
# postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_NETWORK', VALID_NETWORKS, 'Corrected_NETWORK')
# # Apply fuzzy matching to the columns
# try:
# postprocessingfuncs.clean_columns_combined_fuzzy(df, 'CONTRACT_LOB', VALID_LOBS, 0.8)
# except:
# pass
# postprocessingfuncs.clean_columns_combined_fuzzy(df, 'CONTRACT_PROGRAM', VALID_PROGRAMS, 0.8)
# postprocessingfuncs.clean_columns_combined_fuzzy(df, 'CONTRACT_NETWORK', VALID_NETWORKS, 0.8)
# Correct misplaced values across columns
df = postprocessingfuncs.correct_misplaced_values(df, ['CONTRACT_LOB', 'CONTRACT_PROGRAM', 'CONTRACT_NETWORK'], valid_values_dict)
try:
df = postprocessingfuncs.correct_misplaced_values(df, ['CONTRACT_LOB', 'CONTRACT_PROGRAM', 'CONTRACT_NETWORK'], valid_values_dict)
except:
pass
# Clean the specified code columns
df = clean_code_column(df, 'REIMBURSEMENT_PROC_CODES', extract_codes_CPT)
df = clean_code_column(df, 'REIMBURSEMENT_DIAG_CODES', extract_codes_Diagnosis)
df = clean_code_column(df, 'REIMBURSEMENT_REVENUE_CODES', extract_codes_Revenue)
try:
df = clean_code_column(df, 'REIMBURSEMENT_PROC_CODES', extract_codes_CPT)
except Exception as e:
print(f"Error: {e}")
try:
df = clean_code_column(df, 'REIMBURSEMENT_DIAG_CODES', extract_codes_Diagnosis)
except Exception as e:
print(f"Error: {e}")
try:
df = clean_code_column(df, 'REIMBURSEMENT_REVENUE_CODES', extract_codes_Revenue)
except Exception as e:
print(f"Error: {e}")
# Filter out specific SERVICE rows
df = postprocessingfuncs.filter_service_column(df)
try:
df = postprocessingfuncs.filter_service_column(df)
except Exception as e:
print(f"Error: {e}")
# Move percentages and large numbers to correct columns
df = postprocessingfuncs.move_percentage_to_rate(df)
df = postprocessingfuncs.move_large_numbers_to_flat_fee(df)
try:
df = postprocessingfuncs.move_percentage_to_rate(df)
except Exception as e:
print(f"Error: {e}")
try:
df = postprocessingfuncs.move_large_numbers_to_flat_fee(df)
except Exception as e:
print(f"Error: {e}")
clean_df = postprocessingfuncs.clean_pagenumbers(df)
try:
clean_df = postprocessingfuncs.clean_pagenumbers(df)
except Exception as e:
print(f"Error: {e}")
clean_df = df
return clean_df
def postprocess_results(combined_df):
src/postprocessingfuncs.py
+1 -1
View File
@@ -32,7 +32,7 @@ def clean_columns_combined(df, column_name, valid_values, new_column_name):
if match:
return match
return None
df[new_column_name] = df[column_name].apply(update_column)
cleaned_values = df[new_column_name].unique()
return original_values, cleaned_values, changes