From 6889101f6bba9e7fa7cb6c9663399077ccb7d681 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Fri, 7 Jun 2024 08:48:16 -0700 Subject: [PATCH] Add error handling for postprocessing --- src/config.py | 5 ++- src/file_processing.py | 38 +++++++++++--------- src/postprocess.py | 73 ++++++++++++++++++++++++++++++-------- src/postprocessingfuncs.py | 2 +- 4 files changed, 84 insertions(+), 34 deletions(-) diff --git a/src/config.py b/src/config.py index 432bec4..ab922f1 100644 --- a/src/config.py +++ b/src/config.py @@ -34,13 +34,16 @@ RUN_FS = True RUN_EXCEPTION = True RUN_CODES = True +# Postprocessing Settings +FUZZY_MATCH_THRESHOLD = 0.8 + # AWS Keys AWS_ACCESS_KEY_ID="ASIAZTMXAXNXAI7RYNWW" AWS_SECRET_ACCESS_KEY="fPm3Rn3cde8AOVj0hP/n3u1MwSrQU17qxDPRK7ar" AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEGYaCXVzLWVhc3QtMiJHMEUCIEFjYLb1fysSczLo2Ae7zUOD61H0YhfYPtrzlwXGw6TxAiEAnPVt3UX0UGGxJrAS3T14mVud26zn8i2qIbgjUZAkQmIqjAMI7///////////ARAAGgw2NjAxMzEwNjg3ODIiDH2L5zRJcBcGAXDxMSrgAhoB9QUZFydQiXFS7YUNrYBwSRX3fxPM6dml/WETf9qesJ7tkuOlXJIa7ZZLjJreuMCRfS4eUis0mMDZg+8pb1lLYQAAwuCe6JHN757NH7lwOXSdLY/pzqHc+w4atcGfKNsexj6BIGCyOKbqjbA0wcMgz1y5DqEY8lBeU+E/FTMBa6r9SKYH2hyD41KZQrtUAbKeOWrw2lMjxsr4TzlkdHXw8uQJC4WY7uC6iAzZoR+WDQvfOFu1ltzeJlMiJZA74V1pStK0s/f+QtlwmT6KLu8gVafepohAFs4tuWlz5Ohmn0T/hGQURD8Em1dU22Ljj0qfcGejEUFw8HQDI5kYtwFBDU4XLnDh5Z01Dyxidted8xV3DNI6Ok79cAT950jXXSipBuHKxNv9qMAbrgrAir5GUPhs04XnocjIw7FpOoYzglURU+Y/R+wHeOtXRou4VcDLdoOtlrnrKX9/WR5eOI8woZ+MswY6pgG+tl5kEnjwJJDMCErNLFBnAmBHaC+0oBFRPOaX7ZYHY7ryIfo21kqLEkgXD54XY6xRIyfpEMjBIled+AUOJC17BEjZUq3tUUMi1FEo4KFwANQntvQVDHQ+nwmtNQ4NWeCBfBOqqEArsXjG3SROTQOBKr75NHlPFYtsTr2vK03xCe6dVF/Uq0giyXvoEMyT9eCJTFW1Bzs9HUpUhG9h0lgO2q4LKaLd" # File Paths -LOCAL_PATH = 'data/priority_health/' # Replace with local +LOCAL_PATH = 'data/test/' # Replace with local # S3 Settings S3_CLIENT = boto3.client('s3', diff --git a/src/file_processing.py b/src/file_processing.py index e35f8c7..c0afa8e 100644 --- a/src/file_processing.py +++ b/src/file_processing.py @@ -93,45 +93,49 @@ def merge_results(td_results, bu_results, text_dict): def process_file(file_object): + + # Start Processing filename, contract_text = file_object if config.VERBOSE: print(f"Processing {filename}...") + # Create directories + base_filename = os.path.splitext(filename)[0].strip() + output_dir = os.path.join(config.OUTPUT_FOLDER, base_filename) + os.makedirs(output_dir, exist_ok=True) + + #### PREPROCESS #### contract_text = preprocess.clean_newlines(contract_text) text_dict = preprocess.split_text(contract_text) text_dict = table_funcs.align_and_format_tables(text_dict) text_dict = preprocess.highlight_rates(text_dict) - # Run Top Down + #### Run Top Down #### td_results = prompt_funcs.run_top_down(filename, text_dict) # Returns list of dictionaries for each page - print(f"Top Down Complete - {filename} ") + # Write TD results + pd.DataFrame(td_results).to_csv(os.path.join(output_dir, 'td_results.csv'), index=False) + print(f"Top Down Complete - {filename}") - # Run Bottom Up + #### Run Bottom Up #### bu_results = prompt_funcs.run_bottom_up(filename, text_dict) # Returns list of dictionaries + # Write BU results + pd.DataFrame(bu_results).to_csv(os.path.join(output_dir, 'bu_results.csv'), index=False) print(f"Bottom Up Complete - {filename}") - # Combine + #### Combine #### combined_results = merge_results(clean_td(td_results), bu_results, text_dict) print(f"TD/BU Merge Complete - {filename} ") + #### Write unprocessed output #### # Convert to DataFrame combined_df = pd.DataFrame(combined_results) + combined_df.to_csv(os.path.join(output_dir, 'combined_results_unprocessed.csv'), index=False) - # Post-process combined results + #### POSTPROCESSING #### combined_df = combined_df.applymap(postprocessingfuncs.sanitize_value) post_processed_combined_df = postprocess.postprocess_results(combined_df) - print(f"Postprocessing Complete - {filename} ") - - # Create directories - base_filename = os.path.splitext(filename)[0] - output_dir = os.path.join(config.OUTPUT_FOLDER, base_filename) - os.makedirs(output_dir, exist_ok=True) - - # Save results - pd.DataFrame(td_results).to_csv(os.path.join(output_dir, 'td_results.csv'), index=False) - pd.DataFrame(bu_results).to_csv(os.path.join(output_dir, 'bu_results.csv'), index=False) - combined_df.to_csv(os.path.join(output_dir, 'combined_results_unprocessed.csv'), index=False) post_processed_combined_df.to_csv(os.path.join(output_dir, 'combined_results_post_processed.csv'), index=False) - + print(f"Postprocessing Complete - {filename} ") + print(f"Output Complete - {filename} ") \ No newline at end of file diff --git a/src/postprocess.py b/src/postprocess.py index c7e3bc5..e822a0f 100644 --- a/src/postprocess.py +++ b/src/postprocess.py @@ -1,4 +1,5 @@ import pandas as pd +import config import re import postprocessingfuncs @@ -62,31 +63,73 @@ def post_process_combined(df): df = sanitize_combined(df) # Clean specific columns for exact matches - postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_LOB', VALID_LOBS, 'Corrected_LOB') - postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_PROGRAM', VALID_PROGRAMS, 'Corrected_PROGRAM') - postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_NETWORK', VALID_NETWORKS, 'Corrected_NETWORK') + for field_list in [['CONTRACT_LOB', 'Corrected_LOB', VALID_LOBS], ['CONTRACT_PROGRAM', 'Corrected_PROGRAM', VALID_PROGRAMS], ['CONTRACT_NETWORK', 'Corrected_NETWORK', VALID_NETWORKS]]: + try: + postprocessingfuncs.clean_columns_combined(df, field_list[0], field_list[2], field_list[1]) + except: + pass + + try: + postprocessingfuncs.clean_columns_combined_fuzzy(df, field_list[0], field_list[2], config.FUZZY_MATCH_THRESHOLD) + except: + pass + + # postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_LOB', VALID_LOBS, 'Corrected_LOB') - # Apply fuzzy matching to the columns - postprocessingfuncs.clean_columns_combined_fuzzy(df, 'CONTRACT_LOB', VALID_LOBS, 0.8) - postprocessingfuncs.clean_columns_combined_fuzzy(df, 'CONTRACT_PROGRAM', VALID_PROGRAMS, 0.8) - postprocessingfuncs.clean_columns_combined_fuzzy(df, 'CONTRACT_NETWORK', VALID_NETWORKS, 0.8) + # postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_PROGRAM', VALID_PROGRAMS, 'Corrected_PROGRAM') + # postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_NETWORK', VALID_NETWORKS, 'Corrected_NETWORK') + + # # Apply fuzzy matching to the columns + # try: + # postprocessingfuncs.clean_columns_combined_fuzzy(df, 'CONTRACT_LOB', VALID_LOBS, 0.8) + # except: + # pass + # postprocessingfuncs.clean_columns_combined_fuzzy(df, 'CONTRACT_PROGRAM', VALID_PROGRAMS, 0.8) + # postprocessingfuncs.clean_columns_combined_fuzzy(df, 'CONTRACT_NETWORK', VALID_NETWORKS, 0.8) # Correct misplaced values across columns - df = postprocessingfuncs.correct_misplaced_values(df, ['CONTRACT_LOB', 'CONTRACT_PROGRAM', 'CONTRACT_NETWORK'], valid_values_dict) + try: + df = postprocessingfuncs.correct_misplaced_values(df, ['CONTRACT_LOB', 'CONTRACT_PROGRAM', 'CONTRACT_NETWORK'], valid_values_dict) + except: + pass # Clean the specified code columns - df = clean_code_column(df, 'REIMBURSEMENT_PROC_CODES', extract_codes_CPT) - df = clean_code_column(df, 'REIMBURSEMENT_DIAG_CODES', extract_codes_Diagnosis) - df = clean_code_column(df, 'REIMBURSEMENT_REVENUE_CODES', extract_codes_Revenue) + try: + df = clean_code_column(df, 'REIMBURSEMENT_PROC_CODES', extract_codes_CPT) + except Exception as e: + print(f"Error: {e}") + try: + df = clean_code_column(df, 'REIMBURSEMENT_DIAG_CODES', extract_codes_Diagnosis) + except Exception as e: + print(f"Error: {e}") + try: + df = clean_code_column(df, 'REIMBURSEMENT_REVENUE_CODES', extract_codes_Revenue) + except Exception as e: + print(f"Error: {e}") # Filter out specific SERVICE rows - df = postprocessingfuncs.filter_service_column(df) + try: + df = postprocessingfuncs.filter_service_column(df) + except Exception as e: + print(f"Error: {e}") # Move percentages and large numbers to correct columns - df = postprocessingfuncs.move_percentage_to_rate(df) - df = postprocessingfuncs.move_large_numbers_to_flat_fee(df) + try: + df = postprocessingfuncs.move_percentage_to_rate(df) + except Exception as e: + print(f"Error: {e}") + + try: + df = postprocessingfuncs.move_large_numbers_to_flat_fee(df) + except Exception as e: + print(f"Error: {e}") - clean_df = postprocessingfuncs.clean_pagenumbers(df) + try: + clean_df = postprocessingfuncs.clean_pagenumbers(df) + except Exception as e: + print(f"Error: {e}") + clean_df = df + return clean_df def postprocess_results(combined_df): diff --git a/src/postprocessingfuncs.py b/src/postprocessingfuncs.py index 7fbb9eb..ab94eb1 100644 --- a/src/postprocessingfuncs.py +++ b/src/postprocessingfuncs.py @@ -32,7 +32,7 @@ def clean_columns_combined(df, column_name, valid_values, new_column_name): if match: return match return None - + df[new_column_name] = df[column_name].apply(update_column) cleaned_values = df[new_column_name].unique() return original_values, cleaned_values, changes