diff --git a/src/config.py b/src/config.py index c4c65c6..9ba5975 100644 --- a/src/config.py +++ b/src/config.py @@ -35,9 +35,9 @@ RUN_EXCEPTION = True RUN_CODES = True # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXFTD5MQUK" -AWS_SECRET_ACCESS_KEY="yGCDnqRupWsJyABleBXd3NkzySjf2tuU1L5pPsUh" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEDUaCXVzLWVhc3QtMiJIMEYCIQC79KaOSV9wiHBrCPKOdoGQ10Hhl2BlX9kN2LI0AWtbzQIhAOniVNrVlB0wbuaJUVcFjuV8LRO/ZZdCk4ac9O9HftrdKowDCL7//////////wEQABoMNjYwMTMxMDY4NzgyIgz6LOHKqD7pmvQXBf8q4AJkRxmxk5UhNmr6iAx+zsccBvcFR2XBFywrgrPCHk5mUzQXfgPFBmcTW7XTB2F2Rie5y/oZGV1YxK8s2JYXd9gkbJvwUt7K1bNjQYg/IMQ1goecxH2dRHq//JYDs9TQiU01nacnI6eMPQqSCpmtc2yI45GovWwONKJviddUI5Ln/IS7Ktq4dcmy7xx/GqbgqsWb3bCFhmT/9UrOGaBSOHA0fhshKaw8ihbWgz1BtVFSt2i8FsTXRlNXn5l79iwo+vWs4CQzmHgRjw3ZIytrw2V2nppgJlw1Tb+AEwBdbC9wW9WsmDmw9QmBeyxcKAKkmPf66wWqmBqK0/2w8r4q6H7gEta/Ibc08137zd0fLWEu74iZE0I5jhLYOGwh+U0DhsR0ZDufWurcA+YIfWZAsfG0PePonv8+6drcqtNjTqKs0KIfAQ6PHAb4xuoCYEzvLeyMcY1t5sOl/oU+RWxkPrsIMMLLgbMGOqUBfx6TS5599bddNIEV2CH3AHym/18n3lyA19PHcVcfldqU7nMrYTZdgNBMy1Or1OVjcxYV988OI8ZOqWPiYzMGR6cVSzwA1ESjLUVma3vKcOTq4GbgBvRxA44PMfBm/P3ow3Vhx3ONe/oaxqgKre26JBheTN5s0Uua4lEcO4lvgB6/Kap+/aaR9vnEwLtJZ8u5d84k/Ajw1nG8bDMfLPsyWXjtUUZI" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXG7BOCV4A" +AWS_SECRET_ACCESS_KEY="xDx1H2SOL71C1N9jl9pv5HqCyLccRsYs8vQvPU5y" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjED0aCXVzLWVhc3QtMiJIMEYCIQCJo1mxCG+aqqh1B5Jy1vvEcANPejqBUIOVnbPcUs5/+QIhAJmpSAygER0WddjIenyY8hu0V4aqiFCsdBBQbkupzvwzKowDCMf//////////wEQABoMNjYwMTMxMDY4NzgyIgxuroZvh2Y2VPEKEYgq4AK36l5Ls8q/xw+e0A1s6+TpUjvecAr9tUrwEF5MyY0ToWSyac7/N8y/g7p8Kd9lMiJqJ1C7YlePsgGQen9BBxLeLWp/yEWsKuQsyJmh2mcO9DtSFYkWeZgRmxT6KVflCySKzMDOtytwU1vzwrbjQeh8pRG7zijvO0WvGPfBykHYOp9dIGIG8kGa6IUq58EhpzKtCSahOFbaddamvzf+HabTZ45tv5RJbG/Lrw3FqEFAzrj7RkiUS3dnJoScH595PtEHZ3IHDYE2jvyeS/FqV+t454IvTV4avq9SePDtAAjw4CnQVyFR7gchPil+qIKgt2+IDHaqTAIRbR3isC/X2TTS8kw7pkm1Uqgz60DWsFrjrpWpIGqHAYmxmnbEX0rNkGzJgaX6YdDF2o7LTm6z0rBvjmY96uKk1QPs07tVJMxklx1+QFzMzpnZEuDp8KEaKW9xQg3IGP2TuSJM8ZAKDQ4uMNaug7MGOqUBRaIqNEebTiBJ9b1LvNOyonXyhqeq8GsA4y0MPcUDnk5JDFUfUt/1o6gefIV6pFsel5RIwHf49rfsrIsGQs5YzuViwP3whkloYPbg8ZXlQTJRVTXHaOqjQHVfeT26aU9S4VOFfqH8N/dhLVNUMpesutnkzrngyv0nUTubYt7LtBF5A0RvcTqMMfmU9pbjUTNk1fhsd4mOBAuHDrOTMXm+1LMvDJyc" # File Paths LOCAL_PATH = 'data/test/' # Replace with local diff --git a/src/file_processing.py b/src/file_processing.py index 1507097..823c296 100644 --- a/src/file_processing.py +++ b/src/file_processing.py @@ -8,6 +8,8 @@ import table_funcs import prompt_funcs import prompts import claude_funcs +import postprocessingfuncs +import postprocess def clean_td(td): td_clean = [] @@ -104,22 +106,22 @@ def process_file(file_object): # Run Top Down td_results = prompt_funcs.run_top_down(filename, text_dict) # Returns list of dictionaries for each page - print("TD Results:", td_results) + print("Top Down complete...") # Run Bottom Up bu_results = prompt_funcs.run_bottom_up(filename, text_dict) # Returns list of dictionaries - print("BU Results:", bu_results) + print("Bottom Up complete...") # Combine combined_results = merge_results(clean_td(td_results), bu_results, text_dict) - print("Combined Results:", combined_results) + print("TD/BU Merge complete...") # Convert to DataFrame combined_df = pd.DataFrame(combined_results) # Post-process combined results - # post_processed_combined_df = postprocess.postprocess_results(combined_df) - # print("Post-processed Results:", post_processed_combined_df) + combined_df = combined_df.applymap(postprocessingfuncs.sanitize_value) + post_processed_combined_df = postprocess.postprocess_results(combined_df) # Create directories base_filename = os.path.splitext(filename)[0] @@ -129,6 +131,5 @@ def process_file(file_object): # Save results pd.DataFrame(td_results).to_csv(os.path.join(output_dir, 'td_results.csv'), index=False) pd.DataFrame(bu_results).to_csv(os.path.join(output_dir, 'bu_results.csv'), index=False) - combined_df.to_csv(os.path.join(output_dir, 'combined_results.csv'), index=False) - # post_processed_combined_df.to_csv(os.path.join(output_dir, 'combined_results.csv'), index=False) - + post_processed_combined_df.to_csv(os.path.join(output_dir, 'combined_results_post_processed.csv'), index=False) + \ No newline at end of file diff --git a/src/postprocess.py b/src/postprocess.py index ee807f1..787dbc4 100644 --- a/src/postprocess.py +++ b/src/postprocess.py @@ -1,4 +1,5 @@ import pandas as pd +import re import postprocessingfuncs def sanitize_combined(df): @@ -7,17 +8,66 @@ def sanitize_combined(df): df[column] = df[column].apply(postprocessingfuncs.sanitize_value) return df -def post_process_combined(df, ): +def extract_codes(value, pattern): + """ + Extract codes from a given string using a specified regex pattern. + """ + if pd.isna(value): + return value + matches = re.findall(pattern, str(value)) + if matches: + return ', '.join(matches) + else: + # Check if there are any numbers in the value + if any(char.isdigit() for char in value): + return value + else: + return "" + +def clean_code_columns(df, column_name, pattern): + """ + Clean the specified column in the DataFrame by extracting codes based on the provided regex pattern. + """ + df[column_name] = df[column_name].apply(lambda x: extract_codes(x, pattern)) + return df + +def post_process_combined(df): VALID_LOBS = ['MEDICARE', 'MEDICARE ADVANTAGE', 'MEDICAID', 'MARKETPLACE', 'COMMERCIAL', 'GROUP', 'MEDICARE-MEDICAID'] VALID_PROGRAMS = ['CHIP', 'CHIP-P', 'CHIP-PERINATE', 'STAR', 'STAR+PLUS', 'MA', 'DUAL SPECIAL NEEDS PLAN', 'DSNP', 'DUAL'] VALID_NETWORKS = ['HMO', 'PPO', 'EPO', 'POS', 'FFS'] + + # Define valid values dictionary + valid_values_dict = { + 'CONTRACT_LOB': VALID_LOBS, + 'CONTRACT_PROGRAM': VALID_PROGRAMS, + 'CONTRACT_NETWORK': VALID_NETWORKS + } # Sanitize the combined data df = sanitize_combined(df) + # Clean specific columns for exact matches postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_LOB', VALID_LOBS, 'Corrected_LOB') - postprocessingfuncs.clean_columns_combined(df, 'PRODUCT', VALID_PROGRAMS, 'Corrected_PRODUCT') + postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_PROGRAM', VALID_PROGRAMS, 'Corrected_PROGRAM') postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_NETWORK', VALID_NETWORKS, 'Corrected_NETWORK') + + # Apply fuzzy matching to the columns + postprocessingfuncs.clean_columns_combined_fuzzy(df, 'CONTRACT_LOB', VALID_LOBS, 0.8) + postprocessingfuncs.clean_columns_combined_fuzzy(df, 'CONTRACT_PROGRAM', VALID_PROGRAMS, 0.8) + postprocessingfuncs.clean_columns_combined_fuzzy(df, 'CONTRACT_NETWORK', VALID_NETWORKS, 0.8) + + # Correct misplaced values across columns + df = postprocessingfuncs.correct_misplaced_values(df, ['CONTRACT_LOB', 'CONTRACT_PROGRAM', 'CONTRACT_NETWORK'], valid_values_dict) + + # Define the patterns for the different code types + cpt_code_pattern = r'[A-Za-z0-9]{4}[A-Za-z]|[A-Za-z0-9]{3}[A-Za-z][0-9]|[A-Za-z0-9]{2}[A-Za-z][0-9]{2}|[A-Za-z0-9][A-Za-z][0-9]{3}|[A-Za-z][0-9]{4}' + drg_code_pattern = r'\b\d{3}\b' + rev_code_pattern = r'\b\d[A-Za-z]\d{2}|\d{2}[A-Za-z]\d|\d{3}[A-Za-z]|[A-Za-z]\d{3}|\d{4}\b' + + # Clean the specified columns + df = clean_code_columns(df, 'REIMBURSEMENT_PROC_CODES', cpt_code_pattern) + df = clean_code_columns(df, 'REIMBURSEMENT_DIAG_CODES', drg_code_pattern) + df = clean_code_columns(df, 'REIMBURSEMENT_REVENUE_CODES', rev_code_pattern) clean_df = postprocessingfuncs.clean_pagenumbers(df) return clean_df @@ -25,3 +75,4 @@ def post_process_combined(df, ): def postprocess_results(combined_df): combined_df = post_process_combined(combined_df) return combined_df + diff --git a/src/postprocessingfuncs.py b/src/postprocessingfuncs.py index 9dfdd49..0325a76 100644 --- a/src/postprocessingfuncs.py +++ b/src/postprocessingfuncs.py @@ -6,12 +6,11 @@ def sanitize_value(value): """ Remove brackets from list items and clean the values. """ if pd.isna(value): return value + if isinstance(value, list): + value = ', '.join(str(v) for v in value) if isinstance(value, str): value = value.strip('[]') - if ',' in value: - value = ', '.join([item.strip(" '") for item in value.split(',')]) - else: - value = value.strip(" '") + value = ', '.join([item.strip(" '") for item in value.split(',')]) return value def exact_match(val, valid_values): @@ -24,11 +23,12 @@ def exact_match(val, valid_values): def clean_columns_combined(df, column_name, valid_values, new_column_name): """ Cleans a column by applying an exact match check and updates it to a new column. """ changes = {} + df[column_name] = df[column_name].apply(sanitize_value) original_values = df[column_name].unique() def update_column(entry): if pd.notna(entry): - terms = sanitize_value(entry).split(',') + terms = entry.split(',') for term in terms: match = exact_match(term, valid_values) if match: @@ -39,7 +39,7 @@ def clean_columns_combined(df, column_name, valid_values, new_column_name): cleaned_values = df[new_column_name].unique() return original_values, cleaned_values, changes -def get_closest_match(val, valid_values, similarity_threshold=0.5): +def get_closest_match(val, valid_values, similarity_threshold=0.7): if pd.isna(val): return None val = val.strip().upper() @@ -49,11 +49,12 @@ def get_closest_match(val, valid_values, similarity_threshold=0.5): def clean_columns_combined_fuzzy(df, column_name, valid_values, threshold): """ Applies fuzzy matching to a column in the dataframe and logs changes. """ changes = {} + df[column_name] = df[column_name].apply(sanitize_value) original_values = df[column_name].unique() def log_and_clean(entry): if pd.notna(entry): - words = sanitize_value(entry).split(',') + words = entry.split(',') cleaned_words = [] for word in words: cleaned_word = get_closest_match(word.strip(), valid_values, similarity_threshold=threshold) @@ -75,21 +76,29 @@ def extract_page_number(page_text): return page_text def clean_pagenumbers(df): - df['Page'] = df['Page'].apply(extract_page_number) + df['page_num'] = df['page_num'].apply(extract_page_number) return df -def consolidate_csv(df, output_csv='output/consolidated.csv'): - df['page_num'] = df['Page'].apply(lambda x: x.split()[1]) - - grouped = df.groupby(['Filename', 'Corrected_LOB']).agg({ - 'PRODUCT': lambda x: ', '.join(x.dropna().unique()), - 'Corrected_NETWORK': lambda x: ', '.join(x.dropna().unique()), - 'CONTRACT_SERVICE_AREA': lambda x: ', '.join(x.dropna().unique()), - 'LOB_PRICING_TERMS_EFFECTIVE_DT': lambda x: ', '.join(x.dropna().unique()), - 'LOB_PRICING_TERMS_TERMINATION_DT': lambda x: ', '.join(x.dropna().unique()), - 'CONTRACT_MARKETPLACE_METAL_LEVEL': lambda x: ', '.join(x.dropna().unique()), - 'page_num' : lambda x : ', '.join(x.dropna().unique()) - }).reset_index() - - grouped.to_csv(output_csv, index=False) - print(f"Consolidated CSV has been saved to {output_csv}") +def correct_misplaced_values(df, columns, valid_values_dict): + """ + Check and correct misplaced values across specified columns. + """ + for index, row in df.iterrows(): + for col in columns: + if pd.notna(row[col]): + terms = row[col].split(',') + for term in terms: + term = term.strip() + for target_col, valid_values in valid_values_dict.items(): + if target_col != col: + match = exact_match(term, valid_values) + if match: + if pd.isna(row[target_col]) or not row[target_col].strip(): + df.at[index, target_col] = match + df.at[index, col] = None + else: + current_value = row[target_col].strip() + if get_closest_match(match, [current_value], 0.8) is None: + df.at[index, 'Corrected_' + target_col] = f"Found {term} in {col} cell" + df.at[index, col] = None + return df diff --git a/src/prompts.py b/src/prompts.py index 86fffde..0e6567b 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -148,6 +148,8 @@ Read and analyze the page, then populate a JSON dictionary with the following co Return a value for each type of code - return N/A for any values not found. Multiple codes for each type may be found. In this case, return them all. They may also present as a range. If so, return the range. +Ensure you ONLY return codes if they are associated directly with the Service listed. Codes referring to other services should not be included. + Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting. """ diff --git a/src/test.py b/src/test.py index 9d3aa07..86893aa 100644 --- a/src/test.py +++ b/src/test.py @@ -1,5 +1,22 @@ +import utils +import preprocess +import table_funcs +import prompts +import prompt_funcs + +input_dict = utils.read_input() + +filename = list(input_dict.keys())[0] +contract_text = input_dict[filename] + + +contract_text = preprocess.clean_newlines(contract_text) +text_dict = preprocess.split_text(contract_text) +text_dict = table_funcs.align_and_format_tables(text_dict) +text_dict = preprocess.highlight_rates(text_dict) + +bu_results = prompt_funcs.run_bottom_up(filename, text_dict) + +print(bu_results) -#import llama_index -import openai -print(openai.__version__) \ No newline at end of file