diff --git a/fieldExtraction/src/bottom_up_funcs.py b/fieldExtraction/src/bottom_up_funcs.py index 23e49fa..ad40dd0 100644 --- a/fieldExtraction/src/bottom_up_funcs.py +++ b/fieldExtraction/src/bottom_up_funcs.py @@ -28,7 +28,7 @@ def run_bottom_up(filename, text_dict): answer_strings = run_bottom_up_primary(text_dict, 8000) answer_dicts = dict_operations.primary_string_to_dict(answer_strings, filename) answer_dicts_filtered = postprocess_funcs.filter_service_column(answer_dicts) - + # Bottom Up Secondary results_dicts = run_bottom_up_secondary(answer_dicts_filtered, text_dict, 8000) @@ -37,9 +37,9 @@ def run_bottom_up(filename, text_dict): # Add Filename for d in results_dicts: - d['Filename'] = filename - - return results_dicts # List of dictionaries + d["Filename"] = filename + + return results_dicts # List of dictionaries def run_bottom_up_primary(text_dict, tokens): @@ -147,7 +147,6 @@ def run_bottom_up_secondary(answer_dicts, text_dict, tokens): return final_dicts - def run_bottom_up_lesser(d, text_dict, tokens=4000): """ Processes specific clauses or conditions within the document text, such as 'lesser of' or 'greater of', diff --git a/fieldExtraction/src/categorization.py b/fieldExtraction/src/categorization.py index bf4bd98..162dd1e 100644 --- a/fieldExtraction/src/categorization.py +++ b/fieldExtraction/src/categorization.py @@ -1,4 +1,3 @@ - import os import utils @@ -7,28 +6,40 @@ import table_funcs def contains_reimbursement(text_dict, page): - return ('%' in text_dict[page] or '$' in text_dict[page] or 'percent' in text_dict[page].lower()) + return ( + "%" in text_dict[page] + or "$" in text_dict[page] + or "percent" in text_dict[page].lower() + ) -directory_path = "C:\\Users\\kminhas\\Documents\\Doczy\\doczy.ai\\data\\centene_healthnet" -#input_dict = utils.read_input(path=folder_path) + +directory_path = ( + "C:\\Users\\kminhas\\Documents\\Doczy\\doczy.ai\\data\\centene_healthnet" +) +# input_dict = utils.read_input(path=folder_path) all_files = {} reimbursement_count = 0 total_count = 0 # Walk through all directories and files in the directory for root, dirs, files in os.walk(directory_path): for file in files: - if file.endswith('.txt'): + if file.endswith(".txt"): # Construct full file path file_path = os.path.join(root, file) print(file_path) try: - with open(file_path, 'r', encoding='utf-8') as file: + with open(file_path, "r", encoding="utf-8") as file: contract_text = file.read() - if ('%' in contract_text or '$' in contract_text or 'percent' in contract_text.lower()): + if ( + "%" in contract_text + or "$" in contract_text + or "percent" in contract_text.lower() + ): reimbursement_count += 1 total_count += 1 except: continue -print(f"For Centene HealthNet, {reimbursement_count}/{total_count} files contain potential reimbursement terms.") - +print( + f"For Centene HealthNet, {reimbursement_count}/{total_count} files contain potential reimbursement terms." +) diff --git a/fieldExtraction/src/config.py b/fieldExtraction/src/config.py index 8112757..e2094a3 100644 --- a/fieldExtraction/src/config.py +++ b/fieldExtraction/src/config.py @@ -70,22 +70,23 @@ CLIENT = os.getenv("CLIENT", "base") VALID_COLUMNS = columns.VALID_COLUMNS_TEMPLATE[CLIENT] # AWS Keys -AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID') -AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY') -AWS_SESSION_TOKEN = os.getenv('AWS_SESSION_TOKEN') +AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID") +AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY") +AWS_SESSION_TOKEN = os.getenv("AWS_SESSION_TOKEN") # File Paths -LOCAL_PATH = os.getenv('LOCAL_PATH', "data/") +LOCAL_PATH = os.getenv("LOCAL_PATH", "data/") # S3 Settings -S3_CLIENT = boto3.client('s3', - region_name="us-east-2", - aws_access_key_id=AWS_ACCESS_KEY_ID, - aws_secret_access_key=AWS_SECRET_ACCESS_KEY, - aws_session_token=AWS_SESSION_TOKEN - ) -BUCKET = os.getenv('BUCKET', "doczy-dev-infra-textract") -PREFIX = os.getenv('BUCKET_PREFIX') +S3_CLIENT = boto3.client( + "s3", + region_name="us-east-2", + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + aws_session_token=AWS_SESSION_TOKEN, +) +BUCKET = os.getenv("BUCKET", "doczy-dev-infra-textract") +PREFIX = os.getenv("BUCKET_PREFIX") # Bedrock Settings BEDROCK_RUNTIME = boto3.client( @@ -101,9 +102,9 @@ MODEL_ID_CLAUDE2 = "anthropic.claude-instant-v1" # LLama Model # LLAMA_MODEL = Bedrock( - # model=MODEL_ID_CLAUDE3_HAIKU, - # aws_access_key_id=AWS_ACCESS_KEY_ID, - # aws_secret_access_key=AWS_SECRET_ACCESS_KEY, - # aws_session_token=AWS_SESSION_TOKEN, - # region_name="us-east-1" - # ) +# model=MODEL_ID_CLAUDE3_HAIKU, +# aws_access_key_id=AWS_ACCESS_KEY_ID, +# aws_secret_access_key=AWS_SECRET_ACCESS_KEY, +# aws_session_token=AWS_SESSION_TOKEN, +# region_name="us-east-1" +# ) diff --git a/fieldExtraction/src/file_processing.py b/fieldExtraction/src/file_processing.py index c2bcf50..047ad9b 100644 --- a/fieldExtraction/src/file_processing.py +++ b/fieldExtraction/src/file_processing.py @@ -49,20 +49,23 @@ def process_file(file_object): text_dict = preprocess.highlight_rates(text_dict) ################## RUN TOP DOWN ################## - td_results = top_down_funcs.run_top_down(filename, text_dict) # Returns list of dictionaries for each page + td_results = top_down_funcs.run_top_down( + filename, text_dict + ) # Returns list of dictionaries for each page print(f"Top Down Complete - {filename}") ################## RUN BOTTOM UP ################## - bu_results = bottom_up_funcs.run_bottom_up(filename, text_dict) # Returns list of dictionaries + bu_results = bottom_up_funcs.run_bottom_up( + filename, text_dict + ) # Returns list of dictionaries print(f"Bottom Up Complete - {filename}") ################## COMBINE TOP DOWN AND BOTTOM UP ################## combined_results = merge_funcs.merge_results(td_results, bu_results, text_dict) print(f"TD/BU Merge Complete - {filename} ") - + ################## RUN POSTPROCESSING ################## combined_df = pd.DataFrame(combined_results) post_processed_combined_df = postprocess.postprocess_results(combined_df) - + return post_processed_combined_df - diff --git a/fieldExtraction/src/main.py b/fieldExtraction/src/main.py index caae265..ac43128 100644 --- a/fieldExtraction/src/main.py +++ b/fieldExtraction/src/main.py @@ -62,14 +62,23 @@ def main(): # Write Consolidated Output if config.WRITE_OUTPUT: all_results_df = pd.concat(all_results).reset_index(drop=True) - all_results_df.drop('page_num', axis=1, inplace=True) - - # Reorder columns - final_df = all_results_df[[col for col in config.VALID_COLUMNS if col in all_results_df.columns] + [col for col in all_results_df.columns if col not in config.VALID_COLUMNS]] - final_df.to_csv(os.path.join(config.CONSOLIDATED_OUTPUT_DIRECTORY, config.OUTPUT_CSV_PATH)) + all_results_df.drop("page_num", axis=1, inplace=True) + # Reorder columns + final_df = all_results_df[ + [col for col in config.VALID_COLUMNS if col in all_results_df.columns] + + [ + col + for col in all_results_df.columns + if col not in config.VALID_COLUMNS + ] + ] + final_df.to_csv( + os.path.join( + config.CONSOLIDATED_OUTPUT_DIRECTORY, config.OUTPUT_CSV_PATH + ) + ) if __name__ == "__main__": main() - diff --git a/fieldExtraction/src/merge_funcs.py b/fieldExtraction/src/merge_funcs.py index 4a3a2d3..bb18a6a 100644 --- a/fieldExtraction/src/merge_funcs.py +++ b/fieldExtraction/src/merge_funcs.py @@ -7,6 +7,7 @@ def prompt_select_multiple(key, value, bu_dict, page): answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) return answer + def merge_results(td_results, bu_results, text_dict): def find_td_dict(page_num): """Helper function to find dictionary in td_results with matching page_num""" @@ -20,20 +21,26 @@ def merge_results(td_results, bu_results, text_dict): if td_dict is None: # If there's no matching page, use recursion to look one page earlier if int(page_num) > 1: - return get_value(find_td_dict(int(page_num) - 1), key, int(page_num) - 1, text_dict), str(int(page_num) - 1) + return get_value( + find_td_dict(int(page_num) - 1), key, int(page_num) - 1, text_dict + ), str(int(page_num) - 1) else: return "N/A", "N/A" # Base case if no previous pages exist else: value = td_dict.get(key, "N/A") if value == "N/A" or value == "": # If value is 'N/A' or empty, use recursion to look one page earlier - return get_value(find_td_dict(int(page_num) - 1), key, int(page_num) - 1, text_dict), str(int(page_num) - 1) - elif ',' in value and 'DATE' not in key: + return get_value( + find_td_dict(int(page_num) - 1), key, int(page_num) - 1, text_dict + ), str(int(page_num) - 1) + elif "," in value and "DATE" not in key: # Randomly select one of the comma-separated values - return prompt_select_multiple(key, value, bu_dict, text_dict[page_num]), str(int(page_num)) + return prompt_select_multiple( + key, value, bu_dict, text_dict[page_num] + ), str(int(page_num)) else: return value, page_num - + merged_results = [] for bu_dict in bu_results: page_num = bu_dict["page_num"] @@ -43,8 +50,10 @@ def merge_results(td_results, bu_results, text_dict): # Add or overwrite keys from td_dict if td_dict: for key, value in td_dict.items(): - if key not in ['Filename', 'page_num']: - new_dict[key], new_dict[f"{key}_PG"] = get_value(td_dict, key, page_num, text_dict) - + if key not in ["Filename", "page_num"]: + new_dict[key], new_dict[f"{key}_PG"] = get_value( + td_dict, key, page_num, text_dict + ) + merged_results.append(new_dict) return merged_results diff --git a/fieldExtraction/src/postprocess.py b/fieldExtraction/src/postprocess.py index 5fcb9b5..e8168b7 100644 --- a/fieldExtraction/src/postprocess.py +++ b/fieldExtraction/src/postprocess.py @@ -27,18 +27,26 @@ def postprocess_results(combined_df): ["CONTRACT_NETWORK", "Corrected_NETWORK", config.VALID_NETWORKS], ]: try: - postprocess_funcs.clean_columns_combined(df, field_list[0], field_list[2], field_list[1]) + postprocess_funcs.clean_columns_combined( + df, field_list[0], field_list[2], field_list[1] + ) except Exception as e: print(f"Postprocessing Error - clean_columns_combined : {e}") try: - postprocess_funcs.clean_columns_combined_fuzzy(df, field_list[0], field_list[2], config.FUZZY_MATCH_THRESHOLD) + postprocess_funcs.clean_columns_combined_fuzzy( + df, field_list[0], field_list[2], config.FUZZY_MATCH_THRESHOLD + ) except Exception as e: print(f"Postprocessing Error - clean_columns_combined_fuzzy : {e}") # Correct misplaced values across columns try: - df = postprocess_funcs.correct_misplaced_values(df, ['CONTRACT_LOB', 'CONTRACT_PROGRAM', 'CONTRACT_NETWORK'], valid_values_dict) + df = postprocess_funcs.correct_misplaced_values( + df, + ["CONTRACT_LOB", "CONTRACT_PROGRAM", "CONTRACT_NETWORK"], + valid_values_dict, + ) except Exception as e: print(f"Postprocessing Error - correct_misplaced_values : {e}") @@ -73,4 +81,4 @@ def postprocess_results(combined_df): ] df = df[column_order] - return df \ No newline at end of file + return df diff --git a/fieldExtraction/src/postprocess_funcs.py b/fieldExtraction/src/postprocess_funcs.py index 1fec04e..e803cf2 100644 --- a/fieldExtraction/src/postprocess_funcs.py +++ b/fieldExtraction/src/postprocess_funcs.py @@ -1,10 +1,10 @@ - import pandas as pd import re import difflib import config + def sanitize_value(value): """ Sanitizes a given value by handling nulls, lists, and strings to ensure consistency in formatting. @@ -22,15 +22,16 @@ def sanitize_value(value): """ try: if isinstance(value, list): - return ', '.join(str(v) for v in value) + return ", ".join(str(v) for v in value) elif isinstance(value, str): - value = value.strip('[]') - return ', '.join([item.strip(" '") for item in value.split(',')]) + value = value.strip("[]") + return ", ".join([item.strip(" '") for item in value.split(",")]) elif pd.isna(value): return "N/A" except: return value + def exact_match(val, valid_values): """ Compares a given string value against a list of valid values to determine if there is an exact match, case-insensitively. @@ -52,6 +53,7 @@ def exact_match(val, valid_values): return valid_val return None + def clean_columns_combined(df, column_name, valid_values, new_column_name): """ Cleans and standardizes entries in a specified column of a DataFrame based on a list of valid values, creating a new column with standardized values. @@ -75,13 +77,13 @@ def clean_columns_combined(df, column_name, valid_values, new_column_name): def update_column(entry): if pd.notna(entry): - terms = entry.split(',') + terms = entry.split(",") for term in terms: match = exact_match(term, valid_values) if match: return match return None - + df[new_column_name] = df[column_name].apply(update_column) cleaned_values = df[new_column_name].unique() return original_values, cleaned_values, changes @@ -92,7 +94,7 @@ def get_closest_match(val, valid_values, similarity_threshold=0.7): Finds the closest match for a given string from a list of valid values, using a similarity threshold. This function cleans the input value and compares it to each cleaned value in the valid values list using a fuzzy - matching technique. It returns the closest match that meets or exceeds the specified similarity threshold. If no + matching technique. It returns the closest match that meets or exceeds the specified similarity threshold. If no matches meet the threshold, the function returns None. Parameters: @@ -106,7 +108,9 @@ def get_closest_match(val, valid_values, similarity_threshold=0.7): if pd.isna(val): return None val = val.strip().upper() - matches = difflib.get_close_matches(val, [v.upper() for v in valid_values], n=1, cutoff=similarity_threshold) + matches = difflib.get_close_matches( + val, [v.upper() for v in valid_values], n=1, cutoff=similarity_threshold + ) return matches[0] if matches else None @@ -131,17 +135,19 @@ def clean_columns_combined_fuzzy(df, column_name, valid_values, threshold): changes = {} df[column_name] = df[column_name].apply(sanitize_value) original_values = df[column_name].unique() - + def log_and_clean(entry): if pd.notna(entry): - words = entry.split(',') + words = entry.split(",") cleaned_words = [] for word in words: - cleaned_word = get_closest_match(word.strip(), valid_values, similarity_threshold=threshold) + cleaned_word = get_closest_match( + word.strip(), valid_values, similarity_threshold=threshold + ) if cleaned_word and word.strip().upper() != cleaned_word: changes[word.strip()] = cleaned_word cleaned_words.append(cleaned_word if cleaned_word else word.strip()) - return ', '.join(cleaned_words) + return ", ".join(cleaned_words) return None df[column_name] = df[column_name].apply(log_and_clean) @@ -149,7 +155,6 @@ def clean_columns_combined_fuzzy(df, column_name, valid_values, threshold): return original_values, cleaned_values, changes - def correct_misplaced_values(df, columns, valid_values_dict): """ Corrects misplaced values within specified columns of a DataFrame based on a dictionary of valid values for each column. @@ -170,20 +175,32 @@ def correct_misplaced_values(df, columns, valid_values_dict): for index, row in df.iterrows(): for col in columns: if pd.notna(row[col]): - terms = row[col].split(',') + terms = row[col].split(",") for term in terms: term = term.strip() for target_col, valid_values in valid_values_dict.items(): if target_col != col: match = exact_match(term, valid_values) if match: - if pd.isna(row[target_col]) or not row[target_col].strip(): + if ( + pd.isna(row[target_col]) + or not row[target_col].strip() + ): df.at[index, target_col] = match df.at[index, col] = None else: current_value = row[target_col].strip() - if get_closest_match(match, [current_value], config.FUZZY_MATCH_THRESHOLD) is None: - df.at[index, 'Corrected_' + target_col] = f"Found {term} in {col} cell" + if ( + get_closest_match( + match, + [current_value], + config.FUZZY_MATCH_THRESHOLD, + ) + is None + ): + df.at[index, "Corrected_" + target_col] = ( + f"Found {term} in {col} cell" + ) df.at[index, col] = None return df @@ -204,13 +221,28 @@ def filter_service_column(d): list: A new list of dictionaries with items containing specified keywords in the 'SERVICE' key removed. """ keywords = [ - 'LIABILITY', 'RISK', 'LOBBYING', 'DAMAGES', 'CONFIDENTIALITY', 'ARBITRATION', 'FALSE CLAIMS ACT', 'UNSPECIFIED', - 'AUDIT', 'INTEREST', 'N/A', 'BUSINESS', 'COMPLIANCE', 'Medical Assistance Program', 'MATERIAL SUBCONTRACT', 'GIFTS', 'GRATUITIES', + "LIABILITY", + "RISK", + "LOBBYING", + "DAMAGES", + "CONFIDENTIALITY", + "ARBITRATION", + "FALSE CLAIMS ACT", + "UNSPECIFIED", + "AUDIT", + "INTEREST", + "N/A", + "BUSINESS", + "COMPLIANCE", + "Medical Assistance Program", + "MATERIAL SUBCONTRACT", + "GIFTS", + "GRATUITIES", ] - pattern = '|'.join(keywords) + pattern = "|".join(keywords) regex = re.compile(pattern, re.IGNORECASE) - filtered_list = [item for item in d if not regex.search(item.get('SERVICE', ''))] + filtered_list = [item for item in d if not regex.search(item.get("SERVICE", ""))] return filtered_list @@ -229,13 +261,28 @@ def move_percentage_to_rate(df): Returns: pandas.DataFrame: The DataFrame with percentage values moved from the 'REIMBURSEMENT_FLAT_FEE' to the 'REIMBURSEMENT_RATE' column. """ + def move_percentage(value): - if isinstance(value, str) and '%' in value: + if isinstance(value, str) and "%" in value: return True return False - - df['REIMBURSEMENT_RATE'] = df.apply(lambda row: row['REIMBURSEMENT_FLAT_FEE'] if move_percentage(row['REIMBURSEMENT_FLAT_FEE']) else row['REIMBURSEMENT_RATE'], axis=1) - df['REIMBURSEMENT_FLAT_FEE'] = df.apply(lambda row: None if move_percentage(row['REIMBURSEMENT_FLAT_FEE']) else row['REIMBURSEMENT_FLAT_FEE'], axis=1) + + df["REIMBURSEMENT_RATE"] = df.apply( + lambda row: ( + row["REIMBURSEMENT_FLAT_FEE"] + if move_percentage(row["REIMBURSEMENT_FLAT_FEE"]) + else row["REIMBURSEMENT_RATE"] + ), + axis=1, + ) + df["REIMBURSEMENT_FLAT_FEE"] = df.apply( + lambda row: ( + None + if move_percentage(row["REIMBURSEMENT_FLAT_FEE"]) + else row["REIMBURSEMENT_FLAT_FEE"] + ), + axis=1, + ) return df @@ -255,12 +302,15 @@ def set_rate_to_zero_if_not_covered(df): Returns: pandas.DataFrame: The updated DataFrame with adjusted 'REIMURSEMENT_RATE' values where applicable. """ + def check_and_set_rate(row): - if pd.notna(row['FULL_METHODOLOGY']) and re.search(r'not covered', row['FULL_METHODOLOGY'], re.IGNORECASE): + if pd.notna(row["FULL_METHODOLOGY"]) and re.search( + r"not covered", row["FULL_METHODOLOGY"], re.IGNORECASE + ): return 0 - return row['REIMBURSEMENT_RATE'] - - df['REIMBURSEMENT_RATE'] = df.apply(check_and_set_rate, axis=1) + return row["REIMBURSEMENT_RATE"] + + df["REIMBURSEMENT_RATE"] = df.apply(check_and_set_rate, axis=1) return df @@ -279,13 +329,18 @@ def adjust_reimbursement_rate(df): Returns: pandas.DataFrame: The updated DataFrame with recalibrated 'REIMBURSEMENT_RATE' values based on the specified methodology condition. """ + def adjust_rate(row): - if pd.notna(row['FULL_METHODOLOGY']) and re.search(r'case insensitive', row['FULL_METHODOLOGY'], re.IGNORECASE): - if pd.notna(row['REIMBURSEMENT_RATE']) and isinstance(row['REIMBURSEMENT_RATE'], (int, float)): - return 100 - row['REIMBURSEMENT_RATE'] - return row['REIMBURSEMENT_RATE'] - - df['REIMBURSEMENT_RATE'] = df.apply(adjust_rate, axis=1) + if pd.notna(row["FULL_METHODOLOGY"]) and re.search( + r"case insensitive", row["FULL_METHODOLOGY"], re.IGNORECASE + ): + if pd.notna(row["REIMBURSEMENT_RATE"]) and isinstance( + row["REIMBURSEMENT_RATE"], (int, float) + ): + return 100 - row["REIMBURSEMENT_RATE"] + return row["REIMBURSEMENT_RATE"] + + df["REIMBURSEMENT_RATE"] = df.apply(adjust_rate, axis=1) return df @@ -310,12 +365,12 @@ def clean_td(td): for d in td: new_d = {} for k, v in d.items(): - if 'DATE' in k: + if "DATE" in k: new_d[k] = v if isinstance(v, list) else [v] - elif k not in ['page_num', 'Filename']: - if isinstance(v, str) and ',' in v: - new_d[k] = [item.strip() for item in v.split(',')] - elif v == 'N/A': + elif k not in ["page_num", "Filename"]: + if isinstance(v, str) and "," in v: + new_d[k] = [item.strip() for item in v.split(",")] + elif v == "N/A": new_d[k] = [] else: new_d[k] = [v] if isinstance(v, str) else v @@ -325,7 +380,6 @@ def clean_td(td): return td_clean - def sanitize_combined(df): """ Sanitizes all columns in a DataFrame by applying a predefined sanitization function to each value. @@ -345,6 +399,7 @@ def sanitize_combined(df): df[column] = df[column].apply(sanitize_value) return df + def extract_codes_CPT(value): """ Extracts and formats CPT codes from a given input value. @@ -364,16 +419,17 @@ def extract_codes_CPT(value): """ if pd.isna(value): return value - - value = re.sub(r'\bthrough\b', '-', str(value), flags=re.IGNORECASE) - - pattern = r'\b([A-Z]\d{4}|\d{5})(-[A-Z]{2})?\b' + + value = re.sub(r"\bthrough\b", "-", str(value), flags=re.IGNORECASE) + + pattern = r"\b([A-Z]\d{4}|\d{5})(-[A-Z]{2})?\b" matches = re.findall(pattern, value) if matches: - return ', '.join([''.join(match) for match in matches]) + return ", ".join(["".join(match) for match in matches]) else: return "" + def extract_codes_Diagnosis(value): """ Extracts and formats diagnosis codes from a given input value, typically adhering to ICD (International Classification of Diseases) formats. @@ -392,14 +448,15 @@ def extract_codes_Diagnosis(value): """ if pd.isna(value): return value - - pattern = r'\b[A-Z][0-9][A-Z0-9]{1,4}(\.[A-Z0-9]{1,4})?\b' + + pattern = r"\b[A-Z][0-9][A-Z0-9]{1,4}(\.[A-Z0-9]{1,4})?\b" matches = re.findall(pattern, value) if matches: - return ', '.join(matches) + return ", ".join(matches) else: return "" + def extract_codes_Revenue(value): """ Extracts revenue codes from a given input value. Revenue codes are typically numerical codes of three to four digits. @@ -417,14 +474,15 @@ def extract_codes_Revenue(value): """ if pd.isna(value): return value - - pattern = r'\b\d{3,4}\b' + + pattern = r"\b\d{3,4}\b" matches = re.findall(pattern, value) if matches: - return ', '.join(matches) + return ", ".join(matches) else: return "" + def clean_code_column(df, column_name, extract_function): """ Applies a specified function to clean and extract codes from a specific column in a DataFrame. @@ -443,19 +501,23 @@ def clean_code_column(df, column_name, extract_function): pandas.DataFrame: The DataFrame with the specified column updated with cleaned and processed codes. """ df[column_name] = df[column_name].apply(extract_function) - return df + return df def clean_dates(df): for idx, row in df.iterrows(): - if row['LOB_PRICING_TERMS_EFFECTIVE_DATE'] == row['LOB_PRICING_TERMS_TERMINATION_DATE']: - df.loc[idx, 'LOB_PRICING_TERMS_TERMINATION_DATE'] = 'N/A' + if ( + row["LOB_PRICING_TERMS_EFFECTIVE_DATE"] + == row["LOB_PRICING_TERMS_TERMINATION_DATE"] + ): + df.loc[idx, "LOB_PRICING_TERMS_TERMINATION_DATE"] = "N/A" return df + def add_PG_columns(results_dicts): for key in list(results_dicts[0].keys()): - if key != 'page_num': + if key != "page_num": for d in results_dicts: - d[f"{key}_PG"] = d['page_num'] + d[f"{key}_PG"] = d["page_num"] return results_dicts diff --git a/fieldExtraction/src/preprocess.py b/fieldExtraction/src/preprocess.py index 2d0e6d2..77a60f2 100644 --- a/fieldExtraction/src/preprocess.py +++ b/fieldExtraction/src/preprocess.py @@ -133,7 +133,7 @@ def clean_billed_charges(contract_text): "Billed Charges", "the rates set forth in this Exhibit", "the rutes set forth in this Exhibit", - "Provider's usual and customary charge" + "Provider's usual and customary charge", ] max_substring_length = max([len(s) for s in substrings]) diff --git a/fieldExtraction/src/prompts.py b/fieldExtraction/src/prompts.py index c38ffa1..c410498 100644 --- a/fieldExtraction/src/prompts.py +++ b/fieldExtraction/src/prompts.py @@ -44,6 +44,7 @@ Write ONLY the value or values in the list that DIRECTLY applies to the Service If the Service and Methodology are not found on the page, write the LAST item in the list. Do not write any other commentary or explanation. """ + def BOTTOM_UP_PRIMARY(page, payer): return f""" ### PAGE START ### {page} ### PAGE END @@ -221,6 +222,7 @@ Here are the entities, represented as dictionary objects: Write ONLY the number of the dictionary that is most associated with the Service and Methodology of interest. Do not return any additional text beyond the digit itself. """ + def TOP_DOWN_METAL_LEVEL(LOB, page): return f"""### PAGE START ### {page} ### PAGE END @@ -458,6 +460,7 @@ def get_carveout_list(): "Hematology", ] + def prompt_contract_lob(page): return f"""### PAGE START ### {page} ### PAGE END diff --git a/fieldExtraction/src/test.py b/fieldExtraction/src/test.py index 70cd264..ebc0ea8 100644 --- a/fieldExtraction/src/test.py +++ b/fieldExtraction/src/test.py @@ -25,21 +25,21 @@ import shutil # return f""" # ### PAGE START ### {page} ### PAGE END -# The preceding text is one page of a contract between Payer {payer} and a provider in their network. Your job is to extract attributes related to the reimbursement of different services and specialties. +# The preceding text is one page of a contract between Payer {payer} and a provider in their network. Your job is to extract attributes related to the reimbursement of different services and specialties. -# The reimbursement values will be identified with >>> <<< indicators (e.g. >>>105%<<<). Make sure there is at least one dictionary object for EVERY reimbursement value seen (either % or $). +# The reimbursement values will be identified with >>> <<< indicators (e.g. >>>105%<<<). Make sure there is at least one dictionary object for EVERY reimbursement value seen (either % or $). # Some values are found in sections explicitly identified as examples - these must be omitted from output. Other values might be related to liability - these must also be excluded. # If any of the attributes are not found, return N/A. For all attributes, only write what is written on the page. Do not make up new phrases or words. -# Return at least one json object for each combination of attributes seen. It is possible that a page will not have any attributes. When this is the case, return an empty list. +# Return at least one json object for each combination of attributes seen. It is possible that a page will not have any attributes. When this is the case, return an empty list. # Here are the attributes to be included in each dictionary, and instructions on how to correctly answer: # 'SERVICE' : What is the Service that is being reimbursed? Use your best judgement and knowledge of the healthcare industry to answer. Use only exact text from the document. Do not leave this field N/A. # 'REIMBURSEMENT_FLAT_FEE' : If the listed reimbursement is dollar value, return only that dollar value. There can only be one answer for each object. If more than one rate is found, create additional objects for them. # 'REIMBURSEMENT_RATE' : If the listed reimbursement is a percent of something, return only that percent. There can only be one answer for each object. If more than one rate is found, create additional objects for them. Do NOT include multiple percentages in this value. -# 'FULL_METHODOLOGY' : If the listed reimbursement is a percent of something, what is it the percent of? If the listed reimbursement is a direct dollar value, how is that dollar value paid (per diem, per unit, etc)? Write the full sentence in text describing the payment methodology. +# 'FULL_METHODOLOGY' : If the listed reimbursement is a percent of something, what is it the percent of? If the listed reimbursement is a direct dollar value, how is that dollar value paid (per diem, per unit, etc)? Write the full sentence in text describing the payment methodology. # 'DATE_RANGE' : If the listed reimbursement only applies to a specific date range, write that range here. Write only the exact text. Write only the range. If there is none, write 'N/A'. # 'REIMBURSEMENT_TIN' : If the listed reimbursement only applies to a specific TIN, write only the TIN here. If not, write 'N/A' @@ -60,19 +60,22 @@ import shutil # print(answer_dict[page_num]) -batch1_files = pd.read_excel("C:\\Users\\kminhas\\Documents\\Doczy\\Facility TINs_DOCZY AI_DOCUMENT EXTRACTION_ASSIGNMENT_7.1.2024.xlsx", sheet_name='DOCZY AI') -batch1_files = batch1_files['File Name'] +batch1_files = pd.read_excel( + "C:\\Users\\kminhas\\Documents\\Doczy\\Facility TINs_DOCZY AI_DOCUMENT EXTRACTION_ASSIGNMENT_7.1.2024.xlsx", + sheet_name="DOCZY AI", +) +batch1_files = batch1_files["File Name"] # print(list(batch1_files)) tdrive_path = "T:\\AArete Client Work\\Centene\\Restricted\\HealthNet\\Doczy Contract Metadata\\1. Contracts\\05232024\\Extracted" -batch_path = os.path.join(tdrive_path, 'Batch0') +batch_path = os.path.join(tdrive_path, "Batch0") try: os.mkdir(batch_path) except: pass for file in os.listdir(tdrive_path): - if file.endswith('.Pdf') and file.split('.Pdf')[0] in list(batch1_files): + if file.endswith(".Pdf") and file.split(".Pdf")[0] in list(batch1_files): try: source_path = os.path.join(tdrive_path, file) destination_path = os.path.join(batch_path, file) @@ -80,7 +83,3 @@ for file in os.listdir(tdrive_path): print(f"Moved file: {file}") except: pass - - - - diff --git a/fieldExtraction/src/utils.py b/fieldExtraction/src/utils.py index 7a0eed5..da28983 100644 --- a/fieldExtraction/src/utils.py +++ b/fieldExtraction/src/utils.py @@ -139,7 +139,9 @@ def consolidate_csvs( pass concatenated_df = pd.concat(df_list, ignore_index=True) - concatenated_df.to_csv(os.path.join(config.CONSOLIDATED_OUTPUT_DIRECTORY, output_file), index=False) + concatenated_df.to_csv( + os.path.join(config.CONSOLIDATED_OUTPUT_DIRECTORY, output_file), index=False + ) print(f"All CSV files have been consolidated into {output_file}")