diff --git a/src/config.py b/src/config.py index 0e3dd90..bf22093 100644 --- a/src/config.py +++ b/src/config.py @@ -54,9 +54,9 @@ VALID_COLUMNS = ['Filename', 'page_num', 'SERVICE', 'REIMBURSEMENT_FLAT_FEE', 'R 'Corrected_LOB', 'Corrected_PROGRAM', 'Corrected_NETWORK'] # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXFY5DMLIK" -AWS_SECRET_ACCESS_KEY="IN2w1JrXU4f7rKR2IbiOwCZNUrgls3ZK/65w8cCS" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEHMaCXVzLWVhc3QtMiJHMEUCIEhqvUNWd6r+bPwIIFkg242wxJGTn9k7qWiWVZSXn4dLAiEAr01immSRrcDW1oqW0QfN3gqivPNQV8JkTgLipIW0rpsqgwMIHBAAGgw2NjAxMzEwNjg3ODIiDJCiH+aROTYN8KdY2CrgAmCbjJwVL63njz5qxkOtexudA6CZ7Kz5BGpaA2WbdyHHc8j/UB/44MfOAxrTeLiP6w12qpYMc7U5GdZIULvNwWxJhJmnOHwLmLU0XuOtc52l3VCQmXLLMj1Q/dVPm+MVFgTM4RLTTNRGRE9boX1MP92pZlUZ/sjSKBl2ELfiJkrgzOkS3kYADmICJ0wWU1Ndqpi8sGTs8JHCyHCj3TSkJVaeCcwyvWssZHrgOv7/MUdgeHkiJx3m/MJNdOmxyZzr7mWtAFgJMODr8zM/vAPWogzcD0MEoMpa98ag9fcC6nE9us4KpiIIGAJNzXX6YpjkxoU+/tdXwMLsbkqmnqblz3zvB6jEyqTLMeEMYfexlzA7bBrq6Ooh2+nvTpFwF3E4RTdXsOdjgiw2nTwfjBYFT4cAGz9rQl2hZCLdhEhwZu5dmF/oKn6Nd5eec+KV0+Ep/SPor9p/uQh597zDDWt1BMwwurvHswY6pgFAChD2esP7NSzOQ+dIKA0CTZuNwdK7mjfkjtGEPsjDJBF3ueD4CybkHrqM2Ud6xhI8LwZU6DiIdUeqFXQHawdF1wDegBgy1Tymv1nCOZYCyTbW6ihJ+L4oJsr8XJwKpUhRgQQ8ECscAWFGRMN3zPt6T+OAcxJvjnKtdQO+vOCJNMzkr1yb7cuEoUcWPSwFrVkZg5iKP4T9DLLDCfGZUiV7AND6trJ+" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXPGBTIYHY" +AWS_SECRET_ACCESS_KEY="2zV8fp+u6LW8qpr75ZKEG/kWnvbrgQYo6Rg7MDln" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEIr//////////wEaCXVzLWVhc3QtMiJIMEYCIQDZXOwwKnirwI7+o3716cEa6yVDLJiTu3Kzz7kPGooFuAIhAOGrTFPBigoNNA1x1OGJSMZpwGPLmiMxpJIpWbzYxyZbKoMDCDMQABoMNjYwMTMxMDY4NzgyIgwu4eVW0nZEUNi69Pcq4AJw9r1cQpF2vnIdXzmMp18dm9+oB7IZQMFQ9W3LR8R90PY3zk0HzY1UGaTIKnSPsXAdl5bIW2nb7a79RPoiQr4Ro70qM4G0WzHjaP8ottl/YCpiEhFzmNhBamT9lutBSQIyjumzzYa6LX76T2K9zSzumG9+2ubPKLtqLO91d9MV6EDCOt4hBsFM46WT4YBsVxEBQ3ABIsRAVkG2Ff8jgQ01nygd+IYqFjjQBndCxjSqcxM03+MQDBxsaRjNdwbJ/lRcrJpD7PeunPG1d+BIg9u4RfrHuTjJo8wVaZAKUlqjeRPWtS5woCiUEfGsSykWJp+N+x0cv+/tiVxzU1lPSwiLkwTQ5FotFr4dfA7q3NaqkKZ5YUA3GrKrATmsD1UlCRuCx7/jvMgCABFuCOERD7nrY57rgLY167PvKY7/8Fi6mpBgH8xHHtYoICfBQRMz2dLTwHVFC5R4pdA+f7ysF9MZMP+tzLMGOqUBCkhJSij99lLWdoOQQTn67bJHGMFcbXFHkeGnY7mk0jaLnACuodUMgvHPPuHDR6YsIEKk7Yw70pjMDoiXcbDgjbKdYquX86iXKMgv8RP6blXJJSgxsrabyBHTMWITMZDe530iQCoRkwiAo++RQanljFMah0LPCZpfqq6lpeLtws7Ya6SmvBUT6BBrhVDn0FZhKxnuXGJ58tnqaGA2eSYddht5lcDs" # File Paths LOCAL_PATH = 'data/test/' # Replace with local diff --git a/src/postprocessingfuncs.py b/src/postprocessingfuncs.py index 8dd2d64..eae72e7 100644 --- a/src/postprocessingfuncs.py +++ b/src/postprocessingfuncs.py @@ -227,7 +227,8 @@ def filter_service_column(df): df = df.dropna(subset=['SERVICE']) keywords = [ - 'LIABILITY', 'RISK', 'LOBBYING', 'DAMAGES', 'CONFIDENTIALITY', 'AUDIT', 'INTEREST', 'N/A', 'BUSINESS', 'COMPLIANCE', 'Medical Assistance Program' + 'LIABILITY', 'RISK', 'LOBBYING', 'DAMAGES', 'CONFIDENTIALITY', 'ARBITRATION', 'FALSE CLAIMS ACT', 'UNSPECIFIED', + 'AUDIT', 'INTEREST', 'N/A', 'BUSINESS', 'COMPLIANCE', 'Medical Assistance Program', 'MATERIAL SUBCONTRACT', 'GIFTS', 'GRATUITIES', ] pattern = '|'.join(keywords) df = df[~df['SERVICE'].str.contains(pattern, case=False, na=False)] diff --git a/src/preprocess.py b/src/preprocess.py index 54a6b90..caddb66 100644 --- a/src/preprocess.py +++ b/src/preprocess.py @@ -40,7 +40,7 @@ def split_text(text): return text_dict -def highlight_rates(text_dict): +def highlight_rates(text): """ Highlights rate-related terms (percentages and dollar amounts) in the text of each page within a dictionary. @@ -54,15 +54,13 @@ def highlight_rates(text_dict): Returns: dict: The updated dictionary with rate-related terms highlighted in the text of each page. """ - for page, text in text_dict.items(): - words = text.split() - highlighted_words = [] - for word in words: - if '%' in word or '$' in word: - word = f'>>>{word}<<<' - highlighted_words.append(word) - text_dict[page] = ' '.join(highlighted_words) - return text_dict + words = text.split() + highlighted_words = [] + for word in words: + if '%' in word or '$' in word: + word = f'>>>{word}<<<' + highlighted_words.append(word) + return ' '.join(highlighted_words) def chunk_text(text_dict): @@ -97,7 +95,7 @@ def chunk_text(text_dict): return chunk_dict -def clean_billed_charges(text): +def clean_billed_charges(contract_text): """ Cleans occurrences of 'billed charges' in a text by replacing them with '100% of billed charges' unless preceded by a percentage. @@ -111,11 +109,40 @@ def clean_billed_charges(text): Returns: str: The cleaned text with appropriate replacements made for 'billed charges'. """ - def replace(match): - start_pos = match.start() - pre_text = text[max(0, start_pos-15):start_pos] - if '%' in pre_text: - return match.group() - else: - return r'100% of billed charges' - return re.sub(r'\bbilled charges\b', replace, text, flags=re.IGNORECASE) + + substrings = [ + "Physician's Billed Charges", + "Provider's Billed Charges", + "Allowable Billed Charges", + "Hospital's Billed Charges", + "Physician's Charges", + "Provider's Charges", + "Allowable Charges", + "Hospital's Charges", + "Billed Charges" + ] + max_substring_length = max([len(s) for s in substrings]) + + def find_substring_indices(contract_text, s): + indices = [] + lower_contract_text = contract_text.lower() + lower_s = s.lower() + index = lower_contract_text.find(lower_s) + while index != -1: + indices.append(index) + index = lower_contract_text.find(lower_s, index + 1) + return indices + + for s in substrings: + indices = find_substring_indices(contract_text, s) + index_adder = 0 + if indices: + for i in indices: + index = i+index_adder + end_index = index + max_substring_length + match_part = contract_text[index:end_index] + previous = contract_text[max(0, index-30):index] + if '%' not in previous: + contract_text = contract_text[0:index] + f" 100% of {match_part}" + contract_text[end_index:] + index_adder += 9 + return contract_text.replace(' ', ' ') diff --git a/src/prompt_funcs.py b/src/prompt_funcs.py index 686b52d..6831c42 100644 --- a/src/prompt_funcs.py +++ b/src/prompt_funcs.py @@ -107,13 +107,10 @@ def run_bottom_up_secondary(answer_dicts, text_dict, tokens): # Bottom Up Methodology if config.RUN_METHODOLOGY: - if d['REIMBURSEMENT_FLAT_FEE'] == 'N/A': - prompt = prompts.BOTTOM_UP_METHODOLOGY(d) - methodology_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=100) - d['REIMBURSEMENT_METHODOLOGY'] = methodology_answer - else: - d['REIMBURSEMENT_METHODOLOGY'] = 'Flat Fee' - + prompt = prompts.BOTTOM_UP_METHODOLOGY(d) + methodology_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=100) + d['REIMBURSEMENT_METHODOLOGY'] = methodology_answer + # # Bottom Up FS if config.RUN_FS: prompt = prompts.BOTTOM_UP_FS(d) diff --git a/src/prompts.py b/src/prompts.py index c7b4b14..ae8698d 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -38,7 +38,9 @@ def BOTTOM_UP_PRIMARY(page, payer): The preceding text is one page of a contract between Payer {payer} and a provider in their network. Your job is to extract attributes related to the reimbursement of different services and specialties. -The reimbursement values will be identified with >>> <<< indicators (e.g. >>>105%<<<). Make sure there is at least one dictionary object for EVERY reimbursement value seen (either % or $). Some values are found in sections explicitly identified as examples - these must be omitted from output. +The reimbursement values will be identified with >>> <<< indicators (e.g. >>>105%<<<). Make sure there is at least one dictionary object for EVERY reimbursement value seen (either % or $). + +Some values are found in sections explicitly identified as examples - these must be omitted from output. Other values might be related to liability - these must also be excluded. If any of the attributes are not found, return N/A. For all attributes, only write what is written on the page. Do not make up new phrases or words. @@ -122,7 +124,7 @@ def BOTTOM_UP_METHODOLOGY(d): Your job is to identify what the concise methodology is based on the full methodology. Choose the closest option from the following: -Allowed Amount, Allowable Charges, Fee Schedule, Medicare Fee Schedule, Medicare Allowed Amount, Medicaid Fee Schedule, Medicaid Allowed Amount, Provider's Charges, Billed Charges, IME/DME Amount, Cost to Charge Ratio, MSRP, Per Unit, Per Diem, Per Visit, Per Member, Per Member Per Month, Per Hour, Case Rate, Contracted Rate +Allowed Amount, Allowable Charges, Fee Schedule, Medicare Fee Schedule, Medicare Allowed Amount, Medicaid Fee Schedule, Medicaid Allowed Amount, Provider's Charges, Billed Charges, Normal fee for such service, IME/DME Amount, Cost to Charge Ratio, Cost plus %, MSRP, Per Unit, Per Diem, Per Visit, Per Member, Per Member Per Month, Per Hour, Case Rate, Contracted Rate For 'Allowed Amount' and 'Fee Schedule', use the more specific answer (Medicare/Medicaid) if applicable. For the 'Per ___' values, note that this list is not necessarily exhaustive. It could be anything like 'Per ___'. diff --git a/src/table_funcs.py b/src/table_funcs.py index 741a4a8..d6e80e1 100644 --- a/src/table_funcs.py +++ b/src/table_funcs.py @@ -73,21 +73,22 @@ def align_and_format_tables(text_dict): """ aligned_text_dict = {} for key, text in text_dict.items(): + print(key) aligned_text = text if 'Table Start' in text: table_texts = re.findall(r'-------Table Start--------(.*?)-------Table End--------', text, re.DOTALL) for table_text in table_texts: - try: - # Extract pretable text - pretable = table_text.split('{')[0].strip() - # Extract and format table text - #table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0].replace("'", '"') + "}" - table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0] + "}" + + # Extract pretable text + pretable = table_text.split('{')[0].strip() + # Extract and format table text + #table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0].replace("'", '"') + "}" + table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0] + "}" - table, table_size = convert_to_dict(table_only) - table_formatted = format_table(table, table_size) - except: - table_formatted = table_text + table, table_size = convert_to_dict(table_only) + table_formatted = format_table(table, table_size) + + table_formatted = table_text # Align if text.count(pretable) == 2: # One table diff --git a/src/test.py b/src/test.py index 0d04e64..981b967 100644 --- a/src/test.py +++ b/src/test.py @@ -1,4 +1,5 @@ import pandas as pd +import numpy as np import re import json import csv @@ -31,14 +32,55 @@ import claude_funcs input_dict = utils.read_input() - (filename, contract_text) = list(input_dict.items())[0] -contract_text = preprocess.clean_newlines(contract_text) -text_dict = preprocess.split_text(contract_text) -text_dict = table_funcs.align_and_format_tables(text_dict) -text_dict = preprocess.highlight_rates(text_dict) +def clean_billed_charges(contract_text): + substrings = [ + "Physician's Billed Charges", + "Provider's Billed Charges", + "Allowable Billed Charges", + "Hospital's Billed Charges", + "Physician's Charges", + "Provider's Charges", + "Allowable Charges", + "Hospital's Charges", + "Billed Charges" + ] + max_substring_length = np.max([len(s) for s in substrings]) -bu_results = prompt_funcs.run_bottom_up_primary({'5' : text_dict['5']}, 4000) # Returns list of dictionaries + def find_substring_indices(contract_text, s): + indices = [] + lower_contract_text = contract_text.lower() + lower_s = s.lower() + index = lower_contract_text.find(lower_s) + while index != -1: + indices.append(index) + index = lower_contract_text.find(lower_s, index + 1) + return indices + + for s in substrings: + indices = find_substring_indices(contract_text, s) + index_adder = 0 + if indices: + for i in indices: + index = i+index_adder + end_index = index + max_substring_length + match_part = contract_text[index:end_index] + previous = contract_text[max(0, index-30):index] + if '%' not in previous: + contract_text = contract_text[0:index] + f" 100% of {match_part}" + contract_text[end_index:] + index_adder += 9 + return contract_text.replace(' ', ' ') + +contract_text = preprocess.clean_newlines(contract_text) +contract_text = clean_billed_charges(contract_text) +contract_text = preprocess.highlight_rates(contract_text) +text_dict = preprocess.split_text(contract_text) +# text_dict = table_funcs.align_and_format_tables(text_dict) + + +print(text_dict['23']) + +bu_results = prompt_funcs.run_bottom_up_primary({'23' : text_dict['23']}, 4000) # Returns list of dictionaries print(bu_results) diff --git a/src/textract_template.py b/src/textract_template.py new file mode 100644 index 0000000..510d8e1 --- /dev/null +++ b/src/textract_template.py @@ -0,0 +1,236 @@ + +""" +This script was written to extract .txt files from pdf for adhoc client runs. +Ensure you have permissions for API gateway, S3, Lambda function and SQS to execute this (DEVELOPER & above roles in DEV & UAT for Doczy should suffice) +If you have Analyst / Test roles and need the permissions to be elevated or policies to be updated then please ask Sannan Iqbal to make the changes. + +The flow to execute the pipeline is: +Create batch using api req +Add files to the newly created batch along with the batch id as the tag +Construct payload to trigger pipeline +Post request to trigger pipeline with payload +The output text file can be found in the client_bucket/contract_text_file/batch_123456 +And the final LLM parsed outputs can be found in client_bucket/final_output/batch_123456 +""" + + + + +import os +import boto3 +import requests +import json +import boto3 +from datetime import datetime +import os +from botocore.auth import SigV4Auth +from botocore.awsrequest import AWSRequest +from botocore.credentials import get_credentials +from botocore.session import Session +import time + +# Function to upload files to S3 +def upload_files_to_s3(directory, bucket, batch_id): + contract_list = [] + for filename in os.listdir(directory): + if filename.endswith('.pdf'): + file_path = os.path.join(directory, filename) + s3_key = f'contracts-landing-zone/{batch_id}/{filename}' + + # Upload file to S3 + s3_client.upload_file(file_path, bucket, s3_key) + print(f'Uploaded {filename} to S3 bucket\n') + + # Add tags to the uploaded file + s3_client.put_object_tagging( + Bucket=bucket, + Key=s3_key, + Tagging={ + 'TagSet': [ + { + 'Key': 'BatchId', + 'Value': batch_id + } + ] + } + ) + print(f'Added tags to {filename}\n') + # Add file details to contract list + # For now we can leave it as is since only A, C have been operationalized + contract_list.append({ + "contract_name": filename, + "groups": [ + "A", # This needs to be dynamic in UI 1 based on what group has been selected + "C" + ], + "contract_source_path": s3_key + }) + return contract_list + + + + +def create_batch(client_bucket, create_batch_url): + myobj = { "client-bucket-name": client_bucket } + + # Call create batch API endpoint + response = requests.post(create_batch_url, json = myobj) + if response.status_code >= 200 and response.status_code < 300: + try: + new_batch_id = json.loads(json.loads(response.text)['body'])['batch_id'] + landing_zone = json.loads(json.loads(response.text)['body'])['landing_zone'] + except: + print(myobj) + print(response.text) + new_batch_id = 'failed_cases' + landing_zone = 'contracts_landing_zone' + else: + print(response.text) + new_batch_id = 'failed_cases' + landing_zone = 'contracts_landing_zone' + return new_batch_id, landing_zone + + + + +def list_filtered_files(bucket_name, prefix, start_date, end_date, profile_name): + """ + List all files in an S3 bucket filtered by date range. + + Parameters: + - bucket_name: str, the name of the S3 bucket + - prefix: str, the prefix (folder) in the S3 bucket + - start_date: str, the start date in ISO 8601 format + - end_date: str, the end date in ISO 8601 format + - profile_name: str, the AWS profile name + + Returns: + - list of str: the keys of the filtered files + """ + # Parse the dates + start_date = datetime.fromisoformat(start_date.replace('Z', '+00:00')) + end_date = datetime.fromisoformat(end_date.replace('Z', '+00:00')) + + # Initialize a session using the specified profile + session = boto3.Session(profile_name=profile_name) + s3_client = session.client('s3') + + # List objects in the bucket with the specified prefix + paginator = s3_client.get_paginator('list_objects_v2') + page_iterator = paginator.paginate(Bucket=bucket_name, Prefix=prefix) + + # Filter files by date + filtered_files = [] + for page in page_iterator: + if 'Contents' in page: + for obj in page['Contents']: + last_modified = obj['LastModified'] + if start_date <= last_modified <= end_date: + filtered_files.append(obj['Key']) + + return filtered_files + + + +def download_files(bucket_name, file_keys, profile_name, local_directory): + """ + Download files from an S3 bucket. + + Parameters: + - bucket_name: str, the name of the S3 bucket + - file_keys: list of str, the keys of the files to download + - profile_name: str, the AWS profile name + - local_directory: str, the local directory to download files to + """ + # Initialize a session using the specified profile + session = boto3.Session(profile_name=profile_name) + s3_client = session.client('s3') + + # Ensure the local directory exists + if not os.path.exists(local_directory): + os.makedirs(local_directory) + + # Download each file + for key in file_keys: + file_name = os.path.basename(key) # Get only the file name from the key + local_file_path = os.path.join(local_directory, file_name) + print(f"Downloading {key} to {local_file_path}") + s3_client.download_file(bucket_name, key, local_file_path) + + print("Download completed.") + + + +if __name__ == "__main__": + # Define your variables + s3_bucket = 'doczyai-use2-u-cn1-s3-textract-processing-001' + # batch_id = 'batch_100524101551' + client_name = 'Priority Health' + username = 'ADHOC USER' + + # These endpoints are in UAT, please change them to DEV if there are access issues with UAT + api_endpoint = 'https://29gm8cek03.execute-api.us-east-2.amazonaws.com/dev/trigger-pipeline' + create_batch_url = "https://29gm8cek03.execute-api.us-east-2.amazonaws.com/dev/create-batch" + + + + session = boto3.Session(profile_name='temp_cred') # Change the profile name to the one you have in your .aws/credentials file + s3_client = session.client('s3') + + + + # Use current directory as PDF directory + pdf_directory = "C:\\Doczy\\Priority Health\\For_Textract\\For_Textract\\UAT- test East Paris Surgical Center copy" + + + # Create batch + batch_id, landing_zone = create_batch(s3_bucket, create_batch_url) + print(f"Batch ID: {batch_id}") + print(f"Landing Zone: {landing_zone}") + # batch_id = 'batch_110624213433' + # landing_zone = 'contracts_landing_zone/batch_110624213433/' + + if batch_id == 'failed_cases': + print("Batch creation failed. Exiting...") + exit() + # Upload files and get contract list + contract_list = upload_files_to_s3(pdf_directory, s3_bucket, batch_id) + + # Create JSON object + data = { + "s3_bucket": s3_bucket, + "batch_id": batch_id, + "client_name": client_name, + "username": username, + "contract_list": contract_list + } + + # Make POST request to API + response = requests.post(api_endpoint, json=data) + + + + + + # Print response + print(response.status_code) + print(response.json()) + + ################## + # Get text files + + prefix = f'contract-text-file/{batch_id}/' # if you have a specific prefix (folder) in your bucket + + # These dates are to filter the contracts in case there are older contracts in the same batch + start_date = '2024-06-04T00:00:00Z' # ISO 8601 format + end_date = '2024-06-14T23:59:59Z' # ISO 8601 format + profile_name = 'temp_cred' + local_directory = 'C:\\Doczy\\Priority Health\\For_Textract\\For_Textract\\text otuput' # local directory to save files + + # List filtered files + time.sleep(30) # Waiting for the text files to be generated, this may take longer and files may not be available after 30 seconds sometimes + filtered_files = list_filtered_files(s3_bucket, prefix, start_date, end_date, profile_name) + print(f"Filtered files: {len(filtered_files)}") + + # Download files + download_files(s3_bucket, filtered_files, profile_name, local_directory) \ No newline at end of file