Updated billed charges implied %

src/config.py

@@ -54,9 +54,9 @@ VALID_COLUMNS = ['Filename', 'page_num', 'SERVICE', 'REIMBURSEMENT_FLAT_FEE', 'R
 'Corrected_LOB', 'Corrected_PROGRAM', 'Corrected_NETWORK']
 
 # AWS Keys
-AWS_ACCESS_KEY_ID="ASIAZTMXAXNXFY5DMLIK"
-AWS_SECRET_ACCESS_KEY="IN2w1JrXU4f7rKR2IbiOwCZNUrgls3ZK/65w8cCS"
-AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEHMaCXVzLWVhc3QtMiJHMEUCIEhqvUNWd6r+bPwIIFkg242wxJGTn9k7qWiWVZSXn4dLAiEAr01immSRrcDW1oqW0QfN3gqivPNQV8JkTgLipIW0rpsqgwMIHBAAGgw2NjAxMzEwNjg3ODIiDJCiH+aROTYN8KdY2CrgAmCbjJwVL63njz5qxkOtexudA6CZ7Kz5BGpaA2WbdyHHc8j/UB/44MfOAxrTeLiP6w12qpYMc7U5GdZIULvNwWxJhJmnOHwLmLU0XuOtc52l3VCQmXLLMj1Q/dVPm+MVFgTM4RLTTNRGRE9boX1MP92pZlUZ/sjSKBl2ELfiJkrgzOkS3kYADmICJ0wWU1Ndqpi8sGTs8JHCyHCj3TSkJVaeCcwyvWssZHrgOv7/MUdgeHkiJx3m/MJNdOmxyZzr7mWtAFgJMODr8zM/vAPWogzcD0MEoMpa98ag9fcC6nE9us4KpiIIGAJNzXX6YpjkxoU+/tdXwMLsbkqmnqblz3zvB6jEyqTLMeEMYfexlzA7bBrq6Ooh2+nvTpFwF3E4RTdXsOdjgiw2nTwfjBYFT4cAGz9rQl2hZCLdhEhwZu5dmF/oKn6Nd5eec+KV0+Ep/SPor9p/uQh597zDDWt1BMwwurvHswY6pgFAChD2esP7NSzOQ+dIKA0CTZuNwdK7mjfkjtGEPsjDJBF3ueD4CybkHrqM2Ud6xhI8LwZU6DiIdUeqFXQHawdF1wDegBgy1Tymv1nCOZYCyTbW6ihJ+L4oJsr8XJwKpUhRgQQ8ECscAWFGRMN3zPt6T+OAcxJvjnKtdQO+vOCJNMzkr1yb7cuEoUcWPSwFrVkZg5iKP4T9DLLDCfGZUiV7AND6trJ+"
+AWS_ACCESS_KEY_ID="ASIAZTMXAXNXPGBTIYHY"
+AWS_SECRET_ACCESS_KEY="2zV8fp+u6LW8qpr75ZKEG/kWnvbrgQYo6Rg7MDln"
+AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEIr//////////wEaCXVzLWVhc3QtMiJIMEYCIQDZXOwwKnirwI7+o3716cEa6yVDLJiTu3Kzz7kPGooFuAIhAOGrTFPBigoNNA1x1OGJSMZpwGPLmiMxpJIpWbzYxyZbKoMDCDMQABoMNjYwMTMxMDY4NzgyIgwu4eVW0nZEUNi69Pcq4AJw9r1cQpF2vnIdXzmMp18dm9+oB7IZQMFQ9W3LR8R90PY3zk0HzY1UGaTIKnSPsXAdl5bIW2nb7a79RPoiQr4Ro70qM4G0WzHjaP8ottl/YCpiEhFzmNhBamT9lutBSQIyjumzzYa6LX76T2K9zSzumG9+2ubPKLtqLO91d9MV6EDCOt4hBsFM46WT4YBsVxEBQ3ABIsRAVkG2Ff8jgQ01nygd+IYqFjjQBndCxjSqcxM03+MQDBxsaRjNdwbJ/lRcrJpD7PeunPG1d+BIg9u4RfrHuTjJo8wVaZAKUlqjeRPWtS5woCiUEfGsSykWJp+N+x0cv+/tiVxzU1lPSwiLkwTQ5FotFr4dfA7q3NaqkKZ5YUA3GrKrATmsD1UlCRuCx7/jvMgCABFuCOERD7nrY57rgLY167PvKY7/8Fi6mpBgH8xHHtYoICfBQRMz2dLTwHVFC5R4pdA+f7ysF9MZMP+tzLMGOqUBCkhJSij99lLWdoOQQTn67bJHGMFcbXFHkeGnY7mk0jaLnACuodUMgvHPPuHDR6YsIEKk7Yw70pjMDoiXcbDgjbKdYquX86iXKMgv8RP6blXJJSgxsrabyBHTMWITMZDe530iQCoRkwiAo++RQanljFMah0LPCZpfqq6lpeLtws7Ya6SmvBUT6BBrhVDn0FZhKxnuXGJ58tnqaGA2eSYddht5lcDs"
 
 # File Paths
 LOCAL_PATH = 'data/test/' # Replace with local

src/postprocessingfuncs.py

@@ -227,7 +227,8 @@ def filter_service_column(df):
     df = df.dropna(subset=['SERVICE'])
 
     keywords = [
-        'LIABILITY', 'RISK', 'LOBBYING', 'DAMAGES', 'CONFIDENTIALITY', 'AUDIT', 'INTEREST', 'N/A', 'BUSINESS', 'COMPLIANCE', 'Medical Assistance Program'
+        'LIABILITY', 'RISK', 'LOBBYING', 'DAMAGES', 'CONFIDENTIALITY', 'ARBITRATION', 'FALSE CLAIMS ACT', 'UNSPECIFIED',
+        'AUDIT', 'INTEREST', 'N/A', 'BUSINESS', 'COMPLIANCE', 'Medical Assistance Program', 'MATERIAL SUBCONTRACT', 'GIFTS', 'GRATUITIES',
     ]
     pattern = '|'.join(keywords)
     df = df[~df['SERVICE'].str.contains(pattern, case=False, na=False)]

src/preprocess.py

@@ -40,7 +40,7 @@ def split_text(text):
     return text_dict
 
 
-def highlight_rates(text_dict):
+def highlight_rates(text):
     """
     Highlights rate-related terms (percentages and dollar amounts) in the text of each page within a dictionary.
 
@@ -54,15 +54,13 @@ def highlight_rates(text_dict):
     Returns:
         dict: The updated dictionary with rate-related terms highlighted in the text of each page.
     """
-    for page, text in text_dict.items():
-        words = text.split() 
-        highlighted_words = []
-        for word in words:
-            if '%' in word or '$' in word:
-                word = f'>>>{word}<<<'
-            highlighted_words.append(word)
-        text_dict[page] = ' '.join(highlighted_words)
-    return text_dict
+    words = text.split()
+    highlighted_words = []
+    for word in words:
+        if '%' in word or '$' in word:
+            word = f'>>>{word}<<<'
+        highlighted_words.append(word)
+    return ' '.join(highlighted_words)
 
 
 def chunk_text(text_dict):
@@ -97,7 +95,7 @@ def chunk_text(text_dict):
     return chunk_dict
 
 
-def clean_billed_charges(text):
+def clean_billed_charges(contract_text):
     """
     Cleans occurrences of 'billed charges' in a text by replacing them with '100% of billed charges' unless preceded by a percentage.
 
@@ -111,11 +109,40 @@ def clean_billed_charges(text):
     Returns:
         str: The cleaned text with appropriate replacements made for 'billed charges'.
     """
-    def replace(match):
-        start_pos = match.start() 
-        pre_text = text[max(0, start_pos-15):start_pos]
-        if '%' in pre_text:
-            return match.group() 
-        else:
-            return r'100% of billed charges' 
-    return re.sub(r'\bbilled charges\b', replace, text, flags=re.IGNORECASE)
+    
+    substrings = [
+        "Physician's Billed Charges",
+        "Provider's Billed Charges",
+        "Allowable Billed Charges",
+        "Hospital's Billed Charges",
+        "Physician's Charges",
+        "Provider's Charges",
+        "Allowable Charges",
+        "Hospital's Charges",
+        "Billed Charges"
+    ]
+    max_substring_length = max([len(s) for s in substrings])
+
+    def find_substring_indices(contract_text, s):
+        indices = []
+        lower_contract_text = contract_text.lower()
+        lower_s = s.lower()
+        index = lower_contract_text.find(lower_s)
+        while index != -1:
+            indices.append(index)
+            index = lower_contract_text.find(lower_s, index + 1)
+        return indices
+    
+    for s in substrings:
+        indices = find_substring_indices(contract_text, s)
+        index_adder = 0
+        if indices:
+            for i in indices:
+                index = i+index_adder
+                end_index = index + max_substring_length
+                match_part = contract_text[index:end_index]
+                previous = contract_text[max(0, index-30):index]
+                if '%' not in previous:
+                    contract_text = contract_text[0:index] + f" 100% of {match_part}" + contract_text[end_index:]
+                    index_adder += 9
+    return contract_text.replace('  ', ' ')

src/prompt_funcs.py

@@ -107,13 +107,10 @@ def run_bottom_up_secondary(answer_dicts, text_dict, tokens):
             
             # Bottom Up Methodology
             if config.RUN_METHODOLOGY:
-                if d['REIMBURSEMENT_FLAT_FEE'] == 'N/A':
-                    prompt = prompts.BOTTOM_UP_METHODOLOGY(d)
-                    methodology_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=100)
-                    d['REIMBURSEMENT_METHODOLOGY'] = methodology_answer
-                else:
-                    d['REIMBURSEMENT_METHODOLOGY'] = 'Flat Fee'
-
+                prompt = prompts.BOTTOM_UP_METHODOLOGY(d)
+                methodology_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=100)
+                d['REIMBURSEMENT_METHODOLOGY'] = methodology_answer
+                
             # # Bottom Up FS
             if config.RUN_FS:
                 prompt = prompts.BOTTOM_UP_FS(d)

src/prompts.py

@@ -38,7 +38,9 @@ def BOTTOM_UP_PRIMARY(page, payer):
 
 The preceding text is one page of a contract between Payer {payer} and a provider in their network. Your job is to extract attributes related to the reimbursement of different services and specialties. 
 
-The reimbursement values will be identified with >>> <<< indicators (e.g. >>>105%<<<). Make sure there is at least one dictionary object for EVERY reimbursement value seen (either % or $). Some values are found in sections explicitly identified as examples - these must be omitted from output.
+The reimbursement values will be identified with >>> <<< indicators (e.g. >>>105%<<<). Make sure there is at least one dictionary object for EVERY reimbursement value seen (either % or $). 
+
+Some values are found in sections explicitly identified as examples - these must be omitted from output. Other values might be related to liability - these must also be excluded.
 
 If any of the attributes are not found, return N/A. For all attributes, only write what is written on the page. Do not make up new phrases or words.
 
@@ -122,7 +124,7 @@ def BOTTOM_UP_METHODOLOGY(d):
 Your job is to identify what the concise methodology is based on the full methodology.
 
 Choose the closest option from the following:
-Allowed Amount, Allowable Charges, Fee Schedule, Medicare Fee Schedule, Medicare Allowed Amount, Medicaid Fee Schedule, Medicaid Allowed Amount, Provider's Charges, Billed Charges, IME/DME Amount, Cost to Charge Ratio, MSRP, Per Unit, Per Diem, Per Visit, Per Member, Per Member Per Month, Per Hour, Case Rate, Contracted Rate
+Allowed Amount, Allowable Charges, Fee Schedule, Medicare Fee Schedule, Medicare Allowed Amount, Medicaid Fee Schedule, Medicaid Allowed Amount, Provider's Charges, Billed Charges, Normal fee for such service, IME/DME Amount, Cost to Charge Ratio, Cost plus %, MSRP, Per Unit, Per Diem, Per Visit, Per Member, Per Member Per Month, Per Hour, Case Rate, Contracted Rate
 
 For 'Allowed Amount' and 'Fee Schedule', use the more specific answer (Medicare/Medicaid) if applicable.
 For the 'Per ___' values, note that this list is not necessarily exhaustive. It could be anything like 'Per ___'.

src/table_funcs.py

@@ -73,21 +73,22 @@ def align_and_format_tables(text_dict):
     """
     aligned_text_dict = {}
     for key, text in text_dict.items():
+        print(key)
         aligned_text = text
         if 'Table Start' in text:
             table_texts = re.findall(r'-------Table Start--------(.*?)-------Table End--------', text, re.DOTALL)
             for table_text in table_texts:
-                try:
-                    # Extract pretable text
-                    pretable = table_text.split('{')[0].strip()
-                    # Extract and format table text
-                    #table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0].replace("'", '"') + "}"
-                    table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0] + "}"
+                
+                # Extract pretable text
+                pretable = table_text.split('{')[0].strip()
+                # Extract and format table text
+                #table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0].replace("'", '"') + "}"
+                table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0] + "}"
 
-                    table, table_size = convert_to_dict(table_only)
-                    table_formatted = format_table(table, table_size)
-                except:
-                    table_formatted = table_text
+                table, table_size = convert_to_dict(table_only)
+                table_formatted = format_table(table, table_size)
+            
+                table_formatted = table_text
                 
                 # Align
                 if text.count(pretable) == 2: # One table

src/test.py

@@ -1,4 +1,5 @@
 import pandas as pd
+import numpy as np
 import re
 import json
 import csv
@@ -31,14 +32,55 @@ import claude_funcs
 
 
 input_dict = utils.read_input()
-
 (filename, contract_text) = list(input_dict.items())[0]
 
-contract_text = preprocess.clean_newlines(contract_text)
-text_dict = preprocess.split_text(contract_text)
-text_dict = table_funcs.align_and_format_tables(text_dict)
-text_dict = preprocess.highlight_rates(text_dict)
+def clean_billed_charges(contract_text):
+    substrings = [
+        "Physician's Billed Charges",
+        "Provider's Billed Charges",
+        "Allowable Billed Charges",
+        "Hospital's Billed Charges",
+        "Physician's Charges",
+        "Provider's Charges",
+        "Allowable Charges",
+        "Hospital's Charges",
+        "Billed Charges"
+    ]
+    max_substring_length = np.max([len(s) for s in substrings])
 
-bu_results = prompt_funcs.run_bottom_up_primary({'5' : text_dict['5']}, 4000)  # Returns list of dictionaries
+    def find_substring_indices(contract_text, s):
+        indices = []
+        lower_contract_text = contract_text.lower()
+        lower_s = s.lower()
+        index = lower_contract_text.find(lower_s)
+        while index != -1:
+            indices.append(index)
+            index = lower_contract_text.find(lower_s, index + 1)
+        return indices
+    
+    for s in substrings:
+        indices = find_substring_indices(contract_text, s)
+        index_adder = 0
+        if indices:
+            for i in indices:
+                index = i+index_adder
+                end_index = index + max_substring_length
+                match_part = contract_text[index:end_index]
+                previous = contract_text[max(0, index-30):index]
+                if '%' not in previous:
+                    contract_text = contract_text[0:index] + f" 100% of {match_part}" + contract_text[end_index:]
+                    index_adder += 9
+    return contract_text.replace('  ', ' ')
+
+contract_text = preprocess.clean_newlines(contract_text)
+contract_text = clean_billed_charges(contract_text)
+contract_text = preprocess.highlight_rates(contract_text)
+text_dict = preprocess.split_text(contract_text)
+# text_dict = table_funcs.align_and_format_tables(text_dict)
+
+
+print(text_dict['23'])
+
+bu_results = prompt_funcs.run_bottom_up_primary({'23' : text_dict['23']}, 4000)  # Returns list of dictionaries
 print(bu_results)
     

src/textract_template.py

@@ -0,0 +1,236 @@
+
+"""
+This script was written to extract .txt files from pdf for adhoc client runs.
+Ensure you have permissions for API gateway, S3, Lambda function and SQS to execute this (DEVELOPER & above roles in DEV & UAT for Doczy should suffice)
+If you have Analyst / Test roles and need the permissions to be elevated or policies to be updated then please ask Sannan Iqbal to make the changes.
+ 
+The flow to execute the pipeline is:
+Create batch using api req
+Add files to the newly created batch along with the batch id as the tag
+Construct payload to trigger pipeline
+Post request to trigger pipeline with payload
+The output text file can be found in the client_bucket/contract_text_file/batch_123456
+And the final LLM parsed outputs can be found in client_bucket/final_output/batch_123456
+"""
+
+
+
+
+import os
+import boto3
+import requests
+import json
+import boto3
+from datetime import datetime
+import os 
+from botocore.auth import SigV4Auth
+from botocore.awsrequest import AWSRequest
+from botocore.credentials import get_credentials
+from botocore.session import Session
+import time
+
+# Function to upload files to S3
+def upload_files_to_s3(directory, bucket, batch_id):
+    contract_list = []
+    for filename in os.listdir(directory):
+        if filename.endswith('.pdf'):
+            file_path = os.path.join(directory, filename)
+            s3_key = f'contracts-landing-zone/{batch_id}/{filename}'
+            
+            # Upload file to S3
+            s3_client.upload_file(file_path, bucket, s3_key)
+            print(f'Uploaded {filename} to S3 bucket\n')
+
+            # Add tags to the uploaded file
+            s3_client.put_object_tagging(
+                Bucket=bucket,
+                Key=s3_key,
+                Tagging={
+                    'TagSet': [
+                        {
+                            'Key': 'BatchId',
+                            'Value': batch_id
+                        }
+                    ]
+                }
+            )
+            print(f'Added tags to {filename}\n')
+            # Add file details to contract list
+            # For now we can leave it as is since only A, C have been operationalized 
+            contract_list.append({
+                "contract_name": filename,
+                "groups": [
+                    "A", # This needs to be dynamic in UI 1 based on what group has been selected
+                    "C"                    
+                ],
+                "contract_source_path": s3_key
+            })
+    return contract_list
+
+
+
+
+def create_batch(client_bucket, create_batch_url):
+    myobj = { "client-bucket-name": client_bucket }
+
+    # Call create batch API endpoint
+    response = requests.post(create_batch_url, json = myobj)
+    if response.status_code >= 200 and response.status_code < 300:
+        try:
+            new_batch_id = json.loads(json.loads(response.text)['body'])['batch_id']
+            landing_zone = json.loads(json.loads(response.text)['body'])['landing_zone']
+        except:
+            print(myobj)
+            print(response.text)
+            new_batch_id = 'failed_cases'
+            landing_zone = 'contracts_landing_zone'
+    else:
+        print(response.text)
+        new_batch_id = 'failed_cases'
+        landing_zone = 'contracts_landing_zone'
+    return new_batch_id, landing_zone
+
+
+
+
+def list_filtered_files(bucket_name, prefix, start_date, end_date, profile_name):
+    """
+    List all files in an S3 bucket filtered by date range.
+
+    Parameters:
+    - bucket_name: str, the name of the S3 bucket
+    - prefix: str, the prefix (folder) in the S3 bucket
+    - start_date: str, the start date in ISO 8601 format
+    - end_date: str, the end date in ISO 8601 format
+    - profile_name: str, the AWS profile name
+
+    Returns:
+    - list of str: the keys of the filtered files
+    """
+    # Parse the dates
+    start_date = datetime.fromisoformat(start_date.replace('Z', '+00:00'))
+    end_date = datetime.fromisoformat(end_date.replace('Z', '+00:00'))
+
+    # Initialize a session using the specified profile
+    session = boto3.Session(profile_name=profile_name)
+    s3_client = session.client('s3')
+
+    # List objects in the bucket with the specified prefix
+    paginator = s3_client.get_paginator('list_objects_v2')
+    page_iterator = paginator.paginate(Bucket=bucket_name, Prefix=prefix)
+
+    # Filter files by date
+    filtered_files = []
+    for page in page_iterator:
+        if 'Contents' in page:
+            for obj in page['Contents']:
+                last_modified = obj['LastModified']
+                if start_date <= last_modified <= end_date:
+                    filtered_files.append(obj['Key'])
+
+    return filtered_files
+
+
+
+def download_files(bucket_name, file_keys, profile_name, local_directory):
+    """
+    Download files from an S3 bucket.
+
+    Parameters:
+    - bucket_name: str, the name of the S3 bucket
+    - file_keys: list of str, the keys of the files to download
+    - profile_name: str, the AWS profile name
+    - local_directory: str, the local directory to download files to
+    """
+    # Initialize a session using the specified profile
+    session = boto3.Session(profile_name=profile_name)
+    s3_client = session.client('s3')
+
+    # Ensure the local directory exists
+    if not os.path.exists(local_directory):
+        os.makedirs(local_directory)
+
+    # Download each file
+    for key in file_keys:
+        file_name = os.path.basename(key)  # Get only the file name from the key
+        local_file_path = os.path.join(local_directory, file_name)
+        print(f"Downloading {key} to {local_file_path}")
+        s3_client.download_file(bucket_name, key, local_file_path)
+
+    print("Download completed.")
+
+
+
+if __name__ == "__main__":
+    # Define your variables
+    s3_bucket = 'doczyai-use2-u-cn1-s3-textract-processing-001'
+    # batch_id = 'batch_100524101551'
+    client_name = 'Priority Health'
+    username = 'ADHOC USER'
+
+    # These endpoints are in UAT, please change them to DEV if there are access issues with UAT
+    api_endpoint = 'https://29gm8cek03.execute-api.us-east-2.amazonaws.com/dev/trigger-pipeline'
+    create_batch_url = "https://29gm8cek03.execute-api.us-east-2.amazonaws.com/dev/create-batch"
+
+
+    
+    session = boto3.Session(profile_name='temp_cred') # Change the profile name to the one you have in your .aws/credentials file
+    s3_client = session.client('s3')
+
+    
+
+    # Use current directory as PDF directory
+    pdf_directory = "C:\\Doczy\\Priority Health\\For_Textract\\For_Textract\\UAT- test East Paris Surgical Center copy"
+    
+
+    # Create batch
+    batch_id, landing_zone = create_batch(s3_bucket, create_batch_url)
+    print(f"Batch ID: {batch_id}")
+    print(f"Landing Zone: {landing_zone}")
+    # batch_id = 'batch_110624213433'
+    # landing_zone = 'contracts_landing_zone/batch_110624213433/' 
+
+    if batch_id == 'failed_cases':
+        print("Batch creation failed. Exiting...")
+        exit()
+    # Upload files and get contract list
+    contract_list = upload_files_to_s3(pdf_directory, s3_bucket, batch_id)
+
+    # Create JSON object
+    data = {
+        "s3_bucket": s3_bucket,
+        "batch_id": batch_id,
+        "client_name": client_name,
+        "username": username,
+        "contract_list": contract_list
+    }
+
+    # Make POST request to API
+    response = requests.post(api_endpoint, json=data)
+
+
+
+
+
+    # Print response
+    print(response.status_code)
+    print(response.json())
+
+    ##################
+    # Get text files
+
+    prefix = f'contract-text-file/{batch_id}/'  # if you have a specific prefix (folder) in your bucket
+
+    # These dates are to filter the contracts in case there are older contracts in the same batch
+    start_date = '2024-06-04T00:00:00Z'  # ISO 8601 format
+    end_date = '2024-06-14T23:59:59Z'  # ISO 8601 format
+    profile_name = 'temp_cred'
+    local_directory = 'C:\\Doczy\\Priority Health\\For_Textract\\For_Textract\\text otuput'  # local directory to save files
+
+    # List filtered files
+    time.sleep(30) # Waiting for the text files to be generated, this may take longer and files may not be available after 30 seconds sometimes
+    filtered_files = list_filtered_files(s3_bucket, prefix, start_date, end_date, profile_name)
+    print(f"Filtered files: {len(filtered_files)}")
+
+    # Download files
+    download_files(s3_bucket, filtered_files, profile_name, local_directory)