added new postprocessing and bottom up and top down merging. also carveouts
This commit is contained in:
FaizanM2000
committed by
Michael McGuinness
Michael McGuinness
parent
31cb01872c
commit
81f34edc8d
@@ -81,3 +81,6 @@ texts/
|
||||
myenv/
|
||||
*.env
|
||||
*.txt
|
||||
subset/
|
||||
output/
|
||||
docs/
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
import pandas as pd
|
||||
from difflib import SequenceMatcher
|
||||
import re
|
||||
import time
|
||||
import prompts
|
||||
import claude_funcs
|
||||
|
||||
def get_closest_substring_match(val, valid_values):
|
||||
""" Returns the first match from valid_values using a case-insensitive substring match. """
|
||||
if pd.isna(val):
|
||||
return None
|
||||
val = val.strip().upper()
|
||||
for valid_value in valid_values:
|
||||
if valid_value.upper() in val:
|
||||
return valid_value
|
||||
return None
|
||||
|
||||
def get_best_carveout_from_claude(carveout_list, service):
|
||||
print("using claude to get best carveout...")
|
||||
prompt = f"Given the service '{service}', please choose the best matching carveout from the following list: {', '.join(carveout_list)}. ONLY SELECT ONE FROM THE LIST. DO NOT RETURN A SENTENCE"
|
||||
while True:
|
||||
try:
|
||||
response = claude_funcs.invoke_claude_3(prompt, max_tokens=100)
|
||||
return response.strip()
|
||||
except Exception as e:
|
||||
print(f"Error occurred: {e}. Waiting for 60 seconds before retrying...")
|
||||
time.sleep(60)
|
||||
|
||||
def check_prov_type_similarity(prov_type, carveout):
|
||||
print("checking similarity between prov_type and carveout with claude...")
|
||||
prompt = f"Do the provider type '{prov_type}' and the carveout '{carveout}' mean the same thing or are they very similar? ONLY ANSWER WITH 'True' OR 'False'"
|
||||
while True:
|
||||
try:
|
||||
response = claude_funcs.invoke_claude_3(prompt, max_tokens=10)
|
||||
return response.strip()
|
||||
except Exception as e:
|
||||
print(f"Error occurred: {e}. Waiting for 60 seconds before retrying...")
|
||||
time.sleep(60)
|
||||
|
||||
def label_services(filepath, carveout_list, output_filepath):
|
||||
# Read the CSV file
|
||||
df = pd.read_csv(filepath)
|
||||
|
||||
# Select the first 50 rows of the DataFrame
|
||||
df = df.head(500)
|
||||
|
||||
# Define primary service terms
|
||||
primary_terms = ['Covered Services', 'Inpatient Services', 'Outpatient Services', 'Physician Services',
|
||||
'Inpatient', 'Outpatient', 'Physician', 'Medical', 'Surgical', 'Diagnostic', 'Therapeutic']
|
||||
|
||||
# Initialize columns for the results
|
||||
df['carveout_matched'] = ''
|
||||
df['label'] = ''
|
||||
df['IS_CARVEOUT'] = ''
|
||||
|
||||
# Initialize a nested dictionary to track occurrences for each Filename and TD_LOB
|
||||
occurrence_tracker = {}
|
||||
|
||||
carveout_count = 0
|
||||
|
||||
# Iterate through the rows in the DataFrame
|
||||
for index, row in df.iterrows():
|
||||
service = str(row['SERVICE'])
|
||||
prov_type = str(row['PROV_TYPE']) if pd.notna(row['PROV_TYPE']) else ''
|
||||
td_lob = row['TD_LOB']
|
||||
filename = row['Filename']
|
||||
|
||||
# Initialize the nested dictionary for the filename and TD_LOB if not present
|
||||
if filename not in occurrence_tracker:
|
||||
occurrence_tracker[filename] = {}
|
||||
if td_lob not in occurrence_tracker[filename]:
|
||||
occurrence_tracker[filename][td_lob] = {term: 0 for term in primary_terms}
|
||||
|
||||
# Check if the service is a primary or similar to a primary
|
||||
primary = get_closest_substring_match(service, primary_terms)
|
||||
if primary:
|
||||
# Ensure the primary term is initialized in the occurrence tracker
|
||||
if primary not in occurrence_tracker[filename][td_lob]:
|
||||
occurrence_tracker[filename][td_lob][primary] = 0
|
||||
|
||||
# Determine the label based on the count of this primary term for the given Filename and TD_LOB
|
||||
count = occurrence_tracker[filename][td_lob][primary]
|
||||
if count == 0:
|
||||
df.at[index, 'label'] = 'primary'
|
||||
elif count == 1:
|
||||
df.at[index, 'label'] = 'secondary'
|
||||
elif count == 2:
|
||||
df.at[index, 'label'] = 'tertiary'
|
||||
else:
|
||||
df.at[index, 'label'] = 'additional'
|
||||
occurrence_tracker[filename][td_lob][primary] += 1
|
||||
df.at[index, 'IS_CARVEOUT'] = 'N'
|
||||
else:
|
||||
# If PROV_TYPE is blank, continue with carveout determination
|
||||
best_carveout_response = get_best_carveout_from_claude(carveout_list, service)
|
||||
matched_carveout = best_carveout_response # Use the model response directly
|
||||
|
||||
df.at[index, 'carveout_matched'] = matched_carveout
|
||||
|
||||
if matched_carveout:
|
||||
# Check similarity between PROV_TYPE and carveout
|
||||
prov_type_similarity_score = SequenceMatcher(None, prov_type, matched_carveout).ratio()
|
||||
if prov_type_similarity_score < 0.8:
|
||||
prov_type_similarity = check_prov_type_similarity(prov_type, matched_carveout)
|
||||
else:
|
||||
prov_type_similarity = "True"
|
||||
|
||||
if re.search(r'\b(True|Yes)\b', prov_type_similarity, re.IGNORECASE):
|
||||
if matched_carveout not in occurrence_tracker[filename][td_lob]:
|
||||
print('claude returned a match between provider type and service')
|
||||
occurrence_tracker[filename][td_lob][matched_carveout] = 0
|
||||
count = occurrence_tracker[filename][td_lob][matched_carveout]
|
||||
if count == 0:
|
||||
df.at[index, 'label'] = 'primary'
|
||||
elif count == 1:
|
||||
df.at[index, 'label'] = 'secondary'
|
||||
elif count == 2:
|
||||
df.at[index, 'label'] = 'tertiary'
|
||||
else:
|
||||
df.at[index, 'label'] = 'additional'
|
||||
occurrence_tracker[filename][td_lob][matched_carveout] += 1
|
||||
df.at[index, 'IS_CARVEOUT'] = 'N'
|
||||
else:
|
||||
df.at[index, 'IS_CARVEOUT'] = 'Y'
|
||||
else:
|
||||
df.at[index, 'IS_CARVEOUT'] = 'Y'
|
||||
|
||||
carveout_count += 1
|
||||
|
||||
# Save the updated DataFrame to a new CSV file after every 10 carveouts
|
||||
if carveout_count % 10 == 0:
|
||||
df.to_csv(output_filepath, index=False)
|
||||
print(f"Saved progress after processing {carveout_count} carveouts.")
|
||||
|
||||
# Final save of the updated DataFrame to a new CSV file
|
||||
df.to_csv(output_filepath, index=False)
|
||||
|
||||
label_services('service.csv', prompts.get_carveout_list(), 'carveouts50.csv')
|
||||
+5
-4
@@ -3,6 +3,7 @@
|
||||
from datetime import datetime
|
||||
import boto3
|
||||
from llama_index.llms.bedrock import Bedrock
|
||||
#pip install llama-index-llms-bedrock
|
||||
|
||||
# General Settings
|
||||
TEST = False # True to run test prompt - just for testing model connection
|
||||
@@ -28,12 +29,12 @@ RUN_EXCEPTION = True
|
||||
RUN_CODES = True
|
||||
|
||||
# AWS Keys
|
||||
AWS_ACCESS_KEY_ID="ASIAZTMXAXNXNEUYUPUY"
|
||||
AWS_SECRET_ACCESS_KEY="3HljGiz+dqWBtj+dkqs/syyxoguUoTahdLyceEeo"
|
||||
AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEBUaCXVzLWVhc3QtMiJHMEUCIQCX9LcYh9A28cLNA/lDEHJ6ZqrulN0FampT1ptNyX+9JAIgIFrmDiZS5KfHbZWptuvveZt2s7x5r0xDwCU6t6EgBAEqjAMIjv//////////ARAAGgw2NjAxMzEwNjg3ODIiDOLRiVmy5kw46/5mzCrgAscMrf+1rEm7kpwpmd+BHjLNV4ggEy0oqQQ6z5Ny5BV6Ouo0tRmXqOqSaI5/YKXeKsRo8s4lrDNNq8qIK0GxpugvmbqguKYfEzKwyihGi5IVVb2JGL1nBkL495v0w5j1fuXqNwB7VCPkJyJCqxDHeehgtQ1AymuVAiRowkWmr7vVvFQyKKcGsBJlIzBZyguTtJhT2HPCp8IW9LhQMS2XBAa4xCnHqpPZSYIoKyp2zPPwi3pBKm+7eE3NGH7V7/Vrl6IL29zNkk5K/IeqAcyBzaFgaCPilr5N6lSMeKlkz8w0ls3guZeaEjDRf/Mz3NvMD9HeUVqGfi2M+ODxudgEfmfH3Op0EC2g5fsIBu/uvcjJf7PgUo9xT6CH5P5HY/e6TLaZH55ioz468uaTfoNw10TZ/6iymqYEXSpLfqHejC6qCx2pfcYf1UizgD2rn22GfcGs8u6oKFSThJ4fETq1mogwzafCsgY6pgHl1EzoiRwiXqCDl3GsA162dYrapmEPqcIVXTTuNuYXbl7B3GQfxTmjHjqSqKFhieWEvydD4SSQAhSXz9tPsN69Myyelwg3t6r2EWGL+JiZXEFpO1peWJgu3xfd3F1tPnp7Nfv7bVGmuIvft1JYTmx20W/vH9Q4J7jM4tPqEeQs5fRORTdXcm10Yysuq5FYo+9bahUJnPYblcJveO30vVirwzVWvmPm"
|
||||
AWS_ACCESS_KEY_ID="ASIAZTMXAXNXFULN3ZDJ"
|
||||
AWS_SECRET_ACCESS_KEY="biHj+LwFkcquA7hoOgWst66u7AAQcik+A8TnMSoO"
|
||||
AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjELr//////////wEaCXVzLWVhc3QtMiJHMEUCIEIZfQMj70zYiV1czvy0RdOdugmJ9YGZIPBk/wqUE8m1AiEAzA9jGoZfv0ZMZlBKTLHGT3Ax3rAQNVLYWqgo9jy3Ui8qiQMIQxAAGgw2NjAxMzEwNjg3ODIiDOYEfYdB5ykwPYhKOSrmAiHPCbfUR2lXoqyEvURpTFyOqLEmeVFHCHQoVyeFcXwQmjjZor+ui1dclq0asj35ZSScClpyQKZsfk5o5+76pxzTm1tlBez1buIFkel/9OngHNG4MvMwLCMLdI+ILtJy0ufLmky4oVgtSor5n2e5dLENqstqhXEREbGdxZ5gaF7zKg9Dk/VPQ8arzV0oEvxuPGbZ+Z640jtVR27oSU1pQQ4ayK+CM/jBMANlzO9BptRB+Phfm6oOfxYQskyIXRV+aff/3d/MVJs6X08U/YjN+qXx0pDLsyBxYQIpKZXIZvWrYRD/NM7DnPJF8DlTAWhNSDU0QJ4j2oTMKcaG0LHXmd1SejjgHXhK0ZiG2tcpNHPaIBFm/lWnaixOdxM9XUWckoTe+1FRmcblQ7YI2HtWsbgzU3B8BLmLvEPX+lJSDeJBuv1MGFA4ITa4YaRywBBdZTcSl9cwLGB1X9ksTga5cd1XZeW/218w5cfmsgY6pgEsq/5MsjzbPqT20XDAhlC7GUmY3K+glAHfm8nVnihVdfmfU2apbx0bcDu5Er9mngykbj0u24P8VbkmlozS2H2dDmCjqQeE2Ux2pOuUxfqtMlJVSjAO8MCVvjc2nxsv55iHDTHGSlaN8TmsoBqVRbsOmauWFPQ7gFJDqF7YADS+20NDsgblAeeE23eZU4A/mG1dWfIK5E58nAoKWvnumFyQnJ0sMPpN"
|
||||
|
||||
# File Paths
|
||||
LOCAL_PATH = 'docs/priority_health/' # Replace with local
|
||||
LOCAL_PATH = 'subset/' # Replace with local
|
||||
|
||||
# S3 Settings
|
||||
S3_CLIENT = boto3.client('s3',
|
||||
|
||||
@@ -0,0 +1,565 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Imports\n",
|
||||
"import pandas as pd\n",
|
||||
"import re\n",
|
||||
"import ast\n",
|
||||
"import json\n",
|
||||
"\n",
|
||||
"import prompt_funcs\n",
|
||||
"import config\n",
|
||||
"import utils\n",
|
||||
"import preprocess\n",
|
||||
"import table_funcs\n",
|
||||
"import claude_funcs\n",
|
||||
"import prompts"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"function clean_td(td):\n",
|
||||
" Initialize td_clean as an empty list\n",
|
||||
" for each dictionary d in td:\n",
|
||||
" Initialize new_d as an empty dictionary\n",
|
||||
" for each key k and value v in d:\n",
|
||||
" if 'DATE' in k:\n",
|
||||
" new_d[k] = v as a list\n",
|
||||
" else if k is not 'page_num' or 'Filename':\n",
|
||||
" if v is a string with commas:\n",
|
||||
" new_d[k] = split v into a list by commas\n",
|
||||
" else if v is 'N/A':\n",
|
||||
" new_d[k] = an empty list\n",
|
||||
" else:\n",
|
||||
" new_d[k] = v as a list\n",
|
||||
" else:\n",
|
||||
" new_d[k] = v\n",
|
||||
" Append new_d to td_clean\n",
|
||||
" return td_clean\n",
|
||||
"\n",
|
||||
"function get_unique_keys(dicts):\n",
|
||||
" Initialize keys as an empty set\n",
|
||||
" for each dictionary d in dicts:\n",
|
||||
" Add all keys in d to keys\n",
|
||||
" return keys\n",
|
||||
"\n",
|
||||
"function get_unique_date_fields(td_results):\n",
|
||||
" Initialize unique_date_fields as an empty dictionary\n",
|
||||
" for each dictionary td in td_results:\n",
|
||||
" for each key k and value v in td:\n",
|
||||
" if 'DATE' is in k:\n",
|
||||
" if k is not in unique_date_fields:\n",
|
||||
" Initialize unique_date_fields[k] as an empty set\n",
|
||||
" Add v to unique_date_fields[k] as a list\n",
|
||||
" Convert sets in unique_date_fields to lists\n",
|
||||
" return unique_date_fields\n",
|
||||
"\n",
|
||||
"function GENERATE_PROMPT(bu_dict, td_dicts, page, field_name=None, values=None):\n",
|
||||
" if field_name and values:\n",
|
||||
" return a prompt string with page, field_name, and values\n",
|
||||
" else:\n",
|
||||
" return a prompt string with page, SERVICE, and METHODOLOGY from bu_dict\n",
|
||||
"\n",
|
||||
"function merge_results(td_results, bu_results, text_dict):\n",
|
||||
" all_keys = get_unique_keys(td_results) union get_unique_keys(bu_results)\n",
|
||||
" date_fields = get_unique_date_fields(td_results)\n",
|
||||
" Initialize merged_results as an empty list\n",
|
||||
" for each dictionary bu in bu_results:\n",
|
||||
" Initialize merged as a dictionary with keys from all_keys and values from bu\n",
|
||||
" td_on_page = filter td_results by page_num matching bu's page_num\n",
|
||||
" for each dictionary td in td_on_page:\n",
|
||||
" for each key k and value v in td:\n",
|
||||
" if merged[k] is empty:\n",
|
||||
" merged[k] = v\n",
|
||||
" else if v is a list and merged[k] is not a list:\n",
|
||||
" merged[k] = v\n",
|
||||
" else if v is a list:\n",
|
||||
" Extend merged[k] with v\n",
|
||||
" for each date_key and date_values in date_fields:\n",
|
||||
" if date_key is not in merged or merged[date_key] is empty:\n",
|
||||
" merged[date_key] = date_values\n",
|
||||
" else:\n",
|
||||
" Extend merged[date_key] with values not already in merged[date_key]\n",
|
||||
" for each key in all_keys:\n",
|
||||
" if merged[key] is a list with more than one value:\n",
|
||||
" page_text = get text for bu's page_num from text_dict\n",
|
||||
" prompt = GENERATE_PROMPT(bu, td_on_page, page_text, key, merged[key])\n",
|
||||
" response = invoke LLM with prompt\n",
|
||||
" if response is valid:\n",
|
||||
" merged[key] = response\n",
|
||||
" else:\n",
|
||||
" merged[key] = join merged[key] into a string\n",
|
||||
" Append merged to merged_results\n",
|
||||
" return merged_results\n",
|
||||
"\n",
|
||||
"function process_file(filename, input_dict):\n",
|
||||
" contract_text = read contract text from input_dict using filename\n",
|
||||
" Clean contract_text by removing newlines\n",
|
||||
" Split contract_text into pages stored in text_dict\n",
|
||||
" Run Top Down analysis and store results in td_results\n",
|
||||
" Run Bottom Up analysis and store results in bu_results\n",
|
||||
" Combine td_results and bu_results into combined_results\n",
|
||||
" Create output directory based on filename\n",
|
||||
" Save td_results, bu_results, and combined_results as CSV files in the output directory\n",
|
||||
"\n",
|
||||
"function process_all_files(input_dict):\n",
|
||||
" for each filename in input_dict:\n",
|
||||
" Call process_file with filename and input_dict\n",
|
||||
" Print a message indicating the file has been processed\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import pandas as pd\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def clean_td(td):\n",
|
||||
" td_clean = []\n",
|
||||
" for d in td:\n",
|
||||
" new_d = {}\n",
|
||||
" for k, v in d.items():\n",
|
||||
" if 'DATE' in k:\n",
|
||||
" new_d[k] = v if isinstance(v, list) else [v]\n",
|
||||
" elif k not in ['page_num', 'Filename']:\n",
|
||||
" if isinstance(v, str) and ',' in v:\n",
|
||||
" new_d[k] = [item.strip() for item in v.split(',')]\n",
|
||||
" elif v == 'N/A':\n",
|
||||
" new_d[k] = []\n",
|
||||
" else:\n",
|
||||
" new_d[k] = [v] if isinstance(v, str) else v\n",
|
||||
" else:\n",
|
||||
" new_d[k] = v\n",
|
||||
" td_clean.append(new_d)\n",
|
||||
" return td_clean\n",
|
||||
"\n",
|
||||
"def get_unique_keys(dicts):\n",
|
||||
" keys = set()\n",
|
||||
" for d in dicts:\n",
|
||||
" keys.update(d.keys())\n",
|
||||
" return keys\n",
|
||||
"\n",
|
||||
"def get_unique_date_fields(td_results):\n",
|
||||
" unique_date_fields = {}\n",
|
||||
" for td in td_results:\n",
|
||||
" for key, value in td.items():\n",
|
||||
" if 'DATE' in key:\n",
|
||||
" if key not in unique_date_fields:\n",
|
||||
" unique_date_fields[key] = set()\n",
|
||||
" unique_date_fields[key].update(value if isinstance(value, list) else [value])\n",
|
||||
" # Convert sets to lists for consistency\n",
|
||||
" for key in unique_date_fields:\n",
|
||||
" unique_date_fields[key] = list(unique_date_fields[key])\n",
|
||||
" return unique_date_fields\n",
|
||||
"\n",
|
||||
"def GENERATE_PROMPT(bu_dict, td_dicts, page, field_name=None, values=None):\n",
|
||||
" if field_name and values:\n",
|
||||
" return f\"\"\"### PAGE START ### {page} ### PAGE END\n",
|
||||
" \n",
|
||||
"Field: {field_name}\n",
|
||||
"Values: {values}\n",
|
||||
"\n",
|
||||
"Listed above is a page of a contract, as well as the potential values for the {field_name} field.\n",
|
||||
"Your job is to identify which of these values is the most appropriate based on the page content.\n",
|
||||
"\n",
|
||||
"Write ONLY the value that is most appropriate. Do not return any additional text beyond the value itself.\n",
|
||||
"\"\"\"\n",
|
||||
" else:\n",
|
||||
" return f\"\"\"### PAGE START ### {page} ### PAGE END\n",
|
||||
" \n",
|
||||
"Service: {bu_dict['SERVICE']}\n",
|
||||
"Methodology: {bu_dict['FULL_METHODOLOGY']}\n",
|
||||
"\n",
|
||||
"Listed above is a page of a contract, as well as a Service and Methodology found on this page. \n",
|
||||
"Your job is to identify which of the following entities this specific Service is associated with. \n",
|
||||
"\n",
|
||||
"----Change Below----\n",
|
||||
"\n",
|
||||
"Here are the entities, represented as dictionary objects:\n",
|
||||
"\n",
|
||||
"{td_dicts}\n",
|
||||
"\n",
|
||||
"Write ONLY the number of the dictionary that is most associated with the Service and Methodology of interest. Do not return any additional text beyond the digit itself. \n",
|
||||
"\"\"\"\n",
|
||||
"\n",
|
||||
"def merge_results(td_results, bu_results, text_dict):\n",
|
||||
" all_keys = get_unique_keys(td_results) | get_unique_keys(bu_results)\n",
|
||||
" print(\"All Keys:\", all_keys)\n",
|
||||
" \n",
|
||||
" date_fields = get_unique_date_fields(td_results)\n",
|
||||
" print(\"Date Fields:\", date_fields)\n",
|
||||
" \n",
|
||||
" merged_results = []\n",
|
||||
" for bu in bu_results:\n",
|
||||
" merged = {key: bu.get(key, \"\") for key in all_keys}\n",
|
||||
" td_on_page = [td for td in td_results if td['page_num'] == bu['page_num']]\n",
|
||||
" \n",
|
||||
" for td in td_on_page:\n",
|
||||
" for key, value in td.items():\n",
|
||||
" if not merged[key]:\n",
|
||||
" merged[key] = value\n",
|
||||
" elif isinstance(value, list) and value and not isinstance(merged[key], list):\n",
|
||||
" merged[key] = value\n",
|
||||
" elif isinstance(value, list) and value:\n",
|
||||
" merged[key].extend(value)\n",
|
||||
"\n",
|
||||
" # Include unique date fields\n",
|
||||
" for date_key, date_values in date_fields.items():\n",
|
||||
" if date_key not in merged or not merged[date_key]:\n",
|
||||
" merged[date_key] = date_values\n",
|
||||
" else:\n",
|
||||
" merged[date_key].extend([val for val in date_values if val not in merged[date_key]])\n",
|
||||
"\n",
|
||||
" # Use LLM for disambiguation and inference if necessary\n",
|
||||
" for key in all_keys:\n",
|
||||
" if isinstance(merged[key], list) and len(merged[key]) > 1:\n",
|
||||
" page_num = bu['page_num']\n",
|
||||
" page_text = text_dict.get(page_num, \"\")\n",
|
||||
" prompt = GENERATE_PROMPT(bu, td_on_page, page_text, field_name=key, values=merged[key])\n",
|
||||
" response = claude_funcs.invoke_claude_3(prompt, max_tokens=4000)\n",
|
||||
" try:\n",
|
||||
" selected_value = response.strip()\n",
|
||||
" merged[key] = selected_value\n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"Error processing LLM response for key {key}: {e}\")\n",
|
||||
" merged[key] = ', '.join(merged[key])\n",
|
||||
" \n",
|
||||
" merged_results.append(merged)\n",
|
||||
" \n",
|
||||
" return merged_results\n",
|
||||
"\n",
|
||||
"def process_file(filename, input_dict):\n",
|
||||
" contract_text = input_dict[filename]\n",
|
||||
"\n",
|
||||
" # Preprocess\n",
|
||||
" contract_text = preprocess.clean_newlines(contract_text)\n",
|
||||
" text_dict = preprocess.split_text(contract_text)\n",
|
||||
" \n",
|
||||
" # Run Top Down\n",
|
||||
" td_results = prompt_funcs.run_top_down(filename, text_dict) # Returns list of dictionaries for each page\n",
|
||||
" print(\"TD Results:\", td_results)\n",
|
||||
" \n",
|
||||
" # Run Bottom Up\n",
|
||||
" bu_results = prompt_funcs.run_bottom_up(filename, text_dict) # Returns list of dictionaries\n",
|
||||
" print(\"BU Results:\", bu_results)\n",
|
||||
" \n",
|
||||
" # Combine\n",
|
||||
" combined_results = merge_results(clean_td(td_results), bu_results, text_dict)\n",
|
||||
" print(\"Combined Results:\", combined_results)\n",
|
||||
" \n",
|
||||
" # Create directories\n",
|
||||
" base_filename = os.path.splitext(filename)[0]\n",
|
||||
" output_dir = os.path.join('output', base_filename)\n",
|
||||
" os.makedirs(output_dir, exist_ok=True)\n",
|
||||
" \n",
|
||||
" # Save results\n",
|
||||
" pd.DataFrame(td_results).to_csv(os.path.join(output_dir, 'td_results.csv'), index=False)\n",
|
||||
" pd.DataFrame(bu_results).to_csv(os.path.join(output_dir, 'bu_results.csv'), index=False)\n",
|
||||
" pd.DataFrame(combined_results).to_csv(os.path.join(output_dir, 'combined_results.csv'), index=False)\n",
|
||||
" \n",
|
||||
"def process_all_files(input_dict):\n",
|
||||
" for filename in input_dict.keys():\n",
|
||||
" process_file(filename, input_dict)\n",
|
||||
" print(f\"Processed {filename}\")\n",
|
||||
"\n",
|
||||
"# Example usage\n",
|
||||
"input_dict = utils.read_input(path='subset')\n",
|
||||
"process_all_files(input_dict)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Preprocessing:\n",
|
||||
"clean_newlines(contract_text):\n",
|
||||
"\n",
|
||||
"Removes unwanted newline characters from the contract text.\n",
|
||||
"split_text(contract_text):\n",
|
||||
"\n",
|
||||
"Splits the contract text into individual pages and stores them in a dictionary.\n",
|
||||
"Top Down and Bottom Up Analysis:\n",
|
||||
"run_top_down(filename, text_dict):\n",
|
||||
"\n",
|
||||
"Extracts data from each page of the contract text and returns a list of dictionaries.\n",
|
||||
"run_bottom_up(filename, text_dict):\n",
|
||||
"\n",
|
||||
"Extracts data from the entire contract document and returns a list of dictionaries.\n",
|
||||
"Cleaning and Combining Results:\n",
|
||||
"clean_td(td_results):\n",
|
||||
"\n",
|
||||
"Ensures values in the Top Down results are in list format and preserves date fields.\n",
|
||||
"get_unique_keys(td_results, bu_results):\n",
|
||||
"\n",
|
||||
"Collects all unique field names from both Top Down and Bottom Up results.\n",
|
||||
"get_unique_date_fields(td_results):\n",
|
||||
"\n",
|
||||
"Aggregates all unique date fields from the Top Down results.\n",
|
||||
"merge_results(td_results, bu_results, text_dict):\n",
|
||||
"\n",
|
||||
"Combines Top Down and Bottom Up results, ensuring all relevant fields are included.\n",
|
||||
"Uses the LLM to resolve ambiguities and infer the most appropriate values when necessary when multiple are there.\n",
|
||||
"Post-Processing:\n",
|
||||
"post_process_combined(combined_df):\n",
|
||||
"\n",
|
||||
"Cleans and validates specific fields (CONTRACT_LOB, PRODUCT, CONTRACT_NETWORK) in the combined dataset.\n",
|
||||
"Adds new columns (Corrected_LOB, Corrected_PRODUCT, Corrected_NETWORK) to the combined dataset.\n",
|
||||
"Extracts and standardizes page numbers.\n",
|
||||
"clean_columns_combined(df, column_name, valid_values, new_column_name):\n",
|
||||
"\n",
|
||||
"Cleans a column by applying an exact match check and updates it to a new column.\n",
|
||||
"clean_pagenumbers(df):\n",
|
||||
"\n",
|
||||
"Extracts and standardizes page numbers in the combined dataset.\n",
|
||||
"Saving the Results:\n",
|
||||
"process_file(filename, input_dict):\n",
|
||||
"\n",
|
||||
"Runs the entire process for a single file, from reading and preprocessing to saving the final results.\n",
|
||||
"Calls the necessary functions to extract, combine, and post-process the data.\n",
|
||||
"Saves the results as CSV files in the output directory.\n",
|
||||
"process_all_files(input_dict):\n",
|
||||
"\n",
|
||||
"Iterates through all input files and processes each one using process_file."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import pandas as pd\n",
|
||||
"import postprocess\n",
|
||||
"import utils\n",
|
||||
"\n",
|
||||
"def clean_td(td):\n",
|
||||
" td_clean = []\n",
|
||||
" for d in td:\n",
|
||||
" new_d = {}\n",
|
||||
" for k, v in d.items():\n",
|
||||
" if 'DATE' in k:\n",
|
||||
" new_d[k] = v if isinstance(v, list) else [v]\n",
|
||||
" elif k not in ['page_num', 'Filename']:\n",
|
||||
" if isinstance(v, str) and ',' in v:\n",
|
||||
" new_d[k] = [item.strip() for item in v.split(',')]\n",
|
||||
" elif v == 'N/A':\n",
|
||||
" new_d[k] = []\n",
|
||||
" else:\n",
|
||||
" new_d[k] = [v] if isinstance(v, str) else v\n",
|
||||
" else:\n",
|
||||
" new_d[k] = v\n",
|
||||
" td_clean.append(new_d)\n",
|
||||
" return td_clean\n",
|
||||
"\n",
|
||||
"def get_unique_keys(dicts):\n",
|
||||
" keys = set()\n",
|
||||
" for d in dicts:\n",
|
||||
" keys.update(d.keys())\n",
|
||||
" return keys\n",
|
||||
"\n",
|
||||
"def get_unique_date_fields(td_results):\n",
|
||||
" unique_date_fields = {}\n",
|
||||
" for td in td_results:\n",
|
||||
" for key, value in td.items():\n",
|
||||
" if 'DATE' in key:\n",
|
||||
" if key not in unique_date_fields:\n",
|
||||
" unique_date_fields[key] = set()\n",
|
||||
" unique_date_fields[key].update(value if isinstance(value, list) else [value])\n",
|
||||
" for key in unique_date_fields:\n",
|
||||
" unique_date_fields[key] = list(unique_date_fields[key])\n",
|
||||
" return unique_date_fields\n",
|
||||
"\n",
|
||||
"def GENERATE_PROMPT(bu_dict, td_dicts, page, field_name=None, values=None):\n",
|
||||
" if field_name and values:\n",
|
||||
" return f\"\"\"### PAGE START ### {page} ### PAGE END\n",
|
||||
" \n",
|
||||
"Field: {field_name}\n",
|
||||
"Values: {values}\n",
|
||||
"\n",
|
||||
"Listed above is a page of a contract, as well as the potential values for the {field_name} field.\n",
|
||||
"Your job is to identify which of these values is the most appropriate based on the page content.\n",
|
||||
"\n",
|
||||
"Write ONLY the value that is most appropriate. Do not return any additional text beyond the value itself.\n",
|
||||
"\"\"\"\n",
|
||||
" else:\n",
|
||||
" return f\"\"\"### PAGE START ### {page} ### PAGE END\n",
|
||||
" \n",
|
||||
"Service: {bu_dict['SERVICE']}\n",
|
||||
"Methodology: {bu_dict['FULL_METHODOLOGY']}\n",
|
||||
"\n",
|
||||
"Listed above is a page of a contract, as well as a Service and Methodology found on this page. \n",
|
||||
"Your job is to identify which of the following entities this specific Service is associated with. \n",
|
||||
"\n",
|
||||
"----Change Below----\n",
|
||||
"\n",
|
||||
"Here are the entities, represented as dictionary objects:\n",
|
||||
"\n",
|
||||
"{td_dicts}\n",
|
||||
"\n",
|
||||
"Write ONLY the number of the dictionary that is most associated with the Service and Methodology of interest. Do not return any additional text beyond the digit itself. \n",
|
||||
"\"\"\"\n",
|
||||
"\n",
|
||||
"def merge_results(td_results, bu_results, text_dict):\n",
|
||||
" all_keys = get_unique_keys(td_results) | get_unique_keys(bu_results)\n",
|
||||
" print(\"All Keys:\", all_keys)\n",
|
||||
" \n",
|
||||
" date_fields = get_unique_date_fields(td_results)\n",
|
||||
" print(\"Date Fields:\", date_fields)\n",
|
||||
" \n",
|
||||
" merged_results = []\n",
|
||||
" for bu in bu_results:\n",
|
||||
" merged = {key: bu.get(key, \"\") for key in all_keys}\n",
|
||||
" td_on_page = [td for td in td_results if td['page_num'] == bu['page_num']]\n",
|
||||
" \n",
|
||||
" for td in td_on_page:\n",
|
||||
" for key, value in td.items():\n",
|
||||
" if not merged[key]:\n",
|
||||
" merged[key] = value\n",
|
||||
" elif isinstance(value, list) and value and not isinstance(merged[key], list):\n",
|
||||
" merged[key] = value\n",
|
||||
" elif isinstance(value, list) and value:\n",
|
||||
" merged[key].extend(value)\n",
|
||||
"\n",
|
||||
" for date_key, date_values in date_fields.items():\n",
|
||||
" if date_key not in merged or not merged[date_key]:\n",
|
||||
" merged[date_key] = date_values\n",
|
||||
" else:\n",
|
||||
" merged[date_key].extend([val for val in date_values if val not in merged[date_key]])\n",
|
||||
"\n",
|
||||
" for key in all_keys:\n",
|
||||
" if isinstance(merged[key], list) and len(merged[key]) > 1:\n",
|
||||
" page_num = bu['page_num']\n",
|
||||
" page_text = text_dict.get(page_num, \"\")\n",
|
||||
" prompt = GENERATE_PROMPT(bu, td_on_page, page_text, field_name=key, values=merged[key])\n",
|
||||
" response = claude_funcs.invoke_claude_3(prompt, max_tokens=4000)\n",
|
||||
" try:\n",
|
||||
" selected_value = response.strip()\n",
|
||||
" merged[key] = selected_value\n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"Error processing LLM response for key {key}: {e}\")\n",
|
||||
" merged[key] = ', '.join(merged[key])\n",
|
||||
" \n",
|
||||
" merged_results.append(merged)\n",
|
||||
" \n",
|
||||
" return merged_results\n",
|
||||
"\n",
|
||||
"def process_file(filename, input_dict):\n",
|
||||
" contract_text = input_dict[filename]\n",
|
||||
"\n",
|
||||
" # Preprocess\n",
|
||||
" contract_text = preprocess.clean_newlines(contract_text)\n",
|
||||
" text_dict = preprocess.split_text(contract_text)\n",
|
||||
" \n",
|
||||
" # Run Top Down\n",
|
||||
" td_results = prompt_funcs.run_top_down(filename, text_dict) # Returns list of dictionaries for each page\n",
|
||||
" print(\"TD Results:\", td_results)\n",
|
||||
" \n",
|
||||
" # Run Bottom Up\n",
|
||||
" bu_results = prompt_funcs.run_bottom_up(filename, text_dict) # Returns list of dictionaries\n",
|
||||
" print(\"BU Results:\", bu_results)\n",
|
||||
" \n",
|
||||
" # Combine\n",
|
||||
" combined_results = merge_results(clean_td(td_results), bu_results, text_dict)\n",
|
||||
" print(\"Combined Results:\", combined_results)\n",
|
||||
" \n",
|
||||
" # Convert to DataFrame\n",
|
||||
" combined_df = pd.DataFrame(combined_results)\n",
|
||||
" \n",
|
||||
" # Post-process combined results\n",
|
||||
" post_processed_combined_df = postprocess.postprocess_results(combined_df)\n",
|
||||
" print(\"Post-processed Results:\", post_processed_combined_df)\n",
|
||||
" \n",
|
||||
" # Create directories\n",
|
||||
" base_filename = os.path.splitext(filename)[0]\n",
|
||||
" output_dir = os.path.join('output', base_filename)\n",
|
||||
" os.makedirs(output_dir, exist_ok=True)\n",
|
||||
" \n",
|
||||
" # Save results\n",
|
||||
" pd.DataFrame(td_results).to_csv(os.path.join(output_dir, 'td_results.csv'), index=False)\n",
|
||||
" pd.DataFrame(bu_results).to_csv(os.path.join(output_dir, 'bu_results.csv'), index=False)\n",
|
||||
" post_processed_combined_df.to_csv(os.path.join(output_dir, 'combined_results.csv'), index=False)\n",
|
||||
"\n",
|
||||
"def process_all_files(input_dict):\n",
|
||||
" for filename in input_dict.keys():\n",
|
||||
" process_file(filename, input_dict)\n",
|
||||
" print(f\"Processed {filename}\")\n",
|
||||
"\n",
|
||||
"# Example usage\n",
|
||||
"input_dict = utils.read_input(path='subset')\n",
|
||||
"process_all_files(input_dict)\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"\n",
|
||||
"\n",
|
||||
"# Usage example:\n",
|
||||
"#save_combined_to_csv(combined, 'combined_output.csv')\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"[c['CONTRACT_LOB'] for c in combined]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"utils.consolidate_individual(input_folder='../results/', output_folder='../output/')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.4"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import pandas as pd
|
||||
import postprocessingfuncs
|
||||
|
||||
def sanitize_combined(df):
|
||||
""" Sanitize the combined data by removing brackets and cleaning values. """
|
||||
for column in df.columns:
|
||||
df[column] = df[column].apply(postprocessingfuncs.sanitize_value)
|
||||
return df
|
||||
|
||||
def post_process_combined(df):
|
||||
VALID_LOBS = ['MEDICARE', 'MEDICARE ADVANTAGE', 'MEDICAID', 'MARKETPLACE', 'COMMERCIAL', 'GROUP', 'MEDICARE-MEDICAID']
|
||||
VALID_PROGRAMS = ['CHIP', 'CHIP-P', 'CHIP-PERINATE', 'STAR', 'STAR+PLUS', 'MA', 'DUAL SPECIAL NEEDS PLAN', 'DSNP', 'DUAL']
|
||||
VALID_NETWORKS = ['HMO', 'PPO', 'EPO', 'POS', 'FFS']
|
||||
|
||||
# Sanitize the combined data
|
||||
df = sanitize_combined(df)
|
||||
|
||||
postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_LOB', VALID_LOBS, 'Corrected_LOB')
|
||||
postprocessingfuncs.clean_columns_combined(df, 'PRODUCT', VALID_PROGRAMS, 'Corrected_PRODUCT')
|
||||
postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_NETWORK', VALID_NETWORKS, 'Corrected_NETWORK')
|
||||
|
||||
clean_df = postprocessingfuncs.clean_pagenumbers(df)
|
||||
return clean_df
|
||||
|
||||
def postprocess_results(combined_df):
|
||||
combined_df = post_process_combined(combined_df)
|
||||
return combined_df
|
||||
@@ -0,0 +1,95 @@
|
||||
import pandas as pd
|
||||
import re
|
||||
import difflib
|
||||
|
||||
def sanitize_value(value):
|
||||
""" Remove brackets from list items and clean the values. """
|
||||
if pd.isna(value):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
value = value.strip('[]')
|
||||
if ',' in value:
|
||||
value = ', '.join([item.strip(" '") for item in value.split(',')])
|
||||
else:
|
||||
value = value.strip(" '")
|
||||
return value
|
||||
|
||||
def exact_match(val, valid_values):
|
||||
val = val.strip().upper()
|
||||
for valid_val in valid_values:
|
||||
if val == valid_val.upper():
|
||||
return valid_val
|
||||
return None
|
||||
|
||||
def clean_columns_combined(df, column_name, valid_values, new_column_name):
|
||||
""" Cleans a column by applying an exact match check and updates it to a new column. """
|
||||
changes = {}
|
||||
original_values = df[column_name].unique()
|
||||
|
||||
def update_column(entry):
|
||||
if pd.notna(entry):
|
||||
terms = sanitize_value(entry).split(',')
|
||||
for term in terms:
|
||||
match = exact_match(term, valid_values)
|
||||
if match:
|
||||
return match
|
||||
return None
|
||||
|
||||
df[new_column_name] = df[column_name].apply(update_column)
|
||||
cleaned_values = df[new_column_name].unique()
|
||||
return original_values, cleaned_values, changes
|
||||
|
||||
def get_closest_match(val, valid_values, similarity_threshold=0.5):
|
||||
if pd.isna(val):
|
||||
return None
|
||||
val = val.strip().upper()
|
||||
matches = difflib.get_close_matches(val, [v.upper() for v in valid_values], n=1, cutoff=similarity_threshold)
|
||||
return matches[0] if matches else None
|
||||
|
||||
def clean_columns_combined_fuzzy(df, column_name, valid_values, threshold):
|
||||
""" Applies fuzzy matching to a column in the dataframe and logs changes. """
|
||||
changes = {}
|
||||
original_values = df[column_name].unique()
|
||||
|
||||
def log_and_clean(entry):
|
||||
if pd.notna(entry):
|
||||
words = sanitize_value(entry).split(',')
|
||||
cleaned_words = []
|
||||
for word in words:
|
||||
cleaned_word = get_closest_match(word.strip(), valid_values, similarity_threshold=threshold)
|
||||
if cleaned_word and word.strip().upper() != cleaned_word:
|
||||
changes[word.strip()] = cleaned_word
|
||||
cleaned_words.append(cleaned_word if cleaned_word else word.strip())
|
||||
return ', '.join(cleaned_words)
|
||||
return None
|
||||
|
||||
df[column_name] = df[column_name].apply(log_and_clean)
|
||||
cleaned_values = df[column_name].unique()
|
||||
return original_values, cleaned_values, changes
|
||||
|
||||
def extract_page_number(page_text):
|
||||
match = re.search(r'Pages\s+(\d+)-\d+', page_text)
|
||||
if match:
|
||||
return match.group(1)
|
||||
else:
|
||||
return page_text
|
||||
|
||||
def clean_pagenumbers(df):
|
||||
df['Page'] = df['Page'].apply(extract_page_number)
|
||||
return df
|
||||
|
||||
def consolidate_csv(df, output_csv='output/consolidated.csv'):
|
||||
df['page_num'] = df['Page'].apply(lambda x: x.split()[1])
|
||||
|
||||
grouped = df.groupby(['Filename', 'Corrected_LOB']).agg({
|
||||
'PRODUCT': lambda x: ', '.join(x.dropna().unique()),
|
||||
'Corrected_NETWORK': lambda x: ', '.join(x.dropna().unique()),
|
||||
'CONTRACT_SERVICE_AREA': lambda x: ', '.join(x.dropna().unique()),
|
||||
'LOB_PRICING_TERMS_EFFECTIVE_DT': lambda x: ', '.join(x.dropna().unique()),
|
||||
'LOB_PRICING_TERMS_TERMINATION_DT': lambda x: ', '.join(x.dropna().unique()),
|
||||
'CONTRACT_MARKETPLACE_METAL_LEVEL': lambda x: ', '.join(x.dropna().unique()),
|
||||
'page_num' : lambda x : ', '.join(x.dropna().unique())
|
||||
}).reset_index()
|
||||
|
||||
grouped.to_csv(output_csv, index=False)
|
||||
print(f"Consolidated CSV has been saved to {output_csv}")
|
||||
+2
-1
@@ -8,7 +8,7 @@ import json
|
||||
from llama_index.core import VectorStoreIndex, SimpleDirectoryReader, ServiceContext, PromptTemplate
|
||||
from llama_index.core import SummaryIndex, Document
|
||||
from llama_index.llms.bedrock import Bedrock
|
||||
|
||||
#pip install llama-index-llms-bedrock
|
||||
import config
|
||||
import prompts
|
||||
import claude_funcs
|
||||
@@ -227,6 +227,7 @@ def td_bu_combine(td, bu, text_dict):
|
||||
return combined_list # List of dictionaries
|
||||
|
||||
|
||||
|
||||
def run_prompts(file_object):
|
||||
filename, contract_text = file_object
|
||||
if config.VERBOSE: print(f"Processing {filename}...")
|
||||
|
||||
+32
-4
@@ -186,10 +186,38 @@ ONLY return the answer, with no other commentary or explanation.
|
||||
"""
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def get_carveout_list():
|
||||
return ['Emergency Department/Emergency Room', 'Emergency Department', 'Emergency Room', 'Observation', 'Surgery', 'General Surgery',
|
||||
'Ambulatory Surgery', 'Intensive Care', 'Trauma', 'HIV', 'Human Immunodeficiency Virus', 'Major Joint Replacement', 'Transplant', 'OBGYN',
|
||||
'Obstetrician/Gynecologist', 'Obstetrician', 'Gynecologist', 'Opthalmology & Vision', 'Opthalmology', 'Vision', 'Never Events',
|
||||
'Medically Unnecessary', 'Medically Unnecessary Procedures', 'Not Medically Necessary', 'Not Medically Necessary Procedures',
|
||||
'Experimental', 'Investigational', 'Unlisted Codes', 'Durable Medical Equipment', 'DME', 'Prosthetics & Orthotics', 'Prosthetics',
|
||||
'Orthotics', 'Implants', 'Hearing Aids', 'Hearing', 'Anesthesia', 'Anesthesiology', 'Medical Pharmacy', 'Physician Administered Drugs',
|
||||
'Global', 'Bundled or Unbundled Codes', 'Bundled Codes', 'Unbundled Codes', 'Multiple Procedure Reductions', 'Second Surgery', 'Subsequent Surgeries',
|
||||
'Non-Behavioral Health Mid-Level Professionals', 'Physician Assistant', 'PA', 'Nurse Practicioner', 'NP', 'Non-Physician',
|
||||
'Non-Physician Health Professionals', 'Technical Component', 'Professional Component', 'Laboratory', 'Pathology', 'Lab', 'Path', 'Lab/Path',
|
||||
'Laboratory/Pathology', 'Radiology', 'Imaging', 'Radiology/Imaging', 'Mammography', 'Diagnostic', 'Pre-Admission Procedures',
|
||||
'Post-Discharge Procedures', 'Readmission', 'Status Indicators', 'Stop Loss', 'Emergency Medical', 'EMS', 'NICU', 'Neonatal Intensive Care Unit',
|
||||
'Vaccine for Children', 'VFC', 'Surgical Assistant', 'Physicians/Clinical Psychologists', 'Doctor of Nursing Practice', 'Osteopathic Medicine',
|
||||
'Clinical Psychology', 'Audiologist', 'Chiropractors', 'Registered Dietician', 'AUD', 'DC', 'RD', 'Board Certified Behavioral Analysis', 'Behavioral Analysis',
|
||||
'BCBA', 'Independent Licensures', 'Licensed Professional Counselor', 'Marriage and Family Therapist', 'Substance Abuse Counselor', 'Clinical Social Worker',
|
||||
'Behavioral Health Outpatient Clinic', 'Physical Therapist', 'Occupational Therapist', 'Speech Therapist', 'PT', 'OT', 'ST', 'Transportation',
|
||||
'Primary Care', 'Primary Care Behavioral Health', 'Behavioral Physician', 'Clinical Psychologist', 'Mid-Level Practicioner', 'Dentist', 'Dental',
|
||||
'Supplies and Devices', 'Supplies', 'Devices', 'Immunizations', 'Obstetrical Epidural', 'Pediatric Subspecialties', 'Orthopedic Surgery', 'All Other Specialists',
|
||||
'Specialty Care Physician', 'Pediatric Primary Care', 'Nurse Anesthetist', 'Early Periodic Screening, Diagnostic, and Treatment', 'EPSDT',
|
||||
'Ancillary', 'Oncology', 'Cancer', 'Inpatient Physical Rehabilitation', 'Inpatient Rehabilitation', 'Outpatient Rehabilitation', 'Rehabilitation', 'Non-Behavioral Health Rehabilitation',
|
||||
'Extracorporeal Shock Wave Lithotripsy', 'Cardiac', 'Special Care Unit', 'Skilled Nursing', 'Infusion', 'Specialty Care',
|
||||
'Specialty Care Physician', 'Organ Acquisition', 'Blood Products', 'Blood Products Outpatient', 'Blood Products Inpatient', 'High Cost Drugs', 'Sleep Studies',
|
||||
'NICU', 'Newborn Intensive Care Unit', 'Extracorporeal Membrane Oxygenation', 'Burns', 'Kyphoplasty', 'Cryosurgical Ablation of the Prostate',
|
||||
'Transurethral Thermal Ablation', 'TUMT', 'Transurethral Needle Ablation', 'TUNA', 'Hyperbaric Treatment', 'Clinic Visit', 'Boarder Baby',
|
||||
'Pediatric Intensive Care Unit', 'PICU', 'Psychiatric', 'Mental Health', 'Behavioral Health', 'Substance Abuse',
|
||||
'Behavioral Health and Substance Abuse', 'Sub-Acute Facility Care', 'Unrouped Inpatient', 'All Other Acute',
|
||||
'Neurology', 'Neurology Subspecialties', 'Automatic Implantable Cardioverter Defibrillator', 'Percutaneous Transluminal Coronary Angioplasty',
|
||||
'Non-Coronary Angioplasty', 'Cardiac Catheters', 'Cardiovascular Surgery', 'Cardiac Surgery', 'Cesarean Birth', 'Cesarean Section', 'C-Section',
|
||||
'Gamma-Knife Radio-Surgery Outpatient', 'DaVinci Robotic Assisted Surgery', 'Outpatient Electrophysiology with Ablation', 'Outpatient Electrophysiology',
|
||||
'Magnetic Resonance Image', 'MRI', 'Computed Tomography Scan', 'CT Scan', 'Radiation Therapy', 'Dialysis', 'Gastric Bypass', 'Lap Band',
|
||||
'Obesity', 'Laparoscopic Cholecystectomy', 'Lap Chole', 'Laparoscopic Hysterectomy', 'Laparoscopic Prostatectomy', 'Laparoscopic Hysteroscopy',
|
||||
'Treatment Room', 'Wound Care', 'Cardiac Computed Tomograpy & Angiography', 'Positron Emission Tomography Scan', 'PET Scan', 'Hematology']
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -1,359 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Imports\n",
|
||||
"import pandas as pd\n",
|
||||
"import re\n",
|
||||
"import ast\n",
|
||||
"import json\n",
|
||||
"\n",
|
||||
"import prompt_funcs\n",
|
||||
"import config\n",
|
||||
"import utils\n",
|
||||
"import preprocess\n",
|
||||
"import table_funcs\n",
|
||||
"import claude_funcs\n",
|
||||
"import prompts"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 23,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"input_dict = utils.read_input(path='../docs/test/')\n",
|
||||
"input_dict.keys()\n",
|
||||
"filename = '47-1928508 Advanced Anesthesia.txt'"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 24,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"contract_text = input_dict[filename]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 25,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"#### PREPROCESS ####\n",
|
||||
"contract_text = preprocess.clean_newlines(contract_text)\n",
|
||||
"text_dict = preprocess.split_text(contract_text)\n",
|
||||
"#text_dict = table_funcs.align_and_format_tables(text_dict)\n",
|
||||
"#text_dict = preprocess.highlight_rates(text_dict)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"#### TOP DOWN ####\n",
|
||||
"td_results = prompt_funcs.run_top_down(filename, text_dict) # Returns list of list of dictionaries for each page\n",
|
||||
"\n",
|
||||
"#### BOTTOM UP ####\n",
|
||||
"bu_results = prompt_funcs.run_bottom_up(filename, text_dict) # Returns list of dictionaries\n",
|
||||
"\n",
|
||||
"#### COMBINE ####\n",
|
||||
"# combined_results = prompt_funcs.td_bu_combine(td_results, bu_results, text_dict)\n",
|
||||
"\n",
|
||||
"# answer_df = pd.DataFrame(combined_results)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 92,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def clean_td(td):\n",
|
||||
" td_clean = []\n",
|
||||
" for d in td:\n",
|
||||
" new_d = {}\n",
|
||||
" for k in d.keys():\n",
|
||||
" if k not in ['page_num', 'Filename']:\n",
|
||||
" if ',' in d[k]:\n",
|
||||
" if 'DATE' not in k:\n",
|
||||
" new_d[k] = [v.strip() for v in d[k].split(',')]\n",
|
||||
" elif d[k] == 'N/A':\n",
|
||||
" new_d[k] = []\n",
|
||||
" else:\n",
|
||||
" new_d[k] = [d[k]]\n",
|
||||
" else:\n",
|
||||
" new_d[k] = d[k]\n",
|
||||
" td_clean.append(new_d)\n",
|
||||
" return td_clean\n",
|
||||
"\n",
|
||||
"def get_unique_td(td):\n",
|
||||
" unique_td_dict = {}\n",
|
||||
" for d in td:\n",
|
||||
" for key, value in d.items():\n",
|
||||
" if key not in ['page_num', 'Filename']:\n",
|
||||
" for v in value:\n",
|
||||
" if key not in unique_td_dict.keys():\n",
|
||||
" unique_td_dict[key] = [v]\n",
|
||||
" else:\n",
|
||||
" unique_td_dict[key].append(v)\n",
|
||||
" final_dict = {key : list(set(unique_td_dict[key])) for key in unique_td_dict.keys()}\n",
|
||||
" return final_dict\n",
|
||||
"\n",
|
||||
"def td_bu_combine(td, bu, text_dict):\n",
|
||||
" bu_dicts = bu.copy()\n",
|
||||
" td_dicts = td.copy()\n",
|
||||
"\n",
|
||||
" td_dicts_clean = clean_td(td_dicts)\n",
|
||||
" unique_td = get_unique_td(td_dicts_clean)\n",
|
||||
" \n",
|
||||
" combined_list = []\n",
|
||||
" for bud_ in bu_dicts:\n",
|
||||
" bud = bud_.copy()\n",
|
||||
" for key, value in unique_td.items():\n",
|
||||
" if len(value) == 0: # No value in contract\n",
|
||||
" bud.update({key : \"\"})\n",
|
||||
" elif len(value) == 1: # 1 value in contract\n",
|
||||
" bud.update({key : value[0]})\n",
|
||||
" else: # More than one value in contract\n",
|
||||
" # Check for value on the page\n",
|
||||
" page_value = []\n",
|
||||
" for d in td_dicts_clean:\n",
|
||||
" if (d['page_num'] == bud['page_num']) and (key in d.keys()):\n",
|
||||
" for v in d[key]:\n",
|
||||
" page_value.append(v)\n",
|
||||
" \n",
|
||||
" if len(page_value) == 1: # One value on page\n",
|
||||
" bud.update({key : page_value[0]})\n",
|
||||
" elif len(page_value) > 1: # Multiple values on page\n",
|
||||
" bud.update({key : ', '.join(page_value)})\n",
|
||||
" elif len(page_value) == 0:\n",
|
||||
" bud.update({key : \"\"}) \n",
|
||||
" combined_list.append(bud)\n",
|
||||
" return combined_list"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 94,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"combined = td_bu_combine(td_results, bu_results, text_dict)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 95,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"['Medicaid',\n",
|
||||
" 'Medicaid',\n",
|
||||
" 'Medicaid',\n",
|
||||
" 'Medicaid',\n",
|
||||
" 'Medicaid',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" 'Commercial',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '']"
|
||||
]
|
||||
},
|
||||
"execution_count": 95,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"[c['CONTRACT_LOB'] for c in combined]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 98,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"utils.consolidate_individual(input_folder='../results/', output_folder='../output/')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.12.3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
Reference in New Issue
Block a user