Merged in pricing_before_carveouts (pull request #210)
Pricing before carveouts
This commit is contained in:
Michael McGuinness
+11
-1
@@ -73,4 +73,14 @@ streamlit/results.csv
|
||||
streamlit/venv
|
||||
|
||||
*.tfplan
|
||||
*.pem
|
||||
*.pem
|
||||
textfiles/
|
||||
texts/
|
||||
*.csv
|
||||
*.json
|
||||
myenv/
|
||||
*.env
|
||||
*.txt
|
||||
subset/
|
||||
output/
|
||||
docs/
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
# These are some examples of commonly ignored file patterns.
|
||||
# You should customize this list as applicable to your project.
|
||||
# Learn more about .gitignore:
|
||||
# https://www.atlassian.com/git/tutorials/saving-changes/gitignore
|
||||
# Node artifact files
|
||||
node_modules/
|
||||
dist/
|
||||
# Compiled Java class files
|
||||
*.class
|
||||
# Compiled Python bytecode
|
||||
*.py[cod]
|
||||
# Log files
|
||||
*.log
|
||||
# Package files
|
||||
*.jar
|
||||
# Maven
|
||||
target/
|
||||
dist/
|
||||
# JetBrains IDE
|
||||
.idea/
|
||||
# Unit test reports
|
||||
TEST*.xml
|
||||
# Generated by MacOS
|
||||
.DS_Store
|
||||
# Generated by Windows
|
||||
Thumbs.db
|
||||
# Applications
|
||||
*.app
|
||||
*.exe
|
||||
*.war
|
||||
# Large media files
|
||||
*.mp4
|
||||
*.tiff
|
||||
*.avi
|
||||
*.flv
|
||||
*.mov
|
||||
*.wmv
|
||||
# Terraform files
|
||||
.terraform/
|
||||
terraform.tfstate.backup
|
||||
terraform.tfstate
|
||||
.terraform.lock.hcl
|
||||
build/
|
||||
textract-pipeline/terraform/builds/
|
||||
# Data Files
|
||||
streamlit/history.csv
|
||||
streamlit/RESULTS
|
||||
streamlit/DB/
|
||||
streamlit/RAW_DOCUMENTS/
|
||||
streamlit/SOURCE_DOCUMENTS/
|
||||
streamlit/contract_field_values.csv
|
||||
streamlit/contract_fields.csv
|
||||
streamlit/sample.csv
|
||||
streamlit/temp1.csv
|
||||
streamlit/temp2.csv
|
||||
streamlit/results.csv
|
||||
# env
|
||||
streamlit/venv
|
||||
*.tfplan
|
||||
*.pem
|
||||
*.pem
|
||||
textfiles/
|
||||
texts/
|
||||
*.csv
|
||||
*.json
|
||||
myenv/
|
||||
*.env
|
||||
*.txt
|
||||
subset/
|
||||
output/
|
||||
docs/
|
||||
@@ -0,0 +1,214 @@
|
||||
|
||||
import dict_operations
|
||||
import postprocessing_funcs
|
||||
import prompts
|
||||
import config
|
||||
import claude_funcs
|
||||
import utils
|
||||
|
||||
import difflib
|
||||
|
||||
def run_bottom_up(filename, text_dict):
|
||||
"""
|
||||
Processes the text of a document using a two-tiered Bottom Up approach to extract key financial and operational information.
|
||||
|
||||
This function first runs BOTTOM_UP_PRIMARY and BOTTOM_UP_SECONDARY prompts, then processes these initial results to further
|
||||
refine and structure them into a dictionary form.
|
||||
|
||||
Parameters:
|
||||
filename (str): The name of the file being processed, used to tag output data.
|
||||
text_dict (dict): A dictionary of text keyed by page numbers that contains the content to be analyzed.
|
||||
|
||||
Returns:
|
||||
list of dict: A list of dictionaries with each dictionary containing refined and structured information
|
||||
from both primary and secondary Bottom Up analyses, all tagged with the filename.
|
||||
"""
|
||||
|
||||
# Bottom Up Primary - SERVICE, REIMBURSEMENT_FLAT_FEE, REIMBURSEMENT_RATE, METHODOLOGY
|
||||
answer_strings = run_bottom_up_primary(text_dict, 8000)
|
||||
answer_dicts = dict_operations.primary_string_to_dict(answer_strings, filename)
|
||||
answer_dicts_filtered = postprocessing_funcs.filter_service_column(answer_dicts)
|
||||
|
||||
# Bottom Up Secondary
|
||||
results_dicts = run_bottom_up_secondary(answer_dicts_filtered, text_dict, 8000)
|
||||
|
||||
for d in results_dicts:
|
||||
d['Filename'] = filename
|
||||
return results_dicts # List of dictionaries
|
||||
|
||||
|
||||
def run_bottom_up_primary(text_dict, tokens):
|
||||
"""
|
||||
Executes the primary Bottom Up processing for pages that contain reimbursement terms.
|
||||
|
||||
This function scans through a dictionary of page texts, identifies pages containing reimbursement terms,
|
||||
and processes those pages using Claude to extract relevant information.
|
||||
|
||||
Parameters:
|
||||
text_dict (dict): A dictionary containing text content of documents keyed by page numbers.
|
||||
tokens (int): The maximum number of tokens to use in the language model invocation for generating the response.
|
||||
|
||||
Returns:
|
||||
dict: A dictionary where each key is a page number and the value is the response from the language model.
|
||||
"""
|
||||
#chunk_dict = preprocess.chunk_text(text_dict)
|
||||
answer_dict = {}
|
||||
#for page_number in chunk_dict.keys():
|
||||
#if '%' in chunk_dict[page_number] or '$' in chunk_dict[page_number]:
|
||||
#prompt = prompts.BOTTOM_UP_PRIMARY(chunk_dict[page_number], config.CLIENT_NAME)
|
||||
for page_number in text_dict.keys():
|
||||
if utils.contains_reimbursement(text_dict, page_number):
|
||||
# Run Primary
|
||||
prompt = prompts.BOTTOM_UP_PRIMARY(text_dict[page_number], config.CLIENT_NAME)
|
||||
answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens)
|
||||
answer_dict[page_number] = answer
|
||||
return answer_dict
|
||||
|
||||
|
||||
def run_bottom_up_secondary(answer_dicts, text_dict, tokens):
|
||||
"""
|
||||
Executes the secondary Bottom Up processing phase on the results obtained from the primary Bottom Up analysis.
|
||||
|
||||
This function enhances the primary results with additional analyses based on configured conditions. Invokes
|
||||
Claude 3 with tailored prompts to generate structured information that complements the initial results.
|
||||
|
||||
Each piece of data processed possibly undergoes several rounds of checks and transformations, ensuring detailed and
|
||||
comprehensive output.
|
||||
|
||||
Parameters:
|
||||
answer_dicts (list of dict): Initial processed data from the primary Bottom Up analysis.
|
||||
text_dict (dict): Dictionary containing the original document text keyed by page numbers.
|
||||
tokens (int): Token limit for language model invocations.
|
||||
|
||||
Returns:
|
||||
list of dict: A list of dictionaries containing enriched and finalized structured data from both the primary and
|
||||
secondary analyses.
|
||||
"""
|
||||
# New rows for lesser of
|
||||
temp_dicts = []
|
||||
for d in answer_dicts:
|
||||
# Bottom Up Lesser
|
||||
if config.RUN_LESSER:
|
||||
lesser_object = run_bottom_up_lesser(d.copy(), text_dict.copy(), tokens)
|
||||
temp_dicts.append(lesser_object[1]) # Add original object from d
|
||||
if lesser_object[0]: temp_dicts.append(lesser_object[0]) # If lesser, add lesser object
|
||||
else:
|
||||
temp_dicts.append(d)
|
||||
|
||||
# Add additional fields to each row
|
||||
final_dicts = []
|
||||
for d in temp_dicts:
|
||||
if d is not None:
|
||||
page_num = d['page_num']
|
||||
|
||||
# Bottom Up Methodology
|
||||
if config.RUN_METHODOLOGY:
|
||||
prompt = prompts.BOTTOM_UP_METHODOLOGY(d)
|
||||
methodology_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=100)
|
||||
d['REIMBURSEMENT_METHODOLOGY'] = methodology_answer
|
||||
|
||||
# # Bottom Up FS
|
||||
if config.RUN_FS:
|
||||
prompt = prompts.BOTTOM_UP_FS(d)
|
||||
fs_answer = claude_funcs.invoke_claude_3(prompt, model_id=config.MODEL_ID_CLAUDE3_HAIKU, max_tokens=tokens)
|
||||
fs_dict = dict_operations.secondary_string_to_dict(fs_answer)
|
||||
d.update(fs_dict)
|
||||
|
||||
# # Bottom Up Exception/Escalator
|
||||
if config.RUN_EXCEPTION:
|
||||
prompt = prompts.BOTTOM_UP_EXCEPT_ESC(d, text_dict[page_num])
|
||||
exc_answer = claude_funcs.invoke_claude_3(prompt, model_id=config.MODEL_ID_CLAUDE3_HAIKU, max_tokens=tokens)
|
||||
exc_dict = dict_operations.secondary_string_to_dict(exc_answer)
|
||||
d.update(exc_dict)
|
||||
|
||||
# # Bottom Up Codes
|
||||
if config.RUN_CODES:
|
||||
prompt = prompts.BOTTOM_UP_CODES(d, text_dict[page_num])
|
||||
codes_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens)
|
||||
codes_dict = dict_operations.secondary_string_to_dict(codes_answer)
|
||||
d.update(codes_dict)
|
||||
|
||||
final_dicts.append(d)
|
||||
|
||||
return final_dicts
|
||||
|
||||
|
||||
def run_bottom_up_lesser(d, text_dict, tokens=4000):
|
||||
"""
|
||||
Processes specific clauses or conditions within the document text, such as 'lesser of' or 'greater of',
|
||||
using a language model to analyze and extract relevant data from the identified text segment.
|
||||
|
||||
This function takes a dictionary representing initial results and a text dictionary, uses the page number from the initial
|
||||
results to locate the relevant text, and generates a prompt for language model analysis. Based on the response, it
|
||||
determines whether 'lesser of' or 'greater of' language is indicated, updates the results, and returns a new dictionary
|
||||
for the updated data or retains the original if no such language is present.
|
||||
|
||||
Parameters:
|
||||
d (dict): A dictionary containing initial processing results from previous analyses, including a 'page_num' key.
|
||||
text_dict (dict): A dictionary of text keyed by page numbers that contains the content to be analyzed.
|
||||
tokens (int): Token limit for language model invocations, default set to 8000.
|
||||
|
||||
Returns:
|
||||
tuple: A tuple containing two dictionaries, the first for any identified 'lesser of' or 'greater of' data and the second
|
||||
containing the original or updated data depending on the presence of such language.
|
||||
"""
|
||||
|
||||
prompt = prompts.BOTTOM_UP_LESSER(d, text_dict[d['page_num']])
|
||||
lesser_of_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens)
|
||||
lesser_of_dict_list = dict_operations.primary_string_to_dict({d['page_num'] : lesser_of_answer}, d['Filename'])
|
||||
# print(lesser_of_dict_list)
|
||||
# If lesser of is Y
|
||||
if len(lesser_of_dict_list) > 1:
|
||||
lesser_of_dict = get_lesser_of_dict(lesser_of_dict_list, d)
|
||||
else:
|
||||
d['LESSER_OF_LANGUAGE_IND'] = 'N'
|
||||
d['GREATER_OF_LANGUAGE_IND'] = 'N'
|
||||
return ({}, d)
|
||||
|
||||
lesser_of_dict['page_num'] = d['page_num']
|
||||
if lesser_of_dict['LESSER_OF_LANGUAGE_IND'] == 'Y' or lesser_of_dict['GREATER_OF_LANGUAGE_IND'] == 'Y':
|
||||
lesser_of_dict['SERVICE'] = d['SERVICE']
|
||||
return (lesser_of_dict, d)
|
||||
else:
|
||||
d['LESSER_OF_LANGUAGE_IND'] = 'N'
|
||||
d['GREATER_OF_LANGUAGE_IND'] = 'N'
|
||||
return ({}, d)
|
||||
|
||||
|
||||
|
||||
def get_least_similar(dict_list, original_dict, field):
|
||||
# print(f"Running similarity for {field}")
|
||||
|
||||
min_similarity = float('inf')
|
||||
least_similar_dict = None
|
||||
for dictionary in dict_list:
|
||||
# Extract the methodology text
|
||||
field_text = dictionary[field]
|
||||
# print(f"Field text: {field_text}")
|
||||
|
||||
# Compute similarity using difflib
|
||||
similarity = difflib.SequenceMatcher(None, str(field_text), str(original_dict[field])).ratio()
|
||||
# print(f"Simlarity: {similarity}")
|
||||
|
||||
# If the similarity is less than the current minimum, update the minimum and the corresponding dictionary
|
||||
if similarity < min_similarity:
|
||||
min_similarity = similarity
|
||||
least_similar_dict = dictionary
|
||||
|
||||
return least_similar_dict
|
||||
|
||||
def get_lesser_of_dict(dict_list, original_dict):
|
||||
|
||||
unique_rates = list({d['REIMBURSEMENT_RATE'] for d in dict_list})
|
||||
unique_fees = list({d['REIMBURSEMENT_FLAT_FEE'] for d in dict_list})
|
||||
|
||||
# If the rates are different, return the lesser of dict with a different rate than original
|
||||
if len(unique_rates) > 1:
|
||||
least_similar_dict = get_least_similar(dict_list, original_dict, 'REIMBURSEMENT_RATE')
|
||||
elif len(unique_fees) > 1:
|
||||
least_similar_dict = get_least_similar(dict_list, original_dict, 'REIMBURSEMENT_FLAT_FEE')
|
||||
else:
|
||||
least_similar_dict = get_least_similar(dict_list, original_dict, 'FULL_METHODOLOGY')
|
||||
|
||||
return least_similar_dict
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
import pandas as pd
|
||||
from difflib import SequenceMatcher
|
||||
import re
|
||||
import time
|
||||
|
||||
import prompts
|
||||
import claude_funcs
|
||||
|
||||
def get_closest_substring_match(val, valid_values):
|
||||
""" Returns the first match from valid_values using a case-insensitive substring match. """
|
||||
if pd.isna(val):
|
||||
return None
|
||||
val = val.strip().upper()
|
||||
for valid_value in valid_values:
|
||||
if valid_value.upper() in val:
|
||||
return valid_value
|
||||
return None
|
||||
|
||||
def get_best_carveout_from_claude(carveout_list, service):
|
||||
print("using claude to get best carveout...")
|
||||
prompt = f"Given the service '{service}', please choose the best matching carveout from the following list: {', '.join(carveout_list)}. ONLY SELECT ONE FROM THE LIST. DO NOT RETURN A SENTENCE"
|
||||
while True:
|
||||
try:
|
||||
response = claude_funcs.invoke_claude_3(prompt, max_tokens=100)
|
||||
return response.strip()
|
||||
except Exception as e:
|
||||
print(f"Error occurred: {e}. Waiting for 60 seconds before retrying...")
|
||||
time.sleep(60)
|
||||
|
||||
def check_prov_type_similarity(prov_type, carveout):
|
||||
print("checking similarity between prov_type and carveout with claude...")
|
||||
prompt = f"Do the provider type '{prov_type}' and the carveout '{carveout}' mean the same thing or are they very similar? ONLY ANSWER WITH 'True' OR 'False'"
|
||||
while True:
|
||||
try:
|
||||
response = claude_funcs.invoke_claude_3(prompt, max_tokens=10)
|
||||
return response.strip()
|
||||
except Exception as e:
|
||||
print(f"Error occurred: {e}. Waiting for 60 seconds before retrying...")
|
||||
time.sleep(60)
|
||||
|
||||
def label_services(filepath, carveout_list, output_filepath):
|
||||
# Read the CSV file
|
||||
df = pd.read_csv(filepath)
|
||||
|
||||
# Select the first 50 rows of the DataFrame
|
||||
df = df.head(500)
|
||||
|
||||
# Define primary service terms
|
||||
primary_terms = ['Covered Services', 'Inpatient Services', 'Outpatient Services', 'Physician Services',
|
||||
'Inpatient', 'Outpatient', 'Physician', 'Medical', 'Surgical', 'Diagnostic', 'Therapeutic']
|
||||
|
||||
# Initialize columns for the results
|
||||
df['carveout_matched'] = ''
|
||||
df['label'] = ''
|
||||
df['IS_CARVEOUT'] = ''
|
||||
|
||||
# Initialize a nested dictionary to track occurrences for each Filename and TD_LOB
|
||||
occurrence_tracker = {}
|
||||
|
||||
carveout_count = 0
|
||||
|
||||
# Iterate through the rows in the DataFrame
|
||||
for index, row in df.iterrows():
|
||||
service = str(row['SERVICE'])
|
||||
prov_type = str(row['PROV_TYPE']) if pd.notna(row['PROV_TYPE']) else ''
|
||||
td_lob = row['TD_LOB']
|
||||
filename = row['Filename']
|
||||
|
||||
# Initialize the nested dictionary for the filename and TD_LOB if not present
|
||||
if filename not in occurrence_tracker:
|
||||
occurrence_tracker[filename] = {}
|
||||
if td_lob not in occurrence_tracker[filename]:
|
||||
occurrence_tracker[filename][td_lob] = {term: 0 for term in primary_terms}
|
||||
|
||||
# Check if the service is a primary or similar to a primary
|
||||
primary = get_closest_substring_match(service, primary_terms)
|
||||
if primary:
|
||||
# Ensure the primary term is initialized in the occurrence tracker
|
||||
if primary not in occurrence_tracker[filename][td_lob]:
|
||||
occurrence_tracker[filename][td_lob][primary] = 0
|
||||
|
||||
# Determine the label based on the count of this primary term for the given Filename and TD_LOB
|
||||
count = occurrence_tracker[filename][td_lob][primary]
|
||||
if count == 0:
|
||||
df.at[index, 'label'] = 'primary'
|
||||
elif count == 1:
|
||||
df.at[index, 'label'] = 'secondary'
|
||||
elif count == 2:
|
||||
df.at[index, 'label'] = 'tertiary'
|
||||
else:
|
||||
df.at[index, 'label'] = 'additional'
|
||||
occurrence_tracker[filename][td_lob][primary] += 1
|
||||
df.at[index, 'IS_CARVEOUT'] = 'N'
|
||||
else:
|
||||
# If PROV_TYPE is blank, continue with carveout determination
|
||||
best_carveout_response = get_best_carveout_from_claude(carveout_list, service)
|
||||
matched_carveout = best_carveout_response # Use the model response directly
|
||||
|
||||
df.at[index, 'carveout_matched'] = matched_carveout
|
||||
|
||||
if matched_carveout:
|
||||
# Check similarity between PROV_TYPE and carveout
|
||||
prov_type_similarity_score = SequenceMatcher(None, prov_type, matched_carveout).ratio()
|
||||
if prov_type_similarity_score < 0.8:
|
||||
prov_type_similarity = check_prov_type_similarity(prov_type, matched_carveout)
|
||||
else:
|
||||
prov_type_similarity = "True"
|
||||
|
||||
if re.search(r'\b(True|Yes)\b', prov_type_similarity, re.IGNORECASE):
|
||||
if matched_carveout not in occurrence_tracker[filename][td_lob]:
|
||||
print('claude returned a match between provider type and service')
|
||||
occurrence_tracker[filename][td_lob][matched_carveout] = 0
|
||||
count = occurrence_tracker[filename][td_lob][matched_carveout]
|
||||
if count == 0:
|
||||
df.at[index, 'label'] = 'primary'
|
||||
elif count == 1:
|
||||
df.at[index, 'label'] = 'secondary'
|
||||
elif count == 2:
|
||||
df.at[index, 'label'] = 'tertiary'
|
||||
else:
|
||||
df.at[index, 'label'] = 'additional'
|
||||
occurrence_tracker[filename][td_lob][matched_carveout] += 1
|
||||
df.at[index, 'IS_CARVEOUT'] = 'N'
|
||||
else:
|
||||
df.at[index, 'IS_CARVEOUT'] = 'Y'
|
||||
else:
|
||||
df.at[index, 'IS_CARVEOUT'] = 'Y'
|
||||
|
||||
carveout_count += 1
|
||||
|
||||
# Save the updated DataFrame to a new CSV file after every 10 carveouts
|
||||
if carveout_count % 10 == 0:
|
||||
df.to_csv(output_filepath, index=False)
|
||||
print(f"Saved progress after processing {carveout_count} carveouts.")
|
||||
|
||||
# Final save of the updated DataFrame to a new CSV file
|
||||
df.to_csv(output_filepath, index=False)
|
||||
|
||||
label_services('service.csv', prompts.get_carveout_list(), 'carveouts50.csv')
|
||||
@@ -0,0 +1,88 @@
|
||||
|
||||
import json
|
||||
import anthropic
|
||||
|
||||
import config
|
||||
|
||||
# Claude calls
|
||||
def invoke_claude_2(prompt, max_tokens):
|
||||
"""
|
||||
Invokes the Claude 2 language model with specified parameters to generate a response based on the input prompt.
|
||||
|
||||
This function constructs a JSON request with the prompt, modified to include pre-defined introductory and concluding
|
||||
sections for consistency. It specifies the token limit and other parameters to control the generation process. The function
|
||||
sends the request to the model's runtime environment and processes the response to extract and return the generated text.
|
||||
|
||||
Parameters:
|
||||
prompt (str): The text prompt to be processed by the Claude 2 model, describing the information or question addressed.
|
||||
max_tokens (int): The maximum number of tokens that the model is allowed to generate in its response.
|
||||
|
||||
Returns:
|
||||
str: The text generated by the Claude 2 model in response to the input prompt.
|
||||
"""
|
||||
body = json.dumps(
|
||||
{"prompt": anthropic.HUMAN_PROMPT + prompt + anthropic.AI_PROMPT,
|
||||
"max_tokens_to_sample": max_tokens,
|
||||
"temperature":0.0,
|
||||
"top_p":1,
|
||||
"top_k":250,
|
||||
"stop_sequences":[anthropic.HUMAN_PROMPT]
|
||||
}
|
||||
)
|
||||
|
||||
response = config.BEDROCK_RUNTIME.invoke_model(
|
||||
body=body,
|
||||
modelId=config.MODEL_ID_CLAUDE2,
|
||||
accept="application/json",
|
||||
contentType="application/json"
|
||||
)
|
||||
|
||||
response_body = json.loads(response.get("body").read())
|
||||
|
||||
response_text = response_body['completion']
|
||||
return response_text
|
||||
|
||||
def invoke_claude_3(prompt, model_id=config.MODEL_ID_CLAUDE3_SONNET, max_tokens = 0):
|
||||
"""
|
||||
Invokes the Claude 3 language model with specified parameters to generate a response based on the input prompt.
|
||||
|
||||
This function constructs a JSON request with the prompt, modified to include pre-defined introductory and concluding
|
||||
sections for consistency. It specifies the token limit and other parameters to control the generation process. The function
|
||||
sends the request to the model's runtime environment and processes the response to extract and return the generated text.
|
||||
|
||||
Parameters:
|
||||
prompt (str): The text prompt to be processed by the Claude 2 model, describing the information or question addressed.
|
||||
max_tokens (int): The maximum number of tokens that the model is allowed to generate in its response.
|
||||
|
||||
Returns:
|
||||
str: The text generated by the Claude 2 model in response to the input prompt.
|
||||
"""
|
||||
prompt = prompt
|
||||
body = json.dumps({
|
||||
"anthropic_version": "bedrock-2023-05-31",
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": 0.0,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text":prompt
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
response = config.BEDROCK_RUNTIME.invoke_model(
|
||||
body=body,
|
||||
modelId=model_id,
|
||||
accept="application/json",
|
||||
contentType="application/json"
|
||||
)
|
||||
|
||||
response_body = json.loads(response.get("body").read())
|
||||
|
||||
return (response_body['content'][0]['text'])
|
||||
@@ -0,0 +1,96 @@
|
||||
|
||||
|
||||
from datetime import datetime
|
||||
import boto3
|
||||
#from llama_index.llms.bedrock import Bedrock
|
||||
#pip install llama-index-llms-bedrock
|
||||
|
||||
# General Settings
|
||||
TEST = False # True to run test prompt - just for testing model connection
|
||||
VERBOSE = True
|
||||
CLIENT_NAME = ''
|
||||
TODAY = datetime.now().strftime("%Y%m%d")
|
||||
|
||||
# Valid values
|
||||
VALID_LOBS = ['MEDICARE', 'MEDICARE ADVANTAGE', 'MEDICAID', 'MARKETPLACE', 'COMMERCIAL', 'GROUP', 'MEDICARE-MEDICAID']
|
||||
VALID_PROGRAMS = ['CHIP', 'CHIP-P', 'CHIP-PERINATE', 'STAR', 'STAR+PLUS', 'MA', 'DUAL SPECIAL NEEDS PLAN', 'DSNP', 'DUAL']
|
||||
VALID_NETWORKS = ['HMO', 'PPO', 'EPO', 'POS', 'FFS']
|
||||
|
||||
# Input Settings
|
||||
READ_MODE = '_LOCAL_' # OR '_S3_'
|
||||
FILTER_ALREADY_PROCESSED = True
|
||||
|
||||
# Output Settings
|
||||
WRITE_OUTPUT = True # True writes csvs, False prints result in console but no output written
|
||||
OUTPUT_DIRECTORY = 'output'
|
||||
CONSOLIDATED_OUTPUT_DIRECTORY = 'output_consolidated'
|
||||
OUTPUT_CSV_PATH = f'consolidated_output_{TODAY}.csv'
|
||||
TD_RESULTS_NAME = 'td_results.csv'
|
||||
BU_RESULTS_NAME = 'bu_results.csv'
|
||||
UNPROCESSED_RESULTS_NAME = 'combined_results_unprocessed.csv'
|
||||
PROCESSED_RESULTS_NAME = 'combined_results_post_processed.csv'
|
||||
|
||||
# Multithread Settings
|
||||
MAX_WORKERS = 10
|
||||
|
||||
# Prompt Debugging
|
||||
RUN_PRIMARY = True # Always True
|
||||
RUN_LOB = True
|
||||
RUN_LESSER = True
|
||||
RUN_METHODOLOGY = True
|
||||
RUN_FS = True
|
||||
RUN_EXCEPTION = True
|
||||
RUN_CODES = True
|
||||
|
||||
# Postprocessing Settings
|
||||
FUZZY_MATCH_THRESHOLD = 0.8
|
||||
VALID_COLUMNS = ['Filename', 'page_num', 'SERVICE', 'REIMBURSEMENT_FLAT_FEE', 'REIMBURSEMENT_RATE', 'FULL_METHODOLOGY',
|
||||
'REIMBURSEMENT_METHODOLOGY', 'REIMBURSEMENT_FEE_SCHEDULE', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION',
|
||||
'LESSER_OF_LANGUAGE_IND', 'GREATER_OF_LANGUAGE_IND',
|
||||
'CONTRACT_LOB', 'CONTRACT_NETWORK', 'PRODUCT', 'CONTRACT_PROGRAM', 'CONTRACT_MARKETPLACE_METAL_LEVEL', 'PROV_TYPE',
|
||||
'REIMBURSEMENT_PROC_CODES', 'REIMBURSEMENT_PROC_CODE_MODIFIERS', 'REIMBURSEMENT_REVENUE_CODES', 'REIMBURSEMENT_STATUS_INDICATOR_CODES',
|
||||
'REIMBURSEMENT_DIAG_CODES', 'REIMBURSEMENT_GROUPER_CODES', 'REIMBURSEMENT_GROUPER', 'REIMBURSEMENT_PLACEOFSERVICE_CODES', 'REIMBURSEMENT_ADMITTYPE_CODES',
|
||||
'REIMBURSEMENT_EXCEPTION_IND', 'REIMBURSEMENT_DESCRIBE_EXCEPTION',
|
||||
'RATE_ESCALATOR_IND', 'RATE_ESCALATOR_BASIS', 'RATE_ESCALATOR_YEARLY_PERCENT_INCREASE',
|
||||
'Corrected_LOB', 'Corrected_PROGRAM', 'Corrected_NETWORK']
|
||||
|
||||
# AWS Keys
|
||||
AWS_ACCESS_KEY_ID="ASIAZTMXAXNXHQSWW2R5"
|
||||
AWS_SECRET_ACCESS_KEY="W9zH4nDGaae8SuJqcPDXyTHFpQ4xV43eRjbDfPGo"
|
||||
AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEIX//////////wEaCXVzLWVhc3QtMiJIMEYCIQCjQVKSzQSWuO8bDdeGjDUJNUc1NwOA795yLFiBGDSslwIhAOccVQAzlBZyOVtU5mZztHVRbqEbrFigeg2aPzlczG1QKoMDCG8QABoMNjYwMTMxMDY4NzgyIgyP2pSGtJAhII6aZIMq4AI3XnnJgTtwbj6u15RcBlTCYgyn+SqIASHvEaJzyWHP/b7Vf5ZNyWKBsccgIZ590De/biM4Gb7tLsnWYCkFuyj3XswHKgyInBDqDg6b4olgyMGoDxrDmNUI28tQxB5XpwKxsas/azzwNPAgRAyXWMK4b1DcslPGpTRBBi+eBF+hTLiGk1jQnip0qR+GuJnCwXe66gcIGmgscjVxGSg2Q5n0O7VD2UOrr5T7BhvnGLiNtlrXqd3f+BFVk6/SzmuQJgsK33rtsmQUI7R5LXlyZEfH3cEkCXrdch6bFO4Wu53oBmiDtNEkle5n95TXCnXC1EchWzuDA6iP061kLRF3jdihMls8St/RoNACCR41YtOuU5oH1n5qXGvH1HQzOvBuSQBCKIqVGgB7hU63mbJlQCdX+0qd0jgteXFHMyjrwr604zPCxdcfbyehAwpxJR8xLVMimgxJqvyzvAoUjzW1sZPPMKG4rLUGOqUBOobUcGB1mI6GHaV6p6+t+qU/BG9FCt9BZbC/IJsHQkWWemCWpvV6lGVM5JPBmmgGQc3LiwH8g5JSHnoUDjTWlLvlHjXZ2ODLJP+aDFcuS4b1WZmXedL6RTsMvg8qhiELEB7rR+6jEmBmXvh56c+R4mkCG5AiWRBZ/ah910JnvKy5wQTOGJGzNxflyiJmwoO2JOZAKBLT0jg78iG1T3DMJ2qc+Qx7"
|
||||
|
||||
# File Paths
|
||||
LOCAL_PATH = 'data/texas_childrens' # Replace with local
|
||||
|
||||
# S3 Settings
|
||||
S3_CLIENT = boto3.client('s3',
|
||||
region_name="us-east-2",
|
||||
aws_access_key_id=AWS_ACCESS_KEY_ID,
|
||||
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
|
||||
aws_session_token=AWS_SESSION_TOKEN
|
||||
)
|
||||
BUCKET = "doczy-dev-infra-textract"
|
||||
PREFIX = "batches/batch_1/contract-text-file/" # replace with s3 path
|
||||
|
||||
# Bedrock Settings
|
||||
BEDROCK_RUNTIME = boto3.client(service_name="bedrock-runtime",
|
||||
region_name="us-east-1",
|
||||
aws_access_key_id=AWS_ACCESS_KEY_ID,
|
||||
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
|
||||
aws_session_token=AWS_SESSION_TOKEN
|
||||
)
|
||||
MODEL_ID_CLAUDE3_HAIKU = 'anthropic.claude-3-haiku-20240307-v1:0'
|
||||
MODEL_ID_CLAUDE3_SONNET = 'anthropic.claude-3-sonnet-20240229-v1:0'
|
||||
MODEL_ID_CLAUDE2 = 'anthropic.claude-instant-v1'
|
||||
|
||||
# LLama Model
|
||||
# LLAMA_MODEL = Bedrock(
|
||||
# model=MODEL_ID_CLAUDE3_HAIKU,
|
||||
# aws_access_key_id=AWS_ACCESS_KEY_ID,
|
||||
# aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
|
||||
# aws_session_token=AWS_SESSION_TOKEN,
|
||||
# region_name="us-east-1"
|
||||
# )
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
|
||||
import os
|
||||
import pandas as pd
|
||||
|
||||
import config
|
||||
|
||||
# all_dfs = []
|
||||
# for file in os.listdir(config.OUTPUT_DIRECTORY):
|
||||
# if config.PROCESSED_RESULTS_NAME in os.listdir(os.path.join(config.OUTPUT_DIRECTORY, file)):
|
||||
# try:
|
||||
# df = pd.read_csv(f'{config.OUTPUT_DIRECTORY}/{file}/{config.PROCESSED_RESULTS_NAME}')
|
||||
# all_dfs.append(df)
|
||||
# except:
|
||||
# continue
|
||||
|
||||
# final_df = pd.concat(all_dfs, ignore_index=True)
|
||||
# print(final_df.shape)
|
||||
# print(len(final_df.Filename.unique()))
|
||||
|
||||
# final_df.to_csv(os.path.join(config.CONSOLIDATED_OUTPUT_DIRECTORY, config.OUTPUT_CSV_PATH))
|
||||
|
||||
|
||||
ac_path = 'T:\\AArete Client Work\\Texas Children’s Health Plan\\Restricted\\Doczy_Output\\TX_Childrens_1-1.csv'
|
||||
b_path = 'T:\\AArete Client Work\\Texas Children’s Health Plan\\Restricted\\Doczy_Output\\TX_Childrens_1-N.csv'
|
||||
output_path = f'T:\\AArete Client Work\\Texas Children’s Health Plan\\Restricted\\Doczy_Output\\{config.TODAY}_TX_Childrens_460_Doczy_AI_Output.csv'
|
||||
|
||||
# Read the files into pandas DataFrames
|
||||
ac = pd.read_csv(ac_path)
|
||||
b = pd.read_csv(b_path)
|
||||
|
||||
print(b.shape)
|
||||
|
||||
# Merge the DataFrames on the 'Filename' column
|
||||
merged_df = pd.merge(ac, b, on='Filename', how='right')
|
||||
merged_df.to_csv(output_path)
|
||||
|
||||
print(merged_df.shape)
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
|
||||
import re
|
||||
import json
|
||||
from collections import defaultdict
|
||||
|
||||
def secondary_string_to_dict(dict_string):
|
||||
"""
|
||||
Converts a string representation of a dictionary into an actual dictionary object, handling potential formatting issues.
|
||||
|
||||
This function cleans the string by removing specific unwanted characters and markers that may interfere with JSON parsing.
|
||||
It then locates the substring that correctly forms a dictionary format, attempts to parse it as JSON, and handles common
|
||||
parsing errors by replacing problematic single quotes with double quotes before re-parsing.
|
||||
|
||||
Parameters:
|
||||
dict_string (str): The string containing the dictionary-like content, potentially surrounded by extra text or characters.
|
||||
|
||||
Returns:
|
||||
dict: The dictionary obtained from parsing the cleaned and corrected string.
|
||||
"""
|
||||
dict_string = dict_string.replace('<<<', '')
|
||||
dict_string = dict_string.replace('>>>', '')
|
||||
dict_string = dict_string.replace('\n', '') # Remove new line
|
||||
|
||||
start_index = dict_string.find('{')
|
||||
end_index = dict_string.rfind('}') + 1
|
||||
dict_substring = dict_string[start_index:end_index]
|
||||
try:
|
||||
result_dict = json.loads(dict_substring)
|
||||
except:
|
||||
dict_substring = dict_substring.replace("'", '"')
|
||||
result_dict = json.loads(dict_substring)
|
||||
return result_dict
|
||||
|
||||
def primary_string_to_dict(string_dict, filename):
|
||||
"""
|
||||
Converts a dictionary of strings, where each string represents multiple dictionary entries, into a list of dictionaries,
|
||||
augmenting each with metadata such as page number and filename.
|
||||
|
||||
This function iterates over each page number's string data, extracting and converting string representations of
|
||||
dictionaries into actual dictionary objects. It handles and cleans the string format to properly parse it into dictionaries.
|
||||
All dictionaries are then augmented with their respective page number and the filename before being compiled into a list.
|
||||
|
||||
Parameters:
|
||||
string_dict (dict): A dictionary where keys are page numbers and values are strings containing multiple dictionary entries.
|
||||
filename (str): The filename associated with these entries, used to tag each resulting dictionary.
|
||||
|
||||
Returns:
|
||||
list of dict: A list of dictionaries, each representing data extracted and converted from the input string, tagged with
|
||||
their page number and filename.
|
||||
"""
|
||||
data = []
|
||||
pattern = r'\{.*?\}'
|
||||
for page_num in string_dict.keys():
|
||||
primary_list = string_dict[page_num]
|
||||
dicts = primary_list.split('[')[1] # Strip front
|
||||
dicts = dicts.split(']')[0] # Strip back
|
||||
dicts = dicts.replace('\n', '') # Remove new lines
|
||||
dicts = dicts.replace('<<<', '')
|
||||
dicts = dicts.replace('>>>', '')
|
||||
dicts = re.sub(r"(?<!\\)'([^']*?)'(?<!\\):", r'"\1":', dicts)
|
||||
dict_list = re.findall(pattern, dicts)
|
||||
dict_list = [secondary_string_to_dict(d) for d in dict_list]
|
||||
for dict_ in dict_list:
|
||||
dict_['page_num'] = page_num
|
||||
dict_['Filename'] = filename
|
||||
data.append(dict_)
|
||||
return data
|
||||
|
||||
def append_secondary(result_dicts):
|
||||
"""
|
||||
Aggregates and restructures a list of dictionaries by grouping based on specific fields and appending secondary and tertiary
|
||||
entries for specific keys.
|
||||
|
||||
This function organizes dictionaries by creating a unique key from non-'REIMBURSEMENT' fields and groups them accordingly.
|
||||
It then processes each group to combine entries, designating the first three as primary, secondary, and tertiary respectively
|
||||
for any 'REIMBURSEMENT' related fields. The combined results are stored in new dictionaries reflecting this hierarchical structure.
|
||||
|
||||
Parameters:
|
||||
result_dicts (list of dict): A list of dictionaries containing data with fields that may include several related to 'REIMBURSEMENT'.
|
||||
|
||||
Returns:
|
||||
list of dict: A list of new dictionaries where each dictionary represents combined information from the original list,
|
||||
with 'REIMBURSEMENT' fields clearly distinguished as primary, secondary, or tertiary.
|
||||
"""
|
||||
# Helper function to create a key from dict excluding certain keys
|
||||
def create_key(d):
|
||||
return tuple((k, d[k]) for k in d if not k.startswith('REIMBURSEMENT'))
|
||||
|
||||
# Group dictionaries by non-REIMBURSEMENT fields
|
||||
grouped = defaultdict(list)
|
||||
for d in result_dicts:
|
||||
grouped[create_key(d)].append(d)
|
||||
|
||||
# Process groups to create combined entries
|
||||
combined_results = []
|
||||
for key, group in grouped.items():
|
||||
# Create a new dictionary based on non-REIMBURSEMENT fields
|
||||
new_dict = {k: v for k, v in key}
|
||||
# Iterate over REIMBURSEMENT fields and assign them as primary, secondary, tertiary
|
||||
for i, entry in enumerate(group[:3]): # Process only the first three entries
|
||||
for k, v in entry.items():
|
||||
if k.startswith('REIMBURSEMENT'):
|
||||
new_key = f'{["PRIMARY", "SECONDARY", "TERTIARY"][i]}_{k}'
|
||||
new_dict[new_key] = v
|
||||
combined_results.append(new_dict)
|
||||
return combined_results
|
||||
@@ -0,0 +1,81 @@
|
||||
|
||||
import os
|
||||
import pandas as pd
|
||||
|
||||
import config
|
||||
import preprocess
|
||||
import table_funcs
|
||||
import bottom_up_funcs
|
||||
import top_down_funcs
|
||||
import postprocess
|
||||
import merge_funcs
|
||||
|
||||
|
||||
def process_file(file_object):
|
||||
"""
|
||||
file_object = (filename , "file contents in one string"}
|
||||
|
||||
Processes a single file object containing text data from a contract.
|
||||
|
||||
This function orchestrates the full processing workflow for a single document, including:
|
||||
- Directory setup for outputs based on the filename.
|
||||
- Preprocessing text to clean and split it into manageable parts.
|
||||
- Running 'Top Down' and 'Bottom Up' analysis using pre-defined models.
|
||||
- Merging and consolidating results from both analysis methods.
|
||||
- Postprocessing the consolidated results for final output.
|
||||
|
||||
Parameters:
|
||||
file_object (tuple): A tuple containing the filename and the text content of the file.
|
||||
|
||||
Outputs:
|
||||
- Several CSV files are generated in the designated output directory:
|
||||
1. Results from the Top Down analysis.
|
||||
2. Results from the Bottom Up analysis.
|
||||
3. Combined results of both analyses before and after postprocessing.
|
||||
|
||||
Each step of the process is logged verbosely if the configuration is set to verbose mode.
|
||||
"""
|
||||
|
||||
################## INITIATE PROCESSING ##################
|
||||
filename, contract_text = file_object
|
||||
if config.VERBOSE: print(f"Processing {filename}...")
|
||||
|
||||
################## CREATE OUTPUT DIRECTORIES ##################
|
||||
base_filename = os.path.splitext(filename)[0].strip()
|
||||
output_dir = os.path.join(config.OUTPUT_DIRECTORY, base_filename)
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
################## PREPROCESS ##################
|
||||
contract_text = preprocess.clean_newlines(contract_text)
|
||||
contract_text = preprocess.clean_billed_charges(contract_text)
|
||||
text_dict = preprocess.split_text(contract_text)
|
||||
text_dict = table_funcs.align_and_format_tables(text_dict)
|
||||
text_dict = preprocess.highlight_rates(text_dict)
|
||||
|
||||
################## RUN TOP DOWN ##################
|
||||
td_results = top_down_funcs.run_top_down(filename, text_dict) # Returns list of dictionaries for each page
|
||||
pd.DataFrame(td_results).to_csv(os.path.join(output_dir, config.TD_RESULTS_NAME), index=False)
|
||||
print(f"Top Down Complete - {filename}")
|
||||
|
||||
################## RUN BOTTOM UP ##################
|
||||
bu_results = bottom_up_funcs.run_bottom_up(filename, text_dict) # Returns list of dictionaries
|
||||
pd.DataFrame(bu_results).to_csv(os.path.join(output_dir, config.BU_RESULTS_NAME), index=False)
|
||||
print(f"Bottom Up Complete - {filename}")
|
||||
|
||||
################## COMBINE TOP DOWN AND BOTTOM UP ##################
|
||||
combined_results = merge_funcs.merge_results(td_results, bu_results, text_dict)
|
||||
print(f"TD/BU Merge Complete - {filename} ")
|
||||
|
||||
################## WRITE UNPROCESSED OUTPUT ##################
|
||||
combined_df = pd.DataFrame(combined_results)
|
||||
combined_df.to_csv(os.path.join(output_dir, config.UNPROCESSED_RESULTS_NAME), index=False)
|
||||
|
||||
################## RUN POSTPROCESSING ##################
|
||||
# combined_df = combined_df.applymap(postprocessingfuncs.sanitize_value) # Deprecated
|
||||
|
||||
post_processed_combined_df = postprocess.postprocess_results(combined_df)
|
||||
post_processed_combined_df.to_csv(os.path.join(output_dir, config.PROCESSED_RESULTS_NAME), index=False)
|
||||
print(f"Postprocessing Complete - {filename} ")
|
||||
|
||||
print(f"Output Complete - {filename} ")
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,62 @@
|
||||
|
||||
""" main.py
|
||||
Doczy AI - Pricing Before Carveouts: Main Execution Module
|
||||
|
||||
This module is the main entry point for the Priority B: Pricing Before Carveouts section of Doczy AI.
|
||||
|
||||
Functional Overview:
|
||||
- In test mode, a simple Claude 3 model invocation is demonstrated.
|
||||
- In normal mode, the module reads input documents, processes each document concurrently,
|
||||
and manages exceptions and errors during processing.
|
||||
- The processed results can be optionally consolidated into a single CSV file.
|
||||
"""
|
||||
|
||||
# Imports
|
||||
import concurrent.futures
|
||||
import traceback
|
||||
import os
|
||||
|
||||
import utils
|
||||
import config
|
||||
import claude_funcs
|
||||
import file_processing
|
||||
|
||||
def main():
|
||||
if config.TEST:
|
||||
print(claude_funcs.invoke_claude_3("Write 'test', nothing more.", max_tokens = 10))
|
||||
else:
|
||||
# Read contract txt
|
||||
input_dict = utils.read_input()
|
||||
|
||||
if config.FILTER_ALREADY_PROCESSED:
|
||||
input_dict = utils.filter_already_processed(input_dict)
|
||||
|
||||
def process_item(item):
|
||||
key, value = item
|
||||
if utils.contains_reimbursement(item[1],0):
|
||||
try:
|
||||
file_processing.process_file(item)
|
||||
#prompt_funcs.bottom_up(item)
|
||||
except Exception as e:
|
||||
# Print the error message and traceback
|
||||
print(f"Error processing item {key}: {e}")
|
||||
traceback.print_exc()
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor:
|
||||
# Process each item individually
|
||||
futures = [executor.submit(process_item, item) for item in input_dict.items()]
|
||||
|
||||
# Wait for all futures to complete
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
# Retrieve any exceptions raised by the thread
|
||||
try:
|
||||
future.result()
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
|
||||
# Write Consolidated Output
|
||||
# if config.WRITE_OUTPUT:
|
||||
# utils.consolidate_csvs(config.CONSOLIDATED_OUTPUT_DIRECTORY, config.OUTPUT_CSV_PATH)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,52 @@
|
||||
|
||||
import prompts
|
||||
import claude_funcs
|
||||
|
||||
|
||||
def prompt_select_multiple(key, value, bu_dict, page):
|
||||
prompt = prompts.MERGE_SELECT_PROMPT(key, value, bu_dict, page)
|
||||
answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000)
|
||||
return answer
|
||||
|
||||
|
||||
def merge_results(td_results, bu_results, text_dict):
|
||||
def find_td_dict(page_num):
|
||||
""" Helper function to find dictionary in td_results with matching page_num """
|
||||
for td in td_results:
|
||||
if td['page_num'] == page_num:
|
||||
return td
|
||||
return None
|
||||
|
||||
def get_value(td_dict, key, page_num, text_dict):
|
||||
""" Helper function to handle value selection based on the rules specified. """
|
||||
if td_dict is None:
|
||||
# If there's no matching page, use recursion to look one page earlier
|
||||
if int(page_num) > 1:
|
||||
return get_value(find_td_dict(int(page_num) - 1), key, int(page_num) - 1, text_dict)
|
||||
else:
|
||||
return "N/A" # Base case if no previous pages exist
|
||||
else:
|
||||
value = td_dict.get(key, "N/A")
|
||||
if value == "N/A" or value == "":
|
||||
# If value is 'N/A' or empty, use recursion to look one page earlier
|
||||
return get_value(find_td_dict(int(page_num) - 1), key, int(page_num) - 1, text_dict)
|
||||
elif ',' in value and 'DATE' not in key:
|
||||
# Randomly select one of the comma-separated values
|
||||
return prompt_select_multiple(key, value, bu_dict, text_dict[page_num])
|
||||
else:
|
||||
return value
|
||||
|
||||
merged_results = []
|
||||
for bu_dict in bu_results:
|
||||
page_num = bu_dict['page_num']
|
||||
td_dict = find_td_dict(page_num)
|
||||
|
||||
new_dict = bu_dict.copy() # Start with the bu_dict's data
|
||||
# Add or overwrite keys from td_dict
|
||||
if td_dict:
|
||||
for key, value in td_dict.items():
|
||||
if key not in ['Filename', 'page_num']:
|
||||
new_dict[key] = get_value(td_dict, key, page_num, text_dict)
|
||||
|
||||
merged_results.append(new_dict)
|
||||
return merged_results
|
||||
@@ -0,0 +1,70 @@
|
||||
import pandas as pd
|
||||
import re
|
||||
|
||||
import postprocessing_funcs
|
||||
import config
|
||||
|
||||
|
||||
def postprocess_results(combined_df):
|
||||
|
||||
# Define valid values dictionary
|
||||
valid_values_dict = {
|
||||
'CONTRACT_LOB': config.VALID_LOBS,
|
||||
'CONTRACT_PROGRAM': config.VALID_PROGRAMS,
|
||||
'CONTRACT_NETWORK': config.VALID_NETWORKS
|
||||
}
|
||||
|
||||
# Sanitize the combined data
|
||||
try:
|
||||
df = postprocessing_funcs.sanitize_combined(combined_df)
|
||||
except Exception as e:
|
||||
print(f"Postprocessing Error - santize_combined : {e}")
|
||||
|
||||
# Clean specific columns for exact matches
|
||||
for field_list in [['CONTRACT_LOB', 'Corrected_LOB', config.VALID_LOBS], ['CONTRACT_PROGRAM', 'Corrected_PROGRAM', config.VALID_PROGRAMS], ['CONTRACT_NETWORK', 'Corrected_NETWORK', config.VALID_NETWORKS]]:
|
||||
try:
|
||||
postprocessing_funcs.clean_columns_combined(df, field_list[0], field_list[2], field_list[1])
|
||||
except Exception as e:
|
||||
print(f"Postprocessing Error - clean_columns_combined : {e}")
|
||||
|
||||
try:
|
||||
postprocessing_funcs.clean_columns_combined_fuzzy(df, field_list[0], field_list[2], config.FUZZY_MATCH_THRESHOLD)
|
||||
except Exception as e:
|
||||
print(f"Postprocessing Error - clean_columns_combined_fuzzy : {e}")
|
||||
|
||||
# Correct misplaced values across columns
|
||||
try:
|
||||
df = postprocessing_funcs.correct_misplaced_values(df, ['CONTRACT_LOB', 'CONTRACT_PROGRAM', 'CONTRACT_NETWORK'], valid_values_dict)
|
||||
except Exception as e:
|
||||
print(f"Postprocessing Error - correct_misplaced_values : {e}")
|
||||
|
||||
# Move percentages and large numbers to correct columns
|
||||
try:
|
||||
df = postprocessing_funcs.move_percentage_to_rate(df)
|
||||
except Exception as e:
|
||||
print(f"Postprocessing Error - move_percentage_to_rate : {e}")
|
||||
|
||||
# Replace N/As in "Not Covered" with 0%
|
||||
try:
|
||||
df = postprocessing_funcs.set_rate_to_zero_if_not_covered(df)
|
||||
except Exception as e:
|
||||
print(f"Postprocessing Error - set_rate_to_zero_if_not_covered : {e}")
|
||||
|
||||
# Adjust for adding or subtracting
|
||||
try:
|
||||
df = postprocessing_funcs.adjust_reimbursement_rate(df)
|
||||
except Exception as e:
|
||||
print(f"Postprocessing Error - adjust_reimbursement_rate : {e}")
|
||||
|
||||
# -- Deprecated --
|
||||
# Clean dates so that Termination and Effective aren't the same
|
||||
# try:
|
||||
# df = postprocessing_funcs.clean_dates(df)
|
||||
# except Exception as e:
|
||||
# print(f"Postprocessing Error - clean_dates : {e}")
|
||||
|
||||
# Re order columns
|
||||
column_order = [col for col in config.VALID_COLUMNS if col in df.columns] + [col for col in df.columns if col not in config.VALID_COLUMNS]
|
||||
df = df[column_order]
|
||||
|
||||
return df
|
||||
@@ -0,0 +1,455 @@
|
||||
|
||||
import pandas as pd
|
||||
import re
|
||||
import difflib
|
||||
|
||||
import config
|
||||
|
||||
def sanitize_value(value):
|
||||
"""
|
||||
Sanitizes a given value by handling nulls, lists, and strings to ensure consistency in formatting.
|
||||
|
||||
This function performs several cleanliness checks and transformations:
|
||||
- It leaves null (NaN) values unchanged.
|
||||
- Converts lists into comma-separated strings.
|
||||
- Strips unnecessary characters from strings and handles strings that represent lists, ensuring individual elements are properly formatted.
|
||||
|
||||
Parameters:
|
||||
value (varied types): The value to be sanitized, which can be of any type including nulls, lists, or strings.
|
||||
|
||||
Returns:
|
||||
varied types: The sanitized value, which can be a string or the original type if no changes were needed.
|
||||
"""
|
||||
try:
|
||||
if isinstance(value, list):
|
||||
return ', '.join(str(v) for v in value)
|
||||
elif isinstance(value, str):
|
||||
value = value.strip('[]')
|
||||
return ', '.join([item.strip(" '") for item in value.split(',')])
|
||||
elif pd.isna(value):
|
||||
return "N/A"
|
||||
except:
|
||||
return value
|
||||
|
||||
def exact_match(val, valid_values):
|
||||
"""
|
||||
Compares a given string value against a list of valid values to determine if there is an exact match, case-insensitively.
|
||||
|
||||
This function processes the input value by trimming whitespace and converting to uppercase for a case-insensitive comparison.
|
||||
It then checks each item in the list of valid values (also treated case-insensitively) to find an exact match. If a match is found,
|
||||
the function returns the valid value in its original form as specified in the list. If no match is found, it returns None.
|
||||
|
||||
Parameters:
|
||||
val (str): The value to be checked for an exact match.
|
||||
valid_values (list of str): A list of valid values against which the input value is compared.
|
||||
|
||||
Returns:
|
||||
str or None: The matching valid value if found, or None if no exact match is found.
|
||||
"""
|
||||
val = val.strip().upper()
|
||||
for valid_val in valid_values:
|
||||
if val == valid_val.upper():
|
||||
return valid_val
|
||||
return None
|
||||
|
||||
def clean_columns_combined(df, column_name, valid_values, new_column_name):
|
||||
"""
|
||||
Cleans and standardizes entries in a specified column of a DataFrame based on a list of valid values, creating a new column with standardized values.
|
||||
|
||||
This function first sanitizes the values in the specified column. It then examines each entry in that column,
|
||||
looking for any sub-values (separated by commas) that exactly match any of the provided valid values. If a match is found,
|
||||
it is recorded in a new column. The function also tracks changes and compares the original and cleaned unique values.
|
||||
|
||||
Parameters:
|
||||
df (pandas.DataFrame): The DataFrame containing the data to be cleaned.
|
||||
column_name (str): The name of the column to be cleaned.
|
||||
valid_values (list of str): A list of valid values to be used for checking the entries in the specified column.
|
||||
new_column_name (str): The name of the new column where cleaned, standardized values will be stored.
|
||||
|
||||
Returns:
|
||||
tuple: Contains arrays of original unique values, cleaned unique values, and a dictionary of changes from original to new values.
|
||||
"""
|
||||
changes = {}
|
||||
df[column_name] = df[column_name].apply(sanitize_value)
|
||||
original_values = df[column_name].unique()
|
||||
|
||||
def update_column(entry):
|
||||
if pd.notna(entry):
|
||||
terms = entry.split(',')
|
||||
for term in terms:
|
||||
match = exact_match(term, valid_values)
|
||||
if match:
|
||||
return match
|
||||
return None
|
||||
|
||||
df[new_column_name] = df[column_name].apply(update_column)
|
||||
cleaned_values = df[new_column_name].unique()
|
||||
return original_values, cleaned_values, changes
|
||||
|
||||
|
||||
def get_closest_match(val, valid_values, similarity_threshold=0.7):
|
||||
"""
|
||||
Finds the closest match for a given string from a list of valid values, using a similarity threshold.
|
||||
|
||||
This function cleans the input value and compares it to each cleaned value in the valid values list using a fuzzy
|
||||
matching technique. It returns the closest match that meets or exceeds the specified similarity threshold. If no
|
||||
matches meet the threshold, the function returns None.
|
||||
|
||||
Parameters:
|
||||
val (str): The value to be matched.
|
||||
valid_yvalues (list of str): A list of valid values against which the input value is compared.
|
||||
similarity_threshold (float): The cutoff similarity score (between 0 and 1) below which matches are not considered.
|
||||
|
||||
Returns:
|
||||
str or None: The closest valid value if a match is found; otherwise, None.
|
||||
"""
|
||||
if pd.isna(val):
|
||||
return None
|
||||
val = val.strip().upper()
|
||||
matches = difflib.get_close_matches(val, [v.upper() for v in valid_values], n=1, cutoff=similarity_threshold)
|
||||
return matches[0] if matches else None
|
||||
|
||||
|
||||
def clean_columns_combined_fuzzy(df, column_name, valid_values, threshold):
|
||||
"""
|
||||
Applies fuzzy matching to entries in a specified column of a DataFrame to standardize them according to a list of valid values,
|
||||
using a defined similarity threshold to determine matches.
|
||||
|
||||
The function sanitizes the column values and then processes each entry, splitting it into sub-values if multiple are present.
|
||||
Each sub-value is compared against the valid values list using a fuzzy matching algorithm. If a close match is found (above the
|
||||
threshold), it replaces the original sub-value. The function logs any changes made for tracking and verification purposes.
|
||||
|
||||
Parameters:
|
||||
df (pandas.DataFrame): The DataFrame containing the column to be cleaned.
|
||||
column_name (str): The name of the column to be cleaned.
|
||||
valid_values (list of str): A list of valid values used for fuzzy matching.
|
||||
threshold (float): The similarity threshold for accepting matches.
|
||||
|
||||
Returns:
|
||||
tuple: Contains arrays of original unique values, cleaned unique values, and a dictionary of changes (original to new values).
|
||||
"""
|
||||
changes = {}
|
||||
df[column_name] = df[column_name].apply(sanitize_value)
|
||||
original_values = df[column_name].unique()
|
||||
|
||||
def log_and_clean(entry):
|
||||
if pd.notna(entry):
|
||||
words = entry.split(',')
|
||||
cleaned_words = []
|
||||
for word in words:
|
||||
cleaned_word = get_closest_match(word.strip(), valid_values, similarity_threshold=threshold)
|
||||
if cleaned_word and word.strip().upper() != cleaned_word:
|
||||
changes[word.strip()] = cleaned_word
|
||||
cleaned_words.append(cleaned_word if cleaned_word else word.strip())
|
||||
return ', '.join(cleaned_words)
|
||||
return None
|
||||
|
||||
df[column_name] = df[column_name].apply(log_and_clean)
|
||||
cleaned_values = df[column_name].unique()
|
||||
return original_values, cleaned_values, changes
|
||||
|
||||
|
||||
|
||||
def correct_misplaced_values(df, columns, valid_values_dict):
|
||||
"""
|
||||
Corrects misplaced values within specified columns of a DataFrame based on a dictionary of valid values for each column.
|
||||
|
||||
This function iterates through each row of the DataFrame and checks the values of specified columns. If a value
|
||||
is found that matches valid entries for a different column (based on the valid_values_dict), it moves the value to
|
||||
the appropriate column. If the target column already contains a value, it uses fuzzy matching to determine if the
|
||||
misplaced value should replace the existing one. If a replacement isn't suitable, it logs the correction attempt.
|
||||
|
||||
Parameters:
|
||||
df (pandas.DataFrame): The DataFrame containing the data to be corrected.
|
||||
columns (list of str): The columns to check for misplaced values.
|
||||
valid_values_dict (dict): A dictionary mapping column names to lists of valid values for those columns.
|
||||
|
||||
Returns:
|
||||
pandas.DataFrame: The DataFrame with corrected placements of values across the specified columns.
|
||||
"""
|
||||
for index, row in df.iterrows():
|
||||
for col in columns:
|
||||
if pd.notna(row[col]):
|
||||
terms = row[col].split(',')
|
||||
for term in terms:
|
||||
term = term.strip()
|
||||
for target_col, valid_values in valid_values_dict.items():
|
||||
if target_col != col:
|
||||
match = exact_match(term, valid_values)
|
||||
if match:
|
||||
if pd.isna(row[target_col]) or not row[target_col].strip():
|
||||
df.at[index, target_col] = match
|
||||
df.at[index, col] = None
|
||||
else:
|
||||
current_value = row[target_col].strip()
|
||||
if get_closest_match(match, [current_value], config.FUZZY_MATCH_THRESHOLD) is None:
|
||||
df.at[index, 'Corrected_' + target_col] = f"Found {term} in {col} cell"
|
||||
df.at[index, col] = None
|
||||
return df
|
||||
|
||||
|
||||
def filter_service_column(d):
|
||||
"""
|
||||
Filters out items in a list of dictionaries based on specified keywords within the 'SERVICE' key.
|
||||
|
||||
This function uses a list of predefined keywords associated with non-service related terms like 'LIABILITY' or 'RISK'.
|
||||
It constructs a regular expression pattern from these keywords and removes any dictionary items where the 'SERVICE' key contains
|
||||
any of these keywords, effectively filtering the list to exclude items that are likely to be irrelevant or misclassified
|
||||
as services when they pertain to other contractual obligations or conditions.
|
||||
|
||||
Parameters:
|
||||
d (list): A list of dictionaries, each containing a 'SERVICE' key among others.
|
||||
|
||||
Returns:
|
||||
list: A new list of dictionaries with items containing specified keywords in the 'SERVICE' key removed.
|
||||
"""
|
||||
keywords = [
|
||||
'LIABILITY', 'RISK', 'LOBBYING', 'DAMAGES', 'CONFIDENTIALITY', 'ARBITRATION', 'FALSE CLAIMS ACT', 'UNSPECIFIED',
|
||||
'AUDIT', 'INTEREST', 'N/A', 'BUSINESS', 'COMPLIANCE', 'Medical Assistance Program', 'MATERIAL SUBCONTRACT', 'GIFTS', 'GRATUITIES',
|
||||
]
|
||||
pattern = '|'.join(keywords)
|
||||
regex = re.compile(pattern, re.IGNORECASE)
|
||||
|
||||
filtered_list = [item for item in d if not regex.search(item.get('SERVICE', ''))]
|
||||
|
||||
return filtered_list
|
||||
|
||||
|
||||
def move_percentage_to_rate(df):
|
||||
"""
|
||||
Moves percentage values from the 'REIMBURSEMENT_FLAT_FEE' column to the 'REIMBURSEMENT_RATE' column in a DataFrame.
|
||||
|
||||
This function identifies entries in the 'REIMBURSEMENT_FLAT_FEE' column that contain a percentage sign, indicating they are
|
||||
incorrectly placed. It then transfers these percentage values to the 'REIMBURSEMENT_RATE' column. The original 'REIMBURSEMENT_FLAT_FEE'
|
||||
entries from which percentages are moved are set to None, correcting the placement of data within the DataFrame.
|
||||
|
||||
Parameters:
|
||||
df (pandas.DataFrame): The DataFrame containing the reimbursement data to be adjusted.
|
||||
|
||||
Returns:
|
||||
pandas.DataFrame: The DataFrame with percentage values moved from the 'REIMBURSEMENT_FLAT_FEE' to the 'REIMBURSEMENT_RATE' column.
|
||||
"""
|
||||
def move_percentage(value):
|
||||
if isinstance(value, str) and '%' in value:
|
||||
return True
|
||||
return False
|
||||
|
||||
df['REIMBURSEMENT_RATE'] = df.apply(lambda row: row['REIMBURSEMENT_FLAT_FEE'] if move_percentage(row['REIMBURSEMENT_FLAT_FEE']) else row['REIMBURSEMENT_RATE'], axis=1)
|
||||
df['REIMBURSEMENT_FLAT_FEE'] = df.apply(lambda row: None if move_percentage(row['REIMBURSEMENT_FLAT_FEE']) else row['REIMBURSEMENT_FLAT_FEE'], axis=1)
|
||||
return df
|
||||
|
||||
|
||||
def set_rate_to_zero_if_not_covered(df):
|
||||
"""
|
||||
Sets the 'REIMBURSEMENT_RATE' to zero for entries in the DataFrame where the service is explicitly not covered,
|
||||
as indicated by the content of the 'FULL_METHODOLOGY' column.
|
||||
|
||||
This function checks each row's 'FULL_METHODOLOGY' column for phrases indicating that the service is not covered
|
||||
(e.g., "not covered"). If such a phrase is found, the 'REIMBURSEMENT_RATE' for that row is set to zero to reflect that
|
||||
there is no financial reimbursement for these services. This helps ensure that the data accurately represents the terms
|
||||
of service coverage.
|
||||
|
||||
Parameters:
|
||||
df (pandas.DataFrame): The DataFrame containing the reimbursement and methodology data.
|
||||
|
||||
Returns:
|
||||
pandas.DataFrame: The updated DataFrame with adjusted 'REIMURSEMENT_RATE' values where applicable.
|
||||
"""
|
||||
def check_and_set_rate(row):
|
||||
if pd.notna(row['FULL_METHODOLOGY']) and re.search(r'not covered', row['FULL_METHODOLOGY'], re.IGNORECASE):
|
||||
return 0
|
||||
return row['REIMBURSEMENT_RATE']
|
||||
|
||||
df['REIMBURSEMENT_RATE'] = df.apply(check_and_set_rate, axis=1)
|
||||
return df
|
||||
|
||||
|
||||
def adjust_reimbursement_rate(df):
|
||||
"""
|
||||
Adjusts the 'REIMBURSEMENT_RATE' in the DataFrame based on specific conditions mentioned in the 'FULL_METHODOLOGY' column.
|
||||
|
||||
This function reviews each row's 'FULL_METHODOLOGY' column for mentions of being "case insensitive" as a part of the methodology.
|
||||
If this condition is noted and the 'REIMBURSEMENT_RATE' is a numerical value, it adjusts the rate by subtracting it from 100,
|
||||
effectively transforming the rate into a complementary percentage. This modification is intended to correct or recalibrate
|
||||
reimbursement rates under specific contractual terms.
|
||||
|
||||
Parameters:
|
||||
df (pandas.DataFrame): The DataFrame containing the methodology and reimbursement rate data.
|
||||
|
||||
Returns:
|
||||
pandas.DataFrame: The updated DataFrame with recalibrated 'REIMBURSEMENT_RATE' values based on the specified methodology condition.
|
||||
"""
|
||||
def adjust_rate(row):
|
||||
if pd.notna(row['FULL_METHODOLOGY']) and re.search(r'case insensitive', row['FULL_METHODOLOGY'], re.IGNORECASE):
|
||||
if pd.notna(row['REIMBURSEMENT_RATE']) and isinstance(row['REIMBURSEMENT_RATE'], (int, float)):
|
||||
return 100 - row['REIMBURSEMENT_RATE']
|
||||
return row['REIMBURSEMENT_RATE']
|
||||
|
||||
df['REIMBURSEMENT_RATE'] = df.apply(adjust_rate, axis=1)
|
||||
return df
|
||||
|
||||
|
||||
def clean_td(td):
|
||||
"""
|
||||
Cleans the data structure returned from a 'Top Down' analysis by normalizing and formatting its values.
|
||||
|
||||
This function iterates through each entry in the list of dictionaries returned from the top down analysis,
|
||||
modifying data based on specific criteria:
|
||||
- Dates are converted to lists if not already in list form.
|
||||
- Comma-separated strings are split into lists.
|
||||
- 'N/A' values are replaced with empty lists.
|
||||
- Preserves specific keys as they are (e.g., 'page_num', 'Filename').
|
||||
|
||||
Parameters:
|
||||
td (list of dict): The list of dictionaries containing the results from top down analysis.
|
||||
|
||||
Returns:
|
||||
list of dict: A new list of dictionaries with the cleaned and normalized data.
|
||||
"""
|
||||
td_clean = []
|
||||
for d in td:
|
||||
new_d = {}
|
||||
for k, v in d.items():
|
||||
if 'DATE' in k:
|
||||
new_d[k] = v if isinstance(v, list) else [v]
|
||||
elif k not in ['page_num', 'Filename']:
|
||||
if isinstance(v, str) and ',' in v:
|
||||
new_d[k] = [item.strip() for item in v.split(',')]
|
||||
elif v == 'N/A':
|
||||
new_d[k] = []
|
||||
else:
|
||||
new_d[k] = [v] if isinstance(v, str) else v
|
||||
else:
|
||||
new_d[k] = v
|
||||
td_clean.append(new_d)
|
||||
return td_clean
|
||||
|
||||
|
||||
|
||||
def sanitize_combined(df):
|
||||
"""
|
||||
Sanitizes all columns in a DataFrame by applying a predefined sanitization function to each value.
|
||||
|
||||
This function iterates over each column in the DataFrame, applying a sanitization function to each element.
|
||||
The sanitization function is designed to clean or adjust the data in a way that makes it more consistent or appropriate
|
||||
for further processing or analysis. This could include operations like trimming whitespace, correcting formats,
|
||||
or removing unwanted characters.
|
||||
|
||||
Parameters:
|
||||
df (pandas.DataFrame): The DataFrame to be sanitized.
|
||||
|
||||
Returns:
|
||||
pandas.DataFrame: The sanitized DataFrame, with the same structure but cleaned data.
|
||||
"""
|
||||
for column in df.columns:
|
||||
df[column] = df[column].apply(sanitize_value)
|
||||
return df
|
||||
|
||||
def extract_codes_CPT(value):
|
||||
"""
|
||||
Extracts and formats CPT codes from a given input value.
|
||||
|
||||
This function searches for patterns that match CPT codes within the provided input. CPT codes typically consist
|
||||
of a letter followed by four digits, or just five digits, and may include a two-letter state code as a suffix.
|
||||
The function handles cases where the range of codes is specified using 'through' by converting it to a hyphen.
|
||||
If any codes are found, they are returned as a comma-separated string. If no codes are found or the input is null,
|
||||
an appropriate value (empty string or null) is returned.
|
||||
|
||||
Parameters:
|
||||
value (str or NaN): The value from which to extract CPT codes, which could potentially be a string or NaN.
|
||||
|
||||
Returns:
|
||||
str: A comma-separated string of the extracted CPT codes, or an empty string if none are found.
|
||||
If the input is NaN, the function returns the input as is.
|
||||
"""
|
||||
if pd.isna(value):
|
||||
return value
|
||||
|
||||
value = re.sub(r'\bthrough\b', '-', str(value), flags=re.IGNORECASE)
|
||||
|
||||
pattern = r'\b([A-Z]\d{4}|\d{5})(-[A-Z]{2})?\b'
|
||||
matches = re.findall(pattern, value)
|
||||
if matches:
|
||||
return ', '.join([''.join(match) for match in matches])
|
||||
else:
|
||||
return ""
|
||||
|
||||
def extract_codes_Diagnosis(value):
|
||||
"""
|
||||
Extracts and formats diagnosis codes from a given input value, typically adhering to ICD (International Classification of Diseases) formats.
|
||||
|
||||
This function identifies patterns consistent with ICD diagnosis codes, which are generally a letter followed by numbers
|
||||
and may include additional alphanumeric characters and a period for further specification. The function handles null inputs
|
||||
gracefully, returning the input itself if null. If valid diagnosis codes are found, they are returned as a comma-separated
|
||||
string. If no codes are found, the function returns an empty string.
|
||||
|
||||
Parameters:
|
||||
value (str or NaN): The input value from which to extract diagnosis codes, which could be a string or NaN.
|
||||
|
||||
Returns:
|
||||
str: A comma-separated string of the extracted diagnosis codes, or an empty string if none are found.
|
||||
If the input is NaN, the function returns the input as is.
|
||||
"""
|
||||
if pd.isna(value):
|
||||
return value
|
||||
|
||||
pattern = r'\b[A-Z][0-9][A-Z0-9]{1,4}(\.[A-Z0-9]{1,4})?\b'
|
||||
matches = re.findall(pattern, value)
|
||||
if matches:
|
||||
return ', '.join(matches)
|
||||
else:
|
||||
return ""
|
||||
|
||||
def extract_codes_Revenue(value):
|
||||
"""
|
||||
Extracts revenue codes from a given input value. Revenue codes are typically numerical codes of three to four digits.
|
||||
|
||||
This function searches for patterns that match revenue codes in the input provided. It handles cases where the input might be null (NaN),
|
||||
returning the input itself in such cases. If any valid revenue codes are found, they are formatted into a comma-separated string.
|
||||
If no codes are found, the function returns an empty string, indicating the absence of recognizable codes in the input.
|
||||
|
||||
Parameters:
|
||||
value (str or NaN): The input value from which to extract revenue codes, which could be a string or NaN.
|
||||
|
||||
Returns:
|
||||
str: A comma-separated string of the extracted revenue codes, or an empty string if none are found.
|
||||
If the input is NaN, the function returns the input as is.
|
||||
"""
|
||||
if pd.isna(value):
|
||||
return value
|
||||
|
||||
pattern = r'\b\d{3,4}\b'
|
||||
matches = re.findall(pattern, value)
|
||||
if matches:
|
||||
return ', '.join(matches)
|
||||
else:
|
||||
return ""
|
||||
|
||||
def clean_code_column(df, column_name, extract_function):
|
||||
"""
|
||||
Applies a specified function to clean and extract codes from a specific column in a DataFrame.
|
||||
|
||||
This function takes a DataFrame and applies a given extraction function to each element of a specified column.
|
||||
The extraction function is expected to format or extract codes (e.g., CPT, ICD, Revenue codes) from the column's values.
|
||||
This cleaned and extracted data replaces the original values in the column. The function is flexible and can be used
|
||||
with any function designed to process and return cleaned values from a single input.
|
||||
|
||||
Parameters:
|
||||
df (pandas.DataFrame): The DataFrame containing the data to be cleaned.
|
||||
column_name (str): The name of the column to be cleaned.
|
||||
extract_function (function): The function to apply to each value in the specified column, which processes and returns the cleaned value.
|
||||
|
||||
Returns:
|
||||
pandas.DataFrame: The DataFrame with the specified column updated with cleaned and processed codes.
|
||||
"""
|
||||
df[column_name] = df[column_name].apply(extract_function)
|
||||
return df
|
||||
|
||||
|
||||
def clean_dates(df):
|
||||
for idx, row in df.iterrows():
|
||||
if row['LOB_PRICING_TERMS_EFFECTIVE_DATE'] == row['LOB_PRICING_TERMS_TERMINATION_DATE']:
|
||||
df.loc[idx, 'LOB_PRICING_TERMS_TERMINATION_DATE'] = 'N/A'
|
||||
|
||||
return df
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
|
||||
from itertools import groupby, count
|
||||
import re
|
||||
|
||||
def clean_newlines(contract_text):
|
||||
"""
|
||||
Cleans up isolated newlines in a contract text, converting them into spaces to ensure text continuity.
|
||||
|
||||
This function uses a regular expression to identify standalone newlines (those not part of a paragraph break, which
|
||||
typically consists of consecutive newlines) and replaces them with a single space. This process helps in preserving
|
||||
the flow of text and making it more readable and processable, particularly for text extraction and analysis tasks that
|
||||
might be sensitive to abrupt breaks in the text.
|
||||
|
||||
Parameters:
|
||||
contract_text (str): The text of the contract that may contain sporadic newlines.
|
||||
|
||||
Returns:
|
||||
str: The cleaned contract text with isolated newlines replaced by spaces.
|
||||
"""
|
||||
cleaned_text = re.sub(r'(?<!\n)\n(?!\n)', ' ', contract_text)
|
||||
return cleaned_text
|
||||
|
||||
|
||||
def split_text(text):
|
||||
"""
|
||||
Splits a contract text into a dictionary of pages based on a specific delimiter indicating the start of a page.
|
||||
|
||||
This function uses the delimiter 'Start of Page No. = ' to segment the input text into different pages.
|
||||
Each page is stored in a dictionary where the key is the page number (extracted as the first word after the delimiter)
|
||||
and the value is the corresponding page text. This facilitates easier access and processing of individual pages in the contract.
|
||||
|
||||
Parameters:
|
||||
text (str): The contract text containing multiple pages separated by the delimiter.
|
||||
|
||||
Returns:
|
||||
dict: A dictionary with page numbers as keys and the respective page texts as values.
|
||||
"""
|
||||
text_list = text.split('Start of Page No. = ')
|
||||
text_dict = {page.split()[0] : page for page in text_list}
|
||||
return text_dict
|
||||
|
||||
|
||||
def highlight_rates(text_dict):
|
||||
"""
|
||||
Highlights rate-related terms (percentages and dollar amounts) in the text of each page within a dictionary.
|
||||
|
||||
This function processes each page's text in the provided dictionary, identifying words that contain percentage signs ('%')
|
||||
or dollar signs ('$'). It wraps these words with '>>>' and '<<<' to highlight them. The modified text for each page
|
||||
is then updated in the dictionary, allowing for easier identification and processing of rate-related terms in subsequent analyses.
|
||||
|
||||
Parameters:
|
||||
text_dict (dict): A dictionary where keys are page numbers and values are the text content of those pages.
|
||||
|
||||
Returns:
|
||||
dict: The updated dictionary with rate-related terms highlighted in the text of each page.
|
||||
"""
|
||||
for page, text in text_dict.items():
|
||||
words = text.split()
|
||||
highlighted_words = []
|
||||
for word in words:
|
||||
if '%' in word or '$' in word:
|
||||
word = f'>>>{word}<<<'
|
||||
highlighted_words.append(word)
|
||||
text_dict[page] = ' '.join(highlighted_words)
|
||||
return text_dict
|
||||
|
||||
|
||||
def chunk_text(text_dict):
|
||||
"""
|
||||
Creates text chunks from a dictionary of page texts, focusing on pages with special characters (percentages and dollar amounts).
|
||||
|
||||
This function identifies pages that contain '%' or '$' signs and includes those pages along with their immediate neighbors
|
||||
(previous and next pages) to form chunks. The chunks are then grouped and concatenated into single text blocks for easier
|
||||
processing. Each chunk is stored in a new dictionary where the keys represent the range of pages included in the chunk.
|
||||
|
||||
Parameters:
|
||||
text_dict (dict): A dictionary where keys are page numbers (as strings) and values are the text content of those pages.
|
||||
|
||||
Returns:
|
||||
dict: A new dictionary where each key is a string representing the range of pages in a chunk, and each value is the concatenated text of those pages.
|
||||
"""
|
||||
special_pages = {int(page): text for page, text in text_dict.items() if (('%' in text) or ('$' in text) or ('percent' in text))}
|
||||
page_numbers = sorted(special_pages.keys())
|
||||
chunk_page_numbers = []
|
||||
for page in page_numbers:
|
||||
chunk_page_numbers.append(page-1)
|
||||
chunk_page_numbers.append(page)
|
||||
chunk_page_numbers.append(page+1)
|
||||
chunk_page_numbers = list(set(chunk_page_numbers))
|
||||
chunk_page_numbers = [page for page in chunk_page_numbers if str(page) in text_dict.keys()]
|
||||
chunks = [list(group) for key, group in groupby(chunk_page_numbers, lambda x, c=count(): x - next(c))]
|
||||
chunk_dict = {}
|
||||
for item in chunks:
|
||||
dict_key = f'{min(item)}-{max(item)}'
|
||||
text = "".join([text_dict[str(page)] for page in item])
|
||||
chunk_dict[dict_key] = text
|
||||
return chunk_dict
|
||||
|
||||
|
||||
def clean_billed_charges(contract_text):
|
||||
"""
|
||||
Cleans occurrences of 'billed charges' in a text by replacing them with '100% of billed charges' unless preceded by a percentage.
|
||||
|
||||
This function uses a regular expression to find all instances of the phrase 'billed charges'. For each match, it checks the preceding
|
||||
text (up to 15 characters before the match) for a percentage sign ('%'). If a percentage is found, it leaves the match unchanged.
|
||||
Otherwise, it replaces 'billed charges' with '100% of billed charges'.
|
||||
|
||||
Parameters:
|
||||
text (str): The input text to be cleaned.
|
||||
|
||||
Returns:
|
||||
str: The cleaned text with appropriate replacements made for 'billed charges'.
|
||||
"""
|
||||
|
||||
substrings = [
|
||||
"Physician's Billed Charges",
|
||||
"Provider's Billed Charges",
|
||||
"Allowable Billed Charges",
|
||||
"Hospital's Billed Charges",
|
||||
"Physician's Charges",
|
||||
"Provider's Charges",
|
||||
"Allowable Charges",
|
||||
"Hospital's Charges",
|
||||
"Billed Charges",
|
||||
"the rates set forth in this Exhibit",
|
||||
"the rutes set forth in this Exhibit",
|
||||
]
|
||||
max_substring_length = max([len(s) for s in substrings])
|
||||
|
||||
def find_substring_indices(contract_text, s):
|
||||
indices = []
|
||||
lower_contract_text = contract_text.lower()
|
||||
lower_s = s.lower()
|
||||
index = lower_contract_text.find(lower_s)
|
||||
while index != -1:
|
||||
indices.append(index)
|
||||
index = lower_contract_text.find(lower_s, index + 1)
|
||||
return indices
|
||||
|
||||
for s in substrings:
|
||||
indices = find_substring_indices(contract_text, s)
|
||||
index_adder = 0
|
||||
if indices:
|
||||
for i in indices:
|
||||
index = i+index_adder
|
||||
end_index = index + max_substring_length
|
||||
match_part = contract_text[index:end_index]
|
||||
previous = contract_text[max(0, index-30):index]
|
||||
if '%' not in previous:
|
||||
contract_text = contract_text[0:index] + f" 100% of {match_part}" + contract_text[end_index:]
|
||||
index_adder += 9
|
||||
return contract_text.replace(' ', ' ')
|
||||
@@ -0,0 +1,410 @@
|
||||
|
||||
|
||||
def GENERATE_PROMPT(bu_dict, td_dicts, page, field_name=None, values=None):
|
||||
if field_name and values:
|
||||
return f"""### PAGE START ### {page} ### PAGE END
|
||||
|
||||
Field: {field_name}
|
||||
Values: {values}
|
||||
|
||||
Listed above is a page of a contract, as well as the potential values for the {field_name} field.
|
||||
Your job is to identify which of these values is the most appropriate based on the page content.
|
||||
|
||||
Write ONLY the value that is most appropriate. Do not return any additional text beyond the value itself.
|
||||
"""
|
||||
else:
|
||||
return f"""### PAGE START ### {page} ### PAGE END
|
||||
|
||||
Service: {bu_dict['SERVICE']}
|
||||
Methodology: {bu_dict['FULL_METHODOLOGY']}
|
||||
|
||||
Listed above is a page of a contract, as well as a Service and Methodology found on this page.
|
||||
Your job is to identify which of the following entities this specific Service is associated with.
|
||||
|
||||
Here are the entities, represented as dictionary objects:
|
||||
|
||||
{td_dicts}
|
||||
|
||||
Write ONLY the number of the dictionary that is most associated with the Service and Methodology of interest. Do not return any additional text beyond the digit itself.
|
||||
"""
|
||||
|
||||
|
||||
def MERGE_SELECT_PROMPT(key, value, bu_dict, page):
|
||||
return f"""### PAGE START ### {page} ### PAGE END
|
||||
|
||||
Service: {bu_dict['SERVICE']}
|
||||
Methodology: {bu_dict['FULL_METHODOLOGY']}
|
||||
|
||||
Listed above is a page of a contract, as well as a specific Service and Methodology that may be found on this page.
|
||||
Your job is to identify which of the following {key}s this specific Service is associated with.
|
||||
|
||||
Here is a comma-separated list of options:
|
||||
|
||||
{value}
|
||||
|
||||
Write ONLY the value or values in the list that DIRECTLY applies to the Service and Methodology listed above. If none of the values apply, write 'N/A'.
|
||||
If the Service and Methodology are not found on the page, write the LAST item in the list. Do not write any other commentary or explanation.
|
||||
"""
|
||||
|
||||
|
||||
def BOTTOM_UP_PRIMARY(page, payer):
|
||||
return f"""
|
||||
### PAGE START ### {page} ### PAGE END
|
||||
|
||||
The preceding text is one page of a contract between Payer {payer} and a provider in their network. Your job is to extract attributes related to the reimbursement of different services and specialties.
|
||||
|
||||
The reimbursement values will be identified with >>> <<< indicators (e.g. >>>105%<<<). Make sure there is at least one dictionary object for EVERY reimbursement value seen (either % or $).
|
||||
|
||||
Some values are found in sections explicitly identified as examples - these must be omitted from output. Other values might be related to liability - these must also be excluded.
|
||||
|
||||
If any of the attributes are not found, return N/A. For all attributes, only write what is written on the page. Do not make up new phrases or words.
|
||||
|
||||
Return at least one json object for each combination of attributes seen. It is possible that a page will not have any attributes. When this is the case, return an empty list.
|
||||
|
||||
Here are the attributes to be included in each dictionary, and instructions on how to correctly answer:
|
||||
'SERVICE' : What is the Service that is being reimbursed? Use your best judgement and knowledge of the healthcare industry to answer. Use only exact text from the document. Do not leave this field N/A.
|
||||
'REIMBURSEMENT_FLAT_FEE' : If the listed reimbursement is dollar value, return only that dollar value. There can only be one answer for each object. If more than one rate is found, create additional objects for them.
|
||||
'REIMBURSEMENT_RATE' : If the listed reimbursement is a percent of something, return only that percent. There can only be one answer for each object. If more than one rate is found, create additional objects for them. Do NOT include multiple percentages in this value.
|
||||
'FULL_METHODOLOGY' : If the listed reimbursement is a percent of something, what is it the percent of? If the listed reimbursement is a direct dollar value, how is that dollar value paid (per diem, per unit, etc)? Write the full sentence in text describing the payment methodology.
|
||||
'REIMBURSEMENT_DATES' : If the listed reimbursement only applies to a specific date range, write that range here. Write only the exact text. Write only the range. If there is none, write 'N/A'.
|
||||
'REIMBURSEMENT_TIN' : If the listed reimbursement only applies to a specific TIN, write only the TIN here. If not, write 'N/A'
|
||||
|
||||
Note that only one of REIMBURSEMENT_FLAT_FEE or REIMBURSEMENT_RATE should be populated in each dictionary. Ensure all 4 keys are in the dictionary.
|
||||
|
||||
Only return the list, with no other commentary or explanation. Ensure you abide by proper JSON formatting.
|
||||
"""
|
||||
|
||||
|
||||
def BOTTOM_UP_LESSER(d, page):
|
||||
return f"""
|
||||
### PAGE START ### {page} ### PAGE END
|
||||
|
||||
The above text contains a number of reimbursement terms. For the rest of this request, focus only on the specific term listed below:
|
||||
Original_Dictionary:
|
||||
- Original_Service: {d['SERVICE']}
|
||||
- Original_Rate: {d['REIMBURSEMENT_RATE']}
|
||||
- Original_Flat_Fee: {d['REIMBURSEMENT_FLAT_FEE']}
|
||||
- Original_Methodology: {d['FULL_METHODOLOGY']}
|
||||
|
||||
Your job is to identify if there is any 'lesser of' language or 'greater of' language that pertains to the specific service and methodology above.
|
||||
|
||||
Note that a 'lesser of'/'greater of' statement may not be directly written for a given Service, but that does not necessarily mean that it doesn't apply.
|
||||
|
||||
Read and analyze the page, then populate two JSON dictionaries with the following fields (descriptions provided):
|
||||
'LESSER_OF_LANGUAGE_IND' : Write 'Y' if there is 'lesser of' language somewhere on the page that applies to the Service and Original_Methodology. Write 'N' if not.
|
||||
'GREATER_OF_LANGUAGE_IND' : Write 'Y' if there is 'greater of' language somewhere on the page that applies to the Service and Original_Methodology. Write 'N' if not.
|
||||
'FULL_METHODOLOGY' : If either of the above are 'Y', what is the reimbursement the lesser or greater of? Do not write the full sentence.
|
||||
'REIMBURSEMENT_RATE' : If the Lesser/Greater Of Methodology is a percent of something, write just the numeric percent value here. Only include percentages. If not, write N/A.
|
||||
'REIMBURSEMENT_FLAT_FEE' : If the Lesser/Greater Of Methodology is a dollar value, write just the numeric value here. Only include dollar values. If not, write N/A.
|
||||
|
||||
If either LESSER_OF_LANGUAGE_IND or GREATER_OF_LANGUAGE_IND is 'Y', then both dictionaries must be populated. One of the dictionaries will have REIMBURSEMENT_RATE, REIMBURSEMENT_FLAT_FEE, and FULL_METHODOLOGY values equal to the Original term.
|
||||
Fill the first dictionary with the values corresponding to the LEFT half of the Lesser/Greater Of Statement, and the second dictionary with the RIGHT half of the Lesser/Greater Of Statement.
|
||||
|
||||
If both LESSER_OF_LANGUAGE_IND and GREATER_OF_LANGUAGE_IND are 'N', return a list with just the original dictionary.
|
||||
|
||||
Only return the list of dictionaries, with no other commentary or explanation. Ensure you abide by proper JSON formatting.
|
||||
"""
|
||||
|
||||
def BOTTOM_UP_EXCEPT_ESC(d, page):
|
||||
return f"""### PAGE START ### {page} ### PAGE END
|
||||
|
||||
The above text contains a number of reimbursement terms. For the rest of this request, focus only on the specific term listed below:
|
||||
Service: {d['SERVICE']}
|
||||
Methodology: {d['FULL_METHODOLOGY']}
|
||||
|
||||
Your job is to identify if the listed reimbursement contains some sort of exception (like for certain codes, hospitals, or other situations) or an escalation
|
||||
|
||||
Read and analyze the page, then populate a JSON dictionary with the following fields (descriptions provided):
|
||||
REIMBURSEMENT_EXCEPTION_IND: If the listed reimbursement contains some sort of exception (like for certain codes, hospitals, or other situations), return 'Y'. Otherwise, 'N'
|
||||
REIMBURSEMENT_DESCRIBE_EXCEPTION: If the listed reimbursement contains some sort of exception, describe it here. Try to use exact words from the text only. If there is none, just write 'N/A'
|
||||
RATE_ESCALATOR_IND: Does the listed reimbursement include annual rate escalators for year-over-year rate increases? If yes, return 'Y'. Otherwise, 'N'.
|
||||
RATE_ESCALATOR_BASIS: If contract does include annual rate escalators, on what are they based? Do not be overly verbose in your answer.
|
||||
RATE_ESCALATOR_YEARLY_PERCENT_INCREASE: What is the percent value of increase or decrease year-over-year? If a decrease, return a negative percentage.
|
||||
|
||||
Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting.
|
||||
"""
|
||||
|
||||
def BOTTOM_UP_FS(d):
|
||||
return f"""Analyze the reimbursement terms listed below:
|
||||
Service: {d['SERVICE']}
|
||||
Methodology: {d['FULL_METHODOLOGY']}
|
||||
|
||||
Your job is to identify if the reimbursement is based on a Fee Schedule.
|
||||
|
||||
Read and analyze the Methodology, then populate a JSON dictionary with the following fields (descriptions provided):
|
||||
REIMBURSEMENT_FEE_SCHEDULE : If the Methodology is based on a Fee Schedule, what is the name of the fee schedule on which it is based? It could be something like Medicaid, Medicare, CMS, DRG, AWP, ASP, or similar. Do not answer with a complete sentence. If the Methodology is not based on a Fee Schedule, just return 'N/A'
|
||||
REIMBURSEMENT_FEE_SCHEDULE_VERSION : If the Methodology is based on a Fee Schedule, what specific version of the Fee Schedule is it based on. It could be something like 'Current', 'Prevailing', 'at the date of service', a specific year, or similar. If there is no answer, return 'N/A'
|
||||
|
||||
Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting.
|
||||
"""
|
||||
|
||||
def BOTTOM_UP_METHODOLOGY(d):
|
||||
return f"""Analyze the full methodology text listed below:
|
||||
{d['FULL_METHODOLOGY']}
|
||||
|
||||
Your job is to identify what the concise methodology is based on the full methodology.
|
||||
|
||||
Choose the closest option from the following:
|
||||
Allowed Amount, Allowable Charges, Fee Schedule, Medicare Fee Schedule, Medicare Allowed Amount, Medicaid Fee Schedule, Medicaid Allowed Amount, Provider's Charges, Billed Charges, Normal fee for such service, IME/DME Amount, Cost to Charge Ratio, Cost plus %, MSRP, Per Unit, Per Diem, Per Visit, Per Member, Per Member Per Month, Per Hour, Case Rate, Contracted Rate
|
||||
|
||||
For 'Allowed Amount' and 'Fee Schedule', use the more specific answer (Medicare/Medicaid) if applicable.
|
||||
For the 'Per ___' values, note that this list is not necessarily exhaustive. It could be anything like 'Per ___'.
|
||||
|
||||
Return only the answer, without using complete sentences. If there is nothing close to the above options in the full methodology text, simply return 'N/A'
|
||||
"""
|
||||
|
||||
|
||||
def BOTTOM_UP_CODES(d, page):
|
||||
return f"""### PAGE START ### {page} ### PAGE END
|
||||
|
||||
The above text contains a number of reimbursement terms. For the rest of this request, focus only on the specific term listed below:
|
||||
Service: {d['SERVICE']}
|
||||
Methodology: {d['FULL_METHODOLOGY']}
|
||||
|
||||
Your job is to identify codes associated with this specific term.
|
||||
|
||||
Read and analyze the page, then populate a JSON dictionary with the following code types if they are associated with the given Service and Methodology (descriptions provided):
|
||||
'REIMBURSEMENT_ADMITTYPE_CODES' : Single-digit codes that indicate the type of admission to a facility.
|
||||
'REIMBURSEMENT_DIAG_CODES' : Multi-digit alphanumeric codes related to diagnosis. They may contain a '.' after the first 3 characters. These may be labelled as ICD-10 or ICD-10-CM codes.
|
||||
'REIMBURSEMENT_GROUPER_CODES' : 3- or 4-digit codes that may be labelled DRG or or similar.
|
||||
'REIMBURSEMENT_GROUPER' : If there are grouper codes, what type are they? They could be MSDRG, APDRG, APRDRG, APC, APG, or EAPG. Choose only from these options.
|
||||
'REIMBURSEMENT_PLACEOFSERVICE_CODES' : 2-digit codes related to the place of service.
|
||||
'REIMBURSEMENT_PROC_CODES' : 5-digit numeric or alphanumeric codes. They may be labelled as CPT or HCPCS codes. These are related to specific procedures.
|
||||
'REIMBURSEMENT_REVENUE_CODES' : 3- or 4-digit numeric or alphanumeric codes related to revenue. They may be labelled as 'Rev' codes.
|
||||
'REIMBURSEMENT_STATUS_INDICATOR_CODES' : 1- or 2-digit alphanumeric codes indicating the payment status.
|
||||
|
||||
Return a value for each type of code - return N/A for any values not found. Multiple codes for each type may be found. In this case, return them all. They may also present as a range. If so, return the range.
|
||||
|
||||
Note that some PROC_CODES may have 'Modifiers' listed as well. These are 2-character strings. If Modifiers are present, include them as part of this field.
|
||||
|
||||
Ensure you ONLY return codes if they are associated directly with the Service listed. Codes referring to other services should not be included.
|
||||
|
||||
Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting.
|
||||
"""
|
||||
|
||||
|
||||
def TOP_DOWN_PRIMARY(page):
|
||||
return f"""### PAGE START ### {page} ### PAGE END
|
||||
|
||||
The preceding text is one page of a contract between a Payer and a provider in their network. Your job is to extract the names of certain entities listed in the contract and return a single dictionary with the values below:
|
||||
|
||||
Here are the attributes to be included in each dictionary, and instructions on how to correctly answer:
|
||||
'CONTRACT_LOB' : List any Line of Business mentioned on the page. Choose ONLY from the following: Commercial, Group, Marketplace, Marketplace/Exchange, Medicaid, Medicaid-Medicare, Medicare, Medicare Advantage, Medicare Supplemental, Veterans Affairs. If none of these are found on the page, simply write 'N/A' for this answer.
|
||||
'CONTRACT_PROGRAM' : List any Programs mentioned on the page. Some examples of Programs are: CHIP, CHIP-P, CHIP-Perinate, STAR, STAR+PLUS. Do your best to infer the program from this page.
|
||||
'CONTRACT_NETWORK' : List any Networks mentioned on the page. Networks are types of managed care organizations involved in the delivery of healthcare services. Some examples of Networks are: HMO, PPO, EMO, POS, FFS. Do your best to infer the network from this page. If it says anywhere that they are out of network then only return "OUT OF NETWORK" as your answer.
|
||||
'PRODUCT' : List any Products listed on the page. These are typically the brand name of a health plan. You may also use this field as a catch-all for entities that you are unsure of.
|
||||
'PROV_TYPE' : List any Provider Type listed on the page. This may be something like Inpatient, Outpatient, Ambulatory Surgery Center, DME, Home Health, Hospice, Skilled Nursing Facility, or similar.
|
||||
|
||||
If any of the attributes are not found, return N/A. If there are multiple answers for a given attribute, return the answers in a comma-separated list.
|
||||
|
||||
For all attributes, only write what is written on the page. Do not make up new phrases or words.
|
||||
|
||||
Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting.
|
||||
"""
|
||||
|
||||
|
||||
def LOB_CHECK(bu_dict, td_dicts, page):
|
||||
return f"""### PAGE START ### {page} ### PAGE END
|
||||
|
||||
Service: {bu_dict['SERVICE']}
|
||||
Methodology: {bu_dict['FULL_METHODOLOGY']}
|
||||
|
||||
Listed above is page of a contract, as well as a Service and Methodology found on this page.
|
||||
Your job is to identify which of the following entities this specific Service is associated with.
|
||||
|
||||
----Change Below----
|
||||
|
||||
Here are the entities, represented as dictionary objects:
|
||||
|
||||
{td_dicts}
|
||||
|
||||
Write ONLY the number of the dictionary that is most associated with the Service and Methodology of interest. Do not return any additional text beyond the digit itself.
|
||||
"""
|
||||
|
||||
|
||||
def TOP_DOWN_METAL_LEVEL(LOB, page):
|
||||
return f"""### PAGE START ### {page} ### PAGE END
|
||||
|
||||
The above page was identified as applying to a Commercial or Marketplace Line of Business.
|
||||
|
||||
What is the metal level(s) of the line of business. It might be Bronze, Silver, Gold, or Platinum. If multiple are found, return them all in a comma-separated list. If no metal level is found, return 'N/A'.
|
||||
|
||||
Only write what is written on the page. Do not make up new phrases or words.
|
||||
|
||||
ONLY return the metal level, with no other commentary or explanation.
|
||||
"""
|
||||
|
||||
|
||||
def TOP_DOWN_DATE(type_, d, page):
|
||||
return f"""### PAGE START ### {page} ### PAGE END
|
||||
|
||||
Above is a page of a contract. Identify the {type_} Date associated with the information below:
|
||||
|
||||
{d}
|
||||
|
||||
Only write what is written on the page. Do not make up new phrases or words. If no {type_} date is listed, return 'N/A'.
|
||||
|
||||
Ensure that you do not mistake EFFECTIVE date for TERMINATION date, or vice versa. They are not the same thing.
|
||||
The TERMINATION date will NOT be labelled as the EFFECTIVE date, and the EFFECTIVE date will NOT be labelled as the TERMINATION date. This is very important, so pay extra close attention.
|
||||
|
||||
ONLY return the answer, with no other commentary or explanation.
|
||||
"""
|
||||
|
||||
|
||||
def get_carveout_list():
|
||||
return ['Emergency Department/Emergency Room', 'Emergency Department', 'Emergency Room', 'Observation', 'Surgery', 'General Surgery',
|
||||
'Ambulatory Surgery', 'Intensive Care', 'Trauma', 'HIV', 'Human Immunodeficiency Virus', 'Major Joint Replacement', 'Transplant', 'OBGYN',
|
||||
'Obstetrician/Gynecologist', 'Obstetrician', 'Gynecologist', 'Opthalmology & Vision', 'Opthalmology', 'Vision', 'Never Events',
|
||||
'Medically Unnecessary', 'Medically Unnecessary Procedures', 'Not Medically Necessary', 'Not Medically Necessary Procedures',
|
||||
'Experimental', 'Investigational', 'Unlisted Codes', 'Durable Medical Equipment', 'DME', 'Prosthetics & Orthotics', 'Prosthetics',
|
||||
'Orthotics', 'Implants', 'Hearing Aids', 'Hearing', 'Anesthesia', 'Anesthesiology', 'Medical Pharmacy', 'Physician Administered Drugs',
|
||||
'Global', 'Bundled or Unbundled Codes', 'Bundled Codes', 'Unbundled Codes', 'Multiple Procedure Reductions', 'Second Surgery', 'Subsequent Surgeries',
|
||||
'Non-Behavioral Health Mid-Level Professionals', 'Physician Assistant', 'PA', 'Nurse Practicioner', 'NP', 'Non-Physician',
|
||||
'Non-Physician Health Professionals', 'Technical Component', 'Professional Component', 'Laboratory', 'Pathology', 'Lab', 'Path', 'Lab/Path',
|
||||
'Laboratory/Pathology', 'Radiology', 'Imaging', 'Radiology/Imaging', 'Mammography', 'Diagnostic', 'Pre-Admission Procedures',
|
||||
'Post-Discharge Procedures', 'Readmission', 'Status Indicators', 'Stop Loss', 'Emergency Medical', 'EMS', 'NICU', 'Neonatal Intensive Care Unit',
|
||||
'Vaccine for Children', 'VFC', 'Surgical Assistant', 'Physicians/Clinical Psychologists', 'Doctor of Nursing Practice', 'Osteopathic Medicine',
|
||||
'Clinical Psychology', 'Audiologist', 'Chiropractors', 'Registered Dietician', 'AUD', 'DC', 'RD', 'Board Certified Behavioral Analysis', 'Behavioral Analysis',
|
||||
'BCBA', 'Independent Licensures', 'Licensed Professional Counselor', 'Marriage and Family Therapist', 'Substance Abuse Counselor', 'Clinical Social Worker',
|
||||
'Behavioral Health Outpatient Clinic', 'Physical Therapist', 'Occupational Therapist', 'Speech Therapist', 'PT', 'OT', 'ST', 'Transportation',
|
||||
'Primary Care', 'Primary Care Behavioral Health', 'Behavioral Physician', 'Clinical Psychologist', 'Mid-Level Practicioner', 'Dentist', 'Dental',
|
||||
'Supplies and Devices', 'Supplies', 'Devices', 'Immunizations', 'Obstetrical Epidural', 'Pediatric Subspecialties', 'Orthopedic Surgery', 'All Other Specialists',
|
||||
'Specialty Care Physician', 'Pediatric Primary Care', 'Nurse Anesthetist', 'Early Periodic Screening, Diagnostic, and Treatment', 'EPSDT',
|
||||
'Ancillary', 'Oncology', 'Cancer', 'Inpatient Physical Rehabilitation', 'Inpatient Rehabilitation', 'Outpatient Rehabilitation', 'Rehabilitation', 'Non-Behavioral Health Rehabilitation',
|
||||
'Extracorporeal Shock Wave Lithotripsy', 'Cardiac', 'Special Care Unit', 'Skilled Nursing', 'Infusion', 'Specialty Care',
|
||||
'Specialty Care Physician', 'Organ Acquisition', 'Blood Products', 'Blood Products Outpatient', 'Blood Products Inpatient', 'High Cost Drugs', 'Sleep Studies',
|
||||
'NICU', 'Newborn Intensive Care Unit', 'Extracorporeal Membrane Oxygenation', 'Burns', 'Kyphoplasty', 'Cryosurgical Ablation of the Prostate',
|
||||
'Transurethral Thermal Ablation', 'TUMT', 'Transurethral Needle Ablation', 'TUNA', 'Hyperbaric Treatment', 'Clinic Visit', 'Boarder Baby',
|
||||
'Pediatric Intensive Care Unit', 'PICU', 'Psychiatric', 'Mental Health', 'Behavioral Health', 'Substance Abuse',
|
||||
'Behavioral Health and Substance Abuse', 'Sub-Acute Facility Care', 'Unrouped Inpatient', 'All Other Acute',
|
||||
'Neurology', 'Neurology Subspecialties', 'Automatic Implantable Cardioverter Defibrillator', 'Percutaneous Transluminal Coronary Angioplasty',
|
||||
'Non-Coronary Angioplasty', 'Cardiac Catheters', 'Cardiovascular Surgery', 'Cardiac Surgery', 'Cesarean Birth', 'Cesarean Section', 'C-Section',
|
||||
'Gamma-Knife Radio-Surgery Outpatient', 'DaVinci Robotic Assisted Surgery', 'Outpatient Electrophysiology with Ablation', 'Outpatient Electrophysiology',
|
||||
'Magnetic Resonance Image', 'MRI', 'Computed Tomography Scan', 'CT Scan', 'Radiation Therapy', 'Dialysis', 'Gastric Bypass', 'Lap Band',
|
||||
'Obesity', 'Laparoscopic Cholecystectomy', 'Lap Chole', 'Laparoscopic Hysterectomy', 'Laparoscopic Prostatectomy', 'Laparoscopic Hysteroscopy',
|
||||
'Treatment Room', 'Wound Care', 'Cardiac Computed Tomograpy & Angiography', 'Positron Emission Tomography Scan', 'PET Scan', 'Hematology']
|
||||
|
||||
|
||||
def prompt_contract_lob(page):
|
||||
return f"""### PAGE START ### {page} ### PAGE END
|
||||
|
||||
Extract all instances of the Line of Business (LOB) mentioned on the page.
|
||||
Identify each distinct LOB and return them as separate objects. Possible values include:
|
||||
- Commercial
|
||||
- Commercial/Group
|
||||
- MARKETPLACE
|
||||
- MEDICAID
|
||||
- MEDICARE
|
||||
- Marketplace/Exchange
|
||||
- Medicaid/Medicare
|
||||
- Medicare Advantage
|
||||
- Medicare Supplemental
|
||||
- Veterans Affairs (VA)
|
||||
- Civilian Health and Medical Program of the Department of Veterans Affairs (CHAMPVA)
|
||||
- Tricare (health care program for uniformed service members, retirees, and their families around the world)
|
||||
If no LOB is found, return 'N/A'.
|
||||
|
||||
IT IS NOT GOING TO BE THINGS LIKE HMO, PPO, CHIP, STAR, ETC
|
||||
It will not be anything related to a 'Plan', so do not return anything like that.
|
||||
Return the results. If there are multiple identified then return seperated by commas. Only return the extracted information and no other text or sentences or explanations. Only the final values. """
|
||||
|
||||
|
||||
def prompt_product(page):
|
||||
return f"""### PAGE START ### {page} ### PAGE END
|
||||
|
||||
Identify and list all instances of the Plan or Product names mentioned on the page. In the context of health insurance, 'Plan' or 'Product' names are typically brand names of health plans provided by insurers. These names often reflect the type of coverage offered, its scope, or the insurer's branding. Examples of the types of coverage include 'Medical', 'Prescription Drug', 'Mental/Behavioral Health (MH/BH)', 'Substance Abuse (SA)', 'Hearing', 'Dental', and 'Vision'.
|
||||
|
||||
To recognize these names, look for capitalized terms or phrases often preceded by words like 'plan', 'coverage', 'program', or directly followed by 'network'. These names can be unique to the insurer or widely recognized in the industry. For instance, names like 'HealthPlus Medical Plan', 'VisionCare Network', or 'DentalSelect Coverage' are typical examples.
|
||||
|
||||
Extract and list all identified plan or product names. If multiple names are found, return them separated by commas. If no specific plan name is found, return 'N/A'. Only return the extracted information and no other text or sentences or explanations. Only the final values.
|
||||
"""
|
||||
|
||||
def prompt_network_name(page):
|
||||
#MODIFY TO DICTS FOR MAPPING
|
||||
return f"""### PAGE START ### {page} ### PAGE END
|
||||
|
||||
Extract all network names mentioned on the page. A 'network name' in the context of health insurance refers to the type of managed care organization involved in the delivery of healthcare services. These names often indicate the structure or model of care delivery, which may define the relationships between insurers, healthcare providers, and insured individuals.
|
||||
|
||||
Key identifiers of network types include terms that suggest managed care arrangements such as 'Health Maintenance Organization (HMO)', 'Preferred Provider Organization (PPO)', 'Exclusive Provider Organization (EPO)', 'Point of Service (POS)', and 'Indemnity or Fee for Service (FFS)'. Other descriptors might include 'Open Access', 'Exclusive Provider', 'Integrated Delivery Network', and 'Tiered Network'.
|
||||
|
||||
Methodically scan the text for any terms that align with these descriptions. Return all found network names separated by commas if multiple. If no network names are found in the text, return 'N/A'. Only return the extracted information and no other text or sentences or explanations. Only the final values.
|
||||
"""
|
||||
|
||||
|
||||
def prompt_service_area(page):
|
||||
return f"""### PAGE START ### {page} ### PAGE END
|
||||
|
||||
Extract the type of service area mentioned on the page. A 'Service Area' refers to the specific geographic or demographic region type where the health plan provides coverage or services. This could include specific states, counties, cities, or distinctions between urban and rural areas.
|
||||
Select from the following options: Urban, Rural, State, County, City, ZIP, Address
|
||||
|
||||
Extract and return all found attributes separated by commas in a single string. If no specific service areas are mentioned in the text, return 'N/A'. only return the extracted information and no other text or sentences or explanations. Only the final values.
|
||||
"""
|
||||
|
||||
def prompt_effective_date(page):
|
||||
return f"""### PAGE START ### {page} ### PAGE END
|
||||
|
||||
Identify all instances of Effective Dates mentioned within the context of specific compensation exhibits or contracts.
|
||||
The date format should be MM/DD/YYYY.
|
||||
Return all dates seperated by commas. If no effective dates are found, return 'N/A'. only return the extracted information and no other text or sentences or explanations. Only the final values.'
|
||||
DO NOT REPEAT THE SAME DATE MULTIPLE TIMES.
|
||||
"""
|
||||
|
||||
def prompt_termination_date(page):
|
||||
return f"""### PAGE START ### {page} ### PAGE END
|
||||
|
||||
Extract all termination dates mentioned in the text, specifically those related to the end of a compensation period or agreement. Termination dates specify when a contract or policy provision ceases to be effective.
|
||||
|
||||
To identify termination dates, look for dates associated with terms like 'end', 'expiration', 'termination', or similar phrases that indicate the conclusion of an agreement. These dates are typically formatted in MM/DD/YYYY.
|
||||
|
||||
Carefully scan the text for these indications and extract any dates that conform to this format. List each date and return them separated by commas if multiple are found. If no termination dates are found, return 'N/A'. only return the extracted information and no other text or sentences or explanations. Only the final values.
|
||||
DO NOT REPEAT THE SAME DATE MULTIPLE TIMES.
|
||||
"""
|
||||
|
||||
def prompt_metal_level(page):
|
||||
return f"""### PAGE START ### {page} ### PAGE END
|
||||
|
||||
If the Line of Business is identified as Marketplace, extract all instances of metal levels such as Platinum, Gold, Silver, or Bronze.
|
||||
List each metal level. If no metal levels are found or if the LOB is not Marketplace, return 'N/A'. only return the extracted information and no other text or sentences or explanations. Only the final values seperated by commas."""
|
||||
|
||||
def prompt_claim_discount_ind(lob):
|
||||
return f"For this line of business: {lob}, return with a 'Y' or 'NO' only whether the claim for this LOB is discounted. Only return as 'Y' or 'NO'."
|
||||
|
||||
def prompt_claim_discount_percent_rate(lob):
|
||||
return f"For this LOB only: {lob}, extract the claim discount percent rate."
|
||||
|
||||
def prompt_claim_discount_start_date(lob):
|
||||
return f"For this LOB only: {lob}, extract the claim discount start date."
|
||||
|
||||
def prompt_claim_discount_termination_date(lob):
|
||||
return f"For this LOB only: {lob}, extract the claim discount termination date."
|
||||
|
||||
def prompt_premium_ind(lob):
|
||||
return f"""For this Line of Business{lob}, indicate with a 'Y' or 'N' only whether the premium for this LOB is subject to any adjustments. Return only 'Y' or 'N'."""
|
||||
|
||||
def prompt_premium_percent(lob):
|
||||
return f"""For this Line of Business{lob}, extract the percentage rate of premium adjustment if applicable. Provide the percentage as a numerical value only."""
|
||||
|
||||
def prompt_premium_start_date(lob):
|
||||
return f"""For this Line of Business{lob}, identify the start date of the premium adjustments. Return the date in MM/DD/YYYY format only."""
|
||||
|
||||
def prompt_premium_termination_date():
|
||||
return """For this Line of Business, determine the termination date of the premium adjustments. Provide the date in MM/DD/YYYY format only."""
|
||||
|
||||
def prompt_sequestration_ind():
|
||||
return """For this Line of Business, indicate with a 'Y' or 'N' whether there is any sequestration applied. Return only 'Y' or 'N'."""
|
||||
|
||||
def prompt_sequestration_rate():
|
||||
return """For this Line of Business, extract the rate of sequestration as a percentage. Return the rate as a numerical value only."""
|
||||
|
||||
def prompt_sequestration_start_date():
|
||||
return """For this Line of Business, determine the start date for sequestration. Return the date in MM/DD/YYYY format only."""
|
||||
|
||||
def prompt_sequestration_termination_date():
|
||||
return """For this Line of Business, identify the termination date for sequestration. Provide the date in MM/DD/YYYY format only."""
|
||||
|
||||
def prompt_penalties_ind():
|
||||
return """For this Line of Business, indicate with a 'Y' or 'N' only whether there are any penalties applied. Return only 'Y' or 'N'."""
|
||||
|
||||
def prompt_penalties_rate():
|
||||
return """For this Line of Business, extract the rate of any penalties applied as a percentage. Provide the rate as a numerical value only."""
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
|
||||
import re
|
||||
import ast
|
||||
|
||||
def clean_tables(text):
|
||||
def replace_colon(text):
|
||||
# Define a function to use in re.sub to check the context of the match
|
||||
def replacer(match):
|
||||
# Extract the character after ':' to check if it's '['
|
||||
following_text = match.group(1)
|
||||
if following_text.strip().startswith('['):
|
||||
return match.group(0) # Return the original match (':' and whatever follows)
|
||||
else:
|
||||
return '' + match.group(1) # Replace ':' with ';' and return the rest
|
||||
|
||||
# Use a regular expression to find ':' and the text that follows
|
||||
pattern = r':(\s*.)'
|
||||
replaced_text = re.sub(pattern, replacer, text)
|
||||
return replaced_text
|
||||
|
||||
def remove_unquoted_brackets(text):
|
||||
# Define a function to check the content between brackets
|
||||
def replacer(match):
|
||||
content = match.group(1)
|
||||
if "'" not in content and (len(content) > 0):
|
||||
return content # Return just the content without brackets
|
||||
else:
|
||||
return match.group(0) # Return the original match if it contains a quote
|
||||
|
||||
|
||||
# Use a regular expression to find brackets and the text within
|
||||
pattern = r'\[([^\[\]]*?)\]'
|
||||
cleaned_text = re.sub(pattern, replacer, text)
|
||||
return cleaned_text
|
||||
|
||||
# Clean tables
|
||||
text = text.strip('{}') # Remove brackets
|
||||
text = replace_colon(text)
|
||||
text = remove_unquoted_brackets(text)
|
||||
return text
|
||||
|
||||
def convert_to_dict(table_text):
|
||||
"""
|
||||
Converts a formatted table text into a dictionary where each key maps to a list of values.
|
||||
|
||||
This function processes a string representation of a table, extracting key-value pairs and converting them into a dictionary.
|
||||
Each key maps to a list of values parsed from the input string. The function also returns the number of elements in each list.
|
||||
|
||||
Parameters:
|
||||
table_text (str): The input string representing the table, formatted with keys and values.
|
||||
|
||||
Returns:
|
||||
tuple: A dictionary with keys mapping to lists of values, and an integer representing the number of elements in each list.
|
||||
"""
|
||||
table_text = clean_tables(table_text)
|
||||
# print(table_text, '\n')
|
||||
table_text_list = table_text.split(']')
|
||||
table_text_list = [item for item in table_text_list if len(item) > 0]
|
||||
|
||||
final_dict = {}
|
||||
num_elements = 0
|
||||
for key_value_text in table_text_list:
|
||||
key = key_value_text.split(':')[0].strip(', \'')
|
||||
try:
|
||||
value = ':'.join(key_value_text.split(':')[1:]).strip()+']'
|
||||
# print(value, '\n')
|
||||
if len(value) > 1:
|
||||
value_list = ast.literal_eval(value)
|
||||
final_dict[key] = value_list
|
||||
num_elements = len(value_list)
|
||||
except:
|
||||
value = ':'.join(key_value_text.split(':')[1:]).strip()+"']"
|
||||
# print(value, '\n')
|
||||
if len(value) > 1:
|
||||
value_list = ast.literal_eval(value)
|
||||
final_dict[key] = value_list
|
||||
num_elements = len(value_list)
|
||||
|
||||
return final_dict, num_elements
|
||||
|
||||
|
||||
def format_table(table_json, table_size):
|
||||
"""
|
||||
Formats a dictionary of table data into a string representation of the table.
|
||||
|
||||
This function takes a dictionary where each key maps to a list of values and converts it into a formatted string.
|
||||
The keys are used as headers, and each row is constructed by iterating through the lists up to the specified table size.
|
||||
The resulting string separates headers and values with ' : ' and includes newline characters to delineate rows.
|
||||
|
||||
Parameters:
|
||||
table_json (dict): The dictionary containing the table data, with keys as column headers and lists as column values.
|
||||
table_size (int): The number of rows in the table.
|
||||
|
||||
Returns:
|
||||
str: The formatted string representation of the table.
|
||||
"""
|
||||
table_text = ""
|
||||
keys = [key for key in table_json.keys()]
|
||||
table_text += ' : '.join(keys) + "\n"
|
||||
for i in range(table_size):
|
||||
for key in keys:
|
||||
if table_json[key][i]:
|
||||
#table_text += key + ': ' + table_json[key][i] + ', '
|
||||
table_text += table_json[key][i] + ' : '
|
||||
table_text += r'\n'
|
||||
return table_text
|
||||
|
||||
|
||||
def align_and_format_tables(text_dict):
|
||||
"""
|
||||
Aligns and formats tables within the text of each page in a dictionary.
|
||||
|
||||
This function searches for table text within the input text of each page, marked by 'Table Start' and 'Table End' tags.
|
||||
It extracts, converts, and formats these tables, then replaces the original table text with the formatted version while
|
||||
maintaining the alignment with surrounding text. The cleaned and formatted text for each page is then stored in a new dictionary.
|
||||
|
||||
Parameters:
|
||||
text_dict (dict): A dictionary where keys are page numbers and values are the text content of those pages, potentially containing tables.
|
||||
|
||||
Returns:
|
||||
dict: A new dictionary with aligned and formatted tables in the text for each page.
|
||||
"""
|
||||
aligned_text_dict = {}
|
||||
for key, text in text_dict.items():
|
||||
aligned_text = text
|
||||
if 'Table Start' in text:
|
||||
table_texts = re.findall(r'-------Table Start--------(.*?)-------Table End--------', text, re.DOTALL)
|
||||
for table_text in table_texts:
|
||||
# print(table_text, '\n')
|
||||
|
||||
try:
|
||||
# Extract pretable text
|
||||
pretable = table_text.split('{')[0].strip()
|
||||
# Extract and format table text
|
||||
#table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0].replace("'", '"') + "}"
|
||||
table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0] + "}"
|
||||
|
||||
table, table_size = convert_to_dict(table_only)
|
||||
table_formatted = format_table(table, table_size)
|
||||
except:
|
||||
table_formatted = table_text
|
||||
|
||||
# Align
|
||||
if text.count(pretable) == 2: # One table
|
||||
# Remove original table
|
||||
aligned_text = text.replace(table_text, "")
|
||||
# Align new table
|
||||
aligned_text = aligned_text.replace(pretable, pretable + table_formatted)
|
||||
|
||||
else:
|
||||
aligned_text = aligned_text.replace(table_text, pretable + ' ' + table_formatted)
|
||||
aligned_text_dict[key] = aligned_text.replace("-------Table Start--------", "").replace("-------Table End--------", "")
|
||||
else:
|
||||
aligned_text_dict[key] = text
|
||||
|
||||
return aligned_text_dict
|
||||
@@ -0,0 +1,52 @@
|
||||
|
||||
import utils
|
||||
import config
|
||||
|
||||
import os
|
||||
import pandas as pd
|
||||
import boto3
|
||||
|
||||
|
||||
|
||||
|
||||
s3 = boto3.Session(aws_access_key_id='ASIA6GBMBVWOJOOHVXUE',
|
||||
aws_secret_access_key='YsXnoYhY599uOifnQ+hZnqQJpWTyjhZ8XppJr43V',
|
||||
aws_session_token='IQoJb3JpZ2luX2VjEHcaCXVzLWVhc3QtMiJHMEUCIBdFMYRAL20l87z9xZJzlvOW8H7+jOIoI2f6zOkNtMdQAiEAomTpoulfRpLcPhzbNFQVR5atGD8xbM49/gKX0rNDZYgqgQMIYRAAGgw5NzUwNDk5NjA4NjAiDGoxXZfA4RDhWkpflCreAmgKA3i7fZYV6z/e5WejrcnSfffciNhUKpen98pi1gJT6xgB9Cy4k19DAkF8ab8dQcyTO80K20hd2QjzwaTXDDiIpXYJq1TRHiJVN+g0QKOx10l4KOo7g8hGPzD/QPmA80aTxC96PqRklnJkUGNKe9zfZ7nCujts/i/rs0oAeuRCCnEKsh3lOtj4YEhoQGsNlKUsx4Pfh2cTn/JTZ4hma0zO8HnVfPh2f4i4hpa8Ula/0arXZJrkJyHdbQV85w+lmjypILQwAy6kQUd57lURDXDF1uPtziKZ2WqkozpEdeblmUUAT24rXKBhv3m86oqN59pb1IiFoDE7IOmJgRqNbh58OhOwifijMRWYouvSFM8pcBDTD725UmTzNu5GKgGJE00Q4PyBu5K2YXX1rHFmOhr29wq11mPMBPVRTF/zMLjG8ZloHYQWtt8VZwso8WE7ezV7RyScz0SBwBvhOX0mMPSwqbUGOqYBhMdcVuF9yw0Norg5U7G8SDl2WmiYCh/Anfea87h/1KzPs6ZNphtuSLcaH9+C7hVx5DVAJUW7gI0xx5jhPgqHbcptDPkNWCL129URUMPFOmuRiPxyTl0Xl5jSeh0Mj+RJz81OCWaEQyTQYOTtNUir77f1KAm8y+ClFrkYFrk6uz6HyiYknmcxAokjdNMIOS/83O7GTUv1Vo6/fIXo2MKX7gxhuNUoNw=='
|
||||
).client('s3')
|
||||
|
||||
def get_s3_files(bucket_name, folder):
|
||||
paginator = s3.get_paginator('list_objects_v2')
|
||||
result = paginator.paginate(Bucket=bucket_name, Prefix=folder)
|
||||
files = set()
|
||||
|
||||
for page in result:
|
||||
if 'Contents' in page:
|
||||
for obj in page['Contents']:
|
||||
file_name = obj['Key']
|
||||
if file_name: # Ensure it's not a folder
|
||||
files.add(file_name)
|
||||
|
||||
return files
|
||||
|
||||
bucket_name = 'texas-children-files'
|
||||
folder = '510_text_files/'
|
||||
files_in_folder = get_s3_files(bucket_name, folder)
|
||||
|
||||
source_bucket='texas-children-files'
|
||||
for source_key in files_in_folder:
|
||||
filename = source_key.split('510_text_files/')[1]
|
||||
print(filename)
|
||||
|
||||
if filename not in os.listdir('data/texas_childrens'):
|
||||
try:
|
||||
response = s3.get_object(Bucket=source_bucket, Key=source_key)['Body']
|
||||
file_content = response.read().decode('utf-8')
|
||||
|
||||
with open(f'data/texas_childrens/{filename}', 'w') as file:
|
||||
file.write(file_content)
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
|
||||
"""
|
||||
This script was written to extract .txt files from pdf for adhoc client runs.
|
||||
Ensure you have permissions for API gateway, S3, Lambda function and SQS to execute this (DEVELOPER & above roles in DEV & UAT for Doczy should suffice)
|
||||
If you have Analyst / Test roles and need the permissions to be elevated or policies to be updated then please ask Sannan Iqbal to make the changes.
|
||||
|
||||
The flow to execute the pipeline is:
|
||||
Create batch using api req
|
||||
Add files to the newly created batch along with the batch id as the tag
|
||||
Construct payload to trigger pipeline
|
||||
Post request to trigger pipeline with payload
|
||||
The output text file can be found in the client_bucket/contract_text_file/batch_123456
|
||||
And the final LLM parsed outputs can be found in client_bucket/final_output/batch_123456
|
||||
"""
|
||||
|
||||
|
||||
|
||||
|
||||
import os
|
||||
import boto3
|
||||
import requests
|
||||
import json
|
||||
import boto3
|
||||
from datetime import datetime
|
||||
import os
|
||||
from botocore.auth import SigV4Auth
|
||||
from botocore.awsrequest import AWSRequest
|
||||
from botocore.credentials import get_credentials
|
||||
from botocore.session import Session
|
||||
import time
|
||||
|
||||
# Function to upload files to S3
|
||||
def upload_files_to_s3(directory, bucket, batch_id):
|
||||
contract_list = []
|
||||
for filename in os.listdir(directory):
|
||||
if filename.endswith('.pdf'):
|
||||
file_path = os.path.join(directory, filename)
|
||||
s3_key = f'contracts-landing-zone/{batch_id}/{filename}'
|
||||
|
||||
# Upload file to S3
|
||||
s3_client.upload_file(file_path, bucket, s3_key)
|
||||
print(f'Uploaded {filename} to S3 bucket\n')
|
||||
|
||||
# Add tags to the uploaded file
|
||||
s3_client.put_object_tagging(
|
||||
Bucket=bucket,
|
||||
Key=s3_key,
|
||||
Tagging={
|
||||
'TagSet': [
|
||||
{
|
||||
'Key': 'BatchId',
|
||||
'Value': batch_id
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
print(f'Added tags to {filename}\n')
|
||||
# Add file details to contract list
|
||||
# For now we can leave it as is since only A, C have been operationalized
|
||||
contract_list.append({
|
||||
"contract_name": filename,
|
||||
"groups": [
|
||||
"A", # This needs to be dynamic in UI 1 based on what group has been selected
|
||||
"C"
|
||||
],
|
||||
"contract_source_path": s3_key
|
||||
})
|
||||
return contract_list
|
||||
|
||||
|
||||
|
||||
|
||||
def create_batch(client_bucket, create_batch_url):
|
||||
myobj = { "client-bucket-name": client_bucket }
|
||||
|
||||
# Call create batch API endpoint
|
||||
response = requests.post(create_batch_url, json = myobj)
|
||||
if response.status_code >= 200 and response.status_code < 300:
|
||||
try:
|
||||
new_batch_id = json.loads(json.loads(response.text)['body'])['batch_id']
|
||||
landing_zone = json.loads(json.loads(response.text)['body'])['landing_zone']
|
||||
except:
|
||||
print(myobj)
|
||||
print(response.text)
|
||||
new_batch_id = 'failed_cases'
|
||||
landing_zone = 'contracts_landing_zone'
|
||||
else:
|
||||
print(response.text)
|
||||
new_batch_id = 'failed_cases'
|
||||
landing_zone = 'contracts_landing_zone'
|
||||
return new_batch_id, landing_zone
|
||||
|
||||
|
||||
|
||||
|
||||
def list_filtered_files(bucket_name, prefix, start_date, end_date, profile_name):
|
||||
"""
|
||||
List all files in an S3 bucket filtered by date range.
|
||||
|
||||
Parameters:
|
||||
- bucket_name: str, the name of the S3 bucket
|
||||
- prefix: str, the prefix (folder) in the S3 bucket
|
||||
- start_date: str, the start date in ISO 8601 format
|
||||
- end_date: str, the end date in ISO 8601 format
|
||||
- profile_name: str, the AWS profile name
|
||||
|
||||
Returns:
|
||||
- list of str: the keys of the filtered files
|
||||
"""
|
||||
# Parse the dates
|
||||
start_date = datetime.fromisoformat(start_date.replace('Z', '+00:00'))
|
||||
end_date = datetime.fromisoformat(end_date.replace('Z', '+00:00'))
|
||||
|
||||
# Initialize a session using the specified profile
|
||||
session = boto3.Session(profile_name=profile_name)
|
||||
s3_client = session.client('s3')
|
||||
|
||||
# List objects in the bucket with the specified prefix
|
||||
paginator = s3_client.get_paginator('list_objects_v2')
|
||||
page_iterator = paginator.paginate(Bucket=bucket_name, Prefix=prefix)
|
||||
|
||||
# Filter files by date
|
||||
filtered_files = []
|
||||
for page in page_iterator:
|
||||
if 'Contents' in page:
|
||||
for obj in page['Contents']:
|
||||
last_modified = obj['LastModified']
|
||||
if start_date <= last_modified <= end_date:
|
||||
filtered_files.append(obj['Key'])
|
||||
|
||||
return filtered_files
|
||||
|
||||
|
||||
|
||||
def download_files(bucket_name, file_keys, profile_name, local_directory):
|
||||
"""
|
||||
Download files from an S3 bucket.
|
||||
|
||||
Parameters:
|
||||
- bucket_name: str, the name of the S3 bucket
|
||||
- file_keys: list of str, the keys of the files to download
|
||||
- profile_name: str, the AWS profile name
|
||||
- local_directory: str, the local directory to download files to
|
||||
"""
|
||||
# Initialize a session using the specified profile
|
||||
session = boto3.Session(profile_name=profile_name)
|
||||
s3_client = session.client('s3')
|
||||
|
||||
# Ensure the local directory exists
|
||||
if not os.path.exists(local_directory):
|
||||
os.makedirs(local_directory)
|
||||
|
||||
# Download each file
|
||||
for key in file_keys:
|
||||
file_name = os.path.basename(key) # Get only the file name from the key
|
||||
local_file_path = os.path.join(local_directory, file_name)
|
||||
print(f"Downloading {key} to {local_file_path}")
|
||||
s3_client.download_file(bucket_name, key, local_file_path)
|
||||
|
||||
print("Download completed.")
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Define your variables
|
||||
s3_bucket = 'doczyai-use2-u-cn1-s3-textract-processing-001'
|
||||
# batch_id = 'batch_100524101551'
|
||||
client_name = 'Priority Health'
|
||||
username = 'ADHOC USER'
|
||||
|
||||
# These endpoints are in UAT, please change them to DEV if there are access issues with UAT
|
||||
api_endpoint = 'https://29gm8cek03.execute-api.us-east-2.amazonaws.com/dev/trigger-pipeline'
|
||||
create_batch_url = "https://29gm8cek03.execute-api.us-east-2.amazonaws.com/dev/create-batch"
|
||||
|
||||
|
||||
|
||||
session = boto3.Session(profile_name='temp_cred') # Change the profile name to the one you have in your .aws/credentials file
|
||||
s3_client = session.client('s3')
|
||||
|
||||
|
||||
|
||||
# Use current directory as PDF directory
|
||||
pdf_directory = "C:\\Doczy\\Priority Health\\For_Textract\\For_Textract\\UAT- test East Paris Surgical Center copy"
|
||||
|
||||
|
||||
# Create batch
|
||||
batch_id, landing_zone = create_batch(s3_bucket, create_batch_url)
|
||||
print(f"Batch ID: {batch_id}")
|
||||
print(f"Landing Zone: {landing_zone}")
|
||||
# batch_id = 'batch_110624213433'
|
||||
# landing_zone = 'contracts_landing_zone/batch_110624213433/'
|
||||
|
||||
if batch_id == 'failed_cases':
|
||||
print("Batch creation failed. Exiting...")
|
||||
exit()
|
||||
# Upload files and get contract list
|
||||
contract_list = upload_files_to_s3(pdf_directory, s3_bucket, batch_id)
|
||||
|
||||
# Create JSON object
|
||||
data = {
|
||||
"s3_bucket": s3_bucket,
|
||||
"batch_id": batch_id,
|
||||
"client_name": client_name,
|
||||
"username": username,
|
||||
"contract_list": contract_list
|
||||
}
|
||||
|
||||
# Make POST request to API
|
||||
response = requests.post(api_endpoint, json=data)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# Print response
|
||||
print(response.status_code)
|
||||
print(response.json())
|
||||
|
||||
##################
|
||||
# Get text files
|
||||
|
||||
prefix = f'contract-text-file/{batch_id}/' # if you have a specific prefix (folder) in your bucket
|
||||
|
||||
# These dates are to filter the contracts in case there are older contracts in the same batch
|
||||
start_date = '2024-06-04T00:00:00Z' # ISO 8601 format
|
||||
end_date = '2024-06-14T23:59:59Z' # ISO 8601 format
|
||||
profile_name = 'temp_cred'
|
||||
local_directory = 'C:\\Doczy\\Priority Health\\For_Textract\\For_Textract\\text otuput' # local directory to save files
|
||||
|
||||
# List filtered files
|
||||
time.sleep(30) # Waiting for the text files to be generated, this may take longer and files may not be available after 30 seconds sometimes
|
||||
filtered_files = list_filtered_files(s3_bucket, prefix, start_date, end_date, profile_name)
|
||||
print(f"Filtered files: {len(filtered_files)}")
|
||||
|
||||
# Download files
|
||||
download_files(s3_bucket, filtered_files, profile_name, local_directory)
|
||||
@@ -0,0 +1,124 @@
|
||||
|
||||
import prompts
|
||||
import claude_funcs
|
||||
import utils
|
||||
|
||||
import json
|
||||
|
||||
def run_top_down(filename, text_dict):
|
||||
"""
|
||||
Executes the Top Down processing strategy on a provided dictionary of text pages, extracting structured data based on specified prompts.
|
||||
|
||||
The function first filters and processes each page of text that is correctly numbered, using a primary analysis prompt. The results
|
||||
from this analysis are then parsed into dictionaries, tagged with their respective page numbers and filename, and collected.
|
||||
After the initial processing, a secondary Top Down analysis is conducted to refine and possibly expand the extracted data.
|
||||
|
||||
Parameters:
|
||||
filename (str): The filename associated with the text, used to tag the output data.
|
||||
text_dict (dict): A dictionary where keys are page numbers and values are the text on those pages.
|
||||
|
||||
Returns:
|
||||
list of dict: A list of dictionaries where each dictionary contains structured results from both primary and secondary
|
||||
Top Down analyses, associated with a specific page and the overall document.
|
||||
"""
|
||||
|
||||
all_results = []
|
||||
|
||||
# Primary
|
||||
for page_num, page_text in text_dict.items():
|
||||
if not page_num.isdigit():
|
||||
continue
|
||||
prompt = prompts.TOP_DOWN_PRIMARY(page_text)
|
||||
answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000)
|
||||
answer_dict = json.loads(answer)
|
||||
answer_dict.update({'page_num' : page_num, 'Filename' : filename})
|
||||
#answer_dicts = dict_operations.primary_string_to_dict({page_num : answer}, filename) # Convert to dictionaries
|
||||
all_results.append(answer_dict)
|
||||
|
||||
# td_primary = [i for d in all_results for i in d] # Consolidate to one list
|
||||
# print(td_primary)
|
||||
|
||||
# Secondaries
|
||||
td_final = top_down_secondary(all_results, text_dict)
|
||||
|
||||
return td_final # List of list of dictionaries
|
||||
|
||||
|
||||
def top_down_secondary(td_results, text_dict):
|
||||
"""
|
||||
Conducts secondary processing on results obtained from the primary Top Down analysis of document text, enhancing detail and accuracy.
|
||||
|
||||
This function iteratively enhances each dictionary result from the initial analysis by adding or refining information related to:
|
||||
- Market metal levels by running a specific function to determine the contract marketplace metal level.
|
||||
- Effective dates by analyzing the context around specific keywords in the text and extracting these dates.
|
||||
- Termination dates, similarly extracted based on the presence and context of specific keywords.
|
||||
|
||||
Parameters:
|
||||
td_results (list of dict): Initial dictionaries from the primary Top Down processing containing basic extracted data.
|
||||
text_dict (dict): Dictionary keyed by page numbers, providing the full text for corresponding analysis and extraction tasks.
|
||||
|
||||
Returns:
|
||||
list of dict: Updated list of dictionaries, each enhanced with additional detailed attributes like metal level and date specifics.
|
||||
"""
|
||||
updated_dicts = []
|
||||
for d in td_results:
|
||||
# Run Metal Level
|
||||
d['CONTRACT_MARKETPLACE_METAL_LEVEL'] = run_top_down_metal_level(d, text_dict[d['page_num']])
|
||||
|
||||
# Dates
|
||||
# d['LOB_PRICING_TERMS_EFFECTIVE_DATE'] = run_top_down_date('EFFECTIVE' , d, text_dict[d['page_num']])
|
||||
|
||||
# # if auto_renewal == N: else: 'N/A'
|
||||
# d['LOB_PRICING_TERMS_TERMINATION_DATE'] = run_top_down_date('TERMINATION', d, text_dict[d['page_num']])
|
||||
|
||||
updated_dicts.append(d)
|
||||
return updated_dicts
|
||||
|
||||
|
||||
|
||||
def run_top_down_metal_level(d, page):
|
||||
"""
|
||||
Identifies and extracts the metal level of a contract within a specific line of business (LOB) from the provided page text.
|
||||
|
||||
This function determines whether the contract's LOB is associated with marketplace or commercial sectors by examining the
|
||||
'CONTRACT_LOB' key in the dictionary. If relevant, it constructs and sends a specific prompt to a language model to extract the
|
||||
metal level (e.g., Bronze, Silver, Gold, Platinum). If the LOB is not relevant, it directly assigns 'N/A'.
|
||||
|
||||
Parameters:
|
||||
d (dict): A dictionary containing extracted information from primary Top Down analysis, specifically the 'CONTRACT_LOB' key.
|
||||
page (str): The text content of the page that is being analyzed for metal level information.
|
||||
|
||||
Returns:
|
||||
str: The metal level of the contract as determined by the analysis, or 'N/A' if the contract LOB is not applicable.
|
||||
"""
|
||||
|
||||
if 'MARKETPLACE' in str(d['CONTRACT_LOB']).upper() or 'COMMERCIAL' in str(d['CONTRACT_LOB']).upper():
|
||||
prompt = prompts.TOP_DOWN_METAL_LEVEL(d['CONTRACT_LOB'], page)
|
||||
answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000)
|
||||
else:
|
||||
answer = 'N/A'
|
||||
return answer
|
||||
|
||||
|
||||
|
||||
def run_top_down_date(type_, d, page):
|
||||
""" DEPRECATED
|
||||
Extracts specific date-related information from a page of text using a Top Down processing approach.
|
||||
|
||||
This function tailors the extraction to focus on either 'EFFECTIVE' or 'TERMINATION' dates by constructing
|
||||
a prompt that directs a language model to search for and interpret date information relevant to the provided type.
|
||||
It formats the input data to fit the processing needs, sends it along with the page text to the model, and captures
|
||||
the model's response.
|
||||
|
||||
Parameters:
|
||||
type_ (str): Specifies the type of date to extract, 'EFFECTIVE' or 'TERMINATION'.
|
||||
d (dict): A dictionary containing preliminary data extracted from the text, which may include metadata like filename or page number.
|
||||
page (str): The text content of the page from which to extract the date.
|
||||
|
||||
Returns:
|
||||
str: The extracted date as a string, based on the model's interpretation of the input prompt and text context.
|
||||
"""
|
||||
formatted_d = utils.format_td_check([d], ['Filename', 'page_num'])
|
||||
prompt = prompts.TOP_DOWN_DATE('EFFECTIVE', formatted_d, page)
|
||||
answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000)
|
||||
return answer
|
||||
@@ -0,0 +1,152 @@
|
||||
|
||||
import os
|
||||
import re
|
||||
import pandas as pd
|
||||
import shutil
|
||||
|
||||
import config
|
||||
|
||||
def read_local(file_path):
|
||||
# Check if the file is a text file
|
||||
if os.path.isfile(file_path) and file_path.endswith('.txt'):
|
||||
try:
|
||||
# First attempt to open the file with UTF-8 encoding
|
||||
with open(file_path, 'r', encoding='utf-8') as file:
|
||||
file_contents = file.read()
|
||||
return file_contents
|
||||
except UnicodeDecodeError:
|
||||
# If UTF-8 fails, try reading the file with ANSI encoding
|
||||
try:
|
||||
with open(file_path, 'r', encoding='cp1252') as file:
|
||||
file_contents = file.read()
|
||||
return file_contents
|
||||
except UnicodeDecodeError:
|
||||
# If ANSI also fails, log an error message or handle it accordingly
|
||||
print(f"Failed to decode {file_path} with UTF-8 and cp1252 encodings.")
|
||||
|
||||
def read_s3():
|
||||
s3_client = config.S3_CLIENT
|
||||
objects = s3_client.list_objects_v2(Bucket=config.BUCKET, Prefix=config.PREFIX)
|
||||
file_list = []
|
||||
for obj in objects['Contents']:
|
||||
if not obj['Key'].endswith('/'):
|
||||
file_list.append(obj['Key'])
|
||||
contract_list = sorted(file_list)
|
||||
files = {}
|
||||
for contract in contract_list:
|
||||
data = s3_client.get_object(Bucket=config.BUCKET, Key=contract)
|
||||
contents = data['Body'].read()
|
||||
context = contents.decode('utf-8')
|
||||
path, filename = os.path.split(contract)
|
||||
files[filename] = context
|
||||
return files
|
||||
|
||||
def read_input(path=config.LOCAL_PATH, mode=config.READ_MODE):
|
||||
if mode == '_LOCAL_':
|
||||
files = {}
|
||||
for file in os.listdir(path):
|
||||
full_path = os.path.join(path, file)
|
||||
file_text = read_local(full_path)
|
||||
files[file] = file_text
|
||||
return files
|
||||
elif mode == '_S3_':
|
||||
return read_s3()
|
||||
|
||||
def consolidate_individual(input_folder='results', output_folder='output'):
|
||||
dfs = []
|
||||
for filename in os.listdir(input_folder):
|
||||
if filename.endswith('.csv'):
|
||||
filepath = os.path.join(input_folder, filename)
|
||||
dfs.append(pd.read_csv(filepath))
|
||||
|
||||
# Remove temp folder
|
||||
if input_folder=='temp' and os.path.exists(input_folder):
|
||||
shutil.rmtree(input_folder)
|
||||
|
||||
consolidated_df = pd.concat(dfs, ignore_index=True)
|
||||
|
||||
# Write output
|
||||
version = 1
|
||||
existing_files = [filename for filename in os.listdir(output_folder) if filename.startswith(f'consolidated_results_{config.TODAY}')]
|
||||
if existing_files:
|
||||
versions = [int(file.split('_v')[1].split('.')[0]) for file in existing_files if '_v' in file]
|
||||
if versions:
|
||||
version = max(versions) + 1
|
||||
filename = f'consolidated_results_{config.TODAY}_v{version}.csv'
|
||||
consolidated_df.to_csv(os.path.join(output_folder, filename))
|
||||
|
||||
|
||||
def preprocess_text_file(file_path):
|
||||
# Attempt to open the file with UTF-8 encoding first
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as file:
|
||||
text = file.read()
|
||||
except UnicodeDecodeError:
|
||||
# If UTF-8 fails, try reading the file with ANSI (cp1252) encoding
|
||||
try:
|
||||
with open(file_path, 'r', encoding='cp1252') as file:
|
||||
text = file.read()
|
||||
except UnicodeDecodeError:
|
||||
# If ANSI also fails, log an error message or handle it accordingly
|
||||
print(f"Failed to decode {file_path} with UTF-8 and cp1252 encodings.")
|
||||
return []
|
||||
|
||||
# Split the text into pages based on a specific marker
|
||||
pages = re.split(r'Start of Page No\. = \d+', text)
|
||||
return pages
|
||||
|
||||
|
||||
def format_td_check(td_dicts, dont_include_list):
|
||||
final_str = ""
|
||||
dict_count = 1
|
||||
for td_dict in td_dicts:
|
||||
final_str += str(dict_count) + '. '
|
||||
for k in td_dict.keys():
|
||||
if k not in dont_include_list:
|
||||
final_str += k + ': ' + td_dict[k] + ', '
|
||||
final_str += '\n'
|
||||
dict_count += 1
|
||||
return final_str
|
||||
|
||||
|
||||
def consolidate_csvs(output_dir=config.CONSOLIDATED_OUTPUT_DIRECTORY, output_file=config.OUTPUT_CSV_PATH):
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
df_list = []
|
||||
# Walk through each folder in the output directory
|
||||
for root, dirs, files in os.walk(output_dir):
|
||||
for dir_name in dirs:
|
||||
dir_path = os.path.join(root, dir_name)
|
||||
file_path = os.path.join(dir_path, config.PROCESSED_RESULTS_NAME)
|
||||
if os.path.isfile(file_path):
|
||||
try:
|
||||
df = pd.read_csv(file_path)
|
||||
df_list.append(df)
|
||||
except:
|
||||
pass
|
||||
|
||||
concatenated_df = pd.concat(df_list, ignore_index=True)
|
||||
concatenated_df.to_csv(os.path.join(output_dir, output_file), index=False)
|
||||
print(f"All CSV files have been consolidated into {output_file}")
|
||||
|
||||
|
||||
|
||||
def contains_reimbursement(text, page):
|
||||
if isinstance(text, dict):
|
||||
return page.isdigit() and ('%' in text[page] or '$' in text[page] or 'percent' in text[page].lower())
|
||||
elif isinstance(text, str):
|
||||
return ('%' in text or '$' in text or 'percent' in text.lower())
|
||||
else:
|
||||
print("contains_reimbursement - Invalid data type")
|
||||
|
||||
|
||||
|
||||
def filter_already_processed(input_dict):
|
||||
already_processed = []
|
||||
for folder_name in os.listdir(config.OUTPUT_DIRECTORY):
|
||||
if config.PROCESSED_RESULTS_NAME in os.listdir(os.path.join(config.OUTPUT_DIRECTORY, folder_name)):
|
||||
already_processed.append(folder_name+'.txt')
|
||||
|
||||
input_dict = {key : input_dict[key] for key in input_dict.keys() if key not in already_processed}
|
||||
return input_dict
|
||||
|
||||
Reference in New Issue
Block a user