jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in pricing_before_carveouts_priorityhealth (pull request #209)

Browse Source 
Pricing before carveouts priorityhealth

Approved-by: Katon Minhas
This commit is contained in:
Michael McGuinness
2024-09-30 14:46:19 +00:00
parent f6623d9990 7772d1833a
commit 9e632e7fb9
14 changed files with 304 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fieldExtraction/.envrc
+7
View File
@@ -0,0 +1,7 @@
dotenv_if_exists .env
require AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN
require LOCAL_PATH
require BUCKET BUCKET_PREFIX
require READ_MODE MAX_WORKERS
require CLIENT
fieldExtraction/src/bottom_up_funcs.py
+7 -2
View File
@@ -1,5 +1,5 @@
import dict_operations
import postprocessing_funcs
import postprocess_funcs
import prompts
import config
import claude_funcs
@@ -27,13 +27,18 @@ def run_bottom_up(filename, text_dict):
# Bottom Up Primary - SERVICE, REIMBURSEMENT_FLAT_FEE, REIMBURSEMENT_RATE, METHODOLOGY
answer_strings = run_bottom_up_primary(text_dict, 8000)
answer_dicts = dict_operations.primary_string_to_dict(answer_strings, filename)
answer_dicts_filtered = postprocessing_funcs.filter_service_column(answer_dicts)
answer_dicts_filtered = postprocess_funcs.filter_service_column(answer_dicts)
# Bottom Up Secondary
results_dicts = run_bottom_up_secondary(answer_dicts_filtered, text_dict, 8000)
# Add _PG columns
results_dicts = postprocess_funcs.add_PG_columns(results_dicts)
# Add Filename
for d in results_dicts:
d["Filename"] = filename
return results_dicts # List of dictionaries
fieldExtraction/src/categorization.py
+45
View File
@@ -0,0 +1,45 @@
import os
import utils
import preprocess
import table_funcs
def contains_reimbursement(text_dict, page):
return (
"%" in text_dict[page]
or "$" in text_dict[page]
or "percent" in text_dict[page].lower()
)
directory_path = (
"C:\\Users\\kminhas\\Documents\\Doczy\\doczy.ai\\data\\centene_healthnet"
)
# input_dict = utils.read_input(path=folder_path)
all_files = {}
reimbursement_count = 0
total_count = 0
# Walk through all directories and files in the directory
for root, dirs, files in os.walk(directory_path):
for file in files:
if file.endswith(".txt"):
# Construct full file path
file_path = os.path.join(root, file)
print(file_path)
try:
with open(file_path, "r", encoding="utf-8") as file:
contract_text = file.read()
if (
"%" in contract_text
or "$" in contract_text
or "percent" in contract_text.lower()
):
reimbursement_count += 1
total_count += 1
except:
continue
print(
f"For Centene HealthNet, {reimbursement_count}/{total_count} files contain potential reimbursement terms."
)
fieldExtraction/src/columns.py
+104
View File
@@ -0,0 +1,104 @@
VALID_COLUMNS_TEMPLATE = {
"base": [
"Filename",
"page_num",
"SERVICE",
"REIMBURSEMENT_FLAT_FEE",
"REIMBURSEMENT_RATE",
"FULL_METHODOLOGY",
"REIMBURSEMENT_METHODOLOGY",
"REIMBURSEMENT_FEE_SCHEDULE",
"REIMBURSEMENT_FEE_SCHEDULE_VERSION",
"LESSER_OF_LANGUAGE_IND",
"GREATER_OF_LANGUAGE_IND",
"CONTRACT_LOB",
"CONTRACT_NETWORK",
"PRODUCT",
"CONTRACT_PROGRAM",
"CONTRACT_MARKETPLACE_METAL_LEVEL",
"PROV_TYPE",
"REIMBURSEMENT_PROC_CODES",
"REIMBURSEMENT_PROC_CODE_MODIFIERS",
"REIMBURSEMENT_REVENUE_CODES",
"REIMBURSEMENT_STATUS_INDICATOR_CODES",
"REIMBURSEMENT_DIAG_CODES",
"REIMBURSEMENT_GROUPER_CODES",
"REIMBURSEMENT_GROUPER",
"REIMBURSEMENT_PLACEOFSERVICE_CODES",
"REIMBURSEMENT_ADMITTYPE_CODES",
"REIMBURSEMENT_EXCEPTION_IND",
"REIMBURSEMENT_DESCRIBE_EXCEPTION",
"RATE_ESCALATOR_IND",
"RATE_ESCALATOR_BASIS",
"RATE_ESCALATOR_YEARLY_PERCENT_INCREASE",
"Corrected_LOB",
"Corrected_PROGRAM",
"Corrected_NETWORK",
],
"priorityhealth": [
"Filename",
"SERVICE",
"SERVICE_PG",
"REIMBURSEMENT_FLAT_FEE",
"REIMBURSEMENT_FLAT_FEE_PG",
"REIMBURSEMENT_RATE",
"REIMBURSEMENT_RATE_PG",
"FULL_METHODOLOGY",
"FULL_METHODOLOGY_PG",
"REIMBURSEMENT_TIN",
"REIMBURSEMENT_TIN_PG",
"DATE_RANGE",
"DATE_RANGE_PG",
"REIMBURSEMENT_METHODOLOGY",
"REIMBURSEMENT_METHODOLOGY_PG",
"REIMBURSEMENT_FEE_SCHEDULE",
"REIMBURSEMENT_FEE_SCHEDULE_PG",
"REIMBURSEMENT_FEE_SCHEDULE_VERSION",
"REIMBURSEMENT_FEE_SCHEDULE_VERSION_PG",
"LESSER_OF_LANGUAGE_IND",
"LESSER_OF_LANGUAGE_IND_PG",
"GREATER_OF_LANGUAGE_IND",
"GREATER_OF_LANGUAGE_IND_PG",
"CONTRACT_LOB",
"CONTRACT_LOB_PG",
"CONTRACT_MARKETPLACE_METAL_LEVEL",
"CONTRACT_MARKETPLACE_METAL_LEVEL_PG",
"CONTRACT_NETWORK",
"CONTRACT_NETWORK_PG",
"PRODUCT",
"PRODUCT_PG",
"CONTRACT_PROGRAM",
"CONTRACT_PROGRAM_PG",
"REIMBURSEMENT_PROC_CODES",
"REIMBURSEMENT_PROC_CODES_PG",
"REIMBURSEMENT_PROC_CODE_MODIFIERS",
"REIMBURSEMENT_PROC_CODE_MODIFIERS_PG",
"REIMBURSEMENT_REVENUE_CODES",
"REIMBURSEMENT_REVENUE_CODES_PG",
"REIMBURSEMENT_STATUS_INDICATOR_CODES",
"REIMBURSEMENT_STATUS_INDICATOR_CODES_PG",
"REIMBURSEMENT_DIAG_CODES",
"REIMBURSEMENT_DIAG_CODES_PG",
"REIMBURSEMENT_GROUPER_CODES",
"REIMBURSEMENT_GROUPER_CODES_PG",
"REIMBURSEMENT_GROUPER",
"REIMBURSEMENT_GROUPER_PG",
"REIMBURSEMENT_PLACEOFSERVICE_CODES",
"REIMBURSEMENT_PLACEOFSERVICE_CODES_PG",
"REIMBURSEMENT_ADMITTYPE_CODES",
"REIMBURSEMENT_ADMITTYPE_CODES_PG",
"REIMBURSEMENT_EXCEPTION_IND",
"REIMBURSEMENT_EXCEPTION_IND_PG",
"REIMBURSEMENT_DESCRIBE_EXCEPTION",
"REIMBURSEMENT_DESCRIBE_EXCEPTION_PG",
"RATE_ESCALATOR_IND",
"RATE_ESCALATOR_IND_PG",
"RATE_ESCALATOR_BASIS",
"RATE_ESCALATOR_BASIS_PG",
"RATE_ESCALATOR_YEARLY_PERCENT_INCREASE",
"RATE_ESCALATOR_YEARLY_PERCENT_INCREASE_PG",
"Corrected_LOB",
"Corrected_PROGRAM",
"Corrected_NETWORK",
],
}
fieldExtraction/src/config.py
+13 -44
View File
@@ -1,3 +1,5 @@
import os
import columns
from datetime import datetime
import boto3
@@ -34,7 +36,8 @@ VALID_PROGRAMS = [
VALID_NETWORKS = ["HMO", "PPO", "EPO", "POS", "FFS"]
# Input Settings
READ_MODE = "_LOCAL_" # OR '_S3_'
READ_MODE = os.getenv("READ_MODE")
# '_LOCAL_' OR '_S3_'
FILTER_ALREADY_PROCESSED = True
# Output Settings
@@ -50,7 +53,7 @@ UNPROCESSED_RESULTS_NAME = "combined_results_unprocessed.csv"
PROCESSED_RESULTS_NAME = "combined_results_post_processed.csv"
# Multithread Settings
MAX_WORKERS = 10
MAX_WORKERS = os.getenv("MAX_WORKERS", 10)
# Prompt Debugging
RUN_PRIMARY = True # Always True
@@ -63,50 +66,16 @@ RUN_CODES = True
# Postprocessing Settings
FUZZY_MATCH_THRESHOLD = 0.8
VALID_COLUMNS = [
"Filename",
"page_num",
"SERVICE",
"REIMBURSEMENT_FLAT_FEE",
"REIMBURSEMENT_RATE",
"FULL_METHODOLOGY",
"REIMBURSEMENT_METHODOLOGY",
"REIMBURSEMENT_FEE_SCHEDULE",
"REIMBURSEMENT_FEE_SCHEDULE_VERSION",
"LESSER_OF_LANGUAGE_IND",
"GREATER_OF_LANGUAGE_IND",
"CONTRACT_LOB",
"CONTRACT_NETWORK",
"PRODUCT",
"CONTRACT_PROGRAM",
"CONTRACT_MARKETPLACE_METAL_LEVEL",
"PROV_TYPE",
"REIMBURSEMENT_PROC_CODES",
"REIMBURSEMENT_PROC_CODE_MODIFIERS",
"REIMBURSEMENT_REVENUE_CODES",
"REIMBURSEMENT_STATUS_INDICATOR_CODES",
"REIMBURSEMENT_DIAG_CODES",
"REIMBURSEMENT_GROUPER_CODES",
"REIMBURSEMENT_GROUPER",
"REIMBURSEMENT_PLACEOFSERVICE_CODES",
"REIMBURSEMENT_ADMITTYPE_CODES",
"REIMBURSEMENT_EXCEPTION_IND",
"REIMBURSEMENT_DESCRIBE_EXCEPTION",
"RATE_ESCALATOR_IND",
"RATE_ESCALATOR_BASIS",
"RATE_ESCALATOR_YEARLY_PERCENT_INCREASE",
"Corrected_LOB",
"Corrected_PROGRAM",
"Corrected_NETWORK",
]
CLIENT = os.getenv("CLIENT", "base")
VALID_COLUMNS = columns.VALID_COLUMNS_TEMPLATE[CLIENT]
# AWS Keys
AWS_ACCESS_KEY_ID = "ASIAZTMXAXNXHQSWW2R5"
AWS_SECRET_ACCESS_KEY = "W9zH4nDGaae8SuJqcPDXyTHFpQ4xV43eRjbDfPGo"
AWS_SESSION_TOKEN = "IQoJb3JpZ2luX2VjEIX//////////wEaCXVzLWVhc3QtMiJIMEYCIQCjQVKSzQSWuO8bDdeGjDUJNUc1NwOA795yLFiBGDSslwIhAOccVQAzlBZyOVtU5mZztHVRbqEbrFigeg2aPzlczG1QKoMDCG8QABoMNjYwMTMxMDY4NzgyIgyP2pSGtJAhII6aZIMq4AI3XnnJgTtwbj6u15RcBlTCYgyn+SqIASHvEaJzyWHP/b7Vf5ZNyWKBsccgIZ590De/biM4Gb7tLsnWYCkFuyj3XswHKgyInBDqDg6b4olgyMGoDxrDmNUI28tQxB5XpwKxsas/azzwNPAgRAyXWMK4b1DcslPGpTRBBi+eBF+hTLiGk1jQnip0qR+GuJnCwXe66gcIGmgscjVxGSg2Q5n0O7VD2UOrr5T7BhvnGLiNtlrXqd3f+BFVk6/SzmuQJgsK33rtsmQUI7R5LXlyZEfH3cEkCXrdch6bFO4Wu53oBmiDtNEkle5n95TXCnXC1EchWzuDA6iP061kLRF3jdihMls8St/RoNACCR41YtOuU5oH1n5qXGvH1HQzOvBuSQBCKIqVGgB7hU63mbJlQCdX+0qd0jgteXFHMyjrwr604zPCxdcfbyehAwpxJR8xLVMimgxJqvyzvAoUjzW1sZPPMKG4rLUGOqUBOobUcGB1mI6GHaV6p6+t+qU/BG9FCt9BZbC/IJsHQkWWemCWpvV6lGVM5JPBmmgGQc3LiwH8g5JSHnoUDjTWlLvlHjXZ2ODLJP+aDFcuS4b1WZmXedL6RTsMvg8qhiELEB7rR+6jEmBmXvh56c+R4mkCG5AiWRBZ/ah910JnvKy5wQTOGJGzNxflyiJmwoO2JOZAKBLT0jg78iG1T3DMJ2qc+Qx7"
AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID")
AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY")
AWS_SESSION_TOKEN = os.getenv("AWS_SESSION_TOKEN")
# File Paths
LOCAL_PATH = "data/texas_childrens" # Replace with local
LOCAL_PATH = os.getenv("LOCAL_PATH", "data/")
# S3 Settings
S3_CLIENT = boto3.client(
@@ -116,8 +85,8 @@ S3_CLIENT = boto3.client(
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
aws_session_token=AWS_SESSION_TOKEN,
)
BUCKET = "doczy-dev-infra-textract"
PREFIX = "batches/batch_1/contract-text-file/" # replace with s3 path
BUCKET = os.getenv("BUCKET", "doczy-dev-infra-textract")
PREFIX = os.getenv("BUCKET_PREFIX")
# Bedrock Settings
BEDROCK_RUNTIME = boto3.client(
fieldExtraction/src/file_processing.py
+4 -25
View File
@@ -4,8 +4,9 @@ import pandas as pd
import config
import preprocess
import table_funcs
import bottom_up_funcs
import top_down_funcs
import bottom_up_funcs
import postprocess_funcs
import postprocess
import merge_funcs
@@ -40,11 +41,6 @@ def process_file(file_object):
if config.VERBOSE:
print(f"Processing {filename}...")
################## CREATE OUTPUT DIRECTORIES ##################
base_filename = os.path.splitext(filename)[0].strip()
output_dir = os.path.join(config.OUTPUT_DIRECTORY, base_filename)
os.makedirs(output_dir, exist_ok=True)
################## PREPROCESS ##################
contract_text = preprocess.clean_newlines(contract_text)
contract_text = preprocess.clean_billed_charges(contract_text)
@@ -56,37 +52,20 @@ def process_file(file_object):
td_results = top_down_funcs.run_top_down(
filename, text_dict
) # Returns list of dictionaries for each page
pd.DataFrame(td_results).to_csv(
os.path.join(output_dir, config.TD_RESULTS_NAME), index=False
)
print(f"Top Down Complete - {filename}")
################## RUN BOTTOM UP ##################
bu_results = bottom_up_funcs.run_bottom_up(
filename, text_dict
) # Returns list of dictionaries
pd.DataFrame(bu_results).to_csv(
os.path.join(output_dir, config.BU_RESULTS_NAME), index=False
)
print(f"Bottom Up Complete - {filename}")
################## COMBINE TOP DOWN AND BOTTOM UP ##################
combined_results = merge_funcs.merge_results(td_results, bu_results, text_dict)
print(f"TD/BU Merge Complete - {filename} ")
################## WRITE UNPROCESSED OUTPUT ##################
combined_df = pd.DataFrame(combined_results)
combined_df.to_csv(
os.path.join(output_dir, config.UNPROCESSED_RESULTS_NAME), index=False
)
################## RUN POSTPROCESSING ##################
# combined_df = combined_df.applymap(postprocessingfuncs.sanitize_value) # Deprecated
combined_df = pd.DataFrame(combined_results)
post_processed_combined_df = postprocess.postprocess_results(combined_df)
post_processed_combined_df.to_csv(
os.path.join(output_dir, config.PROCESSED_RESULTS_NAME), index=False
)
print(f"Postprocessing Complete - {filename} ")
print(f"Output Complete - {filename} ")
return post_processed_combined_df
fieldExtraction/src/main.py
+19 -4
View File
@@ -11,8 +11,7 @@ Functional Overview:
"""
# Imports
import concurrent.futures
import traceback
import pandas as pd
import os
import utils
@@ -61,8 +60,24 @@ def main():
print(f"Error: {e}")
# Write Consolidated Output
# if config.WRITE_OUTPUT:
# utils.consolidate_csvs(config.CONSOLIDATED_OUTPUT_DIRECTORY, config.OUTPUT_CSV_PATH)
if config.WRITE_OUTPUT:
all_results_df = pd.concat(all_results).reset_index(drop=True)
all_results_df.drop("page_num", axis=1, inplace=True)
# Reorder columns
final_df = all_results_df[
[col for col in config.VALID_COLUMNS if col in all_results_df.columns]
+ [
col
for col in all_results_df.columns
if col not in config.VALID_COLUMNS
]
]
final_df.to_csv(
os.path.join(
config.CONSOLIDATED_OUTPUT_DIRECTORY, config.OUTPUT_CSV_PATH
)
)
if __name__ == "__main__":
fieldExtraction/src/merge_funcs.py
+10 -6
View File
@@ -23,21 +23,23 @@ def merge_results(td_results, bu_results, text_dict):
if int(page_num) > 1:
return get_value(
find_td_dict(int(page_num) - 1), key, int(page_num) - 1, text_dict
)
), str(int(page_num) - 1)
else:
return "N/A" # Base case if no previous pages exist
return "N/A", "N/A" # Base case if no previous pages exist
else:
value = td_dict.get(key, "N/A")
if value == "N/A" or value == "":
# If value is 'N/A' or empty, use recursion to look one page earlier
return get_value(
find_td_dict(int(page_num) - 1), key, int(page_num) - 1, text_dict
)
), str(int(page_num) - 1)
elif "," in value and "DATE" not in key:
# Randomly select one of the comma-separated values
return prompt_select_multiple(key, value, bu_dict, text_dict[page_num])
return prompt_select_multiple(
key, value, bu_dict, text_dict[page_num]
), str(int(page_num))
else:
return value
return value, page_num
merged_results = []
for bu_dict in bu_results:
@@ -49,7 +51,9 @@ def merge_results(td_results, bu_results, text_dict):
if td_dict:
for key, value in td_dict.items():
if key not in ["Filename", "page_num"]:
new_dict[key] = get_value(td_dict, key, page_num, text_dict)
new_dict[key], new_dict[f"{key}_PG"] = get_value(
td_dict, key, page_num, text_dict
)
merged_results.append(new_dict)
return merged_results
fieldExtraction/src/postprocess.py
+8 -8
View File
@@ -1,7 +1,7 @@
import pandas as pd
import re
import postprocessing_funcs
import postprocess_funcs
import config
@@ -16,7 +16,7 @@ def postprocess_results(combined_df):
# Sanitize the combined data
try:
df = postprocessing_funcs.sanitize_combined(combined_df)
df = postprocess_funcs.sanitize_combined(combined_df)
except Exception as e:
print(f"Postprocessing Error - santize_combined : {e}")
@@ -27,14 +27,14 @@ def postprocess_results(combined_df):
["CONTRACT_NETWORK", "Corrected_NETWORK", config.VALID_NETWORKS],
]:
try:
postprocessing_funcs.clean_columns_combined(
postprocess_funcs.clean_columns_combined(
df, field_list[0], field_list[2], field_list[1]
)
except Exception as e:
print(f"Postprocessing Error - clean_columns_combined : {e}")
try:
postprocessing_funcs.clean_columns_combined_fuzzy(
postprocess_funcs.clean_columns_combined_fuzzy(
df, field_list[0], field_list[2], config.FUZZY_MATCH_THRESHOLD
)
except Exception as e:
@@ -42,7 +42,7 @@ def postprocess_results(combined_df):
# Correct misplaced values across columns
try:
df = postprocessing_funcs.correct_misplaced_values(
df = postprocess_funcs.correct_misplaced_values(
df,
["CONTRACT_LOB", "CONTRACT_PROGRAM", "CONTRACT_NETWORK"],
valid_values_dict,
@@ -52,19 +52,19 @@ def postprocess_results(combined_df):
# Move percentages and large numbers to correct columns
try:
df = postprocessing_funcs.move_percentage_to_rate(df)
df = postprocess_funcs.move_percentage_to_rate(df)
except Exception as e:
print(f"Postprocessing Error - move_percentage_to_rate : {e}")
# Replace N/As in "Not Covered" with 0%
try:
df = postprocessing_funcs.set_rate_to_zero_if_not_covered(df)
df = postprocess_funcs.set_rate_to_zero_if_not_covered(df)
except Exception as e:
print(f"Postprocessing Error - set_rate_to_zero_if_not_covered : {e}")
# Adjust for adding or subtracting
try:
df = postprocessing_funcs.adjust_reimbursement_rate(df)
df = postprocess_funcs.adjust_reimbursement_rate(df)
except Exception as e:
print(f"Postprocessing Error - adjust_reimbursement_rate : {e}")
fieldExtraction/src/postprocessing_funcs.py → fieldExtraction/src/postprocess_funcs.py
+8
View File
@@ -513,3 +513,11 @@ def clean_dates(df):
df.loc[idx, "LOB_PRICING_TERMS_TERMINATION_DATE"] = "N/A"
return df
def add_PG_columns(results_dicts):
for key in list(results_dicts[0].keys()):
if key != "page_num":
for d in results_dicts:
d[f"{key}_PG"] = d["page_num"]
return results_dicts
fieldExtraction/src/preprocess.py
+1
View File
@@ -133,6 +133,7 @@ def clean_billed_charges(contract_text):
"Billed Charges",
"the rates set forth in this Exhibit",
"the rutes set forth in this Exhibit",
"Provider's usual and customary charge",
]
max_substring_length = max([len(s) for s in substrings])
fieldExtraction/src/prompts.py
+3 -3
View File
@@ -64,7 +64,7 @@ Here are the attributes to be included in each dictionary, and instructions on h
'REIMBURSEMENT_FLAT_FEE' : If the listed reimbursement is dollar value, return only that dollar value. There can only be one answer for each object. If more than one rate is found, create additional objects for them.
'REIMBURSEMENT_RATE' : If the listed reimbursement is a percent of something, return only that percent. There can only be one answer for each object. If more than one rate is found, create additional objects for them. Do NOT include multiple percentages in this value.
'FULL_METHODOLOGY' : If the listed reimbursement is a percent of something, what is it the percent of? If the listed reimbursement is a direct dollar value, how is that dollar value paid (per diem, per unit, etc)? Write the full sentence in text describing the payment methodology.
'REIMBURSEMENT_DATES' : If the listed reimbursement only applies to a specific date range, write that range here. Write only the exact text. Write only the range. If there is none, write 'N/A'.
'DATE_RANGE' : If the listed reimbursement only applies to a specific date range, write that range here. Write only the exact text. Write only the range. If there is none, write 'N/A'.
'REIMBURSEMENT_TIN' : If the listed reimbursement only applies to a specific TIN, write only the TIN here. If not, write 'N/A'
Note that only one of REIMBURSEMENT_FLAT_FEE or REIMBURSEMENT_RATE should be populated in each dictionary. Ensure all 4 keys are in the dictionary.
@@ -176,7 +176,7 @@ Read and analyze the page, then populate a JSON dictionary with the following co
Return a value for each type of code - return N/A for any values not found. Multiple codes for each type may be found. In this case, return them all. They may also present as a range. If so, return the range.
Note that some PROC_CODES may have 'Modifiers' listed as well. These are 2-character strings. If Modifiers are present, include them as part of this field.
Note that some PROC_CODES may have 'Modifiers' listed as well. These are 2-character strings. If Modifiers are present, include them.
Ensure you ONLY return codes if they are associated directly with the Service listed. Codes referring to other services should not be included.
@@ -194,7 +194,7 @@ Here are the attributes to be included in each dictionary, and instructions on h
'CONTRACT_PROGRAM' : List any Programs mentioned on the page. Some examples of Programs are: CHIP, CHIP-P, CHIP-Perinate, STAR, STAR+PLUS. Do your best to infer the program from this page.
'CONTRACT_NETWORK' : List any Networks mentioned on the page. Networks are types of managed care organizations involved in the delivery of healthcare services. Some examples of Networks are: HMO, PPO, EMO, POS, FFS. Do your best to infer the network from this page. If it says anywhere that they are out of network then only return "OUT OF NETWORK" as your answer.
'PRODUCT' : List any Products listed on the page. These are typically the brand name of a health plan. You may also use this field as a catch-all for entities that you are unsure of.
'PROV_TYPE' : List any Provider Type listed on the page. This may be something like Inpatient, Outpatient, Ambulatory Surgery Center, DME, Home Health, Hospice, Skilled Nursing Facility, or similar.
'PROV_TYPE' : List any Provider Types listed on the page. This can be something like Inpatient, Outpatient, Ancillary, ASC, DME, Hospital, Home Health, Hospice, or similar.
If any of the attributes are not found, return N/A. If there are multiple answers for a given attribute, return the answers in a comma-separated list.
fieldExtraction/src/test.py
+72 -35
View File
@@ -1,48 +1,85 @@
import utils
import preprocess
import table_funcs
import prompts
import claude_funcs
import config
import os
import pandas as pd
import boto3
import os
import shutil
s3 = boto3.Session(
aws_access_key_id="ASIA6GBMBVWOJOOHVXUE",
aws_secret_access_key="YsXnoYhY599uOifnQ+hZnqQJpWTyjhZ8XppJr43V",
aws_session_token="IQoJb3JpZ2luX2VjEHcaCXVzLWVhc3QtMiJHMEUCIBdFMYRAL20l87z9xZJzlvOW8H7+jOIoI2f6zOkNtMdQAiEAomTpoulfRpLcPhzbNFQVR5atGD8xbM49/gKX0rNDZYgqgQMIYRAAGgw5NzUwNDk5NjA4NjAiDGoxXZfA4RDhWkpflCreAmgKA3i7fZYV6z/e5WejrcnSfffciNhUKpen98pi1gJT6xgB9Cy4k19DAkF8ab8dQcyTO80K20hd2QjzwaTXDDiIpXYJq1TRHiJVN+g0QKOx10l4KOo7g8hGPzD/QPmA80aTxC96PqRklnJkUGNKe9zfZ7nCujts/i/rs0oAeuRCCnEKsh3lOtj4YEhoQGsNlKUsx4Pfh2cTn/JTZ4hma0zO8HnVfPh2f4i4hpa8Ula/0arXZJrkJyHdbQV85w+lmjypILQwAy6kQUd57lURDXDF1uPtziKZ2WqkozpEdeblmUUAT24rXKBhv3m86oqN59pb1IiFoDE7IOmJgRqNbh58OhOwifijMRWYouvSFM8pcBDTD725UmTzNu5GKgGJE00Q4PyBu5K2YXX1rHFmOhr29wq11mPMBPVRTF/zMLjG8ZloHYQWtt8VZwso8WE7ezV7RyScz0SBwBvhOX0mMPSwqbUGOqYBhMdcVuF9yw0Norg5U7G8SDl2WmiYCh/Anfea87h/1KzPs6ZNphtuSLcaH9+C7hVx5DVAJUW7gI0xx5jhPgqHbcptDPkNWCL129URUMPFOmuRiPxyTl0Xl5jSeh0Mj+RJz81OCWaEQyTQYOTtNUir77f1KAm8y+ClFrkYFrk6uz6HyiYknmcxAokjdNMIOS/83O7GTUv1Vo6/fIXo2MKX7gxhuNUoNw==",
).client("s3")
# input_dict = utils.read_input()
# filename = list(input_dict.keys())[0]
# contract_text = input_dict[filename]
# contract_text = preprocess.clean_newlines(contract_text)
# contract_text = preprocess.clean_billed_charges(contract_text)
# text_dict = preprocess.split_text(contract_text)
# text_dict = table_funcs.align_and_format_tables(text_dict)
# text_dict = preprocess.highlight_rates(text_dict)
# def BOTTOM_UP_PRIMARY(page, payer):
# return f"""
# ### PAGE START ### {page} ### PAGE END
# The preceding text is one page of a contract between Payer {payer} and a provider in their network. Your job is to extract attributes related to the reimbursement of different services and specialties.
# The reimbursement values will be identified with >>> <<< indicators (e.g. >>>105%<<<). Make sure there is at least one dictionary object for EVERY reimbursement value seen (either % or $).
# Some values are found in sections explicitly identified as examples - these must be omitted from output. Other values might be related to liability - these must also be excluded.
# If any of the attributes are not found, return N/A. For all attributes, only write what is written on the page. Do not make up new phrases or words.
# Return at least one json object for each combination of attributes seen. It is possible that a page will not have any attributes. When this is the case, return an empty list.
# Here are the attributes to be included in each dictionary, and instructions on how to correctly answer:
# 'SERVICE' : What is the Service that is being reimbursed? Use your best judgement and knowledge of the healthcare industry to answer. Use only exact text from the document. Do not leave this field N/A.
# 'REIMBURSEMENT_FLAT_FEE' : If the listed reimbursement is dollar value, return only that dollar value. There can only be one answer for each object. If more than one rate is found, create additional objects for them.
# 'REIMBURSEMENT_RATE' : If the listed reimbursement is a percent of something, return only that percent. There can only be one answer for each object. If more than one rate is found, create additional objects for them. Do NOT include multiple percentages in this value.
# 'FULL_METHODOLOGY' : If the listed reimbursement is a percent of something, what is it the percent of? If the listed reimbursement is a direct dollar value, how is that dollar value paid (per diem, per unit, etc)? Write the full sentence in text describing the payment methodology.
# 'DATE_RANGE' : If the listed reimbursement only applies to a specific date range, write that range here. Write only the exact text. Write only the range. If there is none, write 'N/A'.
# 'REIMBURSEMENT_TIN' : If the listed reimbursement only applies to a specific TIN, write only the TIN here. If not, write 'N/A'
# Note that only one of REIMBURSEMENT_FLAT_FEE or REIMBURSEMENT_RATE should be populated in each dictionary. Ensure all 4 keys are in the dictionary.
# Only return the list, with no other commentary or explanation. Ensure you abide by proper JSON formatting.
# """
# answer_dict = {}
# for page_number in text_dict.keys():
# if utils.contains_reimbursement(text_dict, page_number):
# # Run Primary
# prompt = BOTTOM_UP_PRIMARY(text_dict[page_number], config.CLIENT_NAME)
# answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000)
# answer_dict[page_number] = answer
# for page_num in answer_dict.keys():
# print(f"Page {page_num}\n")
# print(answer_dict[page_num])
def get_s3_files(bucket_name, folder):
paginator = s3.get_paginator("list_objects_v2")
result = paginator.paginate(Bucket=bucket_name, Prefix=folder)
files = set()
batch1_files = pd.read_excel(
"C:\\Users\\kminhas\\Documents\\Doczy\\Facility TINs_DOCZY AI_DOCUMENT EXTRACTION_ASSIGNMENT_7.1.2024.xlsx",
sheet_name="DOCZY AI",
)
batch1_files = batch1_files["File Name"]
# print(list(batch1_files))
for page in result:
if "Contents" in page:
for obj in page["Contents"]:
file_name = obj["Key"]
if file_name: # Ensure it's not a folder
files.add(file_name)
tdrive_path = "T:\\AArete Client Work\\Centene\\Restricted\\HealthNet\\Doczy Contract Metadata\\1. Contracts\\05232024\\Extracted"
batch_path = os.path.join(tdrive_path, "Batch0")
try:
os.mkdir(batch_path)
except:
pass
return files
bucket_name = "texas-children-files"
folder = "510_text_files/"
files_in_folder = get_s3_files(bucket_name, folder)
source_bucket = "texas-children-files"
for source_key in files_in_folder:
filename = source_key.split("510_text_files/")[1]
print(filename)
if filename not in os.listdir("data/texas_childrens"):
for file in os.listdir(tdrive_path):
if file.endswith(".Pdf") and file.split(".Pdf")[0] in list(batch1_files):
try:
response = s3.get_object(Bucket=source_bucket, Key=source_key)["Body"]
file_content = response.read().decode("utf-8")
with open(f"data/texas_childrens/{filename}", "w") as file:
file.write(file_content)
source_path = os.path.join(tdrive_path, file)
destination_path = os.path.join(batch_path, file)
shutil.move(source_path, destination_path)
print(f"Moved file: {file}")
except:
pass
fieldExtraction/src/utils.py
+3 -1
View File
@@ -139,7 +139,9 @@ def consolidate_csvs(
pass
concatenated_df = pd.concat(df_list, ignore_index=True)
concatenated_df.to_csv(os.path.join(output_dir, output_file), index=False)
concatenated_df.to_csv(
os.path.join(config.CONSOLIDATED_OUTPUT_DIRECTORY, output_file), index=False
)
print(f"All CSV files have been consolidated into {output_file}")