jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

TD BU alignment fix

Browse Source
This commit is contained in:
Katon Minhas
2024-05-24 15:28:33 -05:00
committed by Michael McGuinness
parent 548c7d1400
commit 0bbba55705
5 changed files with 376 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/config.py
+4 -4
View File
@@ -28,12 +28,12 @@ RUN_EXCEPTION = True
RUN_CODES = True
# AWS Keys
AWS_ACCESS_KEY_ID="ASIAZTMXAXNXDI7HGA7Q"
AWS_SECRET_ACCESS_KEY="mXAJEp6Ar35G+VEwUfKq/3cu+1Hn721R+pNs+Fxh"
AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEOj//////////wEaCXVzLWVhc3QtMiJHMEUCIQDlpQWz84O6hWg0qtTjfoNx1nIt3Ip82gSG/jdYh05Z/AIgK3N9ViVt3ez4iwTbo7hwOgtnTbeK6Jn5fpCRfcHDv2QqgwMIYhAAGgw2NjAxMzEwNjg3ODIiDC3AfcU4Q5o7CB6tkCrgAjwqqjf8jXvnSMmnxvYmhqy4IwD/N4hyCIwYSHC1iGMOp9QsFyBVKixjAEl6m9dzt24XomQCY/GSoVNchUbqewuFIy5im3nyKlF/tGYj/xi261JTIr0szXk5UDw5twCXHSSdem1PII+C3EmsGl6fLjRFbPNlXi6vY3Z/X6nnhix/J6ZJQzPg5Plj6jGY9uBrtE/CF5DYtfGRPvFT13ZiI+UYJ/97viOM5JOWcT1nHoVY8DCLWicRCZO4aXbrOUs8QtRNns7oJye1sL1mU+0ocgkrXCYOnX54McYj4dViAHy3UUYo9w9u6a2LeW2MfQjvKam0W+q2iHuLs1ydn8ip7WjDzmBzLTAKZPWnPTjKZXlCGUsPXTj7U1rF0wNgBmIpOgMO7d49TsbSL0MWifCXuOF4sgu5ICKt5bcd7N0rvGgQR3I/kAY7abH60EAl37tbhP79tLmPt//hlR4xJ2lKuD0wrrq4sgY6pgEsIEApALn8eIykqKZPFf5jz1QWbhuScAy3Yjh+OjOvk29eUERS3xvIeOgYUF40qlkPboFkCRY3NNgvHLkbfg9I9m929zKk6Tugb6S30Aiqy9prxpPNvIA9/X877EzIb0xoIbxJHb8ucvW1xBkUNEwwZGlA6xbzKXqTHau7/j6odzLd/CPno+IQ9lCBSilCbg/1W1NAiDZC4IwJjpV8rHONSaXVdA44"
AWS_ACCESS_KEY_ID="ASIAZTMXAXNXNEUYUPUY"
AWS_SECRET_ACCESS_KEY="3HljGiz+dqWBtj+dkqs/syyxoguUoTahdLyceEeo"
AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEBUaCXVzLWVhc3QtMiJHMEUCIQCX9LcYh9A28cLNA/lDEHJ6ZqrulN0FampT1ptNyX+9JAIgIFrmDiZS5KfHbZWptuvveZt2s7x5r0xDwCU6t6EgBAEqjAMIjv//////////ARAAGgw2NjAxMzEwNjg3ODIiDOLRiVmy5kw46/5mzCrgAscMrf+1rEm7kpwpmd+BHjLNV4ggEy0oqQQ6z5Ny5BV6Ouo0tRmXqOqSaI5/YKXeKsRo8s4lrDNNq8qIK0GxpugvmbqguKYfEzKwyihGi5IVVb2JGL1nBkL495v0w5j1fuXqNwB7VCPkJyJCqxDHeehgtQ1AymuVAiRowkWmr7vVvFQyKKcGsBJlIzBZyguTtJhT2HPCp8IW9LhQMS2XBAa4xCnHqpPZSYIoKyp2zPPwi3pBKm+7eE3NGH7V7/Vrl6IL29zNkk5K/IeqAcyBzaFgaCPilr5N6lSMeKlkz8w0ls3guZeaEjDRf/Mz3NvMD9HeUVqGfi2M+ODxudgEfmfH3Op0EC2g5fsIBu/uvcjJf7PgUo9xT6CH5P5HY/e6TLaZH55ioz468uaTfoNw10TZ/6iymqYEXSpLfqHejC6qCx2pfcYf1UizgD2rn22GfcGs8u6oKFSThJ4fETq1mogwzafCsgY6pgHl1EzoiRwiXqCDl3GsA162dYrapmEPqcIVXTTuNuYXbl7B3GQfxTmjHjqSqKFhieWEvydD4SSQAhSXz9tPsN69Myyelwg3t6r2EWGL+JiZXEFpO1peWJgu3xfd3F1tPnp7Nfv7bVGmuIvft1JYTmx20W/vH9Q4J7jM4tPqEeQs5fRORTdXcm10Yysuq5FYo+9bahUJnPYblcJveO30vVirwzVWvmPm"
# File Paths
LOCAL_PATH = 'docs/test/' # Replace with local
LOCAL_PATH = 'docs/priority_health/' # Replace with local
# S3 Settings
S3_CLIENT = boto3.client('s3',
src/prompt_funcs.py
+67 -31
View File
@@ -3,6 +3,7 @@
import os
import re
import pandas as pd
import json
from llama_index.core import VectorStoreIndex, SimpleDirectoryReader, ServiceContext, PromptTemplate
from llama_index.core import SummaryIndex, Document
@@ -95,17 +96,6 @@ def run_bottom_up_secondary(answer_dicts, text_dict, tokens):
return final_dicts
def consolidate_lob(lob_dicts, results_dicts):
final_dicts = []
for d in results_dicts:
try:
d.update(lob_dicts[d['page_num']])
final_dicts.append(d)
except:
final_dicts.append(d)
return final_dicts
def run_bottom_up(filename, text_dict):
# Bottom Up Primary - SERVICE, REIMBURSEMENT_FLAT_FEE, REIMBURSEMENT_RATE, METHODOLOGY
@@ -158,33 +148,79 @@ def run_top_down(filename, text_dict):
continue
prompt = prompts.TOP_DOWN_PRIMARY(page_text)
answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000)
answer_dicts = dict_operations.primary_string_to_dict({page_num : answer}, filename) # Convert to list of dictionaries
all_results.append(answer_dicts)
td_primary = [i for d in all_results for i in d] # Consolidate to one list
answer_dict = json.loads(answer)
answer_dict.update({'page_num' : page_num, 'Filename' : filename})
#answer_dicts = dict_operations.primary_string_to_dict({page_num : answer}, filename) # Convert to dictionaries
all_results.append(answer_dict)
# td_primary = [i for d in all_results for i in d] # Consolidate to one list
# print(td_primary)
# Secondaries
td_final = top_down_secondary(td_primary, text_dict)
td_final = top_down_secondary(all_results, text_dict)
return td_final # List of list of dictionaries
def clean_td(td):
td_clean = []
for d in td:
new_d = {}
for k in d.keys():
if k not in ['page_num', 'Filename']:
if ',' in d[k]:
if 'DATE' not in k:
new_d[k] = [v.strip() for v in d[k].split(',')]
elif d[k] == 'N/A':
new_d[k] = []
else:
new_d[k] = [d[k]]
else:
new_d[k] = d[k]
td_clean.append(new_d)
return td_clean
def get_unique_td(td):
unique_td_dict = {}
for d in td:
for key, value in d.items():
if key not in ['page_num', 'Filename']:
for v in value:
if key not in unique_td_dict.keys():
unique_td_dict[key] = [v]
else:
unique_td_dict[key].append(v)
final_dict = {key : list(set(unique_td_dict[key])) for key in unique_td_dict.keys()}
return final_dict
def td_bu_combine(td, bu, text_dict):
bu_dicts = bu.copy()
td_dicts = td.copy()
td_dicts_clean = clean_td(td_dicts)
unique_td = get_unique_td(td_dicts_clean)
combined_list = []
for bud in bu:
td_dicts = [d for d in td if d['page_num'] == bud['page_num']]
# Case 1: Only one option
if len(td_dicts) == 1:
td_dict = td_dicts[0]
bud.update(td_dict)
# Case 2: More than one option
elif len(td_dicts) > 1:
td_check_formatted = utils.format_td_check(td_dicts, ['Filename', 'page_num'])
prompt = prompts.LOB_CHECK(bud, td_check_formatted, text_dict[bud['page_num']])
answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000)
answer_idx = ''.join(re.findall(r'\d', answer))
bud.update(td_dicts[int(answer_idx)-1])
# Case 3: No option for page
else:
continue
for bud_ in bu_dicts:
bud = bud_.copy()
for key, value in unique_td.items():
if len(value) == 0: # No value in contract
bud.update({key : ""})
elif len(value) == 1: # 1 value in contract
bud.update({key : value[0]})
else: # More than one value in contract
# Check for value on the page
page_value = []
for d in td_dicts_clean:
if (d['page_num'] == bud['page_num']) and (key in d.keys()):
for v in d[key]:
page_value.append(v)
if len(page_value) == 1: # One value on page
bud.update({key : page_value[0]})
elif len(page_value) > 1: # Multiple values on page
bud.update({key : ', '.join(page_value)})
elif len(page_value) == 0:
bud.update({key : ""})
combined_list.append(bud)
return combined_list
src/prompts.py
+10 -10
View File
@@ -122,34 +122,34 @@ Only return the dictionary, with no other commentary or explanation. Ensure you
def TOP_DOWN_PRIMARY(page):
return f"""### PAGE START ### {page} ### PAGE END
The preceding text is one page of a contract between a Payer and a provider in their network. Your job is to extract the names of certain entities listed in the contract and return a list of dictionaries with the values below.
The preceding text is one page of a contract between a Payer and a provider in their network. Your job is to extract the names of certain entities listed in the contract and return a single dictionary with the values below:
Here are the attributes to be included in each dictionary, and instructions on how to correctly answer:
'CONTRACT_LOB' : List any Line of Business mentioned on the page. CONTRACT_LOB will ONLY be one of the following: Commercial, Group, Marketplace, Marketplace/Exchange, Medicaid, Medicaid-Medicare, Medicare, Medicare Advantage, Medicare Supplemental, Veterans Affairs
'CONTRACT_LOB' : List any Line of Business mentioned on the page. Choose ONLY from the following: Commercial, Group, Marketplace, Marketplace/Exchange, Medicaid, Medicaid-Medicare, Medicare, Medicare Advantage, Medicare Supplemental, Veterans Affairs. If none of these are found on the page, simply write 'N/A' for this answer.
'CONTRACT_PROGRAM' : List any Programs mentioned on the page. Some examples of Programs are: CHIP, CHIP-P, CHIP-Perinate, STAR, STAR+PLUS
'CONTRACT_NETWORK' : List any Networks mentioned on the page. Networks are types of managed care organizations involved in the delivery of healthcare services. Some examples of Networks are: HMO, PPO, EMO, POS, FFS
'PRODUCT' : List any Products listed on the page. These are typically the brand name of a health plan. You may also use this field as a catch-all for entities that you are unsure of.
If any of the attributes are not found, return N/A. If there are multiple answers for a given attribute, return a separate dictionary for them such that each of the four attributes is associated with the other attributes in the dictionary.
If any of the attributes are not found, return N/A. If there are multiple answers for a given attribute, return the answers in a comma-separated list.
For all attributes, only write what is written on the page. Do not make up new phrases or words.
Only return the list, with no other commentary or explanation. Ensure you abide by proper JSON formatting.
Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting.
"""
def LOB_CHECK(bu_dict, td_dicts, page):
return f"""### PAGE START ### {page} ### PAGE END
Service: {bu_dict['SERVICE']}
Methodology: {bu_dict['FULL_METHODOLOGY']}
Service: {bu_dict['SERVICE']}
Methodology: {bu_dict['FULL_METHODOLOGY']}
Listed above is page of a contract, as well as a Service and Methodology found on this page. Your job is to identify which of the following entities this specific Service is associated with. Here are the entities, represented as dictionary objects:
Listed above is page of a contract, as well as a Service and Methodology found on this page. Your job is to identify which of the following entities this specific Service is associated with. Here are the entities, represented as dictionary objects:
{td_dicts}
{td_dicts}
Write ONLY the number of the dictionary that is most associated with the Service and Methodology of interest. Do not return any additional text beyond the digit itself.
"""
Write ONLY the number of the dictionary that is most associated with the Service and Methodology of interest. Do not return any additional text beyond the digit itself.
"""
def TOP_DOWN_METAL_LEVEL(LOB, page):
src/table_funcs.py
+19 -18
View File
@@ -1,6 +1,7 @@
import re
import pandas as pd
import json
import ast
def extract_tables(text):
@@ -33,29 +34,29 @@ def align_and_format_tables(text_dict):
aligned_text_dict = {}
for key, text in text_dict.items():
if 'Table Start' in text:
print(key)
table_texts = re.findall(r'-------Table Start--------(.*?)-------Table End--------', text, re.DOTALL)
for table_text in table_texts:
# Extract pretable text
pretable = table_text.split('{')[0].strip()
try:
# Extract pretable text
pretable = table_text.split('{')[0].strip()
# Extract and format table text
table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0].replace("'", '"') + "}"
table = json.loads(table_only)
table_formatted = format_table(table)
# Extract and format table text
table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0].replace("'", '"') + "}"
table = json.loads(table_only)
table_formatted = format_table(table)
# Align
if text.count(pretable) == 2:
# Remove original table
aligned_text = text.replace(table_text, "")
# Align new table
aligned_text = aligned_text.replace(pretable, pretable + table_formatted)
else:
aligned_text = text.replace(table_text, pretable + table_formatted)
# Align
if text.count(pretable) == 2:
# Remove original table
aligned_text = text.replace(table_text, "")
# Align new table
aligned_text = aligned_text.replace(pretable, pretable + table_formatted)
else:
aligned_text = text.replace(table_text, pretable + table_formatted)
except:
aligned_text = text
aligned_text_dict[key] = aligned_text.replace("-------Table Start--------", "").replace("-------Table End--------", "")
else:
aligned_text_dict[key] = text
return aligned_text_dict
return aligned_text_dict
src/test_notebook.ipynb
+276 -148
View File
File diff suppressed because one or more lines are too long