jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Bottom Up Primary optimization

Browse Source
This commit is contained in:
Katon Minhas
2024-06-07 13:26:28 -07:00
committed by Michael McGuinness
parent be11899de1
commit 4f531cf8bd
2 changed files with 4 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/prompts.py
+1 -1
View File
@@ -45,7 +45,7 @@ If any of the attributes are not found, return N/A. For all attributes, only wri
Return at least one json object for each combination of attributes seen. It is possible that a page will not have any attributes. When this is the case, return an empty list.
Here are the attributes to be included in each dictionary, and instructions on how to correctly answer:
'SERVICE' : What is the Service that is being reimbursed? Use your best judgement and knowledge of the healthcare industry to answer. Do not leave this field N/A.
'SERVICE' : What is the Service that is being reimbursed? Use your best judgement and knowledge of the healthcare industry to answer. Use only exact text from the document. Do not leave this field N/A.
'REIMBURSEMENT_FLAT_FEE' : If the listed reimbursement is dollar value, return only that dollar value. There can only be one answer for each object. If more than one rate is found, create additional objects for them.
'REIMBURSEMENT_RATE' : If the listed reimbursement is a percent of something, return only that percent. There can only be one answer for each object. If more than one rate is found, create additional objects for them. Do NOT include multiple percentages in this value.
'FULL_METHODOLOGY' : If the listed reimbursement is a percent of something, what is it the percent of? If the listed reimbursement is a direct dollar value, how is that dollar value paid (per diem, per unit, etc)? Write the full sentence in text describing the payment methology.
src/test.py
+3 -67
View File
@@ -27,7 +27,7 @@ If any of the attributes are not found, return N/A. For all attributes, only wri
Return at least one json object for each combination of attributes seen. It is possible that a page will not have any attributes. When this is the case, return an empty list.
Here are the attributes to be included in each dictionary, and instructions on how to correctly answer:
'SERVICE' : What is the Service that is being reimbursed? Use your best judgement and knowledge of the healthcare industry to answer. Do not leave this field N/A.
'SERVICE' : What is the Service that is being reimbursed? Use your best judgement and knowledge of the healthcare industry to answer. Use only exact text from the document. Do not leave this field N/A.
'REIMBURSEMENT_FLAT_FEE' : If the listed reimbursement is dollar value, return only that dollar value. There can only be one answer for each object. If more than one rate is found, create additional objects for them.
'REIMBURSEMENT_RATE' : If the listed reimbursement is a percent of something, return only that percent. There can only be one answer for each object. If more than one rate is found, create additional objects for them. Do NOT include multiple percentages in this value.
'FULL_METHODOLOGY' : If the listed reimbursement is a percent of something, what is it the percent of? If the listed reimbursement is a direct dollar value, how is that dollar value paid (per diem, per unit, etc)? Write the full sentence in text describing the payment methology.
@@ -43,77 +43,13 @@ filename = list(input_dict.keys())[0]
contract_text = input_dict[filename]
print(filename)
def convert_to_dict(table_text):
table_text = table_text.strip('{}')
table_text_list = table_text.split(']')
table_text_list = [item for item in table_text_list if len(item) > 0]
final_dict = {}
num_elements = 0
for key_value_text in table_text_list:
key = key_value_text.split(':')[0].strip(', \'')
value = ':'.join(key_value_text.split(':')[1:]).strip(' \'')+'\']'
value_list = ast.literal_eval(value)
final_dict[key] = value_list
num_elements = len(value_list)
return final_dict, num_elements
def format_table(table_json, table_size):
table_text = ""
keys = [key for key in table_json.keys()]
table_text += ' : '.join(keys) + "\n"
for i in range(table_size):
for key in keys:
if table_json[key][i]:
#table_text += key + ': ' + table_json[key][i] + ', '
table_text += table_json[key][i] + ' : '
table_text += r'\n'
return table_text
def align_and_format_tables(text_dict):
aligned_text_dict = {}
for key, text in text_dict.items():
aligned_text = text
if 'Table Start' in text:
table_texts = re.findall(r'-------Table Start--------(.*?)-------Table End--------', text, re.DOTALL)
for table_text in table_texts:
try:
# Extract pretable text
pretable = table_text.split('{')[0].strip()
# Extract and format table text
#table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0].replace("'", '"') + "}"
table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0] + "}"
table, table_size = convert_to_dict(table_only)
table_formatted = format_table(table, table_size)
except:
table_formatted = table_text
# Align
if text.count(pretable) == 2: # One table
# Remove original table
aligned_text = text.replace(table_text, "")
# Align new table
aligned_text = aligned_text.replace(pretable, pretable + table_formatted)
else:
aligned_text = aligned_text.replace(table_text, pretable + table_formatted)
aligned_text_dict[key] = aligned_text.replace("-------Table Start--------", "").replace("-------Table End--------", "")
else:
aligned_text_dict[key] = text
return aligned_text_dict
contract_text = preprocess.clean_newlines(contract_text)
text_dict = preprocess.split_text(contract_text)
text_dict = align_and_format_tables({'25' : text_dict['25']})
text_dict = table_funcs.align_and_format_tables({'2' : text_dict['2']})
text_dict = preprocess.highlight_rates(text_dict)
prompt = BOTTOM_UP_PRIMARY(text_dict['25'], '')
prompt = BOTTOM_UP_PRIMARY(text_dict['2'], '')
answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000)
print(answer)