Katon Minhas
87 lines
2.6 KiB
Python
87 lines
2.6 KiB
Python
import pandas as pd
|
|
import numpy as np
|
|
import re
|
|
import json
|
|
import csv
|
|
from io import StringIO
|
|
import config
|
|
import ast
|
|
import os
|
|
|
|
import utils
|
|
import postprocess
|
|
import postprocessingfuncs
|
|
import preprocess
|
|
import table_funcs
|
|
import prompts
|
|
import prompt_funcs
|
|
import claude_funcs
|
|
|
|
|
|
# all_dfs = []
|
|
# for file in os.listdir('output'):
|
|
# if 'combined_results_post_processed.csv' in os.listdir(os.path.join('output', file)):
|
|
# try:
|
|
# df = pd.read_csv(f'output/{file}/combined_results_post_processed.csv')
|
|
# all_dfs.append(df)
|
|
# except:
|
|
# continue
|
|
|
|
# final_df = pd.concat(all_dfs, ignore_index=True)
|
|
# final_df.to_excel('output_consolidated/test_20240610_batch1.xlsx')
|
|
|
|
|
|
input_dict = utils.read_input()
|
|
(filename, contract_text) = list(input_dict.items())[0]
|
|
|
|
def clean_billed_charges(contract_text):
|
|
substrings = [
|
|
"Physician's Billed Charges",
|
|
"Provider's Billed Charges",
|
|
"Allowable Billed Charges",
|
|
"Hospital's Billed Charges",
|
|
"Physician's Charges",
|
|
"Provider's Charges",
|
|
"Allowable Charges",
|
|
"Hospital's Charges",
|
|
"Billed Charges"
|
|
]
|
|
max_substring_length = np.max([len(s) for s in substrings])
|
|
|
|
def find_substring_indices(contract_text, s):
|
|
indices = []
|
|
lower_contract_text = contract_text.lower()
|
|
lower_s = s.lower()
|
|
index = lower_contract_text.find(lower_s)
|
|
while index != -1:
|
|
indices.append(index)
|
|
index = lower_contract_text.find(lower_s, index + 1)
|
|
return indices
|
|
|
|
for s in substrings:
|
|
indices = find_substring_indices(contract_text, s)
|
|
index_adder = 0
|
|
if indices:
|
|
for i in indices:
|
|
index = i+index_adder
|
|
end_index = index + max_substring_length
|
|
match_part = contract_text[index:end_index]
|
|
previous = contract_text[max(0, index-30):index]
|
|
if '%' not in previous:
|
|
contract_text = contract_text[0:index] + f" 100% of {match_part}" + contract_text[end_index:]
|
|
index_adder += 9
|
|
return contract_text.replace(' ', ' ')
|
|
|
|
contract_text = preprocess.clean_newlines(contract_text)
|
|
contract_text = clean_billed_charges(contract_text)
|
|
contract_text = preprocess.highlight_rates(contract_text)
|
|
text_dict = preprocess.split_text(contract_text)
|
|
# text_dict = table_funcs.align_and_format_tables(text_dict)
|
|
|
|
|
|
print(text_dict['23'])
|
|
|
|
bu_results = prompt_funcs.run_bottom_up_primary({'23' : text_dict['23']}, 4000) # Returns list of dictionaries
|
|
print(bu_results)
|
|
|