diff --git a/.gitignore b/.gitignore index b24d71e..b3393a2 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,14 @@ Thumbs.db *.mov *.wmv +# Data Files +streamlit/history.csv +streamlit/RESULTS/ +streamlit/DB/ +streamlit/RAW_DOCUMENTS/ +streamlit/SOURCE_DOCUMENTS/ +streamlit/contract_field_values.csv +streamlit/contract_fields.csv +streamlit/sample.csv + + diff --git a/streamlit/constants.py b/streamlit/constants.py new file mode 100644 index 0000000..0f50fb6 --- /dev/null +++ b/streamlit/constants.py @@ -0,0 +1,189 @@ +import os + +# from dotenv import load_dotenv +from chromadb.config import Settings + +# https://python.langchain.com/en/latest/modules/indexes/document_loaders/examples/excel.html?highlight=xlsx#microsoft-excel +from langchain_community.document_loaders import CSVLoader, PDFMinerLoader, TextLoader, UnstructuredExcelLoader, Docx2txtLoader +from langchain_community.document_loaders import UnstructuredFileLoader, UnstructuredMarkdownLoader + + +# load_dotenv() +# ROOT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) +ROOT_DIRECTORY = "\\\\amznfsxuofkyi1z.aarete.local\\SharedFiles\\AArete Client Work\\Modahealth\\Restricted\\Moda Growth\\Artificial Intelligence\\DEFAXXER_20231207" + +# Define the folder for source and output +SOURCE_DIRECTORY = "SOURCE_DOCUMENTS" +OUTPUT_DIRECTORY = f"{ROOT_DIRECTORY}\\Output" + +PERSIST_DIRECTORY = 'DB' + +MODELS_PATH = "C:\\Users\\Public\\models" + +# Can be changed to a specific number +INGEST_THREADS = os.cpu_count() or 8 + +# Define the Chroma settings +CHROMA_SETTINGS = Settings( + anonymized_telemetry=False, + is_persistent=True, + allow_reset=True, +) + +# Context Window and Max New Tokens +CONTEXT_WINDOW_SIZE = 4096 +MAX_NEW_TOKENS = CONTEXT_WINDOW_SIZE # int(CONTEXT_WINDOW_SIZE/4) + +#### If you get a "not enough space in the buffer" error, you should reduce the values below, start with half of the original values and keep halving the value until the error stops appearing + +N_GPU_LAYERS = 100 # Llama-2-70B has 83 layers +N_BATCH = 512 + +### From experimenting with the Llama-2-7B-Chat-GGML model on 8GB VRAM, these values work: +# N_GPU_LAYERS = 20 +# N_BATCH = 512 + + +# https://python.langchain.com/en/latest/_modules/langchain/document_loaders/excel.html#UnstructuredExcelLoader +DOCUMENT_MAP = { + ".txt": TextLoader, + ".md": UnstructuredMarkdownLoader, + ".py": TextLoader, + # ".pdf": PDFMinerLoader, + ".pdf": UnstructuredFileLoader, + ".csv": CSVLoader, + ".xls": UnstructuredExcelLoader, + ".xlsx": UnstructuredExcelLoader, + ".docx": Docx2txtLoader, + ".doc": Docx2txtLoader, +} + +# Default Instructor Model +EMBEDDING_MODEL_NAME = "hkunlp/instructor-large" # Uses 1.5 GB of VRAM (High Accuracy with lower VRAM usage) + +#### +#### OTHER EMBEDDING MODEL OPTIONS +#### + +# EMBEDDING_MODEL_NAME = "hkunlp/instructor-xl" # Uses 5 GB of VRAM (Most Accurate of all models) +# EMBEDDING_MODEL_NAME = "intfloat/e5-large-v2" # Uses 1.5 GB of VRAM (A little less accurate than instructor-large) +# EMBEDDING_MODEL_NAME = "intfloat/e5-base-v2" # Uses 0.5 GB of VRAM (A good model for lower VRAM GPUs) +# EMBEDDING_MODEL_NAME = "all-MiniLM-L6-v2" # Uses 0.2 GB of VRAM (Less accurate but fastest - only requires 150mb of vram) + +#### +#### MULTILINGUAL EMBEDDING MODELS +#### + +# EMBEDDING_MODEL_NAME = "intfloat/multilingual-e5-large" # Uses 2.5 GB of VRAM +# EMBEDDING_MODEL_NAME = "intfloat/multilingual-e5-base" # Uses 1.2 GB of VRAM + + +#### SELECT AN OPEN SOURCE LLM (LARGE LANGUAGE MODEL) +# Select the Model ID and model_basename +# load the LLM for generating Natural Language responses + +#### GPU VRAM Memory required for LLM Models (ONLY) by Billion Parameter value (B Model) +#### Does not include VRAM used by Embedding Models - which use an additional 2GB-7GB of VRAM depending on the model. +#### +#### (B Model) (float32) (float16) (GPTQ 8bit) (GPTQ 4bit) +#### 7b 28 GB 14 GB 7 GB - 9 GB 3.5 GB - 5 GB +#### 13b 52 GB 26 GB 13 GB - 15 GB 6.5 GB - 8 GB +#### 32b 130 GB 65 GB 32.5 GB - 35 GB 16.25 GB - 19 GB +#### 65b 260.8 GB 130.4 GB 65.2 GB - 67 GB 32.6 GB - - 35 GB + +# MODEL_ID = "TheBloke/Llama-2-7B-Chat-GGML" +# MODEL_BASENAME = "llama-2-7b-chat.ggmlv3.q4_0.bin" + +#### +#### (FOR GGUF MODELS) +#### + +# MODEL_ID = "TheBloke/Llama-2-13b-Chat-GGUF" +# MODEL_BASENAME = "llama-2-13b-chat.Q4_K_M.gguf" + +MODEL_ID = "TheBloke/Llama-2-7b-Chat-GGUF" +MODEL_BASENAME = "llama-2-7b-chat.Q4_K_M.gguf" + +# MODEL_ID = "TheBloke/Mistral-7B-Instruct-v0.1-GGUF" +# MODEL_BASENAME = "mistral-7b-instruct-v0.1.Q8_0.gguf" + +# MODEL_ID = "TheBloke/Llama-2-70b-Chat-GGUF" +# MODEL_BASENAME = "llama-2-70b-chat.Q4_K_M.gguf" + +#### +#### (FOR HF MODELS) +#### + +# MODEL_ID = "NousResearch/Llama-2-7b-chat-hf" +# MODEL_BASENAME = None +# MODEL_ID = "TheBloke/vicuna-7B-1.1-HF" +# MODEL_BASENAME = None +# MODEL_ID = "TheBloke/Wizard-Vicuna-7B-Uncensored-HF" +# MODEL_ID = "TheBloke/guanaco-7B-HF" +# MODEL_ID = 'NousResearch/Nous-Hermes-13b' # Requires ~ 23GB VRAM. Using STransformers +# alongside will 100% create OOM on 24GB cards. +# llm = load_model(device_type, model_id=model_id) + +#### +#### (FOR GPTQ QUANTIZED) Select a llm model based on your GPU and VRAM GB. Does not include Embedding Models VRAM usage. +#### + +##### 48GB VRAM Graphics Cards (RTX 6000, RTX A6000 and other 48GB VRAM GPUs) ##### + +### 65b GPTQ LLM Models for 48GB GPUs (*** With best embedding model: hkunlp/instructor-xl ***) +# MODEL_ID = "TheBloke/guanaco-65B-GPTQ" +# MODEL_BASENAME = "model.safetensors" +# MODEL_ID = "TheBloke/Airoboros-65B-GPT4-2.0-GPTQ" +# MODEL_BASENAME = "model.safetensors" +# MODEL_ID = "TheBloke/gpt4-alpaca-lora_mlp-65B-GPTQ" +# MODEL_BASENAME = "model.safetensors" +# MODEL_ID = "TheBloke/Upstage-Llama1-65B-Instruct-GPTQ" +# MODEL_BASENAME = "model.safetensors" + +##### 24GB VRAM Graphics Cards (RTX 3090 - RTX 4090 (35% Faster) - RTX A5000 - RTX A5500) ##### + +### 13b GPTQ Models for 24GB GPUs (*** With best embedding model: hkunlp/instructor-xl ***) +# MODEL_ID = "TheBloke/Wizard-Vicuna-13B-Uncensored-GPTQ" +# MODEL_BASENAME = "Wizard-Vicuna-13B-Uncensored-GPTQ-4bit-128g.compat.no-act-order.safetensors" +# MODEL_ID = "TheBloke/vicuna-13B-v1.5-GPTQ" +# MODEL_BASENAME = "model.safetensors" +# MODEL_ID = "TheBloke/Nous-Hermes-13B-GPTQ" +# MODEL_BASENAME = "nous-hermes-13b-GPTQ-4bit-128g.no-act.order" +# MODEL_ID = "TheBloke/WizardLM-13B-V1.2-GPTQ" +# MODEL_BASENAME = "gptq_model-4bit-128g.safetensors + +### 30b GPTQ Models for 24GB GPUs (*** Requires using intfloat/e5-base-v2 instead of hkunlp/instructor-large as embedding model ***) +# MODEL_ID = "TheBloke/Wizard-Vicuna-30B-Uncensored-GPTQ" +# MODEL_BASENAME = "Wizard-Vicuna-30B-Uncensored-GPTQ-4bit--1g.act.order.safetensors" +# MODEL_ID = "TheBloke/WizardLM-30B-Uncensored-GPTQ" +# MODEL_BASENAME = "WizardLM-30B-Uncensored-GPTQ-4bit.act-order.safetensors" + +##### 8-10GB VRAM Graphics Cards (RTX 3080 - RTX 3080 Ti - RTX 3070 Ti - 3060 Ti - RTX 2000 Series, Quadro RTX 4000, 5000, 6000) ##### +### (*** Requires using intfloat/e5-small-v2 instead of hkunlp/instructor-large as embedding model ***) + +### 7b GPTQ Models for 8GB GPUs +# MODEL_ID = "TheBloke/Wizard-Vicuna-7B-Uncensored-GPTQ" +# MODEL_BASENAME = "Wizard-Vicuna-7B-Uncensored-GPTQ-4bit-128g.no-act.order.safetensors" +# MODEL_ID = "TheBloke/WizardLM-7B-uncensored-GPTQ" +# MODEL_BASENAME = "WizardLM-7B-uncensored-GPTQ-4bit-128g.compat.no-act-order.safetensors" +# MODEL_ID = "TheBloke/wizardLM-7B-GPTQ" +# MODEL_BASENAME = "wizardLM-7B-GPTQ-4bit.compat.no-act-order.safetensors" + +#### +#### (FOR GGML) (Quantized cpu+gpu+mps) models - check if they support llama.cpp +#### + +# MODEL_ID = "TheBloke/wizard-vicuna-13B-GGML" +# MODEL_BASENAME = "wizard-vicuna-13B.ggmlv3.q4_0.bin" +# MODEL_BASENAME = "wizard-vicuna-13B.ggmlv3.q6_K.bin" +# MODEL_BASENAME = "wizard-vicuna-13B.ggmlv3.q2_K.bin" +# MODEL_ID = "TheBloke/orca_mini_3B-GGML" +# MODEL_BASENAME = "orca-mini-3b.ggmlv3.q4_0.bin" + +#### +#### (FOR AWQ QUANTIZED) Select a llm model based on your GPU and VRAM GB. Does not include Embedding Models VRAM usage. +### (*** MODEL_BASENAME is not actually used but have to contain .awq so the correct model loading is used ***) +### (*** Compute capability 7.5 (sm75) and CUDA Toolkit 11.8+ are required ***) +#### +# MODEL_ID = "TheBloke/Llama-2-7B-Chat-AWQ" +# MODEL_BASENAME = "model.safetensors.awq" diff --git a/streamlit/ingest.py b/streamlit/ingest.py new file mode 100644 index 0000000..c5e5833 --- /dev/null +++ b/streamlit/ingest.py @@ -0,0 +1,219 @@ +import logging +import os +from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed + +import click +import torch +from langchain.docstore.document import Document +from langchain.embeddings import HuggingFaceInstructEmbeddings +from langchain.text_splitter import Language, RecursiveCharacterTextSplitter +from langchain.vectorstores import Chroma +import uuid + +from constants import ( + CHROMA_SETTINGS, + DOCUMENT_MAP, + EMBEDDING_MODEL_NAME, + INGEST_THREADS, + PERSIST_DIRECTORY, + SOURCE_DIRECTORY, +) + +import boto3 +from langchain.embeddings.bedrock import BedrockEmbeddings + +def file_log(logentry): + file1 = open("file_ingest.log","a") + file1.write(logentry + "\n") + file1.close() + print(logentry + "\n") + +def load_single_document(file_path: str) -> Document: + # Loads a single document from a file path + try: + file_extension = os.path.splitext(file_path)[1] + loader_class = DOCUMENT_MAP.get(file_extension) + if loader_class: + file_log(file_path + ' loaded.') + loader = loader_class(file_path) + else: + file_log(file_path + ' document type is undefined.') + raise ValueError("Document type is undefined") + return loader.load()[0] + except Exception as ex: + file_log('%s loading error: \n%s' % (file_path, ex)) + return None + +def load_document_batch(filepaths): + logging.info("Loading document batch") + # create a thread pool + with ThreadPoolExecutor(len(filepaths)) as exe: + # load files + futures = [exe.submit(load_single_document, name) for name in filepaths] + # collect data + if futures is None: + file_log(name + ' failed to submit') + return None + else: + data_list = [future.result() for future in futures] + # return data and file paths + return (data_list, filepaths) + + +def load_documents(source_dir: str) -> list[Document]: + # Loads all documents from the source documents directory, including nested folders + paths = [] + for root, _, files in os.walk(source_dir): + for file_name in files: + print('Importing: ' + file_name) + file_extension = os.path.splitext(file_name)[1] + source_file_path = os.path.join(root, file_name) + if file_extension in DOCUMENT_MAP.keys(): + paths.append(source_file_path) + + # Have at least one worker and at most INGEST_THREADS workers + n_workers = min(INGEST_THREADS, max(len(paths), 1)) + chunksize = round(len(paths) / n_workers) + docs = [] + with ProcessPoolExecutor(n_workers) as executor: + futures = [] + # split the load operations into chunks + for i in range(0, len(paths), chunksize): + # select a chunk of filenames + filepaths = paths[i : (i + chunksize)] + # submit the task + try: + future = executor.submit(load_document_batch, filepaths) + except Exception as ex: + file_log('executor task failed: %s' % (ex)) + future = None + if future is not None: + futures.append(future) + # process all results + for future in as_completed(futures): + # open the file and load the data + try: + contents, _ = future.result() + docs.extend(contents) + except Exception as ex: + file_log('Exception: %s' % (ex)) + + return docs + + +def split_documents(documents: list[Document]) -> tuple[list[Document], list[Document]]: + # Splits documents for correct Text Splitter + text_docs, python_docs = [], [] + for doc in documents: + if doc is not None: + file_extension = os.path.splitext(doc.metadata["source"])[1] + if file_extension == ".py": + python_docs.append(doc) + else: + text_docs.append(doc) + return text_docs, python_docs + +def process_in_batches(texts, batch_size): + for i in range(0, len(texts), batch_size): + yield texts[i:i+batch_size] + +@click.command() +@click.option( + "--device_type", + default="cuda" if torch.cuda.is_available() else "cpu", + type=click.Choice( + [ + "cpu", + "cuda", + "ipu", + "xpu", + "mkldnn", + "opengl", + "opencl", + "ideep", + "hip", + "ve", + "fpga", + "ort", + "xla", + "lazy", + "vulkan", + "mps", + "meta", + "hpu", + "mtia", + ], + ), + help="Device to run on. (Default is cuda)", +) + +def main(device_type): + # Load documents and split in chunks + logging.info(f"Loading documents from {SOURCE_DIRECTORY}") + for filename in os.listdir(SOURCE_DIRECTORY): + with open(os.path.join(SOURCE_DIRECTORY, filename), 'r') as infile: + text = infile.read() + text_splitted = [i for i in text.split('Start of Page No. = ')] + for i, txt in enumerate(text_splitted): + if len(txt) > 2: + page_path = os.path.join(SOURCE_DIRECTORY, f'{filename[:-4]}_page{i}.txt') + with open(page_path, 'w') as f: + f.write(txt) + + documents = [load_single_document(page_path)] + os.remove(page_path) + text_documents, python_documents = split_documents(documents) + text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200) + python_splitter = RecursiveCharacterTextSplitter.from_language( + language=Language.PYTHON, chunk_size=880, chunk_overlap=200 + ) + texts = text_splitter.split_documents(text_documents) + texts.extend(python_splitter.split_documents(python_documents)) + logging.info(f"Loaded {len(documents)} documents from {SOURCE_DIRECTORY}") + logging.info(f"Split into {len(texts)} chunks of text") + + # Create embeddings + # embeddings = HuggingFaceInstructEmbeddings( + # model_name=EMBEDDING_MODEL_NAME, + # model_kwargs={"device": device_type}, + # ) + bedrock_runtime = boto3.client( + service_name="bedrock-runtime", + region_name="us-east-1", + aws_access_key_id="ASIAZTMXAXNXD3TOUOAJ", + aws_secret_access_key="pk7k69CqXZPB/bf2hdFsW+47D5WYkoWXFdxQ633X", + aws_session_token="IQoJb3JpZ2luX2VjELj//////////wEaCXVzLWVhc3QtMiJIMEYCIQCqcyLSwLeN9RX6tz+TgB4VMabHYwPcA3bTzx7xZ5b75QIhAMiOKbDlmPZ+eHkeskzelK4M9gWtS7GqlzJq5qmWbbrOKo4DCKH//////////wEQABoMNjYwMTMxMDY4NzgyIgzPuv0vReC65Mo/bQcq4gLXtII+VTaxES+JrjHHWpWywdmpsJneN6bcB3U37z7J8BFA5aaUqkATKmwJI0brmk5ZJKL8SpgBEC2TNdA/V9nzbMlf2HPunhEv6OPLzSWp5iJqaNL945MP764CbkYvfN9QWd6durUv1WgGZRNcbMzXg2UFsxKcRql795vtOmL207+R7uIouWl73So7NaCkEgaj4FdEJ9lbnfvWFeNcBlHbjwUx8e9EJjwm8D60OkTdS4w7Q3EacoEKLO94/kp2RtsaggAUV33OcvO/32VwYJzhRJYuveQUZnIzfkmybGtrXkkWLqMO9pls1bkTmIjaeMwcL8Uo7oowR9sTFCT87rY711yIYBGVOjkN9mfavPH4FSCNeeI6ta5aXqa7iVDU7rPerpFtle2i1VGfTW5bKoaWxjO13IdqI4yI9Pgibl+FuVeRB7md2tDuS6SMAuX4qpWnMVDLKCItlAxXlZdmVhuDFf8wr+7rrgY6pQH02EIrvCkYYXbJd8u7t78hOam3lSNPz+nghQHA5ppl7TnyRBL5/9Rfqp5Ib8y54HdC4hbzz/7w6lvfS0QbHE3Q6r2GC6XWk9L4FAxD8pyZkIYbtwf/WueX1g0r+0x7uLCxKxbWsYum/bigyvNxsDyFIcQcUOm+OJsXGPH3Z7z+qWKni5N9RRA4SEEpdSTRkAWipDtgVNII6kd/eWvyE74d+LYUrmA=" + ) + embeddings = BedrockEmbeddings( + client=bedrock_runtime, + model_id="amazon.titan-embed-text-v1", + ) + + """ + db = Chroma.from_documents( + texts, + embeddings, + persist_directory=PERSIST_DIRECTORY, + client_settings=CHROMA_SETTINGS, + ) + """ + + # for batch_texts in process_in_batches(texts, 20000): # https://github.com/PromtEngineer/localGPT/issues/489y + print(filename) + + db = Chroma.from_documents( + texts, + embeddings, + persist_directory=PERSIST_DIRECTORY, + client_settings=CHROMA_SETTINGS, + collection_metadata={"hnsw:space": "cosine"}, + # ids = [str(filename)] + ) + + + +if __name__ == "__main__": + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(filename)s:%(lineno)s - %(message)s", level=logging.INFO + ) + main() diff --git a/streamlit/interface_1.py b/streamlit/interface_1.py new file mode 100644 index 0000000..400adb0 --- /dev/null +++ b/streamlit/interface_1.py @@ -0,0 +1,112 @@ +import streamlit as st +from streamlit_extras.add_vertical_space import add_vertical_space +import os +import streamlit as st +import pandas as pd +from io import StringIO +from datetime import datetime + +st.set_page_config(layout = "wide") + +# # Sidebar contents +# with st.sidebar: +# st.title("Doczy.AI ™") +# st.markdown( +# """ +# ## About +# This app extracts data from contracts + +# """ +# ) +# add_vertical_space(15) +# # st.write("Doczy") + + +client_row = st.columns([0.1, 0.8]) +with client_row[0]: + st.write("**Client Name**") +with client_row[1]: + client = st.selectbox('Client Name',('Delaware First Health, Inc.', 'Community Health Choice, Inc','CareSource Network Partners LLC', + 'HealthNet of Cali', 'Oklahoma Complete Health, Inc', 'HealthFirst', 'Molina Healthcare of TX', 'AvMed', 'Arizona Care1st', + 'WellCare New Jersey'), label_visibility = "collapsed") + +def file_selector(folder_path='.'): + filenames = os.listdir(folder_path) + selected_filename = st.selectbox('**Path to folder**', filenames, label_visibility = "collapsed") + return os.path.join(folder_path, selected_filename) + # return selected_filename + +path_row = st.columns([0.1, 0.8]) +with path_row[0]: + st.write("**Path to folder**") +with path_row[1]: + # Directory = st.text_input("**Path to folder**", label_visibility = "collapsed") + Directory = file_selector() + +checks = st.columns([0.1, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12]) +with checks[0]: + st.write("**Group No.**") +with checks[1]: + a = st.checkbox('Unique Key', key = str(1)) +with checks[2]: + b = st.checkbox('Pricing Before Carveouts', key = str(2)) +with checks[3]: + c = st.checkbox('Contract Related', key = str(3)) +with checks[4]: + d = st.checkbox('Provider', key = str(4)) +with checks[5]: + e = st.checkbox('Timeline', key = str(5)) +with checks[6]: + f = st.checkbox('Carveout Indicator', key = str(6)) +with checks[7]: + g = st.checkbox('Carveout Methodology', key = str(7)) + +add_vertical_space(1) + +df = pd.DataFrame(columns=['Request ID','Contract ID','Contract Name','Unique Key','Pricing Before Carveouts' + , 'Contract Related', 'Provider', 'Timeline', 'Carveout Indicator', 'Carveout Methodology']) +file_list = [] + +if st.button("Read the contracts from Path"): + for filename in os.listdir(Directory): + # with open(os.path.join(Directory, filename), encoding="utf8") as f: + # context = f.read() + file_list.append(filename) + + df['Contract Name'] = file_list + df['Request ID'] = range(len(file_list)) + df['Contract ID'] = file_list + # df['Folder Name'] = Directory + # df['Updated Group Number'] = updated_group + df['Unique Key'] = a + df['Pricing Before Carveouts'] = b + df['Contract Related'] = c + df['Provider'] = d + df['Timeline'] = e + df['Carveout Indicator'] = f + df['Carveout Methodology'] = g + df.to_csv('temp1.csv', index=False) + +add_vertical_space(1) + +# df_copy = df.set_index(df.columns[0]).copy() +df2 = pd.read_csv('temp1.csv') +edited_df = st.data_editor(df2) + + +@st.cache_data +def convert_df(df): + return df.to_csv(index=False).encode('utf-8') + +csv = convert_df(edited_df) + +buttons = st.columns([0.45, 0.35, 0.2]) +with buttons[0]: + st.button("Save All Edits") +with buttons[1]: + st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv') +with buttons[2]: + st.button("Run Doczy.AI Pipeline") + + + diff --git a/streamlit/interface_2.py b/streamlit/interface_2.py new file mode 100644 index 0000000..9a27c78 --- /dev/null +++ b/streamlit/interface_2.py @@ -0,0 +1,197 @@ +import json + +import boto3 +from langchain.prompts import PromptTemplate +from langchain.embeddings.bedrock import BedrockEmbeddings +from langchain.llms.bedrock import Bedrock +from langchain_community.vectorstores import Chroma +from constants import CHROMA_SETTINGS, EMBEDDING_MODEL_NAME, PERSIST_DIRECTORY, MODEL_ID, MODEL_BASENAME, SOURCE_DIRECTORY +from langchain.chains import RetrievalQA + +import streamlit as st +from streamlit_extras.add_vertical_space import add_vertical_space +import os +import pandas as pd + +st.set_page_config(layout = "wide") + +# Sidebar contents +with st.sidebar: + st.title("Doczy.AI ™") + st.markdown( + """ + ## About + This app extracts data from contracts + + """ + ) + add_vertical_space(15) + # st.write("Doczy") + +sample = pd.read_csv('sample.csv') +sample = sample[['Filename', 'Attribute', 'Query', 'Answer']] +attribute_list = ['Agreement Name', 'Agreement Type', 'Contract State', 'Contract Type', 'Create Date', 'Effective Date', 'Gold Carded', + 'Modify Date', 'National Contract', 'Provider State', 'Summary', 'Termination Date', 'TIN', 'Value-Based Contract'] +sample = sample[sample['Attribute'].isin(attribute_list)] +field_prompt_mapping = sample[['Attribute', 'Query']].drop_duplicates().dropna() +field_prompt_mapping = dict(zip(field_prompt_mapping.Attribute, field_prompt_mapping.Query)) + +def file_selector(folder_path='RAW_DOCUMENTS'): + filenames = os.listdir(folder_path) + selected_filename = st.selectbox('Select a file', filenames, label_visibility = "collapsed") + # return os.path.join(folder_path, selected_filename) + return selected_filename + +file_row = st.columns([0.2, 0.7, 0.1]) +with file_row[0]: + st.write("**Contract Name**") +with file_row[1]: + # file_name = st.text_input("**Contract Name**", label_visibility = "collapsed") + file_name = file_selector() + +lob_row = st.columns([0.2, 0.7, 0.1]) +with lob_row[0]: + st.write("**LOB**") +with lob_row[1]: + lob = st.selectbox('LOB',('Medicare', 'Medicaid'), label_visibility = "collapsed") + +llm_row = st.columns([0.2, 0.7, 0.1]) +with llm_row[0]: + st.write("**Langauge Model**") +with llm_row[1]: + llm_selected = st.selectbox('Langauge Model',('Llama 2 Chat 13B', 'Llama 2 Chat 70B', 'Titan Text Express'), label_visibility = "collapsed") + +page_list = [] +with open(os.path.join(SOURCE_DIRECTORY, file_name[:-4]+'.txt'), 'r') as infile: + text = infile.read() +page_count = text.count('Start of Page No. = ') +for page in range(page_count+1): + file_path = "SOURCE_DOCUMENTS\\" + f'{file_name[:-4]}_page{page}.txt' + dict_with_pages = { 'source': { '$eq': file_path }} + page_list.append(dict_with_pages) + +# AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID') +# AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY') +# AWS_SESSION_TOKEN=os.getenv('AWS_SESSION_TOKEN') + +# Setup bedrock +bedrock_runtime = boto3.client( + service_name="bedrock-runtime", + region_name="us-east-1" +) + +embeddings = BedrockEmbeddings( + client=bedrock_runtime, + model_id="amazon.titan-embed-text-v1", +) +DB = Chroma( + persist_directory=PERSIST_DIRECTORY, + embedding_function=embeddings, + client_settings=CHROMA_SETTINGS, +) +RETRIEVER = DB.as_retriever(search_kwargs={"filter": {"$or": page_list}, "k": 4}) + +if llm_selected == 'Titan Text Express': + LLM = Bedrock( + model_id="amazon.titan-text-express-v1", + client=bedrock_runtime, + model_kwargs={ + "maxTokenCount": 4096, + "stopSequences": [], + "temperature": 0, + "topP": 1, + } + ) +elif llm_selected == 'Llama 2 Chat 70B': + LLM = Bedrock( + model_id="meta.llama2-70b-chat-v1", + client=bedrock_runtime, + model_kwargs={ + "max_gen_len": 512, + "temperature": 0, + # "topP": 0.9, + } + ) +else: + LLM = Bedrock( + model_id="meta.llama2-13b-chat-v1", + client=bedrock_runtime, + model_kwargs={ + "max_gen_len": 512, + "temperature": 0, + # "topP": 0.9, + } + ) + +template = """ + +Use the following pieces of context to answer the question at the end. If you don't know the answer,\ +just say that you don't know, don't try to make up an answer. + +{context} + +Question: {question} +Answer:""" +prompt = PromptTemplate(input_variables=["context", "question"], template=template) + +QA = RetrievalQA.from_chain_type( + llm=LLM, + chain_type="stuff", + retriever=RETRIEVER, + return_source_documents=True, + chain_type_kwargs={"prompt": prompt}, +) + +# query = "In which state or states is the Contract applicable? Answer in one or two words. State name: " +# response = QA({"query":query}) +# st.write(query) +# st.write(response['result']) +# st.write("-----------") +# st.write(response) + +# clicked = st.button("Show Results") +df = pd.DataFrame(columns=['Contract Name','Field Name','Snippet','Page Number','Confidence Level', + 'Field Extracted Value','Imputed Value']) +field_list = list(field_prompt_mapping.keys()) +query_list = [field_prompt_mapping[x] for x in field_list] +score_list = [DB.similarity_search_with_relevance_scores(query, k=4, filter={"$or": page_list}) for query in query_list] +confidence_list = [] +for score in score_list: + confidence_list.append(max(d[1] for d in score)) +# st.write(confidence_list) + +if st.button("Show Results"): + response_list = [QA({"query":query}) for query in query_list] + answer_list = [response['result'] for response in response_list] + doc_list = [response['source_documents'] for response in response_list] + snippet_list = [str(doc[0].page_content) for doc in doc_list] + page_no_list = [int(str(doc[0].metadata["source"]).rsplit('_page')[1].replace('.txt',''))+1 for doc in doc_list] + + df['Field Name'] = field_list + df['Contract Name'] = file_name + df['Snippet'] = snippet_list + df['Page Number'] = page_no_list + df['Confidence Level'] = confidence_list + df['Field Extracted Value'] = answer_list + df.to_csv('temp2.csv', index=False) + +df2 = pd.read_csv('temp2.csv') +df2['Imputed Value'] = '' +edited_df = st.data_editor(df2) + +@st.cache_data +def convert_df(df): + return df.to_csv(index=False).encode('utf-8') + +csv = convert_df(edited_df) + +buttons = st.columns(3) +with buttons[0]: + st.button("Save All Imputations") +with buttons[1]: + st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv') +with buttons[2]: + st.button("Kickoff Database Integration") + + + diff --git a/streamlit/interface_3.py b/streamlit/interface_3.py new file mode 100644 index 0000000..2cb2b06 --- /dev/null +++ b/streamlit/interface_3.py @@ -0,0 +1,334 @@ +import json +import boto3 +from langchain.prompts import PromptTemplate +from langchain.embeddings.bedrock import BedrockEmbeddings +from langchain.llms.bedrock import Bedrock +from langchain_community.vectorstores import Chroma +from constants import CHROMA_SETTINGS, EMBEDDING_MODEL_NAME, PERSIST_DIRECTORY, MODEL_ID, MODEL_BASENAME, SOURCE_DIRECTORY +from langchain.chains import RetrievalQA + +import streamlit as st +from streamlit_extras.add_vertical_space import add_vertical_space + +import pandas as pd +from datetime import datetime +import random +import os +import dateutil + +st.set_page_config(layout = "wide") + +# Sidebar contents +with st.sidebar: + st.title("Doczy.AI ™") + st.markdown( + """ + ## About + This app extracts data from contracts + + """ + ) + add_vertical_space(15) + # st.write("Doczy") + +fields = pd.read_csv('contract_fields.csv', encoding='unicode_escape', skipinitialspace=True) +fields = fields[fields['PRIORITY'].isin(['A','C'])] +field_values = pd.read_csv('contract_field_values.csv', encoding='unicode_escape', skipinitialspace=True) +fields = fields.drop_duplicates(subset='Field Name', keep="first").sort_values('Field Name') +fields = fields[~fields['Field Name'].isnull()] +field_prompt_mapping = dict(zip(fields['Field Name'], fields['Interrogation Question?'])) + +field_row = st.columns([0.15, 0.45, 0.4]) +with field_row[0]: + st.write("**Field Name**") +with field_row[1]: + field = st.selectbox('Field Name',sorted(set(field_prompt_mapping.keys())), index=0, label_visibility = "collapsed") + +contract_count_row = st.columns([0.15, 0.45, 0.4]) +with contract_count_row[0]: + st.write("**# of Contracts**") +with contract_count_row[1]: + contract_count = st.selectbox('Contract count',('10', '20', '30', '50', 'All'), index=0, label_visibility = "collapsed") + +seed_row = st.columns([0.15, 0.45, 0.4]) +with seed_row[0]: + st.write("**Seed Value**") +with seed_row[1]: + seed_value = st.text_input("**Seed Value**", value = 20, label_visibility = "collapsed") + +llm_row = st.columns([0.15, 0.45, 0.4]) +with llm_row[0]: + st.write("**Langauge Model**") +with llm_row[1]: + llm_selected = st.selectbox('Langauge Model',('Claude 2', 'Claude Instant', 'Llama 2 Chat 13B', 'Llama 2 Chat 70B' + , 'Titan Text Express'), label_visibility = "collapsed") + +st.write("**Prompt**") +sequence_input = field_prompt_mapping.get(field) +prompt_row = st.columns([0.8, 0.2]) +with prompt_row[1]: + if st.button("Clear Prompt"): + sequence_input = '' + if st.button("Back to default"): + prompt = sequence_input + st.button("Save Prompt") +with prompt_row[0]: + prompt = st.text_area("**Prompt**", sequence_input, height = 150, label_visibility = "collapsed") + +random.seed(seed_value) +try: + contract_list = sorted(random.choices(os.listdir(SOURCE_DIRECTORY), k=int(contract_count))) +except: + contract_list = sorted(os.listdir(SOURCE_DIRECTORY)) + +# to be deleted later +contract_list = [contract for contract in contract_list if contract.replace(' MU','').replace('_MU','').replace('.txt','') in list(field_values['(internal) Document Name'])] + +page_list_all = [] +for contract in contract_list: + page_list = [] + with open(os.path.join(SOURCE_DIRECTORY, contract[:-4]+'.txt'), 'r') as infile: + text = infile.read() + page_count = text.count('Start of Page No. = ') + for page in range(page_count+1): + file_path = "SOURCE_DOCUMENTS\\" + f'{contract[:-4]}_page{page}.txt' + dict_with_pages = { 'source': { '$eq': file_path }} + page_list.append(dict_with_pages) + page_list_all.append(page_list) +contract_txt_mapping = dict(zip(contract_list, page_list_all)) + +column_name = fields.loc[fields['Field Name'] == field, 'SF_DB_COL_NAME'].iloc[0] +column_list = ['(internal) Document Name', '(Internal) Carveout ID', column_name] +if column_name+'_PG' in list(field_values.columns): + column_list.append(column_name+'_PG') +field_values = field_values[column_list] +field_values.rename(columns={'(internal) Document Name': 'Contract Name', column_name: 'Actual Value Stored' + , '(Internal) Carveout ID': 'Contract ID', column_name+'_PG': 'Original Page Number'}, inplace=True) +field_values = field_values.drop_duplicates(subset='Contract Name', keep="first").sort_values('Contract Name') + +# sample = dict(zip(sample.Filename, sample.Answer)) +# actual_value_list = [sample.get(contract.rsplit('.',1)[0]+'.txt', ' ') for contract in contract_list] + +# AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID') +# AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY') +# AWS_SESSION_TOKEN=os.getenv('AWS_SESSION_TOKEN') + +# Setup bedrock +bedrock_runtime = boto3.client( + service_name="bedrock-runtime", + region_name="us-east-1", +) + +# Define the retreiver +# load the vectorstore +if "EMBEDDINGS" not in st.session_state: + EMBEDDINGS = BedrockEmbeddings( + client=bedrock_runtime, + model_id="amazon.titan-embed-text-v1", + ) + st.session_state.EMBEDDINGS = EMBEDDINGS + +if "DB" not in st.session_state: + DB = Chroma( + persist_directory=PERSIST_DIRECTORY, + embedding_function=st.session_state.EMBEDDINGS, + client_settings=CHROMA_SETTINGS, + ) + st.session_state.DB = DB + +# if "RETRIEVER" not in st.session_state: +# # { "source": { '$eq': "SOURCE_DOCUMENTS\\A.1_UH_Health_System_eff_2_1_08 (1)_page0.txt"} } +# RETRIEVER = DB.as_retriever(search_kwargs={"filter": { "source": { '$eq': "SOURCE_DOCUMENTS\\A.1_UH_Health_System_eff_2_1_08 (1)_page0.txt"} }, "k": 2}) +# st.session_state.RETRIEVER = RETRIEVER + +# if "LLM" not in st.session_state: +if llm_selected == 'Titan Text Express': + LLM = Bedrock( + model_id="amazon.titan-text-express-v1", + client=bedrock_runtime, + model_kwargs={ + "maxTokenCount": 512, + "stopSequences": [], + "temperature": 0, + "topP": 1, + } + ) +elif llm_selected == 'Llama 2 Chat 70B': + LLM = Bedrock( + model_id="meta.llama2-70b-chat-v1", + client=bedrock_runtime, + model_kwargs={ + "max_gen_len": 512, + "temperature": 0, + # "topP": 0.9, + } + ) +elif llm_selected == 'Llama 2 Chat 13B': + LLM = Bedrock( + model_id="meta.llama2-13b-chat-v1", + client=bedrock_runtime, + model_kwargs={ + "max_gen_len": 512, + "temperature": 0, + # "topP": 0.9, + } + ) +elif llm_selected == 'Claude Instant': + LLM = Bedrock( + model_id="anthropic.claude-instant-v1", + client=bedrock_runtime, + model_kwargs={ + # "max_tokens_to_sample": 512, + "temperature": 0, + # "topP": 0.9, + } + ) +elif llm_selected == 'Claude 2': + LLM = Bedrock( + model_id="anthropic.claude-v2:1", + client=bedrock_runtime, + model_kwargs={ + # "max_tokens_to_sample": 512, + "temperature": 0, + # "topP": 0.9, + } + ) +st.session_state["LLM"] = LLM + +# if "QA" not in st.session_state: +# prompt, memory = model_memory() + +# QA = RetrievalQA.from_chain_type( +# llm=LLM, +# chain_type="stuff", +# retriever=RETRIEVER, +# return_source_documents=True, +# chain_type_kwargs={"prompt": prompt, "memory": memory}, +# ) +# st.session_state["QA"] = QA + +# df = pd.DataFrame(columns=['Contract Name','Contract ID','Actual Value Stored','New Extracted value','Confidence Level','Snippet', +# 'Original Page Number', 'New Page Number', 'Revised Prompt', 'Result']) +df = pd.DataFrame(columns=['Contract Name','Raw value','New Extracted value','Confidence Level','Snippet','New Page Number' + , 'Revised Prompt', 'Result']) +try: + history = pd.read_csv('history.csv') +except: + history = pd.DataFrame(columns=['Field Name','# Contracts Tested', 'Username', 'Date/Time', 'Accuracy', 'Attempt #']) +attempt = 0 + +if llm_selected in ['Llama 2 Chat 13B', 'Llama 2 Chat 70B']: + k_value = 14 +else: + k_value = 25 + + +if st.button("Test Configuration"): + + answer_list = [] + doc_list = [] + response_list = [] + score_list = [] + attempt = attempt + 1 + + for page_list in page_list_all: + RETRIEVER = st.session_state.DB.as_retriever(search_kwargs={"filter": {"$or": page_list}, "k": k_value}) + QA = RetrievalQA.from_chain_type( + llm=st.session_state["LLM"], + chain_type="stuff", + retriever=RETRIEVER, + return_source_documents=True, + # chain_type_kwargs={"prompt": prompt, "memory": None}, + ) + score = st.session_state.DB.similarity_search_with_relevance_scores(prompt, k=4, filter={"$or": page_list}) + score_list.append(max(d[1] for d in score)) + response = QA(prompt) + answer, docs = response["result"], response["source_documents"] + answer_list.append(answer) + doc_list.append(docs) + response_list.append(response) + + df['Raw value'] = answer_list + # post-processing + if 'Date' in field: + date_list = [] + for answer in answer_list: + try: + extracted_date = dateutil.parser.parse(str(answer).replace('"',''), fuzzy=True).date() + except: + extracted_date = " " + date_list.append(extracted_date) + answer_list = date_list + elif llm_selected in ['Llama 2 Chat 13B', 'Llama 2 Chat 70B']: + answer_list = [answer.rstrip(".") for answer in answer_list] + answer_list = [answer if "I don't know" not in str(answer) else " " for answer in answer_list] + answer_list = [answer if "N/A" not in str(answer) else " " for answer in answer_list] + answer_list = [answer if "does not contain" not in str(answer) else " " for answer in answer_list] + answer_list = [answer if "None" not in str(answer) else " " for answer in answer_list] + answer_list = [answer if "Not specified in the contract" not in str(answer) else " " for answer in answer_list] + answer_list = [answer if "Not applicable" not in str(answer) else " " for answer in answer_list] + elif llm_selected in ['Claude 2', 'Claude Instant']: + answer_list = [answer if "Unfortunately, I do not have enough context" not in str(answer) else " " for answer in answer_list] + answer_list = [answer.rstrip(".") for answer in answer_list] + else: + answer_list = [answer.rstrip(".") for answer in answer_list] + # answer_list = [str(x).rsplit(':',1)[0] if len(str(x).rsplit(':',1)) < 2 else str(x).rsplit(':',1)[1] for x in answer_list] + + df['Contract Name'] = contract_list + # to be deleted later + df['Contract Name'] = [contract.replace(' MU','').replace('_MU','').replace('.txt','') for contract in contract_list] + + df['New Extracted value'] = answer_list + df['Confidence Level'] = [round(score, 2) for score in score_list] + df['Snippet'] = [str(doc[0].page_content) for doc in doc_list] + df['New Page Number'] = [int(str(doc[0].metadata["source"]).rsplit('_page')[1].replace('.txt',''))+1 for doc in doc_list] + df['Revised Prompt'] = [prompt] * len(contract_list) + + df = pd.merge(df, field_values, how ='left', on ='Contract Name') + answer_list = list(df['New Extracted value']) + actual_value_list = list(df['Actual Value Stored']) + result_list = [i==j for i, j in zip(actual_value_list, answer_list)] + df['Result'] = [str(x) for x in result_list] + df = df[~df['Contract ID'].isnull()] + + if 'Original Page Number' in df.columns: + df = df[['Contract Name','Contract ID','Actual Value Stored','Raw value','New Extracted value','Confidence Level','Snippet' + ,'Original Page Number', 'New Page Number', 'Revised Prompt', 'Result']] + else: + df = df[['Contract Name','Contract ID','Actual Value Stored','Raw value','New Extracted value','Confidence Level','Snippet' + , 'New Page Number', 'Revised Prompt', 'Result']] + + try: + accuracy = round(sum(bool(x) for x in result_list) * 100 / len(list(df['Result'])), 2) + except: + accuracy = 'NA' + + history.loc[len(history.index)] = [field, str(contract_count), None, datetime.now().strftime("%Y-%m-%d %H:%M:%S"), accuracy, attempt] + df.to_csv("RESULTS\\"+field.replace("?","").replace("/","_")+'-'+llm_selected+'.csv', index=False) + history.to_csv('history.csv', index=False) + +# df_copy = df.set_index(df.columns[0]).copy() +# df_2_copy = history.set_index(history.columns[0]).copy() +st.dataframe(df) +st.dataframe(history) + + +# @st.cache_data +# def convert_df(df): +# return df.to_csv(index=False).encode('utf-8') + +# csv = convert_df(edited_df) + +# buttons = st.columns(3) +# with buttons[0]: +# st.button("Save All Imputations") +# with buttons[1]: +# st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv') +# with buttons[2]: +# st.button("Kickoff Database Integration") + + +st.write(column_name) +st.write(len(contract_list)) + + diff --git a/streamlit/requirements.txt b/streamlit/requirements.txt new file mode 100644 index 0000000..18cb8ac --- /dev/null +++ b/streamlit/requirements.txt @@ -0,0 +1,38 @@ +# Natural Language Processing +langchain==0.0.267 +chromadb==0.4.6 +pdfminer.six==20221105 +InstructorEmbedding +sentence-transformers==2.2.2 +faiss-cpu +huggingface_hub +transformers +autoawq +protobuf==3.20.2; sys_platform != 'darwin' +protobuf==3.20.2; sys_platform == 'darwin' and platform_machine != 'arm64' +protobuf==3.20.3; sys_platform == 'darwin' and platform_machine == 'arm64' +auto-gptq==0.2.2 +docx2txt +unstructured +unstructured[pdf] + +# Utilities +urllib3==1.26.6 +accelerate +bitsandbytes ; sys_platform != 'win32' +bitsandbytes-windows ; sys_platform == 'win32' +click +flask +requests + +# Streamlit related +streamlit +Streamlit-extras + +# Excel File Manipulation +openpyxl +numpy>=1.22.2 + +# AWS related +boto3 +awscli