Mayank Aamseek
+11
@@ -48,3 +48,14 @@ Thumbs.db
|
||||
*.mov
|
||||
*.wmv
|
||||
|
||||
# Data Files
|
||||
streamlit/history.csv
|
||||
streamlit/RESULTS/
|
||||
streamlit/DB/
|
||||
streamlit/RAW_DOCUMENTS/
|
||||
streamlit/SOURCE_DOCUMENTS/
|
||||
streamlit/contract_field_values.csv
|
||||
streamlit/contract_fields.csv
|
||||
streamlit/sample.csv
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
import os
|
||||
|
||||
# from dotenv import load_dotenv
|
||||
from chromadb.config import Settings
|
||||
|
||||
# https://python.langchain.com/en/latest/modules/indexes/document_loaders/examples/excel.html?highlight=xlsx#microsoft-excel
|
||||
from langchain_community.document_loaders import CSVLoader, PDFMinerLoader, TextLoader, UnstructuredExcelLoader, Docx2txtLoader
|
||||
from langchain_community.document_loaders import UnstructuredFileLoader, UnstructuredMarkdownLoader
|
||||
|
||||
|
||||
# load_dotenv()
|
||||
# ROOT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
|
||||
ROOT_DIRECTORY = "\\\\amznfsxuofkyi1z.aarete.local\\SharedFiles\\AArete Client Work\\Modahealth\\Restricted\\Moda Growth\\Artificial Intelligence\\DEFAXXER_20231207"
|
||||
|
||||
# Define the folder for source and output
|
||||
SOURCE_DIRECTORY = "SOURCE_DOCUMENTS"
|
||||
OUTPUT_DIRECTORY = f"{ROOT_DIRECTORY}\\Output"
|
||||
|
||||
PERSIST_DIRECTORY = 'DB'
|
||||
|
||||
MODELS_PATH = "C:\\Users\\Public\\models"
|
||||
|
||||
# Can be changed to a specific number
|
||||
INGEST_THREADS = os.cpu_count() or 8
|
||||
|
||||
# Define the Chroma settings
|
||||
CHROMA_SETTINGS = Settings(
|
||||
anonymized_telemetry=False,
|
||||
is_persistent=True,
|
||||
allow_reset=True,
|
||||
)
|
||||
|
||||
# Context Window and Max New Tokens
|
||||
CONTEXT_WINDOW_SIZE = 4096
|
||||
MAX_NEW_TOKENS = CONTEXT_WINDOW_SIZE # int(CONTEXT_WINDOW_SIZE/4)
|
||||
|
||||
#### If you get a "not enough space in the buffer" error, you should reduce the values below, start with half of the original values and keep halving the value until the error stops appearing
|
||||
|
||||
N_GPU_LAYERS = 100 # Llama-2-70B has 83 layers
|
||||
N_BATCH = 512
|
||||
|
||||
### From experimenting with the Llama-2-7B-Chat-GGML model on 8GB VRAM, these values work:
|
||||
# N_GPU_LAYERS = 20
|
||||
# N_BATCH = 512
|
||||
|
||||
|
||||
# https://python.langchain.com/en/latest/_modules/langchain/document_loaders/excel.html#UnstructuredExcelLoader
|
||||
DOCUMENT_MAP = {
|
||||
".txt": TextLoader,
|
||||
".md": UnstructuredMarkdownLoader,
|
||||
".py": TextLoader,
|
||||
# ".pdf": PDFMinerLoader,
|
||||
".pdf": UnstructuredFileLoader,
|
||||
".csv": CSVLoader,
|
||||
".xls": UnstructuredExcelLoader,
|
||||
".xlsx": UnstructuredExcelLoader,
|
||||
".docx": Docx2txtLoader,
|
||||
".doc": Docx2txtLoader,
|
||||
}
|
||||
|
||||
# Default Instructor Model
|
||||
EMBEDDING_MODEL_NAME = "hkunlp/instructor-large" # Uses 1.5 GB of VRAM (High Accuracy with lower VRAM usage)
|
||||
|
||||
####
|
||||
#### OTHER EMBEDDING MODEL OPTIONS
|
||||
####
|
||||
|
||||
# EMBEDDING_MODEL_NAME = "hkunlp/instructor-xl" # Uses 5 GB of VRAM (Most Accurate of all models)
|
||||
# EMBEDDING_MODEL_NAME = "intfloat/e5-large-v2" # Uses 1.5 GB of VRAM (A little less accurate than instructor-large)
|
||||
# EMBEDDING_MODEL_NAME = "intfloat/e5-base-v2" # Uses 0.5 GB of VRAM (A good model for lower VRAM GPUs)
|
||||
# EMBEDDING_MODEL_NAME = "all-MiniLM-L6-v2" # Uses 0.2 GB of VRAM (Less accurate but fastest - only requires 150mb of vram)
|
||||
|
||||
####
|
||||
#### MULTILINGUAL EMBEDDING MODELS
|
||||
####
|
||||
|
||||
# EMBEDDING_MODEL_NAME = "intfloat/multilingual-e5-large" # Uses 2.5 GB of VRAM
|
||||
# EMBEDDING_MODEL_NAME = "intfloat/multilingual-e5-base" # Uses 1.2 GB of VRAM
|
||||
|
||||
|
||||
#### SELECT AN OPEN SOURCE LLM (LARGE LANGUAGE MODEL)
|
||||
# Select the Model ID and model_basename
|
||||
# load the LLM for generating Natural Language responses
|
||||
|
||||
#### GPU VRAM Memory required for LLM Models (ONLY) by Billion Parameter value (B Model)
|
||||
#### Does not include VRAM used by Embedding Models - which use an additional 2GB-7GB of VRAM depending on the model.
|
||||
####
|
||||
#### (B Model) (float32) (float16) (GPTQ 8bit) (GPTQ 4bit)
|
||||
#### 7b 28 GB 14 GB 7 GB - 9 GB 3.5 GB - 5 GB
|
||||
#### 13b 52 GB 26 GB 13 GB - 15 GB 6.5 GB - 8 GB
|
||||
#### 32b 130 GB 65 GB 32.5 GB - 35 GB 16.25 GB - 19 GB
|
||||
#### 65b 260.8 GB 130.4 GB 65.2 GB - 67 GB 32.6 GB - - 35 GB
|
||||
|
||||
# MODEL_ID = "TheBloke/Llama-2-7B-Chat-GGML"
|
||||
# MODEL_BASENAME = "llama-2-7b-chat.ggmlv3.q4_0.bin"
|
||||
|
||||
####
|
||||
#### (FOR GGUF MODELS)
|
||||
####
|
||||
|
||||
# MODEL_ID = "TheBloke/Llama-2-13b-Chat-GGUF"
|
||||
# MODEL_BASENAME = "llama-2-13b-chat.Q4_K_M.gguf"
|
||||
|
||||
MODEL_ID = "TheBloke/Llama-2-7b-Chat-GGUF"
|
||||
MODEL_BASENAME = "llama-2-7b-chat.Q4_K_M.gguf"
|
||||
|
||||
# MODEL_ID = "TheBloke/Mistral-7B-Instruct-v0.1-GGUF"
|
||||
# MODEL_BASENAME = "mistral-7b-instruct-v0.1.Q8_0.gguf"
|
||||
|
||||
# MODEL_ID = "TheBloke/Llama-2-70b-Chat-GGUF"
|
||||
# MODEL_BASENAME = "llama-2-70b-chat.Q4_K_M.gguf"
|
||||
|
||||
####
|
||||
#### (FOR HF MODELS)
|
||||
####
|
||||
|
||||
# MODEL_ID = "NousResearch/Llama-2-7b-chat-hf"
|
||||
# MODEL_BASENAME = None
|
||||
# MODEL_ID = "TheBloke/vicuna-7B-1.1-HF"
|
||||
# MODEL_BASENAME = None
|
||||
# MODEL_ID = "TheBloke/Wizard-Vicuna-7B-Uncensored-HF"
|
||||
# MODEL_ID = "TheBloke/guanaco-7B-HF"
|
||||
# MODEL_ID = 'NousResearch/Nous-Hermes-13b' # Requires ~ 23GB VRAM. Using STransformers
|
||||
# alongside will 100% create OOM on 24GB cards.
|
||||
# llm = load_model(device_type, model_id=model_id)
|
||||
|
||||
####
|
||||
#### (FOR GPTQ QUANTIZED) Select a llm model based on your GPU and VRAM GB. Does not include Embedding Models VRAM usage.
|
||||
####
|
||||
|
||||
##### 48GB VRAM Graphics Cards (RTX 6000, RTX A6000 and other 48GB VRAM GPUs) #####
|
||||
|
||||
### 65b GPTQ LLM Models for 48GB GPUs (*** With best embedding model: hkunlp/instructor-xl ***)
|
||||
# MODEL_ID = "TheBloke/guanaco-65B-GPTQ"
|
||||
# MODEL_BASENAME = "model.safetensors"
|
||||
# MODEL_ID = "TheBloke/Airoboros-65B-GPT4-2.0-GPTQ"
|
||||
# MODEL_BASENAME = "model.safetensors"
|
||||
# MODEL_ID = "TheBloke/gpt4-alpaca-lora_mlp-65B-GPTQ"
|
||||
# MODEL_BASENAME = "model.safetensors"
|
||||
# MODEL_ID = "TheBloke/Upstage-Llama1-65B-Instruct-GPTQ"
|
||||
# MODEL_BASENAME = "model.safetensors"
|
||||
|
||||
##### 24GB VRAM Graphics Cards (RTX 3090 - RTX 4090 (35% Faster) - RTX A5000 - RTX A5500) #####
|
||||
|
||||
### 13b GPTQ Models for 24GB GPUs (*** With best embedding model: hkunlp/instructor-xl ***)
|
||||
# MODEL_ID = "TheBloke/Wizard-Vicuna-13B-Uncensored-GPTQ"
|
||||
# MODEL_BASENAME = "Wizard-Vicuna-13B-Uncensored-GPTQ-4bit-128g.compat.no-act-order.safetensors"
|
||||
# MODEL_ID = "TheBloke/vicuna-13B-v1.5-GPTQ"
|
||||
# MODEL_BASENAME = "model.safetensors"
|
||||
# MODEL_ID = "TheBloke/Nous-Hermes-13B-GPTQ"
|
||||
# MODEL_BASENAME = "nous-hermes-13b-GPTQ-4bit-128g.no-act.order"
|
||||
# MODEL_ID = "TheBloke/WizardLM-13B-V1.2-GPTQ"
|
||||
# MODEL_BASENAME = "gptq_model-4bit-128g.safetensors
|
||||
|
||||
### 30b GPTQ Models for 24GB GPUs (*** Requires using intfloat/e5-base-v2 instead of hkunlp/instructor-large as embedding model ***)
|
||||
# MODEL_ID = "TheBloke/Wizard-Vicuna-30B-Uncensored-GPTQ"
|
||||
# MODEL_BASENAME = "Wizard-Vicuna-30B-Uncensored-GPTQ-4bit--1g.act.order.safetensors"
|
||||
# MODEL_ID = "TheBloke/WizardLM-30B-Uncensored-GPTQ"
|
||||
# MODEL_BASENAME = "WizardLM-30B-Uncensored-GPTQ-4bit.act-order.safetensors"
|
||||
|
||||
##### 8-10GB VRAM Graphics Cards (RTX 3080 - RTX 3080 Ti - RTX 3070 Ti - 3060 Ti - RTX 2000 Series, Quadro RTX 4000, 5000, 6000) #####
|
||||
### (*** Requires using intfloat/e5-small-v2 instead of hkunlp/instructor-large as embedding model ***)
|
||||
|
||||
### 7b GPTQ Models for 8GB GPUs
|
||||
# MODEL_ID = "TheBloke/Wizard-Vicuna-7B-Uncensored-GPTQ"
|
||||
# MODEL_BASENAME = "Wizard-Vicuna-7B-Uncensored-GPTQ-4bit-128g.no-act.order.safetensors"
|
||||
# MODEL_ID = "TheBloke/WizardLM-7B-uncensored-GPTQ"
|
||||
# MODEL_BASENAME = "WizardLM-7B-uncensored-GPTQ-4bit-128g.compat.no-act-order.safetensors"
|
||||
# MODEL_ID = "TheBloke/wizardLM-7B-GPTQ"
|
||||
# MODEL_BASENAME = "wizardLM-7B-GPTQ-4bit.compat.no-act-order.safetensors"
|
||||
|
||||
####
|
||||
#### (FOR GGML) (Quantized cpu+gpu+mps) models - check if they support llama.cpp
|
||||
####
|
||||
|
||||
# MODEL_ID = "TheBloke/wizard-vicuna-13B-GGML"
|
||||
# MODEL_BASENAME = "wizard-vicuna-13B.ggmlv3.q4_0.bin"
|
||||
# MODEL_BASENAME = "wizard-vicuna-13B.ggmlv3.q6_K.bin"
|
||||
# MODEL_BASENAME = "wizard-vicuna-13B.ggmlv3.q2_K.bin"
|
||||
# MODEL_ID = "TheBloke/orca_mini_3B-GGML"
|
||||
# MODEL_BASENAME = "orca-mini-3b.ggmlv3.q4_0.bin"
|
||||
|
||||
####
|
||||
#### (FOR AWQ QUANTIZED) Select a llm model based on your GPU and VRAM GB. Does not include Embedding Models VRAM usage.
|
||||
### (*** MODEL_BASENAME is not actually used but have to contain .awq so the correct model loading is used ***)
|
||||
### (*** Compute capability 7.5 (sm75) and CUDA Toolkit 11.8+ are required ***)
|
||||
####
|
||||
# MODEL_ID = "TheBloke/Llama-2-7B-Chat-AWQ"
|
||||
# MODEL_BASENAME = "model.safetensors.awq"
|
||||
@@ -0,0 +1,219 @@
|
||||
import logging
|
||||
import os
|
||||
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
|
||||
|
||||
import click
|
||||
import torch
|
||||
from langchain.docstore.document import Document
|
||||
from langchain.embeddings import HuggingFaceInstructEmbeddings
|
||||
from langchain.text_splitter import Language, RecursiveCharacterTextSplitter
|
||||
from langchain.vectorstores import Chroma
|
||||
import uuid
|
||||
|
||||
from constants import (
|
||||
CHROMA_SETTINGS,
|
||||
DOCUMENT_MAP,
|
||||
EMBEDDING_MODEL_NAME,
|
||||
INGEST_THREADS,
|
||||
PERSIST_DIRECTORY,
|
||||
SOURCE_DIRECTORY,
|
||||
)
|
||||
|
||||
import boto3
|
||||
from langchain.embeddings.bedrock import BedrockEmbeddings
|
||||
|
||||
def file_log(logentry):
|
||||
file1 = open("file_ingest.log","a")
|
||||
file1.write(logentry + "\n")
|
||||
file1.close()
|
||||
print(logentry + "\n")
|
||||
|
||||
def load_single_document(file_path: str) -> Document:
|
||||
# Loads a single document from a file path
|
||||
try:
|
||||
file_extension = os.path.splitext(file_path)[1]
|
||||
loader_class = DOCUMENT_MAP.get(file_extension)
|
||||
if loader_class:
|
||||
file_log(file_path + ' loaded.')
|
||||
loader = loader_class(file_path)
|
||||
else:
|
||||
file_log(file_path + ' document type is undefined.')
|
||||
raise ValueError("Document type is undefined")
|
||||
return loader.load()[0]
|
||||
except Exception as ex:
|
||||
file_log('%s loading error: \n%s' % (file_path, ex))
|
||||
return None
|
||||
|
||||
def load_document_batch(filepaths):
|
||||
logging.info("Loading document batch")
|
||||
# create a thread pool
|
||||
with ThreadPoolExecutor(len(filepaths)) as exe:
|
||||
# load files
|
||||
futures = [exe.submit(load_single_document, name) for name in filepaths]
|
||||
# collect data
|
||||
if futures is None:
|
||||
file_log(name + ' failed to submit')
|
||||
return None
|
||||
else:
|
||||
data_list = [future.result() for future in futures]
|
||||
# return data and file paths
|
||||
return (data_list, filepaths)
|
||||
|
||||
|
||||
def load_documents(source_dir: str) -> list[Document]:
|
||||
# Loads all documents from the source documents directory, including nested folders
|
||||
paths = []
|
||||
for root, _, files in os.walk(source_dir):
|
||||
for file_name in files:
|
||||
print('Importing: ' + file_name)
|
||||
file_extension = os.path.splitext(file_name)[1]
|
||||
source_file_path = os.path.join(root, file_name)
|
||||
if file_extension in DOCUMENT_MAP.keys():
|
||||
paths.append(source_file_path)
|
||||
|
||||
# Have at least one worker and at most INGEST_THREADS workers
|
||||
n_workers = min(INGEST_THREADS, max(len(paths), 1))
|
||||
chunksize = round(len(paths) / n_workers)
|
||||
docs = []
|
||||
with ProcessPoolExecutor(n_workers) as executor:
|
||||
futures = []
|
||||
# split the load operations into chunks
|
||||
for i in range(0, len(paths), chunksize):
|
||||
# select a chunk of filenames
|
||||
filepaths = paths[i : (i + chunksize)]
|
||||
# submit the task
|
||||
try:
|
||||
future = executor.submit(load_document_batch, filepaths)
|
||||
except Exception as ex:
|
||||
file_log('executor task failed: %s' % (ex))
|
||||
future = None
|
||||
if future is not None:
|
||||
futures.append(future)
|
||||
# process all results
|
||||
for future in as_completed(futures):
|
||||
# open the file and load the data
|
||||
try:
|
||||
contents, _ = future.result()
|
||||
docs.extend(contents)
|
||||
except Exception as ex:
|
||||
file_log('Exception: %s' % (ex))
|
||||
|
||||
return docs
|
||||
|
||||
|
||||
def split_documents(documents: list[Document]) -> tuple[list[Document], list[Document]]:
|
||||
# Splits documents for correct Text Splitter
|
||||
text_docs, python_docs = [], []
|
||||
for doc in documents:
|
||||
if doc is not None:
|
||||
file_extension = os.path.splitext(doc.metadata["source"])[1]
|
||||
if file_extension == ".py":
|
||||
python_docs.append(doc)
|
||||
else:
|
||||
text_docs.append(doc)
|
||||
return text_docs, python_docs
|
||||
|
||||
def process_in_batches(texts, batch_size):
|
||||
for i in range(0, len(texts), batch_size):
|
||||
yield texts[i:i+batch_size]
|
||||
|
||||
@click.command()
|
||||
@click.option(
|
||||
"--device_type",
|
||||
default="cuda" if torch.cuda.is_available() else "cpu",
|
||||
type=click.Choice(
|
||||
[
|
||||
"cpu",
|
||||
"cuda",
|
||||
"ipu",
|
||||
"xpu",
|
||||
"mkldnn",
|
||||
"opengl",
|
||||
"opencl",
|
||||
"ideep",
|
||||
"hip",
|
||||
"ve",
|
||||
"fpga",
|
||||
"ort",
|
||||
"xla",
|
||||
"lazy",
|
||||
"vulkan",
|
||||
"mps",
|
||||
"meta",
|
||||
"hpu",
|
||||
"mtia",
|
||||
],
|
||||
),
|
||||
help="Device to run on. (Default is cuda)",
|
||||
)
|
||||
|
||||
def main(device_type):
|
||||
# Load documents and split in chunks
|
||||
logging.info(f"Loading documents from {SOURCE_DIRECTORY}")
|
||||
for filename in os.listdir(SOURCE_DIRECTORY):
|
||||
with open(os.path.join(SOURCE_DIRECTORY, filename), 'r') as infile:
|
||||
text = infile.read()
|
||||
text_splitted = [i for i in text.split('Start of Page No. = ')]
|
||||
for i, txt in enumerate(text_splitted):
|
||||
if len(txt) > 2:
|
||||
page_path = os.path.join(SOURCE_DIRECTORY, f'{filename[:-4]}_page{i}.txt')
|
||||
with open(page_path, 'w') as f:
|
||||
f.write(txt)
|
||||
|
||||
documents = [load_single_document(page_path)]
|
||||
os.remove(page_path)
|
||||
text_documents, python_documents = split_documents(documents)
|
||||
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
|
||||
python_splitter = RecursiveCharacterTextSplitter.from_language(
|
||||
language=Language.PYTHON, chunk_size=880, chunk_overlap=200
|
||||
)
|
||||
texts = text_splitter.split_documents(text_documents)
|
||||
texts.extend(python_splitter.split_documents(python_documents))
|
||||
logging.info(f"Loaded {len(documents)} documents from {SOURCE_DIRECTORY}")
|
||||
logging.info(f"Split into {len(texts)} chunks of text")
|
||||
|
||||
# Create embeddings
|
||||
# embeddings = HuggingFaceInstructEmbeddings(
|
||||
# model_name=EMBEDDING_MODEL_NAME,
|
||||
# model_kwargs={"device": device_type},
|
||||
# )
|
||||
bedrock_runtime = boto3.client(
|
||||
service_name="bedrock-runtime",
|
||||
region_name="us-east-1",
|
||||
aws_access_key_id="ASIAZTMXAXNXD3TOUOAJ",
|
||||
aws_secret_access_key="pk7k69CqXZPB/bf2hdFsW+47D5WYkoWXFdxQ633X",
|
||||
aws_session_token="IQoJb3JpZ2luX2VjELj//////////wEaCXVzLWVhc3QtMiJIMEYCIQCqcyLSwLeN9RX6tz+TgB4VMabHYwPcA3bTzx7xZ5b75QIhAMiOKbDlmPZ+eHkeskzelK4M9gWtS7GqlzJq5qmWbbrOKo4DCKH//////////wEQABoMNjYwMTMxMDY4NzgyIgzPuv0vReC65Mo/bQcq4gLXtII+VTaxES+JrjHHWpWywdmpsJneN6bcB3U37z7J8BFA5aaUqkATKmwJI0brmk5ZJKL8SpgBEC2TNdA/V9nzbMlf2HPunhEv6OPLzSWp5iJqaNL945MP764CbkYvfN9QWd6durUv1WgGZRNcbMzXg2UFsxKcRql795vtOmL207+R7uIouWl73So7NaCkEgaj4FdEJ9lbnfvWFeNcBlHbjwUx8e9EJjwm8D60OkTdS4w7Q3EacoEKLO94/kp2RtsaggAUV33OcvO/32VwYJzhRJYuveQUZnIzfkmybGtrXkkWLqMO9pls1bkTmIjaeMwcL8Uo7oowR9sTFCT87rY711yIYBGVOjkN9mfavPH4FSCNeeI6ta5aXqa7iVDU7rPerpFtle2i1VGfTW5bKoaWxjO13IdqI4yI9Pgibl+FuVeRB7md2tDuS6SMAuX4qpWnMVDLKCItlAxXlZdmVhuDFf8wr+7rrgY6pQH02EIrvCkYYXbJd8u7t78hOam3lSNPz+nghQHA5ppl7TnyRBL5/9Rfqp5Ib8y54HdC4hbzz/7w6lvfS0QbHE3Q6r2GC6XWk9L4FAxD8pyZkIYbtwf/WueX1g0r+0x7uLCxKxbWsYum/bigyvNxsDyFIcQcUOm+OJsXGPH3Z7z+qWKni5N9RRA4SEEpdSTRkAWipDtgVNII6kd/eWvyE74d+LYUrmA="
|
||||
)
|
||||
embeddings = BedrockEmbeddings(
|
||||
client=bedrock_runtime,
|
||||
model_id="amazon.titan-embed-text-v1",
|
||||
)
|
||||
|
||||
"""
|
||||
db = Chroma.from_documents(
|
||||
texts,
|
||||
embeddings,
|
||||
persist_directory=PERSIST_DIRECTORY,
|
||||
client_settings=CHROMA_SETTINGS,
|
||||
)
|
||||
"""
|
||||
|
||||
# for batch_texts in process_in_batches(texts, 20000): # https://github.com/PromtEngineer/localGPT/issues/489y
|
||||
print(filename)
|
||||
|
||||
db = Chroma.from_documents(
|
||||
texts,
|
||||
embeddings,
|
||||
persist_directory=PERSIST_DIRECTORY,
|
||||
client_settings=CHROMA_SETTINGS,
|
||||
collection_metadata={"hnsw:space": "cosine"},
|
||||
# ids = [str(filename)]
|
||||
)
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(
|
||||
format="%(asctime)s - %(levelname)s - %(filename)s:%(lineno)s - %(message)s", level=logging.INFO
|
||||
)
|
||||
main()
|
||||
@@ -0,0 +1,112 @@
|
||||
import streamlit as st
|
||||
from streamlit_extras.add_vertical_space import add_vertical_space
|
||||
import os
|
||||
import streamlit as st
|
||||
import pandas as pd
|
||||
from io import StringIO
|
||||
from datetime import datetime
|
||||
|
||||
st.set_page_config(layout = "wide")
|
||||
|
||||
# # Sidebar contents
|
||||
# with st.sidebar:
|
||||
# st.title("Doczy.AI ™")
|
||||
# st.markdown(
|
||||
# """
|
||||
# ## About
|
||||
# This app extracts data from contracts
|
||||
|
||||
# """
|
||||
# )
|
||||
# add_vertical_space(15)
|
||||
# # st.write("Doczy")
|
||||
|
||||
|
||||
client_row = st.columns([0.1, 0.8])
|
||||
with client_row[0]:
|
||||
st.write("**Client Name**")
|
||||
with client_row[1]:
|
||||
client = st.selectbox('Client Name',('Delaware First Health, Inc.', 'Community Health Choice, Inc','CareSource Network Partners LLC',
|
||||
'HealthNet of Cali', 'Oklahoma Complete Health, Inc', 'HealthFirst', 'Molina Healthcare of TX', 'AvMed', 'Arizona Care1st',
|
||||
'WellCare New Jersey'), label_visibility = "collapsed")
|
||||
|
||||
def file_selector(folder_path='.'):
|
||||
filenames = os.listdir(folder_path)
|
||||
selected_filename = st.selectbox('**Path to folder**', filenames, label_visibility = "collapsed")
|
||||
return os.path.join(folder_path, selected_filename)
|
||||
# return selected_filename
|
||||
|
||||
path_row = st.columns([0.1, 0.8])
|
||||
with path_row[0]:
|
||||
st.write("**Path to folder**")
|
||||
with path_row[1]:
|
||||
# Directory = st.text_input("**Path to folder**", label_visibility = "collapsed")
|
||||
Directory = file_selector()
|
||||
|
||||
checks = st.columns([0.1, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12])
|
||||
with checks[0]:
|
||||
st.write("**Group No.**")
|
||||
with checks[1]:
|
||||
a = st.checkbox('Unique Key', key = str(1))
|
||||
with checks[2]:
|
||||
b = st.checkbox('Pricing Before Carveouts', key = str(2))
|
||||
with checks[3]:
|
||||
c = st.checkbox('Contract Related', key = str(3))
|
||||
with checks[4]:
|
||||
d = st.checkbox('Provider', key = str(4))
|
||||
with checks[5]:
|
||||
e = st.checkbox('Timeline', key = str(5))
|
||||
with checks[6]:
|
||||
f = st.checkbox('Carveout Indicator', key = str(6))
|
||||
with checks[7]:
|
||||
g = st.checkbox('Carveout Methodology', key = str(7))
|
||||
|
||||
add_vertical_space(1)
|
||||
|
||||
df = pd.DataFrame(columns=['Request ID','Contract ID','Contract Name','Unique Key','Pricing Before Carveouts'
|
||||
, 'Contract Related', 'Provider', 'Timeline', 'Carveout Indicator', 'Carveout Methodology'])
|
||||
file_list = []
|
||||
|
||||
if st.button("Read the contracts from Path"):
|
||||
for filename in os.listdir(Directory):
|
||||
# with open(os.path.join(Directory, filename), encoding="utf8") as f:
|
||||
# context = f.read()
|
||||
file_list.append(filename)
|
||||
|
||||
df['Contract Name'] = file_list
|
||||
df['Request ID'] = range(len(file_list))
|
||||
df['Contract ID'] = file_list
|
||||
# df['Folder Name'] = Directory
|
||||
# df['Updated Group Number'] = updated_group
|
||||
df['Unique Key'] = a
|
||||
df['Pricing Before Carveouts'] = b
|
||||
df['Contract Related'] = c
|
||||
df['Provider'] = d
|
||||
df['Timeline'] = e
|
||||
df['Carveout Indicator'] = f
|
||||
df['Carveout Methodology'] = g
|
||||
df.to_csv('temp1.csv', index=False)
|
||||
|
||||
add_vertical_space(1)
|
||||
|
||||
# df_copy = df.set_index(df.columns[0]).copy()
|
||||
df2 = pd.read_csv('temp1.csv')
|
||||
edited_df = st.data_editor(df2)
|
||||
|
||||
|
||||
@st.cache_data
|
||||
def convert_df(df):
|
||||
return df.to_csv(index=False).encode('utf-8')
|
||||
|
||||
csv = convert_df(edited_df)
|
||||
|
||||
buttons = st.columns([0.45, 0.35, 0.2])
|
||||
with buttons[0]:
|
||||
st.button("Save All Edits")
|
||||
with buttons[1]:
|
||||
st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv')
|
||||
with buttons[2]:
|
||||
st.button("Run Doczy.AI Pipeline")
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
import json
|
||||
|
||||
import boto3
|
||||
from langchain.prompts import PromptTemplate
|
||||
from langchain.embeddings.bedrock import BedrockEmbeddings
|
||||
from langchain.llms.bedrock import Bedrock
|
||||
from langchain_community.vectorstores import Chroma
|
||||
from constants import CHROMA_SETTINGS, EMBEDDING_MODEL_NAME, PERSIST_DIRECTORY, MODEL_ID, MODEL_BASENAME, SOURCE_DIRECTORY
|
||||
from langchain.chains import RetrievalQA
|
||||
|
||||
import streamlit as st
|
||||
from streamlit_extras.add_vertical_space import add_vertical_space
|
||||
import os
|
||||
import pandas as pd
|
||||
|
||||
st.set_page_config(layout = "wide")
|
||||
|
||||
# Sidebar contents
|
||||
with st.sidebar:
|
||||
st.title("Doczy.AI ™")
|
||||
st.markdown(
|
||||
"""
|
||||
## About
|
||||
This app extracts data from contracts
|
||||
|
||||
"""
|
||||
)
|
||||
add_vertical_space(15)
|
||||
# st.write("Doczy")
|
||||
|
||||
sample = pd.read_csv('sample.csv')
|
||||
sample = sample[['Filename', 'Attribute', 'Query', 'Answer']]
|
||||
attribute_list = ['Agreement Name', 'Agreement Type', 'Contract State', 'Contract Type', 'Create Date', 'Effective Date', 'Gold Carded',
|
||||
'Modify Date', 'National Contract', 'Provider State', 'Summary', 'Termination Date', 'TIN', 'Value-Based Contract']
|
||||
sample = sample[sample['Attribute'].isin(attribute_list)]
|
||||
field_prompt_mapping = sample[['Attribute', 'Query']].drop_duplicates().dropna()
|
||||
field_prompt_mapping = dict(zip(field_prompt_mapping.Attribute, field_prompt_mapping.Query))
|
||||
|
||||
def file_selector(folder_path='RAW_DOCUMENTS'):
|
||||
filenames = os.listdir(folder_path)
|
||||
selected_filename = st.selectbox('Select a file', filenames, label_visibility = "collapsed")
|
||||
# return os.path.join(folder_path, selected_filename)
|
||||
return selected_filename
|
||||
|
||||
file_row = st.columns([0.2, 0.7, 0.1])
|
||||
with file_row[0]:
|
||||
st.write("**Contract Name**")
|
||||
with file_row[1]:
|
||||
# file_name = st.text_input("**Contract Name**", label_visibility = "collapsed")
|
||||
file_name = file_selector()
|
||||
|
||||
lob_row = st.columns([0.2, 0.7, 0.1])
|
||||
with lob_row[0]:
|
||||
st.write("**LOB**")
|
||||
with lob_row[1]:
|
||||
lob = st.selectbox('LOB',('Medicare', 'Medicaid'), label_visibility = "collapsed")
|
||||
|
||||
llm_row = st.columns([0.2, 0.7, 0.1])
|
||||
with llm_row[0]:
|
||||
st.write("**Langauge Model**")
|
||||
with llm_row[1]:
|
||||
llm_selected = st.selectbox('Langauge Model',('Llama 2 Chat 13B', 'Llama 2 Chat 70B', 'Titan Text Express'), label_visibility = "collapsed")
|
||||
|
||||
page_list = []
|
||||
with open(os.path.join(SOURCE_DIRECTORY, file_name[:-4]+'.txt'), 'r') as infile:
|
||||
text = infile.read()
|
||||
page_count = text.count('Start of Page No. = ')
|
||||
for page in range(page_count+1):
|
||||
file_path = "SOURCE_DOCUMENTS\\" + f'{file_name[:-4]}_page{page}.txt'
|
||||
dict_with_pages = { 'source': { '$eq': file_path }}
|
||||
page_list.append(dict_with_pages)
|
||||
|
||||
# AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID')
|
||||
# AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY')
|
||||
# AWS_SESSION_TOKEN=os.getenv('AWS_SESSION_TOKEN')
|
||||
|
||||
# Setup bedrock
|
||||
bedrock_runtime = boto3.client(
|
||||
service_name="bedrock-runtime",
|
||||
region_name="us-east-1"
|
||||
)
|
||||
|
||||
embeddings = BedrockEmbeddings(
|
||||
client=bedrock_runtime,
|
||||
model_id="amazon.titan-embed-text-v1",
|
||||
)
|
||||
DB = Chroma(
|
||||
persist_directory=PERSIST_DIRECTORY,
|
||||
embedding_function=embeddings,
|
||||
client_settings=CHROMA_SETTINGS,
|
||||
)
|
||||
RETRIEVER = DB.as_retriever(search_kwargs={"filter": {"$or": page_list}, "k": 4})
|
||||
|
||||
if llm_selected == 'Titan Text Express':
|
||||
LLM = Bedrock(
|
||||
model_id="amazon.titan-text-express-v1",
|
||||
client=bedrock_runtime,
|
||||
model_kwargs={
|
||||
"maxTokenCount": 4096,
|
||||
"stopSequences": [],
|
||||
"temperature": 0,
|
||||
"topP": 1,
|
||||
}
|
||||
)
|
||||
elif llm_selected == 'Llama 2 Chat 70B':
|
||||
LLM = Bedrock(
|
||||
model_id="meta.llama2-70b-chat-v1",
|
||||
client=bedrock_runtime,
|
||||
model_kwargs={
|
||||
"max_gen_len": 512,
|
||||
"temperature": 0,
|
||||
# "topP": 0.9,
|
||||
}
|
||||
)
|
||||
else:
|
||||
LLM = Bedrock(
|
||||
model_id="meta.llama2-13b-chat-v1",
|
||||
client=bedrock_runtime,
|
||||
model_kwargs={
|
||||
"max_gen_len": 512,
|
||||
"temperature": 0,
|
||||
# "topP": 0.9,
|
||||
}
|
||||
)
|
||||
|
||||
template = """
|
||||
|
||||
Use the following pieces of context to answer the question at the end. If you don't know the answer,\
|
||||
just say that you don't know, don't try to make up an answer.
|
||||
|
||||
{context}
|
||||
|
||||
Question: {question}
|
||||
Answer:"""
|
||||
prompt = PromptTemplate(input_variables=["context", "question"], template=template)
|
||||
|
||||
QA = RetrievalQA.from_chain_type(
|
||||
llm=LLM,
|
||||
chain_type="stuff",
|
||||
retriever=RETRIEVER,
|
||||
return_source_documents=True,
|
||||
chain_type_kwargs={"prompt": prompt},
|
||||
)
|
||||
|
||||
# query = "In which state or states is the Contract applicable? Answer in one or two words. State name: "
|
||||
# response = QA({"query":query})
|
||||
# st.write(query)
|
||||
# st.write(response['result'])
|
||||
# st.write("-----------")
|
||||
# st.write(response)
|
||||
|
||||
# clicked = st.button("Show Results")
|
||||
df = pd.DataFrame(columns=['Contract Name','Field Name','Snippet','Page Number','Confidence Level',
|
||||
'Field Extracted Value','Imputed Value'])
|
||||
field_list = list(field_prompt_mapping.keys())
|
||||
query_list = [field_prompt_mapping[x] for x in field_list]
|
||||
score_list = [DB.similarity_search_with_relevance_scores(query, k=4, filter={"$or": page_list}) for query in query_list]
|
||||
confidence_list = []
|
||||
for score in score_list:
|
||||
confidence_list.append(max(d[1] for d in score))
|
||||
# st.write(confidence_list)
|
||||
|
||||
if st.button("Show Results"):
|
||||
response_list = [QA({"query":query}) for query in query_list]
|
||||
answer_list = [response['result'] for response in response_list]
|
||||
doc_list = [response['source_documents'] for response in response_list]
|
||||
snippet_list = [str(doc[0].page_content) for doc in doc_list]
|
||||
page_no_list = [int(str(doc[0].metadata["source"]).rsplit('_page')[1].replace('.txt',''))+1 for doc in doc_list]
|
||||
|
||||
df['Field Name'] = field_list
|
||||
df['Contract Name'] = file_name
|
||||
df['Snippet'] = snippet_list
|
||||
df['Page Number'] = page_no_list
|
||||
df['Confidence Level'] = confidence_list
|
||||
df['Field Extracted Value'] = answer_list
|
||||
df.to_csv('temp2.csv', index=False)
|
||||
|
||||
df2 = pd.read_csv('temp2.csv')
|
||||
df2['Imputed Value'] = ''
|
||||
edited_df = st.data_editor(df2)
|
||||
|
||||
@st.cache_data
|
||||
def convert_df(df):
|
||||
return df.to_csv(index=False).encode('utf-8')
|
||||
|
||||
csv = convert_df(edited_df)
|
||||
|
||||
buttons = st.columns(3)
|
||||
with buttons[0]:
|
||||
st.button("Save All Imputations")
|
||||
with buttons[1]:
|
||||
st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv')
|
||||
with buttons[2]:
|
||||
st.button("Kickoff Database Integration")
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,334 @@
|
||||
import json
|
||||
import boto3
|
||||
from langchain.prompts import PromptTemplate
|
||||
from langchain.embeddings.bedrock import BedrockEmbeddings
|
||||
from langchain.llms.bedrock import Bedrock
|
||||
from langchain_community.vectorstores import Chroma
|
||||
from constants import CHROMA_SETTINGS, EMBEDDING_MODEL_NAME, PERSIST_DIRECTORY, MODEL_ID, MODEL_BASENAME, SOURCE_DIRECTORY
|
||||
from langchain.chains import RetrievalQA
|
||||
|
||||
import streamlit as st
|
||||
from streamlit_extras.add_vertical_space import add_vertical_space
|
||||
|
||||
import pandas as pd
|
||||
from datetime import datetime
|
||||
import random
|
||||
import os
|
||||
import dateutil
|
||||
|
||||
st.set_page_config(layout = "wide")
|
||||
|
||||
# Sidebar contents
|
||||
with st.sidebar:
|
||||
st.title("Doczy.AI ™")
|
||||
st.markdown(
|
||||
"""
|
||||
## About
|
||||
This app extracts data from contracts
|
||||
|
||||
"""
|
||||
)
|
||||
add_vertical_space(15)
|
||||
# st.write("Doczy")
|
||||
|
||||
fields = pd.read_csv('contract_fields.csv', encoding='unicode_escape', skipinitialspace=True)
|
||||
fields = fields[fields['PRIORITY'].isin(['A','C'])]
|
||||
field_values = pd.read_csv('contract_field_values.csv', encoding='unicode_escape', skipinitialspace=True)
|
||||
fields = fields.drop_duplicates(subset='Field Name', keep="first").sort_values('Field Name')
|
||||
fields = fields[~fields['Field Name'].isnull()]
|
||||
field_prompt_mapping = dict(zip(fields['Field Name'], fields['Interrogation Question?']))
|
||||
|
||||
field_row = st.columns([0.15, 0.45, 0.4])
|
||||
with field_row[0]:
|
||||
st.write("**Field Name**")
|
||||
with field_row[1]:
|
||||
field = st.selectbox('Field Name',sorted(set(field_prompt_mapping.keys())), index=0, label_visibility = "collapsed")
|
||||
|
||||
contract_count_row = st.columns([0.15, 0.45, 0.4])
|
||||
with contract_count_row[0]:
|
||||
st.write("**# of Contracts**")
|
||||
with contract_count_row[1]:
|
||||
contract_count = st.selectbox('Contract count',('10', '20', '30', '50', 'All'), index=0, label_visibility = "collapsed")
|
||||
|
||||
seed_row = st.columns([0.15, 0.45, 0.4])
|
||||
with seed_row[0]:
|
||||
st.write("**Seed Value**")
|
||||
with seed_row[1]:
|
||||
seed_value = st.text_input("**Seed Value**", value = 20, label_visibility = "collapsed")
|
||||
|
||||
llm_row = st.columns([0.15, 0.45, 0.4])
|
||||
with llm_row[0]:
|
||||
st.write("**Langauge Model**")
|
||||
with llm_row[1]:
|
||||
llm_selected = st.selectbox('Langauge Model',('Claude 2', 'Claude Instant', 'Llama 2 Chat 13B', 'Llama 2 Chat 70B'
|
||||
, 'Titan Text Express'), label_visibility = "collapsed")
|
||||
|
||||
st.write("**Prompt**")
|
||||
sequence_input = field_prompt_mapping.get(field)
|
||||
prompt_row = st.columns([0.8, 0.2])
|
||||
with prompt_row[1]:
|
||||
if st.button("Clear Prompt"):
|
||||
sequence_input = ''
|
||||
if st.button("Back to default"):
|
||||
prompt = sequence_input
|
||||
st.button("Save Prompt")
|
||||
with prompt_row[0]:
|
||||
prompt = st.text_area("**Prompt**", sequence_input, height = 150, label_visibility = "collapsed")
|
||||
|
||||
random.seed(seed_value)
|
||||
try:
|
||||
contract_list = sorted(random.choices(os.listdir(SOURCE_DIRECTORY), k=int(contract_count)))
|
||||
except:
|
||||
contract_list = sorted(os.listdir(SOURCE_DIRECTORY))
|
||||
|
||||
# to be deleted later
|
||||
contract_list = [contract for contract in contract_list if contract.replace(' MU','').replace('_MU','').replace('.txt','') in list(field_values['(internal) Document Name'])]
|
||||
|
||||
page_list_all = []
|
||||
for contract in contract_list:
|
||||
page_list = []
|
||||
with open(os.path.join(SOURCE_DIRECTORY, contract[:-4]+'.txt'), 'r') as infile:
|
||||
text = infile.read()
|
||||
page_count = text.count('Start of Page No. = ')
|
||||
for page in range(page_count+1):
|
||||
file_path = "SOURCE_DOCUMENTS\\" + f'{contract[:-4]}_page{page}.txt'
|
||||
dict_with_pages = { 'source': { '$eq': file_path }}
|
||||
page_list.append(dict_with_pages)
|
||||
page_list_all.append(page_list)
|
||||
contract_txt_mapping = dict(zip(contract_list, page_list_all))
|
||||
|
||||
column_name = fields.loc[fields['Field Name'] == field, 'SF_DB_COL_NAME'].iloc[0]
|
||||
column_list = ['(internal) Document Name', '(Internal) Carveout ID', column_name]
|
||||
if column_name+'_PG' in list(field_values.columns):
|
||||
column_list.append(column_name+'_PG')
|
||||
field_values = field_values[column_list]
|
||||
field_values.rename(columns={'(internal) Document Name': 'Contract Name', column_name: 'Actual Value Stored'
|
||||
, '(Internal) Carveout ID': 'Contract ID', column_name+'_PG': 'Original Page Number'}, inplace=True)
|
||||
field_values = field_values.drop_duplicates(subset='Contract Name', keep="first").sort_values('Contract Name')
|
||||
|
||||
# sample = dict(zip(sample.Filename, sample.Answer))
|
||||
# actual_value_list = [sample.get(contract.rsplit('.',1)[0]+'.txt', ' ') for contract in contract_list]
|
||||
|
||||
# AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID')
|
||||
# AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY')
|
||||
# AWS_SESSION_TOKEN=os.getenv('AWS_SESSION_TOKEN')
|
||||
|
||||
# Setup bedrock
|
||||
bedrock_runtime = boto3.client(
|
||||
service_name="bedrock-runtime",
|
||||
region_name="us-east-1",
|
||||
)
|
||||
|
||||
# Define the retreiver
|
||||
# load the vectorstore
|
||||
if "EMBEDDINGS" not in st.session_state:
|
||||
EMBEDDINGS = BedrockEmbeddings(
|
||||
client=bedrock_runtime,
|
||||
model_id="amazon.titan-embed-text-v1",
|
||||
)
|
||||
st.session_state.EMBEDDINGS = EMBEDDINGS
|
||||
|
||||
if "DB" not in st.session_state:
|
||||
DB = Chroma(
|
||||
persist_directory=PERSIST_DIRECTORY,
|
||||
embedding_function=st.session_state.EMBEDDINGS,
|
||||
client_settings=CHROMA_SETTINGS,
|
||||
)
|
||||
st.session_state.DB = DB
|
||||
|
||||
# if "RETRIEVER" not in st.session_state:
|
||||
# # { "source": { '$eq': "SOURCE_DOCUMENTS\\A.1_UH_Health_System_eff_2_1_08 (1)_page0.txt"} }
|
||||
# RETRIEVER = DB.as_retriever(search_kwargs={"filter": { "source": { '$eq': "SOURCE_DOCUMENTS\\A.1_UH_Health_System_eff_2_1_08 (1)_page0.txt"} }, "k": 2})
|
||||
# st.session_state.RETRIEVER = RETRIEVER
|
||||
|
||||
# if "LLM" not in st.session_state:
|
||||
if llm_selected == 'Titan Text Express':
|
||||
LLM = Bedrock(
|
||||
model_id="amazon.titan-text-express-v1",
|
||||
client=bedrock_runtime,
|
||||
model_kwargs={
|
||||
"maxTokenCount": 512,
|
||||
"stopSequences": [],
|
||||
"temperature": 0,
|
||||
"topP": 1,
|
||||
}
|
||||
)
|
||||
elif llm_selected == 'Llama 2 Chat 70B':
|
||||
LLM = Bedrock(
|
||||
model_id="meta.llama2-70b-chat-v1",
|
||||
client=bedrock_runtime,
|
||||
model_kwargs={
|
||||
"max_gen_len": 512,
|
||||
"temperature": 0,
|
||||
# "topP": 0.9,
|
||||
}
|
||||
)
|
||||
elif llm_selected == 'Llama 2 Chat 13B':
|
||||
LLM = Bedrock(
|
||||
model_id="meta.llama2-13b-chat-v1",
|
||||
client=bedrock_runtime,
|
||||
model_kwargs={
|
||||
"max_gen_len": 512,
|
||||
"temperature": 0,
|
||||
# "topP": 0.9,
|
||||
}
|
||||
)
|
||||
elif llm_selected == 'Claude Instant':
|
||||
LLM = Bedrock(
|
||||
model_id="anthropic.claude-instant-v1",
|
||||
client=bedrock_runtime,
|
||||
model_kwargs={
|
||||
# "max_tokens_to_sample": 512,
|
||||
"temperature": 0,
|
||||
# "topP": 0.9,
|
||||
}
|
||||
)
|
||||
elif llm_selected == 'Claude 2':
|
||||
LLM = Bedrock(
|
||||
model_id="anthropic.claude-v2:1",
|
||||
client=bedrock_runtime,
|
||||
model_kwargs={
|
||||
# "max_tokens_to_sample": 512,
|
||||
"temperature": 0,
|
||||
# "topP": 0.9,
|
||||
}
|
||||
)
|
||||
st.session_state["LLM"] = LLM
|
||||
|
||||
# if "QA" not in st.session_state:
|
||||
# prompt, memory = model_memory()
|
||||
|
||||
# QA = RetrievalQA.from_chain_type(
|
||||
# llm=LLM,
|
||||
# chain_type="stuff",
|
||||
# retriever=RETRIEVER,
|
||||
# return_source_documents=True,
|
||||
# chain_type_kwargs={"prompt": prompt, "memory": memory},
|
||||
# )
|
||||
# st.session_state["QA"] = QA
|
||||
|
||||
# df = pd.DataFrame(columns=['Contract Name','Contract ID','Actual Value Stored','New Extracted value','Confidence Level','Snippet',
|
||||
# 'Original Page Number', 'New Page Number', 'Revised Prompt', 'Result'])
|
||||
df = pd.DataFrame(columns=['Contract Name','Raw value','New Extracted value','Confidence Level','Snippet','New Page Number'
|
||||
, 'Revised Prompt', 'Result'])
|
||||
try:
|
||||
history = pd.read_csv('history.csv')
|
||||
except:
|
||||
history = pd.DataFrame(columns=['Field Name','# Contracts Tested', 'Username', 'Date/Time', 'Accuracy', 'Attempt #'])
|
||||
attempt = 0
|
||||
|
||||
if llm_selected in ['Llama 2 Chat 13B', 'Llama 2 Chat 70B']:
|
||||
k_value = 14
|
||||
else:
|
||||
k_value = 25
|
||||
|
||||
|
||||
if st.button("Test Configuration"):
|
||||
|
||||
answer_list = []
|
||||
doc_list = []
|
||||
response_list = []
|
||||
score_list = []
|
||||
attempt = attempt + 1
|
||||
|
||||
for page_list in page_list_all:
|
||||
RETRIEVER = st.session_state.DB.as_retriever(search_kwargs={"filter": {"$or": page_list}, "k": k_value})
|
||||
QA = RetrievalQA.from_chain_type(
|
||||
llm=st.session_state["LLM"],
|
||||
chain_type="stuff",
|
||||
retriever=RETRIEVER,
|
||||
return_source_documents=True,
|
||||
# chain_type_kwargs={"prompt": prompt, "memory": None},
|
||||
)
|
||||
score = st.session_state.DB.similarity_search_with_relevance_scores(prompt, k=4, filter={"$or": page_list})
|
||||
score_list.append(max(d[1] for d in score))
|
||||
response = QA(prompt)
|
||||
answer, docs = response["result"], response["source_documents"]
|
||||
answer_list.append(answer)
|
||||
doc_list.append(docs)
|
||||
response_list.append(response)
|
||||
|
||||
df['Raw value'] = answer_list
|
||||
# post-processing
|
||||
if 'Date' in field:
|
||||
date_list = []
|
||||
for answer in answer_list:
|
||||
try:
|
||||
extracted_date = dateutil.parser.parse(str(answer).replace('"',''), fuzzy=True).date()
|
||||
except:
|
||||
extracted_date = " "
|
||||
date_list.append(extracted_date)
|
||||
answer_list = date_list
|
||||
elif llm_selected in ['Llama 2 Chat 13B', 'Llama 2 Chat 70B']:
|
||||
answer_list = [answer.rstrip(".") for answer in answer_list]
|
||||
answer_list = [answer if "I don't know" not in str(answer) else " " for answer in answer_list]
|
||||
answer_list = [answer if "N/A" not in str(answer) else " " for answer in answer_list]
|
||||
answer_list = [answer if "does not contain" not in str(answer) else " " for answer in answer_list]
|
||||
answer_list = [answer if "None" not in str(answer) else " " for answer in answer_list]
|
||||
answer_list = [answer if "Not specified in the contract" not in str(answer) else " " for answer in answer_list]
|
||||
answer_list = [answer if "Not applicable" not in str(answer) else " " for answer in answer_list]
|
||||
elif llm_selected in ['Claude 2', 'Claude Instant']:
|
||||
answer_list = [answer if "Unfortunately, I do not have enough context" not in str(answer) else " " for answer in answer_list]
|
||||
answer_list = [answer.rstrip(".") for answer in answer_list]
|
||||
else:
|
||||
answer_list = [answer.rstrip(".") for answer in answer_list]
|
||||
# answer_list = [str(x).rsplit(':',1)[0] if len(str(x).rsplit(':',1)) < 2 else str(x).rsplit(':',1)[1] for x in answer_list]
|
||||
|
||||
df['Contract Name'] = contract_list
|
||||
# to be deleted later
|
||||
df['Contract Name'] = [contract.replace(' MU','').replace('_MU','').replace('.txt','') for contract in contract_list]
|
||||
|
||||
df['New Extracted value'] = answer_list
|
||||
df['Confidence Level'] = [round(score, 2) for score in score_list]
|
||||
df['Snippet'] = [str(doc[0].page_content) for doc in doc_list]
|
||||
df['New Page Number'] = [int(str(doc[0].metadata["source"]).rsplit('_page')[1].replace('.txt',''))+1 for doc in doc_list]
|
||||
df['Revised Prompt'] = [prompt] * len(contract_list)
|
||||
|
||||
df = pd.merge(df, field_values, how ='left', on ='Contract Name')
|
||||
answer_list = list(df['New Extracted value'])
|
||||
actual_value_list = list(df['Actual Value Stored'])
|
||||
result_list = [i==j for i, j in zip(actual_value_list, answer_list)]
|
||||
df['Result'] = [str(x) for x in result_list]
|
||||
df = df[~df['Contract ID'].isnull()]
|
||||
|
||||
if 'Original Page Number' in df.columns:
|
||||
df = df[['Contract Name','Contract ID','Actual Value Stored','Raw value','New Extracted value','Confidence Level','Snippet'
|
||||
,'Original Page Number', 'New Page Number', 'Revised Prompt', 'Result']]
|
||||
else:
|
||||
df = df[['Contract Name','Contract ID','Actual Value Stored','Raw value','New Extracted value','Confidence Level','Snippet'
|
||||
, 'New Page Number', 'Revised Prompt', 'Result']]
|
||||
|
||||
try:
|
||||
accuracy = round(sum(bool(x) for x in result_list) * 100 / len(list(df['Result'])), 2)
|
||||
except:
|
||||
accuracy = 'NA'
|
||||
|
||||
history.loc[len(history.index)] = [field, str(contract_count), None, datetime.now().strftime("%Y-%m-%d %H:%M:%S"), accuracy, attempt]
|
||||
df.to_csv("RESULTS\\"+field.replace("?","").replace("/","_")+'-'+llm_selected+'.csv', index=False)
|
||||
history.to_csv('history.csv', index=False)
|
||||
|
||||
# df_copy = df.set_index(df.columns[0]).copy()
|
||||
# df_2_copy = history.set_index(history.columns[0]).copy()
|
||||
st.dataframe(df)
|
||||
st.dataframe(history)
|
||||
|
||||
|
||||
# @st.cache_data
|
||||
# def convert_df(df):
|
||||
# return df.to_csv(index=False).encode('utf-8')
|
||||
|
||||
# csv = convert_df(edited_df)
|
||||
|
||||
# buttons = st.columns(3)
|
||||
# with buttons[0]:
|
||||
# st.button("Save All Imputations")
|
||||
# with buttons[1]:
|
||||
# st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv')
|
||||
# with buttons[2]:
|
||||
# st.button("Kickoff Database Integration")
|
||||
|
||||
|
||||
st.write(column_name)
|
||||
st.write(len(contract_list))
|
||||
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
# Natural Language Processing
|
||||
langchain==0.0.267
|
||||
chromadb==0.4.6
|
||||
pdfminer.six==20221105
|
||||
InstructorEmbedding
|
||||
sentence-transformers==2.2.2
|
||||
faiss-cpu
|
||||
huggingface_hub
|
||||
transformers
|
||||
autoawq
|
||||
protobuf==3.20.2; sys_platform != 'darwin'
|
||||
protobuf==3.20.2; sys_platform == 'darwin' and platform_machine != 'arm64'
|
||||
protobuf==3.20.3; sys_platform == 'darwin' and platform_machine == 'arm64'
|
||||
auto-gptq==0.2.2
|
||||
docx2txt
|
||||
unstructured
|
||||
unstructured[pdf]
|
||||
|
||||
# Utilities
|
||||
urllib3==1.26.6
|
||||
accelerate
|
||||
bitsandbytes ; sys_platform != 'win32'
|
||||
bitsandbytes-windows ; sys_platform == 'win32'
|
||||
click
|
||||
flask
|
||||
requests
|
||||
|
||||
# Streamlit related
|
||||
streamlit
|
||||
Streamlit-extras
|
||||
|
||||
# Excel File Manipulation
|
||||
openpyxl
|
||||
numpy>=1.22.2
|
||||
|
||||
# AWS related
|
||||
boto3
|
||||
awscli
|
||||
Reference in New Issue
Block a user