import json import boto3 from langchain.prompts import PromptTemplate from langchain.embeddings.bedrock import BedrockEmbeddings from langchain.llms.bedrock import Bedrock from langchain_community.vectorstores import Chroma # from constants import CHROMA_SETTINGS, EMBEDDING_MODEL_NAME, PERSIST_DIRECTORY, MODEL_ID, MODEL_BASENAME, SOURCE_DIRECTORY, USER_LIST from langchain.chains import RetrievalQA import streamlit as st from streamlit_extras.add_vertical_space import add_vertical_space from streamlit_pdf_viewer import pdf_viewer # needs to be installed on the server import os import pandas as pd import numpy as np # import util import anthropic from pydantic import BaseModel from typing import List import re import base64 # from sf_conn import get_snowflake_conn REDIRECT_URI = 'https://doczydev.aarete.com:8502' # user_list = USER_LIST st.set_page_config(layout = "wide") # Sidebar contents # with st.sidebar: # st.title("Doczy.AI ™") # st.markdown( # """ # ## About # This app extracts data from contracts # """ # ) # add_vertical_space(15) # # st.write("Doczy") _,c1= st.columns([5,1]) # try: # util.setup_page(REDIRECT_URI) # except: # st.write("SSO Failed") # st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} try: c1.write(f"User: **{st.session_state.user_info['displayName']}**") user_mail = st.session_state.user_info['mail'] except KeyError as e: st.write("Session Expired.") st.stop() # remove below try except statement if comparison with actual vales is not required # try: # conn = get_snowflake_conn('STG') # cur = conn.cursor() # query = 'select * from "TRAINING_DATA_RAW"' # cur.execute(query) # field_values = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description]) # field_values['Document_Name'] = field_values['DOCUMENT_NAME'] # except: # # field_values = pd.read_csv('contract_field_values.csv', encoding='unicode_escape', skipinitialspace=True) # # field_values = field_values.loc[:, ~field_values.columns.str.contains('Unnamed:')] # st.write("Conn failed, unable to fetch data from training data table in Snowflake") field_values = pd.read_csv('contract_field_values.csv', encoding='unicode_escape', skipinitialspace=True) field_values = field_values.loc[:, ~field_values.columns.str.contains('Unnamed:')] # try: # query = 'select * from "PROMPT_CONFIG"' # cur.execute(query) # fields = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description]) # fields.rename(columns={'FIELD_DESC': 'Field Name'}, inplace = True) # fields.rename(columns={'PROMPT': 'Interrogation Question?'}, inplace = True) # fields.rename(columns={'GROUP_ID': 'PRIORITY'}, inplace = True) # fields.rename(columns={'FIELD_NAME': 'SF_DB_COL_NAME'}, inplace = True) # fields.rename(columns={'FM_MODEL_ID': 'llm_selected'}, inplace = True) # except Exception as e: # st.write("Unable to fetch data from Snowflake: ",e) # # fields = pd.read_csv('contract_fields.csv', encoding='unicode_escape', skipinitialspace=True) # # fields = fields[~fields['SF_COL_NAME'].str.endswith('_PG', na=None)] fields = pd.read_csv('contract_fields.csv', encoding='unicode_escape', skipinitialspace=True) # fields = fields[fields['SF_DB_COL_NAME'].str.endswith('_PG', na=None)] # change the code below if contract list is fetched from snowflake s3_client = boto3.client('s3', region_name="us-east-2" ) bucket = 'doczy-dev-infra-textract' # objects = s3_client.list_objects_v2(Bucket=bucket, Prefix="training-data/") # file_list = [] # for obj in objects['Contents']: # if not obj['Key'].endswith('/'): # file_list.append(obj['Key']) # # to be replaced with snowflake data client_list = ['doczy-ai-client-1', 'Delaware First Health, Inc.', 'Community Health Choice, Inc','CareSource Network Partners LLC', 'HealthNet of Cali', 'Oklahoma Complete Health, Inc', 'HealthFirst', 'Molina Healthcare of TX', 'AvMed', 'Arizona Care1st', 'WellCare New Jersey'] #Replace client_list with this to get client names from s3 buckets # client_list, s3_paths = get_client_names() # client_s3_paths = dict(zip(client_list, s3_paths)) client_row = st.columns([0.2, 0.7, 0.1]) with client_row[0]: st.write("**Client Name**") with client_row[1]: client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed", index= None) # batch_objects = s3_client.list_objects_v2(Bucket=client_bucket # , Prefix="contracts_landing_zone/", Delimiter='/') batch_list = ['d_12132', 'd_13345','i_23423', 'i_72223', 'b_12345', 'b_33452'] # for prefix in batch_objects['CommonPrefixes']: # batch_list.append(prefix['Prefix'][:-1].split('/')[-1]) path_row = st.columns([0.2, 0.7, 0.1]) with path_row[0]: st.write("**Batch ID**") with path_row[1]: batch_id = st.selectbox('**Batch ID**', batch_list, label_visibility = "collapsed", index=None) #Need to populate file_list with the list of files from the selected client from client_list # client_bucket = client_s3_paths.get(client) # bucket = client_bucket # objects = s3_client.list_objects_v2(Bucket=bucket, Prefix="training-data/") # file_list = [] # for obj in objects['Contents']: # if not obj['Key'].endswith('/'): # file_list.append(obj['Key']) file_list = ['Contract_Training_Exercise_Pricing.pdf', 'Contract_Training_Exercise_SLA.pdf'] contract_list = sorted(file_list) file_row = st.columns([0.2, 0.7, 0.1]) with file_row[0]: st.write("**Contract Name**") with file_row[1]: file_name = st.selectbox('Select a file', ['All'] + contract_list, label_visibility = "collapsed", index= None) # MODIFIED - Append 'All' in the front instead of at the end field_row = st.columns([0.2, 0.7, 0.1]) with field_row[0]: st.write("**Field Group**") with field_row[1]: field_group = st.selectbox('Field Group',('Unique Key', 'Contract Related', 'Pricing Before Carveouts - I' , 'Pricing Before Carveouts - II', 'Carveout Indicator, Code Type and Code #s - I' , 'Carveout Indicator, Code Type and Code #s - II', 'Carveout Indicator, Code Type and Code #s - III' , 'Optimize Carving Indic.', 'Carveout Method - I', 'Carveout Method - II', 'Provider' , 'Timeline'), label_visibility = "collapsed", index = None) if field_group == 'Unique Key': fields = fields[fields['PRIORITY'] == 'A'] elif field_group == 'Contract Related': fields = fields[fields['PRIORITY'] == 'C'] elif field_group == 'Pricing Before Carveouts - I': fields = fields[fields['PRIORITY'] == 'B'] fields = np.array_split(fields, 2)[0] elif field_group == 'Pricing Before Carveouts - II': fields = fields[fields['PRIORITY'] == 'B'] fields = np.array_split(fields, 2)[1] elif field_group == 'Carveout Indicator, Code Type and Code #s - I': fields = fields[fields['PRIORITY'] == 'F'] fields = np.array_split(fields, 3)[0] elif field_group == 'Carveout Indicator, Code Type and Code #s - II': fields = fields[fields['PRIORITY'] == 'F'] fields = np.array_split(fields, 3)[1] elif field_group == 'Carveout Indicator, Code Type and Code #s - III': fields = fields[fields['PRIORITY'] == 'F'] fields = np.array_split(fields, 3)[2] elif field_group == 'Carveout Methodology - I': fields = fields[fields['PRIORITY'] == 'G'] fields = np.array_split(fields, 4)[0] elif field_group == 'Carveout Methodology - II': fields = fields[fields['PRIORITY'] == 'G'] fields = np.array_split(fields, 4)[1] elif field_group == 'Carveout Method - III': fields = fields[fields['PRIORITY'] == 'G'] fields = np.array_split(fields, 4)[2] elif field_group == 'Carveout Method - IV': fields = fields[fields['PRIORITY'] == 'G'] fields = np.array_split(fields, 4)[3] elif field_group == 'Provider': fields = fields[fields['PRIORITY'] == 'D'] elif field_group == 'Timeline': fields = fields[fields['PRIORITY'] == 'E'] if st.button("Show Results"): # query = 'select * from "DOCZY_PIPELINE_RAW_OUTPUT"' # cur.execute(query) # df2 = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description]) # get this dataframe from snowflake table df2 = pd.DataFrame(columns=['Contract Name','Field Name', 'SF_DB_COL_NAME', 'Snippet','Page Number' , 'Field Extracted Value', 'Actual Value','Imputed Value']) df2.to_csv('temp2.csv', index=False) st.subheader("Showing " + client + " : " + file_name) if st.button("Show PDF"): if file_name == None or file_name == "All": st.error("Choose one specific file.") else: with st.sidebar: st.markdown( """ """, unsafe_allow_html=True, ) pdf_viewer(file_name, width=1500) # if st.button("Show PDF"): # if file_name == None or file_name == "All": # st.error("Choose one specific file.") # else: # with st.sidebar: # with open(file_name, "rb") as f: # base64_pdf = base64.b64encode(f.read()).decode('utf-8') # # Embedding PDF in HTML # pdf_display = F'' # # Displaying File # st.markdown( # """ # # """, # unsafe_allow_html=True, # ) # st.markdown(pdf_display, unsafe_allow_html=True) df2 = pd.read_csv('temp2.csv') df2['Imputed Value'] = '' edited_df = st.data_editor(df2) @st.cache_data def convert_df(df): return df.to_csv(index=False).encode('utf-8') csv = convert_df(edited_df) buttons = st.columns(3) with buttons[0]: # st.button("Save All Imputations") st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv') with buttons[1]: # st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv') st.write("") with buttons[2]: if st.button("Kickoff Database Integration"): st.write("Stored in DB")