[DOC-540] Add logging and set environment for services
This commit is contained in:
Grzegorz Huber
@@ -15,8 +15,9 @@ from io import StringIO, BytesIO
|
||||
import time
|
||||
from sf_conn import get_client_names, insert_upload_logs
|
||||
from constants import USER_LIST
|
||||
from util import logger
|
||||
|
||||
(REDIRECT_URI, create_batch_url, doczy_pipeline) = util.load_page_details(0)
|
||||
(redirect_uri, create_batch_url, doczy_pipeline) = util.load_page_details(0)
|
||||
|
||||
user_list = USER_LIST
|
||||
if 'uploading' not in st.session_state:
|
||||
@@ -74,7 +75,7 @@ st.markdown(hide_img_fs, unsafe_allow_html=True)
|
||||
|
||||
_,c1= st.columns([5,1])
|
||||
try:
|
||||
util.setup_page(REDIRECT_URI)
|
||||
util.setup_page(redirect_uri)
|
||||
except Exception as e:
|
||||
st.write(f"SSO Failed = {e}")
|
||||
st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'}
|
||||
@@ -83,7 +84,7 @@ try:
|
||||
user_mail = st.session_state.user_info['mail']
|
||||
except KeyError as e:
|
||||
st.write("Session Expired.")
|
||||
auth_url = security.get_auth_url(REDIRECT_URI)
|
||||
auth_url = security.get_auth_url(redirect_uri)
|
||||
st.markdown(f"<a href='{auth_url}' target='_self'>Sign In</a>", unsafe_allow_html=True)
|
||||
st.stop()
|
||||
|
||||
@@ -99,6 +100,8 @@ client_list = ['doczy-ai-client-1', 'Delaware First Health, Inc.', 'Community He
|
||||
# TODO: Need to update the streamlit code to use the client names from this list
|
||||
# And use the s3 paths to save the objects for the respective client
|
||||
client_list, s3_paths = get_client_names()
|
||||
logger.info(f"client_list={client_list}")
|
||||
logger.info(f"s3_paths={s3_paths}")
|
||||
client_s3_paths = dict(zip(client_list, s3_paths))
|
||||
|
||||
client_row = st.columns([0.1, 0.8])
|
||||
|
||||
+41
-35
@@ -12,11 +12,12 @@ import requests
|
||||
import time
|
||||
from sf_conn import get_client_names, get_secret, save_to_sf
|
||||
from constants import USER_LIST
|
||||
from util import logger
|
||||
|
||||
(REDIRECT_URI, create_batch_url, doczy_pipeline) = util.load_page_details(1)
|
||||
(redirect_uri, create_batch_url, doczy_pipeline) = util.load_page_details(1)
|
||||
|
||||
user_list = USER_LIST
|
||||
st.set_page_config(layout = "wide")
|
||||
st.set_page_config(layout="wide")
|
||||
# Sidebar contents
|
||||
with st.sidebar:
|
||||
st.title("Doczy.AI ™")
|
||||
@@ -24,14 +25,14 @@ with st.sidebar:
|
||||
"""
|
||||
## About
|
||||
This app extracts data from contracts
|
||||
|
||||
|
||||
"""
|
||||
)
|
||||
add_vertical_space(15)
|
||||
# st.write("Doczy")
|
||||
|
||||
# AARETE LOGO
|
||||
x,y,z = st.columns([15,2,15])
|
||||
x, y, z = st.columns([15, 2, 15])
|
||||
with y:
|
||||
st.image('aaretelogo.png')
|
||||
|
||||
@@ -43,9 +44,9 @@ button[title="View fullscreen"]{
|
||||
'''
|
||||
st.markdown(hide_img_fs, unsafe_allow_html=True)
|
||||
|
||||
_,c1= st.columns([5,1])
|
||||
_, c1 = st.columns([5, 1])
|
||||
try:
|
||||
util.setup_page(REDIRECT_URI)
|
||||
util.setup_page(redirect_uri)
|
||||
except:
|
||||
st.write("SSO Failed")
|
||||
st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'}
|
||||
@@ -56,15 +57,13 @@ except KeyError as e:
|
||||
# Do we add a link to get to the login page here?
|
||||
st.write("Session Expired.")
|
||||
# st.write("Please sign-in to use this app.")
|
||||
auth_url = security.get_auth_url(REDIRECT_URI)
|
||||
auth_url = security.get_auth_url(redirect_uri)
|
||||
st.markdown(f"<a href='{auth_url}' target='_self'>Sign In</a>", unsafe_allow_html=True)
|
||||
st.stop()
|
||||
|
||||
|
||||
|
||||
s3_client = boto3.client('s3',
|
||||
region_name="us-east-2",
|
||||
)
|
||||
region_name="us-east-2",
|
||||
)
|
||||
|
||||
# # to be replaced with snowflake data
|
||||
# client_list = ['doczy-ai-client-1', 'Delaware First Health, Inc.', 'Community Health Choice, Inc','CareSource Network Partners LLC',
|
||||
@@ -72,13 +71,16 @@ s3_client = boto3.client('s3',
|
||||
# 'WellCare New Jersey']
|
||||
|
||||
client_list, s3_paths = get_client_names()
|
||||
logger.info(f"client_list={client_list}")
|
||||
logger.info(f"s3_paths={s3_paths}")
|
||||
client_s3_paths = dict(zip(client_list, s3_paths))
|
||||
|
||||
client_row = st.columns([0.1, 0.8])
|
||||
with client_row[0]:
|
||||
st.write("**Client Name**")
|
||||
with client_row[1]:
|
||||
client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344
|
||||
client = st.selectbox('Client Name', (client_list), label_visibility="collapsed",
|
||||
index=None) # MODIFIED for Ticket DOC-344
|
||||
|
||||
client_bucket = client_s3_paths.get(client)
|
||||
|
||||
@@ -86,7 +88,7 @@ client_bucket = client_s3_paths.get(client)
|
||||
client_bucket = 'doczy-dev-infra-textract'
|
||||
|
||||
batch_objects = s3_client.list_objects_v2(Bucket=client_bucket
|
||||
, Prefix="contracts-landing-zone/", Delimiter='/')
|
||||
, Prefix="contracts-landing-zone/", Delimiter='/')
|
||||
|
||||
batch_list = []
|
||||
for prefix in batch_objects['CommonPrefixes']:
|
||||
@@ -100,6 +102,7 @@ for prefix in batch_objects['CommonPrefixes']:
|
||||
if 'sorted_list' not in st.session_state:
|
||||
st.session_state.sorted_list = batch_list
|
||||
|
||||
|
||||
def sort_list(ex_list, sort_by, order):
|
||||
if sort_by == 'Alphabetical':
|
||||
ex_list = sorted(ex_list, reverse=(order == 'Descending'))
|
||||
@@ -108,16 +111,16 @@ def sort_list(ex_list, sort_by, order):
|
||||
|
||||
return ex_list
|
||||
|
||||
|
||||
col1, col2, col3, col4 = st.columns([0.5, 0.5, 0.5, 0.5])
|
||||
|
||||
|
||||
with col1:
|
||||
with col1:
|
||||
sort_by = st.radio("**Sort Batch_IDs**", ('Alphabetical', 'Create Date'))
|
||||
|
||||
with col2:
|
||||
order = st.radio('', ('Ascending','Descending'))
|
||||
order = st.radio('', ('Ascending', 'Descending'))
|
||||
|
||||
with col3:
|
||||
with col3:
|
||||
add_vertical_space(2)
|
||||
if st.button('Apply'):
|
||||
st.session_state.sorted_list = sort_list(batch_list, sort_by, order)
|
||||
@@ -126,7 +129,8 @@ path_row = st.columns([0.1, 0.8])
|
||||
with path_row[0]:
|
||||
st.write("**Batch ID**")
|
||||
with path_row[1]:
|
||||
batch_id = st.selectbox('**Batch ID**', st.session_state.sorted_list, label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344
|
||||
batch_id = st.selectbox('**Batch ID**', st.session_state.sorted_list, label_visibility="collapsed",
|
||||
index=None) # MODIFIED for Ticket DOC-344
|
||||
|
||||
if not batch_id:
|
||||
batch_id = "None"
|
||||
@@ -136,37 +140,37 @@ with checks[0]:
|
||||
st.write("**Group No.**")
|
||||
|
||||
with checks[1]:
|
||||
a = st.checkbox('Unique Key', key = str(1), args="Unique")
|
||||
a = st.checkbox('Unique Key', key=str(1), args="Unique")
|
||||
with checks[2]:
|
||||
b = st.checkbox('Pricing Before Carveouts', key = str(2))
|
||||
b = st.checkbox('Pricing Before Carveouts', key=str(2))
|
||||
with checks[3]:
|
||||
c = st.checkbox('Contract Related', key = str(3))
|
||||
c = st.checkbox('Contract Related', key=str(3))
|
||||
with checks[4]:
|
||||
d = st.checkbox('Provider', key = str(4))
|
||||
d = st.checkbox('Provider', key=str(4))
|
||||
with checks[5]:
|
||||
e = st.checkbox('Timeline', key = str(5))
|
||||
e = st.checkbox('Timeline', key=str(5))
|
||||
with checks[6]:
|
||||
f = st.checkbox('Carveout Indicator', key = str(6))
|
||||
f = st.checkbox('Carveout Indicator', key=str(6))
|
||||
with checks[7]:
|
||||
g = st.checkbox('Carveout Methodology', key = str(7))
|
||||
g = st.checkbox('Carveout Methodology', key=str(7))
|
||||
|
||||
add_vertical_space(1)
|
||||
|
||||
df = pd.DataFrame(columns=['Contract Name', 'Unique Key','Pricing Before Carveouts'
|
||||
df = pd.DataFrame(columns=['Contract Name', 'Unique Key', 'Pricing Before Carveouts'
|
||||
, 'Contract Related', 'Provider', 'Timeline', 'Carveout Indicator', 'Carveout Methodology'])
|
||||
file_list = []
|
||||
file_objects = s3_client.list_objects_v2(Bucket=client_bucket
|
||||
, Prefix="contracts-landing-zone/"+batch_id+"/", Delimiter='/')
|
||||
, Prefix="contracts-landing-zone/" + batch_id + "/", Delimiter='/')
|
||||
|
||||
# Hardcoded file_list for testing purposes
|
||||
# file_list = ['Boilerplate_TX Amendment Mission Health Network effective_040114 MU.pdf', 'Custom_TX - MP AMENDMENT - MISSION HEALTH NETWORK - MU.pdf',
|
||||
# 'Delaware First Health_First State Homecare Agency_212260_7 MU.pdf', 'Molina Healthcare of Texas, Inc. Amendment 4 - HIX ACA__EFF 01012016_MU.pdf']
|
||||
|
||||
if st.button("Read the contracts from Path"):
|
||||
for obj in file_objects.get('Contents',[]):
|
||||
for obj in file_objects.get('Contents', []):
|
||||
if not obj['Key'].endswith('/'):
|
||||
file_list.append(obj['Key'].split('/')[-1])
|
||||
|
||||
|
||||
df['Contract Name'] = file_list
|
||||
# df['Request ID'] = range(len(file_list))
|
||||
# df['Contract ID'] = file_list
|
||||
@@ -190,17 +194,20 @@ edited_df['LATEST_FLAG BOOLEAN'] = True
|
||||
edited_df['PIPELINE_KICKOFF_DATETIME'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
edited_df['REQUEST_DATETIME'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
@st.cache_data
|
||||
|
||||
@st.cache_data
|
||||
def convert_df(df):
|
||||
return df.to_csv(index=False).encode('utf-8')
|
||||
|
||||
|
||||
csv = convert_df(edited_df)
|
||||
# edited_df = edited_df.reset_index() # make sure indexes pair with number of rows
|
||||
|
||||
# additional_info = pd.DataFrame(columns=['REQUEST_ID','T_DRIVE_PATH','CLIENT_NAME'
|
||||
# , 'GROUP_NAME', 'REQUEST_USERNAME', 'REQUEST_DATETIME'])
|
||||
additional_info = pd.DataFrame(columns=['CLIENT_NAME', 'BATCH_ID', 'REQUEST_USERNAME', 'REQUEST_DATETIME'])
|
||||
additional_info.loc[0] = [client, batch_id, st.session_state.user_info['mail'], datetime.now().strftime("%Y-%m-%d %H:%M:%S")]
|
||||
additional_info.loc[0] = [client, batch_id, st.session_state.user_info['mail'],
|
||||
datetime.now().strftime("%Y-%m-%d %H:%M:%S")]
|
||||
st.write(additional_info)
|
||||
|
||||
st.session_state.contract_count = 0
|
||||
@@ -233,7 +240,7 @@ for index, row in edited_df.iterrows():
|
||||
entry_dict = {
|
||||
"contract_name": row['Contract Name'],
|
||||
"groups": group_list,
|
||||
"contract_source_path": "contracts_landing_zone/"+batch_id+"/"+row['Contract Name']
|
||||
"contract_source_path": "contracts_landing_zone/" + batch_id + "/" + row['Contract Name']
|
||||
}
|
||||
contract_list.append(entry_dict)
|
||||
|
||||
@@ -253,7 +260,7 @@ with buttons[1]:
|
||||
if not st.session_state.contract_count == len(edited_df):
|
||||
st.error("Select at least one Group No. for every Contract")
|
||||
else:
|
||||
with st.spinner('Running...'):
|
||||
with st.spinner('Running...'):
|
||||
# csv_buf = StringIO()
|
||||
# additional_info.to_csv(csv_buf, header=True, index=False)
|
||||
# csv_buf.seek(0)
|
||||
@@ -266,11 +273,10 @@ with buttons[1]:
|
||||
# save_to_sf('load_request_and_contract_submissions', request_submission_file_name = "request_submission.csv", contract_config_file_name = "contract_config.csv")
|
||||
# except Exception as e:
|
||||
# st.write(e)
|
||||
response = requests.post(doczy_pipeline, json = myobj)
|
||||
response = requests.post(doczy_pipeline, json=myobj)
|
||||
if response.status_code >= 200 and response.status_code < 300:
|
||||
st.write("Success")
|
||||
# st.write(myobj)
|
||||
else:
|
||||
st.write("Failed")
|
||||
# st.write(response.text)
|
||||
|
||||
|
||||
+42
-36
@@ -5,12 +5,13 @@ from langchain.prompts import PromptTemplate
|
||||
from langchain.embeddings.bedrock import BedrockEmbeddings
|
||||
from langchain.llms.bedrock import Bedrock
|
||||
from langchain_community.vectorstores import Chroma
|
||||
from constants import CHROMA_SETTINGS, EMBEDDING_MODEL_NAME, PERSIST_DIRECTORY, MODEL_ID, MODEL_BASENAME, SOURCE_DIRECTORY, USER_LIST
|
||||
from constants import CHROMA_SETTINGS, EMBEDDING_MODEL_NAME, PERSIST_DIRECTORY, MODEL_ID, MODEL_BASENAME, \
|
||||
SOURCE_DIRECTORY, USER_LIST
|
||||
from langchain.chains import RetrievalQA
|
||||
|
||||
import streamlit as st
|
||||
from streamlit_extras.add_vertical_space import add_vertical_space
|
||||
from streamlit_pdf_viewer import pdf_viewer # needs to be installed on the server
|
||||
from streamlit_pdf_viewer import pdf_viewer # needs to be installed on the server
|
||||
import os
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
@@ -23,11 +24,12 @@ import base64
|
||||
from sf_conn import get_snowflake_conn
|
||||
from sf_conn import get_client_names, get_secret, save_to_sf
|
||||
import io
|
||||
from util import logger
|
||||
|
||||
(REDIRECT_URI, create_batch_url, doczy_pipeline) = util.load_page_details(2)
|
||||
(redirect_uri, create_batch_url, doczy_pipeline) = util.load_page_details(2)
|
||||
user_list = USER_LIST
|
||||
|
||||
st.set_page_config(layout = "wide")
|
||||
st.set_page_config(layout="wide")
|
||||
# Sidebar contents
|
||||
with st.sidebar:
|
||||
st.title("Doczy.AI ™")
|
||||
@@ -35,14 +37,14 @@ with st.sidebar:
|
||||
"""
|
||||
## About
|
||||
This app extracts data from contracts
|
||||
|
||||
|
||||
"""
|
||||
)
|
||||
# add_vertical_space(15)
|
||||
# st.write("Doczy")
|
||||
|
||||
# AARETE LOGO
|
||||
x,y,z = st.columns([15,2,15])
|
||||
x, y, z = st.columns([15, 2, 15])
|
||||
with y:
|
||||
st.image('aaretelogo.png')
|
||||
|
||||
@@ -54,9 +56,9 @@ button[title="View fullscreen"]{
|
||||
'''
|
||||
st.markdown(hide_img_fs, unsafe_allow_html=True)
|
||||
|
||||
_,c1= st.columns([5,1])
|
||||
_, c1 = st.columns([5, 1])
|
||||
try:
|
||||
util.setup_page(REDIRECT_URI)
|
||||
util.setup_page(redirect_uri)
|
||||
except:
|
||||
st.write("SSO Failed")
|
||||
st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'}
|
||||
@@ -65,8 +67,8 @@ try:
|
||||
user_mail = st.session_state.user_info['mail']
|
||||
except KeyError as e:
|
||||
st.write("Session Expired.")
|
||||
#st.write("Please sign-in to use this app.")
|
||||
auth_url = security.get_auth_url(REDIRECT_URI)
|
||||
# st.write("Please sign-in to use this app.")
|
||||
auth_url = security.get_auth_url(redirect_uri)
|
||||
st.markdown(f"<a href='{auth_url}' target='_self'>Sign In</a>", unsafe_allow_html=True)
|
||||
st.stop()
|
||||
|
||||
@@ -88,35 +90,37 @@ try:
|
||||
cur.execute(query)
|
||||
fields = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description])
|
||||
|
||||
fields.rename(columns={'FIELD_DESC': 'Field Name'}, inplace = True)
|
||||
fields.rename(columns={'PROMPT': 'Interrogation Question?'}, inplace = True)
|
||||
fields.rename(columns={'GROUP_ID': 'PRIORITY'}, inplace = True)
|
||||
fields.rename(columns={'FIELD_NAME': 'SF_DB_COL_NAME'}, inplace = True)
|
||||
fields.rename(columns={'FM_MODEL_ID': 'llm_selected'}, inplace = True)
|
||||
fields.rename(columns={'FIELD_DESC': 'Field Name'}, inplace=True)
|
||||
fields.rename(columns={'PROMPT': 'Interrogation Question?'}, inplace=True)
|
||||
fields.rename(columns={'GROUP_ID': 'PRIORITY'}, inplace=True)
|
||||
fields.rename(columns={'FIELD_NAME': 'SF_DB_COL_NAME'}, inplace=True)
|
||||
fields.rename(columns={'FM_MODEL_ID': 'llm_selected'}, inplace=True)
|
||||
except Exception as e:
|
||||
st.write("Unable to fetch data from Snowflake: ",e)
|
||||
st.write("Unable to fetch data from Snowflake: ", e)
|
||||
# fields = pd.read_csv('contract_fields.csv', encoding='unicode_escape', skipinitialspace=True)
|
||||
# fields = fields[~fields['SF_COL_NAME'].str.endswith('_PG', na=None)]
|
||||
|
||||
# change the code below if contract list is fetched from snowflake
|
||||
s3_client = boto3.client('s3',
|
||||
region_name="us-east-2"
|
||||
)
|
||||
region_name="us-east-2"
|
||||
)
|
||||
|
||||
client_list, s3_paths = get_client_names()
|
||||
logger.info(f"client_list={client_list}")
|
||||
logger.info(f"s3_paths={s3_paths}")
|
||||
client_s3_paths = dict(zip(client_list, s3_paths))
|
||||
|
||||
client_row = st.columns([0.2, 0.7, 0.1])
|
||||
with client_row[0]:
|
||||
st.write("**Client Name**")
|
||||
with client_row[1]:
|
||||
client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed", index= None)
|
||||
client = st.selectbox('Client Name', (client_list), label_visibility="collapsed", index=None)
|
||||
|
||||
# client_bucket = client_s3_paths.get(client)
|
||||
client_bucket = 'doczy-dev-infra-textract'
|
||||
|
||||
batch_objects = s3_client.list_objects_v2(Bucket=client_bucket
|
||||
, Prefix="contracts-landing-zone/", Delimiter='/')
|
||||
, Prefix="contracts-landing-zone/", Delimiter='/')
|
||||
|
||||
batch_list = []
|
||||
for prefix in batch_objects['CommonPrefixes']:
|
||||
@@ -126,13 +130,13 @@ path_row = st.columns([0.2, 0.7, 0.1])
|
||||
with path_row[0]:
|
||||
st.write("**Batch ID**")
|
||||
with path_row[1]:
|
||||
batch_id = st.selectbox('**Batch ID**', batch_list, label_visibility = "collapsed", index = None)
|
||||
batch_id = st.selectbox('**Batch ID**', batch_list, label_visibility="collapsed", index=None)
|
||||
|
||||
if batch_id:
|
||||
|
||||
objects = s3_client.list_objects_v2(Bucket=client_bucket, Prefix="contracts-landing-zone/"+batch_id+"/")
|
||||
objects = s3_client.list_objects_v2(Bucket=client_bucket, Prefix="contracts-landing-zone/" + batch_id + "/")
|
||||
file_list = []
|
||||
if 'Contents' in objects:
|
||||
if 'Contents' in objects:
|
||||
for obj in objects['Contents']:
|
||||
if not obj['Key'].endswith('/'):
|
||||
file_list.append(obj['Key'])
|
||||
@@ -145,17 +149,20 @@ if batch_id:
|
||||
with file_row[0]:
|
||||
st.write("**Contract Name**")
|
||||
with file_row[1]:
|
||||
file_name = st.selectbox('Select a file', ['All'] + contract_list, label_visibility = "collapsed", index= None)
|
||||
file_name = st.selectbox('Select a file', ['All'] + contract_list, label_visibility="collapsed", index=None)
|
||||
|
||||
field_row = st.columns([0.2, 0.7, 0.1])
|
||||
with field_row[0]:
|
||||
st.write("**Field Group**")
|
||||
with field_row[1]:
|
||||
field_group = st.selectbox('Field Group',('Unique Key', 'Contract Related', 'Pricing Before Carveouts - I'
|
||||
, 'Pricing Before Carveouts - II', 'Carveout Indicator, Code Type and Code #s - I'
|
||||
, 'Carveout Indicator, Code Type and Code #s - II', 'Carveout Indicator, Code Type and Code #s - III'
|
||||
, 'Optimize Carving Indic.', 'Carveout Method - I', 'Carveout Method - II', 'Provider'
|
||||
, 'Timeline'), label_visibility = "collapsed", index = None)
|
||||
field_group = st.selectbox('Field Group', ('Unique Key', 'Contract Related', 'Pricing Before Carveouts - I'
|
||||
, 'Pricing Before Carveouts - II',
|
||||
'Carveout Indicator, Code Type and Code #s - I'
|
||||
, 'Carveout Indicator, Code Type and Code #s - II',
|
||||
'Carveout Indicator, Code Type and Code #s - III'
|
||||
, 'Optimize Carving Indic.', 'Carveout Method - I',
|
||||
'Carveout Method - II', 'Provider'
|
||||
, 'Timeline'), label_visibility="collapsed", index=None)
|
||||
|
||||
if field_group == 'Unique Key':
|
||||
fields = fields[fields['PRIORITY'] == 'A']
|
||||
@@ -218,8 +225,9 @@ if batch_id:
|
||||
""",
|
||||
unsafe_allow_html=True,
|
||||
)
|
||||
s3_obj = s3_client.get_object(Bucket = client_bucket, Key = file_name)
|
||||
data=s3_obj['Body'].read()
|
||||
s3_obj = s3_client.get_object(Bucket=client_bucket,
|
||||
Key=file_name)
|
||||
data = s3_obj['Body'].read()
|
||||
pdf_viewer(data, width=1500)
|
||||
|
||||
# if st.button("Show PDF"):
|
||||
@@ -248,10 +256,12 @@ if batch_id:
|
||||
df2['Imputed Value'] = ''
|
||||
edited_df = st.data_editor(df2)
|
||||
|
||||
@st.cache_data
|
||||
|
||||
@st.cache_data
|
||||
def convert_df(df):
|
||||
return df.to_csv(index=False).encode('utf-8')
|
||||
|
||||
|
||||
csv = convert_df(edited_df)
|
||||
|
||||
buttons = st.columns(3)
|
||||
@@ -264,7 +274,3 @@ if batch_id:
|
||||
with buttons[2]:
|
||||
if st.button("Kickoff Database Integration"):
|
||||
st.write("Stored in DB")
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+192
-169
@@ -4,7 +4,8 @@ from langchain.prompts import PromptTemplate
|
||||
from langchain.embeddings.bedrock import BedrockEmbeddings
|
||||
from langchain.llms.bedrock import Bedrock
|
||||
from langchain_community.vectorstores import Chroma
|
||||
from constants import CHROMA_SETTINGS, EMBEDDING_MODEL_NAME, PERSIST_DIRECTORY, MODEL_ID, MODEL_BASENAME, SOURCE_DIRECTORY, USER_LIST
|
||||
from constants import CHROMA_SETTINGS, EMBEDDING_MODEL_NAME, PERSIST_DIRECTORY, MODEL_ID, MODEL_BASENAME, \
|
||||
SOURCE_DIRECTORY, USER_LIST
|
||||
from langchain.chains import RetrievalQA
|
||||
|
||||
import streamlit as st
|
||||
@@ -22,12 +23,12 @@ import re
|
||||
import snowflake.connector
|
||||
from sf_conn import get_secret, save_to_sf
|
||||
from io import StringIO
|
||||
from util import logger
|
||||
|
||||
|
||||
(REDIRECT_URI, create_batch_url, doczy_pipeline) = util.load_page_details(3)
|
||||
(redirect_uri, create_batch_url, doczy_pipeline) = util.load_page_details(3)
|
||||
user_list = USER_LIST
|
||||
|
||||
st.set_page_config(layout = "wide")
|
||||
st.set_page_config(layout="wide")
|
||||
# Sidebar contents
|
||||
with st.sidebar:
|
||||
st.title("Doczy.AI ™")
|
||||
@@ -41,10 +42,10 @@ with st.sidebar:
|
||||
add_vertical_space(15)
|
||||
# st.write("Doczy")
|
||||
|
||||
_,c1= st.columns([5,1])
|
||||
# util.setup_page(REDIRECT_URI)
|
||||
_, c1 = st.columns([5, 1])
|
||||
# util.setup_page(redirect_uri)
|
||||
try:
|
||||
util.setup_page(REDIRECT_URI)
|
||||
util.setup_page(redirect_uri)
|
||||
except:
|
||||
st.write("SSO Failed")
|
||||
st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'}
|
||||
@@ -53,7 +54,7 @@ try:
|
||||
user_mail = st.session_state.user_info['mail']
|
||||
except KeyError as e:
|
||||
st.write("Session Expired.")
|
||||
st. stop()
|
||||
st.stop()
|
||||
|
||||
try:
|
||||
sf_secrets = json.loads(get_secret())
|
||||
@@ -61,11 +62,11 @@ try:
|
||||
user=sf_secrets.get('user'),
|
||||
password=sf_secrets.get('password'),
|
||||
account="aarete-doczyai",
|
||||
role = "DEVADMIN",
|
||||
role="DEVADMIN",
|
||||
warehouse="DEV_XS",
|
||||
database="DOCZY_DEV",
|
||||
schema="STG"
|
||||
)
|
||||
)
|
||||
cur = conn.cursor()
|
||||
query = 'select * from "TRAINING_DATA_RAW"'
|
||||
cur.execute(query)
|
||||
@@ -87,9 +88,9 @@ try:
|
||||
query = 'select * from "BUSINESS_CONFIG"'
|
||||
cur.execute(query)
|
||||
fields = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description])
|
||||
fields.rename(columns={'FIELD_NAME': 'Field Name'}, inplace = True)
|
||||
fields.rename(columns={'QUESTION': 'Interrogation Question?'}, inplace = True)
|
||||
fields.rename(columns={'SF_COL_NAME': 'SF_DB_COL_NAME'}, inplace = True)
|
||||
fields.rename(columns={'FIELD_NAME': 'Field Name'}, inplace=True)
|
||||
fields.rename(columns={'QUESTION': 'Interrogation Question?'}, inplace=True)
|
||||
fields.rename(columns={'SF_COL_NAME': 'SF_DB_COL_NAME'}, inplace=True)
|
||||
fields = fields[~fields['SF_DB_COL_NAME'].str.endswith('_PG', na=None)]
|
||||
fields['Field Name'] = fields['SF_DB_COL_NAME']
|
||||
# fields.rename(columns={'FM_MODEL_ID': 'llm_selected'}, inplace = True)
|
||||
@@ -99,9 +100,9 @@ except:
|
||||
# fields = fields.drop_duplicates(subset='Field Name', keep="first").sort_values('Field Name')
|
||||
# fields['Field Name'] = fields['SF_DB_COL_NAME']
|
||||
# fields = fields[~fields['Field Name'].isnull()]
|
||||
fields.rename(columns={'FIELD_NAME': 'Field Name'}, inplace = True)
|
||||
fields.rename(columns={'QUESTION': 'Interrogation Question?'}, inplace = True)
|
||||
fields.rename(columns={'SF_COL_NAME': 'SF_DB_COL_NAME'}, inplace = True)
|
||||
fields.rename(columns={'FIELD_NAME': 'Field Name'}, inplace=True)
|
||||
fields.rename(columns={'QUESTION': 'Interrogation Question?'}, inplace=True)
|
||||
fields.rename(columns={'SF_COL_NAME': 'SF_DB_COL_NAME'}, inplace=True)
|
||||
fields = fields[~fields['SF_DB_COL_NAME'].str.endswith('_PG', na=None)]
|
||||
fields['Field Name'] = fields['SF_DB_COL_NAME']
|
||||
st.write("Local copy of BUSINESS_CONFIG table loaded")
|
||||
@@ -110,33 +111,33 @@ try:
|
||||
query = 'select * from "TRAINING_ATTEMPT_LOGS"'
|
||||
cur.execute(query)
|
||||
history = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description])
|
||||
history.rename(columns={'FIELD_NAME': 'Field Name'}, inplace = True)
|
||||
history.rename(columns={'CONTRACTS_TESTED': '# Contracts Tested'}, inplace = True)
|
||||
history.rename(columns={'USERNAME': 'Username'}, inplace = True)
|
||||
history.rename(columns={'DATE_TIME': 'Date/Time'}, inplace = True)
|
||||
history.rename(columns={'ACCURACY': 'Accuracy'}, inplace = True)
|
||||
history.rename(columns={'ATTEMPT_NUM': 'Attempt #'}, inplace = True)
|
||||
history.rename(columns={'FIELD_NAME': 'Field Name'}, inplace=True)
|
||||
history.rename(columns={'CONTRACTS_TESTED': '# Contracts Tested'}, inplace=True)
|
||||
history.rename(columns={'USERNAME': 'Username'}, inplace=True)
|
||||
history.rename(columns={'DATE_TIME': 'Date/Time'}, inplace=True)
|
||||
history.rename(columns={'ACCURACY': 'Accuracy'}, inplace=True)
|
||||
history.rename(columns={'ATTEMPT_NUM': 'Attempt #'}, inplace=True)
|
||||
history = history[['Field Name', '# Contracts Tested', 'Username', 'Date/Time', 'Accuracy', 'Attempt #']]
|
||||
except:
|
||||
history = pd.read_csv('history.csv')
|
||||
st.write("Local copy of TRAINING_ATTEMPT_LOGS table loaded")
|
||||
|
||||
|
||||
|
||||
field_row = st.columns([0.15, 0.45, 0.4])
|
||||
with field_row[0]:
|
||||
st.write("**Field Group**")
|
||||
with field_row[1]:
|
||||
field_group = st.selectbox('Field Group',('Unique and Contract Related', 'Pricing Before Carveouts - All'
|
||||
, 'Pricing Before Carveouts - I', 'Pricing Before Carveouts - II', 'Carveout Indicator, Code Type and Code #s - I'
|
||||
, 'Carveout Indicator, Code Type and Code #s - II', 'Carveout Indicator, Code Type and Code #s - III'
|
||||
, 'Optimize Carving Indic.', 'Carveout Method - I', 'Carveout Method - II', 'Provider'
|
||||
, 'Timeline'), label_visibility = "collapsed")
|
||||
|
||||
field_group = st.selectbox('Field Group', ('Unique and Contract Related', 'Pricing Before Carveouts - All'
|
||||
, 'Pricing Before Carveouts - I', 'Pricing Before Carveouts - II',
|
||||
'Carveout Indicator, Code Type and Code #s - I'
|
||||
, 'Carveout Indicator, Code Type and Code #s - II',
|
||||
'Carveout Indicator, Code Type and Code #s - III'
|
||||
, 'Optimize Carving Indic.', 'Carveout Method - I',
|
||||
'Carveout Method - II', 'Provider'
|
||||
, 'Timeline'), label_visibility="collapsed")
|
||||
|
||||
# priorty column will be relaced by group_id in snowflake db
|
||||
if field_group == 'Unique and Contract Related':
|
||||
fields = fields[fields['PRIORITY'].isin(['A','C'])]
|
||||
fields = fields[fields['PRIORITY'].isin(['A', 'C'])]
|
||||
elif field_group == 'Pricing Before Carveouts - All':
|
||||
fields = fields[fields['PRIORITY'] == 'B']
|
||||
elif field_group == 'Pricing Before Carveouts - I':
|
||||
@@ -178,28 +179,32 @@ mode_row = st.columns([0.15, 0.45, 0.4])
|
||||
with mode_row[0]:
|
||||
st.write("**Mode**")
|
||||
with mode_row[1]:
|
||||
mode = st.selectbox('Mode',('Single field - Non Empty values', 'Multiple fields', 'One-to-many fields'), index=0, label_visibility = "collapsed")
|
||||
mode = st.selectbox('Mode', ('Single field - Non Empty values', 'Multiple fields', 'One-to-many fields'), index=0,
|
||||
label_visibility="collapsed")
|
||||
|
||||
field_row = st.columns([0.15, 0.45, 0.4])
|
||||
with field_row[0]:
|
||||
st.write("**Field Name**")
|
||||
with field_row[1]:
|
||||
if mode == 'Single field - Non Empty values':
|
||||
field = st.selectbox('Field Name',sorted(set(field_prompt_mapping.keys())), index=0, label_visibility = "collapsed")
|
||||
field = st.selectbox('Field Name', sorted(set(field_prompt_mapping.keys())), index=0,
|
||||
label_visibility="collapsed")
|
||||
else:
|
||||
field = st.multiselect('Field Name',sorted(set(field_prompt_mapping.keys())), sorted(
|
||||
set(field_prompt_mapping.keys())), label_visibility = "collapsed")
|
||||
field = st.multiselect('Field Name', sorted(set(field_prompt_mapping.keys())), sorted(
|
||||
set(field_prompt_mapping.keys())), label_visibility="collapsed")
|
||||
field_prompt_mapping = {key: field_prompt_mapping[key] for key in field}
|
||||
|
||||
contract_count_row = st.columns([0.15, 0.45, 0.4])
|
||||
with contract_count_row[0]:
|
||||
st.write("**# of Contracts**")
|
||||
with contract_count_row[1]:
|
||||
contract_count = st.selectbox('Contract count',('1', '10', '20', '30', '50', '100', '200','Part-1', 'Part-2', 'All'), index=1, label_visibility = "collapsed")
|
||||
contract_count = st.selectbox('Contract count',
|
||||
('1', '10', '20', '30', '50', '100', '200', 'Part-1', 'Part-2', 'All'), index=1,
|
||||
label_visibility="collapsed")
|
||||
|
||||
s3_client = boto3.client('s3',
|
||||
region_name="us-east-2"
|
||||
)
|
||||
region_name="us-east-2"
|
||||
)
|
||||
bucket = 'doczy-dev-infra-textract'
|
||||
objects = s3_client.list_objects_v2(Bucket=bucket, Prefix="training-data/")
|
||||
file_list = []
|
||||
@@ -221,8 +226,9 @@ if mode == 'Single field - Non Empty values':
|
||||
# st.write(df)
|
||||
# st.write(len(contract_list))
|
||||
# contract_list = [contract for contract in contract_list if contract.rsplit('/',1)[1].replace(' MU','').replace('_MU','').replace('.txt','') in list(field_values['Document_Name'])]
|
||||
contract_list = [str(contract)[:-4]+'.pdf' for contract in contract_list]
|
||||
contract_list = [contract for contract in contract_list if contract.rsplit('/',1)[1] in list(field_values['Document_Name'])]
|
||||
contract_list = [str(contract)[:-4] + '.pdf' for contract in contract_list]
|
||||
contract_list = [contract for contract in contract_list if
|
||||
contract.rsplit('/', 1)[1] in list(field_values['Document_Name'])]
|
||||
# st.write(len(contract_list))
|
||||
|
||||
if contract_count == 'All':
|
||||
@@ -242,21 +248,21 @@ with seed_row[0]:
|
||||
st.write("**Contract Name**")
|
||||
with seed_row[1]:
|
||||
if contract_count in ['10', '20', '30', '50', '100', '200']:
|
||||
seed_value = st.text_input("**Seed Value**", value = 20, label_visibility = "collapsed")
|
||||
seed_value = st.text_input("**Seed Value**", value=20, label_visibility="collapsed")
|
||||
random.seed(seed_value)
|
||||
# contract_list = sorted(random.choices(os.listdir(SOURCE_DIRECTORY), k=int(contract_count)))
|
||||
contract_list = sorted(random.choices(contract_list, k=int(contract_count)))
|
||||
elif contract_count == '1':
|
||||
contract_name = st.selectbox('Contract Name', (contract_list), label_visibility = "collapsed")
|
||||
contract_name = st.selectbox('Contract Name', (contract_list), label_visibility="collapsed")
|
||||
contract_list = [contract_name]
|
||||
|
||||
|
||||
llm_row = st.columns([0.15, 0.45, 0.4])
|
||||
with llm_row[0]:
|
||||
st.write("**Langauge Model**")
|
||||
with llm_row[1]:
|
||||
llm_selected = st.selectbox('Langauge Model',('Claude 2', 'Claude 3 - Haiku', 'Claude 3 - Sonnet', 'Claude Instant'
|
||||
, 'Llama 2 Chat 70B', 'Titan Text Express'), index=3, label_visibility = "collapsed")
|
||||
llm_selected = st.selectbox('Langauge Model', ('Claude 2', 'Claude 3 - Haiku', 'Claude 3 - Sonnet', 'Claude Instant'
|
||||
, 'Llama 2 Chat 70B', 'Titan Text Express'), index=3,
|
||||
label_visibility="collapsed")
|
||||
|
||||
st.write("**Prompt**")
|
||||
if mode != 'Single field - Non Empty values':
|
||||
@@ -271,8 +277,7 @@ with prompt_row[1]:
|
||||
prompt = sequence_input
|
||||
# st.button("Save Prompt")
|
||||
with prompt_row[0]:
|
||||
prompt = st.text_area("**Prompt**", sequence_input, height = 100, label_visibility = "collapsed")
|
||||
|
||||
prompt = st.text_area("**Prompt**", sequence_input, height=100, label_visibility="collapsed")
|
||||
|
||||
# column_name = fields.loc[fields['Field Name'] == field, 'SF_DB_COL_NAME'].iloc[0]
|
||||
# column_list = ['Document_Name', column_name]
|
||||
@@ -285,7 +290,8 @@ with prompt_row[0]:
|
||||
field_values.rename(columns={'Document_Name': 'Contract Name'}, inplace=True)
|
||||
if mode != 'One-to-many fields':
|
||||
field_values = field_values.drop_duplicates(subset='Contract Name', keep="first").sort_values('Contract Name')
|
||||
field_values = field_values[field_values['Contract Name'].isin([contract.rsplit('/',1)[1] for contract in contract_list])]
|
||||
field_values = field_values[
|
||||
field_values['Contract Name'].isin([contract.rsplit('/', 1)[1] for contract in contract_list])]
|
||||
|
||||
# Setup bedrock
|
||||
bedrock_runtime = boto3.client(
|
||||
@@ -296,21 +302,22 @@ bedrock_runtime = boto3.client(
|
||||
# question = prompt
|
||||
if mode != 'Single field - Non Empty values':
|
||||
prompt_dict = json.loads(prompt)
|
||||
prompt_dict_pg = prompt_dict | {str(k)+'_PG': "On which page can I find answer to the question - "+str(
|
||||
prompt_dict_pg = prompt_dict | {str(k) + '_PG': "On which page can I find answer to the question - " + str(
|
||||
v) for k, v in prompt_dict.items()}
|
||||
question = json.dumps(dict(sorted(prompt_dict_pg.items())))
|
||||
else:
|
||||
question = json.dumps({field: prompt, str(field)+'_PG': "On which page can I find answer to the question - "+str(prompt)})
|
||||
question = json.dumps(
|
||||
{field: prompt, str(field) + '_PG': "On which page can I find answer to the question - " + str(prompt)})
|
||||
|
||||
# st.write(question)
|
||||
question_with_schema = question
|
||||
attempt = 0
|
||||
|
||||
def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, history):
|
||||
|
||||
|
||||
def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, history):
|
||||
# df = pd.DataFrame(columns=['Contract Name','Contract ID','Actual Value Stored','New Extracted value','Confidence Level','Snippet',
|
||||
# 'Original Page Number', 'New Page Number', 'Revised Prompt', 'Result'])
|
||||
df = pd.DataFrame(columns=['Contract Name' ,'New Extracted value','Confidence Level','Snippet','New Page Number'
|
||||
df = pd.DataFrame(columns=['Contract Name', 'New Extracted value', 'Confidence Level', 'Snippet', 'New Page Number'
|
||||
, 'Revised Prompt', 'Result'])
|
||||
|
||||
field_list = []
|
||||
@@ -323,7 +330,7 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h
|
||||
for contract in contract_list:
|
||||
# with open(os.path.join(SOURCE_DIRECTORY, contract[:-4]+'.txt'), 'r') as infile:
|
||||
# context = infile.read()
|
||||
data = s3_client.get_object(Bucket=bucket, Key=str(contract)[:-4]+'.txt')
|
||||
data = s3_client.get_object(Bucket=bucket, Key=str(contract)[:-4] + '.txt')
|
||||
contents = data['Body'].read()
|
||||
context = contents.decode("utf-8")
|
||||
# st.write(question_with_schema)
|
||||
@@ -340,11 +347,11 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h
|
||||
Question: {question}
|
||||
Answer: Answer in JSON format: {{"""
|
||||
parameters = {
|
||||
"maxTokenCount":2048,
|
||||
"stopSequences":[],
|
||||
"temperature":0,
|
||||
"topP":0.9
|
||||
}
|
||||
"maxTokenCount": 2048,
|
||||
"stopSequences": [],
|
||||
"temperature": 0,
|
||||
"topP": 0.9
|
||||
}
|
||||
|
||||
body = json.dumps({"inputText": prompt_data, "textGenerationConfig": parameters})
|
||||
model_id = "amazon.titan-text-express-v1" # change this to use a different version from the model provider
|
||||
@@ -358,14 +365,14 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h
|
||||
|
||||
Question: {question}
|
||||
Answer: Answer in JSON format: {{"""
|
||||
payload={
|
||||
"prompt":"[INST]"+ prompt_data +"[/INST]",
|
||||
"max_gen_len":2048,
|
||||
"temperature":0.0,
|
||||
"top_p":0.9
|
||||
payload = {
|
||||
"prompt": "[INST]" + prompt_data + "[/INST]",
|
||||
"max_gen_len": 2048,
|
||||
"temperature": 0.0,
|
||||
"top_p": 0.9
|
||||
}
|
||||
body=json.dumps(payload)
|
||||
model_id="meta.llama2-70b-chat-v1"
|
||||
body = json.dumps(payload)
|
||||
model_id = "meta.llama2-70b-chat-v1"
|
||||
|
||||
elif llm_selected in ['Claude Instant', 'Claude 2', 'Claude 3 - Haiku', 'Claude 3 - Sonnet']:
|
||||
if llm_selected == 'Claude Instant':
|
||||
@@ -384,51 +391,51 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h
|
||||
if llm_selected == "Claude 2":
|
||||
model_id = "anthropic.claude-v2:1"
|
||||
body = json.dumps(
|
||||
{"prompt": anthropic.HUMAN_PROMPT + prompt_data + anthropic.AI_PROMPT,
|
||||
"max_tokens_to_sample": 4096,
|
||||
"temperature":0.0,
|
||||
"top_p":1,
|
||||
"top_k":250,
|
||||
"stop_sequences":[anthropic.HUMAN_PROMPT]
|
||||
})
|
||||
{"prompt": anthropic.HUMAN_PROMPT + prompt_data + anthropic.AI_PROMPT,
|
||||
"max_tokens_to_sample": 4096,
|
||||
"temperature": 0.0,
|
||||
"top_p": 1,
|
||||
"top_k": 250,
|
||||
"stop_sequences": [anthropic.HUMAN_PROMPT]
|
||||
})
|
||||
elif llm_selected in ["Claude 3 - Haiku", "Claude 3 - Sonnet"]:
|
||||
if llm_selected == "Claude 3 - Haiku":
|
||||
model_id = 'anthropic.claude-3-haiku-20240307-v1:0'
|
||||
else:
|
||||
model_id = 'anthropic.claude-3-sonnet-20240229-v1:0'
|
||||
body = json.dumps({
|
||||
"anthropic_version": "bedrock-2023-05-31",
|
||||
"max_tokens": 4096,
|
||||
"messages": [
|
||||
"anthropic_version": "bedrock-2023-05-31",
|
||||
"max_tokens": 4096,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text":anthropic.HUMAN_PROMPT + prompt_data + anthropic.AI_PROMPT
|
||||
}
|
||||
]
|
||||
"type": "text",
|
||||
"text": anthropic.HUMAN_PROMPT + prompt_data + anthropic.AI_PROMPT
|
||||
}
|
||||
],
|
||||
"temperature": 0.0
|
||||
]
|
||||
}
|
||||
)
|
||||
],
|
||||
"temperature": 0.0
|
||||
}
|
||||
)
|
||||
else:
|
||||
model_id = "anthropic.claude-instant-v1"
|
||||
body = json.dumps(
|
||||
{"prompt": anthropic.HUMAN_PROMPT + prompt_data + anthropic.AI_PROMPT,
|
||||
"max_tokens_to_sample": 2048,
|
||||
"temperature":0.0,
|
||||
"top_p":1,
|
||||
"top_k":250,
|
||||
"stop_sequences":[anthropic.HUMAN_PROMPT]
|
||||
})
|
||||
{"prompt": anthropic.HUMAN_PROMPT + prompt_data + anthropic.AI_PROMPT,
|
||||
"max_tokens_to_sample": 2048,
|
||||
"temperature": 0.0,
|
||||
"top_p": 1,
|
||||
"top_k": 250,
|
||||
"stop_sequences": [anthropic.HUMAN_PROMPT]
|
||||
})
|
||||
|
||||
try:
|
||||
response = bedrock_runtime.invoke_model(
|
||||
body=body,
|
||||
modelId=model_id,
|
||||
accept="application/json",
|
||||
body=body,
|
||||
modelId=model_id,
|
||||
accept="application/json",
|
||||
contentType="application/json"
|
||||
)
|
||||
response_body = json.loads(response.get("body").read())
|
||||
@@ -449,16 +456,16 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h
|
||||
response_text = response_text.strip()
|
||||
|
||||
try:
|
||||
if response_text.split("{",1)[1].strip()[0] == '"':
|
||||
response_text = "{" + response_text.split("{",1)[1]
|
||||
if response_text.split("{", 1)[1].strip()[0] == '"':
|
||||
response_text = "{" + response_text.split("{", 1)[1]
|
||||
else:
|
||||
response_text = "{" + response_text
|
||||
except:
|
||||
response_text = "{" + response_text
|
||||
|
||||
if len(response_text.split("}",1)) > 1:
|
||||
if response_text.rsplit("}",1)[0].strip()[-1] == '"':
|
||||
response_text = response_text.rsplit("}",1)[0] + "}"
|
||||
if len(response_text.split("}", 1)) > 1:
|
||||
if response_text.rsplit("}", 1)[0].strip()[-1] == '"':
|
||||
response_text = response_text.rsplit("}", 1)[0] + "}"
|
||||
else:
|
||||
response_text = response_text.rstrip(",")
|
||||
response_text = response_text + "}"
|
||||
@@ -469,21 +476,21 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h
|
||||
if mode != 'Single field - Non Empty values':
|
||||
response_dict = {"Test field": "Failed to extract"}
|
||||
else:
|
||||
response_dict = {field:response_text.strip("{").strip("}")}
|
||||
response_dict = {field: response_text.strip("{").strip("}")}
|
||||
|
||||
# if mode == 'Multiple fields':
|
||||
# field_l = list(response_dict.keys())
|
||||
# answer_l = list(response_dict.values())
|
||||
# else:
|
||||
# field_l = [field]
|
||||
# # answer = response_dict.get(field, " ")
|
||||
# try:
|
||||
# if isinstance(response_dict, dict):
|
||||
# answer_l = list(response_dict.values())[:1]
|
||||
# else:
|
||||
# answer_l = list(response_dict)[:1]
|
||||
# except:
|
||||
# answer_l = [response_dict]
|
||||
# field_l = [field]
|
||||
# # answer = response_dict.get(field, " ")
|
||||
# try:
|
||||
# if isinstance(response_dict, dict):
|
||||
# answer_l = list(response_dict.values())[:1]
|
||||
# else:
|
||||
# answer_l = list(response_dict)[:1]
|
||||
# except:
|
||||
# answer_l = [response_dict]
|
||||
if mode != 'Single field - Non Empty values':
|
||||
field_l = list(response_dict.keys())
|
||||
answer_l = list(response_dict.values())
|
||||
@@ -499,11 +506,11 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h
|
||||
try:
|
||||
page_no_l = [list(response_dict.values())[1]]
|
||||
except:
|
||||
page_no_l = ['']
|
||||
page_no_l = ['']
|
||||
|
||||
field_list.extend(field_l)
|
||||
answer_list.extend(answer_l)
|
||||
contract_list_f.extend([contract]*len(field_l))
|
||||
contract_list_f.extend([contract] * len(field_l))
|
||||
# location = context.find(answer) if isinstance(answer, str) and answer != "" else -1
|
||||
# snippet = ' '.join(context[:location].split()[-25:]) + ' ' + ' '.join(context[location:].split()[:30]) if location != -1 else ' '
|
||||
# page_no = " " if location == -1 else context[:location].rsplit("Start of Page No. = ", 1)[1] if len(context[:location].rsplit(
|
||||
@@ -511,12 +518,14 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h
|
||||
# page_no = re.search(r'\d+', page_no).group() if page_no != " " and re.search(r'\d+', page_no) is not None else ""
|
||||
|
||||
try:
|
||||
location_l = [context.find(a, context.find("Start of Page No. = "+str(p))) if isinstance(
|
||||
location_l = [context.find(a, context.find("Start of Page No. = " + str(p))) if isinstance(
|
||||
a, str) and a != "" else -1 for a, p in zip(answer_l, page_no_l)]
|
||||
except:
|
||||
location_l = [context.find(answer) if isinstance(answer, str) and answer != "" else -1 for answer in answer_l]
|
||||
location_l = [context.find(answer) if isinstance(answer, str) and answer != "" else -1 for answer in
|
||||
answer_l]
|
||||
snippet_l = [' '.join(context[:location].split('.')[-4:]) + ' ' + ' '.join(context[location:].split('. ')[:5]
|
||||
) if location != -1 else ' ' for location in location_l]
|
||||
) if location != -1 else ' ' for
|
||||
location in location_l]
|
||||
# page_no_l = [" " if location == -1 else context[:location].rsplit("Start of Page No. = ", 1)[1] if len(context[:location].rsplit(
|
||||
# "Start of Page No. = ", 1)) > 1 else context[:location].rsplit("Start of Page No. = ", 1)[0] for location in location_l]
|
||||
# # st.write(location_list)
|
||||
@@ -537,17 +546,19 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h
|
||||
# date_list.append(extracted_date)
|
||||
# answer_list = date_list
|
||||
try:
|
||||
answer_list = [str(answer).strip("\n").strip().strip("[").strip("]").strip("{").strip("}").strip('"').rstrip('"').strip(
|
||||
' ') if answer is not None else None for answer in answer_list]
|
||||
answer_list = [
|
||||
str(answer).strip("\n").strip().strip("[").strip("]").strip("{").strip("}").strip('"').rstrip('"').strip(
|
||||
' ') if answer is not None else None for answer in answer_list]
|
||||
if llm_selected in ['Llama 2 Chat 13B', 'Llama 2 Chat 70B']:
|
||||
answer_list = [answer.rstrip(".") for answer in answer_list]
|
||||
answer_list = [answer if "I don't know" not in str(answer) else " " for answer in answer_list]
|
||||
answer_list = [answer if "N/A" not in str(answer) else " " for answer in answer_list]
|
||||
answer_list = [answer if "does not contain" not in str(answer) else " " for answer in answer_list]
|
||||
answer_list = [answer if "None" not in str(answer) else " " for answer in answer_list]
|
||||
answer_list = [answer if "Not specified in the contract" not in str(answer) else " " for answer in answer_list]
|
||||
answer_list = [answer if "Not specified in the contract" not in str(answer) else " " for answer in
|
||||
answer_list]
|
||||
answer_list = [answer if "Not applicable" not in str(answer) else " " for answer in answer_list]
|
||||
elif llm_selected in ['Claude 2','Claude 3 - Haiku', 'Claude 3 - Sonnet', 'Claude Instant']:
|
||||
elif llm_selected in ['Claude 2', 'Claude 3 - Haiku', 'Claude 3 - Sonnet', 'Claude Instant']:
|
||||
answer_list = [answer if "do not have" not in str(answer) else " " for answer in answer_list]
|
||||
answer_list = [answer if "do not see" not in str(answer) else " " for answer in answer_list]
|
||||
answer_list = [answer if "does not specify" not in str(answer) else " " for answer in answer_list]
|
||||
@@ -571,7 +582,7 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h
|
||||
|
||||
# to be deleted later
|
||||
# contract_list_f = [contract.rsplit('/',1)[1].replace(' MU','').replace('_MU','').replace('.txt','') for contract in contract_list_f]
|
||||
contract_list_f = [contract.rsplit('/',1)[1] for contract in contract_list_f]
|
||||
contract_list_f = [contract.rsplit('/', 1)[1] for contract in contract_list_f]
|
||||
|
||||
df['Contract Name'] = contract_list_f
|
||||
df['New Extracted value'] = answer_list
|
||||
@@ -579,26 +590,27 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h
|
||||
df['Snippet'] = snippet_list
|
||||
df['New Page Number'] = page_no_list
|
||||
df['New Page Number'] = df['New Page Number'].apply(lambda x: re.search(r'\d+', x).group(
|
||||
) if isinstance(x, str) and re.search(r'\d+', x) is not None else " ")
|
||||
) if isinstance(x, str) and re.search(r'\d+', x) is not None else " ")
|
||||
df['Revised Prompt'] = [prompt] * len(contract_list_f)
|
||||
|
||||
df = pd.merge(df, fields[['Field Name', 'SF_DB_COL_NAME']], how ='left', on ='Field Name')
|
||||
df = pd.merge(df, fields[['Field Name', 'SF_DB_COL_NAME']], how='left', on='Field Name')
|
||||
|
||||
field_values_2 = pd.DataFrame(columns=['Contract Name', 'LOB_Product_Network_Metal_Area_Program_Type_Speciality', 'SF_DB_COL_NAME'
|
||||
, 'Actual Value Stored', 'Original Page Number'])
|
||||
field_values_2 = pd.DataFrame(
|
||||
columns=['Contract Name', 'LOB_Product_Network_Metal_Area_Program_Type_Speciality', 'SF_DB_COL_NAME'
|
||||
, 'Actual Value Stored', 'Original Page Number'])
|
||||
for file_name in contract_list:
|
||||
# document_name = [x for x in list(field_values['Contract Name']) if not pd.isna(x) and file_name.rsplit('/',1)[1].replace(' MU','').replace(
|
||||
# '_MU','').replace('.txt','') in x][0]
|
||||
document_name = [x for x in list(field_values['Contract Name']) if not pd.isna(x) and file_name.rsplit('/',1)[1] in x][0]
|
||||
|
||||
temp_df = field_values[field_values['Contract Name'] == document_name].fillna('NA')
|
||||
unique_identifier = str(temp_df.at[temp_df.index[0],'CONTRACT_LOB']) + '__' + str(
|
||||
temp_df.at[temp_df.index[0],'CONTRACT_PRODUCT']) + '__' + str(
|
||||
temp_df.at[temp_df.index[0],'CONTRACT_NETWORK']) + '__' + str(
|
||||
temp_df.at[temp_df.index[0],'CONTRACT_MARKETPLACE_METAL_LEVEL']) + '__' + str(
|
||||
temp_df.at[temp_df.index[0],'CONTRACT_SERVICE_AREA']) + '__' + str(
|
||||
temp_df.at[temp_df.index[0],'CONTRACT_PROGRAM']) + '__' + str(
|
||||
temp_df.at[temp_df.index[0],'PROV_TYPE']) + '__' + str(temp_df.at[temp_df.index[0],'PROV_SPECIALTY'])
|
||||
unique_identifier = str(temp_df.at[temp_df.index[0], 'CONTRACT_LOB']) + '__' + str(
|
||||
temp_df.at[temp_df.index[0], 'CONTRACT_PRODUCT']) + '__' + str(
|
||||
temp_df.at[temp_df.index[0], 'CONTRACT_NETWORK']) + '__' + str(
|
||||
temp_df.at[temp_df.index[0], 'CONTRACT_MARKETPLACE_METAL_LEVEL']) + '__' + str(
|
||||
temp_df.at[temp_df.index[0], 'CONTRACT_SERVICE_AREA']) + '__' + str(
|
||||
temp_df.at[temp_df.index[0], 'CONTRACT_PROGRAM']) + '__' + str(
|
||||
temp_df.at[temp_df.index[0], 'PROV_TYPE']) + '__' + str(temp_df.at[temp_df.index[0], 'PROV_SPECIALTY'])
|
||||
field_values_1 = field_values[field_values['Contract Name'] == document_name].head(1).transpose().reset_index()
|
||||
field_values_1.columns = ['SF_DB_COL_NAME', 'Actual Value Stored']
|
||||
field_values_p1 = field_values_1[~field_values_1['SF_DB_COL_NAME'].str.endswith('_PG')]
|
||||
@@ -606,36 +618,37 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h
|
||||
field_values_p2.columns = ['SF_DB_COL_NAME', 'Original Page Number']
|
||||
field_values_p2["SF_DB_COL_NAME"] = field_values_p2["SF_DB_COL_NAME"].str.replace("_PG", "")
|
||||
|
||||
field_values_1 = pd.merge(field_values_p1, field_values_p2, how ='left', on =['SF_DB_COL_NAME'])
|
||||
field_values_1 = pd.merge(field_values_p1, field_values_p2, how='left', on=['SF_DB_COL_NAME'])
|
||||
field_values_1['Contract Name'] = document_name
|
||||
field_values_1['LOB_Product_Network_Metal_Area_Program_Type_Speciality'] = unique_identifier
|
||||
field_values_2 = pd.concat([field_values_2, field_values_1], ignore_index = True)
|
||||
field_values_2 = pd.concat([field_values_2, field_values_1], ignore_index=True)
|
||||
|
||||
if mode == 'One-to-many fields':
|
||||
for i in range(1, field_values[field_values['Contract Name'] == document_name].shape[0]):
|
||||
unique_identifier = str(temp_df.at[temp_df.index[i],'CONTRACT_LOB']) + '__' + str(
|
||||
temp_df.at[temp_df.index[i],'CONTRACT_PRODUCT']) + '__' + str(
|
||||
temp_df.at[temp_df.index[i],'CONTRACT_NETWORK']) + '__' + str(
|
||||
temp_df.at[temp_df.index[i],'CONTRACT_MARKETPLACE_METAL_LEVEL']) + '__' + str(
|
||||
temp_df.at[temp_df.index[i],'CONTRACT_SERVICE_AREA']) + '__' + str(
|
||||
temp_df.at[temp_df.index[i],'CONTRACT_PROGRAM']) + '__' + str(
|
||||
temp_df.at[temp_df.index[i],'PROV_TYPE']) + '__' + str(temp_df.at[temp_df.index[i],'PROV_SPECIALTY'])
|
||||
field_values_1 = field_values[field_values['Contract Name'] == document_name].iloc[[i]].transpose().reset_index()
|
||||
unique_identifier = str(temp_df.at[temp_df.index[i], 'CONTRACT_LOB']) + '__' + str(
|
||||
temp_df.at[temp_df.index[i], 'CONTRACT_PRODUCT']) + '__' + str(
|
||||
temp_df.at[temp_df.index[i], 'CONTRACT_NETWORK']) + '__' + str(
|
||||
temp_df.at[temp_df.index[i], 'CONTRACT_MARKETPLACE_METAL_LEVEL']) + '__' + str(
|
||||
temp_df.at[temp_df.index[i], 'CONTRACT_SERVICE_AREA']) + '__' + str(
|
||||
temp_df.at[temp_df.index[i], 'CONTRACT_PROGRAM']) + '__' + str(
|
||||
temp_df.at[temp_df.index[i], 'PROV_TYPE']) + '__' + str(
|
||||
temp_df.at[temp_df.index[i], 'PROV_SPECIALTY'])
|
||||
field_values_1 = field_values[field_values['Contract Name'] == document_name].iloc[
|
||||
[i]].transpose().reset_index()
|
||||
field_values_1.columns = ['SF_DB_COL_NAME', 'Actual Value Stored']
|
||||
field_values_p1 = field_values_1[~field_values_1['SF_DB_COL_NAME'].str.endswith('_PG')]
|
||||
field_values_p2 = field_values_1[field_values_1['SF_DB_COL_NAME'].str.endswith('_PG')]
|
||||
field_values_p2.columns = ['SF_DB_COL_NAME', 'Original Page Number']
|
||||
field_values_p2["SF_DB_COL_NAME"] = field_values_p2["SF_DB_COL_NAME"].str.replace("_PG", "")
|
||||
|
||||
field_values_1 = pd.merge(field_values_p1, field_values_p2, how ='left', on =['SF_DB_COL_NAME'])
|
||||
field_values_1 = pd.merge(field_values_p1, field_values_p2, how='left', on=['SF_DB_COL_NAME'])
|
||||
field_values_1['Contract Name'] = document_name
|
||||
field_values_1['LOB_Product_Network_Metal_Area_Program_Type_Speciality'] = unique_identifier
|
||||
field_values_2 = pd.concat([field_values_2, field_values_1], ignore_index = True)
|
||||
|
||||
field_values_2 = pd.concat([field_values_2, field_values_1], ignore_index=True)
|
||||
|
||||
# st.write(df)
|
||||
# st.write(field_values_2)
|
||||
df = pd.merge(df, field_values_2, how ='right', on =['Contract Name', 'SF_DB_COL_NAME'])
|
||||
df = pd.merge(df, field_values_2, how='right', on=['Contract Name', 'SF_DB_COL_NAME'])
|
||||
if mode != 'Single field - Non Empty values':
|
||||
df = df[df['SF_DB_COL_NAME'].isin(list(field_prompt_mapping.keys()))]
|
||||
else:
|
||||
@@ -643,14 +656,16 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h
|
||||
if mode != 'One-to-many fields':
|
||||
df = df.drop_duplicates(subset=['SF_DB_COL_NAME', 'Contract Name'], keep="first")
|
||||
df['Original Page Number'] = df['Original Page Number'].apply(lambda x: re.search(r'\d+', x).group(
|
||||
) if isinstance(x, str) and re.search(r'\d+', x) is not None else " ")
|
||||
) if isinstance(x, str) and re.search(r'\d+', x) is not None else " ")
|
||||
df['Raw value 2'] = df['New Extracted value']
|
||||
df_date = df[df['SF_DB_COL_NAME'].str.contains('_DT', na=False)]
|
||||
df_others = df[~df['SF_DB_COL_NAME'].str.contains('_DT', na=False)]
|
||||
|
||||
df_date['Actual Value Stored'] = pd.to_datetime(df_date['Actual Value Stored'],errors='coerce').dt.strftime('%Y-%m-%d').fillna(" ")
|
||||
df_date['New Extracted value'] = pd.to_datetime(df_date['New Extracted value'],errors='coerce').dt.strftime('%Y-%m-%d').fillna(" ")
|
||||
df = pd.concat([df_date, df_others], ignore_index = True)
|
||||
df_date['Actual Value Stored'] = pd.to_datetime(df_date['Actual Value Stored'], errors='coerce').dt.strftime(
|
||||
'%Y-%m-%d').fillna(" ")
|
||||
df_date['New Extracted value'] = pd.to_datetime(df_date['New Extracted value'], errors='coerce').dt.strftime(
|
||||
'%Y-%m-%d').fillna(" ")
|
||||
df = pd.concat([df_date, df_others], ignore_index=True)
|
||||
df.sort_values(['SF_DB_COL_NAME', 'Contract Name'], inplace=True)
|
||||
|
||||
df.fillna(" ", inplace=True)
|
||||
@@ -661,7 +676,7 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h
|
||||
actual_value_list = [answer if str(answer) != "12 months" else "1 year" for answer in actual_value_list]
|
||||
actual_value_list = [answer if str(answer) != "Fifth" else "5" for answer in actual_value_list]
|
||||
actual_value_list = [answer if str(answer) != "Seventh" else "7" for answer in actual_value_list]
|
||||
|
||||
|
||||
answer_list = list(df['New Extracted value'])
|
||||
answer_list = [answer if str(answer) != "one-year" else "1 year" for answer in answer_list]
|
||||
answer_list = [answer if str(answer) != "one year" else "1 year" for answer in answer_list]
|
||||
@@ -672,9 +687,10 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h
|
||||
answer_list = [answer if str(answer) != "Third" else "3" for answer in answer_list]
|
||||
answer_list = [answer if str(answer) != "Six" else "6" for answer in answer_list]
|
||||
|
||||
actual_value_list = [s.replace('-', '').replace(' ', '').replace('[', '').replace(']', '').lower() for s in actual_value_list]
|
||||
actual_value_list = [s.replace('-', '').replace(' ', '').replace('[', '').replace(']', '').lower() for s in
|
||||
actual_value_list]
|
||||
answer_list = [s.replace('-', '').replace(' ', '').replace('[', '').replace(']', '').lower() for s in answer_list]
|
||||
result_list = [(i in j) or (j in i) if isinstance(i, str) and isinstance(
|
||||
result_list = [(i in j) or (j in i) if isinstance(i, str) and isinstance(
|
||||
j, str) and ((i != '') == (j != '')) else False for i, j in zip(actual_value_list, answer_list)]
|
||||
df['Result'] = [str(x) for x in result_list]
|
||||
|
||||
@@ -682,8 +698,10 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h
|
||||
# df['Contract ID'] = contract_list_f
|
||||
df['Contract ID'] = range(len(actual_value_list))
|
||||
|
||||
df = df[['Contract Name','Contract ID', 'LOB_Product_Network_Metal_Area_Program_Type_Speciality', 'SF_DB_COL_NAME', 'Actual Value Stored', 'Raw value', 'Raw value 2', 'Raw value 3'
|
||||
, 'New Extracted value','Confidence Level','Snippet','Original Page Number', 'New Page Number', 'Revised Prompt', 'Result']]
|
||||
df = df[['Contract Name', 'Contract ID', 'LOB_Product_Network_Metal_Area_Program_Type_Speciality', 'SF_DB_COL_NAME',
|
||||
'Actual Value Stored', 'Raw value', 'Raw value 2', 'Raw value 3'
|
||||
, 'New Extracted value', 'Confidence Level', 'Snippet', 'Original Page Number', 'New Page Number',
|
||||
'Revised Prompt', 'Result']]
|
||||
|
||||
try:
|
||||
accuracy = round(sum(bool(x) for x in result_list) * 100 / len(list(df['Result'])), 2)
|
||||
@@ -692,38 +710,45 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h
|
||||
|
||||
if mode != 'Single field - Non Empty values':
|
||||
field = field_group
|
||||
history.loc[len(history.index)] = [field, str(contract_count), st.session_state.user_info['mail'], datetime.now().strftime("%Y-%m-%d %H:%M:%S"), accuracy, attempt]
|
||||
history.loc[len(history.index)] = [field, str(contract_count), st.session_state.user_info['mail'],
|
||||
datetime.now().strftime("%Y-%m-%d %H:%M:%S"), accuracy, attempt]
|
||||
# df.to_csv("RESULTS\\"+field.replace("?","").replace("/","_")+'-'+llm_selected+'.csv', index=False)
|
||||
|
||||
return df, history, attempt, raw_response_text
|
||||
|
||||
|
||||
raw_response_text = ''
|
||||
df = pd.DataFrame(columns=['Contract Name','Contract ID', 'Actual Value Stored','New Extracted value','Confidence Level'
|
||||
,'Snippet','New Page Number', 'Revised Prompt', 'Result'])
|
||||
df = pd.DataFrame(
|
||||
columns=['Contract Name', 'Contract ID', 'Actual Value Stored', 'New Extracted value', 'Confidence Level'
|
||||
, 'Snippet', 'New Page Number', 'Revised Prompt', 'Result'])
|
||||
|
||||
if st.button("Test Configuration"):
|
||||
df, history, attempt, raw_response_text = run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, history)
|
||||
df_1 = df[['Contract Name','Contract ID', 'Actual Value Stored','New Extracted value','Confidence Level'
|
||||
,'Snippet','Original Page Number', 'New Page Number', 'Revised Prompt', 'Result']]
|
||||
df, history, attempt, raw_response_text = run_llm(attempt, bucket, contract_list, llm_selected, field, field_values,
|
||||
history)
|
||||
df_1 = df[['Contract Name', 'Contract ID', 'Actual Value Stored', 'New Extracted value', 'Confidence Level'
|
||||
, 'Snippet', 'Original Page Number', 'New Page Number', 'Revised Prompt', 'Result']]
|
||||
df.to_csv('results.csv', index=False)
|
||||
history.to_csv('history.csv', index=False)
|
||||
# s3_client.upload_file('results.csv', bucket, key)
|
||||
csv_buf = StringIO()
|
||||
df_1.to_csv(csv_buf, header=True, index=False)
|
||||
csv_buf.seek(0)
|
||||
s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/results.csv')
|
||||
s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(),
|
||||
Key='training_interface/results.csv')
|
||||
csv_buf = StringIO()
|
||||
history.tail(1).to_csv(csv_buf, header=True, index=False)
|
||||
csv_buf.seek(0)
|
||||
s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/history.csv')
|
||||
df = df[['Contract Name','Contract ID', 'LOB_Product_Network_Metal_Area_Program_Type_Speciality','SF_DB_COL_NAME', 'Actual Value Stored','New Extracted value','Confidence Level'
|
||||
,'Snippet','Original Page Number', 'New Page Number', 'Revised Prompt', 'Result']]
|
||||
s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(),
|
||||
Key='training_interface/history.csv')
|
||||
df = df[['Contract Name', 'Contract ID', 'LOB_Product_Network_Metal_Area_Program_Type_Speciality', 'SF_DB_COL_NAME',
|
||||
'Actual Value Stored', 'New Extracted value', 'Confidence Level'
|
||||
, 'Snippet', 'Original Page Number', 'New Page Number', 'Revised Prompt', 'Result']]
|
||||
|
||||
try:
|
||||
df = pd.read_csv('results.csv')
|
||||
df['Result'] = df['Result'].astype('str')
|
||||
except:
|
||||
df = pd.DataFrame(columns=['Contract Name' ,'New Extracted value','Confidence Level','Snippet','New Page Number'
|
||||
df = pd.DataFrame(columns=['Contract Name', 'New Extracted value', 'Confidence Level', 'Snippet', 'New Page Number'
|
||||
, 'Revised Prompt', 'Result'])
|
||||
history = pd.read_csv('history.csv')
|
||||
st.dataframe(df)
|
||||
@@ -752,8 +777,6 @@ st.write(len(contract_list))
|
||||
st.write(raw_response_text)
|
||||
|
||||
try:
|
||||
save_to_sf('load_training_results', training_results_file_name = "results.csv", attempt_logs_file_name = "history.csv")
|
||||
save_to_sf('load_training_results', training_results_file_name="results.csv", attempt_logs_file_name="history.csv")
|
||||
except:
|
||||
st.write("running locally")
|
||||
|
||||
|
||||
|
||||
+37
-7
@@ -1,17 +1,39 @@
|
||||
|
||||
import streamlit as st
|
||||
import security
|
||||
import os
|
||||
import constants
|
||||
import logging
|
||||
from logging.handlers import RotatingFileHandler
|
||||
|
||||
def setup_page(REDIRECT_URI):
|
||||
# Ensure the log directory exists
|
||||
log_dir = '/home/ubuntu/doczy.ai/streamlit'
|
||||
if not os.path.exists(log_dir):
|
||||
os.makedirs(log_dir, exist_ok=True)
|
||||
|
||||
# Configure logging with RotatingFileHandler
|
||||
log_file = f'{log_dir}/interface.log'
|
||||
rotating_handler = RotatingFileHandler(
|
||||
log_file,
|
||||
maxBytes=10*1024*1024, # 10 MB
|
||||
backupCount=5 # Keep up to 5 backup files
|
||||
)
|
||||
rotating_handler.setLevel(logging.INFO)
|
||||
formatter = logging.Formatter('%(asctime)s %(levelname)s [%(filename)s] %(message)s')
|
||||
rotating_handler.setFormatter(formatter)
|
||||
|
||||
logger = logging.getLogger()
|
||||
logger.setLevel(logging.INFO)
|
||||
logger.addHandler(rotating_handler)
|
||||
|
||||
|
||||
def setup_page(redirect_uril):
|
||||
# st.set_page_config(
|
||||
# page_title=page_title,
|
||||
# page_icon="👋",
|
||||
# )
|
||||
|
||||
if st.query_params.get('code'):
|
||||
security.handle_redirect(REDIRECT_URI)
|
||||
security.handle_redirect(redirect_uril)
|
||||
|
||||
access_token = st.session_state.get('access_token')
|
||||
|
||||
@@ -21,15 +43,23 @@ def setup_page(REDIRECT_URI):
|
||||
return True
|
||||
else:
|
||||
st.write("Please sign-in to use this app.")
|
||||
auth_url = security.get_auth_url(REDIRECT_URI)
|
||||
auth_url = security.get_auth_url(redirect_uril)
|
||||
st.markdown(f"<a href='{auth_url}' target='_self'>Sign In</a>", unsafe_allow_html=True)
|
||||
st.stop()
|
||||
|
||||
|
||||
def load_page_details(interface):
|
||||
env_var = os.environ.get('ENVIRONMENT', 'DEV')
|
||||
logger.info(f"env_var={env_var}")
|
||||
if env_var == 'UAT':
|
||||
return (constants.DOCZY_REDIRECT_URL_UAT + str(interface), constants.DOCZY_CREATE_BATCH_URL_UAT, constants.DOCZY_PIPELINE_URL_UAT)
|
||||
logger.info(constants.DOCZY_REDIRECT_URL_UAT + str(interface))
|
||||
return (constants.DOCZY_REDIRECT_URL_UAT + str(interface), constants.DOCZY_CREATE_BATCH_URL_UAT,
|
||||
constants.DOCZY_PIPELINE_URL_UAT)
|
||||
elif env_var == 'DEV':
|
||||
return (constants.DOCZY_REDIRECT_URL_DEV + str(interface), constants.DOCZY_CREATE_BATCH_URL_DEV, constants.DOCZY_PIPELINE_URL_DEV)
|
||||
logger.info(constants.DOCZY_REDIRECT_URL_DEV + str(interface))
|
||||
return (constants.DOCZY_REDIRECT_URL_DEV + str(interface), constants.DOCZY_CREATE_BATCH_URL_DEV,
|
||||
constants.DOCZY_PIPELINE_URL_DEV)
|
||||
else:
|
||||
return (constants.DOCZY_REDIRECT_URL_DEV + str(interface), constants.DOCZY_CREATE_BATCH_URL_DEV, constants.DOCZY_PIPELINE_URL_DEV)
|
||||
logger.info(constants.DOCZY_REDIRECT_URL_DEV + str(interface))
|
||||
return (constants.DOCZY_REDIRECT_URL_DEV + str(interface), constants.DOCZY_CREATE_BATCH_URL_DEV,
|
||||
constants.DOCZY_PIPELINE_URL_DEV)
|
||||
|
||||
Reference in New Issue
Block a user