diff --git a/streamlit-server/main.tf b/streamlit-server/main.tf index ee5a6a9..3e0540b 100644 --- a/streamlit-server/main.tf +++ b/streamlit-server/main.tf @@ -107,7 +107,6 @@ resource "aws_instance" "streamlit_server" { export ENVIRONMENT=${var.environment} - # Create UI0 systemd service cat < /etc/systemd/system/streamlit-ui0.service [Unit] @@ -118,6 +117,7 @@ resource "aws_instance" "streamlit_server" { Type=simple Restart=always WorkingDirectory=/home/ubuntu/doczy.ai/streamlit + Environment="ENVIRONMENT=${upper(var.environment)}" ExecStart=/home/ubuntu/.local/bin/streamlit run /home/ubuntu/doczy.ai/streamlit/interface_0.py --server.port 8500 [Install] @@ -138,6 +138,7 @@ resource "aws_instance" "streamlit_server" { Type=simple Restart=always WorkingDirectory=/home/ubuntu/doczy.ai/streamlit + Environment="ENVIRONMENT=${upper(var.environment)}" ExecStart=/home/ubuntu/.local/bin/streamlit run /home/ubuntu/doczy.ai/streamlit/interface_1.py --server.port 8501 [Install] @@ -157,6 +158,7 @@ resource "aws_instance" "streamlit_server" { Type=simple Restart=always WorkingDirectory=/home/ubuntu/doczy.ai/streamlit + Environment="ENVIRONMENT=${upper(var.environment)}" ExecStart=/home/ubuntu/.local/bin/streamlit run /home/ubuntu/doczy.ai/streamlit/interface_2.py --server.port 8502 [Install] @@ -176,6 +178,7 @@ resource "aws_instance" "streamlit_server" { Type=simple Restart=always WorkingDirectory=/home/ubuntu/doczy.ai/streamlit + Environment="ENVIRONMENT=${upper(var.environment)}" ExecStart=/home/ubuntu/.local/bin/streamlit run /home/ubuntu/doczy.ai/streamlit/interface_3.py --server.port 8503 [Install] @@ -194,6 +197,7 @@ resource "aws_instance" "streamlit_server" { Type=simple Restart=always WorkingDirectory=/home/ubuntu/doczy.ai/streamlit/multipage + Environment="ENVIRONMENT=${upper(var.environment)}" ExecStart=/home/ubuntu/.local/bin/streamlit run /home/ubuntu/doczy.ai/streamlit/multipage/Interface_0.py --server.port 8505 [Install] diff --git a/streamlit/interface_0.py b/streamlit/interface_0.py index a2809f3..dfb9436 100644 --- a/streamlit/interface_0.py +++ b/streamlit/interface_0.py @@ -15,8 +15,9 @@ from io import StringIO, BytesIO import time from sf_conn import get_client_names, insert_upload_logs from constants import USER_LIST +from util import logger -(REDIRECT_URI, create_batch_url, doczy_pipeline) = util.load_page_details(0) +(redirect_uri, create_batch_url, doczy_pipeline) = util.load_page_details(0) user_list = USER_LIST if 'uploading' not in st.session_state: @@ -74,7 +75,7 @@ st.markdown(hide_img_fs, unsafe_allow_html=True) _,c1= st.columns([5,1]) try: - util.setup_page(REDIRECT_URI) + util.setup_page(redirect_uri) except Exception as e: st.write(f"SSO Failed = {e}") st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} @@ -83,7 +84,7 @@ try: user_mail = st.session_state.user_info['mail'] except KeyError as e: st.write("Session Expired.") - auth_url = security.get_auth_url(REDIRECT_URI) + auth_url = security.get_auth_url(redirect_uri) st.markdown(f"Sign In", unsafe_allow_html=True) st.stop() @@ -99,6 +100,8 @@ client_list = ['doczy-ai-client-1', 'Delaware First Health, Inc.', 'Community He # TODO: Need to update the streamlit code to use the client names from this list # And use the s3 paths to save the objects for the respective client client_list, s3_paths = get_client_names() +logger.info(f"client_list={client_list}") +logger.info(f"s3_paths={s3_paths}") client_s3_paths = dict(zip(client_list, s3_paths)) client_row = st.columns([0.1, 0.8]) diff --git a/streamlit/interface_1.py b/streamlit/interface_1.py index 48fc09d..75a9a4f 100644 --- a/streamlit/interface_1.py +++ b/streamlit/interface_1.py @@ -12,11 +12,12 @@ import requests import time from sf_conn import get_client_names, get_secret, save_to_sf from constants import USER_LIST +from util import logger -(REDIRECT_URI, create_batch_url, doczy_pipeline) = util.load_page_details(1) +(redirect_uri, create_batch_url, doczy_pipeline) = util.load_page_details(1) user_list = USER_LIST -st.set_page_config(layout = "wide") +st.set_page_config(layout="wide") # Sidebar contents with st.sidebar: st.title("Doczy.AI ™") @@ -24,14 +25,14 @@ with st.sidebar: """ ## About This app extracts data from contracts - + """ ) add_vertical_space(15) # st.write("Doczy") # AARETE LOGO -x,y,z = st.columns([15,2,15]) +x, y, z = st.columns([15, 2, 15]) with y: st.image('aaretelogo.png') @@ -43,9 +44,9 @@ button[title="View fullscreen"]{ ''' st.markdown(hide_img_fs, unsafe_allow_html=True) -_,c1= st.columns([5,1]) +_, c1 = st.columns([5, 1]) try: - util.setup_page(REDIRECT_URI) + util.setup_page(redirect_uri) except: st.write("SSO Failed") st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} @@ -56,15 +57,13 @@ except KeyError as e: # Do we add a link to get to the login page here? st.write("Session Expired.") # st.write("Please sign-in to use this app.") - auth_url = security.get_auth_url(REDIRECT_URI) + auth_url = security.get_auth_url(redirect_uri) st.markdown(f"Sign In", unsafe_allow_html=True) st.stop() - - s3_client = boto3.client('s3', - region_name="us-east-2", -) + region_name="us-east-2", + ) # # to be replaced with snowflake data # client_list = ['doczy-ai-client-1', 'Delaware First Health, Inc.', 'Community Health Choice, Inc','CareSource Network Partners LLC', @@ -72,13 +71,16 @@ s3_client = boto3.client('s3', # 'WellCare New Jersey'] client_list, s3_paths = get_client_names() +logger.info(f"client_list={client_list}") +logger.info(f"s3_paths={s3_paths}") client_s3_paths = dict(zip(client_list, s3_paths)) client_row = st.columns([0.1, 0.8]) with client_row[0]: st.write("**Client Name**") with client_row[1]: - client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344 + client = st.selectbox('Client Name', (client_list), label_visibility="collapsed", + index=None) # MODIFIED for Ticket DOC-344 client_bucket = client_s3_paths.get(client) @@ -86,7 +88,7 @@ client_bucket = client_s3_paths.get(client) client_bucket = 'doczy-dev-infra-textract' batch_objects = s3_client.list_objects_v2(Bucket=client_bucket - , Prefix="contracts-landing-zone/", Delimiter='/') + , Prefix="contracts-landing-zone/", Delimiter='/') batch_list = [] for prefix in batch_objects['CommonPrefixes']: @@ -100,6 +102,7 @@ for prefix in batch_objects['CommonPrefixes']: if 'sorted_list' not in st.session_state: st.session_state.sorted_list = batch_list + def sort_list(ex_list, sort_by, order): if sort_by == 'Alphabetical': ex_list = sorted(ex_list, reverse=(order == 'Descending')) @@ -108,16 +111,16 @@ def sort_list(ex_list, sort_by, order): return ex_list + col1, col2, col3, col4 = st.columns([0.5, 0.5, 0.5, 0.5]) - -with col1: +with col1: sort_by = st.radio("**Sort Batch_IDs**", ('Alphabetical', 'Create Date')) with col2: - order = st.radio('', ('Ascending','Descending')) + order = st.radio('', ('Ascending', 'Descending')) -with col3: +with col3: add_vertical_space(2) if st.button('Apply'): st.session_state.sorted_list = sort_list(batch_list, sort_by, order) @@ -126,7 +129,8 @@ path_row = st.columns([0.1, 0.8]) with path_row[0]: st.write("**Batch ID**") with path_row[1]: - batch_id = st.selectbox('**Batch ID**', st.session_state.sorted_list, label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344 + batch_id = st.selectbox('**Batch ID**', st.session_state.sorted_list, label_visibility="collapsed", + index=None) # MODIFIED for Ticket DOC-344 if not batch_id: batch_id = "None" @@ -136,37 +140,37 @@ with checks[0]: st.write("**Group No.**") with checks[1]: - a = st.checkbox('Unique Key', key = str(1), args="Unique") + a = st.checkbox('Unique Key', key=str(1), args="Unique") with checks[2]: - b = st.checkbox('Pricing Before Carveouts', key = str(2)) + b = st.checkbox('Pricing Before Carveouts', key=str(2)) with checks[3]: - c = st.checkbox('Contract Related', key = str(3)) + c = st.checkbox('Contract Related', key=str(3)) with checks[4]: - d = st.checkbox('Provider', key = str(4)) + d = st.checkbox('Provider', key=str(4)) with checks[5]: - e = st.checkbox('Timeline', key = str(5)) + e = st.checkbox('Timeline', key=str(5)) with checks[6]: - f = st.checkbox('Carveout Indicator', key = str(6)) + f = st.checkbox('Carveout Indicator', key=str(6)) with checks[7]: - g = st.checkbox('Carveout Methodology', key = str(7)) + g = st.checkbox('Carveout Methodology', key=str(7)) add_vertical_space(1) -df = pd.DataFrame(columns=['Contract Name', 'Unique Key','Pricing Before Carveouts' +df = pd.DataFrame(columns=['Contract Name', 'Unique Key', 'Pricing Before Carveouts' , 'Contract Related', 'Provider', 'Timeline', 'Carveout Indicator', 'Carveout Methodology']) file_list = [] file_objects = s3_client.list_objects_v2(Bucket=client_bucket - , Prefix="contracts-landing-zone/"+batch_id+"/", Delimiter='/') + , Prefix="contracts-landing-zone/" + batch_id + "/", Delimiter='/') # Hardcoded file_list for testing purposes # file_list = ['Boilerplate_TX Amendment Mission Health Network effective_040114 MU.pdf', 'Custom_TX - MP AMENDMENT - MISSION HEALTH NETWORK - MU.pdf', # 'Delaware First Health_First State Homecare Agency_212260_7 MU.pdf', 'Molina Healthcare of Texas, Inc. Amendment 4 - HIX ACA__EFF 01012016_MU.pdf'] if st.button("Read the contracts from Path"): - for obj in file_objects.get('Contents',[]): + for obj in file_objects.get('Contents', []): if not obj['Key'].endswith('/'): file_list.append(obj['Key'].split('/')[-1]) - + df['Contract Name'] = file_list # df['Request ID'] = range(len(file_list)) # df['Contract ID'] = file_list @@ -190,17 +194,20 @@ edited_df['LATEST_FLAG BOOLEAN'] = True edited_df['PIPELINE_KICKOFF_DATETIME'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") edited_df['REQUEST_DATETIME'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") -@st.cache_data + +@st.cache_data def convert_df(df): return df.to_csv(index=False).encode('utf-8') + csv = convert_df(edited_df) # edited_df = edited_df.reset_index() # make sure indexes pair with number of rows # additional_info = pd.DataFrame(columns=['REQUEST_ID','T_DRIVE_PATH','CLIENT_NAME' # , 'GROUP_NAME', 'REQUEST_USERNAME', 'REQUEST_DATETIME']) additional_info = pd.DataFrame(columns=['CLIENT_NAME', 'BATCH_ID', 'REQUEST_USERNAME', 'REQUEST_DATETIME']) -additional_info.loc[0] = [client, batch_id, st.session_state.user_info['mail'], datetime.now().strftime("%Y-%m-%d %H:%M:%S")] +additional_info.loc[0] = [client, batch_id, st.session_state.user_info['mail'], + datetime.now().strftime("%Y-%m-%d %H:%M:%S")] st.write(additional_info) st.session_state.contract_count = 0 @@ -233,7 +240,7 @@ for index, row in edited_df.iterrows(): entry_dict = { "contract_name": row['Contract Name'], "groups": group_list, - "contract_source_path": "contracts_landing_zone/"+batch_id+"/"+row['Contract Name'] + "contract_source_path": "contracts_landing_zone/" + batch_id + "/" + row['Contract Name'] } contract_list.append(entry_dict) @@ -253,7 +260,7 @@ with buttons[1]: if not st.session_state.contract_count == len(edited_df): st.error("Select at least one Group No. for every Contract") else: - with st.spinner('Running...'): + with st.spinner('Running...'): # csv_buf = StringIO() # additional_info.to_csv(csv_buf, header=True, index=False) # csv_buf.seek(0) @@ -266,11 +273,10 @@ with buttons[1]: # save_to_sf('load_request_and_contract_submissions', request_submission_file_name = "request_submission.csv", contract_config_file_name = "contract_config.csv") # except Exception as e: # st.write(e) - response = requests.post(doczy_pipeline, json = myobj) + response = requests.post(doczy_pipeline, json=myobj) if response.status_code >= 200 and response.status_code < 300: st.write("Success") # st.write(myobj) else: st.write("Failed") # st.write(response.text) - diff --git a/streamlit/interface_2.py b/streamlit/interface_2.py index eb9cb52..fba18b3 100644 --- a/streamlit/interface_2.py +++ b/streamlit/interface_2.py @@ -5,12 +5,13 @@ from langchain.prompts import PromptTemplate from langchain.embeddings.bedrock import BedrockEmbeddings from langchain.llms.bedrock import Bedrock from langchain_community.vectorstores import Chroma -from constants import CHROMA_SETTINGS, EMBEDDING_MODEL_NAME, PERSIST_DIRECTORY, MODEL_ID, MODEL_BASENAME, SOURCE_DIRECTORY, USER_LIST +from constants import CHROMA_SETTINGS, EMBEDDING_MODEL_NAME, PERSIST_DIRECTORY, MODEL_ID, MODEL_BASENAME, \ + SOURCE_DIRECTORY, USER_LIST from langchain.chains import RetrievalQA import streamlit as st from streamlit_extras.add_vertical_space import add_vertical_space -from streamlit_pdf_viewer import pdf_viewer # needs to be installed on the server +from streamlit_pdf_viewer import pdf_viewer # needs to be installed on the server import os import pandas as pd import numpy as np @@ -23,11 +24,12 @@ import base64 from sf_conn import get_snowflake_conn from sf_conn import get_client_names, get_secret, save_to_sf import io +from util import logger -(REDIRECT_URI, create_batch_url, doczy_pipeline) = util.load_page_details(2) +(redirect_uri, create_batch_url, doczy_pipeline) = util.load_page_details(2) user_list = USER_LIST -st.set_page_config(layout = "wide") +st.set_page_config(layout="wide") # Sidebar contents with st.sidebar: st.title("Doczy.AI ™") @@ -35,14 +37,14 @@ with st.sidebar: """ ## About This app extracts data from contracts - + """ ) # add_vertical_space(15) # st.write("Doczy") # AARETE LOGO -x,y,z = st.columns([15,2,15]) +x, y, z = st.columns([15, 2, 15]) with y: st.image('aaretelogo.png') @@ -54,9 +56,9 @@ button[title="View fullscreen"]{ ''' st.markdown(hide_img_fs, unsafe_allow_html=True) -_,c1= st.columns([5,1]) +_, c1 = st.columns([5, 1]) try: - util.setup_page(REDIRECT_URI) + util.setup_page(redirect_uri) except: st.write("SSO Failed") st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} @@ -65,8 +67,8 @@ try: user_mail = st.session_state.user_info['mail'] except KeyError as e: st.write("Session Expired.") - #st.write("Please sign-in to use this app.") - auth_url = security.get_auth_url(REDIRECT_URI) + # st.write("Please sign-in to use this app.") + auth_url = security.get_auth_url(redirect_uri) st.markdown(f"Sign In", unsafe_allow_html=True) st.stop() @@ -88,35 +90,37 @@ try: cur.execute(query) fields = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description]) - fields.rename(columns={'FIELD_DESC': 'Field Name'}, inplace = True) - fields.rename(columns={'PROMPT': 'Interrogation Question?'}, inplace = True) - fields.rename(columns={'GROUP_ID': 'PRIORITY'}, inplace = True) - fields.rename(columns={'FIELD_NAME': 'SF_DB_COL_NAME'}, inplace = True) - fields.rename(columns={'FM_MODEL_ID': 'llm_selected'}, inplace = True) + fields.rename(columns={'FIELD_DESC': 'Field Name'}, inplace=True) + fields.rename(columns={'PROMPT': 'Interrogation Question?'}, inplace=True) + fields.rename(columns={'GROUP_ID': 'PRIORITY'}, inplace=True) + fields.rename(columns={'FIELD_NAME': 'SF_DB_COL_NAME'}, inplace=True) + fields.rename(columns={'FM_MODEL_ID': 'llm_selected'}, inplace=True) except Exception as e: - st.write("Unable to fetch data from Snowflake: ",e) + st.write("Unable to fetch data from Snowflake: ", e) # fields = pd.read_csv('contract_fields.csv', encoding='unicode_escape', skipinitialspace=True) # fields = fields[~fields['SF_COL_NAME'].str.endswith('_PG', na=None)] # change the code below if contract list is fetched from snowflake s3_client = boto3.client('s3', - region_name="us-east-2" -) + region_name="us-east-2" + ) client_list, s3_paths = get_client_names() +logger.info(f"client_list={client_list}") +logger.info(f"s3_paths={s3_paths}") client_s3_paths = dict(zip(client_list, s3_paths)) client_row = st.columns([0.2, 0.7, 0.1]) with client_row[0]: st.write("**Client Name**") with client_row[1]: - client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed", index= None) + client = st.selectbox('Client Name', (client_list), label_visibility="collapsed", index=None) # client_bucket = client_s3_paths.get(client) client_bucket = 'doczy-dev-infra-textract' batch_objects = s3_client.list_objects_v2(Bucket=client_bucket - , Prefix="contracts-landing-zone/", Delimiter='/') + , Prefix="contracts-landing-zone/", Delimiter='/') batch_list = [] for prefix in batch_objects['CommonPrefixes']: @@ -126,13 +130,13 @@ path_row = st.columns([0.2, 0.7, 0.1]) with path_row[0]: st.write("**Batch ID**") with path_row[1]: - batch_id = st.selectbox('**Batch ID**', batch_list, label_visibility = "collapsed", index = None) + batch_id = st.selectbox('**Batch ID**', batch_list, label_visibility="collapsed", index=None) if batch_id: - objects = s3_client.list_objects_v2(Bucket=client_bucket, Prefix="contracts-landing-zone/"+batch_id+"/") + objects = s3_client.list_objects_v2(Bucket=client_bucket, Prefix="contracts-landing-zone/" + batch_id + "/") file_list = [] - if 'Contents' in objects: + if 'Contents' in objects: for obj in objects['Contents']: if not obj['Key'].endswith('/'): file_list.append(obj['Key']) @@ -145,17 +149,20 @@ if batch_id: with file_row[0]: st.write("**Contract Name**") with file_row[1]: - file_name = st.selectbox('Select a file', ['All'] + contract_list, label_visibility = "collapsed", index= None) + file_name = st.selectbox('Select a file', ['All'] + contract_list, label_visibility="collapsed", index=None) field_row = st.columns([0.2, 0.7, 0.1]) with field_row[0]: st.write("**Field Group**") with field_row[1]: - field_group = st.selectbox('Field Group',('Unique Key', 'Contract Related', 'Pricing Before Carveouts - I' - , 'Pricing Before Carveouts - II', 'Carveout Indicator, Code Type and Code #s - I' - , 'Carveout Indicator, Code Type and Code #s - II', 'Carveout Indicator, Code Type and Code #s - III' - , 'Optimize Carving Indic.', 'Carveout Method - I', 'Carveout Method - II', 'Provider' - , 'Timeline'), label_visibility = "collapsed", index = None) + field_group = st.selectbox('Field Group', ('Unique Key', 'Contract Related', 'Pricing Before Carveouts - I' + , 'Pricing Before Carveouts - II', + 'Carveout Indicator, Code Type and Code #s - I' + , 'Carveout Indicator, Code Type and Code #s - II', + 'Carveout Indicator, Code Type and Code #s - III' + , 'Optimize Carving Indic.', 'Carveout Method - I', + 'Carveout Method - II', 'Provider' + , 'Timeline'), label_visibility="collapsed", index=None) if field_group == 'Unique Key': fields = fields[fields['PRIORITY'] == 'A'] @@ -218,8 +225,9 @@ if batch_id: """, unsafe_allow_html=True, ) - s3_obj = s3_client.get_object(Bucket = client_bucket, Key = file_name) - data=s3_obj['Body'].read() + s3_obj = s3_client.get_object(Bucket=client_bucket, + Key=file_name) + data = s3_obj['Body'].read() pdf_viewer(data, width=1500) # if st.button("Show PDF"): @@ -248,10 +256,12 @@ if batch_id: df2['Imputed Value'] = '' edited_df = st.data_editor(df2) - @st.cache_data + + @st.cache_data def convert_df(df): return df.to_csv(index=False).encode('utf-8') + csv = convert_df(edited_df) buttons = st.columns(3) @@ -264,7 +274,3 @@ if batch_id: with buttons[2]: if st.button("Kickoff Database Integration"): st.write("Stored in DB") - - - - diff --git a/streamlit/interface_3.py b/streamlit/interface_3.py index 2da0b40..788357c 100644 --- a/streamlit/interface_3.py +++ b/streamlit/interface_3.py @@ -4,7 +4,8 @@ from langchain.prompts import PromptTemplate from langchain.embeddings.bedrock import BedrockEmbeddings from langchain.llms.bedrock import Bedrock from langchain_community.vectorstores import Chroma -from constants import CHROMA_SETTINGS, EMBEDDING_MODEL_NAME, PERSIST_DIRECTORY, MODEL_ID, MODEL_BASENAME, SOURCE_DIRECTORY, USER_LIST +from constants import CHROMA_SETTINGS, EMBEDDING_MODEL_NAME, PERSIST_DIRECTORY, MODEL_ID, MODEL_BASENAME, \ + SOURCE_DIRECTORY, USER_LIST from langchain.chains import RetrievalQA import streamlit as st @@ -22,12 +23,12 @@ import re import snowflake.connector from sf_conn import get_secret, save_to_sf from io import StringIO +from util import logger - -(REDIRECT_URI, create_batch_url, doczy_pipeline) = util.load_page_details(3) +(redirect_uri, create_batch_url, doczy_pipeline) = util.load_page_details(3) user_list = USER_LIST -st.set_page_config(layout = "wide") +st.set_page_config(layout="wide") # Sidebar contents with st.sidebar: st.title("Doczy.AI ™") @@ -41,10 +42,10 @@ with st.sidebar: add_vertical_space(15) # st.write("Doczy") -_,c1= st.columns([5,1]) -# util.setup_page(REDIRECT_URI) +_, c1 = st.columns([5, 1]) +# util.setup_page(redirect_uri) try: - util.setup_page(REDIRECT_URI) + util.setup_page(redirect_uri) except: st.write("SSO Failed") st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} @@ -53,7 +54,7 @@ try: user_mail = st.session_state.user_info['mail'] except KeyError as e: st.write("Session Expired.") - st. stop() + st.stop() try: sf_secrets = json.loads(get_secret()) @@ -61,11 +62,11 @@ try: user=sf_secrets.get('user'), password=sf_secrets.get('password'), account="aarete-doczyai", - role = "DEVADMIN", + role="DEVADMIN", warehouse="DEV_XS", database="DOCZY_DEV", schema="STG" - ) + ) cur = conn.cursor() query = 'select * from "TRAINING_DATA_RAW"' cur.execute(query) @@ -87,9 +88,9 @@ try: query = 'select * from "BUSINESS_CONFIG"' cur.execute(query) fields = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description]) - fields.rename(columns={'FIELD_NAME': 'Field Name'}, inplace = True) - fields.rename(columns={'QUESTION': 'Interrogation Question?'}, inplace = True) - fields.rename(columns={'SF_COL_NAME': 'SF_DB_COL_NAME'}, inplace = True) + fields.rename(columns={'FIELD_NAME': 'Field Name'}, inplace=True) + fields.rename(columns={'QUESTION': 'Interrogation Question?'}, inplace=True) + fields.rename(columns={'SF_COL_NAME': 'SF_DB_COL_NAME'}, inplace=True) fields = fields[~fields['SF_DB_COL_NAME'].str.endswith('_PG', na=None)] fields['Field Name'] = fields['SF_DB_COL_NAME'] # fields.rename(columns={'FM_MODEL_ID': 'llm_selected'}, inplace = True) @@ -99,9 +100,9 @@ except: # fields = fields.drop_duplicates(subset='Field Name', keep="first").sort_values('Field Name') # fields['Field Name'] = fields['SF_DB_COL_NAME'] # fields = fields[~fields['Field Name'].isnull()] - fields.rename(columns={'FIELD_NAME': 'Field Name'}, inplace = True) - fields.rename(columns={'QUESTION': 'Interrogation Question?'}, inplace = True) - fields.rename(columns={'SF_COL_NAME': 'SF_DB_COL_NAME'}, inplace = True) + fields.rename(columns={'FIELD_NAME': 'Field Name'}, inplace=True) + fields.rename(columns={'QUESTION': 'Interrogation Question?'}, inplace=True) + fields.rename(columns={'SF_COL_NAME': 'SF_DB_COL_NAME'}, inplace=True) fields = fields[~fields['SF_DB_COL_NAME'].str.endswith('_PG', na=None)] fields['Field Name'] = fields['SF_DB_COL_NAME'] st.write("Local copy of BUSINESS_CONFIG table loaded") @@ -110,33 +111,33 @@ try: query = 'select * from "TRAINING_ATTEMPT_LOGS"' cur.execute(query) history = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description]) - history.rename(columns={'FIELD_NAME': 'Field Name'}, inplace = True) - history.rename(columns={'CONTRACTS_TESTED': '# Contracts Tested'}, inplace = True) - history.rename(columns={'USERNAME': 'Username'}, inplace = True) - history.rename(columns={'DATE_TIME': 'Date/Time'}, inplace = True) - history.rename(columns={'ACCURACY': 'Accuracy'}, inplace = True) - history.rename(columns={'ATTEMPT_NUM': 'Attempt #'}, inplace = True) + history.rename(columns={'FIELD_NAME': 'Field Name'}, inplace=True) + history.rename(columns={'CONTRACTS_TESTED': '# Contracts Tested'}, inplace=True) + history.rename(columns={'USERNAME': 'Username'}, inplace=True) + history.rename(columns={'DATE_TIME': 'Date/Time'}, inplace=True) + history.rename(columns={'ACCURACY': 'Accuracy'}, inplace=True) + history.rename(columns={'ATTEMPT_NUM': 'Attempt #'}, inplace=True) history = history[['Field Name', '# Contracts Tested', 'Username', 'Date/Time', 'Accuracy', 'Attempt #']] except: history = pd.read_csv('history.csv') st.write("Local copy of TRAINING_ATTEMPT_LOGS table loaded") - - field_row = st.columns([0.15, 0.45, 0.4]) with field_row[0]: st.write("**Field Group**") with field_row[1]: - field_group = st.selectbox('Field Group',('Unique and Contract Related', 'Pricing Before Carveouts - All' - , 'Pricing Before Carveouts - I', 'Pricing Before Carveouts - II', 'Carveout Indicator, Code Type and Code #s - I' - , 'Carveout Indicator, Code Type and Code #s - II', 'Carveout Indicator, Code Type and Code #s - III' - , 'Optimize Carving Indic.', 'Carveout Method - I', 'Carveout Method - II', 'Provider' - , 'Timeline'), label_visibility = "collapsed") - + field_group = st.selectbox('Field Group', ('Unique and Contract Related', 'Pricing Before Carveouts - All' + , 'Pricing Before Carveouts - I', 'Pricing Before Carveouts - II', + 'Carveout Indicator, Code Type and Code #s - I' + , 'Carveout Indicator, Code Type and Code #s - II', + 'Carveout Indicator, Code Type and Code #s - III' + , 'Optimize Carving Indic.', 'Carveout Method - I', + 'Carveout Method - II', 'Provider' + , 'Timeline'), label_visibility="collapsed") # priorty column will be relaced by group_id in snowflake db if field_group == 'Unique and Contract Related': - fields = fields[fields['PRIORITY'].isin(['A','C'])] + fields = fields[fields['PRIORITY'].isin(['A', 'C'])] elif field_group == 'Pricing Before Carveouts - All': fields = fields[fields['PRIORITY'] == 'B'] elif field_group == 'Pricing Before Carveouts - I': @@ -178,28 +179,32 @@ mode_row = st.columns([0.15, 0.45, 0.4]) with mode_row[0]: st.write("**Mode**") with mode_row[1]: - mode = st.selectbox('Mode',('Single field - Non Empty values', 'Multiple fields', 'One-to-many fields'), index=0, label_visibility = "collapsed") + mode = st.selectbox('Mode', ('Single field - Non Empty values', 'Multiple fields', 'One-to-many fields'), index=0, + label_visibility="collapsed") field_row = st.columns([0.15, 0.45, 0.4]) with field_row[0]: st.write("**Field Name**") with field_row[1]: if mode == 'Single field - Non Empty values': - field = st.selectbox('Field Name',sorted(set(field_prompt_mapping.keys())), index=0, label_visibility = "collapsed") + field = st.selectbox('Field Name', sorted(set(field_prompt_mapping.keys())), index=0, + label_visibility="collapsed") else: - field = st.multiselect('Field Name',sorted(set(field_prompt_mapping.keys())), sorted( - set(field_prompt_mapping.keys())), label_visibility = "collapsed") + field = st.multiselect('Field Name', sorted(set(field_prompt_mapping.keys())), sorted( + set(field_prompt_mapping.keys())), label_visibility="collapsed") field_prompt_mapping = {key: field_prompt_mapping[key] for key in field} contract_count_row = st.columns([0.15, 0.45, 0.4]) with contract_count_row[0]: st.write("**# of Contracts**") with contract_count_row[1]: - contract_count = st.selectbox('Contract count',('1', '10', '20', '30', '50', '100', '200','Part-1', 'Part-2', 'All'), index=1, label_visibility = "collapsed") + contract_count = st.selectbox('Contract count', + ('1', '10', '20', '30', '50', '100', '200', 'Part-1', 'Part-2', 'All'), index=1, + label_visibility="collapsed") s3_client = boto3.client('s3', - region_name="us-east-2" -) + region_name="us-east-2" + ) bucket = 'doczy-dev-infra-textract' objects = s3_client.list_objects_v2(Bucket=bucket, Prefix="training-data/") file_list = [] @@ -221,8 +226,9 @@ if mode == 'Single field - Non Empty values': # st.write(df) # st.write(len(contract_list)) # contract_list = [contract for contract in contract_list if contract.rsplit('/',1)[1].replace(' MU','').replace('_MU','').replace('.txt','') in list(field_values['Document_Name'])] -contract_list = [str(contract)[:-4]+'.pdf' for contract in contract_list] -contract_list = [contract for contract in contract_list if contract.rsplit('/',1)[1] in list(field_values['Document_Name'])] +contract_list = [str(contract)[:-4] + '.pdf' for contract in contract_list] +contract_list = [contract for contract in contract_list if + contract.rsplit('/', 1)[1] in list(field_values['Document_Name'])] # st.write(len(contract_list)) if contract_count == 'All': @@ -242,21 +248,21 @@ with seed_row[0]: st.write("**Contract Name**") with seed_row[1]: if contract_count in ['10', '20', '30', '50', '100', '200']: - seed_value = st.text_input("**Seed Value**", value = 20, label_visibility = "collapsed") + seed_value = st.text_input("**Seed Value**", value=20, label_visibility="collapsed") random.seed(seed_value) # contract_list = sorted(random.choices(os.listdir(SOURCE_DIRECTORY), k=int(contract_count))) contract_list = sorted(random.choices(contract_list, k=int(contract_count))) elif contract_count == '1': - contract_name = st.selectbox('Contract Name', (contract_list), label_visibility = "collapsed") + contract_name = st.selectbox('Contract Name', (contract_list), label_visibility="collapsed") contract_list = [contract_name] - llm_row = st.columns([0.15, 0.45, 0.4]) with llm_row[0]: st.write("**Langauge Model**") with llm_row[1]: - llm_selected = st.selectbox('Langauge Model',('Claude 2', 'Claude 3 - Haiku', 'Claude 3 - Sonnet', 'Claude Instant' - , 'Llama 2 Chat 70B', 'Titan Text Express'), index=3, label_visibility = "collapsed") + llm_selected = st.selectbox('Langauge Model', ('Claude 2', 'Claude 3 - Haiku', 'Claude 3 - Sonnet', 'Claude Instant' + , 'Llama 2 Chat 70B', 'Titan Text Express'), index=3, + label_visibility="collapsed") st.write("**Prompt**") if mode != 'Single field - Non Empty values': @@ -271,8 +277,7 @@ with prompt_row[1]: prompt = sequence_input # st.button("Save Prompt") with prompt_row[0]: - prompt = st.text_area("**Prompt**", sequence_input, height = 100, label_visibility = "collapsed") - + prompt = st.text_area("**Prompt**", sequence_input, height=100, label_visibility="collapsed") # column_name = fields.loc[fields['Field Name'] == field, 'SF_DB_COL_NAME'].iloc[0] # column_list = ['Document_Name', column_name] @@ -285,7 +290,8 @@ with prompt_row[0]: field_values.rename(columns={'Document_Name': 'Contract Name'}, inplace=True) if mode != 'One-to-many fields': field_values = field_values.drop_duplicates(subset='Contract Name', keep="first").sort_values('Contract Name') -field_values = field_values[field_values['Contract Name'].isin([contract.rsplit('/',1)[1] for contract in contract_list])] +field_values = field_values[ + field_values['Contract Name'].isin([contract.rsplit('/', 1)[1] for contract in contract_list])] # Setup bedrock bedrock_runtime = boto3.client( @@ -296,21 +302,22 @@ bedrock_runtime = boto3.client( # question = prompt if mode != 'Single field - Non Empty values': prompt_dict = json.loads(prompt) - prompt_dict_pg = prompt_dict | {str(k)+'_PG': "On which page can I find answer to the question - "+str( + prompt_dict_pg = prompt_dict | {str(k) + '_PG': "On which page can I find answer to the question - " + str( v) for k, v in prompt_dict.items()} question = json.dumps(dict(sorted(prompt_dict_pg.items()))) else: - question = json.dumps({field: prompt, str(field)+'_PG': "On which page can I find answer to the question - "+str(prompt)}) + question = json.dumps( + {field: prompt, str(field) + '_PG': "On which page can I find answer to the question - " + str(prompt)}) # st.write(question) question_with_schema = question attempt = 0 - -def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, history): + +def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, history): # df = pd.DataFrame(columns=['Contract Name','Contract ID','Actual Value Stored','New Extracted value','Confidence Level','Snippet', # 'Original Page Number', 'New Page Number', 'Revised Prompt', 'Result']) - df = pd.DataFrame(columns=['Contract Name' ,'New Extracted value','Confidence Level','Snippet','New Page Number' + df = pd.DataFrame(columns=['Contract Name', 'New Extracted value', 'Confidence Level', 'Snippet', 'New Page Number' , 'Revised Prompt', 'Result']) field_list = [] @@ -323,7 +330,7 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h for contract in contract_list: # with open(os.path.join(SOURCE_DIRECTORY, contract[:-4]+'.txt'), 'r') as infile: # context = infile.read() - data = s3_client.get_object(Bucket=bucket, Key=str(contract)[:-4]+'.txt') + data = s3_client.get_object(Bucket=bucket, Key=str(contract)[:-4] + '.txt') contents = data['Body'].read() context = contents.decode("utf-8") # st.write(question_with_schema) @@ -340,11 +347,11 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h Question: {question} Answer: Answer in JSON format: {{""" parameters = { - "maxTokenCount":2048, - "stopSequences":[], - "temperature":0, - "topP":0.9 - } + "maxTokenCount": 2048, + "stopSequences": [], + "temperature": 0, + "topP": 0.9 + } body = json.dumps({"inputText": prompt_data, "textGenerationConfig": parameters}) model_id = "amazon.titan-text-express-v1" # change this to use a different version from the model provider @@ -358,14 +365,14 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h Question: {question} Answer: Answer in JSON format: {{""" - payload={ - "prompt":"[INST]"+ prompt_data +"[/INST]", - "max_gen_len":2048, - "temperature":0.0, - "top_p":0.9 + payload = { + "prompt": "[INST]" + prompt_data + "[/INST]", + "max_gen_len": 2048, + "temperature": 0.0, + "top_p": 0.9 } - body=json.dumps(payload) - model_id="meta.llama2-70b-chat-v1" + body = json.dumps(payload) + model_id = "meta.llama2-70b-chat-v1" elif llm_selected in ['Claude Instant', 'Claude 2', 'Claude 3 - Haiku', 'Claude 3 - Sonnet']: if llm_selected == 'Claude Instant': @@ -384,51 +391,51 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h if llm_selected == "Claude 2": model_id = "anthropic.claude-v2:1" body = json.dumps( - {"prompt": anthropic.HUMAN_PROMPT + prompt_data + anthropic.AI_PROMPT, - "max_tokens_to_sample": 4096, - "temperature":0.0, - "top_p":1, - "top_k":250, - "stop_sequences":[anthropic.HUMAN_PROMPT] - }) + {"prompt": anthropic.HUMAN_PROMPT + prompt_data + anthropic.AI_PROMPT, + "max_tokens_to_sample": 4096, + "temperature": 0.0, + "top_p": 1, + "top_k": 250, + "stop_sequences": [anthropic.HUMAN_PROMPT] + }) elif llm_selected in ["Claude 3 - Haiku", "Claude 3 - Sonnet"]: if llm_selected == "Claude 3 - Haiku": model_id = 'anthropic.claude-3-haiku-20240307-v1:0' else: model_id = 'anthropic.claude-3-sonnet-20240229-v1:0' body = json.dumps({ - "anthropic_version": "bedrock-2023-05-31", - "max_tokens": 4096, - "messages": [ + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 4096, + "messages": [ + { + "role": "user", + "content": [ { - "role": "user", - "content": [ - { - "type": "text", - "text":anthropic.HUMAN_PROMPT + prompt_data + anthropic.AI_PROMPT - } - ] + "type": "text", + "text": anthropic.HUMAN_PROMPT + prompt_data + anthropic.AI_PROMPT } - ], - "temperature": 0.0 + ] } - ) + ], + "temperature": 0.0 + } + ) else: model_id = "anthropic.claude-instant-v1" body = json.dumps( - {"prompt": anthropic.HUMAN_PROMPT + prompt_data + anthropic.AI_PROMPT, - "max_tokens_to_sample": 2048, - "temperature":0.0, - "top_p":1, - "top_k":250, - "stop_sequences":[anthropic.HUMAN_PROMPT] - }) + {"prompt": anthropic.HUMAN_PROMPT + prompt_data + anthropic.AI_PROMPT, + "max_tokens_to_sample": 2048, + "temperature": 0.0, + "top_p": 1, + "top_k": 250, + "stop_sequences": [anthropic.HUMAN_PROMPT] + }) try: response = bedrock_runtime.invoke_model( - body=body, - modelId=model_id, - accept="application/json", + body=body, + modelId=model_id, + accept="application/json", contentType="application/json" ) response_body = json.loads(response.get("body").read()) @@ -449,16 +456,16 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h response_text = response_text.strip() try: - if response_text.split("{",1)[1].strip()[0] == '"': - response_text = "{" + response_text.split("{",1)[1] + if response_text.split("{", 1)[1].strip()[0] == '"': + response_text = "{" + response_text.split("{", 1)[1] else: response_text = "{" + response_text except: response_text = "{" + response_text - if len(response_text.split("}",1)) > 1: - if response_text.rsplit("}",1)[0].strip()[-1] == '"': - response_text = response_text.rsplit("}",1)[0] + "}" + if len(response_text.split("}", 1)) > 1: + if response_text.rsplit("}", 1)[0].strip()[-1] == '"': + response_text = response_text.rsplit("}", 1)[0] + "}" else: response_text = response_text.rstrip(",") response_text = response_text + "}" @@ -469,21 +476,21 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h if mode != 'Single field - Non Empty values': response_dict = {"Test field": "Failed to extract"} else: - response_dict = {field:response_text.strip("{").strip("}")} + response_dict = {field: response_text.strip("{").strip("}")} # if mode == 'Multiple fields': # field_l = list(response_dict.keys()) # answer_l = list(response_dict.values()) # else: - # field_l = [field] - # # answer = response_dict.get(field, " ") - # try: - # if isinstance(response_dict, dict): - # answer_l = list(response_dict.values())[:1] - # else: - # answer_l = list(response_dict)[:1] - # except: - # answer_l = [response_dict] + # field_l = [field] + # # answer = response_dict.get(field, " ") + # try: + # if isinstance(response_dict, dict): + # answer_l = list(response_dict.values())[:1] + # else: + # answer_l = list(response_dict)[:1] + # except: + # answer_l = [response_dict] if mode != 'Single field - Non Empty values': field_l = list(response_dict.keys()) answer_l = list(response_dict.values()) @@ -499,11 +506,11 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h try: page_no_l = [list(response_dict.values())[1]] except: - page_no_l = [''] + page_no_l = [''] field_list.extend(field_l) answer_list.extend(answer_l) - contract_list_f.extend([contract]*len(field_l)) + contract_list_f.extend([contract] * len(field_l)) # location = context.find(answer) if isinstance(answer, str) and answer != "" else -1 # snippet = ' '.join(context[:location].split()[-25:]) + ' ' + ' '.join(context[location:].split()[:30]) if location != -1 else ' ' # page_no = " " if location == -1 else context[:location].rsplit("Start of Page No. = ", 1)[1] if len(context[:location].rsplit( @@ -511,12 +518,14 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h # page_no = re.search(r'\d+', page_no).group() if page_no != " " and re.search(r'\d+', page_no) is not None else "" try: - location_l = [context.find(a, context.find("Start of Page No. = "+str(p))) if isinstance( + location_l = [context.find(a, context.find("Start of Page No. = " + str(p))) if isinstance( a, str) and a != "" else -1 for a, p in zip(answer_l, page_no_l)] except: - location_l = [context.find(answer) if isinstance(answer, str) and answer != "" else -1 for answer in answer_l] + location_l = [context.find(answer) if isinstance(answer, str) and answer != "" else -1 for answer in + answer_l] snippet_l = [' '.join(context[:location].split('.')[-4:]) + ' ' + ' '.join(context[location:].split('. ')[:5] - ) if location != -1 else ' ' for location in location_l] + ) if location != -1 else ' ' for + location in location_l] # page_no_l = [" " if location == -1 else context[:location].rsplit("Start of Page No. = ", 1)[1] if len(context[:location].rsplit( # "Start of Page No. = ", 1)) > 1 else context[:location].rsplit("Start of Page No. = ", 1)[0] for location in location_l] # # st.write(location_list) @@ -537,17 +546,19 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h # date_list.append(extracted_date) # answer_list = date_list try: - answer_list = [str(answer).strip("\n").strip().strip("[").strip("]").strip("{").strip("}").strip('"').rstrip('"').strip( - ' ') if answer is not None else None for answer in answer_list] + answer_list = [ + str(answer).strip("\n").strip().strip("[").strip("]").strip("{").strip("}").strip('"').rstrip('"').strip( + ' ') if answer is not None else None for answer in answer_list] if llm_selected in ['Llama 2 Chat 13B', 'Llama 2 Chat 70B']: answer_list = [answer.rstrip(".") for answer in answer_list] answer_list = [answer if "I don't know" not in str(answer) else " " for answer in answer_list] answer_list = [answer if "N/A" not in str(answer) else " " for answer in answer_list] answer_list = [answer if "does not contain" not in str(answer) else " " for answer in answer_list] answer_list = [answer if "None" not in str(answer) else " " for answer in answer_list] - answer_list = [answer if "Not specified in the contract" not in str(answer) else " " for answer in answer_list] + answer_list = [answer if "Not specified in the contract" not in str(answer) else " " for answer in + answer_list] answer_list = [answer if "Not applicable" not in str(answer) else " " for answer in answer_list] - elif llm_selected in ['Claude 2','Claude 3 - Haiku', 'Claude 3 - Sonnet', 'Claude Instant']: + elif llm_selected in ['Claude 2', 'Claude 3 - Haiku', 'Claude 3 - Sonnet', 'Claude Instant']: answer_list = [answer if "do not have" not in str(answer) else " " for answer in answer_list] answer_list = [answer if "do not see" not in str(answer) else " " for answer in answer_list] answer_list = [answer if "does not specify" not in str(answer) else " " for answer in answer_list] @@ -571,7 +582,7 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h # to be deleted later # contract_list_f = [contract.rsplit('/',1)[1].replace(' MU','').replace('_MU','').replace('.txt','') for contract in contract_list_f] - contract_list_f = [contract.rsplit('/',1)[1] for contract in contract_list_f] + contract_list_f = [contract.rsplit('/', 1)[1] for contract in contract_list_f] df['Contract Name'] = contract_list_f df['New Extracted value'] = answer_list @@ -579,26 +590,27 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h df['Snippet'] = snippet_list df['New Page Number'] = page_no_list df['New Page Number'] = df['New Page Number'].apply(lambda x: re.search(r'\d+', x).group( - ) if isinstance(x, str) and re.search(r'\d+', x) is not None else " ") + ) if isinstance(x, str) and re.search(r'\d+', x) is not None else " ") df['Revised Prompt'] = [prompt] * len(contract_list_f) - df = pd.merge(df, fields[['Field Name', 'SF_DB_COL_NAME']], how ='left', on ='Field Name') + df = pd.merge(df, fields[['Field Name', 'SF_DB_COL_NAME']], how='left', on='Field Name') - field_values_2 = pd.DataFrame(columns=['Contract Name', 'LOB_Product_Network_Metal_Area_Program_Type_Speciality', 'SF_DB_COL_NAME' - , 'Actual Value Stored', 'Original Page Number']) + field_values_2 = pd.DataFrame( + columns=['Contract Name', 'LOB_Product_Network_Metal_Area_Program_Type_Speciality', 'SF_DB_COL_NAME' + , 'Actual Value Stored', 'Original Page Number']) for file_name in contract_list: # document_name = [x for x in list(field_values['Contract Name']) if not pd.isna(x) and file_name.rsplit('/',1)[1].replace(' MU','').replace( # '_MU','').replace('.txt','') in x][0] document_name = [x for x in list(field_values['Contract Name']) if not pd.isna(x) and file_name.rsplit('/',1)[1] in x][0] temp_df = field_values[field_values['Contract Name'] == document_name].fillna('NA') - unique_identifier = str(temp_df.at[temp_df.index[0],'CONTRACT_LOB']) + '__' + str( - temp_df.at[temp_df.index[0],'CONTRACT_PRODUCT']) + '__' + str( - temp_df.at[temp_df.index[0],'CONTRACT_NETWORK']) + '__' + str( - temp_df.at[temp_df.index[0],'CONTRACT_MARKETPLACE_METAL_LEVEL']) + '__' + str( - temp_df.at[temp_df.index[0],'CONTRACT_SERVICE_AREA']) + '__' + str( - temp_df.at[temp_df.index[0],'CONTRACT_PROGRAM']) + '__' + str( - temp_df.at[temp_df.index[0],'PROV_TYPE']) + '__' + str(temp_df.at[temp_df.index[0],'PROV_SPECIALTY']) + unique_identifier = str(temp_df.at[temp_df.index[0], 'CONTRACT_LOB']) + '__' + str( + temp_df.at[temp_df.index[0], 'CONTRACT_PRODUCT']) + '__' + str( + temp_df.at[temp_df.index[0], 'CONTRACT_NETWORK']) + '__' + str( + temp_df.at[temp_df.index[0], 'CONTRACT_MARKETPLACE_METAL_LEVEL']) + '__' + str( + temp_df.at[temp_df.index[0], 'CONTRACT_SERVICE_AREA']) + '__' + str( + temp_df.at[temp_df.index[0], 'CONTRACT_PROGRAM']) + '__' + str( + temp_df.at[temp_df.index[0], 'PROV_TYPE']) + '__' + str(temp_df.at[temp_df.index[0], 'PROV_SPECIALTY']) field_values_1 = field_values[field_values['Contract Name'] == document_name].head(1).transpose().reset_index() field_values_1.columns = ['SF_DB_COL_NAME', 'Actual Value Stored'] field_values_p1 = field_values_1[~field_values_1['SF_DB_COL_NAME'].str.endswith('_PG')] @@ -606,36 +618,37 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h field_values_p2.columns = ['SF_DB_COL_NAME', 'Original Page Number'] field_values_p2["SF_DB_COL_NAME"] = field_values_p2["SF_DB_COL_NAME"].str.replace("_PG", "") - field_values_1 = pd.merge(field_values_p1, field_values_p2, how ='left', on =['SF_DB_COL_NAME']) + field_values_1 = pd.merge(field_values_p1, field_values_p2, how='left', on=['SF_DB_COL_NAME']) field_values_1['Contract Name'] = document_name field_values_1['LOB_Product_Network_Metal_Area_Program_Type_Speciality'] = unique_identifier - field_values_2 = pd.concat([field_values_2, field_values_1], ignore_index = True) + field_values_2 = pd.concat([field_values_2, field_values_1], ignore_index=True) if mode == 'One-to-many fields': for i in range(1, field_values[field_values['Contract Name'] == document_name].shape[0]): - unique_identifier = str(temp_df.at[temp_df.index[i],'CONTRACT_LOB']) + '__' + str( - temp_df.at[temp_df.index[i],'CONTRACT_PRODUCT']) + '__' + str( - temp_df.at[temp_df.index[i],'CONTRACT_NETWORK']) + '__' + str( - temp_df.at[temp_df.index[i],'CONTRACT_MARKETPLACE_METAL_LEVEL']) + '__' + str( - temp_df.at[temp_df.index[i],'CONTRACT_SERVICE_AREA']) + '__' + str( - temp_df.at[temp_df.index[i],'CONTRACT_PROGRAM']) + '__' + str( - temp_df.at[temp_df.index[i],'PROV_TYPE']) + '__' + str(temp_df.at[temp_df.index[i],'PROV_SPECIALTY']) - field_values_1 = field_values[field_values['Contract Name'] == document_name].iloc[[i]].transpose().reset_index() + unique_identifier = str(temp_df.at[temp_df.index[i], 'CONTRACT_LOB']) + '__' + str( + temp_df.at[temp_df.index[i], 'CONTRACT_PRODUCT']) + '__' + str( + temp_df.at[temp_df.index[i], 'CONTRACT_NETWORK']) + '__' + str( + temp_df.at[temp_df.index[i], 'CONTRACT_MARKETPLACE_METAL_LEVEL']) + '__' + str( + temp_df.at[temp_df.index[i], 'CONTRACT_SERVICE_AREA']) + '__' + str( + temp_df.at[temp_df.index[i], 'CONTRACT_PROGRAM']) + '__' + str( + temp_df.at[temp_df.index[i], 'PROV_TYPE']) + '__' + str( + temp_df.at[temp_df.index[i], 'PROV_SPECIALTY']) + field_values_1 = field_values[field_values['Contract Name'] == document_name].iloc[ + [i]].transpose().reset_index() field_values_1.columns = ['SF_DB_COL_NAME', 'Actual Value Stored'] field_values_p1 = field_values_1[~field_values_1['SF_DB_COL_NAME'].str.endswith('_PG')] field_values_p2 = field_values_1[field_values_1['SF_DB_COL_NAME'].str.endswith('_PG')] field_values_p2.columns = ['SF_DB_COL_NAME', 'Original Page Number'] field_values_p2["SF_DB_COL_NAME"] = field_values_p2["SF_DB_COL_NAME"].str.replace("_PG", "") - field_values_1 = pd.merge(field_values_p1, field_values_p2, how ='left', on =['SF_DB_COL_NAME']) + field_values_1 = pd.merge(field_values_p1, field_values_p2, how='left', on=['SF_DB_COL_NAME']) field_values_1['Contract Name'] = document_name field_values_1['LOB_Product_Network_Metal_Area_Program_Type_Speciality'] = unique_identifier - field_values_2 = pd.concat([field_values_2, field_values_1], ignore_index = True) - + field_values_2 = pd.concat([field_values_2, field_values_1], ignore_index=True) # st.write(df) # st.write(field_values_2) - df = pd.merge(df, field_values_2, how ='right', on =['Contract Name', 'SF_DB_COL_NAME']) + df = pd.merge(df, field_values_2, how='right', on=['Contract Name', 'SF_DB_COL_NAME']) if mode != 'Single field - Non Empty values': df = df[df['SF_DB_COL_NAME'].isin(list(field_prompt_mapping.keys()))] else: @@ -643,14 +656,16 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h if mode != 'One-to-many fields': df = df.drop_duplicates(subset=['SF_DB_COL_NAME', 'Contract Name'], keep="first") df['Original Page Number'] = df['Original Page Number'].apply(lambda x: re.search(r'\d+', x).group( - ) if isinstance(x, str) and re.search(r'\d+', x) is not None else " ") + ) if isinstance(x, str) and re.search(r'\d+', x) is not None else " ") df['Raw value 2'] = df['New Extracted value'] df_date = df[df['SF_DB_COL_NAME'].str.contains('_DT', na=False)] df_others = df[~df['SF_DB_COL_NAME'].str.contains('_DT', na=False)] - df_date['Actual Value Stored'] = pd.to_datetime(df_date['Actual Value Stored'],errors='coerce').dt.strftime('%Y-%m-%d').fillna(" ") - df_date['New Extracted value'] = pd.to_datetime(df_date['New Extracted value'],errors='coerce').dt.strftime('%Y-%m-%d').fillna(" ") - df = pd.concat([df_date, df_others], ignore_index = True) + df_date['Actual Value Stored'] = pd.to_datetime(df_date['Actual Value Stored'], errors='coerce').dt.strftime( + '%Y-%m-%d').fillna(" ") + df_date['New Extracted value'] = pd.to_datetime(df_date['New Extracted value'], errors='coerce').dt.strftime( + '%Y-%m-%d').fillna(" ") + df = pd.concat([df_date, df_others], ignore_index=True) df.sort_values(['SF_DB_COL_NAME', 'Contract Name'], inplace=True) df.fillna(" ", inplace=True) @@ -661,7 +676,7 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h actual_value_list = [answer if str(answer) != "12 months" else "1 year" for answer in actual_value_list] actual_value_list = [answer if str(answer) != "Fifth" else "5" for answer in actual_value_list] actual_value_list = [answer if str(answer) != "Seventh" else "7" for answer in actual_value_list] - + answer_list = list(df['New Extracted value']) answer_list = [answer if str(answer) != "one-year" else "1 year" for answer in answer_list] answer_list = [answer if str(answer) != "one year" else "1 year" for answer in answer_list] @@ -672,9 +687,10 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h answer_list = [answer if str(answer) != "Third" else "3" for answer in answer_list] answer_list = [answer if str(answer) != "Six" else "6" for answer in answer_list] - actual_value_list = [s.replace('-', '').replace(' ', '').replace('[', '').replace(']', '').lower() for s in actual_value_list] + actual_value_list = [s.replace('-', '').replace(' ', '').replace('[', '').replace(']', '').lower() for s in + actual_value_list] answer_list = [s.replace('-', '').replace(' ', '').replace('[', '').replace(']', '').lower() for s in answer_list] - result_list = [(i in j) or (j in i) if isinstance(i, str) and isinstance( + result_list = [(i in j) or (j in i) if isinstance(i, str) and isinstance( j, str) and ((i != '') == (j != '')) else False for i, j in zip(actual_value_list, answer_list)] df['Result'] = [str(x) for x in result_list] @@ -682,8 +698,10 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h # df['Contract ID'] = contract_list_f df['Contract ID'] = range(len(actual_value_list)) - df = df[['Contract Name','Contract ID', 'LOB_Product_Network_Metal_Area_Program_Type_Speciality', 'SF_DB_COL_NAME', 'Actual Value Stored', 'Raw value', 'Raw value 2', 'Raw value 3' - , 'New Extracted value','Confidence Level','Snippet','Original Page Number', 'New Page Number', 'Revised Prompt', 'Result']] + df = df[['Contract Name', 'Contract ID', 'LOB_Product_Network_Metal_Area_Program_Type_Speciality', 'SF_DB_COL_NAME', + 'Actual Value Stored', 'Raw value', 'Raw value 2', 'Raw value 3' + , 'New Extracted value', 'Confidence Level', 'Snippet', 'Original Page Number', 'New Page Number', + 'Revised Prompt', 'Result']] try: accuracy = round(sum(bool(x) for x in result_list) * 100 / len(list(df['Result'])), 2) @@ -692,38 +710,45 @@ def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, h if mode != 'Single field - Non Empty values': field = field_group - history.loc[len(history.index)] = [field, str(contract_count), st.session_state.user_info['mail'], datetime.now().strftime("%Y-%m-%d %H:%M:%S"), accuracy, attempt] + history.loc[len(history.index)] = [field, str(contract_count), st.session_state.user_info['mail'], + datetime.now().strftime("%Y-%m-%d %H:%M:%S"), accuracy, attempt] # df.to_csv("RESULTS\\"+field.replace("?","").replace("/","_")+'-'+llm_selected+'.csv', index=False) return df, history, attempt, raw_response_text + raw_response_text = '' -df = pd.DataFrame(columns=['Contract Name','Contract ID', 'Actual Value Stored','New Extracted value','Confidence Level' - ,'Snippet','New Page Number', 'Revised Prompt', 'Result']) +df = pd.DataFrame( + columns=['Contract Name', 'Contract ID', 'Actual Value Stored', 'New Extracted value', 'Confidence Level' + , 'Snippet', 'New Page Number', 'Revised Prompt', 'Result']) if st.button("Test Configuration"): - df, history, attempt, raw_response_text = run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, history) - df_1 = df[['Contract Name','Contract ID', 'Actual Value Stored','New Extracted value','Confidence Level' - ,'Snippet','Original Page Number', 'New Page Number', 'Revised Prompt', 'Result']] + df, history, attempt, raw_response_text = run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, + history) + df_1 = df[['Contract Name', 'Contract ID', 'Actual Value Stored', 'New Extracted value', 'Confidence Level' + , 'Snippet', 'Original Page Number', 'New Page Number', 'Revised Prompt', 'Result']] df.to_csv('results.csv', index=False) history.to_csv('history.csv', index=False) # s3_client.upload_file('results.csv', bucket, key) csv_buf = StringIO() df_1.to_csv(csv_buf, header=True, index=False) csv_buf.seek(0) - s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/results.csv') + s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), + Key='training_interface/results.csv') csv_buf = StringIO() history.tail(1).to_csv(csv_buf, header=True, index=False) csv_buf.seek(0) - s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/history.csv') - df = df[['Contract Name','Contract ID', 'LOB_Product_Network_Metal_Area_Program_Type_Speciality','SF_DB_COL_NAME', 'Actual Value Stored','New Extracted value','Confidence Level' - ,'Snippet','Original Page Number', 'New Page Number', 'Revised Prompt', 'Result']] + s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), + Key='training_interface/history.csv') + df = df[['Contract Name', 'Contract ID', 'LOB_Product_Network_Metal_Area_Program_Type_Speciality', 'SF_DB_COL_NAME', + 'Actual Value Stored', 'New Extracted value', 'Confidence Level' + , 'Snippet', 'Original Page Number', 'New Page Number', 'Revised Prompt', 'Result']] try: df = pd.read_csv('results.csv') df['Result'] = df['Result'].astype('str') except: - df = pd.DataFrame(columns=['Contract Name' ,'New Extracted value','Confidence Level','Snippet','New Page Number' + df = pd.DataFrame(columns=['Contract Name', 'New Extracted value', 'Confidence Level', 'Snippet', 'New Page Number' , 'Revised Prompt', 'Result']) history = pd.read_csv('history.csv') st.dataframe(df) @@ -752,8 +777,6 @@ st.write(len(contract_list)) st.write(raw_response_text) try: - save_to_sf('load_training_results', training_results_file_name = "results.csv", attempt_logs_file_name = "history.csv") + save_to_sf('load_training_results', training_results_file_name="results.csv", attempt_logs_file_name="history.csv") except: st.write("running locally") - - diff --git a/streamlit/util.py b/streamlit/util.py index 78c2782..89e2d64 100644 --- a/streamlit/util.py +++ b/streamlit/util.py @@ -1,17 +1,39 @@ - import streamlit as st import security import os import constants +import logging +from logging.handlers import RotatingFileHandler -def setup_page(REDIRECT_URI): +# Ensure the log directory exists +log_dir = '/home/ubuntu/doczy.ai/streamlit' +if not os.path.exists(log_dir): + os.makedirs(log_dir, exist_ok=True) + +# Configure logging with RotatingFileHandler +log_file = f'{log_dir}/interface.log' +rotating_handler = RotatingFileHandler( + log_file, + maxBytes=10*1024*1024, # 10 MB + backupCount=5 # Keep up to 5 backup files +) +rotating_handler.setLevel(logging.INFO) +formatter = logging.Formatter('%(asctime)s %(levelname)s [%(filename)s] %(message)s') +rotating_handler.setFormatter(formatter) + +logger = logging.getLogger() +logger.setLevel(logging.INFO) +logger.addHandler(rotating_handler) + + +def setup_page(redirect_uril): # st.set_page_config( # page_title=page_title, # page_icon="👋", # ) if st.query_params.get('code'): - security.handle_redirect(REDIRECT_URI) + security.handle_redirect(redirect_uril) access_token = st.session_state.get('access_token') @@ -21,15 +43,23 @@ def setup_page(REDIRECT_URI): return True else: st.write("Please sign-in to use this app.") - auth_url = security.get_auth_url(REDIRECT_URI) + auth_url = security.get_auth_url(redirect_uril) st.markdown(f"Sign In", unsafe_allow_html=True) st.stop() + def load_page_details(interface): env_var = os.environ.get('ENVIRONMENT', 'DEV') + logger.info(f"env_var={env_var}") if env_var == 'UAT': - return (constants.DOCZY_REDIRECT_URL_UAT + str(interface), constants.DOCZY_CREATE_BATCH_URL_UAT, constants.DOCZY_PIPELINE_URL_UAT) + logger.info(constants.DOCZY_REDIRECT_URL_UAT + str(interface)) + return (constants.DOCZY_REDIRECT_URL_UAT + str(interface), constants.DOCZY_CREATE_BATCH_URL_UAT, + constants.DOCZY_PIPELINE_URL_UAT) elif env_var == 'DEV': - return (constants.DOCZY_REDIRECT_URL_DEV + str(interface), constants.DOCZY_CREATE_BATCH_URL_DEV, constants.DOCZY_PIPELINE_URL_DEV) + logger.info(constants.DOCZY_REDIRECT_URL_DEV + str(interface)) + return (constants.DOCZY_REDIRECT_URL_DEV + str(interface), constants.DOCZY_CREATE_BATCH_URL_DEV, + constants.DOCZY_PIPELINE_URL_DEV) else: - return (constants.DOCZY_REDIRECT_URL_DEV + str(interface), constants.DOCZY_CREATE_BATCH_URL_DEV, constants.DOCZY_PIPELINE_URL_DEV) + logger.info(constants.DOCZY_REDIRECT_URL_DEV + str(interface)) + return (constants.DOCZY_REDIRECT_URL_DEV + str(interface), constants.DOCZY_CREATE_BATCH_URL_DEV, + constants.DOCZY_PIPELINE_URL_DEV)