From b7c1c22d3d6e0d0f4e8fa9bbc88ef0162e5d98df Mon Sep 17 00:00:00 2001 From: "AARETE\\agupta" Date: Tue, 4 Jun 2024 10:28:15 -0500 Subject: [PATCH] Merged Aryan's and Pratham's changes in the local/ folder. Modified interface_0.py to include all changes and continue working on the server including all dependencies. --- streamlit/interface_0.py | 89 +++++++---- streamlit/local/local_interface_0.py | 72 +++++---- streamlit/local/p_local_interface_0.py | 197 ------------------------- 3 files changed, 102 insertions(+), 256 deletions(-) delete mode 100644 streamlit/local/p_local_interface_0.py diff --git a/streamlit/interface_0.py b/streamlit/interface_0.py index f638209..f57fb43 100644 --- a/streamlit/interface_0.py +++ b/streamlit/interface_0.py @@ -18,6 +18,22 @@ create_batch_url = 'https://lfksus2t62.execute-api.us-east-2.amazonaws.com/dev/c REDIRECT_URI = 'https://doczydev.aarete.com:8500' user_list = USER_LIST +if 'uploading' not in st.session_state: + st.session_state.uploading = False + + + +def insert_upload_logs(batch_id, client_name, file_name, upload_datetime, upload_user): + """ + Input: batch_id, client_name, file_name, upload_datetime, upload_user + Output: status of the insert query + """ + try: + return 'Log inserted successfully' + except Exception as e: + return e + + st.set_page_config(layout = "wide") # # Sidebar contents # with st.sidebar: @@ -31,6 +47,7 @@ st.set_page_config(layout = "wide") # ) # add_vertical_space(15) # # st.write("Doczy") +# _,c1= st.columns([5,1]) try: @@ -45,7 +62,7 @@ except KeyError as e: st.write("Session Expired.") st.stop() - +print(st.session_state) s3_client = boto3.client('s3', region_name="us-east-2", @@ -65,7 +82,7 @@ client_row = st.columns([0.1, 0.8]) with client_row[0]: st.write("**Client Name**") with client_row[1]: - client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed") + client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344 client_bucket = client_s3_paths.get(client) @@ -76,42 +93,54 @@ file_row = st.columns([0.1, 0.8]) with file_row[0]: st.write("**Upload Files**") with file_row[1]: - file_list = st.file_uploader("Upload", type=None, accept_multiple_files=True, label_visibility = "collapsed") + file_list = st.file_uploader("Upload", type=['csv','pdf'], accept_multiple_files=True, label_visibility = "collapsed", help="Only .csv and .pdf file formats are supported.", disabled=st.session_state.uploading) add_vertical_space(2) df = pd.DataFrame(columns=['Contract Name']) df['Contract Name'] = file_list file_names = [] buttons = st.columns([0.4, 0.4, 0.2]) -with buttons[1]: - if st.button("Create Batch"): - myobj = { "client-bucket-name": client_bucket } - response = requests.post(create_batch_url, json = myobj) - if response.status_code >= 200 and response.status_code < 300: - try: - batch_id = json.loads(json.loads(response.text)['body'])['batch_id'] - landing_zone = json.loads(json.loads(response.text)['body'])['landing_zone'] - except: - st.write(myobj) - st.write(response.text) - batch_id = 'failed_cases' - landing_zone = 'contracts_landing_zone' +def set_uploading_state(): + if not client == None and not len(file_list) == 0: + st.session_state.uploading = True + +with buttons[1]: + if st.button("Create Batch", on_click = set_uploading_state): + if client == None: + st.error("No Client Name Selected.") + elif len(file_list) == 0: + st.error("No Files Selected.") else: - st.write("Failed") - for uploaded_file in file_list: - stringio = BytesIO(uploaded_file.getvalue()) - stringio.seek(0) - s3_client.put_object(Bucket=client_bucket, Body=stringio.getvalue(), Key= - landing_zone+batch_id+'/'+str(uploaded_file.name)) - - # TODO: Test this insert function with snowflake - upload_log = insert_upload_logs(batch_id, client, str(uploaded_file.name), datetime.now().strftime("%Y-%m-%d %H:%M:%S"), user_mail) - st.write(upload_log) - - file_names.append(str(uploaded_file.name)) - st.write(f"{batch_id} created") - st.write(f"Files uploaded to s3://{client_bucket}/{landing_zone}{batch_id}") + myobj = { "client-bucket-name": client_bucket } + response = requests.post(create_batch_url, json = myobj) + if response.status_code >= 200 and response.status_code < 300: + try: + batch_id = json.loads(json.loads(response.text)['body'])['batch_id'] + landing_zone = json.loads(json.loads(response.text)['body'])['landing_zone'] + except: + st.write(myobj) + st.write(response.text) + batch_id = 'failed_cases' + landing_zone = 'contracts_landing_zone' + else: + st.write("Failed") + for uploaded_file in file_list: + stringio = BytesIO(uploaded_file.getvalue()) + stringio.seek(0) + s3_client.put_object(Bucket=client_bucket, Body=stringio.getvalue(), Key= + landing_zone+batch_id+'/'+str(uploaded_file.name)) + + # TODO: Test this insert function with snowflake + upload_log = insert_upload_logs(batch_id, client, str(uploaded_file.name), datetime.now().strftime("%Y-%m-%d %H:%M:%S"), user_mail) + st.write(upload_log) + + file_names.append(str(uploaded_file.name)) + st.session_state.uploading = False + st.write(f"{batch_id} created") + st.write(f"Files uploaded to s3://{client_bucket}/{landing_zone}{batch_id}") + + # @st.cache_data diff --git a/streamlit/local/local_interface_0.py b/streamlit/local/local_interface_0.py index eba8f5d..55802f2 100644 --- a/streamlit/local/local_interface_0.py +++ b/streamlit/local/local_interface_0.py @@ -11,6 +11,7 @@ import boto3 import requests # from sf_conn import get_secret, save_to_sf from io import StringIO, BytesIO +import time # from sf_conn import get_client_names, insert_upload_logs # from constants import USER_LIST @@ -18,6 +19,10 @@ create_batch_url = 'https://lfksus2t62.execute-api.us-east-2.amazonaws.com/dev/c # REDIRECT_URI = 'https://doczydev.aarete.com:8500' # user_list = USER_LIST +if 'uploading' not in st.session_state: + st.session_state.uploading = False + + def insert_upload_logs(batch_id, client_name, file_name, upload_datetime, upload_user): """ @@ -43,6 +48,7 @@ st.set_page_config(layout = "wide") # # ) # # add_vertical_space(15) # # # st.write("Doczy") +# # _,c1= st.columns([5,1]) # try: @@ -58,7 +64,7 @@ except KeyError as e: st.write("Session Expired.") st.stop() - +print(st.session_state) s3_client = boto3.client('s3', region_name="us-east-2", @@ -89,47 +95,55 @@ file_row = st.columns([0.1, 0.8]) with file_row[0]: st.write("**Upload Files**") with file_row[1]: - file_list = st.file_uploader("Upload", type=['csv','pdf'], accept_multiple_files=True, label_visibility = "collapsed", help="Only .csv and .pdf file formats are supported.") + file_list = st.file_uploader("Upload", type=['csv','pdf'], accept_multiple_files=True, label_visibility = "collapsed", help="Only .csv and .pdf file formats are supported.", disabled=st.session_state.uploading) add_vertical_space(2) df = pd.DataFrame(columns=['Contract Name']) df['Contract Name'] = file_list file_names = [] buttons = st.columns([0.4, 0.4, 0.2]) + +def set_uploading_state(): + if not client == None and not len(file_list) == 0: + st.session_state.uploading = True + with buttons[1]: - if st.button("Create Batch"): + if st.button("Create Batch", on_click = set_uploading_state): if client == None: st.error("No Client Name Selected.") elif len(file_list) == 0: st.error("No Files Selected.") else: + time.sleep(5) + # myobj = { "client-bucket-name": client_bucket } + # response = requests.post(create_batch_url, json = myobj) + # if response.status_code >= 200 and response.status_code < 300: + # try: + # batch_id = json.loads(json.loads(response.text)['body'])['batch_id'] + # landing_zone = json.loads(json.loads(response.text)['body'])['landing_zone'] + # except: + # st.write(myobj) + # st.write(response.text) + # batch_id = 'failed_cases' + # landing_zone = 'contracts_landing_zone' + # else: + # st.write("Failed") + # for uploaded_file in file_list: + # stringio = BytesIO(uploaded_file.getvalue()) + # stringio.seek(0) + # s3_client.put_object(Bucket=client_bucket, Body=stringio.getvalue(), Key= + # landing_zone+batch_id+'/'+str(uploaded_file.name)) + + # # TODO: Test this insert function with snowflake + # upload_log = insert_upload_logs(batch_id, client, str(uploaded_file.name), datetime.now().strftime("%Y-%m-%d %H:%M:%S"), user_mail) + # st.write(upload_log) + + # file_names.append(str(uploaded_file.name)) + # st.write(f"{batch_id} created") + st.session_state.uploading = False + # st.write(f"Files uploaded to s3://{client_bucket}/{landing_zone}{batch_id}") + - myobj = { "client-bucket-name": client_bucket } - response = requests.post(create_batch_url, json = myobj) - if response.status_code >= 200 and response.status_code < 300: - try: - batch_id = json.loads(json.loads(response.text)['body'])['batch_id'] - landing_zone = json.loads(json.loads(response.text)['body'])['landing_zone'] - except: - st.write(myobj) - st.write(response.text) - batch_id = 'failed_cases' - landing_zone = 'contracts_landing_zone' - else: - st.write("Failed") - for uploaded_file in file_list: - stringio = BytesIO(uploaded_file.getvalue()) - stringio.seek(0) - s3_client.put_object(Bucket=client_bucket, Body=stringio.getvalue(), Key= - landing_zone+batch_id+'/'+str(uploaded_file.name)) - - # TODO: Test this insert function with snowflake - upload_log = insert_upload_logs(batch_id, client, str(uploaded_file.name), datetime.now().strftime("%Y-%m-%d %H:%M:%S"), user_mail) - st.write(upload_log) - - file_names.append(str(uploaded_file.name)) - st.write(f"{batch_id} created") - st.write(f"Files uploaded to s3://{client_bucket}/{landing_zone}{batch_id}") # @st.cache_data diff --git a/streamlit/local/p_local_interface_0.py b/streamlit/local/p_local_interface_0.py deleted file mode 100644 index 8024684..0000000 --- a/streamlit/local/p_local_interface_0.py +++ /dev/null @@ -1,197 +0,0 @@ -import json -import streamlit as st -from streamlit_extras.add_vertical_space import add_vertical_space -import os -import streamlit as st -import pandas as pd -from io import StringIO -from datetime import datetime -import boto3 -# import util -import requests -# from sf_conn import get_secret, save_to_sf -from io import StringIO, BytesIO -import time -# from sf_conn import get_client_names, insert_upload_logs -# from constants import USER_LIST - -create_batch_url = 'https://lfksus2t62.execute-api.us-east-2.amazonaws.com/dev/create-batch' - -# REDIRECT_URI = 'https://doczydev.aarete.com:8500' -# user_list = USER_LIST -if 'uploading' not in st.session_state: - st.session_state.uploading = False - - - -def insert_upload_logs(batch_id, client_name, file_name, upload_datetime, upload_user): - """ - Input: batch_id, client_name, file_name, upload_datetime, upload_user - Output: status of the insert query - """ - try: - return 'Log inserted successfully' - except Exception as e: - return e - - -st.set_page_config(layout = "wide") -# # # Sidebar contents -# # with st.sidebar: -# # st.title("Doczy.AI ™") -# # st.markdown( -# # """ -# # ## About -# # This app extracts data from contracts - -# # """ -# # ) -# # add_vertical_space(15) -# # # st.write("Doczy") -# # - -_,c1= st.columns([5,1]) -# try: -# util.setup_page(REDIRECT_URI) -# except Exception as e: -# st.write(f"SSO Failed = {e}") -# st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} -st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} -try: - c1.write(f"User: **{st.session_state.user_info['displayName']}**") - user_mail = st.session_state.user_info['mail'] -except KeyError as e: - st.write("Session Expired.") - st.stop() - - - -s3_client = boto3.client('s3', - region_name="us-east-2", -) - -client_list = ['doczy-ai-client-1', 'Delaware First Health, Inc.', 'Community Health Choice, Inc','CareSource Network Partners LLC', - 'HealthNet of Cali', 'Oklahoma Complete Health, Inc', 'HealthFirst', 'Molina Healthcare of TX', 'AvMed', 'Arizona Care1st', - 'WellCare New Jersey'] - -# This is the list of client fetched from Snowflake -# TODO: Need to update the streamlit code to use the client names from this list -# And use the s3 paths to save the objects for the respective client -# client_list, s3_paths = get_client_names() -# client_s3_paths = dict(zip(client_list, s3_paths)) - -client_row = st.columns([0.1, 0.8]) -with client_row[0]: - st.write("**Client Name**") -with client_row[1]: - client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344 - -# client_bucket = client_s3_paths.get(client) - -# to be deleted when client buckets are created -client_bucket = 'doczy-ai-client-1' - -file_row = st.columns([0.1, 0.8]) -with file_row[0]: - st.write("**Upload Files**") -with file_row[1]: - file_list = st.file_uploader("Upload", type=['csv','pdf'], accept_multiple_files=True, label_visibility = "collapsed", help="Only .csv and .pdf file formats are supported.", disabled=st.session_state.uploading) - -add_vertical_space(2) -df = pd.DataFrame(columns=['Contract Name']) -df['Contract Name'] = file_list -file_names = [] -buttons = st.columns([0.4, 0.4, 0.2]) -with buttons[1]: - if st.button("Create Batch"): - st.session_state.uploading = True - if client == None: - st.session_state.uploading = False - st.error("No Client Name Selected.") - elif len(file_list) == 0: - st.session_state.uploading = False - st.error("No Files Selected.") - else: - myobj = { "client-bucket-name": client_bucket } - response = requests.post(create_batch_url, json = myobj) - if response.status_code >= 200 and response.status_code < 300: - try: - batch_id = json.loads(json.loads(response.text)['body'])['batch_id'] - landing_zone = json.loads(json.loads(response.text)['body'])['landing_zone'] - except: - st.write(myobj) - st.write(response.text) - batch_id = 'failed_cases' - landing_zone = 'contracts_landing_zone' - else: - st.write("Failed") - for uploaded_file in file_list: - stringio = BytesIO(uploaded_file.getvalue()) - stringio.seek(0) - s3_client.put_object(Bucket=client_bucket, Body=stringio.getvalue(), Key= - landing_zone+batch_id+'/'+str(uploaded_file.name)) - - # TODO: Test this insert function with snowflake - upload_log = insert_upload_logs(batch_id, client, str(uploaded_file.name), datetime.now().strftime("%Y-%m-%d %H:%M:%S"), user_mail) - st.write(upload_log) - - file_names.append(str(uploaded_file.name)) - st.write(f"{batch_id} created") - st.session_state.uploading = False - st.write(f"Files uploaded to s3://{client_bucket}/{landing_zone}{batch_id}") - - - - - # @st.cache_data - # def convert_df(df): - # return df.to_csv(index=False).encode('utf-8') - - # csv = convert_df(df) - - # df = df.reset_index() # make sure indexes pair with number of rows - - # contract_list = [] - # for index, row in df.iterrows(): - # group_list = [] - # if row['Unique Key']: - # group_list.append('Unique Key') - # if row['Pricing Before Carveouts']: - # group_list.append('Pricing Before Carveouts') - # if row['Contract Related']: - # group_list.append('Contract Related') - # if row['Provider']: - # group_list.append('Provider') - # if row['Timeline']: - # group_list.append('Timeline') - # if row['Carveout Indicator']: - # group_list.append('Carveout Indicator') - # if row['Carveout Methodology']: - # group_list.append('Carveout Methodology') - # entry_dict = { - # "contract_name": row['Contract Name'], - # "groups": group_list, - # "contract_source_path": "batches/batch_1/"+client+"/"+row['Contract Name'] - # } - # contract_list.append(entry_dict) - # contract_list = list(df['Contract Name']) - - # myobj = { - # "s3_bucket": 'doczy-dev-infra-textract', - # "batch_id": "1", - # "client_name": client, - # "username": user_mail, - # "contract_list": contract_list - # } - - # buttons = st.columns([0.8, 0.2]) - # with buttons[0]: - # st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv') - # with buttons[1]: - # if st.button("Upload to DB"): - # response = requests.post(doczy_pipeline, json = myobj) - # if response.status_code >= 200 and response.status_code < 300: - # st.write("Success") - # else: - # st.write("Failed") - # # st.write(response.text)