From 648fe4c54b0c4b1ee1f1dd58196f196b052cdd8a Mon Sep 17 00:00:00 2001 From: Mayank Aamseek Date: Mon, 18 Mar 2024 17:27:08 +0530 Subject: [PATCH] connected UI3 to S3 --- streamlit/interface_2.py | 2 +- streamlit/interface_3.py | 53 +++++++++++++++++++++++++++------------- streamlit/sf_conn.py | 4 +-- 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/streamlit/interface_2.py b/streamlit/interface_2.py index aa9366e..b4464f3 100644 --- a/streamlit/interface_2.py +++ b/streamlit/interface_2.py @@ -20,7 +20,7 @@ from typing import List import re REDIRECT_URI = 'https://doczy.aarete.com:8502' -user_list = ['maamseek@aarete.com', 'smahdavian@aarete.com', 'ahinge@aarete.com', 'akadam@aarete.com' +user_list = ['maamseek@aarete.com', 'smahdavian@aarete.com', 'ahinge@aarete.com', 'akadam@aarete.com', 'pkatariya@aarete.com' , 'piragavarapu@aarete.com', 'umistry@aarete.com', 'ahutchison@aarete.com', 'bgrunst@aarete.com', 'ddimeglio@aarete.com' , 'vnair@aarete.com', 'kminhas@aarete.com','dculotta@aarete.com','cbull@aarete.com','sclark@aarete.com'] diff --git a/streamlit/interface_3.py b/streamlit/interface_3.py index 842caab..a9a6def 100644 --- a/streamlit/interface_3.py +++ b/streamlit/interface_3.py @@ -24,7 +24,7 @@ from sf_conn import get_secret REDIRECT_URI = 'https://doczy.aarete.com:8503' -user_list = ['maamseek@aarete.com', 'smahdavian@aarete.com', 'ahinge@aarete.com', 'akadam@aarete.com' +user_list = ['maamseek@aarete.com', 'smahdavian@aarete.com', 'ahinge@aarete.com', 'akadam@aarete.com', 'pkatariya@aarete.com' , 'piragavarapu@aarete.com', 'umistry@aarete.com', 'ahutchison@aarete.com', 'bgrunst@aarete.com', 'ddimeglio@aarete.com' , 'vnair@aarete.com'] @@ -66,13 +66,13 @@ try: cur.execute(query) field_values = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description]) # st.write(field_values) - field_values['Document_Name'] = field_values['CONTRACT_TITLE'] - field_values['Contract ID'] = field_values['CONTRACT_TITLE'] - error('table values are incorrect') + field_values['Document_Name'] = field_values['DOCUMENT_NAME'] + # field_values['Contract ID'] = field_values['CONTRACT_TITLE'] + # error('table values are incorrect') except: field_values = pd.read_csv('contract_field_values.csv', encoding='unicode_escape', skipinitialspace=True) field_values.rename(columns={'(internal) Document Name': 'Document_Name'}, inplace = True) - field_values.rename(columns={'(Internal) Carveout ID': 'Contract ID'}, inplace = True) + # field_values.rename(columns={'(Internal) Carveout ID': 'Contract ID'}, inplace = True) # st.write("conn failed") try: @@ -154,26 +154,40 @@ if user_mail in user_list: with contract_count_row[0]: st.write("**# of Contracts**") with contract_count_row[1]: - contract_count = st.selectbox('Contract count',('1', '10', '20', '30', '50', 'All'), index=1, label_visibility = "collapsed") + contract_count = st.selectbox('Contract count',('1', '10', '20', '30', '50', '100', '200', 'All'), index=1, label_visibility = "collapsed") seed_row = st.columns([0.15, 0.45, 0.4]) + s3_client = boto3.client('s3', + region_name="us-east-1" + ) + bucket = 'doczy-dev-infra-textract' + objects = s3_client.list_objects_v2(Bucket=bucket, Prefix="batches/batch_1/contract-text-file/") + file_list = [] + for obj in objects['Contents']: + if not obj['Key'].endswith('/'): + file_list.append(obj['Key']) + # print(os.path.join('RAW_DOCUMENTS', obj['Key'].rsplit('/',1)[1])) + # s3_client.download_file('doczy-dev-infra-textract', obj['Key'], os.path.join('RAW_DOCUMENTS', obj['Key'].rsplit('/',1)[1])) - contract_list = sorted(os.listdir(SOURCE_DIRECTORY)) + contract_list = sorted(file_list) + # contract_list = sorted(os.listdir(SOURCE_DIRECTORY)) # to be deleted later - contract_list = [contract for contract in contract_list if contract.replace(' MU','').replace('_MU','').replace('.txt','') in list(field_values['Document_Name'])] - + contract_list = [contract for contract in contract_list if contract.rsplit('/',1)[1].replace(' MU','').replace('_MU','').replace('.txt','') in list(field_values['Document_Name'])] + if contract_count == 'All': + contract_count = len(contract_list) with seed_row[0]: - if contract_count in ['10', '20', '30', '50']: + if contract_count in ['10', '20', '30', '50', '100', '200']: st.write("**Seed Value**") elif contract_count == '1': st.write("**Contract Name**") with seed_row[1]: - if contract_count in ['10', '20', '30', '50']: + if contract_count in ['10', '20', '30', '50', '100', '200']: seed_value = st.text_input("**Seed Value**", value = 20, label_visibility = "collapsed") random.seed(seed_value) - contract_list = sorted(random.choices(os.listdir(SOURCE_DIRECTORY), k=int(contract_count))) + # contract_list = sorted(random.choices(os.listdir(SOURCE_DIRECTORY), k=int(contract_count))) + contract_list = sorted(random.choices(file_list, k=int(contract_count))) elif contract_count == '1': contract_name = st.selectbox('Contract Name', (contract_list), label_visibility = "collapsed") contract_list = [contract_name] @@ -200,7 +214,8 @@ if user_mail in user_list: column_name = fields.loc[fields['Field Name'] == field, 'SF_DB_COL_NAME'].iloc[0] - column_list = ['Document_Name', 'Contract ID', column_name] + column_list = ['Document_Name', column_name] + # column_list = ['Document_Name', 'Contract ID', column_name] if column_name+'_PG' in list(field_values.columns): column_list.append(column_name+'_PG') field_values = field_values[column_list] @@ -236,8 +251,12 @@ if user_mail in user_list: attempt = attempt + 1 for contract in contract_list: - with open(os.path.join(SOURCE_DIRECTORY, contract[:-4]+'.txt'), 'r') as infile: - context = infile.read() + # with open(os.path.join(SOURCE_DIRECTORY, contract[:-4]+'.txt'), 'r') as infile: + # context = infile.read() + data = s3_client.get_object(Bucket=bucket, Key=contract) + contents = data['Body'].read() + context = contents.decode("utf-8") + print(context) # Add "You must answer in correct JSON format." # Add Answer in JSON format: {{ @@ -364,11 +383,11 @@ if user_mail in user_list: # answer_list = [str(x).rsplit(':',1)[0] if len(str(x).rsplit(':',1)) < 2 else str(x).rsplit(':',1)[1] for x in answer_list] + df['Contract ID'] = contract_list # to be deleted later - contract_list = [contract.replace(' MU','').replace('_MU','').replace('.txt','') for contract in contract_list] + contract_list = [contract.rsplit('/',1)[1].replace(' MU','').replace('_MU','').replace('.txt','') for contract in contract_list] df['Contract Name'] = contract_list - df['New Extracted value'] = answer_list df['Confidence Level'] = ' ' df['Snippet'] = snippet_list diff --git a/streamlit/sf_conn.py b/streamlit/sf_conn.py index 3f20fbb..61ed01d 100644 --- a/streamlit/sf_conn.py +++ b/streamlit/sf_conn.py @@ -33,7 +33,7 @@ def get_secret(): secret = get_secret_value_response['SecretString'] return secret -get_secret() +# get_secret() def save_to_sf(training_results_file, attempt_logs_file): @@ -78,5 +78,5 @@ def save_to_sf(training_results_file, attempt_logs_file): return base64.b64decode(mydata['stdout']) -save_to_sf("2024-03-13T17-36_prompt_results.csv", "2024-03-14T23-19_history.csv") +# save_to_sf("2024-03-13T17-36_prompt_results.csv", "2024-03-14T23-19_history.csv")