jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

connected UI3 to S3

Browse Source
This commit is contained in:
Mayank Aamseek
2024-03-18 17:27:08 +05:30
parent d60e8e0684
commit 648fe4c54b
3 changed files with 39 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
streamlit/interface_2.py
+1 -1
View File
@@ -20,7 +20,7 @@ from typing import List
import re
REDIRECT_URI = 'https://doczy.aarete.com:8502'
user_list = ['maamseek@aarete.com', 'smahdavian@aarete.com', 'ahinge@aarete.com', 'akadam@aarete.com'
user_list = ['maamseek@aarete.com', 'smahdavian@aarete.com', 'ahinge@aarete.com', 'akadam@aarete.com', 'pkatariya@aarete.com'
, 'piragavarapu@aarete.com', 'umistry@aarete.com', 'ahutchison@aarete.com', 'bgrunst@aarete.com', 'ddimeglio@aarete.com'
, 'vnair@aarete.com', 'kminhas@aarete.com','dculotta@aarete.com','cbull@aarete.com','sclark@aarete.com']
streamlit/interface_3.py
+36 -17
View File
@@ -24,7 +24,7 @@ from sf_conn import get_secret
REDIRECT_URI = 'https://doczy.aarete.com:8503'
user_list = ['maamseek@aarete.com', 'smahdavian@aarete.com', 'ahinge@aarete.com', 'akadam@aarete.com'
user_list = ['maamseek@aarete.com', 'smahdavian@aarete.com', 'ahinge@aarete.com', 'akadam@aarete.com', 'pkatariya@aarete.com'
, 'piragavarapu@aarete.com', 'umistry@aarete.com', 'ahutchison@aarete.com', 'bgrunst@aarete.com', 'ddimeglio@aarete.com'
, 'vnair@aarete.com']
@@ -66,13 +66,13 @@ try:
cur.execute(query)
field_values = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description])
# st.write(field_values)
field_values['Document_Name'] = field_values['CONTRACT_TITLE']
field_values['Contract ID'] = field_values['CONTRACT_TITLE']
error('table values are incorrect')
field_values['Document_Name'] = field_values['DOCUMENT_NAME']
# field_values['Contract ID'] = field_values['CONTRACT_TITLE']
# error('table values are incorrect')
except:
field_values = pd.read_csv('contract_field_values.csv', encoding='unicode_escape', skipinitialspace=True)
field_values.rename(columns={'(internal) Document Name': 'Document_Name'}, inplace = True)
field_values.rename(columns={'(Internal) Carveout ID': 'Contract ID'}, inplace = True)
# field_values.rename(columns={'(Internal) Carveout ID': 'Contract ID'}, inplace = True)
# st.write("conn failed")
try:
@@ -154,26 +154,40 @@ if user_mail in user_list:
with contract_count_row[0]:
st.write("**# of Contracts**")
with contract_count_row[1]:
contract_count = st.selectbox('Contract count',('1', '10', '20', '30', '50', 'All'), index=1, label_visibility = "collapsed")
contract_count = st.selectbox('Contract count',('1', '10', '20', '30', '50', '100', '200', 'All'), index=1, label_visibility = "collapsed")
seed_row = st.columns([0.15, 0.45, 0.4])
s3_client = boto3.client('s3',
region_name="us-east-1"
)
bucket = 'doczy-dev-infra-textract'
objects = s3_client.list_objects_v2(Bucket=bucket, Prefix="batches/batch_1/contract-text-file/")
file_list = []
for obj in objects['Contents']:
if not obj['Key'].endswith('/'):
file_list.append(obj['Key'])
# print(os.path.join('RAW_DOCUMENTS', obj['Key'].rsplit('/',1)[1]))
# s3_client.download_file('doczy-dev-infra-textract', obj['Key'], os.path.join('RAW_DOCUMENTS', obj['Key'].rsplit('/',1)[1]))
contract_list = sorted(os.listdir(SOURCE_DIRECTORY))
contract_list = sorted(file_list)
# contract_list = sorted(os.listdir(SOURCE_DIRECTORY))
# to be deleted later
contract_list = [contract for contract in contract_list if contract.replace(' MU','').replace('_MU','').replace('.txt','') in list(field_values['Document_Name'])]
contract_list = [contract for contract in contract_list if contract.rsplit('/',1)[1].replace(' MU','').replace('_MU','').replace('.txt','') in list(field_values['Document_Name'])]
if contract_count == 'All':
contract_count = len(contract_list)
with seed_row[0]:
if contract_count in ['10', '20', '30', '50']:
if contract_count in ['10', '20', '30', '50', '100', '200']:
st.write("**Seed Value**")
elif contract_count == '1':
st.write("**Contract Name**")
with seed_row[1]:
if contract_count in ['10', '20', '30', '50']:
if contract_count in ['10', '20', '30', '50', '100', '200']:
seed_value = st.text_input("**Seed Value**", value = 20, label_visibility = "collapsed")
random.seed(seed_value)
contract_list = sorted(random.choices(os.listdir(SOURCE_DIRECTORY), k=int(contract_count)))
# contract_list = sorted(random.choices(os.listdir(SOURCE_DIRECTORY), k=int(contract_count)))
contract_list = sorted(random.choices(file_list, k=int(contract_count)))
elif contract_count == '1':
contract_name = st.selectbox('Contract Name', (contract_list), label_visibility = "collapsed")
contract_list = [contract_name]
@@ -200,7 +214,8 @@ if user_mail in user_list:
column_name = fields.loc[fields['Field Name'] == field, 'SF_DB_COL_NAME'].iloc[0]
column_list = ['Document_Name', 'Contract ID', column_name]
column_list = ['Document_Name', column_name]
# column_list = ['Document_Name', 'Contract ID', column_name]
if column_name+'_PG' in list(field_values.columns):
column_list.append(column_name+'_PG')
field_values = field_values[column_list]
@@ -236,8 +251,12 @@ if user_mail in user_list:
attempt = attempt + 1
for contract in contract_list:
with open(os.path.join(SOURCE_DIRECTORY, contract[:-4]+'.txt'), 'r') as infile:
context = infile.read()
# with open(os.path.join(SOURCE_DIRECTORY, contract[:-4]+'.txt'), 'r') as infile:
# context = infile.read()
data = s3_client.get_object(Bucket=bucket, Key=contract)
contents = data['Body'].read()
context = contents.decode("utf-8")
print(context)
# Add "You must answer in correct JSON format."
# Add Answer in JSON format: {{
@@ -364,11 +383,11 @@ if user_mail in user_list:
# answer_list = [str(x).rsplit(':',1)[0] if len(str(x).rsplit(':',1)) < 2 else str(x).rsplit(':',1)[1] for x in answer_list]
df['Contract ID'] = contract_list
# to be deleted later
contract_list = [contract.replace(' MU','').replace('_MU','').replace('.txt','') for contract in contract_list]
contract_list = [contract.rsplit('/',1)[1].replace(' MU','').replace('_MU','').replace('.txt','') for contract in contract_list]
df['Contract Name'] = contract_list
df['New Extracted value'] = answer_list
df['Confidence Level'] = ' '
df['Snippet'] = snippet_list
streamlit/sf_conn.py
+2 -2
View File
@@ -33,7 +33,7 @@ def get_secret():
secret = get_secret_value_response['SecretString']
return secret
get_secret()
# get_secret()
def save_to_sf(training_results_file, attempt_logs_file):
@@ -78,5 +78,5 @@ def save_to_sf(training_results_file, attempt_logs_file):
return base64.b64decode(mydata['stdout'])
save_to_sf("2024-03-13T17-36_prompt_results.csv", "2024-03-14T23-19_history.csv")
# save_to_sf("2024-03-13T17-36_prompt_results.csv", "2024-03-14T23-19_history.csv")