jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in DEV-DS (pull request #62)

Browse Source 
DEV DS
This commit is contained in:
Umang Mistry
2024-04-22 16:22:07 +00:00
parent 53ea3392c1 3d5cb15d5a
commit 3ca10b04d6
9 changed files with 115 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+2 -2
View File
@@ -55,11 +55,11 @@ streamlit/DB/
streamlit/RAW_DOCUMENTS/
streamlit/SOURCE_DOCUMENTS/
streamlit/contract_field_values.csv
# streamlit/contract_fields.csv
streamlit/contract_fields.csv
streamlit/sample.csv
streamlit/temp1.csv
streamlit/temp2.csv
# streamlit/results.csv
streamlit/results.csv
# env
streamlit/venv
streamlit/interface_0.py
+8 -11
View File
@@ -18,7 +18,7 @@ create_batch_url = 'https://lfksus2t62.execute-api.us-east-2.amazonaws.com/dev/c
REDIRECT_URI = 'https://doczy.aarete.com:8500'
user_list = ['maamseek@aarete.com', 'smahdavian@aarete.com', 'ahinge@aarete.com', 'akadam@aarete.com', 'pkatariya@aarete.com'
, 'piragavarapu@aarete.com', 'umistry@aarete.com', 'ahutchison@aarete.com', 'bgrunst@aarete.com', 'ddimeglio@aarete.com'
, 'vnair@aarete.com', 'kminhas@aarete.com', 'fmohiuddin@aarete.com', 'slitewka@aarete.com', 'qdoest@aarete.com', 'bkoryga@aarete.com', 'bcielecki@aarete.com', 'mszymanski@aarete.com']
, 'vnair@aarete.com', 'kminhas@aarete.com', 'fmohiuddin@aarete.com', 'slitewka@aarete.com', 'qdoest@aarete.com', 'bkoryga@aarete.com', 'bcielecki@aarete.com', 'mszymanski@aarete.com','hupreti@aarete.com']
st.set_page_config(layout = "wide")
# # Sidebar contents
# with st.sidebar:
@@ -36,8 +36,8 @@ st.set_page_config(layout = "wide")
_,c1= st.columns([5,1])
try:
util.setup_page(REDIRECT_URI)
except:
st.write("SSO Failed")
except Exception as e:
st.write(f"SSO Failed = {e}")
st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'}
try:
c1.write(f"User: **{st.session_state.user_info['displayName']}**")
@@ -68,9 +68,8 @@ if user_mail in user_list:
with client_row[1]:
client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed")
client = client_s3_paths.get(client)
# # to be deleted when buckets for different clients are ready; below line is added only for testing the corresponding DAG
client = 'doczy-ai-client-1'
client_bucket = client_s3_paths.get(client)
# client = 'doczy-ai-client-1'
file_row = st.columns([0.1, 0.8])
with file_row[0]:
@@ -86,7 +85,7 @@ if user_mail in user_list:
with buttons[1]:
if st.button("Create Batch"):
myobj = { "client-bucket-name": client }
myobj = { "client-bucket-name": client_bucket }
response = requests.post(create_batch_url, json = myobj)
if response.status_code >= 200 and response.status_code < 300:
try:
@@ -102,9 +101,7 @@ if user_mail in user_list:
for uploaded_file in file_list:
stringio = BytesIO(uploaded_file.getvalue())
stringio.seek(0)
# s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=stringio.getvalue(), Key=
# 'config_interface/'+str(uploaded_file.name))
s3_client.put_object(Bucket=client, Body=stringio.getvalue(), Key=
s3_client.put_object(Bucket=client_bucket, Body=stringio.getvalue(), Key=
batch_id+'/'+landing_zone+'/'+str(uploaded_file.name))
# TODO: Test this insert function with snowflake
@@ -113,7 +110,7 @@ if user_mail in user_list:
file_names.append(str(uploaded_file.name))
st.write(f"{batch_id} created")
st.write(f"Files uploaded to s3://{client}/{batch_id}/{landing_zone}")
st.write(f"Files uploaded to s3://{client_bucket}/{batch_id}/{landing_zone}")
# @st.cache_data
streamlit/interface_1.py
+1 -1
View File
@@ -15,7 +15,7 @@ doczy_pipeline = 'https://8ir4vi1ri4.execute-api.us-east-2.amazonaws.com/dev/'
REDIRECT_URI = 'https://doczy.aarete.com:8501'
user_list = ['maamseek@aarete.com', 'smahdavian@aarete.com', 'ahinge@aarete.com', 'akadam@aarete.com', 'pkatariya@aarete.com'
, 'piragavarapu@aarete.com', 'umistry@aarete.com', 'ahutchison@aarete.com', 'bgrunst@aarete.com', 'ddimeglio@aarete.com'
, 'vnair@aarete.com', 'kminhas@aarete.com', 'fmohiuddin@aarete.com', 'slitewka@aarete.com', 'qdoest@aarete.com', 'bkoryga@aarete.com', 'bcielecki@aarete.com', 'mszymanski@aarete.com']
, 'vnair@aarete.com', 'kminhas@aarete.com', 'fmohiuddin@aarete.com', 'slitewka@aarete.com', 'qdoest@aarete.com', 'bkoryga@aarete.com', 'bcielecki@aarete.com', 'mszymanski@aarete.com','hupreti@aarete.com']
st.set_page_config(layout = "wide")
# # Sidebar contents
# with st.sidebar:
streamlit/interface_2.py
+2 -2
View File
@@ -22,7 +22,7 @@ import re
REDIRECT_URI = 'https://doczy.aarete.com:8502'
user_list = ['maamseek@aarete.com', 'smahdavian@aarete.com', 'ahinge@aarete.com', 'akadam@aarete.com', 'pkatariya@aarete.com'
, 'piragavarapu@aarete.com', 'umistry@aarete.com', 'ahutchison@aarete.com', 'bgrunst@aarete.com', 'ddimeglio@aarete.com'
, 'vnair@aarete.com', 'kminhas@aarete.com','dculotta@aarete.com','cbull@aarete.com','sclark@aarete.com', 'fmohiuddin@aarete.com']
, 'vnair@aarete.com', 'kminhas@aarete.com','dculotta@aarete.com','cbull@aarete.com','sclark@aarete.com', 'fmohiuddin@aarete.com', 'hupreti@aarete.com']
st.set_page_config(layout = "wide")
# Sidebar contents
@@ -95,7 +95,7 @@ s3_client = boto3.client('s3',
region_name="us-east-2"
)
bucket = 'doczy-dev-infra-textract'
objects = s3_client.list_objects_v2(Bucket=bucket, Prefix="training-data/contract-text-file/")
objects = s3_client.list_objects_v2(Bucket=bucket, Prefix="training-data/")
file_list = []
for obj in objects['Contents']:
if not obj['Key'].endswith('/'):
streamlit/interface_2_rag.py
+1 -1
View File
@@ -17,7 +17,7 @@ import util
REDIRECT_URI = 'https://doczy.aarete.com:8502'
user_list = ['maamseek@aarete.com', 'smahdavian@aarete.com', 'ahinge@aarete.com', 'akadam@aarete.com', 'pkatariya@aarete.com'
, 'piragavarapu@aarete.com', 'umistry@aarete.com', 'ahutchison@aarete.com', 'bgrunst@aarete.com', 'ddimeglio@aarete.com'
, 'vnair@aarete.com', 'kminhas@aarete.com', 'fmohiuddin@aarete.com', 'slitewka@aarete.com', 'qdoest@aarete.com', 'bkoryga@aarete.com', 'bcielecki@aarete.com', 'mszymanski@aarete.com']
, 'vnair@aarete.com', 'kminhas@aarete.com', 'fmohiuddin@aarete.com', 'slitewka@aarete.com', 'qdoest@aarete.com', 'bkoryga@aarete.com', 'bcielecki@aarete.com', 'mszymanski@aarete.com','hupreti@aarete.com']
st.set_page_config(layout = "wide")
# Sidebar contents
streamlit/interface_3.py
+89 -37
View File
@@ -27,7 +27,7 @@ from io import StringIO
REDIRECT_URI = 'https://doczy.aarete.com:8503'
user_list = ['maamseek@aarete.com', 'smahdavian@aarete.com', 'ahinge@aarete.com', 'akadam@aarete.com', 'pkatariya@aarete.com'
, 'piragavarapu@aarete.com', 'umistry@aarete.com', 'ahutchison@aarete.com', 'bgrunst@aarete.com', 'ddimeglio@aarete.com'
, 'vnair@aarete.com', 'kminhas@aarete.com', 'fmohiuddin@aarete.com', 'slitewka@aarete.com', 'qdoest@aarete.com', 'bkoryga@aarete.com', 'bcielecki@aarete.com', 'mszymanski@aarete.com']
, 'vnair@aarete.com', 'kminhas@aarete.com', 'fmohiuddin@aarete.com', 'slitewka@aarete.com', 'qdoest@aarete.com', 'bkoryga@aarete.com', 'bcielecki@aarete.com', 'mszymanski@aarete.com','hupreti@aarete.com']
st.set_page_config(layout = "wide")
# Sidebar contents
@@ -176,13 +176,14 @@ if st.session_state.user_info['mail'] in user_list:
field_row = st.columns([0.15, 0.45, 0.4])
with field_row[0]:
if mode != 'Multiple fields':
st.write("**Field Name**")
st.write("**Field Name**")
with field_row[1]:
if mode != 'Multiple fields':
field = st.selectbox('Field Name',sorted(set(field_prompt_mapping.keys())), index=0, label_visibility = "collapsed")
else:
field = sorted(set(field_prompt_mapping.keys()))
field = st.multiselect('Field Name',sorted(set(field_prompt_mapping.keys())), sorted(
set(field_prompt_mapping.keys())), label_visibility = "collapsed")
field_prompt_mapping = {key: field_prompt_mapping[key] for key in field}
contract_count_row = st.columns([0.15, 0.45, 0.4])
with contract_count_row[0]:
@@ -194,7 +195,7 @@ if st.session_state.user_info['mail'] in user_list:
region_name="us-east-2"
)
bucket = 'doczy-dev-infra-textract'
objects = s3_client.list_objects_v2(Bucket=bucket, Prefix="training-data/contract-text-file/")
objects = s3_client.list_objects_v2(Bucket=bucket, Prefix="training-data/")
file_list = []
for obj in objects['Contents']:
if not obj['Key'].endswith('/'):
@@ -214,7 +215,8 @@ if st.session_state.user_info['mail'] in user_list:
# st.write(df)
# st.write(len(contract_list))
# contract_list = [contract for contract in contract_list if contract.rsplit('/',1)[1].replace(' MU','').replace('_MU','').replace('.txt','') in list(field_values['Document_Name'])]
contract_list = [contract for contract in contract_list if contract.rsplit('/',1)[1].replace('.txt','.pdf') in list(field_values['Document_Name'])]
contract_list = [str(contract)[:-4]+'.pdf' for contract in contract_list]
contract_list = [contract for contract in contract_list if contract.rsplit('/',1)[1] in list(field_values['Document_Name'])]
# st.write(len(contract_list))
if contract_count == 'All':
@@ -247,8 +249,8 @@ if st.session_state.user_info['mail'] in user_list:
with llm_row[0]:
st.write("**Langauge Model**")
with llm_row[1]:
llm_selected = st.selectbox('Langauge Model',('Claude 2', 'Claude Instant', 'Llama 2 Chat 70B'
, 'Titan Text Express'), index=1, label_visibility = "collapsed")
llm_selected = st.selectbox('Langauge Model',('Claude 2', 'Claude 3 - Haiku', 'Claude 3 - Sonnet', 'Claude Instant'
, 'Llama 2 Chat 70B', 'Titan Text Express'), index=3, label_visibility = "collapsed")
st.write("**Prompt**")
if mode == 'Multiple fields':
@@ -276,6 +278,7 @@ if st.session_state.user_info['mail'] in user_list:
# # , column_name+'_PG': 'Original Page Number'}, inplace=True)
field_values.rename(columns={'Document_Name': 'Contract Name'}, inplace=True)
field_values = field_values.drop_duplicates(subset='Contract Name', keep="first").sort_values('Contract Name')
field_values = field_values[field_values['Contract Name'].isin([contract.rsplit('/',1)[1] for contract in contract_list])]
# Setup bedrock
bedrock_runtime = boto3.client(
@@ -313,7 +316,7 @@ if st.session_state.user_info['mail'] in user_list:
for contract in contract_list:
# with open(os.path.join(SOURCE_DIRECTORY, contract[:-4]+'.txt'), 'r') as infile:
# context = infile.read()
data = s3_client.get_object(Bucket=bucket, Key=contract)
data = s3_client.get_object(Bucket=bucket, Key=str(contract)[:-4]+'.txt')
contents = data['Body'].read()
context = contents.decode("utf-8")
# st.write(question_with_schema)
@@ -357,7 +360,7 @@ if st.session_state.user_info['mail'] in user_list:
body=json.dumps(payload)
model_id="meta.llama2-70b-chat-v1"
elif llm_selected in ['Claude Instant', 'Claude 2']:
elif llm_selected in ['Claude Instant', 'Claude 2', 'Claude 3 - Haiku', 'Claude 3 - Sonnet']:
if llm_selected == 'Claude Instant':
context = context[:175000]
@@ -370,18 +373,49 @@ if st.session_state.user_info['mail'] in user_list:
Question: {question_with_schema}
Assistant: Answer in JSON format: {{"""
body = json.dumps(
{"prompt": anthropic.HUMAN_PROMPT + prompt_data + anthropic.AI_PROMPT,
"max_tokens_to_sample": 2048,
"temperature":0.0,
"top_p":1,
"top_k":250,
"stop_sequences":[anthropic.HUMAN_PROMPT]
})
if llm_selected == "Claude 2":
model_id = "anthropic.claude-v2:1"
body = json.dumps(
{"prompt": anthropic.HUMAN_PROMPT + prompt_data + anthropic.AI_PROMPT,
"max_tokens_to_sample": 2048,
"temperature":0.0,
"top_p":1,
"top_k":250,
"stop_sequences":[anthropic.HUMAN_PROMPT]
})
elif llm_selected in ["Claude 3 - Haiku", "Claude 3 - Sonnet"]:
if llm_selected == "Claude 3 - Haiku":
model_id = 'anthropic.claude-3-haiku-20240307-v1:0'
else:
model_id = 'anthropic.claude-3-sonnet-20240229-v1:0'
body = json.dumps({
"anthropic_version": "bedrock-2023-05-31",
"max_tokens": 2048,
"messages": [
{
"role": "user",
"content": [
{
"type": "text",
"text":anthropic.HUMAN_PROMPT + prompt_data + anthropic.AI_PROMPT
}
]
}
],
"temperature": 0.0
}
)
else:
model_id = "anthropic.claude-instant-v1"
body = json.dumps(
{"prompt": anthropic.HUMAN_PROMPT + prompt_data + anthropic.AI_PROMPT,
"max_tokens_to_sample": 2048,
"temperature":0.0,
"top_p":1,
"top_k":250,
"stop_sequences":[anthropic.HUMAN_PROMPT]
})
try:
response = bedrock_runtime.invoke_model(
@@ -398,6 +432,8 @@ if st.session_state.user_info['mail'] in user_list:
response_text = response_body['generation']
elif llm_selected in ['Claude Instant', 'Claude 2']:
response_text = response_body['completion']
elif llm_selected in ['Claude 3 - Haiku', 'Claude 3 - Sonnet']:
response_text = response_body['content'][0]['text']
except:
response_text = "failed"
@@ -494,7 +530,7 @@ if st.session_state.user_info['mail'] in user_list:
# date_list.append(extracted_date)
# answer_list = date_list
try:
answer_list = [answer.strip("\n").strip().strip("{").strip("}").strip('"').rstrip('"').strip(
answer_list = [str(answer).strip("\n").strip().strip("[").strip("]").strip("{").strip("}").strip('"').rstrip('"').strip(
' ') if answer is not None else None for answer in answer_list]
if llm_selected in ['Llama 2 Chat 13B', 'Llama 2 Chat 70B']:
answer_list = [answer.rstrip(".") for answer in answer_list]
@@ -504,7 +540,7 @@ if st.session_state.user_info['mail'] in user_list:
answer_list = [answer if "None" not in str(answer) else " " for answer in answer_list]
answer_list = [answer if "Not specified in the contract" not in str(answer) else " " for answer in answer_list]
answer_list = [answer if "Not applicable" not in str(answer) else " " for answer in answer_list]
elif llm_selected in ['Claude 2', 'Claude Instant']:
elif llm_selected in ['Claude 2','Claude 3 - Haiku', 'Claude 3 - Sonnet', 'Claude Instant']:
answer_list = [answer if "do not have" not in str(answer) else " " for answer in answer_list]
answer_list = [answer if "do not see" not in str(answer) else " " for answer in answer_list]
answer_list = [answer if "does not specify" not in str(answer) else " " for answer in answer_list]
@@ -519,23 +555,16 @@ if st.session_state.user_info['mail'] in user_list:
answer_list = [answer if "don't have" not in str(answer) else " " for answer in answer_list]
answer_list = [answer if "Does not apply" not in str(answer) else " " for answer in answer_list]
answer_list = [answer if "Nothing found" not in str(answer) else " " for answer in answer_list]
answer_list = [answer if str(answer) != "one-year" else "1 year" for answer in answer_list]
answer_list = [answer if str(answer) != "one year" else "1 year" for answer in answer_list]
answer_list = [answer if str(answer) != "one" else "1 year" for answer in answer_list]
answer_list = [answer if str(answer) != "one (1) year" else "1 year" for answer in answer_list]
answer_list = [answer.rstrip(".") for answer in answer_list]
else:
answer_list = [answer.rstrip(".") for answer in answer_list]
# answer_list = [str(x).rsplit(':',1)[0] if len(str(x).rsplit(':',1)) < 2 else str(x).rsplit(':',1)[1] for x in answer_list]
except:
print('post processing failed')
# df['Contract ID'] = contract_list_f
df['Contract ID'] = range(len(contract_list_f))
except Exception as e:
st.write(e)
# to be deleted later
# contract_list_f = [contract.rsplit('/',1)[1].replace(' MU','').replace('_MU','').replace('.txt','') for contract in contract_list_f]
contract_list_f = [contract.rsplit('/',1)[1].replace('.txt','.pdf') for contract in contract_list_f]
contract_list_f = [contract.rsplit('/',1)[1] for contract in contract_list_f]
df['Contract Name'] = contract_list_f
df['New Extracted value'] = answer_list
@@ -552,7 +581,7 @@ if st.session_state.user_info['mail'] in user_list:
for file_name in contract_list:
# document_name = [x for x in list(field_values['Contract Name']) if not pd.isna(x) and file_name.rsplit('/',1)[1].replace(' MU','').replace(
# '_MU','').replace('.txt','') in x][0]
document_name = [x for x in list(field_values['Contract Name']) if not pd.isna(x) and file_name.rsplit('/',1)[1].replace('.txt','.pdf') in x][0]
document_name = [x for x in list(field_values['Contract Name']) if not pd.isna(x) and file_name.rsplit('/',1)[1] in x][0]
field_values_1 = field_values[field_values['Contract Name'] == document_name].head(1).transpose().reset_index()
field_values_1.columns = ['SF_DB_COL_NAME', 'Actual Value Stored']
field_values_p1 = field_values_1[~field_values_1['SF_DB_COL_NAME'].str.endswith('_PG')]
@@ -564,10 +593,17 @@ if st.session_state.user_info['mail'] in user_list:
field_values_1['Contract Name'] = document_name
field_values_2 = pd.concat([field_values_2, field_values_1], ignore_index = True)
df = pd.merge(df, field_values_2, how ='left', on =['Contract Name', 'SF_DB_COL_NAME'])
# st.write(df)
# st.write(field_values_2)
df = pd.merge(df, field_values_2, how ='right', on =['Contract Name', 'SF_DB_COL_NAME'])
if mode == 'Multiple fields':
df = df[df['SF_DB_COL_NAME'].isin(list(field_prompt_mapping.keys()))]
else:
df = df[df['SF_DB_COL_NAME'].isin([field])]
df = df.drop_duplicates(subset=['SF_DB_COL_NAME', 'Contract Name'], keep="first")
df['Original Page Number'] = df['Original Page Number'].apply(lambda x: re.search(r'\d+', x).group(
) if isinstance(x, str) and re.search(r'\d+', x) is not None else " ")
df['Raw value 2'] = df['New Extracted value']
df_date = df[df['SF_DB_COL_NAME'].str.contains('_DT', na=False)]
df_others = df[~df['SF_DB_COL_NAME'].str.contains('_DT', na=False)]
@@ -577,20 +613,36 @@ if st.session_state.user_info['mail'] in user_list:
df.sort_values(['SF_DB_COL_NAME', 'Contract Name'], inplace=True)
df.fillna(" ", inplace=True)
df['Raw value 3'] = df['New Extracted value']
df['Actual Value Stored'] = df['Actual Value Stored'].apply(lambda x: x.strip() if isinstance(x, str) else '')
df['New Extracted value'] = df['New Extracted value'].apply(lambda x: x.strip() if isinstance(x, str) else '')
actual_value_list = list(df['Actual Value Stored'])
actual_value_list = [answer if str(answer) != "12 months" else "1 year" for answer in actual_value_list]
actual_value_list = [answer if str(answer) != "Fifth" else "5" for answer in actual_value_list]
actual_value_list = [answer if str(answer) != "Seventh" else "7" for answer in actual_value_list]
answer_list = list(df['New Extracted value'])
answer_list = [answer if str(answer) != "one-year" else "1 year" for answer in answer_list]
answer_list = [answer if str(answer) != "one year" else "1 year" for answer in answer_list]
answer_list = [answer if str(answer) != "one" else "1 year" for answer in answer_list]
answer_list = [answer if str(answer) != "one (1) year" else "1 year" for answer in answer_list]
answer_list = [answer if str(answer) != "twelve" else "1 year" for answer in answer_list]
answer_list = [answer if str(answer) != "XI" else "11" for answer in answer_list]
answer_list = [answer if str(answer) != "Third" else "3" for answer in answer_list]
answer_list = [answer if str(answer) != "Six" else "6" for answer in answer_list]
actual_value_list = [s.replace('-', '').replace(' ', '').replace('[', '').replace(']', '').lower() for s in actual_value_list]
answer_list = [s.replace('-', '').replace(' ', '').replace('[', '').replace(']', '').lower() for s in answer_list]
result_list = [(i in j) or (j in i) if isinstance(i, str) and isinstance(
j, str) and ((i != '') == (j != '')) else False for i, j in zip(actual_value_list, answer_list)]
df['Result'] = [str(x) for x in result_list]
df = df[~df['Contract ID'].isnull()]
df = df[['Contract Name','Contract ID', 'SF_DB_COL_NAME', 'Actual Value Stored', 'Raw value', 'New Extracted value','Confidence Level'
,'Snippet','Original Page Number', 'New Page Number', 'Revised Prompt', 'Result']]
# df = df[~df['Contract ID'].isnull()]
# df['Contract ID'] = contract_list_f
df['Contract ID'] = range(len(actual_value_list))
df = df[['Contract Name','Contract ID', 'SF_DB_COL_NAME', 'Actual Value Stored', 'Raw value', 'Raw value 2', 'Raw value 3'
, 'New Extracted value','Confidence Level','Snippet','Original Page Number', 'New Page Number', 'Revised Prompt', 'Result']]
try:
accuracy = round(sum(bool(x) for x in result_list) * 100 / len(list(df['Result'])), 2)
streamlit/interface_3_rag.py
+1 -1
View File
@@ -20,7 +20,7 @@ import util
REDIRECT_URI = 'http://172.29.20.126:8503'
user_list = ['maamseek@aarete.com', 'smahdavian@aarete.com', 'ahinge@aarete.com', 'akadam@aarete.com', 'pkatariya@aarete.com'
, 'piragavarapu@aarete.com', 'umistry@aarete.com', 'ahutchison@aarete.com', 'bgrunst@aarete.com', 'ddimeglio@aarete.com'
, 'vnair@aarete.com', 'kminhas@aarete.com', 'fmohiuddin@aarete.com', 'slitewka@aarete.com', 'qdoest@aarete.com', 'bkoryga@aarete.com', 'bcielecki@aarete.com', 'mszymanski@aarete.com']
, 'vnair@aarete.com', 'kminhas@aarete.com', 'fmohiuddin@aarete.com', 'slitewka@aarete.com', 'qdoest@aarete.com', 'bkoryga@aarete.com', 'bcielecki@aarete.com', 'mszymanski@aarete.com','hupreti@aarete.com']
st.set_page_config(layout = "wide")
# Sidebar contents
streamlit/security.py
+9 -7
View File
@@ -3,7 +3,7 @@ import msal
import requests
import boto3
from botocore.exceptions import ClientError
import json
# Replace with your own values
CLIENT_ID = 'effafe90-7ed7-43a3-ab03-19a0be2f1758'
@@ -14,6 +14,7 @@ AUTHORITY = 'https://login.microsoftonline.com/organizations/'
SCOPE = ['User.Read']
REDIRECT_URI = 'https://171.29.20.126:8501'
# Initialize boto3 client to interact with AWS Secrets Manager
@@ -24,12 +25,12 @@ def get_secret():
region_name = "us-east-2"
# Create a Secrets Manager client
session = boto3.session.Session()
client = session.client(
service_name='secretsmanager',
region_name=region_name
)
# session = boto3.session.Session()
# client = session.client(
# service_name='secretsmanager',
# region_name=region_name
# )
client = boto3.client('secretsmanager', region_name=region_name)
try:
get_secret_value_response = client.get_secret_value(
SecretId=secret_name
@@ -38,6 +39,7 @@ def get_secret():
raise e
secret = get_secret_value_response['SecretString']
secret = json.loads(secret)['CLIENT_SECRET']
return secret
streamlit/sf_conn.py
+2 -2
View File
@@ -106,7 +106,7 @@ def get_client_names():
cursor = conn.cursor()
# Query to get client names and their s3_paths
query = "SELECT DISTINCT client_name, s3_bucket_path FROM STG.CLIENT_CONFIG"
query = "SELECT DISTINCT client_name, bucket_name FROM STG.CLIENT_LOGS"
cursor.execute(query)
# Create 2 lists from the query results
client_names = []
@@ -114,7 +114,7 @@ def get_client_names():
for row in cursor:
client_names.append(row[0])
s3_paths.append(row[1].replace('s3://','').split('/')[0]) # Extracting the bucket name from the s3 path
s3_paths.append(row[1]) # Extracting the bucket name from the s3 path
cursor.close()
conn.close()
return client_names, s3_paths