From 4af8a08f622e4a41995b79d90d69d8aa5e0efbf5 Mon Sep 17 00:00:00 2001 From: "AARETE\\agupta" Date: Thu, 27 Jun 2024 16:57:09 -0500 Subject: [PATCH] PROD links added and more error handling done --- streamlit/constants.py | 13 +++++++++++-- streamlit/interface_2.py | 24 +++++++++++++----------- streamlit/multipage/constants.py | 13 +++++++++++-- streamlit/multipage/pages/Interface_1.py | 5 ++++- streamlit/multipage/pages/Interface_2.py | 24 +++++++++++++----------- streamlit/multipage/util.py | 4 ++++ streamlit/util.py | 4 ++++ 7 files changed, 60 insertions(+), 27 deletions(-) diff --git a/streamlit/constants.py b/streamlit/constants.py index 254c509..8ccab24 100644 --- a/streamlit/constants.py +++ b/streamlit/constants.py @@ -212,7 +212,10 @@ DOCZY_PIPELINE_URL_UAT = 'https://29gm8cek03.execute-api.us-east-2.amazonaws.com DOCZY_REDIRECT_URL_UAT = 'https://doczyuat.aarete.com:850' DOCZY_CREATE_BATCH_URL_UAT = 'https://29gm8cek03.execute-api.us-east-2.amazonaws.com/dev/create-batch' -# DOCZY UAT +# DOCZY PROD +DOCZY_PIPELINE_URL_PROD = 'https://d612isd3ja.execute-api.us-east-2.amazonaws.com/dev/trigger-pipeline' +DOCZY_REDIRECT_URL_PROD = 'https://doczy.aarete.com:850' +DOCZY_CREATE_BATCH_URL_PROD = 'https://d612isd3ja.execute-api.us-east-2.amazonaws.com/dev/create-batch' # SNOWFLAKE DEV DATABASE SNOWFLAKE_ACCOUNT_LOCATOR="aarete-doczyai", @@ -223,6 +226,12 @@ DEV_STAGING_SCHEMA="STG" # SNOWFLAKE UAT DATABASE UAT_DB_ROLE = "UATADMIN", -UAT_WH="UAT_XS", +UAT_WH="DEV_XS", UAT_DB="DOCZY_UAT", UAT_STAGING_SCHEMA="STG" + +# SNOWFLAKE PROD DATABASE +PROD_DB_ROLE = "PRODADMIN", +PROD_WH="DEV_XS", +PROD_DB="DOCZY_PROD", +PROD_STAGING_SCHEMA="STG" \ No newline at end of file diff --git a/streamlit/interface_2.py b/streamlit/interface_2.py index fd38dfb..0f1de08 100644 --- a/streamlit/interface_2.py +++ b/streamlit/interface_2.py @@ -189,18 +189,8 @@ if client: elif field_group == 'Timeline': fields = fields[fields['PRIORITY'] == 'E'] - button_cols = st.columns([0.2, 0.2, 0.6]) + button_cols = st.columns([1,1,8]) with button_cols[0]: - if st.button("Show Results"): - query = f'select * from "DOCZY_PIPELINE_RAW_OUTPUT_AC" where batch_id = \'{batch_id}\'' - cur.execute(query) - df2 = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description]) - - # get this dataframe from snowflake table - # df2 = pd.DataFrame(columns=['Contract Name','Field Name', 'SF_DB_COL_NAME', 'Snippet','Page Number' - # , 'Field Extracted Value', 'Actual Value','Imputed Value']) - df2.to_csv('temp2.csv', index=False) - with button_cols[1]: if st.button("Show PDF"): if file_name == None or file_name == "All": st.error("Choose one specific file.") @@ -219,6 +209,18 @@ if client: s3_obj = s3_client.get_object(Bucket = client_bucket, Key = file_name) data=s3_obj['Body'].read() pdf_viewer(data, width=1500) + with button_cols[1]: + if st.button("Show Results"): + try: + query = f'select * from "DOCZY_PIPELINE_RAW_OUTPUT_AC" where batch_id = \'{batch_id}\'' + cur.execute(query) + df2 = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description]) + + except: + df2 = pd.DataFrame(columns=['Contract Name','Field Name', 'SF_DB_COL_NAME', 'Snippet','Page Number' + , 'Field Extracted Value', 'Actual Value','Imputed Value']) + st.error('There was an error in fetching the output.') + df2.to_csv('temp2.csv', index=False) # if st.button("Show PDF"): # if file_name == None or file_name == "All": diff --git a/streamlit/multipage/constants.py b/streamlit/multipage/constants.py index 254c509..8ccab24 100644 --- a/streamlit/multipage/constants.py +++ b/streamlit/multipage/constants.py @@ -212,7 +212,10 @@ DOCZY_PIPELINE_URL_UAT = 'https://29gm8cek03.execute-api.us-east-2.amazonaws.com DOCZY_REDIRECT_URL_UAT = 'https://doczyuat.aarete.com:850' DOCZY_CREATE_BATCH_URL_UAT = 'https://29gm8cek03.execute-api.us-east-2.amazonaws.com/dev/create-batch' -# DOCZY UAT +# DOCZY PROD +DOCZY_PIPELINE_URL_PROD = 'https://d612isd3ja.execute-api.us-east-2.amazonaws.com/dev/trigger-pipeline' +DOCZY_REDIRECT_URL_PROD = 'https://doczy.aarete.com:850' +DOCZY_CREATE_BATCH_URL_PROD = 'https://d612isd3ja.execute-api.us-east-2.amazonaws.com/dev/create-batch' # SNOWFLAKE DEV DATABASE SNOWFLAKE_ACCOUNT_LOCATOR="aarete-doczyai", @@ -223,6 +226,12 @@ DEV_STAGING_SCHEMA="STG" # SNOWFLAKE UAT DATABASE UAT_DB_ROLE = "UATADMIN", -UAT_WH="UAT_XS", +UAT_WH="DEV_XS", UAT_DB="DOCZY_UAT", UAT_STAGING_SCHEMA="STG" + +# SNOWFLAKE PROD DATABASE +PROD_DB_ROLE = "PRODADMIN", +PROD_WH="DEV_XS", +PROD_DB="DOCZY_PROD", +PROD_STAGING_SCHEMA="STG" \ No newline at end of file diff --git a/streamlit/multipage/pages/Interface_1.py b/streamlit/multipage/pages/Interface_1.py index 107b1d8..f24fb69 100644 --- a/streamlit/multipage/pages/Interface_1.py +++ b/streamlit/multipage/pages/Interface_1.py @@ -92,7 +92,10 @@ if client: batch_list = [] for prefix in batch_objects['CommonPrefixes']: - batch_list.append(prefix['Prefix'][:-1].split('/')[-1]) + batch_name = prefix['Prefix'][:-1].split('/')[-1] + batch_objects2 = s3_client.list_objects_v2(Bucket=client_bucket, Prefix="contracts-landing-zone/"+batch_name+"/", Delimiter='/') + if 'Contents' in batch_objects2 and len(batch_objects2['Contents'] > 0): + batch_list.append(prefix['Prefix'][:-1].split('/')[-1]) # Hardcoded batch_list for testing purposes # batch_list = ['batch_020524103737', 'batch_090524131433', 'batch_090524131607', 'batch_100524123000', 'batch_130524064322', diff --git a/streamlit/multipage/pages/Interface_2.py b/streamlit/multipage/pages/Interface_2.py index e3c164c..8e2328a 100644 --- a/streamlit/multipage/pages/Interface_2.py +++ b/streamlit/multipage/pages/Interface_2.py @@ -189,18 +189,8 @@ if client: elif field_group == 'Timeline': fields = fields[fields['PRIORITY'] == 'E'] - button_cols = st.columns([0.2, 0.2, 0.6]) + button_cols = st.columns([1,1,8]) with button_cols[0]: - if st.button("Show Results"): - query = f'select * from "DOCZY_PIPELINE_RAW_OUTPUT_AC" where batch_id = \'{batch_id}\'' - cur.execute(query) - df2 = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description]) - - # get this dataframe from snowflake table - # df2 = pd.DataFrame(columns=['Contract Name','Field Name', 'SF_DB_COL_NAME', 'Snippet','Page Number' - # , 'Field Extracted Value', 'Actual Value','Imputed Value']) - df2.to_csv('temp2.csv', index=False) - with button_cols[1]: if st.button("Show PDF"): if file_name == None or file_name == "All": st.error("Choose one specific file.") @@ -219,6 +209,18 @@ if client: s3_obj = s3_client.get_object(Bucket = client_bucket, Key = file_name) data=s3_obj['Body'].read() pdf_viewer(data, width=1500) + with button_cols[1]: + if st.button("Show Results"): + try: + query = f'select * from "DOCZY_PIPELINE_RAW_OUTPUT_AC" where batch_id = \'{batch_id}\'' + cur.execute(query) + df2 = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description]) + + except: + df2 = pd.DataFrame(columns=['Contract Name','Field Name', 'SF_DB_COL_NAME', 'Snippet','Page Number' + , 'Field Extracted Value', 'Actual Value','Imputed Value']) + st.error('There was an error in fetching the output.') + df2.to_csv('temp2.csv', index=False) # if st.button("Show PDF"): # if file_name == None or file_name == "All": diff --git a/streamlit/multipage/util.py b/streamlit/multipage/util.py index 89e2d64..6497716 100644 --- a/streamlit/multipage/util.py +++ b/streamlit/multipage/util.py @@ -59,6 +59,10 @@ def load_page_details(interface): logger.info(constants.DOCZY_REDIRECT_URL_DEV + str(interface)) return (constants.DOCZY_REDIRECT_URL_DEV + str(interface), constants.DOCZY_CREATE_BATCH_URL_DEV, constants.DOCZY_PIPELINE_URL_DEV) + elif env_var == 'PROD': + logger.info(constants.DOCZY_REDIRECT_URL_PROD + str(interface)) + return (constants.DOCZY_REDIRECT_URL_PROD + str(interface), constants.DOCZY_CREATE_BATCH_URL_PROD, + constants.DOCZY_PIPELINE_URL_PROD) else: logger.info(constants.DOCZY_REDIRECT_URL_DEV + str(interface)) return (constants.DOCZY_REDIRECT_URL_DEV + str(interface), constants.DOCZY_CREATE_BATCH_URL_DEV, diff --git a/streamlit/util.py b/streamlit/util.py index 89e2d64..6497716 100644 --- a/streamlit/util.py +++ b/streamlit/util.py @@ -59,6 +59,10 @@ def load_page_details(interface): logger.info(constants.DOCZY_REDIRECT_URL_DEV + str(interface)) return (constants.DOCZY_REDIRECT_URL_DEV + str(interface), constants.DOCZY_CREATE_BATCH_URL_DEV, constants.DOCZY_PIPELINE_URL_DEV) + elif env_var == 'PROD': + logger.info(constants.DOCZY_REDIRECT_URL_PROD + str(interface)) + return (constants.DOCZY_REDIRECT_URL_PROD + str(interface), constants.DOCZY_CREATE_BATCH_URL_PROD, + constants.DOCZY_PIPELINE_URL_PROD) else: logger.info(constants.DOCZY_REDIRECT_URL_DEV + str(interface)) return (constants.DOCZY_REDIRECT_URL_DEV + str(interface), constants.DOCZY_CREATE_BATCH_URL_DEV,