diff --git a/streamlit/interface_0.py b/streamlit/interface_0.py index 09ccf09..5d12f6c 100644 --- a/streamlit/interface_0.py +++ b/streamlit/interface_0.py @@ -91,12 +91,6 @@ if user_mail in user_list: st.write(f"{batch_id} created") st.write(f"Files uploaded to s3://{client}/{batch_id}/{landing_zone}") - try: - save_to_sf('load_request_and_contract_submissions', "request_submission_file_name", file_names[0] - , "contract_config_file_name", file_names[0]) - except: - st.write("running locally") - # @st.cache_data # def convert_df(df): @@ -152,7 +146,6 @@ if user_mail in user_list: # # st.write(response.text) - else: st.write("Access Denied") diff --git a/streamlit/interface_1.py b/streamlit/interface_1.py index abe548f..fc61e2e 100644 --- a/streamlit/interface_1.py +++ b/streamlit/interface_1.py @@ -30,7 +30,7 @@ st.set_page_config(layout = "wide") _,c1= st.columns([5,1]) try: - util.setup_page(REDIRECT_URI) + putil.setup_page(REDIRECT_URI) except: st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} c1.write(f"User: **{st.session_state.user_info['displayName']}**") @@ -41,15 +41,9 @@ if user_mail in user_list: s3_client = boto3.client('s3', region_name="us-east-1", ) - objects = s3_client.list_objects_v2(Bucket='doczy-dev-infra-textract' - , Prefix="batches/batch_1/", Delimiter='/') - folder_list = [] - for prefix in objects['CommonPrefixes']: - folder_list.append(prefix['Prefix'][:-1].split('/')[-1]) - - # to be deleted later - folder_list = ['Delaware First Health, Inc.', 'Community Health Choice, Inc','CareSource Network Partners LLC', + # to be replaced with snowflake data + client_list = ['doczy-ai-client-1', 'Delaware First Health, Inc.', 'Community Health Choice, Inc','CareSource Network Partners LLC', 'HealthNet of Cali', 'Oklahoma Complete Health, Inc', 'HealthFirst', 'Molina Healthcare of TX', 'AvMed', 'Arizona Care1st', 'WellCare New Jersey'] @@ -57,23 +51,20 @@ if user_mail in user_list: with client_row[0]: st.write("**Client Name**") with client_row[1]: - client = st.selectbox('Client Name',(folder_list), label_visibility = "collapsed") + client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed") - # to be deleted later - client = 'textract-receiver-processed-file' + batch_objects = s3_client.list_objects_v2(Bucket=client + , Prefix="", Delimiter='/') - folder_objects = s3_client.list_objects_v2(Bucket='doczy-dev-infra-textract' - , Prefix="batches/batch_1/"+client+"/", Delimiter='/') - - folder_list_2 = [] - for prefix in folder_objects['CommonPrefixes']: - folder_list_2.append(prefix['Prefix'][:-1].split('/')[-1]) + batch_list = [] + for prefix in batch_objects['CommonPrefixes']: + batch_list.append(prefix['Prefix'][:-1].split('/')[-1]) path_row = st.columns([0.1, 0.8]) with path_row[0]: - st.write("**Path to folder**") + st.write("**Batch ID**") with path_row[1]: - Directory = st.selectbox('**Path to folder**', folder_list_2, label_visibility = "collapsed") + batch_id = st.selectbox('**Batch ID**', batch_list, label_visibility = "collapsed") checks = st.columns([0.1, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12]) with checks[0]: @@ -98,8 +89,8 @@ if user_mail in user_list: df = pd.DataFrame(columns=['Contract Name','Unique Key','Pricing Before Carveouts' , 'Contract Related', 'Provider', 'Timeline', 'Carveout Indicator', 'Carveout Methodology']) file_list = [] - file_objects = s3_client.list_objects_v2(Bucket='doczy-dev-infra-textract' - , Prefix="batches/batch_1/"+client+"/"+Directory+"/", Delimiter='/') + file_objects = s3_client.list_objects_v2(Bucket=client + , Prefix=batch_id+"/contracts_landing_zone/", Delimiter='/') if st.button("Read the contracts from Path"): for obj in file_objects.get('Contents',[]): @@ -121,64 +112,72 @@ if user_mail in user_list: df.to_csv('temp1.csv', index=False) add_vertical_space(1) - - # df_copy = df.set_index(df.columns[0]).copy() df2 = pd.read_csv('temp1.csv') edited_df = st.data_editor(df2) - @st.cache_data def convert_df(df): return df.to_csv(index=False).encode('utf-8') csv = convert_df(edited_df) - edited_df = edited_df.reset_index() # make sure indexes pair with number of rows - contract_list = [] - for index, row in edited_df.iterrows(): - group_list = [] - if row['Unique Key']: - group_list.append('Unique Key') - if row['Pricing Before Carveouts']: - group_list.append('Pricing Before Carveouts') - if row['Contract Related']: - group_list.append('Contract Related') - if row['Provider']: - group_list.append('Provider') - if row['Timeline']: - group_list.append('Timeline') - if row['Carveout Indicator']: - group_list.append('Carveout Indicator') - if row['Carveout Methodology']: - group_list.append('Carveout Methodology') - entry_dict = { - "contract_name": row['Contract Name'], - "groups": group_list, - "contract_source_path": "batches/batch_1/"+client+"/"+row['Contract Name'] - } - contract_list.append(entry_dict) + # additional_info = pd.DataFrame(columns=['REQUEST_ID','T_DRIVE_PATH','CLIENT_NAME' + # , 'GROUP_NAME', 'REQUEST_USERNAME', 'REQUEST_DATETIME']) + additional_info = pd.DataFrame(columns=['CLIENT_NAME', 'BATCH_ID', 'REQUEST_USERNAME', 'REQUEST_DATETIME']) + additional_info.loc[0] = [client, batch_id, st.session_state.user_info['mail'], datetime.now().strftime("%Y-%m-%d %H:%M:%S")] + st.write(additional_info) - myobj = { - "s3_bucket": 'doczy-dev-infra-textract', - "batch_id": "1", - "client_name": client, - "username": user_mail, - "contract_list": contract_list - } + # contract_list = [] + # for index, row in edited_df.iterrows(): + # group_list = [] + # if row['Unique Key']: + # group_list.append('Unique Key') + # if row['Pricing Before Carveouts']: + # group_list.append('Pricing Before Carveouts') + # if row['Contract Related']: + # group_list.append('Contract Related') + # if row['Provider']: + # group_list.append('Provider') + # if row['Timeline']: + # group_list.append('Timeline') + # if row['Carveout Indicator']: + # group_list.append('Carveout Indicator') + # if row['Carveout Methodology']: + # group_list.append('Carveout Methodology') + # entry_dict = { + # "contract_name": row['Contract Name'], + # "groups": group_list, + # "contract_source_path": "batches/batch_1/"+client+"/"+row['Contract Name'] + # } + # contract_list.append(entry_dict) + + # myobj = { + # "s3_bucket": 'doczy-dev-infra-textract', + # "batch_id": "1", + # "client_name": client, + # "username": user_mail, + # "contract_list": contract_list + # } buttons = st.columns([0.8, 0.2]) with buttons[0]: st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv') with buttons[1]: if st.button("Run Doczy.AI Pipeline"): - response = requests.post(doczy_pipeline, json = myobj) - if response.status_code >= 200 and response.status_code < 300: - st.write("Success") - else: - st.write("Failed") - # st.write(response.text) - + csv_buf = StringIO() + additional_info.to_csv(csv_buf, header=True, index=False) + csv_buf.seek(0) + s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/request_submission.csv') + csv_buf = StringIO() + edited_df.to_csv(csv_buf, header=True, index=False) + csv_buf.seek(0) + s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/contract_config.csv') + try: + save_to_sf('load_request_and_contract_submissions', "request_submission_file_name" + , "contract_config_file_name", "request_submission.csv", "contract_config.csv") + except: + st.write("running locally") else: diff --git a/streamlit/interface_3.py b/streamlit/interface_3.py index 9b761bc..09683a0 100644 --- a/streamlit/interface_3.py +++ b/streamlit/interface_3.py @@ -581,7 +581,7 @@ if st.session_state.user_info['mail'] in user_list: st.write(raw_response_text) try: - save_to_sf('load_training_results', "results.csv", "history.csv") + save_to_sf('load_training_results', "training_results_file_name", "attempt_logs_file_name", "results.csv", "history.csv") except: st.write("running locally") diff --git a/streamlit/sf_conn.py b/streamlit/sf_conn.py index f60fdf4..7a728a4 100644 --- a/streamlit/sf_conn.py +++ b/streamlit/sf_conn.py @@ -36,7 +36,7 @@ def get_secret(): # get_secret() -def save_to_sf(dag_name, training_results_file, attempt_logs_file): +def save_to_sf(dag_name, conf_1, conf_2, training_results_file, attempt_logs_file): mwaa_env_name = 'doczy-dev-infra-mwaa' dag_name = dag_name @@ -60,8 +60,8 @@ def save_to_sf(dag_name, training_results_file, attempt_logs_file): # conf = "{\"" + "training_results_file_name" + "\":\"" + {training_results_file} + "\", \"" + "attempt_logs_file_name" + "\":\"" + {attempt_logs_file} + "\"}".format(training_results_file=training_results_file, attempt_logs_file=attempt_logs_file) conf = f"""{{ - "training_results_file_name": "{training_results_file}", - "attempt_logs_file_name": "{attempt_logs_file}" + "{conf_1}": "{training_results_file}", + "{conf_2}": "{attempt_logs_file}" }}"""