interface 0 connected to S3

2024-03-28 17:53:00 +05:30
parent 4f2170fd8a
commit 2726bf6c7b
4 changed files with 63 additions and 46 deletions
@@ -30,7 +30,7 @@ st.set_page_config(layout = "wide")
 #     # st.write("Doczy")

 try:
-    util.setup_page(REDIRECT_URI)
+    # util.setup_page(REDIRECT_URI)
    _,c1= st.columns([5,1])
    c1.write(f"User: **{st.session_state.user_info['displayName']}**")
    user_mail = st.session_state.user_info['mail']
@@ -49,10 +49,10 @@ if user_mail in user_list:
    for prefix in objects['CommonPrefixes']:
        folder_list.append(prefix['Prefix'][:-1].split('/')[-1])

-    # # to be deleted later
-    # folder_list = ['Delaware First Health, Inc.', 'Community Health Choice, Inc','CareSource Network Partners LLC',
-    #     'HealthNet of Cali', 'Oklahoma Complete Health, Inc', 'HealthFirst', 'Molina Healthcare of TX', 'AvMed', 'Arizona Care1st',
-    #     'WellCare New Jersey']
+    # to be deleted later
+    folder_list = ['Delaware First Health, Inc.', 'Community Health Choice, Inc','CareSource Network Partners LLC',
+        'HealthNet of Cali', 'Oklahoma Complete Health, Inc', 'HealthFirst', 'Molina Healthcare of TX', 'AvMed', 'Arizona Care1st',
+        'WellCare New Jersey']

    client_row = st.columns([0.1, 0.8])
    with client_row[0]:
@@ -60,8 +60,8 @@ if user_mail in user_list:
    with client_row[1]:
        client = st.selectbox('Client Name',(folder_list), label_visibility = "collapsed")

-    # # to be deleted later
-    # client = 'textract-receiver-processed-file'
+    # to be deleted later
+    client = 'textract-receiver-processed-file'

    folder_objects = s3_client.list_objects_v2(Bucket='doczy-dev-infra-textract'
        , Prefix="batches/batch_1/"+client+"/", Delimiter='/')      
@@ -72,34 +72,50 @@ if user_mail in user_list:

    path_row = st.columns([0.1, 0.8])
    with path_row[0]:
-        st.write("**Path to folder**")
+        # st.write("**Path to folder**")
+        st.write("**Upload Files**")
    with path_row[1]:
-        Directory = st.selectbox('**Path to folder**', folder_list_2, label_visibility = "collapsed")
+        # Directory = st.selectbox('**Path to folder**', folder_list_2, label_visibility = "collapsed")
+        file_list = st.file_uploader("Upload", type=None, accept_multiple_files=True, label_visibility = "collapsed")

    add_vertical_space(1)

    df = pd.DataFrame(columns=['Contract Name','Unique Key','Pricing Before Carveouts'
        , 'Contract Related', 'Provider', 'Timeline', 'Carveout Indicator', 'Carveout Methodology'])
-    file_list = []
-    file_objects = s3_client.list_objects_v2(Bucket='doczy-dev-infra-textract'
-        , Prefix="batches/batch_1/"+client+"/"+Directory+"/", Delimiter='/')
+    df['Contract Name'] = file_list
+    # file_list = []
+    # file_objects = s3_client.list_objects_v2(Bucket='doczy-dev-infra-textract'
+    #     , Prefix="batches/batch_1/"+client+"/"+Directory+"/", Delimiter='/')

-    if st.button("Create Batch"):
-        for obj in file_objects.get('Contents',[]):
-            if not obj['Key'].endswith('/'):
-                file_list.append(obj['Key'].split('/')[-1])
+    add_vertical_space(2)

-        df['Contract Name'] = file_list
+    buttons = st.columns([0.4, 0.4, 0.2])
+    with buttons[1]:
+        if st.button("Create Batch"):
+            # for obj in file_objects.get('Contents',[]):
+            #     if not obj['Key'].endswith('/'):
+            #         file_list.append(obj['Key'].split('/')[-1])
+            for uploaded_file in file_list:
+                stringio = StringIO(uploaded_file.getvalue().decode("utf-8"))
+                stringio.seek(0)
+                s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=stringio.getvalue(), Key=
+                    'config_interface/'+str(uploaded_file.name))

+        
    add_vertical_space(1)

-    @st.cache_data 
-    def convert_df(df):
-       return df.to_csv(index=False).encode('utf-8')
+    try:
+        save_to_sf('load_request_and_contract_submissions', "results.csv", "history.csv")
+    except:
+        st.write("running locally")

-    csv = convert_df(df)
+    # @st.cache_data 
+    # def convert_df(df):
+    #    return df.to_csv(index=False).encode('utf-8')

-    df = df.reset_index()  # make sure indexes pair with number of rows
+    # csv = convert_df(df)
+
+    # df = df.reset_index()  # make sure indexes pair with number of rows

    # contract_list = []
    # for index, row in df.iterrows():
@@ -124,27 +140,27 @@ if user_mail in user_list:
    #         "contract_source_path": "batches/batch_1/"+client+"/"+row['Contract Name']
    #     }
    #     contract_list.append(entry_dict)
-    contract_list = list(df['Contract Name'])
+    # contract_list = list(df['Contract Name'])

-    myobj = {
-      "s3_bucket": 'doczy-dev-infra-textract',
-      "batch_id": "1",
-      "client_name": client,
-      "username": user_mail,
-      "contract_list": contract_list
-    }
+    # myobj = {
+    #   "s3_bucket": 'doczy-dev-infra-textract',
+    #   "batch_id": "1",
+    #   "client_name": client,
+    #   "username": user_mail,
+    #   "contract_list": contract_list
+    # }

-    buttons = st.columns([0.8, 0.2])
-    with buttons[0]:
-        st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv')
-    with buttons[1]:
-        if st.button("Upload to DB"):
-            response = requests.post(doczy_pipeline, json = myobj)
-            if response.status_code >= 200 and response.status_code < 300:
-                st.write("Success")
-            else:
-                st.write("Failed")
-            # st.write(response.text)
+    # buttons = st.columns([0.8, 0.2])
+    # with buttons[0]:
+    #     st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv')
+    # with buttons[1]:
+    #     if st.button("Upload to DB"):
+    #         response = requests.post(doczy_pipeline, json = myobj)
+    #         if response.status_code >= 200 and response.status_code < 300:
+    #             st.write("Success")
+    #         else:
+    #             st.write("Failed")
+    #         # st.write(response.text)



@@ -433,9 +433,10 @@ if user_mail in user_list:

    buttons = st.columns(3)
    with buttons[0]:
-        st.button("Save All Imputations") 
-    with buttons[1]:
+        # st.button("Save All Imputations") 
        st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv')
+    with buttons[1]:
+        # st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv')
    with buttons[2]:
        st.button("Kickoff Database Integration") 

@@ -568,7 +568,7 @@ if user_mail in user_list:
    st.write(raw_response_text)

    try:
-        save_to_sf("results.csv", "history.csv")
+        save_to_sf('load_training_results', "results.csv", "history.csv")
    except:
        st.write("running locally")

@@ -36,10 +36,10 @@ def get_secret():
 # get_secret()


-def save_to_sf(training_results_file, attempt_logs_file):
+def save_to_sf(dag_name, training_results_file, attempt_logs_file):

    mwaa_env_name = 'doczy-dev-infra-mwaa'
-    dag_name = 'load_training_results'
+    dag_name = dag_name
    mwaa_cli_command = 'dags trigger'

    # Create the client with the specified profile