load history and raw data from sf

streamlit/interface_3.py

@@ -20,7 +20,8 @@ import util
 import anthropic
 import re
 import snowflake.connector
-from sf_conn import get_secret
+from sf_conn import get_secret, save_to_sf
+from io import StringIO
 
 
 REDIRECT_URI = 'https://doczy.aarete.com:8503'
@@ -73,7 +74,7 @@ except:
     field_values = pd.read_csv('contract_field_values.csv', encoding='unicode_escape', skipinitialspace=True)
     field_values.rename(columns={'(internal) Document Name': 'Document_Name'}, inplace = True)
     # field_values.rename(columns={'(Internal) Carveout ID': 'Contract ID'}, inplace = True)
-    st.write("sf conn failed")
+    st.write("Local copy of TRAINING_DATA_RAW table loaded")
 
 try:
     query = 'select * from "BUSINESS_CONFIG"'
@@ -91,8 +92,21 @@ except:
     fields = pd.read_csv('contract_fields.csv', encoding='unicode_escape', skipinitialspace=True)
     fields = fields.drop_duplicates(subset='Field Name', keep="first").sort_values('Field Name')
     fields = fields[~fields['Field Name'].isnull()]
-    st.write("sf conn failed 2")
+    st.write("Local copy of BUSINESS_CONFIG table loaded")
 
+try:
+    query = 'select * from "TRAINING_ATTEMPT_LOGS"'
+    cur.execute(query)
+    history = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description])
+    history.rename(columns={'FIELD_NAME': 'Field Name'}, inplace = True)
+    history.rename(columns={'CONTRACTS_TESTED': '# Contracts Tested'}, inplace = True)
+    history.rename(columns={'USERNAME': 'Username'}, inplace = True)
+    history.rename(columns={'DATE_TIME': 'Date/Time'}, inplace = True)
+    history.rename(columns={'ACCURACY': 'Accuracy'}, inplace = True)
+    history.rename(columns={'ATTEMPT_NUM': 'Attempt #'}, inplace = True)
+except:
+    history = pd.read_csv('history.csv')
+    st.write("Local copy of TRAINING_ATTEMPT_LOGS table loaded")
 
 if user_mail in user_list: 
 
@@ -256,16 +270,12 @@ if user_mail in user_list:
     question_with_schema = question
     attempt = 0
         
-    def run_llm(attempt, bucket, contract_list, llm_selected, field_values):
+    def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, history):
 
         # df = pd.DataFrame(columns=['Contract Name','Contract ID','Actual Value Stored','New Extracted value','Confidence Level','Snippet',
         #     'Original Page Number', 'New Page Number', 'Revised Prompt', 'Result'])
         df = pd.DataFrame(columns=['Contract Name' ,'New Extracted value','Confidence Level','Snippet','New Page Number'
             , 'Revised Prompt', 'Result'])
-        try:
-            history = pd.read_csv('history.csv')
-        except:
-            history = pd.DataFrame(columns=['Field Name','# Contracts Tested', 'Username', 'Date/Time', 'Accuracy', 'Attempt #'])
 
         field_list = []
         answer_list = []
@@ -498,22 +508,36 @@ if user_mail in user_list:
         except:
             accuracy = 'NA'
 
+        if mode == 'Multiple fields':
+            field = field_group
         history.loc[len(history.index)] = [field, str(contract_count), None, datetime.now().strftime("%Y-%m-%d %H:%M:%S"), accuracy, attempt]
         # df.to_csv("RESULTS\\"+field.replace("?","").replace("/","_")+'-'+llm_selected+'.csv', index=False)
 
         return df, history, attempt, raw_response_text
 
     raw_response_text = ''
-    if st.button("Test Configuration"):
+    df = pd.DataFrame(columns=['Contract Name' ,'New Extracted value','Confidence Level','Snippet','New Page Number'
-        df, history, attempt, raw_response_text = run_llm(attempt, bucket, contract_list, llm_selected, field_values)
+    , 'Revised Prompt', 'Result'])
-        df.to_csv('results.csv', index=False)
-        history.to_csv('history.csv', index=False)
 
-    # df_copy = df.set_index(df.columns[0]).copy()
+    if st.button("Test Configuration"):
-    # df_2_copy = history.set_index(history.columns[0]).copy()
+        df, history, attempt, raw_response_text = run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, history)
-    df = pd.read_csv('results.csv')
+        df = df[['Contract Name','Contract ID', 'Actual Value Stored','New Extracted value','Confidence Level'
-    df['Result'] = df['Result'].astype('str')
+        ,'Snippet','Original Page Number', 'New Page Number', 'Revised Prompt', 'Result']]
-    history = pd.read_csv('history.csv')
+        # df.to_csv('results.csv', index=False)
+        # history.to_csv('history.csv', index=False)
+        # s3_client.upload_file('results.csv', bucket, key)
+        csv_buf = StringIO()
+        df.to_csv(csv_buf, header=True, index=False)
+        csv_buf.seek(0)
+        s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/results.csv')
+        csv_buf = StringIO()
+        history.to_csv(csv_buf, header=True, index=False)
+        csv_buf.seek(0)
+        s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/history.csv')
+
+    # df = pd.read_csv('results.csv')
+    # df['Result'] = df['Result'].astype('str')
+    # history = pd.read_csv('history.csv')
     st.dataframe(df)
     st.dataframe(history)
 
@@ -537,6 +561,11 @@ if user_mail in user_list:
     st.write(len(contract_list))
     st.write(raw_response_text)
 
+    try:
+        save_to_sf("results.csv", "history.csv")
+    except:
+        st.write("running locally")
+
 else:
     st.write("Access Denied")