jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

load history and raw data from sf

Browse Source
This commit is contained in:
Mayank Aamseek
2024-03-27 13:24:33 +05:30
parent 860d04cd73
commit e87713bbd2
1 changed files with 46 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
streamlit/interface_3.py
+46 -17
View File
@@ -20,7 +20,8 @@ import util
import anthropic
import re
import snowflake.connector
from sf_conn import get_secret
from sf_conn import get_secret, save_to_sf
from io import StringIO
REDIRECT_URI = 'https://doczy.aarete.com:8503'
@@ -73,7 +74,7 @@ except:
field_values = pd.read_csv('contract_field_values.csv', encoding='unicode_escape', skipinitialspace=True)
field_values.rename(columns={'(internal) Document Name': 'Document_Name'}, inplace = True)
# field_values.rename(columns={'(Internal) Carveout ID': 'Contract ID'}, inplace = True)
st.write("sf conn failed")
st.write("Local copy of TRAINING_DATA_RAW table loaded")
try:
query = 'select * from "BUSINESS_CONFIG"'
@@ -91,8 +92,21 @@ except:
fields = pd.read_csv('contract_fields.csv', encoding='unicode_escape', skipinitialspace=True)
fields = fields.drop_duplicates(subset='Field Name', keep="first").sort_values('Field Name')
fields = fields[~fields['Field Name'].isnull()]
st.write("sf conn failed 2")
st.write("Local copy of BUSINESS_CONFIG table loaded")
try:
query = 'select * from "TRAINING_ATTEMPT_LOGS"'
cur.execute(query)
history = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description])
history.rename(columns={'FIELD_NAME': 'Field Name'}, inplace = True)
history.rename(columns={'CONTRACTS_TESTED': '# Contracts Tested'}, inplace = True)
history.rename(columns={'USERNAME': 'Username'}, inplace = True)
history.rename(columns={'DATE_TIME': 'Date/Time'}, inplace = True)
history.rename(columns={'ACCURACY': 'Accuracy'}, inplace = True)
history.rename(columns={'ATTEMPT_NUM': 'Attempt #'}, inplace = True)
except:
history = pd.read_csv('history.csv')
st.write("Local copy of TRAINING_ATTEMPT_LOGS table loaded")
if user_mail in user_list:
@@ -256,16 +270,12 @@ if user_mail in user_list:
question_with_schema = question
attempt = 0
def run_llm(attempt, bucket, contract_list, llm_selected, field_values):
def run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, history):
# df = pd.DataFrame(columns=['Contract Name','Contract ID','Actual Value Stored','New Extracted value','Confidence Level','Snippet',
# 'Original Page Number', 'New Page Number', 'Revised Prompt', 'Result'])
df = pd.DataFrame(columns=['Contract Name' ,'New Extracted value','Confidence Level','Snippet','New Page Number'
, 'Revised Prompt', 'Result'])
try:
history = pd.read_csv('history.csv')
except:
history = pd.DataFrame(columns=['Field Name','# Contracts Tested', 'Username', 'Date/Time', 'Accuracy', 'Attempt #'])
field_list = []
answer_list = []
@@ -498,22 +508,36 @@ if user_mail in user_list:
except:
accuracy = 'NA'
if mode == 'Multiple fields':
field = field_group
history.loc[len(history.index)] = [field, str(contract_count), None, datetime.now().strftime("%Y-%m-%d %H:%M:%S"), accuracy, attempt]
# df.to_csv("RESULTS\\"+field.replace("?","").replace("/","_")+'-'+llm_selected+'.csv', index=False)
return df, history, attempt, raw_response_text
raw_response_text = ''
if st.button("Test Configuration"):
df, history, attempt, raw_response_text = run_llm(attempt, bucket, contract_list, llm_selected, field_values)
df.to_csv('results.csv', index=False)
history.to_csv('history.csv', index=False)
df = pd.DataFrame(columns=['Contract Name' ,'New Extracted value','Confidence Level','Snippet','New Page Number'
, 'Revised Prompt', 'Result'])
# df_copy = df.set_index(df.columns[0]).copy()
# df_2_copy = history.set_index(history.columns[0]).copy()
df = pd.read_csv('results.csv')
df['Result'] = df['Result'].astype('str')
history = pd.read_csv('history.csv')
if st.button("Test Configuration"):
df, history, attempt, raw_response_text = run_llm(attempt, bucket, contract_list, llm_selected, field, field_values, history)
df = df[['Contract Name','Contract ID', 'Actual Value Stored','New Extracted value','Confidence Level'
,'Snippet','Original Page Number', 'New Page Number', 'Revised Prompt', 'Result']]
# df.to_csv('results.csv', index=False)
# history.to_csv('history.csv', index=False)
# s3_client.upload_file('results.csv', bucket, key)
csv_buf = StringIO()
df.to_csv(csv_buf, header=True, index=False)
csv_buf.seek(0)
s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/results.csv')
csv_buf = StringIO()
history.to_csv(csv_buf, header=True, index=False)
csv_buf.seek(0)
s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/history.csv')
# df = pd.read_csv('results.csv')
# df['Result'] = df['Result'].astype('str')
# history = pd.read_csv('history.csv')
st.dataframe(df)
st.dataframe(history)
@@ -537,6 +561,11 @@ if user_mail in user_list:
st.write(len(contract_list))
st.write(raw_response_text)
try:
save_to_sf("results.csv", "history.csv")
except:
st.write("running locally")
else:
st.write("Access Denied")