jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

save_to_df testing

Browse Source
This commit is contained in:
Mayank Aamseek
2024-04-09 16:46:41 +05:30
parent 95f2ad05bd
commit 380ea3b2f3
1 changed files with 19 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
streamlit/sf_conn.py
+19 -23
View File
@@ -63,32 +63,28 @@ def save_to_sf(dag_name, **kwargs):
# training_results_file = "training_results_sample.csv"
# attempt_logs_file = "attempt_logs_sample.csv"
# conf = "{\"" + "training_results_file_name" + "\":\"" + {training_results_file} + "\", \"" + "attempt_logs_file_name" + "\":\"" + {attempt_logs_file} + "\"}".format(training_results_file=training_results_file, attempt_logs_file=attempt_logs_file)
try:
conf1 = f"""{{
"{list(kwargs.keys())[0]}": "{list(kwargs.values())[0]}",
"{list(kwargs.keys())[1]}": "{list(kwargs.values())[1]}"
}}"""
st.write(conf1)
conf = f"""{{
{json.dumps(kwargs)}
}}"""
st.write(conf)
conf = f"""{{
"{list(kwargs.keys())[0]}": "{list(kwargs.values())[0]}",
"{list(kwargs.keys())[1]}": "{list(kwargs.values())[1]}"
}}"""
st.write(conf)
payload = mwaa_cli_command + " " + dag_name + " --conf '{}'".format(conf)
headers = {
'Authorization': 'Bearer ' + mwaa_cli_token['CliToken'],
'Content-Type': 'text/plain'
}
conn.request("POST", "/aws_mwaa/cli", payload, headers)
res = conn.getresponse()
data = res.read()
dict_str = data.decode("UTF-8")
mydata = ast.literal_eval(dict_str)
return base64.b64decode(mydata['stdout'])
except Exception as e:
return e
conf = f"""{{
{json.dumps(kwargs)}
}}"""
payload = mwaa_cli_command + " " + dag_name + " --conf '{}'".format(conf)
headers = {
'Authorization': 'Bearer ' + mwaa_cli_token['CliToken'],
'Content-Type': 'text/plain'
}
conn.request("POST", "/aws_mwaa/cli", payload, headers)
res = conn.getresponse()
data = res.read()
dict_str = data.decode("UTF-8")
mydata = ast.literal_eval(dict_str)
return base64.b64decode(mydata['stdout'])
# save_to_sf("2024-03-13T17-36_prompt_results.csv", "2024-03-14T23-19_history.csv")