jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

run doczy.ai pipeline updated

Browse Source
This commit is contained in:
Mayank Aamseek
2024-04-02 17:11:56 +05:30
parent 4dd4a44546
commit 0ee9d8eb24
4 changed files with 66 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
streamlit/interface_0.py
-7
View File
@@ -91,12 +91,6 @@ if user_mail in user_list:
st.write(f"{batch_id} created")
st.write(f"Files uploaded to s3://{client}/{batch_id}/{landing_zone}")
try:
save_to_sf('load_request_and_contract_submissions', "request_submission_file_name", file_names[0]
, "contract_config_file_name", file_names[0])
except:
st.write("running locally")
# @st.cache_data
# def convert_df(df):
@@ -152,7 +146,6 @@ if user_mail in user_list:
# # st.write(response.text)
else:
st.write("Access Denied")
streamlit/interface_1.py
+62 -63
View File
@@ -30,7 +30,7 @@ st.set_page_config(layout = "wide")
_,c1= st.columns([5,1])
try:
util.setup_page(REDIRECT_URI)
putil.setup_page(REDIRECT_URI)
except:
st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'}
c1.write(f"User: **{st.session_state.user_info['displayName']}**")
@@ -41,15 +41,9 @@ if user_mail in user_list:
s3_client = boto3.client('s3',
region_name="us-east-1",
)
objects = s3_client.list_objects_v2(Bucket='doczy-dev-infra-textract'
, Prefix="batches/batch_1/", Delimiter='/')
folder_list = []
for prefix in objects['CommonPrefixes']:
folder_list.append(prefix['Prefix'][:-1].split('/')[-1])
# to be deleted later
folder_list = ['Delaware First Health, Inc.', 'Community Health Choice, Inc','CareSource Network Partners LLC',
# to be replaced with snowflake data
client_list = ['doczy-ai-client-1', 'Delaware First Health, Inc.', 'Community Health Choice, Inc','CareSource Network Partners LLC',
'HealthNet of Cali', 'Oklahoma Complete Health, Inc', 'HealthFirst', 'Molina Healthcare of TX', 'AvMed', 'Arizona Care1st',
'WellCare New Jersey']
@@ -57,23 +51,20 @@ if user_mail in user_list:
with client_row[0]:
st.write("**Client Name**")
with client_row[1]:
client = st.selectbox('Client Name',(folder_list), label_visibility = "collapsed")
client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed")
# to be deleted later
client = 'textract-receiver-processed-file'
batch_objects = s3_client.list_objects_v2(Bucket=client
, Prefix="", Delimiter='/')
folder_objects = s3_client.list_objects_v2(Bucket='doczy-dev-infra-textract'
, Prefix="batches/batch_1/"+client+"/", Delimiter='/')
folder_list_2 = []
for prefix in folder_objects['CommonPrefixes']:
folder_list_2.append(prefix['Prefix'][:-1].split('/')[-1])
batch_list = []
for prefix in batch_objects['CommonPrefixes']:
batch_list.append(prefix['Prefix'][:-1].split('/')[-1])
path_row = st.columns([0.1, 0.8])
with path_row[0]:
st.write("**Path to folder**")
st.write("**Batch ID**")
with path_row[1]:
Directory = st.selectbox('**Path to folder**', folder_list_2, label_visibility = "collapsed")
batch_id = st.selectbox('**Batch ID**', batch_list, label_visibility = "collapsed")
checks = st.columns([0.1, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12])
with checks[0]:
@@ -98,8 +89,8 @@ if user_mail in user_list:
df = pd.DataFrame(columns=['Contract Name','Unique Key','Pricing Before Carveouts'
, 'Contract Related', 'Provider', 'Timeline', 'Carveout Indicator', 'Carveout Methodology'])
file_list = []
file_objects = s3_client.list_objects_v2(Bucket='doczy-dev-infra-textract'
, Prefix="batches/batch_1/"+client+"/"+Directory+"/", Delimiter='/')
file_objects = s3_client.list_objects_v2(Bucket=client
, Prefix=batch_id+"/contracts_landing_zone/", Delimiter='/')
if st.button("Read the contracts from Path"):
for obj in file_objects.get('Contents',[]):
@@ -121,64 +112,72 @@ if user_mail in user_list:
df.to_csv('temp1.csv', index=False)
add_vertical_space(1)
# df_copy = df.set_index(df.columns[0]).copy()
df2 = pd.read_csv('temp1.csv')
edited_df = st.data_editor(df2)
@st.cache_data
def convert_df(df):
return df.to_csv(index=False).encode('utf-8')
csv = convert_df(edited_df)
edited_df = edited_df.reset_index() # make sure indexes pair with number of rows
contract_list = []
for index, row in edited_df.iterrows():
group_list = []
if row['Unique Key']:
group_list.append('Unique Key')
if row['Pricing Before Carveouts']:
group_list.append('Pricing Before Carveouts')
if row['Contract Related']:
group_list.append('Contract Related')
if row['Provider']:
group_list.append('Provider')
if row['Timeline']:
group_list.append('Timeline')
if row['Carveout Indicator']:
group_list.append('Carveout Indicator')
if row['Carveout Methodology']:
group_list.append('Carveout Methodology')
entry_dict = {
"contract_name": row['Contract Name'],
"groups": group_list,
"contract_source_path": "batches/batch_1/"+client+"/"+row['Contract Name']
}
contract_list.append(entry_dict)
# additional_info = pd.DataFrame(columns=['REQUEST_ID','T_DRIVE_PATH','CLIENT_NAME'
# , 'GROUP_NAME', 'REQUEST_USERNAME', 'REQUEST_DATETIME'])
additional_info = pd.DataFrame(columns=['CLIENT_NAME', 'BATCH_ID', 'REQUEST_USERNAME', 'REQUEST_DATETIME'])
additional_info.loc[0] = [client, batch_id, st.session_state.user_info['mail'], datetime.now().strftime("%Y-%m-%d %H:%M:%S")]
st.write(additional_info)
myobj = {
"s3_bucket": 'doczy-dev-infra-textract',
"batch_id": "1",
"client_name": client,
"username": user_mail,
"contract_list": contract_list
}
# contract_list = []
# for index, row in edited_df.iterrows():
# group_list = []
# if row['Unique Key']:
# group_list.append('Unique Key')
# if row['Pricing Before Carveouts']:
# group_list.append('Pricing Before Carveouts')
# if row['Contract Related']:
# group_list.append('Contract Related')
# if row['Provider']:
# group_list.append('Provider')
# if row['Timeline']:
# group_list.append('Timeline')
# if row['Carveout Indicator']:
# group_list.append('Carveout Indicator')
# if row['Carveout Methodology']:
# group_list.append('Carveout Methodology')
# entry_dict = {
# "contract_name": row['Contract Name'],
# "groups": group_list,
# "contract_source_path": "batches/batch_1/"+client+"/"+row['Contract Name']
# }
# contract_list.append(entry_dict)
# myobj = {
# "s3_bucket": 'doczy-dev-infra-textract',
# "batch_id": "1",
# "client_name": client,
# "username": user_mail,
# "contract_list": contract_list
# }
buttons = st.columns([0.8, 0.2])
with buttons[0]:
st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv')
with buttons[1]:
if st.button("Run Doczy.AI Pipeline"):
response = requests.post(doczy_pipeline, json = myobj)
if response.status_code >= 200 and response.status_code < 300:
st.write("Success")
else:
st.write("Failed")
# st.write(response.text)
csv_buf = StringIO()
additional_info.to_csv(csv_buf, header=True, index=False)
csv_buf.seek(0)
s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/request_submission.csv')
csv_buf = StringIO()
edited_df.to_csv(csv_buf, header=True, index=False)
csv_buf.seek(0)
s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/contract_config.csv')
try:
save_to_sf('load_request_and_contract_submissions', "request_submission_file_name"
, "contract_config_file_name", "request_submission.csv", "contract_config.csv")
except:
st.write("running locally")
else:
streamlit/interface_3.py
+1 -1
View File
@@ -581,7 +581,7 @@ if st.session_state.user_info['mail'] in user_list:
st.write(raw_response_text)
try:
save_to_sf('load_training_results', "results.csv", "history.csv")
save_to_sf('load_training_results', "training_results_file_name", "attempt_logs_file_name", "results.csv", "history.csv")
except:
st.write("running locally")
streamlit/sf_conn.py
+3 -3
View File
@@ -36,7 +36,7 @@ def get_secret():
# get_secret()
def save_to_sf(dag_name, training_results_file, attempt_logs_file):
def save_to_sf(dag_name, conf_1, conf_2, training_results_file, attempt_logs_file):
mwaa_env_name = 'doczy-dev-infra-mwaa'
dag_name = dag_name
@@ -60,8 +60,8 @@ def save_to_sf(dag_name, training_results_file, attempt_logs_file):
# conf = "{\"" + "training_results_file_name" + "\":\"" + {training_results_file} + "\", \"" + "attempt_logs_file_name" + "\":\"" + {attempt_logs_file} + "\"}".format(training_results_file=training_results_file, attempt_logs_file=attempt_logs_file)
conf = f"""{{
"training_results_file_name": "{training_results_file}",
"attempt_logs_file_name": "{attempt_logs_file}"
"{conf_1}": "{training_results_file}",
"{conf_2}": "{attempt_logs_file}"
}}"""