jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Fixes to Snowflake Connections, API endpoints and UI 1

Browse Source
This commit is contained in:
AARETE\agupta
2024-06-20 16:25:26 -05:00
parent babf0a1228
commit af3387bf1a
10 changed files with 89 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
streamlit/interface_1.py
+36 -41
View File
@@ -14,10 +14,10 @@ from sf_conn import get_client_names, get_secret, save_to_sf
from constants import USER_LIST
from util import logger
(redirect_uri, create_batch_url, doczy_pipeline) = util.load_page_details(1)
(REDIRECT_URI, create_batch_url, doczy_pipeline) = util.load_page_details(1)
user_list = USER_LIST
st.set_page_config(layout="wide")
st.set_page_config(layout = "wide")
# Sidebar contents
with st.sidebar:
st.title("Doczy.AI ™")
@@ -25,14 +25,14 @@ with st.sidebar:
"""
## About
This app extracts data from contracts
"""
)
add_vertical_space(15)
# st.write("Doczy")
# AARETE LOGO
x, y, z = st.columns([15, 2, 15])
x,y,z = st.columns([15,2,15])
with y:
st.image('aaretelogo.png')
@@ -44,9 +44,9 @@ button[title="View fullscreen"]{
'''
st.markdown(hide_img_fs, unsafe_allow_html=True)
_, c1 = st.columns([5, 1])
_,c1= st.columns([5,1])
try:
util.setup_page(redirect_uri)
util.setup_page(REDIRECT_URI)
except:
st.write("SSO Failed")
st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'}
@@ -57,13 +57,15 @@ except KeyError as e:
# Do we add a link to get to the login page here?
st.write("Session Expired.")
# st.write("Please sign-in to use this app.")
auth_url = security.get_auth_url(redirect_uri)
auth_url = security.get_auth_url(REDIRECT_URI)
st.markdown(f"<a href='{auth_url}' target='_self'>Sign In</a>", unsafe_allow_html=True)
st.stop()
s3_client = boto3.client('s3',
region_name="us-east-2",
)
region_name="us-east-2",
)
# # to be replaced with snowflake data
# client_list = ['doczy-ai-client-1', 'Delaware First Health, Inc.', 'Community Health Choice, Inc','CareSource Network Partners LLC',
@@ -71,24 +73,21 @@ s3_client = boto3.client('s3',
# 'WellCare New Jersey']
client_list, s3_paths = get_client_names()
logger.info(f"client_list={client_list}")
logger.info(f"s3_paths={s3_paths}")
client_s3_paths = dict(zip(client_list, s3_paths))
client_row = st.columns([0.1, 0.8])
with client_row[0]:
st.write("**Client Name**")
with client_row[1]:
client = st.selectbox('Client Name', (client_list), label_visibility="collapsed",
index=None) # MODIFIED for Ticket DOC-344
client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed", index = None)
client_bucket = client_s3_paths.get(client)
# to be deleted when buckets for different clients are ready; below line is added only for testing the corresponding DAG
client_bucket = 'doczy-dev-infra-textract'
client_bucket = 'doczyai-use2-d-cn1-s3-textract-processing-001'
batch_objects = s3_client.list_objects_v2(Bucket=client_bucket
, Prefix="contracts-landing-zone/", Delimiter='/')
, Prefix="contracts-landing-zone/", Delimiter='/')
batch_list = []
for prefix in batch_objects['CommonPrefixes']:
@@ -102,7 +101,6 @@ for prefix in batch_objects['CommonPrefixes']:
if 'sorted_list' not in st.session_state:
st.session_state.sorted_list = batch_list
def sort_list(ex_list, sort_by, order):
if sort_by == 'Alphabetical':
ex_list = sorted(ex_list, reverse=(order == 'Descending'))
@@ -111,16 +109,16 @@ def sort_list(ex_list, sort_by, order):
return ex_list
col1, col2, col3, col4 = st.columns([0.5, 0.5, 0.5, 0.5])
with col1:
with col1:
sort_by = st.radio("**Sort Batch_IDs**", ('Alphabetical', 'Create Date'))
with col2:
order = st.radio('', ('Ascending', 'Descending'))
order = st.radio('', ('Ascending','Descending'))
with col3:
with col3:
add_vertical_space(2)
if st.button('Apply'):
st.session_state.sorted_list = sort_list(batch_list, sort_by, order)
@@ -129,8 +127,7 @@ path_row = st.columns([0.1, 0.8])
with path_row[0]:
st.write("**Batch ID**")
with path_row[1]:
batch_id = st.selectbox('**Batch ID**', st.session_state.sorted_list, label_visibility="collapsed",
index=None) # MODIFIED for Ticket DOC-344
batch_id = st.selectbox('**Batch ID**', st.session_state.sorted_list, label_visibility = "collapsed", index = None)
if not batch_id:
batch_id = "None"
@@ -140,37 +137,37 @@ with checks[0]:
st.write("**Group No.**")
with checks[1]:
a = st.checkbox('Unique Key', key=str(1), args="Unique")
a = st.checkbox('Unique Key', key = str(1), args="Unique")
with checks[2]:
b = st.checkbox('Pricing Before Carveouts', key=str(2))
b = st.checkbox('Pricing Before Carveouts', key = str(2))
with checks[3]:
c = st.checkbox('Contract Related', key=str(3))
c = st.checkbox('Contract Related', key = str(3))
with checks[4]:
d = st.checkbox('Provider', key=str(4))
d = st.checkbox('Provider', key = str(4))
with checks[5]:
e = st.checkbox('Timeline', key=str(5))
e = st.checkbox('Timeline', key = str(5))
with checks[6]:
f = st.checkbox('Carveout Indicator', key=str(6))
f = st.checkbox('Carveout Indicator', key = str(6))
with checks[7]:
g = st.checkbox('Carveout Methodology', key=str(7))
g = st.checkbox('Carveout Methodology', key = str(7))
add_vertical_space(1)
df = pd.DataFrame(columns=['Contract Name', 'Unique Key', 'Pricing Before Carveouts'
df = pd.DataFrame(columns=['Contract Name', 'Unique Key','Pricing Before Carveouts'
, 'Contract Related', 'Provider', 'Timeline', 'Carveout Indicator', 'Carveout Methodology'])
file_list = []
file_objects = s3_client.list_objects_v2(Bucket=client_bucket
, Prefix="contracts-landing-zone/" + batch_id + "/", Delimiter='/')
, Prefix="contracts-landing-zone/"+batch_id+"/", Delimiter='/')
# Hardcoded file_list for testing purposes
# file_list = ['Boilerplate_TX Amendment Mission Health Network effective_040114 MU.pdf', 'Custom_TX - MP AMENDMENT - MISSION HEALTH NETWORK - MU.pdf',
# 'Delaware First Health_First State Homecare Agency_212260_7 MU.pdf', 'Molina Healthcare of Texas, Inc. Amendment 4 - HIX ACA__EFF 01012016_MU.pdf']
if st.button("Read the contracts from Path"):
for obj in file_objects.get('Contents', []):
for obj in file_objects.get('Contents',[]):
if not obj['Key'].endswith('/'):
file_list.append(obj['Key'].split('/')[-1])
df['Contract Name'] = file_list
# df['Request ID'] = range(len(file_list))
# df['Contract ID'] = file_list
@@ -194,20 +191,17 @@ edited_df['LATEST_FLAG BOOLEAN'] = True
edited_df['PIPELINE_KICKOFF_DATETIME'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
edited_df['REQUEST_DATETIME'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
@st.cache_data
@st.cache_data
def convert_df(df):
return df.to_csv(index=False).encode('utf-8')
csv = convert_df(edited_df)
# edited_df = edited_df.reset_index() # make sure indexes pair with number of rows
# additional_info = pd.DataFrame(columns=['REQUEST_ID','T_DRIVE_PATH','CLIENT_NAME'
# , 'GROUP_NAME', 'REQUEST_USERNAME', 'REQUEST_DATETIME'])
additional_info = pd.DataFrame(columns=['CLIENT_NAME', 'BATCH_ID', 'REQUEST_USERNAME', 'REQUEST_DATETIME'])
additional_info.loc[0] = [client, batch_id, st.session_state.user_info['mail'],
datetime.now().strftime("%Y-%m-%d %H:%M:%S")]
additional_info.loc[0] = [client, batch_id, st.session_state.user_info['mail'], datetime.now().strftime("%Y-%m-%d %H:%M:%S")]
st.write(additional_info)
st.session_state.contract_count = 0
@@ -240,7 +234,7 @@ for index, row in edited_df.iterrows():
entry_dict = {
"contract_name": row['Contract Name'],
"groups": group_list,
"contract_source_path": "contracts_landing_zone/" + batch_id + "/" + row['Contract Name']
"contract_source_path": "contracts-landing-zone/"+batch_id+"/"+row['Contract Name']
}
contract_list.append(entry_dict)
@@ -260,7 +254,7 @@ with buttons[1]:
if not st.session_state.contract_count == len(edited_df):
st.error("Select at least one Group No. for every Contract")
else:
with st.spinner('Running...'):
with st.spinner('Running...'):
# csv_buf = StringIO()
# additional_info.to_csv(csv_buf, header=True, index=False)
# csv_buf.seek(0)
@@ -273,10 +267,11 @@ with buttons[1]:
# save_to_sf('load_request_and_contract_submissions', request_submission_file_name = "request_submission.csv", contract_config_file_name = "contract_config.csv")
# except Exception as e:
# st.write(e)
response = requests.post(doczy_pipeline, json=myobj)
response = requests.post(doczy_pipeline, json = myobj)
if response.status_code >= 200 and response.status_code < 300:
st.write("Success")
# st.write(myobj)
else:
st.write("Failed")
# st.write(response.text)