Merge branch 'DEV' into feature/terraform-refactor
This commit is contained in:
Grzegorz Huber
+29
-11
@@ -200,31 +200,49 @@ pipelines:
|
||||
name: "Plan & Apply DevOps Pipeline"
|
||||
image: hugree/bitbucket-aws-python38-tf154:latest
|
||||
oidc: true
|
||||
trigger : automatic # Keeping it manual so that it doesnt conflict with existing resources
|
||||
trigger : automatic
|
||||
script:
|
||||
- *aws-context-uat
|
||||
- cd devops-pipeline/terraform-backend-resources/
|
||||
- terraform init
|
||||
- terraform validate
|
||||
# Ensure the backend configuration for s3 obj key is correct
|
||||
- terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/devops-pipeline/terraform.tfstate"
|
||||
- terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat"
|
||||
- terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat"- echo "S3_BUCKET=$(terraform output -raw terraform_s3_bucket_name)" >> backend.env
|
||||
- echo "DYNAMODB_TABLE=$(terraform output -raw terraform_dynamodb_table_name)" >> backend.env
|
||||
artifacts:
|
||||
- backend.env
|
||||
condition:
|
||||
changesets:
|
||||
includePaths:
|
||||
- devops-pipeline/terraform-backend-resources/*
|
||||
- devops-pipeline/terraform-backend-resources/**/*
|
||||
- devops-pipeline/terraform-backend-resources/**/*- step:
|
||||
name: "Plan & Apply DevOps Extra Resource Pipeline"
|
||||
image: hugree/bitbucket-aws-python38-tf154:latest
|
||||
oidc: true
|
||||
trigger : automatic # Keeping it manual so that it doesnt conflict with existing resources
|
||||
script:
|
||||
- *aws-context-uat
|
||||
- cd devops-pipeline/other-resources/
|
||||
- terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/devops-pipeline-other-resources/terraform.tfstate"
|
||||
- terraform validate
|
||||
- terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat"
|
||||
condition:
|
||||
changesets:
|
||||
includePaths:
|
||||
- devops-pipeline/other-resources/*
|
||||
- devops-pipeline/other-resources/**/*
|
||||
- step:
|
||||
name: "Plan & Apply Streamlit Pipeline"
|
||||
image: hugree/bitbucket-aws-python38-tf154:latest
|
||||
oidc: true
|
||||
trigger : manual
|
||||
trigger : automatic
|
||||
script:
|
||||
- *aws-context-uat
|
||||
- cd streamlit-server/
|
||||
- terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/streamlit-server/terraform.tfstate"
|
||||
- terraform validate
|
||||
# Ensure the backend configuration for s3 obj key is correct
|
||||
- terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/streamlit-server/terraform.tfstate"
|
||||
- terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat"
|
||||
- terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat"-var "vpc_id=vpc-0392396d0e7bdd77f"
|
||||
condition:
|
||||
changesets:
|
||||
includePaths:
|
||||
@@ -242,12 +260,12 @@ pipelines:
|
||||
image: python:3.8
|
||||
name: "Deploy streamlit to EC2"
|
||||
oidc: true
|
||||
trigger: manual # Keeping it manual as the server will be created first and then referenced in this
|
||||
trigger: automatic # Keeping it manual as the server will be created first and then referenced in this
|
||||
script:
|
||||
- python -m pip install --upgrade pip
|
||||
- apt-get update && apt-get install -y jq git
|
||||
- pip install awscli
|
||||
- *aws-context-dev
|
||||
- *aws-context-uat
|
||||
- *ssm-send-command-uat
|
||||
condition:
|
||||
changesets:
|
||||
@@ -269,10 +287,10 @@ pipelines:
|
||||
script:
|
||||
- *aws-context-uat
|
||||
- cd textract-pipeline/terraform/
|
||||
- terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/textract-pipeline/terraform.tfstate"
|
||||
- terraform validate
|
||||
# Ensure the backend configuration for s3 obj key is corresponding to the stream name
|
||||
- terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/textract-pipeline/terraform.tfstate"
|
||||
- terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat"
|
||||
- terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat"
|
||||
condition:
|
||||
changesets:
|
||||
includePaths:
|
||||
|
||||
@@ -53,6 +53,7 @@ variable "mwaa_resources_s3_bucket" {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
variable "aws_account_id" {
|
||||
type = string
|
||||
}
|
||||
@@ -8,7 +8,7 @@ terraform {
|
||||
}
|
||||
|
||||
provider "aws" {
|
||||
region = var.aws_region
|
||||
region = var.aws_region
|
||||
}
|
||||
|
||||
module "aws_terraform_remote_backend" {
|
||||
|
||||
+14
@@ -10,6 +10,20 @@ variable "environment" {
|
||||
variable "client_name" {
|
||||
type = string
|
||||
}
|
||||
# required
|
||||
variable "secret_key" {
|
||||
type = string
|
||||
}
|
||||
|
||||
# required
|
||||
variable "access_key" {
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "token" {
|
||||
type = string
|
||||
}
|
||||
|
||||
# dynamoDB table
|
||||
variable "terraform_dynamodb_table" {
|
||||
type = object({
|
||||
|
||||
@@ -9,4 +9,4 @@ output "terraform_s3_bucket_arn" {
|
||||
}
|
||||
output "terraform_s3_bucket_name" {
|
||||
value = module.aws_terraform_remote_backend.terraform_s3_bucket_name
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,4 +16,19 @@ variable "environment" {
|
||||
variable "client_name" {
|
||||
type = string
|
||||
default = "infra"
|
||||
}
|
||||
}
|
||||
|
||||
#required
|
||||
variable "secret_key" {
|
||||
type = string
|
||||
}
|
||||
|
||||
#required
|
||||
variable "access_key" {
|
||||
type = string
|
||||
}
|
||||
|
||||
#required
|
||||
variable "token" {
|
||||
type = string
|
||||
}
|
||||
|
||||
@@ -51,7 +51,7 @@ BEGIN
|
||||
batch_id := (SELECT NULLIF(TRIM($2),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1);
|
||||
request_datetime := (SELECT NULLIF(TRIM($13),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1);
|
||||
|
||||
UPDATE TABLE STG.CONTRACT_CONFIG
|
||||
UPDATE STG.CONTRACT_CONFIG
|
||||
SET LATEST_FLAG = FALSE
|
||||
WHERE BATCH_ID = :batch_id AND CONTRACT_NAME = :contract_name AND REQUEST_DATETIME < :request_datetime AND LATEST_FLAG = TRUE;
|
||||
|
||||
|
||||
@@ -8,4 +8,4 @@ create or replace TABLE STG.LOG_AUDIT (
|
||||
SUB_SECTION_NAME VARCHAR(255) COLLATE 'en-ci',
|
||||
AUDIT_SID NUMBER(38,0),
|
||||
TOTAL_MIN NUMBER(38,0)
|
||||
);
|
||||
);
|
||||
|
||||
@@ -50,8 +50,8 @@ BEGIN
|
||||
contract_name := (SELECT NULLIF(TRIM($1),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1);
|
||||
batch_id := (SELECT NULLIF(TRIM($2),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1);
|
||||
request_datetime := (SELECT NULLIF(TRIM($13),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1);
|
||||
|
||||
UPDATE TABLE STG.CONTRACT_CONFIG
|
||||
|
||||
UPDATE STG.CONTRACT_CONFIG
|
||||
SET LATEST_FLAG = FALSE
|
||||
WHERE BATCH_ID = :batch_id AND CONTRACT_NAME = :contract_name AND REQUEST_DATETIME < :request_datetime AND LATEST_FLAG = TRUE;
|
||||
|
||||
|
||||
@@ -115,7 +115,8 @@ resource "aws_lb_listener" "https_8500" {
|
||||
protocol = "HTTPS"
|
||||
ssl_policy = "ELBSecurityPolicy-2016-08"
|
||||
# Using variable for ARN
|
||||
certificate_arn = var.acm_arn_dev
|
||||
certificate_arn = lookup(var.acm_arns, var.environment)
|
||||
|
||||
|
||||
default_action {
|
||||
type = "forward"
|
||||
@@ -128,7 +129,7 @@ resource "aws_lb_listener" "https_8501" {
|
||||
port = "8501"
|
||||
protocol = "HTTPS"
|
||||
ssl_policy = "ELBSecurityPolicy-2016-08"
|
||||
certificate_arn = var.acm_arn_dev
|
||||
certificate_arn = lookup(var.acm_arns, var.environment)
|
||||
|
||||
default_action {
|
||||
type = "forward"
|
||||
@@ -141,7 +142,7 @@ resource "aws_lb_listener" "https_8502" {
|
||||
port = "8502"
|
||||
protocol = "HTTPS"
|
||||
ssl_policy = "ELBSecurityPolicy-2016-08"
|
||||
certificate_arn = var.acm_arn_dev
|
||||
certificate_arn = lookup(var.acm_arns, var.environment)
|
||||
|
||||
default_action {
|
||||
type = "forward"
|
||||
@@ -154,7 +155,7 @@ resource "aws_lb_listener" "https_8503" {
|
||||
port = "8503"
|
||||
protocol = "HTTPS"
|
||||
ssl_policy = "ELBSecurityPolicy-2016-08"
|
||||
certificate_arn = var.acm_arn_dev
|
||||
certificate_arn = lookup(var.acm_arns, var.environment)
|
||||
|
||||
default_action {
|
||||
type = "forward"
|
||||
|
||||
@@ -81,13 +81,13 @@ resource "aws_iam_instance_profile" "ec2_instance_profile" {
|
||||
}
|
||||
|
||||
resource "aws_instance" "streamlit_server" {
|
||||
ami = var.ubuntu_ami
|
||||
ami = lookup(var.ubuntu_ami, var.environment)
|
||||
instance_type = var.ec2_instance_type
|
||||
subnet_id = data.aws_subnets.subnets.ids[0]
|
||||
vpc_security_group_ids = data.aws_security_groups.security_groups.ids
|
||||
iam_instance_profile = aws_iam_instance_profile.ec2_instance_profile.name
|
||||
associate_public_ip_address = false
|
||||
key_name = "tf-test-key" # Created using TF and uploaded to S3. Prerequisite
|
||||
key_name = "tf-uat-key" # Created using TF and uploaded to S3. Prerequisite
|
||||
|
||||
root_block_device {
|
||||
volume_size = 30 # Variable
|
||||
@@ -101,7 +101,7 @@ resource "aws_instance" "streamlit_server" {
|
||||
apt-get update
|
||||
apt-get install -y python3 python3-pip
|
||||
sudo apt-get install libffi-dev
|
||||
python3 -m pip3 install -r /tmp/requirements.txt
|
||||
python3 -m pip install -r /tmp/requirements.txt
|
||||
sudo apt install openssh-client
|
||||
sudo apt install net-tools
|
||||
|
||||
@@ -197,3 +197,4 @@ resource "aws_instance" "streamlit_server" {
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -30,10 +30,18 @@ variable "vpc_id" {
|
||||
type = string
|
||||
}
|
||||
|
||||
# variable "ubuntu_ami" {
|
||||
# type = string
|
||||
# default = "ami-0de7e97fedfbc6ef6" # Switch to this ami-0b8b44ec9a8f90422
|
||||
|
||||
# }
|
||||
|
||||
variable "ubuntu_ami" {
|
||||
type = string
|
||||
default = "ami-0de7e97fedfbc6ef6" # Switch to this ami-0b8b44ec9a8f90422
|
||||
|
||||
type = map(string)
|
||||
default = {
|
||||
dev = "ami-0de7e97fedfbc6ef6"
|
||||
uat = "ami-0b8b44ec9a8f90422"
|
||||
}
|
||||
}
|
||||
|
||||
variable "ec2_instance_type" {
|
||||
@@ -52,6 +60,15 @@ variable "acm_arn_dev" {
|
||||
default = "arn:aws:acm:us-east-2:660131068782:certificate/947e04b3-be22-43bf-bea2-a0ad3b221d70"
|
||||
}
|
||||
|
||||
variable "acm_arns" {
|
||||
type = map(string)
|
||||
default = {
|
||||
dev = "arn:aws:acm:us-east-2:660131068782:certificate/947e04b3-be22-43bf-bea2-a0ad3b221d70"
|
||||
uat = "arn:aws:acm:us-east-2:975049960860:certificate/e036e76d-1f4f-496e-ac97-5f65ec4a34cc"
|
||||
prod = "arn:aws:acm:region:account-id:certificate/prod-cert-id"
|
||||
}
|
||||
}
|
||||
|
||||
#variable "secret_key" {
|
||||
# type = string
|
||||
#}
|
||||
|
||||
+60
-30
@@ -18,6 +18,22 @@ create_batch_url = 'https://lfksus2t62.execute-api.us-east-2.amazonaws.com/dev/c
|
||||
|
||||
REDIRECT_URI = 'https://doczydev.aarete.com:8500'
|
||||
user_list = USER_LIST
|
||||
if 'uploading' not in st.session_state:
|
||||
st.session_state.uploading = False
|
||||
|
||||
|
||||
|
||||
def insert_upload_logs(batch_id, client_name, file_name, upload_datetime, upload_user):
|
||||
"""
|
||||
Input: batch_id, client_name, file_name, upload_datetime, upload_user
|
||||
Output: status of the insert query
|
||||
"""
|
||||
try:
|
||||
return 'Log inserted successfully'
|
||||
except Exception as e:
|
||||
return e
|
||||
|
||||
|
||||
st.set_page_config(layout = "wide")
|
||||
# # Sidebar contents
|
||||
# with st.sidebar:
|
||||
@@ -31,6 +47,7 @@ st.set_page_config(layout = "wide")
|
||||
# )
|
||||
# add_vertical_space(15)
|
||||
# # st.write("Doczy")
|
||||
#
|
||||
|
||||
_,c1= st.columns([5,1])
|
||||
try:
|
||||
@@ -45,7 +62,7 @@ except KeyError as e:
|
||||
st.write("Session Expired.")
|
||||
st.stop()
|
||||
|
||||
|
||||
print(st.session_state)
|
||||
|
||||
s3_client = boto3.client('s3',
|
||||
region_name="us-east-2",
|
||||
@@ -65,7 +82,7 @@ client_row = st.columns([0.1, 0.8])
|
||||
with client_row[0]:
|
||||
st.write("**Client Name**")
|
||||
with client_row[1]:
|
||||
client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed")
|
||||
client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344
|
||||
|
||||
client_bucket = client_s3_paths.get(client)
|
||||
|
||||
@@ -76,42 +93,55 @@ file_row = st.columns([0.1, 0.8])
|
||||
with file_row[0]:
|
||||
st.write("**Upload Files**")
|
||||
with file_row[1]:
|
||||
file_list = st.file_uploader("Upload", type=None, accept_multiple_files=True, label_visibility = "collapsed")
|
||||
file_list = st.file_uploader("Upload", type=['docx','tiff','pdf'], accept_multiple_files=True, label_visibility = "collapsed", help="Only PDF, TIFF and DOCX file formats are supported.", disabled=st.session_state.uploading)
|
||||
|
||||
add_vertical_space(2)
|
||||
df = pd.DataFrame(columns=['Contract Name'])
|
||||
df['Contract Name'] = file_list
|
||||
file_names = []
|
||||
buttons = st.columns([0.4, 0.4, 0.2])
|
||||
with buttons[1]:
|
||||
if st.button("Create Batch"):
|
||||
|
||||
myobj = { "client-bucket-name": client_bucket }
|
||||
response = requests.post(create_batch_url, json = myobj)
|
||||
if response.status_code >= 200 and response.status_code < 300:
|
||||
try:
|
||||
batch_id = json.loads(json.loads(response.text)['body'])['batch_id']
|
||||
landing_zone = json.loads(json.loads(response.text)['body'])['landing_zone']
|
||||
except:
|
||||
st.write(myobj)
|
||||
st.write(response.text)
|
||||
batch_id = 'failed_cases'
|
||||
landing_zone = 'contracts_landing_zone'
|
||||
def set_uploading_state():
|
||||
if not client == None and not len(file_list) == 0:
|
||||
st.session_state.uploading = True
|
||||
|
||||
with buttons[1]:
|
||||
if st.button("Create Batch", on_click = set_uploading_state):
|
||||
if client == None:
|
||||
st.error("No Client Name Selected.")
|
||||
elif len(file_list) == 0:
|
||||
st.error("No Files Selected.")
|
||||
else:
|
||||
st.write("Failed")
|
||||
for uploaded_file in file_list:
|
||||
stringio = BytesIO(uploaded_file.getvalue())
|
||||
stringio.seek(0)
|
||||
s3_client.put_object(Bucket=client_bucket, Body=stringio.getvalue(), Key=
|
||||
landing_zone+batch_id+'/'+str(uploaded_file.name))
|
||||
|
||||
# TODO: Test this insert function with snowflake
|
||||
upload_log = insert_upload_logs(batch_id, client, str(uploaded_file.name), datetime.now().strftime("%Y-%m-%d %H:%M:%S"), user_mail)
|
||||
st.write(upload_log)
|
||||
|
||||
file_names.append(str(uploaded_file.name))
|
||||
st.write(f"{batch_id} created")
|
||||
st.write(f"Files uploaded to s3://{client_bucket}/{landing_zone}{batch_id}")
|
||||
myobj = { "client-bucket-name": client_bucket }
|
||||
response = requests.post(create_batch_url, json = myobj)
|
||||
if response.status_code >= 200 and response.status_code < 300:
|
||||
try:
|
||||
batch_id = json.loads(json.loads(response.text)['body'])['batch_id']
|
||||
landing_zone = json.loads(json.loads(response.text)['body'])['landing_zone']
|
||||
except:
|
||||
st.write(myobj)
|
||||
st.write(response.text)
|
||||
batch_id = 'failed_cases'
|
||||
landing_zone = 'contracts_landing_zone'
|
||||
else:
|
||||
st.write("Failed")
|
||||
for uploaded_file in file_list:
|
||||
stringio = BytesIO(uploaded_file.getvalue())
|
||||
stringio.seek(0)
|
||||
s3_client.put_object(Bucket=client_bucket, Body=stringio.getvalue(), Key=
|
||||
landing_zone+batch_id+'/'+str(uploaded_file.name))
|
||||
|
||||
# TODO: Test this insert function with snowflake
|
||||
upload_log = insert_upload_logs(batch_id, client, str(uploaded_file.name), datetime.now().strftime("%Y-%m-%d %H:%M:%S"), user_mail)
|
||||
st.write(upload_log)
|
||||
|
||||
file_names.append(str(uploaded_file.name))
|
||||
st.session_state.uploading = False
|
||||
st.write(f"{batch_id} created")
|
||||
st.write(f"Files uploaded to s3://{client_bucket}/{landing_zone}{batch_id}")
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# @st.cache_data
|
||||
|
||||
+77
-23
@@ -8,6 +8,7 @@ from datetime import datetime
|
||||
import boto3
|
||||
import util
|
||||
import requests
|
||||
import time
|
||||
from sf_conn import get_client_names, get_secret, save_to_sf
|
||||
from constants import USER_LIST, DOCZY_PIPELINE_URL_DEV
|
||||
|
||||
@@ -61,10 +62,11 @@ client_row = st.columns([0.1, 0.8])
|
||||
with client_row[0]:
|
||||
st.write("**Client Name**")
|
||||
with client_row[1]:
|
||||
client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed")
|
||||
client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344
|
||||
|
||||
client_bucket = client_s3_paths.get(client)
|
||||
# # to be deleted when buckets for different clients are ready; below line is added only for testing the corresponding DAG
|
||||
|
||||
# to be deleted when buckets for different clients are ready; below line is added only for testing the corresponding DAG
|
||||
client_bucket = 'doczy-ai-client-1'
|
||||
|
||||
batch_objects = s3_client.list_objects_v2(Bucket=client_bucket
|
||||
@@ -74,17 +76,51 @@ batch_list = []
|
||||
for prefix in batch_objects['CommonPrefixes']:
|
||||
batch_list.append(prefix['Prefix'][:-1].split('/')[-1])
|
||||
|
||||
# Hardcoded batch_list for testing purposes
|
||||
# batch_list = ['batch_020524103737', 'batch_090524131433', 'batch_090524131607', 'batch_100524123000', 'batch_130524064322',
|
||||
# 'batch_160524071331', 'batch_200524213550', 'batch_250424112237', 'batch_280524120530', 'batch_280524121721', 'batch_280524144222',
|
||||
# 'batch_290524123926', 'batch_290524164044', 'batch_310524102029', 'batch_310524124050', 'batch_310524162346', 'batch_310524162631']
|
||||
|
||||
if 'sorted_list' not in st.session_state:
|
||||
st.session_state.sorted_list = batch_list
|
||||
|
||||
def sort_list(ex_list, sort_by, order):
|
||||
if sort_by == 'Alphabetical':
|
||||
ex_list = sorted(ex_list, reverse=(order == 'Descending'))
|
||||
elif sort_by == 'Create Date':
|
||||
ex_list = ex_list if order == 'Ascending' else list(reversed(ex_list))
|
||||
|
||||
return ex_list
|
||||
|
||||
col1, col2, col3, col4 = st.columns([0.5, 0.5, 0.5, 0.5])
|
||||
|
||||
|
||||
with col1:
|
||||
sort_by = st.radio("**Sort Batch_IDs**", ('Alphabetical', 'Create Date'))
|
||||
|
||||
with col2:
|
||||
order = st.radio('', ('Ascending','Descending'))
|
||||
|
||||
with col3:
|
||||
add_vertical_space(2)
|
||||
if st.button('Apply'):
|
||||
st.session_state.sorted_list = sort_list(batch_list, sort_by, order)
|
||||
|
||||
path_row = st.columns([0.1, 0.8])
|
||||
with path_row[0]:
|
||||
st.write("**Batch ID**")
|
||||
with path_row[1]:
|
||||
batch_id = st.selectbox('**Batch ID**', batch_list, label_visibility = "collapsed")
|
||||
batch_id = st.selectbox('**Batch ID**', st.session_state.sorted_list, label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344
|
||||
|
||||
if not batch_id:
|
||||
batch_id = "None"
|
||||
|
||||
checks = st.columns([0.1, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12])
|
||||
with checks[0]:
|
||||
st.write("**Group No.**")
|
||||
|
||||
with checks[1]:
|
||||
a = st.checkbox('Unique Key', key = str(1))
|
||||
a = st.checkbox('Unique Key', key = str(1), args="Unique")
|
||||
with checks[2]:
|
||||
b = st.checkbox('Pricing Before Carveouts', key = str(2))
|
||||
with checks[3]:
|
||||
@@ -106,11 +142,15 @@ file_list = []
|
||||
file_objects = s3_client.list_objects_v2(Bucket=client_bucket
|
||||
, Prefix="contracts_landing_zone/"+batch_id+"/", Delimiter='/')
|
||||
|
||||
# Hardcoded file_list for testing purposes
|
||||
# file_list = ['Boilerplate_TX Amendment Mission Health Network effective_040114 MU.pdf', 'Custom_TX - MP AMENDMENT - MISSION HEALTH NETWORK - MU.pdf',
|
||||
# 'Delaware First Health_First State Homecare Agency_212260_7 MU.pdf', 'Molina Healthcare of Texas, Inc. Amendment 4 - HIX ACA__EFF 01012016_MU.pdf']
|
||||
|
||||
if st.button("Read the contracts from Path"):
|
||||
for obj in file_objects.get('Contents',[]):
|
||||
if not obj['Key'].endswith('/'):
|
||||
file_list.append(obj['Key'].split('/')[-1])
|
||||
|
||||
|
||||
df['Contract Name'] = file_list
|
||||
# df['Request ID'] = range(len(file_list))
|
||||
# df['Contract ID'] = file_list
|
||||
@@ -147,23 +187,33 @@ additional_info = pd.DataFrame(columns=['CLIENT_NAME', 'BATCH_ID', 'REQUEST_USER
|
||||
additional_info.loc[0] = [client, batch_id, st.session_state.user_info['mail'], datetime.now().strftime("%Y-%m-%d %H:%M:%S")]
|
||||
st.write(additional_info)
|
||||
|
||||
st.session_state.contract_count = 0
|
||||
contract_list = []
|
||||
for index, row in edited_df.iterrows():
|
||||
allow_run_for_contract = False
|
||||
group_list = []
|
||||
if row['Unique Key']:
|
||||
group_list.append('Unique Key')
|
||||
allow_run_for_contract = True
|
||||
if row['Pricing Before Carveouts']:
|
||||
group_list.append('Pricing Before Carveouts')
|
||||
allow_run_for_contract = True
|
||||
if row['Contract Related']:
|
||||
group_list.append('Contract Related')
|
||||
allow_run_for_contract = True
|
||||
if row['Provider']:
|
||||
group_list.append('Provider')
|
||||
allow_run_for_contract = True
|
||||
if row['Timeline']:
|
||||
group_list.append('Timeline')
|
||||
allow_run_for_contract = True
|
||||
if row['Carveout Indicator']:
|
||||
group_list.append('Carveout Indicator')
|
||||
allow_run_for_contract = True
|
||||
if row['Carveout Methodology']:
|
||||
group_list.append('Carveout Methodology')
|
||||
allow_run_for_contract = True
|
||||
if allow_run_for_contract: st.session_state.contract_count += 1
|
||||
entry_dict = {
|
||||
"contract_name": row['Contract Name'],
|
||||
"groups": group_list,
|
||||
@@ -184,23 +234,27 @@ with buttons[0]:
|
||||
st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv')
|
||||
with buttons[1]:
|
||||
if st.button("Run Doczy.AI Pipeline"):
|
||||
# csv_buf = StringIO()
|
||||
# additional_info.to_csv(csv_buf, header=True, index=False)
|
||||
# csv_buf.seek(0)
|
||||
# s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/request_submission.csv')
|
||||
# csv_buf = StringIO()
|
||||
# edited_df.to_csv(csv_buf, header=True, index=False)
|
||||
# csv_buf.seek(0)
|
||||
# s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/contract_config.csv')
|
||||
# try:
|
||||
# save_to_sf('load_request_and_contract_submissions', request_submission_file_name = "request_submission.csv", contract_config_file_name = "contract_config.csv")
|
||||
# except Exception as e:
|
||||
# st.write(e)
|
||||
response = requests.post(doczy_pipeline, json = myobj)
|
||||
if response.status_code >= 200 and response.status_code < 300:
|
||||
st.write("Success")
|
||||
# st.write(myobj)
|
||||
if not st.session_state.contract_count == len(edited_df):
|
||||
st.error("Select at least one Group No. for every Contract")
|
||||
else:
|
||||
st.write("Failed")
|
||||
# st.write(response.text)
|
||||
with st.spinner('Running...'):
|
||||
# csv_buf = StringIO()
|
||||
# additional_info.to_csv(csv_buf, header=True, index=False)
|
||||
# csv_buf.seek(0)
|
||||
# s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/request_submission.csv')
|
||||
# csv_buf = StringIO()
|
||||
# edited_df.to_csv(csv_buf, header=True, index=False)
|
||||
# csv_buf.seek(0)
|
||||
# s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/contract_config.csv')
|
||||
# try:
|
||||
# save_to_sf('load_request_and_contract_submissions', request_submission_file_name = "request_submission.csv", contract_config_file_name = "contract_config.csv")
|
||||
# except Exception as e:
|
||||
# st.write(e)
|
||||
response = requests.post(doczy_pipeline, json = myobj)
|
||||
if response.status_code >= 200 and response.status_code < 300:
|
||||
st.write("Success")
|
||||
# st.write(myobj)
|
||||
else:
|
||||
st.write("Failed")
|
||||
# st.write(response.text)
|
||||
|
||||
|
||||
+58
-16
@@ -10,6 +10,7 @@ from langchain.chains import RetrievalQA
|
||||
|
||||
import streamlit as st
|
||||
from streamlit_extras.add_vertical_space import add_vertical_space
|
||||
from streamlit_pdf_viewer import pdf_viewer # needs to be installed on the server
|
||||
import os
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
@@ -18,24 +19,26 @@ import anthropic
|
||||
from pydantic import BaseModel
|
||||
from typing import List
|
||||
import re
|
||||
import base64
|
||||
from sf_conn import get_snowflake_conn
|
||||
import io
|
||||
|
||||
REDIRECT_URI = 'https://doczydev.aarete.com:8502'
|
||||
user_list = USER_LIST
|
||||
|
||||
st.set_page_config(layout = "wide")
|
||||
# Sidebar contents
|
||||
with st.sidebar:
|
||||
st.title("Doczy.AI ™")
|
||||
st.markdown(
|
||||
"""
|
||||
## About
|
||||
This app extracts data from contracts
|
||||
# with st.sidebar:
|
||||
# st.title("Doczy.AI ™")
|
||||
# st.markdown(
|
||||
# """
|
||||
# ## About
|
||||
# This app extracts data from contracts
|
||||
|
||||
"""
|
||||
)
|
||||
add_vertical_space(15)
|
||||
# st.write("Doczy")
|
||||
# """
|
||||
# )
|
||||
# add_vertical_space(15)
|
||||
# # st.write("Doczy")
|
||||
|
||||
_,c1= st.columns([5,1])
|
||||
try:
|
||||
@@ -91,13 +94,11 @@ for obj in objects['Contents']:
|
||||
|
||||
contract_list = sorted(file_list)
|
||||
|
||||
|
||||
|
||||
file_row = st.columns([0.2, 0.7, 0.1])
|
||||
with file_row[0]:
|
||||
st.write("**Contract Name**")
|
||||
with file_row[1]:
|
||||
file_name = st.selectbox('Select a file', contract_list + ['All'], label_visibility = "collapsed")
|
||||
file_name = st.selectbox('Select a file', ['All'] + contract_list, label_visibility = "collapsed", index= None) # MODIFIED - Append 'All' in the front instead of at the end
|
||||
|
||||
field_row = st.columns([0.2, 0.7, 0.1])
|
||||
with field_row[0]:
|
||||
@@ -107,7 +108,7 @@ with field_row[1]:
|
||||
, 'Pricing Before Carveouts - II', 'Carveout Indicator, Code Type and Code #s - I'
|
||||
, 'Carveout Indicator, Code Type and Code #s - II', 'Carveout Indicator, Code Type and Code #s - III'
|
||||
, 'Optimize Carving Indic.', 'Carveout Method - I', 'Carveout Method - II', 'Provider'
|
||||
, 'Timeline'), label_visibility = "collapsed")
|
||||
, 'Timeline'), label_visibility = "collapsed", index = None)
|
||||
|
||||
if field_group == 'Unique Key':
|
||||
fields = fields[fields['PRIORITY'] == 'A']
|
||||
@@ -151,10 +152,51 @@ if st.button("Show Results"):
|
||||
df2 = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description])
|
||||
|
||||
# get this dataframe from snowflake table
|
||||
# df2 = pd.DataFrame(columns=['Contract Name','Field Name', 'SF_DB_COL_NAME', 'Snippet','Page Number'
|
||||
# , 'Field Extracted Value', 'Actual Value','Imputed Value'])
|
||||
df2 = pd.DataFrame(columns=['Contract Name','Field Name', 'SF_DB_COL_NAME', 'Snippet','Page Number'
|
||||
, 'Field Extracted Value', 'Actual Value','Imputed Value'])
|
||||
df2.to_csv('temp2.csv', index=False)
|
||||
|
||||
if st.button("Show PDF"):
|
||||
if file_name == None or file_name == "All":
|
||||
st.error("Choose one specific file.")
|
||||
else:
|
||||
with st.sidebar:
|
||||
st.markdown(
|
||||
"""
|
||||
<style>
|
||||
section[data-testid="stSidebar"] {
|
||||
width: 550px !important; # Set the width to your desired value
|
||||
}
|
||||
</style>
|
||||
""",
|
||||
unsafe_allow_html=True,
|
||||
)
|
||||
s3_client.Object(bucket,file_name)
|
||||
data=obj.get()['Body'].read()
|
||||
pdf_viewer(io.BytesIO(data), width=1500)
|
||||
|
||||
# if st.button("Show PDF"):
|
||||
# if file_name == None or file_name == "All":
|
||||
# st.error("Choose one specific file.")
|
||||
# else:
|
||||
# with st.sidebar:
|
||||
# with open(file_name, "rb") as f:
|
||||
# base64_pdf = base64.b64encode(f.read()).decode('utf-8')
|
||||
# # Embedding PDF in HTML
|
||||
# pdf_display = F'<iframe src="data:application/pdf;base64,{base64_pdf}" width="500" height="1000" type="application/pdf"></iframe>'
|
||||
# # Displaying File
|
||||
# st.markdown(
|
||||
# """
|
||||
# <style>
|
||||
# section[data-testid="stSidebar"] {
|
||||
# width: 600px !important; # Set the width to your desired value
|
||||
# }
|
||||
# </style>
|
||||
# """,
|
||||
# unsafe_allow_html=True,
|
||||
# )
|
||||
# st.markdown(pdf_display, unsafe_allow_html=True)
|
||||
|
||||
df2 = pd.read_csv('temp2.csv')
|
||||
df2['Imputed Value'] = ''
|
||||
edited_df = st.data_editor(df2)
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
import json
|
||||
import streamlit as st
|
||||
from streamlit_extras.add_vertical_space import add_vertical_space
|
||||
import os
|
||||
import streamlit as st
|
||||
import pandas as pd
|
||||
from io import StringIO
|
||||
from datetime import datetime
|
||||
import boto3
|
||||
# import util
|
||||
import requests
|
||||
# from sf_conn import get_secret, save_to_sf
|
||||
from io import StringIO, BytesIO
|
||||
import time
|
||||
# from sf_conn import get_client_names, insert_upload_logs
|
||||
# from constants import USER_LIST
|
||||
|
||||
create_batch_url = 'https://lfksus2t62.execute-api.us-east-2.amazonaws.com/dev/create-batch'
|
||||
|
||||
# REDIRECT_URI = 'https://doczydev.aarete.com:8500'
|
||||
# user_list = USER_LIST
|
||||
if 'uploading' not in st.session_state:
|
||||
st.session_state.uploading = False
|
||||
|
||||
|
||||
|
||||
def insert_upload_logs(batch_id, client_name, file_name, upload_datetime, upload_user):
|
||||
"""
|
||||
Input: batch_id, client_name, file_name, upload_datetime, upload_user
|
||||
Output: status of the insert query
|
||||
"""
|
||||
try:
|
||||
return 'Log inserted successfully'
|
||||
except Exception as e:
|
||||
return e
|
||||
|
||||
|
||||
st.set_page_config(layout = "wide")
|
||||
# # # Sidebar contents
|
||||
# # with st.sidebar:
|
||||
# # st.title("Doczy.AI ™")
|
||||
# # st.markdown(
|
||||
# # """
|
||||
# # ## About
|
||||
# # This app extracts data from contracts
|
||||
|
||||
# # """
|
||||
# # )
|
||||
# # add_vertical_space(15)
|
||||
# # # st.write("Doczy")
|
||||
# #
|
||||
|
||||
_,c1= st.columns([5,1])
|
||||
# try:
|
||||
# util.setup_page(REDIRECT_URI)
|
||||
# except Exception as e:
|
||||
# st.write(f"SSO Failed = {e}")
|
||||
# st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'}
|
||||
st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'}
|
||||
try:
|
||||
c1.write(f"User: **{st.session_state.user_info['displayName']}**")
|
||||
user_mail = st.session_state.user_info['mail']
|
||||
except KeyError as e:
|
||||
st.write("Session Expired.")
|
||||
st.stop()
|
||||
|
||||
print(st.session_state)
|
||||
|
||||
s3_client = boto3.client('s3',
|
||||
region_name="us-east-2",
|
||||
)
|
||||
|
||||
client_list = ['doczy-ai-client-1', 'Delaware First Health, Inc.', 'Community Health Choice, Inc','CareSource Network Partners LLC',
|
||||
'HealthNet of Cali', 'Oklahoma Complete Health, Inc', 'HealthFirst', 'Molina Healthcare of TX', 'AvMed', 'Arizona Care1st',
|
||||
'WellCare New Jersey']
|
||||
|
||||
# This is the list of client fetched from Snowflake
|
||||
# TODO: Need to update the streamlit code to use the client names from this list
|
||||
# And use the s3 paths to save the objects for the respective client
|
||||
# client_list, s3_paths = get_client_names()
|
||||
# client_s3_paths = dict(zip(client_list, s3_paths))
|
||||
|
||||
client_row = st.columns([0.1, 0.8])
|
||||
with client_row[0]:
|
||||
st.write("**Client Name**")
|
||||
with client_row[1]:
|
||||
client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344
|
||||
|
||||
# client_bucket = client_s3_paths.get(client)
|
||||
|
||||
# to be deleted when client buckets are created
|
||||
client_bucket = 'doczy-ai-client-1'
|
||||
|
||||
file_row = st.columns([0.1, 0.8])
|
||||
with file_row[0]:
|
||||
st.write("**Upload Files**")
|
||||
with file_row[1]:
|
||||
file_list = st.file_uploader("Upload", type=['docx','tiff','pdf'], accept_multiple_files=True, label_visibility = "collapsed", help="Only PDF, TIFF and DOCX file formats are supported.", disabled=st.session_state.uploading)
|
||||
|
||||
add_vertical_space(2)
|
||||
df = pd.DataFrame(columns=['Contract Name'])
|
||||
df['Contract Name'] = file_list
|
||||
file_names = []
|
||||
buttons = st.columns([0.4, 0.4, 0.2])
|
||||
|
||||
def set_uploading_state():
|
||||
if not client == None and not len(file_list) == 0:
|
||||
st.session_state.uploading = True
|
||||
|
||||
with buttons[1]:
|
||||
if st.button("Create Batch", on_click = set_uploading_state):
|
||||
if client == None:
|
||||
st.error("No Client Name Selected.")
|
||||
elif len(file_list) == 0:
|
||||
st.error("No Files Selected.")
|
||||
else:
|
||||
time.sleep(5)
|
||||
# myobj = { "client-bucket-name": client_bucket }
|
||||
# response = requests.post(create_batch_url, json = myobj)
|
||||
# if response.status_code >= 200 and response.status_code < 300:
|
||||
# try:
|
||||
# batch_id = json.loads(json.loads(response.text)['body'])['batch_id']
|
||||
# landing_zone = json.loads(json.loads(response.text)['body'])['landing_zone']
|
||||
# except:
|
||||
# st.write(myobj)
|
||||
# st.write(response.text)
|
||||
# batch_id = 'failed_cases'
|
||||
# landing_zone = 'contracts_landing_zone'
|
||||
# else:
|
||||
# st.write("Failed")
|
||||
# for uploaded_file in file_list:
|
||||
# stringio = BytesIO(uploaded_file.getvalue())
|
||||
# stringio.seek(0)
|
||||
# s3_client.put_object(Bucket=client_bucket, Body=stringio.getvalue(), Key=
|
||||
# landing_zone+batch_id+'/'+str(uploaded_file.name))
|
||||
|
||||
# # TODO: Test this insert function with snowflake
|
||||
# upload_log = insert_upload_logs(batch_id, client, str(uploaded_file.name), datetime.now().strftime("%Y-%m-%d %H:%M:%S"), user_mail)
|
||||
# st.write(upload_log)
|
||||
|
||||
# file_names.append(str(uploaded_file.name))
|
||||
# st.write(f"{batch_id} created")
|
||||
st.session_state.uploading = False
|
||||
# st.write(f"Files uploaded to s3://{client_bucket}/{landing_zone}{batch_id}")
|
||||
|
||||
|
||||
|
||||
|
||||
# @st.cache_data
|
||||
# def convert_df(df):
|
||||
# return df.to_csv(index=False).encode('utf-8')
|
||||
|
||||
# csv = convert_df(df)
|
||||
|
||||
# df = df.reset_index() # make sure indexes pair with number of rows
|
||||
|
||||
# contract_list = []
|
||||
# for index, row in df.iterrows():
|
||||
# group_list = []
|
||||
# if row['Unique Key']:
|
||||
# group_list.append('Unique Key')
|
||||
# if row['Pricing Before Carveouts']:
|
||||
# group_list.append('Pricing Before Carveouts')
|
||||
# if row['Contract Related']:
|
||||
# group_list.append('Contract Related')
|
||||
# if row['Provider']:
|
||||
# group_list.append('Provider')
|
||||
# if row['Timeline']:
|
||||
# group_list.append('Timeline')
|
||||
# if row['Carveout Indicator']:
|
||||
# group_list.append('Carveout Indicator')
|
||||
# if row['Carveout Methodology']:
|
||||
# group_list.append('Carveout Methodology')
|
||||
# entry_dict = {
|
||||
# "contract_name": row['Contract Name'],
|
||||
# "groups": group_list,
|
||||
# "contract_source_path": "batches/batch_1/"+client+"/"+row['Contract Name']
|
||||
# }
|
||||
# contract_list.append(entry_dict)
|
||||
# contract_list = list(df['Contract Name'])
|
||||
|
||||
# myobj = {
|
||||
# "s3_bucket": 'doczy-dev-infra-textract',
|
||||
# "batch_id": "1",
|
||||
# "client_name": client,
|
||||
# "username": user_mail,
|
||||
# "contract_list": contract_list
|
||||
# }
|
||||
|
||||
# buttons = st.columns([0.8, 0.2])
|
||||
# with buttons[0]:
|
||||
# st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv')
|
||||
# with buttons[1]:
|
||||
# if st.button("Upload to DB"):
|
||||
# response = requests.post(doczy_pipeline, json = myobj)
|
||||
# if response.status_code >= 200 and response.status_code < 300:
|
||||
# st.write("Success")
|
||||
# else:
|
||||
# st.write("Failed")
|
||||
# # st.write(response.text)
|
||||
@@ -0,0 +1,259 @@
|
||||
import streamlit as st
|
||||
from streamlit_extras.add_vertical_space import add_vertical_space
|
||||
import os
|
||||
import streamlit as st
|
||||
import pandas as pd
|
||||
from io import StringIO
|
||||
from datetime import datetime
|
||||
import boto3
|
||||
# import util
|
||||
import requests
|
||||
import time
|
||||
# from sf_conn import get_client_names, get_secret, save_to_sf
|
||||
# from constants import USER_LIST, DOCZY_PIPELINE_URL_DEV
|
||||
|
||||
# doczy_pipeline = DOCZY_PIPELINE_URL_DEV
|
||||
|
||||
# REDIRECT_URI = 'https://doczydev.aarete.com:8501'
|
||||
# user_list = USER_LIST
|
||||
st.set_page_config(layout = "wide")
|
||||
# # # Sidebar contents
|
||||
# # with st.sidebar:
|
||||
# # st.title("Doczy.AI ™")
|
||||
# # st.markdown(
|
||||
# # """
|
||||
# # ## About
|
||||
# # This app extracts data from contracts
|
||||
|
||||
# # """
|
||||
# # )
|
||||
# # add_vertical_space(15)
|
||||
# # # st.write("Doczy")
|
||||
|
||||
_,c1= st.columns([5,1])
|
||||
# try:
|
||||
# util.setup_page(REDIRECT_URI)
|
||||
# except:
|
||||
# st.write("SSO Failed")
|
||||
# st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'}
|
||||
st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'}
|
||||
try:
|
||||
c1.write(f"User: **{st.session_state.user_info['displayName']}**")
|
||||
user_mail = st.session_state.user_info['mail']
|
||||
except KeyError as e:
|
||||
# Do we add a link to get to the login page here?
|
||||
st.write("Session Expired.")
|
||||
st.stop()
|
||||
|
||||
|
||||
|
||||
s3_client = boto3.client('s3',
|
||||
region_name="us-east-2",
|
||||
)
|
||||
|
||||
# # to be replaced with snowflake data
|
||||
client_list = ['doczy-ai-client-1', 'Delaware First Health, Inc.', 'Community Health Choice, Inc','CareSource Network Partners LLC',
|
||||
'HealthNet of Cali', 'Oklahoma Complete Health, Inc', 'HealthFirst', 'Molina Healthcare of TX', 'AvMed', 'Arizona Care1st',
|
||||
'WellCare New Jersey']
|
||||
|
||||
# client_list, s3_paths = get_client_names()
|
||||
# client_s3_paths = dict(zip(client_list, s3_paths))
|
||||
|
||||
client_row = st.columns([0.1, 0.8])
|
||||
with client_row[0]:
|
||||
st.write("**Client Name**")
|
||||
with client_row[1]:
|
||||
client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344
|
||||
|
||||
# client_bucket = client_s3_paths.get(client)
|
||||
|
||||
# # to be deleted when buckets for different clients are ready; below line is added only for testing the corresponding DAG
|
||||
client_bucket = 'doczy-ai-client-1'
|
||||
|
||||
# batch_objects = s3_client.list_objects_v2(Bucket=client_bucket
|
||||
# , Prefix="contracts_landing_zone/", Delimiter='/')
|
||||
|
||||
# batch_list = []
|
||||
# for prefix in batch_objects['CommonPrefixes']:
|
||||
# batch_list.append(prefix['Prefix'][:-1].split('/')[-1])
|
||||
|
||||
batch_list = ['batch_020524103737', 'batch_090524131433', 'batch_090524131607', 'batch_100524123000', 'batch_130524064322',
|
||||
'batch_160524071331', 'batch_200524213550', 'batch_250424112237', 'batch_280524120530', 'batch_280524121721', 'batch_280524144222',
|
||||
'batch_290524123926', 'batch_290524164044', 'batch_310524102029', 'batch_310524124050', 'batch_310524162346', 'batch_310524162631']
|
||||
|
||||
if 'sorted_list' not in st.session_state:
|
||||
st.session_state.sorted_list = batch_list
|
||||
|
||||
def sort_list(ex_list, sort_by, order):
|
||||
if sort_by == 'Alphabetical':
|
||||
ex_list = sorted(ex_list, reverse=(order == 'Descending'))
|
||||
elif sort_by == 'Create Date':
|
||||
ex_list = ex_list if order == 'Ascending' else list(reversed(ex_list))
|
||||
|
||||
return ex_list
|
||||
|
||||
col1, col2, col3, col4 = st.columns([0.5, 0.5, 0.5, 0.5])
|
||||
|
||||
|
||||
with col1:
|
||||
sort_by = st.radio("**Sort Batch_IDs**", ('Alphabetical', 'Create Date'))
|
||||
|
||||
with col2:
|
||||
order = st.radio('', ('Ascending','Descending'))
|
||||
|
||||
with col3:
|
||||
add_vertical_space(2)
|
||||
if st.button('Apply'):
|
||||
st.session_state.sorted_list = sort_list(batch_list, sort_by, order)
|
||||
|
||||
path_row = st.columns([0.1, 0.8])
|
||||
with path_row[0]:
|
||||
st.write("**Batch ID**")
|
||||
with path_row[1]:
|
||||
batch_id = st.selectbox('**Batch ID**', st.session_state.sorted_list, label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344
|
||||
|
||||
if not batch_id:
|
||||
batch_id = "None"
|
||||
|
||||
checks = st.columns([0.1, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12])
|
||||
with checks[0]:
|
||||
st.write("**Group No.**")
|
||||
|
||||
with checks[1]:
|
||||
a = st.checkbox('Unique Key', key = str(1), args="Unique")
|
||||
with checks[2]:
|
||||
b = st.checkbox('Pricing Before Carveouts', key = str(2))
|
||||
with checks[3]:
|
||||
c = st.checkbox('Contract Related', key = str(3))
|
||||
with checks[4]:
|
||||
d = st.checkbox('Provider', key = str(4))
|
||||
with checks[5]:
|
||||
e = st.checkbox('Timeline', key = str(5))
|
||||
with checks[6]:
|
||||
f = st.checkbox('Carveout Indicator', key = str(6))
|
||||
with checks[7]:
|
||||
g = st.checkbox('Carveout Methodology', key = str(7))
|
||||
|
||||
add_vertical_space(1)
|
||||
|
||||
df = pd.DataFrame(columns=['Contract Name', 'Unique Key','Pricing Before Carveouts'
|
||||
, 'Contract Related', 'Provider', 'Timeline', 'Carveout Indicator', 'Carveout Methodology'])
|
||||
# file_list = []
|
||||
# file_objects = s3_client.list_objects_v2(Bucket=client_bucket
|
||||
# , Prefix="contracts_landing_zone/"+batch_id+"/", Delimiter='/')
|
||||
file_list = ['Boilerplate_TX Amendment Mission Health Network effective_040114 MU.pdf', 'Custom_TX - MP AMENDMENT - MISSION HEALTH NETWORK - MU.pdf',
|
||||
'Delaware First Health_First State Homecare Agency_212260_7 MU.pdf', 'Molina Healthcare of Texas, Inc. Amendment 4 - HIX ACA__EFF 01012016_MU.pdf']
|
||||
|
||||
if st.button("Read the contracts from Path"):
|
||||
# for obj in file_objects.get('Contents',[]):
|
||||
# if not obj['Key'].endswith('/'):
|
||||
# file_list.append(obj['Key'].split('/')[-1])
|
||||
df['Contract Name'] = file_list
|
||||
# df['Request ID'] = range(len(file_list))
|
||||
# df['Contract ID'] = file_list
|
||||
df['Unique Key'] = a
|
||||
df['Pricing Before Carveouts'] = b
|
||||
df['Contract Related'] = c
|
||||
df['Provider'] = d
|
||||
df['Timeline'] = e
|
||||
df['Carveout Indicator'] = f
|
||||
df['Carveout Methodology'] = g
|
||||
dir_path = os.path.dirname(os.path.realpath(__file__))
|
||||
print(f'DEBUGGING: PWD= {dir_path}')
|
||||
df.to_csv('temp1.csv', index=False)
|
||||
|
||||
add_vertical_space(1)
|
||||
df2 = pd.read_csv('temp1.csv')
|
||||
edited_df = st.data_editor(df2)
|
||||
|
||||
edited_df['REQUEST_USER'] = user_mail
|
||||
edited_df['LATEST_FLAG BOOLEAN'] = True
|
||||
edited_df['PIPELINE_KICKOFF_DATETIME'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
edited_df['REQUEST_DATETIME'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
@st.cache_data
|
||||
def convert_df(df):
|
||||
return df.to_csv(index=False).encode('utf-8')
|
||||
|
||||
csv = convert_df(edited_df)
|
||||
# edited_df = edited_df.reset_index() # make sure indexes pair with number of rows
|
||||
|
||||
# additional_info = pd.DataFrame(columns=['REQUEST_ID','T_DRIVE_PATH','CLIENT_NAME'
|
||||
# , 'GROUP_NAME', 'REQUEST_USERNAME', 'REQUEST_DATETIME'])
|
||||
additional_info = pd.DataFrame(columns=['CLIENT_NAME', 'BATCH_ID', 'REQUEST_USERNAME', 'REQUEST_DATETIME'])
|
||||
additional_info.loc[0] = [client, batch_id, st.session_state.user_info['mail'], datetime.now().strftime("%Y-%m-%d %H:%M:%S")]
|
||||
st.write(additional_info)
|
||||
|
||||
st.session_state.contract_count = 0
|
||||
contract_list = []
|
||||
for index, row in edited_df.iterrows():
|
||||
allow_run_for_contract = False
|
||||
group_list = []
|
||||
if row['Unique Key']:
|
||||
group_list.append('Unique Key')
|
||||
allow_run_for_contract = True
|
||||
if row['Pricing Before Carveouts']:
|
||||
group_list.append('Pricing Before Carveouts')
|
||||
allow_run_for_contract = True
|
||||
if row['Contract Related']:
|
||||
group_list.append('Contract Related')
|
||||
allow_run_for_contract = True
|
||||
if row['Provider']:
|
||||
group_list.append('Provider')
|
||||
allow_run_for_contract = True
|
||||
if row['Timeline']:
|
||||
group_list.append('Timeline')
|
||||
allow_run_for_contract = True
|
||||
if row['Carveout Indicator']:
|
||||
group_list.append('Carveout Indicator')
|
||||
allow_run_for_contract = True
|
||||
if row['Carveout Methodology']:
|
||||
group_list.append('Carveout Methodology')
|
||||
allow_run_for_contract = True
|
||||
if allow_run_for_contract: st.session_state.contract_count += 1
|
||||
entry_dict = {
|
||||
"contract_name": row['Contract Name'],
|
||||
"groups": group_list,
|
||||
"contract_source_path": "contracts_landing_zone/"+batch_id+"/"+row['Contract Name']
|
||||
}
|
||||
contract_list.append(entry_dict)
|
||||
|
||||
myobj = {
|
||||
"s3_bucket": client_bucket,
|
||||
"batch_id": batch_id,
|
||||
"client_name": client,
|
||||
"username": user_mail,
|
||||
"contract_list": contract_list
|
||||
}
|
||||
|
||||
buttons = st.columns([0.8, 0.2])
|
||||
with buttons[0]:
|
||||
st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv')
|
||||
with buttons[1]:
|
||||
if st.button("Run Doczy.AI Pipeline"):
|
||||
if not st.session_state.contract_count == len(edited_df):
|
||||
st.error("Select at least one Group No. for every Contract")
|
||||
else:
|
||||
with st.spinner('Running...'): # Feedback to User while API endpoint sends response for DOC-342
|
||||
# csv_buf = StringIO()
|
||||
# additional_info.to_csv(csv_buf, header=True, index=False)
|
||||
# csv_buf.seek(0)
|
||||
# s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/request_submission.csv')
|
||||
# csv_buf = StringIO()
|
||||
# edited_df.to_csv(csv_buf, header=True, index=False)
|
||||
# csv_buf.seek(0)
|
||||
# s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/contract_config.csv')
|
||||
# try:
|
||||
# save_to_sf('load_request_and_contract_submissions', request_submission_file_name = "request_submission.csv", contract_config_file_name = "contract_config.csv")
|
||||
# except Exception as e:
|
||||
# st.write(e)
|
||||
# response = requests.post(doczy_pipeline, json = myobj)
|
||||
# if response.status_code >= 200 and response.status_code < 300:
|
||||
# st.write("Success")
|
||||
# # st.write(myobj)
|
||||
# else:
|
||||
# st.write("Failed")
|
||||
# st.write(response.text)
|
||||
time.sleep(5)
|
||||
st.write("Success!")
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
import json
|
||||
|
||||
import boto3
|
||||
from langchain.prompts import PromptTemplate
|
||||
from langchain.embeddings.bedrock import BedrockEmbeddings
|
||||
from langchain.llms.bedrock import Bedrock
|
||||
from langchain_community.vectorstores import Chroma
|
||||
# from constants import CHROMA_SETTINGS, EMBEDDING_MODEL_NAME, PERSIST_DIRECTORY, MODEL_ID, MODEL_BASENAME, SOURCE_DIRECTORY, USER_LIST
|
||||
from langchain.chains import RetrievalQA
|
||||
|
||||
import streamlit as st
|
||||
from streamlit_extras.add_vertical_space import add_vertical_space
|
||||
from streamlit_pdf_viewer import pdf_viewer # needs to be installed on the server
|
||||
import os
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
# import util
|
||||
import anthropic
|
||||
from pydantic import BaseModel
|
||||
from typing import List
|
||||
import re
|
||||
import base64
|
||||
# from sf_conn import get_snowflake_conn
|
||||
|
||||
REDIRECT_URI = 'https://doczydev.aarete.com:8502'
|
||||
# user_list = USER_LIST
|
||||
|
||||
st.set_page_config(layout = "wide")
|
||||
# Sidebar contents
|
||||
# with st.sidebar:
|
||||
# st.title("Doczy.AI ™")
|
||||
# st.markdown(
|
||||
# """
|
||||
# ## About
|
||||
# This app extracts data from contracts
|
||||
|
||||
# """
|
||||
# )
|
||||
# add_vertical_space(15)
|
||||
# # st.write("Doczy")
|
||||
|
||||
_,c1= st.columns([5,1])
|
||||
# try:
|
||||
# util.setup_page(REDIRECT_URI)
|
||||
# except:
|
||||
# st.write("SSO Failed")
|
||||
# st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'}
|
||||
st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'}
|
||||
try:
|
||||
c1.write(f"User: **{st.session_state.user_info['displayName']}**")
|
||||
user_mail = st.session_state.user_info['mail']
|
||||
except KeyError as e:
|
||||
st.write("Session Expired.")
|
||||
st.stop()
|
||||
|
||||
# remove below try except statement if comparison with actual vales is not required
|
||||
# try:
|
||||
# conn = get_snowflake_conn('STG')
|
||||
# cur = conn.cursor()
|
||||
# query = 'select * from "TRAINING_DATA_RAW"'
|
||||
# cur.execute(query)
|
||||
# field_values = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description])
|
||||
# field_values['Document_Name'] = field_values['DOCUMENT_NAME']
|
||||
# except:
|
||||
# # field_values = pd.read_csv('contract_field_values.csv', encoding='unicode_escape', skipinitialspace=True)
|
||||
# # field_values = field_values.loc[:, ~field_values.columns.str.contains('Unnamed:')]
|
||||
# st.write("Conn failed, unable to fetch data from training data table in Snowflake")
|
||||
|
||||
field_values = pd.read_csv('contract_field_values.csv', encoding='unicode_escape', skipinitialspace=True)
|
||||
field_values = field_values.loc[:, ~field_values.columns.str.contains('Unnamed:')]
|
||||
|
||||
# try:
|
||||
# query = 'select * from "PROMPT_CONFIG"'
|
||||
# cur.execute(query)
|
||||
# fields = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description])
|
||||
|
||||
# fields.rename(columns={'FIELD_DESC': 'Field Name'}, inplace = True)
|
||||
# fields.rename(columns={'PROMPT': 'Interrogation Question?'}, inplace = True)
|
||||
# fields.rename(columns={'GROUP_ID': 'PRIORITY'}, inplace = True)
|
||||
# fields.rename(columns={'FIELD_NAME': 'SF_DB_COL_NAME'}, inplace = True)
|
||||
# fields.rename(columns={'FM_MODEL_ID': 'llm_selected'}, inplace = True)
|
||||
# except Exception as e:
|
||||
# st.write("Unable to fetch data from Snowflake: ",e)
|
||||
# # fields = pd.read_csv('contract_fields.csv', encoding='unicode_escape', skipinitialspace=True)
|
||||
# # fields = fields[~fields['SF_COL_NAME'].str.endswith('_PG', na=None)]
|
||||
|
||||
fields = pd.read_csv('contract_fields.csv', encoding='unicode_escape', skipinitialspace=True)
|
||||
# fields = fields[fields['SF_DB_COL_NAME'].str.endswith('_PG', na=None)]
|
||||
|
||||
# change the code below if contract list is fetched from snowflake
|
||||
s3_client = boto3.client('s3',
|
||||
region_name="us-east-2"
|
||||
)
|
||||
bucket = 'doczy-dev-infra-textract'
|
||||
# objects = s3_client.list_objects_v2(Bucket=bucket, Prefix="training-data/")
|
||||
# file_list = []
|
||||
# for obj in objects['Contents']:
|
||||
# if not obj['Key'].endswith('/'):
|
||||
# file_list.append(obj['Key'])
|
||||
|
||||
file_list = ['Contract_Training_Exercise_Pricing.pdf', 'Contract_Training_Exercise_SLA.pdf']
|
||||
contract_list = sorted(file_list)
|
||||
|
||||
file_row = st.columns([0.2, 0.7, 0.1])
|
||||
with file_row[0]:
|
||||
st.write("**Contract Name**")
|
||||
with file_row[1]:
|
||||
file_name = st.selectbox('Select a file', ['All'] + contract_list, label_visibility = "collapsed", index= None) # MODIFIED - Append 'All' in the front instead of at the end
|
||||
|
||||
field_row = st.columns([0.2, 0.7, 0.1])
|
||||
with field_row[0]:
|
||||
st.write("**Field Group**")
|
||||
with field_row[1]:
|
||||
field_group = st.selectbox('Field Group',('Unique Key', 'Contract Related', 'Pricing Before Carveouts - I'
|
||||
, 'Pricing Before Carveouts - II', 'Carveout Indicator, Code Type and Code #s - I'
|
||||
, 'Carveout Indicator, Code Type and Code #s - II', 'Carveout Indicator, Code Type and Code #s - III'
|
||||
, 'Optimize Carving Indic.', 'Carveout Method - I', 'Carveout Method - II', 'Provider'
|
||||
, 'Timeline'), label_visibility = "collapsed", index = None)
|
||||
|
||||
if field_group == 'Unique Key':
|
||||
fields = fields[fields['PRIORITY'] == 'A']
|
||||
elif field_group == 'Contract Related':
|
||||
fields = fields[fields['PRIORITY'] == 'C']
|
||||
elif field_group == 'Pricing Before Carveouts - I':
|
||||
fields = fields[fields['PRIORITY'] == 'B']
|
||||
fields = np.array_split(fields, 2)[0]
|
||||
elif field_group == 'Pricing Before Carveouts - II':
|
||||
fields = fields[fields['PRIORITY'] == 'B']
|
||||
fields = np.array_split(fields, 2)[1]
|
||||
elif field_group == 'Carveout Indicator, Code Type and Code #s - I':
|
||||
fields = fields[fields['PRIORITY'] == 'F']
|
||||
fields = np.array_split(fields, 3)[0]
|
||||
elif field_group == 'Carveout Indicator, Code Type and Code #s - II':
|
||||
fields = fields[fields['PRIORITY'] == 'F']
|
||||
fields = np.array_split(fields, 3)[1]
|
||||
elif field_group == 'Carveout Indicator, Code Type and Code #s - III':
|
||||
fields = fields[fields['PRIORITY'] == 'F']
|
||||
fields = np.array_split(fields, 3)[2]
|
||||
elif field_group == 'Carveout Methodology - I':
|
||||
fields = fields[fields['PRIORITY'] == 'G']
|
||||
fields = np.array_split(fields, 4)[0]
|
||||
elif field_group == 'Carveout Methodology - II':
|
||||
fields = fields[fields['PRIORITY'] == 'G']
|
||||
fields = np.array_split(fields, 4)[1]
|
||||
elif field_group == 'Carveout Method - III':
|
||||
fields = fields[fields['PRIORITY'] == 'G']
|
||||
fields = np.array_split(fields, 4)[2]
|
||||
elif field_group == 'Carveout Method - IV':
|
||||
fields = fields[fields['PRIORITY'] == 'G']
|
||||
fields = np.array_split(fields, 4)[3]
|
||||
elif field_group == 'Provider':
|
||||
fields = fields[fields['PRIORITY'] == 'D']
|
||||
elif field_group == 'Timeline':
|
||||
fields = fields[fields['PRIORITY'] == 'E']
|
||||
|
||||
if st.button("Show Results"):
|
||||
# query = 'select * from "DOCZY_PIPELINE_RAW_OUTPUT"'
|
||||
# cur.execute(query)
|
||||
# df2 = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description])
|
||||
|
||||
# get this dataframe from snowflake table
|
||||
df2 = pd.DataFrame(columns=['Contract Name','Field Name', 'SF_DB_COL_NAME', 'Snippet','Page Number'
|
||||
, 'Field Extracted Value', 'Actual Value','Imputed Value'])
|
||||
df2.to_csv('temp2.csv', index=False)
|
||||
|
||||
if st.button("Show PDF"):
|
||||
if file_name == None or file_name == "All":
|
||||
st.error("Choose one specific file.")
|
||||
else:
|
||||
with st.sidebar:
|
||||
st.markdown(
|
||||
"""
|
||||
<style>
|
||||
section[data-testid="stSidebar"] {
|
||||
width: 550px !important; # Set the width to your desired value
|
||||
}
|
||||
</style>
|
||||
""",
|
||||
unsafe_allow_html=True,
|
||||
)
|
||||
pdf_viewer(file_name, width=1500)
|
||||
|
||||
# if st.button("Show PDF"):
|
||||
# if file_name == None or file_name == "All":
|
||||
# st.error("Choose one specific file.")
|
||||
# else:
|
||||
# with st.sidebar:
|
||||
# with open(file_name, "rb") as f:
|
||||
# base64_pdf = base64.b64encode(f.read()).decode('utf-8')
|
||||
# # Embedding PDF in HTML
|
||||
# pdf_display = F'<iframe src="data:application/pdf;base64,{base64_pdf}" width="500" height="1000" type="application/pdf"></iframe>'
|
||||
# # Displaying File
|
||||
# st.markdown(
|
||||
# """
|
||||
# <style>
|
||||
# section[data-testid="stSidebar"] {
|
||||
# width: 600px !important; # Set the width to your desired value
|
||||
# }
|
||||
# </style>
|
||||
# """,
|
||||
# unsafe_allow_html=True,
|
||||
# )
|
||||
# st.markdown(pdf_display, unsafe_allow_html=True)
|
||||
|
||||
df2 = pd.read_csv('temp2.csv')
|
||||
df2['Imputed Value'] = ''
|
||||
edited_df = st.data_editor(df2)
|
||||
|
||||
@st.cache_data
|
||||
def convert_df(df):
|
||||
return df.to_csv(index=False).encode('utf-8')
|
||||
|
||||
csv = convert_df(edited_df)
|
||||
|
||||
buttons = st.columns(3)
|
||||
with buttons[0]:
|
||||
# st.button("Save All Imputations")
|
||||
st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv')
|
||||
with buttons[1]:
|
||||
# st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv')
|
||||
st.write("")
|
||||
with buttons[2]:
|
||||
if st.button("Kickoff Database Integration"):
|
||||
st.write("Stored in DB")
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -70,9 +70,6 @@ locals {
|
||||
provider "aws" {
|
||||
# profile = var.aws_profile
|
||||
|
||||
# access_key = var.access_key
|
||||
# secret_key = var.secret_key
|
||||
|
||||
default_tags {
|
||||
tags = local.common_tags
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user