diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml
index 7b2856e..06cb48a 100644
--- a/bitbucket-pipelines.yml
+++ b/bitbucket-pipelines.yml
@@ -200,31 +200,49 @@ pipelines:
name: "Plan & Apply DevOps Pipeline"
image: hugree/bitbucket-aws-python38-tf154:latest
oidc: true
- trigger : automatic # Keeping it manual so that it doesnt conflict with existing resources
+ trigger : automatic
script:
- *aws-context-uat
- cd devops-pipeline/terraform-backend-resources/
+ - terraform init
- terraform validate
# Ensure the backend configuration for s3 obj key is correct
- - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/devops-pipeline/terraform.tfstate"
- - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat"
+ - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat"- echo "S3_BUCKET=$(terraform output -raw terraform_s3_bucket_name)" >> backend.env
+ - echo "DYNAMODB_TABLE=$(terraform output -raw terraform_dynamodb_table_name)" >> backend.env
+ artifacts:
+ - backend.env
condition:
changesets:
includePaths:
- devops-pipeline/terraform-backend-resources/*
- - devops-pipeline/terraform-backend-resources/**/*
+ - devops-pipeline/terraform-backend-resources/**/*- step:
+ name: "Plan & Apply DevOps Extra Resource Pipeline"
+ image: hugree/bitbucket-aws-python38-tf154:latest
+ oidc: true
+ trigger : automatic # Keeping it manual so that it doesnt conflict with existing resources
+ script:
+ - *aws-context-uat
+ - cd devops-pipeline/other-resources/
+ - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/devops-pipeline-other-resources/terraform.tfstate"
+ - terraform validate
+ - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat"
+ condition:
+ changesets:
+ includePaths:
+ - devops-pipeline/other-resources/*
+ - devops-pipeline/other-resources/**/*
- step:
name: "Plan & Apply Streamlit Pipeline"
image: hugree/bitbucket-aws-python38-tf154:latest
oidc: true
- trigger : manual
+ trigger : automatic
script:
- *aws-context-uat
- cd streamlit-server/
+ - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/streamlit-server/terraform.tfstate"
- terraform validate
# Ensure the backend configuration for s3 obj key is correct
- - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/streamlit-server/terraform.tfstate"
- - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat"
+ - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat"-var "vpc_id=vpc-0392396d0e7bdd77f"
condition:
changesets:
includePaths:
@@ -242,12 +260,12 @@ pipelines:
image: python:3.8
name: "Deploy streamlit to EC2"
oidc: true
- trigger: manual # Keeping it manual as the server will be created first and then referenced in this
+ trigger: automatic # Keeping it manual as the server will be created first and then referenced in this
script:
- python -m pip install --upgrade pip
- apt-get update && apt-get install -y jq git
- pip install awscli
- - *aws-context-dev
+ - *aws-context-uat
- *ssm-send-command-uat
condition:
changesets:
@@ -269,10 +287,10 @@ pipelines:
script:
- *aws-context-uat
- cd textract-pipeline/terraform/
+ - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/textract-pipeline/terraform.tfstate"
- terraform validate
# Ensure the backend configuration for s3 obj key is corresponding to the stream name
- - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/textract-pipeline/terraform.tfstate"
- - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat"
+ - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat"
condition:
changesets:
includePaths:
diff --git a/devops-pipeline/other-resources/variables.tf b/devops-pipeline/other-resources/variables.tf
index ac98867..97e452b 100644
--- a/devops-pipeline/other-resources/variables.tf
+++ b/devops-pipeline/other-resources/variables.tf
@@ -53,6 +53,7 @@ variable "mwaa_resources_s3_bucket" {
}
}
+
variable "aws_account_id" {
type = string
}
\ No newline at end of file
diff --git a/devops-pipeline/terraform-backend-resources/main.tf b/devops-pipeline/terraform-backend-resources/main.tf
index 1dbd42c..56aec03 100644
--- a/devops-pipeline/terraform-backend-resources/main.tf
+++ b/devops-pipeline/terraform-backend-resources/main.tf
@@ -8,7 +8,7 @@ terraform {
}
provider "aws" {
- region = var.aws_region
+ region = var.aws_region
}
module "aws_terraform_remote_backend" {
diff --git a/devops-pipeline/terraform-backend-resources/modules/aws-terraform-remote-backend/variables.tf b/devops-pipeline/terraform-backend-resources/modules/aws-terraform-remote-backend/variables.tf
index 48cfca4..3483195 100644
--- a/devops-pipeline/terraform-backend-resources/modules/aws-terraform-remote-backend/variables.tf
+++ b/devops-pipeline/terraform-backend-resources/modules/aws-terraform-remote-backend/variables.tf
@@ -10,6 +10,20 @@ variable "environment" {
variable "client_name" {
type = string
}
+# required
+variable "secret_key" {
+ type = string
+}
+
+# required
+variable "access_key" {
+ type = string
+}
+
+variable "token" {
+ type = string
+}
+
# dynamoDB table
variable "terraform_dynamodb_table" {
type = object({
diff --git a/devops-pipeline/terraform-backend-resources/outputs.tf b/devops-pipeline/terraform-backend-resources/outputs.tf
index 58a5ddb..2192baf 100644
--- a/devops-pipeline/terraform-backend-resources/outputs.tf
+++ b/devops-pipeline/terraform-backend-resources/outputs.tf
@@ -9,4 +9,4 @@ output "terraform_s3_bucket_arn" {
}
output "terraform_s3_bucket_name" {
value = module.aws_terraform_remote_backend.terraform_s3_bucket_name
-}
\ No newline at end of file
+}
diff --git a/devops-pipeline/terraform-backend-resources/variables.tf b/devops-pipeline/terraform-backend-resources/variables.tf
index b756d1b..3e52526 100644
--- a/devops-pipeline/terraform-backend-resources/variables.tf
+++ b/devops-pipeline/terraform-backend-resources/variables.tf
@@ -16,4 +16,19 @@ variable "environment" {
variable "client_name" {
type = string
default = "infra"
-}
\ No newline at end of file
+}
+
+#required
+variable "secret_key" {
+ type = string
+}
+
+#required
+variable "access_key" {
+ type = string
+}
+
+#required
+variable "token" {
+ type = string
+}
diff --git a/snowflake/DEV/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql b/snowflake/DEV/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql
index 4e5562c..bd1b22f 100644
--- a/snowflake/DEV/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql
+++ b/snowflake/DEV/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql
@@ -51,7 +51,7 @@ BEGIN
batch_id := (SELECT NULLIF(TRIM($2),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1);
request_datetime := (SELECT NULLIF(TRIM($13),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1);
- UPDATE TABLE STG.CONTRACT_CONFIG
+ UPDATE STG.CONTRACT_CONFIG
SET LATEST_FLAG = FALSE
WHERE BATCH_ID = :batch_id AND CONTRACT_NAME = :contract_name AND REQUEST_DATETIME < :request_datetime AND LATEST_FLAG = TRUE;
diff --git a/snowflake/UAT/R__1002_LOG_AUDIT_TABLE.sql b/snowflake/UAT/R__1002_LOG_AUDIT_TABLE.sql
index 1cc0768..d3ce540 100644
--- a/snowflake/UAT/R__1002_LOG_AUDIT_TABLE.sql
+++ b/snowflake/UAT/R__1002_LOG_AUDIT_TABLE.sql
@@ -8,4 +8,4 @@ create or replace TABLE STG.LOG_AUDIT (
SUB_SECTION_NAME VARCHAR(255) COLLATE 'en-ci',
AUDIT_SID NUMBER(38,0),
TOTAL_MIN NUMBER(38,0)
-);
\ No newline at end of file
+);
diff --git a/snowflake/UAT/config_interface/R__1004_LOAD_CONTRACT_CONFIG_SP.sql b/snowflake/UAT/config_interface/R__1004_LOAD_CONTRACT_CONFIG_SP.sql
index 4e5562c..e00a89c 100644
--- a/snowflake/UAT/config_interface/R__1004_LOAD_CONTRACT_CONFIG_SP.sql
+++ b/snowflake/UAT/config_interface/R__1004_LOAD_CONTRACT_CONFIG_SP.sql
@@ -50,8 +50,8 @@ BEGIN
contract_name := (SELECT NULLIF(TRIM($1),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1);
batch_id := (SELECT NULLIF(TRIM($2),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1);
request_datetime := (SELECT NULLIF(TRIM($13),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1);
-
- UPDATE TABLE STG.CONTRACT_CONFIG
+
+ UPDATE STG.CONTRACT_CONFIG
SET LATEST_FLAG = FALSE
WHERE BATCH_ID = :batch_id AND CONTRACT_NAME = :contract_name AND REQUEST_DATETIME < :request_datetime AND LATEST_FLAG = TRUE;
diff --git a/streamlit-server/alb.tf b/streamlit-server/alb.tf
index a94a31c..48b6bdb 100644
--- a/streamlit-server/alb.tf
+++ b/streamlit-server/alb.tf
@@ -115,7 +115,8 @@ resource "aws_lb_listener" "https_8500" {
protocol = "HTTPS"
ssl_policy = "ELBSecurityPolicy-2016-08"
# Using variable for ARN
- certificate_arn = var.acm_arn_dev
+ certificate_arn = lookup(var.acm_arns, var.environment)
+
default_action {
type = "forward"
@@ -128,7 +129,7 @@ resource "aws_lb_listener" "https_8501" {
port = "8501"
protocol = "HTTPS"
ssl_policy = "ELBSecurityPolicy-2016-08"
- certificate_arn = var.acm_arn_dev
+ certificate_arn = lookup(var.acm_arns, var.environment)
default_action {
type = "forward"
@@ -141,7 +142,7 @@ resource "aws_lb_listener" "https_8502" {
port = "8502"
protocol = "HTTPS"
ssl_policy = "ELBSecurityPolicy-2016-08"
- certificate_arn = var.acm_arn_dev
+ certificate_arn = lookup(var.acm_arns, var.environment)
default_action {
type = "forward"
@@ -154,7 +155,7 @@ resource "aws_lb_listener" "https_8503" {
port = "8503"
protocol = "HTTPS"
ssl_policy = "ELBSecurityPolicy-2016-08"
- certificate_arn = var.acm_arn_dev
+ certificate_arn = lookup(var.acm_arns, var.environment)
default_action {
type = "forward"
diff --git a/streamlit-server/main.tf b/streamlit-server/main.tf
index 75d1352..0a82054 100644
--- a/streamlit-server/main.tf
+++ b/streamlit-server/main.tf
@@ -81,13 +81,13 @@ resource "aws_iam_instance_profile" "ec2_instance_profile" {
}
resource "aws_instance" "streamlit_server" {
- ami = var.ubuntu_ami
+ ami = lookup(var.ubuntu_ami, var.environment)
instance_type = var.ec2_instance_type
subnet_id = data.aws_subnets.subnets.ids[0]
vpc_security_group_ids = data.aws_security_groups.security_groups.ids
iam_instance_profile = aws_iam_instance_profile.ec2_instance_profile.name
associate_public_ip_address = false
- key_name = "tf-test-key" # Created using TF and uploaded to S3. Prerequisite
+ key_name = "tf-uat-key" # Created using TF and uploaded to S3. Prerequisite
root_block_device {
volume_size = 30 # Variable
@@ -101,7 +101,7 @@ resource "aws_instance" "streamlit_server" {
apt-get update
apt-get install -y python3 python3-pip
sudo apt-get install libffi-dev
- python3 -m pip3 install -r /tmp/requirements.txt
+ python3 -m pip install -r /tmp/requirements.txt
sudo apt install openssh-client
sudo apt install net-tools
@@ -197,3 +197,4 @@ resource "aws_instance" "streamlit_server" {
}
+
diff --git a/streamlit-server/variables.tf b/streamlit-server/variables.tf
index c71c156..7e18a5f 100644
--- a/streamlit-server/variables.tf
+++ b/streamlit-server/variables.tf
@@ -30,10 +30,18 @@ variable "vpc_id" {
type = string
}
+# variable "ubuntu_ami" {
+# type = string
+# default = "ami-0de7e97fedfbc6ef6" # Switch to this ami-0b8b44ec9a8f90422
+
+# }
+
variable "ubuntu_ami" {
- type = string
- default = "ami-0de7e97fedfbc6ef6" # Switch to this ami-0b8b44ec9a8f90422
-
+ type = map(string)
+ default = {
+ dev = "ami-0de7e97fedfbc6ef6"
+ uat = "ami-0b8b44ec9a8f90422"
+ }
}
variable "ec2_instance_type" {
@@ -52,6 +60,15 @@ variable "acm_arn_dev" {
default = "arn:aws:acm:us-east-2:660131068782:certificate/947e04b3-be22-43bf-bea2-a0ad3b221d70"
}
+variable "acm_arns" {
+ type = map(string)
+ default = {
+ dev = "arn:aws:acm:us-east-2:660131068782:certificate/947e04b3-be22-43bf-bea2-a0ad3b221d70"
+ uat = "arn:aws:acm:us-east-2:975049960860:certificate/e036e76d-1f4f-496e-ac97-5f65ec4a34cc"
+ prod = "arn:aws:acm:region:account-id:certificate/prod-cert-id"
+ }
+}
+
#variable "secret_key" {
# type = string
#}
diff --git a/streamlit/interface_0.py b/streamlit/interface_0.py
index f638209..656939d 100644
--- a/streamlit/interface_0.py
+++ b/streamlit/interface_0.py
@@ -18,6 +18,22 @@ create_batch_url = 'https://lfksus2t62.execute-api.us-east-2.amazonaws.com/dev/c
REDIRECT_URI = 'https://doczydev.aarete.com:8500'
user_list = USER_LIST
+if 'uploading' not in st.session_state:
+ st.session_state.uploading = False
+
+
+
+def insert_upload_logs(batch_id, client_name, file_name, upload_datetime, upload_user):
+ """
+ Input: batch_id, client_name, file_name, upload_datetime, upload_user
+ Output: status of the insert query
+ """
+ try:
+ return 'Log inserted successfully'
+ except Exception as e:
+ return e
+
+
st.set_page_config(layout = "wide")
# # Sidebar contents
# with st.sidebar:
@@ -31,6 +47,7 @@ st.set_page_config(layout = "wide")
# )
# add_vertical_space(15)
# # st.write("Doczy")
+#
_,c1= st.columns([5,1])
try:
@@ -45,7 +62,7 @@ except KeyError as e:
st.write("Session Expired.")
st.stop()
-
+print(st.session_state)
s3_client = boto3.client('s3',
region_name="us-east-2",
@@ -65,7 +82,7 @@ client_row = st.columns([0.1, 0.8])
with client_row[0]:
st.write("**Client Name**")
with client_row[1]:
- client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed")
+ client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344
client_bucket = client_s3_paths.get(client)
@@ -76,42 +93,55 @@ file_row = st.columns([0.1, 0.8])
with file_row[0]:
st.write("**Upload Files**")
with file_row[1]:
- file_list = st.file_uploader("Upload", type=None, accept_multiple_files=True, label_visibility = "collapsed")
+ file_list = st.file_uploader("Upload", type=['docx','tiff','pdf'], accept_multiple_files=True, label_visibility = "collapsed", help="Only PDF, TIFF and DOCX file formats are supported.", disabled=st.session_state.uploading)
add_vertical_space(2)
df = pd.DataFrame(columns=['Contract Name'])
df['Contract Name'] = file_list
file_names = []
buttons = st.columns([0.4, 0.4, 0.2])
-with buttons[1]:
- if st.button("Create Batch"):
- myobj = { "client-bucket-name": client_bucket }
- response = requests.post(create_batch_url, json = myobj)
- if response.status_code >= 200 and response.status_code < 300:
- try:
- batch_id = json.loads(json.loads(response.text)['body'])['batch_id']
- landing_zone = json.loads(json.loads(response.text)['body'])['landing_zone']
- except:
- st.write(myobj)
- st.write(response.text)
- batch_id = 'failed_cases'
- landing_zone = 'contracts_landing_zone'
+def set_uploading_state():
+ if not client == None and not len(file_list) == 0:
+ st.session_state.uploading = True
+
+with buttons[1]:
+ if st.button("Create Batch", on_click = set_uploading_state):
+ if client == None:
+ st.error("No Client Name Selected.")
+ elif len(file_list) == 0:
+ st.error("No Files Selected.")
else:
- st.write("Failed")
- for uploaded_file in file_list:
- stringio = BytesIO(uploaded_file.getvalue())
- stringio.seek(0)
- s3_client.put_object(Bucket=client_bucket, Body=stringio.getvalue(), Key=
- landing_zone+batch_id+'/'+str(uploaded_file.name))
-
- # TODO: Test this insert function with snowflake
- upload_log = insert_upload_logs(batch_id, client, str(uploaded_file.name), datetime.now().strftime("%Y-%m-%d %H:%M:%S"), user_mail)
- st.write(upload_log)
-
- file_names.append(str(uploaded_file.name))
- st.write(f"{batch_id} created")
- st.write(f"Files uploaded to s3://{client_bucket}/{landing_zone}{batch_id}")
+ myobj = { "client-bucket-name": client_bucket }
+ response = requests.post(create_batch_url, json = myobj)
+ if response.status_code >= 200 and response.status_code < 300:
+ try:
+ batch_id = json.loads(json.loads(response.text)['body'])['batch_id']
+ landing_zone = json.loads(json.loads(response.text)['body'])['landing_zone']
+ except:
+ st.write(myobj)
+ st.write(response.text)
+ batch_id = 'failed_cases'
+ landing_zone = 'contracts_landing_zone'
+ else:
+ st.write("Failed")
+ for uploaded_file in file_list:
+ stringio = BytesIO(uploaded_file.getvalue())
+ stringio.seek(0)
+ s3_client.put_object(Bucket=client_bucket, Body=stringio.getvalue(), Key=
+ landing_zone+batch_id+'/'+str(uploaded_file.name))
+
+ # TODO: Test this insert function with snowflake
+ upload_log = insert_upload_logs(batch_id, client, str(uploaded_file.name), datetime.now().strftime("%Y-%m-%d %H:%M:%S"), user_mail)
+ st.write(upload_log)
+
+ file_names.append(str(uploaded_file.name))
+ st.session_state.uploading = False
+ st.write(f"{batch_id} created")
+ st.write(f"Files uploaded to s3://{client_bucket}/{landing_zone}{batch_id}")
+
+
+
# @st.cache_data
diff --git a/streamlit/interface_1.py b/streamlit/interface_1.py
index 3d3a695..5b3f82a 100644
--- a/streamlit/interface_1.py
+++ b/streamlit/interface_1.py
@@ -8,6 +8,7 @@ from datetime import datetime
import boto3
import util
import requests
+import time
from sf_conn import get_client_names, get_secret, save_to_sf
from constants import USER_LIST, DOCZY_PIPELINE_URL_DEV
@@ -61,10 +62,11 @@ client_row = st.columns([0.1, 0.8])
with client_row[0]:
st.write("**Client Name**")
with client_row[1]:
- client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed")
+ client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344
client_bucket = client_s3_paths.get(client)
-# # to be deleted when buckets for different clients are ready; below line is added only for testing the corresponding DAG
+
+# to be deleted when buckets for different clients are ready; below line is added only for testing the corresponding DAG
client_bucket = 'doczy-ai-client-1'
batch_objects = s3_client.list_objects_v2(Bucket=client_bucket
@@ -74,17 +76,51 @@ batch_list = []
for prefix in batch_objects['CommonPrefixes']:
batch_list.append(prefix['Prefix'][:-1].split('/')[-1])
+# Hardcoded batch_list for testing purposes
+# batch_list = ['batch_020524103737', 'batch_090524131433', 'batch_090524131607', 'batch_100524123000', 'batch_130524064322',
+# 'batch_160524071331', 'batch_200524213550', 'batch_250424112237', 'batch_280524120530', 'batch_280524121721', 'batch_280524144222',
+# 'batch_290524123926', 'batch_290524164044', 'batch_310524102029', 'batch_310524124050', 'batch_310524162346', 'batch_310524162631']
+
+if 'sorted_list' not in st.session_state:
+ st.session_state.sorted_list = batch_list
+
+def sort_list(ex_list, sort_by, order):
+ if sort_by == 'Alphabetical':
+ ex_list = sorted(ex_list, reverse=(order == 'Descending'))
+ elif sort_by == 'Create Date':
+ ex_list = ex_list if order == 'Ascending' else list(reversed(ex_list))
+
+ return ex_list
+
+col1, col2, col3, col4 = st.columns([0.5, 0.5, 0.5, 0.5])
+
+
+with col1:
+ sort_by = st.radio("**Sort Batch_IDs**", ('Alphabetical', 'Create Date'))
+
+with col2:
+ order = st.radio('', ('Ascending','Descending'))
+
+with col3:
+ add_vertical_space(2)
+ if st.button('Apply'):
+ st.session_state.sorted_list = sort_list(batch_list, sort_by, order)
+
path_row = st.columns([0.1, 0.8])
with path_row[0]:
st.write("**Batch ID**")
with path_row[1]:
- batch_id = st.selectbox('**Batch ID**', batch_list, label_visibility = "collapsed")
+ batch_id = st.selectbox('**Batch ID**', st.session_state.sorted_list, label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344
+
+if not batch_id:
+ batch_id = "None"
checks = st.columns([0.1, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12])
with checks[0]:
st.write("**Group No.**")
+
with checks[1]:
- a = st.checkbox('Unique Key', key = str(1))
+ a = st.checkbox('Unique Key', key = str(1), args="Unique")
with checks[2]:
b = st.checkbox('Pricing Before Carveouts', key = str(2))
with checks[3]:
@@ -106,11 +142,15 @@ file_list = []
file_objects = s3_client.list_objects_v2(Bucket=client_bucket
, Prefix="contracts_landing_zone/"+batch_id+"/", Delimiter='/')
+# Hardcoded file_list for testing purposes
+# file_list = ['Boilerplate_TX Amendment Mission Health Network effective_040114 MU.pdf', 'Custom_TX - MP AMENDMENT - MISSION HEALTH NETWORK - MU.pdf',
+# 'Delaware First Health_First State Homecare Agency_212260_7 MU.pdf', 'Molina Healthcare of Texas, Inc. Amendment 4 - HIX ACA__EFF 01012016_MU.pdf']
+
if st.button("Read the contracts from Path"):
for obj in file_objects.get('Contents',[]):
if not obj['Key'].endswith('/'):
file_list.append(obj['Key'].split('/')[-1])
-
+
df['Contract Name'] = file_list
# df['Request ID'] = range(len(file_list))
# df['Contract ID'] = file_list
@@ -147,23 +187,33 @@ additional_info = pd.DataFrame(columns=['CLIENT_NAME', 'BATCH_ID', 'REQUEST_USER
additional_info.loc[0] = [client, batch_id, st.session_state.user_info['mail'], datetime.now().strftime("%Y-%m-%d %H:%M:%S")]
st.write(additional_info)
+st.session_state.contract_count = 0
contract_list = []
for index, row in edited_df.iterrows():
+ allow_run_for_contract = False
group_list = []
if row['Unique Key']:
group_list.append('Unique Key')
+ allow_run_for_contract = True
if row['Pricing Before Carveouts']:
group_list.append('Pricing Before Carveouts')
+ allow_run_for_contract = True
if row['Contract Related']:
group_list.append('Contract Related')
+ allow_run_for_contract = True
if row['Provider']:
group_list.append('Provider')
+ allow_run_for_contract = True
if row['Timeline']:
group_list.append('Timeline')
+ allow_run_for_contract = True
if row['Carveout Indicator']:
group_list.append('Carveout Indicator')
+ allow_run_for_contract = True
if row['Carveout Methodology']:
group_list.append('Carveout Methodology')
+ allow_run_for_contract = True
+ if allow_run_for_contract: st.session_state.contract_count += 1
entry_dict = {
"contract_name": row['Contract Name'],
"groups": group_list,
@@ -184,23 +234,27 @@ with buttons[0]:
st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv')
with buttons[1]:
if st.button("Run Doczy.AI Pipeline"):
- # csv_buf = StringIO()
- # additional_info.to_csv(csv_buf, header=True, index=False)
- # csv_buf.seek(0)
- # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/request_submission.csv')
- # csv_buf = StringIO()
- # edited_df.to_csv(csv_buf, header=True, index=False)
- # csv_buf.seek(0)
- # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/contract_config.csv')
- # try:
- # save_to_sf('load_request_and_contract_submissions', request_submission_file_name = "request_submission.csv", contract_config_file_name = "contract_config.csv")
- # except Exception as e:
- # st.write(e)
- response = requests.post(doczy_pipeline, json = myobj)
- if response.status_code >= 200 and response.status_code < 300:
- st.write("Success")
- # st.write(myobj)
+ if not st.session_state.contract_count == len(edited_df):
+ st.error("Select at least one Group No. for every Contract")
else:
- st.write("Failed")
- # st.write(response.text)
+ with st.spinner('Running...'):
+ # csv_buf = StringIO()
+ # additional_info.to_csv(csv_buf, header=True, index=False)
+ # csv_buf.seek(0)
+ # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/request_submission.csv')
+ # csv_buf = StringIO()
+ # edited_df.to_csv(csv_buf, header=True, index=False)
+ # csv_buf.seek(0)
+ # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/contract_config.csv')
+ # try:
+ # save_to_sf('load_request_and_contract_submissions', request_submission_file_name = "request_submission.csv", contract_config_file_name = "contract_config.csv")
+ # except Exception as e:
+ # st.write(e)
+ response = requests.post(doczy_pipeline, json = myobj)
+ if response.status_code >= 200 and response.status_code < 300:
+ st.write("Success")
+ # st.write(myobj)
+ else:
+ st.write("Failed")
+ # st.write(response.text)
diff --git a/streamlit/interface_2.py b/streamlit/interface_2.py
index ed8c494..824509a 100644
--- a/streamlit/interface_2.py
+++ b/streamlit/interface_2.py
@@ -10,6 +10,7 @@ from langchain.chains import RetrievalQA
import streamlit as st
from streamlit_extras.add_vertical_space import add_vertical_space
+from streamlit_pdf_viewer import pdf_viewer # needs to be installed on the server
import os
import pandas as pd
import numpy as np
@@ -18,24 +19,26 @@ import anthropic
from pydantic import BaseModel
from typing import List
import re
+import base64
from sf_conn import get_snowflake_conn
+import io
REDIRECT_URI = 'https://doczydev.aarete.com:8502'
user_list = USER_LIST
st.set_page_config(layout = "wide")
# Sidebar contents
-with st.sidebar:
- st.title("Doczy.AI ™")
- st.markdown(
- """
- ## About
- This app extracts data from contracts
+# with st.sidebar:
+# st.title("Doczy.AI ™")
+# st.markdown(
+# """
+# ## About
+# This app extracts data from contracts
- """
- )
- add_vertical_space(15)
- # st.write("Doczy")
+# """
+# )
+# add_vertical_space(15)
+# # st.write("Doczy")
_,c1= st.columns([5,1])
try:
@@ -91,13 +94,11 @@ for obj in objects['Contents']:
contract_list = sorted(file_list)
-
-
file_row = st.columns([0.2, 0.7, 0.1])
with file_row[0]:
st.write("**Contract Name**")
with file_row[1]:
- file_name = st.selectbox('Select a file', contract_list + ['All'], label_visibility = "collapsed")
+ file_name = st.selectbox('Select a file', ['All'] + contract_list, label_visibility = "collapsed", index= None) # MODIFIED - Append 'All' in the front instead of at the end
field_row = st.columns([0.2, 0.7, 0.1])
with field_row[0]:
@@ -107,7 +108,7 @@ with field_row[1]:
, 'Pricing Before Carveouts - II', 'Carveout Indicator, Code Type and Code #s - I'
, 'Carveout Indicator, Code Type and Code #s - II', 'Carveout Indicator, Code Type and Code #s - III'
, 'Optimize Carving Indic.', 'Carveout Method - I', 'Carveout Method - II', 'Provider'
- , 'Timeline'), label_visibility = "collapsed")
+ , 'Timeline'), label_visibility = "collapsed", index = None)
if field_group == 'Unique Key':
fields = fields[fields['PRIORITY'] == 'A']
@@ -151,10 +152,51 @@ if st.button("Show Results"):
df2 = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description])
# get this dataframe from snowflake table
- # df2 = pd.DataFrame(columns=['Contract Name','Field Name', 'SF_DB_COL_NAME', 'Snippet','Page Number'
- # , 'Field Extracted Value', 'Actual Value','Imputed Value'])
+ df2 = pd.DataFrame(columns=['Contract Name','Field Name', 'SF_DB_COL_NAME', 'Snippet','Page Number'
+ , 'Field Extracted Value', 'Actual Value','Imputed Value'])
df2.to_csv('temp2.csv', index=False)
+if st.button("Show PDF"):
+ if file_name == None or file_name == "All":
+ st.error("Choose one specific file.")
+ else:
+ with st.sidebar:
+ st.markdown(
+ """
+
+ """,
+ unsafe_allow_html=True,
+ )
+ s3_client.Object(bucket,file_name)
+ data=obj.get()['Body'].read()
+ pdf_viewer(io.BytesIO(data), width=1500)
+
+# if st.button("Show PDF"):
+# if file_name == None or file_name == "All":
+# st.error("Choose one specific file.")
+# else:
+# with st.sidebar:
+# with open(file_name, "rb") as f:
+# base64_pdf = base64.b64encode(f.read()).decode('utf-8')
+# # Embedding PDF in HTML
+# pdf_display = F''
+# # Displaying File
+# st.markdown(
+# """
+#
+# """,
+# unsafe_allow_html=True,
+# )
+# st.markdown(pdf_display, unsafe_allow_html=True)
+
df2 = pd.read_csv('temp2.csv')
df2['Imputed Value'] = ''
edited_df = st.data_editor(df2)
diff --git a/streamlit/local/local_interface_0.py b/streamlit/local/local_interface_0.py
new file mode 100644
index 0000000..1c6a006
--- /dev/null
+++ b/streamlit/local/local_interface_0.py
@@ -0,0 +1,200 @@
+import json
+import streamlit as st
+from streamlit_extras.add_vertical_space import add_vertical_space
+import os
+import streamlit as st
+import pandas as pd
+from io import StringIO
+from datetime import datetime
+import boto3
+# import util
+import requests
+# from sf_conn import get_secret, save_to_sf
+from io import StringIO, BytesIO
+import time
+# from sf_conn import get_client_names, insert_upload_logs
+# from constants import USER_LIST
+
+create_batch_url = 'https://lfksus2t62.execute-api.us-east-2.amazonaws.com/dev/create-batch'
+
+# REDIRECT_URI = 'https://doczydev.aarete.com:8500'
+# user_list = USER_LIST
+if 'uploading' not in st.session_state:
+ st.session_state.uploading = False
+
+
+
+def insert_upload_logs(batch_id, client_name, file_name, upload_datetime, upload_user):
+ """
+ Input: batch_id, client_name, file_name, upload_datetime, upload_user
+ Output: status of the insert query
+ """
+ try:
+ return 'Log inserted successfully'
+ except Exception as e:
+ return e
+
+
+st.set_page_config(layout = "wide")
+# # # Sidebar contents
+# # with st.sidebar:
+# # st.title("Doczy.AI ™")
+# # st.markdown(
+# # """
+# # ## About
+# # This app extracts data from contracts
+
+# # """
+# # )
+# # add_vertical_space(15)
+# # # st.write("Doczy")
+# #
+
+_,c1= st.columns([5,1])
+# try:
+# util.setup_page(REDIRECT_URI)
+# except Exception as e:
+# st.write(f"SSO Failed = {e}")
+# st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'}
+st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'}
+try:
+ c1.write(f"User: **{st.session_state.user_info['displayName']}**")
+ user_mail = st.session_state.user_info['mail']
+except KeyError as e:
+ st.write("Session Expired.")
+ st.stop()
+
+print(st.session_state)
+
+s3_client = boto3.client('s3',
+ region_name="us-east-2",
+)
+
+client_list = ['doczy-ai-client-1', 'Delaware First Health, Inc.', 'Community Health Choice, Inc','CareSource Network Partners LLC',
+ 'HealthNet of Cali', 'Oklahoma Complete Health, Inc', 'HealthFirst', 'Molina Healthcare of TX', 'AvMed', 'Arizona Care1st',
+ 'WellCare New Jersey']
+
+# This is the list of client fetched from Snowflake
+# TODO: Need to update the streamlit code to use the client names from this list
+# And use the s3 paths to save the objects for the respective client
+# client_list, s3_paths = get_client_names()
+# client_s3_paths = dict(zip(client_list, s3_paths))
+
+client_row = st.columns([0.1, 0.8])
+with client_row[0]:
+ st.write("**Client Name**")
+with client_row[1]:
+ client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344
+
+# client_bucket = client_s3_paths.get(client)
+
+# to be deleted when client buckets are created
+client_bucket = 'doczy-ai-client-1'
+
+file_row = st.columns([0.1, 0.8])
+with file_row[0]:
+ st.write("**Upload Files**")
+with file_row[1]:
+ file_list = st.file_uploader("Upload", type=['docx','tiff','pdf'], accept_multiple_files=True, label_visibility = "collapsed", help="Only PDF, TIFF and DOCX file formats are supported.", disabled=st.session_state.uploading)
+
+add_vertical_space(2)
+df = pd.DataFrame(columns=['Contract Name'])
+df['Contract Name'] = file_list
+file_names = []
+buttons = st.columns([0.4, 0.4, 0.2])
+
+def set_uploading_state():
+ if not client == None and not len(file_list) == 0:
+ st.session_state.uploading = True
+
+with buttons[1]:
+ if st.button("Create Batch", on_click = set_uploading_state):
+ if client == None:
+ st.error("No Client Name Selected.")
+ elif len(file_list) == 0:
+ st.error("No Files Selected.")
+ else:
+ time.sleep(5)
+ # myobj = { "client-bucket-name": client_bucket }
+ # response = requests.post(create_batch_url, json = myobj)
+ # if response.status_code >= 200 and response.status_code < 300:
+ # try:
+ # batch_id = json.loads(json.loads(response.text)['body'])['batch_id']
+ # landing_zone = json.loads(json.loads(response.text)['body'])['landing_zone']
+ # except:
+ # st.write(myobj)
+ # st.write(response.text)
+ # batch_id = 'failed_cases'
+ # landing_zone = 'contracts_landing_zone'
+ # else:
+ # st.write("Failed")
+ # for uploaded_file in file_list:
+ # stringio = BytesIO(uploaded_file.getvalue())
+ # stringio.seek(0)
+ # s3_client.put_object(Bucket=client_bucket, Body=stringio.getvalue(), Key=
+ # landing_zone+batch_id+'/'+str(uploaded_file.name))
+
+ # # TODO: Test this insert function with snowflake
+ # upload_log = insert_upload_logs(batch_id, client, str(uploaded_file.name), datetime.now().strftime("%Y-%m-%d %H:%M:%S"), user_mail)
+ # st.write(upload_log)
+
+ # file_names.append(str(uploaded_file.name))
+ # st.write(f"{batch_id} created")
+ st.session_state.uploading = False
+ # st.write(f"Files uploaded to s3://{client_bucket}/{landing_zone}{batch_id}")
+
+
+
+
+ # @st.cache_data
+ # def convert_df(df):
+ # return df.to_csv(index=False).encode('utf-8')
+
+ # csv = convert_df(df)
+
+ # df = df.reset_index() # make sure indexes pair with number of rows
+
+ # contract_list = []
+ # for index, row in df.iterrows():
+ # group_list = []
+ # if row['Unique Key']:
+ # group_list.append('Unique Key')
+ # if row['Pricing Before Carveouts']:
+ # group_list.append('Pricing Before Carveouts')
+ # if row['Contract Related']:
+ # group_list.append('Contract Related')
+ # if row['Provider']:
+ # group_list.append('Provider')
+ # if row['Timeline']:
+ # group_list.append('Timeline')
+ # if row['Carveout Indicator']:
+ # group_list.append('Carveout Indicator')
+ # if row['Carveout Methodology']:
+ # group_list.append('Carveout Methodology')
+ # entry_dict = {
+ # "contract_name": row['Contract Name'],
+ # "groups": group_list,
+ # "contract_source_path": "batches/batch_1/"+client+"/"+row['Contract Name']
+ # }
+ # contract_list.append(entry_dict)
+ # contract_list = list(df['Contract Name'])
+
+ # myobj = {
+ # "s3_bucket": 'doczy-dev-infra-textract',
+ # "batch_id": "1",
+ # "client_name": client,
+ # "username": user_mail,
+ # "contract_list": contract_list
+ # }
+
+ # buttons = st.columns([0.8, 0.2])
+ # with buttons[0]:
+ # st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv')
+ # with buttons[1]:
+ # if st.button("Upload to DB"):
+ # response = requests.post(doczy_pipeline, json = myobj)
+ # if response.status_code >= 200 and response.status_code < 300:
+ # st.write("Success")
+ # else:
+ # st.write("Failed")
+ # # st.write(response.text)
diff --git a/streamlit/local/local_interface_1.py b/streamlit/local/local_interface_1.py
new file mode 100644
index 0000000..fd03888
--- /dev/null
+++ b/streamlit/local/local_interface_1.py
@@ -0,0 +1,259 @@
+import streamlit as st
+from streamlit_extras.add_vertical_space import add_vertical_space
+import os
+import streamlit as st
+import pandas as pd
+from io import StringIO
+from datetime import datetime
+import boto3
+# import util
+import requests
+import time
+# from sf_conn import get_client_names, get_secret, save_to_sf
+# from constants import USER_LIST, DOCZY_PIPELINE_URL_DEV
+
+# doczy_pipeline = DOCZY_PIPELINE_URL_DEV
+
+# REDIRECT_URI = 'https://doczydev.aarete.com:8501'
+# user_list = USER_LIST
+st.set_page_config(layout = "wide")
+# # # Sidebar contents
+# # with st.sidebar:
+# # st.title("Doczy.AI ™")
+# # st.markdown(
+# # """
+# # ## About
+# # This app extracts data from contracts
+
+# # """
+# # )
+# # add_vertical_space(15)
+# # # st.write("Doczy")
+
+_,c1= st.columns([5,1])
+# try:
+# util.setup_page(REDIRECT_URI)
+# except:
+# st.write("SSO Failed")
+# st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'}
+st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'}
+try:
+ c1.write(f"User: **{st.session_state.user_info['displayName']}**")
+ user_mail = st.session_state.user_info['mail']
+except KeyError as e:
+ # Do we add a link to get to the login page here?
+ st.write("Session Expired.")
+ st.stop()
+
+
+
+s3_client = boto3.client('s3',
+ region_name="us-east-2",
+)
+
+# # to be replaced with snowflake data
+client_list = ['doczy-ai-client-1', 'Delaware First Health, Inc.', 'Community Health Choice, Inc','CareSource Network Partners LLC',
+ 'HealthNet of Cali', 'Oklahoma Complete Health, Inc', 'HealthFirst', 'Molina Healthcare of TX', 'AvMed', 'Arizona Care1st',
+ 'WellCare New Jersey']
+
+# client_list, s3_paths = get_client_names()
+# client_s3_paths = dict(zip(client_list, s3_paths))
+
+client_row = st.columns([0.1, 0.8])
+with client_row[0]:
+ st.write("**Client Name**")
+with client_row[1]:
+ client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344
+
+# client_bucket = client_s3_paths.get(client)
+
+# # to be deleted when buckets for different clients are ready; below line is added only for testing the corresponding DAG
+client_bucket = 'doczy-ai-client-1'
+
+# batch_objects = s3_client.list_objects_v2(Bucket=client_bucket
+# , Prefix="contracts_landing_zone/", Delimiter='/')
+
+# batch_list = []
+# for prefix in batch_objects['CommonPrefixes']:
+# batch_list.append(prefix['Prefix'][:-1].split('/')[-1])
+
+batch_list = ['batch_020524103737', 'batch_090524131433', 'batch_090524131607', 'batch_100524123000', 'batch_130524064322',
+ 'batch_160524071331', 'batch_200524213550', 'batch_250424112237', 'batch_280524120530', 'batch_280524121721', 'batch_280524144222',
+ 'batch_290524123926', 'batch_290524164044', 'batch_310524102029', 'batch_310524124050', 'batch_310524162346', 'batch_310524162631']
+
+if 'sorted_list' not in st.session_state:
+ st.session_state.sorted_list = batch_list
+
+def sort_list(ex_list, sort_by, order):
+ if sort_by == 'Alphabetical':
+ ex_list = sorted(ex_list, reverse=(order == 'Descending'))
+ elif sort_by == 'Create Date':
+ ex_list = ex_list if order == 'Ascending' else list(reversed(ex_list))
+
+ return ex_list
+
+col1, col2, col3, col4 = st.columns([0.5, 0.5, 0.5, 0.5])
+
+
+with col1:
+ sort_by = st.radio("**Sort Batch_IDs**", ('Alphabetical', 'Create Date'))
+
+with col2:
+ order = st.radio('', ('Ascending','Descending'))
+
+with col3:
+ add_vertical_space(2)
+ if st.button('Apply'):
+ st.session_state.sorted_list = sort_list(batch_list, sort_by, order)
+
+path_row = st.columns([0.1, 0.8])
+with path_row[0]:
+ st.write("**Batch ID**")
+with path_row[1]:
+ batch_id = st.selectbox('**Batch ID**', st.session_state.sorted_list, label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344
+
+if not batch_id:
+ batch_id = "None"
+
+checks = st.columns([0.1, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12])
+with checks[0]:
+ st.write("**Group No.**")
+
+with checks[1]:
+ a = st.checkbox('Unique Key', key = str(1), args="Unique")
+with checks[2]:
+ b = st.checkbox('Pricing Before Carveouts', key = str(2))
+with checks[3]:
+ c = st.checkbox('Contract Related', key = str(3))
+with checks[4]:
+ d = st.checkbox('Provider', key = str(4))
+with checks[5]:
+ e = st.checkbox('Timeline', key = str(5))
+with checks[6]:
+ f = st.checkbox('Carveout Indicator', key = str(6))
+with checks[7]:
+ g = st.checkbox('Carveout Methodology', key = str(7))
+
+add_vertical_space(1)
+
+df = pd.DataFrame(columns=['Contract Name', 'Unique Key','Pricing Before Carveouts'
+ , 'Contract Related', 'Provider', 'Timeline', 'Carveout Indicator', 'Carveout Methodology'])
+# file_list = []
+# file_objects = s3_client.list_objects_v2(Bucket=client_bucket
+# , Prefix="contracts_landing_zone/"+batch_id+"/", Delimiter='/')
+file_list = ['Boilerplate_TX Amendment Mission Health Network effective_040114 MU.pdf', 'Custom_TX - MP AMENDMENT - MISSION HEALTH NETWORK - MU.pdf',
+ 'Delaware First Health_First State Homecare Agency_212260_7 MU.pdf', 'Molina Healthcare of Texas, Inc. Amendment 4 - HIX ACA__EFF 01012016_MU.pdf']
+
+if st.button("Read the contracts from Path"):
+ # for obj in file_objects.get('Contents',[]):
+ # if not obj['Key'].endswith('/'):
+ # file_list.append(obj['Key'].split('/')[-1])
+ df['Contract Name'] = file_list
+ # df['Request ID'] = range(len(file_list))
+ # df['Contract ID'] = file_list
+ df['Unique Key'] = a
+ df['Pricing Before Carveouts'] = b
+ df['Contract Related'] = c
+ df['Provider'] = d
+ df['Timeline'] = e
+ df['Carveout Indicator'] = f
+ df['Carveout Methodology'] = g
+ dir_path = os.path.dirname(os.path.realpath(__file__))
+ print(f'DEBUGGING: PWD= {dir_path}')
+ df.to_csv('temp1.csv', index=False)
+
+add_vertical_space(1)
+df2 = pd.read_csv('temp1.csv')
+edited_df = st.data_editor(df2)
+
+edited_df['REQUEST_USER'] = user_mail
+edited_df['LATEST_FLAG BOOLEAN'] = True
+edited_df['PIPELINE_KICKOFF_DATETIME'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+edited_df['REQUEST_DATETIME'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+@st.cache_data
+def convert_df(df):
+ return df.to_csv(index=False).encode('utf-8')
+
+csv = convert_df(edited_df)
+# edited_df = edited_df.reset_index() # make sure indexes pair with number of rows
+
+# additional_info = pd.DataFrame(columns=['REQUEST_ID','T_DRIVE_PATH','CLIENT_NAME'
+# , 'GROUP_NAME', 'REQUEST_USERNAME', 'REQUEST_DATETIME'])
+additional_info = pd.DataFrame(columns=['CLIENT_NAME', 'BATCH_ID', 'REQUEST_USERNAME', 'REQUEST_DATETIME'])
+additional_info.loc[0] = [client, batch_id, st.session_state.user_info['mail'], datetime.now().strftime("%Y-%m-%d %H:%M:%S")]
+st.write(additional_info)
+
+st.session_state.contract_count = 0
+contract_list = []
+for index, row in edited_df.iterrows():
+ allow_run_for_contract = False
+ group_list = []
+ if row['Unique Key']:
+ group_list.append('Unique Key')
+ allow_run_for_contract = True
+ if row['Pricing Before Carveouts']:
+ group_list.append('Pricing Before Carveouts')
+ allow_run_for_contract = True
+ if row['Contract Related']:
+ group_list.append('Contract Related')
+ allow_run_for_contract = True
+ if row['Provider']:
+ group_list.append('Provider')
+ allow_run_for_contract = True
+ if row['Timeline']:
+ group_list.append('Timeline')
+ allow_run_for_contract = True
+ if row['Carveout Indicator']:
+ group_list.append('Carveout Indicator')
+ allow_run_for_contract = True
+ if row['Carveout Methodology']:
+ group_list.append('Carveout Methodology')
+ allow_run_for_contract = True
+ if allow_run_for_contract: st.session_state.contract_count += 1
+ entry_dict = {
+ "contract_name": row['Contract Name'],
+ "groups": group_list,
+ "contract_source_path": "contracts_landing_zone/"+batch_id+"/"+row['Contract Name']
+ }
+ contract_list.append(entry_dict)
+
+myobj = {
+ "s3_bucket": client_bucket,
+ "batch_id": batch_id,
+ "client_name": client,
+ "username": user_mail,
+ "contract_list": contract_list
+}
+
+buttons = st.columns([0.8, 0.2])
+with buttons[0]:
+ st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv')
+with buttons[1]:
+ if st.button("Run Doczy.AI Pipeline"):
+ if not st.session_state.contract_count == len(edited_df):
+ st.error("Select at least one Group No. for every Contract")
+ else:
+ with st.spinner('Running...'): # Feedback to User while API endpoint sends response for DOC-342
+ # csv_buf = StringIO()
+ # additional_info.to_csv(csv_buf, header=True, index=False)
+ # csv_buf.seek(0)
+ # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/request_submission.csv')
+ # csv_buf = StringIO()
+ # edited_df.to_csv(csv_buf, header=True, index=False)
+ # csv_buf.seek(0)
+ # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/contract_config.csv')
+ # try:
+ # save_to_sf('load_request_and_contract_submissions', request_submission_file_name = "request_submission.csv", contract_config_file_name = "contract_config.csv")
+ # except Exception as e:
+ # st.write(e)
+ # response = requests.post(doczy_pipeline, json = myobj)
+ # if response.status_code >= 200 and response.status_code < 300:
+ # st.write("Success")
+ # # st.write(myobj)
+ # else:
+ # st.write("Failed")
+ # st.write(response.text)
+ time.sleep(5)
+ st.write("Success!")
+
diff --git a/streamlit/local/local_interface_2.py b/streamlit/local/local_interface_2.py
new file mode 100644
index 0000000..18702f3
--- /dev/null
+++ b/streamlit/local/local_interface_2.py
@@ -0,0 +1,228 @@
+import json
+
+import boto3
+from langchain.prompts import PromptTemplate
+from langchain.embeddings.bedrock import BedrockEmbeddings
+from langchain.llms.bedrock import Bedrock
+from langchain_community.vectorstores import Chroma
+# from constants import CHROMA_SETTINGS, EMBEDDING_MODEL_NAME, PERSIST_DIRECTORY, MODEL_ID, MODEL_BASENAME, SOURCE_DIRECTORY, USER_LIST
+from langchain.chains import RetrievalQA
+
+import streamlit as st
+from streamlit_extras.add_vertical_space import add_vertical_space
+from streamlit_pdf_viewer import pdf_viewer # needs to be installed on the server
+import os
+import pandas as pd
+import numpy as np
+# import util
+import anthropic
+from pydantic import BaseModel
+from typing import List
+import re
+import base64
+# from sf_conn import get_snowflake_conn
+
+REDIRECT_URI = 'https://doczydev.aarete.com:8502'
+# user_list = USER_LIST
+
+st.set_page_config(layout = "wide")
+# Sidebar contents
+# with st.sidebar:
+# st.title("Doczy.AI ™")
+# st.markdown(
+# """
+# ## About
+# This app extracts data from contracts
+
+# """
+# )
+# add_vertical_space(15)
+# # st.write("Doczy")
+
+_,c1= st.columns([5,1])
+# try:
+# util.setup_page(REDIRECT_URI)
+# except:
+# st.write("SSO Failed")
+# st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'}
+st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'}
+try:
+ c1.write(f"User: **{st.session_state.user_info['displayName']}**")
+ user_mail = st.session_state.user_info['mail']
+except KeyError as e:
+ st.write("Session Expired.")
+ st.stop()
+
+# remove below try except statement if comparison with actual vales is not required
+# try:
+# conn = get_snowflake_conn('STG')
+# cur = conn.cursor()
+# query = 'select * from "TRAINING_DATA_RAW"'
+# cur.execute(query)
+# field_values = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description])
+# field_values['Document_Name'] = field_values['DOCUMENT_NAME']
+# except:
+# # field_values = pd.read_csv('contract_field_values.csv', encoding='unicode_escape', skipinitialspace=True)
+# # field_values = field_values.loc[:, ~field_values.columns.str.contains('Unnamed:')]
+# st.write("Conn failed, unable to fetch data from training data table in Snowflake")
+
+field_values = pd.read_csv('contract_field_values.csv', encoding='unicode_escape', skipinitialspace=True)
+field_values = field_values.loc[:, ~field_values.columns.str.contains('Unnamed:')]
+
+# try:
+# query = 'select * from "PROMPT_CONFIG"'
+# cur.execute(query)
+# fields = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description])
+
+# fields.rename(columns={'FIELD_DESC': 'Field Name'}, inplace = True)
+# fields.rename(columns={'PROMPT': 'Interrogation Question?'}, inplace = True)
+# fields.rename(columns={'GROUP_ID': 'PRIORITY'}, inplace = True)
+# fields.rename(columns={'FIELD_NAME': 'SF_DB_COL_NAME'}, inplace = True)
+# fields.rename(columns={'FM_MODEL_ID': 'llm_selected'}, inplace = True)
+# except Exception as e:
+# st.write("Unable to fetch data from Snowflake: ",e)
+# # fields = pd.read_csv('contract_fields.csv', encoding='unicode_escape', skipinitialspace=True)
+# # fields = fields[~fields['SF_COL_NAME'].str.endswith('_PG', na=None)]
+
+fields = pd.read_csv('contract_fields.csv', encoding='unicode_escape', skipinitialspace=True)
+# fields = fields[fields['SF_DB_COL_NAME'].str.endswith('_PG', na=None)]
+
+# change the code below if contract list is fetched from snowflake
+s3_client = boto3.client('s3',
+ region_name="us-east-2"
+)
+bucket = 'doczy-dev-infra-textract'
+# objects = s3_client.list_objects_v2(Bucket=bucket, Prefix="training-data/")
+# file_list = []
+# for obj in objects['Contents']:
+# if not obj['Key'].endswith('/'):
+# file_list.append(obj['Key'])
+
+file_list = ['Contract_Training_Exercise_Pricing.pdf', 'Contract_Training_Exercise_SLA.pdf']
+contract_list = sorted(file_list)
+
+file_row = st.columns([0.2, 0.7, 0.1])
+with file_row[0]:
+ st.write("**Contract Name**")
+with file_row[1]:
+ file_name = st.selectbox('Select a file', ['All'] + contract_list, label_visibility = "collapsed", index= None) # MODIFIED - Append 'All' in the front instead of at the end
+
+field_row = st.columns([0.2, 0.7, 0.1])
+with field_row[0]:
+ st.write("**Field Group**")
+with field_row[1]:
+ field_group = st.selectbox('Field Group',('Unique Key', 'Contract Related', 'Pricing Before Carveouts - I'
+ , 'Pricing Before Carveouts - II', 'Carveout Indicator, Code Type and Code #s - I'
+ , 'Carveout Indicator, Code Type and Code #s - II', 'Carveout Indicator, Code Type and Code #s - III'
+ , 'Optimize Carving Indic.', 'Carveout Method - I', 'Carveout Method - II', 'Provider'
+ , 'Timeline'), label_visibility = "collapsed", index = None)
+
+if field_group == 'Unique Key':
+ fields = fields[fields['PRIORITY'] == 'A']
+elif field_group == 'Contract Related':
+ fields = fields[fields['PRIORITY'] == 'C']
+elif field_group == 'Pricing Before Carveouts - I':
+ fields = fields[fields['PRIORITY'] == 'B']
+ fields = np.array_split(fields, 2)[0]
+elif field_group == 'Pricing Before Carveouts - II':
+ fields = fields[fields['PRIORITY'] == 'B']
+ fields = np.array_split(fields, 2)[1]
+elif field_group == 'Carveout Indicator, Code Type and Code #s - I':
+ fields = fields[fields['PRIORITY'] == 'F']
+ fields = np.array_split(fields, 3)[0]
+elif field_group == 'Carveout Indicator, Code Type and Code #s - II':
+ fields = fields[fields['PRIORITY'] == 'F']
+ fields = np.array_split(fields, 3)[1]
+elif field_group == 'Carveout Indicator, Code Type and Code #s - III':
+ fields = fields[fields['PRIORITY'] == 'F']
+ fields = np.array_split(fields, 3)[2]
+elif field_group == 'Carveout Methodology - I':
+ fields = fields[fields['PRIORITY'] == 'G']
+ fields = np.array_split(fields, 4)[0]
+elif field_group == 'Carveout Methodology - II':
+ fields = fields[fields['PRIORITY'] == 'G']
+ fields = np.array_split(fields, 4)[1]
+elif field_group == 'Carveout Method - III':
+ fields = fields[fields['PRIORITY'] == 'G']
+ fields = np.array_split(fields, 4)[2]
+elif field_group == 'Carveout Method - IV':
+ fields = fields[fields['PRIORITY'] == 'G']
+ fields = np.array_split(fields, 4)[3]
+elif field_group == 'Provider':
+ fields = fields[fields['PRIORITY'] == 'D']
+elif field_group == 'Timeline':
+ fields = fields[fields['PRIORITY'] == 'E']
+
+if st.button("Show Results"):
+ # query = 'select * from "DOCZY_PIPELINE_RAW_OUTPUT"'
+ # cur.execute(query)
+ # df2 = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description])
+
+ # get this dataframe from snowflake table
+ df2 = pd.DataFrame(columns=['Contract Name','Field Name', 'SF_DB_COL_NAME', 'Snippet','Page Number'
+ , 'Field Extracted Value', 'Actual Value','Imputed Value'])
+ df2.to_csv('temp2.csv', index=False)
+
+if st.button("Show PDF"):
+ if file_name == None or file_name == "All":
+ st.error("Choose one specific file.")
+ else:
+ with st.sidebar:
+ st.markdown(
+ """
+
+ """,
+ unsafe_allow_html=True,
+ )
+ pdf_viewer(file_name, width=1500)
+
+# if st.button("Show PDF"):
+# if file_name == None or file_name == "All":
+# st.error("Choose one specific file.")
+# else:
+# with st.sidebar:
+# with open(file_name, "rb") as f:
+# base64_pdf = base64.b64encode(f.read()).decode('utf-8')
+# # Embedding PDF in HTML
+# pdf_display = F''
+# # Displaying File
+# st.markdown(
+# """
+#
+# """,
+# unsafe_allow_html=True,
+# )
+# st.markdown(pdf_display, unsafe_allow_html=True)
+
+df2 = pd.read_csv('temp2.csv')
+df2['Imputed Value'] = ''
+edited_df = st.data_editor(df2)
+
+@st.cache_data
+def convert_df(df):
+ return df.to_csv(index=False).encode('utf-8')
+
+csv = convert_df(edited_df)
+
+buttons = st.columns(3)
+with buttons[0]:
+ # st.button("Save All Imputations")
+ st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv')
+with buttons[1]:
+ # st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv')
+ st.write("")
+with buttons[2]:
+ if st.button("Kickoff Database Integration"):
+ st.write("Stored in DB")
+
+
+
+
diff --git a/textract-pipeline/terraform/main.tf b/textract-pipeline/terraform/main.tf
index 9c7f38b..0284a95 100644
--- a/textract-pipeline/terraform/main.tf
+++ b/textract-pipeline/terraform/main.tf
@@ -70,9 +70,6 @@ locals {
provider "aws" {
# profile = var.aws_profile
- # access_key = var.access_key
- # secret_key = var.secret_key
-
default_tags {
tags = local.common_tags
}