From d9cbfe75a8364eeefd5482bb11a646fd2936240b Mon Sep 17 00:00:00 2001 From: Umang Mistry Date: Thu, 30 May 2024 14:54:15 -0500 Subject: [PATCH 01/37] triggering devops pipeline --- devops-pipeline/terraform-backend-resources/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devops-pipeline/terraform-backend-resources/variables.tf b/devops-pipeline/terraform-backend-resources/variables.tf index 25c303f..c806237 100644 --- a/devops-pipeline/terraform-backend-resources/variables.tf +++ b/devops-pipeline/terraform-backend-resources/variables.tf @@ -22,4 +22,4 @@ variable "environment" { variable "client_name" { type = string default = "infra" -} \ No newline at end of file +} From 003808a256f2495a6dfa08b9695d73f496cce61c Mon Sep 17 00:00:00 2001 From: Umang Mistry Date: Thu, 30 May 2024 14:57:35 -0500 Subject: [PATCH 02/37] removed typo from UAT pipeline.yaml --- bitbucket-pipelines.yml | 2 +- devops-pipeline/terraform-backend-resources/variables.tf | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index ffaecb2..66cfa53 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -10,7 +10,7 @@ definitions: export AWS_REGION='us-east-2'; - script: &aws-context-uat export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO_UAT:role/$OIDC_ROLE_UAT; - export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token;s + export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token; echo $BITBUCKET_STEP_OIDC_TOKEN > $(pwd)/web-identity-token; export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId'); export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey'); diff --git a/devops-pipeline/terraform-backend-resources/variables.tf b/devops-pipeline/terraform-backend-resources/variables.tf index c806237..71eb690 100644 --- a/devops-pipeline/terraform-backend-resources/variables.tf +++ b/devops-pipeline/terraform-backend-resources/variables.tf @@ -23,3 +23,4 @@ variable "client_name" { type = string default = "infra" } + From 2ea5f07957359b7654df0d97453be0dd6ebcaf64 Mon Sep 17 00:00:00 2001 From: Umang Mistry Date: Thu, 30 May 2024 15:00:05 -0500 Subject: [PATCH 03/37] UAT pipeline update --- bitbucket-pipelines.yml | 6 +++--- devops-pipeline/terraform-backend-resources/variables.tf | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 66cfa53..c74018d 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -204,9 +204,9 @@ pipelines: script: - *aws-context-uat - cd devops-pipeline/terraform-backend-resources/ + - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/devops-pipeline/terraform.tfstate" - terraform validate # Ensure the backend configuration for s3 obj key is correct - - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/devops-pipeline/terraform.tfstate" - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" condition: changesets: @@ -221,9 +221,9 @@ pipelines: script: - *aws-context-uat - cd streamlit-server/ + - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/streamlit-server/terraform.tfstate" - terraform validate # Ensure the backend configuration for s3 obj key is correct - - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/streamlit-server/terraform.tfstate" - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" condition: changesets: @@ -269,9 +269,9 @@ pipelines: script: - *aws-context-uat - cd textract-pipeline/terraform/ + - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/textract-pipeline/terraform.tfstate" - terraform validate # Ensure the backend configuration for s3 obj key is corresponding to the stream name - - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/textract-pipeline/terraform.tfstate" - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" condition: changesets: diff --git a/devops-pipeline/terraform-backend-resources/variables.tf b/devops-pipeline/terraform-backend-resources/variables.tf index 71eb690..c806237 100644 --- a/devops-pipeline/terraform-backend-resources/variables.tf +++ b/devops-pipeline/terraform-backend-resources/variables.tf @@ -23,4 +23,3 @@ variable "client_name" { type = string default = "infra" } - From 5ea0f98b19147f6591e12da0ff68d838f618b70c Mon Sep 17 00:00:00 2001 From: Umang Mistry Date: Thu, 30 May 2024 15:54:52 -0500 Subject: [PATCH 04/37] Updated a lot of terraform code to clean up --- bitbucket-pipelines.yml | 10 +++++++++- devops-pipeline/other-resources/main.tf | 5 +++-- devops-pipeline/other-resources/variables.tf | 20 +++++++++++-------- .../terraform-backend-resources/main.tf | 8 ++++++-- .../aws-terraform-remote-backend/variables.tf | 14 +++++++++++++ .../terraform-backend-resources/variables.tf | 15 ++++++++++++++ textract-pipeline/terraform/main.tf | 7 ++++--- textract-pipeline/terraform/variables.tf | 20 +++++++++++-------- 8 files changed, 75 insertions(+), 24 deletions(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index c74018d..c2420da 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -208,6 +208,14 @@ pipelines: - terraform validate # Ensure the backend configuration for s3 obj key is correct - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" + - export DYNAMODB_TABLE_ARN=$(terraform output -raw terraform_dynamodb_table_arn) + - export DYNAMODB_TABLE_NAME=$(terraform output -raw terraform_dynamodb_table_name) + - export S3_BUCKET_ARN=$(terraform output -raw terraform_s3_bucket_arn) + - export S3_BUCKET_NAME=$(terraform output -raw terraform_s3_bucket_name) + - cd .. && cd other-resources/ + - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=$DYNAMODB_TABLE_NAME" -backend-config="bucket=$S3_BUCKET_NAME" -backend-config="key=terraform/devops-pipeline-other-resources/terraform.tfstate" + - terraform validate + - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" condition: changesets: includePaths: @@ -224,7 +232,7 @@ pipelines: - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/streamlit-server/terraform.tfstate" - terraform validate # Ensure the backend configuration for s3 obj key is correct - - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" + - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" -var "vpc_id=vpc-0392396d0e7bdd77f" condition: changesets: includePaths: diff --git a/devops-pipeline/other-resources/main.tf b/devops-pipeline/other-resources/main.tf index bc17c4e..31aa7df 100644 --- a/devops-pipeline/other-resources/main.tf +++ b/devops-pipeline/other-resources/main.tf @@ -17,8 +17,9 @@ terraform { provider "aws" { - # access_key = var.access_key - # secret_key = var.secret_key + access_key = var.access_key + secret_key = var.secret_key + token = var.token profile = var.aws_profile diff --git a/devops-pipeline/other-resources/variables.tf b/devops-pipeline/other-resources/variables.tf index 1035b4d..19b455c 100644 --- a/devops-pipeline/other-resources/variables.tf +++ b/devops-pipeline/other-resources/variables.tf @@ -4,15 +4,19 @@ variable "aws_profile" { # default = "doczyai" } -# # required -# variable "secret_key" { -# type = string -# } +# required +variable "secret_key" { + type = string +} -# # required -# variable "access_key" { -# type = string -# } +# required +variable "access_key" { + type = string +} + +variable "token" { + type = string +} # required variable "aws_region" { diff --git a/devops-pipeline/terraform-backend-resources/main.tf b/devops-pipeline/terraform-backend-resources/main.tf index 885ad4e..7225313 100644 --- a/devops-pipeline/terraform-backend-resources/main.tf +++ b/devops-pipeline/terraform-backend-resources/main.tf @@ -8,8 +8,9 @@ terraform { } provider "aws" { - profile = var.aws_profile - region = var.aws_region + profile = var.aws_profile + region = var.aws_region + # default_tags { # tags = { # Terraform = "true" @@ -23,4 +24,7 @@ module "aws_terraform_remote_backend" { aws_region = var.aws_region environment = var.environment client_name = var.client_name + secret_key = var.secret_key + access_key = var.access_key + token = var.token } \ No newline at end of file diff --git a/devops-pipeline/terraform-backend-resources/modules/aws-terraform-remote-backend/variables.tf b/devops-pipeline/terraform-backend-resources/modules/aws-terraform-remote-backend/variables.tf index 48cfca4..3483195 100644 --- a/devops-pipeline/terraform-backend-resources/modules/aws-terraform-remote-backend/variables.tf +++ b/devops-pipeline/terraform-backend-resources/modules/aws-terraform-remote-backend/variables.tf @@ -10,6 +10,20 @@ variable "environment" { variable "client_name" { type = string } +# required +variable "secret_key" { + type = string +} + +# required +variable "access_key" { + type = string +} + +variable "token" { + type = string +} + # dynamoDB table variable "terraform_dynamodb_table" { type = object({ diff --git a/devops-pipeline/terraform-backend-resources/variables.tf b/devops-pipeline/terraform-backend-resources/variables.tf index c806237..e0dbad2 100644 --- a/devops-pipeline/terraform-backend-resources/variables.tf +++ b/devops-pipeline/terraform-backend-resources/variables.tf @@ -23,3 +23,18 @@ variable "client_name" { type = string default = "infra" } + +#required +variable "secret_key" { + type = string +} + +#required +variable "access_key" { + type = string +} + +#required +variable "token" { + type = string +} \ No newline at end of file diff --git a/textract-pipeline/terraform/main.tf b/textract-pipeline/terraform/main.tf index 3ab3b0f..d6d192c 100644 --- a/textract-pipeline/terraform/main.tf +++ b/textract-pipeline/terraform/main.tf @@ -70,9 +70,10 @@ locals { provider "aws" { profile = var.aws_profile - # access_key = var.access_key - # secret_key = var.secret_key - + access_key = var.access_key + secret_key = var.secret_key + token = var.token + default_tags { tags = local.common_tags } diff --git a/textract-pipeline/terraform/variables.tf b/textract-pipeline/terraform/variables.tf index 276513c..9d542b0 100644 --- a/textract-pipeline/terraform/variables.tf +++ b/textract-pipeline/terraform/variables.tf @@ -1,12 +1,16 @@ -# # required -# variable "secret_key" { -# type = string -# } +# required +variable "secret_key" { + type = string +} -# # required -# variable "access_key" { -# type = string -# } +# required +variable "access_key" { + type = string +} +#required +variable "token" { + type = string +} # required From 0c7faa8900c06a748b2aae6e6dfa7c317fd6db7b Mon Sep 17 00:00:00 2001 From: Umang Mistry Date: Thu, 30 May 2024 15:57:34 -0500 Subject: [PATCH 05/37] removed aws profile --- devops-pipeline/terraform-backend-resources/variables.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/devops-pipeline/terraform-backend-resources/variables.tf b/devops-pipeline/terraform-backend-resources/variables.tf index e0dbad2..52e0276 100644 --- a/devops-pipeline/terraform-backend-resources/variables.tf +++ b/devops-pipeline/terraform-backend-resources/variables.tf @@ -1,8 +1,8 @@ # required -variable "aws_profile" { - type = string - default = "doczy_uat" -} +# variable "aws_profile" { +# type = string +# # default = "doczy_uat" +# } # required variable "aws_region" { type = string From 110cff16266c6fcfd35580b3315bc29bde39da96 Mon Sep 17 00:00:00 2001 From: Umang Mistry Date: Thu, 30 May 2024 16:11:17 -0500 Subject: [PATCH 06/37] Updated backend config --- devops-pipeline/terraform-backend-resources/main.tf | 2 -- devops-pipeline/terraform-backend-resources/variables.tf | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/devops-pipeline/terraform-backend-resources/main.tf b/devops-pipeline/terraform-backend-resources/main.tf index 7225313..5b22d5c 100644 --- a/devops-pipeline/terraform-backend-resources/main.tf +++ b/devops-pipeline/terraform-backend-resources/main.tf @@ -8,9 +8,7 @@ terraform { } provider "aws" { - profile = var.aws_profile region = var.aws_region - # default_tags { # tags = { # Terraform = "true" diff --git a/devops-pipeline/terraform-backend-resources/variables.tf b/devops-pipeline/terraform-backend-resources/variables.tf index 52e0276..af3c916 100644 --- a/devops-pipeline/terraform-backend-resources/variables.tf +++ b/devops-pipeline/terraform-backend-resources/variables.tf @@ -1,7 +1,7 @@ # required # variable "aws_profile" { # type = string -# # default = "doczy_uat" +# # default = "temp_cred" # } # required variable "aws_region" { @@ -16,7 +16,7 @@ variable "project_name" { # required variable "environment" { type = string - default = "uat" + # default = "uat" } # required variable "client_name" { From 23a58780752107b507ce8a0522dd212addbc0dcd Mon Sep 17 00:00:00 2001 From: Umang Mistry Date: Thu, 30 May 2024 16:14:35 -0500 Subject: [PATCH 07/37] removed backend config for devops --- devops-pipeline/terraform-backend-resources/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devops-pipeline/terraform-backend-resources/variables.tf b/devops-pipeline/terraform-backend-resources/variables.tf index af3c916..6bb02a2 100644 --- a/devops-pipeline/terraform-backend-resources/variables.tf +++ b/devops-pipeline/terraform-backend-resources/variables.tf @@ -37,4 +37,4 @@ variable "access_key" { #required variable "token" { type = string -} \ No newline at end of file +} From 5cd2fad5e5baca1181b3d6094b8f5a19fc02177e Mon Sep 17 00:00:00 2001 From: Umang Mistry Date: Thu, 30 May 2024 16:58:13 -0500 Subject: [PATCH 08/37] Updated pipeline to handle state and then apply --- bitbucket-pipelines.yml | 31 +++++++++++++------ devops-pipeline/other-resources/variables.tf | 1 + .../terraform-backend-resources/outputs.tf | 2 +- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index c2420da..0a7a982 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -200,27 +200,40 @@ pipelines: name: "Plan & Apply DevOps Pipeline" image: hugree/bitbucket-aws-python38-tf154:latest oidc: true - trigger : automatic # Keeping it manual so that it doesnt conflict with existing resources + trigger : automatic script: - *aws-context-uat - cd devops-pipeline/terraform-backend-resources/ - - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/devops-pipeline/terraform.tfstate" + - terraform init - terraform validate # Ensure the backend configuration for s3 obj key is correct - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" - - export DYNAMODB_TABLE_ARN=$(terraform output -raw terraform_dynamodb_table_arn) - - export DYNAMODB_TABLE_NAME=$(terraform output -raw terraform_dynamodb_table_name) - - export S3_BUCKET_ARN=$(terraform output -raw terraform_s3_bucket_arn) - - export S3_BUCKET_NAME=$(terraform output -raw terraform_s3_bucket_name) - - cd .. && cd other-resources/ + - echo "S3_BUCKET=$(terraform output -raw terraform_s3_bucket_name)" >> backend.env + - echo "DYNAMODB_TABLE=$(terraform output -raw terraform_dynamodb_table_name)" >> backend.env + artifacts: + - backend.env + condition: + changesets: + includePaths: + - devops-pipeline/terraform-backend-resources/* + - devops-pipeline/terraform-backend-resources/**/* + - step: + name: "Plan & Apply DevOps Extra Resource Pipeline" + image: hugree/bitbucket-aws-python38-tf154:latest + oidc: true + trigger : automatic # Keeping it manual so that it doesnt conflict with existing resources + script: + - source backend.env + - *aws-context-uat + - cd devops-pipeline/other-resources/ - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=$DYNAMODB_TABLE_NAME" -backend-config="bucket=$S3_BUCKET_NAME" -backend-config="key=terraform/devops-pipeline-other-resources/terraform.tfstate" - terraform validate - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" condition: changesets: includePaths: - - devops-pipeline/terraform-backend-resources/* - - devops-pipeline/terraform-backend-resources/**/* + - devops-pipeline/other-resources/* + - devops-pipeline/other-resources/**/* - step: name: "Plan & Apply Streamlit Pipeline" image: hugree/bitbucket-aws-python38-tf154:latest diff --git a/devops-pipeline/other-resources/variables.tf b/devops-pipeline/other-resources/variables.tf index 19b455c..f236a75 100644 --- a/devops-pipeline/other-resources/variables.tf +++ b/devops-pipeline/other-resources/variables.tf @@ -74,3 +74,4 @@ variable "mwaa_resources_s3_bucket" { } } + diff --git a/devops-pipeline/terraform-backend-resources/outputs.tf b/devops-pipeline/terraform-backend-resources/outputs.tf index 58a5ddb..2192baf 100644 --- a/devops-pipeline/terraform-backend-resources/outputs.tf +++ b/devops-pipeline/terraform-backend-resources/outputs.tf @@ -9,4 +9,4 @@ output "terraform_s3_bucket_arn" { } output "terraform_s3_bucket_name" { value = module.aws_terraform_remote_backend.terraform_s3_bucket_name -} \ No newline at end of file +} From c1d7680456bff32b27d52d32beb2909f2fa84bbd Mon Sep 17 00:00:00 2001 From: Umang Mistry Date: Thu, 30 May 2024 17:28:46 -0500 Subject: [PATCH 09/37] Triggering streamlit pipeline creation --- bitbucket-pipelines.yml | 5 ++--- devops-pipeline/other-resources/variables.tf | 1 - streamlit-server/variables.tf | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 0a7a982..563d1b7 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -223,10 +223,9 @@ pipelines: oidc: true trigger : automatic # Keeping it manual so that it doesnt conflict with existing resources script: - - source backend.env - *aws-context-uat - cd devops-pipeline/other-resources/ - - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=$DYNAMODB_TABLE_NAME" -backend-config="bucket=$S3_BUCKET_NAME" -backend-config="key=terraform/devops-pipeline-other-resources/terraform.tfstate" + - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/devops-pipeline-other-resources/terraform.tfstate" - terraform validate - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" condition: @@ -238,7 +237,7 @@ pipelines: name: "Plan & Apply Streamlit Pipeline" image: hugree/bitbucket-aws-python38-tf154:latest oidc: true - trigger : manual + trigger : automatic script: - *aws-context-uat - cd streamlit-server/ diff --git a/devops-pipeline/other-resources/variables.tf b/devops-pipeline/other-resources/variables.tf index f236a75..19b455c 100644 --- a/devops-pipeline/other-resources/variables.tf +++ b/devops-pipeline/other-resources/variables.tf @@ -74,4 +74,3 @@ variable "mwaa_resources_s3_bucket" { } } - diff --git a/streamlit-server/variables.tf b/streamlit-server/variables.tf index 79f9735..61e5d15 100644 --- a/streamlit-server/variables.tf +++ b/streamlit-server/variables.tf @@ -71,4 +71,4 @@ variable "access_key" { # required variable "token" { type = string -} \ No newline at end of file +} From 6f4789c2438dd546618c8251786a52b9fbad0bd7 Mon Sep 17 00:00:00 2001 From: Umang Mistry Date: Thu, 30 May 2024 17:33:10 -0500 Subject: [PATCH 10/37] removed variable from other backend resources --- devops-pipeline/other-resources/main.tf | 1 - streamlit-server/variables.tf | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/devops-pipeline/other-resources/main.tf b/devops-pipeline/other-resources/main.tf index 31aa7df..7995e4d 100644 --- a/devops-pipeline/other-resources/main.tf +++ b/devops-pipeline/other-resources/main.tf @@ -21,7 +21,6 @@ provider "aws" { secret_key = var.secret_key token = var.token - profile = var.aws_profile default_tags { tags = { diff --git a/streamlit-server/variables.tf b/streamlit-server/variables.tf index 61e5d15..960a3f1 100644 --- a/streamlit-server/variables.tf +++ b/streamlit-server/variables.tf @@ -72,3 +72,4 @@ variable "access_key" { variable "token" { type = string } + From 7c32ffe74cb4ec44ed5268ffe244d93c54ee4c36 Mon Sep 17 00:00:00 2001 From: Umang Mistry Date: Thu, 30 May 2024 17:36:38 -0500 Subject: [PATCH 11/37] removed profile --- devops-pipeline/other-resources/variables.tf | 5 ----- devops-pipeline/terraform-backend-resources/variables.tf | 6 +----- textract-pipeline/terraform/main.tf | 1 - textract-pipeline/terraform/variables.tf | 6 +----- 4 files changed, 2 insertions(+), 16 deletions(-) diff --git a/devops-pipeline/other-resources/variables.tf b/devops-pipeline/other-resources/variables.tf index 19b455c..0d87045 100644 --- a/devops-pipeline/other-resources/variables.tf +++ b/devops-pipeline/other-resources/variables.tf @@ -1,8 +1,3 @@ -# required -variable "aws_profile" { - type = string - # default = "doczyai" -} # required variable "secret_key" { diff --git a/devops-pipeline/terraform-backend-resources/variables.tf b/devops-pipeline/terraform-backend-resources/variables.tf index 6bb02a2..deb30e1 100644 --- a/devops-pipeline/terraform-backend-resources/variables.tf +++ b/devops-pipeline/terraform-backend-resources/variables.tf @@ -1,8 +1,4 @@ -# required -# variable "aws_profile" { -# type = string -# # default = "temp_cred" -# } + # required variable "aws_region" { type = string diff --git a/textract-pipeline/terraform/main.tf b/textract-pipeline/terraform/main.tf index d6d192c..705ffd5 100644 --- a/textract-pipeline/terraform/main.tf +++ b/textract-pipeline/terraform/main.tf @@ -68,7 +68,6 @@ locals { } } provider "aws" { - profile = var.aws_profile access_key = var.access_key secret_key = var.secret_key diff --git a/textract-pipeline/terraform/variables.tf b/textract-pipeline/terraform/variables.tf index 9d542b0..9ce3db4 100644 --- a/textract-pipeline/terraform/variables.tf +++ b/textract-pipeline/terraform/variables.tf @@ -13,11 +13,7 @@ variable "token" { } -# required -variable "aws_profile" { - type = string - # default = "doczyai" -} + # required variable "aws_region" { type = string From a102e3fba524b929a78bd569c3565030b1d3da81 Mon Sep 17 00:00:00 2001 From: Umang Mistry Date: Thu, 30 May 2024 17:44:13 -0500 Subject: [PATCH 12/37] Trigger pipeline --- devops-pipeline/other-resources/variables.tf | 1 + streamlit-server/variables.tf | 1 + 2 files changed, 2 insertions(+) diff --git a/devops-pipeline/other-resources/variables.tf b/devops-pipeline/other-resources/variables.tf index 0d87045..db549da 100644 --- a/devops-pipeline/other-resources/variables.tf +++ b/devops-pipeline/other-resources/variables.tf @@ -69,3 +69,4 @@ variable "mwaa_resources_s3_bucket" { } } + diff --git a/streamlit-server/variables.tf b/streamlit-server/variables.tf index 960a3f1..f9b2c57 100644 --- a/streamlit-server/variables.tf +++ b/streamlit-server/variables.tf @@ -73,3 +73,4 @@ variable "token" { type = string } + From 7d04a95c6652e0f7994f2d366766203c8edf6175 Mon Sep 17 00:00:00 2001 From: Umang Mistry Date: Thu, 30 May 2024 18:08:02 -0500 Subject: [PATCH 13/37] Updated logic to handle SSL certs --- streamlit-server/alb.tf | 6 +++--- streamlit-server/variables.tf | 11 ++++++++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/streamlit-server/alb.tf b/streamlit-server/alb.tf index a94a31c..6b9f8a3 100644 --- a/streamlit-server/alb.tf +++ b/streamlit-server/alb.tf @@ -128,7 +128,7 @@ resource "aws_lb_listener" "https_8501" { port = "8501" protocol = "HTTPS" ssl_policy = "ELBSecurityPolicy-2016-08" - certificate_arn = var.acm_arn_dev + certificate_arn = lookup(var.acm_arns, var.environment) default_action { type = "forward" @@ -141,7 +141,7 @@ resource "aws_lb_listener" "https_8502" { port = "8502" protocol = "HTTPS" ssl_policy = "ELBSecurityPolicy-2016-08" - certificate_arn = var.acm_arn_dev + certificate_arn = lookup(var.acm_arns, var.environment) default_action { type = "forward" @@ -154,7 +154,7 @@ resource "aws_lb_listener" "https_8503" { port = "8503" protocol = "HTTPS" ssl_policy = "ELBSecurityPolicy-2016-08" - certificate_arn = var.acm_arn_dev + certificate_arn = lookup(var.acm_arns, var.environment) default_action { type = "forward" diff --git a/streamlit-server/variables.tf b/streamlit-server/variables.tf index f9b2c57..268314e 100644 --- a/streamlit-server/variables.tf +++ b/streamlit-server/variables.tf @@ -12,7 +12,7 @@ variable "project_name" { # required variable "environment" { type = string - default = "dev" + # default = "dev" } @@ -59,6 +59,15 @@ variable "acm_arn_dev" { default = "arn:aws:acm:us-east-2:660131068782:certificate/947e04b3-be22-43bf-bea2-a0ad3b221d70" } +variable "acm_arns" { + type = map(string) + default = { + dev = "arn:aws:acm:us-east-2:660131068782:certificate/947e04b3-be22-43bf-bea2-a0ad3b221d70" + uat = "arn:aws:acm:us-east-2:975049960860:certificate/e036e76d-1f4f-496e-ac97-5f65ec4a34cc" + prod = "arn:aws:acm:region:account-id:certificate/prod-cert-id" + } +} + variable "secret_key" { type = string } From aa81ee96ba6f6ef485d3e6a2455fd48bff5aa275 Mon Sep 17 00:00:00 2001 From: Umang Mistry Date: Thu, 30 May 2024 18:17:09 -0500 Subject: [PATCH 14/37] Updated ui0 listner --- streamlit-server/alb.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/streamlit-server/alb.tf b/streamlit-server/alb.tf index 6b9f8a3..48b6bdb 100644 --- a/streamlit-server/alb.tf +++ b/streamlit-server/alb.tf @@ -115,7 +115,8 @@ resource "aws_lb_listener" "https_8500" { protocol = "HTTPS" ssl_policy = "ELBSecurityPolicy-2016-08" # Using variable for ARN - certificate_arn = var.acm_arn_dev + certificate_arn = lookup(var.acm_arns, var.environment) + default_action { type = "forward" From c4294d607e915bac48c182ec50876e29499c817d Mon Sep 17 00:00:00 2001 From: Umang Mistry Date: Thu, 30 May 2024 18:31:40 -0500 Subject: [PATCH 15/37] Updated key name for ec2 creation --- streamlit-server/main.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/streamlit-server/main.tf b/streamlit-server/main.tf index 1fd8316..b2016ed 100644 --- a/streamlit-server/main.tf +++ b/streamlit-server/main.tf @@ -92,7 +92,7 @@ resource "aws_instance" "streamlit_server" { vpc_security_group_ids = data.aws_security_groups.security_groups.ids iam_instance_profile = aws_iam_instance_profile.ec2_instance_profile.name associate_public_ip_address = false - key_name = "tf-test-key" # Created using TF and uploaded to S3. Prerequisite + key_name = "tf-uat-key" # Created using TF and uploaded to S3. Prerequisite root_block_device { volume_size = 30 # Variable @@ -202,3 +202,4 @@ resource "aws_instance" "streamlit_server" { } + From f79e066c8f8d176d98a8963d0186eec09834861d Mon Sep 17 00:00:00 2001 From: Umang Mistry Date: Thu, 30 May 2024 18:36:31 -0500 Subject: [PATCH 16/37] Updated the ami to fix the error --- streamlit-server/main.tf | 2 +- streamlit-server/variables.tf | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/streamlit-server/main.tf b/streamlit-server/main.tf index b2016ed..5ec66d2 100644 --- a/streamlit-server/main.tf +++ b/streamlit-server/main.tf @@ -86,7 +86,7 @@ resource "aws_iam_instance_profile" "ec2_instance_profile" { } resource "aws_instance" "streamlit_server" { - ami = var.ubuntu_ami + ami = lookup(var.ubuntu_ami, var.environment) instance_type = var.ec2_instance_type subnet_id = data.aws_subnets.subnets.ids[0] vpc_security_group_ids = data.aws_security_groups.security_groups.ids diff --git a/streamlit-server/variables.tf b/streamlit-server/variables.tf index 268314e..2d35966 100644 --- a/streamlit-server/variables.tf +++ b/streamlit-server/variables.tf @@ -31,10 +31,18 @@ variable "vpc_id" { type = string } -variable "ubuntu_ami" { - type = string - default = "ami-0de7e97fedfbc6ef6" # Switch to this ami-0b8b44ec9a8f90422 +# variable "ubuntu_ami" { +# type = string +# default = "ami-0de7e97fedfbc6ef6" # Switch to this ami-0b8b44ec9a8f90422 +# } + +variable "ubuntu_ami" { + type = map(string) + default = { + dev = "ami-0de7e97fedfbc6ef6" + uat = "ami-0b8b44ec9a8f90422" + } } variable "ec2_instance_type" { From 810eb2e98b198e3e287d0ab86e1142948e878405 Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Fri, 31 May 2024 16:17:52 -0500 Subject: [PATCH 17/37] Created a new version of interface 1 with tickets implemented and some processes smoothened --- streamlit/new_interface_1.py | 233 +++++++++++++++++++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100644 streamlit/new_interface_1.py diff --git a/streamlit/new_interface_1.py b/streamlit/new_interface_1.py new file mode 100644 index 0000000..28e562c --- /dev/null +++ b/streamlit/new_interface_1.py @@ -0,0 +1,233 @@ +import streamlit as st +from streamlit_extras.add_vertical_space import add_vertical_space +import os +import streamlit as st +import pandas as pd +from io import StringIO +from datetime import datetime +import boto3 +# import util +import requests +import time +# from sf_conn import get_client_names, get_secret, save_to_sf +# from constants import USER_LIST, DOCZY_PIPELINE_URL_DEV + +# doczy_pipeline = DOCZY_PIPELINE_URL_DEV + +# REDIRECT_URI = 'https://doczydev.aarete.com:8501' +# user_list = USER_LIST +st.set_page_config(layout = "wide") +# # # Sidebar contents +# # with st.sidebar: +# # st.title("Doczy.AI ™") +# # st.markdown( +# # """ +# # ## About +# # This app extracts data from contracts + +# # """ +# # ) +# # add_vertical_space(15) +# # # st.write("Doczy") + +_,c1= st.columns([5,1]) +# try: +# util.setup_page(REDIRECT_URI) +# except: +# st.write("SSO Failed") +# st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} +st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} +try: + c1.write(f"User: **{st.session_state.user_info['displayName']}**") + user_mail = st.session_state.user_info['mail'] +except KeyError as e: + # Do we add a link to get to the login page here? + st.write("Session Expired.") + st.stop() + + + +s3_client = boto3.client('s3', + region_name="us-east-2", +) + +# # to be replaced with snowflake data +client_list = ['doczy-ai-client-1', 'Delaware First Health, Inc.', 'Community Health Choice, Inc','CareSource Network Partners LLC', + 'HealthNet of Cali', 'Oklahoma Complete Health, Inc', 'HealthFirst', 'Molina Healthcare of TX', 'AvMed', 'Arizona Care1st', + 'WellCare New Jersey'] + +# client_list, s3_paths = get_client_names() +# client_s3_paths = dict(zip(client_list, s3_paths)) + +client_row = st.columns([0.1, 0.8]) +with client_row[0]: + st.write("**Client Name**") +with client_row[1]: + client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344 + +# client_bucket = client_s3_paths.get(client) + +# # to be deleted when buckets for different clients are ready; below line is added only for testing the corresponding DAG +client_bucket = 'doczy-ai-client-1' + +# batch_objects = s3_client.list_objects_v2(Bucket=client_bucket +# , Prefix="contracts_landing_zone/", Delimiter='/') + +# batch_list = [] +# for prefix in batch_objects['CommonPrefixes']: +# batch_list.append(prefix['Prefix'][:-1].split('/')[-1]) + +batch_list = ['batch_020524103737', 'batch_090524131433', 'batch_090524131607', 'batch_100524123000', 'batch_130524064322', + 'batch_160524071331', 'batch_200524213550', 'batch_250424112237', 'batch_280524120530', 'batch_280524121721', 'batch_280524144222', + 'batch_290524123926', 'batch_290524164044', 'batch_310524102029', 'batch_310524124050', 'batch_310524162346', 'batch_310524162631'] + +path_row = st.columns([0.1, 0.8]) +with path_row[0]: + st.write("**Batch ID**") +with path_row[1]: + batch_id = st.selectbox('**Batch ID**', batch_list, label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344 + +if not batch_id: + batch_id = "None" + +checks = st.columns([0.1, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12]) +with checks[0]: + st.write("**Group No.**") + +with checks[1]: + a = st.checkbox('Unique Key', key = str(1), args="Unique") +with checks[2]: + b = st.checkbox('Pricing Before Carveouts', key = str(2)) +with checks[3]: + c = st.checkbox('Contract Related', key = str(3)) +with checks[4]: + d = st.checkbox('Provider', key = str(4)) +with checks[5]: + e = st.checkbox('Timeline', key = str(5)) +with checks[6]: + f = st.checkbox('Carveout Indicator', key = str(6)) +with checks[7]: + g = st.checkbox('Carveout Methodology', key = str(7)) + +add_vertical_space(1) + +df = pd.DataFrame(columns=['Contract Name', 'Unique Key','Pricing Before Carveouts' + , 'Contract Related', 'Provider', 'Timeline', 'Carveout Indicator', 'Carveout Methodology']) +# file_list = [] +# file_objects = s3_client.list_objects_v2(Bucket=client_bucket +# , Prefix="contracts_landing_zone/"+batch_id+"/", Delimiter='/') +file_list = ['Boilerplate_TX Amendment Mission Health Network effective_040114 MU.pdf', 'Custom_TX - MP AMENDMENT - MISSION HEALTH NETWORK - MU.pdf', + 'Delaware First Health_First State Homecare Agency_212260_7 MU.pdf', 'Molina Healthcare of Texas, Inc. Amendment 4 - HIX ACA__EFF 01012016_MU.pdf'] + +if st.button("Read the contracts from Path"): + # for obj in file_objects.get('Contents',[]): + # if not obj['Key'].endswith('/'): + # file_list.append(obj['Key'].split('/')[-1]) + df['Contract Name'] = file_list + # df['Request ID'] = range(len(file_list)) + # df['Contract ID'] = file_list + df['Unique Key'] = a + df['Pricing Before Carveouts'] = b + df['Contract Related'] = c + df['Provider'] = d + df['Timeline'] = e + df['Carveout Indicator'] = f + df['Carveout Methodology'] = g + dir_path = os.path.dirname(os.path.realpath(__file__)) + print(f'DEBUGGING: PWD= {dir_path}') + df.to_csv('temp1.csv', index=False) + +add_vertical_space(1) +df2 = pd.read_csv('temp1.csv') +edited_df = st.data_editor(df2) + +edited_df['REQUEST_USER'] = user_mail +edited_df['LATEST_FLAG BOOLEAN'] = True +edited_df['PIPELINE_KICKOFF_DATETIME'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") +edited_df['REQUEST_DATETIME'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + +@st.cache_data +def convert_df(df): + return df.to_csv(index=False).encode('utf-8') + +csv = convert_df(edited_df) +# edited_df = edited_df.reset_index() # make sure indexes pair with number of rows + +# additional_info = pd.DataFrame(columns=['REQUEST_ID','T_DRIVE_PATH','CLIENT_NAME' +# , 'GROUP_NAME', 'REQUEST_USERNAME', 'REQUEST_DATETIME']) +additional_info = pd.DataFrame(columns=['CLIENT_NAME', 'BATCH_ID', 'REQUEST_USERNAME', 'REQUEST_DATETIME']) +additional_info.loc[0] = [client, batch_id, st.session_state.user_info['mail'], datetime.now().strftime("%Y-%m-%d %H:%M:%S")] +st.write(additional_info) + +st.session_state.contract_count = 0 +contract_list = [] +for index, row in edited_df.iterrows(): + allow_run_for_contract = False + group_list = [] + if row['Unique Key']: + group_list.append('Unique Key') + allow_run_for_contract = True + if row['Pricing Before Carveouts']: + group_list.append('Pricing Before Carveouts') + allow_run_for_contract = True + if row['Contract Related']: + group_list.append('Contract Related') + allow_run_for_contract = True + if row['Provider']: + group_list.append('Provider') + allow_run_for_contract = True + if row['Timeline']: + group_list.append('Timeline') + allow_run_for_contract = True + if row['Carveout Indicator']: + group_list.append('Carveout Indicator') + allow_run_for_contract = True + if row['Carveout Methodology']: + group_list.append('Carveout Methodology') + allow_run_for_contract = True + if allow_run_for_contract: st.session_state.contract_count += 1 + entry_dict = { + "contract_name": row['Contract Name'], + "groups": group_list, + "contract_source_path": "contracts_landing_zone/"+batch_id+"/"+row['Contract Name'] + } + contract_list.append(entry_dict) + +myobj = { + "s3_bucket": client_bucket, + "batch_id": batch_id, + "client_name": client, + "username": user_mail, + "contract_list": contract_list +} + +print(st.session_state) + +buttons = st.columns([0.8, 0.2]) +with buttons[0]: + st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv') +with buttons[1]: + if st.button("Run Doczy.AI Pipeline", disabled= not st.session_state.contract_count == len(edited_df)): + with st.spinner('Running...'): # Feedback to User while API endpoint sends response for DOC-342 + # csv_buf = StringIO() + # additional_info.to_csv(csv_buf, header=True, index=False) + # csv_buf.seek(0) + # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/request_submission.csv') + # csv_buf = StringIO() + # edited_df.to_csv(csv_buf, header=True, index=False) + # csv_buf.seek(0) + # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/contract_config.csv') + # try: + # save_to_sf('load_request_and_contract_submissions', request_submission_file_name = "request_submission.csv", contract_config_file_name = "contract_config.csv") + # except Exception as e: + # st.write(e) + # response = requests.post(doczy_pipeline, json = myobj) + # if response.status_code >= 200 and response.status_code < 300: + # st.write("Success") + # # st.write(myobj) + # else: + # st.write("Failed") + # st.write(response.text) + time.sleep(5) + st.write("Success!") + From c8389fca0a20c5d584403b350dbaef2aaeaef1db Mon Sep 17 00:00:00 2001 From: Umang Mistry Date: Fri, 31 May 2024 16:30:58 -0500 Subject: [PATCH 18/37] Triggering snowflake db pipeline --- snowflake/UAT/R__1002_LOG_AUDIT_TABLE.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snowflake/UAT/R__1002_LOG_AUDIT_TABLE.sql b/snowflake/UAT/R__1002_LOG_AUDIT_TABLE.sql index 1cc0768..d3ce540 100644 --- a/snowflake/UAT/R__1002_LOG_AUDIT_TABLE.sql +++ b/snowflake/UAT/R__1002_LOG_AUDIT_TABLE.sql @@ -8,4 +8,4 @@ create or replace TABLE STG.LOG_AUDIT ( SUB_SECTION_NAME VARCHAR(255) COLLATE 'en-ci', AUDIT_SID NUMBER(38,0), TOTAL_MIN NUMBER(38,0) -); \ No newline at end of file +); From 749d34dc8c456890036067dddccfb8ab4132a1f8 Mon Sep 17 00:00:00 2001 From: Umang Mistry Date: Fri, 31 May 2024 17:06:25 -0500 Subject: [PATCH 19/37] Updated SP to load contract config for UAT --- .../UAT/config_interface/R__1004_LOAD_CONTRACT_CONFIG_SP.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/snowflake/UAT/config_interface/R__1004_LOAD_CONTRACT_CONFIG_SP.sql b/snowflake/UAT/config_interface/R__1004_LOAD_CONTRACT_CONFIG_SP.sql index 4e5562c..bc09320 100644 --- a/snowflake/UAT/config_interface/R__1004_LOAD_CONTRACT_CONFIG_SP.sql +++ b/snowflake/UAT/config_interface/R__1004_LOAD_CONTRACT_CONFIG_SP.sql @@ -50,8 +50,7 @@ BEGIN contract_name := (SELECT NULLIF(TRIM($1),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1); batch_id := (SELECT NULLIF(TRIM($2),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1); request_datetime := (SELECT NULLIF(TRIM($13),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1); - - UPDATE TABLE STG.CONTRACT_CONFIG + UPDATE STG.CONTRACT_CONFIG SET LATEST_FLAG = FALSE WHERE BATCH_ID = :batch_id AND CONTRACT_NAME = :contract_name AND REQUEST_DATETIME < :request_datetime AND LATEST_FLAG = TRUE; From 25224ecfee0a4f5e9da74729140eacba287cbaea Mon Sep 17 00:00:00 2001 From: Umang Mistry Date: Fri, 31 May 2024 17:11:39 -0500 Subject: [PATCH 20/37] Updates to load contract config sp --- .../UAT/config_interface/R__1004_LOAD_CONTRACT_CONFIG_SP.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/snowflake/UAT/config_interface/R__1004_LOAD_CONTRACT_CONFIG_SP.sql b/snowflake/UAT/config_interface/R__1004_LOAD_CONTRACT_CONFIG_SP.sql index bc09320..e00a89c 100644 --- a/snowflake/UAT/config_interface/R__1004_LOAD_CONTRACT_CONFIG_SP.sql +++ b/snowflake/UAT/config_interface/R__1004_LOAD_CONTRACT_CONFIG_SP.sql @@ -50,6 +50,7 @@ BEGIN contract_name := (SELECT NULLIF(TRIM($1),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1); batch_id := (SELECT NULLIF(TRIM($2),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1); request_datetime := (SELECT NULLIF(TRIM($13),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1); + UPDATE STG.CONTRACT_CONFIG SET LATEST_FLAG = FALSE WHERE BATCH_ID = :batch_id AND CONTRACT_NAME = :contract_name AND REQUEST_DATETIME < :request_datetime AND LATEST_FLAG = TRUE; From e0c8dc2649861b0bbad3a51752e00d054f7f1fa8 Mon Sep 17 00:00:00 2001 From: Umang Mistry Date: Fri, 31 May 2024 17:15:08 -0500 Subject: [PATCH 21/37] Updates to SP --- .../DEV/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snowflake/DEV/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql b/snowflake/DEV/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql index 4e5562c..bd1b22f 100644 --- a/snowflake/DEV/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql +++ b/snowflake/DEV/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql @@ -51,7 +51,7 @@ BEGIN batch_id := (SELECT NULLIF(TRIM($2),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1); request_datetime := (SELECT NULLIF(TRIM($13),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1); - UPDATE TABLE STG.CONTRACT_CONFIG + UPDATE STG.CONTRACT_CONFIG SET LATEST_FLAG = FALSE WHERE BATCH_ID = :batch_id AND CONTRACT_NAME = :contract_name AND REQUEST_DATETIME < :request_datetime AND LATEST_FLAG = TRUE; From d69c21b67bae98070eac5300257bb3c1b006a6c7 Mon Sep 17 00:00:00 2001 From: Umang Mistry Date: Fri, 31 May 2024 17:31:44 -0500 Subject: [PATCH 22/37] Triggering textract pipeline --- textract-pipeline/terraform/variables.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/textract-pipeline/terraform/variables.tf b/textract-pipeline/terraform/variables.tf index 9ce3db4..ed57e79 100644 --- a/textract-pipeline/terraform/variables.tf +++ b/textract-pipeline/terraform/variables.tf @@ -13,7 +13,6 @@ variable "token" { } - # required variable "aws_region" { type = string From 70b83a99b39a92903ffef27824bebf186006c605 Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Mon, 3 Jun 2024 10:24:38 -0500 Subject: [PATCH 23/37] Created a new version of interface 0 with tickets implemented and some processes smoothened --- streamlit/new_interface_0.py | 186 +++++++++++++++++++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100644 streamlit/new_interface_0.py diff --git a/streamlit/new_interface_0.py b/streamlit/new_interface_0.py new file mode 100644 index 0000000..eba8f5d --- /dev/null +++ b/streamlit/new_interface_0.py @@ -0,0 +1,186 @@ +import json +import streamlit as st +from streamlit_extras.add_vertical_space import add_vertical_space +import os +import streamlit as st +import pandas as pd +from io import StringIO +from datetime import datetime +import boto3 +# import util +import requests +# from sf_conn import get_secret, save_to_sf +from io import StringIO, BytesIO +# from sf_conn import get_client_names, insert_upload_logs +# from constants import USER_LIST + +create_batch_url = 'https://lfksus2t62.execute-api.us-east-2.amazonaws.com/dev/create-batch' + +# REDIRECT_URI = 'https://doczydev.aarete.com:8500' +# user_list = USER_LIST + +def insert_upload_logs(batch_id, client_name, file_name, upload_datetime, upload_user): + """ + Input: batch_id, client_name, file_name, upload_datetime, upload_user + Output: status of the insert query + """ + try: + return 'Log inserted successfully' + except Exception as e: + return e + + +st.set_page_config(layout = "wide") +# # # Sidebar contents +# # with st.sidebar: +# # st.title("Doczy.AI ™") +# # st.markdown( +# # """ +# # ## About +# # This app extracts data from contracts + +# # """ +# # ) +# # add_vertical_space(15) +# # # st.write("Doczy") + +_,c1= st.columns([5,1]) +# try: +# util.setup_page(REDIRECT_URI) +# except Exception as e: +# st.write(f"SSO Failed = {e}") +# st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} +st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} +try: + c1.write(f"User: **{st.session_state.user_info['displayName']}**") + user_mail = st.session_state.user_info['mail'] +except KeyError as e: + st.write("Session Expired.") + st.stop() + + + +s3_client = boto3.client('s3', + region_name="us-east-2", +) + +client_list = ['doczy-ai-client-1', 'Delaware First Health, Inc.', 'Community Health Choice, Inc','CareSource Network Partners LLC', + 'HealthNet of Cali', 'Oklahoma Complete Health, Inc', 'HealthFirst', 'Molina Healthcare of TX', 'AvMed', 'Arizona Care1st', + 'WellCare New Jersey'] + +# This is the list of client fetched from Snowflake +# TODO: Need to update the streamlit code to use the client names from this list +# And use the s3 paths to save the objects for the respective client +# client_list, s3_paths = get_client_names() +# client_s3_paths = dict(zip(client_list, s3_paths)) + +client_row = st.columns([0.1, 0.8]) +with client_row[0]: + st.write("**Client Name**") +with client_row[1]: + client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344 + +# client_bucket = client_s3_paths.get(client) + +# to be deleted when client buckets are created +client_bucket = 'doczy-ai-client-1' + +file_row = st.columns([0.1, 0.8]) +with file_row[0]: + st.write("**Upload Files**") +with file_row[1]: + file_list = st.file_uploader("Upload", type=['csv','pdf'], accept_multiple_files=True, label_visibility = "collapsed", help="Only .csv and .pdf file formats are supported.") + +add_vertical_space(2) +df = pd.DataFrame(columns=['Contract Name']) +df['Contract Name'] = file_list +file_names = [] +buttons = st.columns([0.4, 0.4, 0.2]) +with buttons[1]: + if st.button("Create Batch"): + if client == None: + st.error("No Client Name Selected.") + elif len(file_list) == 0: + st.error("No Files Selected.") + else: + + myobj = { "client-bucket-name": client_bucket } + response = requests.post(create_batch_url, json = myobj) + if response.status_code >= 200 and response.status_code < 300: + try: + batch_id = json.loads(json.loads(response.text)['body'])['batch_id'] + landing_zone = json.loads(json.loads(response.text)['body'])['landing_zone'] + except: + st.write(myobj) + st.write(response.text) + batch_id = 'failed_cases' + landing_zone = 'contracts_landing_zone' + else: + st.write("Failed") + for uploaded_file in file_list: + stringio = BytesIO(uploaded_file.getvalue()) + stringio.seek(0) + s3_client.put_object(Bucket=client_bucket, Body=stringio.getvalue(), Key= + landing_zone+batch_id+'/'+str(uploaded_file.name)) + + # TODO: Test this insert function with snowflake + upload_log = insert_upload_logs(batch_id, client, str(uploaded_file.name), datetime.now().strftime("%Y-%m-%d %H:%M:%S"), user_mail) + st.write(upload_log) + + file_names.append(str(uploaded_file.name)) + st.write(f"{batch_id} created") + st.write(f"Files uploaded to s3://{client_bucket}/{landing_zone}{batch_id}") + + + # @st.cache_data + # def convert_df(df): + # return df.to_csv(index=False).encode('utf-8') + + # csv = convert_df(df) + + # df = df.reset_index() # make sure indexes pair with number of rows + + # contract_list = [] + # for index, row in df.iterrows(): + # group_list = [] + # if row['Unique Key']: + # group_list.append('Unique Key') + # if row['Pricing Before Carveouts']: + # group_list.append('Pricing Before Carveouts') + # if row['Contract Related']: + # group_list.append('Contract Related') + # if row['Provider']: + # group_list.append('Provider') + # if row['Timeline']: + # group_list.append('Timeline') + # if row['Carveout Indicator']: + # group_list.append('Carveout Indicator') + # if row['Carveout Methodology']: + # group_list.append('Carveout Methodology') + # entry_dict = { + # "contract_name": row['Contract Name'], + # "groups": group_list, + # "contract_source_path": "batches/batch_1/"+client+"/"+row['Contract Name'] + # } + # contract_list.append(entry_dict) + # contract_list = list(df['Contract Name']) + + # myobj = { + # "s3_bucket": 'doczy-dev-infra-textract', + # "batch_id": "1", + # "client_name": client, + # "username": user_mail, + # "contract_list": contract_list + # } + + # buttons = st.columns([0.8, 0.2]) + # with buttons[0]: + # st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv') + # with buttons[1]: + # if st.button("Upload to DB"): + # response = requests.post(doczy_pipeline, json = myobj) + # if response.status_code >= 200 and response.status_code < 300: + # st.write("Success") + # else: + # st.write("Failed") + # # st.write(response.text) From 31683043d7e5133cd188ef8f359dd150e9461fb8 Mon Sep 17 00:00:00 2001 From: Umang Mistry Date: Mon, 3 Jun 2024 16:25:37 -0500 Subject: [PATCH 24/37] Updating streamlit code in UAT --- bitbucket-pipelines.yml | 4 ++-- streamlit/interface_0.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 563d1b7..c3366bd 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -262,12 +262,12 @@ pipelines: image: python:3.8 name: "Deploy streamlit to EC2" oidc: true - trigger: manual # Keeping it manual as the server will be created first and then referenced in this + trigger: automatic # Keeping it manual as the server will be created first and then referenced in this script: - python -m pip install --upgrade pip - apt-get update && apt-get install -y jq git - pip install awscli - - *aws-context-dev + - *aws-context-uat - *ssm-send-command-uat condition: changesets: diff --git a/streamlit/interface_0.py b/streamlit/interface_0.py index f638209..b381247 100644 --- a/streamlit/interface_0.py +++ b/streamlit/interface_0.py @@ -114,6 +114,7 @@ with buttons[1]: st.write(f"Files uploaded to s3://{client_bucket}/{landing_zone}{batch_id}") + # @st.cache_data # def convert_df(df): # return df.to_csv(index=False).encode('utf-8') From 7027c9649da880514c508ae8939b898caa2bc663 Mon Sep 17 00:00:00 2001 From: "AARETE\\agupta" Date: Tue, 4 Jun 2024 08:48:45 -0500 Subject: [PATCH 25/37] Created a new version of interface 2, under /local/ that can be run locally and has a version of the PDF viewer implemented --- streamlit/local/local_interface_2.py | 228 +++++++++++++++++++++++++++ 1 file changed, 228 insertions(+) create mode 100644 streamlit/local/local_interface_2.py diff --git a/streamlit/local/local_interface_2.py b/streamlit/local/local_interface_2.py new file mode 100644 index 0000000..7e40615 --- /dev/null +++ b/streamlit/local/local_interface_2.py @@ -0,0 +1,228 @@ +import json + +import boto3 +from langchain.prompts import PromptTemplate +from langchain.embeddings.bedrock import BedrockEmbeddings +from langchain.llms.bedrock import Bedrock +from langchain_community.vectorstores import Chroma +# from constants import CHROMA_SETTINGS, EMBEDDING_MODEL_NAME, PERSIST_DIRECTORY, MODEL_ID, MODEL_BASENAME, SOURCE_DIRECTORY, USER_LIST +from langchain.chains import RetrievalQA + +import streamlit as st +from streamlit_extras.add_vertical_space import add_vertical_space +from streamlit_pdf_viewer import pdf_viewer +import os +import pandas as pd +import numpy as np +# import util +import anthropic +from pydantic import BaseModel +from typing import List +import re +import base64 +# from sf_conn import get_snowflake_conn + +REDIRECT_URI = 'https://doczydev.aarete.com:8502' +# user_list = USER_LIST + +st.set_page_config(layout = "wide") +# Sidebar contents +# with st.sidebar: +# st.title("Doczy.AI ™") +# st.markdown( +# """ +# ## About +# This app extracts data from contracts + +# """ +# ) +# add_vertical_space(15) +# # st.write("Doczy") + +_,c1= st.columns([5,1]) +# try: +# util.setup_page(REDIRECT_URI) +# except: +# st.write("SSO Failed") +# st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} +st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} +try: + c1.write(f"User: **{st.session_state.user_info['displayName']}**") + user_mail = st.session_state.user_info['mail'] +except KeyError as e: + st.write("Session Expired.") + st.stop() + +# remove below try except statement if comparison with actual vales is not required +# try: +# conn = get_snowflake_conn('STG') +# cur = conn.cursor() +# query = 'select * from "TRAINING_DATA_RAW"' +# cur.execute(query) +# field_values = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description]) +# field_values['Document_Name'] = field_values['DOCUMENT_NAME'] +# except: +# # field_values = pd.read_csv('contract_field_values.csv', encoding='unicode_escape', skipinitialspace=True) +# # field_values = field_values.loc[:, ~field_values.columns.str.contains('Unnamed:')] +# st.write("Conn failed, unable to fetch data from training data table in Snowflake") + +field_values = pd.read_csv('contract_field_values.csv', encoding='unicode_escape', skipinitialspace=True) +field_values = field_values.loc[:, ~field_values.columns.str.contains('Unnamed:')] + +# try: +# query = 'select * from "PROMPT_CONFIG"' +# cur.execute(query) +# fields = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description]) + +# fields.rename(columns={'FIELD_DESC': 'Field Name'}, inplace = True) +# fields.rename(columns={'PROMPT': 'Interrogation Question?'}, inplace = True) +# fields.rename(columns={'GROUP_ID': 'PRIORITY'}, inplace = True) +# fields.rename(columns={'FIELD_NAME': 'SF_DB_COL_NAME'}, inplace = True) +# fields.rename(columns={'FM_MODEL_ID': 'llm_selected'}, inplace = True) +# except Exception as e: +# st.write("Unable to fetch data from Snowflake: ",e) +# # fields = pd.read_csv('contract_fields.csv', encoding='unicode_escape', skipinitialspace=True) +# # fields = fields[~fields['SF_COL_NAME'].str.endswith('_PG', na=None)] + +fields = pd.read_csv('contract_fields.csv', encoding='unicode_escape', skipinitialspace=True) +# fields = fields[fields['SF_DB_COL_NAME'].str.endswith('_PG', na=None)] + +# change the code below if contract list is fetched from snowflake +s3_client = boto3.client('s3', + region_name="us-east-2" +) +bucket = 'doczy-dev-infra-textract' +# objects = s3_client.list_objects_v2(Bucket=bucket, Prefix="training-data/") +# file_list = [] +# for obj in objects['Contents']: +# if not obj['Key'].endswith('/'): +# file_list.append(obj['Key']) + +file_list = ['Contract_Training_Exercise_Pricing.pdf', 'Contract_Training_Exercise_SLA.pdf'] +contract_list = sorted(file_list) + +file_row = st.columns([0.2, 0.7, 0.1]) +with file_row[0]: + st.write("**Contract Name**") +with file_row[1]: + file_name = st.selectbox('Select a file', ['All'] + contract_list, label_visibility = "collapsed", index= None) # MODIFIED - Append 'All' in the front instead of at the end + +field_row = st.columns([0.2, 0.7, 0.1]) +with field_row[0]: + st.write("**Field Group**") +with field_row[1]: + field_group = st.selectbox('Field Group',('Unique Key', 'Contract Related', 'Pricing Before Carveouts - I' + , 'Pricing Before Carveouts - II', 'Carveout Indicator, Code Type and Code #s - I' + , 'Carveout Indicator, Code Type and Code #s - II', 'Carveout Indicator, Code Type and Code #s - III' + , 'Optimize Carving Indic.', 'Carveout Method - I', 'Carveout Method - II', 'Provider' + , 'Timeline'), label_visibility = "collapsed", index = None) + +if field_group == 'Unique Key': + fields = fields[fields['PRIORITY'] == 'A'] +elif field_group == 'Contract Related': + fields = fields[fields['PRIORITY'] == 'C'] +elif field_group == 'Pricing Before Carveouts - I': + fields = fields[fields['PRIORITY'] == 'B'] + fields = np.array_split(fields, 2)[0] +elif field_group == 'Pricing Before Carveouts - II': + fields = fields[fields['PRIORITY'] == 'B'] + fields = np.array_split(fields, 2)[1] +elif field_group == 'Carveout Indicator, Code Type and Code #s - I': + fields = fields[fields['PRIORITY'] == 'F'] + fields = np.array_split(fields, 3)[0] +elif field_group == 'Carveout Indicator, Code Type and Code #s - II': + fields = fields[fields['PRIORITY'] == 'F'] + fields = np.array_split(fields, 3)[1] +elif field_group == 'Carveout Indicator, Code Type and Code #s - III': + fields = fields[fields['PRIORITY'] == 'F'] + fields = np.array_split(fields, 3)[2] +elif field_group == 'Carveout Methodology - I': + fields = fields[fields['PRIORITY'] == 'G'] + fields = np.array_split(fields, 4)[0] +elif field_group == 'Carveout Methodology - II': + fields = fields[fields['PRIORITY'] == 'G'] + fields = np.array_split(fields, 4)[1] +elif field_group == 'Carveout Method - III': + fields = fields[fields['PRIORITY'] == 'G'] + fields = np.array_split(fields, 4)[2] +elif field_group == 'Carveout Method - IV': + fields = fields[fields['PRIORITY'] == 'G'] + fields = np.array_split(fields, 4)[3] +elif field_group == 'Provider': + fields = fields[fields['PRIORITY'] == 'D'] +elif field_group == 'Timeline': + fields = fields[fields['PRIORITY'] == 'E'] + +if st.button("Show Results"): + # query = 'select * from "DOCZY_PIPELINE_RAW_OUTPUT"' + # cur.execute(query) + # df2 = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description]) + + # get this dataframe from snowflake table + df2 = pd.DataFrame(columns=['Contract Name','Field Name', 'SF_DB_COL_NAME', 'Snippet','Page Number' + , 'Field Extracted Value', 'Actual Value','Imputed Value']) + df2.to_csv('temp2.csv', index=False) + +if st.button("Show PDF"): + if file_name == None or file_name == "All": + st.error("Choose one specific file.") + else: + with st.sidebar: + st.markdown( + """ + + """, + unsafe_allow_html=True, + ) + pdf_viewer(file_name, width=1500) + +# if st.button("Show PDF"): +# if file_name == None or file_name == "All": +# st.error("Choose one specific file.") +# else: +# with st.sidebar: +# with open(file_name, "rb") as f: +# base64_pdf = base64.b64encode(f.read()).decode('utf-8') +# # Embedding PDF in HTML +# pdf_display = F'' +# # Displaying File +# st.markdown( +# """ +# +# """, +# unsafe_allow_html=True, +# ) +# st.markdown(pdf_display, unsafe_allow_html=True) + +df2 = pd.read_csv('temp2.csv') +df2['Imputed Value'] = '' +edited_df = st.data_editor(df2) + +@st.cache_data +def convert_df(df): + return df.to_csv(index=False).encode('utf-8') + +csv = convert_df(edited_df) + +buttons = st.columns(3) +with buttons[0]: + # st.button("Save All Imputations") + st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv') +with buttons[1]: + # st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv') + st.write("") +with buttons[2]: + if st.button("Kickoff Database Integration"): + st.write("Stored in DB") + + + + From 8b97614d3fa4340079dbd70122cf9b3ca0048242 Mon Sep 17 00:00:00 2001 From: "AARETE\\agupta" Date: Tue, 4 Jun 2024 08:53:38 -0500 Subject: [PATCH 26/37] Added a local/ subfolder that consists of the most recently developed file for local development purposes --- streamlit/{new_interface_0.py => local/local_interface_0.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename streamlit/{new_interface_0.py => local/local_interface_0.py} (100%) diff --git a/streamlit/new_interface_0.py b/streamlit/local/local_interface_0.py similarity index 100% rename from streamlit/new_interface_0.py rename to streamlit/local/local_interface_0.py From 96eb109d8d398f2b8377442756042e1da9024f3d Mon Sep 17 00:00:00 2001 From: "AARETE\\agupta" Date: Tue, 4 Jun 2024 08:54:36 -0500 Subject: [PATCH 27/37] Added a local/ subfolder that consists of the most recently developed file for local development purposes --- streamlit/{new_interface_1.py => local/local_interface_1.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename streamlit/{new_interface_1.py => local/local_interface_1.py} (100%) diff --git a/streamlit/new_interface_1.py b/streamlit/local/local_interface_1.py similarity index 100% rename from streamlit/new_interface_1.py rename to streamlit/local/local_interface_1.py From 441044b48118c7d056bf6cba9699316493bd8f79 Mon Sep 17 00:00:00 2001 From: Pratham Soni Date: Tue, 4 Jun 2024 09:04:17 -0500 Subject: [PATCH 28/37] Pratham's changes for UI 1 --- streamlit/local/p_local_interface1.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 streamlit/local/p_local_interface1.py diff --git a/streamlit/local/p_local_interface1.py b/streamlit/local/p_local_interface1.py new file mode 100644 index 0000000..e69de29 From 8893734027c416994f47dae1370b7ef8d92b8983 Mon Sep 17 00:00:00 2001 From: Pratham Soni Date: Tue, 4 Jun 2024 09:06:42 -0500 Subject: [PATCH 29/37] Pratham's changes to UI 1 --- streamlit/local/p_local_interface1.py | 230 ++++++++++++++++++++++++++ 1 file changed, 230 insertions(+) diff --git a/streamlit/local/p_local_interface1.py b/streamlit/local/p_local_interface1.py index e69de29..3337414 100644 --- a/streamlit/local/p_local_interface1.py +++ b/streamlit/local/p_local_interface1.py @@ -0,0 +1,230 @@ +import streamlit as st +from streamlit_extras.add_vertical_space import add_vertical_space +import os +import streamlit as st +import pandas as pd +from io import StringIO +from datetime import datetime +import boto3 +#import util +import requests +#from sf_conn import get_client_names, get_secret, save_to_sf +#from constants import USER_LIST, DOCZY_PIPELINE_URL_DEV + +#doczy_pipeline = DOCZY_PIPELINE_URL_DEV + +REDIRECT_URI = 'https://doczydev.aarete.com:8501' +#user_list = USER_LIST +st.set_page_config(layout = "wide") +# # Sidebar contents +# with st.sidebar: +# st.title("Doczy.AI ™") +# st.markdown( +# """ +# ## About +# This app extracts data from contracts + +# """ +# ) +# add_vertical_space(15) +# # st.write("Doczy") + +_,c1= st.columns([5,1]) +# try: +# util.setup_page(REDIRECT_URI) +# except: +# st.write("SSO Failed") +# st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} +# try: +# c1.write(f"User: **{st.session_state.user_info['displayName']}**") +# user_mail = st.session_state.user_info['mail'] +# except KeyError as e: +# # Do we add a link to get to the login page here? +# st.write("Session Expired.") +# st.stop() + + + +s3_client = boto3.client('s3', + region_name="us-east-2", +) + +# # to be replaced with snowflake data +client_list = ['doczy-ai-client-1', 'Delaware First Health, Inc.', 'Community Health Choice, Inc','CareSource Network Partners LLC', + 'HealthNet of Cali', 'Oklahoma Complete Health, Inc', 'HealthFirst', 'Molina Healthcare of TX', 'AvMed', 'Arizona Care1st', + 'WellCare New Jersey'] + +# client_list, s3_paths = get_client_names() +# client_s3_paths = dict(zip(client_list, s3_paths)) + +client_row = st.columns([0.1, 0.8]) +with client_row[0]: + st.write("**Client Name**") +with client_row[1]: + client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed") + +# client_bucket = client_s3_paths.get(client) +# # to be deleted when buckets for different clients are ready; below line is added only for testing the corresponding DAG +client_bucket = 'doczy-ai-client-1' + +# batch_objects = s3_client.list_objects_v2(Bucket=client_bucket +# , Prefix="contracts_landing_zone/", Delimiter='/') + +batch_list = ['d_12132', 'd_13345','i_23423', 'i_72223', 'b_12345', 'b_33452'] + +if 'sorted_list' not in st.session_state: + st.session_state.sorted_list = batch_list + +def sort_list(ex_list, sort_by, order): + if sort_by == 'Alphabetical': + ex_list = sorted(ex_list, reverse=(order == 'Descending')) + elif sort_by == 'Create Date': + ex_list = ex_list if order == 'Ascending' else list(reversed(ex_list)) + + return ex_list + +col1, col2, col3, col4 = st.columns([0.5, 0.5, 0.5, 0.5]) + + +with col1: + sort_by = st.radio("**Sort Batch_IDs**", ('Alphabetical', 'Create Date')) + +with col2: + order = st.radio('', ('Ascending','Descending')) + +with col3: + add_vertical_space(2) + if st.button('Apply'): + st.session_state.sorted_list = sort_list(batch_list, sort_by, order) +# for prefix in batch_objects['CommonPrefixes']: +# batch_list.append(prefix['Prefix'][:-1].split('/')[-1]) + +path_row = st.columns([0.1, 0.8]) +with path_row[0]: + st.write("**Batch ID**") +with path_row[1]: + batch_id = st.selectbox('**Batch ID**', st.session_state.sorted_list, label_visibility = "collapsed") + +checks = st.columns([0.1, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12]) +with checks[0]: + st.write("**Group No.**") +with checks[1]: + a = st.checkbox('Unique Key', key = str(1)) +with checks[2]: + b = st.checkbox('Pricing Before Carveouts', key = str(2)) +with checks[3]: + c = st.checkbox('Contract Related', key = str(3)) +with checks[4]: + d = st.checkbox('Provider', key = str(4)) +with checks[5]: + e = st.checkbox('Timeline', key = str(5)) +with checks[6]: + f = st.checkbox('Carveout Indicator', key = str(6)) +with checks[7]: + g = st.checkbox('Carveout Methodology', key = str(7)) + +add_vertical_space(1) + +df = pd.DataFrame(columns=['Contract Name', 'Unique Key','Pricing Before Carveouts' + , 'Contract Related', 'Provider', 'Timeline', 'Carveout Indicator', 'Carveout Methodology']) +file_list = [] +# file_objects = s3_client.list_objects_v2(Bucket=client_bucket +# , Prefix="contracts_landing_zone/"+batch_id+"/", Delimiter='/') + +if st.button("Read the contracts from Path"): + for obj in file_objects.get('Contents',[]): + if not obj['Key'].endswith('/'): + file_list.append(obj['Key'].split('/')[-1]) + + df['Contract Name'] = file_list + # df['Request ID'] = range(len(file_list)) + # df['Contract ID'] = file_list + df['Unique Key'] = a + df['Pricing Before Carveouts'] = b + df['Contract Related'] = c + df['Provider'] = d + df['Timeline'] = e + df['Carveout Indicator'] = f + df['Carveout Methodology'] = g + dir_path = os.path.dirname(os.path.realpath(__file__)) + print(f'DEBUGGING: PWD= {dir_path}') + df.to_csv('temp1.csv', index=False) + +add_vertical_space(1) +#df2 = pd.read_csv('temp1.csv') +#edited_df = st.data_editor(df2) + +# edited_df['REQUEST_USER'] = user_mail +# edited_df['LATEST_FLAG BOOLEAN'] = True +# edited_df['PIPELINE_KICKOFF_DATETIME'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") +# edited_df['REQUEST_DATETIME'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + +@st.cache_data +def convert_df(df): + return df.to_csv(index=False).encode('utf-8') + +#csv = convert_df(edited_df) +# edited_df = edited_df.reset_index() # make sure indexes pair with number of rows + +# additional_info = pd.DataFrame(columns=['REQUEST_ID','T_DRIVE_PATH','CLIENT_NAME' +# , 'GROUP_NAME', 'REQUEST_USERNAME', 'REQUEST_DATETIME']) +additional_info = pd.DataFrame(columns=['CLIENT_NAME', 'BATCH_ID', 'REQUEST_USERNAME', 'REQUEST_DATETIME']) +additional_info.loc[0] = [client, batch_id, st.session_state.user_info['mail'], datetime.now().strftime("%Y-%m-%d %H:%M:%S")] +st.write(additional_info) + +contract_list = [] +for index, row in edited_df.iterrows(): + group_list = [] + if row['Unique Key']: + group_list.append('Unique Key') + if row['Pricing Before Carveouts']: + group_list.append('Pricing Before Carveouts') + if row['Contract Related']: + group_list.append('Contract Related') + if row['Provider']: + group_list.append('Provider') + if row['Timeline']: + group_list.append('Timeline') + if row['Carveout Indicator']: + group_list.append('Carveout Indicator') + if row['Carveout Methodology']: + group_list.append('Carveout Methodology') + entry_dict = { + "contract_name": row['Contract Name'], + "groups": group_list, + "contract_source_path": "contracts_landing_zone/"+batch_id+"/"+row['Contract Name'] + } + contract_list.append(entry_dict) + +myobj = { + "s3_bucket": client_bucket, + "batch_id": batch_id, + "client_name": client, + #"username": user_mail, + "contract_list": contract_list +} + +buttons = st.columns([0.8, 0.2]) +with buttons[0]: + st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv') +with buttons[1]: + if st.button("Run Doczy.AI Pipeline"): + # csv_buf = StringIO() + # additional_info.to_csv(csv_buf, header=True, index=False) + # csv_buf.seek(0) + # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/request_submission.csv') + # csv_buf = StringIO() + # edited_df.to_csv(csv_buf, header=True, index=False) + # csv_buf.seek(0) + # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/contract_config.csv') + # try: + # save_to_sf('load_request_and_contract_submissions', request_submission_file_name = "request_submission.csv", contract_config_file_name = "contract_config.csv") + # except Exception as e: + # st.write(e) + response = requests.post(doczy_pipeline, json = myobj) + if response.status_code >= 200 and response.status_code < 300: + st.write("Success") + # st.write(myobj) + else: + st.write("Failed") + # st.write(response.text) \ No newline at end of file From 8da81d4c7c4b0c46d5ddda711e659180b880326b Mon Sep 17 00:00:00 2001 From: "AARETE\\agupta" Date: Tue, 4 Jun 2024 09:32:26 -0500 Subject: [PATCH 30/37] Merged Aryan's and Pratham's changes in the local/ folder. Modified interface_1.py to include all changes and continue working on the server including all dependencies. --- streamlit/interface_1.py | 89 +++++++--- streamlit/local/local_interface_1.py | 29 +++- streamlit/local/p_local_interface1.py | 230 -------------------------- 3 files changed, 96 insertions(+), 252 deletions(-) delete mode 100644 streamlit/local/p_local_interface1.py diff --git a/streamlit/interface_1.py b/streamlit/interface_1.py index 3d3a695..42b3768 100644 --- a/streamlit/interface_1.py +++ b/streamlit/interface_1.py @@ -8,6 +8,7 @@ from datetime import datetime import boto3 import util import requests +import time from sf_conn import get_client_names, get_secret, save_to_sf from constants import USER_LIST, DOCZY_PIPELINE_URL_DEV @@ -61,10 +62,11 @@ client_row = st.columns([0.1, 0.8]) with client_row[0]: st.write("**Client Name**") with client_row[1]: - client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed") + client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344 client_bucket = client_s3_paths.get(client) -# # to be deleted when buckets for different clients are ready; below line is added only for testing the corresponding DAG + +# to be deleted when buckets for different clients are ready; below line is added only for testing the corresponding DAG client_bucket = 'doczy-ai-client-1' batch_objects = s3_client.list_objects_v2(Bucket=client_bucket @@ -74,17 +76,51 @@ batch_list = [] for prefix in batch_objects['CommonPrefixes']: batch_list.append(prefix['Prefix'][:-1].split('/')[-1]) +# Hardcoded batch_list for testing purposes +# batch_list = ['batch_020524103737', 'batch_090524131433', 'batch_090524131607', 'batch_100524123000', 'batch_130524064322', +# 'batch_160524071331', 'batch_200524213550', 'batch_250424112237', 'batch_280524120530', 'batch_280524121721', 'batch_280524144222', +# 'batch_290524123926', 'batch_290524164044', 'batch_310524102029', 'batch_310524124050', 'batch_310524162346', 'batch_310524162631'] + +if 'sorted_list' not in st.session_state: + st.session_state.sorted_list = batch_list + +def sort_list(ex_list, sort_by, order): + if sort_by == 'Alphabetical': + ex_list = sorted(ex_list, reverse=(order == 'Descending')) + elif sort_by == 'Create Date': + ex_list = ex_list if order == 'Ascending' else list(reversed(ex_list)) + + return ex_list + +col1, col2, col3, col4 = st.columns([0.5, 0.5, 0.5, 0.5]) + + +with col1: + sort_by = st.radio("**Sort Batch_IDs**", ('Alphabetical', 'Create Date')) + +with col2: + order = st.radio('', ('Ascending','Descending')) + +with col3: + add_vertical_space(2) + if st.button('Apply'): + st.session_state.sorted_list = sort_list(batch_list, sort_by, order) + path_row = st.columns([0.1, 0.8]) with path_row[0]: st.write("**Batch ID**") with path_row[1]: - batch_id = st.selectbox('**Batch ID**', batch_list, label_visibility = "collapsed") + batch_id = st.selectbox('**Batch ID**', st.session_state.sorted_list, label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344 + +if not batch_id: + batch_id = "None" checks = st.columns([0.1, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12]) with checks[0]: st.write("**Group No.**") + with checks[1]: - a = st.checkbox('Unique Key', key = str(1)) + a = st.checkbox('Unique Key', key = str(1), args="Unique") with checks[2]: b = st.checkbox('Pricing Before Carveouts', key = str(2)) with checks[3]: @@ -106,11 +142,15 @@ file_list = [] file_objects = s3_client.list_objects_v2(Bucket=client_bucket , Prefix="contracts_landing_zone/"+batch_id+"/", Delimiter='/') +# Hardcoded file_list for testing purposes +# file_list = ['Boilerplate_TX Amendment Mission Health Network effective_040114 MU.pdf', 'Custom_TX - MP AMENDMENT - MISSION HEALTH NETWORK - MU.pdf', +# 'Delaware First Health_First State Homecare Agency_212260_7 MU.pdf', 'Molina Healthcare of Texas, Inc. Amendment 4 - HIX ACA__EFF 01012016_MU.pdf'] + if st.button("Read the contracts from Path"): for obj in file_objects.get('Contents',[]): if not obj['Key'].endswith('/'): file_list.append(obj['Key'].split('/')[-1]) - + df['Contract Name'] = file_list # df['Request ID'] = range(len(file_list)) # df['Contract ID'] = file_list @@ -147,23 +187,33 @@ additional_info = pd.DataFrame(columns=['CLIENT_NAME', 'BATCH_ID', 'REQUEST_USER additional_info.loc[0] = [client, batch_id, st.session_state.user_info['mail'], datetime.now().strftime("%Y-%m-%d %H:%M:%S")] st.write(additional_info) +st.session_state.contract_count = 0 contract_list = [] for index, row in edited_df.iterrows(): + allow_run_for_contract = False group_list = [] if row['Unique Key']: group_list.append('Unique Key') + allow_run_for_contract = True if row['Pricing Before Carveouts']: group_list.append('Pricing Before Carveouts') + allow_run_for_contract = True if row['Contract Related']: group_list.append('Contract Related') + allow_run_for_contract = True if row['Provider']: group_list.append('Provider') + allow_run_for_contract = True if row['Timeline']: group_list.append('Timeline') + allow_run_for_contract = True if row['Carveout Indicator']: group_list.append('Carveout Indicator') + allow_run_for_contract = True if row['Carveout Methodology']: group_list.append('Carveout Methodology') + allow_run_for_contract = True + if allow_run_for_contract: st.session_state.contract_count += 1 entry_dict = { "contract_name": row['Contract Name'], "groups": group_list, @@ -183,20 +233,21 @@ buttons = st.columns([0.8, 0.2]) with buttons[0]: st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv') with buttons[1]: - if st.button("Run Doczy.AI Pipeline"): - # csv_buf = StringIO() - # additional_info.to_csv(csv_buf, header=True, index=False) - # csv_buf.seek(0) - # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/request_submission.csv') - # csv_buf = StringIO() - # edited_df.to_csv(csv_buf, header=True, index=False) - # csv_buf.seek(0) - # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/contract_config.csv') - # try: - # save_to_sf('load_request_and_contract_submissions', request_submission_file_name = "request_submission.csv", contract_config_file_name = "contract_config.csv") - # except Exception as e: - # st.write(e) - response = requests.post(doczy_pipeline, json = myobj) + if st.button("Run Doczy.AI Pipeline", disabled= not st.session_state.contract_count == len(edited_df)): + with st.spinner('Running...'): # Feedback to User while API endpoint sends response for DOC-342 + # csv_buf = StringIO() + # additional_info.to_csv(csv_buf, header=True, index=False) + # csv_buf.seek(0) + # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/request_submission.csv') + # csv_buf = StringIO() + # edited_df.to_csv(csv_buf, header=True, index=False) + # csv_buf.seek(0) + # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/contract_config.csv') + # try: + # save_to_sf('load_request_and_contract_submissions', request_submission_file_name = "request_submission.csv", contract_config_file_name = "contract_config.csv") + # except Exception as e: + # st.write(e) + response = requests.post(doczy_pipeline, json = myobj) if response.status_code >= 200 and response.status_code < 300: st.write("Success") # st.write(myobj) diff --git a/streamlit/local/local_interface_1.py b/streamlit/local/local_interface_1.py index 28e562c..d9e6dd8 100644 --- a/streamlit/local/local_interface_1.py +++ b/streamlit/local/local_interface_1.py @@ -81,11 +81,36 @@ batch_list = ['batch_020524103737', 'batch_090524131433', 'batch_090524131607', 'batch_160524071331', 'batch_200524213550', 'batch_250424112237', 'batch_280524120530', 'batch_280524121721', 'batch_280524144222', 'batch_290524123926', 'batch_290524164044', 'batch_310524102029', 'batch_310524124050', 'batch_310524162346', 'batch_310524162631'] +if 'sorted_list' not in st.session_state: + st.session_state.sorted_list = batch_list + +def sort_list(ex_list, sort_by, order): + if sort_by == 'Alphabetical': + ex_list = sorted(ex_list, reverse=(order == 'Descending')) + elif sort_by == 'Create Date': + ex_list = ex_list if order == 'Ascending' else list(reversed(ex_list)) + + return ex_list + +col1, col2, col3, col4 = st.columns([0.5, 0.5, 0.5, 0.5]) + + +with col1: + sort_by = st.radio("**Sort Batch_IDs**", ('Alphabetical', 'Create Date')) + +with col2: + order = st.radio('', ('Ascending','Descending')) + +with col3: + add_vertical_space(2) + if st.button('Apply'): + st.session_state.sorted_list = sort_list(batch_list, sort_by, order) + path_row = st.columns([0.1, 0.8]) with path_row[0]: st.write("**Batch ID**") with path_row[1]: - batch_id = st.selectbox('**Batch ID**', batch_list, label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344 + batch_id = st.selectbox('**Batch ID**', st.session_state.sorted_list, label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344 if not batch_id: batch_id = "None" @@ -201,8 +226,6 @@ myobj = { "contract_list": contract_list } -print(st.session_state) - buttons = st.columns([0.8, 0.2]) with buttons[0]: st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv') diff --git a/streamlit/local/p_local_interface1.py b/streamlit/local/p_local_interface1.py deleted file mode 100644 index 3337414..0000000 --- a/streamlit/local/p_local_interface1.py +++ /dev/null @@ -1,230 +0,0 @@ -import streamlit as st -from streamlit_extras.add_vertical_space import add_vertical_space -import os -import streamlit as st -import pandas as pd -from io import StringIO -from datetime import datetime -import boto3 -#import util -import requests -#from sf_conn import get_client_names, get_secret, save_to_sf -#from constants import USER_LIST, DOCZY_PIPELINE_URL_DEV - -#doczy_pipeline = DOCZY_PIPELINE_URL_DEV - -REDIRECT_URI = 'https://doczydev.aarete.com:8501' -#user_list = USER_LIST -st.set_page_config(layout = "wide") -# # Sidebar contents -# with st.sidebar: -# st.title("Doczy.AI ™") -# st.markdown( -# """ -# ## About -# This app extracts data from contracts - -# """ -# ) -# add_vertical_space(15) -# # st.write("Doczy") - -_,c1= st.columns([5,1]) -# try: -# util.setup_page(REDIRECT_URI) -# except: -# st.write("SSO Failed") -# st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} -# try: -# c1.write(f"User: **{st.session_state.user_info['displayName']}**") -# user_mail = st.session_state.user_info['mail'] -# except KeyError as e: -# # Do we add a link to get to the login page here? -# st.write("Session Expired.") -# st.stop() - - - -s3_client = boto3.client('s3', - region_name="us-east-2", -) - -# # to be replaced with snowflake data -client_list = ['doczy-ai-client-1', 'Delaware First Health, Inc.', 'Community Health Choice, Inc','CareSource Network Partners LLC', - 'HealthNet of Cali', 'Oklahoma Complete Health, Inc', 'HealthFirst', 'Molina Healthcare of TX', 'AvMed', 'Arizona Care1st', - 'WellCare New Jersey'] - -# client_list, s3_paths = get_client_names() -# client_s3_paths = dict(zip(client_list, s3_paths)) - -client_row = st.columns([0.1, 0.8]) -with client_row[0]: - st.write("**Client Name**") -with client_row[1]: - client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed") - -# client_bucket = client_s3_paths.get(client) -# # to be deleted when buckets for different clients are ready; below line is added only for testing the corresponding DAG -client_bucket = 'doczy-ai-client-1' - -# batch_objects = s3_client.list_objects_v2(Bucket=client_bucket -# , Prefix="contracts_landing_zone/", Delimiter='/') - -batch_list = ['d_12132', 'd_13345','i_23423', 'i_72223', 'b_12345', 'b_33452'] - -if 'sorted_list' not in st.session_state: - st.session_state.sorted_list = batch_list - -def sort_list(ex_list, sort_by, order): - if sort_by == 'Alphabetical': - ex_list = sorted(ex_list, reverse=(order == 'Descending')) - elif sort_by == 'Create Date': - ex_list = ex_list if order == 'Ascending' else list(reversed(ex_list)) - - return ex_list - -col1, col2, col3, col4 = st.columns([0.5, 0.5, 0.5, 0.5]) - - -with col1: - sort_by = st.radio("**Sort Batch_IDs**", ('Alphabetical', 'Create Date')) - -with col2: - order = st.radio('', ('Ascending','Descending')) - -with col3: - add_vertical_space(2) - if st.button('Apply'): - st.session_state.sorted_list = sort_list(batch_list, sort_by, order) -# for prefix in batch_objects['CommonPrefixes']: -# batch_list.append(prefix['Prefix'][:-1].split('/')[-1]) - -path_row = st.columns([0.1, 0.8]) -with path_row[0]: - st.write("**Batch ID**") -with path_row[1]: - batch_id = st.selectbox('**Batch ID**', st.session_state.sorted_list, label_visibility = "collapsed") - -checks = st.columns([0.1, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12]) -with checks[0]: - st.write("**Group No.**") -with checks[1]: - a = st.checkbox('Unique Key', key = str(1)) -with checks[2]: - b = st.checkbox('Pricing Before Carveouts', key = str(2)) -with checks[3]: - c = st.checkbox('Contract Related', key = str(3)) -with checks[4]: - d = st.checkbox('Provider', key = str(4)) -with checks[5]: - e = st.checkbox('Timeline', key = str(5)) -with checks[6]: - f = st.checkbox('Carveout Indicator', key = str(6)) -with checks[7]: - g = st.checkbox('Carveout Methodology', key = str(7)) - -add_vertical_space(1) - -df = pd.DataFrame(columns=['Contract Name', 'Unique Key','Pricing Before Carveouts' - , 'Contract Related', 'Provider', 'Timeline', 'Carveout Indicator', 'Carveout Methodology']) -file_list = [] -# file_objects = s3_client.list_objects_v2(Bucket=client_bucket -# , Prefix="contracts_landing_zone/"+batch_id+"/", Delimiter='/') - -if st.button("Read the contracts from Path"): - for obj in file_objects.get('Contents',[]): - if not obj['Key'].endswith('/'): - file_list.append(obj['Key'].split('/')[-1]) - - df['Contract Name'] = file_list - # df['Request ID'] = range(len(file_list)) - # df['Contract ID'] = file_list - df['Unique Key'] = a - df['Pricing Before Carveouts'] = b - df['Contract Related'] = c - df['Provider'] = d - df['Timeline'] = e - df['Carveout Indicator'] = f - df['Carveout Methodology'] = g - dir_path = os.path.dirname(os.path.realpath(__file__)) - print(f'DEBUGGING: PWD= {dir_path}') - df.to_csv('temp1.csv', index=False) - -add_vertical_space(1) -#df2 = pd.read_csv('temp1.csv') -#edited_df = st.data_editor(df2) - -# edited_df['REQUEST_USER'] = user_mail -# edited_df['LATEST_FLAG BOOLEAN'] = True -# edited_df['PIPELINE_KICKOFF_DATETIME'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") -# edited_df['REQUEST_DATETIME'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - -@st.cache_data -def convert_df(df): - return df.to_csv(index=False).encode('utf-8') - -#csv = convert_df(edited_df) -# edited_df = edited_df.reset_index() # make sure indexes pair with number of rows - -# additional_info = pd.DataFrame(columns=['REQUEST_ID','T_DRIVE_PATH','CLIENT_NAME' -# , 'GROUP_NAME', 'REQUEST_USERNAME', 'REQUEST_DATETIME']) -additional_info = pd.DataFrame(columns=['CLIENT_NAME', 'BATCH_ID', 'REQUEST_USERNAME', 'REQUEST_DATETIME']) -additional_info.loc[0] = [client, batch_id, st.session_state.user_info['mail'], datetime.now().strftime("%Y-%m-%d %H:%M:%S")] -st.write(additional_info) - -contract_list = [] -for index, row in edited_df.iterrows(): - group_list = [] - if row['Unique Key']: - group_list.append('Unique Key') - if row['Pricing Before Carveouts']: - group_list.append('Pricing Before Carveouts') - if row['Contract Related']: - group_list.append('Contract Related') - if row['Provider']: - group_list.append('Provider') - if row['Timeline']: - group_list.append('Timeline') - if row['Carveout Indicator']: - group_list.append('Carveout Indicator') - if row['Carveout Methodology']: - group_list.append('Carveout Methodology') - entry_dict = { - "contract_name": row['Contract Name'], - "groups": group_list, - "contract_source_path": "contracts_landing_zone/"+batch_id+"/"+row['Contract Name'] - } - contract_list.append(entry_dict) - -myobj = { - "s3_bucket": client_bucket, - "batch_id": batch_id, - "client_name": client, - #"username": user_mail, - "contract_list": contract_list -} - -buttons = st.columns([0.8, 0.2]) -with buttons[0]: - st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv') -with buttons[1]: - if st.button("Run Doczy.AI Pipeline"): - # csv_buf = StringIO() - # additional_info.to_csv(csv_buf, header=True, index=False) - # csv_buf.seek(0) - # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/request_submission.csv') - # csv_buf = StringIO() - # edited_df.to_csv(csv_buf, header=True, index=False) - # csv_buf.seek(0) - # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/contract_config.csv') - # try: - # save_to_sf('load_request_and_contract_submissions', request_submission_file_name = "request_submission.csv", contract_config_file_name = "contract_config.csv") - # except Exception as e: - # st.write(e) - response = requests.post(doczy_pipeline, json = myobj) - if response.status_code >= 200 and response.status_code < 300: - st.write("Success") - # st.write(myobj) - else: - st.write("Failed") - # st.write(response.text) \ No newline at end of file From 93dd5f4178ba5aa0cc1c732891991867e39732a4 Mon Sep 17 00:00:00 2001 From: Pratham Soni Date: Tue, 4 Jun 2024 09:44:47 -0500 Subject: [PATCH 31/37] changes to UI0 --- streamlit/local/p_local_interface_0.py | 196 +++++++++++++++++++++++++ 1 file changed, 196 insertions(+) create mode 100644 streamlit/local/p_local_interface_0.py diff --git a/streamlit/local/p_local_interface_0.py b/streamlit/local/p_local_interface_0.py new file mode 100644 index 0000000..6bbac49 --- /dev/null +++ b/streamlit/local/p_local_interface_0.py @@ -0,0 +1,196 @@ +import json +import streamlit as st +from streamlit_extras.add_vertical_space import add_vertical_space +import os +import streamlit as st +import pandas as pd +from io import StringIO +from datetime import datetime +import boto3 +# import util +import requests +# from sf_conn import get_secret, save_to_sf +from io import StringIO, BytesIO +import time +# from sf_conn import get_client_names, insert_upload_logs +# from constants import USER_LIST + +create_batch_url = 'https://lfksus2t62.execute-api.us-east-2.amazonaws.com/dev/create-batch' + +# REDIRECT_URI = 'https://doczydev.aarete.com:8500' +# user_list = USER_LIST +if 'uploading' not in st.session_state: + st.session_state.uploading = False + + + +def insert_upload_logs(batch_id, client_name, file_name, upload_datetime, upload_user): + """ + Input: batch_id, client_name, file_name, upload_datetime, upload_user + Output: status of the insert query + """ + try: + return 'Log inserted successfully' + except Exception as e: + return e + + +st.set_page_config(layout = "wide") +# # # Sidebar contents +# # with st.sidebar: +# # st.title("Doczy.AI ™") +# # st.markdown( +# # """ +# # ## About +# # This app extracts data from contracts + +# # """ +# # ) +# # add_vertical_space(15) +# # # st.write("Doczy") + +_,c1= st.columns([5,1]) +# try: +# util.setup_page(REDIRECT_URI) +# except Exception as e: +# st.write(f"SSO Failed = {e}") +# st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} +st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} +try: + c1.write(f"User: **{st.session_state.user_info['displayName']}**") + user_mail = st.session_state.user_info['mail'] +except KeyError as e: + st.write("Session Expired.") + st.stop() + + + +s3_client = boto3.client('s3', + region_name="us-east-2", +) + +client_list = ['doczy-ai-client-1', 'Delaware First Health, Inc.', 'Community Health Choice, Inc','CareSource Network Partners LLC', + 'HealthNet of Cali', 'Oklahoma Complete Health, Inc', 'HealthFirst', 'Molina Healthcare of TX', 'AvMed', 'Arizona Care1st', + 'WellCare New Jersey'] + +# This is the list of client fetched from Snowflake +# TODO: Need to update the streamlit code to use the client names from this list +# And use the s3 paths to save the objects for the respective client +# client_list, s3_paths = get_client_names() +# client_s3_paths = dict(zip(client_list, s3_paths)) + +client_row = st.columns([0.1, 0.8]) +with client_row[0]: + st.write("**Client Name**") +with client_row[1]: + client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344 + +# client_bucket = client_s3_paths.get(client) + +# to be deleted when client buckets are created +client_bucket = 'doczy-ai-client-1' + +file_row = st.columns([0.1, 0.8]) +with file_row[0]: + st.write("**Upload Files**") +with file_row[1]: + file_list = st.file_uploader("Upload", type=['csv','pdf'], accept_multiple_files=True, label_visibility = "collapsed", help="Only .csv and .pdf file formats are supported.", disabled=st.session_state.uploading) + +add_vertical_space(2) +df = pd.DataFrame(columns=['Contract Name']) +df['Contract Name'] = file_list +file_names = [] +buttons = st.columns([0.4, 0.4, 0.2]) +with buttons[1]: + if st.button("Create Batch"): + st.session_state.uploading = True + if client == None: + st.session_state.uploading = False + st.error("No Client Name Selected.") + elif len(file_list) == 0: + st.session_state.uploading = False + st.error("No Files Selected.") + else: + myobj = { "client-bucket-name": client_bucket } + response = requests.post(create_batch_url, json = myobj) + if response.status_code >= 200 and response.status_code < 300: + try: + batch_id = json.loads(json.loads(response.text)['body'])['batch_id'] + landing_zone = json.loads(json.loads(response.text)['body'])['landing_zone'] + except: + st.write(myobj) + st.write(response.text) + batch_id = 'failed_cases' + landing_zone = 'contracts_landing_zone' + else: + st.write("Failed") + for uploaded_file in file_list: + stringio = BytesIO(uploaded_file.getvalue()) + stringio.seek(0) + s3_client.put_object(Bucket=client_bucket, Body=stringio.getvalue(), Key= + landing_zone+batch_id+'/'+str(uploaded_file.name)) + + # TODO: Test this insert function with snowflake + upload_log = insert_upload_logs(batch_id, client, str(uploaded_file.name), datetime.now().strftime("%Y-%m-%d %H:%M:%S"), user_mail) + st.write(upload_log) + + file_names.append(str(uploaded_file.name)) + st.write(f"{batch_id} created") + st.session_state.uploading = False + st.write(f"Files uploaded to s3://{client_bucket}/{landing_zone}{batch_id}") + + + + + # @st.cache_data + # def convert_df(df): + # return df.to_csv(index=False).encode('utf-8') + + # csv = convert_df(df) + + # df = df.reset_index() # make sure indexes pair with number of rows + + # contract_list = [] + # for index, row in df.iterrows(): + # group_list = [] + # if row['Unique Key']: + # group_list.append('Unique Key') + # if row['Pricing Before Carveouts']: + # group_list.append('Pricing Before Carveouts') + # if row['Contract Related']: + # group_list.append('Contract Related') + # if row['Provider']: + # group_list.append('Provider') + # if row['Timeline']: + # group_list.append('Timeline') + # if row['Carveout Indicator']: + # group_list.append('Carveout Indicator') + # if row['Carveout Methodology']: + # group_list.append('Carveout Methodology') + # entry_dict = { + # "contract_name": row['Contract Name'], + # "groups": group_list, + # "contract_source_path": "batches/batch_1/"+client+"/"+row['Contract Name'] + # } + # contract_list.append(entry_dict) + # contract_list = list(df['Contract Name']) + + # myobj = { + # "s3_bucket": 'doczy-dev-infra-textract', + # "batch_id": "1", + # "client_name": client, + # "username": user_mail, + # "contract_list": contract_list + # } + + # buttons = st.columns([0.8, 0.2]) + # with buttons[0]: + # st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv') + # with buttons[1]: + # if st.button("Upload to DB"): + # response = requests.post(doczy_pipeline, json = myobj) + # if response.status_code >= 200 and response.status_code < 300: + # st.write("Success") + # else: + # st.write("Failed") + # # st.write(response.text) From 56b7657a74c7fb1a8de76114b8415457ae728037 Mon Sep 17 00:00:00 2001 From: Pratham Soni Date: Tue, 4 Jun 2024 09:50:11 -0500 Subject: [PATCH 32/37] pratham changes to ui0 --- streamlit/local/p_local_interface_0.py | 1 + 1 file changed, 1 insertion(+) diff --git a/streamlit/local/p_local_interface_0.py b/streamlit/local/p_local_interface_0.py index 6bbac49..8024684 100644 --- a/streamlit/local/p_local_interface_0.py +++ b/streamlit/local/p_local_interface_0.py @@ -48,6 +48,7 @@ st.set_page_config(layout = "wide") # # ) # # add_vertical_space(15) # # # st.write("Doczy") +# # _,c1= st.columns([5,1]) # try: From 0801a24448ad8ad8e5fbb775147772fc16dd09da Mon Sep 17 00:00:00 2001 From: "AARETE\\agupta" Date: Tue, 4 Jun 2024 09:56:27 -0500 Subject: [PATCH 33/37] Added an error message for Group No. not being selected, instead of just disabling the button. --- streamlit/interface_1.py | 43 ++++++++++++------------ streamlit/local/local_interface_1.py | 49 +++++++++++++++------------- 2 files changed, 49 insertions(+), 43 deletions(-) diff --git a/streamlit/interface_1.py b/streamlit/interface_1.py index 42b3768..5b3f82a 100644 --- a/streamlit/interface_1.py +++ b/streamlit/interface_1.py @@ -233,25 +233,28 @@ buttons = st.columns([0.8, 0.2]) with buttons[0]: st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv') with buttons[1]: - if st.button("Run Doczy.AI Pipeline", disabled= not st.session_state.contract_count == len(edited_df)): - with st.spinner('Running...'): # Feedback to User while API endpoint sends response for DOC-342 - # csv_buf = StringIO() - # additional_info.to_csv(csv_buf, header=True, index=False) - # csv_buf.seek(0) - # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/request_submission.csv') - # csv_buf = StringIO() - # edited_df.to_csv(csv_buf, header=True, index=False) - # csv_buf.seek(0) - # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/contract_config.csv') - # try: - # save_to_sf('load_request_and_contract_submissions', request_submission_file_name = "request_submission.csv", contract_config_file_name = "contract_config.csv") - # except Exception as e: - # st.write(e) - response = requests.post(doczy_pipeline, json = myobj) - if response.status_code >= 200 and response.status_code < 300: - st.write("Success") - # st.write(myobj) + if st.button("Run Doczy.AI Pipeline"): + if not st.session_state.contract_count == len(edited_df): + st.error("Select at least one Group No. for every Contract") else: - st.write("Failed") - # st.write(response.text) + with st.spinner('Running...'): + # csv_buf = StringIO() + # additional_info.to_csv(csv_buf, header=True, index=False) + # csv_buf.seek(0) + # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/request_submission.csv') + # csv_buf = StringIO() + # edited_df.to_csv(csv_buf, header=True, index=False) + # csv_buf.seek(0) + # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/contract_config.csv') + # try: + # save_to_sf('load_request_and_contract_submissions', request_submission_file_name = "request_submission.csv", contract_config_file_name = "contract_config.csv") + # except Exception as e: + # st.write(e) + response = requests.post(doczy_pipeline, json = myobj) + if response.status_code >= 200 and response.status_code < 300: + st.write("Success") + # st.write(myobj) + else: + st.write("Failed") + # st.write(response.text) diff --git a/streamlit/local/local_interface_1.py b/streamlit/local/local_interface_1.py index d9e6dd8..fd03888 100644 --- a/streamlit/local/local_interface_1.py +++ b/streamlit/local/local_interface_1.py @@ -230,27 +230,30 @@ buttons = st.columns([0.8, 0.2]) with buttons[0]: st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv') with buttons[1]: - if st.button("Run Doczy.AI Pipeline", disabled= not st.session_state.contract_count == len(edited_df)): - with st.spinner('Running...'): # Feedback to User while API endpoint sends response for DOC-342 - # csv_buf = StringIO() - # additional_info.to_csv(csv_buf, header=True, index=False) - # csv_buf.seek(0) - # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/request_submission.csv') - # csv_buf = StringIO() - # edited_df.to_csv(csv_buf, header=True, index=False) - # csv_buf.seek(0) - # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/contract_config.csv') - # try: - # save_to_sf('load_request_and_contract_submissions', request_submission_file_name = "request_submission.csv", contract_config_file_name = "contract_config.csv") - # except Exception as e: - # st.write(e) - # response = requests.post(doczy_pipeline, json = myobj) - # if response.status_code >= 200 and response.status_code < 300: - # st.write("Success") - # # st.write(myobj) - # else: - # st.write("Failed") - # st.write(response.text) - time.sleep(5) - st.write("Success!") + if st.button("Run Doczy.AI Pipeline"): + if not st.session_state.contract_count == len(edited_df): + st.error("Select at least one Group No. for every Contract") + else: + with st.spinner('Running...'): # Feedback to User while API endpoint sends response for DOC-342 + # csv_buf = StringIO() + # additional_info.to_csv(csv_buf, header=True, index=False) + # csv_buf.seek(0) + # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/request_submission.csv') + # csv_buf = StringIO() + # edited_df.to_csv(csv_buf, header=True, index=False) + # csv_buf.seek(0) + # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/contract_config.csv') + # try: + # save_to_sf('load_request_and_contract_submissions', request_submission_file_name = "request_submission.csv", contract_config_file_name = "contract_config.csv") + # except Exception as e: + # st.write(e) + # response = requests.post(doczy_pipeline, json = myobj) + # if response.status_code >= 200 and response.status_code < 300: + # st.write("Success") + # # st.write(myobj) + # else: + # st.write("Failed") + # st.write(response.text) + time.sleep(5) + st.write("Success!") From b7c1c22d3d6e0d0f4e8fa9bbc88ef0162e5d98df Mon Sep 17 00:00:00 2001 From: "AARETE\\agupta" Date: Tue, 4 Jun 2024 10:28:15 -0500 Subject: [PATCH 34/37] Merged Aryan's and Pratham's changes in the local/ folder. Modified interface_0.py to include all changes and continue working on the server including all dependencies. --- streamlit/interface_0.py | 89 +++++++---- streamlit/local/local_interface_0.py | 72 +++++---- streamlit/local/p_local_interface_0.py | 197 ------------------------- 3 files changed, 102 insertions(+), 256 deletions(-) delete mode 100644 streamlit/local/p_local_interface_0.py diff --git a/streamlit/interface_0.py b/streamlit/interface_0.py index f638209..f57fb43 100644 --- a/streamlit/interface_0.py +++ b/streamlit/interface_0.py @@ -18,6 +18,22 @@ create_batch_url = 'https://lfksus2t62.execute-api.us-east-2.amazonaws.com/dev/c REDIRECT_URI = 'https://doczydev.aarete.com:8500' user_list = USER_LIST +if 'uploading' not in st.session_state: + st.session_state.uploading = False + + + +def insert_upload_logs(batch_id, client_name, file_name, upload_datetime, upload_user): + """ + Input: batch_id, client_name, file_name, upload_datetime, upload_user + Output: status of the insert query + """ + try: + return 'Log inserted successfully' + except Exception as e: + return e + + st.set_page_config(layout = "wide") # # Sidebar contents # with st.sidebar: @@ -31,6 +47,7 @@ st.set_page_config(layout = "wide") # ) # add_vertical_space(15) # # st.write("Doczy") +# _,c1= st.columns([5,1]) try: @@ -45,7 +62,7 @@ except KeyError as e: st.write("Session Expired.") st.stop() - +print(st.session_state) s3_client = boto3.client('s3', region_name="us-east-2", @@ -65,7 +82,7 @@ client_row = st.columns([0.1, 0.8]) with client_row[0]: st.write("**Client Name**") with client_row[1]: - client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed") + client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344 client_bucket = client_s3_paths.get(client) @@ -76,42 +93,54 @@ file_row = st.columns([0.1, 0.8]) with file_row[0]: st.write("**Upload Files**") with file_row[1]: - file_list = st.file_uploader("Upload", type=None, accept_multiple_files=True, label_visibility = "collapsed") + file_list = st.file_uploader("Upload", type=['csv','pdf'], accept_multiple_files=True, label_visibility = "collapsed", help="Only .csv and .pdf file formats are supported.", disabled=st.session_state.uploading) add_vertical_space(2) df = pd.DataFrame(columns=['Contract Name']) df['Contract Name'] = file_list file_names = [] buttons = st.columns([0.4, 0.4, 0.2]) -with buttons[1]: - if st.button("Create Batch"): - myobj = { "client-bucket-name": client_bucket } - response = requests.post(create_batch_url, json = myobj) - if response.status_code >= 200 and response.status_code < 300: - try: - batch_id = json.loads(json.loads(response.text)['body'])['batch_id'] - landing_zone = json.loads(json.loads(response.text)['body'])['landing_zone'] - except: - st.write(myobj) - st.write(response.text) - batch_id = 'failed_cases' - landing_zone = 'contracts_landing_zone' +def set_uploading_state(): + if not client == None and not len(file_list) == 0: + st.session_state.uploading = True + +with buttons[1]: + if st.button("Create Batch", on_click = set_uploading_state): + if client == None: + st.error("No Client Name Selected.") + elif len(file_list) == 0: + st.error("No Files Selected.") else: - st.write("Failed") - for uploaded_file in file_list: - stringio = BytesIO(uploaded_file.getvalue()) - stringio.seek(0) - s3_client.put_object(Bucket=client_bucket, Body=stringio.getvalue(), Key= - landing_zone+batch_id+'/'+str(uploaded_file.name)) - - # TODO: Test this insert function with snowflake - upload_log = insert_upload_logs(batch_id, client, str(uploaded_file.name), datetime.now().strftime("%Y-%m-%d %H:%M:%S"), user_mail) - st.write(upload_log) - - file_names.append(str(uploaded_file.name)) - st.write(f"{batch_id} created") - st.write(f"Files uploaded to s3://{client_bucket}/{landing_zone}{batch_id}") + myobj = { "client-bucket-name": client_bucket } + response = requests.post(create_batch_url, json = myobj) + if response.status_code >= 200 and response.status_code < 300: + try: + batch_id = json.loads(json.loads(response.text)['body'])['batch_id'] + landing_zone = json.loads(json.loads(response.text)['body'])['landing_zone'] + except: + st.write(myobj) + st.write(response.text) + batch_id = 'failed_cases' + landing_zone = 'contracts_landing_zone' + else: + st.write("Failed") + for uploaded_file in file_list: + stringio = BytesIO(uploaded_file.getvalue()) + stringio.seek(0) + s3_client.put_object(Bucket=client_bucket, Body=stringio.getvalue(), Key= + landing_zone+batch_id+'/'+str(uploaded_file.name)) + + # TODO: Test this insert function with snowflake + upload_log = insert_upload_logs(batch_id, client, str(uploaded_file.name), datetime.now().strftime("%Y-%m-%d %H:%M:%S"), user_mail) + st.write(upload_log) + + file_names.append(str(uploaded_file.name)) + st.session_state.uploading = False + st.write(f"{batch_id} created") + st.write(f"Files uploaded to s3://{client_bucket}/{landing_zone}{batch_id}") + + # @st.cache_data diff --git a/streamlit/local/local_interface_0.py b/streamlit/local/local_interface_0.py index eba8f5d..55802f2 100644 --- a/streamlit/local/local_interface_0.py +++ b/streamlit/local/local_interface_0.py @@ -11,6 +11,7 @@ import boto3 import requests # from sf_conn import get_secret, save_to_sf from io import StringIO, BytesIO +import time # from sf_conn import get_client_names, insert_upload_logs # from constants import USER_LIST @@ -18,6 +19,10 @@ create_batch_url = 'https://lfksus2t62.execute-api.us-east-2.amazonaws.com/dev/c # REDIRECT_URI = 'https://doczydev.aarete.com:8500' # user_list = USER_LIST +if 'uploading' not in st.session_state: + st.session_state.uploading = False + + def insert_upload_logs(batch_id, client_name, file_name, upload_datetime, upload_user): """ @@ -43,6 +48,7 @@ st.set_page_config(layout = "wide") # # ) # # add_vertical_space(15) # # # st.write("Doczy") +# # _,c1= st.columns([5,1]) # try: @@ -58,7 +64,7 @@ except KeyError as e: st.write("Session Expired.") st.stop() - +print(st.session_state) s3_client = boto3.client('s3', region_name="us-east-2", @@ -89,47 +95,55 @@ file_row = st.columns([0.1, 0.8]) with file_row[0]: st.write("**Upload Files**") with file_row[1]: - file_list = st.file_uploader("Upload", type=['csv','pdf'], accept_multiple_files=True, label_visibility = "collapsed", help="Only .csv and .pdf file formats are supported.") + file_list = st.file_uploader("Upload", type=['csv','pdf'], accept_multiple_files=True, label_visibility = "collapsed", help="Only .csv and .pdf file formats are supported.", disabled=st.session_state.uploading) add_vertical_space(2) df = pd.DataFrame(columns=['Contract Name']) df['Contract Name'] = file_list file_names = [] buttons = st.columns([0.4, 0.4, 0.2]) + +def set_uploading_state(): + if not client == None and not len(file_list) == 0: + st.session_state.uploading = True + with buttons[1]: - if st.button("Create Batch"): + if st.button("Create Batch", on_click = set_uploading_state): if client == None: st.error("No Client Name Selected.") elif len(file_list) == 0: st.error("No Files Selected.") else: + time.sleep(5) + # myobj = { "client-bucket-name": client_bucket } + # response = requests.post(create_batch_url, json = myobj) + # if response.status_code >= 200 and response.status_code < 300: + # try: + # batch_id = json.loads(json.loads(response.text)['body'])['batch_id'] + # landing_zone = json.loads(json.loads(response.text)['body'])['landing_zone'] + # except: + # st.write(myobj) + # st.write(response.text) + # batch_id = 'failed_cases' + # landing_zone = 'contracts_landing_zone' + # else: + # st.write("Failed") + # for uploaded_file in file_list: + # stringio = BytesIO(uploaded_file.getvalue()) + # stringio.seek(0) + # s3_client.put_object(Bucket=client_bucket, Body=stringio.getvalue(), Key= + # landing_zone+batch_id+'/'+str(uploaded_file.name)) + + # # TODO: Test this insert function with snowflake + # upload_log = insert_upload_logs(batch_id, client, str(uploaded_file.name), datetime.now().strftime("%Y-%m-%d %H:%M:%S"), user_mail) + # st.write(upload_log) + + # file_names.append(str(uploaded_file.name)) + # st.write(f"{batch_id} created") + st.session_state.uploading = False + # st.write(f"Files uploaded to s3://{client_bucket}/{landing_zone}{batch_id}") + - myobj = { "client-bucket-name": client_bucket } - response = requests.post(create_batch_url, json = myobj) - if response.status_code >= 200 and response.status_code < 300: - try: - batch_id = json.loads(json.loads(response.text)['body'])['batch_id'] - landing_zone = json.loads(json.loads(response.text)['body'])['landing_zone'] - except: - st.write(myobj) - st.write(response.text) - batch_id = 'failed_cases' - landing_zone = 'contracts_landing_zone' - else: - st.write("Failed") - for uploaded_file in file_list: - stringio = BytesIO(uploaded_file.getvalue()) - stringio.seek(0) - s3_client.put_object(Bucket=client_bucket, Body=stringio.getvalue(), Key= - landing_zone+batch_id+'/'+str(uploaded_file.name)) - - # TODO: Test this insert function with snowflake - upload_log = insert_upload_logs(batch_id, client, str(uploaded_file.name), datetime.now().strftime("%Y-%m-%d %H:%M:%S"), user_mail) - st.write(upload_log) - - file_names.append(str(uploaded_file.name)) - st.write(f"{batch_id} created") - st.write(f"Files uploaded to s3://{client_bucket}/{landing_zone}{batch_id}") # @st.cache_data diff --git a/streamlit/local/p_local_interface_0.py b/streamlit/local/p_local_interface_0.py deleted file mode 100644 index 8024684..0000000 --- a/streamlit/local/p_local_interface_0.py +++ /dev/null @@ -1,197 +0,0 @@ -import json -import streamlit as st -from streamlit_extras.add_vertical_space import add_vertical_space -import os -import streamlit as st -import pandas as pd -from io import StringIO -from datetime import datetime -import boto3 -# import util -import requests -# from sf_conn import get_secret, save_to_sf -from io import StringIO, BytesIO -import time -# from sf_conn import get_client_names, insert_upload_logs -# from constants import USER_LIST - -create_batch_url = 'https://lfksus2t62.execute-api.us-east-2.amazonaws.com/dev/create-batch' - -# REDIRECT_URI = 'https://doczydev.aarete.com:8500' -# user_list = USER_LIST -if 'uploading' not in st.session_state: - st.session_state.uploading = False - - - -def insert_upload_logs(batch_id, client_name, file_name, upload_datetime, upload_user): - """ - Input: batch_id, client_name, file_name, upload_datetime, upload_user - Output: status of the insert query - """ - try: - return 'Log inserted successfully' - except Exception as e: - return e - - -st.set_page_config(layout = "wide") -# # # Sidebar contents -# # with st.sidebar: -# # st.title("Doczy.AI ™") -# # st.markdown( -# # """ -# # ## About -# # This app extracts data from contracts - -# # """ -# # ) -# # add_vertical_space(15) -# # # st.write("Doczy") -# # - -_,c1= st.columns([5,1]) -# try: -# util.setup_page(REDIRECT_URI) -# except Exception as e: -# st.write(f"SSO Failed = {e}") -# st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} -st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} -try: - c1.write(f"User: **{st.session_state.user_info['displayName']}**") - user_mail = st.session_state.user_info['mail'] -except KeyError as e: - st.write("Session Expired.") - st.stop() - - - -s3_client = boto3.client('s3', - region_name="us-east-2", -) - -client_list = ['doczy-ai-client-1', 'Delaware First Health, Inc.', 'Community Health Choice, Inc','CareSource Network Partners LLC', - 'HealthNet of Cali', 'Oklahoma Complete Health, Inc', 'HealthFirst', 'Molina Healthcare of TX', 'AvMed', 'Arizona Care1st', - 'WellCare New Jersey'] - -# This is the list of client fetched from Snowflake -# TODO: Need to update the streamlit code to use the client names from this list -# And use the s3 paths to save the objects for the respective client -# client_list, s3_paths = get_client_names() -# client_s3_paths = dict(zip(client_list, s3_paths)) - -client_row = st.columns([0.1, 0.8]) -with client_row[0]: - st.write("**Client Name**") -with client_row[1]: - client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344 - -# client_bucket = client_s3_paths.get(client) - -# to be deleted when client buckets are created -client_bucket = 'doczy-ai-client-1' - -file_row = st.columns([0.1, 0.8]) -with file_row[0]: - st.write("**Upload Files**") -with file_row[1]: - file_list = st.file_uploader("Upload", type=['csv','pdf'], accept_multiple_files=True, label_visibility = "collapsed", help="Only .csv and .pdf file formats are supported.", disabled=st.session_state.uploading) - -add_vertical_space(2) -df = pd.DataFrame(columns=['Contract Name']) -df['Contract Name'] = file_list -file_names = [] -buttons = st.columns([0.4, 0.4, 0.2]) -with buttons[1]: - if st.button("Create Batch"): - st.session_state.uploading = True - if client == None: - st.session_state.uploading = False - st.error("No Client Name Selected.") - elif len(file_list) == 0: - st.session_state.uploading = False - st.error("No Files Selected.") - else: - myobj = { "client-bucket-name": client_bucket } - response = requests.post(create_batch_url, json = myobj) - if response.status_code >= 200 and response.status_code < 300: - try: - batch_id = json.loads(json.loads(response.text)['body'])['batch_id'] - landing_zone = json.loads(json.loads(response.text)['body'])['landing_zone'] - except: - st.write(myobj) - st.write(response.text) - batch_id = 'failed_cases' - landing_zone = 'contracts_landing_zone' - else: - st.write("Failed") - for uploaded_file in file_list: - stringio = BytesIO(uploaded_file.getvalue()) - stringio.seek(0) - s3_client.put_object(Bucket=client_bucket, Body=stringio.getvalue(), Key= - landing_zone+batch_id+'/'+str(uploaded_file.name)) - - # TODO: Test this insert function with snowflake - upload_log = insert_upload_logs(batch_id, client, str(uploaded_file.name), datetime.now().strftime("%Y-%m-%d %H:%M:%S"), user_mail) - st.write(upload_log) - - file_names.append(str(uploaded_file.name)) - st.write(f"{batch_id} created") - st.session_state.uploading = False - st.write(f"Files uploaded to s3://{client_bucket}/{landing_zone}{batch_id}") - - - - - # @st.cache_data - # def convert_df(df): - # return df.to_csv(index=False).encode('utf-8') - - # csv = convert_df(df) - - # df = df.reset_index() # make sure indexes pair with number of rows - - # contract_list = [] - # for index, row in df.iterrows(): - # group_list = [] - # if row['Unique Key']: - # group_list.append('Unique Key') - # if row['Pricing Before Carveouts']: - # group_list.append('Pricing Before Carveouts') - # if row['Contract Related']: - # group_list.append('Contract Related') - # if row['Provider']: - # group_list.append('Provider') - # if row['Timeline']: - # group_list.append('Timeline') - # if row['Carveout Indicator']: - # group_list.append('Carveout Indicator') - # if row['Carveout Methodology']: - # group_list.append('Carveout Methodology') - # entry_dict = { - # "contract_name": row['Contract Name'], - # "groups": group_list, - # "contract_source_path": "batches/batch_1/"+client+"/"+row['Contract Name'] - # } - # contract_list.append(entry_dict) - # contract_list = list(df['Contract Name']) - - # myobj = { - # "s3_bucket": 'doczy-dev-infra-textract', - # "batch_id": "1", - # "client_name": client, - # "username": user_mail, - # "contract_list": contract_list - # } - - # buttons = st.columns([0.8, 0.2]) - # with buttons[0]: - # st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv') - # with buttons[1]: - # if st.button("Upload to DB"): - # response = requests.post(doczy_pipeline, json = myobj) - # if response.status_code >= 200 and response.status_code < 300: - # st.write("Success") - # else: - # st.write("Failed") - # # st.write(response.text) From 139af89785bb6fc4601d3859dc15967dc783c79f Mon Sep 17 00:00:00 2001 From: "AARETE\\agupta" Date: Tue, 4 Jun 2024 10:35:56 -0500 Subject: [PATCH 35/37] Small change to change the list of acceptable file formats --- streamlit/interface_0.py | 2 +- streamlit/local/local_interface_0.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/streamlit/interface_0.py b/streamlit/interface_0.py index f57fb43..a4ad0f2 100644 --- a/streamlit/interface_0.py +++ b/streamlit/interface_0.py @@ -93,7 +93,7 @@ file_row = st.columns([0.1, 0.8]) with file_row[0]: st.write("**Upload Files**") with file_row[1]: - file_list = st.file_uploader("Upload", type=['csv','pdf'], accept_multiple_files=True, label_visibility = "collapsed", help="Only .csv and .pdf file formats are supported.", disabled=st.session_state.uploading) + file_list = st.file_uploader("Upload", type=['docx','tiff','pdf'], accept_multiple_files=True, label_visibility = "collapsed", help="Only PDF, TIFF and DOCX file formats are supported.", disabled=st.session_state.uploading) add_vertical_space(2) df = pd.DataFrame(columns=['Contract Name']) diff --git a/streamlit/local/local_interface_0.py b/streamlit/local/local_interface_0.py index 55802f2..1c6a006 100644 --- a/streamlit/local/local_interface_0.py +++ b/streamlit/local/local_interface_0.py @@ -95,7 +95,7 @@ file_row = st.columns([0.1, 0.8]) with file_row[0]: st.write("**Upload Files**") with file_row[1]: - file_list = st.file_uploader("Upload", type=['csv','pdf'], accept_multiple_files=True, label_visibility = "collapsed", help="Only .csv and .pdf file formats are supported.", disabled=st.session_state.uploading) + file_list = st.file_uploader("Upload", type=['docx','tiff','pdf'], accept_multiple_files=True, label_visibility = "collapsed", help="Only PDF, TIFF and DOCX file formats are supported.", disabled=st.session_state.uploading) add_vertical_space(2) df = pd.DataFrame(columns=['Contract Name']) From 8c762424281f162c31f7efd3005d4dd88a02c663 Mon Sep 17 00:00:00 2001 From: "AARETE\\agupta" Date: Tue, 4 Jun 2024 12:57:14 -0500 Subject: [PATCH 36/37] Made changes for the PDF viewer to be able to pull the bytes from the S3 Bucket and display it. --- streamlit/interface_2.py | 74 ++++++++++++++++++++++------ streamlit/local/local_interface_2.py | 2 +- 2 files changed, 59 insertions(+), 17 deletions(-) diff --git a/streamlit/interface_2.py b/streamlit/interface_2.py index ed8c494..824509a 100644 --- a/streamlit/interface_2.py +++ b/streamlit/interface_2.py @@ -10,6 +10,7 @@ from langchain.chains import RetrievalQA import streamlit as st from streamlit_extras.add_vertical_space import add_vertical_space +from streamlit_pdf_viewer import pdf_viewer # needs to be installed on the server import os import pandas as pd import numpy as np @@ -18,24 +19,26 @@ import anthropic from pydantic import BaseModel from typing import List import re +import base64 from sf_conn import get_snowflake_conn +import io REDIRECT_URI = 'https://doczydev.aarete.com:8502' user_list = USER_LIST st.set_page_config(layout = "wide") # Sidebar contents -with st.sidebar: - st.title("Doczy.AI ™") - st.markdown( - """ - ## About - This app extracts data from contracts +# with st.sidebar: +# st.title("Doczy.AI ™") +# st.markdown( +# """ +# ## About +# This app extracts data from contracts - """ - ) - add_vertical_space(15) - # st.write("Doczy") +# """ +# ) +# add_vertical_space(15) +# # st.write("Doczy") _,c1= st.columns([5,1]) try: @@ -91,13 +94,11 @@ for obj in objects['Contents']: contract_list = sorted(file_list) - - file_row = st.columns([0.2, 0.7, 0.1]) with file_row[0]: st.write("**Contract Name**") with file_row[1]: - file_name = st.selectbox('Select a file', contract_list + ['All'], label_visibility = "collapsed") + file_name = st.selectbox('Select a file', ['All'] + contract_list, label_visibility = "collapsed", index= None) # MODIFIED - Append 'All' in the front instead of at the end field_row = st.columns([0.2, 0.7, 0.1]) with field_row[0]: @@ -107,7 +108,7 @@ with field_row[1]: , 'Pricing Before Carveouts - II', 'Carveout Indicator, Code Type and Code #s - I' , 'Carveout Indicator, Code Type and Code #s - II', 'Carveout Indicator, Code Type and Code #s - III' , 'Optimize Carving Indic.', 'Carveout Method - I', 'Carveout Method - II', 'Provider' - , 'Timeline'), label_visibility = "collapsed") + , 'Timeline'), label_visibility = "collapsed", index = None) if field_group == 'Unique Key': fields = fields[fields['PRIORITY'] == 'A'] @@ -151,10 +152,51 @@ if st.button("Show Results"): df2 = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description]) # get this dataframe from snowflake table - # df2 = pd.DataFrame(columns=['Contract Name','Field Name', 'SF_DB_COL_NAME', 'Snippet','Page Number' - # , 'Field Extracted Value', 'Actual Value','Imputed Value']) + df2 = pd.DataFrame(columns=['Contract Name','Field Name', 'SF_DB_COL_NAME', 'Snippet','Page Number' + , 'Field Extracted Value', 'Actual Value','Imputed Value']) df2.to_csv('temp2.csv', index=False) +if st.button("Show PDF"): + if file_name == None or file_name == "All": + st.error("Choose one specific file.") + else: + with st.sidebar: + st.markdown( + """ + + """, + unsafe_allow_html=True, + ) + s3_client.Object(bucket,file_name) + data=obj.get()['Body'].read() + pdf_viewer(io.BytesIO(data), width=1500) + +# if st.button("Show PDF"): +# if file_name == None or file_name == "All": +# st.error("Choose one specific file.") +# else: +# with st.sidebar: +# with open(file_name, "rb") as f: +# base64_pdf = base64.b64encode(f.read()).decode('utf-8') +# # Embedding PDF in HTML +# pdf_display = F'' +# # Displaying File +# st.markdown( +# """ +# +# """, +# unsafe_allow_html=True, +# ) +# st.markdown(pdf_display, unsafe_allow_html=True) + df2 = pd.read_csv('temp2.csv') df2['Imputed Value'] = '' edited_df = st.data_editor(df2) diff --git a/streamlit/local/local_interface_2.py b/streamlit/local/local_interface_2.py index 7e40615..18702f3 100644 --- a/streamlit/local/local_interface_2.py +++ b/streamlit/local/local_interface_2.py @@ -10,7 +10,7 @@ from langchain.chains import RetrievalQA import streamlit as st from streamlit_extras.add_vertical_space import add_vertical_space -from streamlit_pdf_viewer import pdf_viewer +from streamlit_pdf_viewer import pdf_viewer # needs to be installed on the server import os import pandas as pd import numpy as np From ecea8f1145d023be2eb4aac951cdc95bd444927f Mon Sep 17 00:00:00 2001 From: Umang Mistry Date: Thu, 6 Jun 2024 09:52:21 -0500 Subject: [PATCH 37/37] Updated EC2 server user_data --- streamlit-server/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streamlit-server/main.tf b/streamlit-server/main.tf index 5ec66d2..db081fc 100644 --- a/streamlit-server/main.tf +++ b/streamlit-server/main.tf @@ -106,7 +106,7 @@ resource "aws_instance" "streamlit_server" { apt-get update apt-get install -y python3 python3-pip sudo apt-get install libffi-dev - python3 -m pip3 install -r /tmp/requirements.txt + python3 -m pip install -r /tmp/requirements.txt sudo apt install openssh-client sudo apt install net-tools