diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 41000ea..fc0e6e8 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -292,13 +292,13 @@ pipelines: name: Run Snowsql on DEV oidc: true script: -# - *aws-context-dev - - export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO:role/$OIDC_ROLE - - export STS_OUTPUT=$(aws sts assume-role-with-web-identity --role-arn $AWS_ROLE_ARN --role-session-name BitbucketPipeline --web-identity-token "$BITBUCKET_STEP_OIDC_TOKEN" --duration-seconds 3600) - - export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId') - - export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey') - - export AWS_SESSION_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken') - - aws sts get-caller-identity --output text + - *aws-context-dev +# - export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO:role/$OIDC_ROLE +# - export STS_OUTPUT=$(aws sts assume-role-with-web-identity --role-arn $AWS_ROLE_ARN --role-session-name BitbucketPipeline --web-identity-token "$BITBUCKET_STEP_OIDC_TOKEN" --duration-seconds 3600) +# - export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId') +# - export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey') +# - export AWS_SESSION_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken') +# - aws sts get-caller-identity --output text # - bash snowsql-cicd.sh dev - IS_DEPLOY_ALL=true python terraform.py apply --environment=dev --module=textract-pipeline/terraform --apply-extra-args="-target=null_resource.snowsql_client_create" @@ -307,7 +307,7 @@ pipelines: name: Run Snowsql on UAT oidc: true script: - # - *aws-context-dev + # - *aws-context-uat - export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO_UAT:role/$OIDC_ROLE_UAT - export STS_OUTPUT=$(aws sts assume-role-with-web-identity --role-arn $AWS_ROLE_ARN --role-session-name BitbucketPipeline --web-identity-token "$BITBUCKET_STEP_OIDC_TOKEN" --duration-seconds 3600) - export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId') diff --git a/textract-pipeline/terraform/main.tf b/textract-pipeline/terraform/main.tf index 07a5403..4c7a8ac 100644 --- a/textract-pipeline/terraform/main.tf +++ b/textract-pipeline/terraform/main.tf @@ -7,11 +7,6 @@ terraform { } backend "s3" { - bucket = "doczyai-use2-d-infra-s3-terraform-state" # Parameterize using -backend-config flag with "terraform init" - key = "terraform/textract-pipeline/terraform.tfstate" # Parameterize - region = "us-east-2" # Parameterize - # profile = "doczyai" # Parameterize - dynamodb_table = "doczyai-use2-d-infra-dyd-terraform-lock" # Parameterize encrypt = true } } @@ -66,7 +61,11 @@ locals { Environment = var.environment Terraform = "true" } + + s3_prefix = "${local.global_prefix}-s3" + processing_s3_bucket_name = "${local.s3_prefix}-textract-processing-001" } + provider "aws" { default_tags { @@ -80,7 +79,7 @@ module "textract_pipeline" { # for each for multiple resources for_each = var.client_list - client_name = each.value.client_name + client_name = each.value.client_id aws_account_id = var.aws_account_id @@ -113,7 +112,11 @@ resource "null_resource" "snowsql_client_create" { provisioner "local-exec" { environment = { } - command = "./snowsql-cicd.sh dev ${each.value.client_name}" +# doczyai-use2-u-cn1-s3-textract-processing-001 +# doczyai-use2-d-infra-s3-textract-processing-001 +# doczyai-use2-d-cn1-s3-textract-processing-001 + + command = "./snowsql-cicd.sh dev ${each.value.client_id} ${each.value.full_name} ${replace(local.processing_s3_bucket_name, "infra", each.value.client_id)}" } } diff --git a/textract-pipeline/terraform/snowsql-cicd.sh b/textract-pipeline/terraform/snowsql-cicd.sh index 84e0880..f2753b7 100755 --- a/textract-pipeline/terraform/snowsql-cicd.sh +++ b/textract-pipeline/terraform/snowsql-cicd.sh @@ -3,9 +3,11 @@ set -euo pipefail ENV_GROUP=$1 -CLIENT_NAME=$2 +CLIENT_ID=$2 +FULL_NAME=$3 +BUCKET_NAME=$4 -echo "ENV_GROUP=${ENV_GROUP}, CLIENT_NAME=${CLIENT_NAME}" +echo "ENV_GROUP=${ENV_GROUP}, CLIENT_ID=${CLIENT_ID}, FULL_NAME=${FULL_NAME}, BUCKET_NAME=${BUCKET_NAME}" readonly SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" @@ -47,4 +49,15 @@ export SNOWSQL_PWD=${SNOWFLAKE_PASSWORD} #snowsql -a "${SNOWFLAKE_ACCOUNT}" -u "${SNOWFLAKE_USER}" -d "${SNOWFLAKE_DATABASE}" -r "${SNOWFLAKE_ROLE}" -o exit_on_error=true -f "${SCRIPT_DIR}/snowsql-query.sql" snowsql -a "${SNOWFLAKE_ACCOUNT}" -u "${SNOWFLAKE_USER}" -d "${SNOWFLAKE_DATABASE}" -r "${SNOWFLAKE_ROLE}" -o exit_on_error=true -q "SELECT DISTINCT client_id, client_name, bucket_name FROM STG.CLIENT_LOGS" -#snowsql -a "${SNOWFLAKE_ACCOUNT}" -u "${SNOWFLAKE_USER}" -d "${SNOWFLAKE_DATABASE}" -r "${SNOWFLAKE_ROLE}" -o exit_on_error=true -q "INSERT INTO STG.CLIENT_LOGS(CLIENT_ID, CLIENT_NAME, BUCKET_NAME) VALUES ('001','CLIENT_NAME','doczyai-use2-d-${CLIENT_NAME}')" + +#QUERY="INSERT INTO STG.CLIENT_LOGS(CLIENT_ID, CLIENT_NAME, BUCKET_NAME) VALUES ('${CLIENT_ID}','${FULL_NAME}','${BUCKET_NAME}')" +#echo "QUERY=${QUERY}" +#snowsql -a "${SNOWFLAKE_ACCOUNT}" -u "${SNOWFLAKE_USER}" -d "${SNOWFLAKE_DATABASE}" -r "${SNOWFLAKE_ROLE}" -o exit_on_error=true -q "${QUERY}" + +#INSERT INTO STG.CLIENT_LOGS (CLIENT_ID, CLIENT_NAME, BUCKET_NAME) +#SELECT '001', 'CLIENT_NAME', 'doczyai-use2-d-CLIENT_NAME' +#WHERE NOT EXISTS ( +# SELECT 1 +# FROM STG.CLIENT_LOGS +# WHERE CLIENT_ID = '001' +#); \ No newline at end of file diff --git a/textract-pipeline/terraform/variables.tf b/textract-pipeline/terraform/variables.tf index c29c60c..c4fb014 100644 --- a/textract-pipeline/terraform/variables.tf +++ b/textract-pipeline/terraform/variables.tf @@ -17,7 +17,8 @@ variable "environment" { variable "client_list" { type = map(object({ - client_name = string + client_id = string + full_name = string })) } diff --git a/textract-pipeline/terraform/vars-dev.tfvars b/textract-pipeline/terraform/vars-dev.tfvars index eee4077..4ca3365 100644 --- a/textract-pipeline/terraform/vars-dev.tfvars +++ b/textract-pipeline/terraform/vars-dev.tfvars @@ -3,6 +3,7 @@ text_creation_lambda_additional_layer_arn = "arn:aws:lambda:us-east-2:3363929483 client_list = { client1 = { - client_name = "cn1" + client_id = "cn1" + full_name = "Centene" } } \ No newline at end of file diff --git a/textract-pipeline/terraform/vars-uat.tfvars b/textract-pipeline/terraform/vars-uat.tfvars index 8223bd2..b7365e0 100644 --- a/textract-pipeline/terraform/vars-uat.tfvars +++ b/textract-pipeline/terraform/vars-uat.tfvars @@ -4,6 +4,7 @@ text_creation_lambda_additional_layer_arn = "arn:aws:lambda:us-east-2:3363929483 client_list = { client1 = { - client_name = "cn1" + client_id = "cn1" + full_name = "Centene" } } \ No newline at end of file