jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

[DOC-535] Snowsql test

Browse Source
This commit is contained in:
Grzegorz Huber
2024-06-25 17:06:46 +02:00
parent bb518543a8
commit a791b89125
6 changed files with 40 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bitbucket-pipelines.yml
+8 -8
View File
@@ -292,13 +292,13 @@ pipelines:
name: Run Snowsql on DEV
oidc: true
script:
# - *aws-context-dev
- export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO:role/$OIDC_ROLE
- export STS_OUTPUT=$(aws sts assume-role-with-web-identity --role-arn $AWS_ROLE_ARN --role-session-name BitbucketPipeline --web-identity-token "$BITBUCKET_STEP_OIDC_TOKEN" --duration-seconds 3600)
- export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId')
- export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey')
- export AWS_SESSION_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken')
- aws sts get-caller-identity --output text
- *aws-context-dev
# - export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO:role/$OIDC_ROLE
# - export STS_OUTPUT=$(aws sts assume-role-with-web-identity --role-arn $AWS_ROLE_ARN --role-session-name BitbucketPipeline --web-identity-token "$BITBUCKET_STEP_OIDC_TOKEN" --duration-seconds 3600)
# - export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId')
# - export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey')
# - export AWS_SESSION_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken')
# - aws sts get-caller-identity --output text
# - bash snowsql-cicd.sh dev
- IS_DEPLOY_ALL=true python terraform.py apply --environment=dev --module=textract-pipeline/terraform --apply-extra-args="-target=null_resource.snowsql_client_create"
@@ -307,7 +307,7 @@ pipelines:
name: Run Snowsql on UAT
oidc: true
script:
# - *aws-context-dev
# - *aws-context-uat
- export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO_UAT:role/$OIDC_ROLE_UAT
- export STS_OUTPUT=$(aws sts assume-role-with-web-identity --role-arn $AWS_ROLE_ARN --role-session-name BitbucketPipeline --web-identity-token "$BITBUCKET_STEP_OIDC_TOKEN" --duration-seconds 3600)
- export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId')
textract-pipeline/terraform/main.tf
+10 -7
View File
@@ -7,11 +7,6 @@ terraform {
}
backend "s3" {
bucket = "doczyai-use2-d-infra-s3-terraform-state" # Parameterize using -backend-config flag with "terraform init"
key = "terraform/textract-pipeline/terraform.tfstate" # Parameterize
region = "us-east-2" # Parameterize
# profile = "doczyai" # Parameterize
dynamodb_table = "doczyai-use2-d-infra-dyd-terraform-lock" # Parameterize
encrypt = true
}
}
@@ -66,7 +61,11 @@ locals {
Environment = var.environment
Terraform = "true"
}
s3_prefix = "${local.global_prefix}-s3"
processing_s3_bucket_name = "${local.s3_prefix}-textract-processing-001"
}
provider "aws" {
default_tags {
@@ -80,7 +79,7 @@ module "textract_pipeline" {
# for each for multiple resources
for_each = var.client_list
client_name = each.value.client_name
client_name = each.value.client_id
aws_account_id = var.aws_account_id
@@ -113,7 +112,11 @@ resource "null_resource" "snowsql_client_create" {
provisioner "local-exec" {
environment = {
}
command = "./snowsql-cicd.sh dev ${each.value.client_name}"
# doczyai-use2-u-cn1-s3-textract-processing-001
# doczyai-use2-d-infra-s3-textract-processing-001
# doczyai-use2-d-cn1-s3-textract-processing-001
command = "./snowsql-cicd.sh dev ${each.value.client_id} ${each.value.full_name} ${replace(local.processing_s3_bucket_name, "infra", each.value.client_id)}"
}
}
textract-pipeline/terraform/snowsql-cicd.sh
+16 -3
View File
@@ -3,9 +3,11 @@
set -euo pipefail
ENV_GROUP=$1
CLIENT_NAME=$2
CLIENT_ID=$2
FULL_NAME=$3
BUCKET_NAME=$4
echo "ENV_GROUP=${ENV_GROUP}, CLIENT_NAME=${CLIENT_NAME}"
echo "ENV_GROUP=${ENV_GROUP}, CLIENT_ID=${CLIENT_ID}, FULL_NAME=${FULL_NAME}, BUCKET_NAME=${BUCKET_NAME}"
readonly SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
@@ -47,4 +49,15 @@ export SNOWSQL_PWD=${SNOWFLAKE_PASSWORD}
#snowsql -a "${SNOWFLAKE_ACCOUNT}" -u "${SNOWFLAKE_USER}" -d "${SNOWFLAKE_DATABASE}" -r "${SNOWFLAKE_ROLE}" -o exit_on_error=true -f "${SCRIPT_DIR}/snowsql-query.sql"
snowsql -a "${SNOWFLAKE_ACCOUNT}" -u "${SNOWFLAKE_USER}" -d "${SNOWFLAKE_DATABASE}" -r "${SNOWFLAKE_ROLE}" -o exit_on_error=true -q "SELECT DISTINCT client_id, client_name, bucket_name FROM STG.CLIENT_LOGS"
#snowsql -a "${SNOWFLAKE_ACCOUNT}" -u "${SNOWFLAKE_USER}" -d "${SNOWFLAKE_DATABASE}" -r "${SNOWFLAKE_ROLE}" -o exit_on_error=true -q "INSERT INTO STG.CLIENT_LOGS(CLIENT_ID, CLIENT_NAME, BUCKET_NAME) VALUES ('001','CLIENT_NAME','doczyai-use2-d-${CLIENT_NAME}')"
#QUERY="INSERT INTO STG.CLIENT_LOGS(CLIENT_ID, CLIENT_NAME, BUCKET_NAME) VALUES ('${CLIENT_ID}','${FULL_NAME}','${BUCKET_NAME}')"
#echo "QUERY=${QUERY}"
#snowsql -a "${SNOWFLAKE_ACCOUNT}" -u "${SNOWFLAKE_USER}" -d "${SNOWFLAKE_DATABASE}" -r "${SNOWFLAKE_ROLE}" -o exit_on_error=true -q "${QUERY}"
#INSERT INTO STG.CLIENT_LOGS (CLIENT_ID, CLIENT_NAME, BUCKET_NAME)
#SELECT '001', 'CLIENT_NAME', 'doczyai-use2-d-CLIENT_NAME'
#WHERE NOT EXISTS (
# SELECT 1
# FROM STG.CLIENT_LOGS
# WHERE CLIENT_ID = '001'
#);
textract-pipeline/terraform/variables.tf
+2 -1
View File
@@ -17,7 +17,8 @@ variable "environment" {
variable "client_list" {
type = map(object({
client_name = string
client_id = string
full_name = string
}))
}
textract-pipeline/terraform/vars-dev.tfvars
+2 -1
View File
@@ -3,6 +3,7 @@ text_creation_lambda_additional_layer_arn = "arn:aws:lambda:us-east-2:3363929483
client_list = {
client1 = {
client_name = "cn1"
client_id = "cn1"
full_name = "Centene"
}
}
textract-pipeline/terraform/vars-uat.tfvars
+2 -1
View File
@@ -4,6 +4,7 @@ text_creation_lambda_additional_layer_arn = "arn:aws:lambda:us-east-2:3363929483
client_list = {
client1 = {
client_name = "cn1"
client_id = "cn1"
full_name = "Centene"
}
}