jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

[DOC-535] Snowsql test

Browse Source
This commit is contained in:
Grzegorz Huber
2024-06-25 16:11:07 +02:00
parent 4fd8758c7a
commit bb518543a8
6 changed files with 32 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bitbucket-pipelines.yml
+17 -1
View File
@@ -286,9 +286,10 @@ pipelines:
feature/DOC-535-snowsql-in-cicd:
- parallel:
- step:
image: hugree/terraform-with-snowsql:latest
name: Run Snowsql in CICD
name: Run Snowsql on DEV
oidc: true
script:
# - *aws-context-dev
@@ -300,3 +301,18 @@ pipelines:
- aws sts get-caller-identity --output text
# - bash snowsql-cicd.sh dev
- IS_DEPLOY_ALL=true python terraform.py apply --environment=dev --module=textract-pipeline/terraform --apply-extra-args="-target=null_resource.snowsql_client_create"
- step:
image: hugree/terraform-with-snowsql:latest
name: Run Snowsql on UAT
oidc: true
script:
# - *aws-context-dev
- export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO_UAT:role/$OIDC_ROLE_UAT
- export STS_OUTPUT=$(aws sts assume-role-with-web-identity --role-arn $AWS_ROLE_ARN --role-session-name BitbucketPipeline --web-identity-token "$BITBUCKET_STEP_OIDC_TOKEN" --duration-seconds 3600)
- export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId')
- export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey')
- export AWS_SESSION_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken')
- aws sts get-caller-identity --output text
# - bash snowsql-cicd.sh dev
- IS_DEPLOY_ALL=true python terraform.py apply --environment=uat --module=textract-pipeline/terraform --apply-extra-args="-target=null_resource.snowsql_client_create"
textract-pipeline/terraform/snowsql-cicd.sh
+2 -1
View File
@@ -45,5 +45,6 @@ echo "SNOWFLAKE_USER=${SNOWFLAKE_USER}"
echo "SNOWFLAKE_ROLE=${SNOWFLAKE_ROLE}"
export SNOWSQL_PWD=${SNOWFLAKE_PASSWORD}
snowsql -a "${SNOWFLAKE_ACCOUNT}" -u "${SNOWFLAKE_USER}" -d "${SNOWFLAKE_DATABASE}" -r "${SNOWFLAKE_ROLE}" -o exit_on_error=true -f "${SCRIPT_DIR}/snowsql-query.sql"
#snowsql -a "${SNOWFLAKE_ACCOUNT}" -u "${SNOWFLAKE_USER}" -d "${SNOWFLAKE_DATABASE}" -r "${SNOWFLAKE_ROLE}" -o exit_on_error=true -f "${SCRIPT_DIR}/snowsql-query.sql"
snowsql -a "${SNOWFLAKE_ACCOUNT}" -u "${SNOWFLAKE_USER}" -d "${SNOWFLAKE_DATABASE}" -r "${SNOWFLAKE_ROLE}" -o exit_on_error=true -q "SELECT DISTINCT client_id, client_name, bucket_name FROM STG.CLIENT_LOGS"
#snowsql -a "${SNOWFLAKE_ACCOUNT}" -u "${SNOWFLAKE_USER}" -d "${SNOWFLAKE_DATABASE}" -r "${SNOWFLAKE_ROLE}" -o exit_on_error=true -q "INSERT INTO STG.CLIENT_LOGS(CLIENT_ID, CLIENT_NAME, BUCKET_NAME) VALUES ('001','CLIENT_NAME','doczyai-use2-d-${CLIENT_NAME}')"
textract-pipeline/terraform/snowsql-query.sql
-2
View File
@@ -1,2 +0,0 @@
SELECT DISTINCT client_id, client_name, bucket_name FROM STG.CLIENT_LOGS
--INSERT INTO STG.CLIENT_LOGS(CLIENT_ID, CLIENT_NAME, BUCKET_NAME) VALUES ('001','CLIENT_NAME','doczyai-use2-d-cn1')
textract-pipeline/terraform/variables.tf
-5
View File
@@ -19,11 +19,6 @@ variable "client_list" {
type = map(object({
client_name = string
}))
default = {
client1 = {
client_name = "cn1"
}
}
}
variable "aws_account_id" {
textract-pipeline/terraform/vars-dev.tfvars
+7 -1
View File
@@ -1,2 +1,8 @@
aws_account_id = 660131068782
text_creation_lambda_additional_layer_arn = "arn:aws:lambda:us-east-2:336392948345:layer:AWSSDKPandas-Python312:8"
text_creation_lambda_additional_layer_arn = "arn:aws:lambda:us-east-2:336392948345:layer:AWSSDKPandas-Python312:8"
client_list = {
client1 = {
client_name = "cn1"
}
}
textract-pipeline/terraform/vars-uat.tfvars
+6
View File
@@ -1,3 +1,9 @@
aws_account_id = 975049960860
# TODO provide new layer arn for UAT? 336392948345 was used in DEV but not sure what account that is?!
text_creation_lambda_additional_layer_arn = "arn:aws:lambda:us-east-2:336392948345:layer:AWSSDKPandas-Python312:8"
client_list = {
client1 = {
client_name = "cn1"
}
}