definitions: steps: - step: &terraform_plan_apply name: "Plan and Apply Terraform" image: zenika/terraform-aws-cli:latest script: - terraform --version # terraform steps here trigger: automatic # uncomment after testing # trigger: manual - step: &snowflake_deploy name: "Deploy Snowflake Changes" image: python:3.7 size: 2x script: - python -m pip install --upgrade pip - pip install snowflake-connector-python - pip install schemachange --upgrade # Setting the database name dynamically based on the branch name - if [ "$BITBUCKET_BRANCH" == "DEV" ]; then export SNOWFLAKE_DATABASE=$DEV_SNOWFLAKE_DATABASE; export SNOWFLAKE_ROLE=$DEV_SNOWFLAKE_ROLE; fi - if [ "$BITBUCKET_BRANCH" == "UAT" ]; then export SNOWFLAKE_DATABASE=$UAT_SNOWFLAKE_DATABASE; export SNOWFLAKE_ROLE=$UAT_SNOWFLAKE_ROLE; fi - if [ "$BITBUCKET_BRANCH" == "PROD" ]; then export SNOWFLAKE_DATABASE=$PROD_SNOWFLAKE_DATABASE; export SNOWFLAKE_ROLE=$PROD_SNOWFLAKE_ROLE; fi - schemachange -a $SNOWFLAKE_ACCOUNT -u $SNOWFLAKE_USER -r $SNOWFLAKE_ROLE -w $SNOWFLAKE_WAREHOUSE -d $SNOWFLAKE_DATABASE -c $SNOWFLAKE_DATABASE.SCHEMACHANGE.CHANGE_HISTORY --create-change-history-table -v trigger: automatic # trigger: manual caches: - pip - step: &streamlit_deploy name: "Deploy streamlit to EC2" oidc: true image: python:3.8 script: - python -m pip install --upgrade pip - apt-get update && apt-get install -y jq git - pip install awscli - export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO:role/$OIDC_ROLE - export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token - echo $BITBUCKET_STEP_OIDC_TOKEN > $(pwd)/web-identity-token - export STS_OUTPUT=$(aws sts assume-role-with-web-identity --role-arn $AWS_ROLE_ARN --role-session-name BitbucketPipeline --web-identity-token "$BITBUCKET_STEP_OIDC_TOKEN" --duration-seconds 3600) - export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId') - export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey') - export AWS_SESSION_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken') - aws sts get-caller-identity - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $DEV_INSTANCE_ID --region $AWS_DEFAULT_REGION --parameters commands='["cd /home/ubuntu/streamlit","(if [ -d doczy.ai ]; then cd doczy.ai && git fetch && git pull; else git clone git@bitbucket.org:aarete/doczy.ai.git; fi)"]' trigger: automatic # uncomment after testing - step: &airflow_dags_deploy name: "Deploy Airflow DAGs to S3 bucket" image: python:3.8 oidc: true script: - python -m pip install --upgrade pip - apt-get update && apt-get install -y jq git - pip install awscli - export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO:role/$OIDC_ROLE - export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token - echo $BITBUCKET_STEP_OIDC_TOKEN > $(pwd)/web-identity-token - export STS_OUTPUT=$(aws sts assume-role-with-web-identity --role-arn $AWS_ROLE_ARN --role-session-name BitbucketPipeline --web-identity-token "$BITBUCKET_STEP_OIDC_TOKEN" --duration-seconds 3600) - export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId') - export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey') - export AWS_SESSION_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken') - aws sts get-caller-identity - aws s3 sync ./airflow/dags s3://doczy-dev-infra-mwaa-resources/dags trigger: automatic # uncomment after testing pipelines: branches: DEV: - step: <<: *terraform_plan_apply condition: changesets: includePaths: - terraform/environments/dev/**/* - step: <<: *snowflake_deploy condition: changesets: includePaths: - snowflake/**/* - step: <<: *streamlit_deploy condition: changesets: includePaths: - "streamlit/*" - "streamlit/**/*" - step: <<: *airflow_dags_deploy condition: changesets: includePaths: - airflow/dags/* # UAT and PROD pipelines will be updated with Streamlit steps once it is tested and ready UAT: - step: <<: *terraform_plan_apply condition: paths: - terraform/uat/**/* - step: <<: *snowflake_deploy condition: paths: - snowflake/**/*.sql PROD: - step: <<: *terraform_plan_apply condition: paths: - terraform/prod//**/* - step: <<: *snowflake_deploy condition: paths: - snowflake/**/*.sql