doczyai-pipelines/bitbucket-pipelines.yml

definitions:
  steps:
    - step: &terraform_plan_apply
        name: "Plan and Apply Terraform"
        image: zenika/terraform-aws-cli:latest
        script:
          - terraform --version
          # terraform steps here
        trigger: automatic # uncomment after testing
        # trigger: manual
    - step: &snowflake_deploy
        name: "Deploy Snowflake Changes"
        image: python:3.7
        size: 2x
        script:
          - python -m pip install --upgrade pip
          - pip install snowflake-connector-python
          - pip install schemachange --upgrade
          # Setting the database name dynamically based on the branch name
          - if [ "$BITBUCKET_BRANCH" == "DEV" ]; then export SNOWFLAKE_DATABASE=$DEV_SNOWFLAKE_DATABASE; export SNOWFLAKE_ROLE=$DEV_SNOWFLAKE_ROLE; fi
          - if [ "$BITBUCKET_BRANCH" == "UAT" ]; then export SNOWFLAKE_DATABASE=$UAT_SNOWFLAKE_DATABASE; export SNOWFLAKE_ROLE=$UAT_SNOWFLAKE_ROLE; fi
          - if [ "$BITBUCKET_BRANCH" == "PROD" ]; then export SNOWFLAKE_DATABASE=$PROD_SNOWFLAKE_DATABASE; export SNOWFLAKE_ROLE=$PROD_SNOWFLAKE_ROLE; fi
          - schemachange -a $SNOWFLAKE_ACCOUNT -u $SNOWFLAKE_USER -r $SNOWFLAKE_ROLE -w $SNOWFLAKE_WAREHOUSE -d $SNOWFLAKE_DATABASE -c $SNOWFLAKE_DATABASE.SCHEMACHANGE.CHANGE_HISTORY --create-change-history-table -v
        trigger: automatic
        # trigger: manual
        caches:
          - pip
    - step: &streamlit_deploy
        name: "Deploy streamlit to EC2"
        oidc: true
        image: python:3.8
        script:
          - python -m pip install --upgrade pip 
          - apt-get update && apt-get install -y jq git
          - pip install awscli
          - export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO:role/$OIDC_ROLE
          - export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token 
          - echo $BITBUCKET_STEP_OIDC_TOKEN > $(pwd)/web-identity-token
          - export STS_OUTPUT=$(aws sts assume-role-with-web-identity --role-arn $AWS_ROLE_ARN --role-session-name BitbucketPipeline --web-identity-token "$BITBUCKET_STEP_OIDC_TOKEN"  --duration-seconds 3600)
          - export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId')
          - export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey')
          - export AWS_SESSION_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken')
          - aws sts get-caller-identity
          - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $DEV_INSTANCE_ID --region $AWS_DEFAULT_REGION --parameters commands='["cd /home/ubuntu/streamlit","(if [ -d doczy.ai ]; then cd doczy.ai && git fetch && git pull; else git clone git@bitbucket.org:aarete/doczy.ai.git; fi)"]'
        trigger: automatic # uncomment after testing
    - step: &airflow_dags_deploy
        name: "Deploy Airflow DAGs to S3 bucket"
        image: python:3.8
        oidc: true
        script:
          - python -m pip install --upgrade pip 
          - apt-get update && apt-get install -y jq git
          - pip install awscli
          - export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO:role/$OIDC_ROLE
          - export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token 
          - echo $BITBUCKET_STEP_OIDC_TOKEN > $(pwd)/web-identity-token
          - export STS_OUTPUT=$(aws sts assume-role-with-web-identity --role-arn $AWS_ROLE_ARN --role-session-name BitbucketPipeline --web-identity-token "$BITBUCKET_STEP_OIDC_TOKEN"  --duration-seconds 3600)
          - export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId')
          - export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey')
          - export AWS_SESSION_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken')
          - aws sts get-caller-identity
          - aws s3 sync ./airflow/dags s3://doczy-dev-infra-mwaa-resources/dags         
        trigger: automatic # uncomment after testing

pipelines:
  branches:
    DEV: 
      - step:
          <<: *terraform_plan_apply
          condition:
            changesets:
              includePaths:
                - terraform/environments/dev/**/*
      - step:
          <<: *snowflake_deploy
          condition:
            changesets:
              includePaths:
              - snowflake/**/*
      - step:
          <<: *streamlit_deploy
          condition:
            changesets:
              includePaths:
              - streamlit/**/*
      - step:
          <<: *airflow_dags_deploy
          condition:
            changesets:
              includePaths:
              - airflow/dags/*
      - step:
          <<: *streamlit_deploy
          condition:
            paths:
              - streamlit/**/*
    # UAT and PROD pipelines will be updated with Streamlit steps once it is tested and ready
    UAT:
      - step:
          <<: *terraform_plan_apply
          condition:
            paths:
              - terraform/uat/**/*
      - step:
          <<: *snowflake_deploy
          condition:
            paths:
              - snowflake/**/*.sql
    PROD:
      - step:
          <<: *terraform_plan_apply
          condition:
            paths:
              - terraform/prod//**/*
      - step:
          <<: *snowflake_deploy
          condition:
            paths:
              - snowflake/**/*.sql