doczyai-pipelines/bitbucket-pipelines.yml

definitions:
  scripts:
    - script: &aws-context
        export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO:role/$AWS_OIDC_ROLE;
        export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token;
        echo $BITBUCKET_STEP_OIDC_TOKEN > $(pwd)/web-identity-token;
        export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId');
        export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey');
        export AWS_SECURITY_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken');
        export AWS_REGION='us-east-2';
  stage: &deployment
    name: deployment
    deployment: development
    trigger: manual
    environment:
      SNOWFLAKE_DATABASE: $DEV_SNOWFLAKE_DATABASE
      SNOWFLAKE_ROLE: $DEV_SNOWFLAKE_ROLE
      AWS_IODC_ROLE: $IODC_ROLE
      INSTANCE_ID_TF: $DEV_INSTANCE_ID_TF
      IS_DEPLOY_ALL: true
      BB_PIPELINE_BRANCH: $BITBUCKET_BRANCH
      ENVIRONMENT: dev
    steps:
      - step:
          name: "Deploy Snowflake Changes"
          image: python:3.7
          size: 2x
          script:
            - python -m pip install --upgrade pip
            - pip install snowflake-connector-python
            - pip install schemachange --upgrade
            - schemachange -a $SNOWFLAKE_ACCOUNT -u $SNOWFLAKE_USER -r $SNOWFLAKE_ROLE -w $SNOWFLAKE_WAREHOUSE -d $SNOWFLAKE_DATABASE -c $SNOWFLAKE_DATABASE.SCHEMACHANGE.CHANGE_HISTORY --create-change-history-table -v
          caches:
           - pip
          condition:
           changesets:
              includePaths:
              - snowflake/**/*
              - snowflake/**/**/*
              - snowflake/**/**/**/*
      - step:
          image: python:3.8
          name: "Deploy streamlit to EC2"
          oidc: true
          script:
            - python -m pip install --upgrade pip
            - apt-get update && apt-get install -y jq git
            - pip install awscli
            - *aws-context
            - >-
              aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID_TF --region $AWS_DEFAULT_REGION --parameters commands='[
                "sudo -u ubuntu -i <<EOF",
                "export HOME=/home/ubuntu",
                "git config --global --add safe.directory /home/ubuntu/doczy.ai",
                "if [ -d /home/ubuntu/doczy.ai ]; then cd /home/ubuntu/doczy.ai && git fetch && git pull; else cd /home/ubuntu && git clone git@bitbucket.org:aarete/doczy.ai.git; fi",
                "EOF"
              ]'
          condition:
            changesets:
              includePaths:
              - streamlit/*
              - streamlit/**/*
      - step:
          name: "Deploy Airflow DAGs to S3 bucket"
          image: python:3.8
          oidc: true
          script:
            - python -m pip install --upgrade pip
            - apt-get update && apt-get install -y jq git
            - pip install awscli
            - *aws-context
            - aws sts get-caller-identity
            - aws s3 sync ./airflow/dags s3://doczy-dev-infra-mwaa-resources/dags
          condition:
            changesets:
              includePaths:
              - airflow/dags/*
      - step:
        image: hugree/terraform-with-snowsql:latest
        name: Run Snowsql
        oidc: true
        script:
          - *aws-context
          - pip install -r requirements.txt
          - IS_DEPLOY_ALL=true python terraform.py apply --environment=$ENVIRONMENT --module=textract-pipeline/terraform --apply-extra-args="-target=null_resource.snowsql_client_create"
      - step:
          name: Apply textract-pipeline
          image: hugree/terraform-with-snowsql:latest
          oidc: true
          script:
            - *aws-context
            - pip install -r requirements.txt
            - python terraform.py apply --environment=$ENVIRONMENT --module=textract-pipeline/terraform
      - step:
          name: Apply devops-pipeline
          image: hugree/bitbucket-aws-python38-tf154:latest
          oidc: true
          script:
            - *aws-context
            - pip install -r requirements.txt
            - python terraform.py apply --environment=$ENVIRONMENT --module=devops-pipeline/other-resources
      - step:
          name: Apply streamlit-server
          image: hugree/bitbucket-aws-python38-tf154:latest
          oidc: true
          script:
            - *aws-context
            - pip install -r requirements.txt
            - python terraform.py apply --environment=$ENVIRONMENT --module=streamlit-server
  stage: &example
    name: example
    deployment: test
    steps:
      - step:
          name: "Example"
          script:
            - echo $BITBUCKET_DEPLOYMENT_ENVIRONMENT
            - echo $CUSTOM_ENV HEELOO

pipelines:
  default:
    - stage: *example
  branches:
    main:
      - step: Build and Test
        script: echo Build and test pipeline will go here
      - stage: *deployment
        name: deployment-dev
        deployment: development
        #trigger: automatic
        environment:
          SNOWFLAKE_DATABASE: $DEV_SNOWFLAKE_DATABASE
          SNOWFLAKE_ROLE: $DEV_SNOWFLAKE_ROLE
          AWS_IODC_ROLE: $IODC_ROLE
          INSTANCE_ID_TF: $DEV_INSTANCE_ID_TF
          IS_DEPLOY_ALL: true
          ENVIRONMENT: dev
      - stage: *deployment
        name: deployment-uat
        deployment: staging
        #trigger: automatic
        environment:
          SNOWFLAKE_DATABASE: $UAT_SNOWFLAKE_DATABASE
          SNOWFLAKE_ROLE: $UAT_SNOWFLAKE_ROLE
          AWS_IODC_ROLE: $IODC_ROLE_UAT
          INSTANCE_ID_TF: $UAT_INSTANCE_ID_TF
          IS_DEPLOY_ALL: true
          ENVIRONMENT: uat
      - include: cicd/deployment.yml
        name: deployment-prod
        deployment: production
        environment:
          SNOWFLAKE_DATABASE: $PROD_SNOWFLAKE_DATABASE
          SNOWFLAKE_ROLE: $PROD_SNOWFLAKE_ROLE
          AWS_IODC_ROLE: $IODC_ROLE_PROD
          INSTANCE_ID_TF: $PROD_INSTANCE_ID_TF
          IS_DEPLOY_ALL: true
          ENVIRONMENT: prod