doczyai-pipelines/bitbucket-pipelines.yml

definitions:
  scripts:
    - script: &aws-context-dev
        export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO:role/$OIDC_ROLE;
        export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token;
        echo $BITBUCKET_STEP_OIDC_TOKEN > $(pwd)/web-identity-token;
        export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId');
        export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey');
        export AWS_SECURITY_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken');
        export AWS_REGION='us-east-2';
    - script: &aws-context-uat
        export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO_UAT:role/$OIDC_ROLE_UAT;
        export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token;
        echo $BITBUCKET_STEP_OIDC_TOKEN > $(pwd)/web-identity-token;
        export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId');
        export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey');
        export AWS_SECURITY_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken');
        export AWS_REGION='us-east-2';
    - script: &ssm-send-command-dev
        aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $DEV_INSTANCE_ID_TF --region $AWS_DEFAULT_REGION --parameters commands='[
          "sudo -u ubuntu -i <<EOF",
          "export HOME=/home/ubuntu",
          "git config --global --add safe.directory /home/ubuntu/doczy.ai",
          "if [ -d /home/ubuntu/doczy.ai ]; then cd /home/ubuntu/doczy.ai && git fetch && git pull; else cd /home/ubuntu && git clone git@bitbucket.org:aarete/doczy.ai.git; fi",
          "EOF"
          ]'
    - script: &ssm-send-command-uat
        aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $UAT_INSTANCE_ID_TF --region $AWS_DEFAULT_REGION --parameters commands='[
          "sudo -u ubuntu -i <<EOF",
          "export HOME=/home/ubuntu",
          "git config --global --add safe.directory /home/ubuntu/doczy.ai",
          "if [ -d /home/ubuntu/doczy.ai ]; then cd /home/ubuntu/doczy.ai && git fetch && git pull; else cd /home/ubuntu && git clone git@bitbucket.org:aarete/doczy.ai.git; fi",
          "EOF"
          ]'

    - step: &snowflake_deploy
        name: "Deploy Snowflake Changes"
        image: python:3.7
        size: 2x
        script:
          - python -m pip install --upgrade pip
          - pip install snowflake-connector-python
          - pip install schemachange --upgrade
          # Setting the database name dynamically based on the branch name
          - if [ "$BITBUCKET_BRANCH" == "DEV" ]; then export SNOWFLAKE_DATABASE=$DEV_SNOWFLAKE_DATABASE; export SNOWFLAKE_ROLE=$DEV_SNOWFLAKE_ROLE; fi
          - if [ "$BITBUCKET_BRANCH" == "UAT" ]; then export SNOWFLAKE_DATABASE=$UAT_SNOWFLAKE_DATABASE; export SNOWFLAKE_ROLE=$UAT_SNOWFLAKE_ROLE; fi
          - if [ "$BITBUCKET_BRANCH" == "PROD" ]; then export SNOWFLAKE_DATABASE=$PROD_SNOWFLAKE_DATABASE; export SNOWFLAKE_ROLE=$PROD_SNOWFLAKE_ROLE; fi
          - schemachange -a $SNOWFLAKE_ACCOUNT -u $SNOWFLAKE_USER -r $SNOWFLAKE_ROLE -w $SNOWFLAKE_WAREHOUSE -d $SNOWFLAKE_DATABASE -c $SNOWFLAKE_DATABASE.SCHEMACHANGE.CHANGE_HISTORY --create-change-history-table -v
        trigger: automatic
        # trigger: manual
        caches:
          - pip
    - step: &terraform_plan_apply
        name: "Plan and Apply Terraform"
        image: python:3.8
        script:
          - python -m pip install --upgrade pip
          - apt-get update && apt-get install -y jq git
          - pip install awscli
          - pip install terraform
          - export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO:role/$OIDC_ROLE
          - export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token
          - echo $BITBUCKET_STEP_OIDC_TOKEN > $(pwd)/web-identity-token
          - export STS_OUTPUT=$(aws sts assume-role-with-web-identity --role-arn $AWS_ROLE_ARN --role-session-name BitbucketPipeline --web-identity-token "$BITBUCKET_STEP_OIDC_TOKEN"  --duration-seconds 3600)
          - export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId')
          - export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey')
          - export AWS_SESSION_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken')
          - aws sts get-caller-identity
          - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="region=us-east-2" -backend-config="dynamodb_table=doczyai-use2-d-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-d-infra-s3-terraform-state" -backend-config="key=devops-pipeline/terraform.tfstate"
          - terraform validate
          - BRANCH_NAME_LOWER=$(echo $BITBUCKET_BRANCH | tr '[:upper:]' '[:lower:]')
          - terraform apply -auto-approve -no-color -var 'access_key=$AWS_ACCESS_KEY_ID' -var 'secret_key=$AWS_SECRET_ACCESS_KEY' -var 'aws_region=us-east-2' -var 'environment=$BRANCH_NAME_LOWER'
        trigger: automatic # uncomment after testing
        # trigger: manual
    - step: &streamlit_deploy
        name: "Deploy streamlit to EC2"
        oidc: true
        image: python:3.8
        script:
          - python -m pip install --upgrade pip
          - apt-get update && apt-get install -y jq git
          - pip install awscli
          - export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO:role/$OIDC_ROLE
          - export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token
          - echo $BITBUCKET_STEP_OIDC_TOKEN > $(pwd)/web-identity-token
          - export STS_OUTPUT=$(aws sts assume-role-with-web-identity --role-arn $AWS_ROLE_ARN --role-session-name BitbucketPipeline --web-identity-token "$BITBUCKET_STEP_OIDC_TOKEN"  --duration-seconds 3600)
          - export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId')
          - export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey')
          - export AWS_SESSION_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken')
          - aws sts get-caller-identity
          - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $DEV_INSTANCE_ID --region $AWS_DEFAULT_REGION --parameters commands='["cd /home/ubuntu/streamlit","su -l ubuntu","(if [ -d doczy.ai ]; then cd doczy.ai && git fetch && git pull; else git clone git@bitbucket.org:aarete/doczy.ai.git; fi)"]'
        trigger: automatic # uncomment after testing
    - step: &airflow_dags_deploy
        name: "Deploy Airflow DAGs to S3 bucket"
        image: python:3.8
        oidc: true
        script:
          - python -m pip install --upgrade pip
          - apt-get update && apt-get install -y jq git
          - pip install awscli
          - export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO:role/$OIDC_ROLE
          - export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token
          - echo $BITBUCKET_STEP_OIDC_TOKEN > $(pwd)/web-identity-token
          - export STS_OUTPUT=$(aws sts assume-role-with-web-identity --role-arn $AWS_ROLE_ARN --role-session-name BitbucketPipeline --web-identity-token "$BITBUCKET_STEP_OIDC_TOKEN"  --duration-seconds 3600)
          - export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId')
          - export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey')
          - export AWS_SESSION_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken')
          - aws sts get-caller-identity
          - aws s3 sync ./airflow/dags s3://doczy-dev-infra-mwaa-resources/dags
        trigger: automatic # uncomment after testing

pipelines:
  branches:
    DEV:
      - step:
          <<: *snowflake_deploy
          condition:
            changesets:
              includePaths:
              - snowflake/DEV/*
              - snowflake/DEV/**/*
              - snowflake/DEV/**/**/*
      - step:
          image: python:3.8
          name: "Deploy streamlit to EC2"
          oidc: true
          script:
            - python -m pip install --upgrade pip
            - apt-get update && apt-get install -y jq git
            - pip install awscli
            - *aws-context-dev
            - *ssm-send-command-dev
          condition:
            changesets:
              includePaths:
              - streamlit/*
              - streamlit/**/*
      - step:
          <<: *airflow_dags_deploy
          condition:
            changesets:
              includePaths:
              - airflow/dags/*
      - step:
          name: "Plan & Apply Textract Pipeline"
          image: hugree/bitbucket-aws-python38-tf154:latest
          oidc: true
          trigger : manual
          script:
            - *aws-context-dev
            - cd textract-pipeline/terraform/
            # Ensure the backend configuration for s3 obj key is corresponding to the stream name 
            - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-d-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-d-infra-s3-terraform-state" -backend-config="key=terraform/textract-pipeline/terraform.tfstate"
            - terraform validate
            - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=dev"
          condition:
            changesets:
              includePaths:
              - textract-pipeline/terraform/*
              - textract-pipeline/terraform/**/*
      - step:
          name: "Plan & Apply DevOps Pipeline"
          image: hugree/bitbucket-aws-python38-tf154:latest
          oidc: true
          trigger : manual # Keeping it manual so that it doesnt conflict with existing resources
          script:
            - *aws-context-dev
            - cd devops-pipeline/terraform-backend-resources/
            # Ensure the backend configuration for s3 obj key is correct
            - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-d-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-d-infra-s3-terraform-state" -backend-config="key=terraform/devops-pipeline/terraform.tfstate"
            - terraform validate
            - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=dev"
          condition:
            changesets:
              includePaths:
              - devops-pipeline/terraform-backend-resources/*
              - devops-pipeline/terraform-backend-resources/**/*
      - step:
          name: "Plan & Apply Streamlit Pipeline"
          image: hugree/bitbucket-aws-python38-tf154:latest
          oidc: true
          trigger : automatic
          script:
            - *aws-context-dev
            - cd streamlit-server/
            # Ensure the backend configuration for s3 obj key is correct
            - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-d-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-d-infra-s3-terraform-state" -backend-config="key=terraform/streamlit-pipeline/terraform.tfstate" -backend-config="region=us-east-2"
            - terraform validate
            - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=dev" -var "vpc_id=vpc-0f9d7c913f6522079"
          condition:
            changesets:
              includePaths:
              - streamlit-server/*
              - streamlit-server/**/*


    # UAT and PROD pipelines will be updated with Streamlit steps once it is tested and ready
    UAT:
      - step:
          name: "Plan & Apply DevOps Pipeline"
          image: hugree/bitbucket-aws-python38-tf154:latest
          oidc: true
          trigger : automatic
          script:
            - *aws-context-uat
            - cd devops-pipeline/terraform-backend-resources/
            - terraform init
            - terraform validate
            # Ensure the backend configuration for s3 obj key is correct
            -  terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat"- echo "S3_BUCKET=$(terraform output -raw terraform_s3_bucket_name)" >> backend.env
            - echo "DYNAMODB_TABLE=$(terraform output -raw terraform_dynamodb_table_name)" >> backend.env
          artifacts:
            - backend.env
          condition:
            changesets:
              includePaths:
              - devops-pipeline/terraform-backend-resources/*
              - devops-pipeline/terraform-backend-resources/**/*- step:
          name: "Plan & Apply DevOps Extra Resource Pipeline"
          image: hugree/bitbucket-aws-python38-tf154:latest
          oidc: true
          trigger : automatic # Keeping it manual so that it doesnt conflict with existing resources
          script:
            - *aws-context-uat
            - cd devops-pipeline/other-resources/
            - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/devops-pipeline-other-resources/terraform.tfstate"
            - terraform validate
            - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat"
          condition:
            changesets:
              includePaths:
              - devops-pipeline/other-resources/*
              - devops-pipeline/other-resources/**/*
      - step:
          name: "Plan & Apply Streamlit Pipeline"
          image: hugree/bitbucket-aws-python38-tf154:latest
          oidc: true
          trigger : automatic
          script:
            - *aws-context-uat
            - cd streamlit-server/
            - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/streamlit-server/terraform.tfstate"
            - terraform validate
            # Ensure the backend configuration for s3 obj key is correct
            -  terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat"-var "vpc_id=vpc-0392396d0e7bdd77f"
          condition:
            changesets:
              includePaths:
              - streamlit-server/*
              - streamlit-server/**/*
      - step:
          <<: *snowflake_deploy
          condition:
            changesets:
              includePaths:
              - snowflake/UAT/*
              - snowflake/UAT/**/*
              - snowflake/UAT/**/**/*
      - step:
          image: python:3.8
          name: "Deploy streamlit to EC2"
          oidc: true
          trigger: automatic # Keeping it manual as the server will be created first and then referenced in this
          script:
            - python -m pip install --upgrade pip
            - apt-get update && apt-get install -y jq git
            - pip install awscli
            - *aws-context-uat
            - *ssm-send-command-uat
          condition:
            changesets:
              includePaths:
              - streamlit/*
              - streamlit/**/*
      - step:
          <<: *airflow_dags_deploy
          trigger: manual
          condition:
            changesets:
              includePaths:
              - airflow/dags/*
      - step:
          name: "Plan & Apply Textract Pipeline"
          image: hugree/bitbucket-aws-python38-tf154:latest
          oidc: true
          trigger : manual
          script:
            - *aws-context-uat
            - cd textract-pipeline/terraform/
            - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/textract-pipeline/terraform.tfstate"
            - terraform validate
            # Ensure the backend configuration for s3 obj key is corresponding to the stream name 
            -  terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat"
          condition:
            changesets:
              includePaths:
              - textract-pipeline/terraform/*
              - textract-pipeline/terraform/**/*


    PROD:
      - step:
          <<: *terraform_plan_apply
          condition:
            paths:
              - terraform/prod//**/*
      - step:
          <<: *snowflake_deploy
          condition:
            paths:
              - snowflake/**/*.sql


    feature/terraform-refactor:
      - parallel:
          - step:
              name: Plan devops-pipeline on DEV
              image: hugree/bitbucket-aws-python38-tf154:latest
              oidc: true
              script:
                - *aws-context-uat
                - python terraform.py plan --environment=dev --module=devops-pipeline/other-resources
          - step:
              name: Plan streamlit-server on DEV
              image: hugree/bitbucket-aws-python38-tf154:latest
              oidc: true
              script:
                - *aws-context-uat
                - python terraform.py plan --environment=dev --module=streamlit-server
          - step:
              name: Plan textract-pipeline on DEV
              image: hugree/bitbucket-aws-python38-tf154:latest
              oidc: true
              script:
                - *aws-context-uat
                - python terraform.py plan --environment=dev --module=textract-pipeline/terraform
          - step:
              name: Plan devops-pipeline on UAT
              image: hugree/bitbucket-aws-python38-tf154:latest
              oidc: true
              script:
                - *aws-context-uat
                - python terraform.py plan --environment=uat --module=devops-pipeline/other-resources
          - step:
              name: Plan streamlit-server on UAT
              image: hugree/bitbucket-aws-python38-tf154:latest
              oidc: true
              script:
                - *aws-context-uat
                - python terraform.py plan --environment=uat --module=streamlit-server
          - step:
              name: Plan textract-pipeline on UAT
              image: hugree/bitbucket-aws-python38-tf154:latest
              oidc: true
              script:
                - *aws-context-uat
                - python terraform.py plan --environment=uat --module=textract-pipeline/terraform