doczyai-pipelines/bitbucket-pipelines.yml

definitions:
  scripts:
    - script: &prep-dir
        DIR=${DIR:=.};
        cd $DIR;
        pip install poetry;
        poetry install;
    - script: &lint
        poetry run black .
    - script: &validate-static
        poetry run mypy .
    - script: &unit-test
        poetry run pytest
    - script: &aws-context
        apt-get update;
        apt-get install -y jq;
        pip install awscli;
        export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO:role/$OIDC_ROLE;
        export AWS_ROLE_SESSION_NAME=$OIDC_ROLE;
        export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token;
        echo $BITBUCKET_STEP_OIDC_TOKEN > $(pwd)/web-identity-token;
        export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId');
        export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey');
        export AWS_SECURITY_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken');
        export AWS_REGION='us-east-2';
        aws sts get-caller-identity;
        export IS_DEPLOY_ALL=true; 
  steps: &deployment-plan-steps
    - step:
        name: Plan devops-pipeline
        image: hugree/bitbucket-aws-python38-tf154:latest
        oidc: true
        script:
          - *prep-dir
          - *aws-context
          - python terraform.py plan --environment=$ENVIRONMENT --module=devops-pipeline/other-resources
        caches:
          - pip
    - step:
        name: Plan textract-pipeline
        image: hugree/terraform-with-snowsql:latest
        oidc: true
        script:
          - *prep-dir
          - *aws-context
          - python terraform.py plan --environment=$ENVIRONMENT --module=textract-pipeline/terraform
        caches:
          - pip
    - step:
        name: Plan streamlit-server
        image: hugree/bitbucket-aws-python38-tf154:latest
        oidc: true
        script:
          - *prep-dir
          - *aws-context
          - python terraform.py plan --environment=$ENVIRONMENT --module=streamlit-server
        caches:
          - pip
    - step:
        name: Plan Snowflake Changes
        image: python:3.7
        size: 2x
        script:
          - pip install snowflake-connector-python
          - pip install schemachange --upgrade
          - |
            schemachange -a $SNOWFLAKE_ACCOUNT -u $SNOWFLAKE_USER \
            -f snowflake/ \
            -r $SNOWFLAKE_ROLE -w $SNOWFLAKE_WAREHOUSE \
            -d $SNOWFLAKE_DATABASE -c $SNOWFLAKE_DATABASE.SCHEMACHANGE.CHANGE_HISTORY \
            --create-change-history-table -v \
            --dry-run
        caches:
          - pip
    - step:
        image: hugree/terraform-with-snowsql:latest
        name: Plan Snowsql
        oidc: true
        script:
          - *prep-dir
          - *aws-context
          - python terraform.py plan --environment=$ENVIRONMENT --module=textract-pipeline/terraform --apply-extra-args="-target=null_resource.snowsql_client_create"
        caches:
          - pip
    - step:
        image: python:3.8
        name: Plan streamlit to EC2
        oidc: true
        script:
          - *aws-context
          - apt-get install -y git
          - >-
            aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID_TF --region $AWS_DEFAULT_REGION --parameters commands='[
              "sudo -u ubuntu -i <<EOF",
              "export HOME=/home/ubuntu",
              "git config --global --add safe.directory /home/ubuntu/doczy.ai",
              "if [ -d /home/ubuntu/doczy.ai ]; then cd /home/ubuntu/doczy.ai && git fetch && git status; else cd /home/ubuntu && echo \"Doczy.AI does not exist\"; fi",
              "EOF"
            ]'
        caches:
          - pip
  steps: &deployment-steps
    - step:
        name: Apply devops-pipeline
        image: hugree/bitbucket-aws-python38-tf154:latest
        oidc: true
        script:
          - *prep-dir
          - *aws-context
          - python terraform.py apply --environment=$ENVIRONMENT --module=devops-pipeline/other-resources
        caches:
          - pip
    - step:
        name: Apply textract-pipeline
        image: hugree/terraform-with-snowsql:latest
        oidc: true
        script:
          - *prep-dir
          - *aws-context
          - python terraform.py apply --environment=$ENVIRONMENT --module=textract-pipeline/terraform
        caches:
          - pip
    - step:
        name: Apply streamlit-server
        image: hugree/bitbucket-aws-python38-tf154:latest
        oidc: true
        script:
          - *prep-dir
          - *aws-context
          - python terraform.py apply --environment=$ENVIRONMENT --module=streamlit-server
        caches:
          - pip
    - step:
        name: Apply Snowflake Changes
        image: python:3.7
        size: 2x
        script:
          - pip install snowflake-connector-python
          - pip install schemachange --upgrade
          - |
            schemachange -a $SNOWFLAKE_ACCOUNT -u $SNOWFLAKE_USER \
            -f snowflake/ \
            -r $SNOWFLAKE_ROLE -w $SNOWFLAKE_WAREHOUSE \
            -d $SNOWFLAKE_DATABASE -c $SNOWFLAKE_DATABASE.SCHEMACHANGE.CHANGE_HISTORY \
            --create-change-history-table -v
        caches:
          - pip
    - step:
        image: hugree/terraform-with-snowsql:latest
        name: Apply Snowsql
        oidc: true
        script:
          - *prep-dir
          - *aws-context
          - python terraform.py apply --environment=$ENVIRONMENT --module=textract-pipeline/terraform --apply-extra-args="-target=null_resource.snowsql_client_create"
        caches:
          - pip
    - step:
        image: python:3.8
        name: Apply streamlit to EC2
        oidc: true
        script:
          - *aws-context
          - apt-get install -y git
          - >-
            aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID_TF --region $AWS_DEFAULT_REGION --parameters commands='[
              "sudo -u ubuntu -i <<EOF",
              "export HOME=/home/ubuntu",
              "git config --global --add safe.directory /home/ubuntu/doczy.ai",
              "if [ -d /home/ubuntu/doczy.ai ]; then cd /home/ubuntu/doczy.ai && git fetch && git pull; else cd /home/ubuntu && git clone git@bitbucket.org:aarete/doczy.ai.git; fi",
              "EOF"
            ]'
        caches:
          - pip

pipelines:
  default:
    - step:
        image: python:3.12
        name: Install Pip
        script:
          - python -m pip install --upgrade pip
        caches:
          - pip
    - parallel:
        - step:
            name: Lint check tiff-to-pdf
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/tiff-to-pdf
              - *prep-dir
              - *lint
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/tiff-to-pdf/**
        - step:
            name: Lint check batch-creation
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/batch-creation
              - *prep-dir
              - *lint
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/batch-creation/**
        - step:
            name: Lint check call-bedrock
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/call-bedrock
              - *prep-dir
              - *lint
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/call-bedrock/**
        - step:
            name: Lint check database-interface
            image: python:3.11.10
            script:
              - DIR=textract-pipeline/src/lambda/database-interface
              - *prep-dir
              - *lint
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/database-interface/**
        - step:
            name: Lint check database-interface-get
            image: python:3.11.10
            script:
              - DIR=textract-pipeline/src/lambda/database-interface-get
              - *prep-dir
              - *lint
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/database-interface-get/**
        - step:
            name: Lint check docx-to-pdf
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/docx-to-pdf
              - *prep-dir
              - *lint
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/docx-to-pdf/**
        - step:
            name: Lint check insert-record
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/insert-record
              - *prep-dir
              - *lint
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/insert-record/**
        - step:
            name: Lint check pdf-validation
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/pdf-validation
              - *prep-dir
              - *lint
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/pdf-validation/**
        - step:
            name: Lint check prompt-orchestrator
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/prompt-processor
              - *prep-dir
              - *lint
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/prompt-processor/**
        - step:
            name: Lint check s3-folder-details
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/s3-folder-details
              - *prep-dir
              - *lint
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/s3-folder-details/**
        - step:
            name: Lint check text-creation
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/text-creation
              - *prep-dir
              - *lint
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/text-creation/**
        - step:
            name: Lint check textract-receiver
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/textract-receiver
              - *prep-dir
              - *lint
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/textract-receiver/**
        - step:
            name: Lint check textract-sender
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/textract-sender
              - *prep-dir
              - *lint
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/textract-sender/**
        - step:
            name: Lint check trigger-pipeline
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/trigger-pipeline
              - *prep-dir
              - *lint
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/trigger-pipeline/**
        - step:
            name: Lint check streamlit
            image: python:3.11.10
            size: 4x
            script:
              - DIR=streamlit
              - *prep-dir
              - *lint
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - streamlit/**
        - step:
            name: Lint check streamlit_multipage
            image: python:3.12.7
            script:
              - DIR=streamlit_multipage
              - *prep-dir
              - *lint
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - streamlit_multipage/**
        - step:
            name: Lint check fieldExtraction
            image: python:3.12.7
            script:
              - DIR=fieldExtraction
              - *prep-dir
              - *lint
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - fieldExtraction/**
        - step:
            name: Lint check Doczy.AI_Automation
            image: python:3.11.10
            script:
              - DIR=Doczy.AI_Automation
              - *prep-dir
              - *lint
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - Doczy.AI_Automation/**
        - step:
            name: Lint check airflow
            image: python:3.12.7
            script:
              - DIR=airflow
              - *prep-dir
              - *lint
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - airflow/**
    - parallel:
        - step:
            name: Static check tiff-to-pdf
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/tiff-to-pdf
              - *prep-dir
              - *validate-static
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/tiff-to-pdf/**
        - step:
            name: Static check batch-creation
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/batch-creation
              - *prep-dir
              - *validate-static
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/batch-creation/**
        - step:
            name: Static check call-bedrock
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/call-bedrock
              - *prep-dir
              - *validate-static
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/call-bedrock/**
        - step:
            name: Static check database-interface
            image: python:3.11.10
            script:
              - DIR=textract-pipeline/src/lambda/database-interface
              - *prep-dir
              - *validate-static
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/database-interface/**
        - step:
            name: Static check database-interface-get
            image: python:3.11.10
            script:
              - DIR=textract-pipeline/src/lambda/database-interface-get
              - *prep-dir
              - *validate-static
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/database-interface-get/**
        - step:
            name: Static check docx-to-pdf
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/docx-to-pdf
              - *prep-dir
              - *validate-static
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/docx-to-pdf/**
        - step:
            name: Static check insert-record
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/insert-record
              - *prep-dir
              - *validate-static
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/insert-record/**
        - step:
            name: Static check pdf-validation
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/pdf-validation
              - *prep-dir
              - *validate-static
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/pdf-validation/**
        - step:
            name: Static check prompt-orchestrator
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/prompt-processor
              - *prep-dir
              - *validate-static
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/prompt-processor/**
        - step:
            name: Static check s3-folder-details
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/s3-folder-details
              - *prep-dir
              - *validate-static
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/s3-folder-details/**
        - step:
            name: Static check text-creation
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/text-creation
              - *prep-dir
              - *validate-static
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/text-creation/**
        - step:
            name: Static check textract-receiver
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/textract-receiver
              - *prep-dir
              - *validate-static
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/textract-receiver/**
        - step:
            name: Static check textract-sender
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/textract-sender
              - *prep-dir
              - *validate-static
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/textract-sender/**
        - step:
            name: Static check trigger-pipeline
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/trigger-pipeline
              - *prep-dir
              - *validate-static
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/trigger-pipeline/**
        - step:
            name: Static check streamlit
            image: python:3.11.10
            size: 4x
            script:
              - DIR=streamlit
              - echo Must be in usage
              - exit 0
              - *prep-dir
              - *validate-static
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - streamlit/**
        - step:
            name: Static check streamlit_multipage
            image: python:3.12.7
            script:
              - DIR=streamlit_multipage
              - *prep-dir
              - *validate-static
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - streamlit_multipage/**
        - step:
            name: Static check fieldExtraction
            image: python:3.12.7
            script:
              - DIR=fieldExtraction
              - *prep-dir
              - *validate-static
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - fieldExtraction/**
        - step:
            name: Static check Doczy.AI_Automation
            image: python:3.11.10
            script:
              - DIR=Doczy.AI_Automation
              - echo Must be in usage
              - exit 0
              - *prep-dir
              - *validate-static
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - Doczy.AI_Automation/**
        - step:
            name: Static check airflow
            image: python:3.12.7
            script:
              - DIR=airflow
              - *prep-dir
              - *validate-static
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - airflow/**
    - parallel:
        - step:
            name: Unit Test check tiff-to-pdf
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/tiff-to-pdf
              - *prep-dir
              - *unit-test
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/tiff-to-pdf/**
        - step:
            name: Unit Test check batch-creation
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/batch-creation
              - *prep-dir
              - *unit-test
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/batch-creation/**
        - step:
            name: Unit Test check call-bedrock
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/call-bedrock
              - *prep-dir
              - *unit-test
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/call-bedrock/**
        - step:
            name: Unit Test check database-interface
            image: python:3.11.10
            script:
              - DIR=textract-pipeline/src/lambda/database-interface
              - *prep-dir
              - *unit-test
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/database-interface/**
        - step:
            name: Unit Test check database-interface-get
            image: python:3.11.10
            script:
              - DIR=textract-pipeline/src/lambda/database-interface-get
              - *prep-dir
              - *unit-test
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/database-interface-get/**
        - step:
            name: Unit Test check docx-to-pdf
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/docx-to-pdf
              - *prep-dir
              - *unit-test
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/docx-to-pdf/**
        - step:
            name: Unit Test check insert-record
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/insert-record
              - *prep-dir
              - *unit-test
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/insert-record/**
        - step:
            name: Unit Test check pdf-validation
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/pdf-validation
              - *prep-dir
              - *unit-test
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/pdf-validation/**
        - step:
            name: Unit Test check prompt-orchestrator
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/prompt-processor
              - *prep-dir
              - *unit-test
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/prompt-processor/**
        - step:
            name: Unit Test check s3-folder-details
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/s3-folder-details
              - *prep-dir
              - *unit-test
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/s3-folder-details/**
        - step:
            name: Unit Test check text-creation
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/text-creation
              - *prep-dir
              - *unit-test
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/text-creation/**
        - step:
            name: Unit Test check textract-receiver
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/textract-receiver
              - *prep-dir
              - *unit-test
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/textract-receiver/**
        - step:
            name: Unit Test check textract-sender
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/textract-sender
              - *prep-dir
              - *unit-test
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/textract-sender/**
        - step:
            name: Unit Test check trigger-pipeline
            image: python:3.12.7
            script:
              - DIR=textract-pipeline/src/lambda/trigger-pipeline
              - *prep-dir
              - *unit-test
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - textract-pipeline/src/lambda/trigger-pipeline/**
        - step:
            name: Unit Test check streamlit
            image: python:3.11.10
            size: 4x
            script:
              - DIR=streamlit
              - *prep-dir
              - *unit-test
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - streamlit/**
        - step:
            name: Unit Test check streamlit_multipage
            image: python:3.12.7
            script:
              - DIR=streamlit_multipage
              - *prep-dir
              - *unit-test
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - streamlit_multipage/**
        - step:
            name: Unit Test check fieldExtraction
            image: python:3.12.7
            script:
              - DIR=fieldExtraction
              - *prep-dir
              - *unit-test
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - fieldExtraction/**
        - step:
            name: Unit Test check Doczy.AI_Automation
            image: python:3.11.10
            script:
              - DIR=Doczy.AI_Automation
              - echo Must be in usage
              - exit 0
              - *prep-dir
              - *unit-test
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - Doczy.AI_Automation/**
        - step:
            name: Unit Test check airflow
            image: python:3.12.7
            script:
              - DIR=airflow
              - *prep-dir
              - *unit-test
            caches:
              - pip
            condition:
              changesets:
                includePaths:
                  - airflow/**
  branches:
    main:
      - step:
          name: Build and Test
          script:
            - echo Build and test pipeline will go here
      - stage:
          name: deployment-dev
          deployment: development
          steps: *deployment-steps
      - stage:
          name: deployment-plan-uat
          deployment: uat-plan
          trigger: manual
          steps: *deployment-plan-steps
      - stage:
          name: deployment-uat
          deployment: uat
          trigger: manual
          steps: *deployment-steps
      - stage:
          name: deployment-prod
          deployment: production
          trigger: manual
          steps: *deployment-steps