isolatesteps

2024-09-27 17:14:05 +01:00
parent d1afc1fc3e
commit bb5b010cde
1 changed files with 98 additions and 96 deletions
@@ -8,96 +8,94 @@ definitions:
        export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey');
        export AWS_SECURITY_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken');
        export AWS_REGION='us-east-2';
-  stage: &deployment
-    trigger: manual
-    steps:
-      - step:
-          name: "Deploy Snowflake Changes"
-          image: python:3.7
-          size: 2x
-          script:
-            - python -m pip install --upgrade pip
-            - pip install snowflake-connector-python
-            - pip install schemachange --upgrade
-            - echo $SNOWFLAKE_ACCOUNT $SNOWFLAKE_USER $SNOWFLAKE_ROLE $SNOWFLAKE_WAREHOUSE $SNOWFLAKE_DATABASE
-            - schemachange -a $SNOWFLAKE_ACCOUNT -u $SNOWFLAKE_USER -r $SNOWFLAKE_ROLE -w $SNOWFLAKE_WAREHOUSE -d $SNOWFLAKE_DATABASE -c $SNOWFLAKE_DATABASE.SCHEMACHANGE.CHANGE_HISTORY --create-change-history-table -v
-          caches:
-           - pip
-          #condition:
-          # changesets:
-          #    includePaths:
-          #    - snowflake/**/*
-          #    - snowflake/**/**/*
-          #    - snowflake/**/**/**/*
-      - step:
-          image: python:3.8
-          name: "Deploy streamlit to EC2"
-          oidc: true
-          script:
-            - python -m pip install --upgrade pip
-            - apt-get update && apt-get install -y jq git
-            - pip install awscli
-            - *aws-context
-            - >-
-              aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID_TF --region $AWS_DEFAULT_REGION --parameters commands='[
-                "sudo -u ubuntu -i <<EOF",
-                "export HOME=/home/ubuntu",
-                "git config --global --add safe.directory /home/ubuntu/doczy.ai",
-                "if [ -d /home/ubuntu/doczy.ai ]; then cd /home/ubuntu/doczy.ai && git fetch && git pull; else cd /home/ubuntu && git clone git@bitbucket.org:aarete/doczy.ai.git; fi",
-                "EOF"
-              ]'
-          #condition:
-          #  changesets:
-          #    includePaths:
-          #    - streamlit/*
-          #    - streamlit/**/*
-      - step:
-          name: "Deploy Airflow DAGs to S3 bucket"
-          image: python:3.8
-          oidc: true
-          script:
-            - python -m pip install --upgrade pip
-            - apt-get update && apt-get install -y jq git
-            - pip install awscli
-            - *aws-context
-            - aws sts get-caller-identity
-            - aws s3 sync ./airflow/dags s3://doczy-dev-infra-mwaa-resources/dags
-          #condition:
-          #  changesets:
-          #    includePaths:
-          #    - airflow/dags/*
-      - step:
-          image: hugree/terraform-with-snowsql:latest
-          name: Run Snowsql
-          oidc: true
-          script:
-            - *aws-context
-            - pip install -r requirements.txt
-            - IS_DEPLOY_ALL=true python terraform.py apply --environment=$ENVIRONMENT --module=textract-pipeline/terraform --apply-extra-args="-target=null_resource.snowsql_client_create"
-      - step:
-          name: Apply textract-pipeline
-          image: hugree/terraform-with-snowsql:latest
-          oidc: true
-          script:
-            - *aws-context
-            - pip install -r requirements.txt
-            - python terraform.py apply --environment=$ENVIRONMENT --module=textract-pipeline/terraform
-      - step:
-          name: Apply devops-pipeline
-          image: hugree/bitbucket-aws-python38-tf154:latest
-          oidc: true
-          script:
-            - *aws-context
-            - pip install -r requirements.txt
-            - python terraform.py apply --environment=$ENVIRONMENT --module=devops-pipeline/other-resources
-      - step:
-          name: Apply streamlit-server
-          image: hugree/bitbucket-aws-python38-tf154:latest
-          oidc: true
-          script:
-            - *aws-context
-            - pip install -r requirements.txt
-            - python terraform.py apply --environment=$ENVIRONMENT --module=streamlit-server
+  steps: &deployment-steps
+    - step:
+        name: "Deploy Snowflake Changes"
+        image: python:3.7
+        size: 2x
+        script:
+          - python -m pip install --upgrade pip
+          - pip install snowflake-connector-python
+          - pip install schemachange --upgrade
+          - echo $SNOWFLAKE_ACCOUNT $SNOWFLAKE_USER $SNOWFLAKE_ROLE $SNOWFLAKE_WAREHOUSE $SNOWFLAKE_DATABASE
+          - schemachange -a $SNOWFLAKE_ACCOUNT -u $SNOWFLAKE_USER -r $SNOWFLAKE_ROLE -w $SNOWFLAKE_WAREHOUSE -d $SNOWFLAKE_DATABASE -c $SNOWFLAKE_DATABASE.SCHEMACHANGE.CHANGE_HISTORY --create-change-history-table -v
+        caches:
+          - pip
+        #condition:
+        # changesets:
+        #    includePaths:
+        #    - snowflake/**/*
+        #    - snowflake/**/**/*
+        #    - snowflake/**/**/**/*
+    - step:
+        image: python:3.8
+        name: "Deploy streamlit to EC2"
+        oidc: true
+        script:
+          - python -m pip install --upgrade pip
+          - apt-get update && apt-get install -y jq git
+          - pip install awscli
+          - *aws-context
+          - >-
+            aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID_TF --region $AWS_DEFAULT_REGION --parameters commands='[
+              "sudo -u ubuntu -i <<EOF",
+              "export HOME=/home/ubuntu",
+              "git config --global --add safe.directory /home/ubuntu/doczy.ai",
+              "if [ -d /home/ubuntu/doczy.ai ]; then cd /home/ubuntu/doczy.ai && git fetch && git pull; else cd /home/ubuntu && git clone git@bitbucket.org:aarete/doczy.ai.git; fi",
+              "EOF"
+            ]'
+        #condition:
+        #  changesets:
+        #    includePaths:
+        #    - streamlit/*
+        #    - streamlit/**/*
+    - step:
+        name: "Deploy Airflow DAGs to S3 bucket"
+        image: python:3.8
+        oidc: true
+        script:
+          - python -m pip install --upgrade pip
+          - apt-get update && apt-get install -y jq git
+          - pip install awscli
+          - *aws-context
+          - aws sts get-caller-identity
+          - aws s3 sync ./airflow/dags s3://doczy-dev-infra-mwaa-resources/dags
+        #condition:
+        #  changesets:
+        #    includePaths:
+        #    - airflow/dags/*
+    - step:
+        image: hugree/terraform-with-snowsql:latest
+        name: Run Snowsql
+        oidc: true
+        script:
+          - *aws-context
+          - pip install -r requirements.txt
+          - IS_DEPLOY_ALL=true python terraform.py apply --environment=$ENVIRONMENT --module=textract-pipeline/terraform --apply-extra-args="-target=null_resource.snowsql_client_create"
+    - step:
+        name: Apply textract-pipeline
+        image: hugree/terraform-with-snowsql:latest
+        oidc: true
+        script:
+          - *aws-context
+          - pip install -r requirements.txt
+          - python terraform.py apply --environment=$ENVIRONMENT --module=textract-pipeline/terraform
+    - step:
+        name: Apply devops-pipeline
+        image: hugree/bitbucket-aws-python38-tf154:latest
+        oidc: true
+        script:
+          - *aws-context
+          - pip install -r requirements.txt
+          - python terraform.py apply --environment=$ENVIRONMENT --module=devops-pipeline/other-resources
+    - step:
+        name: Apply streamlit-server
+        image: hugree/bitbucket-aws-python38-tf154:latest
+        oidc: true
+        script:
+          - *aws-context
+          - pip install -r requirements.txt
+          - python terraform.py apply --environment=$ENVIRONMENT --module=streamlit-server

 pipelines:
  default:
@@ -113,14 +111,18 @@ pipelines:
          name: Build and Test
          script:
            - echo Build and test pipeline will go here
-      - stage: *deployment
+      - stage:
        name: deployment-dev
        deployment: development
-        #trigger: automatic
-      - stage: *deployment
+        trigger: automatic
+        steps: *deployment-steps
+      - stage:
        name: deployment-uat
        deployment: staging
-        #trigger: automatic
-      - stage: *deployment
+        trigger: manual
+        steps: *deployment-steps
+      - stage:
        name: deployment-prod
-        deployment: production
+        deployment: production
+        trigger: manual
+        steps: *deployment-steps