From b5d400dfe2a35242172cd93f571db0f65cf45eec Mon Sep 17 00:00:00 2001
From: Michael McGuinness <mmcguinness@aarete.com>
Date: Fri, 4 Oct 2024 15:26:57 +0000
Subject: [PATCH] Merged in bugfix/deployConfigProperties (pull request #224)

Bugfix/deployConfigProperties

* syncconfig

* copydir

* specbucket

* pipplusdevdeploy

* struct

* script

* image

* cleanup

* testplan

* addallplans

* stepnames
---
 bitbucket-pipelines.yml | 105 ++++++++++++++++++++--------------------
 1 file changed, 52 insertions(+), 53 deletions(-)

diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml
index 69f2176..cdf5de6 100644
--- a/bitbucket-pipelines.yml
+++ b/bitbucket-pipelines.yml
@@ -1,6 +1,9 @@
 definitions:
   scripts:
     - script: &aws-context
+        apt-get update;
+        apt-get install -y jq;
+        pip install awscli;
         export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO:role/$OIDC_ROLE;
         export AWS_ROLE_SESSION_NAME=$OIDC_ROLE;
         export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token;
@@ -10,6 +13,7 @@ definitions:
         export AWS_SECURITY_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken');
         export AWS_REGION='us-east-2';
         aws sts get-caller-identity;
+        export IS_DEPLOY_ALL=true; 
   steps: &deployment-plan-steps
     - step:
         name: Plan devops-pipeline
@@ -19,6 +23,8 @@ definitions:
           - *aws-context
           - pip install -r requirements.txt
           - python terraform.py plan --environment=$ENVIRONMENT --module=devops-pipeline/other-resources
+        caches:
+          - pip
     - step:
         name: Plan textract-pipeline
         image: hugree/terraform-with-snowsql:latest
@@ -27,6 +33,8 @@ definitions:
           - *aws-context
           - pip install -r requirements.txt
           - python terraform.py plan --environment=$ENVIRONMENT --module=textract-pipeline/terraform
+        caches:
+          - pip
     - step:
         name: Plan streamlit-server
         image: hugree/bitbucket-aws-python38-tf154:latest
@@ -35,17 +43,8 @@ definitions:
           - *aws-context
           - pip install -r requirements.txt
           - python terraform.py plan --environment=$ENVIRONMENT --module=streamlit-server
-#    - step:
-#        name: Plan Snowflake Changes
-#        image: python:3.7
-#        size: 2x
-#        script:
-#          - python -m pip install --upgrade pip
-#          - pip install snowflake-connector-python
-#          - pip install schemachange --upgrade
-#          - schemachange -a $SNOWFLAKE_ACCOUNT -u $SNOWFLAKE_USER -r $SNOWFLAKE_ROLE -w $SNOWFLAKE_WAREHOUSE -d $SNOWFLAKE_DATABASE -c $SNOWFLAKE_DATABASE.SCHEMACHANGE.CHANGE_HISTORY --create-change-history-table -v
-#        caches:
-#          - pip
+        caches:
+          - pip
     - step:
         image: hugree/terraform-with-snowsql:latest
         name: Plan Snowsql
@@ -53,24 +52,9 @@ definitions:
         script:
           - *aws-context
           - pip install -r requirements.txt
-          - IS_DEPLOY_ALL=true python terraform.py plan --environment=$ENVIRONMENT --module=textract-pipeline/terraform --apply-extra-args="-target=null_resource.snowsql_client_create"
-#    - step:
-#        image: python:3.8
-#        name: "Deploy streamlit to EC2"
-#        oidc: true
-#        script:
-#          - python -m pip install --upgrade pip
-#          - apt-get update && apt-get install -y jq git
-#          - pip install awscli
-#          - *aws-context
-#          - >-
-#            aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID_TF --region $AWS_DEFAULT_REGION --parameters commands='[
-#              "sudo -u ubuntu -i <<EOF",
-#              "export HOME=/home/ubuntu",
-#              "git config --global --add safe.directory /home/ubuntu/doczy.ai",
-#              "if [ -d /home/ubuntu/doczy.ai ]; then cd /home/ubuntu/doczy.ai && git fetch && git pull; else cd /home/ubuntu && git clone git@bitbucket.org:aarete/doczy.ai.git; fi",
-#              "EOF"
-#            ]'
+          - python terraform.py plan --environment=$ENVIRONMENT --module=textract-pipeline/terraform --apply-extra-args="-target=null_resource.snowsql_client_create"
+        caches:
+          - pip
   steps: &deployment-steps
     - step:
         name: Apply devops-pipeline
@@ -80,6 +64,17 @@ definitions:
           - *aws-context
           - pip install -r requirements.txt
           - python terraform.py apply --environment=$ENVIRONMENT --module=devops-pipeline/other-resources
+        caches:
+          - pip
+    - step:
+        name: Sync S3 Config
+        image: python:3.8
+        oidc: true
+        script:
+          - *aws-context
+          - aws s3 sync ./textract-pipeline/src/s3-files s3://${TEXTRACT_BUCKET}
+        caches:
+          - pip
     - step:
         name: Apply textract-pipeline
         image: hugree/terraform-with-snowsql:latest
@@ -88,6 +83,8 @@ definitions:
           - *aws-context
           - pip install -r requirements.txt
           - python terraform.py apply --environment=$ENVIRONMENT --module=textract-pipeline/terraform
+        caches:
+          - pip
     - step:
         name: Apply streamlit-server
         image: hugree/bitbucket-aws-python38-tf154:latest
@@ -96,40 +93,35 @@ definitions:
           - *aws-context
           - pip install -r requirements.txt
           - python terraform.py apply --environment=$ENVIRONMENT --module=streamlit-server
+        caches:
+          - pip
     - step:
         name: "Deploy Snowflake Changes"
         image: python:3.7
         size: 2x
         script:
-          - python -m pip install --upgrade pip
           - pip install snowflake-connector-python
           - pip install schemachange --upgrade
           - schemachange -a $SNOWFLAKE_ACCOUNT -u $SNOWFLAKE_USER -r $SNOWFLAKE_ROLE -w $SNOWFLAKE_WAREHOUSE -d $SNOWFLAKE_DATABASE -c $SNOWFLAKE_DATABASE.SCHEMACHANGE.CHANGE_HISTORY --create-change-history-table -v
         caches:
           - pip
-        #condition:
-        # changesets:
-        #    includePaths:
-        #    - snowflake/**/*
-        #    - snowflake/**/**/*
-        #    - snowflake/**/**/**/*
     - step:
         image: hugree/terraform-with-snowsql:latest
-        name: Run Snowsql
+        name: Apply Snowsql
         oidc: true
         script:
           - *aws-context
           - pip install -r requirements.txt
-          - IS_DEPLOY_ALL=true python terraform.py apply --environment=$ENVIRONMENT --module=textract-pipeline/terraform --apply-extra-args="-target=null_resource.snowsql_client_create"
+          - python terraform.py apply --environment=$ENVIRONMENT --module=textract-pipeline/terraform --apply-extra-args="-target=null_resource.snowsql_client_create"
+        caches:
+          - pip
     - step:
         image: python:3.8
         name: "Deploy streamlit to EC2"
         oidc: true
         script:
-          - python -m pip install --upgrade pip
-          - apt-get update && apt-get install -y jq git
-          - pip install awscli
           - *aws-context
+          - apt-get install -y git
           - >-
             aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID_TF --region $AWS_DEFAULT_REGION --parameters commands='[
               "sudo -u ubuntu -i <<EOF",
@@ -138,35 +130,43 @@ definitions:
               "if [ -d /home/ubuntu/doczy.ai ]; then cd /home/ubuntu/doczy.ai && git fetch && git pull; else cd /home/ubuntu && git clone git@bitbucket.org:aarete/doczy.ai.git; fi",
               "EOF"
             ]'
-        #condition:
-        #  changesets:
-        #    includePaths:
-        #    - streamlit/*
-        #    - streamlit/**/*
+        caches:
+          - pip
 
 pipelines:
   default:
+    - step:
+        image: python:3.8
+        name: Install Pip
+        script:
+          - python -m pip install --upgrade pip
+        caches:
+          - pip
     - step:
         name: Lint the code
         image: hugree/bitbucket-aws-python38-tf154:latest
         script:
           - pip install -r dev-requirements.txt
           - python -m black --check .
+        caches:
+          - pip
   branches:
     main:
       - step:
           name: Build and Test
           script:
             - echo Build and test pipeline will go here
+      - stage:
+          name: deployment-dev-plan
+          deployment: development-plan
+          steps: *deployment-plan-steps
       - stage:
           name: deployment-dev
           deployment: development
-          trigger: automatic
           steps: *deployment-steps
       - stage:
-          name: deployment-plan-uat
-          deployment: uat
-          trigger: manual
+          name: deployment-uat-plan
+          deployment: uat-plan
           steps: *deployment-plan-steps
       - stage:
           name: deployment-uat
@@ -174,9 +174,8 @@ pipelines:
           trigger: manual
           steps: *deployment-steps
       - stage:
-          name: deployment-plan-prod
-          deployment: production
-          trigger: manual
+          name: deployment-prod-plan
+          deployment: production-plan
           steps: *deployment-plan-steps
       - stage:
           name: deployment-prod