fullrambo

2024-09-23 16:28:03 +01:00
parent 6736b91a24
commit 8cf1b4b4d6
2 changed files with 146 additions and 51 deletions
@@ -1,49 +1,156 @@
-export: true
-
 definitions:
-  stage: &example
+  scripts:
+    - script: &aws-context
+        export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO:role/$AWS_OIDC_ROLE;
+        export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token;
+        echo $BITBUCKET_STEP_OIDC_TOKEN > $(pwd)/web-identity-token;
+        export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId');
+        export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey');
+        export AWS_SECURITY_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken');
+        export AWS_REGION='us-east-2';
+  stage: &deployment
    name: deployment
+    deployment: development
+    trigger: manual
+    environment:
+      SNOWFLAKE_DATABASE: $DEV_SNOWFLAKE_DATABASE
+      SNOWFLAKE_ROLE: $DEV_SNOWFLAKE_ROLE
+      AWS_IODC_ROLE: $IODC_ROLE
+      INSTANCE_ID_TF: $DEV_INSTANCE_ID_TF
+      IS_DEPLOY_ALL: true
+      BB_PIPELINE_BRANCH: $BITBUCKET_BRANCH
+      ENVIRONMENT: dev
+    steps:
+      - step:
+          name: "Deploy Snowflake Changes"
+          image: python:3.7
+          size: 2x
+          script:
+            - python -m pip install --upgrade pip
+            - pip install snowflake-connector-python
+            - pip install schemachange --upgrade
+            - schemachange -a $SNOWFLAKE_ACCOUNT -u $SNOWFLAKE_USER -r $SNOWFLAKE_ROLE -w $SNOWFLAKE_WAREHOUSE -d $SNOWFLAKE_DATABASE -c $SNOWFLAKE_DATABASE.SCHEMACHANGE.CHANGE_HISTORY --create-change-history-table -v
+          caches:
+           - pip
+          condition:
+           changesets:
+              includePaths:
+              - snowflake/**/*
+              - snowflake/**/**/*
+              - snowflake/**/**/**/*
+      - step:
+          image: python:3.8
+          name: "Deploy streamlit to EC2"
+          oidc: true
+          script:
+            - python -m pip install --upgrade pip
+            - apt-get update && apt-get install -y jq git
+            - pip install awscli
+            - *aws-context
+            - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID_TF --region $AWS_DEFAULT_REGION --parameters commands='[
+              "sudo -u ubuntu -i <<EOF",
+              "export HOME=/home/ubuntu",
+              "git config --global --add safe.directory /home/ubuntu/doczy.ai",
+              "if [ -d /home/ubuntu/doczy.ai ]; then cd /home/ubuntu/doczy.ai && git fetch && git pull; else cd /home/ubuntu && git clone git@bitbucket.org:aarete/doczy.ai.git; fi",
+              "EOF"
+            \]'
+          condition:
+            changesets:
+              includePaths:
+              - streamlit/*
+              - streamlit/**/*
+      - step:
+          name: "Deploy Airflow DAGs to S3 bucket"
+          image: python:3.8
+          oidc: true
+          script:
+            - python -m pip install --upgrade pip
+            - apt-get update && apt-get install -y jq git
+            - pip install awscli
+            - *aws-context
+            - aws sts get-caller-identity
+            - aws s3 sync ./airflow/dags s3://doczy-dev-infra-mwaa-resources/dags
+          condition:
+            changesets:
+              includePaths:
+              - airflow/dags/*
+      - step:
+        image: hugree/terraform-with-snowsql:latest
+        name: Run Snowsql
+        oidc: true
+        script:
+          - *aws-context
+          - pip install -r requirements.txt
+          - IS_DEPLOY_ALL=true python terraform.py apply --environment=$ENVIRONMENT --module=textract-pipeline/terraform --apply-extra-args="-target=null_resource.snowsql_client_create"
+      - step:
+          name: Apply textract-pipeline
+          image: hugree/terraform-with-snowsql:latest
+          oidc: true
+          script:
+            - *aws-context
+            - pip install -r requirements.txt
+            - python terraform.py apply --environment=$ENVIRONMENT --module=textract-pipeline/terraform
+      - step:
+          name: Apply devops-pipeline
+          image: hugree/bitbucket-aws-python38-tf154:latest
+          oidc: true
+          script:
+            - *aws-context
+            - pip install -r requirements.txt
+            - python terraform.py apply --environment=$ENVIRONMENT --module=devops-pipeline/other-resources
+      - step:
+          name: Apply streamlit-server
+          image: hugree/bitbucket-aws-python38-tf154:latest
+          oidc: true
+          script:
+            - *aws-context
+            - pip install -r requirements.txt
+            - python terraform.py apply --environment=$ENVIRONMENT --module=streamlit-server
+  stage: &example
+    name: example
+    trigger: manual
+    environment:
+      CUSTOM_ENV: HEYHEY
    steps:
      - step:
          name: "Example"
          script:
-            - echo "HEELOO"
+            - echo "$CUSTOM_ENV HEELOO"

 pipelines:
  default:
    - stage: *example
-#  branches:
-#  main:
-#    - include: cicd/deployment.yml
-#      name: deployment-dev
-#      deployment: development
-#      trigger: automatic
-#      environment:
-#        SNOWFLAKE_DATABASE: $DEV_SNOWFLAKE_DATABASE
-#        SNOWFLAKE_ROLE: $DEV_SNOWFLAKE_ROLE
-#        AWS_IODC_ROLE: $IODC_ROLE
-#        INSTANCE_ID_TF: $DEV_INSTANCE_ID_TF
-#        IS_DEPLOY_ALL: true
-#        ENVIRONMENT: dev
-#    - include: cicd/deployment.yml
-#      name: deployment-uat
-#      deployment: staging
-#      trigger: automatic
-#      environment:
-#        SNOWFLAKE_DATABASE: $UAT_SNOWFLAKE_DATABASE
-#        SNOWFLAKE_ROLE: $UAT_SNOWFLAKE_ROLE
-#        AWS_IODC_ROLE: $IODC_ROLE_UAT
-#        INSTANCE_ID_TF: $UAT_INSTANCE_ID_TF
-#        IS_DEPLOY_ALL: true
-#        ENVIRONMENT: uat
-#    - include: cicd/deployment.yml
-#      name: deployment-prod
-#      deployment: production
-#      trigger: manual
-#      environment:
-#        SNOWFLAKE_DATABASE: $PROD_SNOWFLAKE_DATABASE
-#        SNOWFLAKE_ROLE: $PROD_SNOWFLAKE_ROLE
-#        AWS_IODC_ROLE: $IODC_ROLE_PROD
-#        INSTANCE_ID_TF: $PROD_INSTANCE_ID_TF
-#        IS_DEPLOY_ALL: true
-#        ENVIRONMENT: prod
+  branches:
+    main:
+      - stage: *deployment
+        name: deployment-dev
+        deployment: development
+        #trigger: automatic
+        environment:
+          SNOWFLAKE_DATABASE: $DEV_SNOWFLAKE_DATABASE
+          SNOWFLAKE_ROLE: $DEV_SNOWFLAKE_ROLE
+          AWS_IODC_ROLE: $IODC_ROLE
+          INSTANCE_ID_TF: $DEV_INSTANCE_ID_TF
+          IS_DEPLOY_ALL: true
+          ENVIRONMENT: dev
+      - include: cicd/deployment.yml
+        name: deployment-uat
+        deployment: staging
+        #trigger: automatic
+        environment:
+          SNOWFLAKE_DATABASE: $UAT_SNOWFLAKE_DATABASE
+          SNOWFLAKE_ROLE: $UAT_SNOWFLAKE_ROLE
+          AWS_IODC_ROLE: $IODC_ROLE_UAT
+          INSTANCE_ID_TF: $UAT_INSTANCE_ID_TF
+          IS_DEPLOY_ALL: true
+          ENVIRONMENT: uat
+      - include: cicd/deployment.yml
+        name: deployment-prod
+        deployment: production
+        environment:
+          SNOWFLAKE_DATABASE: $PROD_SNOWFLAKE_DATABASE
+          SNOWFLAKE_ROLE: $PROD_SNOWFLAKE_ROLE
+          AWS_IODC_ROLE: $IODC_ROLE_PROD
+          INSTANCE_ID_TF: $PROD_INSTANCE_ID_TF
+          IS_DEPLOY_ALL: true
+          ENVIRONMENT: prod
@@ -1,12 +0,0 @@
-export: true
-
-definitions:
-  pipelines:
-    example:
-      - stage:
-        name: deployment
-        steps:
-          - step:
-              name: "Example"
-              script:
-                - echo "HEELOO"