[DOC-542] Use terraform wrapper in CICD pipeline

2024-06-11 15:54:47 +02:00
parent b403013ea9
commit 80ef476172
1 changed files with 32 additions and 104 deletions
@@ -50,28 +50,6 @@ definitions:
        # trigger: manual
        caches:
          - pip
-    - step: &terraform_plan_apply
-        name: "Plan and Apply Terraform"
-        image: python:3.8
-        script:
-          - python -m pip install --upgrade pip
-          - apt-get update && apt-get install -y jq git
-          - pip install awscli
-          - pip install terraform
-          - export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO:role/$OIDC_ROLE
-          - export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token
-          - echo $BITBUCKET_STEP_OIDC_TOKEN > $(pwd)/web-identity-token
-          - export STS_OUTPUT=$(aws sts assume-role-with-web-identity --role-arn $AWS_ROLE_ARN --role-session-name BitbucketPipeline --web-identity-token "$BITBUCKET_STEP_OIDC_TOKEN"  --duration-seconds 3600)
-          - export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId')
-          - export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey')
-          - export AWS_SESSION_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken')
-          - aws sts get-caller-identity
-          - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="region=us-east-2" -backend-config="dynamodb_table=doczyai-use2-d-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-d-infra-s3-terraform-state" -backend-config="key=devops-pipeline/terraform.tfstate"
-          - terraform validate
-          - BRANCH_NAME_LOWER=$(echo $BITBUCKET_BRANCH | tr '[:upper:]' '[:lower:]')
-          - terraform apply -auto-approve -no-color -var 'access_key=$AWS_ACCESS_KEY_ID' -var 'secret_key=$AWS_SECRET_ACCESS_KEY' -var 'aws_region=us-east-2' -var 'environment=$BRANCH_NAME_LOWER'
-        trigger: automatic # uncomment after testing
-        # trigger: manual
    - step: &streamlit_deploy
        name: "Deploy streamlit to EC2"
        oidc: true
@@ -142,51 +120,38 @@ pipelines:
              includePaths:
              - airflow/dags/*
      - step:
-          name: "Plan & Apply Textract Pipeline"
+          name: Apply textract-pipeline on DEV
          image: hugree/bitbucket-aws-python38-tf154:latest
          oidc: true
-          trigger : manual
          script:
            - *aws-context-dev
-            - cd textract-pipeline/terraform/
-            # Ensure the backend configuration for s3 obj key is corresponding to the stream name 
-            - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-d-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-d-infra-s3-terraform-state" -backend-config="key=terraform/textract-pipeline/terraform.tfstate"
-            - terraform validate
-            - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=dev"
+            - python terraform.py apply --environment=dev --module=textract-pipeline/terraform
          condition:
            changesets:
              includePaths:
+              - textract-pipeline/src/*
+              - textract-pipeline/src/**/*
              - textract-pipeline/terraform/*
              - textract-pipeline/terraform/**/*
      - step:
-          name: "Plan & Apply DevOps Pipeline"
+          name: Apply devops-pipeline on DEV
          image: hugree/bitbucket-aws-python38-tf154:latest
          oidc: true
-          trigger : manual # Keeping it manual so that it doesnt conflict with existing resources
          script:
            - *aws-context-dev
-            - cd devops-pipeline/terraform-backend-resources/
-            # Ensure the backend configuration for s3 obj key is correct
-            - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-d-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-d-infra-s3-terraform-state" -backend-config="key=terraform/devops-pipeline/terraform.tfstate"
-            - terraform validate
-            - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=dev"
+            - python terraform.py apply --environment=dev --module=devops-pipeline/other-resources
          condition:
            changesets:
              includePaths:
-              - devops-pipeline/terraform-backend-resources/*
-              - devops-pipeline/terraform-backend-resources/**/*
+              - devops-pipeline/other-resources/*
+              - devops-pipeline/other-resources/**/*
      - step:
-          name: "Plan & Apply Streamlit Pipeline"
+          name: Apply streamlit-server on DEV
          image: hugree/bitbucket-aws-python38-tf154:latest
          oidc: true
-          trigger : automatic
          script:
            - *aws-context-dev
-            - cd streamlit-server/
-            # Ensure the backend configuration for s3 obj key is correct
-            - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-d-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-d-infra-s3-terraform-state" -backend-config="key=terraform/streamlit-pipeline/terraform.tfstate" -backend-config="region=us-east-2"
-            - terraform validate
-            - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=dev" -var "vpc_id=vpc-0f9d7c913f6522079"
+            - python terraform.py apply --environment=dev --module=streamlit-server
          condition:
            changesets:
              includePaths:
@@ -197,57 +162,43 @@ pipelines:
    # UAT and PROD pipelines will be updated with Streamlit steps once it is tested and ready
    UAT:
      - step:
-          name: "Plan & Apply DevOps Pipeline"
+          name: Apply textract-pipeline on UAT
          image: hugree/bitbucket-aws-python38-tf154:latest
          oidc: true
-          trigger : automatic
          script:
            - *aws-context-uat
-            - cd devops-pipeline/terraform-backend-resources/
-            - terraform init
-            - terraform validate
-            # Ensure the backend configuration for s3 obj key is correct
-            -  terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat"- echo "S3_BUCKET=$(terraform output -raw terraform_s3_bucket_name)" >> backend.env
-            - echo "DYNAMODB_TABLE=$(terraform output -raw terraform_dynamodb_table_name)" >> backend.env
-          artifacts:
-            - backend.env
+            - python terraform.py apply --environment=uat --module=textract-pipeline/terraform
          condition:
            changesets:
              includePaths:
-              - devops-pipeline/terraform-backend-resources/*
-              - devops-pipeline/terraform-backend-resources/**/*- step:
-          name: "Plan & Apply DevOps Extra Resource Pipeline"
-          image: hugree/bitbucket-aws-python38-tf154:latest
-          oidc: true
-          trigger : automatic # Keeping it manual so that it doesnt conflict with existing resources
-          script:
-            - *aws-context-uat
-            - cd devops-pipeline/other-resources/
-            - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/devops-pipeline-other-resources/terraform.tfstate"
-            - terraform validate
-            - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat"
-          condition:
-            changesets:
-              includePaths:
-              - devops-pipeline/other-resources/*
-              - devops-pipeline/other-resources/**/*
+                - textract-pipeline/src/*
+                - textract-pipeline/src/**/*
+                - textract-pipeline/terraform/*
+                - textract-pipeline/terraform/**/*
      - step:
-          name: "Plan & Apply Streamlit Pipeline"
+          name: Apply devops-pipeline on UAT
          image: hugree/bitbucket-aws-python38-tf154:latest
          oidc: true
-          trigger : automatic
          script:
            - *aws-context-uat
-            - cd streamlit-server/
-            - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/streamlit-server/terraform.tfstate"
-            - terraform validate
-            # Ensure the backend configuration for s3 obj key is correct
-            -  terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat"-var "vpc_id=vpc-0392396d0e7bdd77f"
+            - python terraform.py apply --environment=uat --module=devops-pipeline/other-resources
          condition:
            changesets:
              includePaths:
-              - streamlit-server/*
-              - streamlit-server/**/*
+                - devops-pipeline/other-resources/*
+                - devops-pipeline/other-resources/**/*
+      - step:
+          name: Apply streamlit-server on UAT
+          image: hugree/bitbucket-aws-python38-tf154:latest
+          oidc: true
+          script:
+            - *aws-context-uat
+            - python terraform.py apply --environment=uat --module=streamlit-server
+          condition:
+            changesets:
+              includePaths:
+                - streamlit-server/*
+                - streamlit-server/**/*
      - step:
          <<: *snowflake_deploy
          condition:
@@ -279,31 +230,8 @@ pipelines:
            changesets:
              includePaths:
              - airflow/dags/*
-      - step:
-          name: "Plan & Apply Textract Pipeline"
-          image: hugree/bitbucket-aws-python38-tf154:latest
-          oidc: true
-          trigger : manual
-          script:
-            - *aws-context-uat
-            - cd textract-pipeline/terraform/
-            - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/textract-pipeline/terraform.tfstate"
-            - terraform validate
-            # Ensure the backend configuration for s3 obj key is corresponding to the stream name 
-            -  terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat"
-          condition:
-            changesets:
-              includePaths:
-              - textract-pipeline/terraform/*
-              - textract-pipeline/terraform/**/*
-

    PROD:
-      - step:
-          <<: *terraform_plan_apply
-          condition:
-            paths:
-              - terraform/prod//**/*
      - step:
          <<: *snowflake_deploy
          condition: