Updated bitbucket pipeline for all terraform streams

2024-05-09 17:30:31 -05:00
parent 4fe037bf57
commit 6bf61e4783
2 changed files with 144 additions and 28 deletions
@@ -1,4 +1,21 @@
 definitions:
+  scripts:
+    - script: &aws-context-dev
+        export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO:role/$OIDC_ROLE;
+        export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token;
+        echo $BITBUCKET_STEP_OIDC_TOKEN > $(pwd)/web-identity-token;
+        export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId');
+        export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey');
+        export AWS_SECURITY_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken');
+        export AWS_REGION='us-east-2';
+    - script: &aws-context-uat
+        export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO_UAT:role/$OIDC_ROLE_UAT;
+        export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token;s
+        echo $BITBUCKET_STEP_OIDC_TOKEN > $(pwd)/web-identity-token;
+        export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId');
+        export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey');
+        export AWS_SECURITY_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken');
+        export AWS_REGION='us-east-2';
  steps:
    - step: &snowflake_deploy
        name: "Deploy Snowflake Changes"
@@ -79,12 +96,6 @@ definitions:
 pipelines:
  branches:
    DEV: 
-      # - step:
-      #     <<: *terraform_plan_apply
-      #     condition:
-      #       changesets:
-      #         includePaths:
-      #           - terraform/environments/dev/**/*
      - step:
          <<: *snowflake_deploy
          condition:
@@ -105,27 +116,130 @@ pipelines:
              includePaths:
              - airflow/dags/*
      - step:
-          <<: *terraform_plan_apply
-          name: Plan and Apply Textract Pipeline
+          name: "Plan & Apply Textract Pipeline"
+          image: hugree/bitbucket-aws-python38-tf154:latest
+          oidc: true
+          trigger : manual
+          script:
+            - *aws-context-dev
+            - cd textract-pipeline/terraform/
+            - terraform validate
+            # Ensure the backend configuration for s3 obj key is corresponding to the stream name 
+            - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-d-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-d-infra-s3-terraform-state" -backend-config="key=textract-pipeline/terraform.tfstate" 
+            - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=dev" 
          condition:
            changesets:
              includePaths:
-                - textract-pipeline/terraform/**/*
-                - textract-pipeline/terraform/*
-                
+              - textract-pipeline/terraform/*
+              - textract-pipeline/terraform/**/* 
+      - step:
+          name: "Plan & Apply DevOps Pipeline"
+          image: hugree/bitbucket-aws-python38-tf154:latest
+          oidc: true
+          trigger : manual # Keeping it manual so that it doesnt conflict with existing resources
+          script:
+            - *aws-context-dev
+            - cd devops-pipeline/terraform-backend-resources/
+            - terraform validate
+            # Ensure the backend configuration for s3 obj key is correct
+            - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-d-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-d-infra-s3-terraform-state" -backend-config="key=devops-pipeline/terraform.tfstate" 
+            - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=dev" 
+          condition:
+            changesets:
+              includePaths:
+              - devops-pipeline/terraform-backend-resources/*
+              - devops-pipeline/terraform-backend-resources/**/* 
+      - step:
+          name: "Plan & Apply Streamlit Pipeline"
+          image: hugree/bitbucket-aws-python38-tf154:latest
+          oidc: true
+          trigger : manual # Keeping it manual so that it doesnt conflict with existing resources
+          script:
+            - *aws-context-dev
+            - cd streamlit-server/
+            - terraform validate
+            # Ensure the backend configuration for s3 obj key is correct
+            - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-d-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-d-infra-s3-terraform-state" -backend-config="key=streamlit-pipeline/terraform.tfstate" 
+            - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=dev" 
+          condition:
+            changesets:
+              includePaths:
+              - streamlit-server/*
+              - streamlit-server/**/* 

+                
    # UAT and PROD pipelines will be updated with Streamlit steps once it is tested and ready
    UAT:
-      - step:
-          <<: *terraform_plan_apply
-          condition:
-            paths:
-              - terraform/uat/**/*
      - step:
          <<: *snowflake_deploy
          condition:
-            paths:
-              - snowflake/**/*.sql
+            changesets:
+              includePaths:
+              - snowflake/**/*
+      - step:
+          <<: *streamlit_deploy
+          condition:
+            changesets:
+              includePaths:
+              - "streamlit/*"
+              - "streamlit/**/*"
+      - step:
+          <<: *airflow_dags_deploy
+          condition:
+            changesets:
+              includePaths:
+              - airflow/dags/*
+      - step:
+          name: "Plan & Apply Textract Pipeline"
+          image: hugree/bitbucket-aws-python38-tf154:latest
+          oidc: true
+          trigger : automatic
+          script:
+            - *aws-context-uat
+            - cd textract-pipeline/terraform/
+            - terraform validate
+            # Ensure the backend configuration for s3 obj key is corresponding to the stream name 
+            - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=textract-pipeline/terraform.tfstate" 
+            - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" 
+          condition:
+            changesets:
+              includePaths:
+              - textract-pipeline/terraform/*
+              - textract-pipeline/terraform/**/* 
+      - step:
+          name: "Plan & Apply DevOps Pipeline"
+          image: hugree/bitbucket-aws-python38-tf154:latest
+          oidc: true
+          trigger : automatic # Keeping it manual so that it doesnt conflict with existing resources
+          script:
+            - *aws-context-uat
+            - cd devops-pipeline/terraform-backend-resources/
+            - terraform validate
+            # Ensure the backend configuration for s3 obj key is correct
+            - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=textract-pipeline/terraform.tfstate" 
+            - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" 
+          condition:
+            changesets:
+              includePaths:
+              - devops-pipeline/terraform-backend-resources/*
+              - devops-pipeline/terraform-backend-resources/**/* 
+      - step:
+          name: "Plan & Apply Streamlit Pipeline"
+          image: hugree/bitbucket-aws-python38-tf154:latest
+          oidc: true
+          trigger : automatic
+          script:
+            - *aws-context-uat
+            - cd streamlit-server/
+            - terraform validate
+            # Ensure the backend configuration for s3 obj key is correct
+            - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=textract-pipeline/terraform.tfstate" 
+            - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" 
+          condition:
+            changesets:
+              includePaths:
+              - streamlit-server/*
+              - streamlit-server/**/* 
    PROD:
      - step:
          <<: *terraform_plan_apply
@@ -1,13 +1,15 @@
 -- CREATING THE CONFIG TABLE
 CREATE TABLE IF NOT EXISTS STG.DOCZY_PIPELINE_RAW_OUTPUT (
-    CONTRACT_NAME VARCHAR(255),
-    SF_DB_COL_NAME VARCHAR(255),
-    ACTUAL_VALUE_STORED VARCHAR(255),
-    RAW_VALUE VARCHAR(255),
-    NEW_EXTRACTED_VALUE VARCHAR(255),
-    SNIPPET VARCHAR(255),
-    ORIGINAL_PAGE_NUMBER VARCHAR(255),
-    NEW_PAGE_NUMBER VARCHAR(255),
-    RESULT VARCHAR(255),
-    BATCH_ID VARCHAR(255)
+    DOCUMENT_ID VARCHAR(16777216) NOT NULL,
+	SF_DB_COL_NAME VARCHAR(255),
+	ACTUAL_VALUE_STORED VARCHAR(16777216),
+	RAW_VALUE VARCHAR(16777216),
+	NEW_EXTRACTED_VALUE VARCHAR(16777216),
+	SNIPPET VARCHAR(16777216),
+	ORIGINAL_PAGE_NUMBER VARCHAR(255),
+	NEW_PAGE_NUMBER VARCHAR(255),
+	RESULT VARCHAR(16777216),
+	BATCH_ID VARCHAR(255),
+	CREATED_TIME TIMESTAMP_NTZ(9),
+	MODIFIED_TIME TIMESTAMP_NTZ(9)
 );