diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 970f9d7..ba20121 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -50,28 +50,6 @@ definitions: # trigger: manual caches: - pip - - step: &terraform_plan_apply - name: "Plan and Apply Terraform" - image: python:3.8 - script: - - python -m pip install --upgrade pip - - apt-get update && apt-get install -y jq git - - pip install awscli - - pip install terraform - - export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO:role/$OIDC_ROLE - - export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token - - echo $BITBUCKET_STEP_OIDC_TOKEN > $(pwd)/web-identity-token - - export STS_OUTPUT=$(aws sts assume-role-with-web-identity --role-arn $AWS_ROLE_ARN --role-session-name BitbucketPipeline --web-identity-token "$BITBUCKET_STEP_OIDC_TOKEN" --duration-seconds 3600) - - export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId') - - export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey') - - export AWS_SESSION_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken') - - aws sts get-caller-identity - - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="region=us-east-2" -backend-config="dynamodb_table=doczyai-use2-d-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-d-infra-s3-terraform-state" -backend-config="key=devops-pipeline/terraform.tfstate" - - terraform validate - - BRANCH_NAME_LOWER=$(echo $BITBUCKET_BRANCH | tr '[:upper:]' '[:lower:]') - - terraform apply -auto-approve -no-color -var 'access_key=$AWS_ACCESS_KEY_ID' -var 'secret_key=$AWS_SECRET_ACCESS_KEY' -var 'aws_region=us-east-2' -var 'environment=$BRANCH_NAME_LOWER' - trigger: automatic # uncomment after testing - # trigger: manual - step: &streamlit_deploy name: "Deploy streamlit to EC2" oidc: true @@ -142,51 +120,38 @@ pipelines: includePaths: - airflow/dags/* - step: - name: "Plan & Apply Textract Pipeline" + name: Apply textract-pipeline on DEV image: hugree/bitbucket-aws-python38-tf154:latest oidc: true - trigger : manual script: - *aws-context-dev - - cd textract-pipeline/terraform/ - # Ensure the backend configuration for s3 obj key is corresponding to the stream name - - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-d-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-d-infra-s3-terraform-state" -backend-config="key=terraform/textract-pipeline/terraform.tfstate" - - terraform validate - - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=dev" + - python terraform.py apply --environment=dev --module=textract-pipeline/terraform condition: changesets: includePaths: + - textract-pipeline/src/* + - textract-pipeline/src/**/* - textract-pipeline/terraform/* - textract-pipeline/terraform/**/* - step: - name: "Plan & Apply DevOps Pipeline" + name: Apply devops-pipeline on DEV image: hugree/bitbucket-aws-python38-tf154:latest oidc: true - trigger : manual # Keeping it manual so that it doesnt conflict with existing resources script: - *aws-context-dev - - cd devops-pipeline/terraform-backend-resources/ - # Ensure the backend configuration for s3 obj key is correct - - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-d-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-d-infra-s3-terraform-state" -backend-config="key=terraform/devops-pipeline/terraform.tfstate" - - terraform validate - - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=dev" + - python terraform.py apply --environment=dev --module=devops-pipeline/other-resources condition: changesets: includePaths: - - devops-pipeline/terraform-backend-resources/* - - devops-pipeline/terraform-backend-resources/**/* + - devops-pipeline/other-resources/* + - devops-pipeline/other-resources/**/* - step: - name: "Plan & Apply Streamlit Pipeline" + name: Apply streamlit-server on DEV image: hugree/bitbucket-aws-python38-tf154:latest oidc: true - trigger : automatic script: - *aws-context-dev - - cd streamlit-server/ - # Ensure the backend configuration for s3 obj key is correct - - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-d-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-d-infra-s3-terraform-state" -backend-config="key=terraform/streamlit-pipeline/terraform.tfstate" -backend-config="region=us-east-2" - - terraform validate - - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=dev" -var "vpc_id=vpc-0f9d7c913f6522079" + - python terraform.py apply --environment=dev --module=streamlit-server condition: changesets: includePaths: @@ -197,57 +162,43 @@ pipelines: # UAT and PROD pipelines will be updated with Streamlit steps once it is tested and ready UAT: - step: - name: "Plan & Apply DevOps Pipeline" + name: Apply textract-pipeline on UAT image: hugree/bitbucket-aws-python38-tf154:latest oidc: true - trigger : automatic script: - *aws-context-uat - - cd devops-pipeline/terraform-backend-resources/ - - terraform init - - terraform validate - # Ensure the backend configuration for s3 obj key is correct - - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat"- echo "S3_BUCKET=$(terraform output -raw terraform_s3_bucket_name)" >> backend.env - - echo "DYNAMODB_TABLE=$(terraform output -raw terraform_dynamodb_table_name)" >> backend.env - artifacts: - - backend.env + - python terraform.py apply --environment=uat --module=textract-pipeline/terraform condition: changesets: includePaths: - - devops-pipeline/terraform-backend-resources/* - - devops-pipeline/terraform-backend-resources/**/*- step: - name: "Plan & Apply DevOps Extra Resource Pipeline" - image: hugree/bitbucket-aws-python38-tf154:latest - oidc: true - trigger : automatic # Keeping it manual so that it doesnt conflict with existing resources - script: - - *aws-context-uat - - cd devops-pipeline/other-resources/ - - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/devops-pipeline-other-resources/terraform.tfstate" - - terraform validate - - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" - condition: - changesets: - includePaths: - - devops-pipeline/other-resources/* - - devops-pipeline/other-resources/**/* + - textract-pipeline/src/* + - textract-pipeline/src/**/* + - textract-pipeline/terraform/* + - textract-pipeline/terraform/**/* - step: - name: "Plan & Apply Streamlit Pipeline" + name: Apply devops-pipeline on UAT image: hugree/bitbucket-aws-python38-tf154:latest oidc: true - trigger : automatic script: - *aws-context-uat - - cd streamlit-server/ - - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/streamlit-server/terraform.tfstate" - - terraform validate - # Ensure the backend configuration for s3 obj key is correct - - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat"-var "vpc_id=vpc-0392396d0e7bdd77f" + - python terraform.py apply --environment=uat --module=devops-pipeline/other-resources condition: changesets: includePaths: - - streamlit-server/* - - streamlit-server/**/* + - devops-pipeline/other-resources/* + - devops-pipeline/other-resources/**/* + - step: + name: Apply streamlit-server on UAT + image: hugree/bitbucket-aws-python38-tf154:latest + oidc: true + script: + - *aws-context-uat + - python terraform.py apply --environment=uat --module=streamlit-server + condition: + changesets: + includePaths: + - streamlit-server/* + - streamlit-server/**/* - step: <<: *snowflake_deploy condition: @@ -279,31 +230,8 @@ pipelines: changesets: includePaths: - airflow/dags/* - - step: - name: "Plan & Apply Textract Pipeline" - image: hugree/bitbucket-aws-python38-tf154:latest - oidc: true - trigger : manual - script: - - *aws-context-uat - - cd textract-pipeline/terraform/ - - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/textract-pipeline/terraform.tfstate" - - terraform validate - # Ensure the backend configuration for s3 obj key is corresponding to the stream name - - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" - condition: - changesets: - includePaths: - - textract-pipeline/terraform/* - - textract-pipeline/terraform/**/* - PROD: - - step: - <<: *terraform_plan_apply - condition: - paths: - - terraform/prod//**/* - step: <<: *snowflake_deploy condition: