jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/DOC-595-cicd-for-prod (pull request #149)

Browse Source 
[DOC-595] cicd for prod

Approved-by: Umang Mistry
This commit is contained in:
Grzegorz Huber
2024-06-28 18:30:25 +00:00
committed by Umang Mistry
parent 34cd07eaea 04a3568163
commit 47403cf4ff
13 changed files with 153 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bitbucket-pipelines.yml
+106 -3
View File
@@ -16,6 +16,14 @@ definitions:
export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey');
export AWS_SECURITY_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken');
export AWS_REGION='us-east-2';
- script: &aws-context-prod
export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO_PROD:role/$OIDC_ROLE_PROD;
export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token;
echo $BITBUCKET_STEP_OIDC_TOKEN > $(pwd)/web-identity-token;
export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId');
export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey');
export AWS_SECURITY_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken');
export AWS_REGION='us-east-2';
- script: &ssm-send-command-dev
aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $DEV_INSTANCE_ID_TF --region $AWS_DEFAULT_REGION --parameters commands='[
"sudo -u ubuntu -i <<EOF",
@@ -32,6 +40,15 @@ definitions:
"if [ -d /home/ubuntu/doczy.ai ]; then cd /home/ubuntu/doczy.ai && git fetch && git pull; else cd /home/ubuntu && git clone git@bitbucket.org:aarete/doczy.ai.git; fi",
"EOF"
]'
- script: &ssm-send-command-prod
aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $PROD_INSTANCE_ID_TF --region $AWS_DEFAULT_REGION --parameters commands='[
"sudo -u ubuntu -i <<EOF",
"export HOME=/home/ubuntu",
"git config --global --add safe.directory /home/ubuntu/doczy.ai",
"if [ -d /home/ubuntu/doczy.ai ]; then cd /home/ubuntu/doczy.ai && git fetch && git pull; else cd /home/ubuntu && git clone git@bitbucket.org:aarete/doczy.ai.git; fi",
"EOF"
]'
- step: &snowflake_deploy
name: "Deploy Snowflake Changes"
@@ -204,13 +221,100 @@ pipelines:
- airflow/dags/*
PROD:
- step:
name: Apply devops-pipeline on PROD
image: hugree/bitbucket-aws-python38-tf154:latest
oidc: true
script:
- *aws-context-prod
- aws sts get-caller-identity --output text
- pip install -r requirements.txt
- BB_PIPELINE_BRANCH="DEV" python terraform.py apply --environment=prod --module=devops-pipeline/other-resources
- step:
name: Apply streamlit-server on PROD
image: hugree/bitbucket-aws-python38-tf154:latest
oidc: true
script:
- *aws-context-prod
- pip install -r requirements.txt
- BB_PIPELINE_BRANCH="DEV" python terraform.py apply --environment=prod --module=streamlit-server
- step:
name: Apply textract-pipeline on PROD
image: hugree/terraform-with-snowsql:latest
oidc: true
script:
- *aws-context-prod
- pip install -r requirements.txt
- BB_PIPELINE_BRANCH="DEV" python terraform.py apply --environment=prod --module=textract-pipeline/terraform
- step:
<<: *snowflake_deploy
condition:
paths:
- snowflake/**/*.sql
changesets:
includePaths:
- snowflake/PROD/*
- snowflake/PROD/**/*
- snowflake/PROD/**/**/*
- step:
image: python:3.8
name: "Deploy streamlit to EC2"
oidc: true
trigger: automatic # Keeping it manual as the server will be created first and then referenced in this
script:
- python -m pip install --upgrade pip
- apt-get update && apt-get install -y jq git
- pip install awscli
- *aws-context-prod
- *ssm-send-command-prod
condition:
changesets:
includePaths:
- streamlit/*
- streamlit/**/*
- step:
<<: *airflow_dags_deploy
trigger: manual
condition:
changesets:
includePaths:
- airflow/dags/*
feature/DOC-595-cicd-for-prod:
- parallel:
# - step:
# name: Plan devops-pipeline on PROD
# image: hugree/bitbucket-aws-python38-tf154:latest
# oidc: true
# script:
# - *aws-context-prod
# - aws sts get-caller-identity --output text
# - pip install -r requirements.txt
# - BB_PIPELINE_BRANCH="DEV" python terraform.py plan --environment=prod --module=devops-pipeline/other-resources
# - step:
# name: Plan streamlit-server on PROD
# image: hugree/bitbucket-aws-python38-tf154:latest
# oidc: true
# script:
# - *aws-context-prod
# - pip install -r requirements.txt
# - BB_PIPELINE_BRANCH="DEV" python terraform.py plan --environment=prod --module=streamlit-server
# - step:
# name: Plan textract-pipeline on PROD
# image: hugree/terraform-with-snowsql:latest
# oidc: true
# script:
# - *aws-context-prod
# - pip install -r requirements.txt
# - BB_PIPELINE_BRANCH="DEV" python terraform.py plan --environment=prod --module=textract-pipeline/terraform
- step:
name: Apply textract-pipeline on UAT
image: hugree/terraform-with-snowsql:latest
oidc: true
script:
- *aws-context-uat
- pip install -r requirements.txt
- BB_PIPELINE_BRANCH="DEV" python terraform.py apply --environment=uat --module=textract-pipeline/terraform
feature/DOC-536-smart-git-diff:
- parallel:
- step:
@@ -273,7 +377,6 @@ pipelines:
- *aws-context-dev
- pip install -r requirements.txt
- IS_DEPLOY_ALL=true python terraform.py apply --environment=dev --module=textract-pipeline/terraform --apply-extra-args="-target=null_resource.snowsql_client_create"
- step:
image: hugree/terraform-with-snowsql:latest
name: Run Snowsql on UAT
devops-pipeline/other-resources/vars-prod.tfvars
+4
View File
@@ -0,0 +1,4 @@
aws_account_id = 211125720533
# TODO change to proper prod value
storage_integration_external_id = "OQ11564_SFCRole=2_L4NUsKqd3NIBmrATa1djc/T3ECQ="
cross_account_target_account_id = 660131068782
devops-pipeline/other-resources/vars-uat.tfvars
+1 -1
View File
@@ -1,3 +1,3 @@
aws_account_id = 975049960860
storage_integration_external_id = "OQ11564_SFCRole=2_L4NUsKqd3NIBmrATa1djc/T3ECQ="
storage_integration_external_id = "OQ11564_SFCRole=2_yIGmS6lIr9sc1V6IdTqkl/8l1e4="
cross_account_target_account_id = 660131068782
devops-pipeline/terraform-backend-resources/main.tf
+1 -1
View File
@@ -8,7 +8,7 @@ terraform {
}
provider "aws" {
region = var.aws_region
region = var.aws_region
}
module "aws_terraform_remote_backend" {
devops-pipeline/terraform-backend-resources/modules/aws-terraform-remote-backend/variables.tf
-13
View File
@@ -10,19 +10,6 @@ variable "environment" {
variable "client_name" {
type = string
}
# required
variable "secret_key" {
type = string
}
# required
variable "access_key" {
type = string
}
variable "token" {
type = string
}
# dynamoDB table
variable "terraform_dynamodb_table" {
devops-pipeline/terraform-backend-resources/variables.tf
-15
View File
@@ -17,18 +17,3 @@ variable "client_name" {
type = string
default = "infra"
}
#required
variable "secret_key" {
type = string
}
#required
variable "access_key" {
type = string
}
#required
variable "token" {
type = string
}
devops-pipeline/terraform-backend-resources/vars-prod.tfvars
View File
streamlit-server/variables.tf
+10 -16
View File
@@ -1,4 +1,3 @@
# required
variable "aws_region" {
type = string
@@ -11,7 +10,7 @@ variable "project_name" {
}
# required
variable "environment" {
type = string
type = string
}
@@ -30,42 +29,37 @@ variable "vpc_id" {
type = string
}
# variable "ubuntu_ami" {
# type = string
# default = "ami-0de7e97fedfbc6ef6" # Switch to this ami-0b8b44ec9a8f90422
# }
variable "ubuntu_ami" {
type = map(string)
type = map(string)
default = {
dev = "ami-0de7e97fedfbc6ef6"
uat = "ami-0b8b44ec9a8f90422"
prod = "ami-0b8b44ec9a8f90422"
}
}
variable "ec2_instance_type" {
type = string
default = "t2.large"
type = string
default = "t2.large"
}
# Cerate an array of ARNS
variable "SecretsNames"{
type = list(string)
variable "SecretsNames" {
type = list(string)
default = []
}
variable "acm_arn_dev" {
type = string
type = string
default = "arn:aws:acm:us-east-2:660131068782:certificate/947e04b3-be22-43bf-bea2-a0ad3b221d70"
}
variable "acm_arns" {
type = map(string)
type = map(string)
default = {
dev = "arn:aws:acm:us-east-2:660131068782:certificate/947e04b3-be22-43bf-bea2-a0ad3b221d70"
uat = "arn:aws:acm:us-east-2:975049960860:certificate/e036e76d-1f4f-496e-ac97-5f65ec4a34cc"
prod = "arn:aws:acm:region:account-id:certificate/prod-cert-id"
prod = "arn:aws:acm:us-east-2:211125720533:certificate/45617883-689b-404c-a16c-019d4083bf7e"
}
}
streamlit-server/vars-prod.tfvars
+2
View File
@@ -0,0 +1,2 @@
vpc_id = "vpc-00cfffdf5b3ff8aec"
key_name = "tf-prod-kp"
textract-pipeline/terraform/main.tf
+7 -1
View File
@@ -39,6 +39,13 @@ locals {
devops_s3_bucket_name = "doczyai-use2-u-infra-s3-devops-resources"
api_access_required_roles_arns = ["arn:aws:iam::975049960860:role/doczyai-use2-u-rol-streamlit-server"]
}
prod = {
vpc_id = "vpc-00cfffdf5b3ff8aec"
secret_manager_name = "doczy_dev_db_creds"
devops_s3_bucket_name = "doczyai-use2-p-infra-s3-devops-resources"
api_access_required_roles_arns = ["arn:aws:iam::211125720533:role/doczyai-use2-p-rol-streamlit-server"]
}
}
# region abbreviation
@@ -120,7 +127,6 @@ resource "null_resource" "snowsql_client_create" {
}
}
# Create lambda_role
resource "aws_iam_role" "lambda_role" {
name = local.lambda_role_name
textract-pipeline/terraform/snowsql-cicd.sh
+8 -3
View File
@@ -11,11 +11,11 @@ echo "ENV_GROUP=${ENV_GROUP}, CLIENT_ID=${CLIENT_ID}, FULL_NAME=${FULL_NAME}, BU
readonly SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
echo "# Getting snowflake creds from AWS Secret Manager..."
snowflake_secret_name="doczy-${ENV_GROUP}-db-svc-acc"
snowflake_secret_name="bb_svc_acc"
echo "# Getting snowflake creds from AWS Secret Manager... snowflake_secret_name=${snowflake_secret_name}"
snowflake_secret_value=$(aws secretsmanager get-secret-value --secret-id "$snowflake_secret_name" --query 'SecretString' --output text)
SNOWFLAKE_ACCOUNT=$(echo $snowflake_secret_value | jq -r '.account_locator')
SNOWFLAKE_ACCOUNT=$(echo $snowflake_secret_value | jq -r '.account_alias')
SNOWFLAKE_DATABASE=$(echo $snowflake_secret_value | jq -r '.database')
SNOWFLAKE_USER=$(echo $snowflake_secret_value | jq -r '.user')
SNOWFLAKE_ROLE=$(echo $snowflake_secret_value | jq -r '.role')
@@ -34,6 +34,11 @@ export SNOWSQL_PWD=${SNOWFLAKE_PASSWORD}
#echo "QUERY=${QUERY}"
#snowsql -a "${SNOWFLAKE_ACCOUNT}" -u "${SNOWFLAKE_USER}" -d "${SNOWFLAKE_DATABASE}" -r "${SNOWFLAKE_ROLE}" -o exit_on_error=true -q "${QUERY}"
## SHOW THE VALUES
#SELECT_QUERY="SELECT DISTINCT client_id, client_name, bucket_name FROM STG.CLIENT_LOGS"
#echo "Client=${CLIENT_ID}: SELECT_QUERY=${SELECT_QUERY}"
#snowsql -a "${SNOWFLAKE_ACCOUNT}" -u "${SNOWFLAKE_USER}" -d "${SNOWFLAKE_DATABASE}" -r "${SNOWFLAKE_ROLE}" -o exit_on_error=true -q "${SELECT_QUERY}"
INSERT_QUERY=$(cat <<EOF
INSERT INTO STG.CLIENT_LOGS (CLIENT_ID, CLIENT_NAME, BUCKET_NAME)
SELECT '${CLIENT_ID}', '${FULL_NAME}', '${BUCKET_NAME}'
textract-pipeline/terraform/vars-prod.tfvars
+10
View File
@@ -0,0 +1,10 @@
aws_account_id = 211125720533
# TODO provide new layer arn for UAT? 336392948345 was used in DEV but not sure what account that is?!
text_creation_lambda_additional_layer_arn = "arn:aws:lambda:us-east-2:336392948345:layer:AWSSDKPandas-Python312:8"
client_list = {
client1 = {
client_id = "cn1"
full_name = "Centene"
}
}
textract-pipeline/terraform/vars-uat.tfvars
+4
View File
@@ -6,5 +6,9 @@ client_list = {
client1 = {
client_id = "cn1"
full_name = "Centene"
},
client2 = {
client_id = "ph1"
full_name = "Priority Health"
}
}