jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in DEV (pull request #146)

Browse Source 
DEV and UAT sync

Approved-by: Umang Mistry
This commit is contained in:
Grzegorz Huber
2024-06-26 14:06:26 +00:00
committed by Umang Mistry
parent 6824013059 648aaac752
commit 913c0835b5
15 changed files with 253 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+3
View File
@@ -71,3 +71,6 @@ streamlit/results.csv
# env
streamlit/venv
*.tfplan
*.pem
bitbucket-pipelines.yml
+43 -44
View File
@@ -121,84 +121,56 @@ pipelines:
- airflow/dags/*
- step:
name: Apply textract-pipeline on DEV
image: hugree/bitbucket-aws-python38-tf154:latest
image: hugree/terraform-with-snowsql:latest
oidc: true
script:
- *aws-context-dev
- pip install -r requirements.txt
- python terraform.py apply --environment=dev --module=textract-pipeline/terraform
condition:
changesets:
includePaths:
- textract-pipeline/src/*
- textract-pipeline/src/**/*
- textract-pipeline/terraform/*
- textract-pipeline/terraform/**/*
- step:
name: Apply devops-pipeline on DEV
image: hugree/bitbucket-aws-python38-tf154:latest
oidc: true
script:
- *aws-context-dev
- pip install -r requirements.txt
- python terraform.py apply --environment=dev --module=devops-pipeline/other-resources
condition:
changesets:
includePaths:
- devops-pipeline/other-resources/*
- devops-pipeline/other-resources/**/*
- step:
name: Apply streamlit-server on DEV
image: hugree/bitbucket-aws-python38-tf154:latest
oidc: true
script:
- *aws-context-dev
- pip install -r requirements.txt
- python terraform.py apply --environment=dev --module=streamlit-server
condition:
changesets:
includePaths:
- streamlit-server/*
- streamlit-server/**/*
# UAT and PROD pipelines will be updated with Streamlit steps once it is tested and ready
UAT:
- step:
name: Apply textract-pipeline on UAT
image: hugree/bitbucket-aws-python38-tf154:latest
image: hugree/terraform-with-snowsql:latest
oidc: true
script:
- *aws-context-uat
- pip install -r requirements.txt
- python terraform.py apply --environment=uat --module=textract-pipeline/terraform
condition:
changesets:
includePaths:
- textract-pipeline/src/*
- textract-pipeline/src/**/*
- textract-pipeline/terraform/*
- textract-pipeline/terraform/**/*
- step:
name: Apply devops-pipeline on UAT
image: hugree/bitbucket-aws-python38-tf154:latest
oidc: true
script:
- *aws-context-uat
- pip install -r requirements.txt
- python terraform.py apply --environment=uat --module=devops-pipeline/other-resources
condition:
changesets:
includePaths:
- devops-pipeline/other-resources/*
- devops-pipeline/other-resources/**/*
- step:
name: Apply streamlit-server on UAT
image: hugree/bitbucket-aws-python38-tf154:latest
oidc: true
script:
- *aws-context-uat
- pip install -r requirements.txt
- python terraform.py apply --environment=uat --module=streamlit-server
condition:
changesets:
includePaths:
- streamlit-server/*
- streamlit-server/**/*
- step:
<<: *snowflake_deploy
condition:
@@ -239,7 +211,7 @@ pipelines:
- snowflake/**/*.sql
feature/terraform-refactor:
feature/DOC-536-smart-git-diff:
- parallel:
- step:
name: Plan devops-pipeline on DEV
@@ -247,39 +219,66 @@ pipelines:
oidc: true
script:
- *aws-context-dev
- python terraform.py plan --environment=dev --module=devops-pipeline/other-resources
- pip install -r requirements.txt
- BB_PIPELINE_BRANCH="DEV" python terraform.py plan --environment=dev --module=devops-pipeline/other-resources
- step:
name: Plan streamlit-server on DEV
image: hugree/bitbucket-aws-python38-tf154:latest
oidc: true
script:
- *aws-context-dev
- python terraform.py plan --environment=dev --module=streamlit-server
- pip install -r requirements.txt
- BB_PIPELINE_BRANCH="DEV" python terraform.py plan --environment=dev --module=streamlit-server
- step:
name: Plan textract-pipeline on DEV
image: hugree/bitbucket-aws-python38-tf154:latest
image: hugree/terraform-with-snowsql:latest
oidc: true
script:
- *aws-context-dev
- python terraform.py plan --environment=dev --module=textract-pipeline/terraform
- pip install -r requirements.txt
- BB_PIPELINE_BRANCH="DEV" python terraform.py plan --environment=dev --module=textract-pipeline/terraform
- step:
name: Plan devops-pipeline on UAT
image: hugree/bitbucket-aws-python38-tf154:latest
oidc: true
script:
- *aws-context-uat
- python terraform.py plan --environment=uat --module=devops-pipeline/other-resources
- pip install -r requirements.txt
- BB_PIPELINE_BRANCH="UAT" python terraform.py plan --environment=uat --module=devops-pipeline/other-resources
- step:
name: Plan streamlit-server on UAT
image: hugree/bitbucket-aws-python38-tf154:latest
oidc: true
script:
- *aws-context-uat
- python terraform.py plan --environment=uat --module=streamlit-server
- pip install -r requirements.txt
- BB_PIPELINE_BRANCH="UAT" python terraform.py plan --environment=uat --module=streamlit-server
- step:
name: Plan textract-pipeline on UAT
image: hugree/bitbucket-aws-python38-tf154:latest
oidc: true
script:
- *aws-context-uat
- python terraform.py plan --environment=uat --module=textract-pipeline/terraform
- pip install -r requirements.txt
- BB_PIPELINE_BRANCH="UAT" python terraform.py plan --environment=uat --module=textract-pipeline/terraform
feature/DOC-535-snowsql-in-cicd:
- parallel:
- step:
image: hugree/terraform-with-snowsql:latest
name: Run Snowsql on DEV
oidc: true
script:
- *aws-context-dev
- pip install -r requirements.txt
- IS_DEPLOY_ALL=true python terraform.py apply --environment=dev --module=textract-pipeline/terraform --apply-extra-args="-target=null_resource.snowsql_client_create"
- step:
image: hugree/terraform-with-snowsql:latest
name: Run Snowsql on UAT
oidc: true
script:
- *aws-context-uat
- pip install -r requirements.txt
- IS_DEPLOY_ALL=true python terraform.py apply --environment=uat --module=textract-pipeline/terraform --apply-extra-args="-target=null_resource.snowsql_client_create"
git_diff_changes.py
+90
View File
@@ -0,0 +1,90 @@
import os
import subprocess
import requests
BITBUCKET_REPO_OWNER = "aarete"
BITBUCKET_REPO_SLUG = "doczy.ai"
BB_PIPELINE_BRANCH = os.getenv("BB_PIPELINE_BRANCH", "DEV")
BITBUCKET_AUTH_HEADER = os.getenv("BITBUCKET_AUTH_HEADER")
from terminal import run_command, prepare_logger, decorate_warn
LOGGER = prepare_logger()
def check_bitbucket_auth_header():
if os.getenv("BITBUCKET_CI") == "true" and not BITBUCKET_AUTH_HEADER:
raise EnvironmentError("Error: BITBUCKET_AUTH_HEADER is not set.")
def get_last_successful_commit(module):
if os.getenv("BITBUCKET_CI") != "true":
rc, result = run_command("git rev-parse HEAD", log_output=True, log_cmd=True, log_prefix=f"[{module}]")
return result
for commit in subprocess.check_output(["git", "log", "--format=%h", "-n", "30"]).decode().split():
url = f"https://api.bitbucket.org/2.0/repositories/{BITBUCKET_REPO_OWNER}/{BITBUCKET_REPO_SLUG}/commit/{commit}/statuses/"
LOGGER.info(f"commit={commit}, url={url}")
basic_url_headers = {
BITBUCKET_AUTH_HEADER.split(": ")[0]: BITBUCKET_AUTH_HEADER.split(": ")[1]
}
response = requests.get(url, headers=basic_url_headers).json()
LOGGER.debug(f"response={response}")
for item in response.get('values', []):
name = item['name']
if "security/snyk" in name or "iacbot" in name:
continue
commit_state = item['state']
ref_name = item['refname']
updated_on = item['updated_on']
LOGGER.info(
f"COMMIT={commit}, COMMIT_STATE={commit_state}, REF_NAME={ref_name}, NAME={name}, UPDATED_ON={updated_on}")
if commit_state == "SUCCESSFUL" and ref_name.startswith(BB_PIPELINE_BRANCH):
return commit
return commit
def get_changes_output(module, last_successful_commit):
if os.getenv("BITBUCKET_CI") == "true":
_, result = run_command(f"git diff --dirstat=files,0 {last_successful_commit} | sed -E 's/^[ 0-9.]+% //g'",
log_output=True, log_cmd=True, log_prefix=f"[{module}]")
return result
else:
_, result = run_command("git diff --dirstat=files,0 HEAD~1 | sed -E 's/^[ 0-9.]+% //g'",
log_output=True, log_cmd=True, log_prefix=f"[{module}]")
return result
def is_deploy_module(module, dir_prefix):
check_bitbucket_auth_header()
if os.getenv("IS_DEPLOY_ALL") == "true":
return True
if os.getenv("SKIP_DEPLOY"):
return False
if module == os.getenv("MODULE_NAME"):
return True
elif os.getenv("MODULE_NAME"):
return False
changes_output = get_changes_output(module, get_last_successful_commit(module))
LOGGER.info(f"changes_output={changes_output}")
special_checks = [
("textract-pipeline/terraform", "textract-pipeline/src"),
]
for mod, pattern in special_checks:
pattern_found = any(pattern in change for change in changes_output)
if module == mod and pattern_found:
LOGGER.info(decorate_warn(f"Marking {module} as changed based on the special pattern={pattern}"))
return True
if f"{dir_prefix}{module}/" in changes_output:
return True
return False
requirements.txt
+1
View File
@@ -0,0 +1 @@
requests
streamlit-server/main.tf
-3
View File
@@ -220,6 +220,3 @@ resource "aws_instance" "streamlit_server" {
Environment = var.environment
}
}
streamlit/interface_2.py
+1 -1
View File
@@ -196,7 +196,7 @@ if client:
fields = fields[fields['PRIORITY'] == 'E']
if st.button("Show Results"):
query = 'select * from "DOCZY_PIPELINE_RAW_OUTPUT"'
query = 'select * from "DOCZY_PIPELINE_RAW_OUTPUT_AC"'
cur.execute(query)
df2 = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description])
streamlit/multipage/pages/Interface_2.py
+1 -1
View File
@@ -196,7 +196,7 @@ if client:
fields = fields[fields['PRIORITY'] == 'E']
if st.button("Show Results"):
query = 'select * from "DOCZY_PIPELINE_RAW_OUTPUT"'
query = 'select * from "DOCZY_PIPELINE_RAW_OUTPUT_AC"'
cur.execute(query)
df2 = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description])
streamlit/security.py
+1 -1
View File
@@ -57,7 +57,7 @@ app = msal.ConfidentialClientApplication(CLIENT_ID, authority=AUTHORITY, client_
def get_auth_url(REDIRECT_URI):
auth_url = app.get_authorization_request_url(SCOPE, redirect_uri=REDIRECT_URI)
auth_url = app.get_authorization_request_url(SCOPE, redirect_uri=REDIRECT_URI )
return auth_url
@st.cache_data
terraform.py
+11 -3
View File
@@ -1,4 +1,5 @@
from terminal import run_command, prepare_logger
from terminal import run_command, prepare_logger, decorate_white_bold, decorate_warn
from git_diff_changes import is_deploy_module
import os
import argparse
@@ -99,7 +100,13 @@ tf_root_module_path = os.getenv('TF_ROOT_MODULE_PATH') if not module_name else m
def do_terraform(stack_path):
LOGGER.info(f"stack_path={stack_path}")
LOGGER.info(f"[{stack_path}]: stack_path={stack_path}")
is_deploy_module_path = is_deploy_module(stack_path, '')
LOGGER.info(f"[{stack_path}]: is_deploy_module_path={decorate_white_bold(is_deploy_module_path)}")
if not is_deploy_module_path:
LOGGER.info(decorate_warn(f"[{stack_path}]: Module is not found in git changes. Skipping deploy."))
return True, "OK"
absolute_path = change_dir(stack_path)
# TF INIT
state_environment = "dev-new" if environment == "dev" else environment
@@ -224,4 +231,5 @@ if __name__ == "__main__":
paths = find_paths_with_file('provider.tf')
for path in paths:
LOGGER.warning(f"Discovered: {path}")
execute_with_dependencies(paths)
# TODO finish later if we want multi modulue deploy in one go
# execute_with_dependencies(paths)
textract-pipeline/terraform/main.tf
+26 -6
View File
@@ -7,11 +7,6 @@ terraform {
}
backend "s3" {
bucket = "doczyai-use2-d-infra-s3-terraform-state" # Parameterize using -backend-config flag with "terraform init"
key = "terraform/textract-pipeline/terraform.tfstate" # Parameterize
region = "us-east-2" # Parameterize
# profile = "doczyai" # Parameterize
dynamodb_table = "doczyai-use2-d-infra-dyd-terraform-lock" # Parameterize
encrypt = true
}
}
@@ -66,7 +61,11 @@ locals {
Environment = var.environment
Terraform = "true"
}
s3_prefix = "${local.global_prefix}-s3"
processing_s3_bucket_name = "${local.s3_prefix}-textract-processing-001"
}
provider "aws" {
default_tags {
@@ -80,7 +79,7 @@ module "textract_pipeline" {
# for each for multiple resources
for_each = var.client_list
client_name = each.value.client_name
client_name = each.value.client_id
aws_account_id = var.aws_account_id
@@ -101,6 +100,27 @@ module "textract_pipeline" {
text_creation_lambda_additional_layer_arn = var.text_creation_lambda_additional_layer_arn
}
resource "null_resource" "snowsql_client_create" {
# for each for multiple resources
for_each = var.client_list
triggers = {
is_changed = uuid()
}
provisioner "local-exec" {
environment = {
}
# doczyai-use2-u-cn1-s3-textract-processing-001
# doczyai-use2-d-infra-s3-textract-processing-001
# doczyai-use2-d-cn1-s3-textract-processing-001
command = "./snowsql-cicd.sh dev ${each.value.client_id} '${each.value.full_name}' ${replace(local.processing_s3_bucket_name, "infra", each.value.client_id)}"
}
}
# Create lambda_role
resource "aws_iam_role" "lambda_role" {
name = local.lambda_role_name
textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf
+6 -3
View File
@@ -909,7 +909,8 @@ resource "aws_api_gateway_method" "create_batch_method" {
rest_api_id = "${aws_api_gateway_rest_api.project_api.id}"
resource_id = "${aws_api_gateway_resource.create_batch.id}"
http_method = "POST"
authorization = "AWS_IAM"
#authorization = "AWS_IAM"
authorization = "NONE"
depends_on = [
aws_api_gateway_rest_api.project_api,
@@ -997,7 +998,8 @@ resource "aws_api_gateway_method" "s3_folder_detail_method" {
rest_api_id = "${aws_api_gateway_rest_api.project_api.id}"
resource_id = "${aws_api_gateway_resource.s3_folder_detail.id}"
http_method = "POST"
authorization = "AWS_IAM"
#authorization = "AWS_IAM"
authorization = "NONE"
}
# API Gateway - s3 folder detail - create method response
resource "aws_api_gateway_method_response" "s3_folder_detail_method_response_200" {
@@ -1059,7 +1061,8 @@ resource "aws_api_gateway_method" "trigger_pipeline_method" {
rest_api_id = "${aws_api_gateway_rest_api.project_api.id}"
resource_id = "${aws_api_gateway_resource.trigger_pipeline.id}"
http_method = "POST"
authorization = "AWS_IAM"
#authorization = "AWS_IAM"
authorization = "NONE"
}
# API Gateway - trigger pipeline - create method response
resource "aws_api_gateway_method_response" "trigger_pipeline_method_response_200" {
textract-pipeline/terraform/snowsql-cicd.sh
Executable
+53
View File
@@ -0,0 +1,53 @@
#!/usr/bin/env bash
set -euo pipefail
ENV_GROUP="$1"
CLIENT_ID="$2"
FULL_NAME="$3"
BUCKET_NAME="$4"
echo "ENV_GROUP=${ENV_GROUP}, CLIENT_ID=${CLIENT_ID}, FULL_NAME=${FULL_NAME}, BUCKET_NAME=${BUCKET_NAME}"
readonly SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
echo "# Getting snowflake creds from AWS Secret Manager..."
snowflake_secret_name="doczy-${ENV_GROUP}-db-svc-acc"
snowflake_secret_value=$(aws secretsmanager get-secret-value --secret-id "$snowflake_secret_name" --query 'SecretString' --output text)
SNOWFLAKE_ACCOUNT=$(echo $snowflake_secret_value | jq -r '.account_locator')
SNOWFLAKE_DATABASE=$(echo $snowflake_secret_value | jq -r '.database')
SNOWFLAKE_USER=$(echo $snowflake_secret_value | jq -r '.user')
SNOWFLAKE_ROLE=$(echo $snowflake_secret_value | jq -r '.role')
SNOWFLAKE_PASSWORD=$(echo $snowflake_secret_value | jq -r '.password')
echo "SNOWFLAKE_ACCOUNT=${SNOWFLAKE_ACCOUNT}"
echo "SNOWFLAKE_DATABASE=${SNOWFLAKE_DATABASE}"
echo "SNOWFLAKE_USER=${SNOWFLAKE_USER}"
echo "SNOWFLAKE_ROLE=${SNOWFLAKE_ROLE}"
export SNOWSQL_PWD=${SNOWFLAKE_PASSWORD}
#snowsql -a "${SNOWFLAKE_ACCOUNT}" -u "${SNOWFLAKE_USER}" -d "${SNOWFLAKE_DATABASE}" -r "${SNOWFLAKE_ROLE}" -o exit_on_error=true -f "${SCRIPT_DIR}/snowsql-query.sql"
#QUERY="INSERT INTO STG.CLIENT_LOGS(CLIENT_ID, CLIENT_NAME, BUCKET_NAME) VALUES ('${CLIENT_ID}','${FULL_NAME}','${BUCKET_NAME}')"
#echo "QUERY=${QUERY}"
#snowsql -a "${SNOWFLAKE_ACCOUNT}" -u "${SNOWFLAKE_USER}" -d "${SNOWFLAKE_DATABASE}" -r "${SNOWFLAKE_ROLE}" -o exit_on_error=true -q "${QUERY}"
INSERT_QUERY=$(cat <<EOF
INSERT INTO STG.CLIENT_LOGS (CLIENT_ID, CLIENT_NAME, BUCKET_NAME)
SELECT '${CLIENT_ID}', '${FULL_NAME}', '${BUCKET_NAME}'
WHERE NOT EXISTS (
SELECT 1
FROM STG.CLIENT_LOGS
WHERE CLIENT_ID = '${CLIENT_ID}'
);
EOF
)
echo "Client=${CLIENT_ID}: INSERT_QUERY=${INSERT_QUERY}"
snowsql -a "${SNOWFLAKE_ACCOUNT}" -u "${SNOWFLAKE_USER}" -d "${SNOWFLAKE_DATABASE}" -r "${SNOWFLAKE_ROLE}" -o exit_on_error=true -q "${INSERT_QUERY}"
# SHOW THE VALUES
SELECT_QUERY="SELECT DISTINCT client_id, client_name, bucket_name FROM STG.CLIENT_LOGS"
echo "Client=${CLIENT_ID}: SELECT_QUERY=${SELECT_QUERY}"
snowsql -a "${SNOWFLAKE_ACCOUNT}" -u "${SNOWFLAKE_USER}" -d "${SNOWFLAKE_DATABASE}" -r "${SNOWFLAKE_ROLE}" -o exit_on_error=true -q "${SELECT_QUERY}"
textract-pipeline/terraform/variables.tf
+2 -6
View File
@@ -17,13 +17,9 @@ variable "environment" {
variable "client_list" {
type = map(object({
client_name = string
client_id = string
full_name = string
}))
default = {
client1 = {
client_name = "cn1"
}
}
}
variable "aws_account_id" {
textract-pipeline/terraform/vars-dev.tfvars
+8 -1
View File
@@ -1,2 +1,9 @@
aws_account_id = 660131068782
text_creation_lambda_additional_layer_arn = "arn:aws:lambda:us-east-2:336392948345:layer:AWSSDKPandas-Python312:8"
text_creation_lambda_additional_layer_arn = "arn:aws:lambda:us-east-2:336392948345:layer:AWSSDKPandas-Python312:8"
client_list = {
client1 = {
client_id = "cn1"
full_name = "Centene"
},
}
textract-pipeline/terraform/vars-uat.tfvars
+7
View File
@@ -1,3 +1,10 @@
aws_account_id = 975049960860
# TODO provide new layer arn for UAT? 336392948345 was used in DEV but not sure what account that is?!
text_creation_lambda_additional_layer_arn = "arn:aws:lambda:us-east-2:336392948345:layer:AWSSDKPandas-Python312:8"
client_list = {
client1 = {
client_id = "cn1"
full_name = "Centene"
}
}