diff --git a/.gitignore b/.gitignore index e49dd36..dd5df53 100644 --- a/.gitignore +++ b/.gitignore @@ -71,3 +71,6 @@ streamlit/results.csv # env streamlit/venv + +*.tfplan +*.pem \ No newline at end of file diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index ba20121..e01bf18 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -121,84 +121,56 @@ pipelines: - airflow/dags/* - step: name: Apply textract-pipeline on DEV - image: hugree/bitbucket-aws-python38-tf154:latest + image: hugree/terraform-with-snowsql:latest oidc: true script: - *aws-context-dev + - pip install -r requirements.txt - python terraform.py apply --environment=dev --module=textract-pipeline/terraform - condition: - changesets: - includePaths: - - textract-pipeline/src/* - - textract-pipeline/src/**/* - - textract-pipeline/terraform/* - - textract-pipeline/terraform/**/* - step: name: Apply devops-pipeline on DEV image: hugree/bitbucket-aws-python38-tf154:latest oidc: true script: - *aws-context-dev + - pip install -r requirements.txt - python terraform.py apply --environment=dev --module=devops-pipeline/other-resources - condition: - changesets: - includePaths: - - devops-pipeline/other-resources/* - - devops-pipeline/other-resources/**/* - step: name: Apply streamlit-server on DEV image: hugree/bitbucket-aws-python38-tf154:latest oidc: true script: - *aws-context-dev + - pip install -r requirements.txt - python terraform.py apply --environment=dev --module=streamlit-server - condition: - changesets: - includePaths: - - streamlit-server/* - - streamlit-server/**/* # UAT and PROD pipelines will be updated with Streamlit steps once it is tested and ready UAT: - step: name: Apply textract-pipeline on UAT - image: hugree/bitbucket-aws-python38-tf154:latest + image: hugree/terraform-with-snowsql:latest oidc: true script: - *aws-context-uat + - pip install -r requirements.txt - python terraform.py apply --environment=uat --module=textract-pipeline/terraform - condition: - changesets: - includePaths: - - textract-pipeline/src/* - - textract-pipeline/src/**/* - - textract-pipeline/terraform/* - - textract-pipeline/terraform/**/* - step: name: Apply devops-pipeline on UAT image: hugree/bitbucket-aws-python38-tf154:latest oidc: true script: - *aws-context-uat + - pip install -r requirements.txt - python terraform.py apply --environment=uat --module=devops-pipeline/other-resources - condition: - changesets: - includePaths: - - devops-pipeline/other-resources/* - - devops-pipeline/other-resources/**/* - step: name: Apply streamlit-server on UAT image: hugree/bitbucket-aws-python38-tf154:latest oidc: true script: - *aws-context-uat + - pip install -r requirements.txt - python terraform.py apply --environment=uat --module=streamlit-server - condition: - changesets: - includePaths: - - streamlit-server/* - - streamlit-server/**/* - step: <<: *snowflake_deploy condition: @@ -239,7 +211,7 @@ pipelines: - snowflake/**/*.sql - feature/terraform-refactor: + feature/DOC-536-smart-git-diff: - parallel: - step: name: Plan devops-pipeline on DEV @@ -247,39 +219,66 @@ pipelines: oidc: true script: - *aws-context-dev - - python terraform.py plan --environment=dev --module=devops-pipeline/other-resources + - pip install -r requirements.txt + - BB_PIPELINE_BRANCH="DEV" python terraform.py plan --environment=dev --module=devops-pipeline/other-resources - step: name: Plan streamlit-server on DEV image: hugree/bitbucket-aws-python38-tf154:latest oidc: true script: - *aws-context-dev - - python terraform.py plan --environment=dev --module=streamlit-server + - pip install -r requirements.txt + - BB_PIPELINE_BRANCH="DEV" python terraform.py plan --environment=dev --module=streamlit-server - step: name: Plan textract-pipeline on DEV - image: hugree/bitbucket-aws-python38-tf154:latest + image: hugree/terraform-with-snowsql:latest oidc: true script: - *aws-context-dev - - python terraform.py plan --environment=dev --module=textract-pipeline/terraform + - pip install -r requirements.txt + - BB_PIPELINE_BRANCH="DEV" python terraform.py plan --environment=dev --module=textract-pipeline/terraform - step: name: Plan devops-pipeline on UAT image: hugree/bitbucket-aws-python38-tf154:latest oidc: true script: - *aws-context-uat - - python terraform.py plan --environment=uat --module=devops-pipeline/other-resources + - pip install -r requirements.txt + - BB_PIPELINE_BRANCH="UAT" python terraform.py plan --environment=uat --module=devops-pipeline/other-resources - step: name: Plan streamlit-server on UAT image: hugree/bitbucket-aws-python38-tf154:latest oidc: true script: - *aws-context-uat - - python terraform.py plan --environment=uat --module=streamlit-server + - pip install -r requirements.txt + - BB_PIPELINE_BRANCH="UAT" python terraform.py plan --environment=uat --module=streamlit-server - step: name: Plan textract-pipeline on UAT image: hugree/bitbucket-aws-python38-tf154:latest oidc: true script: - *aws-context-uat - - python terraform.py plan --environment=uat --module=textract-pipeline/terraform + - pip install -r requirements.txt + - BB_PIPELINE_BRANCH="UAT" python terraform.py plan --environment=uat --module=textract-pipeline/terraform + + + feature/DOC-535-snowsql-in-cicd: + - parallel: + - step: + image: hugree/terraform-with-snowsql:latest + name: Run Snowsql on DEV + oidc: true + script: + - *aws-context-dev + - pip install -r requirements.txt + - IS_DEPLOY_ALL=true python terraform.py apply --environment=dev --module=textract-pipeline/terraform --apply-extra-args="-target=null_resource.snowsql_client_create" + + - step: + image: hugree/terraform-with-snowsql:latest + name: Run Snowsql on UAT + oidc: true + script: + - *aws-context-uat + - pip install -r requirements.txt + - IS_DEPLOY_ALL=true python terraform.py apply --environment=uat --module=textract-pipeline/terraform --apply-extra-args="-target=null_resource.snowsql_client_create" diff --git a/git_diff_changes.py b/git_diff_changes.py new file mode 100644 index 0000000..948c28a --- /dev/null +++ b/git_diff_changes.py @@ -0,0 +1,90 @@ +import os +import subprocess +import requests + +BITBUCKET_REPO_OWNER = "aarete" +BITBUCKET_REPO_SLUG = "doczy.ai" +BB_PIPELINE_BRANCH = os.getenv("BB_PIPELINE_BRANCH", "DEV") +BITBUCKET_AUTH_HEADER = os.getenv("BITBUCKET_AUTH_HEADER") + +from terminal import run_command, prepare_logger, decorate_warn + +LOGGER = prepare_logger() + + +def check_bitbucket_auth_header(): + if os.getenv("BITBUCKET_CI") == "true" and not BITBUCKET_AUTH_HEADER: + raise EnvironmentError("Error: BITBUCKET_AUTH_HEADER is not set.") + + +def get_last_successful_commit(module): + if os.getenv("BITBUCKET_CI") != "true": + rc, result = run_command("git rev-parse HEAD", log_output=True, log_cmd=True, log_prefix=f"[{module}]") + return result + + for commit in subprocess.check_output(["git", "log", "--format=%h", "-n", "30"]).decode().split(): + url = f"https://api.bitbucket.org/2.0/repositories/{BITBUCKET_REPO_OWNER}/{BITBUCKET_REPO_SLUG}/commit/{commit}/statuses/" + LOGGER.info(f"commit={commit}, url={url}") + basic_url_headers = { + BITBUCKET_AUTH_HEADER.split(": ")[0]: BITBUCKET_AUTH_HEADER.split(": ")[1] + } + response = requests.get(url, headers=basic_url_headers).json() + LOGGER.debug(f"response={response}") + + for item in response.get('values', []): + name = item['name'] + if "security/snyk" in name or "iacbot" in name: + continue + commit_state = item['state'] + ref_name = item['refname'] + updated_on = item['updated_on'] + + LOGGER.info( + f"COMMIT={commit}, COMMIT_STATE={commit_state}, REF_NAME={ref_name}, NAME={name}, UPDATED_ON={updated_on}") + + if commit_state == "SUCCESSFUL" and ref_name.startswith(BB_PIPELINE_BRANCH): + return commit + + return commit + + +def get_changes_output(module, last_successful_commit): + if os.getenv("BITBUCKET_CI") == "true": + _, result = run_command(f"git diff --dirstat=files,0 {last_successful_commit} | sed -E 's/^[ 0-9.]+% //g'", + log_output=True, log_cmd=True, log_prefix=f"[{module}]") + return result + else: + _, result = run_command("git diff --dirstat=files,0 HEAD~1 | sed -E 's/^[ 0-9.]+% //g'", + log_output=True, log_cmd=True, log_prefix=f"[{module}]") + return result + + +def is_deploy_module(module, dir_prefix): + check_bitbucket_auth_header() + if os.getenv("IS_DEPLOY_ALL") == "true": + return True + if os.getenv("SKIP_DEPLOY"): + return False + if module == os.getenv("MODULE_NAME"): + return True + elif os.getenv("MODULE_NAME"): + return False + + changes_output = get_changes_output(module, get_last_successful_commit(module)) + + LOGGER.info(f"changes_output={changes_output}") + + special_checks = [ + ("textract-pipeline/terraform", "textract-pipeline/src"), + ] + + for mod, pattern in special_checks: + pattern_found = any(pattern in change for change in changes_output) + if module == mod and pattern_found: + LOGGER.info(decorate_warn(f"Marking {module} as changed based on the special pattern={pattern}")) + return True + + if f"{dir_prefix}{module}/" in changes_output: + return True + + return False diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..f229360 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +requests diff --git a/streamlit-server/main.tf b/streamlit-server/main.tf index 3e0540b..0213521 100644 --- a/streamlit-server/main.tf +++ b/streamlit-server/main.tf @@ -220,6 +220,3 @@ resource "aws_instance" "streamlit_server" { Environment = var.environment } } - - - diff --git a/streamlit/interface_2.py b/streamlit/interface_2.py index ec38e22..26a5e88 100644 --- a/streamlit/interface_2.py +++ b/streamlit/interface_2.py @@ -196,7 +196,7 @@ if client: fields = fields[fields['PRIORITY'] == 'E'] if st.button("Show Results"): - query = 'select * from "DOCZY_PIPELINE_RAW_OUTPUT"' + query = 'select * from "DOCZY_PIPELINE_RAW_OUTPUT_AC"' cur.execute(query) df2 = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description]) diff --git a/streamlit/multipage/pages/Interface_2.py b/streamlit/multipage/pages/Interface_2.py index 7323247..85a8a81 100644 --- a/streamlit/multipage/pages/Interface_2.py +++ b/streamlit/multipage/pages/Interface_2.py @@ -196,7 +196,7 @@ if client: fields = fields[fields['PRIORITY'] == 'E'] if st.button("Show Results"): - query = 'select * from "DOCZY_PIPELINE_RAW_OUTPUT"' + query = 'select * from "DOCZY_PIPELINE_RAW_OUTPUT_AC"' cur.execute(query) df2 = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description]) diff --git a/streamlit/security.py b/streamlit/security.py index d759570..d7465d2 100644 --- a/streamlit/security.py +++ b/streamlit/security.py @@ -57,7 +57,7 @@ app = msal.ConfidentialClientApplication(CLIENT_ID, authority=AUTHORITY, client_ def get_auth_url(REDIRECT_URI): - auth_url = app.get_authorization_request_url(SCOPE, redirect_uri=REDIRECT_URI) + auth_url = app.get_authorization_request_url(SCOPE, redirect_uri=REDIRECT_URI ) return auth_url @st.cache_data diff --git a/terraform.py b/terraform.py index 0d6faaf..3ab19f9 100644 --- a/terraform.py +++ b/terraform.py @@ -1,4 +1,5 @@ -from terminal import run_command, prepare_logger +from terminal import run_command, prepare_logger, decorate_white_bold, decorate_warn +from git_diff_changes import is_deploy_module import os import argparse @@ -99,7 +100,13 @@ tf_root_module_path = os.getenv('TF_ROOT_MODULE_PATH') if not module_name else m def do_terraform(stack_path): - LOGGER.info(f"stack_path={stack_path}") + LOGGER.info(f"[{stack_path}]: stack_path={stack_path}") + is_deploy_module_path = is_deploy_module(stack_path, '') + LOGGER.info(f"[{stack_path}]: is_deploy_module_path={decorate_white_bold(is_deploy_module_path)}") + if not is_deploy_module_path: + LOGGER.info(decorate_warn(f"[{stack_path}]: Module is not found in git changes. Skipping deploy.")) + return True, "OK" + absolute_path = change_dir(stack_path) # TF INIT state_environment = "dev-new" if environment == "dev" else environment @@ -224,4 +231,5 @@ if __name__ == "__main__": paths = find_paths_with_file('provider.tf') for path in paths: LOGGER.warning(f"Discovered: {path}") - execute_with_dependencies(paths) + # TODO finish later if we want multi modulue deploy in one go + # execute_with_dependencies(paths) diff --git a/textract-pipeline/terraform/main.tf b/textract-pipeline/terraform/main.tf index d3f8114..15d3f1b 100644 --- a/textract-pipeline/terraform/main.tf +++ b/textract-pipeline/terraform/main.tf @@ -7,11 +7,6 @@ terraform { } backend "s3" { - bucket = "doczyai-use2-d-infra-s3-terraform-state" # Parameterize using -backend-config flag with "terraform init" - key = "terraform/textract-pipeline/terraform.tfstate" # Parameterize - region = "us-east-2" # Parameterize - # profile = "doczyai" # Parameterize - dynamodb_table = "doczyai-use2-d-infra-dyd-terraform-lock" # Parameterize encrypt = true } } @@ -66,7 +61,11 @@ locals { Environment = var.environment Terraform = "true" } + + s3_prefix = "${local.global_prefix}-s3" + processing_s3_bucket_name = "${local.s3_prefix}-textract-processing-001" } + provider "aws" { default_tags { @@ -80,7 +79,7 @@ module "textract_pipeline" { # for each for multiple resources for_each = var.client_list - client_name = each.value.client_name + client_name = each.value.client_id aws_account_id = var.aws_account_id @@ -101,6 +100,27 @@ module "textract_pipeline" { text_creation_lambda_additional_layer_arn = var.text_creation_lambda_additional_layer_arn } +resource "null_resource" "snowsql_client_create" { + + # for each for multiple resources + for_each = var.client_list + + triggers = { + is_changed = uuid() + } + + provisioner "local-exec" { + environment = { + } +# doczyai-use2-u-cn1-s3-textract-processing-001 +# doczyai-use2-d-infra-s3-textract-processing-001 +# doczyai-use2-d-cn1-s3-textract-processing-001 + + command = "./snowsql-cicd.sh dev ${each.value.client_id} '${each.value.full_name}' ${replace(local.processing_s3_bucket_name, "infra", each.value.client_id)}" + } +} + + # Create lambda_role resource "aws_iam_role" "lambda_role" { name = local.lambda_role_name diff --git a/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf b/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf index 51534a4..00623f5 100644 --- a/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf +++ b/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf @@ -909,7 +909,8 @@ resource "aws_api_gateway_method" "create_batch_method" { rest_api_id = "${aws_api_gateway_rest_api.project_api.id}" resource_id = "${aws_api_gateway_resource.create_batch.id}" http_method = "POST" - authorization = "AWS_IAM" + #authorization = "AWS_IAM" + authorization = "NONE" depends_on = [ aws_api_gateway_rest_api.project_api, @@ -997,7 +998,8 @@ resource "aws_api_gateway_method" "s3_folder_detail_method" { rest_api_id = "${aws_api_gateway_rest_api.project_api.id}" resource_id = "${aws_api_gateway_resource.s3_folder_detail.id}" http_method = "POST" - authorization = "AWS_IAM" + #authorization = "AWS_IAM" + authorization = "NONE" } # API Gateway - s3 folder detail - create method response resource "aws_api_gateway_method_response" "s3_folder_detail_method_response_200" { @@ -1059,7 +1061,8 @@ resource "aws_api_gateway_method" "trigger_pipeline_method" { rest_api_id = "${aws_api_gateway_rest_api.project_api.id}" resource_id = "${aws_api_gateway_resource.trigger_pipeline.id}" http_method = "POST" - authorization = "AWS_IAM" + #authorization = "AWS_IAM" + authorization = "NONE" } # API Gateway - trigger pipeline - create method response resource "aws_api_gateway_method_response" "trigger_pipeline_method_response_200" { diff --git a/textract-pipeline/terraform/snowsql-cicd.sh b/textract-pipeline/terraform/snowsql-cicd.sh new file mode 100755 index 0000000..1f854be --- /dev/null +++ b/textract-pipeline/terraform/snowsql-cicd.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ENV_GROUP="$1" +CLIENT_ID="$2" +FULL_NAME="$3" +BUCKET_NAME="$4" + +echo "ENV_GROUP=${ENV_GROUP}, CLIENT_ID=${CLIENT_ID}, FULL_NAME=${FULL_NAME}, BUCKET_NAME=${BUCKET_NAME}" + +readonly SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" + +echo "# Getting snowflake creds from AWS Secret Manager..." +snowflake_secret_name="doczy-${ENV_GROUP}-db-svc-acc" +snowflake_secret_value=$(aws secretsmanager get-secret-value --secret-id "$snowflake_secret_name" --query 'SecretString' --output text) + +SNOWFLAKE_ACCOUNT=$(echo $snowflake_secret_value | jq -r '.account_locator') +SNOWFLAKE_DATABASE=$(echo $snowflake_secret_value | jq -r '.database') +SNOWFLAKE_USER=$(echo $snowflake_secret_value | jq -r '.user') +SNOWFLAKE_ROLE=$(echo $snowflake_secret_value | jq -r '.role') + +SNOWFLAKE_PASSWORD=$(echo $snowflake_secret_value | jq -r '.password') + +echo "SNOWFLAKE_ACCOUNT=${SNOWFLAKE_ACCOUNT}" +echo "SNOWFLAKE_DATABASE=${SNOWFLAKE_DATABASE}" +echo "SNOWFLAKE_USER=${SNOWFLAKE_USER}" +echo "SNOWFLAKE_ROLE=${SNOWFLAKE_ROLE}" +export SNOWSQL_PWD=${SNOWFLAKE_PASSWORD} + +#snowsql -a "${SNOWFLAKE_ACCOUNT}" -u "${SNOWFLAKE_USER}" -d "${SNOWFLAKE_DATABASE}" -r "${SNOWFLAKE_ROLE}" -o exit_on_error=true -f "${SCRIPT_DIR}/snowsql-query.sql" + +#QUERY="INSERT INTO STG.CLIENT_LOGS(CLIENT_ID, CLIENT_NAME, BUCKET_NAME) VALUES ('${CLIENT_ID}','${FULL_NAME}','${BUCKET_NAME}')" +#echo "QUERY=${QUERY}" +#snowsql -a "${SNOWFLAKE_ACCOUNT}" -u "${SNOWFLAKE_USER}" -d "${SNOWFLAKE_DATABASE}" -r "${SNOWFLAKE_ROLE}" -o exit_on_error=true -q "${QUERY}" + +INSERT_QUERY=$(cat <