jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

test new terraform script

Browse Source
This commit is contained in:
Grzegorz Huber
2024-06-06 09:54:14 +02:00
parent 80d2a6f733
commit 7fe7612f4b
7 changed files with 391 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bitbucket-pipelines.yml
+16
View File
@@ -291,3 +291,19 @@ pipelines:
condition:
paths:
- snowflake/**/*.sql
feature/terraform-refactor:
- parallel:
- step:
name: Plan streamlit-server on DEV
oidc: true
script:
- *aws-context-dev
- python terraform.py plan --environment=dev --module=streamlit-server
- step:
name: Plan streamlit-server on UAT
oidc: true
script:
- *aws-context-uat
- python terraform.py plan --environment=uat --module=streamlit-server
streamlit-server/main.tf
+8 -13
View File
@@ -1,5 +1,5 @@
provider "aws" {
region = "us-east-2"
region = "us-east-2"
}
terraform {
@@ -11,11 +11,6 @@ terraform {
}
backend "s3" {
# bucket = "doczyai-use2-d-infra-s3-terraform-state" # Parameterize using -backend-config flag with "terraform init"
# key = "terraform/streamlit-server/terraform.tfstate" # Parameterize
# region = "us-east-2" # Parameterize
# profile = "temp_cred" # Parameterize
# dynamodb_table = "doczyai-use2-d-infra-dyd-terraform-lock" # Parameterize
encrypt = true
}
}
@@ -43,7 +38,7 @@ locals {
# global prefix
global_prefix = "${var.project_name}-${local.region_short_name}-${local.environment_short_name}"
# Resource prefix
iam_policy_prefix = "${local.global_prefix}-pol"
@@ -65,14 +60,14 @@ data "aws_subnets" "subnets" {
values = [var.vpc_id]
}
}
# vpc security groups
data "aws_security_groups" "security_groups" {
filter {
name = "group-name"
values = ["*default*"]
}
filter {
name = "vpc-id"
values = [var.vpc_id]
@@ -89,8 +84,8 @@ resource "aws_instance" "streamlit_server" {
ami = var.ubuntu_ami
instance_type = var.ec2_instance_type
subnet_id = data.aws_subnets.subnets.ids[0]
vpc_security_group_ids = data.aws_security_groups.security_groups.ids
iam_instance_profile = aws_iam_instance_profile.ec2_instance_profile.name
vpc_security_group_ids = data.aws_security_groups.security_groups.ids
iam_instance_profile = aws_iam_instance_profile.ec2_instance_profile.name
associate_public_ip_address = false
key_name = "tf-test-key" # Created using TF and uploaded to S3. Prerequisite
@@ -98,8 +93,8 @@ resource "aws_instance" "streamlit_server" {
volume_size = 30 # Variable
encrypted = true # Mandatory
}
# Setup SSH authentication for bitbucket access to pull code
# Setup SSL certificate for HTTPS access for SSO to work
# Setup SSH authentication for bitbucket access to pull code
# Setup SSL certificate for HTTPS access for SSO to work
user_data = <<-EOF
#!/bin/bash
echo '${file("${path.module}/requirements.txt")}' > /tmp/requirements.txt
streamlit-server/variables.tf
+13 -13
View File
@@ -59,16 +59,16 @@ variable "acm_arn_dev" {
default = "arn:aws:acm:us-east-2:660131068782:certificate/947e04b3-be22-43bf-bea2-a0ad3b221d70"
}
variable "secret_key" {
type = string
}
# required
variable "access_key" {
type = string
}
# required
variable "token" {
type = string
}
#variable "secret_key" {
# type = string
#}
#
## required
#variable "access_key" {
# type = string
#}
#
## required
#variable "token" {
# type = string
#}
terminal.py
+125
View File
@@ -0,0 +1,125 @@
#!/usr/bin/env python3
import os
import re
import subprocess
import logging
import signal
from typing import Callable, Optional, Dict, Tuple, List
LOGGER = logging.getLogger(__name__)
LOGGER.setLevel(logging.INFO)
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
LOGGER.addHandler(handler)
class SensitiveFormatter(logging.Formatter):
"""Formatter that removes sensitive information from logs."""
@staticmethod
def mask_password(log: str) -> str:
return re.sub(r'password=([^\s]+)', r'password=*****', log)
@staticmethod
def mask_api_key(log: str) -> str:
return re.sub(r'api_key=([^\s]+)', r'api_key=*****', log)
@staticmethod
def _mask(s: str) -> str:
filtered = SensitiveFormatter.mask_password(s)
filtered = SensitiveFormatter.mask_api_key(filtered)
return filtered
def format(self, record) -> str:
original = super().format(record)
return self._mask(original)
def prepare_logger() -> logging.Logger:
global LOGGER
if LOGGER is not None:
return LOGGER
LOGGER = logging.getLogger(__name__)
LOGGER.setLevel(logging.INFO)
log_format = '%(asctime)s %(filename)s:%(lineno)-4s [%(levelname)s] %(message)s'
handler = logging.StreamHandler()
handler.setFormatter(SensitiveFormatter(log_format))
LOGGER.addHandler(handler)
return LOGGER
def get_blue_shade(log_prefix: str) -> str:
"""Returns an ANSI color code for a shade of blue based on the hash of the log_prefix."""
blue_shades = [81, 87, 117, 153, 159, 195, 111, 45, 39]
hash_value = hash(log_prefix)
blue_index = hash_value % len(blue_shades)
return f"\033[38;5;{blue_shades[blue_index]}m"
def signal_handler(sig, frame, process):
print("Ctrl+C pressed! Sending SIGTERM to Terraform process...")
process.terminate()
def run_command(command: str, log_output: bool = False, decorate_logs: bool = True, log_cmd: bool = False,
log_prefix: str = "", envs: Optional[str] = None, secrets: Optional[Dict[str, str]] = None,
failure_callback: Optional[Callable[[str], None]] = None,
cwd: Optional[str] = None) -> Tuple[int, List[str]]:
new_env = os.environ.copy()
if secrets:
for key, value in secrets.items():
new_env[key] = os.path.expandvars(value)
full_command = f"{envs} {command}" if envs else command
process = subprocess.Popen(full_command, shell=True, cwd=cwd, env=new_env,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
# Setup signal handler
# original_sigint_handler = signal.getsignal(signal.SIGINT)
# signal.signal(signal.SIGINT, lambda sig, frame: signal_handler(sig, frame, process))
log_prefix_env = os.getenv("LOG_PREFIX", "")
log_prefix_full = f"[{log_prefix_env}] {log_prefix}" if log_prefix_env else log_prefix
if log_cmd:
LOGGER.info(
f"{get_blue_shade(log_prefix_full)}{log_prefix_full}\033[0m: Running command: \033[1;34m{command}\033[0m")
if envs:
LOGGER.info(f"Using additional envs: \033[1;34m{envs}\033[0m")
if secrets:
LOGGER.info(f"Using additional environment variables with secrets: \033[1;34m{list(secrets.keys())}\033[0m")
cmd_output = []
while True:
output = process.stdout.readline() if process.stdout else ''
if output:
output_line = output.strip()
if log_output:
log_statement = (
f"{get_blue_shade(log_prefix_full)}{log_prefix_full}\033[0m: "
f"{output_line}") if log_prefix else output_line
LOGGER.info(log_statement) if decorate_logs else print(log_statement)
cmd_output.append(output_line)
elif process.poll() is not None:
break
# Restore original signal handler
# signal.signal(signal.SIGINT, original_sigint_handler)
rc = process.poll() or 0 # Ensure rc is an int, default to 0 if None
if rc != 0 and failure_callback:
failure_callback(f"Command returned code: {rc}")
return rc, cmd_output
def decorate_successful(text: str) -> str:
return f"\033[32;1m{text}\033[0m"
def decorate_white_bold(text: str) -> str:
return f"\033[37;1m{text}\033[0m"
def decorate_warn(text: str) -> str:
return f"\033[33;1m{text}\033[0m"
terraform.py
+224
View File
@@ -0,0 +1,224 @@
from terminal import run_command, prepare_logger
import os
import argparse
import sys
import subprocess
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
script_dir = os.path.dirname(os.path.abspath(__file__))
LOGGER = prepare_logger()
LOGGER.info(sys.argv)
parser = argparse.ArgumentParser(description='Run Terraform init with dynamic backend configuration.')
parser.add_argument('terraform_command', type=str, choices=['plan', 'apply'],
help='The Terraform command to execute (plan, apply, init).')
parser.add_argument('--environment', type=str, required=True, help='The environment name i.e. dev, prod.')
parser.add_argument('--module', type=str, required=False, help='The module name.')
parser.add_argument('--init-extra-args', type=str, required=False, help='Terraform init extra arguments')
parser.add_argument('--plan-extra-args', type=str, required=False, help='Terraform plan extra arguments')
parser.add_argument('--apply-extra-args', type=str, required=False, help='Terraform plan extra arguments')
parser.add_argument('--threads', type=int, default=4, help='Number of threads to use (default: 4)')
args = parser.parse_args()
raw_terraform_command = args.terraform_command
environment = args.environment
init_extra_args = args.init_extra_args
plan_extra_args = args.plan_extra_args
apply_extra_args = args.apply_extra_args
terraform_threads = args.threads
module_name = args.module
bitbucket_ci = os.getenv('BITBUCKET_CI')
def change_dir(path):
if path:
_, git_root = run_command('git rev-parse --show-toplevel', log_cmd=True, log_output=True)
LOGGER.info(f"git_root={git_root}")
absolute_path = os.path.join(git_root[0], path)
return absolute_path
# os.chdir(absolute_path)
else:
LOGGER.error("TF_ROOT_MODULE_PATH environment variable is not set.")
sys.exit(1)
def select_vars_file():
tf_vars_file = os.getenv('TFVARS_FILE_PATH', f'vars-{environment}.tfvars')
if tf_vars_file:
LOGGER.info(f"Using {tf_vars_file} as vars-file...")
else:
LOGGER.error("TFVARS_FILE_PATH variable is not set.")
sys.exit(1)
return tf_vars_file
def prepare_tf_init_command(stack_path, init_extra_args, state_environment, tf_state_bucket_name):
command = (f'terraform init --reconfigure '
f'-backend-config="dynamodb_table=doczyai-use2-{get_environment_value(environment)}-infra-dyd-terraform-lock" '
f'-backend-config="bucket=doczyai-use2-{get_environment_value(environment)}-infra-s3-terraform-state" '
f'-backend-config="key=terraform/{stack_path}/terraform.tfstate" -backend-config="region=us-east-2"')
if init_extra_args:
command = f"{command} {init_extra_args}"
return command
def get_environment_value(environment):
environment_map = {
'prod': 'p',
'uat': 'u',
'qa': 'q',
'dev': 'd'
}
return environment_map.get(environment, None) # Returns None if the environment is not found
def prepare_tf_main_command(environment, stack_path, state_environment, tf_state_bucket_name, vars_file,
plan_extra_args,
apply_extra_args):
common_command = f'-var "aws_region=us-east-2" -var "environment={environment}" -var "vpc_id=vpc-0f9d7c913f6522079"'
if raw_terraform_command == "plan":
command = f"terraform plan -out {stack_path}.{environment}.tfplan {common_command}"
if plan_extra_args:
command = f"{command} {plan_extra_args}"
elif raw_terraform_command == "apply":
command = f"terraform apply -auto-approve {common_command}"
if apply_extra_args:
command = f"{command} {apply_extra_args}"
return command
tf_root_module_path = os.getenv('TF_ROOT_MODULE_PATH') if not module_name else module_name
def do_terraform(stack_path):
LOGGER.info(f"stack_path={stack_path}")
absolute_path = change_dir(stack_path)
# TF INIT
state_environment = "dev-new" if environment == "dev" else environment
tf_state_bucket_name = f"pi2new-{state_environment}-terraform-state-file"
terraform_init_command = prepare_tf_init_command(stack_path, init_extra_args, state_environment,
tf_state_bucket_name)
rc_init, init_result = run_command(terraform_init_command, log_output=True, log_cmd=True, log_prefix=stack_path,
cwd=absolute_path)
if rc_init > 0:
raise Exception(f"{stack_path}: Failed to run terraform init")
vars_file = select_vars_file()
terraform_main_command = prepare_tf_main_command(environment, stack_path, state_environment, tf_state_bucket_name,
vars_file, plan_extra_args, apply_extra_args)
rc_main, command_result = run_command(terraform_main_command, log_output=True, log_cmd=True,
log_prefix=stack_path, cwd=absolute_path)
if rc_init or rc_main > 0:
raise RuntimeError(f"Unexpected Terraform error for command: {terraform_main_command}")
else:
return True, "OK"
# return True, "OK"
def find_paths_with_file(file_name):
try:
completed_process = subprocess.run(['git', 'ls-files', f'*{file_name}'], check=True, text=True,
stdout=subprocess.PIPE)
paths = set()
for file_path in completed_process.stdout.splitlines():
dir_path = '/'.join(file_path.split('/')[:-1])
if dir_path.startswith("modules/"):
continue
if dir_path.startswith("infra/"):
dir_path = dir_path[len("infra/"):]
if dir_path.startswith("kinesis") or dir_path.startswith("aws-transfer-sftp"):
paths.add(dir_path)
return paths
except subprocess.CalledProcessError as e:
LOGGER.error(f"Error executing git command: {e}")
return set()
def read_dependencies(path):
depends_on_file = os.path.join(path, ".depends-on.txt")
if os.path.exists(depends_on_file):
with open(depends_on_file, "r") as f:
dependencies = [line.strip() for line in f.readlines()]
return dependencies
return []
def wait_for_dependencies(dependencies, completed, failed, path):
"""
Waits for all dependencies to be satisfied before returning.
Now also checks for failed dependencies and raises an exception if found.
"""
while dependencies:
pending_dependencies = [dep for dep in dependencies if dep not in completed]
if any(dep in failed for dep in pending_dependencies):
failed_deps = [dep for dep in pending_dependencies if dep in failed]
raise Exception(f"{path}: Cannot proceed. Dependencies failed: {failed_deps}")
if not pending_dependencies:
return
LOGGER.warning(f"{path}: Waiting for dependencies to complete: {pending_dependencies}")
time.sleep(5)
def execute_with_dependencies(paths):
completed = set() # Track completed paths
errored_modules = set() # Track paths of modules where an error occurred
failed = {} # Track failures
with ThreadPoolExecutor(max_workers=terraform_threads) as executor:
future_to_path = {}
for path in paths:
dependencies = read_dependencies(path)
if dependencies:
# Submit a future for waiting dependencies
dep_future = executor.submit(wait_for_dependencies, dependencies, completed, failed, path)
# Submit the main task to be executed after dependencies are resolved
future = executor.submit(lambda p=dep_future, x=path: do_terraform(x) if p.result() is None else None)
else:
future = executor.submit(do_terraform, path)
future_to_path[future] = path
for future in as_completed(future_to_path):
path = future_to_path[future]
try:
result = future.result()
if result is None or not result[0]: # Assuming do_terraform returns (success: bool, message: str)
errored_modules.add(path)
failed[path] = "Failed due to internal error" # Mark as failed
else:
completed.add(path) # Mark as completed
LOGGER.info(f"Thread result for {path}: {result}")
except Exception as exc:
errored_modules.add(path)
failed[path] = str(exc) # Mark as failed
LOGGER.error(f"{path} generated an exception: {exc}")
# Check if any errors occurred and print the errored module paths
if errored_modules:
LOGGER.error("One or more modules failed to process correctly.")
for module_path in errored_modules:
LOGGER.error(f"Module with error: {module_path}")
sys.exit(1)
# Your main execution logic here
if __name__ == "__main__":
if tf_root_module_path:
do_terraform(tf_root_module_path)
else:
paths = find_paths_with_file('provider.tf')
for path in paths:
LOGGER.warning(f"Discovered: {path}")
execute_with_dependencies(paths)
textract-pipeline/terraform/main.tf
+1 -1
View File
@@ -68,7 +68,7 @@ locals {
}
}
provider "aws" {
profile = var.aws_profile
# profile = var.aws_profile
# access_key = var.access_key
# secret_key = var.secret_key
textract-pipeline/terraform/variables.tf
+4 -4
View File
@@ -10,10 +10,10 @@
# required
variable "aws_profile" {
type = string
# default = "doczyai"
}
#variable "aws_profile" {
# type = string
# # default = "doczyai"
#}
# required
variable "aws_region" {
type = string