From 7fe7612f4b7de2234965d901721c04a16bc4df1f Mon Sep 17 00:00:00 2001 From: Grzegorz Huber Date: Thu, 6 Jun 2024 09:54:14 +0200 Subject: [PATCH] test new terraform script --- bitbucket-pipelines.yml | 16 ++ streamlit-server/main.tf | 21 +-- streamlit-server/variables.tf | 26 +-- terminal.py | 125 +++++++++++++ terraform.py | 224 +++++++++++++++++++++++ textract-pipeline/terraform/main.tf | 2 +- textract-pipeline/terraform/variables.tf | 8 +- 7 files changed, 391 insertions(+), 31 deletions(-) create mode 100644 terminal.py create mode 100644 terraform.py diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index ffaecb2..33599cf 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -291,3 +291,19 @@ pipelines: condition: paths: - snowflake/**/*.sql + + + feature/terraform-refactor: + - parallel: + - step: + name: Plan streamlit-server on DEV + oidc: true + script: + - *aws-context-dev + - python terraform.py plan --environment=dev --module=streamlit-server + - step: + name: Plan streamlit-server on UAT + oidc: true + script: + - *aws-context-uat + - python terraform.py plan --environment=uat --module=streamlit-server diff --git a/streamlit-server/main.tf b/streamlit-server/main.tf index 1fd8316..75d1352 100644 --- a/streamlit-server/main.tf +++ b/streamlit-server/main.tf @@ -1,5 +1,5 @@ provider "aws" { - region = "us-east-2" + region = "us-east-2" } terraform { @@ -11,11 +11,6 @@ terraform { } backend "s3" { - # bucket = "doczyai-use2-d-infra-s3-terraform-state" # Parameterize using -backend-config flag with "terraform init" - # key = "terraform/streamlit-server/terraform.tfstate" # Parameterize - # region = "us-east-2" # Parameterize - # profile = "temp_cred" # Parameterize - # dynamodb_table = "doczyai-use2-d-infra-dyd-terraform-lock" # Parameterize encrypt = true } } @@ -43,7 +38,7 @@ locals { # global prefix global_prefix = "${var.project_name}-${local.region_short_name}-${local.environment_short_name}" - + # Resource prefix iam_policy_prefix = "${local.global_prefix}-pol" @@ -65,14 +60,14 @@ data "aws_subnets" "subnets" { values = [var.vpc_id] } } - + # vpc security groups data "aws_security_groups" "security_groups" { filter { name = "group-name" values = ["*default*"] } - + filter { name = "vpc-id" values = [var.vpc_id] @@ -89,8 +84,8 @@ resource "aws_instance" "streamlit_server" { ami = var.ubuntu_ami instance_type = var.ec2_instance_type subnet_id = data.aws_subnets.subnets.ids[0] - vpc_security_group_ids = data.aws_security_groups.security_groups.ids - iam_instance_profile = aws_iam_instance_profile.ec2_instance_profile.name + vpc_security_group_ids = data.aws_security_groups.security_groups.ids + iam_instance_profile = aws_iam_instance_profile.ec2_instance_profile.name associate_public_ip_address = false key_name = "tf-test-key" # Created using TF and uploaded to S3. Prerequisite @@ -98,8 +93,8 @@ resource "aws_instance" "streamlit_server" { volume_size = 30 # Variable encrypted = true # Mandatory } -# Setup SSH authentication for bitbucket access to pull code -# Setup SSL certificate for HTTPS access for SSO to work +# Setup SSH authentication for bitbucket access to pull code +# Setup SSL certificate for HTTPS access for SSO to work user_data = <<-EOF #!/bin/bash echo '${file("${path.module}/requirements.txt")}' > /tmp/requirements.txt diff --git a/streamlit-server/variables.tf b/streamlit-server/variables.tf index 79f9735..9bc76e7 100644 --- a/streamlit-server/variables.tf +++ b/streamlit-server/variables.tf @@ -59,16 +59,16 @@ variable "acm_arn_dev" { default = "arn:aws:acm:us-east-2:660131068782:certificate/947e04b3-be22-43bf-bea2-a0ad3b221d70" } -variable "secret_key" { - type = string -} - -# required -variable "access_key" { - type = string -} - -# required -variable "token" { - type = string -} \ No newline at end of file +#variable "secret_key" { +# type = string +#} +# +## required +#variable "access_key" { +# type = string +#} +# +## required +#variable "token" { +# type = string +#} \ No newline at end of file diff --git a/terminal.py b/terminal.py new file mode 100644 index 0000000..8c5fcfb --- /dev/null +++ b/terminal.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +import os +import re +import subprocess +import logging +import signal +from typing import Callable, Optional, Dict, Tuple, List + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.INFO) +handler = logging.StreamHandler() +handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')) +LOGGER.addHandler(handler) + + +class SensitiveFormatter(logging.Formatter): + """Formatter that removes sensitive information from logs.""" + + @staticmethod + def mask_password(log: str) -> str: + return re.sub(r'password=([^\s]+)', r'password=*****', log) + + @staticmethod + def mask_api_key(log: str) -> str: + return re.sub(r'api_key=([^\s]+)', r'api_key=*****', log) + + @staticmethod + def _mask(s: str) -> str: + filtered = SensitiveFormatter.mask_password(s) + filtered = SensitiveFormatter.mask_api_key(filtered) + return filtered + + def format(self, record) -> str: + original = super().format(record) + return self._mask(original) + + +def prepare_logger() -> logging.Logger: + global LOGGER + if LOGGER is not None: + return LOGGER + LOGGER = logging.getLogger(__name__) + LOGGER.setLevel(logging.INFO) + log_format = '%(asctime)s %(filename)s:%(lineno)-4s [%(levelname)s] %(message)s' + handler = logging.StreamHandler() + handler.setFormatter(SensitiveFormatter(log_format)) + LOGGER.addHandler(handler) + return LOGGER + + +def get_blue_shade(log_prefix: str) -> str: + """Returns an ANSI color code for a shade of blue based on the hash of the log_prefix.""" + blue_shades = [81, 87, 117, 153, 159, 195, 111, 45, 39] + hash_value = hash(log_prefix) + blue_index = hash_value % len(blue_shades) + return f"\033[38;5;{blue_shades[blue_index]}m" + + +def signal_handler(sig, frame, process): + print("Ctrl+C pressed! Sending SIGTERM to Terraform process...") + process.terminate() + + +def run_command(command: str, log_output: bool = False, decorate_logs: bool = True, log_cmd: bool = False, + log_prefix: str = "", envs: Optional[str] = None, secrets: Optional[Dict[str, str]] = None, + failure_callback: Optional[Callable[[str], None]] = None, + cwd: Optional[str] = None) -> Tuple[int, List[str]]: + new_env = os.environ.copy() + if secrets: + for key, value in secrets.items(): + new_env[key] = os.path.expandvars(value) + + full_command = f"{envs} {command}" if envs else command + process = subprocess.Popen(full_command, shell=True, cwd=cwd, env=new_env, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + + # Setup signal handler + # original_sigint_handler = signal.getsignal(signal.SIGINT) + # signal.signal(signal.SIGINT, lambda sig, frame: signal_handler(sig, frame, process)) + + log_prefix_env = os.getenv("LOG_PREFIX", "") + log_prefix_full = f"[{log_prefix_env}] {log_prefix}" if log_prefix_env else log_prefix + + if log_cmd: + LOGGER.info( + f"{get_blue_shade(log_prefix_full)}{log_prefix_full}\033[0m: Running command: \033[1;34m{command}\033[0m") + if envs: + LOGGER.info(f"Using additional envs: \033[1;34m{envs}\033[0m") + if secrets: + LOGGER.info(f"Using additional environment variables with secrets: \033[1;34m{list(secrets.keys())}\033[0m") + + cmd_output = [] + while True: + output = process.stdout.readline() if process.stdout else '' + if output: + output_line = output.strip() + if log_output: + log_statement = ( + f"{get_blue_shade(log_prefix_full)}{log_prefix_full}\033[0m: " + f"{output_line}") if log_prefix else output_line + LOGGER.info(log_statement) if decorate_logs else print(log_statement) + cmd_output.append(output_line) + elif process.poll() is not None: + break + + # Restore original signal handler + # signal.signal(signal.SIGINT, original_sigint_handler) + + rc = process.poll() or 0 # Ensure rc is an int, default to 0 if None + if rc != 0 and failure_callback: + failure_callback(f"Command returned code: {rc}") + + return rc, cmd_output + + +def decorate_successful(text: str) -> str: + return f"\033[32;1m{text}\033[0m" + + +def decorate_white_bold(text: str) -> str: + return f"\033[37;1m{text}\033[0m" + + +def decorate_warn(text: str) -> str: + return f"\033[33;1m{text}\033[0m" diff --git a/terraform.py b/terraform.py new file mode 100644 index 0000000..aaf308e --- /dev/null +++ b/terraform.py @@ -0,0 +1,224 @@ +from terminal import run_command, prepare_logger + +import os +import argparse +import sys +import subprocess +import time +from concurrent.futures import ThreadPoolExecutor, as_completed + +script_dir = os.path.dirname(os.path.abspath(__file__)) + +LOGGER = prepare_logger() + +LOGGER.info(sys.argv) + +parser = argparse.ArgumentParser(description='Run Terraform init with dynamic backend configuration.') +parser.add_argument('terraform_command', type=str, choices=['plan', 'apply'], + help='The Terraform command to execute (plan, apply, init).') +parser.add_argument('--environment', type=str, required=True, help='The environment name i.e. dev, prod.') +parser.add_argument('--module', type=str, required=False, help='The module name.') +parser.add_argument('--init-extra-args', type=str, required=False, help='Terraform init extra arguments') +parser.add_argument('--plan-extra-args', type=str, required=False, help='Terraform plan extra arguments') +parser.add_argument('--apply-extra-args', type=str, required=False, help='Terraform plan extra arguments') +parser.add_argument('--threads', type=int, default=4, help='Number of threads to use (default: 4)') + +args = parser.parse_args() + +raw_terraform_command = args.terraform_command +environment = args.environment +init_extra_args = args.init_extra_args +plan_extra_args = args.plan_extra_args +apply_extra_args = args.apply_extra_args +terraform_threads = args.threads +module_name = args.module + +bitbucket_ci = os.getenv('BITBUCKET_CI') + + +def change_dir(path): + if path: + _, git_root = run_command('git rev-parse --show-toplevel', log_cmd=True, log_output=True) + LOGGER.info(f"git_root={git_root}") + absolute_path = os.path.join(git_root[0], path) + return absolute_path + # os.chdir(absolute_path) + else: + LOGGER.error("TF_ROOT_MODULE_PATH environment variable is not set.") + sys.exit(1) + + +def select_vars_file(): + tf_vars_file = os.getenv('TFVARS_FILE_PATH', f'vars-{environment}.tfvars') + if tf_vars_file: + LOGGER.info(f"Using {tf_vars_file} as vars-file...") + else: + LOGGER.error("TFVARS_FILE_PATH variable is not set.") + sys.exit(1) + return tf_vars_file + + +def prepare_tf_init_command(stack_path, init_extra_args, state_environment, tf_state_bucket_name): + command = (f'terraform init --reconfigure ' + f'-backend-config="dynamodb_table=doczyai-use2-{get_environment_value(environment)}-infra-dyd-terraform-lock" ' + f'-backend-config="bucket=doczyai-use2-{get_environment_value(environment)}-infra-s3-terraform-state" ' + f'-backend-config="key=terraform/{stack_path}/terraform.tfstate" -backend-config="region=us-east-2"') + if init_extra_args: + command = f"{command} {init_extra_args}" + return command + + +def get_environment_value(environment): + environment_map = { + 'prod': 'p', + 'uat': 'u', + 'qa': 'q', + 'dev': 'd' + } + + return environment_map.get(environment, None) # Returns None if the environment is not found + + +def prepare_tf_main_command(environment, stack_path, state_environment, tf_state_bucket_name, vars_file, + plan_extra_args, + apply_extra_args): + common_command = f'-var "aws_region=us-east-2" -var "environment={environment}" -var "vpc_id=vpc-0f9d7c913f6522079"' + if raw_terraform_command == "plan": + command = f"terraform plan -out {stack_path}.{environment}.tfplan {common_command}" + if plan_extra_args: + command = f"{command} {plan_extra_args}" + elif raw_terraform_command == "apply": + command = f"terraform apply -auto-approve {common_command}" + if apply_extra_args: + command = f"{command} {apply_extra_args}" + return command + + +tf_root_module_path = os.getenv('TF_ROOT_MODULE_PATH') if not module_name else module_name + + +def do_terraform(stack_path): + LOGGER.info(f"stack_path={stack_path}") + absolute_path = change_dir(stack_path) + # TF INIT + state_environment = "dev-new" if environment == "dev" else environment + tf_state_bucket_name = f"pi2new-{state_environment}-terraform-state-file" + + terraform_init_command = prepare_tf_init_command(stack_path, init_extra_args, state_environment, + tf_state_bucket_name) + rc_init, init_result = run_command(terraform_init_command, log_output=True, log_cmd=True, log_prefix=stack_path, + cwd=absolute_path) + if rc_init > 0: + raise Exception(f"{stack_path}: Failed to run terraform init") + + vars_file = select_vars_file() + terraform_main_command = prepare_tf_main_command(environment, stack_path, state_environment, tf_state_bucket_name, + vars_file, plan_extra_args, apply_extra_args) + + rc_main, command_result = run_command(terraform_main_command, log_output=True, log_cmd=True, + log_prefix=stack_path, cwd=absolute_path) + if rc_init or rc_main > 0: + raise RuntimeError(f"Unexpected Terraform error for command: {terraform_main_command}") + else: + return True, "OK" + + # return True, "OK" + + +def find_paths_with_file(file_name): + try: + completed_process = subprocess.run(['git', 'ls-files', f'*{file_name}'], check=True, text=True, + stdout=subprocess.PIPE) + + paths = set() + for file_path in completed_process.stdout.splitlines(): + dir_path = '/'.join(file_path.split('/')[:-1]) + if dir_path.startswith("modules/"): + continue + if dir_path.startswith("infra/"): + dir_path = dir_path[len("infra/"):] + if dir_path.startswith("kinesis") or dir_path.startswith("aws-transfer-sftp"): + paths.add(dir_path) + + return paths + except subprocess.CalledProcessError as e: + LOGGER.error(f"Error executing git command: {e}") + return set() + + +def read_dependencies(path): + depends_on_file = os.path.join(path, ".depends-on.txt") + if os.path.exists(depends_on_file): + with open(depends_on_file, "r") as f: + dependencies = [line.strip() for line in f.readlines()] + return dependencies + return [] + + +def wait_for_dependencies(dependencies, completed, failed, path): + """ + Waits for all dependencies to be satisfied before returning. + Now also checks for failed dependencies and raises an exception if found. + """ + while dependencies: + pending_dependencies = [dep for dep in dependencies if dep not in completed] + if any(dep in failed for dep in pending_dependencies): + failed_deps = [dep for dep in pending_dependencies if dep in failed] + raise Exception(f"{path}: Cannot proceed. Dependencies failed: {failed_deps}") + if not pending_dependencies: + return + LOGGER.warning(f"{path}: Waiting for dependencies to complete: {pending_dependencies}") + time.sleep(5) + + +def execute_with_dependencies(paths): + completed = set() # Track completed paths + errored_modules = set() # Track paths of modules where an error occurred + failed = {} # Track failures + + with ThreadPoolExecutor(max_workers=terraform_threads) as executor: + future_to_path = {} + + for path in paths: + dependencies = read_dependencies(path) + if dependencies: + # Submit a future for waiting dependencies + dep_future = executor.submit(wait_for_dependencies, dependencies, completed, failed, path) + # Submit the main task to be executed after dependencies are resolved + future = executor.submit(lambda p=dep_future, x=path: do_terraform(x) if p.result() is None else None) + else: + future = executor.submit(do_terraform, path) + future_to_path[future] = path + + for future in as_completed(future_to_path): + path = future_to_path[future] + try: + result = future.result() + if result is None or not result[0]: # Assuming do_terraform returns (success: bool, message: str) + errored_modules.add(path) + failed[path] = "Failed due to internal error" # Mark as failed + else: + completed.add(path) # Mark as completed + LOGGER.info(f"Thread result for {path}: {result}") + except Exception as exc: + errored_modules.add(path) + failed[path] = str(exc) # Mark as failed + LOGGER.error(f"{path} generated an exception: {exc}") + + # Check if any errors occurred and print the errored module paths + if errored_modules: + LOGGER.error("One or more modules failed to process correctly.") + for module_path in errored_modules: + LOGGER.error(f"Module with error: {module_path}") + sys.exit(1) + + +# Your main execution logic here +if __name__ == "__main__": + if tf_root_module_path: + do_terraform(tf_root_module_path) + else: + paths = find_paths_with_file('provider.tf') + for path in paths: + LOGGER.warning(f"Discovered: {path}") + execute_with_dependencies(paths) diff --git a/textract-pipeline/terraform/main.tf b/textract-pipeline/terraform/main.tf index 3ab3b0f..2161404 100644 --- a/textract-pipeline/terraform/main.tf +++ b/textract-pipeline/terraform/main.tf @@ -68,7 +68,7 @@ locals { } } provider "aws" { - profile = var.aws_profile +# profile = var.aws_profile # access_key = var.access_key # secret_key = var.secret_key diff --git a/textract-pipeline/terraform/variables.tf b/textract-pipeline/terraform/variables.tf index 276513c..8be092e 100644 --- a/textract-pipeline/terraform/variables.tf +++ b/textract-pipeline/terraform/variables.tf @@ -10,10 +10,10 @@ # required -variable "aws_profile" { - type = string - # default = "doczyai" -} +#variable "aws_profile" { +# type = string +# # default = "doczyai" +#} # required variable "aws_region" { type = string