From c5508fe1c4bc3b6e3ac24ee5e0a5d0e9beeaa0bb Mon Sep 17 00:00:00 2001 From: Pankaj Katariya Date: Thu, 9 May 2024 19:19:57 +0530 Subject: [PATCH] Added additional devops resources as mentioned on DOC-428 --- devops-pipeline/other-resources/locals.tf | 21 +- devops-pipeline/other-resources/main.tf | 322 ++++++++++++++++++- devops-pipeline/other-resources/outputs.tf | 22 ++ devops-pipeline/other-resources/variables.tf | 53 ++- 4 files changed, 392 insertions(+), 26 deletions(-) diff --git a/devops-pipeline/other-resources/locals.tf b/devops-pipeline/other-resources/locals.tf index be68b3a..77203e7 100644 --- a/devops-pipeline/other-resources/locals.tf +++ b/devops-pipeline/other-resources/locals.tf @@ -27,12 +27,23 @@ locals { global_prefix = "${var.project_name}-${local.region_short_name}-${local.environment_short_name}-${var.client_name}" # Resource prefix - lambda_prefix = "${local.global_prefix}-lmb" - sqs_prefix = "${local.global_prefix}-sqs" - sns_prefix = "${local.global_prefix}-sns" - dynamodb_prefix = "${local.global_prefix}-dyd" s3_prefix = "${local.global_prefix}-s3" + iam_role_prefix = "${local.global_prefix}-rol" + iam_policy_prefix = "${local.global_prefix}-pol" - # Terraform locking table + # s3 bukect name devops_s3_bucket_name = "${local.s3_prefix}-${var.devops_s3_bucket.name}" + mwaa_resources_s3_bucket_name = "${local.s3_prefix}-${var.mwaa_resources_s3_bucket.name}" + raw_data_ingestion_s3_bucket_name = "${local.s3_prefix}-${var.raw_data_ingestion_s3_bucket.name}" + + # IAM role name + snowflake_integration_role_name = "${local.iam_role_prefix}-snowflake-integration-role" + cross_account_role_name = "${local.iam_role_prefix}-cross-account-role" + mwaa_exec_role_role_name = "${local.iam_role_prefix}-mwaa-exec-role" + + # IAM policy name + snowflake_integration_policy_name = "${local.iam_policy_prefix}-snowflake-integration-policy" + cross_account_policy_name = "${local.iam_policy_prefix}-cross-account-policy" + mwaa_exec_policy_name = "${local.iam_policy_prefix}-mwaa-exec-policy" + } \ No newline at end of file diff --git a/devops-pipeline/other-resources/main.tf b/devops-pipeline/other-resources/main.tf index 0b5463c..d9febef 100644 --- a/devops-pipeline/other-resources/main.tf +++ b/devops-pipeline/other-resources/main.tf @@ -6,19 +6,21 @@ terraform { } } backend "s3" { - # bucket = "doczyai-use2-d-infra-s3-terraform-state" # Parameterize using -backend-config flag with "terraform init" - # key = "terraform/devops-pipeline/devops.tfstate" # Parameterize - # region = "us-east-2" # Parameterize - # profile = "doczyai" # Parameterize - # dynamodb_table = "doczyai-use2-d-infra-dyd-terraform-lock" # Parameterize + bucket = "doczyai-use2-d-infra-s3-terraform-state" # Parameterize using -backend-config flag with "terraform init" + key = "terraform/devops-pipeline/devops.tfstate" # Parameterize + region = "us-east-2" # Parameterize + profile = "doczyai" # Parameterize + dynamodb_table = "doczyai-use2-d-infra-dyd-terraform-lock" # Parameterize encrypt = true } } provider "aws" { - access_key = var.access_key - secret_key = var.secret_key + # access_key = var.access_key + # secret_key = var.secret_key + + profile = var.aws_profile default_tags { tags = { @@ -27,6 +29,7 @@ provider "aws" { } } +# s3 bucket - devops resource "aws_s3_bucket" "devops" { bucket = local.devops_s3_bucket_name tags = local.common_tags @@ -37,6 +40,29 @@ resource "aws_s3_bucket_policy" "deny_insecure_communication" { policy = data.aws_iam_policy_document.deny_insecure_communication.json } +# s3 bucket - raw-data-ingestion +resource "aws_s3_bucket" "raw_data_ingestion" { + bucket = local.raw_data_ingestion_s3_bucket_name + tags = local.common_tags +} + +resource "aws_s3_bucket_policy" "raw_data_ingestion_policy" { + bucket = aws_s3_bucket.raw_data_ingestion.id + policy = data.aws_iam_policy_document.raw_data_ingestion_deny_insecure_communication.json +} + +# s3 bucket - mwaa_resources +resource "aws_s3_bucket" "mwaa_resources" { + bucket = local.mwaa_resources_s3_bucket_name + tags = local.common_tags +} + +resource "aws_s3_bucket_policy" "mwaa_resources_policy" { + bucket = aws_s3_bucket.mwaa_resources.id + policy = data.aws_iam_policy_document.mwaa_resources_deny_insecure_communication.json +} + +# Deny insecure s3 bucket policy data "aws_iam_policy_document" "deny_insecure_communication" { statement { sid = "DenyInsecureCommunications" @@ -62,3 +88,285 @@ data "aws_iam_policy_document" "deny_insecure_communication" { ] } } + +# Deny insecure s3 bucket policy - raw_data_ingestion +data "aws_iam_policy_document" "raw_data_ingestion_deny_insecure_communication" { + statement { + sid = "DenyInsecureCommunications" + + principals { + type = "*" + identifiers = ["*"] + } + + condition { + test = "Bool" + variable = "aws:SecureTransport" + values = ["false"] + } + + effect = "Deny" + + actions = ["s3:*"] + + resources = [ + aws_s3_bucket.raw_data_ingestion.arn, + "${aws_s3_bucket.raw_data_ingestion.arn}/*", + ] + } +} + +# Deny insecure s3 bucket policy - mwaa_resources +data "aws_iam_policy_document" "mwaa_resources_deny_insecure_communication" { + statement { + sid = "DenyInsecureCommunications" + + principals { + type = "*" + identifiers = ["*"] + } + + condition { + test = "Bool" + variable = "aws:SecureTransport" + values = ["false"] + } + + effect = "Deny" + + actions = ["s3:*"] + + resources = [ + aws_s3_bucket.mwaa_resources.arn, + "${aws_s3_bucket.mwaa_resources.arn}/*", + ] + } +} + +# IAM role - snowflake-integration-role +resource "aws_iam_role" "snowflake_integration_role" { + name = local.snowflake_integration_role_name + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Sid = "" + Principal = { + AWS = "arn:aws:iam::851725635820:user/3nmi0000-s" + } + Condition = { + StringEquals = { + "sts:ExternalId" = "OQ11564_SFCRole=2_lNKzpyFn/e/xQgvGGpusVbEZA0A=" + } + } + }, + ] + }) +} + +# Construct IAM Policy for snowflake-integration-role +data "aws_iam_policy_document" "snowflake_integration_policy_document" { + statement { + effect = "Allow" + actions = [ + "s3:PutObject", + "s3:GetObject", + "s3:GetObjectVersion", + "s3:DeleteObject", + "s3:DeleteObjectVersion" + ] + resources = ["${aws_s3_bucket.raw_data_ingestion.arn}*"] + } + + statement { + effect = "Allow" + actions = [ + "s3:ListBucket", + "s3:GetBucketLocation" + ] + resources = ["${aws_s3_bucket.raw_data_ingestion.arn}"] + + condition { + test = "StringLike" + variable = "s3:prefix" + values = ["*"] + } + } +} + +# IAM Policy for snowflake-integration-role +resource "aws_iam_policy" "snowflake_integration_policy" { + name = local.snowflake_integration_policy_name + description = "Client textract permissions" + policy = data.aws_iam_policy_document.snowflake_integration_policy_document.json +} + +# Attach IAM Policy to snowflake-integration-role +resource "aws_iam_role_policy_attachment" "snowflake_integration_policy_attachment" { + role = aws_iam_role.snowflake_integration_role.name + policy_arn = aws_iam_policy.snowflake_integration_policy.arn +} + + +# IAM role - cross-account-role +resource "aws_iam_role" "cross_account_role" { + name = local.cross_account_role_name + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Sid = "" + Principal = { + AWS = "arn:aws:iam::873115228912:root" + } + }, + ] + }) +} + +# Construct IAM Policy for cross-account-role +data "aws_iam_policy_document" "cross_account_policy_document" { + statement { + effect = "Allow" + actions = [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucket", + "s3:PutObjectAcl", + "s3:GetObjectVersion" + ] + resources = [ + "${aws_s3_bucket.raw_data_ingestion.arn}", + "${aws_s3_bucket.raw_data_ingestion.arn}/*" + ] + } +} + +# IAM Policy for cross-account-role +resource "aws_iam_policy" "cross_account_policy" { + name = local.cross_account_policy_name + description = "This role is used for other AWS account to assume inorder to drop files to our data ingestion bucket. At the time of creation, this is being used by CODE DEV to drop all clients list." + policy = data.aws_iam_policy_document.cross_account_policy_document.json +} + +# Attach IAM Policy to cross-account-role +resource "aws_iam_role_policy_attachment" "cross_account_policy_attachment" { + role = aws_iam_role.cross_account_role.name + policy_arn = aws_iam_policy.cross_account_policy.arn +} + +# IAM role - mwaa-exec-role +resource "aws_iam_role" "mwaa_exec_role" { + name = local.mwaa_exec_role_role_name + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Sid = "" + Principal = { + Service = ["airflow-env.amazonaws.com","airflow.amazonaws.com"] + } + }, + ] + }) +} + +# Construct IAM Policy for mwaa-exec-role +data "aws_iam_policy_document" "mwaa_exec_policy_document" { + statement { + effect = "Allow" + actions = ["s3:*"] + resources = [ + "${aws_s3_bucket.raw_data_ingestion.arn}/*" + ] + } + statement { + effect = "Allow" + actions = ["airflow:CreateCliToken"] + resources = [ + "arn:aws:airflow:us-east-2:660131068782:environment/doczy-dev-infra-mwaa" + ] + } + statement { + effect = "Allow" + actions = ["airflow:PublishMetrics"] + resources = [ + "arn:aws:airflow:us-east-2:660131068782:environment/doczy-dev-infra-mwaa" + ] + } + statement { + effect = "Deny" + actions = ["s3:ListAllMyBuckets"] + resources = [ + "${aws_s3_bucket.mwaa_resources.arn}", + "${aws_s3_bucket.mwaa_resources.arn}/*" + ] + } + statement { + effect = "Allow" + actions = [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + "s3:PutObject" + ] + resources = [ + "${aws_s3_bucket.mwaa_resources.arn}", + "${aws_s3_bucket.mwaa_resources.arn}/*" + ] + } + + statement { + effect = "Allow" + actions = [ + "logs:CreateLogStream", + "logs:CreateLogGroup", + "logs:PutLogEvents", + "logs:GetLogEvents", + "logs:GetLogRecord", + "logs:GetLogGroupFields", + "logs:GetQueryResults" + ] + resources = [ + "arn:aws:logs:us-east-2:660131068782:log-group:airflow-doczy-dev-infra-mwaa-*" + ] + } + + statement { + effect = "Allow" + actions = [ + "logs:DescribeLogGroups" + ] + resources = ["*"] + } + + statement { + effect = "Allow" + actions = [ + "cloudwatch:PutMetricData" + ] + resources = ["*"] + } +} + +# IAM Policy for mwaa-exec-role +resource "aws_iam_policy" "mwaa_exec_policy" { + name = local.mwaa_exec_policy_name + description = "" + policy = data.aws_iam_policy_document.mwaa_exec_policy_document.json +} + +# Attach IAM Policy to mwaa-exec-role +resource "aws_iam_role_policy_attachment" "mwaa_exec_policy_attachment" { + role = aws_iam_role.mwaa_exec_role.name + policy_arn = aws_iam_policy.mwaa_exec_policy.arn +} \ No newline at end of file diff --git a/devops-pipeline/other-resources/outputs.tf b/devops-pipeline/other-resources/outputs.tf index f27f57c..a8e15a4 100644 --- a/devops-pipeline/other-resources/outputs.tf +++ b/devops-pipeline/other-resources/outputs.tf @@ -1,3 +1,25 @@ +# S3 bucket name output "devops_s3_bucket_name" { value = aws_s3_bucket.devops.id +} + +output "raw_data_ingestion_bucket_name" { + value = aws_s3_bucket.raw_data_ingestion.id +} + +output "mwaa_resources_bucket_name" { + value = aws_s3_bucket.mwaa_resources.id +} + +# IAM Role ARN +output "snowflake_integration_role_arn" { + value = aws_iam_role.snowflake_integration_role.arn +} + +output "cross_account_role_arn" { + value = aws_iam_role.cross_account_role.id +} + +output "mwaa_exec_role_arn" { + value = aws_iam_role.mwaa_exec_role.id } \ No newline at end of file diff --git a/devops-pipeline/other-resources/variables.tf b/devops-pipeline/other-resources/variables.tf index 1cd1239..dbbfd87 100644 --- a/devops-pipeline/other-resources/variables.tf +++ b/devops-pipeline/other-resources/variables.tf @@ -1,23 +1,23 @@ # required -# variable "aws_profile" { +variable "aws_profile" { + type = string + default = "doczyai" +} + +# # required +# variable "secret_key" { # type = string -# default = "doczyai" # } -# required -variable "secret_key" { - type = string -} - -# required -variable "access_key" { - type = string -} +# # required +# variable "access_key" { +# type = string +# } # required variable "aws_region" { type = string - # default = "us-east-2" + default = "us-east-2" } # required variable "project_name" { @@ -27,7 +27,7 @@ variable "project_name" { # required variable "environment" { type = string - # default = "dev" + default = "dev" } # required variable "client_name" { @@ -44,4 +44,29 @@ variable "devops_s3_bucket" { name = "devops-resources" } -} \ No newline at end of file +} + +# s3 bucket - raw-data-ingestion +variable "raw_data_ingestion_s3_bucket" { + type = object({ + name = string + + }) + default = { + name = "raw-data-ingestion" + + } +} + +# s3 bucket - mwaa-resources +variable "mwaa_resources_s3_bucket" { + type = object({ + name = string + + }) + default = { + name = "mwaa-resources" + + } +} +