diff --git a/devops-pipeline/other-resources/locals.tf b/devops-pipeline/other-resources/locals.tf index be68b3a..77203e7 100644 --- a/devops-pipeline/other-resources/locals.tf +++ b/devops-pipeline/other-resources/locals.tf @@ -27,12 +27,23 @@ locals { global_prefix = "${var.project_name}-${local.region_short_name}-${local.environment_short_name}-${var.client_name}" # Resource prefix - lambda_prefix = "${local.global_prefix}-lmb" - sqs_prefix = "${local.global_prefix}-sqs" - sns_prefix = "${local.global_prefix}-sns" - dynamodb_prefix = "${local.global_prefix}-dyd" s3_prefix = "${local.global_prefix}-s3" + iam_role_prefix = "${local.global_prefix}-rol" + iam_policy_prefix = "${local.global_prefix}-pol" - # Terraform locking table + # s3 bukect name devops_s3_bucket_name = "${local.s3_prefix}-${var.devops_s3_bucket.name}" + mwaa_resources_s3_bucket_name = "${local.s3_prefix}-${var.mwaa_resources_s3_bucket.name}" + raw_data_ingestion_s3_bucket_name = "${local.s3_prefix}-${var.raw_data_ingestion_s3_bucket.name}" + + # IAM role name + snowflake_integration_role_name = "${local.iam_role_prefix}-snowflake-integration-role" + cross_account_role_name = "${local.iam_role_prefix}-cross-account-role" + mwaa_exec_role_role_name = "${local.iam_role_prefix}-mwaa-exec-role" + + # IAM policy name + snowflake_integration_policy_name = "${local.iam_policy_prefix}-snowflake-integration-policy" + cross_account_policy_name = "${local.iam_policy_prefix}-cross-account-policy" + mwaa_exec_policy_name = "${local.iam_policy_prefix}-mwaa-exec-policy" + } \ No newline at end of file diff --git a/devops-pipeline/other-resources/main.tf b/devops-pipeline/other-resources/main.tf index 0b5463c..d9febef 100644 --- a/devops-pipeline/other-resources/main.tf +++ b/devops-pipeline/other-resources/main.tf @@ -6,19 +6,21 @@ terraform { } } backend "s3" { - # bucket = "doczyai-use2-d-infra-s3-terraform-state" # Parameterize using -backend-config flag with "terraform init" - # key = "terraform/devops-pipeline/devops.tfstate" # Parameterize - # region = "us-east-2" # Parameterize - # profile = "doczyai" # Parameterize - # dynamodb_table = "doczyai-use2-d-infra-dyd-terraform-lock" # Parameterize + bucket = "doczyai-use2-d-infra-s3-terraform-state" # Parameterize using -backend-config flag with "terraform init" + key = "terraform/devops-pipeline/devops.tfstate" # Parameterize + region = "us-east-2" # Parameterize + profile = "doczyai" # Parameterize + dynamodb_table = "doczyai-use2-d-infra-dyd-terraform-lock" # Parameterize encrypt = true } } provider "aws" { - access_key = var.access_key - secret_key = var.secret_key + # access_key = var.access_key + # secret_key = var.secret_key + + profile = var.aws_profile default_tags { tags = { @@ -27,6 +29,7 @@ provider "aws" { } } +# s3 bucket - devops resource "aws_s3_bucket" "devops" { bucket = local.devops_s3_bucket_name tags = local.common_tags @@ -37,6 +40,29 @@ resource "aws_s3_bucket_policy" "deny_insecure_communication" { policy = data.aws_iam_policy_document.deny_insecure_communication.json } +# s3 bucket - raw-data-ingestion +resource "aws_s3_bucket" "raw_data_ingestion" { + bucket = local.raw_data_ingestion_s3_bucket_name + tags = local.common_tags +} + +resource "aws_s3_bucket_policy" "raw_data_ingestion_policy" { + bucket = aws_s3_bucket.raw_data_ingestion.id + policy = data.aws_iam_policy_document.raw_data_ingestion_deny_insecure_communication.json +} + +# s3 bucket - mwaa_resources +resource "aws_s3_bucket" "mwaa_resources" { + bucket = local.mwaa_resources_s3_bucket_name + tags = local.common_tags +} + +resource "aws_s3_bucket_policy" "mwaa_resources_policy" { + bucket = aws_s3_bucket.mwaa_resources.id + policy = data.aws_iam_policy_document.mwaa_resources_deny_insecure_communication.json +} + +# Deny insecure s3 bucket policy data "aws_iam_policy_document" "deny_insecure_communication" { statement { sid = "DenyInsecureCommunications" @@ -62,3 +88,285 @@ data "aws_iam_policy_document" "deny_insecure_communication" { ] } } + +# Deny insecure s3 bucket policy - raw_data_ingestion +data "aws_iam_policy_document" "raw_data_ingestion_deny_insecure_communication" { + statement { + sid = "DenyInsecureCommunications" + + principals { + type = "*" + identifiers = ["*"] + } + + condition { + test = "Bool" + variable = "aws:SecureTransport" + values = ["false"] + } + + effect = "Deny" + + actions = ["s3:*"] + + resources = [ + aws_s3_bucket.raw_data_ingestion.arn, + "${aws_s3_bucket.raw_data_ingestion.arn}/*", + ] + } +} + +# Deny insecure s3 bucket policy - mwaa_resources +data "aws_iam_policy_document" "mwaa_resources_deny_insecure_communication" { + statement { + sid = "DenyInsecureCommunications" + + principals { + type = "*" + identifiers = ["*"] + } + + condition { + test = "Bool" + variable = "aws:SecureTransport" + values = ["false"] + } + + effect = "Deny" + + actions = ["s3:*"] + + resources = [ + aws_s3_bucket.mwaa_resources.arn, + "${aws_s3_bucket.mwaa_resources.arn}/*", + ] + } +} + +# IAM role - snowflake-integration-role +resource "aws_iam_role" "snowflake_integration_role" { + name = local.snowflake_integration_role_name + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Sid = "" + Principal = { + AWS = "arn:aws:iam::851725635820:user/3nmi0000-s" + } + Condition = { + StringEquals = { + "sts:ExternalId" = "OQ11564_SFCRole=2_lNKzpyFn/e/xQgvGGpusVbEZA0A=" + } + } + }, + ] + }) +} + +# Construct IAM Policy for snowflake-integration-role +data "aws_iam_policy_document" "snowflake_integration_policy_document" { + statement { + effect = "Allow" + actions = [ + "s3:PutObject", + "s3:GetObject", + "s3:GetObjectVersion", + "s3:DeleteObject", + "s3:DeleteObjectVersion" + ] + resources = ["${aws_s3_bucket.raw_data_ingestion.arn}*"] + } + + statement { + effect = "Allow" + actions = [ + "s3:ListBucket", + "s3:GetBucketLocation" + ] + resources = ["${aws_s3_bucket.raw_data_ingestion.arn}"] + + condition { + test = "StringLike" + variable = "s3:prefix" + values = ["*"] + } + } +} + +# IAM Policy for snowflake-integration-role +resource "aws_iam_policy" "snowflake_integration_policy" { + name = local.snowflake_integration_policy_name + description = "Client textract permissions" + policy = data.aws_iam_policy_document.snowflake_integration_policy_document.json +} + +# Attach IAM Policy to snowflake-integration-role +resource "aws_iam_role_policy_attachment" "snowflake_integration_policy_attachment" { + role = aws_iam_role.snowflake_integration_role.name + policy_arn = aws_iam_policy.snowflake_integration_policy.arn +} + + +# IAM role - cross-account-role +resource "aws_iam_role" "cross_account_role" { + name = local.cross_account_role_name + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Sid = "" + Principal = { + AWS = "arn:aws:iam::873115228912:root" + } + }, + ] + }) +} + +# Construct IAM Policy for cross-account-role +data "aws_iam_policy_document" "cross_account_policy_document" { + statement { + effect = "Allow" + actions = [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucket", + "s3:PutObjectAcl", + "s3:GetObjectVersion" + ] + resources = [ + "${aws_s3_bucket.raw_data_ingestion.arn}", + "${aws_s3_bucket.raw_data_ingestion.arn}/*" + ] + } +} + +# IAM Policy for cross-account-role +resource "aws_iam_policy" "cross_account_policy" { + name = local.cross_account_policy_name + description = "This role is used for other AWS account to assume inorder to drop files to our data ingestion bucket. At the time of creation, this is being used by CODE DEV to drop all clients list." + policy = data.aws_iam_policy_document.cross_account_policy_document.json +} + +# Attach IAM Policy to cross-account-role +resource "aws_iam_role_policy_attachment" "cross_account_policy_attachment" { + role = aws_iam_role.cross_account_role.name + policy_arn = aws_iam_policy.cross_account_policy.arn +} + +# IAM role - mwaa-exec-role +resource "aws_iam_role" "mwaa_exec_role" { + name = local.mwaa_exec_role_role_name + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Sid = "" + Principal = { + Service = ["airflow-env.amazonaws.com","airflow.amazonaws.com"] + } + }, + ] + }) +} + +# Construct IAM Policy for mwaa-exec-role +data "aws_iam_policy_document" "mwaa_exec_policy_document" { + statement { + effect = "Allow" + actions = ["s3:*"] + resources = [ + "${aws_s3_bucket.raw_data_ingestion.arn}/*" + ] + } + statement { + effect = "Allow" + actions = ["airflow:CreateCliToken"] + resources = [ + "arn:aws:airflow:us-east-2:660131068782:environment/doczy-dev-infra-mwaa" + ] + } + statement { + effect = "Allow" + actions = ["airflow:PublishMetrics"] + resources = [ + "arn:aws:airflow:us-east-2:660131068782:environment/doczy-dev-infra-mwaa" + ] + } + statement { + effect = "Deny" + actions = ["s3:ListAllMyBuckets"] + resources = [ + "${aws_s3_bucket.mwaa_resources.arn}", + "${aws_s3_bucket.mwaa_resources.arn}/*" + ] + } + statement { + effect = "Allow" + actions = [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + "s3:PutObject" + ] + resources = [ + "${aws_s3_bucket.mwaa_resources.arn}", + "${aws_s3_bucket.mwaa_resources.arn}/*" + ] + } + + statement { + effect = "Allow" + actions = [ + "logs:CreateLogStream", + "logs:CreateLogGroup", + "logs:PutLogEvents", + "logs:GetLogEvents", + "logs:GetLogRecord", + "logs:GetLogGroupFields", + "logs:GetQueryResults" + ] + resources = [ + "arn:aws:logs:us-east-2:660131068782:log-group:airflow-doczy-dev-infra-mwaa-*" + ] + } + + statement { + effect = "Allow" + actions = [ + "logs:DescribeLogGroups" + ] + resources = ["*"] + } + + statement { + effect = "Allow" + actions = [ + "cloudwatch:PutMetricData" + ] + resources = ["*"] + } +} + +# IAM Policy for mwaa-exec-role +resource "aws_iam_policy" "mwaa_exec_policy" { + name = local.mwaa_exec_policy_name + description = "" + policy = data.aws_iam_policy_document.mwaa_exec_policy_document.json +} + +# Attach IAM Policy to mwaa-exec-role +resource "aws_iam_role_policy_attachment" "mwaa_exec_policy_attachment" { + role = aws_iam_role.mwaa_exec_role.name + policy_arn = aws_iam_policy.mwaa_exec_policy.arn +} \ No newline at end of file diff --git a/devops-pipeline/other-resources/outputs.tf b/devops-pipeline/other-resources/outputs.tf index f27f57c..a8e15a4 100644 --- a/devops-pipeline/other-resources/outputs.tf +++ b/devops-pipeline/other-resources/outputs.tf @@ -1,3 +1,25 @@ +# S3 bucket name output "devops_s3_bucket_name" { value = aws_s3_bucket.devops.id +} + +output "raw_data_ingestion_bucket_name" { + value = aws_s3_bucket.raw_data_ingestion.id +} + +output "mwaa_resources_bucket_name" { + value = aws_s3_bucket.mwaa_resources.id +} + +# IAM Role ARN +output "snowflake_integration_role_arn" { + value = aws_iam_role.snowflake_integration_role.arn +} + +output "cross_account_role_arn" { + value = aws_iam_role.cross_account_role.id +} + +output "mwaa_exec_role_arn" { + value = aws_iam_role.mwaa_exec_role.id } \ No newline at end of file diff --git a/devops-pipeline/other-resources/variables.tf b/devops-pipeline/other-resources/variables.tf index 1cd1239..dbbfd87 100644 --- a/devops-pipeline/other-resources/variables.tf +++ b/devops-pipeline/other-resources/variables.tf @@ -1,23 +1,23 @@ # required -# variable "aws_profile" { +variable "aws_profile" { + type = string + default = "doczyai" +} + +# # required +# variable "secret_key" { # type = string -# default = "doczyai" # } -# required -variable "secret_key" { - type = string -} - -# required -variable "access_key" { - type = string -} +# # required +# variable "access_key" { +# type = string +# } # required variable "aws_region" { type = string - # default = "us-east-2" + default = "us-east-2" } # required variable "project_name" { @@ -27,7 +27,7 @@ variable "project_name" { # required variable "environment" { type = string - # default = "dev" + default = "dev" } # required variable "client_name" { @@ -44,4 +44,29 @@ variable "devops_s3_bucket" { name = "devops-resources" } -} \ No newline at end of file +} + +# s3 bucket - raw-data-ingestion +variable "raw_data_ingestion_s3_bucket" { + type = object({ + name = string + + }) + default = { + name = "raw-data-ingestion" + + } +} + +# s3 bucket - mwaa-resources +variable "mwaa_resources_s3_bucket" { + type = object({ + name = string + + }) + default = { + name = "mwaa-resources" + + } +} +