jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Added additional devops resources as mentioned on DOC-428

Browse Source
This commit is contained in:
Pankaj Katariya
2024-05-09 19:19:57 +05:30
parent 1958cccf7b
commit c5508fe1c4
4 changed files with 392 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
devops-pipeline/other-resources/locals.tf
+16 -5
View File
@@ -27,12 +27,23 @@ locals {
global_prefix = "${var.project_name}-${local.region_short_name}-${local.environment_short_name}-${var.client_name}"
# Resource prefix
lambda_prefix = "${local.global_prefix}-lmb"
sqs_prefix = "${local.global_prefix}-sqs"
sns_prefix = "${local.global_prefix}-sns"
dynamodb_prefix = "${local.global_prefix}-dyd"
s3_prefix = "${local.global_prefix}-s3"
iam_role_prefix = "${local.global_prefix}-rol"
iam_policy_prefix = "${local.global_prefix}-pol"
# Terraform locking table
# s3 bukect name
devops_s3_bucket_name = "${local.s3_prefix}-${var.devops_s3_bucket.name}"
mwaa_resources_s3_bucket_name = "${local.s3_prefix}-${var.mwaa_resources_s3_bucket.name}"
raw_data_ingestion_s3_bucket_name = "${local.s3_prefix}-${var.raw_data_ingestion_s3_bucket.name}"
# IAM role name
snowflake_integration_role_name = "${local.iam_role_prefix}-snowflake-integration-role"
cross_account_role_name = "${local.iam_role_prefix}-cross-account-role"
mwaa_exec_role_role_name = "${local.iam_role_prefix}-mwaa-exec-role"
# IAM policy name
snowflake_integration_policy_name = "${local.iam_policy_prefix}-snowflake-integration-policy"
cross_account_policy_name = "${local.iam_policy_prefix}-cross-account-policy"
mwaa_exec_policy_name = "${local.iam_policy_prefix}-mwaa-exec-policy"
}
devops-pipeline/other-resources/main.tf
+315 -7
View File
@@ -6,19 +6,21 @@ terraform {
}
}
backend "s3" {
# bucket = "doczyai-use2-d-infra-s3-terraform-state" # Parameterize using -backend-config flag with "terraform init"
# key = "terraform/devops-pipeline/devops.tfstate" # Parameterize
# region = "us-east-2" # Parameterize
# profile = "doczyai" # Parameterize
# dynamodb_table = "doczyai-use2-d-infra-dyd-terraform-lock" # Parameterize
bucket = "doczyai-use2-d-infra-s3-terraform-state" # Parameterize using -backend-config flag with "terraform init"
key = "terraform/devops-pipeline/devops.tfstate" # Parameterize
region = "us-east-2" # Parameterize
profile = "doczyai" # Parameterize
dynamodb_table = "doczyai-use2-d-infra-dyd-terraform-lock" # Parameterize
encrypt = true
}
}
provider "aws" {
access_key = var.access_key
secret_key = var.secret_key
# access_key = var.access_key
# secret_key = var.secret_key
profile = var.aws_profile
default_tags {
tags = {
@@ -27,6 +29,7 @@ provider "aws" {
}
}
# s3 bucket - devops
resource "aws_s3_bucket" "devops" {
bucket = local.devops_s3_bucket_name
tags = local.common_tags
@@ -37,6 +40,29 @@ resource "aws_s3_bucket_policy" "deny_insecure_communication" {
policy = data.aws_iam_policy_document.deny_insecure_communication.json
}
# s3 bucket - raw-data-ingestion
resource "aws_s3_bucket" "raw_data_ingestion" {
bucket = local.raw_data_ingestion_s3_bucket_name
tags = local.common_tags
}
resource "aws_s3_bucket_policy" "raw_data_ingestion_policy" {
bucket = aws_s3_bucket.raw_data_ingestion.id
policy = data.aws_iam_policy_document.raw_data_ingestion_deny_insecure_communication.json
}
# s3 bucket - mwaa_resources
resource "aws_s3_bucket" "mwaa_resources" {
bucket = local.mwaa_resources_s3_bucket_name
tags = local.common_tags
}
resource "aws_s3_bucket_policy" "mwaa_resources_policy" {
bucket = aws_s3_bucket.mwaa_resources.id
policy = data.aws_iam_policy_document.mwaa_resources_deny_insecure_communication.json
}
# Deny insecure s3 bucket policy
data "aws_iam_policy_document" "deny_insecure_communication" {
statement {
sid = "DenyInsecureCommunications"
@@ -62,3 +88,285 @@ data "aws_iam_policy_document" "deny_insecure_communication" {
]
}
}
# Deny insecure s3 bucket policy - raw_data_ingestion
data "aws_iam_policy_document" "raw_data_ingestion_deny_insecure_communication" {
statement {
sid = "DenyInsecureCommunications"
principals {
type = "*"
identifiers = ["*"]
}
condition {
test = "Bool"
variable = "aws:SecureTransport"
values = ["false"]
}
effect = "Deny"
actions = ["s3:*"]
resources = [
aws_s3_bucket.raw_data_ingestion.arn,
"${aws_s3_bucket.raw_data_ingestion.arn}/*",
]
}
}
# Deny insecure s3 bucket policy - mwaa_resources
data "aws_iam_policy_document" "mwaa_resources_deny_insecure_communication" {
statement {
sid = "DenyInsecureCommunications"
principals {
type = "*"
identifiers = ["*"]
}
condition {
test = "Bool"
variable = "aws:SecureTransport"
values = ["false"]
}
effect = "Deny"
actions = ["s3:*"]
resources = [
aws_s3_bucket.mwaa_resources.arn,
"${aws_s3_bucket.mwaa_resources.arn}/*",
]
}
}
# IAM role - snowflake-integration-role
resource "aws_iam_role" "snowflake_integration_role" {
name = local.snowflake_integration_role_name
assume_role_policy = jsonencode({
Version = "2012-10-17"
Statement = [
{
Action = "sts:AssumeRole"
Effect = "Allow"
Sid = ""
Principal = {
AWS = "arn:aws:iam::851725635820:user/3nmi0000-s"
}
Condition = {
StringEquals = {
"sts:ExternalId" = "OQ11564_SFCRole=2_lNKzpyFn/e/xQgvGGpusVbEZA0A="
}
}
},
]
})
}
# Construct IAM Policy for snowflake-integration-role
data "aws_iam_policy_document" "snowflake_integration_policy_document" {
statement {
effect = "Allow"
actions = [
"s3:PutObject",
"s3:GetObject",
"s3:GetObjectVersion",
"s3:DeleteObject",
"s3:DeleteObjectVersion"
]
resources = ["${aws_s3_bucket.raw_data_ingestion.arn}*"]
}
statement {
effect = "Allow"
actions = [
"s3:ListBucket",
"s3:GetBucketLocation"
]
resources = ["${aws_s3_bucket.raw_data_ingestion.arn}"]
condition {
test = "StringLike"
variable = "s3:prefix"
values = ["*"]
}
}
}
# IAM Policy for snowflake-integration-role
resource "aws_iam_policy" "snowflake_integration_policy" {
name = local.snowflake_integration_policy_name
description = "Client textract permissions"
policy = data.aws_iam_policy_document.snowflake_integration_policy_document.json
}
# Attach IAM Policy to snowflake-integration-role
resource "aws_iam_role_policy_attachment" "snowflake_integration_policy_attachment" {
role = aws_iam_role.snowflake_integration_role.name
policy_arn = aws_iam_policy.snowflake_integration_policy.arn
}
# IAM role - cross-account-role
resource "aws_iam_role" "cross_account_role" {
name = local.cross_account_role_name
assume_role_policy = jsonencode({
Version = "2012-10-17"
Statement = [
{
Action = "sts:AssumeRole"
Effect = "Allow"
Sid = ""
Principal = {
AWS = "arn:aws:iam::873115228912:root"
}
},
]
})
}
# Construct IAM Policy for cross-account-role
data "aws_iam_policy_document" "cross_account_policy_document" {
statement {
effect = "Allow"
actions = [
"s3:PutObject",
"s3:GetObject",
"s3:ListBucket",
"s3:PutObjectAcl",
"s3:GetObjectVersion"
]
resources = [
"${aws_s3_bucket.raw_data_ingestion.arn}",
"${aws_s3_bucket.raw_data_ingestion.arn}/*"
]
}
}
# IAM Policy for cross-account-role
resource "aws_iam_policy" "cross_account_policy" {
name = local.cross_account_policy_name
description = "This role is used for other AWS account to assume inorder to drop files to our data ingestion bucket. At the time of creation, this is being used by CODE DEV to drop all clients list."
policy = data.aws_iam_policy_document.cross_account_policy_document.json
}
# Attach IAM Policy to cross-account-role
resource "aws_iam_role_policy_attachment" "cross_account_policy_attachment" {
role = aws_iam_role.cross_account_role.name
policy_arn = aws_iam_policy.cross_account_policy.arn
}
# IAM role - mwaa-exec-role
resource "aws_iam_role" "mwaa_exec_role" {
name = local.mwaa_exec_role_role_name
assume_role_policy = jsonencode({
Version = "2012-10-17"
Statement = [
{
Action = "sts:AssumeRole"
Effect = "Allow"
Sid = ""
Principal = {
Service = ["airflow-env.amazonaws.com","airflow.amazonaws.com"]
}
},
]
})
}
# Construct IAM Policy for mwaa-exec-role
data "aws_iam_policy_document" "mwaa_exec_policy_document" {
statement {
effect = "Allow"
actions = ["s3:*"]
resources = [
"${aws_s3_bucket.raw_data_ingestion.arn}/*"
]
}
statement {
effect = "Allow"
actions = ["airflow:CreateCliToken"]
resources = [
"arn:aws:airflow:us-east-2:660131068782:environment/doczy-dev-infra-mwaa"
]
}
statement {
effect = "Allow"
actions = ["airflow:PublishMetrics"]
resources = [
"arn:aws:airflow:us-east-2:660131068782:environment/doczy-dev-infra-mwaa"
]
}
statement {
effect = "Deny"
actions = ["s3:ListAllMyBuckets"]
resources = [
"${aws_s3_bucket.mwaa_resources.arn}",
"${aws_s3_bucket.mwaa_resources.arn}/*"
]
}
statement {
effect = "Allow"
actions = [
"s3:GetObject*",
"s3:GetBucket*",
"s3:List*",
"s3:PutObject"
]
resources = [
"${aws_s3_bucket.mwaa_resources.arn}",
"${aws_s3_bucket.mwaa_resources.arn}/*"
]
}
statement {
effect = "Allow"
actions = [
"logs:CreateLogStream",
"logs:CreateLogGroup",
"logs:PutLogEvents",
"logs:GetLogEvents",
"logs:GetLogRecord",
"logs:GetLogGroupFields",
"logs:GetQueryResults"
]
resources = [
"arn:aws:logs:us-east-2:660131068782:log-group:airflow-doczy-dev-infra-mwaa-*"
]
}
statement {
effect = "Allow"
actions = [
"logs:DescribeLogGroups"
]
resources = ["*"]
}
statement {
effect = "Allow"
actions = [
"cloudwatch:PutMetricData"
]
resources = ["*"]
}
}
# IAM Policy for mwaa-exec-role
resource "aws_iam_policy" "mwaa_exec_policy" {
name = local.mwaa_exec_policy_name
description = ""
policy = data.aws_iam_policy_document.mwaa_exec_policy_document.json
}
# Attach IAM Policy to mwaa-exec-role
resource "aws_iam_role_policy_attachment" "mwaa_exec_policy_attachment" {
role = aws_iam_role.mwaa_exec_role.name
policy_arn = aws_iam_policy.mwaa_exec_policy.arn
}
devops-pipeline/other-resources/outputs.tf
+22
View File
@@ -1,3 +1,25 @@
# S3 bucket name
output "devops_s3_bucket_name" {
value = aws_s3_bucket.devops.id
}
output "raw_data_ingestion_bucket_name" {
value = aws_s3_bucket.raw_data_ingestion.id
}
output "mwaa_resources_bucket_name" {
value = aws_s3_bucket.mwaa_resources.id
}
# IAM Role ARN
output "snowflake_integration_role_arn" {
value = aws_iam_role.snowflake_integration_role.arn
}
output "cross_account_role_arn" {
value = aws_iam_role.cross_account_role.id
}
output "mwaa_exec_role_arn" {
value = aws_iam_role.mwaa_exec_role.id
}
devops-pipeline/other-resources/variables.tf
+39 -14
View File
@@ -1,23 +1,23 @@
# required
# variable "aws_profile" {
variable "aws_profile" {
type = string
default = "doczyai"
}
# # required
# variable "secret_key" {
# type = string
# default = "doczyai"
# }
# required
variable "secret_key" {
type = string
}
# required
variable "access_key" {
type = string
}
# # required
# variable "access_key" {
# type = string
# }
# required
variable "aws_region" {
type = string
# default = "us-east-2"
default = "us-east-2"
}
# required
variable "project_name" {
@@ -27,7 +27,7 @@ variable "project_name" {
# required
variable "environment" {
type = string
# default = "dev"
default = "dev"
}
# required
variable "client_name" {
@@ -44,4 +44,29 @@ variable "devops_s3_bucket" {
name = "devops-resources"
}
}
}
# s3 bucket - raw-data-ingestion
variable "raw_data_ingestion_s3_bucket" {
type = object({
name = string
})
default = {
name = "raw-data-ingestion"
}
}
# s3 bucket - mwaa-resources
variable "mwaa_resources_s3_bucket" {
type = object({
name = string
})
default = {
name = "mwaa-resources"
}
}