Added terraform approach folder

2024-03-29 16:31:28 +05:30
parent b260c23ebe
commit e652b63800
13 changed files with 484 additions and 0 deletions
@@ -0,0 +1,3 @@
+# AWS Lambda handler function
+def lambda_handler(event, context):
+    print("Receiver Lambda")
@@ -0,0 +1,3 @@
+# AWS Lambda handler function
+def lambda_handler(event, context):
+    print("Sender Lambda")
@@ -0,0 +1,42 @@
+terraform {
+  required_providers {
+    aws = {
+      source = "hashicorp/aws"
+      version = "5.39.0"
+    }
+  }
+
+  backend "s3" {
+    bucket         = "doczy-test-bucket" # Parameterize using -backend-config flag with "terraform init"
+    key            = "terraform/terraform_new2.tfstate" # Parameterize
+    region         = "us-east-2" # Parameterize 
+    profile        = "doczyai" # Parameterize
+    dynamodb_table = "terraform-lock" # Parameterize
+    encrypt        = true
+  }
+}
+
+provider "aws" {
+  profile = var.aws_profile
+}
+
+module "textract_pipeline" {
+  source = "./modules/aws-textract-pipeline"
+  project_name = var.project_name
+  aws_region = var.aws_region
+  environment = var.environment
+
+  # for each for multiple resources
+  for_each = var.client_list
+
+  client_name = each.value.client_name
+  s3_bucket_name = each.value.s3_bucket_name
+
+  lambda_role = "arn:aws:iam::660131068782:role/doczy-dev-infra-lambda-universal-role"
+  
+}
+
+
+# List - multiple client, separate state files
+# client abbrv
+# stream-wise folders or repo
@@ -0,0 +1,212 @@
+locals {
+  region_map = {
+    us-east-1 = "use1"
+    us-east-2 = "use2"
+    us-west-1 = "usw1"
+    us-west-2 = "usw2"
+  }
+  environment_map = {
+    prod    = "p"
+    uat     = "u"
+    qa      = "q"
+    dev     = "d"
+  }
+
+  common_tags = {
+    environment = var.environment
+
+  }
+
+  # region abbreviation
+  region_short_name   = local.region_map[var.aws_region]
+
+  # environment abbreviation
+  environment_short_name   = local.environment_map[var.environment]
+
+  # global prefix
+  global_prefix = "${var.project_name}-${local.region_short_name}-${local.environment_short_name}-${var.client_name}"
+  
+  # Resource prefix
+  lambda_prefix = "${local.global_prefix}-lmb"
+  sqs_prefix = "${local.global_prefix}-sqs"
+  sns_prefix = "${local.global_prefix}-sns"
+  
+  # Lambda names
+  sender_lambda_name = "${local.lambda_prefix}-${var.textract_sender_lambda.name}"
+  receiver_lambda_name = "${local.lambda_prefix}-${var.textract_receiver_lambda.name}"
+
+  # SQS names
+  sender_queue_name = "${local.sqs_prefix}-${var.sender_sqs_queue.name}"
+  receiver_queue_name = "${local.sqs_prefix}-${var.receiver_sqs_queue.name}"
+
+  # SNS names
+  textract_notification_sns_name = "${local.sns_prefix}-textract-notification"
+
+  # property file path
+  property_file_s3_path = "${var.s3_bucket_name}/config.properties"
+}
+
+# sender lambda function
+module "textract_sender_lambda_function" {
+  source = "terraform-aws-modules/lambda/aws"
+  version = "7.2.2"
+
+  function_name = local.sender_lambda_name
+  description   = var.textract_sender_lambda.description
+  handler       = var.textract_sender_lambda.handler
+  runtime       = var.textract_sender_lambda.runtime
+  timeout       = var.textract_sender_lambda.timeout
+  memory_size   = var.textract_sender_lambda.memory_size
+  ephemeral_storage_size  = var.textract_sender_lambda.ephemeral_storage_size
+  architectures = var.textract_sender_lambda.architectures
+  
+  environment_variables = {
+    PROPERTY_FILE_S3_PATH = local.property_file_s3_path
+  }
+
+  source_path   = var.textract_sender_lambda.source_path
+
+  event_source_mapping = {
+    sqs = {
+      event_source_arn        = module.textract_sender_sqs.queue_arn
+      function_response_types = ["ReportBatchItemFailures"]
+      scaling_config = {
+        maximum_concurrency = 20
+        
+      }
+      batch_size = 1
+    }
+  }
+
+  lambda_role  = var.lambda_role
+  create_role = false
+
+  tags    = local.common_tags
+}
+
+# Sender SQS Queue
+module "textract_sender_sqs" {
+  source  = "terraform-aws-modules/sqs/aws"
+  version = "4.1.1"
+  name    = local.sender_queue_name
+  delay_seconds             = var.sender_sqs_queue.delay_seconds
+  max_message_size          = var.sender_sqs_queue.max_message_size
+  message_retention_seconds = var.sender_sqs_queue.message_retention_seconds
+  visibility_timeout_seconds= var.sender_sqs_queue.visibility_timeout_seconds
+  tags    = local.common_tags
+}
+
+# receiver lambda function
+module "textract_receiver_lambda_function" {
+  source = "terraform-aws-modules/lambda/aws"
+  version = "7.2.2"
+
+  function_name = local.receiver_lambda_name
+  description   = var.textract_receiver_lambda.description
+  handler       = var.textract_receiver_lambda.handler
+  runtime       = var.textract_receiver_lambda.runtime
+  timeout       = var.textract_receiver_lambda.timeout
+  memory_size   = var.textract_receiver_lambda.memory_size
+  ephemeral_storage_size  = var.textract_receiver_lambda.ephemeral_storage_size
+  architectures = var.textract_receiver_lambda.architectures
+  
+  environment_variables = {
+    PROPERTY_FILE_S3_PATH = local.property_file_s3_path
+  }
+
+  source_path   = var.textract_receiver_lambda.source_path
+
+  event_source_mapping = {
+    sqs = {
+      event_source_arn        = module.textract_receiver_sqs.queue_arn
+      function_response_types = ["ReportBatchItemFailures"]
+      scaling_config = {
+        maximum_concurrency = 20 
+      }
+      batch_size = 1
+    }
+  }
+
+  lambda_role  = var.lambda_role
+  create_role = false
+
+  tags    = local.common_tags
+}
+
+# textract notification sns 
+module "textract_notification_sns" {
+  source  = "terraform-aws-modules/sns/aws"
+  version = ">= 5.0"
+
+  name = local.textract_notification_sns_name
+
+  topic_policy_statements = {
+    sqs = {
+      sid = "SQSSubscribe"
+      actions = [
+        "sns:Subscribe",
+        "sns:Receive",
+      ]
+
+      principals = [{
+        type        = "AWS"
+        identifiers = ["*"]
+      }]
+
+      conditions = [{
+        test     = "StringLike"
+        variable = "sns:Endpoint"
+        values   = [module.textract_receiver_sqs.queue_arn]
+      }]
+    }
+  }
+
+  subscriptions = {
+    sqs = {
+      protocol = "sqs"
+      endpoint = module.textract_receiver_sqs.queue_arn
+    }
+  }
+
+  tags = {
+    Environment = "dev"
+  }
+}
+
+# Receiver SQS Queue
+module "textract_receiver_sqs" {
+  source  = "terraform-aws-modules/sqs/aws"
+  version = "4.1.1"
+  name    = local.receiver_queue_name
+  delay_seconds             = var.receiver_sqs_queue.delay_seconds
+  max_message_size          = var.receiver_sqs_queue.max_message_size
+  message_retention_seconds = var.receiver_sqs_queue.message_retention_seconds
+  visibility_timeout_seconds= var.receiver_sqs_queue.visibility_timeout_seconds
+  tags    = local.common_tags
+
+  create_queue_policy = true
+  queue_policy_statements = {
+    sns = {
+      sid     = "SNSPublish"
+      actions = ["sqs:SendMessage"]
+
+      principals = [
+        {
+          type        = "Service"
+          identifiers = ["sns.amazonaws.com"]
+        }
+      ]
+
+      conditions = [{
+        test     = "ArnEquals"
+        variable = "aws:SourceArn"
+        values   = [module.textract_notification_sns.topic_arn]
+      }]
+    }
+  }
+}
+
+
+
+
+
@@ -0,0 +1,26 @@
+# Output variable definitions
+
+output "sender_lambda_arn" {
+  description = "Sender lambda arn"
+  value       = module.textract_sender_lambda_function.lambda_function_arn
+}
+
+output "receiver_lambda_arn" {
+  description = "Receiver lambda arn"
+  value       = module.textract_receiver_lambda_function.lambda_function_arn
+}
+
+output "sender_queue_arn" {
+  description = "Sender queue arn"
+  value       = module.textract_sender_sqs.queue_arn
+}
+
+output "receiver_queue_arn" {
+  description = "Receiver queue arn"
+  value       = module.textract_receiver_sqs.queue_arn
+}
+
+output "textract_notification_sns_arn" {
+  description = "value"
+  value = module.textract_notification_sns.topic_arn
+}
@@ -0,0 +1,130 @@
+# Required
+variable "project_name" {
+  type = string
+}
+
+# Required
+variable "client_name" {
+  type = string
+}
+
+
+# Required
+variable "environment" {
+  type = string
+}
+
+# Required
+variable "aws_region" {
+  type = string
+}
+
+# Required
+variable "lambda_role" {
+  type = string
+}
+
+# Required
+variable "s3_bucket_name" {
+  type = string
+}
+
+# textract_sender_lambda
+variable "textract_sender_lambda" {
+  type = object({
+    name    = string
+    description = string
+    handler = string
+    runtime = string
+    timeout = number
+    memory_size = number
+    ephemeral_storage_size = number
+    architectures = list(string)
+    layers = list(string)
+    environment_variables = map(string)
+    source_path = string
+  })
+  default = {
+    name = "textract-sender"
+    description = "This function sends async request to TEXTRACT service"
+    handler = "index.lambda_handler"
+    runtime = "python3.12"
+    timeout = 60
+    memory_size = 256
+    ephemeral_storage_size = 512
+    architectures = ["x86_64"]
+    layers = []
+    environment_variables = {
+      PROPERTY_FILE_S3_PATH = "doczy-dev-infra-textract/config.properties"
+    }
+    source_path = "../src/lambda/textract-sender"
+  }
+}
+
+# textract sender queue
+variable "sender_sqs_queue" {
+    type = object({
+      name = string
+      delay_seconds = number
+      max_message_size          = number
+      message_retention_seconds = number
+      visibility_timeout_seconds= number
+    })
+    default = {
+      name                      = "textract-sender-queue"
+      delay_seconds             = 90
+      max_message_size          = 2048
+      message_retention_seconds = 86400
+      visibility_timeout_seconds= 901
+    } 
+}
+
+# textract_receiver_lambda
+variable "textract_receiver_lambda" {
+  type = object({
+    name    = string
+    description = string
+    handler = string
+    runtime = string
+    timeout = number
+    memory_size = number
+    ephemeral_storage_size = number
+    architectures = list(string)
+    layers = list(string)
+    environment_variables = map(string)
+    source_path = string
+  })
+  default = {
+    name = "textract-receiver"
+    description = "This function receive async request to TEXTRACT service"
+    handler = "index.lambda_handler"
+    runtime = "python3.12"
+    timeout = 60
+    memory_size = 256
+    ephemeral_storage_size = 512
+    architectures = ["x86_64"]
+    layers = []
+    environment_variables = {
+      PROPERTY_FILE_S3_PATH = "doczy-dev-infra-textract/config.properties"
+    }
+    source_path = "../src/lambda/textract-receiver"
+  }
+}
+
+# textract receiver queue 
+variable "receiver_sqs_queue" {
+    type = object({
+      name = string
+      delay_seconds = number
+      max_message_size          = number
+      message_retention_seconds = number
+      visibility_timeout_seconds= number
+    })
+    default = {
+      name                      = "textract-receiver-queue"
+      delay_seconds             = 90
+      max_message_size          = 2048
+      message_retention_seconds = 86400
+      visibility_timeout_seconds= 901
+    } 
+}
@@ -0,0 +1,25 @@
+# Output variable definitions
+
+output "sender_lambda_arn" {
+  description = "Sender lambda arn"
+  value = [ for output_list in module.textract_pipeline : output_list.sender_lambda_arn]
+}
+
+output "receiver_lambda_arn" {
+  description = "Receiver lambda arn"
+  value = [ for output_list in module.textract_pipeline : output_list.receiver_lambda_arn]
+}
+
+output "sender_queue_arn" {
+  description = "Sender queue arn"
+  value = [ for output_list in module.textract_pipeline : output_list.sender_queue_arn]
+}
+
+output "receiver_queue_arn" {
+  description = "Receiver queue arn"
+  value = [ for output_list in module.textract_pipeline : output_list.receiver_queue_arn]
+}
+
+output "textract_notification_sns_arn" {
+    value = [ for output_list in module.textract_pipeline : output_list.textract_notification_sns_arn]
+}
@@ -0,0 +1,37 @@
+variable "aws_profile" {
+  type = string
+  default = "doczyai"
+}
+
+variable "aws_region" {
+  type = string
+  default = "us-east-2"
+}
+
+variable "project_name" {
+  type = string
+  default = "doczyai"
+}
+
+variable "environment" {
+  type = string
+  default = "dev"
+}
+
+variable "client_list" {
+  type = map(object({
+    client_name = string
+    s3_bucket_name = string
+  }))
+  default = {
+    client1 = {
+      s3_bucket_name  = "doczy-dev-infra-textract"
+      client_name = "cn1"
+    },
+    client2 = {
+      s3_bucket_name  = "doczy-dev-infra-textract"
+      client_name = "cn2"
+    }
+    
+  }
+}
@@ -0,0 +1,3 @@
+# AWS Lambda handler function
+def lambda_handler(event, context):
+    print("Receiver Lambda")
@@ -0,0 +1,3 @@
+# AWS Lambda handler function
+def lambda_handler(event, context):
+    print("Sender Lambda")