diff --git a/terraform-approach/src/lambda-layer/libreoffice/lo.tar.zip b/terraform-approach/src/lambda-layer/libreoffice/lo.tar.zip new file mode 100644 index 0000000..6a2b9a6 Binary files /dev/null and b/terraform-approach/src/lambda-layer/libreoffice/lo.tar.zip differ diff --git a/terraform-approach/src/lambda/textract-receiver/index.py b/terraform-approach/src/lambda/textract-receiver/index.py new file mode 100644 index 0000000..1f11d80 --- /dev/null +++ b/terraform-approach/src/lambda/textract-receiver/index.py @@ -0,0 +1,3 @@ +# AWS Lambda handler function +def lambda_handler(event, context): + print("Receiver Lambda") \ No newline at end of file diff --git a/terraform-approach/src/lambda/textract-sender/index.py b/terraform-approach/src/lambda/textract-sender/index.py new file mode 100644 index 0000000..015fbee --- /dev/null +++ b/terraform-approach/src/lambda/textract-sender/index.py @@ -0,0 +1,3 @@ +# AWS Lambda handler function +def lambda_handler(event, context): + print("Sender Lambda") \ No newline at end of file diff --git a/terraform-approach/terraform/config.tfvars b/terraform-approach/terraform/config.tfvars new file mode 100644 index 0000000..e69de29 diff --git a/terraform-approach/terraform/main.tf b/terraform-approach/terraform/main.tf new file mode 100644 index 0000000..7e1b0e3 --- /dev/null +++ b/terraform-approach/terraform/main.tf @@ -0,0 +1,42 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "5.39.0" + } + } + + backend "s3" { + bucket = "doczy-test-bucket" # Parameterize using -backend-config flag with "terraform init" + key = "terraform/terraform_new2.tfstate" # Parameterize + region = "us-east-2" # Parameterize + profile = "doczyai" # Parameterize + dynamodb_table = "terraform-lock" # Parameterize + encrypt = true + } +} + +provider "aws" { + profile = var.aws_profile +} + +module "textract_pipeline" { + source = "./modules/aws-textract-pipeline" + project_name = var.project_name + aws_region = var.aws_region + environment = var.environment + + # for each for multiple resources + for_each = var.client_list + + client_name = each.value.client_name + s3_bucket_name = each.value.s3_bucket_name + + lambda_role = "arn:aws:iam::660131068782:role/doczy-dev-infra-lambda-universal-role" + +} + + +# List - multiple client, separate state files +# client abbrv +# stream-wise folders or repo \ No newline at end of file diff --git a/terraform-approach/terraform/modules/aws-textract-pipeline-part2/main.tf b/terraform-approach/terraform/modules/aws-textract-pipeline-part2/main.tf new file mode 100644 index 0000000..1da8a80 --- /dev/null +++ b/terraform-approach/terraform/modules/aws-textract-pipeline-part2/main.tf @@ -0,0 +1,212 @@ +locals { + region_map = { + us-east-1 = "use1" + us-east-2 = "use2" + us-west-1 = "usw1" + us-west-2 = "usw2" + } + environment_map = { + prod = "p" + uat = "u" + qa = "q" + dev = "d" + } + + common_tags = { + environment = var.environment + + } + + # region abbreviation + region_short_name = local.region_map[var.aws_region] + + # environment abbreviation + environment_short_name = local.environment_map[var.environment] + + # global prefix + global_prefix = "${var.project_name}-${local.region_short_name}-${local.environment_short_name}-${var.client_name}" + + # Resource prefix + lambda_prefix = "${local.global_prefix}-lmb" + sqs_prefix = "${local.global_prefix}-sqs" + sns_prefix = "${local.global_prefix}-sns" + + # Lambda names + sender_lambda_name = "${local.lambda_prefix}-${var.textract_sender_lambda.name}" + receiver_lambda_name = "${local.lambda_prefix}-${var.textract_receiver_lambda.name}" + + # SQS names + sender_queue_name = "${local.sqs_prefix}-${var.sender_sqs_queue.name}" + receiver_queue_name = "${local.sqs_prefix}-${var.receiver_sqs_queue.name}" + + # SNS names + textract_notification_sns_name = "${local.sns_prefix}-textract-notification" + + # property file path + property_file_s3_path = "${var.s3_bucket_name}/config.properties" +} + +# sender lambda function +module "textract_sender_lambda_function" { + source = "terraform-aws-modules/lambda/aws" + version = "7.2.2" + + function_name = local.sender_lambda_name + description = var.textract_sender_lambda.description + handler = var.textract_sender_lambda.handler + runtime = var.textract_sender_lambda.runtime + timeout = var.textract_sender_lambda.timeout + memory_size = var.textract_sender_lambda.memory_size + ephemeral_storage_size = var.textract_sender_lambda.ephemeral_storage_size + architectures = var.textract_sender_lambda.architectures + + environment_variables = { + PROPERTY_FILE_S3_PATH = local.property_file_s3_path + } + + source_path = var.textract_sender_lambda.source_path + + event_source_mapping = { + sqs = { + event_source_arn = module.textract_sender_sqs.queue_arn + function_response_types = ["ReportBatchItemFailures"] + scaling_config = { + maximum_concurrency = 20 + + } + batch_size = 1 + } + } + + lambda_role = var.lambda_role + create_role = false + + tags = local.common_tags +} + +# Sender SQS Queue +module "textract_sender_sqs" { + source = "terraform-aws-modules/sqs/aws" + version = "4.1.1" + name = local.sender_queue_name + delay_seconds = var.sender_sqs_queue.delay_seconds + max_message_size = var.sender_sqs_queue.max_message_size + message_retention_seconds = var.sender_sqs_queue.message_retention_seconds + visibility_timeout_seconds= var.sender_sqs_queue.visibility_timeout_seconds + tags = local.common_tags +} + +# receiver lambda function +module "textract_receiver_lambda_function" { + source = "terraform-aws-modules/lambda/aws" + version = "7.2.2" + + function_name = local.receiver_lambda_name + description = var.textract_receiver_lambda.description + handler = var.textract_receiver_lambda.handler + runtime = var.textract_receiver_lambda.runtime + timeout = var.textract_receiver_lambda.timeout + memory_size = var.textract_receiver_lambda.memory_size + ephemeral_storage_size = var.textract_receiver_lambda.ephemeral_storage_size + architectures = var.textract_receiver_lambda.architectures + + environment_variables = { + PROPERTY_FILE_S3_PATH = local.property_file_s3_path + } + + source_path = var.textract_receiver_lambda.source_path + + event_source_mapping = { + sqs = { + event_source_arn = module.textract_receiver_sqs.queue_arn + function_response_types = ["ReportBatchItemFailures"] + scaling_config = { + maximum_concurrency = 20 + } + batch_size = 1 + } + } + + lambda_role = var.lambda_role + create_role = false + + tags = local.common_tags +} + +# textract notification sns +module "textract_notification_sns" { + source = "terraform-aws-modules/sns/aws" + version = ">= 5.0" + + name = local.textract_notification_sns_name + + topic_policy_statements = { + sqs = { + sid = "SQSSubscribe" + actions = [ + "sns:Subscribe", + "sns:Receive", + ] + + principals = [{ + type = "AWS" + identifiers = ["*"] + }] + + conditions = [{ + test = "StringLike" + variable = "sns:Endpoint" + values = [module.textract_receiver_sqs.queue_arn] + }] + } + } + + subscriptions = { + sqs = { + protocol = "sqs" + endpoint = module.textract_receiver_sqs.queue_arn + } + } + + tags = { + Environment = "dev" + } +} + +# Receiver SQS Queue +module "textract_receiver_sqs" { + source = "terraform-aws-modules/sqs/aws" + version = "4.1.1" + name = local.receiver_queue_name + delay_seconds = var.receiver_sqs_queue.delay_seconds + max_message_size = var.receiver_sqs_queue.max_message_size + message_retention_seconds = var.receiver_sqs_queue.message_retention_seconds + visibility_timeout_seconds= var.receiver_sqs_queue.visibility_timeout_seconds + tags = local.common_tags + + create_queue_policy = true + queue_policy_statements = { + sns = { + sid = "SNSPublish" + actions = ["sqs:SendMessage"] + + principals = [ + { + type = "Service" + identifiers = ["sns.amazonaws.com"] + } + ] + + conditions = [{ + test = "ArnEquals" + variable = "aws:SourceArn" + values = [module.textract_notification_sns.topic_arn] + }] + } + } +} + + + + + diff --git a/terraform-approach/terraform/modules/aws-textract-pipeline-part2/outputs.tf b/terraform-approach/terraform/modules/aws-textract-pipeline-part2/outputs.tf new file mode 100644 index 0000000..52923fc --- /dev/null +++ b/terraform-approach/terraform/modules/aws-textract-pipeline-part2/outputs.tf @@ -0,0 +1,26 @@ +# Output variable definitions + +output "sender_lambda_arn" { + description = "Sender lambda arn" + value = module.textract_sender_lambda_function.lambda_function_arn +} + +output "receiver_lambda_arn" { + description = "Receiver lambda arn" + value = module.textract_receiver_lambda_function.lambda_function_arn +} + +output "sender_queue_arn" { + description = "Sender queue arn" + value = module.textract_sender_sqs.queue_arn +} + +output "receiver_queue_arn" { + description = "Receiver queue arn" + value = module.textract_receiver_sqs.queue_arn +} + +output "textract_notification_sns_arn" { + description = "value" + value = module.textract_notification_sns.topic_arn +} \ No newline at end of file diff --git a/terraform-approach/terraform/modules/aws-textract-pipeline-part2/variables.tf b/terraform-approach/terraform/modules/aws-textract-pipeline-part2/variables.tf new file mode 100644 index 0000000..a29e909 --- /dev/null +++ b/terraform-approach/terraform/modules/aws-textract-pipeline-part2/variables.tf @@ -0,0 +1,130 @@ +# Required +variable "project_name" { + type = string +} + +# Required +variable "client_name" { + type = string +} + + +# Required +variable "environment" { + type = string +} + +# Required +variable "aws_region" { + type = string +} + +# Required +variable "lambda_role" { + type = string +} + +# Required +variable "s3_bucket_name" { + type = string +} + +# textract_sender_lambda +variable "textract_sender_lambda" { + type = object({ + name = string + description = string + handler = string + runtime = string + timeout = number + memory_size = number + ephemeral_storage_size = number + architectures = list(string) + layers = list(string) + environment_variables = map(string) + source_path = string + }) + default = { + name = "textract-sender" + description = "This function sends async request to TEXTRACT service" + handler = "index.lambda_handler" + runtime = "python3.12" + timeout = 60 + memory_size = 256 + ephemeral_storage_size = 512 + architectures = ["x86_64"] + layers = [] + environment_variables = { + PROPERTY_FILE_S3_PATH = "doczy-dev-infra-textract/config.properties" + } + source_path = "../src/lambda/textract-sender" + } +} + +# textract sender queue +variable "sender_sqs_queue" { + type = object({ + name = string + delay_seconds = number + max_message_size = number + message_retention_seconds = number + visibility_timeout_seconds= number + }) + default = { + name = "textract-sender-queue" + delay_seconds = 90 + max_message_size = 2048 + message_retention_seconds = 86400 + visibility_timeout_seconds= 901 + } +} + +# textract_receiver_lambda +variable "textract_receiver_lambda" { + type = object({ + name = string + description = string + handler = string + runtime = string + timeout = number + memory_size = number + ephemeral_storage_size = number + architectures = list(string) + layers = list(string) + environment_variables = map(string) + source_path = string + }) + default = { + name = "textract-receiver" + description = "This function receive async request to TEXTRACT service" + handler = "index.lambda_handler" + runtime = "python3.12" + timeout = 60 + memory_size = 256 + ephemeral_storage_size = 512 + architectures = ["x86_64"] + layers = [] + environment_variables = { + PROPERTY_FILE_S3_PATH = "doczy-dev-infra-textract/config.properties" + } + source_path = "../src/lambda/textract-receiver" + } +} + +# textract receiver queue +variable "receiver_sqs_queue" { + type = object({ + name = string + delay_seconds = number + max_message_size = number + message_retention_seconds = number + visibility_timeout_seconds= number + }) + default = { + name = "textract-receiver-queue" + delay_seconds = 90 + max_message_size = 2048 + message_retention_seconds = 86400 + visibility_timeout_seconds= 901 + } +} diff --git a/terraform-approach/terraform/outputs.tf b/terraform-approach/terraform/outputs.tf new file mode 100644 index 0000000..7caebf7 --- /dev/null +++ b/terraform-approach/terraform/outputs.tf @@ -0,0 +1,25 @@ +# Output variable definitions + +output "sender_lambda_arn" { + description = "Sender lambda arn" + value = [ for output_list in module.textract_pipeline : output_list.sender_lambda_arn] +} + +output "receiver_lambda_arn" { + description = "Receiver lambda arn" + value = [ for output_list in module.textract_pipeline : output_list.receiver_lambda_arn] +} + +output "sender_queue_arn" { + description = "Sender queue arn" + value = [ for output_list in module.textract_pipeline : output_list.sender_queue_arn] +} + +output "receiver_queue_arn" { + description = "Receiver queue arn" + value = [ for output_list in module.textract_pipeline : output_list.receiver_queue_arn] +} + +output "textract_notification_sns_arn" { + value = [ for output_list in module.textract_pipeline : output_list.textract_notification_sns_arn] +} \ No newline at end of file diff --git a/terraform-approach/terraform/variables.tf b/terraform-approach/terraform/variables.tf new file mode 100644 index 0000000..0153b6b --- /dev/null +++ b/terraform-approach/terraform/variables.tf @@ -0,0 +1,37 @@ +variable "aws_profile" { + type = string + default = "doczyai" +} + +variable "aws_region" { + type = string + default = "us-east-2" +} + +variable "project_name" { + type = string + default = "doczyai" +} + +variable "environment" { + type = string + default = "dev" +} + +variable "client_list" { + type = map(object({ + client_name = string + s3_bucket_name = string + })) + default = { + client1 = { + s3_bucket_name = "doczy-dev-infra-textract" + client_name = "cn1" + }, + client2 = { + s3_bucket_name = "doczy-dev-infra-textract" + client_name = "cn2" + } + + } +} \ No newline at end of file diff --git a/terraform/textract-approach3/src/lambda-layer/libreoffice/lo.tar.zip b/terraform/textract-approach3/src/lambda-layer/libreoffice/lo.tar.zip new file mode 100644 index 0000000..6a2b9a6 Binary files /dev/null and b/terraform/textract-approach3/src/lambda-layer/libreoffice/lo.tar.zip differ diff --git a/terraform/textract-approach3/src/lambda/textract-receiver/index.py b/terraform/textract-approach3/src/lambda/textract-receiver/index.py new file mode 100644 index 0000000..1f11d80 --- /dev/null +++ b/terraform/textract-approach3/src/lambda/textract-receiver/index.py @@ -0,0 +1,3 @@ +# AWS Lambda handler function +def lambda_handler(event, context): + print("Receiver Lambda") \ No newline at end of file diff --git a/terraform/textract-approach3/src/lambda/textract-sender/index.py b/terraform/textract-approach3/src/lambda/textract-sender/index.py new file mode 100644 index 0000000..015fbee --- /dev/null +++ b/terraform/textract-approach3/src/lambda/textract-sender/index.py @@ -0,0 +1,3 @@ +# AWS Lambda handler function +def lambda_handler(event, context): + print("Sender Lambda") \ No newline at end of file