diff --git a/textract-pipeline/src/lambda-layer/brotli-layer/brotlipy-layer.zip b/textract-pipeline/src/lambda-layer/brotli-layer/brotlipy-layer.zip new file mode 100644 index 0000000..2294e36 Binary files /dev/null and b/textract-pipeline/src/lambda-layer/brotli-layer/brotlipy-layer.zip differ diff --git a/textract-pipeline/src/lambda-layer/libre-office/lo.tar.zip b/textract-pipeline/src/lambda-layer/libre-office/lo.tar.zip new file mode 100644 index 0000000..6a2b9a6 Binary files /dev/null and b/textract-pipeline/src/lambda-layer/libre-office/lo.tar.zip differ diff --git a/textract-pipeline/src/lambda-layer/pypdf2-layer/PyPDF2.zip b/textract-pipeline/src/lambda-layer/pypdf2-layer/PyPDF2.zip new file mode 100644 index 0000000..334c6d0 Binary files /dev/null and b/textract-pipeline/src/lambda-layer/pypdf2-layer/PyPDF2.zip differ diff --git a/textract-pipeline/src/lambda/batch-creation/index.py b/textract-pipeline/src/lambda/batch-creation/index.py new file mode 100644 index 0000000..015fbee --- /dev/null +++ b/textract-pipeline/src/lambda/batch-creation/index.py @@ -0,0 +1,3 @@ +# AWS Lambda handler function +def lambda_handler(event, context): + print("Sender Lambda") \ No newline at end of file diff --git a/textract-pipeline/src/lambda/database-interface/index.py b/textract-pipeline/src/lambda/database-interface/index.py new file mode 100644 index 0000000..015fbee --- /dev/null +++ b/textract-pipeline/src/lambda/database-interface/index.py @@ -0,0 +1,3 @@ +# AWS Lambda handler function +def lambda_handler(event, context): + print("Sender Lambda") \ No newline at end of file diff --git a/textract-pipeline/src/lambda/docx-to-pdf/index.py b/textract-pipeline/src/lambda/docx-to-pdf/index.py new file mode 100644 index 0000000..015fbee --- /dev/null +++ b/textract-pipeline/src/lambda/docx-to-pdf/index.py @@ -0,0 +1,3 @@ +# AWS Lambda handler function +def lambda_handler(event, context): + print("Sender Lambda") \ No newline at end of file diff --git a/textract-pipeline/src/lambda/insert-record/index.py b/textract-pipeline/src/lambda/insert-record/index.py new file mode 100644 index 0000000..015fbee --- /dev/null +++ b/textract-pipeline/src/lambda/insert-record/index.py @@ -0,0 +1,3 @@ +# AWS Lambda handler function +def lambda_handler(event, context): + print("Sender Lambda") \ No newline at end of file diff --git a/textract-pipeline/src/lambda/pdf-validation/index.py b/textract-pipeline/src/lambda/pdf-validation/index.py new file mode 100644 index 0000000..015fbee --- /dev/null +++ b/textract-pipeline/src/lambda/pdf-validation/index.py @@ -0,0 +1,3 @@ +# AWS Lambda handler function +def lambda_handler(event, context): + print("Sender Lambda") \ No newline at end of file diff --git a/textract-pipeline/src/lambda/text-creation/index.py b/textract-pipeline/src/lambda/text-creation/index.py new file mode 100644 index 0000000..015fbee --- /dev/null +++ b/textract-pipeline/src/lambda/text-creation/index.py @@ -0,0 +1,3 @@ +# AWS Lambda handler function +def lambda_handler(event, context): + print("Sender Lambda") \ No newline at end of file diff --git a/textract-pipeline/src/lambda/textract-receiver/index.py b/textract-pipeline/src/lambda/textract-receiver/index.py new file mode 100644 index 0000000..1f11d80 --- /dev/null +++ b/textract-pipeline/src/lambda/textract-receiver/index.py @@ -0,0 +1,3 @@ +# AWS Lambda handler function +def lambda_handler(event, context): + print("Receiver Lambda") \ No newline at end of file diff --git a/textract-pipeline/src/lambda/textract-sender/index.py b/textract-pipeline/src/lambda/textract-sender/index.py new file mode 100644 index 0000000..015fbee --- /dev/null +++ b/textract-pipeline/src/lambda/textract-sender/index.py @@ -0,0 +1,3 @@ +# AWS Lambda handler function +def lambda_handler(event, context): + print("Sender Lambda") \ No newline at end of file diff --git a/textract-pipeline/src/lambda/tiff-to-pdf/index.py b/textract-pipeline/src/lambda/tiff-to-pdf/index.py new file mode 100644 index 0000000..015fbee --- /dev/null +++ b/textract-pipeline/src/lambda/tiff-to-pdf/index.py @@ -0,0 +1,3 @@ +# AWS Lambda handler function +def lambda_handler(event, context): + print("Sender Lambda") \ No newline at end of file diff --git a/textract-pipeline/src/lambda/trigger-pipeline/index.py b/textract-pipeline/src/lambda/trigger-pipeline/index.py new file mode 100644 index 0000000..015fbee --- /dev/null +++ b/textract-pipeline/src/lambda/trigger-pipeline/index.py @@ -0,0 +1,3 @@ +# AWS Lambda handler function +def lambda_handler(event, context): + print("Sender Lambda") \ No newline at end of file diff --git a/textract-pipeline/src/s3-files/config.properties b/textract-pipeline/src/s3-files/config.properties new file mode 100644 index 0000000..85035dc --- /dev/null +++ b/textract-pipeline/src/s3-files/config.properties @@ -0,0 +1,28 @@ +[RESOURCES] +SNS_TOPIC_ARN=arn:aws:sns:{aws_region}:{aws_account_id}:doczy-textract-sender-sns-dev +TEXTRACT_ROLE_ARN=arn:aws:iam::{aws_account_id}:role/doczy-dev-textract-role +SQS_QUEUE_URL=https://sqs.{aws_region}.amazonaws.com/{aws_account_id}/doczy-textract-sender-sqs-dev +PDF_SQS_QUEUE_URL=https://sqs.{aws_region}.amazonaws.com/{aws_account_id}/doczy-textract-pdf-sqs-dev + +[FOLDER_LOCATIONS] +CONTRACTS_LANDNING_ZONE=contracts_landing_zone/ +ALL_PDF_LOCATION=batches/{}/all_pdf_files/ +SOURCE_LOCATION=batches/{}/source-documents-pdf/ +SOURCE_DOCX_LOCATION=batches/{}/source-documents-docx/ +SOURCE_DOCX_PROCESSED_LOCATION=batches/{}/source-documents-docx-processed/ +SOURCE_DOCX_UNPROCESSED_LOCATION=batches/{}/source-documents-docx-unprocessed/ +STAGING_LOCATION=batches/{}/textract-sender-staging-pdf/ +OUTPUT_LOCATION=batches/{}/textract-output-json/ +PROCESSED_LOCATION=batches/{}/textract-receiver-processed-file/ +UNPROCESSED_LOCATION=batches/{}/textract-receiver-unprocessed-file/ +CONTRACT_TEXT_FILE_LOCATION=batches/{}/contract-text-file/ +LLM_RESPONSE_FILE_LOCATION=batches/{}/llm-response-file/ +INVALID_PDF_FILE_LOCATION=batches/{}/invalid-pdf-file/ + + +[OTHERS] +SENDER_MAX_FILES=10 +PROCESS_TYPE=ANALYSIS +ANALYSIS_FEATURE_TYPE=LAYOUT +JOB_TAG=healthcare-contract +LOG_LEVEL=DEBUG \ No newline at end of file diff --git a/textract-pipeline/terraform/main.tf b/textract-pipeline/terraform/main.tf new file mode 100644 index 0000000..d0b0acc --- /dev/null +++ b/textract-pipeline/terraform/main.tf @@ -0,0 +1,128 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "5.39.0" + } + } + + backend "s3" { + bucket = "doczyai-use2-d-infra-s3-terraform-state" # Parameterize using -backend-config flag with "terraform init" + key = "terraform/textract-pipeline/terraform.tfstate" # Parameterize + region = "us-east-2" # Parameterize + profile = "doczyai" # Parameterize + dynamodb_table = "doczyai-use2-d-infra-dyd-terraform-lock" # Parameterize + encrypt = true + } +} + +locals { + region_map = { + us-east-1 = "use1" + us-east-2 = "use2" + us-west-1 = "usw1" + us-west-2 = "usw2" + } + environment_map = { + prod = "p" + uat = "u" + qa = "q" + dev = "d" + } + + environment_resource_map = { + dev = { + vpc_id = "vpc-0f9d7c913f6522079" + secret_manager_name = "doczy-dev-db-svc-acc" + devops_s3_bucket_name = "doczyai-use2-d-infra-s3-devops-resources" + } + } + + # region abbreviation + region_short_name = local.region_map[var.aws_region] + + # environment abbreviation + environment_short_name = local.environment_map[var.environment] + + # global prefix + global_prefix = "${var.project_name}-${local.region_short_name}-${local.environment_short_name}-infra" + + # Resource prefix + iam_role_prefix = "${local.global_prefix}-rol" + + # IAM role name + lambda_role_name = "${local.iam_role_prefix}-textract-lambda-role" + textract_service_role_name = "${local.iam_role_prefix}-textract-service-role" + + common_tags = { + Environment = var.environment + Terraform = "true" + } +} +provider "aws" { + profile = var.aws_profile + + default_tags { + tags = local.common_tags + } +} + +module "textract_pipeline" { + source = "./modules/aws-textract-pipeline" + + # for each for multiple resources + for_each = var.client_list + + client_name = each.value.client_name + + project_name = var.project_name + aws_region = var.aws_region + environment = var.environment + vpc_id = local.environment_resource_map[var.environment].vpc_id + secret_manager_name = local.environment_resource_map[var.environment].secret_manager_name + devops_s3_bucket = local.environment_resource_map[var.environment].devops_s3_bucket_name + + lambda_role_arn = aws_iam_role.lambda_role.arn + textract_role_arn = aws_iam_role.textract_role.arn + + textract_role_name = aws_iam_role.textract_role.name + lambda_role_name = aws_iam_role.lambda_role.name +} + +# Create lambda_role +resource "aws_iam_role" "lambda_role" { + name = local.lambda_role_name + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Sid = "" + Principal = { + Service = "lambda.amazonaws.com" + } + }, + ] + }) +} + +# Create textract_role +resource "aws_iam_role" "textract_role" { + name = local.textract_service_role_name + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Sid = "" + Principal = { + Service = "textract.amazonaws.com" + } + }, + ] + }) +} \ No newline at end of file diff --git a/textract-pipeline/terraform/modules/aws-textract-pipeline-part1/main.tf b/textract-pipeline/terraform/modules/aws-textract-pipeline-part1/main.tf new file mode 100644 index 0000000..4e9b6a3 --- /dev/null +++ b/textract-pipeline/terraform/modules/aws-textract-pipeline-part1/main.tf @@ -0,0 +1,377 @@ +locals { + region_map = { + us-east-1 = "use1" + us-east-2 = "use2" + us-west-1 = "usw1" + us-west-2 = "usw2" + } + environment_map = { + prod = "p" + uat = "u" + qa = "q" + dev = "d" + } + + common_tags = { + Environment = var.environment + ClientName = var.client_name + } + + # region abbreviation + region_short_name = local.region_map[var.aws_region] + + # environment abbreviation + environment_short_name = local.environment_map[var.environment] + + # global prefix + global_prefix = "${var.project_name}-${local.region_short_name}-${local.environment_short_name}-${var.client_name}" + + # Resource prefix + sqs_prefix = "${local.global_prefix}-sqs" + sns_prefix = "${local.global_prefix}-sns" + s3_prefix = "${local.global_prefix}-s3" + + # SQS names + insert_record_sqs_queue_name = "${local.sqs_prefix}-${var.insert_record_sqs_queue.name}" + docx_to_pdf_sqs_queue_name = "${local.sqs_prefix}-${var.docx_to_pdf_sqs_queue.name}" + tiff_to_pdf_sqs_queue_name = "${local.sqs_prefix}-${var.tiff_to_pdf_sqs_queue.name}" + pdf_validations_sqs_queue_name = "${local.sqs_prefix}-${var.pdf_validations_sqs_queue.name}" + sender_queue_name = "${local.sqs_prefix}-${var.sender_sqs_queue.name}" + receiver_queue_name = "${local.sqs_prefix}-${var.receiver_sqs_queue.name}" + text_creation_queue_name = "${local.sqs_prefix}-${var.text_creation_sqs_queue.name}" + + # SNS names + textract_notification_sns_name = "${local.sns_prefix}-${var.textract_notification_sns.name}" + + # S3 names + processing_s3_bucket_name = "${local.s3_prefix}-textract-processing-001" +} + +# insert_record_sqs_queue_name SQS Queue +module "insert_record_sqs" { + source = "terraform-aws-modules/sqs/aws" + version = "4.1.1" + name = local.insert_record_sqs_queue_name + delay_seconds = var.insert_record_sqs_queue.delay_seconds + max_message_size = var.insert_record_sqs_queue.max_message_size + message_retention_seconds = var.insert_record_sqs_queue.message_retention_seconds + visibility_timeout_seconds= var.insert_record_sqs_queue.visibility_timeout_seconds + + create_queue_policy = true + + queue_policy_statements = { + s3 = { + + sid = "AllowS3SendMessage" + effect = "Allow" + + principals = [{ + type = "Service" + identifiers = ["s3.amazonaws.com"] + }] + + actions = ["sqs:SendMessage"] + + conditions = [{ + test = "ArnEquals" + variable = "aws:SourceArn" + values = [aws_s3_bucket.textract_processing.arn] + }] + } + } + + tags = local.common_tags +} + +# docx_to_pdf_sqs_queue SQS Queue +module "docx_to_pdf_sqs" { + source = "terraform-aws-modules/sqs/aws" + version = "4.1.1" + name = local.docx_to_pdf_sqs_queue_name + delay_seconds = var.docx_to_pdf_sqs_queue.delay_seconds + max_message_size = var.docx_to_pdf_sqs_queue.max_message_size + message_retention_seconds = var.docx_to_pdf_sqs_queue.message_retention_seconds + visibility_timeout_seconds= var.docx_to_pdf_sqs_queue.visibility_timeout_seconds + + create_queue_policy = true + + queue_policy_statements = { + s3 = { + + sid = "AllowS3SendMessage" + effect = "Allow" + + principals = [{ + type = "Service" + identifiers = ["s3.amazonaws.com"] + }] + + actions = ["sqs:SendMessage"] + + conditions = [{ + test = "ArnEquals" + variable = "aws:SourceArn" + values = [aws_s3_bucket.textract_processing.arn] + }] + } + } + + tags = local.common_tags +} + +# tiff_to_pdf_sqs_queue SQS Queue +module "tiff_to_pdf_sqs" { + source = "terraform-aws-modules/sqs/aws" + version = "4.1.1" + name = local.tiff_to_pdf_sqs_queue_name + delay_seconds = var.tiff_to_pdf_sqs_queue.delay_seconds + max_message_size = var.tiff_to_pdf_sqs_queue.max_message_size + message_retention_seconds = var.tiff_to_pdf_sqs_queue.message_retention_seconds + visibility_timeout_seconds= var.tiff_to_pdf_sqs_queue.visibility_timeout_seconds + + create_queue_policy = true + + queue_policy_statements = { + s3 = { + + sid = "AllowS3SendMessage" + effect = "Allow" + + principals = [{ + type = "Service" + identifiers = ["s3.amazonaws.com"] + }] + + actions = ["sqs:SendMessage"] + + conditions = [{ + test = "ArnEquals" + variable = "aws:SourceArn" + values = [aws_s3_bucket.textract_processing.arn] + }] + } + } + + tags = local.common_tags +} + +# pdf_validations_sqs_queue SQS Queue +module "pdf_validations_sqs" { + source = "terraform-aws-modules/sqs/aws" + version = "4.1.1" + name = local.pdf_validations_sqs_queue_name + delay_seconds = var.pdf_validations_sqs_queue.delay_seconds + max_message_size = var.pdf_validations_sqs_queue.max_message_size + message_retention_seconds = var.pdf_validations_sqs_queue.message_retention_seconds + visibility_timeout_seconds= var.pdf_validations_sqs_queue.visibility_timeout_seconds + + create_queue_policy = true + + queue_policy_statements = { + s3 = { + + sid = "AllowS3SendMessage" + effect = "Allow" + + principals = [{ + type = "Service" + identifiers = ["s3.amazonaws.com"] + }] + + actions = ["sqs:SendMessage"] + + conditions = [{ + test = "ArnEquals" + variable = "aws:SourceArn" + values = [aws_s3_bucket.textract_processing.arn] + }] + } + } + + tags = local.common_tags +} + +# Sender SQS Queue +module "textract_sender_sqs" { + source = "terraform-aws-modules/sqs/aws" + version = "4.1.1" + name = local.sender_queue_name + delay_seconds = var.sender_sqs_queue.delay_seconds + max_message_size = var.sender_sqs_queue.max_message_size + message_retention_seconds = var.sender_sqs_queue.message_retention_seconds + visibility_timeout_seconds= var.sender_sqs_queue.visibility_timeout_seconds + + create_queue_policy = true + + queue_policy_statements = { + s3 = { + + sid = "AllowS3SendMessage" + effect = "Allow" + + principals = [{ + type = "Service" + identifiers = ["s3.amazonaws.com"] + }] + + actions = ["sqs:SendMessage"] + + conditions = [{ + test = "ArnEquals" + variable = "aws:SourceArn" + values = [aws_s3_bucket.textract_processing.arn] + }] + } + } + + tags = local.common_tags +} + +# textract notification sns +module "textract_notification_sns" { + source = "terraform-aws-modules/sns/aws" + version = ">= 5.0" + + name = local.textract_notification_sns_name + + topic_policy_statements = { + sqs = { + sid = "SQSSubscribe" + actions = [ + "sns:Subscribe", + "sns:Receive", + ] + + principals = [{ + type = "AWS" + identifiers = ["*"] + }] + + conditions = [{ + test = "StringLike" + variable = "sns:Endpoint" + values = [module.textract_receiver_sqs.queue_arn] + }] + } + } + + subscriptions = { + sqs = { + protocol = "sqs" + endpoint = module.textract_receiver_sqs.queue_arn + } + } + + tags = local.common_tags +} + +# Receiver SQS Queue +module "textract_receiver_sqs" { + source = "terraform-aws-modules/sqs/aws" + version = "4.1.1" + name = local.receiver_queue_name + delay_seconds = var.receiver_sqs_queue.delay_seconds + max_message_size = var.receiver_sqs_queue.max_message_size + message_retention_seconds = var.receiver_sqs_queue.message_retention_seconds + visibility_timeout_seconds= var.receiver_sqs_queue.visibility_timeout_seconds + tags = local.common_tags + + create_queue_policy = true + queue_policy_statements = { + sns = { + sid = "SNSPublish" + actions = ["sqs:SendMessage"] + + principals = [ + { + type = "Service" + identifiers = ["sns.amazonaws.com"] + } + ] + + conditions = [{ + test = "ArnEquals" + variable = "aws:SourceArn" + values = [module.textract_notification_sns.topic_arn] + }] + } + } +} + +# Text-creation SQS Queue +module "text_creation_sqs" { + source = "terraform-aws-modules/sqs/aws" + version = "4.1.1" + name = local.text_creation_queue_name + delay_seconds = var.text_creation_sqs_queue.delay_seconds + max_message_size = var.text_creation_sqs_queue.max_message_size + message_retention_seconds = var.text_creation_sqs_queue.message_retention_seconds + visibility_timeout_seconds= var.text_creation_sqs_queue.visibility_timeout_seconds + tags = local.common_tags + + create_queue_policy = true + + queue_policy_statements = { + s3 = { + + sid = "AllowS3SendMessage" + effect = "Allow" + + principals = [{ + type = "Service" + identifiers = ["s3.amazonaws.com"] + }] + + actions = ["sqs:SendMessage"] + + conditions = [{ + test = "ArnEquals" + variable = "aws:SourceArn" + values = [aws_s3_bucket.textract_processing.arn] + }] + } + } +} + +resource "aws_s3_bucket" "textract_processing" { + bucket = local.processing_s3_bucket_name + + tags = local.common_tags +} + +resource "aws_s3_bucket_policy" "deny_insecure_communication" { + bucket = aws_s3_bucket.textract_processing.id + policy = data.aws_iam_policy_document.deny_insecure_communication.json +} + +data "aws_iam_policy_document" "deny_insecure_communication" { + statement { + sid = "DenyInsecureCommunications" + + principals { + type = "*" + identifiers = ["*"] + } + + condition { + test = "Bool" + variable = "aws:SecureTransport" + values = ["false"] + } + + effect = "Deny" + + actions = ["s3:*"] + + resources = [ + aws_s3_bucket.textract_processing.arn, + "${aws_s3_bucket.textract_processing.arn}/*", + ] + } +} + +resource "aws_s3_object" "config" { + key = "config.properties" + bucket = aws_s3_bucket.textract_processing.id + source = "../src/s3-files/config.properties" +} \ No newline at end of file diff --git a/textract-pipeline/terraform/modules/aws-textract-pipeline-part1/outputs.tf b/textract-pipeline/terraform/modules/aws-textract-pipeline-part1/outputs.tf new file mode 100644 index 0000000..23aff5d --- /dev/null +++ b/textract-pipeline/terraform/modules/aws-textract-pipeline-part1/outputs.tf @@ -0,0 +1,47 @@ +# Queue arn's +output "insert_record_queue_arn" { + description = "Insert record queue arn" + value = module.insert_record_sqs.queue_arn +} + +output "docx_to_pdf_queue_arn" { + description = "Docx to PDF queue arn" + value = module.docx_to_pdf_sqs.queue_arn +} + +output "tiff_to_pdf_queue_arn" { + description = "Docx to PDF queue arn" + value = module.tiff_to_pdf_sqs.queue_arn +} + +output "pdf_validation_queue_arn" { + description = "Docx to PDF queue arn" + value = module.pdf_validations_sqs.queue_arn +} + +output "sender_queue_arn" { + description = "Sender queue arn" + value = module.textract_sender_sqs.queue_arn +} + +output "receiver_queue_arn" { + description = "Receiver queue arn" + value = module.textract_receiver_sqs.queue_arn +} + +output "text_creation_queue_arn" { + description = "Text creation queue arn" + value = module.text_creation_sqs.queue_arn +} + +# SNS arn's +output "textract_notification_sns_arn" { + description = "sns topic arn for textract service" + value = module.textract_notification_sns.topic_arn +} + +# S3 bucket name +output "textract_s3_bucket_name" { + description = "sns topic arn for textract service" + value = aws_s3_bucket.textract_processing.id +} \ No newline at end of file diff --git a/textract-pipeline/terraform/modules/aws-textract-pipeline-part1/variables.tf b/textract-pipeline/terraform/modules/aws-textract-pipeline-part1/variables.tf new file mode 100644 index 0000000..0e3501f --- /dev/null +++ b/textract-pipeline/terraform/modules/aws-textract-pipeline-part1/variables.tf @@ -0,0 +1,166 @@ +# Required +variable "project_name" { + type = string +} + +# Required +variable "client_name" { + type = string +} + +# Required +variable "environment" { + type = string +} + +# Required +variable "aws_region" { + type = string +} + +# # Required +# variable "lambda_role" { +# type = string +# } + +# Required +# variable "s3_bucket_name" { +# type = string +# } + +# textract notification sns topic +variable "textract_notification_sns" { + type = object({ + name = string + }) + default = { + name = "textract-notification" + } +} + +# insert-record-queue +variable "insert_record_sqs_queue" { + type = object({ + name = string + delay_seconds = number + max_message_size = number + message_retention_seconds = number + visibility_timeout_seconds= number + }) + default = { + name = "insert-record-queue" + delay_seconds = 90 + max_message_size = 2048 + message_retention_seconds = 86400 + visibility_timeout_seconds= 901 + } +} + + +# docx-to-pdf queue +variable "docx_to_pdf_sqs_queue" { + type = object({ + name = string + delay_seconds = number + max_message_size = number + message_retention_seconds = number + visibility_timeout_seconds= number + }) + default = { + name = "docx-to-pdf-queue" + delay_seconds = 90 + max_message_size = 2048 + message_retention_seconds = 86400 + visibility_timeout_seconds= 901 + } +} + +# tiff-to-pdf queue +variable "tiff_to_pdf_sqs_queue" { + type = object({ + name = string + delay_seconds = number + max_message_size = number + message_retention_seconds = number + visibility_timeout_seconds= number + }) + default = { + name = "tiff-to-pdf-queue" + delay_seconds = 90 + max_message_size = 2048 + message_retention_seconds = 86400 + visibility_timeout_seconds= 901 + } +} + +# pdf-validation-queue queue +variable "pdf_validations_sqs_queue" { + type = object({ + name = string + delay_seconds = number + max_message_size = number + message_retention_seconds = number + visibility_timeout_seconds= number + }) + default = { + name = "pdf-validation-queue" + delay_seconds = 90 + max_message_size = 2048 + message_retention_seconds = 86400 + visibility_timeout_seconds= 901 + } +} + +# textract sender queue +variable "sender_sqs_queue" { + type = object({ + name = string + delay_seconds = number + max_message_size = number + message_retention_seconds = number + visibility_timeout_seconds= number + }) + default = { + name = "textract-sender-queue" + delay_seconds = 90 + max_message_size = 2048 + message_retention_seconds = 86400 + visibility_timeout_seconds= 901 + } +} + +# textract receiver queue +variable "receiver_sqs_queue" { + type = object({ + name = string + delay_seconds = number + max_message_size = number + message_retention_seconds = number + visibility_timeout_seconds= number + }) + default = { + name = "textract-receiver-queue" + delay_seconds = 90 + max_message_size = 2048 + message_retention_seconds = 86400 + visibility_timeout_seconds= 901 + } +} + +# textract receiver queue +variable "text_creation_sqs_queue" { + type = object({ + name = string + delay_seconds = number + max_message_size = number + message_retention_seconds = number + visibility_timeout_seconds= number + }) + default = { + name = "text-creation-queue" + delay_seconds = 90 + max_message_size = 2048 + message_retention_seconds = 86400 + visibility_timeout_seconds= 901 + } +} \ No newline at end of file diff --git a/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf b/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf new file mode 100644 index 0000000..e84331c --- /dev/null +++ b/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf @@ -0,0 +1,653 @@ +locals { + region_map = { + us-east-1 = "use1" + us-east-2 = "use2" + us-west-1 = "usw1" + us-west-2 = "usw2" + } + environment_map = { + prod = "p" + uat = "u" + qa = "q" + dev = "d" + } + + common_tags = { + Environment = var.environment + ClientName = var.client_name + } + + # region abbreviation + region_short_name = local.region_map[var.aws_region] + + # environment abbreviation + environment_short_name = local.environment_map[var.environment] + + # global prefix + global_prefix = "${var.project_name}-${local.region_short_name}-${local.environment_short_name}-${var.client_name}" + + # Resource prefix + lambda_prefix = "${local.global_prefix}-lmb" + api_gateway_prefix = "${local.global_prefix}-api" + lambda_layer_prefix = "${local.global_prefix}-lyr" + + # Lambda names + batch_creation_lambda_name = "${local.lambda_prefix}-${var.batch_creation_lambda.name}" + trigger_pipeline_lambda_name = "${local.lambda_prefix}-${var.trigger_pipeline_lambda.name}" + insert_record_lambda_name = "${local.lambda_prefix}-${var.insert_record_lambda.name}" + docx_to_pdf_lambda_name = "${local.lambda_prefix}-${var.docx_to_pdf_lambda.name}" + tiff_to_pdf_lambda_name = "${local.lambda_prefix}-${var.tiff_to_pdf_lambda.name}" + pdf_validation_lambda_name = "${local.lambda_prefix}-${var.pdf_validation_lambda.name}" + sender_lambda_name = "${local.lambda_prefix}-${var.textract_sender_lambda.name}" + receiver_lambda_name = "${local.lambda_prefix}-${var.textract_receiver_lambda.name}" + text_creation_lambda_name = "${local.lambda_prefix}-${var.text_creation_lambda.name}" + database_interface_lambda_name = "${local.lambda_prefix}-${var.database_interface_lambda.name}" + + # Lambda Layer names + libre_office_lambda_layer_name = "${local.lambda_layer_prefix}-${var.libre_office_lambda_layer.name}" + brotli_lambda_layer_name = "${local.lambda_layer_prefix}-${var.brotli_lambda_layer.name}" + pypdf2_lambda_layer_name = "${local.lambda_layer_prefix}-${var.pypdf2_lambda_layer.name}" + + # API Gateway configuration + api_description = "API's used for DoczyAI pipeline" + batch_creation_api_path = "create-batch-new" + trigger_pipeline_api_path = "trigger-pipeline" + stage_name = "dev" + + # property file path + property_file_s3_path = "${var.s3_bucket_name}/config.properties" + + # common env variables + environment_variables = { + PROPERTY_FILE_S3_PATH = local.property_file_s3_path + DATABASE_INTERFACE_FUNCTION_NAME = local.database_interface_lambda_name + } + + # S3 bucket and prefix for lambda's + lambda_layer_s3_bucket = var.devops_s3_bucket + lambda_layer_s3_prefix = "lambda-layer-builds/" +} + +# vpc subnets +data "aws_subnets" "subnets" { + filter { + name = "vpc-id" + values = [var.vpc_id] + } +} + +# vpc security groups +data "aws_security_groups" "security_groups" { + filter { + name = "group-name" + values = ["*default*"] + } + + filter { + name = "vpc-id" + values = [var.vpc_id] + } +} + +# Lambda Layers - Libre Office - create package from source code +module "lambda_layer_libre_office" { + source = "terraform-aws-modules/lambda/aws" + + create_layer = true + create_package = var.libre_office_lambda_layer.create_package + + layer_name = local.libre_office_lambda_layer_name + description = var.libre_office_lambda_layer.description + compatible_runtimes = var.libre_office_lambda_layer.compatible_runtimes + compatible_architectures = var.libre_office_lambda_layer.compatible_architectures + + source_path = var.libre_office_lambda_layer.source_path + + store_on_s3 = true + s3_bucket = local.lambda_layer_s3_bucket + s3_prefix = local.lambda_layer_s3_prefix +} + +# Lambda Layers - brotli layer - create layer from zip +module "lambda_layer_brotli" { + source = "terraform-aws-modules/lambda/aws" + + create_layer = true + create_package = false + + layer_name = local.brotli_lambda_layer_name + description = var.brotli_lambda_layer.description + compatible_runtimes = var.brotli_lambda_layer.compatible_runtimes + compatible_architectures = var.brotli_lambda_layer.compatible_architectures + + source_path = null + + local_existing_package = var.brotli_lambda_layer.local_existing_package + +} + +# Lambda Layers - pypdf2 - create layer from zip +module "lambda_layer_pypdf2" { + source = "terraform-aws-modules/lambda/aws" + + create_layer = true + create_package = false + + layer_name = local.pypdf2_lambda_layer_name + description = var.pypdf2_lambda_layer.description + compatible_runtimes = var.pypdf2_lambda_layer.compatible_runtimes + compatible_architectures = var.pypdf2_lambda_layer.compatible_architectures + + source_path = var.pypdf2_lambda_layer.source_path + + local_existing_package = var.pypdf2_lambda_layer.local_existing_package +} + +# batch creation lambda function +module "batch_creation_lambda_function" { + source = "terraform-aws-modules/lambda/aws" + version = "7.2.2" + + function_name = local.batch_creation_lambda_name + description = var.batch_creation_lambda.description + handler = var.batch_creation_lambda.handler + runtime = var.batch_creation_lambda.runtime + timeout = var.batch_creation_lambda.timeout + memory_size = var.batch_creation_lambda.memory_size + ephemeral_storage_size = var.batch_creation_lambda.ephemeral_storage_size + architectures = var.batch_creation_lambda.architectures + + environment_variables = local.environment_variables + + source_path = var.batch_creation_lambda.source_path + + lambda_role = var.lambda_role_arn + create_role = false + + vpc_subnet_ids = data.aws_subnets.subnets.ids + vpc_security_group_ids = data.aws_security_groups.security_groups.ids + + tags = local.common_tags +} + +# trigger pipeline lambda function +module "trigger_pipeline_lambda_function" { + source = "terraform-aws-modules/lambda/aws" + version = "7.2.2" + + function_name = local.trigger_pipeline_lambda_name + description = var.trigger_pipeline_lambda.description + handler = var.trigger_pipeline_lambda.handler + runtime = var.trigger_pipeline_lambda.runtime + timeout = var.trigger_pipeline_lambda.timeout + memory_size = var.trigger_pipeline_lambda.memory_size + ephemeral_storage_size = var.trigger_pipeline_lambda.ephemeral_storage_size + architectures = var.trigger_pipeline_lambda.architectures + + environment_variables = local.environment_variables + + source_path = var.trigger_pipeline_lambda.source_path + + lambda_role = var.lambda_role_arn + create_role = false + + vpc_subnet_ids = data.aws_subnets.subnets.ids + vpc_security_group_ids = data.aws_security_groups.security_groups.ids + + tags = local.common_tags +} + +# insert record lambda function +module "insert_record_lambda_function" { + source = "terraform-aws-modules/lambda/aws" + version = "7.2.2" + + function_name = local.insert_record_lambda_name + description = var.insert_record_lambda.description + handler = var.insert_record_lambda.handler + runtime = var.insert_record_lambda.runtime + timeout = var.insert_record_lambda.timeout + memory_size = var.insert_record_lambda.memory_size + ephemeral_storage_size = var.insert_record_lambda.ephemeral_storage_size + architectures = var.insert_record_lambda.architectures + + environment_variables = local.environment_variables + + source_path = var.insert_record_lambda.source_path + + event_source_mapping = { + sqs = { + event_source_arn = var.insert_record_queue_arn + function_response_types = ["ReportBatchItemFailures"] + scaling_config = { + maximum_concurrency = 20 + + } + batch_size = 1 + } + } + + lambda_role = var.lambda_role_arn + create_role = false + + vpc_subnet_ids = data.aws_subnets.subnets.ids + vpc_security_group_ids = data.aws_security_groups.security_groups.ids + + tags = local.common_tags +} + +# docx to pdf lambda function +module "docx_to_pdf_lambda_function" { + source = "terraform-aws-modules/lambda/aws" + version = "7.2.2" + + function_name = local.docx_to_pdf_lambda_name + description = var.docx_to_pdf_lambda.description + handler = var.docx_to_pdf_lambda.handler + runtime = var.docx_to_pdf_lambda.runtime + timeout = var.docx_to_pdf_lambda.timeout + memory_size = var.docx_to_pdf_lambda.memory_size + ephemeral_storage_size = var.docx_to_pdf_lambda.ephemeral_storage_size + architectures = var.docx_to_pdf_lambda.architectures + + layers = [ + module.lambda_layer_libre_office.lambda_layer_arn, + module.lambda_layer_brotli.lambda_layer_arn, + ] + + environment_variables = local.environment_variables + + source_path = var.docx_to_pdf_lambda.source_path + + event_source_mapping = { + sqs = { + event_source_arn = var.docx_to_pdf_queue_arn + function_response_types = ["ReportBatchItemFailures"] + scaling_config = { + maximum_concurrency = 20 + + } + batch_size = 1 + } + } + + lambda_role = var.lambda_role_arn + create_role = false + + vpc_subnet_ids = data.aws_subnets.subnets.ids + vpc_security_group_ids = data.aws_security_groups.security_groups.ids + + tags = local.common_tags +} + +# tiff to pdf lambda function +module "tiff_to_pdf_lambda_function" { + source = "terraform-aws-modules/lambda/aws" + version = "7.2.2" + + function_name = local.tiff_to_pdf_lambda_name + description = var.tiff_to_pdf_lambda.description + handler = var.tiff_to_pdf_lambda.handler + runtime = var.tiff_to_pdf_lambda.runtime + timeout = var.tiff_to_pdf_lambda.timeout + memory_size = var.tiff_to_pdf_lambda.memory_size + ephemeral_storage_size = var.tiff_to_pdf_lambda.ephemeral_storage_size + architectures = var.tiff_to_pdf_lambda.architectures + + environment_variables = local.environment_variables + + source_path = var.tiff_to_pdf_lambda.source_path + + event_source_mapping = { + sqs = { + event_source_arn = var.tiff_to_pdf_queue_arn + function_response_types = ["ReportBatchItemFailures"] + scaling_config = { + maximum_concurrency = 20 + + } + batch_size = 1 + } + } + + lambda_role = var.lambda_role_arn + create_role = false + + vpc_subnet_ids = data.aws_subnets.subnets.ids + vpc_security_group_ids = data.aws_security_groups.security_groups.ids + + tags = local.common_tags +} + +# pdf validation lambda function +module "pdf_validation_lambda_function" { + source = "terraform-aws-modules/lambda/aws" + version = "7.2.2" + + function_name = local.pdf_validation_lambda_name + description = var.pdf_validation_lambda.description + handler = var.pdf_validation_lambda.handler + runtime = var.pdf_validation_lambda.runtime + timeout = var.pdf_validation_lambda.timeout + memory_size = var.pdf_validation_lambda.memory_size + ephemeral_storage_size = var.pdf_validation_lambda.ephemeral_storage_size + architectures = var.pdf_validation_lambda.architectures + + layers = [ + module.lambda_layer_pypdf2.lambda_layer_arn + ] + + environment_variables = local.environment_variables + + source_path = var.pdf_validation_lambda.source_path + + event_source_mapping = { + sqs = { + event_source_arn = var.pdf_validations_queue_arn + function_response_types = ["ReportBatchItemFailures"] + scaling_config = { + maximum_concurrency = 20 + + } + batch_size = 1 + } + } + + lambda_role = var.lambda_role_arn + create_role = false + + vpc_subnet_ids = data.aws_subnets.subnets.ids + vpc_security_group_ids = data.aws_security_groups.security_groups.ids + + tags = local.common_tags +} + +# sender lambda function +module "textract_sender_lambda_function" { + source = "terraform-aws-modules/lambda/aws" + version = "7.2.2" + + function_name = local.sender_lambda_name + description = var.textract_sender_lambda.description + handler = var.textract_sender_lambda.handler + runtime = var.textract_sender_lambda.runtime + timeout = var.textract_sender_lambda.timeout + memory_size = var.textract_sender_lambda.memory_size + ephemeral_storage_size = var.textract_sender_lambda.ephemeral_storage_size + architectures = var.textract_sender_lambda.architectures + + environment_variables = merge(local.environment_variables,{ + TEXTRACT_ROLE_ARN = var.textract_role_arn + SNS_TOPIC_ARN = var.textract_sns_topic_arn + }) + + source_path = var.textract_sender_lambda.source_path + + event_source_mapping = { + sqs = { + event_source_arn = var.textract_sender_queue_arn + function_response_types = ["ReportBatchItemFailures"] + scaling_config = { + maximum_concurrency = 20 + + } + batch_size = 1 + } + } + + lambda_role = var.lambda_role_arn + create_role = false + + vpc_subnet_ids = data.aws_subnets.subnets.ids + vpc_security_group_ids = data.aws_security_groups.security_groups.ids + + tags = local.common_tags +} + +# receiver lambda function +module "textract_receiver_lambda_function" { + source = "terraform-aws-modules/lambda/aws" + version = "7.2.2" + + function_name = local.receiver_lambda_name + description = var.textract_receiver_lambda.description + handler = var.textract_receiver_lambda.handler + runtime = var.textract_receiver_lambda.runtime + timeout = var.textract_receiver_lambda.timeout + memory_size = var.textract_receiver_lambda.memory_size + ephemeral_storage_size = var.textract_receiver_lambda.ephemeral_storage_size + architectures = var.textract_receiver_lambda.architectures + + environment_variables = local.environment_variables + + source_path = var.textract_receiver_lambda.source_path + + event_source_mapping = { + sqs = { + event_source_arn = var.textract_receiver_queue_arn + function_response_types = ["ReportBatchItemFailures"] + scaling_config = { + maximum_concurrency = 20 + + } + batch_size = 1 + } + } + + lambda_role = var.lambda_role_arn + create_role = false + + vpc_subnet_ids = data.aws_subnets.subnets.ids + vpc_security_group_ids = data.aws_security_groups.security_groups.ids + + tags = local.common_tags +} + +# text creation lambda function +module "text_creation_lambda_function" { + source = "terraform-aws-modules/lambda/aws" + version = "7.2.2" + + function_name = local.text_creation_lambda_name + description = var.text_creation_lambda.description + handler = var.text_creation_lambda.handler + runtime = var.text_creation_lambda.runtime + timeout = var.text_creation_lambda.timeout + memory_size = var.text_creation_lambda.memory_size + ephemeral_storage_size = var.text_creation_lambda.ephemeral_storage_size + architectures = var.text_creation_lambda.architectures + + layers = [ + module.lambda_layer_pypdf2.lambda_layer_arn + ] + + environment_variables = local.environment_variables + + source_path = var.text_creation_lambda.source_path + + event_source_mapping = { + sqs = { + event_source_arn = var.text_creation_queue_arn + function_response_types = ["ReportBatchItemFailures"] + scaling_config = { + maximum_concurrency = 20 + + } + batch_size = 1 + } + } + + lambda_role = var.lambda_role_arn + create_role = false + + vpc_subnet_ids = data.aws_subnets.subnets.ids + vpc_security_group_ids = data.aws_security_groups.security_groups.ids + + tags = local.common_tags +} + +# database interface lambda function +module "database_interface_lambda_function" { + source = "terraform-aws-modules/lambda/aws" + version = "7.2.2" + + function_name = local.database_interface_lambda_name + description = var.database_interface_lambda.description + handler = var.database_interface_lambda.handler + runtime = var.database_interface_lambda.runtime + timeout = var.database_interface_lambda.timeout + memory_size = var.database_interface_lambda.memory_size + ephemeral_storage_size = var.database_interface_lambda.ephemeral_storage_size + architectures = var.database_interface_lambda.architectures + + environment_variables = merge(local.environment_variables,{ + SECRET_MANAGER_NAME = var.secret_manager_name + }) + + source_path = var.database_interface_lambda.source_path + + lambda_role = var.lambda_role_arn + create_role = false + + vpc_subnet_ids = data.aws_subnets.subnets.ids + vpc_security_group_ids = data.aws_security_groups.security_groups.ids + + tags = local.common_tags +} + +# S3 notification +resource "aws_s3_bucket_notification" "bucket_notification" { + bucket = var.s3_bucket_name + + queue { + id = "docx-upload-event" + queue_arn = var.docx_to_pdf_queue_arn + events = ["s3:ObjectCreated:*"] + filter_prefix = "source-documents-docx/" + filter_suffix = ".docx" + } + + queue { + id = "doc-upload-event" + queue_arn = var.docx_to_pdf_queue_arn + events = ["s3:ObjectCreated:*"] + filter_prefix = "source-documents-docx/" + filter_suffix = ".doc" + } + + queue { + id = "tiff-upload-event" + queue_arn = var.tiff_to_pdf_queue_arn + events = ["s3:ObjectCreated:*"] + filter_prefix = "source-documents-tiff/" + filter_suffix = ".tiff" + } + + queue { + id = "all-pdf-upload-event" + queue_arn = var.pdf_validations_queue_arn + events = ["s3:ObjectCreated:*"] + filter_prefix = "all-pdfs/" + filter_suffix = ".pdf" + } + + queue { + id = "valid-pdf-upload-event" + queue_arn = var.textract_sender_queue_arn + events = ["s3:ObjectCreated:*"] + filter_prefix = "valid-pdfs/" + filter_suffix = ".pdf" + } + + queue { + id = "textract-json-upload-event" + queue_arn = var.text_creation_queue_arn + events = ["s3:ObjectCreated:*"] + filter_prefix = "textract-output-json/" + filter_suffix = ".json" + } +} + +# API Gateway resources +resource "aws_api_gateway_rest_api" "project_api" { + name = local.api_gateway_prefix + description = local.api_description + + endpoint_configuration { + types = ["REGIONAL"] + } + + tags = local.common_tags +} + +# API Gateway - Create batch - create resource +resource "aws_api_gateway_resource" "create_batch" { + rest_api_id = aws_api_gateway_rest_api.project_api.id + parent_id = aws_api_gateway_rest_api.project_api.root_resource_id + path_part = local.batch_creation_api_path +} +# API Gateway - Create batch - create method +resource "aws_api_gateway_method" "create_batch_method" { + rest_api_id = "${aws_api_gateway_rest_api.project_api.id}" + resource_id = "${aws_api_gateway_resource.create_batch.id}" + http_method = "POST" + authorization = "NONE" +} + +# API Gateway - Create batch - integration with lambda +resource "aws_api_gateway_integration" "create_batch_integration" { + rest_api_id = aws_api_gateway_rest_api.project_api.id + resource_id = aws_api_gateway_resource.create_batch.id + http_method = aws_api_gateway_method.create_batch_method.http_method + integration_http_method = "POST" + type = "AWS" + uri = module.batch_creation_lambda_function.lambda_function_invoke_arn +} +# API Gateway - Create batch - add permissions for lambda +resource "aws_lambda_permission" "create_batch_lambda_permission" { + statement_id = "AllowMyDemoAPIInvoke" + action = "lambda:InvokeFunction" + function_name = "${module.batch_creation_lambda_function.lambda_function_name}" + principal = "apigateway.amazonaws.com" + source_arn = "${aws_api_gateway_rest_api.project_api.execution_arn}/*/${aws_api_gateway_method.create_batch_method.http_method}${aws_api_gateway_resource.create_batch.path}" +} +# API Gateway - trigger pipeline - create resource +resource "aws_api_gateway_resource" "trigger_pipeline" { + rest_api_id = aws_api_gateway_rest_api.project_api.id + parent_id = aws_api_gateway_rest_api.project_api.root_resource_id + path_part = local.trigger_pipeline_api_path +} +# API Gateway - trigger pipeline - create method +resource "aws_api_gateway_method" "trigger_pipeline_method" { + rest_api_id = "${aws_api_gateway_rest_api.project_api.id}" + resource_id = "${aws_api_gateway_resource.trigger_pipeline.id}" + http_method = "POST" + authorization = "NONE" +} +# API Gateway - trigger pipeline - integration with lambda +resource "aws_api_gateway_integration" "trigger_pipeline_integration" { + rest_api_id = aws_api_gateway_rest_api.project_api.id + resource_id = aws_api_gateway_resource.trigger_pipeline.id + http_method = aws_api_gateway_method.trigger_pipeline_method.http_method + integration_http_method = "POST" + type = "AWS" + uri = module.trigger_pipeline_lambda_function.lambda_function_invoke_arn +} +# API Gateway - trigger pipeline - add permissions for lambda +resource "aws_lambda_permission" "trigger_pipeline_lambda_permission" { + statement_id = "AllowMyDemoAPIInvoke" + action = "lambda:InvokeFunction" + function_name = "${module.trigger_pipeline_lambda_function.lambda_function_name}" + principal = "apigateway.amazonaws.com" + source_arn = "${aws_api_gateway_rest_api.project_api.execution_arn}/*/${aws_api_gateway_method.trigger_pipeline_method.http_method}${aws_api_gateway_resource.trigger_pipeline.path}" +} +# API Gateway - deploy api gateway +resource "aws_api_gateway_deployment" "deployment" { + depends_on = [ + aws_api_gateway_integration.create_batch_integration, + aws_api_gateway_integration.trigger_pipeline_integration, + ] + + rest_api_id = "${aws_api_gateway_rest_api.project_api.id}" + stage_name = local.stage_name +} \ No newline at end of file diff --git a/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/outputs.tf b/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/outputs.tf new file mode 100644 index 0000000..46678f9 --- /dev/null +++ b/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/outputs.tf @@ -0,0 +1,71 @@ +# Lambda Layer Arn's +output "lambda_layer_libre_office_arn" { + description = "batch creation lambda fucntion arn" + value = module.lambda_layer_libre_office.lambda_layer_arn +} + +output "lambda_layer_brotli_arn" { + description = "batch creation lambda fucntion arn" + value = module.lambda_layer_brotli.lambda_layer_arn +} + +output "lambda_layer_pypdf2_arn" { + description = "batch creation lambda fucntion arn" + value = module.lambda_layer_pypdf2.lambda_layer_arn +} + +# Lambda Arn's +output "batch_creation_lambda_function_arn" { + description = "batch creation lambda fucntion arn" + value = module.batch_creation_lambda_function.lambda_function_arn +} + +output "trigger_pipeline_lambda_function_arn" { + description = "trigger pipeline lambda fucntion arn" + value = module.trigger_pipeline_lambda_function.lambda_function_arn +} + +output "insert_record_lambda_function_arn" { + description = "insert record lambda fucntion arn" + value = module.insert_record_lambda_function.lambda_function_arn +} + +output "docx_to_pdf_lambda_function_arn" { + description = "docx to pdf lambda fucntion arn" + value = module.docx_to_pdf_lambda_function.lambda_function_arn +} + +output "tiff_to_pdf_lambda_function_arn" { + description = "tiff to pdf lambda fucntion arn" + value = module.tiff_to_pdf_lambda_function.lambda_function_arn +} + +output "pdf_validation_lambda_function_arn" { + description = "pdf validation lambda fucntion arn" + value = module.pdf_validation_lambda_function.lambda_function_arn +} + +output "textract_sender_lambda_function_arn" { + description = "textract sender lambda fucntion arn" + value = module.textract_sender_lambda_function.lambda_function_arn +} + +output "textract_receiver_lambda_function_arn" { + description = "textract receiver lambda fucntion arn" + value = module.textract_receiver_lambda_function.lambda_function_arn +} + +output "text_creation_lambda_function_arn" { + description = "text creation lambda fucntion arn" + value = module.text_creation_lambda_function.lambda_function_arn +} + +output "database_interface_lambda_function_arn" { + description = "database interface lambda fucntion arn" + value = module.database_interface_lambda_function.lambda_function_arn +} + +# API Gateway invoke_url +output "api_invoke_url" { + value = aws_api_gateway_deployment.deployment.invoke_url +} diff --git a/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/variables.tf b/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/variables.tf new file mode 100644 index 0000000..da61de4 --- /dev/null +++ b/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/variables.tf @@ -0,0 +1,442 @@ +# Required +variable "project_name" { + type = string +} + +# Required +variable "client_name" { + type = string +} + +# Required +variable "environment" { + type = string +} + +# Required +variable "aws_region" { + type = string +} + +# Required +variable "vpc_id" { + type = string +} + +# Required +variable "secret_manager_name" { + type = string +} + +# Required +variable "textract_role_arn" { + type = string +} + +# Required - temp +variable "lambda_role_arn" { + type = string +} + +# Required +variable "s3_bucket_name" { + type = string +} + +# Required +variable "devops_s3_bucket" { + type = string +} + +# SQS Required +variable "insert_record_queue_arn" { + type = string +} +variable "docx_to_pdf_queue_arn" { + type = string +} +variable "tiff_to_pdf_queue_arn" { + type = string +} +variable "pdf_validations_queue_arn" { + type = string +} +variable "textract_sender_queue_arn"{ + type = string +} +variable "textract_receiver_queue_arn" { + type = string +} +variable "text_creation_queue_arn" { + type = string +} + +# SNS Required +variable "textract_sns_topic_arn" { + type = string +} + +# Lambda layer - Libre Office +variable "libre_office_lambda_layer" { + type = object({ + name = string + description = string + compatible_runtimes = list(string) + compatible_architectures = list(string) + source_path = string + local_existing_package = string + create_package = bool + }) + default = { + name = "libre-office" + description = "Libre office library for docx to pdf conversion" + compatible_runtimes = ["python3.8"] + compatible_architectures = ["x86_64"] + source_path = "../src/lambda-layer/libre-office" + local_existing_package = null + create_package = true + } +} + +# Lambda layer - brotli +variable "brotli_lambda_layer" { + type = object({ + name = string + description = string + compatible_runtimes = list(string) + compatible_architectures = list(string) + source_path = string + local_existing_package = string + create_package = bool + }) + default = { + name = "brotli" + description = "This layer provides brotli library" + compatible_runtimes = ["python3.8"] + compatible_architectures = ["x86_64"] + source_path = null + local_existing_package = "../src/lambda-layer/brotli-layer/brotlipy-layer.zip" + create_package = false + } +} + +# Lambda layer - PyPDF2 +variable "pypdf2_lambda_layer" { + type = object({ + name = string + description = string + compatible_runtimes = list(string) + compatible_architectures = list(string) + source_path = string + local_existing_package = string + create_package = bool + }) + default = { + name = "pypdf2" + description = "This layer provides PyPDF2 library" + compatible_runtimes = ["python3.12"] + compatible_architectures = ["x86_64"] + source_path = null + local_existing_package = "../src/lambda-layer/pypdf2-layer/PyPDF2.zip" + create_package = false + } +} + +# batch-creation +variable "batch_creation_lambda" { + type = object({ + name = string + description = string + handler = string + runtime = string + timeout = number + memory_size = number + ephemeral_storage_size = number + architectures = list(string) + layers = list(string) + environment_variables = map(string) + source_path = string + }) + default = { + name = "batch-creation" + description = "This function creates batch folder" + handler = "index.lambda_handler" + runtime = "python3.12" + timeout = 60 + memory_size = 256 + ephemeral_storage_size = 512 + architectures = ["x86_64"] + layers = [] + environment_variables = {} + source_path = "../src/lambda/batch-creation" + } +} +# trigger-pipeline +variable "trigger_pipeline_lambda" { + type = object({ + name = string + description = string + handler = string + runtime = string + timeout = number + memory_size = number + ephemeral_storage_size = number + architectures = list(string) + layers = list(string) + environment_variables = map(string) + source_path = string + }) + default = { + name = "trigger-pipeline" + description = "This function triggers doczy.ai pipeline" + handler = "index.lambda_handler" + runtime = "python3.12" + timeout = 60 + memory_size = 256 + ephemeral_storage_size = 512 + architectures = ["x86_64"] + layers = [] + environment_variables = {} + source_path = "../src/lambda/trigger-pipeline" + } +} + +# insert record +variable "insert_record_lambda" { + type = object({ + name = string + description = string + handler = string + runtime = string + timeout = number + memory_size = number + ephemeral_storage_size = number + architectures = list(string) + layers = list(string) + environment_variables = map(string) + source_path = string + }) + default = { + name = "insert-record" + description = "This function inserts record in database" + handler = "index.lambda_handler" + runtime = "python3.12" + timeout = 60 + memory_size = 256 + ephemeral_storage_size = 512 + architectures = ["x86_64"] + layers = [] + environment_variables = {} + source_path = "../src/lambda/insert-record" + } +} + +# docx-to-pdf +variable "docx_to_pdf_lambda" { + type = object({ + name = string + description = string + handler = string + runtime = string + timeout = number + memory_size = number + ephemeral_storage_size = number + architectures = list(string) + layers = list(string) + environment_variables = map(string) + source_path = string + }) + default = { + name = "docx-to-pdf" + description = "This function converts docx file to pdf file" + handler = "index.lambda_handler" + runtime = "python3.8" + timeout = 60 + memory_size = 256 + ephemeral_storage_size = 512 + architectures = ["x86_64"] + layers = [] + environment_variables = {} + source_path = "../src/lambda/docx-to-pdf" + } +} + +# tiff-to-pdf +variable "tiff_to_pdf_lambda" { + type = object({ + name = string + description = string + handler = string + runtime = string + timeout = number + memory_size = number + ephemeral_storage_size = number + architectures = list(string) + layers = list(string) + environment_variables = map(string) + source_path = string + }) + default = { + name = "tiff-to-pdf" + description = "This function converts tiff file to pdf file" + handler = "index.lambda_handler" + runtime = "python3.12" + timeout = 60 + memory_size = 256 + ephemeral_storage_size = 512 + architectures = ["x86_64"] + layers = [] + environment_variables = {} + source_path = "../src/lambda/tiff-to-pdf" + } +} + +# pdf-validation lambda +variable "pdf_validation_lambda" { + type = object({ + name = string + description = string + handler = string + runtime = string + timeout = number + memory_size = number + ephemeral_storage_size = number + architectures = list(string) + layers = list(string) + environment_variables = map(string) + source_path = string + }) + default = { + name = "pdf-validation" + description = "This function valids pdf against textract limitations" + handler = "index.lambda_handler" + runtime = "python3.12" + timeout = 60 + memory_size = 256 + ephemeral_storage_size = 512 + architectures = ["x86_64"] + layers = [] + environment_variables = {} + source_path = "../src/lambda/pdf-validation" + } +} + +# textract_sender_lambda +variable "textract_sender_lambda" { + type = object({ + name = string + description = string + handler = string + runtime = string + timeout = number + memory_size = number + ephemeral_storage_size = number + architectures = list(string) + layers = list(string) + environment_variables = map(string) + source_path = string + }) + default = { + name = "textract-sender" + description = "This function sends async request to TEXTRACT service" + handler = "index.lambda_handler" + runtime = "python3.12" + timeout = 60 + memory_size = 256 + ephemeral_storage_size = 512 + architectures = ["x86_64"] + layers = [] + environment_variables = {} + source_path = "../src/lambda/textract-sender" + } +} + +# textract_receiver_lambda +variable "textract_receiver_lambda" { + type = object({ + name = string + description = string + handler = string + runtime = string + timeout = number + memory_size = number + ephemeral_storage_size = number + architectures = list(string) + layers = list(string) + environment_variables = map(string) + source_path = string + }) + default = { + name = "textract-receiver" + description = "This function receives async request from TEXTRACT service" + handler = "index.lambda_handler" + runtime = "python3.12" + timeout = 60 + memory_size = 256 + ephemeral_storage_size = 512 + architectures = ["x86_64"] + layers = [] + environment_variables = {} + source_path = "../src/lambda/textract-receiver" + } +} + +# text-creation lambda +variable "text_creation_lambda" { + type = object({ + name = string + description = string + handler = string + runtime = string + timeout = number + memory_size = number + ephemeral_storage_size = number + architectures = list(string) + layers = list(string) + environment_variables = map(string) + source_path = string + }) + default = { + name = "text-creation" + description = "This function creates text from Textract JSON" + handler = "index.lambda_handler" + runtime = "python3.12" + timeout = 60 + memory_size = 256 + ephemeral_storage_size = 512 + architectures = ["x86_64"] + layers = [] + environment_variables = {} + source_path = "../src/lambda/text-creation" + } +} + +# database-interface lambda +variable "database_interface_lambda" { + type = object({ + name = string + description = string + handler = string + runtime = string + timeout = number + memory_size = number + ephemeral_storage_size = number + architectures = list(string) + layers = list(string) + environment_variables = map(string) + source_path = string + }) + default = { + name = "database-interface" + description = "This function acts as an interface to communicate with database" + handler = "index.lambda_handler" + runtime = "python3.12" + timeout = 60 + memory_size = 256 + ephemeral_storage_size = 512 + architectures = ["x86_64"] + layers = [] + environment_variables = {} + source_path = "../src/lambda/database-interface" + } +} \ No newline at end of file diff --git a/textract-pipeline/terraform/modules/aws-textract-pipeline/main.tf b/textract-pipeline/terraform/modules/aws-textract-pipeline/main.tf new file mode 100644 index 0000000..83ae4ab --- /dev/null +++ b/textract-pipeline/terraform/modules/aws-textract-pipeline/main.tf @@ -0,0 +1,227 @@ +locals { + region_map = { + us-east-1 = "use1" + us-east-2 = "use2" + us-west-1 = "usw1" + us-west-2 = "usw2" + } + environment_map = { + prod = "p" + uat = "u" + qa = "q" + dev = "d" + } + + # region abbreviation + region_short_name = local.region_map[var.aws_region] + + # environment abbreviation + environment_short_name = local.environment_map[var.environment] + + # global prefix + global_prefix = "${var.project_name}-${local.region_short_name}-${local.environment_short_name}-${var.client_name}" + + # Resource prefix + iam_policy_prefix = "${local.global_prefix}-pol" + + # IAM policy name + lambda_policy_name = "${local.iam_policy_prefix}-textract-lambda-role" + textract_service_policy_name = "${local.iam_policy_prefix}-textract-service-role" + + common_tags = { + ClientName = var.client_name + } +} + +# Deploy SQS, SNS, S3, +module "textract_pipeline_part1" { + source = "../../modules/aws-textract-pipeline-part1" + + client_name = var.client_name + + project_name = var.project_name + aws_region = var.aws_region + environment = var.environment +} + +# Deploy Lambda, Lambda Layer, API Gateway +module "textract_pipeline_part3" { + source = "../../modules/aws-textract-pipeline-part3" + + client_name = var.client_name + + project_name = var.project_name + aws_region = var.aws_region + environment = var.environment + vpc_id = var.vpc_id + secret_manager_name = var.secret_manager_name + devops_s3_bucket = var.devops_s3_bucket + + s3_bucket_name = module.textract_pipeline_part1.textract_s3_bucket_name + + insert_record_queue_arn = module.textract_pipeline_part1.insert_record_queue_arn + docx_to_pdf_queue_arn = module.textract_pipeline_part1.docx_to_pdf_queue_arn + tiff_to_pdf_queue_arn = module.textract_pipeline_part1.tiff_to_pdf_queue_arn + pdf_validations_queue_arn = module.textract_pipeline_part1.pdf_validation_queue_arn + textract_sender_queue_arn = module.textract_pipeline_part1.sender_queue_arn + textract_receiver_queue_arn = module.textract_pipeline_part1.receiver_queue_arn + text_creation_queue_arn = module.textract_pipeline_part1.text_creation_queue_arn + + textract_role_arn = var.textract_role_arn + textract_sns_topic_arn = module.textract_pipeline_part1.textract_notification_sns_arn + + # lambda_role_arn = var.lambda_role_arn + lambda_role_arn = "arn:aws:iam::660131068782:role/doczy-dev-infra-lambda-universal-role" + +} + +# Construct IAM Policy for textract role +data "aws_iam_policy_document" "textract_policy" { + statement { + effect = "Allow" + actions = ["sns:Publish"] + resources = ["${module.textract_pipeline_part1.textract_notification_sns_arn}"] + } +} + +# Construct IAM Policy for Lambda role +data "aws_iam_policy_document" "lambda_policy" { + statement { + sid = "AllowListBucket" + + actions = [ + "s3:ListBucket", + "s3:ListAllMyBuckets", + "s3:GetObject", + "s3:PutObject", + "s3:PutBucketNotification", + "s3:GetBucketNotification", + "s3:DeleteObject", + "s3:DeleteObjectTagging", + "s3:DeleteObjectVersion", + "s3:DeleteObjectVersionTagging", + "s3:GetObjectTagging", + "s3:PutObjectTagging" + ] + + effect = "Allow" + resources = ["*"] + + } + + statement { + sid = "LambdaPermissions" + + actions = [ + "lambda:CreateFunction", + "lambda:DeleteFunction", + "lambda:DeleteLayerVersion", + "lambda:Get*", + "lambda:InvokeFunction", + "lambda:List*", + "lambda:PublishLayerVersion", + "lambda:PublishVersion", + "lambda:UpdateFunctionCode", + "lambda:UpdateFunctionConfiguration" + ] + + effect = "Allow" + resources = ["*"] + } + + statement { + sid = "IamPermissions" + + actions = [ + "iam:Get*", + "iam:List*" + ] + + effect = "Allow" + resources = ["*"] + } + + statement { + sid = "TextractPermissions" + + actions = [ + "textract:*" + ] + + effect = "Allow" + resources = ["*"] + } + + statement { + sid = "SnsPermissions" + + actions = [ + "sns:*" + ] + + effect = "Allow" + resources = ["*"] + } + + statement { + sid = "SqsPermissions" + + actions = [ + "sqs:*" + ] + + effect = "Allow" + resources = ["*"] + } + + statement { + sid = "CloudWatchLogsPermissions" + + actions = [ + "logs:*", + "cloudwatch:*" + ] + + effect = "Allow" + resources = ["*"] + } + + statement { + sid = "SecretManagersPermissions" + + actions = [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + "secretsmanager:ListSecretVersionIds" + ] + + effect = "Allow" + resources = ["arn:aws:secretsmanager:*:660131068782:secret:*"] + } +} + +# IAM Policy for textract role +resource "aws_iam_policy" "textract_policy" { + name = local.textract_service_policy_name + description = "Client textract permissions" + policy = data.aws_iam_policy_document.textract_policy.json +} + +# IAM Policy for lambda role +resource "aws_iam_policy" "lambda_policy" { + name = local.lambda_policy_name + description = "Doczy Pipeline Lambda permissions" + policy = data.aws_iam_policy_document.lambda_policy.json +} + +# Attach IAM Policy to textract role +resource "aws_iam_role_policy_attachment" "textract_policy" { + role = var.textract_role_name + policy_arn = aws_iam_policy.textract_policy.arn +} + +# Attach IAM Policy to lambda role +resource "aws_iam_role_policy_attachment" "lambda_policy" { + role = var.lambda_role_name + policy_arn = aws_iam_policy.lambda_policy.arn +} \ No newline at end of file diff --git a/textract-pipeline/terraform/modules/aws-textract-pipeline/outputs.tf b/textract-pipeline/terraform/modules/aws-textract-pipeline/outputs.tf new file mode 100644 index 0000000..363d292 --- /dev/null +++ b/textract-pipeline/terraform/modules/aws-textract-pipeline/outputs.tf @@ -0,0 +1,119 @@ +# Queue arn's +output "insert_record_queue_arn" { + description = "Insert record queue arn" + value = module.textract_pipeline_part1.insert_record_queue_arn +} + +output "docx_to_pdf_queue_arn" { + description = "Docx to PDF queue arn" + value = module.textract_pipeline_part1.docx_to_pdf_queue_arn +} + +output "tiff_to_pdf_queue_arn" { + description = "Docx to PDF queue arn" + value = module.textract_pipeline_part1.tiff_to_pdf_queue_arn +} + +output "pdf_validation_queue_arn" { + description = "Docx to PDF queue arn" + value = module.textract_pipeline_part1.pdf_validation_queue_arn +} + +output "sender_queue_arn" { + description = "Sender queue arn" + value = module.textract_pipeline_part1.sender_queue_arn +} + +output "receiver_queue_arn" { + description = "Receiver queue arn" + value = module.textract_pipeline_part1.receiver_queue_arn +} + +output "text_creation_queue_arn" { + description = "Text creation queue arn" + value = module.textract_pipeline_part1.text_creation_queue_arn +} + +# SNS arn's +output "textract_notification_sns_arn" { + description = "sns topic arn for textract service" + value = module.textract_pipeline_part1.textract_notification_sns_arn +} + +# S3 bucket name +output "textract_s3_bucket_name" { + description = "sns topic arn for textract service" + value = module.textract_pipeline_part1.textract_s3_bucket_name +} + +# Lambda Layer Arn's +output "lambda_layer_libre_office_arn" { + description = "batch creation lambda fucntion arn" + value = module.textract_pipeline_part3.lambda_layer_libre_office_arn +} + +output "lambda_layer_brotli_arn" { + description = "batch creation lambda fucntion arn" + value = module.textract_pipeline_part3.lambda_layer_brotli_arn +} + +output "lambda_layer_pypdf2_arn" { + description = "batch creation lambda fucntion arn" + value = module.textract_pipeline_part3.lambda_layer_pypdf2_arn +} + +# Lambda Arn's +output "batch_creation_lambda_function_arn" { + description = "batch creation lambda fucntion arn" + value = module.textract_pipeline_part3.batch_creation_lambda_function_arn +} + +output "trigger_pipeline_lambda_function_arn" { + description = "trigger pipeline lambda fucntion arn" + value = module.textract_pipeline_part3.trigger_pipeline_lambda_function_arn +} + +output "insert_record_lambda_function_arn" { + description = "insert record lambda fucntion arn" + value = module.textract_pipeline_part3.insert_record_lambda_function_arn +} + +output "docx_to_pdf_lambda_function_arn" { + description = "docx to pdf lambda fucntion arn" + value = module.textract_pipeline_part3.docx_to_pdf_lambda_function_arn +} + +output "tiff_to_pdf_lambda_function_arn" { + description = "tiff to pdf lambda fucntion arn" + value = module.textract_pipeline_part3.tiff_to_pdf_lambda_function_arn +} + +output "pdf_validation_lambda_function_arn" { + description = "pdf validation lambda fucntion arn" + value = module.textract_pipeline_part3.pdf_validation_lambda_function_arn +} + +output "textract_sender_lambda_function_arn" { + description = "textract sender lambda fucntion arn" + value = module.textract_pipeline_part3.textract_sender_lambda_function_arn +} + +output "textract_receiver_lambda_function_arn" { + description = "textract receiver lambda fucntion arn" + value = module.textract_pipeline_part3.textract_receiver_lambda_function_arn +} + +output "text_creation_lambda_function_arn" { + description = "text creation lambda fucntion arn" + value = module.textract_pipeline_part3.text_creation_lambda_function_arn +} + +output "database_interface_lambda_function_arn" { + description = "database interface lambda fucntion arn" + value = module.textract_pipeline_part3.database_interface_lambda_function_arn +} + +# API Gateway invoke_url +output "api_invoke_url" { + value = module.textract_pipeline_part3.api_invoke_url +} \ No newline at end of file diff --git a/textract-pipeline/terraform/modules/aws-textract-pipeline/variables.tf b/textract-pipeline/terraform/modules/aws-textract-pipeline/variables.tf new file mode 100644 index 0000000..a12b3c5 --- /dev/null +++ b/textract-pipeline/terraform/modules/aws-textract-pipeline/variables.tf @@ -0,0 +1,54 @@ +# Required +variable "project_name" { + type = string +} + +# Required +variable "client_name" { + type = string +} + +# Required +variable "environment" { + type = string +} + +# Required +variable "aws_region" { + type = string +} + +# Required +variable "vpc_id" { + type = string +} + +# Required +variable "secret_manager_name" { + type = string +} + +# Required +variable "devops_s3_bucket" { + type = string +} + +# Required +variable "lambda_role_arn" { + type = string +} + +# Required +variable "textract_role_arn" { + type = string +} + +# Required +variable "textract_role_name" { + type = string +} + +# Required +variable "lambda_role_name" { + type = string +} \ No newline at end of file diff --git a/textract-pipeline/terraform/outputs.tf b/textract-pipeline/terraform/outputs.tf new file mode 100644 index 0000000..a634dfc --- /dev/null +++ b/textract-pipeline/terraform/outputs.tf @@ -0,0 +1,125 @@ +# Queue arn's +output "insert_record_queue_arn" { + description = "Insert record queue arn" + value = [ for output_list in module.textract_pipeline : output_list.insert_record_queue_arn] +} + +output "docx_to_pdf_queue_arn" { + description = "Docx to PDF queue arn" + value = [ for output_list in module.textract_pipeline : output_list.docx_to_pdf_queue_arn] +} + +output "tiff_to_pdf_queue_arn" { + description = "Docx to PDF queue arn" + value = [ for output_list in module.textract_pipeline : output_list.tiff_to_pdf_queue_arn] +} + +output "pdf_validation_queue_arn" { + description = "Docx to PDF queue arn" + value = [ for output_list in module.textract_pipeline : output_list.pdf_validation_queue_arn] +} + +output "sender_queue_arn" { + description = "Sender queue arn" + value = [ for output_list in module.textract_pipeline : output_list.sender_queue_arn] +} + +output "receiver_queue_arn" { + description = "Receiver queue arn" + value = [ for output_list in module.textract_pipeline : output_list.receiver_queue_arn] +} + +output "text_creation_queue_arn" { + description = "Text creation queue arn" + value = [ for output_list in module.textract_pipeline : output_list.text_creation_queue_arn] +} + +# SNS arn's +output "textract_notification_sns_arn" { + description = "sns topic arn for textract service" + value = [ for output_list in module.textract_pipeline : output_list.textract_notification_sns_arn] +} + +# S3 bucket name +output "textract_s3_bucket_name" { + description = "sns topic arn for textract service" + value = [ for output_list in module.textract_pipeline : output_list.textract_s3_bucket_name] +} + +# Lambda Layer Arn's +output "lambda_layer_libre_office_arn" { + value = [ for output_list in module.textract_pipeline : output_list.lambda_layer_libre_office_arn] +} + +output "lambda_layer_brotli_arn" { + value = [ for output_list in module.textract_pipeline : output_list.lambda_layer_brotli_arn] +} + +output "lambda_layer_pypdf2_arn" { + value = [ for output_list in module.textract_pipeline : output_list.lambda_layer_pypdf2_arn] +} + +# Lambda Arn's +output "batch_creation_lambda_function_arn" { + description = "batch creation lambda fucntion arn" + value = [ for output_list in module.textract_pipeline : output_list.batch_creation_lambda_function_arn] +} + +output "trigger_pipeline_lambda_function_arn" { + description = "trigger pipeline lambda fucntion arn" + value = [ for output_list in module.textract_pipeline : output_list.trigger_pipeline_lambda_function_arn] +} + +output "insert_record_lambda_function_arn" { + description = "insert record lambda fucntion arn" + value = [ for output_list in module.textract_pipeline : output_list.insert_record_lambda_function_arn] +} + +output "docx_to_pdf_lambda_function_arn" { + description = "docx to pdf lambda fucntion arn" + value = [ for output_list in module.textract_pipeline : output_list.docx_to_pdf_lambda_function_arn] +} + +output "tiff_to_pdf_lambda_function_arn" { + description = "tiff to pdf lambda fucntion arn" + value = [ for output_list in module.textract_pipeline : output_list.tiff_to_pdf_lambda_function_arn] +} + +output "pdf_validation_lambda_function_arn" { + description = "pdf validation lambda fucntion arn" + value = [ for output_list in module.textract_pipeline : output_list.pdf_validation_lambda_function_arn] +} + +output "textract_sender_lambda_function_arn" { + description = "textract sender lambda fucntion arn" + value = [ for output_list in module.textract_pipeline : output_list.textract_sender_lambda_function_arn] +} + +output "textract_receiver_lambda_function_arn" { + description = "textract receiver lambda fucntion arn" + value = [ for output_list in module.textract_pipeline : output_list.textract_receiver_lambda_function_arn] +} + +output "text_creation_lambda_function_arn" { + description = "text creation lambda fucntion arn" + value = [ for output_list in module.textract_pipeline : output_list.text_creation_lambda_function_arn] +} + +output "database_interface_lambda_function_arn" { + description = "database interface lambda fucntion arn" + value = [ for output_list in module.textract_pipeline : output_list.database_interface_lambda_function_arn] +} + +# API Gateway invoke_url +output "api_invoke_url" { + value = [ for output_list in module.textract_pipeline : output_list.api_invoke_url] +} + +# IAM Roles +output "lambda_iam_role_arn"{ + value = aws_iam_role.lambda_role.arn +} + +output "textract_iam_role_arn"{ + value = aws_iam_role.textract_role.arn +} \ No newline at end of file diff --git a/textract-pipeline/terraform/variables.tf b/textract-pipeline/terraform/variables.tf new file mode 100644 index 0000000..0787a8b --- /dev/null +++ b/textract-pipeline/terraform/variables.tf @@ -0,0 +1,49 @@ +# required +variable "aws_profile" { + type = string + default = "doczyai" +} +# required +variable "aws_region" { + type = string + default = "us-east-2" +} +# required +variable "project_name" { + type = string + default = "doczyai" +} +# required +variable "environment" { + type = string + default = "dev" +} +# Required +variable "vpc_id" { + type = string + default = "vpc-0f9d7c913f6522079" +} + +# Required +variable "secret_manager_name" { + type = string + default = "doczy-dev-db-svc-acc" +} + +# Required +variable "devops_s3_bucket" { + type = string + default = "doczy-test-bucket" +} + + +variable "client_list" { + type = map(object({ + client_name = string + })) + default = { + client1 = { + client_name = "cn1" + } + } +} \ No newline at end of file