Added textract pipeline terraform code

textract-pipeline/src/lambda/batch-creation/index.py

@@ -0,0 +1,3 @@
+# AWS Lambda handler function
+def lambda_handler(event, context):
+    print("Sender Lambda")

textract-pipeline/src/lambda/database-interface/index.py

@@ -0,0 +1,3 @@
+# AWS Lambda handler function
+def lambda_handler(event, context):
+    print("Sender Lambda")

textract-pipeline/src/lambda/docx-to-pdf/index.py

@@ -0,0 +1,3 @@
+# AWS Lambda handler function
+def lambda_handler(event, context):
+    print("Sender Lambda")

textract-pipeline/src/lambda/insert-record/index.py

@@ -0,0 +1,3 @@
+# AWS Lambda handler function
+def lambda_handler(event, context):
+    print("Sender Lambda")

textract-pipeline/src/lambda/pdf-validation/index.py

@@ -0,0 +1,3 @@
+# AWS Lambda handler function
+def lambda_handler(event, context):
+    print("Sender Lambda")

textract-pipeline/src/lambda/text-creation/index.py

@@ -0,0 +1,3 @@
+# AWS Lambda handler function
+def lambda_handler(event, context):
+    print("Sender Lambda")

textract-pipeline/src/lambda/textract-receiver/index.py

@@ -0,0 +1,3 @@
+# AWS Lambda handler function
+def lambda_handler(event, context):
+    print("Receiver Lambda")

textract-pipeline/src/lambda/textract-sender/index.py

@@ -0,0 +1,3 @@
+# AWS Lambda handler function
+def lambda_handler(event, context):
+    print("Sender Lambda")

textract-pipeline/src/lambda/tiff-to-pdf/index.py

@@ -0,0 +1,3 @@
+# AWS Lambda handler function
+def lambda_handler(event, context):
+    print("Sender Lambda")

textract-pipeline/src/lambda/trigger-pipeline/index.py

@@ -0,0 +1,3 @@
+# AWS Lambda handler function
+def lambda_handler(event, context):
+    print("Sender Lambda")

textract-pipeline/src/s3-files/config.properties

@@ -0,0 +1,28 @@
+[RESOURCES]
+SNS_TOPIC_ARN=arn:aws:sns:{aws_region}:{aws_account_id}:doczy-textract-sender-sns-dev
+TEXTRACT_ROLE_ARN=arn:aws:iam::{aws_account_id}:role/doczy-dev-textract-role
+SQS_QUEUE_URL=https://sqs.{aws_region}.amazonaws.com/{aws_account_id}/doczy-textract-sender-sqs-dev
+PDF_SQS_QUEUE_URL=https://sqs.{aws_region}.amazonaws.com/{aws_account_id}/doczy-textract-pdf-sqs-dev
+
+[FOLDER_LOCATIONS]
+CONTRACTS_LANDNING_ZONE=contracts_landing_zone/
+ALL_PDF_LOCATION=batches/{}/all_pdf_files/
+SOURCE_LOCATION=batches/{}/source-documents-pdf/
+SOURCE_DOCX_LOCATION=batches/{}/source-documents-docx/
+SOURCE_DOCX_PROCESSED_LOCATION=batches/{}/source-documents-docx-processed/
+SOURCE_DOCX_UNPROCESSED_LOCATION=batches/{}/source-documents-docx-unprocessed/
+STAGING_LOCATION=batches/{}/textract-sender-staging-pdf/
+OUTPUT_LOCATION=batches/{}/textract-output-json/
+PROCESSED_LOCATION=batches/{}/textract-receiver-processed-file/
+UNPROCESSED_LOCATION=batches/{}/textract-receiver-unprocessed-file/
+CONTRACT_TEXT_FILE_LOCATION=batches/{}/contract-text-file/
+LLM_RESPONSE_FILE_LOCATION=batches/{}/llm-response-file/
+INVALID_PDF_FILE_LOCATION=batches/{}/invalid-pdf-file/
+
+
+[OTHERS]
+SENDER_MAX_FILES=10
+PROCESS_TYPE=ANALYSIS
+ANALYSIS_FEATURE_TYPE=LAYOUT
+JOB_TAG=healthcare-contract
+LOG_LEVEL=DEBUG

textract-pipeline/terraform/main.tf

@@ -0,0 +1,128 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "5.39.0"
+    }
+  }
+
+  backend "s3" {
+    bucket         = "doczyai-use2-d-infra-s3-terraform-state"                       # Parameterize using -backend-config flag with "terraform init"
+    key            = "terraform/textract-pipeline/terraform.tfstate" # Parameterize
+    region         = "us-east-2"                               # Parameterize 
+    profile        = "doczyai"                                 # Parameterize
+    dynamodb_table = "doczyai-use2-d-infra-dyd-terraform-lock"                          # Parameterize
+    encrypt        = true
+  }
+}
+
+locals {
+  region_map = {
+    us-east-1 = "use1"
+    us-east-2 = "use2"
+    us-west-1 = "usw1"
+    us-west-2 = "usw2"
+  }
+  environment_map = {
+    prod    = "p"
+    uat     = "u"
+    qa      = "q"
+    dev     = "d"
+  }
+
+  environment_resource_map = {
+    dev = {
+      vpc_id = "vpc-0f9d7c913f6522079"
+      secret_manager_name = "doczy-dev-db-svc-acc"
+      devops_s3_bucket_name = "doczyai-use2-d-infra-s3-devops-resources"
+    }
+  }
+
+  # region abbreviation
+  region_short_name   = local.region_map[var.aws_region]
+
+  # environment abbreviation
+  environment_short_name   = local.environment_map[var.environment]
+
+  # global prefix
+  global_prefix = "${var.project_name}-${local.region_short_name}-${local.environment_short_name}-infra"
+  
+  # Resource prefix
+  iam_role_prefix = "${local.global_prefix}-rol"
+
+  # IAM role name
+  lambda_role_name = "${local.iam_role_prefix}-textract-lambda-role"
+  textract_service_role_name = "${local.iam_role_prefix}-textract-service-role"
+
+  common_tags = {
+    Environment = var.environment
+    Terraform = "true"
+  }
+}
+provider "aws" {
+  profile = var.aws_profile
+
+  default_tags {
+    tags = local.common_tags
+  }
+}
+
+module "textract_pipeline" {
+  source = "./modules/aws-textract-pipeline"
+
+  # for each for multiple resources
+  for_each = var.client_list
+
+  client_name = each.value.client_name
+
+  project_name        = var.project_name
+  aws_region          = var.aws_region
+  environment         = var.environment
+  vpc_id              = local.environment_resource_map[var.environment].vpc_id
+  secret_manager_name = local.environment_resource_map[var.environment].secret_manager_name
+  devops_s3_bucket    = local.environment_resource_map[var.environment].devops_s3_bucket_name
+
+  lambda_role_arn = aws_iam_role.lambda_role.arn
+  textract_role_arn = aws_iam_role.textract_role.arn
+
+  textract_role_name = aws_iam_role.textract_role.name
+  lambda_role_name = aws_iam_role.lambda_role.name
+}
+
+# Create lambda_role
+resource "aws_iam_role" "lambda_role" {
+  name = local.lambda_role_name
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = "sts:AssumeRole"
+        Effect = "Allow"
+        Sid    = ""
+        Principal = {
+          Service = "lambda.amazonaws.com"
+        }
+      },
+    ]
+  })
+}
+
+# Create textract_role
+resource "aws_iam_role" "textract_role" {
+  name = local.textract_service_role_name
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = "sts:AssumeRole"
+        Effect = "Allow"
+        Sid    = ""
+        Principal = {
+          Service = "textract.amazonaws.com"
+        }
+      },
+    ]
+  })
+}

textract-pipeline/terraform/modules/aws-textract-pipeline-part1/main.tf

@@ -0,0 +1,377 @@
+locals {
+  region_map = {
+    us-east-1 = "use1"
+    us-east-2 = "use2"
+    us-west-1 = "usw1"
+    us-west-2 = "usw2"
+  }
+  environment_map = {
+    prod    = "p"
+    uat     = "u"
+    qa      = "q"
+    dev     = "d"
+  }
+
+  common_tags = {
+    Environment = var.environment
+    ClientName = var.client_name
+  }
+
+  # region abbreviation
+  region_short_name   = local.region_map[var.aws_region]
+
+  # environment abbreviation
+  environment_short_name   = local.environment_map[var.environment]
+
+  # global prefix
+  global_prefix = "${var.project_name}-${local.region_short_name}-${local.environment_short_name}-${var.client_name}"
+  
+  # Resource prefix
+  sqs_prefix = "${local.global_prefix}-sqs"
+  sns_prefix = "${local.global_prefix}-sns"
+  s3_prefix = "${local.global_prefix}-s3"
+  
+  # SQS names
+  insert_record_sqs_queue_name = "${local.sqs_prefix}-${var.insert_record_sqs_queue.name}"
+  docx_to_pdf_sqs_queue_name = "${local.sqs_prefix}-${var.docx_to_pdf_sqs_queue.name}"
+  tiff_to_pdf_sqs_queue_name = "${local.sqs_prefix}-${var.tiff_to_pdf_sqs_queue.name}"
+  pdf_validations_sqs_queue_name = "${local.sqs_prefix}-${var.pdf_validations_sqs_queue.name}"
+  sender_queue_name = "${local.sqs_prefix}-${var.sender_sqs_queue.name}"
+  receiver_queue_name = "${local.sqs_prefix}-${var.receiver_sqs_queue.name}"
+  text_creation_queue_name = "${local.sqs_prefix}-${var.text_creation_sqs_queue.name}"
+
+  # SNS names
+  textract_notification_sns_name = "${local.sns_prefix}-${var.textract_notification_sns.name}"
+
+  # S3 names
+  processing_s3_bucket_name = "${local.s3_prefix}-textract-processing-001"
+}
+
+# insert_record_sqs_queue_name SQS Queue
+module "insert_record_sqs" {
+  source  = "terraform-aws-modules/sqs/aws"
+  version = "4.1.1"
+  name    = local.insert_record_sqs_queue_name
+  delay_seconds             = var.insert_record_sqs_queue.delay_seconds
+  max_message_size          = var.insert_record_sqs_queue.max_message_size
+  message_retention_seconds = var.insert_record_sqs_queue.message_retention_seconds
+  visibility_timeout_seconds= var.insert_record_sqs_queue.visibility_timeout_seconds
+
+  create_queue_policy = true
+
+  queue_policy_statements = {
+    s3 = {
+
+      sid = "AllowS3SendMessage"
+      effect = "Allow"
+
+      principals = [{
+        type        = "Service"
+        identifiers = ["s3.amazonaws.com"]
+      }]
+
+      actions   = ["sqs:SendMessage"]
+
+      conditions = [{
+        test     = "ArnEquals"
+        variable = "aws:SourceArn"
+        values   = [aws_s3_bucket.textract_processing.arn]
+      }]
+    }
+  }
+
+  tags    = local.common_tags
+}
+
+# docx_to_pdf_sqs_queue SQS Queue
+module "docx_to_pdf_sqs" {
+  source  = "terraform-aws-modules/sqs/aws"
+  version = "4.1.1"
+  name    = local.docx_to_pdf_sqs_queue_name
+  delay_seconds             = var.docx_to_pdf_sqs_queue.delay_seconds
+  max_message_size          = var.docx_to_pdf_sqs_queue.max_message_size
+  message_retention_seconds = var.docx_to_pdf_sqs_queue.message_retention_seconds
+  visibility_timeout_seconds= var.docx_to_pdf_sqs_queue.visibility_timeout_seconds
+
+  create_queue_policy = true
+
+  queue_policy_statements = {
+    s3 = {
+
+      sid = "AllowS3SendMessage"
+      effect = "Allow"
+
+      principals = [{
+        type        = "Service"
+        identifiers = ["s3.amazonaws.com"]
+      }]
+
+      actions   = ["sqs:SendMessage"]
+
+      conditions = [{
+        test     = "ArnEquals"
+        variable = "aws:SourceArn"
+        values   = [aws_s3_bucket.textract_processing.arn]
+      }]
+    }
+  }
+
+  tags    = local.common_tags
+}
+
+# tiff_to_pdf_sqs_queue SQS Queue
+module "tiff_to_pdf_sqs" {
+  source  = "terraform-aws-modules/sqs/aws"
+  version = "4.1.1"
+  name    = local.tiff_to_pdf_sqs_queue_name
+  delay_seconds             = var.tiff_to_pdf_sqs_queue.delay_seconds
+  max_message_size          = var.tiff_to_pdf_sqs_queue.max_message_size
+  message_retention_seconds = var.tiff_to_pdf_sqs_queue.message_retention_seconds
+  visibility_timeout_seconds= var.tiff_to_pdf_sqs_queue.visibility_timeout_seconds
+
+  create_queue_policy = true
+
+  queue_policy_statements = {
+    s3 = {
+
+      sid = "AllowS3SendMessage"
+      effect = "Allow"
+
+      principals = [{
+        type        = "Service"
+        identifiers = ["s3.amazonaws.com"]
+      }]
+
+      actions   = ["sqs:SendMessage"]
+
+      conditions = [{
+        test     = "ArnEquals"
+        variable = "aws:SourceArn"
+        values   = [aws_s3_bucket.textract_processing.arn]
+      }]
+    }
+  }
+
+  tags    = local.common_tags
+}
+
+# pdf_validations_sqs_queue SQS Queue
+module "pdf_validations_sqs" {
+  source  = "terraform-aws-modules/sqs/aws"
+  version = "4.1.1"
+  name    = local.pdf_validations_sqs_queue_name
+  delay_seconds             = var.pdf_validations_sqs_queue.delay_seconds
+  max_message_size          = var.pdf_validations_sqs_queue.max_message_size
+  message_retention_seconds = var.pdf_validations_sqs_queue.message_retention_seconds
+  visibility_timeout_seconds= var.pdf_validations_sqs_queue.visibility_timeout_seconds
+
+  create_queue_policy = true
+
+  queue_policy_statements = {
+    s3 = {
+
+      sid = "AllowS3SendMessage"
+      effect = "Allow"
+
+      principals = [{
+        type        = "Service"
+        identifiers = ["s3.amazonaws.com"]
+      }]
+
+      actions   = ["sqs:SendMessage"]
+
+      conditions = [{
+        test     = "ArnEquals"
+        variable = "aws:SourceArn"
+        values   = [aws_s3_bucket.textract_processing.arn]
+      }]
+    }
+  }
+
+  tags    = local.common_tags
+}
+
+# Sender SQS Queue
+module "textract_sender_sqs" {
+  source  = "terraform-aws-modules/sqs/aws"
+  version = "4.1.1"
+  name    = local.sender_queue_name
+  delay_seconds             = var.sender_sqs_queue.delay_seconds
+  max_message_size          = var.sender_sqs_queue.max_message_size
+  message_retention_seconds = var.sender_sqs_queue.message_retention_seconds
+  visibility_timeout_seconds= var.sender_sqs_queue.visibility_timeout_seconds
+
+  create_queue_policy = true
+
+  queue_policy_statements = {
+    s3 = {
+
+      sid = "AllowS3SendMessage"
+      effect = "Allow"
+
+      principals = [{
+        type        = "Service"
+        identifiers = ["s3.amazonaws.com"]
+      }]
+
+      actions   = ["sqs:SendMessage"]
+
+      conditions = [{
+        test     = "ArnEquals"
+        variable = "aws:SourceArn"
+        values   = [aws_s3_bucket.textract_processing.arn]
+      }]
+    }
+  }
+
+  tags    = local.common_tags
+}
+
+# textract notification sns 
+module "textract_notification_sns" {
+  source  = "terraform-aws-modules/sns/aws"
+  version = ">= 5.0"
+
+  name = local.textract_notification_sns_name
+
+  topic_policy_statements = {
+    sqs = {
+      sid = "SQSSubscribe"
+      actions = [
+        "sns:Subscribe",
+        "sns:Receive",
+      ]
+
+      principals = [{
+        type        = "AWS"
+        identifiers = ["*"]
+      }]
+
+      conditions = [{
+        test     = "StringLike"
+        variable = "sns:Endpoint"
+        values   = [module.textract_receiver_sqs.queue_arn]
+      }]
+    }
+  }
+
+  subscriptions = {
+    sqs = {
+      protocol = "sqs"
+      endpoint = module.textract_receiver_sqs.queue_arn
+    }
+  }
+
+  tags    = local.common_tags
+}
+
+# Receiver SQS Queue
+module "textract_receiver_sqs" {
+  source  = "terraform-aws-modules/sqs/aws"
+  version = "4.1.1"
+  name    = local.receiver_queue_name
+  delay_seconds             = var.receiver_sqs_queue.delay_seconds
+  max_message_size          = var.receiver_sqs_queue.max_message_size
+  message_retention_seconds = var.receiver_sqs_queue.message_retention_seconds
+  visibility_timeout_seconds= var.receiver_sqs_queue.visibility_timeout_seconds
+  tags    = local.common_tags
+
+  create_queue_policy = true
+  queue_policy_statements = {
+    sns = {
+      sid     = "SNSPublish"
+      actions = ["sqs:SendMessage"]
+
+      principals = [
+        {
+          type        = "Service"
+          identifiers = ["sns.amazonaws.com"]
+        }
+      ]
+
+      conditions = [{
+        test     = "ArnEquals"
+        variable = "aws:SourceArn"
+        values   = [module.textract_notification_sns.topic_arn]
+      }]
+    }
+  }
+}
+
+# Text-creation SQS Queue
+module "text_creation_sqs" {
+  source  = "terraform-aws-modules/sqs/aws"
+  version = "4.1.1"
+  name    = local.text_creation_queue_name
+  delay_seconds             = var.text_creation_sqs_queue.delay_seconds
+  max_message_size          = var.text_creation_sqs_queue.max_message_size
+  message_retention_seconds = var.text_creation_sqs_queue.message_retention_seconds
+  visibility_timeout_seconds= var.text_creation_sqs_queue.visibility_timeout_seconds
+  tags    = local.common_tags
+
+  create_queue_policy = true
+
+  queue_policy_statements = {
+    s3 = {
+
+      sid = "AllowS3SendMessage"
+      effect = "Allow"
+
+      principals = [{
+        type        = "Service"
+        identifiers = ["s3.amazonaws.com"]
+      }]
+
+      actions   = ["sqs:SendMessage"]
+
+      conditions = [{
+        test     = "ArnEquals"
+        variable = "aws:SourceArn"
+        values   = [aws_s3_bucket.textract_processing.arn]
+      }]
+    }
+  }
+}
+
+resource "aws_s3_bucket" "textract_processing" {
+  bucket = local.processing_s3_bucket_name
+
+  tags = local.common_tags
+}
+
+resource "aws_s3_bucket_policy" "deny_insecure_communication" {
+  bucket = aws_s3_bucket.textract_processing.id
+  policy = data.aws_iam_policy_document.deny_insecure_communication.json
+}
+
+data "aws_iam_policy_document" "deny_insecure_communication" {
+  statement {
+    sid = "DenyInsecureCommunications"
+   
+    principals {
+      type        = "*"
+      identifiers = ["*"]
+    }
+
+    condition {
+        test     = "Bool"
+        variable = "aws:SecureTransport"
+        values   = ["false"]
+    }
+
+    effect = "Deny"
+
+    actions = ["s3:*"]
+
+    resources = [
+      aws_s3_bucket.textract_processing.arn,
+      "${aws_s3_bucket.textract_processing.arn}/*",
+    ]
+  }
+}
+
+resource "aws_s3_object" "config" {
+  key        = "config.properties"
+  bucket     = aws_s3_bucket.textract_processing.id
+  source     = "../src/s3-files/config.properties"
+}

textract-pipeline/terraform/modules/aws-textract-pipeline-part1/outputs.tf

@@ -0,0 +1,47 @@
+# Queue arn's
+output "insert_record_queue_arn" {
+  description = "Insert record queue arn"
+  value       = module.insert_record_sqs.queue_arn
+}
+
+output "docx_to_pdf_queue_arn" {
+  description = "Docx to PDF queue arn"
+  value       = module.docx_to_pdf_sqs.queue_arn
+}
+
+output "tiff_to_pdf_queue_arn" {
+  description = "Docx to PDF queue arn"
+  value       = module.tiff_to_pdf_sqs.queue_arn
+}
+
+output "pdf_validation_queue_arn" {
+  description = "Docx to PDF queue arn"
+  value       = module.pdf_validations_sqs.queue_arn
+}
+
+output "sender_queue_arn" {
+  description = "Sender queue arn"
+  value       = module.textract_sender_sqs.queue_arn
+}
+
+output "receiver_queue_arn" {
+  description = "Receiver queue arn"
+  value       = module.textract_receiver_sqs.queue_arn
+}
+
+output "text_creation_queue_arn" {
+  description = "Text creation queue arn"
+  value       = module.text_creation_sqs.queue_arn
+}
+
+# SNS arn's
+output "textract_notification_sns_arn" {
+  description = "sns topic arn for textract service"
+  value = module.textract_notification_sns.topic_arn
+}
+
+# S3 bucket name
+output "textract_s3_bucket_name" {
+  description = "sns topic arn for textract service"
+  value = aws_s3_bucket.textract_processing.id
+}

textract-pipeline/terraform/modules/aws-textract-pipeline-part1/variables.tf

@@ -0,0 +1,166 @@
+# Required
+variable "project_name" {
+  type = string
+}
+
+# Required
+variable "client_name" {
+  type = string
+}
+
+# Required
+variable "environment" {
+  type = string
+}
+
+# Required
+variable "aws_region" {
+  type = string
+}
+
+# # Required
+# variable "lambda_role" {
+#   type = string
+# }
+
+# Required
+# variable "s3_bucket_name" {
+#   type = string
+# }
+
+# textract notification sns topic
+variable "textract_notification_sns" {
+    type = object({
+      name = string
+    })
+    default = {
+      name = "textract-notification"
+    } 
+}
+
+# insert-record-queue
+variable "insert_record_sqs_queue" {
+    type = object({
+      name = string
+      delay_seconds = number
+      max_message_size          = number
+      message_retention_seconds = number
+      visibility_timeout_seconds= number
+    })
+    default = {
+      name                      = "insert-record-queue"
+      delay_seconds             = 90
+      max_message_size          = 2048
+      message_retention_seconds = 86400
+      visibility_timeout_seconds= 901
+    } 
+}
+
+
+# docx-to-pdf queue 
+variable "docx_to_pdf_sqs_queue" {
+    type = object({
+      name = string
+      delay_seconds = number
+      max_message_size          = number
+      message_retention_seconds = number
+      visibility_timeout_seconds= number
+    })
+    default = {
+      name                      = "docx-to-pdf-queue"
+      delay_seconds             = 90
+      max_message_size          = 2048
+      message_retention_seconds = 86400
+      visibility_timeout_seconds= 901
+    } 
+}
+
+# tiff-to-pdf queue 
+variable "tiff_to_pdf_sqs_queue" {
+    type = object({
+      name = string
+      delay_seconds = number
+      max_message_size          = number
+      message_retention_seconds = number
+      visibility_timeout_seconds= number
+    })
+    default = {
+      name                      = "tiff-to-pdf-queue"
+      delay_seconds             = 90
+      max_message_size          = 2048
+      message_retention_seconds = 86400
+      visibility_timeout_seconds= 901
+    } 
+}
+
+# pdf-validation-queue queue 
+variable "pdf_validations_sqs_queue" {
+    type = object({
+      name = string
+      delay_seconds = number
+      max_message_size          = number
+      message_retention_seconds = number
+      visibility_timeout_seconds= number
+    })
+    default = {
+      name                      = "pdf-validation-queue"
+      delay_seconds             = 90
+      max_message_size          = 2048
+      message_retention_seconds = 86400
+      visibility_timeout_seconds= 901
+    } 
+}
+
+# textract sender queue
+variable "sender_sqs_queue" {
+    type = object({
+      name = string
+      delay_seconds = number
+      max_message_size          = number
+      message_retention_seconds = number
+      visibility_timeout_seconds= number
+    })
+    default = {
+      name                      = "textract-sender-queue"
+      delay_seconds             = 90
+      max_message_size          = 2048
+      message_retention_seconds = 86400
+      visibility_timeout_seconds= 901
+    } 
+}
+
+# textract receiver queue 
+variable "receiver_sqs_queue" {
+    type = object({
+      name = string
+      delay_seconds = number
+      max_message_size          = number
+      message_retention_seconds = number
+      visibility_timeout_seconds= number
+    })
+    default = {
+      name                      = "textract-receiver-queue"
+      delay_seconds             = 90
+      max_message_size          = 2048
+      message_retention_seconds = 86400
+      visibility_timeout_seconds= 901
+    } 
+}
+
+# textract receiver queue 
+variable "text_creation_sqs_queue" {
+    type = object({
+      name = string
+      delay_seconds = number
+      max_message_size          = number
+      message_retention_seconds = number
+      visibility_timeout_seconds= number
+    })
+    default = {
+      name                      = "text-creation-queue"
+      delay_seconds             = 90
+      max_message_size          = 2048
+      message_retention_seconds = 86400
+      visibility_timeout_seconds= 901
+    } 
+}

textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf

@@ -0,0 +1,653 @@
+locals {
+  region_map = {
+    us-east-1 = "use1"
+    us-east-2 = "use2"
+    us-west-1 = "usw1"
+    us-west-2 = "usw2"
+  }
+  environment_map = {
+    prod    = "p"
+    uat     = "u"
+    qa      = "q"
+    dev     = "d"
+  }
+
+  common_tags = {
+    Environment = var.environment
+    ClientName = var.client_name
+  }
+
+  # region abbreviation
+  region_short_name   = local.region_map[var.aws_region]
+
+  # environment abbreviation
+  environment_short_name   = local.environment_map[var.environment]
+
+  # global prefix
+  global_prefix = "${var.project_name}-${local.region_short_name}-${local.environment_short_name}-${var.client_name}"
+  
+  # Resource prefix
+  lambda_prefix = "${local.global_prefix}-lmb"
+  api_gateway_prefix = "${local.global_prefix}-api"
+  lambda_layer_prefix = "${local.global_prefix}-lyr"
+  
+  # Lambda names
+  batch_creation_lambda_name = "${local.lambda_prefix}-${var.batch_creation_lambda.name}"
+  trigger_pipeline_lambda_name = "${local.lambda_prefix}-${var.trigger_pipeline_lambda.name}"
+  insert_record_lambda_name = "${local.lambda_prefix}-${var.insert_record_lambda.name}"
+  docx_to_pdf_lambda_name = "${local.lambda_prefix}-${var.docx_to_pdf_lambda.name}"
+  tiff_to_pdf_lambda_name = "${local.lambda_prefix}-${var.tiff_to_pdf_lambda.name}"
+  pdf_validation_lambda_name = "${local.lambda_prefix}-${var.pdf_validation_lambda.name}"
+  sender_lambda_name = "${local.lambda_prefix}-${var.textract_sender_lambda.name}"
+  receiver_lambda_name = "${local.lambda_prefix}-${var.textract_receiver_lambda.name}"
+  text_creation_lambda_name = "${local.lambda_prefix}-${var.text_creation_lambda.name}"
+  database_interface_lambda_name = "${local.lambda_prefix}-${var.database_interface_lambda.name}"
+
+  # Lambda Layer names
+  libre_office_lambda_layer_name = "${local.lambda_layer_prefix}-${var.libre_office_lambda_layer.name}"
+  brotli_lambda_layer_name = "${local.lambda_layer_prefix}-${var.brotli_lambda_layer.name}"
+  pypdf2_lambda_layer_name = "${local.lambda_layer_prefix}-${var.pypdf2_lambda_layer.name}"
+
+  # API Gateway configuration
+  api_description = "API's used for DoczyAI pipeline"
+  batch_creation_api_path = "create-batch-new"
+  trigger_pipeline_api_path = "trigger-pipeline"
+  stage_name = "dev"
+
+  # property file path
+  property_file_s3_path = "${var.s3_bucket_name}/config.properties"
+
+  # common env variables
+  environment_variables = {
+    PROPERTY_FILE_S3_PATH = local.property_file_s3_path
+    DATABASE_INTERFACE_FUNCTION_NAME = local.database_interface_lambda_name
+  }
+
+  # S3 bucket and prefix for lambda's
+  lambda_layer_s3_bucket = var.devops_s3_bucket
+  lambda_layer_s3_prefix = "lambda-layer-builds/"
+}
+
+# vpc subnets
+data "aws_subnets" "subnets" {
+  filter {
+    name   = "vpc-id"
+    values = [var.vpc_id]
+  }
+}
+
+# vpc security groups
+data "aws_security_groups" "security_groups" {
+  filter {
+    name   = "group-name"
+    values = ["*default*"]
+  }
+
+  filter {
+    name   = "vpc-id"
+    values = [var.vpc_id]
+  }
+}
+
+# Lambda Layers - Libre Office - create package from source code
+module "lambda_layer_libre_office" {
+  source = "terraform-aws-modules/lambda/aws"
+
+  create_layer = true
+  create_package = var.libre_office_lambda_layer.create_package
+
+  layer_name          = local.libre_office_lambda_layer_name
+  description         = var.libre_office_lambda_layer.description
+  compatible_runtimes = var.libre_office_lambda_layer.compatible_runtimes
+  compatible_architectures       = var.libre_office_lambda_layer.compatible_architectures
+
+  source_path = var.libre_office_lambda_layer.source_path
+
+  store_on_s3 = true
+  s3_bucket   = local.lambda_layer_s3_bucket
+  s3_prefix = local.lambda_layer_s3_prefix
+}
+
+# Lambda Layers - brotli layer - create layer from zip
+module "lambda_layer_brotli" {
+  source = "terraform-aws-modules/lambda/aws"
+
+  create_layer = true
+  create_package = false
+
+  layer_name          = local.brotli_lambda_layer_name
+  description         = var.brotli_lambda_layer.description
+  compatible_runtimes = var.brotli_lambda_layer.compatible_runtimes
+  compatible_architectures       =  var.brotli_lambda_layer.compatible_architectures
+
+  source_path = null
+
+  local_existing_package  = var.brotli_lambda_layer.local_existing_package
+
+}
+
+# Lambda Layers - pypdf2 - create layer from zip
+module "lambda_layer_pypdf2" {
+  source = "terraform-aws-modules/lambda/aws"
+
+  create_layer = true
+  create_package = false
+
+  layer_name          = local.pypdf2_lambda_layer_name
+  description         = var.pypdf2_lambda_layer.description
+  compatible_runtimes = var.pypdf2_lambda_layer.compatible_runtimes
+  compatible_architectures       =  var.pypdf2_lambda_layer.compatible_architectures
+
+  source_path = var.pypdf2_lambda_layer.source_path
+
+  local_existing_package  = var.pypdf2_lambda_layer.local_existing_package
+}
+
+# batch creation lambda function
+module "batch_creation_lambda_function" {
+  source = "terraform-aws-modules/lambda/aws"
+  version = "7.2.2"
+
+  function_name = local.batch_creation_lambda_name
+  description   = var.batch_creation_lambda.description
+  handler       = var.batch_creation_lambda.handler
+  runtime       = var.batch_creation_lambda.runtime
+  timeout       = var.batch_creation_lambda.timeout
+  memory_size   = var.batch_creation_lambda.memory_size
+  ephemeral_storage_size  = var.batch_creation_lambda.ephemeral_storage_size
+  architectures = var.batch_creation_lambda.architectures
+  
+  environment_variables = local.environment_variables
+
+  source_path   = var.batch_creation_lambda.source_path
+
+  lambda_role  = var.lambda_role_arn
+  create_role = false
+
+  vpc_subnet_ids         = data.aws_subnets.subnets.ids
+  vpc_security_group_ids = data.aws_security_groups.security_groups.ids
+
+  tags    = local.common_tags
+}
+
+# trigger pipeline lambda function
+module "trigger_pipeline_lambda_function" {
+  source = "terraform-aws-modules/lambda/aws"
+  version = "7.2.2"
+
+  function_name = local.trigger_pipeline_lambda_name
+  description   = var.trigger_pipeline_lambda.description
+  handler       = var.trigger_pipeline_lambda.handler
+  runtime       = var.trigger_pipeline_lambda.runtime
+  timeout       = var.trigger_pipeline_lambda.timeout
+  memory_size   = var.trigger_pipeline_lambda.memory_size
+  ephemeral_storage_size  = var.trigger_pipeline_lambda.ephemeral_storage_size
+  architectures = var.trigger_pipeline_lambda.architectures
+  
+  environment_variables = local.environment_variables
+
+  source_path   = var.trigger_pipeline_lambda.source_path
+
+  lambda_role  = var.lambda_role_arn
+  create_role = false
+
+  vpc_subnet_ids         = data.aws_subnets.subnets.ids
+  vpc_security_group_ids = data.aws_security_groups.security_groups.ids
+
+  tags    = local.common_tags
+}
+
+# insert record lambda function
+module "insert_record_lambda_function" {
+  source = "terraform-aws-modules/lambda/aws"
+  version = "7.2.2"
+
+  function_name = local.insert_record_lambda_name
+  description   = var.insert_record_lambda.description
+  handler       = var.insert_record_lambda.handler
+  runtime       = var.insert_record_lambda.runtime
+  timeout       = var.insert_record_lambda.timeout
+  memory_size   = var.insert_record_lambda.memory_size
+  ephemeral_storage_size  = var.insert_record_lambda.ephemeral_storage_size
+  architectures = var.insert_record_lambda.architectures
+  
+  environment_variables = local.environment_variables
+
+  source_path   = var.insert_record_lambda.source_path
+
+  event_source_mapping = {
+    sqs = {
+      event_source_arn        = var.insert_record_queue_arn
+      function_response_types = ["ReportBatchItemFailures"]
+      scaling_config = {
+        maximum_concurrency = 20
+        
+      }
+      batch_size = 1
+    }
+  }
+
+  lambda_role  = var.lambda_role_arn
+  create_role = false
+
+  vpc_subnet_ids         = data.aws_subnets.subnets.ids
+  vpc_security_group_ids = data.aws_security_groups.security_groups.ids
+
+  tags    = local.common_tags
+}
+
+# docx to pdf lambda function
+module "docx_to_pdf_lambda_function" {
+  source = "terraform-aws-modules/lambda/aws"
+  version = "7.2.2"
+
+  function_name = local.docx_to_pdf_lambda_name
+  description   = var.docx_to_pdf_lambda.description
+  handler       = var.docx_to_pdf_lambda.handler
+  runtime       = var.docx_to_pdf_lambda.runtime
+  timeout       = var.docx_to_pdf_lambda.timeout
+  memory_size   = var.docx_to_pdf_lambda.memory_size
+  ephemeral_storage_size  = var.docx_to_pdf_lambda.ephemeral_storage_size
+  architectures = var.docx_to_pdf_lambda.architectures
+
+  layers = [
+    module.lambda_layer_libre_office.lambda_layer_arn,
+    module.lambda_layer_brotli.lambda_layer_arn,
+  ]
+  
+  environment_variables = local.environment_variables
+
+  source_path   = var.docx_to_pdf_lambda.source_path
+
+  event_source_mapping = {
+    sqs = {
+      event_source_arn        = var.docx_to_pdf_queue_arn
+      function_response_types = ["ReportBatchItemFailures"]
+      scaling_config = {
+        maximum_concurrency = 20
+        
+      }
+      batch_size = 1
+    }
+  }
+
+  lambda_role  = var.lambda_role_arn
+  create_role = false
+
+  vpc_subnet_ids         = data.aws_subnets.subnets.ids
+  vpc_security_group_ids = data.aws_security_groups.security_groups.ids
+
+  tags    = local.common_tags
+}
+
+# tiff to pdf lambda function
+module "tiff_to_pdf_lambda_function" {
+  source = "terraform-aws-modules/lambda/aws"
+  version = "7.2.2"
+
+  function_name = local.tiff_to_pdf_lambda_name
+  description   = var.tiff_to_pdf_lambda.description
+  handler       = var.tiff_to_pdf_lambda.handler
+  runtime       = var.tiff_to_pdf_lambda.runtime
+  timeout       = var.tiff_to_pdf_lambda.timeout
+  memory_size   = var.tiff_to_pdf_lambda.memory_size
+  ephemeral_storage_size  = var.tiff_to_pdf_lambda.ephemeral_storage_size
+  architectures = var.tiff_to_pdf_lambda.architectures
+  
+  environment_variables = local.environment_variables
+
+  source_path   = var.tiff_to_pdf_lambda.source_path
+
+  event_source_mapping = {
+    sqs = {
+      event_source_arn        = var.tiff_to_pdf_queue_arn
+      function_response_types = ["ReportBatchItemFailures"]
+      scaling_config = {
+        maximum_concurrency = 20
+        
+      }
+      batch_size = 1
+    }
+  }
+
+  lambda_role  = var.lambda_role_arn
+  create_role = false
+
+  vpc_subnet_ids         = data.aws_subnets.subnets.ids
+  vpc_security_group_ids = data.aws_security_groups.security_groups.ids
+
+  tags    = local.common_tags
+}
+
+# pdf validation lambda function
+module "pdf_validation_lambda_function" {
+  source = "terraform-aws-modules/lambda/aws"
+  version = "7.2.2"
+
+  function_name = local.pdf_validation_lambda_name
+  description   = var.pdf_validation_lambda.description
+  handler       = var.pdf_validation_lambda.handler
+  runtime       = var.pdf_validation_lambda.runtime
+  timeout       = var.pdf_validation_lambda.timeout
+  memory_size   = var.pdf_validation_lambda.memory_size
+  ephemeral_storage_size  = var.pdf_validation_lambda.ephemeral_storage_size
+  architectures = var.pdf_validation_lambda.architectures
+  
+  layers = [
+    module.lambda_layer_pypdf2.lambda_layer_arn
+  ]
+
+  environment_variables = local.environment_variables
+
+  source_path   = var.pdf_validation_lambda.source_path
+
+  event_source_mapping = {
+    sqs = {
+      event_source_arn        = var.pdf_validations_queue_arn
+      function_response_types = ["ReportBatchItemFailures"]
+      scaling_config = {
+        maximum_concurrency = 20
+        
+      }
+      batch_size = 1
+    }
+  }
+
+  lambda_role  = var.lambda_role_arn
+  create_role = false
+
+  vpc_subnet_ids         = data.aws_subnets.subnets.ids
+  vpc_security_group_ids = data.aws_security_groups.security_groups.ids
+
+  tags    = local.common_tags
+}
+
+# sender lambda function
+module "textract_sender_lambda_function" {
+  source = "terraform-aws-modules/lambda/aws"
+  version = "7.2.2"
+
+  function_name = local.sender_lambda_name
+  description   = var.textract_sender_lambda.description
+  handler       = var.textract_sender_lambda.handler
+  runtime       = var.textract_sender_lambda.runtime
+  timeout       = var.textract_sender_lambda.timeout
+  memory_size   = var.textract_sender_lambda.memory_size
+  ephemeral_storage_size  = var.textract_sender_lambda.ephemeral_storage_size
+  architectures = var.textract_sender_lambda.architectures
+  
+  environment_variables = merge(local.environment_variables,{
+    TEXTRACT_ROLE_ARN = var.textract_role_arn
+    SNS_TOPIC_ARN = var.textract_sns_topic_arn
+  })
+
+  source_path   = var.textract_sender_lambda.source_path
+
+  event_source_mapping = {
+    sqs = {
+      event_source_arn        = var.textract_sender_queue_arn
+      function_response_types = ["ReportBatchItemFailures"]
+      scaling_config = {
+        maximum_concurrency = 20
+        
+      }
+      batch_size = 1
+    }
+  }
+
+  lambda_role  = var.lambda_role_arn
+  create_role = false
+
+  vpc_subnet_ids         = data.aws_subnets.subnets.ids
+  vpc_security_group_ids = data.aws_security_groups.security_groups.ids
+
+  tags    = local.common_tags
+}
+
+# receiver lambda function
+module "textract_receiver_lambda_function" {
+  source = "terraform-aws-modules/lambda/aws"
+  version = "7.2.2"
+
+  function_name = local.receiver_lambda_name
+  description   = var.textract_receiver_lambda.description
+  handler       = var.textract_receiver_lambda.handler
+  runtime       = var.textract_receiver_lambda.runtime
+  timeout       = var.textract_receiver_lambda.timeout
+  memory_size   = var.textract_receiver_lambda.memory_size
+  ephemeral_storage_size  = var.textract_receiver_lambda.ephemeral_storage_size
+  architectures = var.textract_receiver_lambda.architectures
+  
+  environment_variables = local.environment_variables
+
+  source_path   = var.textract_receiver_lambda.source_path
+
+  event_source_mapping = {
+    sqs = {
+      event_source_arn        = var.textract_receiver_queue_arn
+      function_response_types = ["ReportBatchItemFailures"]
+      scaling_config = {
+        maximum_concurrency = 20
+        
+      }
+      batch_size = 1
+    }
+  }
+
+  lambda_role  = var.lambda_role_arn
+  create_role = false
+
+  vpc_subnet_ids         = data.aws_subnets.subnets.ids
+  vpc_security_group_ids = data.aws_security_groups.security_groups.ids
+
+  tags    = local.common_tags
+}
+
+# text creation lambda function
+module "text_creation_lambda_function" {
+  source = "terraform-aws-modules/lambda/aws"
+  version = "7.2.2"
+
+  function_name = local.text_creation_lambda_name
+  description   = var.text_creation_lambda.description
+  handler       = var.text_creation_lambda.handler
+  runtime       = var.text_creation_lambda.runtime
+  timeout       = var.text_creation_lambda.timeout
+  memory_size   = var.text_creation_lambda.memory_size
+  ephemeral_storage_size  = var.text_creation_lambda.ephemeral_storage_size
+  architectures = var.text_creation_lambda.architectures
+
+  layers = [
+    module.lambda_layer_pypdf2.lambda_layer_arn
+  ]
+  
+  environment_variables = local.environment_variables
+
+  source_path   = var.text_creation_lambda.source_path
+
+  event_source_mapping = {
+    sqs = {
+      event_source_arn        = var.text_creation_queue_arn
+      function_response_types = ["ReportBatchItemFailures"]
+      scaling_config = {
+        maximum_concurrency = 20
+        
+      }
+      batch_size = 1
+    }
+  }
+
+  lambda_role  = var.lambda_role_arn
+  create_role = false
+
+  vpc_subnet_ids         = data.aws_subnets.subnets.ids
+  vpc_security_group_ids = data.aws_security_groups.security_groups.ids
+
+  tags    = local.common_tags
+}
+
+# database interface lambda function
+module "database_interface_lambda_function" {
+  source = "terraform-aws-modules/lambda/aws"
+  version = "7.2.2"
+
+  function_name = local.database_interface_lambda_name
+  description   = var.database_interface_lambda.description
+  handler       = var.database_interface_lambda.handler
+  runtime       = var.database_interface_lambda.runtime
+  timeout       = var.database_interface_lambda.timeout
+  memory_size   = var.database_interface_lambda.memory_size
+  ephemeral_storage_size  = var.database_interface_lambda.ephemeral_storage_size
+  architectures = var.database_interface_lambda.architectures
+  
+  environment_variables = merge(local.environment_variables,{
+    SECRET_MANAGER_NAME = var.secret_manager_name
+  })
+
+  source_path   = var.database_interface_lambda.source_path
+
+  lambda_role  = var.lambda_role_arn
+  create_role = false
+
+  vpc_subnet_ids         = data.aws_subnets.subnets.ids
+  vpc_security_group_ids = data.aws_security_groups.security_groups.ids
+
+  tags    = local.common_tags
+}
+
+# S3 notification
+resource "aws_s3_bucket_notification" "bucket_notification" {
+  bucket = var.s3_bucket_name
+
+  queue {
+    id            = "docx-upload-event"
+    queue_arn     = var.docx_to_pdf_queue_arn
+    events        = ["s3:ObjectCreated:*"]
+    filter_prefix = "source-documents-docx/"
+    filter_suffix = ".docx"
+  }
+
+  queue {
+    id            = "doc-upload-event"
+    queue_arn     = var.docx_to_pdf_queue_arn
+    events        = ["s3:ObjectCreated:*"]
+    filter_prefix = "source-documents-docx/"
+    filter_suffix = ".doc"
+  }
+
+  queue {
+    id            = "tiff-upload-event"
+    queue_arn     = var.tiff_to_pdf_queue_arn
+    events        = ["s3:ObjectCreated:*"]
+    filter_prefix = "source-documents-tiff/"
+    filter_suffix = ".tiff"
+  }
+
+  queue {
+    id            = "all-pdf-upload-event"
+    queue_arn     = var.pdf_validations_queue_arn
+    events        = ["s3:ObjectCreated:*"]
+    filter_prefix = "all-pdfs/"
+    filter_suffix = ".pdf"
+  }
+
+  queue {
+    id            = "valid-pdf-upload-event"
+    queue_arn     = var.textract_sender_queue_arn
+    events        = ["s3:ObjectCreated:*"]
+    filter_prefix = "valid-pdfs/"
+    filter_suffix = ".pdf"
+  }
+
+  queue {
+    id            = "textract-json-upload-event"
+    queue_arn     = var.text_creation_queue_arn
+    events        = ["s3:ObjectCreated:*"]
+    filter_prefix = "textract-output-json/"
+    filter_suffix = ".json"
+  }  
+}
+
+# API Gateway resources
+resource "aws_api_gateway_rest_api" "project_api" {
+  name        = local.api_gateway_prefix
+  description = local.api_description
+
+  endpoint_configuration {
+    types = ["REGIONAL"]
+  }
+
+  tags    = local.common_tags
+}
+
+# API Gateway - Create batch - create resource
+resource "aws_api_gateway_resource" "create_batch" {
+  rest_api_id = aws_api_gateway_rest_api.project_api.id
+  parent_id   = aws_api_gateway_rest_api.project_api.root_resource_id
+  path_part   = local.batch_creation_api_path
+}
+# API Gateway - Create batch - create method
+resource "aws_api_gateway_method" "create_batch_method" {
+  rest_api_id   = "${aws_api_gateway_rest_api.project_api.id}"
+  resource_id   = "${aws_api_gateway_resource.create_batch.id}"
+  http_method   = "POST"
+  authorization = "NONE"
+}
+
+# API Gateway - Create batch - integration with lambda
+resource "aws_api_gateway_integration" "create_batch_integration" {
+  rest_api_id             = aws_api_gateway_rest_api.project_api.id
+  resource_id             = aws_api_gateway_resource.create_batch.id
+  http_method             = aws_api_gateway_method.create_batch_method.http_method
+  integration_http_method = "POST"
+  type                    = "AWS"
+  uri                     = module.batch_creation_lambda_function.lambda_function_invoke_arn
+}
+# API Gateway - Create batch - add permissions for lambda
+resource "aws_lambda_permission" "create_batch_lambda_permission" {
+  statement_id  = "AllowMyDemoAPIInvoke"
+  action        = "lambda:InvokeFunction"
+  function_name = "${module.batch_creation_lambda_function.lambda_function_name}"
+  principal     = "apigateway.amazonaws.com"
+  source_arn = "${aws_api_gateway_rest_api.project_api.execution_arn}/*/${aws_api_gateway_method.create_batch_method.http_method}${aws_api_gateway_resource.create_batch.path}"
+}
+# API Gateway - trigger pipeline - create resource
+resource "aws_api_gateway_resource" "trigger_pipeline" {
+  rest_api_id = aws_api_gateway_rest_api.project_api.id
+  parent_id   = aws_api_gateway_rest_api.project_api.root_resource_id
+  path_part   = local.trigger_pipeline_api_path
+}
+# API Gateway - trigger pipeline - create method
+resource "aws_api_gateway_method" "trigger_pipeline_method" {
+  rest_api_id   = "${aws_api_gateway_rest_api.project_api.id}"
+  resource_id   = "${aws_api_gateway_resource.trigger_pipeline.id}"
+  http_method   = "POST"
+  authorization = "NONE"
+}
+# API Gateway - trigger pipeline - integration with lambda
+resource "aws_api_gateway_integration" "trigger_pipeline_integration" {
+  rest_api_id             = aws_api_gateway_rest_api.project_api.id
+  resource_id             = aws_api_gateway_resource.trigger_pipeline.id
+  http_method             = aws_api_gateway_method.trigger_pipeline_method.http_method
+  integration_http_method = "POST"
+  type                    = "AWS"
+  uri                     = module.trigger_pipeline_lambda_function.lambda_function_invoke_arn
+}
+# API Gateway - trigger pipeline - add permissions for lambda
+resource "aws_lambda_permission" "trigger_pipeline_lambda_permission" {
+  statement_id  = "AllowMyDemoAPIInvoke"
+  action        = "lambda:InvokeFunction"
+  function_name = "${module.trigger_pipeline_lambda_function.lambda_function_name}"
+  principal     = "apigateway.amazonaws.com"
+  source_arn = "${aws_api_gateway_rest_api.project_api.execution_arn}/*/${aws_api_gateway_method.trigger_pipeline_method.http_method}${aws_api_gateway_resource.trigger_pipeline.path}"
+}
+# API Gateway - deploy api gateway
+resource "aws_api_gateway_deployment" "deployment" {
+  depends_on = [
+    aws_api_gateway_integration.create_batch_integration,
+    aws_api_gateway_integration.trigger_pipeline_integration,
+  ]
+
+  rest_api_id = "${aws_api_gateway_rest_api.project_api.id}"
+  stage_name  = local.stage_name
+}

textract-pipeline/terraform/modules/aws-textract-pipeline-part3/outputs.tf

@@ -0,0 +1,71 @@
+# Lambda Layer Arn's
+output "lambda_layer_libre_office_arn" {
+  description = "batch creation lambda fucntion arn"
+  value       = module.lambda_layer_libre_office.lambda_layer_arn
+}
+
+output "lambda_layer_brotli_arn" {
+  description = "batch creation lambda fucntion arn"
+  value       = module.lambda_layer_brotli.lambda_layer_arn
+}
+
+output "lambda_layer_pypdf2_arn" {
+  description = "batch creation lambda fucntion arn"
+  value       = module.lambda_layer_pypdf2.lambda_layer_arn
+}
+
+# Lambda Arn's
+output "batch_creation_lambda_function_arn" {
+  description = "batch creation lambda fucntion arn"
+  value       = module.batch_creation_lambda_function.lambda_function_arn
+}
+
+output "trigger_pipeline_lambda_function_arn" {
+  description = "trigger pipeline lambda fucntion arn"
+  value       = module.trigger_pipeline_lambda_function.lambda_function_arn
+}
+
+output "insert_record_lambda_function_arn" {
+  description = "insert record lambda fucntion arn"
+  value       = module.insert_record_lambda_function.lambda_function_arn
+}
+
+output "docx_to_pdf_lambda_function_arn" {
+  description = "docx to pdf lambda fucntion arn"
+  value       = module.docx_to_pdf_lambda_function.lambda_function_arn
+}
+
+output "tiff_to_pdf_lambda_function_arn" {
+  description = "tiff to pdf lambda fucntion arn"
+  value       = module.tiff_to_pdf_lambda_function.lambda_function_arn
+}
+
+output "pdf_validation_lambda_function_arn" {
+  description = "pdf validation lambda fucntion arn"
+  value       = module.pdf_validation_lambda_function.lambda_function_arn
+}
+
+output "textract_sender_lambda_function_arn" {
+  description = "textract sender lambda fucntion arn"
+  value       = module.textract_sender_lambda_function.lambda_function_arn
+}
+
+output "textract_receiver_lambda_function_arn" {
+  description = "textract receiver lambda fucntion arn"
+  value       = module.textract_receiver_lambda_function.lambda_function_arn
+}
+
+output "text_creation_lambda_function_arn" {
+  description = "text creation lambda fucntion arn"
+  value       = module.text_creation_lambda_function.lambda_function_arn
+}
+
+output "database_interface_lambda_function_arn" {
+  description = "database interface lambda fucntion arn"
+  value       = module.database_interface_lambda_function.lambda_function_arn
+}
+
+# API Gateway invoke_url
+output "api_invoke_url" {
+  value = aws_api_gateway_deployment.deployment.invoke_url
+}

textract-pipeline/terraform/modules/aws-textract-pipeline-part3/variables.tf

@@ -0,0 +1,442 @@
+# Required
+variable "project_name" {
+  type = string
+}
+
+# Required
+variable "client_name" {
+  type = string
+}
+
+# Required
+variable "environment" {
+  type = string
+}
+
+# Required
+variable "aws_region" {
+  type = string
+}
+
+# Required
+variable "vpc_id" {
+  type = string
+}
+
+# Required
+variable "secret_manager_name" {
+  type = string
+}
+
+# Required
+variable "textract_role_arn" {
+  type = string
+}
+
+# Required - temp
+variable "lambda_role_arn" {
+  type = string
+}
+
+# Required
+variable "s3_bucket_name" {
+  type = string
+}
+
+# Required
+variable "devops_s3_bucket" {
+  type = string
+}
+
+# SQS Required 
+variable "insert_record_queue_arn" {
+  type = string
+}
+variable "docx_to_pdf_queue_arn" {
+  type = string
+}
+variable "tiff_to_pdf_queue_arn" {
+  type = string
+}
+variable "pdf_validations_queue_arn" {
+  type = string
+}
+variable "textract_sender_queue_arn"{
+  type = string
+}
+variable "textract_receiver_queue_arn" {
+  type = string
+}
+variable "text_creation_queue_arn" {
+  type = string
+}
+
+# SNS Required
+variable "textract_sns_topic_arn" {
+  type = string
+}
+
+# Lambda layer - Libre Office
+variable "libre_office_lambda_layer" {
+  type = object({
+    name    = string
+    description = string
+    compatible_runtimes = list(string)
+    compatible_architectures = list(string)
+    source_path = string
+    local_existing_package = string
+    create_package = bool
+  })
+  default = {
+    name = "libre-office"
+    description = "Libre office library for docx to pdf conversion"
+    compatible_runtimes = ["python3.8"]
+    compatible_architectures = ["x86_64"]
+    source_path = "../src/lambda-layer/libre-office"
+    local_existing_package = null
+    create_package = true
+  }
+}
+
+# Lambda layer - brotli
+variable "brotli_lambda_layer" {
+  type = object({
+    name    = string
+    description = string
+    compatible_runtimes = list(string)
+    compatible_architectures = list(string)
+    source_path = string
+    local_existing_package = string
+    create_package = bool
+  })
+  default = {
+    name = "brotli"
+    description = "This layer provides brotli library"
+    compatible_runtimes = ["python3.8"]
+    compatible_architectures = ["x86_64"]
+    source_path = null
+    local_existing_package = "../src/lambda-layer/brotli-layer/brotlipy-layer.zip"
+    create_package = false
+  }
+}
+
+# Lambda layer - PyPDF2
+variable "pypdf2_lambda_layer" {
+  type = object({
+    name    = string
+    description = string
+    compatible_runtimes = list(string)
+    compatible_architectures = list(string)
+    source_path = string
+    local_existing_package = string
+    create_package = bool
+  })
+  default = {
+    name = "pypdf2"
+    description = "This layer provides PyPDF2 library"
+    compatible_runtimes = ["python3.12"]
+    compatible_architectures = ["x86_64"]
+    source_path = null
+    local_existing_package = "../src/lambda-layer/pypdf2-layer/PyPDF2.zip"
+    create_package = false
+  }
+}
+
+# batch-creation
+variable "batch_creation_lambda" {
+  type = object({
+    name    = string
+    description = string
+    handler = string
+    runtime = string
+    timeout = number
+    memory_size = number
+    ephemeral_storage_size = number
+    architectures = list(string)
+    layers = list(string)
+    environment_variables = map(string)
+    source_path = string
+  })
+  default = {
+    name = "batch-creation"
+    description = "This function creates batch folder"
+    handler = "index.lambda_handler"
+    runtime = "python3.12"
+    timeout = 60
+    memory_size = 256
+    ephemeral_storage_size = 512
+    architectures = ["x86_64"]
+    layers = []
+    environment_variables = {}
+    source_path = "../src/lambda/batch-creation"
+  }
+}
+# trigger-pipeline
+variable "trigger_pipeline_lambda" {
+  type = object({
+    name    = string
+    description = string
+    handler = string
+    runtime = string
+    timeout = number
+    memory_size = number
+    ephemeral_storage_size = number
+    architectures = list(string)
+    layers = list(string)
+    environment_variables = map(string)
+    source_path = string
+  })
+  default = {
+    name = "trigger-pipeline"
+    description = "This function triggers doczy.ai pipeline"
+    handler = "index.lambda_handler"
+    runtime = "python3.12"
+    timeout = 60
+    memory_size = 256
+    ephemeral_storage_size = 512
+    architectures = ["x86_64"]
+    layers = []
+    environment_variables = {}
+    source_path = "../src/lambda/trigger-pipeline"
+  }
+}
+
+# insert record
+variable "insert_record_lambda" {
+  type = object({
+    name    = string
+    description = string
+    handler = string
+    runtime = string
+    timeout = number
+    memory_size = number
+    ephemeral_storage_size = number
+    architectures = list(string)
+    layers = list(string)
+    environment_variables = map(string)
+    source_path = string
+  })
+  default = {
+    name = "insert-record"
+    description = "This function inserts record in database"
+    handler = "index.lambda_handler"
+    runtime = "python3.12"
+    timeout = 60
+    memory_size = 256
+    ephemeral_storage_size = 512
+    architectures = ["x86_64"]
+    layers = []
+    environment_variables = {}
+    source_path = "../src/lambda/insert-record"
+  }
+}
+
+# docx-to-pdf
+variable "docx_to_pdf_lambda" {
+  type = object({
+    name    = string
+    description = string
+    handler = string
+    runtime = string
+    timeout = number
+    memory_size = number
+    ephemeral_storage_size = number
+    architectures = list(string)
+    layers = list(string)
+    environment_variables = map(string)
+    source_path = string
+  })
+  default = {
+    name = "docx-to-pdf"
+    description = "This function converts docx file to pdf file"
+    handler = "index.lambda_handler"
+    runtime = "python3.8"
+    timeout = 60
+    memory_size = 256
+    ephemeral_storage_size = 512
+    architectures = ["x86_64"]
+    layers = []
+    environment_variables = {}
+    source_path = "../src/lambda/docx-to-pdf"
+  }
+}
+
+# tiff-to-pdf
+variable "tiff_to_pdf_lambda" {
+  type = object({
+    name    = string
+    description = string
+    handler = string
+    runtime = string
+    timeout = number
+    memory_size = number
+    ephemeral_storage_size = number
+    architectures = list(string)
+    layers = list(string)
+    environment_variables = map(string)
+    source_path = string
+  })
+  default = {
+    name = "tiff-to-pdf"
+    description = "This function converts tiff file to pdf file"
+    handler = "index.lambda_handler"
+    runtime = "python3.12"
+    timeout = 60
+    memory_size = 256
+    ephemeral_storage_size = 512
+    architectures = ["x86_64"]
+    layers = []
+    environment_variables = {}
+    source_path = "../src/lambda/tiff-to-pdf"
+  }
+}
+
+# pdf-validation lambda
+variable "pdf_validation_lambda" {
+  type = object({
+    name    = string
+    description = string
+    handler = string
+    runtime = string
+    timeout = number
+    memory_size = number
+    ephemeral_storage_size = number
+    architectures = list(string)
+    layers = list(string)
+    environment_variables = map(string)
+    source_path = string
+  })
+  default = {
+    name = "pdf-validation"
+    description = "This function valids pdf against textract limitations"
+    handler = "index.lambda_handler"
+    runtime = "python3.12"
+    timeout = 60
+    memory_size = 256
+    ephemeral_storage_size = 512
+    architectures = ["x86_64"]
+    layers = []
+    environment_variables = {}
+    source_path = "../src/lambda/pdf-validation"
+  }
+}
+
+# textract_sender_lambda
+variable "textract_sender_lambda" {
+  type = object({
+    name    = string
+    description = string
+    handler = string
+    runtime = string
+    timeout = number
+    memory_size = number
+    ephemeral_storage_size = number
+    architectures = list(string)
+    layers = list(string)
+    environment_variables = map(string)
+    source_path = string
+  })
+  default = {
+    name = "textract-sender"
+    description = "This function sends async request to TEXTRACT service"
+    handler = "index.lambda_handler"
+    runtime = "python3.12"
+    timeout = 60
+    memory_size = 256
+    ephemeral_storage_size = 512
+    architectures = ["x86_64"]
+    layers = []
+    environment_variables = {}
+    source_path = "../src/lambda/textract-sender"
+  }
+}
+
+# textract_receiver_lambda
+variable "textract_receiver_lambda" {
+  type = object({
+    name    = string
+    description = string
+    handler = string
+    runtime = string
+    timeout = number
+    memory_size = number
+    ephemeral_storage_size = number
+    architectures = list(string)
+    layers = list(string)
+    environment_variables = map(string)
+    source_path = string
+  })
+  default = {
+    name = "textract-receiver"
+    description = "This function receives async request from TEXTRACT service"
+    handler = "index.lambda_handler"
+    runtime = "python3.12"
+    timeout = 60
+    memory_size = 256
+    ephemeral_storage_size = 512
+    architectures = ["x86_64"]
+    layers = []
+    environment_variables = {}
+    source_path = "../src/lambda/textract-receiver"
+  }
+}
+
+# text-creation lambda
+variable "text_creation_lambda" {
+  type = object({
+    name    = string
+    description = string
+    handler = string
+    runtime = string
+    timeout = number
+    memory_size = number
+    ephemeral_storage_size = number
+    architectures = list(string)
+    layers = list(string)
+    environment_variables = map(string)
+    source_path = string
+  })
+  default = {
+    name = "text-creation"
+    description = "This function creates text from Textract JSON"
+    handler = "index.lambda_handler"
+    runtime = "python3.12"
+    timeout = 60
+    memory_size = 256
+    ephemeral_storage_size = 512
+    architectures = ["x86_64"]
+    layers = []
+    environment_variables = {}
+    source_path = "../src/lambda/text-creation"
+  }
+}
+
+# database-interface lambda
+variable "database_interface_lambda" {
+  type = object({
+    name    = string
+    description = string
+    handler = string
+    runtime = string
+    timeout = number
+    memory_size = number
+    ephemeral_storage_size = number
+    architectures = list(string)
+    layers = list(string)
+    environment_variables = map(string)
+    source_path = string
+  })
+  default = {
+    name = "database-interface"
+    description = "This function acts as an interface to communicate with database"
+    handler = "index.lambda_handler"
+    runtime = "python3.12"
+    timeout = 60
+    memory_size = 256
+    ephemeral_storage_size = 512
+    architectures = ["x86_64"]
+    layers = []
+    environment_variables = {}
+    source_path = "../src/lambda/database-interface"
+  }
+}

textract-pipeline/terraform/modules/aws-textract-pipeline/main.tf

@@ -0,0 +1,227 @@
+locals {
+  region_map = {
+    us-east-1 = "use1"
+    us-east-2 = "use2"
+    us-west-1 = "usw1"
+    us-west-2 = "usw2"
+  }
+  environment_map = {
+    prod    = "p"
+    uat     = "u"
+    qa      = "q"
+    dev     = "d"
+  }
+
+  # region abbreviation
+  region_short_name   = local.region_map[var.aws_region]
+
+  # environment abbreviation
+  environment_short_name   = local.environment_map[var.environment]
+
+  # global prefix
+  global_prefix = "${var.project_name}-${local.region_short_name}-${local.environment_short_name}-${var.client_name}"
+  
+  # Resource prefix
+  iam_policy_prefix = "${local.global_prefix}-pol"
+
+  # IAM policy name
+  lambda_policy_name = "${local.iam_policy_prefix}-textract-lambda-role"
+  textract_service_policy_name = "${local.iam_policy_prefix}-textract-service-role"
+
+  common_tags = {
+    ClientName = var.client_name
+  }
+}
+
+# Deploy SQS, SNS, S3, 
+module "textract_pipeline_part1" {
+  source       = "../../modules/aws-textract-pipeline-part1"
+
+  client_name = var.client_name
+
+  project_name = var.project_name
+  aws_region   = var.aws_region
+  environment  = var.environment
+}
+
+# Deploy Lambda, Lambda Layer, API Gateway
+module "textract_pipeline_part3" {
+  source       = "../../modules/aws-textract-pipeline-part3"
+
+  client_name = var.client_name
+
+  project_name = var.project_name
+  aws_region   = var.aws_region
+  environment  = var.environment
+  vpc_id       = var.vpc_id
+  secret_manager_name = var.secret_manager_name
+  devops_s3_bucket = var.devops_s3_bucket
+
+  s3_bucket_name = module.textract_pipeline_part1.textract_s3_bucket_name
+
+  insert_record_queue_arn = module.textract_pipeline_part1.insert_record_queue_arn
+  docx_to_pdf_queue_arn = module.textract_pipeline_part1.docx_to_pdf_queue_arn
+  tiff_to_pdf_queue_arn = module.textract_pipeline_part1.tiff_to_pdf_queue_arn
+  pdf_validations_queue_arn = module.textract_pipeline_part1.pdf_validation_queue_arn
+  textract_sender_queue_arn = module.textract_pipeline_part1.sender_queue_arn
+  textract_receiver_queue_arn = module.textract_pipeline_part1.receiver_queue_arn
+  text_creation_queue_arn = module.textract_pipeline_part1.text_creation_queue_arn
+
+  textract_role_arn = var.textract_role_arn
+  textract_sns_topic_arn = module.textract_pipeline_part1.textract_notification_sns_arn
+
+  # lambda_role_arn = var.lambda_role_arn
+  lambda_role_arn = "arn:aws:iam::660131068782:role/doczy-dev-infra-lambda-universal-role"
+
+}
+
+# Construct IAM Policy for textract role
+data "aws_iam_policy_document" "textract_policy" {
+  statement {
+    effect    = "Allow"
+    actions   = ["sns:Publish"]
+    resources = ["${module.textract_pipeline_part1.textract_notification_sns_arn}"]
+  }
+}
+
+# Construct IAM Policy for Lambda role
+data "aws_iam_policy_document" "lambda_policy" {
+  statement {
+    sid = "AllowListBucket"
+
+    actions = [
+      "s3:ListBucket",
+      "s3:ListAllMyBuckets",
+      "s3:GetObject",
+      "s3:PutObject",
+      "s3:PutBucketNotification",
+      "s3:GetBucketNotification",
+      "s3:DeleteObject",
+      "s3:DeleteObjectTagging",
+      "s3:DeleteObjectVersion",
+      "s3:DeleteObjectVersionTagging",
+      "s3:GetObjectTagging",
+      "s3:PutObjectTagging"
+    ]
+
+    effect = "Allow"
+    resources = ["*"]
+
+  }
+
+  statement {
+    sid = "LambdaPermissions"
+
+    actions = [
+      "lambda:CreateFunction",
+      "lambda:DeleteFunction",
+      "lambda:DeleteLayerVersion",
+      "lambda:Get*",
+      "lambda:InvokeFunction",
+      "lambda:List*",
+      "lambda:PublishLayerVersion",
+      "lambda:PublishVersion",
+      "lambda:UpdateFunctionCode",
+      "lambda:UpdateFunctionConfiguration"
+    ]
+
+    effect = "Allow"
+    resources = ["*"]    
+  }
+
+  statement {
+    sid = "IamPermissions"
+
+    actions = [
+      "iam:Get*",
+      "iam:List*"
+    ]
+
+    effect = "Allow"
+    resources = ["*"] 
+  }
+
+  statement {
+    sid = "TextractPermissions"
+
+    actions = [
+      "textract:*"
+    ]
+
+    effect = "Allow"
+    resources = ["*"] 
+  }
+
+  statement {
+    sid = "SnsPermissions"
+
+    actions = [
+      "sns:*"
+    ]
+
+    effect = "Allow"
+    resources = ["*"] 
+  }
+
+  statement {
+    sid = "SqsPermissions"
+
+    actions = [
+      "sqs:*"
+    ]
+
+    effect = "Allow"
+    resources = ["*"] 
+  }
+
+  statement {
+    sid = "CloudWatchLogsPermissions"
+
+    actions = [
+      "logs:*",
+      "cloudwatch:*"
+    ]
+
+    effect = "Allow"
+    resources = ["*"] 
+  }
+
+  statement {
+    sid = "SecretManagersPermissions"
+
+    actions = [
+      "secretsmanager:GetSecretValue",
+      "secretsmanager:DescribeSecret",
+      "secretsmanager:ListSecretVersionIds"
+    ]
+
+    effect = "Allow"
+    resources = ["arn:aws:secretsmanager:*:660131068782:secret:*"] 
+  }
+}
+
+# IAM Policy for textract role 
+resource "aws_iam_policy" "textract_policy" {
+  name        = local.textract_service_policy_name
+  description = "Client textract permissions"
+  policy      = data.aws_iam_policy_document.textract_policy.json
+}
+
+# IAM Policy for lambda role
+resource "aws_iam_policy" "lambda_policy" {
+  name        = local.lambda_policy_name
+  description = "Doczy Pipeline Lambda permissions"
+  policy      = data.aws_iam_policy_document.lambda_policy.json
+}
+
+# Attach IAM Policy to textract role
+resource "aws_iam_role_policy_attachment" "textract_policy" {
+  role       = var.textract_role_name
+  policy_arn = aws_iam_policy.textract_policy.arn
+}
+
+# Attach IAM Policy to lambda role
+resource "aws_iam_role_policy_attachment" "lambda_policy" {
+  role       = var.lambda_role_name
+  policy_arn = aws_iam_policy.lambda_policy.arn
+}

textract-pipeline/terraform/modules/aws-textract-pipeline/outputs.tf

@@ -0,0 +1,119 @@
+# Queue arn's
+output "insert_record_queue_arn" {
+  description = "Insert record queue arn"
+  value       = module.textract_pipeline_part1.insert_record_queue_arn
+}
+
+output "docx_to_pdf_queue_arn" {
+  description = "Docx to PDF queue arn"
+  value       = module.textract_pipeline_part1.docx_to_pdf_queue_arn
+}
+
+output "tiff_to_pdf_queue_arn" {
+  description = "Docx to PDF queue arn"
+  value       = module.textract_pipeline_part1.tiff_to_pdf_queue_arn
+}
+
+output "pdf_validation_queue_arn" {
+  description = "Docx to PDF queue arn"
+  value       = module.textract_pipeline_part1.pdf_validation_queue_arn
+}
+
+output "sender_queue_arn" {
+  description = "Sender queue arn"
+  value       = module.textract_pipeline_part1.sender_queue_arn
+}
+
+output "receiver_queue_arn" {
+  description = "Receiver queue arn"
+  value       = module.textract_pipeline_part1.receiver_queue_arn
+}
+
+output "text_creation_queue_arn" {
+  description = "Text creation queue arn"
+  value       = module.textract_pipeline_part1.text_creation_queue_arn
+}
+
+# SNS arn's
+output "textract_notification_sns_arn" {
+  description = "sns topic arn for textract service"
+  value = module.textract_pipeline_part1.textract_notification_sns_arn
+}
+
+# S3 bucket name
+output "textract_s3_bucket_name" {
+  description = "sns topic arn for textract service"
+  value = module.textract_pipeline_part1.textract_s3_bucket_name
+}
+
+# Lambda Layer Arn's
+output "lambda_layer_libre_office_arn" {
+  description = "batch creation lambda fucntion arn"
+  value       = module.textract_pipeline_part3.lambda_layer_libre_office_arn
+}
+
+output "lambda_layer_brotli_arn" {
+  description = "batch creation lambda fucntion arn"
+  value       = module.textract_pipeline_part3.lambda_layer_brotli_arn
+}
+
+output "lambda_layer_pypdf2_arn" {
+  description = "batch creation lambda fucntion arn"
+  value       = module.textract_pipeline_part3.lambda_layer_pypdf2_arn
+}
+
+# Lambda Arn's
+output "batch_creation_lambda_function_arn" {
+  description = "batch creation lambda fucntion arn"
+  value       = module.textract_pipeline_part3.batch_creation_lambda_function_arn
+}
+
+output "trigger_pipeline_lambda_function_arn" {
+  description = "trigger pipeline lambda fucntion arn"
+  value       = module.textract_pipeline_part3.trigger_pipeline_lambda_function_arn
+}
+
+output "insert_record_lambda_function_arn" {
+  description = "insert record lambda fucntion arn"
+  value       = module.textract_pipeline_part3.insert_record_lambda_function_arn
+}
+
+output "docx_to_pdf_lambda_function_arn" {
+  description = "docx to pdf lambda fucntion arn"
+  value       = module.textract_pipeline_part3.docx_to_pdf_lambda_function_arn
+}
+
+output "tiff_to_pdf_lambda_function_arn" {
+  description = "tiff to pdf lambda fucntion arn"
+  value       = module.textract_pipeline_part3.tiff_to_pdf_lambda_function_arn
+}
+
+output "pdf_validation_lambda_function_arn" {
+  description = "pdf validation lambda fucntion arn"
+  value       = module.textract_pipeline_part3.pdf_validation_lambda_function_arn
+}
+
+output "textract_sender_lambda_function_arn" {
+  description = "textract sender lambda fucntion arn"
+  value       = module.textract_pipeline_part3.textract_sender_lambda_function_arn
+}
+
+output "textract_receiver_lambda_function_arn" {
+  description = "textract receiver lambda fucntion arn"
+  value       = module.textract_pipeline_part3.textract_receiver_lambda_function_arn
+}
+
+output "text_creation_lambda_function_arn" {
+  description = "text creation lambda fucntion arn"
+  value       = module.textract_pipeline_part3.text_creation_lambda_function_arn
+}
+
+output "database_interface_lambda_function_arn" {
+  description = "database interface lambda fucntion arn"
+  value       = module.textract_pipeline_part3.database_interface_lambda_function_arn
+}
+
+# API Gateway invoke_url
+output "api_invoke_url" {
+  value = module.textract_pipeline_part3.api_invoke_url
+}

textract-pipeline/terraform/modules/aws-textract-pipeline/variables.tf

@@ -0,0 +1,54 @@
+# Required
+variable "project_name" {
+  type = string
+}
+
+# Required
+variable "client_name" {
+  type = string
+}
+
+# Required
+variable "environment" {
+  type = string
+}
+
+# Required
+variable "aws_region" {
+  type = string
+}
+
+# Required
+variable "vpc_id" {
+  type = string
+}
+
+# Required
+variable "secret_manager_name" {
+  type = string
+}
+
+# Required
+variable "devops_s3_bucket" {
+  type = string
+}
+
+# Required
+variable "lambda_role_arn" {
+  type = string
+}
+
+# Required
+variable "textract_role_arn" {
+  type = string
+}
+
+# Required
+variable "textract_role_name" {
+  type = string
+}
+
+# Required
+variable "lambda_role_name" {
+  type = string
+}

textract-pipeline/terraform/outputs.tf

@@ -0,0 +1,125 @@
+# Queue arn's
+output "insert_record_queue_arn" {
+  description = "Insert record queue arn"
+  value = [ for output_list in module.textract_pipeline : output_list.insert_record_queue_arn]
+}
+
+output "docx_to_pdf_queue_arn" {
+  description = "Docx to PDF queue arn"
+  value = [ for output_list in module.textract_pipeline : output_list.docx_to_pdf_queue_arn]
+}
+
+output "tiff_to_pdf_queue_arn" {
+  description = "Docx to PDF queue arn"
+  value = [ for output_list in module.textract_pipeline : output_list.tiff_to_pdf_queue_arn]
+}
+
+output "pdf_validation_queue_arn" {
+  description = "Docx to PDF queue arn"
+  value = [ for output_list in module.textract_pipeline : output_list.pdf_validation_queue_arn]
+}
+
+output "sender_queue_arn" {
+  description = "Sender queue arn"
+  value = [ for output_list in module.textract_pipeline : output_list.sender_queue_arn]
+}
+
+output "receiver_queue_arn" {
+  description = "Receiver queue arn"
+  value = [ for output_list in module.textract_pipeline : output_list.receiver_queue_arn]
+}
+
+output "text_creation_queue_arn" {
+  description = "Text creation queue arn"
+  value = [ for output_list in module.textract_pipeline : output_list.text_creation_queue_arn]
+}
+
+# SNS arn's
+output "textract_notification_sns_arn" {
+  description = "sns topic arn for textract service"
+  value = [ for output_list in module.textract_pipeline : output_list.textract_notification_sns_arn]
+}
+
+# S3 bucket name
+output "textract_s3_bucket_name" {
+  description = "sns topic arn for textract service"
+  value = [ for output_list in module.textract_pipeline : output_list.textract_s3_bucket_name]
+}
+
+# Lambda Layer Arn's
+output "lambda_layer_libre_office_arn" {
+  value = [ for output_list in module.textract_pipeline : output_list.lambda_layer_libre_office_arn]
+}
+
+output "lambda_layer_brotli_arn" {
+  value = [ for output_list in module.textract_pipeline : output_list.lambda_layer_brotli_arn]
+}
+
+output "lambda_layer_pypdf2_arn" {
+  value = [ for output_list in module.textract_pipeline : output_list.lambda_layer_pypdf2_arn]
+}
+
+# Lambda Arn's
+output "batch_creation_lambda_function_arn" {
+  description = "batch creation lambda fucntion arn"
+  value = [ for output_list in module.textract_pipeline : output_list.batch_creation_lambda_function_arn]
+}
+
+output "trigger_pipeline_lambda_function_arn" {
+  description = "trigger pipeline lambda fucntion arn"
+  value = [ for output_list in module.textract_pipeline : output_list.trigger_pipeline_lambda_function_arn]
+}
+
+output "insert_record_lambda_function_arn" {
+  description = "insert record lambda fucntion arn"
+  value = [ for output_list in module.textract_pipeline : output_list.insert_record_lambda_function_arn]
+}
+
+output "docx_to_pdf_lambda_function_arn" {
+  description = "docx to pdf lambda fucntion arn"
+  value = [ for output_list in module.textract_pipeline : output_list.docx_to_pdf_lambda_function_arn]
+}
+
+output "tiff_to_pdf_lambda_function_arn" {
+  description = "tiff to pdf lambda fucntion arn"
+  value = [ for output_list in module.textract_pipeline : output_list.tiff_to_pdf_lambda_function_arn]
+}
+
+output "pdf_validation_lambda_function_arn" {
+  description = "pdf validation lambda fucntion arn"
+  value = [ for output_list in module.textract_pipeline : output_list.pdf_validation_lambda_function_arn]
+}
+
+output "textract_sender_lambda_function_arn" {
+  description = "textract sender lambda fucntion arn"
+  value = [ for output_list in module.textract_pipeline : output_list.textract_sender_lambda_function_arn]
+}
+
+output "textract_receiver_lambda_function_arn" {
+  description = "textract receiver lambda fucntion arn"
+  value = [ for output_list in module.textract_pipeline : output_list.textract_receiver_lambda_function_arn]
+}
+
+output "text_creation_lambda_function_arn" {
+  description = "text creation lambda fucntion arn"
+  value = [ for output_list in module.textract_pipeline : output_list.text_creation_lambda_function_arn]
+}
+
+output "database_interface_lambda_function_arn" {
+  description = "database interface lambda fucntion arn"
+  value = [ for output_list in module.textract_pipeline : output_list.database_interface_lambda_function_arn]
+}
+
+# API Gateway invoke_url
+output "api_invoke_url" {
+  value = [ for output_list in module.textract_pipeline : output_list.api_invoke_url]
+}
+
+# IAM Roles
+output "lambda_iam_role_arn"{
+  value = aws_iam_role.lambda_role.arn
+}
+
+output "textract_iam_role_arn"{
+  value = aws_iam_role.textract_role.arn
+}

textract-pipeline/terraform/variables.tf

@@ -0,0 +1,49 @@
+# required
+variable "aws_profile" {
+  type    = string
+  default = "doczyai"
+}
+# required
+variable "aws_region" {
+  type    = string
+  default = "us-east-2"
+}
+# required
+variable "project_name" {
+  type    = string
+  default = "doczyai"
+}
+# required
+variable "environment" {
+  type    = string
+  default = "dev"
+}
+# Required
+variable "vpc_id" {
+  type    = string
+  default = "vpc-0f9d7c913f6522079"
+}
+
+# Required
+variable "secret_manager_name" {
+  type    = string
+  default = "doczy-dev-db-svc-acc"
+}
+
+# Required
+variable "devops_s3_bucket" {
+  type    = string
+  default = "doczy-test-bucket"
+}
+
+
+variable "client_list" {
+  type = map(object({
+    client_name = string
+  }))
+  default = {
+    client1 = {
+      client_name = "cn1"
+    }
+  }
+}