jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/DOC-542-add-missing-lambda-layer (pull request #89)

Browse Source 
[DOC-542] Add missing layer to text_creation_lambda

Approved-by: Umang Mistry
This commit is contained in:
Grzegorz Huber
2024-06-12 20:00:44 +00:00
committed by Umang Mistry
parent 5c0f0e82bd 71e7d5b11c
commit 4e99ece63c
8 changed files with 64 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
textract-pipeline/terraform/main.tf
+2
View File
@@ -97,6 +97,8 @@ module "textract_pipeline" {
textract_role_name = aws_iam_role.textract_role.name
lambda_role_name = aws_iam_role.lambda_role.name
text_creation_lambda_additional_layer_arn = var.text_creation_lambda_additional_layer_arn
}
# Create lambda_role
textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf
+44 -43
View File
@@ -25,12 +25,12 @@ locals {
# global prefix
global_prefix = "${var.project_name}-${local.region_short_name}-${local.environment_short_name}-${var.client_name}"
# Resource prefix
lambda_prefix = "${local.global_prefix}-lmb"
api_gateway_prefix = "${local.global_prefix}-api"
lambda_layer_prefix = "${local.global_prefix}-lyr"
# Lambda names
s3_folder_details_lambda_name = "${local.lambda_prefix}-${var.s3_folder_detail_lambda.name}"
batch_creation_lambda_name = "${local.lambda_prefix}-${var.batch_creation_lambda.name}"
@@ -183,7 +183,7 @@ module "s3_folder_detail_lambda_function" {
memory_size = var.s3_folder_detail_lambda.memory_size
ephemeral_storage_size = var.s3_folder_detail_lambda.ephemeral_storage_size
architectures = var.s3_folder_detail_lambda.architectures
environment_variables = {}
source_path = var.s3_folder_detail_lambda.source_path
@@ -210,7 +210,7 @@ module "batch_creation_lambda_function" {
memory_size = var.batch_creation_lambda.memory_size
ephemeral_storage_size = var.batch_creation_lambda.ephemeral_storage_size
architectures = var.batch_creation_lambda.architectures
environment_variables = local.environment_variables
source_path = var.batch_creation_lambda.source_path
@@ -237,7 +237,7 @@ module "trigger_pipeline_lambda_function" {
memory_size = var.trigger_pipeline_lambda.memory_size
ephemeral_storage_size = var.trigger_pipeline_lambda.ephemeral_storage_size
architectures = var.trigger_pipeline_lambda.architectures
environment_variables = merge(local.environment_variables,{
INITIATE_DB_SQS_URL = var.insert_record_queue_url
})
@@ -266,7 +266,7 @@ module "insert_record_lambda_function" {
memory_size = var.insert_record_lambda.memory_size
ephemeral_storage_size = var.insert_record_lambda.ephemeral_storage_size
architectures = var.insert_record_lambda.architectures
environment_variables = local.environment_variables
source_path = var.insert_record_lambda.source_path
@@ -277,7 +277,7 @@ module "insert_record_lambda_function" {
function_response_types = ["ReportBatchItemFailures"]
scaling_config = {
maximum_concurrency = 20
}
batch_size = 1
}
@@ -310,7 +310,7 @@ module "docx_to_pdf_lambda_function" {
module.lambda_layer_libre_office.lambda_layer_arn,
module.lambda_layer_brotli.lambda_layer_arn,
]
environment_variables = local.environment_variables
source_path = var.docx_to_pdf_lambda.source_path
@@ -321,7 +321,7 @@ module "docx_to_pdf_lambda_function" {
function_response_types = ["ReportBatchItemFailures"]
scaling_config = {
maximum_concurrency = 20
}
batch_size = 1
}
@@ -349,7 +349,7 @@ module "tiff_to_pdf_lambda_function" {
memory_size = var.tiff_to_pdf_lambda.memory_size
ephemeral_storage_size = var.tiff_to_pdf_lambda.ephemeral_storage_size
architectures = var.tiff_to_pdf_lambda.architectures
environment_variables = local.environment_variables
source_path = var.tiff_to_pdf_lambda.source_path
@@ -360,7 +360,7 @@ module "tiff_to_pdf_lambda_function" {
function_response_types = ["ReportBatchItemFailures"]
scaling_config = {
maximum_concurrency = 20
}
batch_size = 1
}
@@ -388,7 +388,7 @@ module "pdf_validation_lambda_function" {
memory_size = var.pdf_validation_lambda.memory_size
ephemeral_storage_size = var.pdf_validation_lambda.ephemeral_storage_size
architectures = var.pdf_validation_lambda.architectures
layers = [
module.lambda_layer_pypdf2.lambda_layer_arn
]
@@ -403,7 +403,7 @@ module "pdf_validation_lambda_function" {
function_response_types = ["ReportBatchItemFailures"]
scaling_config = {
maximum_concurrency = 20
}
batch_size = 1
}
@@ -431,7 +431,7 @@ module "textract_sender_lambda_function" {
memory_size = var.textract_sender_lambda.memory_size
ephemeral_storage_size = var.textract_sender_lambda.ephemeral_storage_size
architectures = var.textract_sender_lambda.architectures
environment_variables = merge(local.environment_variables,{
TEXTRACT_ROLE_ARN = var.textract_role_arn
SNS_TOPIC_ARN = var.textract_sns_topic_arn
@@ -445,7 +445,7 @@ module "textract_sender_lambda_function" {
function_response_types = ["ReportBatchItemFailures"]
scaling_config = {
maximum_concurrency = 20
}
batch_size = 1
}
@@ -473,7 +473,7 @@ module "textract_receiver_lambda_function" {
memory_size = var.textract_receiver_lambda.memory_size
ephemeral_storage_size = var.textract_receiver_lambda.ephemeral_storage_size
architectures = var.textract_receiver_lambda.architectures
environment_variables = local.environment_variables
source_path = var.textract_receiver_lambda.source_path
@@ -484,7 +484,7 @@ module "textract_receiver_lambda_function" {
function_response_types = ["ReportBatchItemFailures"]
scaling_config = {
maximum_concurrency = 20
}
batch_size = 1
}
@@ -514,9 +514,10 @@ module "text_creation_lambda_function" {
architectures = var.text_creation_lambda.architectures
layers = [
module.lambda_layer_pypdf2.lambda_layer_arn
module.lambda_layer_pypdf2.lambda_layer_arn,
var.text_creation_lambda_additional_layer_arn
]
environment_variables = local.environment_variables
source_path = var.text_creation_lambda.source_path
@@ -527,7 +528,7 @@ module "text_creation_lambda_function" {
function_response_types = ["ReportBatchItemFailures"]
scaling_config = {
maximum_concurrency = 20
}
batch_size = 1
}
@@ -557,7 +558,7 @@ module "prompt_orchestrator_lambda_function" {
architectures = local.prompt_orchestrator_lambda.architectures
layers = []
environment_variables = merge(local.environment_variables, local.prompt_orchestrator_lambda.environment_variables, {
PROMPT_PROCESSOR_QUEUE_URL = var.prompt_processor_queue_url
MAX_PROMPT_LENGTH_PER_GROUP = "5000"
@@ -571,7 +572,7 @@ module "prompt_orchestrator_lambda_function" {
function_response_types = ["ReportBatchItemFailures"]
scaling_config = {
maximum_concurrency = 20
}
batch_size = 1
enabled = false
@@ -602,7 +603,7 @@ module "prompt_processor_lambda_function" {
architectures = var.prompt_processor_lambda.architectures
layers = []
environment_variables = merge(local.environment_variables,{
CALL_BEDROCK_LAMBDA_FUNCTION_NAME = local.call_bedrock_lambda_name
})
@@ -615,7 +616,7 @@ module "prompt_processor_lambda_function" {
function_response_types = ["ReportBatchItemFailures"]
scaling_config = {
maximum_concurrency = 20
}
batch_size = 1
}
@@ -645,7 +646,7 @@ module "call_bedrock_lambda_function" {
architectures = var.call_bedrock_lambda.architectures
layers = []
environment_variables = local.environment_variables
source_path = var.call_bedrock_lambda.source_path
@@ -672,7 +673,7 @@ module "database_interface_lambda_function" {
memory_size = var.database_interface_lambda.memory_size
ephemeral_storage_size = var.database_interface_lambda.ephemeral_storage_size
architectures = var.database_interface_lambda.architectures
environment_variables = merge(local.environment_variables,{
SECRET_MANAGER_NAME = var.secret_manager_name
})
@@ -705,7 +706,7 @@ module "database_interface_get_lambda_function" {
memory_size = var.database_interface_get_lambda.memory_size
ephemeral_storage_size = var.database_interface_get_lambda.ephemeral_storage_size
architectures = var.database_interface_get_lambda.architectures
environment_variables = merge(local.environment_variables,{
SECRET_MANAGER_NAME = var.secret_manager_name
})
@@ -775,7 +776,7 @@ resource "aws_s3_bucket_notification" "bucket_notification" {
events = ["s3:ObjectCreated:*"]
filter_prefix = "textract-output-json/"
filter_suffix = ".json"
}
}
queue {
id = "text-file-upload-event"
@@ -847,9 +848,9 @@ resource "aws_api_gateway_method" "create_batch_method" {
http_method = "POST"
authorization = "AWS_IAM"
depends_on = [
aws_api_gateway_rest_api.project_api,
aws_api_gateway_resource.create_batch
depends_on = [
aws_api_gateway_rest_api.project_api,
aws_api_gateway_resource.create_batch
]
}
@@ -863,8 +864,8 @@ resource "aws_api_gateway_method_response" "create_batch_method_response_200" {
"application/json" = "Empty"
}
depends_on = [
aws_api_gateway_rest_api.project_api,
depends_on = [
aws_api_gateway_rest_api.project_api,
aws_api_gateway_resource.create_batch,
aws_api_gateway_method.create_batch_method
]
@@ -879,8 +880,8 @@ resource "aws_api_gateway_integration" "create_batch_integration" {
type = "AWS"
uri = module.batch_creation_lambda_function.lambda_function_invoke_arn
depends_on = [
aws_api_gateway_rest_api.project_api,
depends_on = [
aws_api_gateway_rest_api.project_api,
aws_api_gateway_resource.create_batch,
aws_api_gateway_method.create_batch_method,
module.batch_creation_lambda_function
@@ -894,8 +895,8 @@ resource "aws_api_gateway_integration_response" "create_batch_integration_respon
http_method = aws_api_gateway_method.create_batch_method.http_method
status_code = aws_api_gateway_method_response.create_batch_method_response_200.status_code
depends_on = [
aws_api_gateway_rest_api.project_api,
depends_on = [
aws_api_gateway_rest_api.project_api,
aws_api_gateway_resource.create_batch,
aws_api_gateway_method.create_batch_method,
aws_api_gateway_method_response.create_batch_method_response_200,
@@ -912,8 +913,8 @@ resource "aws_lambda_permission" "create_batch_lambda_permission" {
principal = "apigateway.amazonaws.com"
source_arn = "${aws_api_gateway_rest_api.project_api.execution_arn}/*/${aws_api_gateway_method.create_batch_method.http_method}${aws_api_gateway_resource.create_batch.path}"
depends_on = [
aws_api_gateway_rest_api.project_api,
depends_on = [
aws_api_gateway_rest_api.project_api,
aws_api_gateway_resource.create_batch,
aws_api_gateway_method.create_batch_method,
module.batch_creation_lambda_function
@@ -963,8 +964,8 @@ resource "aws_api_gateway_integration_response" "s3_folder_detail_integration_re
http_method = aws_api_gateway_method.s3_folder_detail_method.http_method
status_code = aws_api_gateway_method_response.s3_folder_detail_method_response_200.status_code
depends_on = [
aws_api_gateway_rest_api.project_api,
depends_on = [
aws_api_gateway_rest_api.project_api,
aws_api_gateway_resource.s3_folder_detail,
aws_api_gateway_method.s3_folder_detail_method,
aws_api_gateway_method_response.s3_folder_detail_method_response_200,
@@ -1023,8 +1024,8 @@ resource "aws_api_gateway_integration_response" "trigger_pipeline_integration_re
http_method = aws_api_gateway_method.trigger_pipeline_method.http_method
status_code = aws_api_gateway_method_response.trigger_pipeline_method_response_200.status_code
depends_on = [
aws_api_gateway_rest_api.project_api,
depends_on = [
aws_api_gateway_rest_api.project_api,
aws_api_gateway_resource.trigger_pipeline,
aws_api_gateway_method.trigger_pipeline_method,
aws_api_gateway_method_response.trigger_pipeline_method_response_200,
textract-pipeline/terraform/modules/aws-textract-pipeline-part3/variables.tf
+4
View File
@@ -485,6 +485,10 @@ variable "text_creation_lambda" {
}
}
variable "text_creation_lambda_additional_layer_arn" {
type = string
}
# prompt-orchestrator lambda
# Use locals to merge default values with variable values
textract-pipeline/terraform/modules/aws-textract-pipeline/main.tf
+1
View File
@@ -69,6 +69,7 @@ module "textract_pipeline_part3" {
textract_sender_queue_arn = module.textract_pipeline_part1.sender_queue_arn
textract_receiver_queue_arn = module.textract_pipeline_part1.receiver_queue_arn
text_creation_queue_arn = module.textract_pipeline_part1.text_creation_queue_arn
text_creation_lambda_additional_layer_arn = var.text_creation_lambda_additional_layer_arn
prompt_orchestrator_queue_arn = module.textract_pipeline_part1.prompt_orchestrator_queue_arn
prompt_processor_queue_arn = module.textract_pipeline_part1.prompt_processor_queue_arn
prompt_processor_queue_url = module.textract_pipeline_part1.prompt_processor_queue_url
textract-pipeline/terraform/modules/aws-textract-pipeline/variables.tf
+4
View File
@@ -60,4 +60,8 @@ variable "lambda_role_name" {
variable "aws_account_id" {
type = string
}
variable "text_creation_lambda_additional_layer_arn" {
type = string
}
textract-pipeline/terraform/variables.tf
+4
View File
@@ -29,3 +29,7 @@ variable "client_list" {
variable "aws_account_id" {
type = string
}
variable "text_creation_lambda_additional_layer_arn" {
type = string
}
textract-pipeline/terraform/vars-dev.tfvars
+2 -1
View File
@@ -1 +1,2 @@
aws_account_id = 660131068782
aws_account_id = 660131068782
text_creation_lambda_additional_layer_arn = "arn:aws:lambda:us-east-2:336392948345:layer:AWSSDKPandas-Python312:8"
textract-pipeline/terraform/vars-uat.tfvars
+3 -1
View File
@@ -1 +1,3 @@
aws_account_id = 975049960860
aws_account_id = 975049960860
# TODO provide new layer arn for UAT? 336392948345 was used in DEV but not sure what account that is?!
text_creation_lambda_additional_layer_arn = "arn:aws:lambda:us-east-2:336392948345:layer:AWSSDKPandas-Python312:8"