diff --git a/textract-pipeline/terraform/main.tf b/textract-pipeline/terraform/main.tf index bc472fb..d3f8114 100644 --- a/textract-pipeline/terraform/main.tf +++ b/textract-pipeline/terraform/main.tf @@ -97,6 +97,8 @@ module "textract_pipeline" { textract_role_name = aws_iam_role.textract_role.name lambda_role_name = aws_iam_role.lambda_role.name + + text_creation_lambda_additional_layer_arn = var.text_creation_lambda_additional_layer_arn } # Create lambda_role diff --git a/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf b/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf index 95fe99f..b0e8a3c 100644 --- a/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf +++ b/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf @@ -25,12 +25,12 @@ locals { # global prefix global_prefix = "${var.project_name}-${local.region_short_name}-${local.environment_short_name}-${var.client_name}" - + # Resource prefix lambda_prefix = "${local.global_prefix}-lmb" api_gateway_prefix = "${local.global_prefix}-api" lambda_layer_prefix = "${local.global_prefix}-lyr" - + # Lambda names s3_folder_details_lambda_name = "${local.lambda_prefix}-${var.s3_folder_detail_lambda.name}" batch_creation_lambda_name = "${local.lambda_prefix}-${var.batch_creation_lambda.name}" @@ -183,7 +183,7 @@ module "s3_folder_detail_lambda_function" { memory_size = var.s3_folder_detail_lambda.memory_size ephemeral_storage_size = var.s3_folder_detail_lambda.ephemeral_storage_size architectures = var.s3_folder_detail_lambda.architectures - + environment_variables = {} source_path = var.s3_folder_detail_lambda.source_path @@ -210,7 +210,7 @@ module "batch_creation_lambda_function" { memory_size = var.batch_creation_lambda.memory_size ephemeral_storage_size = var.batch_creation_lambda.ephemeral_storage_size architectures = var.batch_creation_lambda.architectures - + environment_variables = local.environment_variables source_path = var.batch_creation_lambda.source_path @@ -237,7 +237,7 @@ module "trigger_pipeline_lambda_function" { memory_size = var.trigger_pipeline_lambda.memory_size ephemeral_storage_size = var.trigger_pipeline_lambda.ephemeral_storage_size architectures = var.trigger_pipeline_lambda.architectures - + environment_variables = merge(local.environment_variables,{ INITIATE_DB_SQS_URL = var.insert_record_queue_url }) @@ -266,7 +266,7 @@ module "insert_record_lambda_function" { memory_size = var.insert_record_lambda.memory_size ephemeral_storage_size = var.insert_record_lambda.ephemeral_storage_size architectures = var.insert_record_lambda.architectures - + environment_variables = local.environment_variables source_path = var.insert_record_lambda.source_path @@ -277,7 +277,7 @@ module "insert_record_lambda_function" { function_response_types = ["ReportBatchItemFailures"] scaling_config = { maximum_concurrency = 20 - + } batch_size = 1 } @@ -310,7 +310,7 @@ module "docx_to_pdf_lambda_function" { module.lambda_layer_libre_office.lambda_layer_arn, module.lambda_layer_brotli.lambda_layer_arn, ] - + environment_variables = local.environment_variables source_path = var.docx_to_pdf_lambda.source_path @@ -321,7 +321,7 @@ module "docx_to_pdf_lambda_function" { function_response_types = ["ReportBatchItemFailures"] scaling_config = { maximum_concurrency = 20 - + } batch_size = 1 } @@ -349,7 +349,7 @@ module "tiff_to_pdf_lambda_function" { memory_size = var.tiff_to_pdf_lambda.memory_size ephemeral_storage_size = var.tiff_to_pdf_lambda.ephemeral_storage_size architectures = var.tiff_to_pdf_lambda.architectures - + environment_variables = local.environment_variables source_path = var.tiff_to_pdf_lambda.source_path @@ -360,7 +360,7 @@ module "tiff_to_pdf_lambda_function" { function_response_types = ["ReportBatchItemFailures"] scaling_config = { maximum_concurrency = 20 - + } batch_size = 1 } @@ -388,7 +388,7 @@ module "pdf_validation_lambda_function" { memory_size = var.pdf_validation_lambda.memory_size ephemeral_storage_size = var.pdf_validation_lambda.ephemeral_storage_size architectures = var.pdf_validation_lambda.architectures - + layers = [ module.lambda_layer_pypdf2.lambda_layer_arn ] @@ -403,7 +403,7 @@ module "pdf_validation_lambda_function" { function_response_types = ["ReportBatchItemFailures"] scaling_config = { maximum_concurrency = 20 - + } batch_size = 1 } @@ -431,7 +431,7 @@ module "textract_sender_lambda_function" { memory_size = var.textract_sender_lambda.memory_size ephemeral_storage_size = var.textract_sender_lambda.ephemeral_storage_size architectures = var.textract_sender_lambda.architectures - + environment_variables = merge(local.environment_variables,{ TEXTRACT_ROLE_ARN = var.textract_role_arn SNS_TOPIC_ARN = var.textract_sns_topic_arn @@ -445,7 +445,7 @@ module "textract_sender_lambda_function" { function_response_types = ["ReportBatchItemFailures"] scaling_config = { maximum_concurrency = 20 - + } batch_size = 1 } @@ -473,7 +473,7 @@ module "textract_receiver_lambda_function" { memory_size = var.textract_receiver_lambda.memory_size ephemeral_storage_size = var.textract_receiver_lambda.ephemeral_storage_size architectures = var.textract_receiver_lambda.architectures - + environment_variables = local.environment_variables source_path = var.textract_receiver_lambda.source_path @@ -484,7 +484,7 @@ module "textract_receiver_lambda_function" { function_response_types = ["ReportBatchItemFailures"] scaling_config = { maximum_concurrency = 20 - + } batch_size = 1 } @@ -514,9 +514,10 @@ module "text_creation_lambda_function" { architectures = var.text_creation_lambda.architectures layers = [ - module.lambda_layer_pypdf2.lambda_layer_arn + module.lambda_layer_pypdf2.lambda_layer_arn, + var.text_creation_lambda_additional_layer_arn ] - + environment_variables = local.environment_variables source_path = var.text_creation_lambda.source_path @@ -527,7 +528,7 @@ module "text_creation_lambda_function" { function_response_types = ["ReportBatchItemFailures"] scaling_config = { maximum_concurrency = 20 - + } batch_size = 1 } @@ -557,7 +558,7 @@ module "prompt_orchestrator_lambda_function" { architectures = local.prompt_orchestrator_lambda.architectures layers = [] - + environment_variables = merge(local.environment_variables, local.prompt_orchestrator_lambda.environment_variables, { PROMPT_PROCESSOR_QUEUE_URL = var.prompt_processor_queue_url MAX_PROMPT_LENGTH_PER_GROUP = "5000" @@ -571,7 +572,7 @@ module "prompt_orchestrator_lambda_function" { function_response_types = ["ReportBatchItemFailures"] scaling_config = { maximum_concurrency = 20 - + } batch_size = 1 enabled = false @@ -602,7 +603,7 @@ module "prompt_processor_lambda_function" { architectures = var.prompt_processor_lambda.architectures layers = [] - + environment_variables = merge(local.environment_variables,{ CALL_BEDROCK_LAMBDA_FUNCTION_NAME = local.call_bedrock_lambda_name }) @@ -615,7 +616,7 @@ module "prompt_processor_lambda_function" { function_response_types = ["ReportBatchItemFailures"] scaling_config = { maximum_concurrency = 20 - + } batch_size = 1 } @@ -645,7 +646,7 @@ module "call_bedrock_lambda_function" { architectures = var.call_bedrock_lambda.architectures layers = [] - + environment_variables = local.environment_variables source_path = var.call_bedrock_lambda.source_path @@ -672,7 +673,7 @@ module "database_interface_lambda_function" { memory_size = var.database_interface_lambda.memory_size ephemeral_storage_size = var.database_interface_lambda.ephemeral_storage_size architectures = var.database_interface_lambda.architectures - + environment_variables = merge(local.environment_variables,{ SECRET_MANAGER_NAME = var.secret_manager_name }) @@ -705,7 +706,7 @@ module "database_interface_get_lambda_function" { memory_size = var.database_interface_get_lambda.memory_size ephemeral_storage_size = var.database_interface_get_lambda.ephemeral_storage_size architectures = var.database_interface_get_lambda.architectures - + environment_variables = merge(local.environment_variables,{ SECRET_MANAGER_NAME = var.secret_manager_name }) @@ -775,7 +776,7 @@ resource "aws_s3_bucket_notification" "bucket_notification" { events = ["s3:ObjectCreated:*"] filter_prefix = "textract-output-json/" filter_suffix = ".json" - } + } queue { id = "text-file-upload-event" @@ -847,9 +848,9 @@ resource "aws_api_gateway_method" "create_batch_method" { http_method = "POST" authorization = "AWS_IAM" - depends_on = [ - aws_api_gateway_rest_api.project_api, - aws_api_gateway_resource.create_batch + depends_on = [ + aws_api_gateway_rest_api.project_api, + aws_api_gateway_resource.create_batch ] } @@ -863,8 +864,8 @@ resource "aws_api_gateway_method_response" "create_batch_method_response_200" { "application/json" = "Empty" } - depends_on = [ - aws_api_gateway_rest_api.project_api, + depends_on = [ + aws_api_gateway_rest_api.project_api, aws_api_gateway_resource.create_batch, aws_api_gateway_method.create_batch_method ] @@ -879,8 +880,8 @@ resource "aws_api_gateway_integration" "create_batch_integration" { type = "AWS" uri = module.batch_creation_lambda_function.lambda_function_invoke_arn - depends_on = [ - aws_api_gateway_rest_api.project_api, + depends_on = [ + aws_api_gateway_rest_api.project_api, aws_api_gateway_resource.create_batch, aws_api_gateway_method.create_batch_method, module.batch_creation_lambda_function @@ -894,8 +895,8 @@ resource "aws_api_gateway_integration_response" "create_batch_integration_respon http_method = aws_api_gateway_method.create_batch_method.http_method status_code = aws_api_gateway_method_response.create_batch_method_response_200.status_code - depends_on = [ - aws_api_gateway_rest_api.project_api, + depends_on = [ + aws_api_gateway_rest_api.project_api, aws_api_gateway_resource.create_batch, aws_api_gateway_method.create_batch_method, aws_api_gateway_method_response.create_batch_method_response_200, @@ -912,8 +913,8 @@ resource "aws_lambda_permission" "create_batch_lambda_permission" { principal = "apigateway.amazonaws.com" source_arn = "${aws_api_gateway_rest_api.project_api.execution_arn}/*/${aws_api_gateway_method.create_batch_method.http_method}${aws_api_gateway_resource.create_batch.path}" - depends_on = [ - aws_api_gateway_rest_api.project_api, + depends_on = [ + aws_api_gateway_rest_api.project_api, aws_api_gateway_resource.create_batch, aws_api_gateway_method.create_batch_method, module.batch_creation_lambda_function @@ -963,8 +964,8 @@ resource "aws_api_gateway_integration_response" "s3_folder_detail_integration_re http_method = aws_api_gateway_method.s3_folder_detail_method.http_method status_code = aws_api_gateway_method_response.s3_folder_detail_method_response_200.status_code - depends_on = [ - aws_api_gateway_rest_api.project_api, + depends_on = [ + aws_api_gateway_rest_api.project_api, aws_api_gateway_resource.s3_folder_detail, aws_api_gateway_method.s3_folder_detail_method, aws_api_gateway_method_response.s3_folder_detail_method_response_200, @@ -1023,8 +1024,8 @@ resource "aws_api_gateway_integration_response" "trigger_pipeline_integration_re http_method = aws_api_gateway_method.trigger_pipeline_method.http_method status_code = aws_api_gateway_method_response.trigger_pipeline_method_response_200.status_code - depends_on = [ - aws_api_gateway_rest_api.project_api, + depends_on = [ + aws_api_gateway_rest_api.project_api, aws_api_gateway_resource.trigger_pipeline, aws_api_gateway_method.trigger_pipeline_method, aws_api_gateway_method_response.trigger_pipeline_method_response_200, diff --git a/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/variables.tf b/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/variables.tf index 2ec776b..4e7447b 100644 --- a/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/variables.tf +++ b/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/variables.tf @@ -485,6 +485,10 @@ variable "text_creation_lambda" { } } +variable "text_creation_lambda_additional_layer_arn" { + type = string +} + # prompt-orchestrator lambda # Use locals to merge default values with variable values diff --git a/textract-pipeline/terraform/modules/aws-textract-pipeline/main.tf b/textract-pipeline/terraform/modules/aws-textract-pipeline/main.tf index 3d3366b..0489c33 100644 --- a/textract-pipeline/terraform/modules/aws-textract-pipeline/main.tf +++ b/textract-pipeline/terraform/modules/aws-textract-pipeline/main.tf @@ -69,6 +69,7 @@ module "textract_pipeline_part3" { textract_sender_queue_arn = module.textract_pipeline_part1.sender_queue_arn textract_receiver_queue_arn = module.textract_pipeline_part1.receiver_queue_arn text_creation_queue_arn = module.textract_pipeline_part1.text_creation_queue_arn + text_creation_lambda_additional_layer_arn = var.text_creation_lambda_additional_layer_arn prompt_orchestrator_queue_arn = module.textract_pipeline_part1.prompt_orchestrator_queue_arn prompt_processor_queue_arn = module.textract_pipeline_part1.prompt_processor_queue_arn prompt_processor_queue_url = module.textract_pipeline_part1.prompt_processor_queue_url diff --git a/textract-pipeline/terraform/modules/aws-textract-pipeline/variables.tf b/textract-pipeline/terraform/modules/aws-textract-pipeline/variables.tf index 84d3b1d..9546ffe 100644 --- a/textract-pipeline/terraform/modules/aws-textract-pipeline/variables.tf +++ b/textract-pipeline/terraform/modules/aws-textract-pipeline/variables.tf @@ -60,4 +60,8 @@ variable "lambda_role_name" { variable "aws_account_id" { type = string +} + +variable "text_creation_lambda_additional_layer_arn" { + type = string } \ No newline at end of file diff --git a/textract-pipeline/terraform/variables.tf b/textract-pipeline/terraform/variables.tf index c918b11..73ee0fc 100644 --- a/textract-pipeline/terraform/variables.tf +++ b/textract-pipeline/terraform/variables.tf @@ -29,3 +29,7 @@ variable "client_list" { variable "aws_account_id" { type = string } + +variable "text_creation_lambda_additional_layer_arn" { + type = string +} \ No newline at end of file diff --git a/textract-pipeline/terraform/vars-dev.tfvars b/textract-pipeline/terraform/vars-dev.tfvars index 9e8b1cd..02453a3 100644 --- a/textract-pipeline/terraform/vars-dev.tfvars +++ b/textract-pipeline/terraform/vars-dev.tfvars @@ -1 +1,2 @@ -aws_account_id = 660131068782 \ No newline at end of file +aws_account_id = 660131068782 +text_creation_lambda_additional_layer_arn = "arn:aws:lambda:us-east-2:336392948345:layer:AWSSDKPandas-Python312:8" \ No newline at end of file diff --git a/textract-pipeline/terraform/vars-uat.tfvars b/textract-pipeline/terraform/vars-uat.tfvars index 53f4392..b6e909a 100644 --- a/textract-pipeline/terraform/vars-uat.tfvars +++ b/textract-pipeline/terraform/vars-uat.tfvars @@ -1 +1,3 @@ -aws_account_id = 975049960860 \ No newline at end of file +aws_account_id = 975049960860 +# TODO provide new layer arn for UAT? 336392948345 was used in DEV but not sure what account that is?! +text_creation_lambda_additional_layer_arn = "arn:aws:lambda:us-east-2:336392948345:layer:AWSSDKPandas-Python312:8"