diff --git a/textract-pipeline/terraform/main.tf b/textract-pipeline/terraform/main.tf index ca35b17..f086ac3 100644 --- a/textract-pipeline/terraform/main.tf +++ b/textract-pipeline/terraform/main.tf @@ -148,6 +148,9 @@ data "aws_iam_policy" "S3FullAccess" { data "aws_iam_policy" "CloudWatchFullAccess" { arn = "arn:aws:iam::aws:policy/CloudWatchLogsFullAccess" } +data "aws_iam_policy" "AWSLambdaVPCAccessExecutionRole" { + arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" +} resource "aws_iam_role_policy_attachment" "bedrock-policy-attach" { role = "${aws_iam_role.lambda_role.name}" policy_arn = "${data.aws_iam_policy.BedrockFullAccess.arn}" @@ -159,4 +162,8 @@ resource "aws_iam_role_policy_attachment" "s3-policy-attach" { resource "aws_iam_role_policy_attachment" "cloudwatch-policy-attach" { role = "${aws_iam_role.lambda_role.name}" policy_arn = "${data.aws_iam_policy.CloudWatchFullAccess.arn}" +} +resource "aws_iam_role_policy_attachment" "VPC-policy-attach" { + role = "${aws_iam_role.lambda_role.name}" + policy_arn = "${data.aws_iam_policy.AWSLambdaVPCAccessExecutionRole.arn}" } \ No newline at end of file diff --git a/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf b/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf index 7769a3f..a30f011 100644 --- a/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf +++ b/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf @@ -52,6 +52,7 @@ locals { libre_office_lambda_layer_name = "${local.lambda_layer_prefix}-${var.libre_office_lambda_layer.name}" brotli_lambda_layer_name = "${local.lambda_layer_prefix}-${var.brotli_lambda_layer.name}" pypdf2_lambda_layer_name = "${local.lambda_layer_prefix}-${var.pypdf2_lambda_layer.name}" + snowflake_connector_lambda_layer_name = "${local.lambda_layer_prefix}-${var.snowflake_connector_lambda_layer.name}" # API Gateway configuration api_description = "API's used for DoczyAI pipeline" @@ -150,6 +151,25 @@ module "lambda_layer_pypdf2" { local_existing_package = var.pypdf2_lambda_layer.local_existing_package } +# Lambda Layers - snowflake connector - create package from source code +module "lambda_layer_snowflake_connector" { + source = "terraform-aws-modules/lambda/aws" + + create_layer = true + create_package = var.snowflake_connector_lambda_layer.create_package + + layer_name = local.snowflake_connector_lambda_layer_name + description = var.snowflake_connector_lambda_layer.description + compatible_runtimes = var.snowflake_connector_lambda_layer.compatible_runtimes + compatible_architectures = var.snowflake_connector_lambda_layer.compatible_architectures + + source_path = var.snowflake_connector_lambda_layer.source_path + + # store_on_s3 = true + s3_bucket = local.lambda_layer_s3_bucket + s3_prefix = local.lambda_layer_s3_prefix +} + # batch creation lambda function module "s3_folder_detail_lambda_function" { source = "terraform-aws-modules/lambda/aws" @@ -171,8 +191,8 @@ module "s3_folder_detail_lambda_function" { lambda_role = var.lambda_role_arn create_role = false - # vpc_subnet_ids = data.aws_subnets.subnets.ids - # vpc_security_group_ids = data.aws_security_groups.security_groups.ids + vpc_subnet_ids = data.aws_subnets.subnets.ids + vpc_security_group_ids = data.aws_security_groups.security_groups.ids tags = local.common_tags } @@ -198,8 +218,8 @@ module "batch_creation_lambda_function" { lambda_role = var.lambda_role_arn create_role = false - # vpc_subnet_ids = data.aws_subnets.subnets.ids - # vpc_security_group_ids = data.aws_security_groups.security_groups.ids + vpc_subnet_ids = data.aws_subnets.subnets.ids + vpc_security_group_ids = data.aws_security_groups.security_groups.ids tags = local.common_tags } @@ -227,8 +247,8 @@ module "trigger_pipeline_lambda_function" { lambda_role = var.lambda_role_arn create_role = false - # vpc_subnet_ids = data.aws_subnets.subnets.ids - # vpc_security_group_ids = data.aws_security_groups.security_groups.ids + vpc_subnet_ids = data.aws_subnets.subnets.ids + vpc_security_group_ids = data.aws_security_groups.security_groups.ids tags = local.common_tags } @@ -266,8 +286,8 @@ module "insert_record_lambda_function" { lambda_role = var.lambda_role_arn create_role = false - # vpc_subnet_ids = data.aws_subnets.subnets.ids - # vpc_security_group_ids = data.aws_security_groups.security_groups.ids + vpc_subnet_ids = data.aws_subnets.subnets.ids + vpc_security_group_ids = data.aws_security_groups.security_groups.ids tags = local.common_tags } @@ -310,8 +330,8 @@ module "docx_to_pdf_lambda_function" { lambda_role = var.lambda_role_arn create_role = false - # vpc_subnet_ids = data.aws_subnets.subnets.ids - # vpc_security_group_ids = data.aws_security_groups.security_groups.ids + vpc_subnet_ids = data.aws_subnets.subnets.ids + vpc_security_group_ids = data.aws_security_groups.security_groups.ids tags = local.common_tags } @@ -349,8 +369,8 @@ module "tiff_to_pdf_lambda_function" { lambda_role = var.lambda_role_arn create_role = false - # vpc_subnet_ids = data.aws_subnets.subnets.ids - # vpc_security_group_ids = data.aws_security_groups.security_groups.ids + vpc_subnet_ids = data.aws_subnets.subnets.ids + vpc_security_group_ids = data.aws_security_groups.security_groups.ids tags = local.common_tags } @@ -392,8 +412,8 @@ module "pdf_validation_lambda_function" { lambda_role = var.lambda_role_arn create_role = false - # vpc_subnet_ids = data.aws_subnets.subnets.ids - # vpc_security_group_ids = data.aws_security_groups.security_groups.ids + vpc_subnet_ids = data.aws_subnets.subnets.ids + vpc_security_group_ids = data.aws_security_groups.security_groups.ids tags = local.common_tags } @@ -434,8 +454,8 @@ module "textract_sender_lambda_function" { lambda_role = var.lambda_role_arn create_role = false - # vpc_subnet_ids = data.aws_subnets.subnets.ids - # vpc_security_group_ids = data.aws_security_groups.security_groups.ids + vpc_subnet_ids = data.aws_subnets.subnets.ids + vpc_security_group_ids = data.aws_security_groups.security_groups.ids tags = local.common_tags } @@ -473,8 +493,8 @@ module "textract_receiver_lambda_function" { lambda_role = var.lambda_role_arn create_role = false - # vpc_subnet_ids = data.aws_subnets.subnets.ids - # vpc_security_group_ids = data.aws_security_groups.security_groups.ids + vpc_subnet_ids = data.aws_subnets.subnets.ids + vpc_security_group_ids = data.aws_security_groups.security_groups.ids tags = local.common_tags } @@ -516,8 +536,8 @@ module "text_creation_lambda_function" { lambda_role = var.lambda_role_arn create_role = false - # vpc_subnet_ids = data.aws_subnets.subnets.ids - # vpc_security_group_ids = data.aws_security_groups.security_groups.ids + vpc_subnet_ids = data.aws_subnets.subnets.ids + vpc_security_group_ids = data.aws_security_groups.security_groups.ids tags = local.common_tags } @@ -561,8 +581,8 @@ module "prompt_orchestrator_lambda_function" { lambda_role = var.lambda_role_arn create_role = false - # vpc_subnet_ids = data.aws_subnets.subnets.ids - # vpc_security_group_ids = data.aws_security_groups.security_groups.ids + vpc_subnet_ids = data.aws_subnets.subnets.ids + vpc_security_group_ids = data.aws_security_groups.security_groups.ids tags = local.common_tags } @@ -604,8 +624,8 @@ module "prompt_processor_lambda_function" { lambda_role = var.lambda_role_arn create_role = false - # vpc_subnet_ids = data.aws_subnets.subnets.ids - # vpc_security_group_ids = data.aws_security_groups.security_groups.ids + vpc_subnet_ids = data.aws_subnets.subnets.ids + vpc_security_group_ids = data.aws_security_groups.security_groups.ids tags = local.common_tags } @@ -633,8 +653,8 @@ module "call_bedrock_lambda_function" { lambda_role = var.lambda_role_arn create_role = false - # vpc_subnet_ids = data.aws_subnets.subnets.ids - # vpc_security_group_ids = data.aws_security_groups.security_groups.ids + vpc_subnet_ids = data.aws_subnets.subnets.ids + vpc_security_group_ids = data.aws_security_groups.security_groups.ids tags = local.common_tags } @@ -662,8 +682,12 @@ module "database_interface_lambda_function" { lambda_role = var.lambda_role_arn create_role = false - # vpc_subnet_ids = data.aws_subnets.subnets.ids - # vpc_security_group_ids = data.aws_security_groups.security_groups.ids + layers = [ + module.lambda_layer_snowflake_connector.lambda_layer_arn + ] + + vpc_subnet_ids = data.aws_subnets.subnets.ids + vpc_security_group_ids = data.aws_security_groups.security_groups.ids tags = local.common_tags } @@ -691,8 +715,12 @@ module "database_interface_get_lambda_function" { lambda_role = var.lambda_role_arn create_role = false - # vpc_subnet_ids = data.aws_subnets.subnets.ids - # vpc_security_group_ids = data.aws_security_groups.security_groups.ids + layers = [ + module.lambda_layer_snowflake_connector.lambda_layer_arn + ] + + vpc_subnet_ids = data.aws_subnets.subnets.ids + vpc_security_group_ids = data.aws_security_groups.security_groups.ids tags = local.common_tags } @@ -798,7 +826,7 @@ data "aws_iam_policy_document" "api_gateway_policy" { } } } -# # API Gateway - attach resource policy +# API Gateway - attach resource policy # resource "aws_api_gateway_rest_api_policy" "project_api_policy" { # rest_api_id = aws_api_gateway_rest_api.project_api.id # policy = data.aws_iam_policy_document.api_gateway_policy.json diff --git a/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/variables.tf b/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/variables.tf index 3a7c776..82718ad 100644 --- a/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/variables.tf +++ b/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/variables.tf @@ -159,6 +159,28 @@ variable "pypdf2_lambda_layer" { } } +# Lambda layer - Snowflake connector +variable "snowflake_connector_lambda_layer" { + type = object({ + name = string + description = string + compatible_runtimes = list(string) + compatible_architectures = list(string) + source_path = string + local_existing_package = string + create_package = bool + }) + default = { + name = "snowflake-connector" + description = "Snowflake connector layer used to log pipeline progress" + compatible_runtimes = ["python3.9"] + compatible_architectures = ["x86_64"] + source_path = "../src/lambda-layer/snowflake-connector/" + local_existing_package = null + create_package = true + } +} + # s3-folder-details variable "s3_folder_detail_lambda" { type = object({ @@ -567,7 +589,7 @@ variable "database_interface_lambda" { name = "database-interface" description = "This function acts as an interface to communicate with database" handler = "index.lambda_handler" - runtime = "python3.12" + runtime = "python3.9" timeout = 300 memory_size = 256 ephemeral_storage_size = 512 @@ -597,7 +619,7 @@ variable "database_interface_get_lambda" { name = "database-interface-get" description = "This function acts as an interface to get records from Snowflake database" handler = "index.lambda_handler" - runtime = "python3.12" + runtime = "python3.9" timeout = 300 memory_size = 256 ephemeral_storage_size = 512