Added snowflake-connector lambda layer to terraform

textract-pipeline/terraform/main.tf

@@ -148,6 +148,9 @@ data "aws_iam_policy" "S3FullAccess" {
 data "aws_iam_policy" "CloudWatchFullAccess" {
   arn = "arn:aws:iam::aws:policy/CloudWatchLogsFullAccess"
 }
+data "aws_iam_policy" "AWSLambdaVPCAccessExecutionRole" {
+  arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole"
+}
 resource "aws_iam_role_policy_attachment" "bedrock-policy-attach" {
   role       = "${aws_iam_role.lambda_role.name}"
   policy_arn = "${data.aws_iam_policy.BedrockFullAccess.arn}"
@@ -159,4 +162,8 @@ resource "aws_iam_role_policy_attachment" "s3-policy-attach" {
 resource "aws_iam_role_policy_attachment" "cloudwatch-policy-attach" {
   role       = "${aws_iam_role.lambda_role.name}"
   policy_arn = "${data.aws_iam_policy.CloudWatchFullAccess.arn}"
+}
+resource "aws_iam_role_policy_attachment" "VPC-policy-attach" {
+  role       = "${aws_iam_role.lambda_role.name}"
+  policy_arn = "${data.aws_iam_policy.AWSLambdaVPCAccessExecutionRole.arn}"
 }

textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf

@@ -52,6 +52,7 @@ locals {
   libre_office_lambda_layer_name = "${local.lambda_layer_prefix}-${var.libre_office_lambda_layer.name}"
   brotli_lambda_layer_name = "${local.lambda_layer_prefix}-${var.brotli_lambda_layer.name}"
   pypdf2_lambda_layer_name = "${local.lambda_layer_prefix}-${var.pypdf2_lambda_layer.name}"
+  snowflake_connector_lambda_layer_name = "${local.lambda_layer_prefix}-${var.snowflake_connector_lambda_layer.name}"
 
   # API Gateway configuration
   api_description = "API's used for DoczyAI pipeline"
@@ -150,6 +151,25 @@ module "lambda_layer_pypdf2" {
   local_existing_package  = var.pypdf2_lambda_layer.local_existing_package
 }
 
+# Lambda Layers - snowflake connector - create package from source code
+module "lambda_layer_snowflake_connector" {
+  source = "terraform-aws-modules/lambda/aws"
+
+  create_layer = true
+  create_package = var.snowflake_connector_lambda_layer.create_package
+
+  layer_name          = local.snowflake_connector_lambda_layer_name
+  description         = var.snowflake_connector_lambda_layer.description
+  compatible_runtimes = var.snowflake_connector_lambda_layer.compatible_runtimes
+  compatible_architectures       = var.snowflake_connector_lambda_layer.compatible_architectures
+
+  source_path = var.snowflake_connector_lambda_layer.source_path
+
+  # store_on_s3 = true
+  s3_bucket   = local.lambda_layer_s3_bucket
+  s3_prefix = local.lambda_layer_s3_prefix
+}
+
 # batch creation lambda function
 module "s3_folder_detail_lambda_function" {
   source = "terraform-aws-modules/lambda/aws"
@@ -171,8 +191,8 @@ module "s3_folder_detail_lambda_function" {
   lambda_role  = var.lambda_role_arn
   create_role = false
 
-  # vpc_subnet_ids         = data.aws_subnets.subnets.ids
-  # vpc_security_group_ids = data.aws_security_groups.security_groups.ids
+  vpc_subnet_ids         = data.aws_subnets.subnets.ids
+  vpc_security_group_ids = data.aws_security_groups.security_groups.ids
 
   tags    = local.common_tags
 }
@@ -198,8 +218,8 @@ module "batch_creation_lambda_function" {
   lambda_role  = var.lambda_role_arn
   create_role = false
 
-  # vpc_subnet_ids         = data.aws_subnets.subnets.ids
-  # vpc_security_group_ids = data.aws_security_groups.security_groups.ids
+  vpc_subnet_ids         = data.aws_subnets.subnets.ids
+  vpc_security_group_ids = data.aws_security_groups.security_groups.ids
 
   tags    = local.common_tags
 }
@@ -227,8 +247,8 @@ module "trigger_pipeline_lambda_function" {
   lambda_role  = var.lambda_role_arn
   create_role = false
 
-  # vpc_subnet_ids         = data.aws_subnets.subnets.ids
-  # vpc_security_group_ids = data.aws_security_groups.security_groups.ids
+  vpc_subnet_ids         = data.aws_subnets.subnets.ids
+  vpc_security_group_ids = data.aws_security_groups.security_groups.ids
 
   tags    = local.common_tags
 }
@@ -266,8 +286,8 @@ module "insert_record_lambda_function" {
   lambda_role  = var.lambda_role_arn
   create_role = false
 
-  # vpc_subnet_ids         = data.aws_subnets.subnets.ids
-  # vpc_security_group_ids = data.aws_security_groups.security_groups.ids
+  vpc_subnet_ids         = data.aws_subnets.subnets.ids
+  vpc_security_group_ids = data.aws_security_groups.security_groups.ids
 
   tags    = local.common_tags
 }
@@ -310,8 +330,8 @@ module "docx_to_pdf_lambda_function" {
   lambda_role  = var.lambda_role_arn
   create_role = false
 
-  # vpc_subnet_ids         = data.aws_subnets.subnets.ids
-  # vpc_security_group_ids = data.aws_security_groups.security_groups.ids
+  vpc_subnet_ids         = data.aws_subnets.subnets.ids
+  vpc_security_group_ids = data.aws_security_groups.security_groups.ids
 
   tags    = local.common_tags
 }
@@ -349,8 +369,8 @@ module "tiff_to_pdf_lambda_function" {
   lambda_role  = var.lambda_role_arn
   create_role = false
 
-  # vpc_subnet_ids         = data.aws_subnets.subnets.ids
-  # vpc_security_group_ids = data.aws_security_groups.security_groups.ids
+  vpc_subnet_ids         = data.aws_subnets.subnets.ids
+  vpc_security_group_ids = data.aws_security_groups.security_groups.ids
 
   tags    = local.common_tags
 }
@@ -392,8 +412,8 @@ module "pdf_validation_lambda_function" {
   lambda_role  = var.lambda_role_arn
   create_role = false
 
-  # vpc_subnet_ids         = data.aws_subnets.subnets.ids
-  # vpc_security_group_ids = data.aws_security_groups.security_groups.ids
+  vpc_subnet_ids         = data.aws_subnets.subnets.ids
+  vpc_security_group_ids = data.aws_security_groups.security_groups.ids
 
   tags    = local.common_tags
 }
@@ -434,8 +454,8 @@ module "textract_sender_lambda_function" {
   lambda_role  = var.lambda_role_arn
   create_role = false
 
-  # vpc_subnet_ids         = data.aws_subnets.subnets.ids
-  # vpc_security_group_ids = data.aws_security_groups.security_groups.ids
+  vpc_subnet_ids         = data.aws_subnets.subnets.ids
+  vpc_security_group_ids = data.aws_security_groups.security_groups.ids
 
   tags    = local.common_tags
 }
@@ -473,8 +493,8 @@ module "textract_receiver_lambda_function" {
   lambda_role  = var.lambda_role_arn
   create_role = false
 
-  # vpc_subnet_ids         = data.aws_subnets.subnets.ids
-  # vpc_security_group_ids = data.aws_security_groups.security_groups.ids
+  vpc_subnet_ids         = data.aws_subnets.subnets.ids
+  vpc_security_group_ids = data.aws_security_groups.security_groups.ids
 
   tags    = local.common_tags
 }
@@ -516,8 +536,8 @@ module "text_creation_lambda_function" {
   lambda_role  = var.lambda_role_arn
   create_role = false
 
-  # vpc_subnet_ids         = data.aws_subnets.subnets.ids
-  # vpc_security_group_ids = data.aws_security_groups.security_groups.ids
+  vpc_subnet_ids         = data.aws_subnets.subnets.ids
+  vpc_security_group_ids = data.aws_security_groups.security_groups.ids
 
   tags    = local.common_tags
 }
@@ -561,8 +581,8 @@ module "prompt_orchestrator_lambda_function" {
   lambda_role  = var.lambda_role_arn
   create_role = false
 
-  # vpc_subnet_ids         = data.aws_subnets.subnets.ids
-  # vpc_security_group_ids = data.aws_security_groups.security_groups.ids
+  vpc_subnet_ids         = data.aws_subnets.subnets.ids
+  vpc_security_group_ids = data.aws_security_groups.security_groups.ids
 
   tags    = local.common_tags
 }
@@ -604,8 +624,8 @@ module "prompt_processor_lambda_function" {
   lambda_role  = var.lambda_role_arn
   create_role = false
 
-  # vpc_subnet_ids         = data.aws_subnets.subnets.ids
-  # vpc_security_group_ids = data.aws_security_groups.security_groups.ids
+  vpc_subnet_ids         = data.aws_subnets.subnets.ids
+  vpc_security_group_ids = data.aws_security_groups.security_groups.ids
 
   tags    = local.common_tags
 }
@@ -633,8 +653,8 @@ module "call_bedrock_lambda_function" {
   lambda_role  = var.lambda_role_arn
   create_role = false
 
-  # vpc_subnet_ids         = data.aws_subnets.subnets.ids
-  # vpc_security_group_ids = data.aws_security_groups.security_groups.ids
+  vpc_subnet_ids         = data.aws_subnets.subnets.ids
+  vpc_security_group_ids = data.aws_security_groups.security_groups.ids
 
   tags    = local.common_tags
 }
@@ -662,8 +682,12 @@ module "database_interface_lambda_function" {
   lambda_role  = var.lambda_role_arn
   create_role = false
 
-  # vpc_subnet_ids         = data.aws_subnets.subnets.ids
-  # vpc_security_group_ids = data.aws_security_groups.security_groups.ids
+  layers = [
+    module.lambda_layer_snowflake_connector.lambda_layer_arn
+  ]
+
+  vpc_subnet_ids         = data.aws_subnets.subnets.ids
+  vpc_security_group_ids = data.aws_security_groups.security_groups.ids
 
   tags    = local.common_tags
 }
@@ -691,8 +715,12 @@ module "database_interface_get_lambda_function" {
   lambda_role  = var.lambda_role_arn
   create_role = false
 
-  # vpc_subnet_ids         = data.aws_subnets.subnets.ids
-  # vpc_security_group_ids = data.aws_security_groups.security_groups.ids
+  layers = [
+    module.lambda_layer_snowflake_connector.lambda_layer_arn
+  ]
+
+  vpc_subnet_ids         = data.aws_subnets.subnets.ids
+  vpc_security_group_ids = data.aws_security_groups.security_groups.ids
 
   tags    = local.common_tags
 }
@@ -798,7 +826,7 @@ data "aws_iam_policy_document" "api_gateway_policy" {
     }
   }
 }
-# # API Gateway - attach resource policy
+# API Gateway - attach resource policy
 # resource "aws_api_gateway_rest_api_policy" "project_api_policy" {
 #   rest_api_id = aws_api_gateway_rest_api.project_api.id
 #   policy      = data.aws_iam_policy_document.api_gateway_policy.json

textract-pipeline/terraform/modules/aws-textract-pipeline-part3/variables.tf

@@ -159,6 +159,28 @@ variable "pypdf2_lambda_layer" {
   }
 }
 
+# Lambda layer - Snowflake connector
+variable "snowflake_connector_lambda_layer" {
+  type = object({
+    name    = string
+    description = string
+    compatible_runtimes = list(string)
+    compatible_architectures = list(string)
+    source_path = string
+    local_existing_package = string
+    create_package = bool
+  })
+  default = {
+    name = "snowflake-connector"
+    description = "Snowflake connector layer used to log pipeline progress"
+    compatible_runtimes = ["python3.9"]
+    compatible_architectures = ["x86_64"]
+    source_path = "../src/lambda-layer/snowflake-connector/"
+    local_existing_package = null
+    create_package = true
+  }
+}
+
 # s3-folder-details
 variable "s3_folder_detail_lambda" {
   type = object({
@@ -567,7 +589,7 @@ variable "database_interface_lambda" {
     name = "database-interface"
     description = "This function acts as an interface to communicate with database"
     handler = "index.lambda_handler"
-    runtime = "python3.12"
+    runtime = "python3.9"
     timeout = 300
     memory_size = 256
     ephemeral_storage_size = 512
@@ -597,7 +619,7 @@ variable "database_interface_get_lambda" {
     name = "database-interface-get"
     description = "This function acts as an interface to get records from Snowflake database"
     handler = "index.lambda_handler"
-    runtime = "python3.12"
+    runtime = "python3.9"
     timeout = 300
     memory_size = 256
     ephemeral_storage_size = 512