From 7b057e6b47463881f30f79d234736e16a32d15c0 Mon Sep 17 00:00:00 2001 From: Umang Mistry Date: Wed, 12 Jun 2024 19:17:04 -0500 Subject: [PATCH 1/3] Updated layers & lambda code --- ...ayer.zip => snowflake-connector-39-verified.zip} | Bin .../src/lambda/prompt-orchestrator/index.py | 8 ++++---- .../modules/aws-textract-pipeline-part3/main.tf | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) rename textract-pipeline/src/lambda-layer/snowflake-connector/{sf-39-layer.zip => snowflake-connector-39-verified.zip} (100%) diff --git a/textract-pipeline/src/lambda-layer/snowflake-connector/sf-39-layer.zip b/textract-pipeline/src/lambda-layer/snowflake-connector/snowflake-connector-39-verified.zip similarity index 100% rename from textract-pipeline/src/lambda-layer/snowflake-connector/sf-39-layer.zip rename to textract-pipeline/src/lambda-layer/snowflake-connector/snowflake-connector-39-verified.zip diff --git a/textract-pipeline/src/lambda/prompt-orchestrator/index.py b/textract-pipeline/src/lambda/prompt-orchestrator/index.py index 4bd563e..b42ac74 100644 --- a/textract-pipeline/src/lambda/prompt-orchestrator/index.py +++ b/textract-pipeline/src/lambda/prompt-orchestrator/index.py @@ -20,7 +20,7 @@ BEDROCK_REGION = "us-east-1" SNOWFLAKE_SCHEMA = "STG" MAX_PROMPT_LENGTH_PER_GROUP = 20000 # Adjust as needed LLM_SELECTED = "Claude 3 - Sonnet" -SNOWFLAKE_CONN_SECRET_NAME = "doczy_dev_db_creds" +SNOWFLAKE_CONN_SECRET_NAME = "doczy-dev-db-svc-acc" # Get variable ENVIRONMENT from env variables using os env = os.environ.get('ENVIRONMENT') @@ -70,8 +70,8 @@ sf_secrets = json.loads(get_secret()) snowflake_conn = snowflake.connector.connect( user=sf_secrets.get('user'), password=sf_secrets.get('password'), - account=sf_secrets.get('account_alias'), - role=sf_secrets.get('ROLE'), + account=sf_secrets.get('account_locator'), + role=sf_secrets.get('role'), warehouse=sf_secrets.get('warehouse'), database=sf_secrets.get('database'), schema=SNOWFLAKE_SCHEMA @@ -494,7 +494,7 @@ def main(bucket, object_key, field_groups): question = prepare_prompt(*field_groups) response_text = invoke_llm(context, question, llm_selected=LLM_SELECTED) - logger.info(f"Response text: {response_text}, CONTEXT: {context}") + # logger.info(f"Response text: {response_text}, CONTEXT: {context}") field_l, answer_l, page_no_l, location_l, snippet_l = json_parsing(response_text, context) field_list.extend(field_l) answer_list.extend(answer_l) diff --git a/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf b/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf index f41ab84..604f58a 100644 --- a/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf +++ b/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf @@ -655,7 +655,7 @@ module "prompt_orchestrator_lambda_function" { environment_variables = merge(local.environment_variables, local.prompt_orchestrator_lambda.environment_variables, { PROMPT_PROCESSOR_QUEUE_URL = var.prompt_processor_queue_url - MAX_PROMPT_LENGTH_PER_GROUP = "5000" + MAX_PROMPT_LENGTH_PER_GROUP = "50000" }) source_path = local.prompt_orchestrator_lambda.source_path From 94c5f2a68db63bd9331d1f638270df639d5acd62 Mon Sep 17 00:00:00 2001 From: Umang Mistry Date: Thu, 13 Jun 2024 16:13:25 -0500 Subject: [PATCH 2/3] Fixed message parsing issue in orchestrator --- textract-pipeline/src/lambda/prompt-orchestrator/index.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/textract-pipeline/src/lambda/prompt-orchestrator/index.py b/textract-pipeline/src/lambda/prompt-orchestrator/index.py index b42ac74..8416fc3 100644 --- a/textract-pipeline/src/lambda/prompt-orchestrator/index.py +++ b/textract-pipeline/src/lambda/prompt-orchestrator/index.py @@ -132,6 +132,8 @@ def lambda_handler(event, context): # Process each message from the SQS event for record in records['Records']: # Extract the message body from the record + logger.info(f"RECORD:: {record}") + sqs_record = record['s3'] bucket_name = sqs_record['bucket']['name'] object_key = unquote_plus(sqs_record['object']['key']) From e717e4033c29eabb8f2372bde1602b557a36c096 Mon Sep 17 00:00:00 2001 From: Michael McGuinness Date: Fri, 27 Sep 2024 12:38:03 +0100 Subject: [PATCH 3/3] comma --- .../terraform/modules/aws-textract-pipeline-part3/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf b/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf index 604f58a..6480d23 100644 --- a/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf +++ b/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf @@ -649,6 +649,7 @@ module "prompt_orchestrator_lambda_function" { layers = [ module.anthropic_sf_lambda_layer.lambda_layer_arn, + # module.pandas_numpy_lambda_layer.lambda_layer_arn, module.pydantic_lambda_layer.lambda_layer_arn, var.pandas_numpy_lambda_layer_additional_arn ]