From d77753931fa2cb2b29f584ff31f8fb4e3d1abf06 Mon Sep 17 00:00:00 2001 From: Umang Mistry Date: Mon, 24 Jun 2024 15:27:31 -0500 Subject: [PATCH] updated pipeline output table and sp --- .../UAT/config_interface/R__1006_PIPELINE_OUTPUT.sql | 1 - .../R__1007_LOAD_DOCZY_PIPELINE_RAW_OPUTPUTS.sql | 10 ++++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/snowflake/UAT/config_interface/R__1006_PIPELINE_OUTPUT.sql b/snowflake/UAT/config_interface/R__1006_PIPELINE_OUTPUT.sql index 407a17c..b9849d1 100644 --- a/snowflake/UAT/config_interface/R__1006_PIPELINE_OUTPUT.sql +++ b/snowflake/UAT/config_interface/R__1006_PIPELINE_OUTPUT.sql @@ -6,6 +6,5 @@ CREATE TABLE IF NOT EXISTS STG.DOCZY_PIPELINE_RAW_OUTPUT_AC ( NEW_EXTRACTED_VALUE VARCHAR(16777216), SNIPPET VARCHAR(16777216), PAGE_NUMBER VARCHAR(255), - REVISED_PROMPT VARCHAR(16777216), BATCH_ID VARCHAR(255) ); \ No newline at end of file diff --git a/snowflake/UAT/config_interface/R__1007_LOAD_DOCZY_PIPELINE_RAW_OPUTPUTS.sql b/snowflake/UAT/config_interface/R__1007_LOAD_DOCZY_PIPELINE_RAW_OPUTPUTS.sql index 94a9c9f..02898eb 100644 --- a/snowflake/UAT/config_interface/R__1007_LOAD_DOCZY_PIPELINE_RAW_OPUTPUTS.sql +++ b/snowflake/UAT/config_interface/R__1007_LOAD_DOCZY_PIPELINE_RAW_OPUTPUTS.sql @@ -16,7 +16,7 @@ BEGIN -- TODO: Update for UAT -- s3://doczyai-use2-u-infra-s3-raw-data-ingestion/ EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.DOCZY_PIPELINE_RAW_OUTPUT_STAGE - STORAGE_INTEGRATION = dev_bucket_integration + STORAGE_INTEGRATION = UAT_BUCKET_INTEGRATION URL = ''s3://doczyai-use2-u-infra-s3-raw-data-ingestion/doczy_pipeline_output/' || :file_name || ''' FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');'; @@ -31,9 +31,8 @@ BEGIN NULLIF(TRIM($4),'') AS RAW_VALUE, NULLIF(TRIM($5),'') AS NEW_EXTRACTED_VALUE, NULLIF(TRIM($6),'') AS SNIPPET, - NULLIF(TRIM($8),'') AS PAGE_NUMBER, - NULLIF(TRIM($9),'') AS REVISED_PROMPT, - NULLIF(TRIM($10),'') AS BATCH_ID + NULLIF(TRIM($7),'') AS PAGE_NUMBER, + NULLIF(TRIM($8),'') AS BATCH_ID FROM @STG.DOCZY_PIPELINE_RAW_OUTPUT_STAGE ) @@ -59,5 +58,4 @@ BEGIN END; -$$; - +$$; \ No newline at end of file