updated pipeline output table and sp

This commit is contained in:
Umang Mistry
2024-06-24 15:27:31 -05:00
parent ab4abdd28d
commit d77753931f
2 changed files with 4 additions and 7 deletions
@@ -6,6 +6,5 @@ CREATE TABLE IF NOT EXISTS STG.DOCZY_PIPELINE_RAW_OUTPUT_AC (
NEW_EXTRACTED_VALUE VARCHAR(16777216),
SNIPPET VARCHAR(16777216),
PAGE_NUMBER VARCHAR(255),
REVISED_PROMPT VARCHAR(16777216),
BATCH_ID VARCHAR(255)
);
@@ -16,7 +16,7 @@ BEGIN
-- TODO: Update for UAT
-- s3://doczyai-use2-u-infra-s3-raw-data-ingestion/
EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.DOCZY_PIPELINE_RAW_OUTPUT_STAGE
STORAGE_INTEGRATION = dev_bucket_integration
STORAGE_INTEGRATION = UAT_BUCKET_INTEGRATION
URL = ''s3://doczyai-use2-u-infra-s3-raw-data-ingestion/doczy_pipeline_output/' || :file_name || '''
FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');';
@@ -31,9 +31,8 @@ BEGIN
NULLIF(TRIM($4),'') AS RAW_VALUE,
NULLIF(TRIM($5),'') AS NEW_EXTRACTED_VALUE,
NULLIF(TRIM($6),'') AS SNIPPET,
NULLIF(TRIM($8),'') AS PAGE_NUMBER,
NULLIF(TRIM($9),'') AS REVISED_PROMPT,
NULLIF(TRIM($10),'') AS BATCH_ID
NULLIF(TRIM($7),'') AS PAGE_NUMBER,
NULLIF(TRIM($8),'') AS BATCH_ID
FROM @STG.DOCZY_PIPELINE_RAW_OUTPUT_STAGE
)
@@ -59,5 +58,4 @@ BEGIN
END;
$$;
$$;