updated pipeline output table and sp
This commit is contained in:
@@ -6,6 +6,5 @@ CREATE TABLE IF NOT EXISTS STG.DOCZY_PIPELINE_RAW_OUTPUT_AC (
|
||||
NEW_EXTRACTED_VALUE VARCHAR(16777216),
|
||||
SNIPPET VARCHAR(16777216),
|
||||
PAGE_NUMBER VARCHAR(255),
|
||||
REVISED_PROMPT VARCHAR(16777216),
|
||||
BATCH_ID VARCHAR(255)
|
||||
);
|
||||
@@ -16,7 +16,7 @@ BEGIN
|
||||
-- TODO: Update for UAT
|
||||
-- s3://doczyai-use2-u-infra-s3-raw-data-ingestion/
|
||||
EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.DOCZY_PIPELINE_RAW_OUTPUT_STAGE
|
||||
STORAGE_INTEGRATION = dev_bucket_integration
|
||||
STORAGE_INTEGRATION = UAT_BUCKET_INTEGRATION
|
||||
URL = ''s3://doczyai-use2-u-infra-s3-raw-data-ingestion/doczy_pipeline_output/' || :file_name || '''
|
||||
FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');';
|
||||
|
||||
@@ -31,9 +31,8 @@ BEGIN
|
||||
NULLIF(TRIM($4),'') AS RAW_VALUE,
|
||||
NULLIF(TRIM($5),'') AS NEW_EXTRACTED_VALUE,
|
||||
NULLIF(TRIM($6),'') AS SNIPPET,
|
||||
NULLIF(TRIM($8),'') AS PAGE_NUMBER,
|
||||
NULLIF(TRIM($9),'') AS REVISED_PROMPT,
|
||||
NULLIF(TRIM($10),'') AS BATCH_ID
|
||||
NULLIF(TRIM($7),'') AS PAGE_NUMBER,
|
||||
NULLIF(TRIM($8),'') AS BATCH_ID
|
||||
|
||||
FROM @STG.DOCZY_PIPELINE_RAW_OUTPUT_STAGE
|
||||
)
|
||||
@@ -59,5 +58,4 @@ BEGIN
|
||||
|
||||
|
||||
END;
|
||||
$$;
|
||||
|
||||
$$;
|
||||
Reference in New Issue
Block a user