jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

updated training interface sp

Browse Source
This commit is contained in:
Umang Mistry
2024-03-15 17:02:13 -05:00
parent 6f6ca9af83
commit 95b8ed8595
2 changed files with 6 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
snowflake/scripts/training_interface/R__005_LOAD_COLUMN_CONFIG_SP.sql
+3 -3
View File
@@ -16,7 +16,7 @@ BEGIN
EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.COLUMN_CONFIG_STAGE
STORAGE_INTEGRATION = dev_bucket_integration
URL = ''s3://doczy-dev-infra-raw-data-ingestion/training_data_raw/' || :file_name || '''
FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');';
FILE_FORMAT = (FORMAT_NAME = ''STG.RAW_TRAINING_DATA_FORMAT'');';
call stg.log_audit(:procedure_name, 'Section 1', 99, 'END');
@@ -36,7 +36,7 @@ BEGIN
NULLIF(TRIM($2), '') AS COLUMN_DATATYPE
FROM @STG.COLUMN_CONFIG_STAGE
)
FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER')
FILE_FORMAT = (FORMAT_NAME = 'STG.RAW_TRAINING_DATA_FORMAT')
ON_ERROR = ABORT_STATEMENT;
call stg.log_audit(:procedure_name, 'Section 3', 99, 'END');
@@ -44,7 +44,7 @@ BEGIN
call stg.log_audit(:procedure_name, 'Section 4', 99, 'START');
INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT)
SELECT STG.AUDIT_SID.NEXTVAL, 'STG.TRAINING_DATA_COLUMN_CONFIG',*
SELECT STG.AUDIT_SID.NEXTVAL, :procedure_name,*
FROM
(SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.COLUMN_CONFIG_STAGE group by 1,2);
snowflake/scripts/training_interface/R__006_LOAD_TRAINING_DATA_RAW_SP.sql
+3 -8
View File
@@ -30,14 +30,9 @@ BEGIN
call stg.log_audit(:procedure_name, 'Section 3', 99, 'START');
-- Copy command to load data
-- COPY INTO STG.TRAINING_DATA_COLUMN_CONFIG FROM (
-- SELECT
-- NULLIF(TRIM($1), '') AS COLUMN_NAME,
-- NULLIF(TRIM($2), '') AS COLUMN_DATATYPE
-- FROM @STG.RAW_TRAINING_DATA_STAGE
-- )
-- FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER')
-- ON_ERROR = ABORT_STATEMENT;
COPY INTO STG.TRAINING_DATA_RAW FROM FROM @STG.RAW_TRAINING_DATA_STAGE
FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER')
ON_ERROR = ABORT_STATEMENT;
call stg.log_audit(:procedure_name, 'Section 3', 99, 'END');