diff --git a/snowflake/scripts/training_interface/R__005_LOAD_COLUMN_CONFIG_SP.sql b/snowflake/scripts/training_interface/R__005_LOAD_COLUMN_CONFIG_SP.sql index b65b42f..950c02f 100644 --- a/snowflake/scripts/training_interface/R__005_LOAD_COLUMN_CONFIG_SP.sql +++ b/snowflake/scripts/training_interface/R__005_LOAD_COLUMN_CONFIG_SP.sql @@ -16,7 +16,7 @@ BEGIN EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.COLUMN_CONFIG_STAGE STORAGE_INTEGRATION = dev_bucket_integration URL = ''s3://doczy-dev-infra-raw-data-ingestion/training_data_raw/' || :file_name || ''' - FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');'; + FILE_FORMAT = (FORMAT_NAME = ''STG.RAW_TRAINING_DATA_FORMAT'');'; call stg.log_audit(:procedure_name, 'Section 1', 99, 'END'); @@ -36,7 +36,7 @@ BEGIN NULLIF(TRIM($2), '') AS COLUMN_DATATYPE FROM @STG.COLUMN_CONFIG_STAGE ) - FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER') + FILE_FORMAT = (FORMAT_NAME = 'STG.RAW_TRAINING_DATA_FORMAT') ON_ERROR = ABORT_STATEMENT; call stg.log_audit(:procedure_name, 'Section 3', 99, 'END'); @@ -44,7 +44,7 @@ BEGIN call stg.log_audit(:procedure_name, 'Section 4', 99, 'START'); INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) - SELECT STG.AUDIT_SID.NEXTVAL, 'STG.TRAINING_DATA_COLUMN_CONFIG',* + SELECT STG.AUDIT_SID.NEXTVAL, :procedure_name,* FROM (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.COLUMN_CONFIG_STAGE group by 1,2); diff --git a/snowflake/scripts/training_interface/R__006_LOAD_TRAINING_DATA_RAW_SP.sql b/snowflake/scripts/training_interface/R__006_LOAD_TRAINING_DATA_RAW_SP.sql index b893c32..3af335a 100644 --- a/snowflake/scripts/training_interface/R__006_LOAD_TRAINING_DATA_RAW_SP.sql +++ b/snowflake/scripts/training_interface/R__006_LOAD_TRAINING_DATA_RAW_SP.sql @@ -30,14 +30,9 @@ BEGIN call stg.log_audit(:procedure_name, 'Section 3', 99, 'START'); -- Copy command to load data - -- COPY INTO STG.TRAINING_DATA_COLUMN_CONFIG FROM ( - -- SELECT - -- NULLIF(TRIM($1), '') AS COLUMN_NAME, - -- NULLIF(TRIM($2), '') AS COLUMN_DATATYPE - -- FROM @STG.RAW_TRAINING_DATA_STAGE - -- ) - -- FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER') - -- ON_ERROR = ABORT_STATEMENT; + COPY INTO STG.TRAINING_DATA_RAW FROM FROM @STG.RAW_TRAINING_DATA_STAGE + FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER') + ON_ERROR = ABORT_STATEMENT; call stg.log_audit(:procedure_name, 'Section 3', 99, 'END');