diff --git a/snowflake/scripts/config_interface/R__002_CONTRACT_CONFIG_TABLE.sql b/snowflake/scripts/config_interface/R__002_CONTRACT_CONFIG_TABLE.sql index 1758057..e963e50 100644 --- a/snowflake/scripts/config_interface/R__002_CONTRACT_CONFIG_TABLE.sql +++ b/snowflake/scripts/config_interface/R__002_CONTRACT_CONFIG_TABLE.sql @@ -1,5 +1,5 @@ -- CREATING THE CONFIG TABLE -CREATE TABLE IF NOT EXISTS CONTRACT_CONFIG ( +CREATE TABLE IF NOT EXISTS STG.CONTRACT_CONFIG ( REQUEST_ID NUMERIC, CONTRACT_NAME VARCHAR, GROUP_1 BOOLEAN, diff --git a/snowflake/scripts/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql b/snowflake/scripts/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql new file mode 100644 index 0000000..fe8b1f4 --- /dev/null +++ b/snowflake/scripts/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql @@ -0,0 +1,69 @@ +CREATE OR REPLACE PROCEDURE STG.LOAD_CONTRACT_CONFIG(file_name VARCHAR) +RETURNS STRING +LANGUAGE SQL +EXECUTE AS CALLER +AS +$$ +DECLARE + procedure_name varchar; +BEGIN + + procedure_name := 'LOAD_CONTRACT_CONFIG'; + + call stg.log_audit(:procedure_name, 'Section 1', 99, 'START'); + + -- Create or replace stage with dynamic file name + EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.CONTRACT_CONFIG_STAGE + STORAGE_INTEGRATION = dev_bucket_integration + URL = ''s3://doczy-dev-infra-raw-data-ingestion/config_interface/' || :file_name || ''' + FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');'; + + call stg.log_audit(:procedure_name, 'Section 1', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 2', 99, 'START'); + + COPY INTO STG.CONTRACT_CONFIG FROM ( + SELECT + NULLIF(TRIM($1),'') AS REQUEST_ID, + NULLIF(TRIM($2),'') AS CONTRACT_NAME, + NULLIF(TRIM($3),'') AS GROUP_1, + NULLIF(TRIM($4),'') AS GROUP_1_OVERRIDE, + NULLIF(TRIM($5),'') AS GROUP_2, + NULLIF(TRIM($6),'') AS GROUP_2_OVERRIDE, + NULLIF(TRIM($7),'') AS GROUP_3, + NULLIF(TRIM($8),'') AS GROUP_3_OVERRIDE, + NULLIF(TRIM($9),'') AS GROUP_4, + NULLIF(TRIM($10),'') AS GROUP_4_OVERRIDE, + NULLIF(TRIM($11),'') AS GROUP_5, + NULLIF(TRIM($12),'') AS GROUP_5_OVERRIDE, + NULLIF(TRIM($13),'') AS REQUEST_DATETIME, + NULLIF(TRIM($14),'') AS REQUEST_USER, + NULLIF(TRIM($15),'') AS OVERRIDE_DATETIME, + NULLIF(TRIM($16),'') AS LATEST_FLAG DEFAULT TRUE, + NULLIF(TRIM($17),'') AS PIPELINE_KICKOFF_DATETIME + FROM @STG.CONTRACT_CONFIG_STAGE + ) + FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER') + ON_ERROR = ABORT_STATEMENT; + + call stg.log_audit(:procedure_name, 'Section 2', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 3', 99, 'START'); + + INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) + SELECT STG.AUDIT_SID.NEXTVAL, 'STG.CONTRACT_CONFIG',* + FROM + (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.CONTRACT_CONFIG_STAGE group by 1,2); + + + + RETURN 'Setup, Load, and Audit Complete'; + + call stg.log_audit(:procedure_name, 'Section 3', 99, 'END'); + + + + +END; +$$; + diff --git a/snowflake/scripts/training_interface/R__001_INIT_OBJS.sql b/snowflake/scripts/training_interface/R__001_INIT_OBJS.sql index 7b7bb3c..64eda50 100644 --- a/snowflake/scripts/training_interface/R__001_INIT_OBJS.sql +++ b/snowflake/scripts/training_interface/R__001_INIT_OBJS.sql @@ -23,7 +23,7 @@ CREATE FILE FORMAT IF NOT EXISTS STG.CSV_HEADER RECORD_DELIMITER = '\\n' DATE_FORMAT = AUTO TRIM_SPACE = TRUE - NULL_IF = ('NULL', '', 'N/A','?','~') + NULL_IF = ('NULL', '', 'N/A','?','~','\\N') SKIP_HEADER = 1 EMPTY_FIELD_AS_NULL = TRUE FIELD_OPTIONALLY_ENCLOSED_BY = '"' @@ -31,7 +31,6 @@ CREATE FILE FORMAT IF NOT EXISTS STG.CSV_HEADER -- Create staging table for attempt logs - CREATE TABLE IF NOT EXISTS STG.TRAINING_ATTEMPT_LOGS ( FIELD_NAME VARCHAR, @@ -44,3 +43,10 @@ CREATE TABLE IF NOT EXISTS STG.TRAINING_ATTEMPT_LOGS ); + +-- Create table for column config that will be used by SP to create raw training data table +CREATE TABLE IF NOT EXISTS STG.TRAINING_DATA_COLUMN_CONFIG( +COLUMN_NAME VARCHAR, +COLUMN_DATATYPE VARCHAR +); +-- Raw training data table is created dynamically with the help of stored procedure diff --git a/snowflake/scripts/training_interface/R__004_CREATE_TRAINING_DATA_TABLE_SP.sql b/snowflake/scripts/training_interface/R__004_CREATE_TRAINING_DATA_TABLE_SP.sql new file mode 100644 index 0000000..6a49bc9 --- /dev/null +++ b/snowflake/scripts/training_interface/R__004_CREATE_TRAINING_DATA_TABLE_SP.sql @@ -0,0 +1,28 @@ +CREATE OR REPLACE PROCEDURE STG.CREATE_TRAINING_DATA_TABLE() +RETURNS VARCHAR(16777216) +LANGUAGE SQL +EXECUTE AS CALLER +AS ' +DECLARE + dynamic_ddl STRING := ''CREATE OR REPLACE TABLE STG.TRAINING_DATA_RAW (''; + column_details RESULTSET; + first_column BOOLEAN := TRUE; + cur_config cursor FOR + SELECT column_name, column_datatype FROM STG.TRAINING_DATA_COLUMN_CONFIG; +BEGIN + + OPEN cur_config; + FOR rec IN cur_config DO + dynamic_ddl := dynamic_ddl || rec.column_name || '' '' || rec.column_datatype || '',''; + END FOR; + + dynamic_ddl := LEFT(dynamic_ddl, LENGTH(dynamic_ddl) - 1); + + dynamic_ddl := dynamic_ddl || '');''; + + EXECUTE IMMEDIATE dynamic_ddl; + + RETURN dynamic_ddl; +END; + +'; \ No newline at end of file diff --git a/snowflake/scripts/training_interface/R__005_LOAD_COLUMN_CONFIG_SP.sql b/snowflake/scripts/training_interface/R__005_LOAD_COLUMN_CONFIG_SP.sql new file mode 100644 index 0000000..b65b42f --- /dev/null +++ b/snowflake/scripts/training_interface/R__005_LOAD_COLUMN_CONFIG_SP.sql @@ -0,0 +1,56 @@ +CREATE OR REPLACE PROCEDURE STG.LOAD_COLUMN_CONFIG(file_name VARCHAR) +RETURNS STRING +LANGUAGE SQL +EXECUTE AS CALLER +AS +$$ +DECLARE + procedure_name varchar; +BEGIN + + procedure_name := 'LOAD_COLUMN_CONFIG'; + + call stg.log_audit(:procedure_name, 'Section 1', 99, 'START'); + + -- Create or replace stage with dynamic file name + EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.COLUMN_CONFIG_STAGE + STORAGE_INTEGRATION = dev_bucket_integration + URL = ''s3://doczy-dev-infra-raw-data-ingestion/training_data_raw/' || :file_name || ''' + FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');'; + + call stg.log_audit(:procedure_name, 'Section 1', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 2', 99, 'START'); + + -- Truncate table as we are using the KILL & FILL approach + TRUNCATE TABLE STG.TRAINING_DATA_COLUMN_CONFIG; + + call stg.log_audit(:procedure_name, 'Section 2', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 3', 99, 'START'); + + -- Copy command to load data + COPY INTO STG.TRAINING_DATA_COLUMN_CONFIG FROM ( + SELECT + NULLIF(TRIM($1), '') AS COLUMN_NAME, + NULLIF(TRIM($2), '') AS COLUMN_DATATYPE + FROM @STG.COLUMN_CONFIG_STAGE + ) + FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER') + ON_ERROR = ABORT_STATEMENT; + + call stg.log_audit(:procedure_name, 'Section 3', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 4', 99, 'START'); + + INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) + SELECT STG.AUDIT_SID.NEXTVAL, 'STG.TRAINING_DATA_COLUMN_CONFIG',* + FROM + (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.COLUMN_CONFIG_STAGE group by 1,2); + + call stg.log_audit(:procedure_name, 'Section 4', 99, 'END'); + + RETURN 'Setup, Load, and Audit Complete'; +END; +$$; + diff --git a/snowflake/scripts/training_interface/R__006_LOAD_TRAINING_DATA_RAW_SP.sql b/snowflake/scripts/training_interface/R__006_LOAD_TRAINING_DATA_RAW_SP.sql new file mode 100644 index 0000000..4580a54 --- /dev/null +++ b/snowflake/scripts/training_interface/R__006_LOAD_TRAINING_DATA_RAW_SP.sql @@ -0,0 +1,56 @@ +CREATE OR REPLACE PROCEDURE STG.LOAD_TRAINING_DATA_RAW(file_name VARCHAR) +RETURNS STRING +LANGUAGE SQL +EXECUTE AS CALLER +AS +$$ +DECLARE + procedure_name varchar; +BEGIN + + procedure_name := 'LOAD_TRAINING_DATA_RAW'; + + call stg.log_audit(:procedure_name, 'Section 1', 99, 'START'); + + -- Create or replace stage with dynamic file name + EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.RAW_TRAINING_DATA_STAGE + STORAGE_INTEGRATION = dev_bucket_integration + URL = ''s3://doczy-dev-infra-raw-data-ingestion/training_data_raw/' || :file_name || ''' + FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');'; + + call stg.log_audit(:procedure_name, 'Section 1', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 2', 99, 'START'); + + -- Truncate table as we are using the KILL & FILL approach + TRUNCATE TABLE STG.TRAINING_DATA_RAW; + + call stg.log_audit(:procedure_name, 'Section 2', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 3', 99, 'START'); + + -- Copy command to load data + COPY INTO STG.TRAINING_DATA_COLUMN_CONFIG FROM ( + SELECT + NULLIF(TRIM($1), '') AS COLUMN_NAME, + NULLIF(TRIM($2), '') AS COLUMN_DATATYPE + FROM @STG.RAW_TRAINING_DATA_STAGE + ) + FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER') + ON_ERROR = ABORT_STATEMENT; + + call stg.log_audit(:procedure_name, 'Section 3', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 4', 99, 'START'); + + INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) + SELECT STG.AUDIT_SID.NEXTVAL, 'STG.TRAINING_DATA_RAW',* + FROM + (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.RAW_TRAINING_DATA_STAGE group by 1,2); + + call stg.log_audit(:procedure_name, 'Section 4', 99, 'END'); + + RETURN 'Setup, Load, and Audit Complete'; +END; +$$; +