Added training data new script, 1st draft of logging lambda and some updates to SF objs

snowflake/scripts/training_interface/R__001_INIT_OBJS.sql

@@ -49,3 +49,18 @@ CREATE TABLE IF NOT EXISTS STG.TRAINING_DATA_COLUMN_CONFIG(
 COLUMN_NAME VARCHAR,
 COLUMN_DATATYPE VARCHAR
 );
+
+-- Creating a separate file format for the training data
+CREATE FILE FORMAT IF NOT EXISTS STG.TRAINING_DATA_FILE_FORMAT
+        TYPE = 'CSV'
+        FIELD_DELIMITER = ','
+        FILE_EXTENSION = '.csv'
+        RECORD_DELIMITER = '\\n'           
+        DATE_FORMAT = AUTO
+        TRIM_SPACE = TRUE
+        NULL_IF = ('NULL', '', 'N/A','?','~','\\N')
+        SKIP_HEADER = 1        
+        EMPTY_FIELD_AS_NULL = TRUE 
+        FIELD_OPTIONALLY_ENCLOSED_BY = '"'
+        error_on_column_count_mismatch=false
+        SKIP_BLANK_LINES = TRUE; 

snowflake/scripts/training_interface/R__004_CREATE_TRAINING_DATA_TABLE_SP.sql

@@ -10,10 +10,10 @@ DECLARE
     cur_config cursor FOR 
         SELECT column_name, column_datatype FROM STG.TRAINING_DATA_COLUMN_CONFIG;
 BEGIN
-    
+    -- Creating the raw training data table with all columns as VARCHAR due to the dynamic nature of the columns and fields 
     OPEN cur_config;
     FOR rec IN cur_config DO
-        dynamic_ddl := dynamic_ddl || rec.column_name || '' '' || rec.column_datatype || '','';
+        dynamic_ddl := dynamic_ddl || rec.column_name || '' '' || ''VARCHAR'' || '','';
     END FOR;
 
     dynamic_ddl := LEFT(dynamic_ddl, LENGTH(dynamic_ddl) - 1);

snowflake/scripts/training_interface/R__006_LOAD_TRAINING_DATA_RAW_SP.sql

@@ -22,8 +22,8 @@ BEGIN
 
     call stg.log_audit(:procedure_name, 'Section 2', 99, 'START');
     
-    -- Truncate table as we are using the KILL & FILL approach
-    TRUNCATE TABLE STG.TRAINING_DATA_RAW;
+    -- Recreating the training data table by calling the SP. This will replace the existing table with new column definitions
+        call STG.CREATE_TRAINING_DATA_TABLE()
     
     call stg.log_audit(:procedure_name, 'Section 2', 99, 'END');
  
@@ -31,7 +31,7 @@ BEGIN
 
     -- Copy command to load data
     COPY INTO STG.TRAINING_DATA_RAW FROM @STG.RAW_TRAINING_DATA_STAGE
-        FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER')
+        FILE_FORMAT = (FORMAT_NAME = 'STG.TRAINING_DATA_FILE_FORMAT')
         ON_ERROR = ABORT_STATEMENT;
     
     call stg.log_audit(:procedure_name, 'Section 3', 99, 'END');