Updated init objects in Snowflake

2024-03-15 16:31:43 -05:00
parent e5d710bd7a
commit f8a255f743
5 changed files with 191 additions and 22 deletions
@@ -0,0 +1,69 @@
+CREATE OR REPLACE PROCEDURE STG.LOAD_REQUEST_SUBMISSION(file_name VARCHAR)
+RETURNS STRING
+LANGUAGE SQL
+EXECUTE AS CALLER
+AS
+$$
+DECLARE
+    procedure_name varchar;
+BEGIN
+    
+    procedure_name := 'LOAD_CONTRACT_CONFIG';
+
+    call stg.log_audit(:procedure_name, 'Section 1', 99, 'START');
+        
+    -- Create or replace stage with dynamic file name
+    EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.CONTRACT_CONFIG_STAGE
+    STORAGE_INTEGRATION = dev_bucket_integration
+    URL = ''s3://doczy-dev-infra-raw-data-ingestion/config_interface/' || :file_name || '''
+    FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');';
+
+    call stg.log_audit(:procedure_name, 'Section 1', 99, 'END');
+
+    call stg.log_audit(:procedure_name, 'Section 2', 99, 'START');
+    
+    COPY INTO STG.CONTRACT_CONFIG FROM (
+        SELECT 
+        NULLIF(TRIM($1),'') AS REQUEST_ID,
+        NULLIF(TRIM($2),'') AS CONTRACT_NAME,
+        NULLIF(TRIM($3),'') AS GROUP_1,
+        NULLIF(TRIM($4),'') AS GROUP_1_OVERRIDE,
+        NULLIF(TRIM($5),'') AS GROUP_2,
+        NULLIF(TRIM($6),'') AS GROUP_2_OVERRIDE,
+        NULLIF(TRIM($7),'') AS GROUP_3,
+        NULLIF(TRIM($8),'') AS GROUP_3_OVERRIDE,
+        NULLIF(TRIM($9),'') AS GROUP_4,
+        NULLIF(TRIM($10),'') AS GROUP_4_OVERRIDE,
+        NULLIF(TRIM($11),'') AS GROUP_5,
+        NULLIF(TRIM($12),'') AS GROUP_5_OVERRIDE,
+        NULLIF(TRIM($13),'') AS REQUEST_DATETIME,
+        NULLIF(TRIM($14),'') AS REQUEST_USER,
+        NULLIF(TRIM($15),'') AS OVERRIDE_DATETIME,
+        NULLIF(TRIM($16),'') AS LATEST_FLAG DEFAULT TRUE,
+        NULLIF(TRIM($17),'') AS PIPELINE_KICKOFF_DATETIME
+        FROM @STG.CONTRACT_CONFIG_STAGE
+    )
+    FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER')
+    ON_ERROR = ABORT_STATEMENT;
+    
+    call stg.log_audit(:procedure_name, 'Section 2', 99, 'END');
+ 
+    call stg.log_audit(:procedure_name, 'Section 3', 99, 'START');
+
+     INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT)
+    SELECT STG.AUDIT_SID.NEXTVAL, 'STG.CONTRACT_CONFIG',* 
+    FROM 
+    (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER)  from @STG.CONTRACT_CONFIG_STAGE group by 1,2);
+
+    
+
+    RETURN 'Setup, Load, and Audit Complete';
+    
+    call stg.log_audit(:procedure_name, 'Section 3', 99, 'END');
+
+
+
+   
+END;
+$$;
+
@@ -16,18 +16,18 @@ CREATE TABLE IF NOT EXISTS STG.TRAINING_RESULTS


 -- Create  file format
-CREATE FILE FORMAT IF NOT EXISTS STG.CSV_HEADER
-        TYPE = 'CSV'
-        FIELD_DELIMITER = ','
-        FILE_EXTENSION = '.csv'
-        RECORD_DELIMITER = '\\n'           
-        DATE_FORMAT = AUTO
-        TRIM_SPACE = TRUE
-        NULL_IF = ('NULL', '', 'N/A','?','~','\\N')
-        SKIP_HEADER = 1        
-        EMPTY_FIELD_AS_NULL = TRUE 
-        FIELD_OPTIONALLY_ENCLOSED_BY = '"'
-        SKIP_BLANK_LINES = TRUE;
+-- CREATE FILE FORMAT IF NOT EXISTS STG.CSV_HEADER
+--     TYPE = 'CSV'
+--         FIELD_DELIMITER = ','
+--         FILE_EXTENSION = '.csv'
+--         RECORD_DELIMITER = '\\n'           
+--         DATE_FORMAT = AUTO
+--         TRIM_SPACE = TRUE
+--         NULL_IF = ('NULL', '', 'N/A','?','~','\\N')
+--         SKIP_HEADER = 0        
+--         EMPTY_FIELD_AS_NULL = TRUE 
+--         FIELD_OPTIONALLY_ENCLOSED_BY = '"'
+--         SKIP_BLANK_LINES = TRUE;


 -- Create staging table for attempt logs
@@ -30,14 +30,14 @@ BEGIN
    call stg.log_audit(:procedure_name, 'Section 3', 99, 'START');

    -- Copy command to load data
-    COPY INTO STG.TRAINING_DATA_COLUMN_CONFIG FROM (
-                SELECT 
-                NULLIF(TRIM($1), '') AS COLUMN_NAME,
-                NULLIF(TRIM($2), '') AS COLUMN_DATATYPE
-                FROM @STG.RAW_TRAINING_DATA_STAGE
-        )
-        FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER')
-        ON_ERROR = ABORT_STATEMENT;
+    -- COPY INTO STG.TRAINING_DATA_COLUMN_CONFIG FROM (
+    --             SELECT 
+    --             NULLIF(TRIM($1), '') AS COLUMN_NAME,
+    --             NULLIF(TRIM($2), '') AS COLUMN_DATATYPE
+    --             FROM @STG.RAW_TRAINING_DATA_STAGE
+    --     )
+    --     FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER')
+    --     ON_ERROR = ABORT_STATEMENT;
    
    call stg.log_audit(:procedure_name, 'Section 3', 99, 'END');