jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Added UI 1 tables and SPs

Browse Source
This commit is contained in:
Umang Mistry
2024-03-14 14:23:20 -05:00
parent f65dcf304f
commit e5d710bd7a
6 changed files with 218 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
snowflake/scripts/config_interface/R__002_CONTRACT_CONFIG_TABLE.sql
+1 -1
View File
@@ -1,5 +1,5 @@
-- CREATING THE CONFIG TABLE
CREATE TABLE IF NOT EXISTS CONTRACT_CONFIG (
CREATE TABLE IF NOT EXISTS STG.CONTRACT_CONFIG (
REQUEST_ID NUMERIC,
CONTRACT_NAME VARCHAR,
GROUP_1 BOOLEAN,
snowflake/scripts/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql
+69
View File
@@ -0,0 +1,69 @@
CREATE OR REPLACE PROCEDURE STG.LOAD_CONTRACT_CONFIG(file_name VARCHAR)
RETURNS STRING
LANGUAGE SQL
EXECUTE AS CALLER
AS
$$
DECLARE
procedure_name varchar;
BEGIN
procedure_name := 'LOAD_CONTRACT_CONFIG';
call stg.log_audit(:procedure_name, 'Section 1', 99, 'START');
-- Create or replace stage with dynamic file name
EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.CONTRACT_CONFIG_STAGE
STORAGE_INTEGRATION = dev_bucket_integration
URL = ''s3://doczy-dev-infra-raw-data-ingestion/config_interface/' || :file_name || '''
FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');';
call stg.log_audit(:procedure_name, 'Section 1', 99, 'END');
call stg.log_audit(:procedure_name, 'Section 2', 99, 'START');
COPY INTO STG.CONTRACT_CONFIG FROM (
SELECT
NULLIF(TRIM($1),'') AS REQUEST_ID,
NULLIF(TRIM($2),'') AS CONTRACT_NAME,
NULLIF(TRIM($3),'') AS GROUP_1,
NULLIF(TRIM($4),'') AS GROUP_1_OVERRIDE,
NULLIF(TRIM($5),'') AS GROUP_2,
NULLIF(TRIM($6),'') AS GROUP_2_OVERRIDE,
NULLIF(TRIM($7),'') AS GROUP_3,
NULLIF(TRIM($8),'') AS GROUP_3_OVERRIDE,
NULLIF(TRIM($9),'') AS GROUP_4,
NULLIF(TRIM($10),'') AS GROUP_4_OVERRIDE,
NULLIF(TRIM($11),'') AS GROUP_5,
NULLIF(TRIM($12),'') AS GROUP_5_OVERRIDE,
NULLIF(TRIM($13),'') AS REQUEST_DATETIME,
NULLIF(TRIM($14),'') AS REQUEST_USER,
NULLIF(TRIM($15),'') AS OVERRIDE_DATETIME,
NULLIF(TRIM($16),'') AS LATEST_FLAG DEFAULT TRUE,
NULLIF(TRIM($17),'') AS PIPELINE_KICKOFF_DATETIME
FROM @STG.CONTRACT_CONFIG_STAGE
)
FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER')
ON_ERROR = ABORT_STATEMENT;
call stg.log_audit(:procedure_name, 'Section 2', 99, 'END');
call stg.log_audit(:procedure_name, 'Section 3', 99, 'START');
INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT)
SELECT STG.AUDIT_SID.NEXTVAL, 'STG.CONTRACT_CONFIG',*
FROM
(SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.CONTRACT_CONFIG_STAGE group by 1,2);
RETURN 'Setup, Load, and Audit Complete';
call stg.log_audit(:procedure_name, 'Section 3', 99, 'END');
END;
$$;
snowflake/scripts/training_interface/R__001_INIT_OBJS.sql
+8 -2
View File
@@ -23,7 +23,7 @@ CREATE FILE FORMAT IF NOT EXISTS STG.CSV_HEADER
RECORD_DELIMITER = '\\n'
DATE_FORMAT = AUTO
TRIM_SPACE = TRUE
NULL_IF = ('NULL', '', 'N/A','?','~')
NULL_IF = ('NULL', '', 'N/A','?','~','\\N')
SKIP_HEADER = 1
EMPTY_FIELD_AS_NULL = TRUE
FIELD_OPTIONALLY_ENCLOSED_BY = '"'
@@ -31,7 +31,6 @@ CREATE FILE FORMAT IF NOT EXISTS STG.CSV_HEADER
-- Create staging table for attempt logs
CREATE TABLE IF NOT EXISTS STG.TRAINING_ATTEMPT_LOGS
(
FIELD_NAME VARCHAR,
@@ -44,3 +43,10 @@ CREATE TABLE IF NOT EXISTS STG.TRAINING_ATTEMPT_LOGS
);
-- Create table for column config that will be used by SP to create raw training data table
CREATE TABLE IF NOT EXISTS STG.TRAINING_DATA_COLUMN_CONFIG(
COLUMN_NAME VARCHAR,
COLUMN_DATATYPE VARCHAR
);
-- Raw training data table is created dynamically with the help of stored procedure
snowflake/scripts/training_interface/R__004_CREATE_TRAINING_DATA_TABLE_SP.sql
+28
View File
@@ -0,0 +1,28 @@
CREATE OR REPLACE PROCEDURE STG.CREATE_TRAINING_DATA_TABLE()
RETURNS VARCHAR(16777216)
LANGUAGE SQL
EXECUTE AS CALLER
AS '
DECLARE
dynamic_ddl STRING := ''CREATE OR REPLACE TABLE STG.TRAINING_DATA_RAW ('';
column_details RESULTSET;
first_column BOOLEAN := TRUE;
cur_config cursor FOR
SELECT column_name, column_datatype FROM STG.TRAINING_DATA_COLUMN_CONFIG;
BEGIN
OPEN cur_config;
FOR rec IN cur_config DO
dynamic_ddl := dynamic_ddl || rec.column_name || '' '' || rec.column_datatype || '','';
END FOR;
dynamic_ddl := LEFT(dynamic_ddl, LENGTH(dynamic_ddl) - 1);
dynamic_ddl := dynamic_ddl || '');'';
EXECUTE IMMEDIATE dynamic_ddl;
RETURN dynamic_ddl;
END;
';
snowflake/scripts/training_interface/R__005_LOAD_COLUMN_CONFIG_SP.sql
+56
View File
@@ -0,0 +1,56 @@
CREATE OR REPLACE PROCEDURE STG.LOAD_COLUMN_CONFIG(file_name VARCHAR)
RETURNS STRING
LANGUAGE SQL
EXECUTE AS CALLER
AS
$$
DECLARE
procedure_name varchar;
BEGIN
procedure_name := 'LOAD_COLUMN_CONFIG';
call stg.log_audit(:procedure_name, 'Section 1', 99, 'START');
-- Create or replace stage with dynamic file name
EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.COLUMN_CONFIG_STAGE
STORAGE_INTEGRATION = dev_bucket_integration
URL = ''s3://doczy-dev-infra-raw-data-ingestion/training_data_raw/' || :file_name || '''
FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');';
call stg.log_audit(:procedure_name, 'Section 1', 99, 'END');
call stg.log_audit(:procedure_name, 'Section 2', 99, 'START');
-- Truncate table as we are using the KILL & FILL approach
TRUNCATE TABLE STG.TRAINING_DATA_COLUMN_CONFIG;
call stg.log_audit(:procedure_name, 'Section 2', 99, 'END');
call stg.log_audit(:procedure_name, 'Section 3', 99, 'START');
-- Copy command to load data
COPY INTO STG.TRAINING_DATA_COLUMN_CONFIG FROM (
SELECT
NULLIF(TRIM($1), '') AS COLUMN_NAME,
NULLIF(TRIM($2), '') AS COLUMN_DATATYPE
FROM @STG.COLUMN_CONFIG_STAGE
)
FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER')
ON_ERROR = ABORT_STATEMENT;
call stg.log_audit(:procedure_name, 'Section 3', 99, 'END');
call stg.log_audit(:procedure_name, 'Section 4', 99, 'START');
INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT)
SELECT STG.AUDIT_SID.NEXTVAL, 'STG.TRAINING_DATA_COLUMN_CONFIG',*
FROM
(SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.COLUMN_CONFIG_STAGE group by 1,2);
call stg.log_audit(:procedure_name, 'Section 4', 99, 'END');
RETURN 'Setup, Load, and Audit Complete';
END;
$$;
snowflake/scripts/training_interface/R__006_LOAD_TRAINING_DATA_RAW_SP.sql
+56
View File
@@ -0,0 +1,56 @@
CREATE OR REPLACE PROCEDURE STG.LOAD_TRAINING_DATA_RAW(file_name VARCHAR)
RETURNS STRING
LANGUAGE SQL
EXECUTE AS CALLER
AS
$$
DECLARE
procedure_name varchar;
BEGIN
procedure_name := 'LOAD_TRAINING_DATA_RAW';
call stg.log_audit(:procedure_name, 'Section 1', 99, 'START');
-- Create or replace stage with dynamic file name
EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.RAW_TRAINING_DATA_STAGE
STORAGE_INTEGRATION = dev_bucket_integration
URL = ''s3://doczy-dev-infra-raw-data-ingestion/training_data_raw/' || :file_name || '''
FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');';
call stg.log_audit(:procedure_name, 'Section 1', 99, 'END');
call stg.log_audit(:procedure_name, 'Section 2', 99, 'START');
-- Truncate table as we are using the KILL & FILL approach
TRUNCATE TABLE STG.TRAINING_DATA_RAW;
call stg.log_audit(:procedure_name, 'Section 2', 99, 'END');
call stg.log_audit(:procedure_name, 'Section 3', 99, 'START');
-- Copy command to load data
COPY INTO STG.TRAINING_DATA_COLUMN_CONFIG FROM (
SELECT
NULLIF(TRIM($1), '') AS COLUMN_NAME,
NULLIF(TRIM($2), '') AS COLUMN_DATATYPE
FROM @STG.RAW_TRAINING_DATA_STAGE
)
FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER')
ON_ERROR = ABORT_STATEMENT;
call stg.log_audit(:procedure_name, 'Section 3', 99, 'END');
call stg.log_audit(:procedure_name, 'Section 4', 99, 'START');
INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT)
SELECT STG.AUDIT_SID.NEXTVAL, 'STG.TRAINING_DATA_RAW',*
FROM
(SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.RAW_TRAINING_DATA_STAGE group by 1,2);
call stg.log_audit(:procedure_name, 'Section 4', 99, 'END');
RETURN 'Setup, Load, and Audit Complete';
END;
$$;