jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in dev_umistry (pull request #8)

Browse Source 
Added log audit steps to SP
This commit is contained in:
Umang Mistry
2024-02-27 21:40:19 +00:00
parent 706c81427f df10aeff36
commit 958a929762
2 changed files with 30 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
snowflake/scripts/training_interface/R__002_LOAD_TRAINING_RESULTS_SP.SQL
+15 -3
View File
@@ -1,18 +1,25 @@
CREATE PROCEDURE IF NOT EXISTS LOAD_TRAINING_RESULTS(file_name VARCHAR)
CREATE PROCEDURE IF NOT EXISTS STG.LOAD_TRAINING_RESULTS(file_name VARCHAR)
RETURNS STRING
LANGUAGE SQL
EXECUTE AS CALLER
AS
$$
DECLARE
procedure_name varchar;
BEGIN
procedure_name := 'LOAD_TRAINING_RESULTS';
call stg.log_audit(procedure_name, 'Section 1', 99, 'START');
-- Create or replace stage with dynamic file name
EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.TRAINING_RESULTS_STAGE
STORAGE_INTEGRATION = dev_bucket_integration
URL = ''s3://doczy-dev-infra-raw-data-ingestion/training_interface/' || :file_name || '''
FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');';
call stg.log_audit(procedure_name, 'Section 1', 99, 'END');
call stg.log_audit(procedure_name, 'Section 2', 99, 'START');
-- Copy command to load data
COPY INTO STG.TRAINING_RESULTS FROM
(
@@ -33,12 +40,17 @@ BEGIN
FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER')
ON_ERROR = CONTINUE;
call stg.log_audit(procedure_name, 'Section 2', 99, 'END');
call stg.log_audit(procedure_name, 'Section 3', 99, 'START');
INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT)
SELECT STG.AUDIT_SID.NEXTVAL, 'STG.TRAINING_RESULTS',*
FROM
(SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @DOCZY_DEV.STG.TRAINING_RESULTS_STAGE group by 1,2);
call stg.log_audit(procedure_name, 'Section 3', 99, 'END');
RETURN 'Setup, Load, and Audit Complete';
END;
snowflake/scripts/training_interface/R__003_LOAD_ATTEMPTS_SP.sql
+15 -3
View File
@@ -4,14 +4,22 @@ LANGUAGE SQL
EXECUTE AS CALLER
AS
$$
DECLARE
prcedure_name varchar;
BEGIN
procedure_name := 'LOAD_ATTEMPT_LOGS';
call stg.log_audit(procedure_name, 'Section 1', 99, 'START');
-- Create or replace stage with dynamic file name
EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.ATTEMPT_LOGS_STAGE
STORAGE_INTEGRATION = dev_bucket_integration
URL = ''s3://doczy-dev-infra-raw-data-ingestion/training_interface/' || :file_name || '''
FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');';
call stg.log_audit(procedure_name, 'Section 1', 99, 'END');
call stg.log_audit(procedure_name, 'Section 2', 99, 'START');
-- Copy command to load data
COPY INTO STG.TRAINING_ATTEMPT_LOGS FROM
@@ -29,13 +37,17 @@ BEGIN
FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER')
ON_ERROR = CONTINUE;
call stg.log_audit(procedure_name, 'Section 2', 99, 'END');
call stg.log_audit(procedure_name, 'Section 3', 99, 'START');
INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT)
SELECT STG.AUDIT_SID.NEXTVAL, 'STG.TRAINING_ATTEMPT_LOGS',*
FROM
(SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @DOCZY_DEV.STG.TRAINING_RESULTS_STAGE group by 1,2);
call stg.log_audit(procedure_name, 'Section 3', 99, 'END');
RETURN 'Setup, Load, and Audit Complete';
END;
$$;
$$;