From 7440778c6b6cdc253cca15e8a2e3dda71b673f6b Mon Sep 17 00:00:00 2001 From: Umang Mistry Date: Tue, 27 Feb 2024 14:29:22 -0600 Subject: [PATCH] Added training I/F tables & SP --- snowflake/scripts/placeholder.sql | 0 .../training_interface/R__001_INIT_OBJS.sql | 46 +++++++++++++++++++ .../R__002_LOAD_TRAINING_RESULTS_SP.SQL | 45 ++++++++++++++++++ .../R__003_LOAD_ATTEMPTS_SP.sql | 41 +++++++++++++++++ 4 files changed, 132 insertions(+) delete mode 100644 snowflake/scripts/placeholder.sql create mode 100644 snowflake/scripts/training_interface/R__001_INIT_OBJS.sql create mode 100644 snowflake/scripts/training_interface/R__002_LOAD_TRAINING_RESULTS_SP.SQL create mode 100644 snowflake/scripts/training_interface/R__003_LOAD_ATTEMPTS_SP.sql diff --git a/snowflake/scripts/placeholder.sql b/snowflake/scripts/placeholder.sql deleted file mode 100644 index e69de29..0000000 diff --git a/snowflake/scripts/training_interface/R__001_INIT_OBJS.sql b/snowflake/scripts/training_interface/R__001_INIT_OBJS.sql new file mode 100644 index 0000000..7b7bb3c --- /dev/null +++ b/snowflake/scripts/training_interface/R__001_INIT_OBJS.sql @@ -0,0 +1,46 @@ +-- Create staging table for results +CREATE TABLE IF NOT EXISTS STG.TRAINING_RESULTS +( + CONTRACT_NAME VARCHAR, + CONTRACT_ID NUMERIC, + ACTUAL_VALUE_STORED VARCHAR, + NEW_EXTRACTED_VALUE VARCHAR, + CONFIDENCE_LEVEL NUMERIC, + SNIPPET VARCHAR, + ORIGINAL_PAGE_NUMBER VARCHAR, + NEW_PAGE_NUMBER VARCHAR, + REVISED_PROMPT VARCHAR, + RESULT BOOLEAN, + LOAD_DT TIMESTAMP +); + + + -- Create file format +CREATE FILE FORMAT IF NOT EXISTS STG.CSV_HEADER + TYPE = 'CSV' + FIELD_DELIMITER = ',' + FILE_EXTENSION = '.csv' + RECORD_DELIMITER = '\\n' + DATE_FORMAT = AUTO + TRIM_SPACE = TRUE + NULL_IF = ('NULL', '', 'N/A','?','~') + SKIP_HEADER = 1 + EMPTY_FIELD_AS_NULL = TRUE + FIELD_OPTIONALLY_ENCLOSED_BY = '"' + SKIP_BLANK_LINES = TRUE; + + +-- Create staging table for attempt logs + +CREATE TABLE IF NOT EXISTS STG.TRAINING_ATTEMPT_LOGS +( + FIELD_NAME VARCHAR, + CONTRACTS_TESTED NUMERIC, + USERNAME VARCHAR, + DATE_TIME TIMESTAMP, + ACCURACY NUMERIC, + ATTEMPT_NUM NUMERIC, + LOAD_DT TIMESTAMP +); + + diff --git a/snowflake/scripts/training_interface/R__002_LOAD_TRAINING_RESULTS_SP.SQL b/snowflake/scripts/training_interface/R__002_LOAD_TRAINING_RESULTS_SP.SQL new file mode 100644 index 0000000..c40a99e --- /dev/null +++ b/snowflake/scripts/training_interface/R__002_LOAD_TRAINING_RESULTS_SP.SQL @@ -0,0 +1,45 @@ +CREATE PROCEDURE IF NOT EXISTS LOAD_TRAINING_RESULTS(file_name VARCHAR) +RETURNS STRING +LANGUAGE SQL +EXECUTE AS CALLER +AS +$$ +BEGIN + + + -- Create or replace stage with dynamic file name + EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.TRAINING_RESULTS_STAGE + STORAGE_INTEGRATION = dev_bucket_integration + URL = ''s3://doczy-dev-infra-raw-data-ingestion/training_interface/' || :file_name || ''' + FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');'; + + -- Copy command to load data + COPY INTO STG.TRAINING_RESULTS FROM + ( + SELECT + NULLIF(TRIM($1), '') AS Contract_Name, + NULLIF(TRIM($2), '') AS Contract_ID, + NULLIF(TRIM($3), '') AS Actual_Value_Stored, + NULLIF(TRIM($4), '') AS New_Extracted_Value, + NULLIF(TRIM($5), '') AS Confidence_Level, + NULLIF(TRIM($6), '') AS Snippet, + NULLIF(TRIM($7), '') AS Original_Page_Number, + NULLIF(TRIM($8), '') AS New_Page_Number, + NULLIF(TRIM($9), '') AS Revised_Prompt, + IFF(TRIM($10) = 'TRUE', TRUE, FALSE) AS Result, + CURRENT_TIMESTAMP(2) AS LOAD_DT + FROM @STG.TRAINING_RESULTS_STAGE + ) + FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER') + ON_ERROR = CONTINUE; + + INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) + SELECT STG.AUDIT_SID.NEXTVAL, 'STG.TRAINING_RESULTS',* + FROM + (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @DOCZY_DEV.STG.TRAINING_RESULTS_STAGE group by 1,2); + + + + RETURN 'Setup, Load, and Audit Complete'; +END; +$$; \ No newline at end of file diff --git a/snowflake/scripts/training_interface/R__003_LOAD_ATTEMPTS_SP.sql b/snowflake/scripts/training_interface/R__003_LOAD_ATTEMPTS_SP.sql new file mode 100644 index 0000000..798f18a --- /dev/null +++ b/snowflake/scripts/training_interface/R__003_LOAD_ATTEMPTS_SP.sql @@ -0,0 +1,41 @@ +CREATE PROCEDURE IF NOT EXISTS STG.LOAD_ATTEMPT_LOGS(file_name VARCHAR) +RETURNS STRING +LANGUAGE SQL +EXECUTE AS CALLER +AS +$$ +BEGIN + + + -- Create or replace stage with dynamic file name + EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.ATTEMPT_LOGS_STAGE + STORAGE_INTEGRATION = dev_bucket_integration + URL = ''s3://doczy-dev-infra-raw-data-ingestion/training_interface/' || :file_name || ''' + FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');'; + + -- Copy command to load data + COPY INTO STG.TRAINING_ATTEMPT_LOGS FROM + ( + SELECT + NULLIF(TRIM($1), '') AS FIELD_NAME, + NULLIF(TRIM($2), '') AS CONTRACTS_TESTED, + NULLIF(TRIM($3), '') AS USERNAME, + NULLIF(TRIM($4), '') AS DATE_TIME, + NULLIF(TRIM($5), '') AS ACCURACY, + NULLIF(TRIM($6), '') AS ATTEMPT_NUM, + CURRENT_TIMESTAMP(2) AS LOAD_DT + FROM @STG.ATTEMPT_LOGS_STAGE + ) + FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER') + ON_ERROR = CONTINUE; + + INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) + SELECT STG.AUDIT_SID.NEXTVAL, 'STG.TRAINING_ATTEMPT_LOGS',* + FROM + (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @DOCZY_DEV.STG.TRAINING_RESULTS_STAGE group by 1,2); + + + + RETURN 'Setup, Load, and Audit Complete'; +END; +$$; \ No newline at end of file