Merged in dev_umistry (pull request #7)

Added training I/F tables & SP
2024-02-27 20:30:32 +00:00
parent 360ed5ccb2 7440778c6b
commit 706c81427f
4 changed files with 132 additions and 0 deletions
@@ -0,0 +1,46 @@
+-- Create staging table for results
+CREATE TABLE IF NOT EXISTS STG.TRAINING_RESULTS 
+(
+    CONTRACT_NAME VARCHAR,
+    CONTRACT_ID NUMERIC,
+    ACTUAL_VALUE_STORED VARCHAR,
+    NEW_EXTRACTED_VALUE VARCHAR,
+    CONFIDENCE_LEVEL NUMERIC,
+    SNIPPET VARCHAR,
+    ORIGINAL_PAGE_NUMBER VARCHAR,
+    NEW_PAGE_NUMBER VARCHAR,
+    REVISED_PROMPT VARCHAR,
+    RESULT BOOLEAN,
+    LOAD_DT TIMESTAMP
+);
+
+
+ -- Create  file format
+CREATE FILE FORMAT IF NOT EXISTS STG.CSV_HEADER
+        TYPE = 'CSV'
+        FIELD_DELIMITER = ','
+        FILE_EXTENSION = '.csv'
+        RECORD_DELIMITER = '\\n'           
+        DATE_FORMAT = AUTO
+        TRIM_SPACE = TRUE
+        NULL_IF = ('NULL', '', 'N/A','?','~')
+        SKIP_HEADER = 1        
+        EMPTY_FIELD_AS_NULL = TRUE 
+        FIELD_OPTIONALLY_ENCLOSED_BY = '"'
+        SKIP_BLANK_LINES = TRUE;
+
+
+-- Create staging table for attempt logs
+
+CREATE TABLE IF NOT EXISTS STG.TRAINING_ATTEMPT_LOGS 
+(
+    FIELD_NAME VARCHAR,
+    CONTRACTS_TESTED NUMERIC,
+    USERNAME VARCHAR,
+    DATE_TIME TIMESTAMP,
+    ACCURACY NUMERIC,
+    ATTEMPT_NUM NUMERIC,
+    LOAD_DT TIMESTAMP
+);
+
+
@@ -0,0 +1,45 @@
+CREATE PROCEDURE IF NOT EXISTS LOAD_TRAINING_RESULTS(file_name VARCHAR)
+RETURNS STRING
+LANGUAGE SQL
+EXECUTE AS CALLER
+AS
+$$
+BEGIN
+    
+    
+    -- Create or replace stage with dynamic file name
+    EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.TRAINING_RESULTS_STAGE
+    STORAGE_INTEGRATION = dev_bucket_integration
+    URL = ''s3://doczy-dev-infra-raw-data-ingestion/training_interface/' || :file_name || '''
+    FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');';
+    
+    -- Copy command to load data
+    COPY INTO STG.TRAINING_RESULTS FROM 
+    (
+        SELECT 
+            NULLIF(TRIM($1), '') AS Contract_Name,
+            NULLIF(TRIM($2), '') AS Contract_ID,
+            NULLIF(TRIM($3), '') AS Actual_Value_Stored,
+            NULLIF(TRIM($4), '') AS New_Extracted_Value,
+            NULLIF(TRIM($5), '') AS Confidence_Level,
+            NULLIF(TRIM($6), '') AS Snippet,
+            NULLIF(TRIM($7), '') AS Original_Page_Number,
+            NULLIF(TRIM($8), '') AS New_Page_Number,
+            NULLIF(TRIM($9), '') AS Revised_Prompt,
+            IFF(TRIM($10) = 'TRUE', TRUE, FALSE) AS Result,
+            CURRENT_TIMESTAMP(2) AS LOAD_DT
+        FROM @STG.TRAINING_RESULTS_STAGE
+    )
+    FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER')
+    ON_ERROR = CONTINUE;
+
+    INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT)
+    SELECT STG.AUDIT_SID.NEXTVAL, 'STG.TRAINING_RESULTS',* 
+    FROM 
+    (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER)  from @DOCZY_DEV.STG.TRAINING_RESULTS_STAGE group by 1,2);
+    
+    
+
+    RETURN 'Setup, Load, and Audit Complete';
+END;
+$$;
@@ -0,0 +1,41 @@
+CREATE PROCEDURE IF NOT EXISTS STG.LOAD_ATTEMPT_LOGS(file_name VARCHAR)
+RETURNS STRING
+LANGUAGE SQL
+EXECUTE AS CALLER
+AS
+$$
+BEGIN
+    
+    
+    -- Create or replace stage with dynamic file name
+    EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.ATTEMPT_LOGS_STAGE
+    STORAGE_INTEGRATION = dev_bucket_integration
+    URL = ''s3://doczy-dev-infra-raw-data-ingestion/training_interface/' || :file_name || '''
+    FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');';
+    
+    -- Copy command to load data
+    COPY INTO STG.TRAINING_ATTEMPT_LOGS FROM 
+    (
+        SELECT 
+            NULLIF(TRIM($1), '') AS FIELD_NAME,
+            NULLIF(TRIM($2), '') AS CONTRACTS_TESTED,
+            NULLIF(TRIM($3), '') AS USERNAME,
+            NULLIF(TRIM($4), '') AS DATE_TIME,
+            NULLIF(TRIM($5), '') AS ACCURACY,
+            NULLIF(TRIM($6), '') AS ATTEMPT_NUM,
+            CURRENT_TIMESTAMP(2) AS LOAD_DT
+        FROM @STG.ATTEMPT_LOGS_STAGE
+    )
+    FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER')
+    ON_ERROR = CONTINUE;
+
+    INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT)
+    SELECT STG.AUDIT_SID.NEXTVAL, 'STG.TRAINING_ATTEMPT_LOGS',* 
+    FROM 
+    (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER)  from @DOCZY_DEV.STG.TRAINING_RESULTS_STAGE group by 1,2);
+    
+    
+
+    RETURN 'Setup, Load, and Audit Complete';
+END;
+$$;