66 lines
1.8 KiB
SQL
66 lines
1.8 KiB
SQL
|
-- Create staging table for results
|
||
|
|
CREATE TABLE IF NOT EXISTS STG.TRAINING_RESULTS
|
||
|
|
(
|
||
|
|
CONTRACT_NAME VARCHAR,
|
||
|
|
CONTRACT_ID NUMERIC,
|
||
|
|
ACTUAL_VALUE_STORED VARCHAR,
|
||
|
|
NEW_EXTRACTED_VALUE VARCHAR,
|
||
|
|
CONFIDENCE_LEVEL NUMERIC,
|
||
|
|
SNIPPET VARCHAR,
|
||
|
|
ORIGINAL_PAGE_NUMBER VARCHAR,
|
||
|
|
NEW_PAGE_NUMBER VARCHAR,
|
||
|
|
REVISED_PROMPT VARCHAR,
|
||
|
|
RESULT BOOLEAN,
|
||
|
|
LOAD_DT TIMESTAMP
|
||
|
|
);
|
||
|
|
|
||
|
|
|
||
|
|
-- Create file format
|
||
|
|
CREATE FILE FORMAT IF NOT EXISTS STG.CSV_HEADER
|
||
|
|
TYPE = 'CSV'
|
||
|
|
FIELD_DELIMITER = ','
|
||
|
|
FILE_EXTENSION = '.csv'
|
||
|
|
RECORD_DELIMITER = '\\n'
|
||
|
|
DATE_FORMAT = AUTO
|
||
|
|
TRIM_SPACE = TRUE
|
||
|
|
NULL_IF = ('NULL', '', 'N/A','?','~','\\N')
|
||
|
|
SKIP_HEADER = 1
|
||
|
|
EMPTY_FIELD_AS_NULL = TRUE
|
||
|
|
FIELD_OPTIONALLY_ENCLOSED_BY = '"'
|
||
|
|
SKIP_BLANK_LINES = TRUE;
|
||
|
|
|
||
|
|
|
||
|
|
-- Create staging table for attempt logs
|
||
|
|
CREATE TABLE IF NOT EXISTS STG.TRAINING_ATTEMPT_LOGS
|
||
|
|
(
|
||
|
|
FIELD_NAME VARCHAR,
|
||
|
|
CONTRACTS_TESTED NUMERIC,
|
||
|
|
USERNAME VARCHAR,
|
||
|
|
DATE_TIME TIMESTAMP,
|
||
|
|
ACCURACY NUMERIC,
|
||
|
|
ATTEMPT_NUM NUMERIC,
|
||
|
|
LOAD_DT TIMESTAMP
|
||
|
|
);
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
-- Create table for column config that will be used by SP to create raw training data table
|
||
|
|
CREATE TABLE IF NOT EXISTS STG.TRAINING_DATA_COLUMN_CONFIG(
|
||
|
|
COLUMN_NAME VARCHAR,
|
||
|
|
COLUMN_DATATYPE VARCHAR
|
||
|
|
);
|
||
|
|
|
||
|
|
-- Creating a separate file format for the training data
|
||
|
|
CREATE FILE FORMAT IF NOT EXISTS STG.TRAINING_DATA_FILE_FORMAT
|
||
|
|
TYPE = 'CSV'
|
||
|
|
FIELD_DELIMITER = ','
|
||
|
|
FILE_EXTENSION = '.csv'
|
||
|
|
RECORD_DELIMITER = '\\n'
|
||
|
|
DATE_FORMAT = AUTO
|
||
|
|
TRIM_SPACE = TRUE
|
||
|
|
NULL_IF = ('NULL', '', 'N/A','?','~','\\N')
|
||
|
|
SKIP_HEADER = 1
|
||
|
|
EMPTY_FIELD_AS_NULL = TRUE
|
||
|
|
FIELD_OPTIONALLY_ENCLOSED_BY = '"'
|
||
|
|
error_on_column_count_mismatch=false
|
||
|
|
SKIP_BLANK_LINES = TRUE;
|