diff --git a/snowflake/scripts/config_interface/R__001_PROMPT_CONFIG_TABLE.sql b/snowflake/scripts/config_interface/R__001_PROMPT_CONFIG_TABLE.sql index 1e927d7..b94878b 100644 --- a/snowflake/scripts/config_interface/R__001_PROMPT_CONFIG_TABLE.sql +++ b/snowflake/scripts/config_interface/R__001_PROMPT_CONFIG_TABLE.sql @@ -1,5 +1,5 @@ -- Separation of single vs multi prompt using a flag in the table -CREATE TABLE STG.PROMPT_CONFIG ( +CREATE TABLE IF NOT EXISTS STG.PROMPT_CONFIG ( FIELD_NAME VARCHAR, FIELD_DESC VARCHAR, IS_REQUIRED BOOLEAN, diff --git a/snowflake/scripts/config_interface/R__002_CONTRACT_CONFIG_TABLE.sql b/snowflake/scripts/config_interface/R__002_CONTRACT_CONFIG_TABLE.sql index 2889556..eba2a95 100644 --- a/snowflake/scripts/config_interface/R__002_CONTRACT_CONFIG_TABLE.sql +++ b/snowflake/scripts/config_interface/R__002_CONTRACT_CONFIG_TABLE.sql @@ -3,27 +3,14 @@ CREATE TABLE IF NOT EXISTS STG.CONTRACT_CONFIG ( CONTRACT_NAME VARCHAR, BATCH_ID VARCHAR, UNIQUE_KEY BOOLEAN, - UNIQUE_KEY_OVERRIDE BOOLEAN DEFAULT FALSE, PRICING_BEFORE_CARVEOUTS BOOLEAN, - PRICING_BEFORE_CARVEOUTS_OVERRIDE BOOLEAN DEFAULT FALSE, CONTRACT_RELATED BOOLEAN, - CONTRACT_RELATED_OVERRIDE BOOLEAN DEFAULT FALSE, PROVIDER BOOLEAN, - PROVIDER_OVERRIDE BOOLEAN DEFAULT FALSE, TIMELINE BOOLEAN, - TIMELINE_OVERRIDE BOOLEAN DEFAULT FALSE, CARVEOUT_INDICATOR BOOLEAN, - CARVEOUT_INDICATOR_OVERRIDE BOOLEAN DEFAULT FALSE, CARVEOUT_METHODOLOGY BOOLEAN - CARVEOUT_METHODOLOGY_OVERRIDE BOOLEAN DEFAULT FALSE, - REQUEST_DATETIME DATETIME, REQUEST_USER VARCHAR, - OVERRIDE_DATETIME DATETIME, LATEST_FLAG BOOLEAN DEFAULT TRUE, PIPELINE_KICKOFF_DATETIME DATETIME, - REQUEST_DATETIME DATETIME DEFAULT CURRENT_TIMESTAMP(), - REQUEST_USER VARCHAR, - OVERRIDE_DATETIME DATETIME DEFAULT NULL, - LATEST_FLAG BOOLEAN DEFAULT TRUE, - PIPELINE_KICKOFF_DATETIME DATETIME + REQUEST_DATETIME DATETIME DEFAULT CURRENT_TIMESTAMP() ); \ No newline at end of file diff --git a/snowflake/scripts/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql b/snowflake/scripts/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql index 223b7aa..802c843 100644 --- a/snowflake/scripts/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql +++ b/snowflake/scripts/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql @@ -23,32 +23,19 @@ BEGIN call stg.log_audit(:procedure_name, 'Section 2', 99, 'START'); COPY INTO STG.CONTRACT_CONFIG FROM ( SELECT - NULLIF(TRIM($2),'') AS CONTRACT_NAME, - NULLIF(TRIM($3),'') AS BATCH_ID, - NULLIF(TRIM($4),'') AS UNIQUE_KEY, - NULLIF(TRIM($5),'') AS UNIQUE_KEY_OVERRIDE, - NULLIF(TRIM($6),'') AS PRICING_BEFORE_CARVEOUTS, - NULLIF(TRIM($7),'') AS PRICING_BEFORE_CARVEOUTS_OVERRIDE, - NULLIF(TRIM($8),'') AS CONTRACT_RELATED, - NULLIF(TRIM($9),'') AS CONTRACT_RELATED_OVERRIDE, - NULLIF(TRIM($10),'') AS PROVIDER, - NULLIF(TRIM($11),'') AS PROVIDER_OVERRIDE, - NULLIF(TRIM($12),'') AS TIMELINE, - NULLIF(TRIM($13),'') AS TIMELINE_OVERRIDE, - NULLIF(TRIM($14),'') AS CARVEOUT_INDICATOR, - NULLIF(TRIM($15),'') AS CARVEOUT_INDICATOR_OVERRIDE, - NULLIF(TRIM($16),'') AS CARVEOUT_METHODOLOGY, - NULLIF(TRIM($17),'') AS CARVEOUT_METHODOLOGY_OVERRIDE, - NULLIF(TRIM($18),'') AS REQUEST_DATETIME, - NULLIF(TRIM($19),'') AS REQUEST_USER, - NULLIF(TRIM($20),'') AS OVERRIDE_DATETIME, - NULLIF(TRIM($21),'') AS LATEST_FLAG, - NULLIF(TRIM($22),'') AS PIPELINE_KICKOFF_DATETIME - NULLIF(TRIM($23),'') AS REQUEST_DATETIME, - NULLIF(TRIM($24),'') AS REQUEST_USER, - NULLIF(TRIM($25),'') AS OVERRIDE_DATETIME, - NULLIF(TRIM($26),'') AS LATEST_FLAG, - NULLIF(TRIM($27),'') AS PIPELINE_KICKOFF_DATETIME + NULLIF(TRIM($1),'') AS CONTRACT_NAME, + NULLIF(TRIM($2),'') AS BATCH_ID + NULLIF(TRIM($3),'') AS UNIQUE_KEY, + NULLIF(TRIM($4),'') AS PRICING_BEFORE_CARVEOUTS, + NULLIF(TRIM($5),'') AS CONTRACT_RELATED, + NULLIF(TRIM($6),'') AS PROVIDER, + NULLIF(TRIM($7),'') AS TIMELINE, + NULLIF(TRIM($8),'') AS CARVEOUT_INDICATOR, + NULLIF(TRIM($9),'') AS CARVEOUT_METHODOLOGY, + NULLIF(TRIM($10),'') AS REQUEST_USER, + NULLIF(TRIM($11),'') AS LATEST_FLAG, + NULLIF(TRIM($12),'') AS PIPELINE_KICKOFF_DATETIME, + NULLIF(TRIM($13),'') AS REQUEST_DATETIME FROM @STG.CONTRACT_CONFIG_STAGE ) FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER') @@ -58,19 +45,27 @@ BEGIN call stg.log_audit(:procedure_name, 'Section 3', 99, 'START'); + -- Once the new data is loaded to the stage, we need to update the existing records in the table to set the LATEST_FLAG to FALSE + + contract_name := (SELECT NULLIF(TRIM($1),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1); + batch_id := (SELECT NULLIF(TRIM($2),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1); + request_datetime := (SELECT NULLIF(TRIM($13),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1); + + UPDATE TABLE STG.CONTRACT_CONFIG + SET LATEST_FLAG = FALSE + WHERE BATCH_ID = :batch_id AND CONTRACT_NAME = :contract_name AND REQUEST_DATETIME < :request_datetime AND LATEST_FLAG = TRUE; + + call stg.log_audit(:procedure_name, 'Section 3', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 4', 99, 'START'); INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) SELECT STG.AUDIT_SID.NEXTVAL, :procedure_name,* FROM (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.CONTRACT_CONFIG_STAGE group by 1,2); - - RETURN 'Setup, Load, and Audit Complete'; - - call stg.log_audit(:procedure_name, 'Section 3', 99, 'END'); - - + call stg.log_audit(:procedure_name, 'Section 4', 99, 'END'); END; $$; diff --git a/snowflake/scripts/upload_interface/R__003_CONTRACT_UPLOAD_LOGS.sql b/snowflake/scripts/upload_interface/R__003_CONTRACT_UPLOAD_LOGS.sql new file mode 100644 index 0000000..ac7126e --- /dev/null +++ b/snowflake/scripts/upload_interface/R__003_CONTRACT_UPLOAD_LOGS.sql @@ -0,0 +1,10 @@ +-- This table is used to store the logs of all the files that were uploaded by the user to S3 +-- Following fields are to be stored in the table: +-- Batch id & client name & file name, date time, user +CREATE TABLE IF NOT EXISTS STG.CONTRACT_UPLOAD_LOGS ( + BATCH_ID VARCHAR, + CLIENT_NAME VARCHAR, + FILE_NAME VARCHAR, + UPLOAD_DATETIME DATETIME DEFAULT CURRENT_TIMESTAMP(), + UPLOAD_USER VARCHAR +); \ No newline at end of file