diff --git a/snowflake/scripts/config_interface/R__001_PROMPT_CONFIG_TABLE.sql b/snowflake/scripts/config_interface/R__001_PROMPT_CONFIG_TABLE.sql index f4890fb..1e927d7 100644 --- a/snowflake/scripts/config_interface/R__001_PROMPT_CONFIG_TABLE.sql +++ b/snowflake/scripts/config_interface/R__001_PROMPT_CONFIG_TABLE.sql @@ -1,3 +1,4 @@ +-- Separation of single vs multi prompt using a flag in the table CREATE TABLE STG.PROMPT_CONFIG ( FIELD_NAME VARCHAR, FIELD_DESC VARCHAR, @@ -9,4 +10,4 @@ CREATE TABLE STG.PROMPT_CONFIG ( FIELD_MAX_LENGTH NUMBER, SAMPLE_VALUE VARCHAR, CONTRACT_SECTION VARCHAR -); +); diff --git a/snowflake/scripts/config_interface/R__002_CONTRACT_CONFIG_TABLE.sql b/snowflake/scripts/config_interface/R__002_CONTRACT_CONFIG_TABLE.sql index e963e50..2889556 100644 --- a/snowflake/scripts/config_interface/R__002_CONTRACT_CONFIG_TABLE.sql +++ b/snowflake/scripts/config_interface/R__002_CONTRACT_CONFIG_TABLE.sql @@ -1,20 +1,29 @@ -- CREATING THE CONFIG TABLE CREATE TABLE IF NOT EXISTS STG.CONTRACT_CONFIG ( - REQUEST_ID NUMERIC, CONTRACT_NAME VARCHAR, - GROUP_1 BOOLEAN, - GROUP_1_OVERRIDE BOOLEAN, - GROUP_2 BOOLEAN, - GROUP_2_OVERRIDE BOOLEAN, - GROUP_3 BOOLEAN, - GROUP_3_OVERRIDE BOOLEAN, - GROUP_4 BOOLEAN, - GROUP_4_OVERRIDE BOOLEAN, - GROUP_5 BOOLEAN, - GROUP_5_OVERRIDE BOOLEAN, + BATCH_ID VARCHAR, + UNIQUE_KEY BOOLEAN, + UNIQUE_KEY_OVERRIDE BOOLEAN DEFAULT FALSE, + PRICING_BEFORE_CARVEOUTS BOOLEAN, + PRICING_BEFORE_CARVEOUTS_OVERRIDE BOOLEAN DEFAULT FALSE, + CONTRACT_RELATED BOOLEAN, + CONTRACT_RELATED_OVERRIDE BOOLEAN DEFAULT FALSE, + PROVIDER BOOLEAN, + PROVIDER_OVERRIDE BOOLEAN DEFAULT FALSE, + TIMELINE BOOLEAN, + TIMELINE_OVERRIDE BOOLEAN DEFAULT FALSE, + CARVEOUT_INDICATOR BOOLEAN, + CARVEOUT_INDICATOR_OVERRIDE BOOLEAN DEFAULT FALSE, + CARVEOUT_METHODOLOGY BOOLEAN + CARVEOUT_METHODOLOGY_OVERRIDE BOOLEAN DEFAULT FALSE, REQUEST_DATETIME DATETIME, REQUEST_USER VARCHAR, OVERRIDE_DATETIME DATETIME, LATEST_FLAG BOOLEAN DEFAULT TRUE, - PIPELINE_KICKOFF_DATETIME DATETIME + PIPELINE_KICKOFF_DATETIME DATETIME, + REQUEST_DATETIME DATETIME DEFAULT CURRENT_TIMESTAMP(), + REQUEST_USER VARCHAR, + OVERRIDE_DATETIME DATETIME DEFAULT NULL, + LATEST_FLAG BOOLEAN DEFAULT TRUE, + PIPELINE_KICKOFF_DATETIME DATETIME ); \ No newline at end of file diff --git a/snowflake/scripts/config_interface/R__003_REQUEST_SUBMISSION_TABLE.sql b/snowflake/scripts/config_interface/R__003_REQUEST_SUBMISSION_TABLE.sql index c3eae3a..2107d73 100644 --- a/snowflake/scripts/config_interface/R__003_REQUEST_SUBMISSION_TABLE.sql +++ b/snowflake/scripts/config_interface/R__003_REQUEST_SUBMISSION_TABLE.sql @@ -1,9 +1,6 @@ CREATE TABLE IF NOT EXISTS STG.REQUEST_SUBMISSION ( - REQUEST_ID NUMERIC, - T_DRIVE_PATH VARCHAR, CLIENT_NAME VARCHAR, - GROUP_NAME VARIANT, + BATCH_ID VARCHAR, REQUEST_USERNAME VARCHAR, - REQUEST_DATETIME DATETIME, - OVERRIDE_DATETIME DATETIME DEFAULT NULL -); \ No newline at end of file + REQUEST_DATETIME DATETIME +); diff --git a/snowflake/scripts/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql b/snowflake/scripts/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql index bc65615..223b7aa 100644 --- a/snowflake/scripts/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql +++ b/snowflake/scripts/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql @@ -21,26 +21,34 @@ BEGIN call stg.log_audit(:procedure_name, 'Section 1', 99, 'END'); call stg.log_audit(:procedure_name, 'Section 2', 99, 'START'); - COPY INTO STG.CONTRACT_CONFIG FROM ( - SELECT - NULLIF(TRIM($1),'') AS REQUEST_ID, - NULLIF(TRIM($2),'') AS CONTRACT_NAME, - NULLIF(TRIM($3),'') AS GROUP_1, - NULLIF(TRIM($4),'') AS GROUP_1_OVERRIDE, - NULLIF(TRIM($5),'') AS GROUP_2, - NULLIF(TRIM($6),'') AS GROUP_2_OVERRIDE, - NULLIF(TRIM($7),'') AS GROUP_3, - NULLIF(TRIM($8),'') AS GROUP_3_OVERRIDE, - NULLIF(TRIM($9),'') AS GROUP_4, - NULLIF(TRIM($10),'') AS GROUP_4_OVERRIDE, - NULLIF(TRIM($11),'') AS GROUP_5, - NULLIF(TRIM($12),'') AS GROUP_5_OVERRIDE, - NULLIF(TRIM($13),'') AS REQUEST_DATETIME, - NULLIF(TRIM($14),'') AS REQUEST_USER, - NULLIF(TRIM($15),'') AS OVERRIDE_DATETIME, - NULLIF(TRIM($16),'') AS LATEST_FLAG, - NULLIF(TRIM($17),'') AS PIPELINE_KICKOFF_DATETIME + SELECT + NULLIF(TRIM($2),'') AS CONTRACT_NAME, + NULLIF(TRIM($3),'') AS BATCH_ID, + NULLIF(TRIM($4),'') AS UNIQUE_KEY, + NULLIF(TRIM($5),'') AS UNIQUE_KEY_OVERRIDE, + NULLIF(TRIM($6),'') AS PRICING_BEFORE_CARVEOUTS, + NULLIF(TRIM($7),'') AS PRICING_BEFORE_CARVEOUTS_OVERRIDE, + NULLIF(TRIM($8),'') AS CONTRACT_RELATED, + NULLIF(TRIM($9),'') AS CONTRACT_RELATED_OVERRIDE, + NULLIF(TRIM($10),'') AS PROVIDER, + NULLIF(TRIM($11),'') AS PROVIDER_OVERRIDE, + NULLIF(TRIM($12),'') AS TIMELINE, + NULLIF(TRIM($13),'') AS TIMELINE_OVERRIDE, + NULLIF(TRIM($14),'') AS CARVEOUT_INDICATOR, + NULLIF(TRIM($15),'') AS CARVEOUT_INDICATOR_OVERRIDE, + NULLIF(TRIM($16),'') AS CARVEOUT_METHODOLOGY, + NULLIF(TRIM($17),'') AS CARVEOUT_METHODOLOGY_OVERRIDE, + NULLIF(TRIM($18),'') AS REQUEST_DATETIME, + NULLIF(TRIM($19),'') AS REQUEST_USER, + NULLIF(TRIM($20),'') AS OVERRIDE_DATETIME, + NULLIF(TRIM($21),'') AS LATEST_FLAG, + NULLIF(TRIM($22),'') AS PIPELINE_KICKOFF_DATETIME + NULLIF(TRIM($23),'') AS REQUEST_DATETIME, + NULLIF(TRIM($24),'') AS REQUEST_USER, + NULLIF(TRIM($25),'') AS OVERRIDE_DATETIME, + NULLIF(TRIM($26),'') AS LATEST_FLAG, + NULLIF(TRIM($27),'') AS PIPELINE_KICKOFF_DATETIME FROM @STG.CONTRACT_CONFIG_STAGE ) FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER') diff --git a/snowflake/scripts/config_interface/R__005_LOAD_REQUEST_SUBMISSION_SP.sql b/snowflake/scripts/config_interface/R__005_LOAD_REQUEST_SUBMISSION_SP.sql index e215ea7..419305b 100644 --- a/snowflake/scripts/config_interface/R__005_LOAD_REQUEST_SUBMISSION_SP.sql +++ b/snowflake/scripts/config_interface/R__005_LOAD_REQUEST_SUBMISSION_SP.sql @@ -8,12 +8,12 @@ DECLARE procedure_name varchar; BEGIN - procedure_name := 'LOAD_CONTRACT_CONFIG'; + procedure_name := 'LOAD_REQUEST_SUBMISSION'; call stg.log_audit(:procedure_name, 'Section 1', 99, 'START'); -- Create or replace stage with dynamic file name - EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.CONTRACT_CONFIG_STAGE + EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.REQUEST_SUBMISSION_STAGE STORAGE_INTEGRATION = dev_bucket_integration URL = ''s3://doczy-dev-infra-raw-data-ingestion/config_interface/' || :file_name || ''' FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');'; @@ -22,26 +22,14 @@ BEGIN call stg.log_audit(:procedure_name, 'Section 2', 99, 'START'); - COPY INTO STG.CONTRACT_CONFIG FROM ( + COPY INTO STG.REQUEST_SUBMISSION FROM ( SELECT - NULLIF(TRIM($1),'') AS REQUEST_ID, - NULLIF(TRIM($2),'') AS CONTRACT_NAME, - NULLIF(TRIM($3),'') AS GROUP_1, - NULLIF(TRIM($4),'') AS GROUP_1_OVERRIDE, - NULLIF(TRIM($5),'') AS GROUP_2, - NULLIF(TRIM($6),'') AS GROUP_2_OVERRIDE, - NULLIF(TRIM($7),'') AS GROUP_3, - NULLIF(TRIM($8),'') AS GROUP_3_OVERRIDE, - NULLIF(TRIM($9),'') AS GROUP_4, - NULLIF(TRIM($10),'') AS GROUP_4_OVERRIDE, - NULLIF(TRIM($11),'') AS GROUP_5, - NULLIF(TRIM($12),'') AS GROUP_5_OVERRIDE, - NULLIF(TRIM($13),'') AS REQUEST_DATETIME, - NULLIF(TRIM($14),'') AS REQUEST_USER, - NULLIF(TRIM($15),'') AS OVERRIDE_DATETIME, - NULLIF(TRIM($16),'') AS LATEST_FLAG , - NULLIF(TRIM($17),'') AS PIPELINE_KICKOFF_DATETIME - FROM @STG.CONTRACT_CONFIG_STAGE + NULLIF(TRIM($2),'') AS CLIENT_NAME, + NULLIF(TRIM($3),'') AS BATCH_ID, + NULLIF(TRIM($4),'') AS REQUEST_USERNAME, + NULLIF(TRIM($5),'') AS REQUEST_DATETIME + + FROM @STG.REQUEST_SUBMISSION_STAGE ) FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER') ON_ERROR = ABORT_STATEMENT; @@ -50,10 +38,10 @@ BEGIN call stg.log_audit(:procedure_name, 'Section 3', 99, 'START'); - INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) - SELECT STG.AUDIT_SID.NEXTVAL, 'STG.CONTRACT_CONFIG',* + INSERT INTO STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) + SELECT STG.AUDIT_SID.NEXTVAL, 'STG.REQUEST_SUBMISSION',* FROM - (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.CONTRACT_CONFIG_STAGE group by 1,2); + (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.REQUEST_SUBMISSION_STAGE group by 1,2);