Merged in dev_umistry (pull request #55)
Updated contract config and request submission tables pipeline with the sample provided by Mayank
This commit is contained in:
Umang Mistry
@@ -1,3 +1,4 @@
|
||||
-- Separation of single vs multi prompt using a flag in the table
|
||||
CREATE TABLE STG.PROMPT_CONFIG (
|
||||
FIELD_NAME VARCHAR,
|
||||
FIELD_DESC VARCHAR,
|
||||
@@ -9,4 +10,4 @@ CREATE TABLE STG.PROMPT_CONFIG (
|
||||
FIELD_MAX_LENGTH NUMBER,
|
||||
SAMPLE_VALUE VARCHAR,
|
||||
CONTRACT_SECTION VARCHAR
|
||||
);
|
||||
);
|
||||
|
||||
@@ -1,20 +1,29 @@
|
||||
-- CREATING THE CONFIG TABLE
|
||||
CREATE TABLE IF NOT EXISTS STG.CONTRACT_CONFIG (
|
||||
REQUEST_ID NUMERIC,
|
||||
CONTRACT_NAME VARCHAR,
|
||||
GROUP_1 BOOLEAN,
|
||||
GROUP_1_OVERRIDE BOOLEAN,
|
||||
GROUP_2 BOOLEAN,
|
||||
GROUP_2_OVERRIDE BOOLEAN,
|
||||
GROUP_3 BOOLEAN,
|
||||
GROUP_3_OVERRIDE BOOLEAN,
|
||||
GROUP_4 BOOLEAN,
|
||||
GROUP_4_OVERRIDE BOOLEAN,
|
||||
GROUP_5 BOOLEAN,
|
||||
GROUP_5_OVERRIDE BOOLEAN,
|
||||
BATCH_ID VARCHAR,
|
||||
UNIQUE_KEY BOOLEAN,
|
||||
UNIQUE_KEY_OVERRIDE BOOLEAN DEFAULT FALSE,
|
||||
PRICING_BEFORE_CARVEOUTS BOOLEAN,
|
||||
PRICING_BEFORE_CARVEOUTS_OVERRIDE BOOLEAN DEFAULT FALSE,
|
||||
CONTRACT_RELATED BOOLEAN,
|
||||
CONTRACT_RELATED_OVERRIDE BOOLEAN DEFAULT FALSE,
|
||||
PROVIDER BOOLEAN,
|
||||
PROVIDER_OVERRIDE BOOLEAN DEFAULT FALSE,
|
||||
TIMELINE BOOLEAN,
|
||||
TIMELINE_OVERRIDE BOOLEAN DEFAULT FALSE,
|
||||
CARVEOUT_INDICATOR BOOLEAN,
|
||||
CARVEOUT_INDICATOR_OVERRIDE BOOLEAN DEFAULT FALSE,
|
||||
CARVEOUT_METHODOLOGY BOOLEAN
|
||||
CARVEOUT_METHODOLOGY_OVERRIDE BOOLEAN DEFAULT FALSE,
|
||||
REQUEST_DATETIME DATETIME,
|
||||
REQUEST_USER VARCHAR,
|
||||
OVERRIDE_DATETIME DATETIME,
|
||||
LATEST_FLAG BOOLEAN DEFAULT TRUE,
|
||||
PIPELINE_KICKOFF_DATETIME DATETIME
|
||||
PIPELINE_KICKOFF_DATETIME DATETIME,
|
||||
REQUEST_DATETIME DATETIME DEFAULT CURRENT_TIMESTAMP(),
|
||||
REQUEST_USER VARCHAR,
|
||||
OVERRIDE_DATETIME DATETIME DEFAULT NULL,
|
||||
LATEST_FLAG BOOLEAN DEFAULT TRUE,
|
||||
PIPELINE_KICKOFF_DATETIME DATETIME
|
||||
);
|
||||
@@ -1,9 +1,6 @@
|
||||
CREATE TABLE IF NOT EXISTS STG.REQUEST_SUBMISSION (
|
||||
REQUEST_ID NUMERIC,
|
||||
T_DRIVE_PATH VARCHAR,
|
||||
CLIENT_NAME VARCHAR,
|
||||
GROUP_NAME VARIANT,
|
||||
BATCH_ID VARCHAR,
|
||||
REQUEST_USERNAME VARCHAR,
|
||||
REQUEST_DATETIME DATETIME,
|
||||
OVERRIDE_DATETIME DATETIME DEFAULT NULL
|
||||
);
|
||||
REQUEST_DATETIME DATETIME
|
||||
);
|
||||
|
||||
@@ -21,26 +21,34 @@ BEGIN
|
||||
call stg.log_audit(:procedure_name, 'Section 1', 99, 'END');
|
||||
|
||||
call stg.log_audit(:procedure_name, 'Section 2', 99, 'START');
|
||||
|
||||
COPY INTO STG.CONTRACT_CONFIG FROM (
|
||||
SELECT
|
||||
NULLIF(TRIM($1),'') AS REQUEST_ID,
|
||||
NULLIF(TRIM($2),'') AS CONTRACT_NAME,
|
||||
NULLIF(TRIM($3),'') AS GROUP_1,
|
||||
NULLIF(TRIM($4),'') AS GROUP_1_OVERRIDE,
|
||||
NULLIF(TRIM($5),'') AS GROUP_2,
|
||||
NULLIF(TRIM($6),'') AS GROUP_2_OVERRIDE,
|
||||
NULLIF(TRIM($7),'') AS GROUP_3,
|
||||
NULLIF(TRIM($8),'') AS GROUP_3_OVERRIDE,
|
||||
NULLIF(TRIM($9),'') AS GROUP_4,
|
||||
NULLIF(TRIM($10),'') AS GROUP_4_OVERRIDE,
|
||||
NULLIF(TRIM($11),'') AS GROUP_5,
|
||||
NULLIF(TRIM($12),'') AS GROUP_5_OVERRIDE,
|
||||
NULLIF(TRIM($13),'') AS REQUEST_DATETIME,
|
||||
NULLIF(TRIM($14),'') AS REQUEST_USER,
|
||||
NULLIF(TRIM($15),'') AS OVERRIDE_DATETIME,
|
||||
NULLIF(TRIM($16),'') AS LATEST_FLAG,
|
||||
NULLIF(TRIM($17),'') AS PIPELINE_KICKOFF_DATETIME
|
||||
SELECT
|
||||
NULLIF(TRIM($2),'') AS CONTRACT_NAME,
|
||||
NULLIF(TRIM($3),'') AS BATCH_ID,
|
||||
NULLIF(TRIM($4),'') AS UNIQUE_KEY,
|
||||
NULLIF(TRIM($5),'') AS UNIQUE_KEY_OVERRIDE,
|
||||
NULLIF(TRIM($6),'') AS PRICING_BEFORE_CARVEOUTS,
|
||||
NULLIF(TRIM($7),'') AS PRICING_BEFORE_CARVEOUTS_OVERRIDE,
|
||||
NULLIF(TRIM($8),'') AS CONTRACT_RELATED,
|
||||
NULLIF(TRIM($9),'') AS CONTRACT_RELATED_OVERRIDE,
|
||||
NULLIF(TRIM($10),'') AS PROVIDER,
|
||||
NULLIF(TRIM($11),'') AS PROVIDER_OVERRIDE,
|
||||
NULLIF(TRIM($12),'') AS TIMELINE,
|
||||
NULLIF(TRIM($13),'') AS TIMELINE_OVERRIDE,
|
||||
NULLIF(TRIM($14),'') AS CARVEOUT_INDICATOR,
|
||||
NULLIF(TRIM($15),'') AS CARVEOUT_INDICATOR_OVERRIDE,
|
||||
NULLIF(TRIM($16),'') AS CARVEOUT_METHODOLOGY,
|
||||
NULLIF(TRIM($17),'') AS CARVEOUT_METHODOLOGY_OVERRIDE,
|
||||
NULLIF(TRIM($18),'') AS REQUEST_DATETIME,
|
||||
NULLIF(TRIM($19),'') AS REQUEST_USER,
|
||||
NULLIF(TRIM($20),'') AS OVERRIDE_DATETIME,
|
||||
NULLIF(TRIM($21),'') AS LATEST_FLAG,
|
||||
NULLIF(TRIM($22),'') AS PIPELINE_KICKOFF_DATETIME
|
||||
NULLIF(TRIM($23),'') AS REQUEST_DATETIME,
|
||||
NULLIF(TRIM($24),'') AS REQUEST_USER,
|
||||
NULLIF(TRIM($25),'') AS OVERRIDE_DATETIME,
|
||||
NULLIF(TRIM($26),'') AS LATEST_FLAG,
|
||||
NULLIF(TRIM($27),'') AS PIPELINE_KICKOFF_DATETIME
|
||||
FROM @STG.CONTRACT_CONFIG_STAGE
|
||||
)
|
||||
FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER')
|
||||
|
||||
@@ -8,12 +8,12 @@ DECLARE
|
||||
procedure_name varchar;
|
||||
BEGIN
|
||||
|
||||
procedure_name := 'LOAD_CONTRACT_CONFIG';
|
||||
procedure_name := 'LOAD_REQUEST_SUBMISSION';
|
||||
|
||||
call stg.log_audit(:procedure_name, 'Section 1', 99, 'START');
|
||||
|
||||
-- Create or replace stage with dynamic file name
|
||||
EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.CONTRACT_CONFIG_STAGE
|
||||
EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.REQUEST_SUBMISSION_STAGE
|
||||
STORAGE_INTEGRATION = dev_bucket_integration
|
||||
URL = ''s3://doczy-dev-infra-raw-data-ingestion/config_interface/' || :file_name || '''
|
||||
FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');';
|
||||
@@ -22,26 +22,14 @@ BEGIN
|
||||
|
||||
call stg.log_audit(:procedure_name, 'Section 2', 99, 'START');
|
||||
|
||||
COPY INTO STG.CONTRACT_CONFIG FROM (
|
||||
COPY INTO STG.REQUEST_SUBMISSION FROM (
|
||||
SELECT
|
||||
NULLIF(TRIM($1),'') AS REQUEST_ID,
|
||||
NULLIF(TRIM($2),'') AS CONTRACT_NAME,
|
||||
NULLIF(TRIM($3),'') AS GROUP_1,
|
||||
NULLIF(TRIM($4),'') AS GROUP_1_OVERRIDE,
|
||||
NULLIF(TRIM($5),'') AS GROUP_2,
|
||||
NULLIF(TRIM($6),'') AS GROUP_2_OVERRIDE,
|
||||
NULLIF(TRIM($7),'') AS GROUP_3,
|
||||
NULLIF(TRIM($8),'') AS GROUP_3_OVERRIDE,
|
||||
NULLIF(TRIM($9),'') AS GROUP_4,
|
||||
NULLIF(TRIM($10),'') AS GROUP_4_OVERRIDE,
|
||||
NULLIF(TRIM($11),'') AS GROUP_5,
|
||||
NULLIF(TRIM($12),'') AS GROUP_5_OVERRIDE,
|
||||
NULLIF(TRIM($13),'') AS REQUEST_DATETIME,
|
||||
NULLIF(TRIM($14),'') AS REQUEST_USER,
|
||||
NULLIF(TRIM($15),'') AS OVERRIDE_DATETIME,
|
||||
NULLIF(TRIM($16),'') AS LATEST_FLAG ,
|
||||
NULLIF(TRIM($17),'') AS PIPELINE_KICKOFF_DATETIME
|
||||
FROM @STG.CONTRACT_CONFIG_STAGE
|
||||
NULLIF(TRIM($2),'') AS CLIENT_NAME,
|
||||
NULLIF(TRIM($3),'') AS BATCH_ID,
|
||||
NULLIF(TRIM($4),'') AS REQUEST_USERNAME,
|
||||
NULLIF(TRIM($5),'') AS REQUEST_DATETIME
|
||||
|
||||
FROM @STG.REQUEST_SUBMISSION_STAGE
|
||||
)
|
||||
FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER')
|
||||
ON_ERROR = ABORT_STATEMENT;
|
||||
@@ -50,10 +38,10 @@ BEGIN
|
||||
|
||||
call stg.log_audit(:procedure_name, 'Section 3', 99, 'START');
|
||||
|
||||
INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT)
|
||||
SELECT STG.AUDIT_SID.NEXTVAL, 'STG.CONTRACT_CONFIG',*
|
||||
INSERT INTO STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT)
|
||||
SELECT STG.AUDIT_SID.NEXTVAL, 'STG.REQUEST_SUBMISSION',*
|
||||
FROM
|
||||
(SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.CONTRACT_CONFIG_STAGE group by 1,2);
|
||||
(SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.REQUEST_SUBMISSION_STAGE group by 1,2);
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user