Merged in dev_umistry (pull request #56)

Created contract upload logs, updated config interface pipeline
2024-04-04 22:05:14 +00:00
parent 93c44bdf09 09f16c45d0
commit aafcfa28e3
4 changed files with 39 additions and 47 deletions
@@ -1,5 +1,5 @@
 -- Separation of single vs multi prompt using a flag in the table
-CREATE TABLE STG.PROMPT_CONFIG (
+CREATE TABLE IF NOT EXISTS STG.PROMPT_CONFIG (
    FIELD_NAME VARCHAR,
    FIELD_DESC VARCHAR,
    IS_REQUIRED BOOLEAN,
@@ -3,27 +3,14 @@ CREATE TABLE IF NOT EXISTS STG.CONTRACT_CONFIG (
    CONTRACT_NAME VARCHAR,
    BATCH_ID VARCHAR,
    UNIQUE_KEY BOOLEAN,
-    UNIQUE_KEY_OVERRIDE BOOLEAN DEFAULT FALSE,    
    PRICING_BEFORE_CARVEOUTS BOOLEAN,
-    PRICING_BEFORE_CARVEOUTS_OVERRIDE BOOLEAN DEFAULT FALSE,
    CONTRACT_RELATED BOOLEAN,
-    CONTRACT_RELATED_OVERRIDE BOOLEAN DEFAULT FALSE,
    PROVIDER BOOLEAN,
-    PROVIDER_OVERRIDE BOOLEAN DEFAULT FALSE,
    TIMELINE BOOLEAN,
-    TIMELINE_OVERRIDE BOOLEAN DEFAULT FALSE,
    CARVEOUT_INDICATOR BOOLEAN,
-    CARVEOUT_INDICATOR_OVERRIDE BOOLEAN DEFAULT FALSE,
    CARVEOUT_METHODOLOGY BOOLEAN
-    CARVEOUT_METHODOLOGY_OVERRIDE BOOLEAN DEFAULT FALSE,
-    REQUEST_DATETIME DATETIME,
    REQUEST_USER VARCHAR,
-    OVERRIDE_DATETIME DATETIME,
    LATEST_FLAG BOOLEAN DEFAULT TRUE,
    PIPELINE_KICKOFF_DATETIME DATETIME,
-    REQUEST_DATETIME DATETIME DEFAULT CURRENT_TIMESTAMP(), 
-    REQUEST_USER VARCHAR,
-    OVERRIDE_DATETIME DATETIME DEFAULT NULL,
-    LATEST_FLAG BOOLEAN DEFAULT TRUE,
-    PIPELINE_KICKOFF_DATETIME DATETIME   
+    REQUEST_DATETIME DATETIME DEFAULT CURRENT_TIMESTAMP()  
 );
@@ -23,32 +23,19 @@ BEGIN
    call stg.log_audit(:procedure_name, 'Section 2', 99, 'START');
    COPY INTO STG.CONTRACT_CONFIG FROM (
        SELECT  
-        NULLIF(TRIM($2),'') AS CONTRACT_NAME,   
-        NULLIF(TRIM($3),'') AS BATCH_ID,
-        NULLIF(TRIM($4),'') AS UNIQUE_KEY,
-        NULLIF(TRIM($5),'') AS UNIQUE_KEY_OVERRIDE,
-        NULLIF(TRIM($6),'') AS PRICING_BEFORE_CARVEOUTS,
-        NULLIF(TRIM($7),'') AS PRICING_BEFORE_CARVEOUTS_OVERRIDE,
-        NULLIF(TRIM($8),'') AS CONTRACT_RELATED,
-        NULLIF(TRIM($9),'') AS CONTRACT_RELATED_OVERRIDE,
-        NULLIF(TRIM($10),'') AS PROVIDER,
-        NULLIF(TRIM($11),'') AS PROVIDER_OVERRIDE,
-        NULLIF(TRIM($12),'') AS TIMELINE,
-        NULLIF(TRIM($13),'') AS TIMELINE_OVERRIDE,
-        NULLIF(TRIM($14),'') AS CARVEOUT_INDICATOR,
-        NULLIF(TRIM($15),'') AS CARVEOUT_INDICATOR_OVERRIDE,
-        NULLIF(TRIM($16),'') AS CARVEOUT_METHODOLOGY,
-        NULLIF(TRIM($17),'') AS CARVEOUT_METHODOLOGY_OVERRIDE,
-        NULLIF(TRIM($18),'') AS REQUEST_DATETIME,
-        NULLIF(TRIM($19),'') AS REQUEST_USER,
-        NULLIF(TRIM($20),'') AS OVERRIDE_DATETIME,
-        NULLIF(TRIM($21),'') AS LATEST_FLAG,
-        NULLIF(TRIM($22),'') AS PIPELINE_KICKOFF_DATETIME
-        NULLIF(TRIM($23),'') AS REQUEST_DATETIME,
-        NULLIF(TRIM($24),'') AS REQUEST_USER,
-        NULLIF(TRIM($25),'') AS OVERRIDE_DATETIME,
-        NULLIF(TRIM($26),'') AS LATEST_FLAG,
-        NULLIF(TRIM($27),'') AS PIPELINE_KICKOFF_DATETIME
+        NULLIF(TRIM($1),'') AS CONTRACT_NAME,
+        NULLIF(TRIM($2),'') AS BATCH_ID 
+        NULLIF(TRIM($3),'') AS UNIQUE_KEY,
+        NULLIF(TRIM($4),'') AS PRICING_BEFORE_CARVEOUTS,
+        NULLIF(TRIM($5),'') AS CONTRACT_RELATED,
+        NULLIF(TRIM($6),'') AS PROVIDER,
+        NULLIF(TRIM($7),'') AS TIMELINE,
+        NULLIF(TRIM($8),'') AS CARVEOUT_INDICATOR,
+        NULLIF(TRIM($9),'') AS CARVEOUT_METHODOLOGY,
+        NULLIF(TRIM($10),'') AS REQUEST_USER,
+        NULLIF(TRIM($11),'') AS LATEST_FLAG,
+        NULLIF(TRIM($12),'') AS PIPELINE_KICKOFF_DATETIME,
+        NULLIF(TRIM($13),'') AS REQUEST_DATETIME
        FROM @STG.CONTRACT_CONFIG_STAGE
    )
    FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER')
@@ -58,19 +45,27 @@ BEGIN
 
    call stg.log_audit(:procedure_name, 'Section 3', 99, 'START');

+    -- Once the new data is loaded to the stage, we need to update the existing records in the table to set the LATEST_FLAG to FALSE
+
+    contract_name := (SELECT NULLIF(TRIM($1),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1);
+    batch_id := (SELECT NULLIF(TRIM($2),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1);
+    request_datetime := (SELECT NULLIF(TRIM($13),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1);
+
+    UPDATE TABLE STG.CONTRACT_CONFIG 
+    SET LATEST_FLAG = FALSE 
+    WHERE BATCH_ID = :batch_id AND CONTRACT_NAME = :contract_name AND REQUEST_DATETIME < :request_datetime AND LATEST_FLAG = TRUE;  
+     
+    call stg.log_audit(:procedure_name, 'Section 3', 99, 'END');
+    
+    call stg.log_audit(:procedure_name, 'Section 4', 99, 'START');
     INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT)
    SELECT STG.AUDIT_SID.NEXTVAL, :procedure_name,* 
    FROM 
    (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER)  from @STG.CONTRACT_CONFIG_STAGE group by 1,2);

-    
-
    RETURN 'Setup, Load, and Audit Complete';
-    
-    call stg.log_audit(:procedure_name, 'Section 3', 99, 'END');
-
-

+    call stg.log_audit(:procedure_name, 'Section 4', 99, 'END');
   
 END;
 $$;
@@ -0,0 +1,10 @@
+-- This table is used to store the logs of all the files that were uploaded by the user to S3
+-- Following fields are to be stored in the table:
+-- Batch id & client name & file name, date time, user
+CREATE TABLE IF NOT EXISTS STG.CONTRACT_UPLOAD_LOGS (
+    BATCH_ID VARCHAR,
+    CLIENT_NAME VARCHAR,
+    FILE_NAME VARCHAR,
+    UPLOAD_DATETIME DATETIME DEFAULT CURRENT_TIMESTAMP(),
+    UPLOAD_USER VARCHAR
+);