Created contract upload logs, updated config interface pipeline

snowflake/scripts/config_interface/R__001_PROMPT_CONFIG_TABLE.sql

@@ -1,5 +1,5 @@
 -- Separation of single vs multi prompt using a flag in the table
-CREATE TABLE STG.PROMPT_CONFIG (
+CREATE TABLE IF NOT EXISTS STG.PROMPT_CONFIG (
     FIELD_NAME VARCHAR,
     FIELD_DESC VARCHAR,
     IS_REQUIRED BOOLEAN,

snowflake/scripts/config_interface/R__002_CONTRACT_CONFIG_TABLE.sql

@@ -3,27 +3,14 @@ CREATE TABLE IF NOT EXISTS STG.CONTRACT_CONFIG (
     CONTRACT_NAME VARCHAR,
     BATCH_ID VARCHAR,
     UNIQUE_KEY BOOLEAN,
-    UNIQUE_KEY_OVERRIDE BOOLEAN DEFAULT FALSE,    
     PRICING_BEFORE_CARVEOUTS BOOLEAN,
-    PRICING_BEFORE_CARVEOUTS_OVERRIDE BOOLEAN DEFAULT FALSE,
     CONTRACT_RELATED BOOLEAN,
-    CONTRACT_RELATED_OVERRIDE BOOLEAN DEFAULT FALSE,
     PROVIDER BOOLEAN,
-    PROVIDER_OVERRIDE BOOLEAN DEFAULT FALSE,
     TIMELINE BOOLEAN,
-    TIMELINE_OVERRIDE BOOLEAN DEFAULT FALSE,
     CARVEOUT_INDICATOR BOOLEAN,
-    CARVEOUT_INDICATOR_OVERRIDE BOOLEAN DEFAULT FALSE,
     CARVEOUT_METHODOLOGY BOOLEAN
-    CARVEOUT_METHODOLOGY_OVERRIDE BOOLEAN DEFAULT FALSE,
-    REQUEST_DATETIME DATETIME,
     REQUEST_USER VARCHAR,
-    OVERRIDE_DATETIME DATETIME,
     LATEST_FLAG BOOLEAN DEFAULT TRUE,
     PIPELINE_KICKOFF_DATETIME DATETIME,
-    REQUEST_DATETIME DATETIME DEFAULT CURRENT_TIMESTAMP(), 
+    REQUEST_DATETIME DATETIME DEFAULT CURRENT_TIMESTAMP()  
-    REQUEST_USER VARCHAR,
-    OVERRIDE_DATETIME DATETIME DEFAULT NULL,
-    LATEST_FLAG BOOLEAN DEFAULT TRUE,
-    PIPELINE_KICKOFF_DATETIME DATETIME   
 );

snowflake/scripts/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql

@@ -23,32 +23,19 @@ BEGIN
     call stg.log_audit(:procedure_name, 'Section 2', 99, 'START');
     COPY INTO STG.CONTRACT_CONFIG FROM (
         SELECT  
-        NULLIF(TRIM($2),'') AS CONTRACT_NAME,   
+        NULLIF(TRIM($1),'') AS CONTRACT_NAME,
-        NULLIF(TRIM($3),'') AS BATCH_ID,
+        NULLIF(TRIM($2),'') AS BATCH_ID 
-        NULLIF(TRIM($4),'') AS UNIQUE_KEY,
+        NULLIF(TRIM($3),'') AS UNIQUE_KEY,
-        NULLIF(TRIM($5),'') AS UNIQUE_KEY_OVERRIDE,
+        NULLIF(TRIM($4),'') AS PRICING_BEFORE_CARVEOUTS,
-        NULLIF(TRIM($6),'') AS PRICING_BEFORE_CARVEOUTS,
+        NULLIF(TRIM($5),'') AS CONTRACT_RELATED,
-        NULLIF(TRIM($7),'') AS PRICING_BEFORE_CARVEOUTS_OVERRIDE,
+        NULLIF(TRIM($6),'') AS PROVIDER,
-        NULLIF(TRIM($8),'') AS CONTRACT_RELATED,
+        NULLIF(TRIM($7),'') AS TIMELINE,
-        NULLIF(TRIM($9),'') AS CONTRACT_RELATED_OVERRIDE,
+        NULLIF(TRIM($8),'') AS CARVEOUT_INDICATOR,
-        NULLIF(TRIM($10),'') AS PROVIDER,
+        NULLIF(TRIM($9),'') AS CARVEOUT_METHODOLOGY,
-        NULLIF(TRIM($11),'') AS PROVIDER_OVERRIDE,
+        NULLIF(TRIM($10),'') AS REQUEST_USER,
-        NULLIF(TRIM($12),'') AS TIMELINE,
+        NULLIF(TRIM($11),'') AS LATEST_FLAG,
-        NULLIF(TRIM($13),'') AS TIMELINE_OVERRIDE,
+        NULLIF(TRIM($12),'') AS PIPELINE_KICKOFF_DATETIME,
-        NULLIF(TRIM($14),'') AS CARVEOUT_INDICATOR,
+        NULLIF(TRIM($13),'') AS REQUEST_DATETIME
-        NULLIF(TRIM($15),'') AS CARVEOUT_INDICATOR_OVERRIDE,
-        NULLIF(TRIM($16),'') AS CARVEOUT_METHODOLOGY,
-        NULLIF(TRIM($17),'') AS CARVEOUT_METHODOLOGY_OVERRIDE,
-        NULLIF(TRIM($18),'') AS REQUEST_DATETIME,
-        NULLIF(TRIM($19),'') AS REQUEST_USER,
-        NULLIF(TRIM($20),'') AS OVERRIDE_DATETIME,
-        NULLIF(TRIM($21),'') AS LATEST_FLAG,
-        NULLIF(TRIM($22),'') AS PIPELINE_KICKOFF_DATETIME
-        NULLIF(TRIM($23),'') AS REQUEST_DATETIME,
-        NULLIF(TRIM($24),'') AS REQUEST_USER,
-        NULLIF(TRIM($25),'') AS OVERRIDE_DATETIME,
-        NULLIF(TRIM($26),'') AS LATEST_FLAG,
-        NULLIF(TRIM($27),'') AS PIPELINE_KICKOFF_DATETIME
         FROM @STG.CONTRACT_CONFIG_STAGE
     )
     FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER')
@@ -58,19 +45,27 @@ BEGIN
  
     call stg.log_audit(:procedure_name, 'Section 3', 99, 'START');
 
+    -- Once the new data is loaded to the stage, we need to update the existing records in the table to set the LATEST_FLAG to FALSE
+
+    contract_name := (SELECT NULLIF(TRIM($1),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1);
+    batch_id := (SELECT NULLIF(TRIM($2),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1);
+    request_datetime := (SELECT NULLIF(TRIM($13),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1);
+
+    UPDATE TABLE STG.CONTRACT_CONFIG 
+    SET LATEST_FLAG = FALSE 
+    WHERE BATCH_ID = :batch_id AND CONTRACT_NAME = :contract_name AND REQUEST_DATETIME < :request_datetime AND LATEST_FLAG = TRUE;  
+     
+    call stg.log_audit(:procedure_name, 'Section 3', 99, 'END');
+    
+    call stg.log_audit(:procedure_name, 'Section 4', 99, 'START');
      INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT)
     SELECT STG.AUDIT_SID.NEXTVAL, :procedure_name,* 
     FROM 
     (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER)  from @STG.CONTRACT_CONFIG_STAGE group by 1,2);
 
-    
-
     RETURN 'Setup, Load, and Audit Complete';
-    
-    call stg.log_audit(:procedure_name, 'Section 3', 99, 'END');
-
-
 
+    call stg.log_audit(:procedure_name, 'Section 4', 99, 'END');
    
 END;
 $$;

snowflake/scripts/upload_interface/R__003_CONTRACT_UPLOAD_LOGS.sql

@@ -0,0 +1,10 @@
+-- This table is used to store the logs of all the files that were uploaded by the user to S3
+-- Following fields are to be stored in the table:
+-- Batch id & client name & file name, date time, user
+CREATE TABLE IF NOT EXISTS STG.CONTRACT_UPLOAD_LOGS (
+    BATCH_ID VARCHAR,
+    CLIENT_NAME VARCHAR,
+    FILE_NAME VARCHAR,
+    UPLOAD_DATETIME DATETIME DEFAULT CURRENT_TIMESTAMP(),
+    UPLOAD_USER VARCHAR
+);