diff --git a/snowflake/UAT/config_interface/R__1001_UAT_PROMPT_CONFIG_TABLE.sql b/snowflake/UAT/config_interface/R__1001_UAT_PROMPT_CONFIG_TABLE.sql index b94878b..01cfcdc 100644 --- a/snowflake/UAT/config_interface/R__1001_UAT_PROMPT_CONFIG_TABLE.sql +++ b/snowflake/UAT/config_interface/R__1001_UAT_PROMPT_CONFIG_TABLE.sql @@ -7,6 +7,7 @@ CREATE TABLE IF NOT EXISTS STG.PROMPT_CONFIG ( PROMPT VARCHAR, FM_MODEL_ID VARCHAR, GROUP_ID NUMERIC, + DEPENDENT_FIELDS VARCHAR, FIELD_MAX_LENGTH NUMBER, SAMPLE_VALUE VARCHAR, CONTRACT_SECTION VARCHAR diff --git a/snowflake/UAT/config_interface/R__1006_PIPELINE_OUTPUT.sql b/snowflake/UAT/config_interface/R__1006_PIPELINE_OUTPUT.sql index b08d1d8..407a17c 100644 --- a/snowflake/UAT/config_interface/R__1006_PIPELINE_OUTPUT.sql +++ b/snowflake/UAT/config_interface/R__1006_PIPELINE_OUTPUT.sql @@ -1,15 +1,11 @@ -- CREATING THE CONFIG TABLE -CREATE TABLE IF NOT EXISTS STG.DOCZY_PIPELINE_RAW_OUTPUT ( - DOCUMENT_ID VARCHAR(16777216) NOT NULL, - SF_DB_COL_NAME VARCHAR(255), - ACTUAL_VALUE_STORED VARCHAR(16777216), +CREATE TABLE IF NOT EXISTS STG.DOCZY_PIPELINE_RAW_OUTPUT_AC ( + CONTRACT_NAME VARCHAR(16777216) NOT NULL, + FIELD_NAME VARCHAR(255), RAW_VALUE VARCHAR(16777216), NEW_EXTRACTED_VALUE VARCHAR(16777216), SNIPPET VARCHAR(16777216), - ORIGINAL_PAGE_NUMBER VARCHAR(255), - NEW_PAGE_NUMBER VARCHAR(255), - RESULT VARCHAR(16777216), - BATCH_ID VARCHAR(255), - CREATED_TIME TIMESTAMP_NTZ(9), - MODIFIED_TIME TIMESTAMP_NTZ(9) + PAGE_NUMBER VARCHAR(255), + REVISED_PROMPT VARCHAR(16777216), + BATCH_ID VARCHAR(255) ); \ No newline at end of file diff --git a/snowflake/UAT/config_interface/R__1007_LOAD_DOCZY_PIPELINE_RAW_OPUTPUTS.sql b/snowflake/UAT/config_interface/R__1007_LOAD_DOCZY_PIPELINE_RAW_OPUTPUTS.sql index 2729f1d..c23d4f0 100644 --- a/snowflake/UAT/config_interface/R__1007_LOAD_DOCZY_PIPELINE_RAW_OPUTPUTS.sql +++ b/snowflake/UAT/config_interface/R__1007_LOAD_DOCZY_PIPELINE_RAW_OPUTPUTS.sql @@ -1,4 +1,4 @@ -CREATE OR REPLACE PROCEDURE STG.LOAD_DOCZY_PIPELINE_RAW_OUTPUTS(file_name VARCHAR) +CREATE OR REPLACE PROCEDURE STG.LOAD_DOCZY_PIPELINE_RAW_OUTPUTS(file_name VARCHAR, bucket_name VARCHAR) RETURNS STRING LANGUAGE SQL EXECUTE AS CALLER @@ -14,26 +14,25 @@ BEGIN -- Create or replace stage with dynamic file name -- TODO: Update for UAT + -- s3://doczyai-use2-u-infra-s3-raw-data-ingestion/ EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.DOCZY_PIPELINE_RAW_OUTPUT_STAGE STORAGE_INTEGRATION = dev_bucket_integration - URL = ''s3://doczy-dev-infra-raw-data-ingestion/doczy_pipeline_output/' || :file_name || ''' + URL = ''s3://doczyai-use2-u-infra-s3-raw-data-ingestion/doczy_pipeline_output/' || :file_name || ''' FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');'; call stg.log_audit(:procedure_name, 'Section 1', 99, 'END'); call stg.log_audit(:procedure_name, 'Section 2', 99, 'START'); - COPY INTO STG.DOCZY_PIPELINE_RAW_OUTPUT FROM ( + COPY INTO STG.DOCZY_PIPELINE_RAW_OUTPUT_AC FROM ( SELECT NULLIF(TRIM($1),'') AS CONTRACT_NAME, - NULLIF(TRIM($2),'') AS SF_DB_COL_NAME, - NULLIF(TRIM($3),'') AS ACTUAL_VALUE_STORED, + NULLIF(TRIM($2),'') AS FIELD_NAME, NULLIF(TRIM($4),'') AS RAW_VALUE, NULLIF(TRIM($5),'') AS NEW_EXTRACTED_VALUE, NULLIF(TRIM($6),'') AS SNIPPET, - NULLIF(TRIM($7),'') AS ORIGINAL_PAGE_NUMBER, - NULLIF(TRIM($8),'') AS NEW_PAGE_NUMBER, - NULLIF(TRIM($9),'') AS RESULT + NULLIF(TRIM($8),'') AS PAGE_NUMBER, + NULLIF(TRIM($9),'') AS REVISED_PROMPT NULLIF(TRIM($10),'') AS BATCH_ID FROM @STG.DOCZY_PIPELINE_RAW_OUTPUT_STAGE