diff --git a/devops-pipeline/other-resources/variables.tf b/devops-pipeline/other-resources/variables.tf index dbbfd87..1035b4d 100644 --- a/devops-pipeline/other-resources/variables.tf +++ b/devops-pipeline/other-resources/variables.tf @@ -1,7 +1,7 @@ # required variable "aws_profile" { type = string - default = "doczyai" + # default = "doczyai" } # # required diff --git a/snowflake/scripts/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql b/snowflake/scripts/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql index cc1340b..4e5562c 100644 --- a/snowflake/scripts/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql +++ b/snowflake/scripts/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql @@ -58,7 +58,7 @@ BEGIN call stg.log_audit(:procedure_name, 'Section 3', 99, 'END'); call stg.log_audit(:procedure_name, 'Section 4', 99, 'START'); - INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) + INSERT INTO STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) SELECT STG.AUDIT_SID.NEXTVAL, :procedure_name,* FROM (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.CONTRACT_CONFIG_STAGE group by 1,2); diff --git a/snowflake/scripts/training_interface/R__002_LOAD_TRAINING_RESULTS_SP.SQL b/snowflake/scripts/training_interface/R__002_LOAD_TRAINING_RESULTS_SP.SQL index e3f20af..bd66aab 100644 --- a/snowflake/scripts/training_interface/R__002_LOAD_TRAINING_RESULTS_SP.SQL +++ b/snowflake/scripts/training_interface/R__002_LOAD_TRAINING_RESULTS_SP.SQL @@ -45,10 +45,10 @@ BEGIN call stg.log_audit(:procedure_name, 'Section 3', 99, 'START'); - INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) + INSERT INTO STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) SELECT STG.AUDIT_SID.NEXTVAL, 'STG.TRAINING_RESULTS',* FROM - (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @DOCZY_DEV.STG.TRAINING_RESULTS_STAGE group by 1,2); + (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.TRAINING_RESULTS_STAGE group by 1,2); call stg.log_audit(:procedure_name, 'Section 3', 99, 'END'); diff --git a/snowflake/scripts/training_interface/R__003_LOAD_ATTEMPTS_SP.sql b/snowflake/scripts/training_interface/R__003_LOAD_ATTEMPTS_SP.sql index 3699dcc..3bcac99 100644 --- a/snowflake/scripts/training_interface/R__003_LOAD_ATTEMPTS_SP.sql +++ b/snowflake/scripts/training_interface/R__003_LOAD_ATTEMPTS_SP.sql @@ -41,10 +41,10 @@ BEGIN call stg.log_audit(:procedure_name, 'Section 3', 99, 'START'); - INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) + INSERT INTO STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) SELECT STG.AUDIT_SID.NEXTVAL, 'STG.TRAINING_ATTEMPT_LOGS',* FROM - (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @DOCZY_DEV.STG.ATTEMPT_LOGS_STAGE group by 1,2); + (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.ATTEMPT_LOGS_STAGE group by 1,2); call stg.log_audit(:procedure_name, 'Section 3', 99, 'END'); diff --git a/snowflake/scripts/training_interface/R__005_LOAD_COLUMN_CONFIG_SP.sql b/snowflake/scripts/training_interface/R__005_LOAD_COLUMN_CONFIG_SP.sql index 5248a83..3226f74 100644 --- a/snowflake/scripts/training_interface/R__005_LOAD_COLUMN_CONFIG_SP.sql +++ b/snowflake/scripts/training_interface/R__005_LOAD_COLUMN_CONFIG_SP.sql @@ -43,7 +43,7 @@ BEGIN call stg.log_audit(:procedure_name, 'Section 4', 99, 'START'); - INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) + INSERT INTO STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) SELECT STG.AUDIT_SID.NEXTVAL, :procedure_name,* FROM (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.COLUMN_CONFIG_STAGE group by 1,2); diff --git a/snowflake/scripts/training_interface/R__006_LOAD_TRAINING_DATA_RAW_SP.sql b/snowflake/scripts/training_interface/R__006_LOAD_TRAINING_DATA_RAW_SP.sql index f1bbc08..51ad3c7 100644 --- a/snowflake/scripts/training_interface/R__006_LOAD_TRAINING_DATA_RAW_SP.sql +++ b/snowflake/scripts/training_interface/R__006_LOAD_TRAINING_DATA_RAW_SP.sql @@ -38,7 +38,7 @@ BEGIN call stg.log_audit(:procedure_name, 'Section 4', 99, 'START'); - INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) + INSERT INTO STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) SELECT STG.AUDIT_SID.NEXTVAL, :procedure_name,* FROM (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.RAW_TRAINING_DATA_STAGE group by 1,2); diff --git a/snowflake/scripts/training_interface/R__008_LOAD_BUSINESS_CONFIG_SP.sql b/snowflake/scripts/training_interface/R__008_LOAD_BUSINESS_CONFIG_SP.sql index 13a750e..7c5aa0b 100644 --- a/snowflake/scripts/training_interface/R__008_LOAD_BUSINESS_CONFIG_SP.sql +++ b/snowflake/scripts/training_interface/R__008_LOAD_BUSINESS_CONFIG_SP.sql @@ -48,7 +48,7 @@ BEGIN call stg.log_audit(:procedure_name, 'Section 4', 99, 'START'); - INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) + INSERT INTO STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) SELECT STG.AUDIT_SID.NEXTVAL, :procedure_name,* FROM (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.BUSINESS_CONFIG_STAGE group by 1,2); diff --git a/snowflake/scripts/upload_interface/R__002_LOAD_CLIENT_CONFIG_SP.sql b/snowflake/scripts/upload_interface/R__002_LOAD_CLIENT_CONFIG_SP.sql index f5b268f..c398598 100644 --- a/snowflake/scripts/upload_interface/R__002_LOAD_CLIENT_CONFIG_SP.sql +++ b/snowflake/scripts/upload_interface/R__002_LOAD_CLIENT_CONFIG_SP.sql @@ -45,7 +45,7 @@ BEGIN call stg.log_audit(:procedure_name, 'Section 4', 99, 'START'); - INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) + INSERT INTO STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) SELECT STG.AUDIT_SID.NEXTVAL, :procedure_name,* FROM (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.CLIENT_CONFIG_STAGE group by 1,2); diff --git a/textract-pipeline/src/lambda/prompt-processor/index.py b/textract-pipeline/src/lambda/prompt-processor/index.py index a8413a3..30cce0f 100644 --- a/textract-pipeline/src/lambda/prompt-processor/index.py +++ b/textract-pipeline/src/lambda/prompt-processor/index.py @@ -108,7 +108,7 @@ def lambda_handler(event, context): # Download the file content from S3 file_content = read_text_from_s3(file_name_with_path) - prompt_config_query = "SELECT LISTAGG(CONCAT('[{\"FIELD_NAME:\"',FIELD_NAME,'\"},{\"PROMPT\":\"',PROMPT,'\"}]'),',') FROM DOCZY_DEV.STG.PROMPT_CONFIG WHERE FIELD_NAME IN ("+required_field_list+")" + prompt_config_query = "SELECT LISTAGG(CONCAT('[{\"FIELD_NAME:\"',FIELD_NAME,'\"},{\"PROMPT\":\"',PROMPT,'\"}]'),',') FROM STG.PROMPT_CONFIG WHERE FIELD_NAME IN ("+required_field_list+")" field_prompt_group_response = call_snowflake_db_select_lambda(prompt_config_query) data_dict = json.loads(field_prompt_group_response)