jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Removed explicit databases from snowflake SPs

Browse Source
This commit is contained in:
Umang Mistry
2024-05-23 15:10:43 -05:00
parent 5cf104d40e
commit bca6288b1c
9 changed files with 11 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
devops-pipeline/other-resources/variables.tf
+1 -1
View File
@@ -1,7 +1,7 @@
# required
variable "aws_profile" {
type = string
default = "doczyai"
# default = "doczyai"
}
# # required
snowflake/scripts/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql
+1 -1
View File
@@ -58,7 +58,7 @@ BEGIN
call stg.log_audit(:procedure_name, 'Section 3', 99, 'END');
call stg.log_audit(:procedure_name, 'Section 4', 99, 'START');
INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT)
INSERT INTO STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT)
SELECT STG.AUDIT_SID.NEXTVAL, :procedure_name,*
FROM
(SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.CONTRACT_CONFIG_STAGE group by 1,2);
snowflake/scripts/training_interface/R__002_LOAD_TRAINING_RESULTS_SP.SQL
+2 -2
View File
@@ -45,10 +45,10 @@ BEGIN
call stg.log_audit(:procedure_name, 'Section 3', 99, 'START');
INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT)
INSERT INTO STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT)
SELECT STG.AUDIT_SID.NEXTVAL, 'STG.TRAINING_RESULTS',*
FROM
(SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @DOCZY_DEV.STG.TRAINING_RESULTS_STAGE group by 1,2);
(SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.TRAINING_RESULTS_STAGE group by 1,2);
call stg.log_audit(:procedure_name, 'Section 3', 99, 'END');
snowflake/scripts/training_interface/R__003_LOAD_ATTEMPTS_SP.sql
+2 -2
View File
@@ -41,10 +41,10 @@ BEGIN
call stg.log_audit(:procedure_name, 'Section 3', 99, 'START');
INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT)
INSERT INTO STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT)
SELECT STG.AUDIT_SID.NEXTVAL, 'STG.TRAINING_ATTEMPT_LOGS',*
FROM
(SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @DOCZY_DEV.STG.ATTEMPT_LOGS_STAGE group by 1,2);
(SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.ATTEMPT_LOGS_STAGE group by 1,2);
call stg.log_audit(:procedure_name, 'Section 3', 99, 'END');
snowflake/scripts/training_interface/R__005_LOAD_COLUMN_CONFIG_SP.sql
+1 -1
View File
@@ -43,7 +43,7 @@ BEGIN
call stg.log_audit(:procedure_name, 'Section 4', 99, 'START');
INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT)
INSERT INTO STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT)
SELECT STG.AUDIT_SID.NEXTVAL, :procedure_name,*
FROM
(SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.COLUMN_CONFIG_STAGE group by 1,2);
snowflake/scripts/training_interface/R__006_LOAD_TRAINING_DATA_RAW_SP.sql
+1 -1
View File
@@ -38,7 +38,7 @@ BEGIN
call stg.log_audit(:procedure_name, 'Section 4', 99, 'START');
INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT)
INSERT INTO STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT)
SELECT STG.AUDIT_SID.NEXTVAL, :procedure_name,*
FROM
(SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.RAW_TRAINING_DATA_STAGE group by 1,2);
snowflake/scripts/training_interface/R__008_LOAD_BUSINESS_CONFIG_SP.sql
+1 -1
View File
@@ -48,7 +48,7 @@ BEGIN
call stg.log_audit(:procedure_name, 'Section 4', 99, 'START');
INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT)
INSERT INTO STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT)
SELECT STG.AUDIT_SID.NEXTVAL, :procedure_name,*
FROM
(SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.BUSINESS_CONFIG_STAGE group by 1,2);
snowflake/scripts/upload_interface/R__002_LOAD_CLIENT_CONFIG_SP.sql
+1 -1
View File
@@ -45,7 +45,7 @@ BEGIN
call stg.log_audit(:procedure_name, 'Section 4', 99, 'START');
INSERT INTO DOCZY_DEV.STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT)
INSERT INTO STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT)
SELECT STG.AUDIT_SID.NEXTVAL, :procedure_name,*
FROM
(SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.CLIENT_CONFIG_STAGE group by 1,2);
textract-pipeline/src/lambda/prompt-processor/index.py
+1 -1
View File
@@ -108,7 +108,7 @@ def lambda_handler(event, context):
# Download the file content from S3
file_content = read_text_from_s3(file_name_with_path)
prompt_config_query = "SELECT LISTAGG(CONCAT('[{\"FIELD_NAME:\"',FIELD_NAME,'\"},{\"PROMPT\":\"',PROMPT,'\"}]'),',') FROM DOCZY_DEV.STG.PROMPT_CONFIG WHERE FIELD_NAME IN ("+required_field_list+")"
prompt_config_query = "SELECT LISTAGG(CONCAT('[{\"FIELD_NAME:\"',FIELD_NAME,'\"},{\"PROMPT\":\"',PROMPT,'\"}]'),',') FROM STG.PROMPT_CONFIG WHERE FIELD_NAME IN ("+required_field_list+")"
field_prompt_group_response = call_snowflake_db_select_lambda(prompt_config_query)
data_dict = json.loads(field_prompt_group_response)