From 9968b8c898fcca78c24adee2e08d8295669599ba Mon Sep 17 00:00:00 2001 From: Umang Mistry Date: Fri, 24 May 2024 16:53:54 -0500 Subject: [PATCH] Separated envs for DEV & UAT for Snowflake. Removed explicit role in TF configs. Made changes in pipeline.yml. Added branching readme --- bitbucket-pipelines.yml | 85 ++++++++++--------- devops-pipeline/other-resources/main.tf | 2 +- readme.md | 65 ++++++++++++++ .../{scripts => DEV}/R__001_LOG_AUDIT.sql | 0 .../R__002_LOG_AUDIT_TABLE.sql | 0 .../R__001_PROMPT_CONFIG_TABLE.sql | 0 .../R__002_CONTRACT_CONFIG_TABLE.sql | 0 .../R__003_REQUEST_SUBMISSION_TABLE.sql | 0 .../R__004_LOAD_CONTRACT_CONFIG_SP.sql | 0 .../R__005_LOAD_REQUEST_SUBMISSION_SP.sql | 0 .../R__006_PIPELINE_OUTPUT.sql | 0 ...__007_LOAD_DOCZY_PIPELINE_RAW_OPUTPUTS.sql | 0 .../R__001_SERVERLESS_LOGS_TABLE.sql | 0 .../training_interface/R__001_INIT_OBJS.sql | 0 .../R__002_LOAD_TRAINING_RESULTS_SP.SQL | 0 .../R__003_LOAD_ATTEMPTS_SP.sql | 0 .../R__004_CREATE_TRAINING_DATA_TABLE_SP.sql | 0 .../R__005_LOAD_COLUMN_CONFIG_SP.sql | 0 .../R__006_LOAD_TRAINING_DATA_RAW_SP.sql | 0 .../R__007_CREATE_BUSINESS_CONFIG_objects.sql | 0 .../R__008_LOAD_BUSINESS_CONFIG_SP.sql | 0 .../R__001_CLIENT_CONFIG_TABLE.sql | 0 .../R__002_LOAD_CLIENT_CONFIG_SP.sql | 0 .../R__003_CONTRACT_UPLOAD_LOGS.sql | 0 snowflake/UAT/R__001_LOG_AUDIT.sql | 29 +++++++ snowflake/UAT/R__002_LOG_AUDIT_TABLE.sql | 11 +++ .../R__001_PROMPT_CONFIG_TABLE.sql | 13 +++ .../R__002_CONTRACT_CONFIG_TABLE.sql | 16 ++++ .../R__003_REQUEST_SUBMISSION_TABLE.sql | 6 ++ .../R__004_LOAD_CONTRACT_CONFIG_SP.sql | 72 ++++++++++++++++ .../R__005_LOAD_REQUEST_SUBMISSION_SP.sql | 57 +++++++++++++ .../R__006_PIPELINE_OUTPUT.sql | 15 ++++ ...__007_LOAD_DOCZY_PIPELINE_RAW_OPUTPUTS.sql | 64 ++++++++++++++ .../R__001_SERVERLESS_LOGS_TABLE.sql | 44 ++++++++++ .../training_interface/R__001_INIT_OBJS.sql | 66 ++++++++++++++ .../R__002_LOAD_TRAINING_RESULTS_SP.SQL | 58 +++++++++++++ .../R__003_LOAD_ATTEMPTS_SP.sql | 55 ++++++++++++ .../R__004_CREATE_TRAINING_DATA_TABLE_SP.sql | 28 ++++++ .../R__005_LOAD_COLUMN_CONFIG_SP.sql | 57 +++++++++++++ .../R__006_LOAD_TRAINING_DATA_RAW_SP.sql | 52 ++++++++++++ .../R__007_CREATE_BUSINESS_CONFIG_objects.sql | 8 ++ .../R__008_LOAD_BUSINESS_CONFIG_SP.sql | 62 ++++++++++++++ .../R__001_CLIENT_CONFIG_TABLE.sql | 7 ++ .../R__002_LOAD_CLIENT_CONFIG_SP.sql | 59 +++++++++++++ .../R__003_CONTRACT_UPLOAD_LOGS.sql | 10 +++ terraform-approach/terraform/main.tf | 2 +- terraform-approach/terraform/variables.tf | 2 +- textract-pipeline/terraform/main.tf | 2 +- textract-pipeline/terraform/variables.tf | 2 +- 49 files changed, 905 insertions(+), 44 deletions(-) create mode 100644 readme.md rename snowflake/{scripts => DEV}/R__001_LOG_AUDIT.sql (100%) rename snowflake/{scripts => DEV}/R__002_LOG_AUDIT_TABLE.sql (100%) rename snowflake/{scripts => DEV}/config_interface/R__001_PROMPT_CONFIG_TABLE.sql (100%) rename snowflake/{scripts => DEV}/config_interface/R__002_CONTRACT_CONFIG_TABLE.sql (100%) rename snowflake/{scripts => DEV}/config_interface/R__003_REQUEST_SUBMISSION_TABLE.sql (100%) rename snowflake/{scripts => DEV}/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql (100%) rename snowflake/{scripts => DEV}/config_interface/R__005_LOAD_REQUEST_SUBMISSION_SP.sql (100%) rename snowflake/{scripts => DEV}/config_interface/R__006_PIPELINE_OUTPUT.sql (100%) rename snowflake/{scripts => DEV}/config_interface/R__007_LOAD_DOCZY_PIPELINE_RAW_OPUTPUTS.sql (100%) rename snowflake/{scripts => DEV}/system_wide/R__001_SERVERLESS_LOGS_TABLE.sql (100%) rename snowflake/{scripts => DEV}/training_interface/R__001_INIT_OBJS.sql (100%) rename snowflake/{scripts => DEV}/training_interface/R__002_LOAD_TRAINING_RESULTS_SP.SQL (100%) rename snowflake/{scripts => DEV}/training_interface/R__003_LOAD_ATTEMPTS_SP.sql (100%) rename snowflake/{scripts => DEV}/training_interface/R__004_CREATE_TRAINING_DATA_TABLE_SP.sql (100%) rename snowflake/{scripts => DEV}/training_interface/R__005_LOAD_COLUMN_CONFIG_SP.sql (100%) rename snowflake/{scripts => DEV}/training_interface/R__006_LOAD_TRAINING_DATA_RAW_SP.sql (100%) rename snowflake/{scripts => DEV}/training_interface/R__007_CREATE_BUSINESS_CONFIG_objects.sql (100%) rename snowflake/{scripts => DEV}/training_interface/R__008_LOAD_BUSINESS_CONFIG_SP.sql (100%) rename snowflake/{scripts => DEV}/upload_interface/R__001_CLIENT_CONFIG_TABLE.sql (100%) rename snowflake/{scripts => DEV}/upload_interface/R__002_LOAD_CLIENT_CONFIG_SP.sql (100%) rename snowflake/{scripts => DEV}/upload_interface/R__003_CONTRACT_UPLOAD_LOGS.sql (100%) create mode 100644 snowflake/UAT/R__001_LOG_AUDIT.sql create mode 100644 snowflake/UAT/R__002_LOG_AUDIT_TABLE.sql create mode 100644 snowflake/UAT/config_interface/R__001_PROMPT_CONFIG_TABLE.sql create mode 100644 snowflake/UAT/config_interface/R__002_CONTRACT_CONFIG_TABLE.sql create mode 100644 snowflake/UAT/config_interface/R__003_REQUEST_SUBMISSION_TABLE.sql create mode 100644 snowflake/UAT/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql create mode 100644 snowflake/UAT/config_interface/R__005_LOAD_REQUEST_SUBMISSION_SP.sql create mode 100644 snowflake/UAT/config_interface/R__006_PIPELINE_OUTPUT.sql create mode 100644 snowflake/UAT/config_interface/R__007_LOAD_DOCZY_PIPELINE_RAW_OPUTPUTS.sql create mode 100644 snowflake/UAT/system_wide/R__001_SERVERLESS_LOGS_TABLE.sql create mode 100644 snowflake/UAT/training_interface/R__001_INIT_OBJS.sql create mode 100644 snowflake/UAT/training_interface/R__002_LOAD_TRAINING_RESULTS_SP.SQL create mode 100644 snowflake/UAT/training_interface/R__003_LOAD_ATTEMPTS_SP.sql create mode 100644 snowflake/UAT/training_interface/R__004_CREATE_TRAINING_DATA_TABLE_SP.sql create mode 100644 snowflake/UAT/training_interface/R__005_LOAD_COLUMN_CONFIG_SP.sql create mode 100644 snowflake/UAT/training_interface/R__006_LOAD_TRAINING_DATA_RAW_SP.sql create mode 100644 snowflake/UAT/training_interface/R__007_CREATE_BUSINESS_CONFIG_objects.sql create mode 100644 snowflake/UAT/training_interface/R__008_LOAD_BUSINESS_CONFIG_SP.sql create mode 100644 snowflake/UAT/upload_interface/R__001_CLIENT_CONFIG_TABLE.sql create mode 100644 snowflake/UAT/upload_interface/R__002_LOAD_CLIENT_CONFIG_SP.sql create mode 100644 snowflake/UAT/upload_interface/R__003_CONTRACT_UPLOAD_LOGS.sql diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 71453c7..78856de 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -109,7 +109,9 @@ pipelines: condition: changesets: includePaths: - - snowflake/**/* + - snowflake/DEV/* + - snowflake/DEV/**/* + - snowflake/DEV/**/**/* - step: image: python:3.8 name: "Deploy streamlit to EC2" @@ -186,47 +188,11 @@ pipelines: # UAT and PROD pipelines will be updated with Streamlit steps once it is tested and ready UAT: - - step: - <<: *snowflake_deploy - condition: - changesets: - includePaths: - - snowflake/**/* - - step: - <<: *streamlit_deploy - condition: - changesets: - includePaths: - - "streamlit/*" - - "streamlit/**/*" - - step: - <<: *airflow_dags_deploy - condition: - changesets: - includePaths: - - airflow/dags/* - - step: - name: "Plan & Apply Textract Pipeline" - image: hugree/bitbucket-aws-python38-tf154:latest - oidc: true - trigger : automatic - script: - - *aws-context-uat - - cd textract-pipeline/terraform/ - - terraform validate - # Ensure the backend configuration for s3 obj key is corresponding to the stream name - - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/textract-pipeline/terraform.tfstate" - - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" - condition: - changesets: - includePaths: - - textract-pipeline/terraform/* - - textract-pipeline/terraform/**/* - step: name: "Plan & Apply DevOps Pipeline" image: hugree/bitbucket-aws-python38-tf154:latest oidc: true - trigger : automatic # Keeping it manual so that it doesnt conflict with existing resources + trigger : manual # Keeping it manual so that it doesnt conflict with existing resources script: - *aws-context-uat - cd devops-pipeline/terraform-backend-resources/ @@ -243,7 +209,7 @@ pipelines: name: "Plan & Apply Streamlit Pipeline" image: hugree/bitbucket-aws-python38-tf154:latest oidc: true - trigger : automatic + trigger : manual script: - *aws-context-uat - cd streamlit-server/ @@ -256,6 +222,47 @@ pipelines: includePaths: - streamlit-server/* - streamlit-server/**/* + - step: + <<: *snowflake_deploy + condition: + changesets: + includePaths: + - snowflake/UAT/* + - snowflake/UAT/**/* + - snowflake/UAT/**/**/* + - step: + <<: *streamlit_deploy + condition: + changesets: + includePaths: + - "streamlit/*" + - "streamlit/**/*" + - step: + <<: *airflow_dags_deploy + trigger: manual + condition: + changesets: + includePaths: + - airflow/dags/* + - step: + name: "Plan & Apply Textract Pipeline" + image: hugree/bitbucket-aws-python38-tf154:latest + oidc: true + trigger : manual + script: + - *aws-context-uat + - cd textract-pipeline/terraform/ + - terraform validate + # Ensure the backend configuration for s3 obj key is corresponding to the stream name + - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/textract-pipeline/terraform.tfstate" + - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" + condition: + changesets: + includePaths: + - textract-pipeline/terraform/* + - textract-pipeline/terraform/**/* + + PROD: - step: <<: *terraform_plan_apply diff --git a/devops-pipeline/other-resources/main.tf b/devops-pipeline/other-resources/main.tf index d9febef..bc17c4e 100644 --- a/devops-pipeline/other-resources/main.tf +++ b/devops-pipeline/other-resources/main.tf @@ -9,7 +9,7 @@ terraform { bucket = "doczyai-use2-d-infra-s3-terraform-state" # Parameterize using -backend-config flag with "terraform init" key = "terraform/devops-pipeline/devops.tfstate" # Parameterize region = "us-east-2" # Parameterize - profile = "doczyai" # Parameterize + # profile = "doczyai" # Parameterize dynamodb_table = "doczyai-use2-d-infra-dyd-terraform-lock" # Parameterize encrypt = true } diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..441375e --- /dev/null +++ b/readme.md @@ -0,0 +1,65 @@ +# Branching and Release Management Guide + +## Overview + +This guide outlines the naming conventions, branching strategy, and merging process for our Git repository. + +## Branch Naming Conventions + +### Feature Branches + +- **Naming Convention**: `feature/_` + - Example: `feature/streamlit_UI0_1.0`, `feature/textract_1.5` + +## Branching Strategy + +### Creating Feature Branches + +1. **From Release Branch**: Create a feature branch from `release/uat_` + - Example: To develop Streamlit UI1 v1.5, create `feature/streamlit_UI1_1.5` from `release/uat_1.0` + +### Merging Process + +#### Merging to DEV + +1. **Initial Merge**: Once the feature is ready, merge the feature branch into DEV. +2. **Verification**: Ensure the functionality is verified in DEV. + +#### Merging to UAT + +1. **Approval Requirement**: Merge to UAT requires approval from default reviewers (Anup / Priya / Umang). +2. **Post-Approval Merge**: After approval, merge the same feature branch that was verified in DEV into UAT. + +#### Merging to PROD + +1. **Final Merge**: Once the feature is tested and approved in UAT, merge it into PROD. +2. **Complete Versions**: Deployment to PROD should only be done with complete versions (e.g., v1.0, v2.0) with an appropriate release branch in place. + +## State of Branches + +### After Initial Release (v1.0) +| Workstream | DEV | UAT | PROD | +|------------|-----|-----|------| +| Streamlit | 1.0 | 1.0 | 1.0 | +| Textract | 1.0 | 1.0 | 1.0 | +| DevOps | 1.0 | 1.0 | 1.0 | +| Data | 1.0 | 1.0 | 1.0 | + +### After Streamlit 1.5 and Textract 1.5 +| Workstream | DEV | UAT | PROD | +|------------|-----|-----|------| +| Streamlit | 1.5 | 1.5 | 1.0 | +| Textract | 1.5 | 1.5 | 1.0 | +| DevOps | 1.0 | 1.0 | 1.0 | +| Data | 1.0 | 1.0 | 1.0 | + +## Merge Restrictions + +- **Workstream Specific Changes**: For feature branches, only changes in the respective folder are allowed. Other changes require a separate feature branch. This ensures that workstream changes are deployed to the respective infra. +- **Merge Approval**: Merging into DEV, UAT or PROD requires approval from default reviewers. + +## Refer to the branching guide in Confluence for more info on this + +--- + +This guide ensures that all team members follow the best practices for branch management and releases. diff --git a/snowflake/scripts/R__001_LOG_AUDIT.sql b/snowflake/DEV/R__001_LOG_AUDIT.sql similarity index 100% rename from snowflake/scripts/R__001_LOG_AUDIT.sql rename to snowflake/DEV/R__001_LOG_AUDIT.sql diff --git a/snowflake/scripts/R__002_LOG_AUDIT_TABLE.sql b/snowflake/DEV/R__002_LOG_AUDIT_TABLE.sql similarity index 100% rename from snowflake/scripts/R__002_LOG_AUDIT_TABLE.sql rename to snowflake/DEV/R__002_LOG_AUDIT_TABLE.sql diff --git a/snowflake/scripts/config_interface/R__001_PROMPT_CONFIG_TABLE.sql b/snowflake/DEV/config_interface/R__001_PROMPT_CONFIG_TABLE.sql similarity index 100% rename from snowflake/scripts/config_interface/R__001_PROMPT_CONFIG_TABLE.sql rename to snowflake/DEV/config_interface/R__001_PROMPT_CONFIG_TABLE.sql diff --git a/snowflake/scripts/config_interface/R__002_CONTRACT_CONFIG_TABLE.sql b/snowflake/DEV/config_interface/R__002_CONTRACT_CONFIG_TABLE.sql similarity index 100% rename from snowflake/scripts/config_interface/R__002_CONTRACT_CONFIG_TABLE.sql rename to snowflake/DEV/config_interface/R__002_CONTRACT_CONFIG_TABLE.sql diff --git a/snowflake/scripts/config_interface/R__003_REQUEST_SUBMISSION_TABLE.sql b/snowflake/DEV/config_interface/R__003_REQUEST_SUBMISSION_TABLE.sql similarity index 100% rename from snowflake/scripts/config_interface/R__003_REQUEST_SUBMISSION_TABLE.sql rename to snowflake/DEV/config_interface/R__003_REQUEST_SUBMISSION_TABLE.sql diff --git a/snowflake/scripts/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql b/snowflake/DEV/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql similarity index 100% rename from snowflake/scripts/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql rename to snowflake/DEV/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql diff --git a/snowflake/scripts/config_interface/R__005_LOAD_REQUEST_SUBMISSION_SP.sql b/snowflake/DEV/config_interface/R__005_LOAD_REQUEST_SUBMISSION_SP.sql similarity index 100% rename from snowflake/scripts/config_interface/R__005_LOAD_REQUEST_SUBMISSION_SP.sql rename to snowflake/DEV/config_interface/R__005_LOAD_REQUEST_SUBMISSION_SP.sql diff --git a/snowflake/scripts/config_interface/R__006_PIPELINE_OUTPUT.sql b/snowflake/DEV/config_interface/R__006_PIPELINE_OUTPUT.sql similarity index 100% rename from snowflake/scripts/config_interface/R__006_PIPELINE_OUTPUT.sql rename to snowflake/DEV/config_interface/R__006_PIPELINE_OUTPUT.sql diff --git a/snowflake/scripts/config_interface/R__007_LOAD_DOCZY_PIPELINE_RAW_OPUTPUTS.sql b/snowflake/DEV/config_interface/R__007_LOAD_DOCZY_PIPELINE_RAW_OPUTPUTS.sql similarity index 100% rename from snowflake/scripts/config_interface/R__007_LOAD_DOCZY_PIPELINE_RAW_OPUTPUTS.sql rename to snowflake/DEV/config_interface/R__007_LOAD_DOCZY_PIPELINE_RAW_OPUTPUTS.sql diff --git a/snowflake/scripts/system_wide/R__001_SERVERLESS_LOGS_TABLE.sql b/snowflake/DEV/system_wide/R__001_SERVERLESS_LOGS_TABLE.sql similarity index 100% rename from snowflake/scripts/system_wide/R__001_SERVERLESS_LOGS_TABLE.sql rename to snowflake/DEV/system_wide/R__001_SERVERLESS_LOGS_TABLE.sql diff --git a/snowflake/scripts/training_interface/R__001_INIT_OBJS.sql b/snowflake/DEV/training_interface/R__001_INIT_OBJS.sql similarity index 100% rename from snowflake/scripts/training_interface/R__001_INIT_OBJS.sql rename to snowflake/DEV/training_interface/R__001_INIT_OBJS.sql diff --git a/snowflake/scripts/training_interface/R__002_LOAD_TRAINING_RESULTS_SP.SQL b/snowflake/DEV/training_interface/R__002_LOAD_TRAINING_RESULTS_SP.SQL similarity index 100% rename from snowflake/scripts/training_interface/R__002_LOAD_TRAINING_RESULTS_SP.SQL rename to snowflake/DEV/training_interface/R__002_LOAD_TRAINING_RESULTS_SP.SQL diff --git a/snowflake/scripts/training_interface/R__003_LOAD_ATTEMPTS_SP.sql b/snowflake/DEV/training_interface/R__003_LOAD_ATTEMPTS_SP.sql similarity index 100% rename from snowflake/scripts/training_interface/R__003_LOAD_ATTEMPTS_SP.sql rename to snowflake/DEV/training_interface/R__003_LOAD_ATTEMPTS_SP.sql diff --git a/snowflake/scripts/training_interface/R__004_CREATE_TRAINING_DATA_TABLE_SP.sql b/snowflake/DEV/training_interface/R__004_CREATE_TRAINING_DATA_TABLE_SP.sql similarity index 100% rename from snowflake/scripts/training_interface/R__004_CREATE_TRAINING_DATA_TABLE_SP.sql rename to snowflake/DEV/training_interface/R__004_CREATE_TRAINING_DATA_TABLE_SP.sql diff --git a/snowflake/scripts/training_interface/R__005_LOAD_COLUMN_CONFIG_SP.sql b/snowflake/DEV/training_interface/R__005_LOAD_COLUMN_CONFIG_SP.sql similarity index 100% rename from snowflake/scripts/training_interface/R__005_LOAD_COLUMN_CONFIG_SP.sql rename to snowflake/DEV/training_interface/R__005_LOAD_COLUMN_CONFIG_SP.sql diff --git a/snowflake/scripts/training_interface/R__006_LOAD_TRAINING_DATA_RAW_SP.sql b/snowflake/DEV/training_interface/R__006_LOAD_TRAINING_DATA_RAW_SP.sql similarity index 100% rename from snowflake/scripts/training_interface/R__006_LOAD_TRAINING_DATA_RAW_SP.sql rename to snowflake/DEV/training_interface/R__006_LOAD_TRAINING_DATA_RAW_SP.sql diff --git a/snowflake/scripts/training_interface/R__007_CREATE_BUSINESS_CONFIG_objects.sql b/snowflake/DEV/training_interface/R__007_CREATE_BUSINESS_CONFIG_objects.sql similarity index 100% rename from snowflake/scripts/training_interface/R__007_CREATE_BUSINESS_CONFIG_objects.sql rename to snowflake/DEV/training_interface/R__007_CREATE_BUSINESS_CONFIG_objects.sql diff --git a/snowflake/scripts/training_interface/R__008_LOAD_BUSINESS_CONFIG_SP.sql b/snowflake/DEV/training_interface/R__008_LOAD_BUSINESS_CONFIG_SP.sql similarity index 100% rename from snowflake/scripts/training_interface/R__008_LOAD_BUSINESS_CONFIG_SP.sql rename to snowflake/DEV/training_interface/R__008_LOAD_BUSINESS_CONFIG_SP.sql diff --git a/snowflake/scripts/upload_interface/R__001_CLIENT_CONFIG_TABLE.sql b/snowflake/DEV/upload_interface/R__001_CLIENT_CONFIG_TABLE.sql similarity index 100% rename from snowflake/scripts/upload_interface/R__001_CLIENT_CONFIG_TABLE.sql rename to snowflake/DEV/upload_interface/R__001_CLIENT_CONFIG_TABLE.sql diff --git a/snowflake/scripts/upload_interface/R__002_LOAD_CLIENT_CONFIG_SP.sql b/snowflake/DEV/upload_interface/R__002_LOAD_CLIENT_CONFIG_SP.sql similarity index 100% rename from snowflake/scripts/upload_interface/R__002_LOAD_CLIENT_CONFIG_SP.sql rename to snowflake/DEV/upload_interface/R__002_LOAD_CLIENT_CONFIG_SP.sql diff --git a/snowflake/scripts/upload_interface/R__003_CONTRACT_UPLOAD_LOGS.sql b/snowflake/DEV/upload_interface/R__003_CONTRACT_UPLOAD_LOGS.sql similarity index 100% rename from snowflake/scripts/upload_interface/R__003_CONTRACT_UPLOAD_LOGS.sql rename to snowflake/DEV/upload_interface/R__003_CONTRACT_UPLOAD_LOGS.sql diff --git a/snowflake/UAT/R__001_LOG_AUDIT.sql b/snowflake/UAT/R__001_LOG_AUDIT.sql new file mode 100644 index 0000000..75fc24c --- /dev/null +++ b/snowflake/UAT/R__001_LOG_AUDIT.sql @@ -0,0 +1,29 @@ +CREATE OR REPLACE PROCEDURE STG.LOG_AUDIT("PROC_NAME" VARCHAR(255), "SUB_SECTION_NAME" VARCHAR(255), "AUDIT_SID" FLOAT, "ACTION_FLAG" VARCHAR(255)) +RETURNS VARCHAR(16777216) +LANGUAGE SQL +EXECUTE AS CALLER +AS +' +BEGIN + BEGIN TRANSACTION; + + IF (:ACTION_FLAG = ''START'') THEN + INSERT INTO STG.LOG_AUDIT (EXECUTION_DT, START_DT, PROCEDURE_NAME, SUB_SECTION_NAME, AUDIT_SID) + VALUES (CURRENT_DATE(), CURRENT_TIMESTAMP(), :PROC_NAME, :SUB_SECTION_NAME, :AUDIT_SID); + END IF; + + IF (:ACTION_FLAG = ''END'') THEN + UPDATE STG.LOG_AUDIT + SET QUERY_ID = LAST_QUERY_ID(-2), END_DT = CURRENT_TIMESTAMP(), TOTAL_MIN = DATEDIFF(Minute, START_DT, CURRENT_TIMESTAMP()) + WHERE PROCEDURE_NAME = :PROC_NAME AND SUB_SECTION_NAME = :SUB_SECTION_NAME AND AUDIT_SID = :AUDIT_SID AND END_DT IS NULL; + + ELSE + COMMIT; + RETURN ''Incorrect Action Flag Used''; + + END IF; + + COMMIT; + RETURN ''Completed''; +END;' +; \ No newline at end of file diff --git a/snowflake/UAT/R__002_LOG_AUDIT_TABLE.sql b/snowflake/UAT/R__002_LOG_AUDIT_TABLE.sql new file mode 100644 index 0000000..1cc0768 --- /dev/null +++ b/snowflake/UAT/R__002_LOG_AUDIT_TABLE.sql @@ -0,0 +1,11 @@ +create or replace TABLE STG.LOG_AUDIT ( + UNIQUE_KEY NUMBER(38,0) NOT NULL autoincrement start 1 increment 1 order, + EXECUTION_DT DATE, + QUERY_ID VARCHAR(100) COLLATE 'en-ci', + START_DT TIMESTAMP_NTZ(9), + END_DT TIMESTAMP_NTZ(9), + PROCEDURE_NAME VARCHAR(255) COLLATE 'en-ci', + SUB_SECTION_NAME VARCHAR(255) COLLATE 'en-ci', + AUDIT_SID NUMBER(38,0), + TOTAL_MIN NUMBER(38,0) +); \ No newline at end of file diff --git a/snowflake/UAT/config_interface/R__001_PROMPT_CONFIG_TABLE.sql b/snowflake/UAT/config_interface/R__001_PROMPT_CONFIG_TABLE.sql new file mode 100644 index 0000000..b94878b --- /dev/null +++ b/snowflake/UAT/config_interface/R__001_PROMPT_CONFIG_TABLE.sql @@ -0,0 +1,13 @@ +-- Separation of single vs multi prompt using a flag in the table +CREATE TABLE IF NOT EXISTS STG.PROMPT_CONFIG ( + FIELD_NAME VARCHAR, + FIELD_DESC VARCHAR, + IS_REQUIRED BOOLEAN, + FIELD_DATA_TYPE VARCHAR, + PROMPT VARCHAR, + FM_MODEL_ID VARCHAR, + GROUP_ID NUMERIC, + FIELD_MAX_LENGTH NUMBER, + SAMPLE_VALUE VARCHAR, + CONTRACT_SECTION VARCHAR +); diff --git a/snowflake/UAT/config_interface/R__002_CONTRACT_CONFIG_TABLE.sql b/snowflake/UAT/config_interface/R__002_CONTRACT_CONFIG_TABLE.sql new file mode 100644 index 0000000..4770866 --- /dev/null +++ b/snowflake/UAT/config_interface/R__002_CONTRACT_CONFIG_TABLE.sql @@ -0,0 +1,16 @@ +-- CREATING THE CONFIG TABLE +CREATE TABLE IF NOT EXISTS STG.CONTRACT_CONFIG ( + CONTRACT_NAME VARCHAR, + BATCH_ID VARCHAR, + UNIQUE_KEY BOOLEAN, + PRICING_BEFORE_CARVEOUTS BOOLEAN, + CONTRACT_RELATED BOOLEAN, + PROVIDER BOOLEAN, + TIMELINE BOOLEAN, + CARVEOUT_INDICATOR BOOLEAN, + CARVEOUT_METHODOLOGY BOOLEAN, + REQUEST_USER VARCHAR, + LATEST_FLAG BOOLEAN DEFAULT TRUE, + PIPELINE_KICKOFF_DATETIME DATETIME, + REQUEST_DATETIME DATETIME DEFAULT CURRENT_TIMESTAMP() +); \ No newline at end of file diff --git a/snowflake/UAT/config_interface/R__003_REQUEST_SUBMISSION_TABLE.sql b/snowflake/UAT/config_interface/R__003_REQUEST_SUBMISSION_TABLE.sql new file mode 100644 index 0000000..2107d73 --- /dev/null +++ b/snowflake/UAT/config_interface/R__003_REQUEST_SUBMISSION_TABLE.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS STG.REQUEST_SUBMISSION ( + CLIENT_NAME VARCHAR, + BATCH_ID VARCHAR, + REQUEST_USERNAME VARCHAR, + REQUEST_DATETIME DATETIME +); diff --git a/snowflake/UAT/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql b/snowflake/UAT/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql new file mode 100644 index 0000000..4e5562c --- /dev/null +++ b/snowflake/UAT/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql @@ -0,0 +1,72 @@ +CREATE OR REPLACE PROCEDURE STG.LOAD_CONTRACT_CONFIG(file_name VARCHAR) +RETURNS STRING +LANGUAGE SQL +EXECUTE AS CALLER +AS +$$ +DECLARE + procedure_name varchar; +BEGIN + + procedure_name := 'LOAD_CONTRACT_CONFIG'; + + call stg.log_audit(:procedure_name, 'Section 1', 99, 'START'); + + -- Create or replace stage with dynamic file name + EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.CONTRACT_CONFIG_STAGE + STORAGE_INTEGRATION = dev_bucket_integration + URL = ''s3://doczy-dev-infra-raw-data-ingestion/config_interface/' || :file_name || ''' + FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');'; + + call stg.log_audit(:procedure_name, 'Section 1', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 2', 99, 'START'); + COPY INTO STG.CONTRACT_CONFIG FROM ( + SELECT + NULLIF(TRIM($1),'') AS CONTRACT_NAME, + NULLIF(TRIM($2),'') AS BATCH_ID, + NULLIF(TRIM($3),'') AS UNIQUE_KEY, + NULLIF(TRIM($4),'') AS PRICING_BEFORE_CARVEOUTS, + NULLIF(TRIM($5),'') AS CONTRACT_RELATED, + NULLIF(TRIM($6),'') AS PROVIDER, + NULLIF(TRIM($7),'') AS TIMELINE, + NULLIF(TRIM($8),'') AS CARVEOUT_INDICATOR, + NULLIF(TRIM($9),'') AS CARVEOUT_METHODOLOGY, + NULLIF(TRIM($10),'') AS REQUEST_USER, + NULLIF(TRIM($11),'') AS LATEST_FLAG, + NULLIF(TRIM($12),'') AS PIPELINE_KICKOFF_DATETIME, + NULLIF(TRIM($13),'') AS REQUEST_DATETIME + FROM @STG.CONTRACT_CONFIG_STAGE + ) + FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER') + ON_ERROR = ABORT_STATEMENT; + + call stg.log_audit(:procedure_name, 'Section 2', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 3', 99, 'START'); + + -- Once the new data is loaded to the stage, we need to update the existing records in the table to set the LATEST_FLAG to FALSE + + contract_name := (SELECT NULLIF(TRIM($1),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1); + batch_id := (SELECT NULLIF(TRIM($2),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1); + request_datetime := (SELECT NULLIF(TRIM($13),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1); + + UPDATE TABLE STG.CONTRACT_CONFIG + SET LATEST_FLAG = FALSE + WHERE BATCH_ID = :batch_id AND CONTRACT_NAME = :contract_name AND REQUEST_DATETIME < :request_datetime AND LATEST_FLAG = TRUE; + + call stg.log_audit(:procedure_name, 'Section 3', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 4', 99, 'START'); + INSERT INTO STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) + SELECT STG.AUDIT_SID.NEXTVAL, :procedure_name,* + FROM + (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.CONTRACT_CONFIG_STAGE group by 1,2); + + RETURN 'Setup, Load, and Audit Complete'; + + call stg.log_audit(:procedure_name, 'Section 4', 99, 'END'); + +END; +$$; + diff --git a/snowflake/UAT/config_interface/R__005_LOAD_REQUEST_SUBMISSION_SP.sql b/snowflake/UAT/config_interface/R__005_LOAD_REQUEST_SUBMISSION_SP.sql new file mode 100644 index 0000000..419305b --- /dev/null +++ b/snowflake/UAT/config_interface/R__005_LOAD_REQUEST_SUBMISSION_SP.sql @@ -0,0 +1,57 @@ +CREATE OR REPLACE PROCEDURE STG.LOAD_REQUEST_SUBMISSION(file_name VARCHAR) +RETURNS STRING +LANGUAGE SQL +EXECUTE AS CALLER +AS +$$ +DECLARE + procedure_name varchar; +BEGIN + + procedure_name := 'LOAD_REQUEST_SUBMISSION'; + + call stg.log_audit(:procedure_name, 'Section 1', 99, 'START'); + + -- Create or replace stage with dynamic file name + EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.REQUEST_SUBMISSION_STAGE + STORAGE_INTEGRATION = dev_bucket_integration + URL = ''s3://doczy-dev-infra-raw-data-ingestion/config_interface/' || :file_name || ''' + FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');'; + + call stg.log_audit(:procedure_name, 'Section 1', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 2', 99, 'START'); + + COPY INTO STG.REQUEST_SUBMISSION FROM ( + SELECT + NULLIF(TRIM($2),'') AS CLIENT_NAME, + NULLIF(TRIM($3),'') AS BATCH_ID, + NULLIF(TRIM($4),'') AS REQUEST_USERNAME, + NULLIF(TRIM($5),'') AS REQUEST_DATETIME + + FROM @STG.REQUEST_SUBMISSION_STAGE + ) + FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER') + ON_ERROR = ABORT_STATEMENT; + + call stg.log_audit(:procedure_name, 'Section 2', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 3', 99, 'START'); + + INSERT INTO STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) + SELECT STG.AUDIT_SID.NEXTVAL, 'STG.REQUEST_SUBMISSION',* + FROM + (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.REQUEST_SUBMISSION_STAGE group by 1,2); + + + + RETURN 'Setup, Load, and Audit Complete'; + + call stg.log_audit(:procedure_name, 'Section 3', 99, 'END'); + + + + +END; +$$; + diff --git a/snowflake/UAT/config_interface/R__006_PIPELINE_OUTPUT.sql b/snowflake/UAT/config_interface/R__006_PIPELINE_OUTPUT.sql new file mode 100644 index 0000000..b08d1d8 --- /dev/null +++ b/snowflake/UAT/config_interface/R__006_PIPELINE_OUTPUT.sql @@ -0,0 +1,15 @@ +-- CREATING THE CONFIG TABLE +CREATE TABLE IF NOT EXISTS STG.DOCZY_PIPELINE_RAW_OUTPUT ( + DOCUMENT_ID VARCHAR(16777216) NOT NULL, + SF_DB_COL_NAME VARCHAR(255), + ACTUAL_VALUE_STORED VARCHAR(16777216), + RAW_VALUE VARCHAR(16777216), + NEW_EXTRACTED_VALUE VARCHAR(16777216), + SNIPPET VARCHAR(16777216), + ORIGINAL_PAGE_NUMBER VARCHAR(255), + NEW_PAGE_NUMBER VARCHAR(255), + RESULT VARCHAR(16777216), + BATCH_ID VARCHAR(255), + CREATED_TIME TIMESTAMP_NTZ(9), + MODIFIED_TIME TIMESTAMP_NTZ(9) +); \ No newline at end of file diff --git a/snowflake/UAT/config_interface/R__007_LOAD_DOCZY_PIPELINE_RAW_OPUTPUTS.sql b/snowflake/UAT/config_interface/R__007_LOAD_DOCZY_PIPELINE_RAW_OPUTPUTS.sql new file mode 100644 index 0000000..2729f1d --- /dev/null +++ b/snowflake/UAT/config_interface/R__007_LOAD_DOCZY_PIPELINE_RAW_OPUTPUTS.sql @@ -0,0 +1,64 @@ +CREATE OR REPLACE PROCEDURE STG.LOAD_DOCZY_PIPELINE_RAW_OUTPUTS(file_name VARCHAR) +RETURNS STRING +LANGUAGE SQL +EXECUTE AS CALLER +AS +$$ +DECLARE + procedure_name varchar; +BEGIN + + procedure_name := 'LOAD_DOCZY_PIPELINE_RAW_OUTPUTS'; + + call stg.log_audit(:procedure_name, 'Section 1', 99, 'START'); + + -- Create or replace stage with dynamic file name + -- TODO: Update for UAT + EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.DOCZY_PIPELINE_RAW_OUTPUT_STAGE + STORAGE_INTEGRATION = dev_bucket_integration + URL = ''s3://doczy-dev-infra-raw-data-ingestion/doczy_pipeline_output/' || :file_name || ''' + FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');'; + + call stg.log_audit(:procedure_name, 'Section 1', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 2', 99, 'START'); + + COPY INTO STG.DOCZY_PIPELINE_RAW_OUTPUT FROM ( + SELECT + NULLIF(TRIM($1),'') AS CONTRACT_NAME, + NULLIF(TRIM($2),'') AS SF_DB_COL_NAME, + NULLIF(TRIM($3),'') AS ACTUAL_VALUE_STORED, + NULLIF(TRIM($4),'') AS RAW_VALUE, + NULLIF(TRIM($5),'') AS NEW_EXTRACTED_VALUE, + NULLIF(TRIM($6),'') AS SNIPPET, + NULLIF(TRIM($7),'') AS ORIGINAL_PAGE_NUMBER, + NULLIF(TRIM($8),'') AS NEW_PAGE_NUMBER, + NULLIF(TRIM($9),'') AS RESULT + NULLIF(TRIM($10),'') AS BATCH_ID + + FROM @STG.DOCZY_PIPELINE_RAW_OUTPUT_STAGE + ) + FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER') + ON_ERROR = ABORT_STATEMENT; + + call stg.log_audit(:procedure_name, 'Section 2', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 3', 99, 'START'); + + INSERT INTO STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) + SELECT STG.AUDIT_SID.NEXTVAL, 'STG.DOCZY_PIPELINE_RAW_OUTPUT',* + FROM + (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.DOCZY_PIPELINE_RAW_OUTPUT_STAGE group by 1,2); + + + + RETURN 'Setup, Load, and Audit Complete'; + + call stg.log_audit(:procedure_name, 'Section 3', 99, 'END'); + + + + +END; +$$; + diff --git a/snowflake/UAT/system_wide/R__001_SERVERLESS_LOGS_TABLE.sql b/snowflake/UAT/system_wide/R__001_SERVERLESS_LOGS_TABLE.sql new file mode 100644 index 0000000..63b03cf --- /dev/null +++ b/snowflake/UAT/system_wide/R__001_SERVERLESS_LOGS_TABLE.sql @@ -0,0 +1,44 @@ + + +-- Creating the 'document_logs' table +CREATE TABLE IF NOT EXISTS STG.DOCUMENT_LOGS +( + DOCUMENT_ID VARCHAR NOT NULL, + BATCH_ID VARCHAR NOT NULL, + JOB_ID VARCHAR, + STAGE VARCHAR, + TEXTRACT_STATUS VARCHAR, + BUCKET_NAME VARCHAR NOT NULL, + FILE_NAME VARCHAR NOT NULL, + FILE_PATH VARCHAR, + DOCUMENT_TYPE VARCHAR, + PAYER_SIGNED BOOLEAN, + PROVIDER_SIGNED BOOLEAN, + GROUP_ID VARCHAR, + CREATED_TIME TIMESTAMP, + MODIFIED_TIME TIMESTAMP, + CREATED_BY VARCHAR, + MODIFIED_BY VARCHAR, + ORIGINAL_FILE_EXTENSION VARCHAR, + NO_OF_PAGES NUMERIC, + FILE_SIZE NUMERIC, + PRIMARY KEY (DOCUMENT_ID) +); + +-- Creating the 'batch_logs' table +CREATE TABLE IF NOT EXISTS STG.BATCH_LOGS ( + BATCH_ID VARCHAR NOT NULL, + CLIENT_ID VARCHAR, + EXECUTION_START_TIME TIMESTAMP, + NO_OF_DOCUMENTS NUMBER, + USER_NAME VARCHAR, + PRIMARY KEY (BATCH_ID) +); + +-- Creating the 'client_logs' table +CREATE TABLE IF NOT EXISTS STG.CLIENT_LOGS ( + CLIENT_ID VARCHAR , + CLIENT_NAME VARCHAR, + BUCKET_NAME VARCHAR, + PRIMARY KEY (CLIENT_ID) +); diff --git a/snowflake/UAT/training_interface/R__001_INIT_OBJS.sql b/snowflake/UAT/training_interface/R__001_INIT_OBJS.sql new file mode 100644 index 0000000..992ece8 --- /dev/null +++ b/snowflake/UAT/training_interface/R__001_INIT_OBJS.sql @@ -0,0 +1,66 @@ +-- Create staging table for results +CREATE TABLE IF NOT EXISTS STG.TRAINING_RESULTS +( + CONTRACT_NAME VARCHAR, + CONTRACT_ID NUMERIC, + ACTUAL_VALUE_STORED VARCHAR, + NEW_EXTRACTED_VALUE VARCHAR, + CONFIDENCE_LEVEL NUMERIC, + SNIPPET VARCHAR, + ORIGINAL_PAGE_NUMBER VARCHAR, + NEW_PAGE_NUMBER VARCHAR, + REVISED_PROMPT VARCHAR, + RESULT BOOLEAN, + LOAD_DT TIMESTAMP +); + + + -- Create file format +CREATE FILE FORMAT IF NOT EXISTS STG.CSV_HEADER + TYPE = 'CSV' + FIELD_DELIMITER = ',' + FILE_EXTENSION = '.csv' + RECORD_DELIMITER = '\\n' + DATE_FORMAT = AUTO + TRIM_SPACE = TRUE + NULL_IF = ('NULL', '', 'N/A','?','~','\\N') + SKIP_HEADER = 1 + EMPTY_FIELD_AS_NULL = TRUE + FIELD_OPTIONALLY_ENCLOSED_BY = '"' + SKIP_BLANK_LINES = TRUE; + + +-- Create staging table for attempt logs +CREATE TABLE IF NOT EXISTS STG.TRAINING_ATTEMPT_LOGS +( + FIELD_NAME VARCHAR, + CONTRACTS_TESTED NUMERIC, + USERNAME VARCHAR, + DATE_TIME TIMESTAMP, + ACCURACY NUMERIC, + ATTEMPT_NUM NUMERIC, + LOAD_DT TIMESTAMP +); + + + +-- Create table for column config that will be used by SP to create raw training data table +CREATE TABLE IF NOT EXISTS STG.TRAINING_DATA_COLUMN_CONFIG( +COLUMN_NAME VARCHAR, +COLUMN_DATATYPE VARCHAR +); + +-- Creating a separate file format for the training data +CREATE FILE FORMAT IF NOT EXISTS STG.TRAINING_DATA_FILE_FORMAT + TYPE = 'CSV' + FIELD_DELIMITER = ',' + FILE_EXTENSION = '.csv' + RECORD_DELIMITER = '\\n' + DATE_FORMAT = AUTO + TRIM_SPACE = TRUE + NULL_IF = ('NULL', '', 'N/A','?','~','\\N') + SKIP_HEADER = 1 + EMPTY_FIELD_AS_NULL = TRUE + FIELD_OPTIONALLY_ENCLOSED_BY = '"' + error_on_column_count_mismatch=false + SKIP_BLANK_LINES = TRUE; \ No newline at end of file diff --git a/snowflake/UAT/training_interface/R__002_LOAD_TRAINING_RESULTS_SP.SQL b/snowflake/UAT/training_interface/R__002_LOAD_TRAINING_RESULTS_SP.SQL new file mode 100644 index 0000000..23d3e9c --- /dev/null +++ b/snowflake/UAT/training_interface/R__002_LOAD_TRAINING_RESULTS_SP.SQL @@ -0,0 +1,58 @@ +CREATE OR REPLACE PROCEDURE STG.LOAD_TRAINING_RESULTS(file_name VARCHAR) +RETURNS STRING +LANGUAGE SQL +EXECUTE AS CALLER +AS +$$ +DECLARE + procedure_name varchar; +BEGIN + + procedure_name := 'LOAD_TRAINING_RESULTS'; + + call stg.log_audit(:procedure_name, 'Section 1', 99, 'START'); + + -- Create or replace stage with dynamic file name + -- TODO: Update for UAT + EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.TRAINING_RESULTS_STAGE + STORAGE_INTEGRATION = dev_bucket_integration + URL = ''s3://doczy-dev-infra-raw-data-ingestion/training_interface/' || :file_name || ''' + FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');'; + + call stg.log_audit(:procedure_name, 'Section 1', 99, 'END'); + call stg.log_audit(:procedure_name, 'Section 2', 99, 'START'); + -- Copy command to load data + COPY INTO STG.TRAINING_RESULTS FROM + ( + SELECT + NULLIF(TRIM($1), '') AS Contract_Name, + NULLIF(TRIM($2), '') AS Contract_ID, + NULLIF(TRIM($3), '') AS Actual_Value_Stored, + NULLIF(TRIM($4), '') AS New_Extracted_Value, + NULLIF(TRIM($5), '') AS Confidence_Level, + NULLIF(TRIM($6), '') AS Snippet, + NULLIF(TRIM($7), '') AS Original_Page_Number, + NULLIF(TRIM($8), '') AS New_Page_Number, + NULLIF(TRIM($9), '') AS Revised_Prompt, + IFF(TRIM($10) = 'TRUE', TRUE, FALSE) AS Result, + CURRENT_TIMESTAMP(2) AS LOAD_DT + FROM @STG.TRAINING_RESULTS_STAGE + ) + FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER') + ON_ERROR = ABORT_STATEMENT; + + call stg.log_audit(:procedure_name, 'Section 2', 99, 'END'); + + + call stg.log_audit(:procedure_name, 'Section 3', 99, 'START'); + + INSERT INTO STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) + SELECT STG.AUDIT_SID.NEXTVAL, 'STG.TRAINING_RESULTS',* + FROM + (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.TRAINING_RESULTS_STAGE group by 1,2); + + call stg.log_audit(:procedure_name, 'Section 3', 99, 'END'); + + RETURN 'Setup, Load, and Audit Complete'; +END; +$$; \ No newline at end of file diff --git a/snowflake/UAT/training_interface/R__003_LOAD_ATTEMPTS_SP.sql b/snowflake/UAT/training_interface/R__003_LOAD_ATTEMPTS_SP.sql new file mode 100644 index 0000000..1b70044 --- /dev/null +++ b/snowflake/UAT/training_interface/R__003_LOAD_ATTEMPTS_SP.sql @@ -0,0 +1,55 @@ +CREATE OR REPLACE PROCEDURE STG.LOAD_ATTEMPT_LOGS(file_name VARCHAR) +RETURNS STRING +LANGUAGE SQL +EXECUTE AS CALLER +AS +$$ +DECLARE + procedure_name varchar; +BEGIN + + procedure_name := 'LOAD_ATTEMPT_LOGS'; + + call stg.log_audit(:procedure_name, 'Section 1', 99, 'START'); + -- Create or replace stage with dynamic file name + -- TODO: Update for UAT + EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.ATTEMPT_LOGS_STAGE + STORAGE_INTEGRATION = dev_bucket_integration + URL = ''s3://doczy-dev-infra-raw-data-ingestion/training_interface/' || :file_name || ''' + FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');'; + + call stg.log_audit(:procedure_name, 'Section 1', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 2', 99, 'START'); + + -- Copy command to load data + COPY INTO STG.TRAINING_ATTEMPT_LOGS FROM + ( + SELECT + NULLIF(TRIM($1), '') AS FIELD_NAME, + NULLIF(TRIM($2), '') AS CONTRACTS_TESTED, + NULLIF(TRIM($3), '') AS USERNAME, + NULLIF(TRIM($4), '') AS DATE_TIME, + NULLIF(TRIM($5), '') AS ACCURACY, + NULLIF(TRIM($6), '') AS ATTEMPT_NUM, + CURRENT_TIMESTAMP(2) AS LOAD_DT + FROM @STG.ATTEMPT_LOGS_STAGE + ) + FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER') + ON_ERROR = ABORT_STATEMENT; + + call stg.log_audit(:procedure_name, 'Section 2', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 3', 99, 'START'); + + INSERT INTO STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) + SELECT STG.AUDIT_SID.NEXTVAL, 'STG.TRAINING_ATTEMPT_LOGS',* + FROM + (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.ATTEMPT_LOGS_STAGE group by 1,2); + + call stg.log_audit(:procedure_name, 'Section 3', 99, 'END'); + + RETURN 'Setup, Load, and Audit Complete'; +END; +$$; + diff --git a/snowflake/UAT/training_interface/R__004_CREATE_TRAINING_DATA_TABLE_SP.sql b/snowflake/UAT/training_interface/R__004_CREATE_TRAINING_DATA_TABLE_SP.sql new file mode 100644 index 0000000..eeb3dbc --- /dev/null +++ b/snowflake/UAT/training_interface/R__004_CREATE_TRAINING_DATA_TABLE_SP.sql @@ -0,0 +1,28 @@ +CREATE OR REPLACE PROCEDURE STG.CREATE_TRAINING_DATA_TABLE() +RETURNS VARCHAR(16777216) +LANGUAGE SQL +EXECUTE AS CALLER +AS ' +DECLARE + dynamic_ddl STRING := ''CREATE OR REPLACE TABLE STG.TRAINING_DATA_RAW (''; + column_details RESULTSET; + first_column BOOLEAN := TRUE; + cur_config cursor FOR + SELECT column_name, column_datatype FROM STG.TRAINING_DATA_COLUMN_CONFIG; +BEGIN + -- Creating the raw training data table with all columns as VARCHAR due to the dynamic nature of the columns and fields + OPEN cur_config; + FOR rec IN cur_config DO + dynamic_ddl := dynamic_ddl || rec.column_name || '' '' || ''VARCHAR'' || '',''; + END FOR; + + dynamic_ddl := LEFT(dynamic_ddl, LENGTH(dynamic_ddl) - 1); + + dynamic_ddl := dynamic_ddl || '');''; + + EXECUTE IMMEDIATE dynamic_ddl; + + RETURN dynamic_ddl; +END; + +'; \ No newline at end of file diff --git a/snowflake/UAT/training_interface/R__005_LOAD_COLUMN_CONFIG_SP.sql b/snowflake/UAT/training_interface/R__005_LOAD_COLUMN_CONFIG_SP.sql new file mode 100644 index 0000000..884b6e8 --- /dev/null +++ b/snowflake/UAT/training_interface/R__005_LOAD_COLUMN_CONFIG_SP.sql @@ -0,0 +1,57 @@ +CREATE OR REPLACE PROCEDURE STG.LOAD_COLUMN_CONFIG(file_name VARCHAR) +RETURNS STRING +LANGUAGE SQL +EXECUTE AS CALLER +AS +$$ +DECLARE + procedure_name varchar; +BEGIN + + procedure_name := 'LOAD_COLUMN_CONFIG'; + + call stg.log_audit(:procedure_name, 'Section 1', 99, 'START'); + + -- Create or replace stage with dynamic file name + -- TODO: Update for UAT + EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.COLUMN_CONFIG_STAGE + STORAGE_INTEGRATION = dev_bucket_integration + URL = ''s3://doczy-dev-infra-raw-data-ingestion/training_data_raw/' || :file_name || ''' + FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');'; + + call stg.log_audit(:procedure_name, 'Section 1', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 2', 99, 'START'); + + -- Truncate table as we are using the KILL & FILL approach + TRUNCATE TABLE STG.TRAINING_DATA_COLUMN_CONFIG; + + call stg.log_audit(:procedure_name, 'Section 2', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 3', 99, 'START'); + + -- Copy command to load data + COPY INTO STG.TRAINING_DATA_COLUMN_CONFIG FROM ( + SELECT + NULLIF(TRIM($1), '') AS COLUMN_NAME, + NULLIF(TRIM($2), '') AS COLUMN_DATATYPE + FROM @STG.COLUMN_CONFIG_STAGE + ) + FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER') + ON_ERROR = ABORT_STATEMENT; + + call stg.log_audit(:procedure_name, 'Section 3', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 4', 99, 'START'); + + INSERT INTO STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) + SELECT STG.AUDIT_SID.NEXTVAL, :procedure_name,* + FROM + (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.COLUMN_CONFIG_STAGE group by 1,2); + + call stg.log_audit(:procedure_name, 'Section 4', 99, 'END'); + + RETURN 'Setup, Load, and Audit Complete'; +END; +$$; + diff --git a/snowflake/UAT/training_interface/R__006_LOAD_TRAINING_DATA_RAW_SP.sql b/snowflake/UAT/training_interface/R__006_LOAD_TRAINING_DATA_RAW_SP.sql new file mode 100644 index 0000000..cda7539 --- /dev/null +++ b/snowflake/UAT/training_interface/R__006_LOAD_TRAINING_DATA_RAW_SP.sql @@ -0,0 +1,52 @@ +CREATE OR REPLACE PROCEDURE STG.LOAD_TRAINING_DATA_RAW(file_name VARCHAR) +RETURNS STRING +LANGUAGE SQL +EXECUTE AS CALLER +AS +$$ +DECLARE + procedure_name varchar; +BEGIN + + procedure_name := 'LOAD_TRAINING_DATA_RAW'; + + call stg.log_audit(:procedure_name, 'Section 1', 99, 'START'); + + -- Create or replace stage with dynamic file name + -- TODO: Update for UAT + EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.RAW_TRAINING_DATA_STAGE + STORAGE_INTEGRATION = dev_bucket_integration + URL = ''s3://doczy-dev-infra-raw-data-ingestion/training_data_raw/' || :file_name || ''' + FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');'; + + call stg.log_audit(:procedure_name, 'Section 1', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 2', 99, 'START'); + + -- Recreating the training data table by calling the SP. This will replace the existing table with new column definitions + call STG.CREATE_TRAINING_DATA_TABLE(); + + call stg.log_audit(:procedure_name, 'Section 2', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 3', 99, 'START'); + + -- Copy command to load data + COPY INTO STG.TRAINING_DATA_RAW FROM @STG.RAW_TRAINING_DATA_STAGE + FILE_FORMAT = (FORMAT_NAME = 'STG.TRAINING_DATA_FILE_FORMAT') + ON_ERROR = ABORT_STATEMENT; + + call stg.log_audit(:procedure_name, 'Section 3', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 4', 99, 'START'); + + INSERT INTO STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) + SELECT STG.AUDIT_SID.NEXTVAL, :procedure_name,* + FROM + (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.RAW_TRAINING_DATA_STAGE group by 1,2); + + call stg.log_audit(:procedure_name, 'Section 4', 99, 'END'); + + RETURN 'Setup, Load, and Audit Complete'; +END; +$$; + diff --git a/snowflake/UAT/training_interface/R__007_CREATE_BUSINESS_CONFIG_objects.sql b/snowflake/UAT/training_interface/R__007_CREATE_BUSINESS_CONFIG_objects.sql new file mode 100644 index 0000000..2601408 --- /dev/null +++ b/snowflake/UAT/training_interface/R__007_CREATE_BUSINESS_CONFIG_objects.sql @@ -0,0 +1,8 @@ +CREATE TABLE IF NOT EXISTS STG.BUSINESS_CONFIG ( + FIELD_NAME VARCHAR , + SF_COL_NAME VARCHAR, + QUESTION VARCHAR, + PRIORITY VARCHAR, + GROUP_NO VARCHAR, + THEME VARCHAR +); \ No newline at end of file diff --git a/snowflake/UAT/training_interface/R__008_LOAD_BUSINESS_CONFIG_SP.sql b/snowflake/UAT/training_interface/R__008_LOAD_BUSINESS_CONFIG_SP.sql new file mode 100644 index 0000000..50e3d0c --- /dev/null +++ b/snowflake/UAT/training_interface/R__008_LOAD_BUSINESS_CONFIG_SP.sql @@ -0,0 +1,62 @@ + +CREATE OR REPLACE PROCEDURE STG.LOAD_BUSINESS_CONFIG(file_name VARCHAR) +RETURNS STRING +LANGUAGE SQL +EXECUTE AS CALLER +AS +$$ +DECLARE + procedure_name varchar; +BEGIN + + procedure_name := 'LOAD_BUSINESS_CONFIG'; + + call stg.log_audit(:procedure_name, 'Section 1', 99, 'START'); + + -- Create or replace stage with dynamic file name + -- TODO: Update for UAT + EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.BUSINESS_CONFIG_STAGE + STORAGE_INTEGRATION = dev_bucket_integration + URL = ''s3://doczy-dev-infra-raw-data-ingestion/training_data_raw/' || :file_name || ''' + FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');'; + + call stg.log_audit(:procedure_name, 'Section 1', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 2', 99, 'START'); + + -- Truncate table as we are using the KILL & FILL approach + TRUNCATE TABLE STG.BUSINESS_CONFIG; + + call stg.log_audit(:procedure_name, 'Section 2', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 3', 99, 'START'); + + -- Copy command to load data + COPY INTO STG.BUSINESS_CONFIG FROM ( + SELECT + NULLIF(TRIM($1), '') AS FIELD_NAME, + NULLIF(TRIM($2), '') AS SF_COL_NAME, + NULLIF(TRIM($3), '') AS QUESTION, + NULLIF(TRIM($4), '') AS PRIORITY, + NULLIF(TRIM($5), '') AS GROUP_NO, + NULLIF(TRIM($6), '') AS THEME + FROM @STG.BUSINESS_CONFIG_STAGE + ) + FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER') + ON_ERROR = ABORT_STATEMENT; + + call stg.log_audit(:procedure_name, 'Section 3', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 4', 99, 'START'); + + INSERT INTO STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) + SELECT STG.AUDIT_SID.NEXTVAL, :procedure_name,* + FROM + (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.BUSINESS_CONFIG_STAGE group by 1,2); + + call stg.log_audit(:procedure_name, 'Section 4', 99, 'END'); + + RETURN 'Setup, Load, and Audit Complete'; +END; +$$; + diff --git a/snowflake/UAT/upload_interface/R__001_CLIENT_CONFIG_TABLE.sql b/snowflake/UAT/upload_interface/R__001_CLIENT_CONFIG_TABLE.sql new file mode 100644 index 0000000..42b03e7 --- /dev/null +++ b/snowflake/UAT/upload_interface/R__001_CLIENT_CONFIG_TABLE.sql @@ -0,0 +1,7 @@ +-- Create the table to store the client configuration details for Upload UI & Config UI +CREATE TABLE IF NOT EXISTS STG.CLIENT_CONFIG( + OA_CLIENT_ID NUMERIC, + CLIENT_NAME VARCHAR, + ACTIVE_PROJECT_COUNT NUMERIC, + S3_BUCKET_PATH VARCHAR +); \ No newline at end of file diff --git a/snowflake/UAT/upload_interface/R__002_LOAD_CLIENT_CONFIG_SP.sql b/snowflake/UAT/upload_interface/R__002_LOAD_CLIENT_CONFIG_SP.sql new file mode 100644 index 0000000..6ef75a1 --- /dev/null +++ b/snowflake/UAT/upload_interface/R__002_LOAD_CLIENT_CONFIG_SP.sql @@ -0,0 +1,59 @@ +CREATE OR REPLACE PROCEDURE STG.LOAD_CLIENT_CONFIG(file_name VARCHAR) +RETURNS STRING +LANGUAGE SQL +EXECUTE AS CALLER +AS +$$ +DECLARE + procedure_name varchar; +BEGIN + + procedure_name := 'LOAD_CLIENT_CONFIG'; + + call stg.log_audit(:procedure_name, 'Section 1', 99, 'START'); + + -- Create or replace stage with dynamic file name + -- TODO: Update for UAT + EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.CLIENT_CONFIG_STAGE + STORAGE_INTEGRATION = dev_bucket_integration + URL = ''s3://doczy-dev-infra-raw-data-ingestion/client_names_openair/' || :file_name || ''' + FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');'; + + call stg.log_audit(:procedure_name, 'Section 1', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 2', 99, 'START'); + + -- Truncate table as we are using the KILL & FILL approach + TRUNCATE TABLE STG.CLIENT_CONFIG; + + call stg.log_audit(:procedure_name, 'Section 2', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 3', 99, 'START'); + + -- Copy command to load data + COPY INTO STG.CLIENT_CONFIG FROM ( + SELECT + NULLIF(TRIM($1), '') AS OA_CLIENT_ID, + NULLIF(TRIM($2), '') AS CLIENT_NAME, + NULLIF(TRIM($3), '') AS ACTIVE_PROJECT_COUNT, + NULLIF(TRIM($4), '') AS S3_BUCKET_PATH, + FROM @STG.CLIENT_CONFIG_STAGE + ) + FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER') + ON_ERROR = ABORT_STATEMENT; + + call stg.log_audit(:procedure_name, 'Section 3', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 4', 99, 'START'); + + INSERT INTO STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) + SELECT STG.AUDIT_SID.NEXTVAL, :procedure_name,* + FROM + (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.CLIENT_CONFIG_STAGE group by 1,2); + + call stg.log_audit(:procedure_name, 'Section 4', 99, 'END'); + + RETURN 'Setup, Load, and Audit Complete'; +END; +$$; + diff --git a/snowflake/UAT/upload_interface/R__003_CONTRACT_UPLOAD_LOGS.sql b/snowflake/UAT/upload_interface/R__003_CONTRACT_UPLOAD_LOGS.sql new file mode 100644 index 0000000..ac7126e --- /dev/null +++ b/snowflake/UAT/upload_interface/R__003_CONTRACT_UPLOAD_LOGS.sql @@ -0,0 +1,10 @@ +-- This table is used to store the logs of all the files that were uploaded by the user to S3 +-- Following fields are to be stored in the table: +-- Batch id & client name & file name, date time, user +CREATE TABLE IF NOT EXISTS STG.CONTRACT_UPLOAD_LOGS ( + BATCH_ID VARCHAR, + CLIENT_NAME VARCHAR, + FILE_NAME VARCHAR, + UPLOAD_DATETIME DATETIME DEFAULT CURRENT_TIMESTAMP(), + UPLOAD_USER VARCHAR +); \ No newline at end of file diff --git a/terraform-approach/terraform/main.tf b/terraform-approach/terraform/main.tf index 7e1b0e3..8e8b181 100644 --- a/terraform-approach/terraform/main.tf +++ b/terraform-approach/terraform/main.tf @@ -10,7 +10,7 @@ terraform { bucket = "doczy-test-bucket" # Parameterize using -backend-config flag with "terraform init" key = "terraform/terraform_new2.tfstate" # Parameterize region = "us-east-2" # Parameterize - profile = "doczyai" # Parameterize + # profile = "doczyai" # Parameterize dynamodb_table = "terraform-lock" # Parameterize encrypt = true } diff --git a/terraform-approach/terraform/variables.tf b/terraform-approach/terraform/variables.tf index 0153b6b..440eeda 100644 --- a/terraform-approach/terraform/variables.tf +++ b/terraform-approach/terraform/variables.tf @@ -1,6 +1,6 @@ variable "aws_profile" { type = string - default = "doczyai" + # default = "doczyai" } variable "aws_region" { diff --git a/textract-pipeline/terraform/main.tf b/textract-pipeline/terraform/main.tf index f086ac3..3ab3b0f 100644 --- a/textract-pipeline/terraform/main.tf +++ b/textract-pipeline/terraform/main.tf @@ -10,7 +10,7 @@ terraform { bucket = "doczyai-use2-d-infra-s3-terraform-state" # Parameterize using -backend-config flag with "terraform init" key = "terraform/textract-pipeline/terraform.tfstate" # Parameterize region = "us-east-2" # Parameterize - profile = "doczyai" # Parameterize + # profile = "doczyai" # Parameterize dynamodb_table = "doczyai-use2-d-infra-dyd-terraform-lock" # Parameterize encrypt = true } diff --git a/textract-pipeline/terraform/variables.tf b/textract-pipeline/terraform/variables.tf index bf55392..276513c 100644 --- a/textract-pipeline/terraform/variables.tf +++ b/textract-pipeline/terraform/variables.tf @@ -12,7 +12,7 @@ # required variable "aws_profile" { type = string - default = "doczyai" + # default = "doczyai" } # required variable "aws_region" {