diff --git a/airflow/dags/config_interface_dag.py b/airflow/dags/config_interface_dag.py new file mode 100644 index 0000000..dcbc12a --- /dev/null +++ b/airflow/dags/config_interface_dag.py @@ -0,0 +1,100 @@ + + +from __future__ import annotations + +import os +from datetime import datetime +import logging + +from airflow import DAG +from airflow.providers.snowflake.operators.snowflake import SnowflakeOperator +from airflow.providers.snowflake.hooks.snowflake import SnowflakeHook +from airflow.operators.python import PythonOperator +from airflow.utils.trigger_rule import TriggerRule +from airflow.operators.empty import EmptyOperator +from airflow.models import Variable +from airflow.exceptions import AirflowFailException + + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + + +SNOWFLAKE_CONN_ID = "doczy_dev_snowflake" +DAG_ID = "load_request_and_contract_submissions" +DATABASE="DOCZY_DEV" +# bucket = "airflow-data-ingestion" + +TAGS=["dev","contract_config_interface","dataload"] + +# Trigger rules +ALL_SUCCESS = 'all_success' +ALL_FAILED = 'all_failed' +ALL_DONE = 'all_done' +ONE_SUCCESS = 'one_success' +ONE_FAILED = 'one_failed' + +# Passing empty params for now, this will be overridden by the payload from the trigger +# These params can also be set from the Airflow UI while manually triggering the DAG +default_params = {"request_submission_file_name": "", "contract_config_file_name":""} +# This will be replaced with the payload from the event after API connection is setup +# training_results_file_name = "training_results_sample.csv" +# attempt_logs_file_name = "attempt_logs_sample.csv" + + +def call_stored_proc(proc_name,file_type, params): + dwh_hook = SnowflakeHook(snowflake_conn_id=SNOWFLAKE_CONN_ID) + with dwh_hook.get_conn() as conn: + # dwh_hook.set_autocommit(conn,autocommit=False) + cur = conn.cursor() + + # Added new parameter file_type to determine the file name to be passed to the stored procedure + # The bucket name is set by default to "doczy-dev-infra-raw-data-ingestion" and the files should be ALWAYS save under training_interface/ path for now + if file_type == 'request_submission': + file_name = params['request_submission_file_name'] + elif file_type == 'contract_config': + file_name = params['contract_config_file_name'] + + cur.execute(f"CALL {DATABASE}.STG.{proc_name}('{file_name}');") + result = cur.fetchone() + if result[0] == 'Setup, Load, and Audit Complete': + logger.info('PROCEDURE EXECUTED SUCCESSFULLY') + else: + raise AirflowFailException("Check the DAG logs for more information. ERROR FROM SNOWFLAKE: ", result) + logger.info(f"QUERY EXECUTION RESULT: {str(result)}") + +dag = DAG( + DAG_ID, + start_date=datetime(2024, 1, 1), + default_args={"snowflake_conn_id": SNOWFLAKE_CONN_ID, "retries":0}, + tags=TAGS, + catchup=False, + schedule=None, + params = default_params +) + +begin_job = EmptyOperator(task_id='Begin') + + +load_training_results = PythonOperator( + task_id="load_request_submissions", + python_callable=call_stored_proc, + dag=dag, + op_kwargs={'proc_name':'LOAD_TRAINING_RESULTS', 'file_type':'request_submission'} +) + +load_attempt_logs_sp = PythonOperator( + task_id="load_contract_config", + python_callable=call_stored_proc, + dag=dag, + op_kwargs={'proc_name':'LOAD_ATTEMPT_LOGS', 'file_type':'contract_config'} +) + + + + +end_job = EmptyOperator(task_id='End') + + +begin_job >> load_training_results >> load_attempt_logs_sp >> end_job \ No newline at end of file diff --git a/snowflake/scripts/config_interface/R__002_CONTRACT_CONFIG_TABLE.sql b/snowflake/scripts/config_interface/R__002_CONTRACT_CONFIG_TABLE.sql new file mode 100644 index 0000000..1758057 --- /dev/null +++ b/snowflake/scripts/config_interface/R__002_CONTRACT_CONFIG_TABLE.sql @@ -0,0 +1,20 @@ +-- CREATING THE CONFIG TABLE +CREATE TABLE IF NOT EXISTS CONTRACT_CONFIG ( + REQUEST_ID NUMERIC, + CONTRACT_NAME VARCHAR, + GROUP_1 BOOLEAN, + GROUP_1_OVERRIDE BOOLEAN, + GROUP_2 BOOLEAN, + GROUP_2_OVERRIDE BOOLEAN, + GROUP_3 BOOLEAN, + GROUP_3_OVERRIDE BOOLEAN, + GROUP_4 BOOLEAN, + GROUP_4_OVERRIDE BOOLEAN, + GROUP_5 BOOLEAN, + GROUP_5_OVERRIDE BOOLEAN, + REQUEST_DATETIME DATETIME, + REQUEST_USER VARCHAR, + OVERRIDE_DATETIME DATETIME, + LATEST_FLAG BOOLEAN DEFAULT TRUE, + PIPELINE_KICKOFF_DATETIME DATETIME +); \ No newline at end of file diff --git a/snowflake/scripts/config_interface/R__003_REQUEST_SUBMISSION_TABLE.sql b/snowflake/scripts/config_interface/R__003_REQUEST_SUBMISSION_TABLE.sql new file mode 100644 index 0000000..c3eae3a --- /dev/null +++ b/snowflake/scripts/config_interface/R__003_REQUEST_SUBMISSION_TABLE.sql @@ -0,0 +1,9 @@ +CREATE TABLE IF NOT EXISTS STG.REQUEST_SUBMISSION ( + REQUEST_ID NUMERIC, + T_DRIVE_PATH VARCHAR, + CLIENT_NAME VARCHAR, + GROUP_NAME VARIANT, + REQUEST_USERNAME VARCHAR, + REQUEST_DATETIME DATETIME, + OVERRIDE_DATETIME DATETIME DEFAULT NULL +); \ No newline at end of file diff --git a/streamlit/requirements.txt b/streamlit/requirements.txt index 18cb8ac..94f93cf 100644 --- a/streamlit/requirements.txt +++ b/streamlit/requirements.txt @@ -36,3 +36,4 @@ numpy>=1.22.2 # AWS related boto3 awscli +