Katon Minhas
afb6d5185d
Feature/lesser table caching refactor hybrid * chore: Remove unused duplicate main.py from shared pipeline * fix: Correct crosswalk paths in aarete_derived.py * chore: Remove unused documentation files from fieldExtraction * docs: Add documentation files to documentation folder * docs: Update README with uv setup, expanded project structure, and branching conventions * docs: Add uv installation steps with Ubuntu/WSL emphasis * Enable prompt caching for all remaining LLM calls - Add _INSTRUCTION() functions for: EXHIBIT_HEADER, EXHIBIT_LINKAGE, EXHIBIT_TITLE_MATCH, DATE_FIX, DERIVED_TERM_DATE, CHECK_PROVIDER_NAME_MATCH, SPECIAL_CASE_ASSIGNMENT - Update all invoke_claude() calls in saas and clover pipelines to use cache=True with corresponding _INSTRUCTION() functions - Add new instructions to get_cacheable_instructions() for cache warming - Update tests for new instruction functions Functions now using caching: - prompt_exhibit_level - prompt_exhibit_lesser (EXHIBIT_LEVEL_LESSER_OF) - prompt_fee_schedule_breakout - prompt_grouper_breakout - prompt_special_case_assignment - prompt_exhibit_linkage - prompt_exhibit_header - prompt_smart_chunked (ONE_TO_ONE templates) - prompt_date_fix - prompt_derived_term_date - prompt_exhibit_title_match - provider_name_match_check 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Reorder * feat: Add bcbs_promise client pipeline with OFFSET_TERM extraction - Add new bcbs_promise client with HSC-based OFFSET_TERM field extraction - Extract full paragraph text of offset/recoupment provisions from contracts - Derive OFFSET_INDICATOR (Y/N) from OFFSET_TERM presence - Fix reorder_columns to preserve extra columns not in COLUMN_ORDER - Update QC/QA output path to outputs/qc_qa/ * fix: Update dev deps and test assertions for QC/QA output path - Add pytest/pytest-mock to dev dependencies for mypy type checking - Update test assertions to expect outputs/qc_qa instead of qa_qc_output * style: Apply black formatting to prompt_templates.py * Merge main, move scripts * Archive some scripts * update py version * remove .py version file * Remove ASCII characters * Restore testbed code * restore tracking * Update testbed metrics * Enable prompt caching for CODE_LAST_CHECK, FILL_BILL_TYPE, DUAL_LOB_CHECK, and GROUPER_BREAKOUT - Add CODE_LAST_CHECK_INSTRUCTION() for service specificity classification - Add FILL_BILL_TYPE_INSTRUCTION() for bill type code determination - Add DUAL_LOB_CHECK_INSTRUCTION() for Medicare/Medicaid classification - Update code_funcs.py to use caching for CODE_LAST_CHECK, FILL_BILL_TYPE, GROUPER_BREAKOUT - Update postprocessing_funcs.py to use caching for DUAL_LOB_CHECK - Add new instructions to get_cacheable_instructions() for cache warming - Add unit tests for new instruction functions 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Fix postprocessing_funcs to remove invalid columns * Merge branch 'main' into feature/lesser-table-caching-refactor-hybrid * Revert prompt caching changes from aed1b73c * update formatting * Update imports Approved-by: Sha Brown Approved-by: Praneel Panchigar
100 lines
3.3 KiB
Python
100 lines
3.3 KiB
Python
from __future__ import annotations
|
|
|
|
import os
|
|
from datetime import datetime
|
|
import logging
|
|
|
|
from airflow import DAG
|
|
from airflow.providers.snowflake.operators.snowflake import SnowflakeOperator
|
|
from airflow.providers.snowflake.hooks.snowflake import SnowflakeHook
|
|
from airflow.operators.python import PythonOperator
|
|
from airflow.utils.trigger_rule import TriggerRule
|
|
from airflow.operators.empty import EmptyOperator
|
|
from airflow.models import Variable
|
|
from airflow.exceptions import AirflowFailException
|
|
|
|
|
|
logging.basicConfig(level=logging.INFO)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
SNOWFLAKE_CONN_ID = "doczy_dev_snowflake"
|
|
DAG_ID = "load_training_results"
|
|
DATABASE = "DOCZY_DEV"
|
|
# bucket = "airflow-data-ingestion"
|
|
|
|
TAGS = ["dev", "training_interface", "dataload"]
|
|
|
|
# Trigger rules
|
|
ALL_SUCCESS = "all_success"
|
|
ALL_FAILED = "all_failed"
|
|
ALL_DONE = "all_done"
|
|
ONE_SUCCESS = "one_success"
|
|
ONE_FAILED = "one_failed"
|
|
|
|
# Passing empty params for now, this will be overridden by the payload from the trigger
|
|
# These params can also be set from the Airflow UI while manually triggering the DAG
|
|
default_params = {"training_results_file_name": "", "attempt_logs_file_name": ""}
|
|
# This will be replaced with the payload from the event after API connection is setup
|
|
# training_results_file_name = "training_results_sample.csv"
|
|
# attempt_logs_file_name = "attempt_logs_sample.csv"
|
|
|
|
|
|
def call_stored_proc(proc_name, file_type, params):
|
|
dwh_hook = SnowflakeHook(snowflake_conn_id=SNOWFLAKE_CONN_ID)
|
|
with dwh_hook.get_conn() as conn:
|
|
# dwh_hook.set_autocommit(conn,autocommit=False)
|
|
cur = conn.cursor()
|
|
|
|
# Added new parameter file_type to determine the file name to be passed to the stored procedure
|
|
# The bucket name is set by default to "doczy-dev-infra-raw-data-ingestion" and the files should be ALWAYS save under training_interface/ path for now
|
|
if file_type == "training_results":
|
|
file_name = params["training_results_file_name"]
|
|
elif file_type == "attempt_logs":
|
|
file_name = params["attempt_logs_file_name"]
|
|
|
|
cur.execute(f"CALL {DATABASE}.STG.{proc_name}('{file_name}');")
|
|
result = cur.fetchone()
|
|
if result[0] == "Setup, Load, and Audit Complete":
|
|
logger.info("PROCEDURE EXECUTED SUCCESSFULLY")
|
|
else:
|
|
raise AirflowFailException(
|
|
"Check the DAG logs for more information. ERROR FROM SNOWFLAKE: ",
|
|
result,
|
|
)
|
|
logger.info(f"QUERY EXECUTION RESULT: {str(result)}")
|
|
|
|
|
|
dag = DAG(
|
|
DAG_ID,
|
|
start_date=datetime(2024, 1, 1),
|
|
default_args={"snowflake_conn_id": SNOWFLAKE_CONN_ID, "retries": 0},
|
|
tags=TAGS,
|
|
catchup=False,
|
|
schedule=None,
|
|
params=default_params,
|
|
)
|
|
|
|
begin_job = EmptyOperator(task_id="Begin")
|
|
|
|
|
|
load_training_results = PythonOperator(
|
|
task_id="load_training_results",
|
|
python_callable=call_stored_proc,
|
|
dag=dag,
|
|
op_kwargs={"proc_name": "LOAD_TRAINING_RESULTS", "file_type": "training_results"},
|
|
)
|
|
|
|
load_attempt_logs_sp = PythonOperator(
|
|
task_id="load_attempt_logs",
|
|
python_callable=call_stored_proc,
|
|
dag=dag,
|
|
op_kwargs={"proc_name": "LOAD_ATTEMPT_LOGS", "file_type": "attempt_logs"},
|
|
)
|
|
|
|
|
|
end_job = EmptyOperator(task_id="End")
|
|
|
|
|
|
begin_job >> load_training_results >> load_attempt_logs_sp >> end_job
|