From 05b00ab6a1daf1b45da5986b510f523fa4ec91a1 Mon Sep 17 00:00:00 2001 From: Sha Brown Date: Tue, 3 Feb 2026 21:12:27 +0000 Subject: [PATCH] Merged in bugfix/max_workers_and_batch_id (pull request #864) Trigger error when batch_id is not specified or when max_workers is negative * Trigger error when batch_id is not specified or when max_workers is negative Approved-by: Katon Minhas --- src/config.py | 2 +- src/pipelines/runner.py | 8 ++++++++ src/pipelines/saas/main.py | 8 ++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/config.py b/src/config.py index e7cb152..631159f 100644 --- a/src/config.py +++ b/src/config.py @@ -93,7 +93,7 @@ def get_arg_value(arg_name, default): # Run config args RUN_MODE = get_arg_value("run_mode", "ec2") # Valid: local or ec2 READ_MODE = get_arg_value("read_mode", "s3") # Valid: local or s3 -BATCH_ID = get_arg_value("batch_id", "test_batch") +BATCH_ID = get_arg_value("batch_id", None) # BATCH_ID is mandatory # Per-file logging settings (for debugging) ENABLE_PER_FILE_LOGGING = get_arg_value("enable_per_file_logging", "False") == "True" diff --git a/src/pipelines/runner.py b/src/pipelines/runner.py index 3cf1770..08629f3 100644 --- a/src/pipelines/runner.py +++ b/src/pipelines/runner.py @@ -106,6 +106,14 @@ def main(client: str = "saas", testing=False, test_params={}): testing: Whether running in test mode test_params: Test parameters if testing=True """ + # Check if batch_id is specified + if not config.BATCH_ID: + raise ValueError("batch_id is required. Please specify batch_id in your command") + + # Check if max_workers is not negative + if config.MAX_WORKERS < 0: + raise ValueError(f"Invalid configuration: MAX_WORKERS must be non-negative (got {config.MAX_WORKERS})") + # Validate client registry = get_registry() if client != "saas" and not registry.has_client(client): diff --git a/src/pipelines/saas/main.py b/src/pipelines/saas/main.py index 679cfdc..243bf64 100644 --- a/src/pipelines/saas/main.py +++ b/src/pipelines/saas/main.py @@ -84,6 +84,14 @@ def safe_process_file(item, constants, run_timestamp): def main(testing=False, test_params={}): + # Check if batch_id is specified + if not config.BATCH_ID: + raise ValueError("batch_id is required. Please specify batch_id in your command") + + # Check if max_workers is not negative + if config.MAX_WORKERS < 0: + raise ValueError(f"Invalid configuration: MAX_WORKERS must be non-negative (got {config.MAX_WORKERS})") + # Set up per-file logging (creates separate log files for each document) logging_utils.setup_per_file_logging()