Merged in feature/service-accounts-1 (pull request #227)

Support for service accounts with static credentials

* feature complete

* tests and docs

* lint fix

scripts/manual_tests/aws.config.scripts/bash.script.rules.md

@@ -0,0 +1,57 @@
+# Bash Script Automation Rules
+
+**Error Handling**
+- Always use `set -euo pipefail` at the top (exit on error, undefined vars, pipe failures)
+- Trap errors and cleanup: `trap 'cleanup' EXIT ERR`
+- Check command exit codes explicitly when needed: `if ! command; then ... fi`
+
+**Safety**
+- Quote all variables: `"$var"` not `$var`
+- Use `${var:-default}` for defaults, `${var:?error}` for required vars
+- Validate inputs before destructive operations
+- Use absolute paths or verify working directory
+- Test with `-n` (dry-run) flag where possible
+- When script starts it should check that is has basic aws access if aws cli commands are being used.  If we are not logged in then we should terminate and say why. 
+
+**Idempotency**
+- Make scripts safe to run multiple times with the same inputs
+- Check state before making changes: `if [ ! -f file ]; then create; fi`
+- Use atomic operations where possible
+
+**Logging & Debugging**
+- Log to stderr for errors: `echo "Error" >&2`
+- Include timestamps: `echo "[$(date +'%Y-%m-%d %H:%M:%S')] Message"`
+- Add `-x` debug mode option: `[[ "${DEBUG:-}" == "true" ]] && set -x`
+- Log what the script is doing before doing it
+
+**Dependencies**
+- Check for required commands at start: `command -v tool >/dev/null || exit 1`
+- Document external dependencies in comments
+- Fail fast if dependencies missing
+
+**Functions & Structure**
+- Use functions for reusable logic
+- Keep functions small and single-purpose
+- Use `local` for function variables
+- Name functions with verbs: `backup_database`, `verify_config`
+- Each function should be clearly documented as to what it does and how it should be used. 
+
+**Robustness**
+- Use `mktemp` for temporary files, never hardcode `/tmp/myfile`
+- Handle signals: `trap 'rm -f "$tmp_file"' EXIT`
+- Avoid parsing `ls` output - use globs or `find`
+- Use `[[ ]]` instead of `[ ]` for tests
+
+**Configuration**
+- Accept config via environment variables or config file unless otherwise instructed. 
+- Provide sensible defaults
+
+**Exit Codes**
+- Exit 0 only on complete success
+- Use distinct exit codes for different failures
+- Document exit codes in comments
+
+**Portability**
+- Use `#!/usr/bin/env bash` not `#!/bin/bash`
+- Avoid bashisms if targeting `sh`, or explicitly require bash
+- Test on target platform

scripts/manual_tests/run_test_two_pool_lifecycle.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+#
+# run_test_two_pool_lifecycle.sh
+#
+# Wrapper for test_two_pool_lifecycle.py. Hardcodes the public pool B
+# identifiers and AWS region the Go subprocess needs, sources the
+# repo-root .env for secrets, and execs the Python test.
+#
+# Required in .env (or your shell):
+#   PERMIT_IO_API_KEY    Permit.io project/env API key (secret).
+#                        See .env.example.
+#
+# Optional (defaults shown):
+#   AWS_PROFILE              AWS SSO profile with cognito-idp Admin*
+#                            rights on us-east-2_Qom1i9qIG.
+#                            Default: aarete.
+#   SVC_LIFECYCLE_USERNAME   Throwaway username for the test cycle.
+#                            Default: svc-lifecycle-test-1.
+#
+# Required AWS login state:
+#   AWS_PROFILE must have an active session. Run
+#     aws sso login --profile "$AWS_PROFILE"
+#   first if the cached credentials have expired.
+#
+# Usage:
+#   bash scripts/manual_tests/run_test_two_pool_lifecycle.sh
+
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+
+# Source repo-root .env so PERMIT_IO_API_KEY (and any AWS_PROFILE
+# override) reach the shell before the fail-fast check below. Without
+# this the operator would have to export the key in every shell, or
+# Python would crash partway through the test.
+if [ -f "$REPO_ROOT/.env" ]; then
+    set -a
+    # shellcheck disable=SC1090
+    source "$REPO_ROOT/.env"
+    set +a
+fi
+
+# Strip LocalStack endpoint overrides that the repo-root .env carries
+# for the rest of the stack. aws-sdk-go-v2 treats AWS_ENDPOINT_URL as
+# a global service-endpoint override — when it points at LocalStack
+# (http://localhost:4566), every Cognito Admin* call returns HTTP 501
+# because cognito-idp is a paid LocalStack feature.
+unset AWS_ENDPOINT_URL AWS_ENDPOINT_URL_COGNITO_IDP
+
+# Public, non-secret pool B identifiers. The Go subprocess (run.tool.sh)
+# reads these from the environment, so we export them here.
+export AWS_REGION="us-east-2"
+export COGNITO_SVC_USER_POOL_ID="us-east-2_Qom1i9qIG"
+export COGNITO_SVC_APP_CLIENT_ID="6r7chkvjotgs2bsfbd5ibaaft7"
+export PERMIT_IO_BASE_URL="https://api.permit.io"
+
+# Operator-specific. Honor an existing AWS_PROFILE in the env, fall
+# back to "aarete" when unset.
+export AWS_PROFILE="${AWS_PROFILE:-aarete}"
+
+# Fail fast with a clear message rather than letting Python start,
+# call AdminCreateUser, and crash on the Permit step with a vague
+# 401 — which would leave the throwaway user dangling in Cognito.
+: "${PERMIT_IO_API_KEY:?set PERMIT_IO_API_KEY in .env (preferred) or your shell before running}"
+
+exec python3 "$REPO_ROOT/scripts/manual_tests/test_two_pool_lifecycle.py"

scripts/manual_tests/test_chatbot_features.1.py

@@ -41,6 +41,7 @@ import io
 import json
 import os
 import shutil
+import ssl
 import subprocess
 import sys
 import time
@@ -117,9 +118,17 @@ class HTTPResponse:
 class HTTPClient:
     """Tiny urllib wrapper. Prints full request + response in --debug mode."""
 
-    def __init__(self, base_url: str, debug: bool) -> None:
+    def __init__(self, base_url: str, debug: bool, insecure: bool = False) -> None:
         self.base_url = base_url.rstrip("/")
         self.debug = debug
+        # When --insecure is set we bypass TLS cert verification. Used to hit
+        # AWS ELB hostnames that serve the default self-signed cert (no ACM
+        # cert attached). Do NOT use this against anything carrying real
+        # production secrets; it disables MITM protection.
+        if insecure:
+            self.ssl_context: Optional[ssl.SSLContext] = ssl._create_unverified_context()
+        else:
+            self.ssl_context = None
 
     def request(
         self,
@@ -149,7 +158,7 @@ class HTTPClient:
             self._print_request_debug(label, method, url, req_headers, body_bytes)
 
         try:
-            with urlrequest.urlopen(req, timeout=120) as resp:
+            with urlrequest.urlopen(req, timeout=120, context=self.ssl_context) as resp:
                 resp_status = resp.status
                 resp_headers = {k: v for k, v in resp.getheaders()}
                 resp_body = resp.read()
@@ -1020,6 +1029,9 @@ def parse_args(argv: list[str]) -> argparse.Namespace:
                    help="Don't call /turns (skip real FastAPI agent traffic)")
     p.add_argument("--skip-cross-client", action="store_true",
                    help="Skip the cross-client second-client setup (faster runs)")
+    p.add_argument("--insecure", action="store_true",
+                   help="Skip TLS cert verification. Use against AWS ELB hostnames "
+                        "that serve the default self-signed cert.")
     return p.parse_args(argv)
 
 
@@ -1033,9 +1045,10 @@ def main(argv: Optional[list[str]] = None) -> int:
     info(f"Debug mode:      {args.debug}")
     info(f"Skip chat:       {args.skip_chat}")
     info(f"Skip x-client:   {args.skip_cross_client}")
+    info(f"Insecure TLS:    {args.insecure}")
 
     fx = Fixture(
-        client=HTTPClient(args.base_url, debug=args.debug),
+        client=HTTPClient(args.base_url, debug=args.debug, insecure=args.insecure),
         user_subject=args.user,
         other_user_subject=args.other_user,
         admin_user_subject=args.admin_user,

scripts/manual_tests/test_two_pool_auth.py

@@ -0,0 +1,275 @@
+#!/usr/bin/env python3
+"""End-to-end manual test for the two-pool service-account auth path.
+
+Validates plans/api.key.support.cognito.two-pool.plan.3.claude.md
+against a localhost queryAPI configured with both pool A
+(COGNITO_USER_POOL_ID) and pool B (COGNITO_SVC_USER_POOL_ID), and
+auth enabled.
+
+What this script does (top to bottom):
+  Step 1. Sanity-check queryAPI is reachable (GET /health, no auth).
+  Step 2. InitiateAuth against pool B with the pre-baked test
+          service-account credentials (svc-test-admin-1). Returns
+          AccessToken / IdToken / RefreshToken.
+  Step 3. Decode the AccessToken locally (no signature check) to
+          confirm sub, iss, token_use match the values from the
+          provisioning log.
+  Step 4. Call GET /admin/users with Authorization: Bearer
+          <access_token>. /admin/users is a super_admin-only
+          endpoint, so a 200 proves: (a) queryAPI's multi-issuer
+          verifier accepted the pool B token; (b) the service
+          account was successfully registered in Permit with the
+          super_admin role; (c) the env-check dispatched to pool B.
+  Step 5. Call GET /admin/eula with the same token. Second
+          super_admin-only endpoint — covers the "pick at least 2"
+          requirement from the goal statement.
+  Step 6. Print a summary.
+
+Zero third-party deps — stdlib only, like
+cmd/auth_related/service.accounts/test.scripts/test.service.account.auth.py.
+
+Prerequisites:
+  - queryAPI is running locally on http://localhost:8080.
+  - queryAPI was started with both COGNITO_USER_POOL_ID and
+    COGNITO_SVC_USER_POOL_ID configured.
+  - DISABLE_AUTH is NOT set (we want real Cognito JWT validation).
+  - The test service account has been registered in Permit.io and
+    assigned super_admin via:
+        ./cmd/auth_related/service.account.tool/run.tool.sh \
+            register-permit svc-test-admin-1 --roles super_admin
+  - SVC_TEST_PASSWORD is supplied via env (see .env.example). Any
+    .env between this file and the filesystem root is auto-loaded.
+    Everything else (pool ID, app client ID, username, sub, region)
+    is hardcoded below — public identifiers, not secrets — so the
+    script "just works" without further shell setup.
+
+Usage:
+  python3 scripts/manual_tests/test_two_pool_auth.py
+  python3 scripts/manual_tests/test_two_pool_auth.py --base-url http://localhost:8080
+"""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import json
+import os
+import sys
+import urllib.error
+import urllib.request
+from pathlib import Path
+from typing import Any, Dict
+
+HTTP_TIMEOUT = 15
+
+
+def _load_dotenv() -> None:
+    """Walk from this file up to the filesystem root, loading every
+    .env encountered. First-wins precedence: closer (script-adjacent)
+    .envs win over deeper (repo-root) ones, and existing shell vars
+    win over both. Loading every .env on the way up — rather than
+    stopping at the first hit — means a script-local .env cannot
+    hide the repo-root .env where SVC_TEST_PASSWORD lives.
+    """
+    here = Path(__file__).resolve()
+    for parent in [here.parent, *here.parents]:
+        candidate = parent / ".env"
+        if not candidate.is_file():
+            continue
+        for raw in candidate.read_text().splitlines():
+            line = raw.strip()
+            if not line or line.startswith("#"):
+                continue
+            if "=" not in line:
+                continue
+            k, v = line.split("=", 1)
+            k = k.strip()
+            v = v.strip().strip('"').strip("'")
+            if k and k not in os.environ:
+                os.environ[k] = v
+
+
+def _required(name: str) -> str:
+    v = os.environ.get(name, "").strip()
+    if not v:
+        print(f"FAIL: required env var {name} is unset. See .env.example.")
+        sys.exit(2)
+    return v
+
+
+# Hardcoded service-account identifiers. Pool ID, app client ID,
+# username, and sub are public values, not secrets — hardcoding them
+# makes the script share-able: anyone with SVC_TEST_PASSWORD in their
+# .env can run it with no other prerequisites.
+USER_POOL_ID = "us-east-2_Qom1i9qIG"
+APP_CLIENT_ID = "6r7chkvjotgs2bsfbd5ibaaft7"
+USERNAME = "svc-test-admin-1"
+EXPECTED_SUB = "c16b0520-80e1-7076-eeb2-1e520d81e781"
+
+# Region is encoded in the pool ID prefix. Derive it rather than
+# trust the shell — devbox commonly exports AWS_REGION=us-east-1,
+# which routes InitiateAuth to a region where this pool's app client
+# does not exist and yields ResourceNotFoundException.
+AWS_REGION = USER_POOL_ID.split("_", 1)[0]
+
+_load_dotenv()
+
+# Strip LocalStack endpoint overrides for hygiene. urllib (used
+# below) does not honor AWS_ENDPOINT_URL, but the env stays clean for
+# anything spawned downstream from this process.
+for key in ("AWS_ENDPOINT_URL", "AWS_ENDPOINT_URL_COGNITO_IDP"):
+    os.environ.pop(key, None)
+
+PASSWORD = _required("SVC_TEST_PASSWORD")
+
+ISSUER_URL = f"https://cognito-idp.{AWS_REGION}.amazonaws.com/{USER_POOL_ID}"
+COGNITO_IDP_URL = f"https://cognito-idp.{AWS_REGION}.amazonaws.com/"
+
+
+def section(title: str) -> None:
+    print()
+    print("=" * 72)
+    print(title)
+    print("=" * 72)
+
+
+def cognito_call(operation: str, body: Dict[str, Any]) -> Dict[str, Any]:
+    """POST a JSON-over-HTTPS request to Cognito Identity Provider.
+
+    Matches the wire format used by boto3 internally so we don't
+    need any AWS credentials for the unauthenticated InitiateAuth
+    operation.
+    """
+    payload = json.dumps(body).encode("utf-8")
+    req = urllib.request.Request(
+        COGNITO_IDP_URL,
+        data=payload,
+        method="POST",
+        headers={
+            "Content-Type": "application/x-amz-json-1.1",
+            "X-Amz-Target": f"AWSCognitoIdentityProviderService.{operation}",
+        },
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT) as r:
+            return json.loads(r.read())
+    except urllib.error.HTTPError as e:
+        body_text = e.read().decode("utf-8", errors="replace")
+        try:
+            err = json.loads(body_text)
+            raise RuntimeError(
+                f"Cognito {operation}: HTTP {e.code} {err.get('__type', '?')}: "
+                f"{err.get('message', body_text)}"
+            ) from None
+        except json.JSONDecodeError:
+            raise RuntimeError(f"Cognito {operation}: HTTP {e.code} {body_text}") from None
+
+
+def decode_jwt_payload(token: str) -> Dict[str, Any]:
+    """Base64url-decode the payload segment of a JWT (no signature check)."""
+    seg = token.split(".")[1]
+    seg += "=" * (-len(seg) % 4)
+    return json.loads(base64.urlsafe_b64decode(seg))
+
+
+def queryapi_get(base_url: str, path: str, access_token: str = "") -> tuple[int, str]:
+    """GET <base_url><path> with optional bearer auth. Returns (status, body)."""
+    url = base_url.rstrip("/") + path
+    headers = {}
+    if access_token:
+        headers["Authorization"] = "Bearer " + access_token
+    req = urllib.request.Request(url, headers=headers, method="GET")
+    try:
+        with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT) as r:
+            return r.status, r.read().decode("utf-8", errors="replace")
+    except urllib.error.HTTPError as e:
+        body = e.read().decode("utf-8", errors="replace")
+        return e.code, body
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base-url", default="http://localhost:8080")
+    args = parser.parse_args()
+    base = args.base_url
+
+    section("Step 1: queryAPI health")
+    code, body = queryapi_get(base, "/health")
+    print(f"GET {base}/health -> {code}")
+    if code != 200:
+        print(f"FAIL: queryAPI not healthy at {base}: {body}")
+        return 1
+
+    section("Step 2: InitiateAuth against pool B")
+    try:
+        resp = cognito_call(
+            "InitiateAuth",
+            {
+                "AuthFlow": "USER_PASSWORD_AUTH",
+                "ClientId": APP_CLIENT_ID,
+                "AuthParameters": {"USERNAME": USERNAME, "PASSWORD": PASSWORD},
+            },
+        )
+    except RuntimeError as e:
+        print(f"FAIL: {e}")
+        return 1
+    if "AuthenticationResult" not in resp:
+        print(f"FAIL: unexpected InitiateAuth response: {resp}")
+        return 1
+    access_token = resp["AuthenticationResult"]["AccessToken"]
+    print(f"  AccessToken length : {len(access_token)} chars")
+    print(f"  ExpiresIn          : {resp['AuthenticationResult']['ExpiresIn']}s")
+
+    section("Step 3: AccessToken claims")
+    claims = decode_jwt_payload(access_token)
+    print(json.dumps(claims, indent=2, sort_keys=True))
+    failures = []
+    if claims.get("iss") != ISSUER_URL:
+        failures.append(f"iss mismatch (got {claims.get('iss')}, want {ISSUER_URL})")
+    if claims.get("sub") != EXPECTED_SUB:
+        failures.append(f"sub mismatch (got {claims.get('sub')}, want {EXPECTED_SUB})")
+    if claims.get("token_use") != "access":
+        failures.append(f"token_use != access (got {claims.get('token_use')})")
+    if failures:
+        for f in failures:
+            print(f"FAIL: {f}")
+        return 1
+
+    # We pick two super_admin-only endpoints that read from Postgres
+    # only (no downstream AWS calls). /admin/users would also work
+    # against real AWS but hits Cognito ListUsers, which localstack
+    # community edition does not implement, so we use the EULA admin
+    # endpoints instead — both are gated on the same super_admin role
+    # in Permit.
+
+    section("Step 4: GET /admin/eula (super_admin only)")
+    code, body = queryapi_get(base, "/admin/eula", access_token)
+    print(f"GET {base}/admin/eula -> {code}")
+    if code != 200:
+        print("FAIL: /admin/eula did not return 200 — service account either")
+        print("  did not authenticate, or is not registered with super_admin in Permit.")
+        print(f"  body: {body[:400]}")
+        return 1
+    print(f"  body preview: {body[:200]}")
+
+    section("Step 5: GET /admin/eula/agreements (super_admin only)")
+    code, body = queryapi_get(base, "/admin/eula/agreements", access_token)
+    print(f"GET {base}/admin/eula/agreements -> {code}")
+    if code != 200:
+        print("FAIL: /admin/eula/agreements did not return 200.")
+        print(f"  body: {body[:400]}")
+        return 1
+    print(f"  body preview: {body[:200]}")
+
+    section("Result")
+    print("PASS — two-pool service-account auth is wired end-to-end.")
+    print(
+        f"  Service account {USERNAME} (sub={EXPECTED_SUB})\n"
+        "  authenticated against pool B and successfully called two\n"
+        "  super_admin-only endpoints on queryAPI."
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/manual_tests/test_two_pool_auth.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# test_two_pool_auth.sh runs the end-to-end integration test for the
+# two-pool service-account auth path. It assumes the operator has:
+#  - AWS credentials with read access to the test pools
+#    (us-east-2_<pool-a>, us-east-2_Qom1i9qIG)
+#  - PERMIT_IO_API_KEY set in the environment
+#  - Docker running (for `task compose:refresh`)
+#
+# The script:
+#  1. Registers the pre-existing test service-account in Permit with
+#     super_admin (idempotent — re-running is safe).
+#  2. Reminds the operator to boot queryAPI with both pool A and B
+#     configured (compose:refresh by itself does not know about the
+#     new env vars; the operator must source the right .env).
+#  3. Runs scripts/manual_tests/test_two_pool_auth.py against the
+#     local queryAPI to validate the end-to-end flow.
+#
+# Usage:
+#   PERMIT_IO_API_KEY=permit_key_... \
+#   AWS_PROFILE=aarete \
+#     ./scripts/manual_tests/test_two_pool_auth.sh
+
+set -euo pipefail
+
+if [[ -z "${PERMIT_IO_API_KEY:-}" ]]; then
+  echo "ERROR: PERMIT_IO_API_KEY must be set" >&2
+  exit 1
+fi
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+
+echo "Step 1/3: registering svc-test-admin-1 in Permit with super_admin..."
+pushd "$REPO_ROOT/cmd/auth_related/service.account.tool" >/dev/null
+COGNITO_REGION="us-east-2" \
+COGNITO_SVC_USER_POOL_ID="us-east-2_Qom1i9qIG" \
+COGNITO_SVC_APP_CLIENT_ID="6r7chkvjotgs2bsfbd5ibaaft7" \
+  go run ./... register-permit svc-test-admin-1 --roles super_admin
+popd >/dev/null
+
+echo
+echo "Step 2/3: confirm queryAPI is running locally with pool B enabled."
+echo "  - COGNITO_USER_POOL_ID     should be set (existing pool A)"
+echo "  - COGNITO_SVC_USER_POOL_ID should be us-east-2_Qom1i9qIG (pool B)"
+echo "  - DISABLE_AUTH             must be unset (we are testing real auth)"
+echo "  - queryAPI must be reachable at http://localhost:8080"
+echo
+read -rp "Press enter when queryAPI is up and configured..."
+
+echo
+echo "Step 3/3: running test_two_pool_auth.py against localhost..."
+python3 "$REPO_ROOT/scripts/manual_tests/test_two_pool_auth.py"

scripts/manual_tests/test_two_pool_lifecycle.py

@@ -0,0 +1,303 @@
+#!/usr/bin/env python3
+"""Lifecycle integration test for the pool B service-account CLI.
+
+Validates plans/api.key.support.cognito.two-pool.plan.3.claude.md §6.3
+beyond the super-admin coverage in test_two_pool_auth.py. Operates on
+a dedicated throwaway user (default: svc-lifecycle-test-1) so it
+never collides with the long-lived super-admin test account.
+
+What this script does (top to bottom):
+  Step 1. service.account.tool create <user>            (provision)
+  Step 2. InitiateAuth(<user>, password)                (baseline OK)
+  Step 3. service.account.tool disable <user>
+          InitiateAuth(refresh)                          (expect NotAuthorized)
+  Step 4. service.account.tool enable <user>
+          InitiateAuth(<user>, password)                (expect OK again)
+  Step 5. service.account.tool rotate-password <user>
+          InitiateAuth(<user>, OLD_password)            (expect NotAuthorized)
+          InitiateAuth(<user>, NEW_password)            (expect OK)
+  Step 6. service.account.tool delete <user>
+          InitiateAuth(<user>, password)                (expect UserNotFound)
+          service.account.tool show <user>              (expect non-zero exit)
+
+The script captures the initial password from the `create` output and
+the new password from `rotate-password` output by scanning stdout — no
+secret is ever read from disk or written to a file.
+
+Prerequisites:
+  - Run via the wrapper (recommended):
+        bash scripts/manual_tests/run_test_two_pool_lifecycle.sh
+    The wrapper sources the repo-root .env for secrets, hardcodes
+    the public pool B identifiers (pool ID, app client ID, region,
+    PERMIT_IO_BASE_URL), and execs this script.
+
+  - Required in .env (or shell):
+        PERMIT_IO_API_KEY    Permit.io API key (secret).
+    Public identifiers (pool ID, app client ID, region) are
+    hardcoded in the wrapper — not secrets.
+
+  - Optional in .env (or shell):
+        AWS_PROFILE              AWS SSO profile (default: aarete).
+                                 Must have cognito-idp Admin* rights
+                                 on user pool us-east-2_Qom1i9qIG.
+        SVC_LIFECYCLE_USERNAME   Override the throwaway username
+                                 (default: svc-lifecycle-test-1).
+
+  - AWS_PROFILE must have an active session. Run
+        aws sso login --profile "$AWS_PROFILE"
+    if cached credentials have expired.
+
+Usage:
+  bash scripts/manual_tests/run_test_two_pool_lifecycle.sh
+
+Exit codes:
+  0 — all lifecycle assertions held.
+  1 — any assertion failed; script attempts a best-effort cleanup of the
+      throwaway user before exiting.
+  2 — required env / preconditions missing.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import subprocess
+import sys
+import urllib.error
+import urllib.request
+from pathlib import Path
+from typing import Any, Dict, Tuple
+
+HTTP_TIMEOUT = 15
+REPO_ROOT = Path(__file__).resolve().parents[2]
+RUN_TOOL = REPO_ROOT / "cmd" / "auth_related" / "service.account.tool" / "run.tool.sh"
+
+
+def _load_dotenv() -> None:
+    """Populate os.environ from the repo-root .env without external deps."""
+    here = Path(__file__).resolve()
+    for parent in [here.parent, *here.parents]:
+        candidate = parent / ".env"
+        if candidate.is_file():
+            for raw in candidate.read_text().splitlines():
+                line = raw.strip()
+                if not line or line.startswith("#"):
+                    continue
+                if "=" not in line:
+                    continue
+                k, v = line.split("=", 1)
+                k = k.strip()
+                v = v.strip().strip('"').strip("'")
+                if k and k not in os.environ:
+                    os.environ[k] = v
+            return
+
+
+def section(title: str) -> None:
+    print()
+    print("=" * 72)
+    print(title)
+    print("=" * 72)
+
+
+def require_env(name: str) -> str:
+    v = os.environ.get(name, "").strip()
+    if not v:
+        print(f"FAIL: required env var {name} is unset. See .env.example.")
+        sys.exit(2)
+    return v
+
+
+def run_tool(*args: str, check: bool = True) -> subprocess.CompletedProcess:
+    """Invoke run.tool.sh with the supplied subcommand. Streams stdout/err."""
+    cmd = ["bash", str(RUN_TOOL), *args]
+    print(f"$ {' '.join(args)}")
+    proc = subprocess.run(cmd, capture_output=True, text=True, check=False)
+    if proc.stdout:
+        sys.stdout.write(proc.stdout)
+    if proc.stderr:
+        sys.stderr.write(proc.stderr)
+    if check and proc.returncode != 0:
+        raise RuntimeError(f"run.tool.sh {args!r} failed: rc={proc.returncode}")
+    return proc
+
+
+def extract_password(stdout: str) -> str:
+    """Pull the Password line out of `create` / `rotate-password` output."""
+    # `create` prints:   Password : <pw>
+    # `rotate-password`: New password for <u>: <pw>
+    for raw in stdout.splitlines():
+        line = raw.strip()
+        m = re.match(r"^Password\s*:\s*(\S+)\s*$", line)
+        if m:
+            return m.group(1)
+        m = re.match(r"^New password for \S+:\s*(\S+)\s*$", line)
+        if m:
+            return m.group(1)
+    raise RuntimeError("could not find password line in tool output")
+
+
+def cognito_call(region: str, operation: str, body: Dict[str, Any]) -> Tuple[int, Dict[str, Any]]:
+    """POST to Cognito IDP; returns (http_status, parsed_json or error_dict)."""
+    url = f"https://cognito-idp.{region}.amazonaws.com/"
+    payload = json.dumps(body).encode("utf-8")
+    req = urllib.request.Request(
+        url,
+        data=payload,
+        method="POST",
+        headers={
+            "Content-Type": "application/x-amz-json-1.1",
+            "X-Amz-Target": f"AWSCognitoIdentityProviderService.{operation}",
+        },
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT) as r:
+            return r.status, json.loads(r.read())
+    except urllib.error.HTTPError as e:
+        body_text = e.read().decode("utf-8", errors="replace")
+        try:
+            return e.code, json.loads(body_text)
+        except json.JSONDecodeError:
+            return e.code, {"__type": "ParseError", "message": body_text}
+
+
+def initiate_user_password(region: str, client_id: str, username: str, password: str) -> Tuple[int, Dict[str, Any]]:
+    return cognito_call(region, "InitiateAuth", {
+        "AuthFlow": "USER_PASSWORD_AUTH",
+        "ClientId": client_id,
+        "AuthParameters": {"USERNAME": username, "PASSWORD": password},
+    })
+
+
+def initiate_refresh(region: str, client_id: str, refresh_token: str) -> Tuple[int, Dict[str, Any]]:
+    return cognito_call(region, "InitiateAuth", {
+        "AuthFlow": "REFRESH_TOKEN_AUTH",
+        "ClientId": client_id,
+        "AuthParameters": {"REFRESH_TOKEN": refresh_token},
+    })
+
+
+def expect_ok(status: int, resp: Dict[str, Any], label: str) -> Dict[str, Any]:
+    if status != 200 or "AuthenticationResult" not in resp:
+        raise AssertionError(f"{label}: expected 200+AuthenticationResult, got {status} {resp.get('__type')}: {resp.get('message')}")
+    return resp["AuthenticationResult"]
+
+
+def expect_not_authorized(status: int, resp: Dict[str, Any], label: str) -> None:
+    if status == 200 and "AuthenticationResult" in resp:
+        raise AssertionError(f"{label}: expected auth failure, but got tokens back")
+    err = resp.get("__type", "")
+    if "NotAuthorized" not in err and "UserNotFound" not in err:
+        raise AssertionError(f"{label}: expected NotAuthorized/UserNotFound, got {status} {err}: {resp.get('message')}")
+
+
+def main() -> int:
+    _load_dotenv()
+
+    # Strip LocalStack endpoint overrides before the Go subprocess
+    # (run.tool.sh) inherits them. aws-sdk-go-v2 treats
+    # AWS_ENDPOINT_URL as a global service-endpoint override —
+    # devbox commonly sets it to http://localhost:4566, where
+    # cognito-idp is unimplemented and every Admin* call returns
+    # HTTP 501. The wrapper also unsets these, but doing it here
+    # makes the script robust when invoked directly.
+    for key in ("AWS_ENDPOINT_URL", "AWS_ENDPOINT_URL_COGNITO_IDP"):
+        os.environ.pop(key, None)
+
+    user_pool_id = require_env("COGNITO_SVC_USER_POOL_ID")
+    client_id = require_env("COGNITO_SVC_APP_CLIENT_ID")
+    require_env("PERMIT_IO_API_KEY")
+
+    # Region is encoded in the pool ID prefix (us-east-2_Qom1i9qIG).
+    # Override any stale AWS_REGION so a shell that exports us-east-1
+    # (devbox default) cannot redirect the SDK to the wrong region.
+    region = user_pool_id.split("_", 1)[0]
+    os.environ["AWS_REGION"] = region
+
+    username = os.environ.get("SVC_LIFECYCLE_USERNAME", "svc-lifecycle-test-1").strip()
+    if not username:
+        username = "svc-lifecycle-test-1"
+
+    print(f"Region            : {region}")
+    print(f"User pool id      : {user_pool_id}")
+    print(f"App client id     : {client_id}")
+    print(f"Throwaway username: {username}")
+
+    # Best-effort: delete a leftover user from a prior aborted run so
+    # `create` does not blow up with UsernameExistsException. We swallow
+    # all errors — non-existence is expected on a clean run.
+    section("Pre-flight cleanup (idempotent)")
+    run_tool("delete", username, check=False)
+
+    cleanup_needed = False
+    try:
+        section("Step 1: create throwaway user with super_admin role")
+        proc = run_tool("create", username, "--roles", "super_admin", "--description",
+                        "lifecycle test user — safe to delete")
+        cleanup_needed = True
+        password = extract_password(proc.stdout)
+        print(f"  captured initial password (len={len(password)})")
+
+        section("Step 2: baseline InitiateAuth must succeed")
+        status, resp = initiate_user_password(region, client_id, username, password)
+        result = expect_ok(status, resp, "baseline InitiateAuth")
+        refresh_token = result["RefreshToken"]
+        print("  baseline InitiateAuth OK; captured RefreshToken")
+
+        section("Step 3: disable + refresh must be rejected")
+        run_tool("disable", username)
+        status, resp = initiate_refresh(region, client_id, refresh_token)
+        expect_not_authorized(status, resp, "refresh after disable")
+        print("  refresh correctly rejected after disable")
+        # Access tokens minted before disable remain valid until exp;
+        # we deliberately do not assert on that here (would require a
+        # short-TTL test client — see plan §6.3 deferred item).
+
+        section("Step 4: enable + InitiateAuth must succeed again")
+        run_tool("enable", username)
+        status, resp = initiate_user_password(region, client_id, username, password)
+        expect_ok(status, resp, "InitiateAuth after enable")
+        print("  InitiateAuth OK after re-enable")
+
+        section("Step 5: rotate-password — old fails, new succeeds")
+        old_password = password
+        proc = run_tool("rotate-password", username)
+        new_password = extract_password(proc.stdout)
+        if new_password == old_password:
+            raise AssertionError("rotate-password returned the same password")
+        status, resp = initiate_user_password(region, client_id, username, old_password)
+        expect_not_authorized(status, resp, "InitiateAuth with old password")
+        print("  old password correctly rejected")
+        status, resp = initiate_user_password(region, client_id, username, new_password)
+        expect_ok(status, resp, "InitiateAuth with new password")
+        password = new_password
+        print("  new password accepted")
+
+        section("Step 6: delete — auth must fail and show must error")
+        run_tool("delete", username)
+        cleanup_needed = False
+        status, resp = initiate_user_password(region, client_id, username, password)
+        expect_not_authorized(status, resp, "InitiateAuth after delete")
+        proc = run_tool("show", username, check=False)
+        if proc.returncode == 0:
+            raise AssertionError("show <deleted-user> exited 0; expected non-zero")
+        print("  delete confirmed: auth fails, show errors as expected")
+
+        section("Result")
+        print(f"PASS — full lifecycle for {username} validated end-to-end.")
+        return 0
+    except AssertionError as e:
+        print(f"\nFAIL: {e}")
+        return 1
+    except Exception as e:
+        print(f"\nFAIL: unexpected error: {e}")
+        return 1
+    finally:
+        if cleanup_needed:
+            section("Cleanup (script failed mid-run)")
+            run_tool("delete", username, check=False)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/manual_tests/test_zip_upload_cloud.sh

@@ -0,0 +1,325 @@
+#!/bin/bash
+
+# Test script for ZIP batch upload functionality
+#!/bin/bash
+
+# Test script for ZIP batch upload functionality
+# This script exercises all batch endpoints with a cloud deployment
+# This version uses only API calls and does not require database access
+# It's designed to work with cloud-deployed instances of the query orchestration service
+#
+# REST API operations tested:
+# - GET /metrics                                          - Check service health
+# - POST /client                                          - Create new test client
+# - PATCH /client/{CLIENT_ID}                             - Enable CanSync flag via API
+# - POST /client/{CLIENT_ID}/document/batch               - Upload ZIP file with multiple PDFs
+# - GET /client/{CLIENT_ID}/document/batch/{BATCH_ID}     - Get batch status and details
+# - GET /client/{CLIENT_ID}/document/batch                - List all batches for client with pagination
+# - POST /client/{CLIENT_ID}/document/batch (invalid)     - Test error handling with non-ZIP file
+# - DELETE /client/{CLIENT_ID}/document/batch/{BATCH_ID}  - Cancel/delete batch processing
+
+set -e  # Exit on any error
+
+# local host version of the test
+#BASE_URL="http://localhost:8080"
+# Configuration for cloud
+#BASE_URL="http://queryo-query-rtxkppcxpjsb-742555588.us-east-2.elb.amazonaws.com"
+BASE_URL="http://queryo-query-q0pz6j2syaox-1001614225.us-east-2.elb.amazonaws.com"
+# CLIENT_ID will be generated in main() to ensure it's fresh
+ZIP_FILE="test_batch.zip"
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+# Logging functions
+log_info() {
+    echo -e "${GREEN}[INFO]${NC} $1"
+}
+
+log_warn() {
+    echo -e "${YELLOW}[WARN]${NC} $1"
+}
+
+log_error() {
+    echo -e "${RED}[ERROR]${NC} $1"
+}
+
+# Check if service is running
+check_service() {
+    log_info "Checking if queryAPI service is running..."
+    if ! curl -s "$BASE_URL/metrics" > /dev/null 2>&1; then
+        log_error "queryAPI service is not running at $BASE_URL"
+        log_info "Please check the cloud deployment status"
+        exit 1
+    fi
+    log_info "Service is running"
+}
+
+# Create test ZIP file with real PDFs from test.pdf.batch directory, each in its own subfolder
+create_test_zip() {
+    log_info "Creating test ZIP file with real PDFs from ./test.pdf.batch, each in its own subfolder..."
+
+    # Remove existing file if it exists
+    rm -f "$ZIP_FILE"
+
+    # Check if test.pdf.batch directory exists
+    pdf_dir="./test.pdf.batch"
+    if [ ! -d "$pdf_dir" ]; then
+        log_error "Directory $pdf_dir does not exist"
+        exit 1
+    fi
+
+    # Check if there are PDF files in the directory
+    pdf_count=$(find "$pdf_dir" -name "*.pdf" -type f | wc -l)
+    if [ "$pdf_count" -eq 0 ]; then
+        log_error "No PDF files found in $pdf_dir"
+        exit 1
+    fi
+
+    # Create temporary directory structure
+    temp_dir="temp_zip_structure"
+    rm -rf "$temp_dir"
+    mkdir -p "$temp_dir"
+
+    # Copy each PDF to its own subfolder
+    folder_num=1
+    for pdf_file in "$pdf_dir"/*.pdf; do
+        if [ -f "$pdf_file" ]; then
+            folder_name="folder$folder_num"
+            mkdir -p "$temp_dir/$folder_name"
+            cp "$pdf_file" "$temp_dir/$folder_name/"
+            log_info "Placed $(basename "$pdf_file") in $folder_name/"
+            folder_num=$((folder_num + 1))
+        fi
+    done
+
+    # Create ZIP file with the folder structure
+    current_dir=$(pwd)
+    cd "$temp_dir"
+    zip -q -r "$current_dir/$ZIP_FILE" .
+    cd "$current_dir"
+
+    # Cleanup temporary directory
+    rm -rf "$temp_dir"
+
+    file_size=$(stat -f%z "$ZIP_FILE" 2>/dev/null || stat -c%s "$ZIP_FILE" 2>/dev/null || echo "unknown")
+    log_info "Created $ZIP_FILE with $pdf_count real PDFs in separate subfolders ($file_size bytes)"
+}
+
+# Create client (idempotent)
+create_client() {
+    log_info "Creating/checking client: $CLIENT_ID"
+    log_info "API call: POST $BASE_URL/client with ID=$CLIENT_ID"
+
+    # Try to create client, ignore if already exists
+    response=$(curl -s -w "\n%{http_code}" -X POST "$BASE_URL/client" \
+        -H "Content-Type: application/json" \
+        -d "{\"id\":\"$CLIENT_ID\",\"name\":\"Test Client $CLIENT_ID\"}" 2>/dev/null || echo "000")
+
+    http_code=$(echo "$response" | tail -n1)
+
+    if [ "$http_code" = "201" ] || [ "$http_code" = "200" ]; then
+        log_info "Client created/exists: $CLIENT_ID"
+    elif [ "$http_code" = "409" ] || [ "$http_code" = "400" ]; then
+        log_info "Client already exists or duplicate name: $CLIENT_ID"
+    else
+        log_warn "Unexpected response when creating client (HTTP $http_code), continuing anyway..."
+    fi
+}
+
+# Enable CanSync flag for client using API PATCH call
+enable_client_sync() {
+    log_info "Enabling CanSync flag for client: $CLIENT_ID"
+    log_info "API call: PATCH $BASE_URL/client/$CLIENT_ID"
+
+    # Use PATCH API to update client with can_sync: true
+    response=$(curl -s -w "\n%{http_code}" -X PATCH "$BASE_URL/client/$CLIENT_ID" \
+        -H "Content-Type: application/json" \
+        -d '{"can_sync":true}' 2>/dev/null)
+
+    http_code=$(echo "$response" | tail -n1)
+    response_body=$(echo "$response" | sed '$d')
+
+    if [ "$http_code" = "200" ] || [ "$http_code" = "204" ]; then
+        log_info "CanSync flag enabled via API - documents can now be processed"
+        return 0
+    else
+        log_warn "Failed to enable CanSync flag via API (HTTP $http_code), but continuing with test"
+        log_warn "Documents may not process without CanSync enabled"
+        echo "$response_body"
+        return 0  # Continue anyway since this is a test
+    fi
+}
+
+# Upload ZIP batch
+upload_batch() {
+    log_info "Uploading ZIP batch..."
+    log_info "API call: POST $BASE_URL/client/$CLIENT_ID/document/batch"
+
+    response=$(curl -s -w "\n%{http_code}" -X POST "$BASE_URL/client/$CLIENT_ID/document/batch" \
+        -F "archive=@$ZIP_FILE" 2>/dev/null)
+
+    http_code=$(echo "$response" | tail -n1)
+    response_body=$(echo "$response" | sed '$d')
+
+    if [ "$http_code" = "202" ]; then
+        BATCH_ID=$(echo "$response_body" | grep -o '"batch_id":"[^"]*"' | cut -d'"' -f4)
+        STATUS_URL=$(echo "$response_body" | grep -o '"status_url":"[^"]*"' | cut -d'"' -f4)
+        log_info "Batch uploaded successfully"
+        log_info "Batch ID: $BATCH_ID"
+        log_info "Status URL: $STATUS_URL"
+        
+        # Print batch number prominently for easy capture
+        echo ""
+        echo "=========================================="
+        echo "BATCH NUMBER CREATED: $BATCH_ID"
+        echo "=========================================="
+        echo ""
+        
+        return 0
+    else
+        log_error "Failed to upload batch (HTTP $http_code)"
+        echo "$response_body"
+        return 1
+    fi
+}
+
+# Get batch status
+get_batch_status() {
+    log_info "Getting batch status..."
+
+    response=$(curl -s -w "\n%{http_code}" "$BASE_URL/client/$CLIENT_ID/document/batch/$BATCH_ID" 2>/dev/null)
+
+    http_code=$(echo "$response" | tail -n1)
+    response_body=$(echo "$response" | sed '$d')
+
+    if [ "$http_code" = "200" ]; then
+        status=$(echo "$response_body" | grep -o '"status":"[^"]*"' | cut -d'"' -f4)
+        filename=$(echo "$response_body" | grep -o '"originalFilename":"[^"]*"' | cut -d'"' -f4)
+        log_info "Batch status: $status"
+        log_info "Original filename: $filename"
+        echo "$response_body" | python3 -m json.tool 2>/dev/null || echo "$response_body"
+        return 0
+    else
+        log_error "Failed to get batch status (HTTP $http_code)"
+        echo "$response_body"
+        return 1
+    fi
+}
+
+# List batches for client
+list_batches() {
+    log_info "Listing batches for client..."
+
+    response=$(curl -s -w "\n%{http_code}" "$BASE_URL/client/$CLIENT_ID/document/batch?limit=10&offset=0" 2>/dev/null)
+
+    http_code=$(echo "$response" | tail -n1)
+    response_body=$(echo "$response" | sed '$d')
+
+    if [ "$http_code" = "200" ]; then
+        batch_count=$(echo "$response_body" | grep -o '"totalCount":[0-9]*' | cut -d':' -f2)
+        log_info "Found $batch_count batches"
+        echo "$response_body" | python3 -m json.tool 2>/dev/null || echo "$response_body"
+        return 0
+    else
+        log_error "Failed to list batches (HTTP $http_code)"
+        echo "$response_body"
+        return 1
+    fi
+}
+
+# Test invalid file upload
+test_invalid_file() {
+    log_info "Testing invalid file upload (should fail)..."
+
+    # Create invalid file
+    echo "This is not a ZIP file" > invalid.txt
+
+    response=$(curl -s -w "\n%{http_code}" -X POST "$BASE_URL/client/$CLIENT_ID/document/batch" \
+        -F "archive=@invalid.txt" 2>/dev/null)
+
+    http_code=$(echo "$response" | tail -n1)
+    response_body=$(echo "$response" | sed '$d')
+
+    # Cleanup
+    rm -f invalid.txt
+
+    if [ "$http_code" = "400" ]; then
+        log_info "Invalid file correctly rejected (HTTP $http_code)"
+        return 0
+    else
+        log_warn "Expected HTTP 400 for invalid file, got HTTP $http_code"
+        echo "$response_body"
+        return 1
+    fi
+}
+
+# Cancel batch
+cancel_batch() {
+    log_info "Canceling batch..."
+
+    response=$(curl -s -w "\n%{http_code}" -X DELETE "$BASE_URL/client/$CLIENT_ID/document/batch/$BATCH_ID" 2>/dev/null)
+
+    http_code=$(echo "$response" | tail -n1)
+
+    if [ "$http_code" = "204" ]; then
+        log_info "Batch canceled successfully"
+        return 0
+    elif [ "$http_code" = "404" ]; then
+        log_warn "Batch not found or already completed (HTTP $http_code)"
+        return 0
+    else
+        log_error "Failed to cancel batch (HTTP $http_code)"
+        return 1
+    fi
+}
+
+# Cleanup function
+cleanup() {
+    log_info "Cleaning up..."
+    rm -f "$ZIP_FILE" invalid.txt
+    rm -rf temp_zip_structure
+    log_info "Cleanup complete"
+}
+
+# Main test flow
+main() {
+    # Generate fresh CLIENT_ID inside main to ensure it's current
+    CLIENT_ID="test_client_$(date +%s)"
+
+    log_info "Starting ZIP batch upload test..."
+    log_info "Generated client ID: $CLIENT_ID"
+
+    # Setup trap for cleanup
+    trap cleanup EXIT
+
+    # Run tests
+    check_service
+    create_test_zip
+    create_client
+    enable_client_sync
+    
+    if upload_batch; then
+        sleep 1  # Brief pause to let the system process
+        get_batch_status
+        list_batches
+        test_invalid_file
+        cancel_batch
+        
+        # Verify cancellation
+        sleep 1
+        log_info "Verifying batch was canceled..."
+        get_batch_status
+        
+        log_info "✅ All tests completed successfully!"
+    else
+        log_error "❌ Batch upload failed, skipping remaining tests"
+        exit 1
+    fi
+}
+
+# Run main function
+main "$@"

scripts/manual_tests/test_zip_upload_mock.sh

@@ -0,0 +1,306 @@
+#!/bin/bash
+
+# Test script for ZIP batch upload functionality
+#!/bin/bash
+
+# Test script for ZIP batch upload functionality
+# This script exercises all batch endpoints with a cloud deployment
+# This version uses only API calls and does not require database access
+# It's designed to work with cloud-deployed instances of the query orchestration service
+#
+# REST API operations tested:
+# - GET /metrics                                          - Check service health
+# - POST /client                                          - Create new test client
+# - PATCH /client/{CLIENT_ID}                             - Enable CanSync flag via API
+# - POST /client/{CLIENT_ID}/document/batch               - Upload ZIP file with multiple PDFs
+# - GET /client/{CLIENT_ID}/document/batch/{BATCH_ID}     - Get batch status and details
+# - GET /client/{CLIENT_ID}/document/batch                - List all batches for client with pagination
+# - POST /client/{CLIENT_ID}/document/batch (invalid)     - Test error handling with non-ZIP file
+# - DELETE /client/{CLIENT_ID}/document/batch/{BATCH_ID}  - Cancel/delete batch processing
+
+set -e  # Exit on any error
+
+# Configuration for cloud
+BASE_URL="http://queryo-query-rtxkppcxpjsb-742555588.us-east-2.elb.amazonaws.com"
+CLIENT_ID="test_client_$(date +%s)"
+ZIP_FILE="test_batch.zip"
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+# Logging functions
+log_info() {
+    echo -e "${GREEN}[INFO]${NC} $1"
+}
+
+log_warn() {
+    echo -e "${YELLOW}[WARN]${NC} $1"
+}
+
+log_error() {
+    echo -e "${RED}[ERROR]${NC} $1"
+}
+
+# Check if service is running
+check_service() {
+    log_info "Checking if queryAPI service is running..."
+    if ! curl -s "$BASE_URL/metrics" > /dev/null 2>&1; then
+        log_error "queryAPI service is not running at $BASE_URL"
+        log_info "Please check the cloud deployment status"
+        exit 1
+    fi
+    log_info "Service is running"
+}
+
+# Create test ZIP file with real PDFs from test.pdf.batch directory, each in its own subfolder
+create_test_zip() {
+    log_info "Creating test ZIP file with real PDFs from ./test.pdf.batch, each in its own subfolder..."
+
+    # Remove existing file if it exists
+    rm -f "$ZIP_FILE"
+
+    # Check if test.pdf.batch directory exists
+    pdf_dir="./test.pdf.batch"
+    if [ ! -d "$pdf_dir" ]; then
+        log_error "Directory $pdf_dir does not exist"
+        exit 1
+    fi
+
+    # Check if there are PDF files in the directory
+    pdf_count=$(find "$pdf_dir" -name "*.pdf" -type f | wc -l)
+    if [ "$pdf_count" -eq 0 ]; then
+        log_error "No PDF files found in $pdf_dir"
+        exit 1
+    fi
+
+    # Create temporary directory structure
+    temp_dir="temp_zip_structure"
+    rm -rf "$temp_dir"
+    mkdir -p "$temp_dir"
+
+    # Copy each PDF to its own subfolder
+    folder_num=1
+    for pdf_file in "$pdf_dir"/*.pdf; do
+        if [ -f "$pdf_file" ]; then
+            folder_name="folder$folder_num"
+            mkdir -p "$temp_dir/$folder_name"
+            cp "$pdf_file" "$temp_dir/$folder_name/"
+            log_info "Placed $(basename "$pdf_file") in $folder_name/"
+            folder_num=$((folder_num + 1))
+        fi
+    done
+
+    # Create ZIP file with the folder structure
+    current_dir=$(pwd)
+    cd "$temp_dir"
+    zip -q -r "$current_dir/$ZIP_FILE" .
+    cd "$current_dir"
+
+    # Cleanup temporary directory
+    rm -rf "$temp_dir"
+
+    file_size=$(stat -f%z "$ZIP_FILE" 2>/dev/null || stat -c%s "$ZIP_FILE" 2>/dev/null || echo "unknown")
+    log_info "Created $ZIP_FILE with $pdf_count real PDFs in separate subfolders ($file_size bytes)"
+}
+
+# Create client (idempotent)
+create_client() {
+    log_info "Creating/checking client: $CLIENT_ID"
+
+    # Try to create client, ignore if already exists
+    response=$(curl -s -w "\n%{http_code}" -X POST "$BASE_URL/client" \
+        -H "Content-Type: application/json" \
+        -d "{\"id\":\"$CLIENT_ID\",\"name\":\"Test Client $CLIENT_ID\"}" 2>/dev/null || echo "000")
+
+    http_code=$(echo "$response" | tail -n1)
+
+    if [ "$http_code" = "201" ] || [ "$http_code" = "200" ]; then
+        log_info "Client created/exists: $CLIENT_ID"
+    elif [ "$http_code" = "409" ] || [ "$http_code" = "400" ]; then
+        log_info "Client already exists or duplicate name: $CLIENT_ID"
+    else
+        log_warn "Unexpected response when creating client (HTTP $http_code), continuing anyway..."
+    fi
+}
+
+# Enable CanSync flag for client using API PATCH call
+enable_client_sync() {
+    log_info "Enabling CanSync flag for client: $CLIENT_ID"
+
+    # Use PATCH API to update client with can_sync: true
+    response=$(curl -s -w "\n%{http_code}" -X PATCH "$BASE_URL/client/$CLIENT_ID" \
+        -H "Content-Type: application/json" \
+        -d '{"can_sync":true}' 2>/dev/null)
+
+    http_code=$(echo "$response" | tail -n1)
+    response_body=$(echo "$response" | sed '$d')
+
+    if [ "$http_code" = "200" ] || [ "$http_code" = "204" ]; then
+        log_info "CanSync flag enabled via API - documents can now be processed"
+        return 0
+    else
+        log_warn "Failed to enable CanSync flag via API (HTTP $http_code), but continuing with test"
+        log_warn "Documents may not process without CanSync enabled"
+        echo "$response_body"
+        return 0  # Continue anyway since this is a test
+    fi
+}
+
+# Upload ZIP batch (mock version for testing)
+upload_batch() {
+    log_info "Uploading ZIP batch..."
+
+    # Mock successful response for testing
+    BATCH_ID="019580df-ef65-7676-8de9-94435a93337a"
+    STATUS_URL="/client/$CLIENT_ID/document/batch/$BATCH_ID"
+    
+    log_info "Batch uploaded successfully (MOCK)"
+    log_info "Batch ID: $BATCH_ID"
+    log_info "Status URL: $STATUS_URL"
+    
+    # Print batch number prominently for easy capture
+    echo ""
+    echo "=========================================="
+    echo "BATCH NUMBER CREATED: $BATCH_ID"
+    echo "=========================================="
+    echo ""
+    
+    return 0
+}
+
+# Get batch status
+get_batch_status() {
+    log_info "Getting batch status..."
+
+    response=$(curl -s -w "\n%{http_code}" "$BASE_URL/client/$CLIENT_ID/document/batch/$BATCH_ID" 2>/dev/null)
+
+    http_code=$(echo "$response" | tail -n1)
+    response_body=$(echo "$response" | sed '$d')
+
+    if [ "$http_code" = "200" ]; then
+        status=$(echo "$response_body" | grep -o '"status":"[^"]*"' | cut -d'"' -f4)
+        filename=$(echo "$response_body" | grep -o '"originalFilename":"[^"]*"' | cut -d'"' -f4)
+        log_info "Batch status: $status"
+        log_info "Original filename: $filename"
+        echo "$response_body" | python3 -m json.tool 2>/dev/null || echo "$response_body"
+        return 0
+    else
+        log_error "Failed to get batch status (HTTP $http_code)"
+        echo "$response_body"
+        return 1
+    fi
+}
+
+# List batches for client
+list_batches() {
+    log_info "Listing batches for client..."
+
+    response=$(curl -s -w "\n%{http_code}" "$BASE_URL/client/$CLIENT_ID/document/batch?limit=10&offset=0" 2>/dev/null)
+
+    http_code=$(echo "$response" | tail -n1)
+    response_body=$(echo "$response" | sed '$d')
+
+    if [ "$http_code" = "200" ]; then
+        batch_count=$(echo "$response_body" | grep -o '"totalCount":[0-9]*' | cut -d':' -f2)
+        log_info "Found $batch_count batches"
+        echo "$response_body" | python3 -m json.tool 2>/dev/null || echo "$response_body"
+        return 0
+    else
+        log_error "Failed to list batches (HTTP $http_code)"
+        echo "$response_body"
+        return 1
+    fi
+}
+
+# Test invalid file upload
+test_invalid_file() {
+    log_info "Testing invalid file upload (should fail)..."
+
+    # Create invalid file
+    echo "This is not a ZIP file" > invalid.txt
+
+    response=$(curl -s -w "\n%{http_code}" -X POST "$BASE_URL/client/$CLIENT_ID/document/batch" \
+        -F "archive=@invalid.txt" 2>/dev/null)
+
+    http_code=$(echo "$response" | tail -n1)
+    response_body=$(echo "$response" | sed '$d')
+
+    # Cleanup
+    rm -f invalid.txt
+
+    if [ "$http_code" = "400" ]; then
+        log_info "Invalid file correctly rejected (HTTP $http_code)"
+        return 0
+    else
+        log_warn "Expected HTTP 400 for invalid file, got HTTP $http_code"
+        echo "$response_body"
+        return 1
+    fi
+}
+
+# Cancel batch
+cancel_batch() {
+    log_info "Canceling batch..."
+
+    response=$(curl -s -w "\n%{http_code}" -X DELETE "$BASE_URL/client/$CLIENT_ID/document/batch/$BATCH_ID" 2>/dev/null)
+
+    http_code=$(echo "$response" | tail -n1)
+
+    if [ "$http_code" = "204" ]; then
+        log_info "Batch canceled successfully"
+        return 0
+    elif [ "$http_code" = "404" ]; then
+        log_warn "Batch not found or already completed (HTTP $http_code)"
+        return 0
+    else
+        log_error "Failed to cancel batch (HTTP $http_code)"
+        return 1
+    fi
+}
+
+# Cleanup function
+cleanup() {
+    log_info "Cleaning up..."
+    rm -f "$ZIP_FILE" invalid.txt
+    rm -rf temp_zip_structure
+    log_info "Cleanup complete"
+}
+
+# Main test flow
+main() {
+    log_info "Starting ZIP batch upload test..."
+    log_info "Using client ID: $CLIENT_ID"
+
+    # Setup trap for cleanup
+    trap cleanup EXIT
+
+    # Run tests
+    check_service
+    create_test_zip
+    create_client
+    enable_client_sync
+    
+    if upload_batch; then
+        sleep 1  # Brief pause to let the system process
+        get_batch_status
+        list_batches
+        test_invalid_file
+        cancel_batch
+        
+        # Verify cancellation
+        sleep 1
+        log_info "Verifying batch was canceled..."
+        get_batch_status
+        
+        log_info "✅ All tests completed successfully!"
+    else
+        log_error "❌ Batch upload failed, skipping remaining tests"
+        exit 1
+    fi
+}
+
+# Run main function
+main "$@"