query-orchestration/scripts/manual_tests/test_chatbot_features.1.py

#!/usr/bin/env python3
"""End-to-end manual test for the chatbot v8 features.

Exercises the chatbot endpoints against a localhost stack with auth disabled.
Pretends to be real users via the X-Test-User-Subject / X-Test-User-Email
headers honored by internal/cognitoauth/test_middleware.go when
DISABLE_AUTH=true.

What it covers:
  - queryAPI health check
  - Fresh client setup (POST /client) and clientcansync flip via docker
  - Folder + document seeding under that client (POST /folders, batch upload)
  - Bot session lifecycle (create / get / list / pagination)
  - Bot session scope patch (add / remove / dedupe / conflict / cross-client)
  - Bot turn lifecycle (post turn, list turns, title derivation, state populated,
    same-prompt -> already_complete, prompt empty / too long)
  - M5 document reads (/schema, /all-metadata) including unbound + cross-client
  - Owner-only isolation: user B cannot see user A's session
  - Super-admin: list / get / delete and post-delete invisibility

Prerequisites:
  - task compose:refresh has been run and the local stack is up
  - queryAPI runs with DISABLE_AUTH=true
  - The chatbot FastAPI agent that CHATBOT_SERVICE_URL points at is reachable
    (the stack is configured for that; this script does not need to know).

Usage:
  python3 scripts/manual_tests/test_chatbot_features.1.py
  python3 scripts/manual_tests/test_chatbot_features.1.py --debug
  python3 scripts/manual_tests/test_chatbot_features.1.py --base-url http://localhost:8080
  python3 scripts/manual_tests/test_chatbot_features.1.py --skip-chat   # don't call /turns

Stdlib only. No third-party dependencies.
"""

from __future__ import annotations

import argparse
import dataclasses
import io
import json
import os
import shutil
import ssl
import subprocess
import sys
import time
import traceback
import uuid
import zipfile
from pathlib import Path
from typing import Any, Callable, Iterable, Optional
from urllib import error as urlerror
from urllib import request as urlrequest

SCRIPT_DIR = Path(__file__).resolve().parent
PDF_BATCH_DIR = SCRIPT_DIR / "test.pdf.batch"

DEFAULT_BASE_URL = "http://localhost:8080"
DEFAULT_USER_SUBJECT = "john.smith@gmail.com"
DEFAULT_OTHER_USER_SUBJECT = "jane.doe@gmail.com"
DEFAULT_ADMIN_USER_SUBJECT = "super.admin@test.local"

BATCH_POLL_INTERVAL_SECONDS = 3
BATCH_POLL_MAX_SECONDS = 240


# ---------- ANSI color helpers (no third-party libs) ----------

class Color:
    RED = "\033[0;31m"
    GREEN = "\033[0;32m"
    YELLOW = "\033[1;33m"
    BLUE = "\033[0;34m"
    MAGENTA = "\033[0;35m"
    CYAN = "\033[0;36m"
    DIM = "\033[2m"
    RESET = "\033[0m"


def info(msg: str) -> None:
    print(f"{Color.GREEN}[INFO]{Color.RESET} {msg}")


def step(msg: str) -> None:
    print(f"{Color.BLUE}[STEP]{Color.RESET} {msg}")


def warn(msg: str) -> None:
    print(f"{Color.YELLOW}[WARN]{Color.RESET} {msg}")


def error(msg: str) -> None:
    print(f"{Color.RED}[ERROR]{Color.RESET} {msg}")


# ---------- HTTP client ----------

@dataclasses.dataclass
class HTTPResponse:
    status: int
    headers: dict[str, str]
    body_bytes: bytes

    @property
    def body_text(self) -> str:
        try:
            return self.body_bytes.decode("utf-8")
        except UnicodeDecodeError:
            return repr(self.body_bytes)

    def json(self) -> Any:
        if not self.body_bytes:
            return None
        return json.loads(self.body_bytes)


class HTTPClient:
    """Tiny urllib wrapper. Prints full request + response in --debug mode."""

    def __init__(self, base_url: str, debug: bool, insecure: bool = False) -> None:
        self.base_url = base_url.rstrip("/")
        self.debug = debug
        # When --insecure is set we bypass TLS cert verification. Used to hit
        # AWS ELB hostnames that serve the default self-signed cert (no ACM
        # cert attached). Do NOT use this against anything carrying real
        # production secrets; it disables MITM protection.
        if insecure:
            self.ssl_context: Optional[ssl.SSLContext] = ssl._create_unverified_context()
        else:
            self.ssl_context = None

    def request(
        self,
        method: str,
        path: str,
        *,
        headers: Optional[dict[str, str]] = None,
        json_body: Any = None,
        raw_body: Optional[bytes] = None,
        content_type: Optional[str] = None,
        label: str = "",
    ) -> HTTPResponse:
        url = self.base_url + path
        req_headers = dict(headers or {})
        body_bytes: Optional[bytes] = None
        if json_body is not None:
            body_bytes = json.dumps(json_body).encode("utf-8")
            req_headers.setdefault("Content-Type", "application/json")
        elif raw_body is not None:
            body_bytes = raw_body
            if content_type:
                req_headers.setdefault("Content-Type", content_type)

        req = urlrequest.Request(url=url, data=body_bytes, method=method, headers=req_headers)

        if self.debug:
            self._print_request_debug(label, method, url, req_headers, body_bytes)

        try:
            with urlrequest.urlopen(req, timeout=120, context=self.ssl_context) as resp:
                resp_status = resp.status
                resp_headers = {k: v for k, v in resp.getheaders()}
                resp_body = resp.read()
        except urlerror.HTTPError as exc:
            resp_status = exc.code
            resp_headers = {k: v for k, v in (exc.headers.items() if exc.headers else [])}
            resp_body = exc.read() if hasattr(exc, "read") else b""

        result = HTTPResponse(status=resp_status, headers=resp_headers, body_bytes=resp_body)

        if self.debug:
            self._print_response_debug(resp_status, resp_headers, resp_body)
        return result

    def _print_request_debug(
        self,
        label: str,
        method: str,
        url: str,
        headers: dict[str, str],
        body: Optional[bytes],
    ) -> None:
        print()
        title = f"REQUEST  {method} {url}"
        if label:
            title = f"REQUEST  [{label}]  {method} {url}"
        print(f"{Color.MAGENTA}{title}{Color.RESET}")
        for key, value in sorted(headers.items()):
            shown = value
            if key.lower() in {"x-api-key", "authorization"} and value:
                shown = "<set>"
            print(f"  {Color.DIM}{key}{Color.RESET}: {shown}")
        if body is None:
            print(f"  {Color.DIM}<no body>{Color.RESET}")
            return
        print(f"  {Color.DIM}body ({len(body)} bytes):{Color.RESET}")
        print(_pretty(body))

    def _print_response_debug(
        self,
        status: int,
        headers: dict[str, str],
        body: bytes,
    ) -> None:
        color = Color.GREEN if 200 <= status < 300 else Color.YELLOW if status < 500 else Color.RED
        print(f"{color}RESPONSE HTTP {status}{Color.RESET}")
        for key, value in sorted(headers.items()):
            print(f"  {Color.DIM}{key}{Color.RESET}: {value}")
        if not body:
            print(f"  {Color.DIM}<no body>{Color.RESET}")
            return
        print(f"  {Color.DIM}body ({len(body)} bytes):{Color.RESET}")
        print(_pretty(body))


def _pretty(body_bytes: bytes) -> str:
    try:
        decoded = body_bytes.decode("utf-8")
    except UnicodeDecodeError:
        return f"  <{len(body_bytes)} non-utf8 bytes>"
    try:
        parsed = json.loads(decoded)
        return json.dumps(parsed, indent=2)
    except json.JSONDecodeError:
        return decoded


def headers_for(subject: str, email: Optional[str] = None) -> dict[str, str]:
    return {
        "X-Test-User-Subject": subject,
        "X-Test-User-Email": email or subject,
    }


# ---------- Test runner ----------

@dataclasses.dataclass
class TestResult:
    name: str
    passed: bool
    duration_seconds: float
    message: str = ""


class TestRunner:
    def __init__(self) -> None:
        self.results: list[TestResult] = []

    def run(self, name: str, fn: Callable[[], None]) -> bool:
        step(f"Running: {name}")
        start = time.monotonic()
        try:
            fn()
        except AssertionError as exc:
            duration = time.monotonic() - start
            tb = traceback.format_exc().strip().splitlines()
            location = tb[-2] if len(tb) >= 2 else tb[-1]
            self.results.append(TestResult(name, False, duration, f"AssertionError: {exc} ({location})"))
            error(f"FAIL: {name}: {exc}")
            return False
        except Exception as exc:  # noqa: BLE001 - surface any unexpected failure
            duration = time.monotonic() - start
            self.results.append(TestResult(name, False, duration, f"{type(exc).__name__}: {exc}"))
            error(f"FAIL: {name}: {type(exc).__name__}: {exc}")
            traceback.print_exc()
            return False
        duration = time.monotonic() - start
        self.results.append(TestResult(name, True, duration))
        info(f"PASS: {name} ({duration:.2f}s)")
        return True

    def summary(self) -> int:
        total = len(self.results)
        passed = sum(1 for r in self.results if r.passed)
        failed = total - passed
        print()
        print(f"{Color.CYAN}========== Summary ==========={Color.RESET}")
        for r in self.results:
            mark = f"{Color.GREEN}PASS{Color.RESET}" if r.passed else f"{Color.RED}FAIL{Color.RESET}"
            line = f"  [{mark}] {r.name} ({r.duration_seconds:.2f}s)"
            if not r.passed:
                line += f"  {Color.DIM}{r.message}{Color.RESET}"
            print(line)
        print(f"{Color.CYAN}=============================={Color.RESET}")
        print(f"Total: {total}  Passed: {Color.GREEN}{passed}{Color.RESET}  Failed: {Color.RED}{failed}{Color.RESET}")
        return 0 if failed == 0 else 1


# ---------- Test fixtures (state shared across tests) ----------

@dataclasses.dataclass
class Fixture:
    client: HTTPClient
    user_subject: str
    other_user_subject: str
    admin_user_subject: str
    skip_chat: bool

    # Populated during setup:
    test_client_id: str = ""
    other_test_client_id: str = ""
    folder_id: str = ""
    document_id: str = ""
    other_client_document_id: str = ""

    # Populated during tests:
    session_id_a: str = ""        # owned by user_subject, used for most tests
    session_id_b: str = ""        # owned by other_user_subject
    session_id_for_admin: str = ""  # used in super-admin tests, gets soft-deleted
    turn1_completed: bool = False  # gate for downstream turn tests


# ---------- Setup helpers ----------

def assert_status(resp: HTTPResponse, *expected: int, label: str = "") -> None:
    if resp.status not in expected:
        msg = (
            f"{label or 'request'}: expected HTTP {','.join(str(e) for e in expected)}, "
            f"got {resp.status}: {resp.body_text[:600]}"
        )
        raise AssertionError(msg)


def health_check(fx: Fixture) -> None:
    resp = fx.client.request(
        "GET", "/health",
        headers=headers_for(fx.user_subject),
        label="health",
    )
    assert_status(resp, 200, label="GET /health")
    payload = resp.json()
    if payload.get("status") != "healthy":
        raise AssertionError(f"queryAPI not healthy: {payload}")
    info(f"queryAPI healthy at version {payload.get('version', '?')}")


def create_test_client(fx: Fixture, suffix: str = "") -> str:
    cid = f"chatbot{suffix}{int(time.time() * 1000) % 10_000_000_000}"
    payload = {"id": cid, "name": f"Chatbot Manual Test {cid}"}
    resp = fx.client.request(
        "POST", "/client",
        headers=headers_for(fx.user_subject),
        json_body=payload,
        label="create client",
    )
    assert_status(resp, 201, label="POST /client")
    info(f"Created test client: {cid}")
    return cid


def enable_client_sync(client_id: str) -> None:
    """Mirror the test_custom_metadata.1.sh pattern so the batch upload
    can reach clean_passed. Uses docker exec against the existing
    deployments-db-1 container - does not modify the host environment."""
    container = "deployments-db-1"
    db = "query_orchestration"
    select_cmd = [
        "docker", "exec", container,
        "psql", "-U", "postgres", "-d", db, "-tA", "-c",
        f"SELECT COUNT(*) FROM clientcansync WHERE clientid='{client_id}';",
    ]
    result = subprocess.run(select_cmd, capture_output=True, text=True, check=False)
    if result.returncode != 0:
        raise RuntimeError(f"docker exec failed: {result.stderr.strip()}")
    existing = (result.stdout or "0").strip() or "0"
    if existing == "0":
        sql = f"INSERT INTO clientcansync (clientid, cansync) VALUES ('{client_id}', true);"
    else:
        sql = f"UPDATE clientcansync SET cansync = true WHERE clientid='{client_id}';"
    upsert_cmd = [
        "docker", "exec", container,
        "psql", "-U", "postgres", "-d", db, "-c", sql,
    ]
    upsert = subprocess.run(upsert_cmd, capture_output=True, text=True, check=False)
    if upsert.returncode != 0:
        raise RuntimeError(f"clientcansync upsert failed: {upsert.stderr.strip()}")
    info(f"Enabled clientcansync for {client_id}")


def create_folder(fx: Fixture, client_id: str, path: str = "/chatbot-test") -> str:
    resp = fx.client.request(
        "POST", "/folders",
        headers=headers_for(fx.user_subject),
        json_body={"path": path, "clientId": client_id, "createdBy": fx.user_subject},
        label="create folder",
    )
    assert_status(resp, 201, label="POST /folders")
    folder = resp.json()
    info(f"Created folder: {folder['id']} ({folder['path']})")
    return folder["id"]


def build_pdf_zip(folder_name: str = "chatbot-batch") -> Path:
    if not PDF_BATCH_DIR.is_dir():
        raise RuntimeError(f"missing PDF batch dir: {PDF_BATCH_DIR}")
    sample_pdf = PDF_BATCH_DIR / "test.pdf"
    if not sample_pdf.is_file():
        raise RuntimeError(f"missing sample PDF: {sample_pdf}")
    tmp = Path(os.environ.get("TMPDIR", "/tmp")) / f"chatbot_test_{uuid.uuid4().hex}.zip"
    with zipfile.ZipFile(tmp, "w", zipfile.ZIP_DEFLATED) as zf:
        zf.write(sample_pdf, arcname=f"{folder_name}/sample_one.pdf")
    info(f"Built PDF zip: {tmp} ({tmp.stat().st_size} bytes)")
    return tmp


def upload_batch(fx: Fixture, client_id: str, zip_path: Path) -> str:
    """Send the zip via multipart/form-data using stdlib only."""
    boundary = f"----chatbotmanualtest{uuid.uuid4().hex}"
    body = io.BytesIO()
    body.write(f"--{boundary}\r\n".encode("utf-8"))
    body.write(
        b'Content-Disposition: form-data; name="archive"; filename="batch.zip"\r\n'
    )
    body.write(b"Content-Type: application/zip\r\n\r\n")
    body.write(zip_path.read_bytes())
    body.write(f"\r\n--{boundary}--\r\n".encode("utf-8"))
    resp = fx.client.request(
        "POST", f"/client/{client_id}/document/batch",
        headers=headers_for(fx.user_subject),
        raw_body=body.getvalue(),
        content_type=f"multipart/form-data; boundary={boundary}",
        label="upload batch",
    )
    assert_status(resp, 202, label="POST /document/batch")
    payload = resp.json()
    batch_id = payload.get("batch_id")
    if not batch_id:
        raise AssertionError(f"upload response missing batch_id: {payload}")
    info(f"Batch uploaded: {batch_id}")
    return batch_id


def wait_for_batch_completion(fx: Fixture, client_id: str, batch_id: str) -> dict[str, Any]:
    deadline = time.monotonic() + BATCH_POLL_MAX_SECONDS
    last_status = ""
    while time.monotonic() < deadline:
        resp = fx.client.request(
            "GET", f"/client/{client_id}/document/batch/{batch_id}",
            headers=headers_for(fx.user_subject),
            label="poll batch",
        )
        assert_status(resp, 200, label="GET /document/batch/{id}")
        body = resp.json()
        status = body.get("status", "?")
        outcomes = body.get("document_outcomes", []) or []
        clean = sum(1 for o in outcomes if o.get("outcome") == "clean_passed")
        progress = f"status={status}, clean_passed={clean}/{len(outcomes)}"
        if progress != last_status:
            info(progress)
            last_status = progress
        if status == "completed":
            return body
        time.sleep(BATCH_POLL_INTERVAL_SECONDS)
    raise AssertionError(f"batch {batch_id} did not complete within {BATCH_POLL_MAX_SECONDS}s")


def pick_document_id(batch_body: dict[str, Any]) -> str:
    for outcome in batch_body.get("document_outcomes", []) or []:
        if outcome.get("outcome") == "clean_passed" and outcome.get("document_id"):
            return outcome["document_id"]
    raise AssertionError(f"no clean_passed document in batch outcomes: {batch_body}")


# ---------- Tests: bot session lifecycle ----------

def test_create_session(fx: Fixture) -> None:
    resp = fx.client.request(
        "POST", f"/client/{fx.test_client_id}/bot/sessions",
        headers=headers_for(fx.user_subject),
        json_body={},
        label="create session A",
    )
    assert_status(resp, 201, label="POST /bot/sessions")
    body = resp.json()
    for field in ("id", "clientId", "createdBy", "title", "lastTurn", "scope", "recentTurns", "state"):
        if field not in body:
            raise AssertionError(f"create session response missing {field!r}: {body}")
    if body["clientId"] != fx.test_client_id:
        raise AssertionError(f"clientId mismatch: {body['clientId']!r} != {fx.test_client_id!r}")
    if body["createdBy"] != fx.user_subject:
        raise AssertionError(f"createdBy mismatch: {body['createdBy']!r} != {fx.user_subject!r}")
    if body["lastTurn"] != 0:
        raise AssertionError(f"lastTurn should be 0 on fresh session, got {body['lastTurn']!r}")
    if body["title"] != "":
        raise AssertionError(f"title should be empty on fresh session, got {body['title']!r}")
    if body["scope"]["documentIds"] or body["scope"]["folderIds"]:
        raise AssertionError(f"scope should be empty on fresh session, got {body['scope']}")
    fx.session_id_a = body["id"]
    info(f"Session A: {fx.session_id_a}")


def test_get_session_by_id(fx: Fixture) -> None:
    resp = fx.client.request(
        "GET", f"/client/{fx.test_client_id}/bot/sessions/{fx.session_id_a}",
        headers=headers_for(fx.user_subject),
        label="get session A",
    )
    assert_status(resp, 200, label="GET /bot/sessions/{id}")
    body = resp.json()
    if body["id"] != fx.session_id_a:
        raise AssertionError(f"id mismatch on GET: {body['id']!r} != {fx.session_id_a!r}")


def test_list_sessions(fx: Fixture) -> None:
    resp = fx.client.request(
        "GET", f"/client/{fx.test_client_id}/bot/sessions?limit=20&offset=0",
        headers=headers_for(fx.user_subject),
        label="list sessions",
    )
    assert_status(resp, 200, label="GET /bot/sessions")
    body = resp.json()
    if "sessions" not in body or not isinstance(body["sessions"], list):
        raise AssertionError(f"list response shape unexpected: {body}")
    ids = [s["id"] for s in body["sessions"]]
    if fx.session_id_a not in ids:
        raise AssertionError(f"session A {fx.session_id_a} not in list: {ids}")


def test_list_sessions_pagination_bounds(fx: Fixture) -> None:
    bad_limit = fx.client.request(
        "GET", f"/client/{fx.test_client_id}/bot/sessions?limit=201",
        headers=headers_for(fx.user_subject),
        label="list sessions limit=201",
    )
    assert_status(bad_limit, 400, label="GET /bot/sessions?limit=201")
    bad_offset = fx.client.request(
        "GET", f"/client/{fx.test_client_id}/bot/sessions?limit=20&offset=-1",
        headers=headers_for(fx.user_subject),
        label="list sessions offset=-1",
    )
    assert_status(bad_offset, 400, label="GET /bot/sessions?offset=-1")


def test_session_404_for_unknown_id(fx: Fixture) -> None:
    bogus = str(uuid.uuid4())
    resp = fx.client.request(
        "GET", f"/client/{fx.test_client_id}/bot/sessions/{bogus}",
        headers=headers_for(fx.user_subject),
        label="get unknown session",
    )
    assert_status(resp, 404, label="GET unknown session")


# ---------- Tests: bot session scope ----------

def test_scope_add_documents(fx: Fixture) -> None:
    resp = fx.client.request(
        "PATCH", f"/client/{fx.test_client_id}/bot/sessions/{fx.session_id_a}/scope",
        headers=headers_for(fx.user_subject),
        json_body={"addDocuments": [fx.document_id]},
        label="scope add doc",
    )
    assert_status(resp, 200, label="PATCH scope addDocuments")
    body = resp.json()
    doc_ids = body["scope"]["documentIds"]
    if fx.document_id not in doc_ids:
        raise AssertionError(f"document {fx.document_id} not in scope after add: {doc_ids}")


def test_scope_add_folders(fx: Fixture) -> None:
    resp = fx.client.request(
        "PATCH", f"/client/{fx.test_client_id}/bot/sessions/{fx.session_id_a}/scope",
        headers=headers_for(fx.user_subject),
        json_body={"addFolders": [fx.folder_id]},
        label="scope add folder",
    )
    assert_status(resp, 200, label="PATCH scope addFolders")
    body = resp.json()
    folder_ids = body["scope"]["folderIds"]
    if fx.folder_id not in folder_ids:
        raise AssertionError(f"folder {fx.folder_id} not in scope after add: {folder_ids}")


def test_scope_dedupe(fx: Fixture) -> None:
    """Submitting the same id twice in a single PATCH should be deduplicated server-side."""
    resp = fx.client.request(
        "PATCH", f"/client/{fx.test_client_id}/bot/sessions/{fx.session_id_a}/scope",
        headers=headers_for(fx.user_subject),
        json_body={"addDocuments": [fx.document_id, fx.document_id]},
        label="scope dedupe",
    )
    assert_status(resp, 200, label="PATCH scope dedupe")
    body = resp.json()
    matching = [d for d in body["scope"]["documentIds"] if d == fx.document_id]
    if len(matching) != 1:
        raise AssertionError(f"dedupe failed: doc id appears {len(matching)} times: {body['scope']['documentIds']}")


def test_scope_add_remove_conflict(fx: Fixture) -> None:
    resp = fx.client.request(
        "PATCH", f"/client/{fx.test_client_id}/bot/sessions/{fx.session_id_a}/scope",
        headers=headers_for(fx.user_subject),
        json_body={
            "addDocuments": [fx.document_id],
            "removeDocuments": [fx.document_id],
        },
        label="scope add+remove conflict",
    )
    assert_status(resp, 400, label="PATCH scope add+remove conflict")
    if "add_remove_conflict" not in resp.body_text:
        raise AssertionError(f"response missing add_remove_conflict marker: {resp.body_text}")


def test_scope_cross_client_document(fx: Fixture) -> None:
    if not fx.other_client_document_id:
        warn("skipping cross-client doc test: no other-client document available")
        return
    resp = fx.client.request(
        "PATCH", f"/client/{fx.test_client_id}/bot/sessions/{fx.session_id_a}/scope",
        headers=headers_for(fx.user_subject),
        json_body={"addDocuments": [fx.other_client_document_id]},
        label="scope cross-client doc",
    )
    assert_status(resp, 400, label="PATCH scope cross-client doc")
    if "document_cross_client" not in resp.body_text:
        raise AssertionError(f"response missing document_cross_client marker: {resp.body_text}")


def test_scope_remove_documents(fx: Fixture) -> None:
    resp = fx.client.request(
        "PATCH", f"/client/{fx.test_client_id}/bot/sessions/{fx.session_id_a}/scope",
        headers=headers_for(fx.user_subject),
        json_body={"removeDocuments": [fx.document_id]},
        label="scope remove doc",
    )
    assert_status(resp, 200, label="PATCH scope removeDocuments")
    body = resp.json()
    if fx.document_id in body["scope"]["documentIds"]:
        raise AssertionError(f"document {fx.document_id} still in scope after remove")


# ---------- Tests: bot turns (real chat through the agent) ----------

def test_post_first_turn_and_title_derivation(fx: Fixture) -> None:
    if fx.skip_chat:
        info("skipping turn test (--skip-chat)")
        return
    prompt = "Hello, please summarize what kinds of contracts you can see in scope."
    resp = fx.client.request(
        "POST", f"/client/{fx.test_client_id}/bot/sessions/{fx.session_id_a}/turns",
        headers=headers_for(fx.user_subject),
        json_body={"prompt": prompt},
        label="post turn 1",
    )
    if resp.status == 502:
        warn(
            "FastAPI agent returned 502 - see body for code. "
            "Treating as soft-fail: agent unavailability does not break this test."
        )
        warn(f"body: {resp.body_text[:500]}")
        return
    assert_status(resp, 201, label="POST /turns (1)")
    body = resp.json()
    if body.get("status") != "completed":
        raise AssertionError(f"first turn did not complete: status={body.get('status')!r} body={body}")
    if body.get("ordinal") != 1:
        raise AssertionError(f"first turn ordinal should be 1, got {body.get('ordinal')!r}")
    if not body.get("completion"):
        raise AssertionError(f"first turn missing completion: {body}")
    fx.turn1_completed = True

    # Verify session title was derived from prompt (rune-truncated to 120).
    sresp = fx.client.request(
        "GET", f"/client/{fx.test_client_id}/bot/sessions/{fx.session_id_a}",
        headers=headers_for(fx.user_subject),
        label="get session after turn 1",
    )
    assert_status(sresp, 200, label="GET session after turn 1")
    sbody = sresp.json()
    if sbody.get("title", "") == "":
        raise AssertionError("session title was not derived after first turn")
    info(f"derived session title: {sbody['title']!r}")
    if sbody.get("lastTurn") != 1:
        raise AssertionError(f"lastTurn should be 1 after first completed turn, got {sbody.get('lastTurn')!r}")


def test_list_turns(fx: Fixture) -> None:
    if fx.skip_chat:
        info("skipping list-turns test (--skip-chat)")
        return
    resp = fx.client.request(
        "GET", f"/client/{fx.test_client_id}/bot/sessions/{fx.session_id_a}/turns",
        headers=headers_for(fx.user_subject),
        label="list turns",
    )
    assert_status(resp, 200, label="GET /turns")
    body = resp.json()
    if "turns" not in body or not isinstance(body["turns"], list):
        raise AssertionError(f"list turns shape unexpected: {body}")
    if len(body["turns"]) < 1:
        warn("turn list empty - skip-chat or first-turn agent error")
        return
    ordinals = [t["ordinal"] for t in body["turns"]]
    if ordinals != sorted(ordinals):
        raise AssertionError(f"turns not ordered ASC by ordinal: {ordinals}")


def test_post_second_turn_keeps_title(fx: Fixture) -> None:
    if fx.skip_chat:
        info("skipping turn-2 test (--skip-chat)")
        return
    # Per fix.chat.bugs.1.md §3 Option A, posting a different prompt
    # after turn 1 errored advances to ordinal 2 instead of wedging on
    # 409 prompt_mismatch. The title is derived in tx1 of turn 1
    # regardless of FastAPI outcome, so the gate below is best-effort.
    pre = fx.client.request(
        "GET", f"/client/{fx.test_client_id}/bot/sessions/{fx.session_id_a}",
        headers=headers_for(fx.user_subject),
        label="get session pre turn 2",
    )
    if pre.status != 200:
        warn("pre-turn-2 session GET failed; skipping")
        return
    pre_title = pre.json().get("title", "")
    if pre_title == "":
        warn("title is empty pre-turn-2; skipping turn 2")
        return

    prompt = "Out of those contracts, are any near termination?"
    resp = fx.client.request(
        "POST", f"/client/{fx.test_client_id}/bot/sessions/{fx.session_id_a}/turns",
        headers=headers_for(fx.user_subject),
        json_body={"prompt": prompt},
        label="post turn 2",
    )
    if resp.status == 502:
        warn(f"agent 502 on turn 2: {resp.body_text[:500]}")
        return
    assert_status(resp, 201, label="POST /turns (2)")
    body = resp.json()
    if body.get("ordinal") != 2:
        raise AssertionError(f"second turn ordinal should be 2, got {body.get('ordinal')!r}")

    post = fx.client.request(
        "GET", f"/client/{fx.test_client_id}/bot/sessions/{fx.session_id_a}",
        headers=headers_for(fx.user_subject),
        label="get session post turn 2",
    )
    assert_status(post, 200, label="GET session after turn 2")
    if post.json().get("title", "") != pre_title:
        raise AssertionError(
            f"title changed across turns: was {pre_title!r}, now {post.json().get('title')!r}"
        )


def test_resubmit_same_prompt_already_complete(fx: Fixture) -> None:
    """Re-submitting the same prompt that was already completed should
    map (via FindBotTurnByPrompt -> existing completed row) to 409
    already_complete."""
    if fx.skip_chat:
        info("skipping already-complete test (--skip-chat)")
        return
    list_resp = fx.client.request(
        "GET", f"/client/{fx.test_client_id}/bot/sessions/{fx.session_id_a}/turns",
        headers=headers_for(fx.user_subject),
        label="list turns for resubmit",
    )
    if list_resp.status != 200:
        warn("could not list turns for resubmit test; skipping")
        return
    completed = [t for t in list_resp.json().get("turns", []) if t.get("status") == "completed"]
    if not completed:
        warn("no completed turn to resubmit; skipping already-complete test")
        return
    prompt = completed[0]["prompt"]
    resp = fx.client.request(
        "POST", f"/client/{fx.test_client_id}/bot/sessions/{fx.session_id_a}/turns",
        headers=headers_for(fx.user_subject),
        json_body={"prompt": prompt},
        label="resubmit completed prompt",
    )
    assert_status(resp, 409, label="POST /turns (resubmit)")
    if "already_complete" not in resp.body_text:
        raise AssertionError(f"resubmit should signal already_complete: body={resp.body_text}")


def test_prompt_empty_rejected(fx: Fixture) -> None:
    if fx.skip_chat:
        info("skipping empty-prompt test (--skip-chat)")
        return
    resp = fx.client.request(
        "POST", f"/client/{fx.test_client_id}/bot/sessions/{fx.session_id_a}/turns",
        headers=headers_for(fx.user_subject),
        json_body={"prompt": "   "},
        label="empty prompt",
    )
    assert_status(resp, 400, label="POST /turns empty prompt")
    if "prompt_empty" not in resp.body_text:
        raise AssertionError(f"expected prompt_empty marker: {resp.body_text}")


def test_prompt_too_long_rejected(fx: Fixture) -> None:
    if fx.skip_chat:
        info("skipping prompt-too-long test (--skip-chat)")
        return
    resp = fx.client.request(
        "POST", f"/client/{fx.test_client_id}/bot/sessions/{fx.session_id_a}/turns",
        headers=headers_for(fx.user_subject),
        json_body={"prompt": "x" * 3000},  # CHATBOT_MAX_TURN_CHARS default = 2000
        label="prompt too long",
    )
    assert_status(resp, 400, label="POST /turns prompt too long")
    if "prompt_too_long" not in resp.body_text:
        raise AssertionError(f"expected prompt_too_long marker: {resp.body_text}")


# ---------- Tests: M5 document reads ----------

def test_document_schema_unbound(fx: Fixture) -> None:
    resp = fx.client.request(
        "GET", f"/client/{fx.test_client_id}/documents/{fx.document_id}/schema",
        headers=headers_for(fx.user_subject),
        label="get doc schema (unbound)",
    )
    assert_status(resp, 200, label="GET /documents/{id}/schema")
    body = resp.json()
    if body.get("documentId") != fx.document_id:
        raise AssertionError(f"documentId mismatch: {body}")
    if body.get("schema") not in (None, {}, ):
        # bound schemas are objects; this fresh document has no schema
        info(f"document has a bound schema (unexpected for fresh doc but acceptable): {body.get('schema')}")


def test_document_all_metadata(fx: Fixture) -> None:
    resp = fx.client.request(
        "GET", f"/client/{fx.test_client_id}/documents/{fx.document_id}/all-metadata",
        headers=headers_for(fx.user_subject),
        label="get doc all-metadata",
    )
    assert_status(resp, 200, label="GET /documents/{id}/all-metadata")
    body = resp.json()
    if "document" not in body or "labels" not in body:
        raise AssertionError(f"all-metadata missing required fields: {body}")
    if body["document"].get("id") != fx.document_id:
        raise AssertionError(f"all-metadata document.id mismatch: {body['document']}")
    if not isinstance(body["labels"], list):
        raise AssertionError(f"labels should always be a list: {body['labels']!r}")
    # customMetadata may be null or absent on a fresh, schema-unbound doc.


def test_document_schema_cross_client_404(fx: Fixture) -> None:
    if not fx.other_client_document_id:
        warn("skipping cross-client doc-schema test: no other-client doc")
        return
    # Try to read the OTHER client's document via OUR client's URL.
    resp = fx.client.request(
        "GET",
        f"/client/{fx.test_client_id}/documents/{fx.other_client_document_id}/schema",
        headers=headers_for(fx.user_subject),
        label="cross-client doc schema",
    )
    assert_status(resp, 404, label="GET cross-client doc schema")


def test_document_all_metadata_cross_client_404(fx: Fixture) -> None:
    if not fx.other_client_document_id:
        warn("skipping cross-client all-metadata test: no other-client doc")
        return
    resp = fx.client.request(
        "GET",
        f"/client/{fx.test_client_id}/documents/{fx.other_client_document_id}/all-metadata",
        headers=headers_for(fx.user_subject),
        label="cross-client all-metadata",
    )
    assert_status(resp, 404, label="GET cross-client all-metadata")


# ---------- Tests: owner-only isolation ----------

def test_owner_only_isolation(fx: Fixture) -> None:
    """User B should not be able to GET user A's session under the same client."""
    resp = fx.client.request(
        "GET", f"/client/{fx.test_client_id}/bot/sessions/{fx.session_id_a}",
        headers=headers_for(fx.other_user_subject),
        label="other-user GET user-A's session",
    )
    assert_status(resp, 404, label="other-user GET user-A's session")


def test_other_user_list_excludes_user_a_session(fx: Fixture) -> None:
    resp = fx.client.request(
        "GET", f"/client/{fx.test_client_id}/bot/sessions",
        headers=headers_for(fx.other_user_subject),
        label="other-user list sessions",
    )
    assert_status(resp, 200, label="other-user list")
    ids = [s["id"] for s in resp.json().get("sessions", [])]
    if fx.session_id_a in ids:
        raise AssertionError(
            f"user B should not see user A's session in their list, but it is present: {ids}"
        )


def test_other_user_can_create_own_session(fx: Fixture) -> None:
    resp = fx.client.request(
        "POST", f"/client/{fx.test_client_id}/bot/sessions",
        headers=headers_for(fx.other_user_subject),
        json_body={},
        label="user B create session",
    )
    assert_status(resp, 201, label="POST /bot/sessions as other user")
    fx.session_id_b = resp.json()["id"]
    info(f"Session B (other user): {fx.session_id_b}")


# ---------- Tests: super-admin ----------

def test_super_admin_list_requires_clientid(fx: Fixture) -> None:
    resp = fx.client.request(
        "GET", "/super-admin/bot/sessions",
        headers=headers_for(fx.admin_user_subject),
        label="super-admin list w/o clientId",
    )
    assert_status(resp, 400, label="super-admin list w/o clientId")


def test_super_admin_list_includes_session_a(fx: Fixture) -> None:
    resp = fx.client.request(
        "GET", f"/super-admin/bot/sessions?clientId={fx.test_client_id}",
        headers=headers_for(fx.admin_user_subject),
        label="super-admin list",
    )
    assert_status(resp, 200, label="super-admin list with clientId")
    ids = [s["id"] for s in resp.json().get("sessions", [])]
    if fx.session_id_a not in ids:
        raise AssertionError(f"super-admin list missing session A: ids={ids}")


def test_super_admin_get_session(fx: Fixture) -> None:
    # Create a fresh session that we'll soft-delete in the next test.
    create = fx.client.request(
        "POST", f"/client/{fx.test_client_id}/bot/sessions",
        headers=headers_for(fx.user_subject),
        json_body={},
        label="create soon-to-be-deleted session",
    )
    assert_status(create, 201, label="create session for admin-delete")
    fx.session_id_for_admin = create.json()["id"]

    resp = fx.client.request(
        "GET",
        f"/super-admin/bot/sessions/{fx.session_id_for_admin}?clientId={fx.test_client_id}",
        headers=headers_for(fx.admin_user_subject),
        label="super-admin get session",
    )
    assert_status(resp, 200, label="super-admin GET session")


def test_super_admin_soft_delete(fx: Fixture) -> None:
    if not fx.session_id_for_admin:
        raise AssertionError("session_id_for_admin not seeded; can't run delete test")
    resp = fx.client.request(
        "DELETE",
        f"/super-admin/bot/sessions/{fx.session_id_for_admin}?clientId={fx.test_client_id}",
        headers=headers_for(fx.admin_user_subject),
        label="super-admin delete session",
    )
    assert_status(resp, 204, label="super-admin DELETE session")


def test_owner_cant_get_after_admin_delete(fx: Fixture) -> None:
    if not fx.session_id_for_admin:
        warn("no admin-deleted session to verify; skipping")
        return
    resp = fx.client.request(
        "GET",
        f"/client/{fx.test_client_id}/bot/sessions/{fx.session_id_for_admin}",
        headers=headers_for(fx.user_subject),
        label="owner GET after admin delete",
    )
    assert_status(resp, 404, label="owner GET after admin-delete")


def test_super_admin_get_after_delete_404(fx: Fixture) -> None:
    if not fx.session_id_for_admin:
        warn("no admin-deleted session to verify; skipping")
        return
    resp = fx.client.request(
        "GET",
        f"/super-admin/bot/sessions/{fx.session_id_for_admin}?clientId={fx.test_client_id}",
        headers=headers_for(fx.admin_user_subject),
        label="super-admin GET after delete",
    )
    assert_status(resp, 404, label="super-admin GET after delete (is_deleted filter)")


# ---------- Setup: cross-client doc seeding ----------

def setup_other_client_document(fx: Fixture) -> None:
    """Create a SECOND test client with one document so the cross-client
    isolation tests have something to point at. Best-effort - on failure
    the cross-client tests log a warning and skip."""
    try:
        cid = create_test_client(fx, suffix="x")
        enable_client_sync(cid)
        zip_path = build_pdf_zip(folder_name="cross-client-batch")
        try:
            batch_id = upload_batch(fx, cid, zip_path)
            body = wait_for_batch_completion(fx, cid, batch_id)
            doc_id = pick_document_id(body)
            fx.other_test_client_id = cid
            fx.other_client_document_id = doc_id
            info(f"Cross-client document: {doc_id} under client {cid}")
        finally:
            try:
                zip_path.unlink()
            except OSError:
                pass
    except Exception as exc:  # noqa: BLE001 - tolerate seed failure, just skip cross-client tests
        warn(f"cross-client setup skipped: {type(exc).__name__}: {exc}")


# ---------- Main entry ----------

def parse_args(argv: list[str]) -> argparse.Namespace:
    p = argparse.ArgumentParser(description="End-to-end manual test for chatbot v8 features.")
    p.add_argument("--base-url", default=DEFAULT_BASE_URL,
                   help=f"queryAPI base URL (default {DEFAULT_BASE_URL})")
    p.add_argument("--debug", action="store_true",
                   help="Print full request and response (headers + body) for every API call.")
    p.add_argument("--user", default=DEFAULT_USER_SUBJECT,
                   help=f"Acting user subject/email (default {DEFAULT_USER_SUBJECT})")
    p.add_argument("--other-user", default=DEFAULT_OTHER_USER_SUBJECT,
                   help="A second user used for owner-only isolation tests")
    p.add_argument("--admin-user", default=DEFAULT_ADMIN_USER_SUBJECT,
                   help="Subject used for super-admin endpoints")
    p.add_argument("--skip-chat", action="store_true",
                   help="Don't call /turns (skip real FastAPI agent traffic)")
    p.add_argument("--skip-cross-client", action="store_true",
                   help="Skip the cross-client second-client setup (faster runs)")
    p.add_argument("--insecure", action="store_true",
                   help="Skip TLS cert verification. Use against AWS ELB hostnames "
                        "that serve the default self-signed cert.")
    return p.parse_args(argv)


def main(argv: Optional[list[str]] = None) -> int:
    args = parse_args(argv if argv is not None else sys.argv[1:])

    info(f"Base URL:        {args.base_url}")
    info(f"User:            {args.user}")
    info(f"Other user:      {args.other_user}")
    info(f"Admin user:      {args.admin_user}")
    info(f"Debug mode:      {args.debug}")
    info(f"Skip chat:       {args.skip_chat}")
    info(f"Skip x-client:   {args.skip_cross_client}")
    info(f"Insecure TLS:    {args.insecure}")

    fx = Fixture(
        client=HTTPClient(args.base_url, debug=args.debug, insecure=args.insecure),
        user_subject=args.user,
        other_user_subject=args.other_user,
        admin_user_subject=args.admin_user,
        skip_chat=args.skip_chat,
    )

    runner = TestRunner()

    # ---- Hard prerequisites: stop early if the basics fail ----
    if not runner.run("queryAPI healthy", lambda: health_check(fx)):
        return runner.summary()

    step("Setup: create test client, enable sync, seed folder + document")
    try:
        fx.test_client_id = create_test_client(fx)
        enable_client_sync(fx.test_client_id)
        fx.folder_id = create_folder(fx, fx.test_client_id)
        zip_path = build_pdf_zip()
        try:
            batch_id = upload_batch(fx, fx.test_client_id, zip_path)
            body = wait_for_batch_completion(fx, fx.test_client_id, batch_id)
            fx.document_id = pick_document_id(body)
            info(f"Seeded document: {fx.document_id}")
        finally:
            try:
                zip_path.unlink()
            except OSError:
                pass
    except Exception as exc:  # noqa: BLE001
        error(f"setup failed: {exc}")
        traceback.print_exc()
        runner.results.append(TestResult("setup", False, 0.0, str(exc)))
        return runner.summary()

    if not args.skip_cross_client:
        step("Setup: seed a second client + document for cross-client tests")
        setup_other_client_document(fx)

    # ---- Bot session lifecycle ----
    runner.run("create bot session", lambda: test_create_session(fx))
    runner.run("get session by id", lambda: test_get_session_by_id(fx))
    runner.run("list sessions includes the new session", lambda: test_list_sessions(fx))
    runner.run("list sessions rejects bad pagination", lambda: test_list_sessions_pagination_bounds(fx))
    runner.run("404 for unknown session id", lambda: test_session_404_for_unknown_id(fx))

    # ---- Scope ----
    runner.run("scope: add documents", lambda: test_scope_add_documents(fx))
    runner.run("scope: add folders", lambda: test_scope_add_folders(fx))
    runner.run("scope: dedupe duplicate ids", lambda: test_scope_dedupe(fx))
    runner.run("scope: add+remove same id rejected", lambda: test_scope_add_remove_conflict(fx))
    runner.run("scope: cross-client document rejected", lambda: test_scope_cross_client_document(fx))
    runner.run("scope: remove documents", lambda: test_scope_remove_documents(fx))

    # ---- Turns ----
    runner.run("turn 1 + title derivation", lambda: test_post_first_turn_and_title_derivation(fx))
    runner.run("list turns ordered ASC", lambda: test_list_turns(fx))
    runner.run("turn 2 keeps the derived title", lambda: test_post_second_turn_keeps_title(fx))
    runner.run("resubmit completed prompt -> 409 already_complete", lambda: test_resubmit_same_prompt_already_complete(fx))
    runner.run("prompt empty -> 400 prompt_empty", lambda: test_prompt_empty_rejected(fx))
    runner.run("prompt too long -> 400 prompt_too_long", lambda: test_prompt_too_long_rejected(fx))

    # ---- M5 document reads ----
    runner.run("GET /documents/{id}/schema (unbound)", lambda: test_document_schema_unbound(fx))
    runner.run("GET /documents/{id}/all-metadata", lambda: test_document_all_metadata(fx))
    runner.run("cross-client schema -> 404", lambda: test_document_schema_cross_client_404(fx))
    runner.run("cross-client all-metadata -> 404", lambda: test_document_all_metadata_cross_client_404(fx))

    # ---- Owner-only isolation ----
    runner.run("owner-only: user B can't GET user A's session", lambda: test_owner_only_isolation(fx))
    runner.run("owner-only: user B's list excludes user A", lambda: test_other_user_list_excludes_user_a_session(fx))
    runner.run("user B creates own session", lambda: test_other_user_can_create_own_session(fx))

    # ---- Super-admin ----
    runner.run("super-admin list missing clientId -> 400", lambda: test_super_admin_list_requires_clientid(fx))
    runner.run("super-admin list includes session A", lambda: test_super_admin_list_includes_session_a(fx))
    runner.run("super-admin GET session", lambda: test_super_admin_get_session(fx))
    runner.run("super-admin DELETE session", lambda: test_super_admin_soft_delete(fx))
    runner.run("owner cannot GET admin-deleted session", lambda: test_owner_cant_get_after_admin_delete(fx))
    runner.run("admin GET deleted session -> 404", lambda: test_super_admin_get_after_delete_404(fx))

    return runner.summary()


if __name__ == "__main__":
    sys.exit(main())