feat: deploy training data to zoe root

.codex/environments/environment.toml

@@ -30,6 +30,11 @@ name = "Training Data Dry Run"
 icon = "tool"
 command = "python3 scripts/generate_training_data.py --dry-run"
 
+[[actions]]
+name = "Deploy Training Data"
+icon = "tool"
+command = "scripts/deploy_training_data.sh"
+
 [[actions]]
 name = "Run Logger"
 icon = "run"

.woodpecker.yml

@@ -10,3 +10,12 @@ steps:
       - rustc --version
       - cargo --version
       - scripts/precommit-check.sh
+
+  deploy_training_data:
+    image: rust:1.95
+    commands:
+      - apt-get update && apt-get install -y python3
+      - scripts/deploy_training_data.sh
+    when:
+      - event: push
+        branch: main

AGENTS.md

@@ -21,6 +21,7 @@
 - Codex environment: `.codex/environments/environment.toml`
 - Training data dry run: `python3 scripts/generate_training_data.py --dry-run`
 - Local training data bridge: `python3 scripts/generate_training_data.py`
+- Deploy training data to Zoe: `scripts/deploy_training_data.sh`
 
 ## Environment
 
@@ -52,12 +53,14 @@ Use `.env` for local values and keep it out of git. Update `.env.example`, `READ
 
 - `SDG_BRIDGE_PLAN.md` documents the local-to-Data-Designer bridge design.
 - `scripts/generate_training_data.py` reads generated dictionary artifacts and produces instruction-format seed datasets.
-- Default output is `../workspace_Data/data/module_controller_intents_<unix_ts>/`.
+- Default output is `/Volumes/Zoe/custom-local-llm/training-data/module_controller_intents_<unix_ts>/`.
-- Default manifest patch target is `../workspace_Data/manifest.llm.json`.
+- Default manifest patch target is `/Volumes/Zoe/custom-local-llm/manifest.llm.json`.
+- `/Volumes/Zoe/custom-local-llm` has its own `README.md`, `AGENTS.md`, `manifest.llm.json`, and `llm.txt`.
 - Use `--dry-run` for validation; it must not write output data or patch manifests.
 - Mimir/Data Designer expansion only runs when `--sdg-host`, `--sdg-user`, and `--sdg-key` are all provided.
 - Do not start or restart Mimir llama.cpp/Keiro from this bridge.
 - If Mimir expansion fails, the bridge keeps the local seed dataset, writes `SDG_ERROR.txt`, patches the manifest with `generated_local_remote_failed`, and exits non-zero.
+- Woodpecker deploys training data on main-branch pushes with `scripts/deploy_training_data.sh`.
 
 ## Dispatch Rules
 

README.md

@@ -26,6 +26,7 @@ cargo run -- run to_slug --payload '{"text":"Hello, World!"}'
 cargo run -- run write_note --payload '{"filename":"hello.txt","content":"hello"}'
 scripts/precommit-check.sh
 python3 scripts/generate_training_data.py --dry-run
+scripts/deploy_training_data.sh
 ```
 
 `serve` starts:
@@ -116,7 +117,7 @@ git config core.hooksPath .githooks
 
 ## SDG Bridge
 
-`scripts/generate_training_data.py` generates instruction-format seed data from current dictionary artifacts.
+`scripts/generate_training_data.py` generates instruction-format seed data from current dictionary artifacts. The default export root is `/Volumes/Zoe/custom-local-llm`.
 
 Dry run:
 
@@ -129,10 +130,16 @@ Local dataset generation:
 ```sh
 python3 scripts/generate_training_data.py \
   --dictionary-dir dictionary/ \
-  --output-dir ../workspace_Data/data/ \
+  --output-dir /Volumes/Zoe/custom-local-llm/training-data/ \
-  --manifest ../workspace_Data/manifest.llm.json
+  --manifest /Volumes/Zoe/custom-local-llm/manifest.llm.json
 ```
 
 Optional Mimir/Data Designer expansion is enabled only when `--sdg-host`, `--sdg-user`, and `--sdg-key` are supplied. The bridge does not start or restart Mimir inference services.
 
 If remote expansion fails, the timestamped local seed dataset remains in place, `SDG_ERROR.txt` is written in that dataset directory, and the manifest entry is marked `generated_local_remote_failed`.
+
+Woodpecker deploys a new training-data export to Zoe on `main` pushes by running:
+
+```sh
+scripts/deploy_training_data.sh
+```

SDG_BRIDGE_PLAN.md

@@ -8,8 +8,9 @@ scripts/generate_training_data.py
 
 This script reads the generated dictionary artifacts from this project, optionally runs a
 NeMo Data Designer expansion pass on Mimir via SSH, and writes a timestamped raw training
-dataset into the `workspace_Data` pipeline, then registers it in `manifest.llm.json` so the
+dataset into `/Volumes/Zoe/custom-local-llm/training-data`, then registers it in
-Training Monitor picks it up automatically.
+`/Volumes/Zoe/custom-local-llm/manifest.llm.json` so local training tools can point at one
+stable artifact root.
 
 ---
 
@@ -46,10 +47,10 @@ Step 2 (remote — Mimir, optional):             │
          --dataset-name module_controller_intents_<timestamp>
   SCP expanded dataset back
                                                │
-Step 3 (local — workspace_Data):               │
+Step 3 (local — Zoe custom-local-llm):         │
-  Write to data/module_controller_intents_<timestamp>/ ◄──────────────┘
+  Write to training-data/module_controller_intents_<timestamp>/ ◄─────┘
   Patch manifest.llm.json with timestamped entry
-  Training Monitor picks it up as a new Data Source
+  Training tools pick it up from the Zoe artifact root
 ```
 
 ---
@@ -59,7 +60,7 @@ Step 3 (local — workspace_Data):               │
 Every dataset generated by this script is tagged with a **Unix timestamp at generation time**.
 The timestamp is embedded in:
 
-1. **Output directory name:** `data/module_controller_intents_<unix_ts>/`
+1. **Output directory name:** `training-data/module_controller_intents_<unix_ts>/`
 2. **Manifest entry key:** `"module_controller_intents_<unix_ts>"`
 3. **Manifest entry field:** `"generated_at": <unix_ts>`
 4. **Every JSONL record's metadata:** `"generated_at": <unix_ts>`
@@ -167,7 +168,7 @@ columns:
   "redistribution_allowed": false,
   "estimated_tokens": null,
   "status": "generated_local",
-  "local_path": "data/module_controller_intents_<unix_ts>",
+  "local_path": "training-data/module_controller_intents_<unix_ts>",
   "local_format": "jsonl",
   "generator": "mp-ai-module-controller/scripts/generate_training_data.py",
   "sdg_engine": "data-designer",
@@ -191,14 +192,14 @@ columns:
 # Seed only (no NeMo expansion — fast, local, 260 records):
 python3 scripts/generate_training_data.py \
   --dictionary-dir dictionary/ \
-  --output-dir ../workspace_Data/data/ \
+  --output-dir /Volumes/Zoe/custom-local-llm/training-data/ \
-  --manifest ../workspace_Data/manifest.llm.json
+  --manifest /Volumes/Zoe/custom-local-llm/manifest.llm.json
 
 # Full SDG run via Mimir (requires NVIDIA_API_KEY on Mimir):
 python3 scripts/generate_training_data.py \
   --dictionary-dir dictionary/ \
-  --output-dir ../workspace_Data/data/ \
+  --output-dir /Volumes/Zoe/custom-local-llm/training-data/ \
-  --manifest ../workspace_Data/manifest.llm.json \
+  --manifest /Volumes/Zoe/custom-local-llm/manifest.llm.json \
   --sdg-host 100.80.52.47 \
   --sdg-user aaron-pressey \
   --sdg-key ~/.ssh/silma_orson_ed25519 \
@@ -206,6 +207,9 @@ python3 scripts/generate_training_data.py \
 
 # Dry run (prints what would be generated, writes nothing):
 python3 scripts/generate_training_data.py --dry-run
+
+# Woodpecker deploy target:
+scripts/deploy_training_data.sh
 ```
 
 ---

llm.txt

@@ -32,6 +32,7 @@ Commands:
 - `scripts/codex-run-logger.sh`
 - `scripts/precommit-check.sh`
 - `scripts/generate_training_data.py --dry-run`
+- `scripts/deploy_training_data.sh`
 - `git config core.hooksPath .githooks`
 - `.codex/environments/environment.toml`
 
@@ -59,4 +60,4 @@ Codex environment: `.codex/environments/environment.toml` defines setup plus Bui
 
 CI and hooks: `.gitea/workflows/ci.yml`, `.woodpecker.yml`, and `.githooks/pre-commit` all use `scripts/precommit-check.sh` to verify formatting, tests, and dictionary generation.
 
-SDG bridge: `scripts/generate_training_data.py` reads `dictionary/actions.index.json`, `dictionary/model.codebook.json`, and `dictionary/model.examples.jsonl`, then writes instruction-format datasets to `../workspace_Data/data/module_controller_intents_<unix_ts>/` and patches `../workspace_Data/manifest.llm.json`. Use `--dry-run` to validate without writes. Optional Mimir expansion requires `--sdg-host`, `--sdg-user`, and `--sdg-key`; failed remote expansion leaves local seeds and writes `SDG_ERROR.txt`.
+SDG bridge: `scripts/generate_training_data.py` reads `dictionary/actions.index.json`, `dictionary/model.codebook.json`, and `dictionary/model.examples.jsonl`, then writes instruction-format datasets to `/Volumes/Zoe/custom-local-llm/training-data/module_controller_intents_<unix_ts>/` and patches `/Volumes/Zoe/custom-local-llm/manifest.llm.json`. Use `--dry-run` to validate without writes. Optional Mimir expansion requires `--sdg-host`, `--sdg-user`, and `--sdg-key`; failed remote expansion leaves local seeds and writes `SDG_ERROR.txt`. Woodpecker deploys the export with `scripts/deploy_training_data.sh`.

manifest.llm.json

@@ -29,6 +29,7 @@
     "precommit_check": "scripts/precommit-check.sh",
     "training_data_dry_run": "python3 scripts/generate_training_data.py --dry-run",
     "generate_training_data": "python3 scripts/generate_training_data.py",
+    "deploy_training_data": "scripts/deploy_training_data.sh",
     "install_hooks": "git config core.hooksPath .githooks"
   },
   "environment": {
@@ -230,8 +231,10 @@
   "sdg_bridge": {
     "design_doc": "SDG_BRIDGE_PLAN.md",
     "script": "scripts/generate_training_data.py",
-    "default_output_dir": "../workspace_Data/data/",
+    "deploy_script": "scripts/deploy_training_data.sh",
-    "default_manifest": "../workspace_Data/manifest.llm.json",
+    "custom_local_llm_root": "/Volumes/Zoe/custom-local-llm",
+    "default_output_dir": "/Volumes/Zoe/custom-local-llm/training-data/",
+    "default_manifest": "/Volumes/Zoe/custom-local-llm/manifest.llm.json",
     "dataset_prefix": "module_controller_intents",
     "schema_profile": "instruction",
     "dry_run": "python3 scripts/generate_training_data.py --dry-run",
@@ -242,6 +245,7 @@
     "codex_environment": ".codex/environments/environment.toml",
     "gitea_actions": ".gitea/workflows/ci.yml",
     "woodpecker": ".woodpecker.yml",
+    "woodpecker_deploy": "scripts/deploy_training_data.sh",
     "precommit_hook": ".githooks/pre-commit",
     "shared_check": "scripts/precommit-check.sh",
     "checks": [

scripts/deploy_training_data.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+cd "$(dirname "$0")/.."
+
+export COPYFILE_DISABLE=1
+
+CUSTOM_LOCAL_LLM_ROOT="${CUSTOM_LOCAL_LLM_ROOT:-/Volumes/Zoe/custom-local-llm}"
+TRAINING_DATA_DIR="${TRAINING_DATA_DIR:-$CUSTOM_LOCAL_LLM_ROOT/training-data}"
+TRAINING_DATA_MANIFEST="${TRAINING_DATA_MANIFEST:-$CUSTOM_LOCAL_LLM_ROOT/manifest.llm.json}"
+
+if [[ ! -d "$CUSTOM_LOCAL_LLM_ROOT" ]]; then
+  echo "missing custom local LLM root: $CUSTOM_LOCAL_LLM_ROOT" >&2
+  exit 1
+fi
+
+mkdir -p "$TRAINING_DATA_DIR" "$CUSTOM_LOCAL_LLM_ROOT/logs" "$CUSTOM_LOCAL_LLM_ROOT/models" "$CUSTOM_LOCAL_LLM_ROOT/benchmarks"
+
+if [[ ! -s "$TRAINING_DATA_MANIFEST" ]]; then
+  echo "missing training data manifest: $TRAINING_DATA_MANIFEST" >&2
+  exit 1
+fi
+
+cargo run -- dictionary generate
+python3 scripts/generate_training_data.py \
+  --dictionary-dir dictionary/ \
+  --output-dir "$TRAINING_DATA_DIR" \
+  --manifest "$TRAINING_DATA_MANIFEST"
+
+python3 - <<'PY'
+import os
+from pathlib import Path
+
+root = Path(os.environ.get("CUSTOM_LOCAL_LLM_ROOT", "/Volumes/Zoe/custom-local-llm"))
+for path in root.rglob("._*"):
+    if path.is_file():
+        path.unlink()
+PY

scripts/generate_training_data.py

@@ -15,6 +15,9 @@ from typing import Any
 INSTRUCTION = "Map the following natural language request to the correct action and payload."
 DATASET_PREFIX = "module_controller_intents"
 DEFAULT_MIN_RECORDS_PER_ACTION = 20
+CUSTOM_LOCAL_LLM_ROOT = Path("/Volumes/Zoe/custom-local-llm")
+DEFAULT_OUTPUT_DIR = CUSTOM_LOCAL_LLM_ROOT / "training-data"
+DEFAULT_MANIFEST = CUSTOM_LOCAL_LLM_ROOT / "manifest.llm.json"
 
 
 def main() -> int:
@@ -32,8 +35,8 @@ def main() -> int:
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--dictionary-dir", default="dictionary/")
-    parser.add_argument("--output-dir", default="../workspace_Data/data/")
+    parser.add_argument("--output-dir", default=str(DEFAULT_OUTPUT_DIR))
-    parser.add_argument("--manifest", default="../workspace_Data/manifest.llm.json")
+    parser.add_argument("--manifest", default=str(DEFAULT_MANIFEST))
     parser.add_argument("--dry-run", action="store_true")
     parser.add_argument("--sdg-host")
     parser.add_argument("--sdg-user")
@@ -469,6 +472,9 @@ def patch_manifest(
 
 def relative_data_path(dataset_dir: Path) -> str:
     parts = dataset_dir.parts
+    if "training-data" in parts:
+        index = len(parts) - 1 - list(reversed(parts)).index("training-data")
+        return "/".join(parts[index:])
     if "data" in parts:
         index = len(parts) - 1 - list(reversed(parts)).index("data")
         return "/".join(parts[index:])

scripts/precommit-check.sh

@@ -32,5 +32,6 @@ PY
 
 python3 -m unittest tests/test_generate_training_data.py
 python3 scripts/generate_training_data.py --dry-run
+test -x scripts/deploy_training_data.sh
 
 echo "precommit check complete"

tests/test_generate_training_data.py

@@ -1,6 +1,7 @@
 import argparse
 import json
 import shutil
+import subprocess
 import tempfile
 import unittest
 from pathlib import Path
@@ -14,7 +15,23 @@ class GenerateTrainingDataTests(unittest.TestCase):
         self.temp = tempfile.TemporaryDirectory()
         self.root = Path(self.temp.name)
         self.dictionary_dir = self.root / "dictionary"
-        shutil.copytree(self.repo / "dictionary", self.dictionary_dir)
+        self.dictionary_dir.mkdir()
+        shutil.copy2(self.repo / "dictionary" / "static-base.json", self.dictionary_dir / "static-base.json")
+        subprocess.run(
+            [
+                "cargo",
+                "run",
+                "--",
+                "dictionary",
+                "generate",
+                "--out-dir",
+                str(self.dictionary_dir),
+            ],
+            cwd=self.repo,
+            check=True,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
         self.output_dir = self.root / "workspace_Data" / "data"
         self.output_dir.mkdir(parents=True)
         self.manifest = self.root / "workspace_Data" / "manifest.llm.json"
@@ -96,6 +113,13 @@ class GenerateTrainingDataTests(unittest.TestCase):
         self.assertIn("codebook_checksum", dataset)
         self.assertTrue((self.output_dir / "module_controller_intents_1234567890" / "train.jsonl").exists())
 
+    def test_training_data_relative_path_prefers_custom_root_shape(self):
+        path = Path("/Volumes/Zoe/custom-local-llm/training-data/module_controller_intents_123")
+        self.assertEqual(
+            bridge.relative_data_path(path),
+            "training-data/module_controller_intents_123",
+        )
+
 
 if __name__ == "__main__":
     unittest.main()