From 8a697cf64f6d05d8ea9ac6d6d2a70013119cef19 Mon Sep 17 00:00:00 2001 From: Jacob Mathison Date: Wed, 22 Jul 2026 09:13:54 -0500 Subject: [PATCH] feat: deploy training data to zoe root --- .codex/environments/environment.toml | 5 ++++ .woodpecker.yml | 9 +++++++ AGENTS.md | 7 +++-- README.md | 13 +++++++--- SDG_BRIDGE_PLAN.md | 26 +++++++++++-------- llm.txt | 3 ++- manifest.llm.json | 8 ++++-- scripts/deploy_training_data.sh | 38 ++++++++++++++++++++++++++++ scripts/generate_training_data.py | 10 ++++++-- scripts/precommit-check.sh | 1 + tests/test_generate_training_data.py | 26 ++++++++++++++++++- 11 files changed, 124 insertions(+), 22 deletions(-) create mode 100755 scripts/deploy_training_data.sh diff --git a/.codex/environments/environment.toml b/.codex/environments/environment.toml index 5c34e35..4218894 100644 --- a/.codex/environments/environment.toml +++ b/.codex/environments/environment.toml @@ -30,6 +30,11 @@ name = "Training Data Dry Run" icon = "tool" command = "python3 scripts/generate_training_data.py --dry-run" +[[actions]] +name = "Deploy Training Data" +icon = "tool" +command = "scripts/deploy_training_data.sh" + [[actions]] name = "Run Logger" icon = "run" diff --git a/.woodpecker.yml b/.woodpecker.yml index 26dbea7..9ac6919 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -10,3 +10,12 @@ steps: - rustc --version - cargo --version - scripts/precommit-check.sh + + deploy_training_data: + image: rust:1.95 + commands: + - apt-get update && apt-get install -y python3 + - scripts/deploy_training_data.sh + when: + - event: push + branch: main diff --git a/AGENTS.md b/AGENTS.md index c33e957..eb71801 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,6 +21,7 @@ - Codex environment: `.codex/environments/environment.toml` - Training data dry run: `python3 scripts/generate_training_data.py --dry-run` - Local training data bridge: `python3 scripts/generate_training_data.py` +- Deploy training data to Zoe: `scripts/deploy_training_data.sh` ## Environment @@ -52,12 +53,14 @@ Use `.env` for local values and keep it out of git. Update `.env.example`, `READ - `SDG_BRIDGE_PLAN.md` documents the local-to-Data-Designer bridge design. - `scripts/generate_training_data.py` reads generated dictionary artifacts and produces instruction-format seed datasets. -- Default output is `../workspace_Data/data/module_controller_intents_/`. -- Default manifest patch target is `../workspace_Data/manifest.llm.json`. +- Default output is `/Volumes/Zoe/custom-local-llm/training-data/module_controller_intents_/`. +- Default manifest patch target is `/Volumes/Zoe/custom-local-llm/manifest.llm.json`. +- `/Volumes/Zoe/custom-local-llm` has its own `README.md`, `AGENTS.md`, `manifest.llm.json`, and `llm.txt`. - Use `--dry-run` for validation; it must not write output data or patch manifests. - Mimir/Data Designer expansion only runs when `--sdg-host`, `--sdg-user`, and `--sdg-key` are all provided. - Do not start or restart Mimir llama.cpp/Keiro from this bridge. - If Mimir expansion fails, the bridge keeps the local seed dataset, writes `SDG_ERROR.txt`, patches the manifest with `generated_local_remote_failed`, and exits non-zero. +- Woodpecker deploys training data on main-branch pushes with `scripts/deploy_training_data.sh`. ## Dispatch Rules diff --git a/README.md b/README.md index 5437f0b..bcb789b 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ cargo run -- run to_slug --payload '{"text":"Hello, World!"}' cargo run -- run write_note --payload '{"filename":"hello.txt","content":"hello"}' scripts/precommit-check.sh python3 scripts/generate_training_data.py --dry-run +scripts/deploy_training_data.sh ``` `serve` starts: @@ -116,7 +117,7 @@ git config core.hooksPath .githooks ## SDG Bridge -`scripts/generate_training_data.py` generates instruction-format seed data from current dictionary artifacts. +`scripts/generate_training_data.py` generates instruction-format seed data from current dictionary artifacts. The default export root is `/Volumes/Zoe/custom-local-llm`. Dry run: @@ -129,10 +130,16 @@ Local dataset generation: ```sh python3 scripts/generate_training_data.py \ --dictionary-dir dictionary/ \ - --output-dir ../workspace_Data/data/ \ - --manifest ../workspace_Data/manifest.llm.json + --output-dir /Volumes/Zoe/custom-local-llm/training-data/ \ + --manifest /Volumes/Zoe/custom-local-llm/manifest.llm.json ``` Optional Mimir/Data Designer expansion is enabled only when `--sdg-host`, `--sdg-user`, and `--sdg-key` are supplied. The bridge does not start or restart Mimir inference services. If remote expansion fails, the timestamped local seed dataset remains in place, `SDG_ERROR.txt` is written in that dataset directory, and the manifest entry is marked `generated_local_remote_failed`. + +Woodpecker deploys a new training-data export to Zoe on `main` pushes by running: + +```sh +scripts/deploy_training_data.sh +``` diff --git a/SDG_BRIDGE_PLAN.md b/SDG_BRIDGE_PLAN.md index a30b086..424fad8 100644 --- a/SDG_BRIDGE_PLAN.md +++ b/SDG_BRIDGE_PLAN.md @@ -8,8 +8,9 @@ scripts/generate_training_data.py This script reads the generated dictionary artifacts from this project, optionally runs a NeMo Data Designer expansion pass on Mimir via SSH, and writes a timestamped raw training -dataset into the `workspace_Data` pipeline, then registers it in `manifest.llm.json` so the -Training Monitor picks it up automatically. +dataset into `/Volumes/Zoe/custom-local-llm/training-data`, then registers it in +`/Volumes/Zoe/custom-local-llm/manifest.llm.json` so local training tools can point at one +stable artifact root. --- @@ -46,10 +47,10 @@ Step 2 (remote — Mimir, optional): │ --dataset-name module_controller_intents_ SCP expanded dataset back │ -Step 3 (local — workspace_Data): │ - Write to data/module_controller_intents_/ ◄──────────────┘ +Step 3 (local — Zoe custom-local-llm): │ + Write to training-data/module_controller_intents_/ ◄─────┘ Patch manifest.llm.json with timestamped entry - Training Monitor picks it up as a new Data Source + Training tools pick it up from the Zoe artifact root ``` --- @@ -59,7 +60,7 @@ Step 3 (local — workspace_Data): │ Every dataset generated by this script is tagged with a **Unix timestamp at generation time**. The timestamp is embedded in: -1. **Output directory name:** `data/module_controller_intents_/` +1. **Output directory name:** `training-data/module_controller_intents_/` 2. **Manifest entry key:** `"module_controller_intents_"` 3. **Manifest entry field:** `"generated_at": ` 4. **Every JSONL record's metadata:** `"generated_at": ` @@ -167,7 +168,7 @@ columns: "redistribution_allowed": false, "estimated_tokens": null, "status": "generated_local", - "local_path": "data/module_controller_intents_", + "local_path": "training-data/module_controller_intents_", "local_format": "jsonl", "generator": "mp-ai-module-controller/scripts/generate_training_data.py", "sdg_engine": "data-designer", @@ -191,14 +192,14 @@ columns: # Seed only (no NeMo expansion — fast, local, 260 records): python3 scripts/generate_training_data.py \ --dictionary-dir dictionary/ \ - --output-dir ../workspace_Data/data/ \ - --manifest ../workspace_Data/manifest.llm.json + --output-dir /Volumes/Zoe/custom-local-llm/training-data/ \ + --manifest /Volumes/Zoe/custom-local-llm/manifest.llm.json # Full SDG run via Mimir (requires NVIDIA_API_KEY on Mimir): python3 scripts/generate_training_data.py \ --dictionary-dir dictionary/ \ - --output-dir ../workspace_Data/data/ \ - --manifest ../workspace_Data/manifest.llm.json \ + --output-dir /Volumes/Zoe/custom-local-llm/training-data/ \ + --manifest /Volumes/Zoe/custom-local-llm/manifest.llm.json \ --sdg-host 100.80.52.47 \ --sdg-user aaron-pressey \ --sdg-key ~/.ssh/silma_orson_ed25519 \ @@ -206,6 +207,9 @@ python3 scripts/generate_training_data.py \ # Dry run (prints what would be generated, writes nothing): python3 scripts/generate_training_data.py --dry-run + +# Woodpecker deploy target: +scripts/deploy_training_data.sh ``` --- diff --git a/llm.txt b/llm.txt index 348c1b2..38d04e3 100644 --- a/llm.txt +++ b/llm.txt @@ -32,6 +32,7 @@ Commands: - `scripts/codex-run-logger.sh` - `scripts/precommit-check.sh` - `scripts/generate_training_data.py --dry-run` +- `scripts/deploy_training_data.sh` - `git config core.hooksPath .githooks` - `.codex/environments/environment.toml` @@ -59,4 +60,4 @@ Codex environment: `.codex/environments/environment.toml` defines setup plus Bui CI and hooks: `.gitea/workflows/ci.yml`, `.woodpecker.yml`, and `.githooks/pre-commit` all use `scripts/precommit-check.sh` to verify formatting, tests, and dictionary generation. -SDG bridge: `scripts/generate_training_data.py` reads `dictionary/actions.index.json`, `dictionary/model.codebook.json`, and `dictionary/model.examples.jsonl`, then writes instruction-format datasets to `../workspace_Data/data/module_controller_intents_/` and patches `../workspace_Data/manifest.llm.json`. Use `--dry-run` to validate without writes. Optional Mimir expansion requires `--sdg-host`, `--sdg-user`, and `--sdg-key`; failed remote expansion leaves local seeds and writes `SDG_ERROR.txt`. +SDG bridge: `scripts/generate_training_data.py` reads `dictionary/actions.index.json`, `dictionary/model.codebook.json`, and `dictionary/model.examples.jsonl`, then writes instruction-format datasets to `/Volumes/Zoe/custom-local-llm/training-data/module_controller_intents_/` and patches `/Volumes/Zoe/custom-local-llm/manifest.llm.json`. Use `--dry-run` to validate without writes. Optional Mimir expansion requires `--sdg-host`, `--sdg-user`, and `--sdg-key`; failed remote expansion leaves local seeds and writes `SDG_ERROR.txt`. Woodpecker deploys the export with `scripts/deploy_training_data.sh`. diff --git a/manifest.llm.json b/manifest.llm.json index a13e738..018b67b 100644 --- a/manifest.llm.json +++ b/manifest.llm.json @@ -29,6 +29,7 @@ "precommit_check": "scripts/precommit-check.sh", "training_data_dry_run": "python3 scripts/generate_training_data.py --dry-run", "generate_training_data": "python3 scripts/generate_training_data.py", + "deploy_training_data": "scripts/deploy_training_data.sh", "install_hooks": "git config core.hooksPath .githooks" }, "environment": { @@ -230,8 +231,10 @@ "sdg_bridge": { "design_doc": "SDG_BRIDGE_PLAN.md", "script": "scripts/generate_training_data.py", - "default_output_dir": "../workspace_Data/data/", - "default_manifest": "../workspace_Data/manifest.llm.json", + "deploy_script": "scripts/deploy_training_data.sh", + "custom_local_llm_root": "/Volumes/Zoe/custom-local-llm", + "default_output_dir": "/Volumes/Zoe/custom-local-llm/training-data/", + "default_manifest": "/Volumes/Zoe/custom-local-llm/manifest.llm.json", "dataset_prefix": "module_controller_intents", "schema_profile": "instruction", "dry_run": "python3 scripts/generate_training_data.py --dry-run", @@ -242,6 +245,7 @@ "codex_environment": ".codex/environments/environment.toml", "gitea_actions": ".gitea/workflows/ci.yml", "woodpecker": ".woodpecker.yml", + "woodpecker_deploy": "scripts/deploy_training_data.sh", "precommit_hook": ".githooks/pre-commit", "shared_check": "scripts/precommit-check.sh", "checks": [ diff --git a/scripts/deploy_training_data.sh b/scripts/deploy_training_data.sh new file mode 100755 index 0000000..8d8b858 --- /dev/null +++ b/scripts/deploy_training_data.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." + +export COPYFILE_DISABLE=1 + +CUSTOM_LOCAL_LLM_ROOT="${CUSTOM_LOCAL_LLM_ROOT:-/Volumes/Zoe/custom-local-llm}" +TRAINING_DATA_DIR="${TRAINING_DATA_DIR:-$CUSTOM_LOCAL_LLM_ROOT/training-data}" +TRAINING_DATA_MANIFEST="${TRAINING_DATA_MANIFEST:-$CUSTOM_LOCAL_LLM_ROOT/manifest.llm.json}" + +if [[ ! -d "$CUSTOM_LOCAL_LLM_ROOT" ]]; then + echo "missing custom local LLM root: $CUSTOM_LOCAL_LLM_ROOT" >&2 + exit 1 +fi + +mkdir -p "$TRAINING_DATA_DIR" "$CUSTOM_LOCAL_LLM_ROOT/logs" "$CUSTOM_LOCAL_LLM_ROOT/models" "$CUSTOM_LOCAL_LLM_ROOT/benchmarks" + +if [[ ! -s "$TRAINING_DATA_MANIFEST" ]]; then + echo "missing training data manifest: $TRAINING_DATA_MANIFEST" >&2 + exit 1 +fi + +cargo run -- dictionary generate +python3 scripts/generate_training_data.py \ + --dictionary-dir dictionary/ \ + --output-dir "$TRAINING_DATA_DIR" \ + --manifest "$TRAINING_DATA_MANIFEST" + +python3 - <<'PY' +import os +from pathlib import Path + +root = Path(os.environ.get("CUSTOM_LOCAL_LLM_ROOT", "/Volumes/Zoe/custom-local-llm")) +for path in root.rglob("._*"): + if path.is_file(): + path.unlink() +PY diff --git a/scripts/generate_training_data.py b/scripts/generate_training_data.py index ebb3632..e1c834c 100755 --- a/scripts/generate_training_data.py +++ b/scripts/generate_training_data.py @@ -15,6 +15,9 @@ from typing import Any INSTRUCTION = "Map the following natural language request to the correct action and payload." DATASET_PREFIX = "module_controller_intents" DEFAULT_MIN_RECORDS_PER_ACTION = 20 +CUSTOM_LOCAL_LLM_ROOT = Path("/Volumes/Zoe/custom-local-llm") +DEFAULT_OUTPUT_DIR = CUSTOM_LOCAL_LLM_ROOT / "training-data" +DEFAULT_MANIFEST = CUSTOM_LOCAL_LLM_ROOT / "manifest.llm.json" def main() -> int: @@ -32,8 +35,8 @@ def main() -> int: def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--dictionary-dir", default="dictionary/") - parser.add_argument("--output-dir", default="../workspace_Data/data/") - parser.add_argument("--manifest", default="../workspace_Data/manifest.llm.json") + parser.add_argument("--output-dir", default=str(DEFAULT_OUTPUT_DIR)) + parser.add_argument("--manifest", default=str(DEFAULT_MANIFEST)) parser.add_argument("--dry-run", action="store_true") parser.add_argument("--sdg-host") parser.add_argument("--sdg-user") @@ -469,6 +472,9 @@ def patch_manifest( def relative_data_path(dataset_dir: Path) -> str: parts = dataset_dir.parts + if "training-data" in parts: + index = len(parts) - 1 - list(reversed(parts)).index("training-data") + return "/".join(parts[index:]) if "data" in parts: index = len(parts) - 1 - list(reversed(parts)).index("data") return "/".join(parts[index:]) diff --git a/scripts/precommit-check.sh b/scripts/precommit-check.sh index ff6eeb8..c57623a 100755 --- a/scripts/precommit-check.sh +++ b/scripts/precommit-check.sh @@ -32,5 +32,6 @@ PY python3 -m unittest tests/test_generate_training_data.py python3 scripts/generate_training_data.py --dry-run +test -x scripts/deploy_training_data.sh echo "precommit check complete" diff --git a/tests/test_generate_training_data.py b/tests/test_generate_training_data.py index 1b38b21..53ca5df 100644 --- a/tests/test_generate_training_data.py +++ b/tests/test_generate_training_data.py @@ -1,6 +1,7 @@ import argparse import json import shutil +import subprocess import tempfile import unittest from pathlib import Path @@ -14,7 +15,23 @@ class GenerateTrainingDataTests(unittest.TestCase): self.temp = tempfile.TemporaryDirectory() self.root = Path(self.temp.name) self.dictionary_dir = self.root / "dictionary" - shutil.copytree(self.repo / "dictionary", self.dictionary_dir) + self.dictionary_dir.mkdir() + shutil.copy2(self.repo / "dictionary" / "static-base.json", self.dictionary_dir / "static-base.json") + subprocess.run( + [ + "cargo", + "run", + "--", + "dictionary", + "generate", + "--out-dir", + str(self.dictionary_dir), + ], + cwd=self.repo, + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) self.output_dir = self.root / "workspace_Data" / "data" self.output_dir.mkdir(parents=True) self.manifest = self.root / "workspace_Data" / "manifest.llm.json" @@ -96,6 +113,13 @@ class GenerateTrainingDataTests(unittest.TestCase): self.assertIn("codebook_checksum", dataset) self.assertTrue((self.output_dir / "module_controller_intents_1234567890" / "train.jsonl").exists()) + def test_training_data_relative_path_prefers_custom_root_shape(self): + path = Path("/Volumes/Zoe/custom-local-llm/training-data/module_controller_intents_123") + self.assertEqual( + bridge.relative_data_path(path), + "training-data/module_controller_intents_123", + ) + if __name__ == "__main__": unittest.main()