feat: add sdg training data bridge

.codex/environments/environment.toml

@@ -25,6 +25,11 @@ name = "Precommit Check"
 icon = "tool"
 command = "scripts/precommit-check.sh"
 
+[[actions]]
+name = "Training Data Dry Run"
+icon = "tool"
+command = "python3 scripts/generate_training_data.py --dry-run"
+
 [[actions]]
 name = "Run Logger"
 icon = "run"

.gitea/workflows/ci.yml

@@ -27,6 +27,10 @@ jobs:
           rustc --version
           cargo --version
 
+      - name: Show Python version
+        shell: bash
+        run: python3 --version
+
       - name: Run precommit checks
         shell: bash
         run: scripts/precommit-check.sh

.gitignore

@@ -11,3 +11,5 @@
 /dictionary/model.examples.jsonl
 *.log
 .DS_Store
+__pycache__/
+*.py[cod]

.woodpecker.yml

@@ -6,6 +6,7 @@ steps:
   validate:
     image: rust:1.95
     commands:
+      - apt-get update && apt-get install -y python3
       - rustc --version
       - cargo --version
       - scripts/precommit-check.sh

AGENTS.md

@@ -19,6 +19,8 @@
 - Codex logger smoke test: `scripts/codex-run-logger.sh`
 - Precommit/CI check: `scripts/precommit-check.sh`
 - Codex environment: `.codex/environments/environment.toml`
+- Training data dry run: `python3 scripts/generate_training_data.py --dry-run`
+- Local training data bridge: `python3 scripts/generate_training_data.py`
 
 ## Environment
 
@@ -46,6 +48,17 @@ Use `.env` for local values and keep it out of git. Update `.env.example`, `READ
 - Do not manually edit generated dictionary files.
 - Generated metadata includes `dictionary_version`, `registry_checksum`, `static_base_checksum`, and `codebook_checksum`.
 
+## SDG Bridge Rules
+
+- `SDG_BRIDGE_PLAN.md` documents the local-to-Data-Designer bridge design.
+- `scripts/generate_training_data.py` reads generated dictionary artifacts and produces instruction-format seed datasets.
+- Default output is `../workspace_Data/data/module_controller_intents_<unix_ts>/`.
+- Default manifest patch target is `../workspace_Data/manifest.llm.json`.
+- Use `--dry-run` for validation; it must not write output data or patch manifests.
+- Mimir/Data Designer expansion only runs when `--sdg-host`, `--sdg-user`, and `--sdg-key` are all provided.
+- Do not start or restart Mimir llama.cpp/Keiro from this bridge.
+- If Mimir expansion fails, the bridge keeps the local seed dataset, writes `SDG_ERROR.txt`, patches the manifest with `generated_local_remote_failed`, and exits non-zero.
+
 ## Dispatch Rules
 
 - Dispatch only exact `action_id` values present in `dictionary/actions.index.json`.
@@ -81,4 +94,5 @@ Use `.env` for local values and keep it out of git. Update `.env.example`, `READ
 - Keep the CLI Rust-first and small.
 - Prefer adding scripts under `src/scripts/` and registering them in `src/registry.rs`.
 - Add focused tests for parsing, dictionary generation, action lookup, and payload validation.
+- Add stdlib Python tests for bridge behavior in `tests/` when changing `scripts/generate_training_data.py`.
 - Run `cargo test` before handing off code changes.

README.md

@@ -25,6 +25,7 @@ cargo run -- run calculate --payload '{"expression":"2 + 2 * 3"}'
 cargo run -- run to_slug --payload '{"text":"Hello, World!"}'
 cargo run -- run write_note --payload '{"filename":"hello.txt","content":"hello"}'
 scripts/precommit-check.sh
+python3 scripts/generate_training_data.py --dry-run
 ```
 
 `serve` starts:
@@ -112,3 +113,26 @@ Enable the hook after cloning:
 ```sh
 git config core.hooksPath .githooks
 ```
+
+## SDG Bridge
+
+`scripts/generate_training_data.py` generates instruction-format seed data from current dictionary artifacts.
+
+Dry run:
+
+```sh
+python3 scripts/generate_training_data.py --dry-run
+```
+
+Local dataset generation:
+
+```sh
+python3 scripts/generate_training_data.py \
+  --dictionary-dir dictionary/ \
+  --output-dir ../workspace_Data/data/ \
+  --manifest ../workspace_Data/manifest.llm.json
+```
+
+Optional Mimir/Data Designer expansion is enabled only when `--sdg-host`, `--sdg-user`, and `--sdg-key` are supplied. The bridge does not start or restart Mimir inference services.
+
+If remote expansion fails, the timestamped local seed dataset remains in place, `SDG_ERROR.txt` is written in that dataset directory, and the manifest entry is marked `generated_local_remote_failed`.

SDG_BRIDGE_PLAN.md

@@ -0,0 +1,232 @@
+# SDG Bridge Plan
+
+## Bridge Script Filename
+
+```
+scripts/generate_training_data.py
+```
+
+This script reads the generated dictionary artifacts from this project, optionally runs a
+NeMo Data Designer expansion pass on Mimir via SSH, and writes a timestamped raw training
+dataset into the `workspace_Data` pipeline, then registers it in `manifest.llm.json` so the
+Training Monitor picks it up automatically.
+
+---
+
+## Mimir — Data Designer Status
+
+Mimir (`100.80.52.47`) has **Data Designer 0.8.0** installed and ready:
+
+| Component | Location | Status |
+|-----------|----------|--------|
+| CLI | `/home/aaron-pressey/.venvs/data-designer/bin/data-designer` | ✅ installed |
+| Backend adapter | `/home/aaron-pressey/.local/share/home-grown-llm-data/nemo_data_designer_backend.py` | ✅ present |
+| Storage | `/mnt/storage/data-designer/` | ✅ mounted |
+| Model configs | `/mnt/storage/data-designer/model_configs.yaml` | ✅ configured |
+| Config env | `/home/aaron-pressey/.config/home-grown-llm-data/data-designer.env` | ✅ present |
+
+**Current model provider:** NVIDIA API (`nvidia-text` alias → `nvidia/nemotron-3-nano-30b-a3b`).
+Local llama.cpp on port 8081 is configured but not currently running — cloud provider is used
+for generation unless a local inference server is started separately.
+
+---
+
+## What the Bridge Does
+
+```
+Step 1 (local — this project):
+  dictionary/actions.index.json     ─┐
+  dictionary/model.codebook.json     ├─► generate_training_data.py ─► seed JSONL
+  dictionary/model.examples.jsonl   ─┘     (20 records/action, 260 total)
+                                               │
+Step 2 (remote — Mimir, optional):             │
+  SCP seed JSONL + DD config to Mimir ◄────────┘
+  SSH: data-designer create module_controller.yaml
+         --num-records <N>
+         --dataset-name module_controller_intents_<timestamp>
+  SCP expanded dataset back
+                                               │
+Step 3 (local — workspace_Data):               │
+  Write to data/module_controller_intents_<timestamp>/ ◄──────────────┘
+  Patch manifest.llm.json with timestamped entry
+  Training Monitor picks it up as a new Data Source
+```
+
+---
+
+## Timestamp Tagging
+
+Every dataset generated by this script is tagged with a **Unix timestamp at generation time**.
+The timestamp is embedded in:
+
+1. **Output directory name:** `data/module_controller_intents_<unix_ts>/`
+2. **Manifest entry key:** `"module_controller_intents_<unix_ts>"`
+3. **Manifest entry field:** `"generated_at": <unix_ts>`
+4. **Every JSONL record's metadata:** `"generated_at": <unix_ts>`
+
+This means you can run the bridge multiple times (after adding actions, changing examples,
+or expanding via NeMo) and the Training Monitor will show each run as a distinct, traceable
+Data Source. No clobbering, no ambiguity.
+
+---
+
+## Output Record Schema
+
+Each output record uses the `instruction` schema profile:
+
+```json
+{
+  "instruction": "Map the following natural language request to the correct action and payload.",
+  "input": "hash hello with sha256",
+  "output": "{\"action_id\": \"hash_string\", \"payload\": {\"text\": \"hello\", \"algorithm\": \"sha256\"}}",
+  "metadata": {
+    "action_id": "hash_string",
+    "compact": "{\"v\":\"1.0.0\",\"c\":\"<codebook_checksum>\",\"a\":\"A007\",\"p\":{\"F010\":\"hello\",\"F001\":\"sha256\"}}",
+    "source": "seed_intent_example",
+    "split": "train",
+    "generated_at": 1784660000
+  }
+}
+```
+
+---
+
+## Generation Strategy
+
+For each action in `actions.index.json`:
+
+1. **Seed examples** — emit each `intent_example` directly (source: `seed_intent_example`)
+2. **Alias variants** — one phrase per alias using the alias as the verb (source: `alias_variant`)
+3. **Payload field variants** — for enum fields, one record per enum value; for numeric fields,
+   low/mid/high values (source: `field_variant`)
+4. **Template expansion** — 5 records per action from `synthetic_template` with filled placeholders
+   (source: `template_expansion`)
+
+**Minimum seed target:** ~20 records × 13 actions = 260 records.
+**After NeMo expansion:** configurable via `--num-records` (default: 1000).
+
+Split: 90% train / 10% validation, partitioned by action so all 13 actions appear in both splits.
+
+---
+
+## Mimir Data Designer Config (auto-generated by bridge)
+
+The bridge writes this config to Mimir before running generation:
+
+```yaml
+# auto-generated by generate_training_data.py — do not edit by hand
+# generated_at: <unix_ts>
+
+model_config_path: /mnt/storage/data-designer/model_configs.yaml
+model_providers_path: /mnt/storage/data-designer/model_providers.yaml
+
+dataset:
+  name: module_controller_intents_<unix_ts>
+  schema_profile: instruction
+  seed_file: /mnt/storage/data-designer/managed-assets/module_controller_seed_<unix_ts>.jsonl
+
+columns:
+  - name: input
+    type: seed_passthrough
+
+  - name: output
+    type: llm_text
+    model_alias: nvidia-text
+    prompt: |
+      You are generating training data for a local action dispatcher.
+      Given the natural language request below, produce valid JSON with
+      exactly two fields: "action_id" (string) and "payload" (object).
+      The action_id must be one of: {action_ids}.
+      Vary the phrasing of the input naturally but keep the output schema strict.
+
+      Request: {{input}}
+    output_schema:
+      type: object
+      required: [action_id, payload]
+      additionalProperties: false
+      properties:
+        action_id:
+          type: string
+          enum: {action_ids}
+        payload:
+          type: object
+```
+
+---
+
+## Manifest Entry (auto-patched by bridge)
+
+```json
+"module_controller_intents_<unix_ts>": {
+  "name": "Module Controller Intents",
+  "category": "synthetic_sft",
+  "schema_profile": "instruction",
+  "source_url": null,
+  "acquisition_method": "local_synthetic_generator",
+  "license_spdx": "UNLICENSED",
+  "redistribution_allowed": false,
+  "estimated_tokens": null,
+  "status": "generated_local",
+  "local_path": "data/module_controller_intents_<unix_ts>",
+  "local_format": "jsonl",
+  "generator": "mp-ai-module-controller/scripts/generate_training_data.py",
+  "sdg_engine": "data-designer",
+  "sdg_host": "mimir (100.80.52.47)",
+  "sdg_model": "nvidia/nemotron-3-nano-30b-a3b",
+  "codebook_version": "1.0.0",
+  "codebook_checksum": "<from model.codebook.json>",
+  "registry_checksum": "<from actions.index.json>",
+  "generated_at": <unix_ts>,
+  "doc": "data/module_controller_intents_<unix_ts>/README.md"
+}
+```
+
+---
+
+## CLI Usage
+
+```bash
+# From mp-ai-module-controller project root:
+
+# Seed only (no NeMo expansion — fast, local, 260 records):
+python3 scripts/generate_training_data.py \
+  --dictionary-dir dictionary/ \
+  --output-dir ../workspace_Data/data/ \
+  --manifest ../workspace_Data/manifest.llm.json
+
+# Full SDG run via Mimir (requires NVIDIA_API_KEY on Mimir):
+python3 scripts/generate_training_data.py \
+  --dictionary-dir dictionary/ \
+  --output-dir ../workspace_Data/data/ \
+  --manifest ../workspace_Data/manifest.llm.json \
+  --sdg-host 100.80.52.47 \
+  --sdg-user aaron-pressey \
+  --sdg-key ~/.ssh/silma_orson_ed25519 \
+  --num-records 1000
+
+# Dry run (prints what would be generated, writes nothing):
+python3 scripts/generate_training_data.py --dry-run
+```
+
+---
+
+## Prerequisites
+
+- Python 3.11+ (stdlib only: `json`, `pathlib`, `hashlib`, `time`, `argparse`, `subprocess`)
+- SSH access to Mimir via `~/.ssh/silma_orson_ed25519` (for NeMo expansion pass)
+- `NVIDIA_API_KEY` set in Mimir's environment (for cloud Nemotron generation)
+- `dictionary/` files current — run `cargo run -- dictionary generate` first if registry changed
+
+---
+
+## Staleness Check
+
+The manifest entry embeds both `codebook_checksum` and `generated_at`. If you add actions:
+
+```bash
+cargo run -- dictionary generate          # updates codebook_checksum
+python3 scripts/generate_training_data.py # new timestamp → new manifest entry
+```
+
+The Training Monitor will show both the old and new dataset. You can deprecate the old one
+by setting its `status` to `"superseded"` in the manifest.

llm.txt

@@ -31,6 +31,7 @@ Commands:
 - `scripts/codex-setup.sh`
 - `scripts/codex-run-logger.sh`
 - `scripts/precommit-check.sh`
+- `scripts/generate_training_data.py --dry-run`
 - `git config core.hooksPath .githooks`
 - `.codex/environments/environment.toml`
 
@@ -57,3 +58,5 @@ File operations are sandboxed to project-local `notes/`.
 Codex environment: `.codex/environments/environment.toml` defines setup plus Build, Test, Generate Dictionary, Precommit Check, Run Logger, Query Logger, and Serve Llama actions.
 
 CI and hooks: `.gitea/workflows/ci.yml`, `.woodpecker.yml`, and `.githooks/pre-commit` all use `scripts/precommit-check.sh` to verify formatting, tests, and dictionary generation.
+
+SDG bridge: `scripts/generate_training_data.py` reads `dictionary/actions.index.json`, `dictionary/model.codebook.json`, and `dictionary/model.examples.jsonl`, then writes instruction-format datasets to `../workspace_Data/data/module_controller_intents_<unix_ts>/` and patches `../workspace_Data/manifest.llm.json`. Use `--dry-run` to validate without writes. Optional Mimir expansion requires `--sdg-host`, `--sdg-user`, and `--sdg-key`; failed remote expansion leaves local seeds and writes `SDG_ERROR.txt`.

manifest.llm.json

@@ -27,6 +27,8 @@
     "codex_setup": "scripts/codex-setup.sh",
     "codex_logger_smoke": "scripts/codex-run-logger.sh",
     "precommit_check": "scripts/precommit-check.sh",
+    "training_data_dry_run": "python3 scripts/generate_training_data.py --dry-run",
+    "generate_training_data": "python3 scripts/generate_training_data.py",
     "install_hooks": "git config core.hooksPath .githooks"
   },
   "environment": {
@@ -225,6 +227,17 @@
     "dictionary/model.codebook.json",
     "dictionary/model.examples.jsonl"
   ],
+  "sdg_bridge": {
+    "design_doc": "SDG_BRIDGE_PLAN.md",
+    "script": "scripts/generate_training_data.py",
+    "default_output_dir": "../workspace_Data/data/",
+    "default_manifest": "../workspace_Data/manifest.llm.json",
+    "dataset_prefix": "module_controller_intents",
+    "schema_profile": "instruction",
+    "dry_run": "python3 scripts/generate_training_data.py --dry-run",
+    "optional_sdg_host": "mimir (100.80.52.47)",
+    "sdg_engine": "data-designer"
+  },
   "ci": {
     "codex_environment": ".codex/environments/environment.toml",
     "gitea_actions": ".gitea/workflows/ci.yml",
@@ -239,7 +252,9 @@
       "test -s dictionary/actions.index.json",
       "test -s dictionary/model.codebook.json",
       "test -s dictionary/model.examples.jsonl",
-      "test -s dictionary/static-base.json"
+      "test -s dictionary/static-base.json",
+      "python3 -m unittest tests/test_generate_training_data.py",
+      "python3 scripts/generate_training_data.py --dry-run"
     ]
   },
   "agent_guidance": "See AGENTS.md. Keep src/registry.rs as the source of truth, regenerate dictionary files after action changes, and reject non-exact LLM action outputs."

scripts/__init__.py

@@ -0,0 +1 @@
+"""Project-local helper scripts."""

scripts/generate_training_data.py

@@ -0,0 +1,527 @@
+#!/usr/bin/env python3
+"""Generate module-controller training data from dictionary artifacts."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+import tempfile
+import time
+from pathlib import Path
+from typing import Any
+
+INSTRUCTION = "Map the following natural language request to the correct action and payload."
+DATASET_PREFIX = "module_controller_intents"
+DEFAULT_MIN_RECORDS_PER_ACTION = 20
+
+
+def main() -> int:
+    args = parse_args()
+    try:
+        summary = run_bridge(args)
+    except Exception as exc:  # noqa: BLE001 - CLI should report any bridge failure clearly.
+        print(f"error: {exc}", file=sys.stderr)
+        return 1
+
+    print(json.dumps(summary, indent=2, sort_keys=True))
+    return 0
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--dictionary-dir", default="dictionary/")
+    parser.add_argument("--output-dir", default="../workspace_Data/data/")
+    parser.add_argument("--manifest", default="../workspace_Data/manifest.llm.json")
+    parser.add_argument("--dry-run", action="store_true")
+    parser.add_argument("--sdg-host")
+    parser.add_argument("--sdg-user")
+    parser.add_argument("--sdg-key")
+    parser.add_argument("--num-records", type=int, default=1000)
+    parser.add_argument(
+        "--timestamp",
+        type=int,
+        help="Override generation timestamp. Intended for deterministic tests.",
+    )
+    return parser.parse_args()
+
+
+def run_bridge(args: argparse.Namespace) -> dict[str, Any]:
+    dictionary_dir = Path(args.dictionary_dir)
+    output_root = Path(args.output_dir)
+    manifest_path = Path(args.manifest)
+    generated_at = args.timestamp or int(time.time())
+    dataset_key = f"{DATASET_PREFIX}_{generated_at}"
+
+    artifacts = load_dictionary_artifacts(dictionary_dir)
+    records = build_seed_records(artifacts, generated_at)
+    train_records, validation_records = split_records_by_action(records)
+
+    summary = {
+        "dataset_key": dataset_key,
+        "generated_at": generated_at,
+        "dry_run": bool(args.dry_run),
+        "action_count": len(artifacts["actions"]),
+        "train_records": len(train_records),
+        "validation_records": len(validation_records),
+        "output_dir": str(output_root / dataset_key),
+        "manifest": str(manifest_path),
+        "sdg_requested": has_sdg_args(args),
+    }
+
+    if args.dry_run:
+        return summary
+
+    dataset_dir = output_root / dataset_key
+    dataset_dir.mkdir(parents=True, exist_ok=False)
+    write_jsonl(dataset_dir / "train.jsonl", train_records)
+    write_jsonl(dataset_dir / "validation.jsonl", validation_records)
+    write_dataset_readme(dataset_dir / "README.md", summary, artifacts)
+
+    expanded_path = None
+    sdg_error = None
+    if has_sdg_args(args):
+        try:
+            expanded_path = run_mimir_expansion(
+                args=args,
+                dataset_dir=dataset_dir,
+                dataset_key=dataset_key,
+                generated_at=generated_at,
+                seed_records=train_records + validation_records,
+                action_ids=sorted(artifacts["actions"].keys()),
+            )
+        except Exception as exc:  # noqa: BLE001 - keep local seed output and report remote failure.
+            sdg_error = str(exc)
+            (dataset_dir / "SDG_ERROR.txt").write_text(
+                f"Mimir/Data Designer expansion failed after local seed generation.\n\n{sdg_error}\n"
+            )
+
+    patch_manifest(
+        manifest_path=manifest_path,
+        dataset_key=dataset_key,
+        dataset_dir=dataset_dir,
+        generated_at=generated_at,
+        artifacts=artifacts,
+        expanded_path=expanded_path,
+        sdg_requested=has_sdg_args(args),
+        sdg_error=sdg_error,
+    )
+    summary["expanded_path"] = str(expanded_path) if expanded_path else None
+    summary["sdg_error"] = sdg_error
+    if sdg_error:
+        raise RuntimeError(f"local seed dataset was written, but Mimir expansion failed: {sdg_error}")
+    return summary
+
+
+def load_dictionary_artifacts(dictionary_dir: Path) -> dict[str, Any]:
+    actions_index = read_json(dictionary_dir / "actions.index.json")
+    codebook = read_json(dictionary_dir / "model.codebook.json")
+    examples = read_jsonl(dictionary_dir / "model.examples.jsonl")
+    actions = actions_index.get("actions", {})
+    if not actions:
+        raise ValueError("actions.index.json contains no actions")
+    return {
+        "actions_index": actions_index,
+        "codebook": codebook,
+        "examples": examples,
+        "actions": actions,
+        "action_to_code": invert_unique(codebook["adaptive"]["actions"], "action codes"),
+        "field_to_code": invert_unique(codebook["adaptive"]["fields"], "field codes"),
+    }
+
+
+def build_seed_records(artifacts: dict[str, Any], generated_at: int) -> list[dict[str, Any]]:
+    records: list[dict[str, Any]] = []
+    for action_id in sorted(artifacts["actions"]):
+        action = artifacts["actions"][action_id]
+        action_records: list[dict[str, Any]] = []
+        sample_payload = sample_payload_for_schema(action.get("payload_schema", {}))
+
+        for phrase in action.get("intent_examples", []):
+            action_records.append(
+                build_record(artifacts, action_id, phrase, sample_payload, "seed_intent_example", generated_at)
+            )
+
+        for alias in action.get("aliases", []):
+            phrase = f"{alias} {payload_phrase(action_id, sample_payload)}".strip()
+            action_records.append(
+                build_record(artifacts, action_id, phrase, sample_payload, "alias_variant", generated_at)
+            )
+
+        for phrase, payload in field_variants(action_id, action.get("payload_schema", {})):
+            action_records.append(
+                build_record(artifacts, action_id, phrase, payload, "field_variant", generated_at)
+            )
+
+        for phrase, payload in template_variants(action_id, sample_payload, artifacts["examples"]):
+            action_records.append(
+                build_record(artifacts, action_id, phrase, payload, "template_expansion", generated_at)
+            )
+
+        while len(action_records) < DEFAULT_MIN_RECORDS_PER_ACTION:
+            ordinal = len(action_records) + 1
+            phrase = f"please run {action_id} example {ordinal} with {payload_phrase(action_id, sample_payload)}"
+            action_records.append(
+                build_record(artifacts, action_id, phrase, sample_payload, "template_expansion", generated_at)
+            )
+
+        records.extend(dedupe_records(action_records)[:DEFAULT_MIN_RECORDS_PER_ACTION])
+
+    return records
+
+
+def build_record(
+    artifacts: dict[str, Any],
+    action_id: str,
+    input_text: str,
+    payload: dict[str, Any],
+    source: str,
+    generated_at: int,
+) -> dict[str, Any]:
+    verbose = {"action_id": action_id, "payload": payload}
+    compact = compact_target(artifacts, action_id, payload)
+    return {
+        "instruction": INSTRUCTION,
+        "input": input_text,
+        "output": json.dumps(verbose, sort_keys=True),
+        "metadata": {
+            "action_id": action_id,
+            "compact": json.dumps(compact, separators=(",", ":"), sort_keys=True),
+            "source": source,
+            "split": "",
+            "generated_at": generated_at,
+        },
+    }
+
+
+def compact_target(artifacts: dict[str, Any], action_id: str, payload: dict[str, Any]) -> dict[str, Any]:
+    codebook = artifacts["codebook"]
+    action_to_code = artifacts["action_to_code"]
+    field_to_code = artifacts["field_to_code"]
+    if action_id not in action_to_code:
+        raise ValueError(f"missing compact action code for {action_id}")
+
+    compact_payload = {}
+    for field, value in sorted(payload.items()):
+        if field not in field_to_code:
+            raise ValueError(f"missing compact field code for {field}")
+        compact_payload[field_to_code[field]] = value
+
+    return {
+        "v": codebook["dictionary_version"],
+        "c": codebook["codebook_checksum"],
+        "a": action_to_code[action_id],
+        "p": compact_payload,
+    }
+
+
+def split_records_by_action(records: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    grouped: dict[str, list[dict[str, Any]]] = {}
+    for record in records:
+        grouped.setdefault(record["metadata"]["action_id"], []).append(record)
+
+    train: list[dict[str, Any]] = []
+    validation: list[dict[str, Any]] = []
+    for action_id in sorted(grouped):
+        action_records = grouped[action_id]
+        validation_count = max(1, round(len(action_records) * 0.1)) if len(action_records) > 1 else 0
+        split_at = len(action_records) - validation_count
+        for record in action_records[:split_at]:
+            train.append(with_split(record, "train"))
+        for record in action_records[split_at:]:
+            validation.append(with_split(record, "validation"))
+    return train, validation
+
+
+def with_split(record: dict[str, Any], split: str) -> dict[str, Any]:
+    cloned = json.loads(json.dumps(record))
+    cloned["metadata"]["split"] = split
+    return cloned
+
+
+def sample_payload_for_schema(schema: dict[str, Any]) -> dict[str, Any]:
+    properties = schema.get("properties", {})
+    return {field: sample_value(field, field_schema) for field, field_schema in sorted(properties.items())}
+
+
+def sample_value(field: str, schema: dict[str, Any]) -> Any:
+    if "enum" in schema:
+        return schema["enum"][0]
+    if field == "expression":
+        return "2 + 2 * 3"
+    if field == "value":
+        return 100
+    if field == "from":
+        return "km"
+    if field == "to":
+        return "miles"
+    if field == "min":
+        return 1
+    if field == "max":
+        return 100
+    if field == "max_chars":
+        return 100
+    if field == "filename":
+        return "hello.txt"
+    if field == "content":
+        return "hello from codex"
+    if field in {"text", "message"}:
+        return "hello from codex"
+    if schema.get("type") in {"number", "integer"}:
+        return 1
+    return f"{field} value"
+
+
+def field_variants(action_id: str, schema: dict[str, Any]) -> list[tuple[str, dict[str, Any]]]:
+    properties = schema.get("properties", {})
+    if not properties:
+        return [(f"{action_id} with empty payload", {})]
+
+    variants: list[tuple[str, dict[str, Any]]] = []
+    base = sample_payload_for_schema(schema)
+    for field, field_schema in sorted(properties.items()):
+        if "enum" in field_schema:
+            for enum_value in field_schema["enum"]:
+                payload = dict(base)
+                payload[field] = enum_value
+                variants.append((f"{action_id} where {field} is {enum_value}", payload))
+        elif field_schema.get("type") in {"number", "integer"}:
+            for value in [0, sample_value(field, field_schema), 999]:
+                payload = dict(base)
+                payload[field] = value
+                variants.append((f"{action_id} with {field} {value}", payload))
+        else:
+            payload = dict(base)
+            payload[field] = sample_value(field, field_schema)
+            variants.append((f"{action_id} using {field} {payload[field]}", payload))
+    return variants
+
+
+def template_variants(
+    action_id: str,
+    sample_payload: dict[str, Any],
+    examples: list[dict[str, Any]],
+) -> list[tuple[str, dict[str, Any]]]:
+    template_count = max(1, sum(1 for example in examples if example.get("kind") == "synthetic_template"))
+    variants = []
+    for index in range(1, max(5, template_count) + 1):
+        variants.append((f"template request {index} for {action_id}: {payload_phrase(action_id, sample_payload)}", sample_payload))
+    return variants
+
+
+def payload_phrase(action_id: str, payload: dict[str, Any]) -> str:
+    if not payload:
+        return "with empty payload"
+    parts = [f"{key} {value}" for key, value in sorted(payload.items())]
+    return f"{action_id} with " + ", ".join(parts)
+
+
+def dedupe_records(records: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    seen = set()
+    unique = []
+    for record in records:
+        key = (record["input"], record["output"])
+        if key in seen:
+            continue
+        seen.add(key)
+        unique.append(record)
+    return unique
+
+
+def run_mimir_expansion(
+    args: argparse.Namespace,
+    dataset_dir: Path,
+    dataset_key: str,
+    generated_at: int,
+    seed_records: list[dict[str, Any]],
+    action_ids: list[str],
+) -> Path:
+    require_sdg_args(args)
+    remote_base = "/mnt/storage/data-designer/managed-assets"
+    remote_seed = f"{remote_base}/module_controller_seed_{generated_at}.jsonl"
+    remote_config = f"{remote_base}/module_controller_{generated_at}.yaml"
+    expanded_path = dataset_dir / "expanded.jsonl"
+
+    with tempfile.TemporaryDirectory() as tmp:
+        tmp_path = Path(tmp)
+        local_seed = tmp_path / "seed.jsonl"
+        local_config = tmp_path / "module_controller.yaml"
+        write_jsonl(local_seed, seed_records)
+        local_config.write_text(data_designer_config(dataset_key, generated_at, remote_seed, action_ids))
+
+        target = f"{args.sdg_user}@{args.sdg_host}"
+        scp_base = ["scp", "-i", str(Path(args.sdg_key).expanduser())]
+        ssh_base = ["ssh", "-i", str(Path(args.sdg_key).expanduser()), target]
+        subprocess.run([*scp_base, str(local_seed), f"{target}:{remote_seed}"], check=True)
+        subprocess.run([*scp_base, str(local_config), f"{target}:{remote_config}"], check=True)
+        subprocess.run(
+            [
+                *ssh_base,
+                "/home/aaron-pressey/.venvs/data-designer/bin/data-designer",
+                "create",
+                remote_config,
+                "--num-records",
+                str(args.num_records),
+                "--dataset-name",
+                dataset_key,
+            ],
+            check=True,
+        )
+        subprocess.run([*scp_base, f"{target}:{remote_base}/{dataset_key}.jsonl", str(expanded_path)], check=True)
+    return expanded_path
+
+
+def data_designer_config(dataset_key: str, generated_at: int, remote_seed: str, action_ids: list[str]) -> str:
+    action_ids_json = json.dumps(action_ids)
+    return f"""# auto-generated by generate_training_data.py - do not edit by hand
+# generated_at: {generated_at}
+
+model_config_path: /mnt/storage/data-designer/model_configs.yaml
+model_providers_path: /mnt/storage/data-designer/model_providers.yaml
+
+dataset:
+  name: {dataset_key}
+  schema_profile: instruction
+  seed_file: {remote_seed}
+
+columns:
+  - name: input
+    type: seed_passthrough
+
+  - name: output
+    type: llm_text
+    model_alias: nvidia-text
+    prompt: |
+      You are generating training data for a local action dispatcher.
+      Given the natural language request below, produce valid JSON with
+      exactly two fields: "action_id" (string) and "payload" (object).
+      The action_id must be one of: {action_ids_json}.
+      Vary the phrasing of the input naturally but keep the output schema strict.
+
+      Request: {{{{input}}}}
+    output_schema:
+      type: object
+      required: [action_id, payload]
+      additionalProperties: false
+      properties:
+        action_id:
+          type: string
+          enum: {action_ids_json}
+        payload:
+          type: object
+"""
+
+
+def has_sdg_args(args: argparse.Namespace) -> bool:
+    return bool(args.sdg_host or args.sdg_user or args.sdg_key)
+
+
+def require_sdg_args(args: argparse.Namespace) -> None:
+    missing = [name for name in ["sdg_host", "sdg_user", "sdg_key"] if not getattr(args, name)]
+    if missing:
+        raise ValueError(f"Mimir expansion requires all SDG flags; missing {', '.join('--' + name.replace('_', '-') for name in missing)}")
+
+
+def patch_manifest(
+    manifest_path: Path,
+    dataset_key: str,
+    dataset_dir: Path,
+    generated_at: int,
+    artifacts: dict[str, Any],
+    expanded_path: Path | None,
+    sdg_requested: bool,
+    sdg_error: str | None = None,
+) -> None:
+    manifest = read_json(manifest_path)
+    datasets = manifest.setdefault("datasets", {})
+    if dataset_key in datasets:
+        raise ValueError(f"manifest already contains dataset {dataset_key}")
+
+    codebook = artifacts["codebook"]
+    actions_index = artifacts["actions_index"]
+    datasets[dataset_key] = {
+        "name": "Module Controller Intents",
+        "category": "synthetic_sft",
+        "schema_profile": "instruction",
+        "source_url": None,
+        "acquisition_method": "local_synthetic_generator",
+        "license_spdx": "UNLICENSED",
+        "redistribution_allowed": False,
+        "estimated_tokens": None,
+        "status": "generated_local_remote_failed" if sdg_error else "generated_local",
+        "local_path": relative_data_path(dataset_dir),
+        "local_format": "jsonl",
+        "generator": "mp-ai-module-controller/scripts/generate_training_data.py",
+        "sdg_engine": "data-designer" if sdg_requested else None,
+        "sdg_host": "mimir (100.80.52.47)" if sdg_requested else None,
+        "sdg_model": "nvidia/nemotron-3-nano-30b-a3b" if sdg_requested else None,
+        "expanded_path": str(expanded_path) if expanded_path else None,
+        "sdg_error": sdg_error,
+        "codebook_version": codebook["dictionary_version"],
+        "codebook_checksum": codebook["codebook_checksum"],
+        "registry_checksum": actions_index["registry_checksum"],
+        "generated_at": generated_at,
+        "doc": f"{relative_data_path(dataset_dir)}/README.md",
+    }
+    manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=False) + "\n")
+
+
+def relative_data_path(dataset_dir: Path) -> str:
+    parts = dataset_dir.parts
+    if "data" in parts:
+        index = len(parts) - 1 - list(reversed(parts)).index("data")
+        return "/".join(parts[index:])
+    return str(dataset_dir)
+
+
+def write_dataset_readme(path: Path, summary: dict[str, Any], artifacts: dict[str, Any]) -> None:
+    codebook = artifacts["codebook"]
+    path.write_text(
+        "\n".join(
+            [
+                f"# {summary['dataset_key']}",
+                "",
+                "Instruction-format seed dataset for module-controller action dispatch.",
+                "",
+                f"- Generated at: `{summary['generated_at']}`",
+                f"- Actions: `{summary['action_count']}`",
+                f"- Train records: `{summary['train_records']}`",
+                f"- Validation records: `{summary['validation_records']}`",
+                f"- Codebook version: `{codebook['dictionary_version']}`",
+                f"- Codebook checksum: `{codebook['codebook_checksum']}`",
+                "",
+            ]
+        )
+    )
+
+
+def read_json(path: Path) -> Any:
+    return json.loads(path.read_text())
+
+
+def read_jsonl(path: Path) -> list[dict[str, Any]]:
+    rows = []
+    for line in path.read_text().splitlines():
+        if line.strip():
+            rows.append(json.loads(line))
+    return rows
+
+
+def write_jsonl(path: Path, records: list[dict[str, Any]]) -> None:
+    with path.open("w") as file:
+        for record in records:
+            file.write(json.dumps(record, sort_keys=True) + "\n")
+
+
+def invert_unique(mapping: dict[str, str], label: str) -> dict[str, str]:
+    inverted = {}
+    for code, value in mapping.items():
+        if value in inverted:
+            raise ValueError(f"duplicate {label} value {value}")
+        inverted[value] = code
+    return inverted
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/precommit-check.sh

@@ -30,4 +30,7 @@ for path in ["dictionary/actions.jsonl", "dictionary/model.examples.jsonl"]:
             json.loads(line)
 PY
 
+python3 -m unittest tests/test_generate_training_data.py
+python3 scripts/generate_training_data.py --dry-run
+
 echo "precommit check complete"

tests/test_generate_training_data.py

@@ -0,0 +1,101 @@
+import argparse
+import json
+import shutil
+import tempfile
+import unittest
+from pathlib import Path
+
+from scripts import generate_training_data as bridge
+
+
+class GenerateTrainingDataTests(unittest.TestCase):
+    def setUp(self):
+        self.repo = Path(__file__).resolve().parents[1]
+        self.temp = tempfile.TemporaryDirectory()
+        self.root = Path(self.temp.name)
+        self.dictionary_dir = self.root / "dictionary"
+        shutil.copytree(self.repo / "dictionary", self.dictionary_dir)
+        self.output_dir = self.root / "workspace_Data" / "data"
+        self.output_dir.mkdir(parents=True)
+        self.manifest = self.root / "workspace_Data" / "manifest.llm.json"
+        self.manifest.write_text(json.dumps({"manifest_version": "0.0.0", "datasets": {}}))
+
+    def tearDown(self):
+        self.temp.cleanup()
+
+    def args(self, **overrides):
+        values = {
+            "dictionary_dir": str(self.dictionary_dir),
+            "output_dir": str(self.output_dir),
+            "manifest": str(self.manifest),
+            "dry_run": False,
+            "sdg_host": None,
+            "sdg_user": None,
+            "sdg_key": None,
+            "num_records": 1000,
+            "timestamp": 1234567890,
+        }
+        values.update(overrides)
+        return argparse.Namespace(**values)
+
+    def test_seed_generation_covers_current_dictionary(self):
+        artifacts = bridge.load_dictionary_artifacts(self.dictionary_dir)
+        records = bridge.build_seed_records(artifacts, generated_at=123)
+        actions = {record["metadata"]["action_id"] for record in records}
+        self.assertEqual(len(actions), 13)
+        self.assertIn("calculate", actions)
+        self.assertIn("delete_file", actions)
+        self.assertEqual(len(records), 260)
+
+    def test_compact_output_uses_codebook_codes(self):
+        artifacts = bridge.load_dictionary_artifacts(self.dictionary_dir)
+        compact = bridge.compact_target(
+            artifacts,
+            "hash_string",
+            {"text": "hello", "algorithm": "sha256"},
+        )
+        self.assertEqual(compact["a"], artifacts["action_to_code"]["hash_string"])
+        self.assertIn(artifacts["field_to_code"]["text"], compact["p"])
+        self.assertIn(artifacts["field_to_code"]["algorithm"], compact["p"])
+
+    def test_empty_payload_action(self):
+        artifacts = bridge.load_dictionary_artifacts(self.dictionary_dir)
+        records = bridge.build_seed_records(artifacts, generated_at=123)
+        uuid_records = [
+            record for record in records if record["metadata"]["action_id"] == "generate_uuid"
+        ]
+        self.assertTrue(uuid_records)
+        output = json.loads(uuid_records[0]["output"])
+        compact = json.loads(uuid_records[0]["metadata"]["compact"])
+        self.assertEqual(output["payload"], {})
+        self.assertEqual(compact["p"], {})
+
+    def test_split_includes_every_action(self):
+        artifacts = bridge.load_dictionary_artifacts(self.dictionary_dir)
+        records = bridge.build_seed_records(artifacts, generated_at=123)
+        train, validation = bridge.split_records_by_action(records)
+        train_actions = {record["metadata"]["action_id"] for record in train}
+        validation_actions = {record["metadata"]["action_id"] for record in validation}
+        self.assertEqual(train_actions, validation_actions)
+        self.assertEqual(len(train_actions), 13)
+
+    def test_dry_run_writes_nothing(self):
+        summary = bridge.run_bridge(self.args(dry_run=True))
+        self.assertTrue(summary["dry_run"])
+        self.assertFalse((self.output_dir / "module_controller_intents_1234567890").exists())
+        manifest = json.loads(self.manifest.read_text())
+        self.assertEqual(manifest["datasets"], {})
+
+    def test_manifest_patch_adds_dataset_without_damaging_existing_keys(self):
+        bridge.run_bridge(self.args())
+        manifest = json.loads(self.manifest.read_text())
+        self.assertEqual(manifest["manifest_version"], "0.0.0")
+        dataset = manifest["datasets"]["module_controller_intents_1234567890"]
+        self.assertEqual(dataset["schema_profile"], "instruction")
+        self.assertEqual(dataset["codebook_version"], "1.0.0")
+        self.assertIn("codebook_checksum", dataset)
+        self.assertTrue((self.output_dir / "module_controller_intents_1234567890" / "train.jsonl").exists())
+
+
+if __name__ == "__main__":
+    unittest.main()