diff --git a/.codex/environments/environment.toml b/.codex/environments/environment.toml index 6dbce5b..5c34e35 100644 --- a/.codex/environments/environment.toml +++ b/.codex/environments/environment.toml @@ -25,6 +25,11 @@ name = "Precommit Check" icon = "tool" command = "scripts/precommit-check.sh" +[[actions]] +name = "Training Data Dry Run" +icon = "tool" +command = "python3 scripts/generate_training_data.py --dry-run" + [[actions]] name = "Run Logger" icon = "run" diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index db43ceb..209a4f0 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -27,6 +27,10 @@ jobs: rustc --version cargo --version + - name: Show Python version + shell: bash + run: python3 --version + - name: Run precommit checks shell: bash run: scripts/precommit-check.sh diff --git a/.gitignore b/.gitignore index eaebfb1..274f6b1 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,5 @@ /dictionary/model.examples.jsonl *.log .DS_Store +__pycache__/ +*.py[cod] diff --git a/.woodpecker.yml b/.woodpecker.yml index a5f6909..26dbea7 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -6,6 +6,7 @@ steps: validate: image: rust:1.95 commands: + - apt-get update && apt-get install -y python3 - rustc --version - cargo --version - scripts/precommit-check.sh diff --git a/AGENTS.md b/AGENTS.md index 5b5fb02..c33e957 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,6 +19,8 @@ - Codex logger smoke test: `scripts/codex-run-logger.sh` - Precommit/CI check: `scripts/precommit-check.sh` - Codex environment: `.codex/environments/environment.toml` +- Training data dry run: `python3 scripts/generate_training_data.py --dry-run` +- Local training data bridge: `python3 scripts/generate_training_data.py` ## Environment @@ -46,6 +48,17 @@ Use `.env` for local values and keep it out of git. Update `.env.example`, `READ - Do not manually edit generated dictionary files. - Generated metadata includes `dictionary_version`, `registry_checksum`, `static_base_checksum`, and `codebook_checksum`. +## SDG Bridge Rules + +- `SDG_BRIDGE_PLAN.md` documents the local-to-Data-Designer bridge design. +- `scripts/generate_training_data.py` reads generated dictionary artifacts and produces instruction-format seed datasets. +- Default output is `../workspace_Data/data/module_controller_intents_/`. +- Default manifest patch target is `../workspace_Data/manifest.llm.json`. +- Use `--dry-run` for validation; it must not write output data or patch manifests. +- Mimir/Data Designer expansion only runs when `--sdg-host`, `--sdg-user`, and `--sdg-key` are all provided. +- Do not start or restart Mimir llama.cpp/Keiro from this bridge. +- If Mimir expansion fails, the bridge keeps the local seed dataset, writes `SDG_ERROR.txt`, patches the manifest with `generated_local_remote_failed`, and exits non-zero. + ## Dispatch Rules - Dispatch only exact `action_id` values present in `dictionary/actions.index.json`. @@ -81,4 +94,5 @@ Use `.env` for local values and keep it out of git. Update `.env.example`, `READ - Keep the CLI Rust-first and small. - Prefer adding scripts under `src/scripts/` and registering them in `src/registry.rs`. - Add focused tests for parsing, dictionary generation, action lookup, and payload validation. +- Add stdlib Python tests for bridge behavior in `tests/` when changing `scripts/generate_training_data.py`. - Run `cargo test` before handing off code changes. diff --git a/README.md b/README.md index 0114109..5437f0b 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ cargo run -- run calculate --payload '{"expression":"2 + 2 * 3"}' cargo run -- run to_slug --payload '{"text":"Hello, World!"}' cargo run -- run write_note --payload '{"filename":"hello.txt","content":"hello"}' scripts/precommit-check.sh +python3 scripts/generate_training_data.py --dry-run ``` `serve` starts: @@ -112,3 +113,26 @@ Enable the hook after cloning: ```sh git config core.hooksPath .githooks ``` + +## SDG Bridge + +`scripts/generate_training_data.py` generates instruction-format seed data from current dictionary artifacts. + +Dry run: + +```sh +python3 scripts/generate_training_data.py --dry-run +``` + +Local dataset generation: + +```sh +python3 scripts/generate_training_data.py \ + --dictionary-dir dictionary/ \ + --output-dir ../workspace_Data/data/ \ + --manifest ../workspace_Data/manifest.llm.json +``` + +Optional Mimir/Data Designer expansion is enabled only when `--sdg-host`, `--sdg-user`, and `--sdg-key` are supplied. The bridge does not start or restart Mimir inference services. + +If remote expansion fails, the timestamped local seed dataset remains in place, `SDG_ERROR.txt` is written in that dataset directory, and the manifest entry is marked `generated_local_remote_failed`. diff --git a/SDG_BRIDGE_PLAN.md b/SDG_BRIDGE_PLAN.md new file mode 100644 index 0000000..a30b086 --- /dev/null +++ b/SDG_BRIDGE_PLAN.md @@ -0,0 +1,232 @@ +# SDG Bridge Plan + +## Bridge Script Filename + +``` +scripts/generate_training_data.py +``` + +This script reads the generated dictionary artifacts from this project, optionally runs a +NeMo Data Designer expansion pass on Mimir via SSH, and writes a timestamped raw training +dataset into the `workspace_Data` pipeline, then registers it in `manifest.llm.json` so the +Training Monitor picks it up automatically. + +--- + +## Mimir — Data Designer Status + +Mimir (`100.80.52.47`) has **Data Designer 0.8.0** installed and ready: + +| Component | Location | Status | +|-----------|----------|--------| +| CLI | `/home/aaron-pressey/.venvs/data-designer/bin/data-designer` | ✅ installed | +| Backend adapter | `/home/aaron-pressey/.local/share/home-grown-llm-data/nemo_data_designer_backend.py` | ✅ present | +| Storage | `/mnt/storage/data-designer/` | ✅ mounted | +| Model configs | `/mnt/storage/data-designer/model_configs.yaml` | ✅ configured | +| Config env | `/home/aaron-pressey/.config/home-grown-llm-data/data-designer.env` | ✅ present | + +**Current model provider:** NVIDIA API (`nvidia-text` alias → `nvidia/nemotron-3-nano-30b-a3b`). +Local llama.cpp on port 8081 is configured but not currently running — cloud provider is used +for generation unless a local inference server is started separately. + +--- + +## What the Bridge Does + +``` +Step 1 (local — this project): + dictionary/actions.index.json ─┐ + dictionary/model.codebook.json ├─► generate_training_data.py ─► seed JSONL + dictionary/model.examples.jsonl ─┘ (20 records/action, 260 total) + │ +Step 2 (remote — Mimir, optional): │ + SCP seed JSONL + DD config to Mimir ◄────────┘ + SSH: data-designer create module_controller.yaml + --num-records + --dataset-name module_controller_intents_ + SCP expanded dataset back + │ +Step 3 (local — workspace_Data): │ + Write to data/module_controller_intents_/ ◄──────────────┘ + Patch manifest.llm.json with timestamped entry + Training Monitor picks it up as a new Data Source +``` + +--- + +## Timestamp Tagging + +Every dataset generated by this script is tagged with a **Unix timestamp at generation time**. +The timestamp is embedded in: + +1. **Output directory name:** `data/module_controller_intents_/` +2. **Manifest entry key:** `"module_controller_intents_"` +3. **Manifest entry field:** `"generated_at": ` +4. **Every JSONL record's metadata:** `"generated_at": ` + +This means you can run the bridge multiple times (after adding actions, changing examples, +or expanding via NeMo) and the Training Monitor will show each run as a distinct, traceable +Data Source. No clobbering, no ambiguity. + +--- + +## Output Record Schema + +Each output record uses the `instruction` schema profile: + +```json +{ + "instruction": "Map the following natural language request to the correct action and payload.", + "input": "hash hello with sha256", + "output": "{\"action_id\": \"hash_string\", \"payload\": {\"text\": \"hello\", \"algorithm\": \"sha256\"}}", + "metadata": { + "action_id": "hash_string", + "compact": "{\"v\":\"1.0.0\",\"c\":\"\",\"a\":\"A007\",\"p\":{\"F010\":\"hello\",\"F001\":\"sha256\"}}", + "source": "seed_intent_example", + "split": "train", + "generated_at": 1784660000 + } +} +``` + +--- + +## Generation Strategy + +For each action in `actions.index.json`: + +1. **Seed examples** — emit each `intent_example` directly (source: `seed_intent_example`) +2. **Alias variants** — one phrase per alias using the alias as the verb (source: `alias_variant`) +3. **Payload field variants** — for enum fields, one record per enum value; for numeric fields, + low/mid/high values (source: `field_variant`) +4. **Template expansion** — 5 records per action from `synthetic_template` with filled placeholders + (source: `template_expansion`) + +**Minimum seed target:** ~20 records × 13 actions = 260 records. +**After NeMo expansion:** configurable via `--num-records` (default: 1000). + +Split: 90% train / 10% validation, partitioned by action so all 13 actions appear in both splits. + +--- + +## Mimir Data Designer Config (auto-generated by bridge) + +The bridge writes this config to Mimir before running generation: + +```yaml +# auto-generated by generate_training_data.py — do not edit by hand +# generated_at: + +model_config_path: /mnt/storage/data-designer/model_configs.yaml +model_providers_path: /mnt/storage/data-designer/model_providers.yaml + +dataset: + name: module_controller_intents_ + schema_profile: instruction + seed_file: /mnt/storage/data-designer/managed-assets/module_controller_seed_.jsonl + +columns: + - name: input + type: seed_passthrough + + - name: output + type: llm_text + model_alias: nvidia-text + prompt: | + You are generating training data for a local action dispatcher. + Given the natural language request below, produce valid JSON with + exactly two fields: "action_id" (string) and "payload" (object). + The action_id must be one of: {action_ids}. + Vary the phrasing of the input naturally but keep the output schema strict. + + Request: {{input}} + output_schema: + type: object + required: [action_id, payload] + additionalProperties: false + properties: + action_id: + type: string + enum: {action_ids} + payload: + type: object +``` + +--- + +## Manifest Entry (auto-patched by bridge) + +```json +"module_controller_intents_": { + "name": "Module Controller Intents", + "category": "synthetic_sft", + "schema_profile": "instruction", + "source_url": null, + "acquisition_method": "local_synthetic_generator", + "license_spdx": "UNLICENSED", + "redistribution_allowed": false, + "estimated_tokens": null, + "status": "generated_local", + "local_path": "data/module_controller_intents_", + "local_format": "jsonl", + "generator": "mp-ai-module-controller/scripts/generate_training_data.py", + "sdg_engine": "data-designer", + "sdg_host": "mimir (100.80.52.47)", + "sdg_model": "nvidia/nemotron-3-nano-30b-a3b", + "codebook_version": "1.0.0", + "codebook_checksum": "", + "registry_checksum": "", + "generated_at": , + "doc": "data/module_controller_intents_/README.md" +} +``` + +--- + +## CLI Usage + +```bash +# From mp-ai-module-controller project root: + +# Seed only (no NeMo expansion — fast, local, 260 records): +python3 scripts/generate_training_data.py \ + --dictionary-dir dictionary/ \ + --output-dir ../workspace_Data/data/ \ + --manifest ../workspace_Data/manifest.llm.json + +# Full SDG run via Mimir (requires NVIDIA_API_KEY on Mimir): +python3 scripts/generate_training_data.py \ + --dictionary-dir dictionary/ \ + --output-dir ../workspace_Data/data/ \ + --manifest ../workspace_Data/manifest.llm.json \ + --sdg-host 100.80.52.47 \ + --sdg-user aaron-pressey \ + --sdg-key ~/.ssh/silma_orson_ed25519 \ + --num-records 1000 + +# Dry run (prints what would be generated, writes nothing): +python3 scripts/generate_training_data.py --dry-run +``` + +--- + +## Prerequisites + +- Python 3.11+ (stdlib only: `json`, `pathlib`, `hashlib`, `time`, `argparse`, `subprocess`) +- SSH access to Mimir via `~/.ssh/silma_orson_ed25519` (for NeMo expansion pass) +- `NVIDIA_API_KEY` set in Mimir's environment (for cloud Nemotron generation) +- `dictionary/` files current — run `cargo run -- dictionary generate` first if registry changed + +--- + +## Staleness Check + +The manifest entry embeds both `codebook_checksum` and `generated_at`. If you add actions: + +```bash +cargo run -- dictionary generate # updates codebook_checksum +python3 scripts/generate_training_data.py # new timestamp → new manifest entry +``` + +The Training Monitor will show both the old and new dataset. You can deprecate the old one +by setting its `status` to `"superseded"` in the manifest. diff --git a/llm.txt b/llm.txt index aaa9d17..348c1b2 100644 --- a/llm.txt +++ b/llm.txt @@ -31,6 +31,7 @@ Commands: - `scripts/codex-setup.sh` - `scripts/codex-run-logger.sh` - `scripts/precommit-check.sh` +- `scripts/generate_training_data.py --dry-run` - `git config core.hooksPath .githooks` - `.codex/environments/environment.toml` @@ -57,3 +58,5 @@ File operations are sandboxed to project-local `notes/`. Codex environment: `.codex/environments/environment.toml` defines setup plus Build, Test, Generate Dictionary, Precommit Check, Run Logger, Query Logger, and Serve Llama actions. CI and hooks: `.gitea/workflows/ci.yml`, `.woodpecker.yml`, and `.githooks/pre-commit` all use `scripts/precommit-check.sh` to verify formatting, tests, and dictionary generation. + +SDG bridge: `scripts/generate_training_data.py` reads `dictionary/actions.index.json`, `dictionary/model.codebook.json`, and `dictionary/model.examples.jsonl`, then writes instruction-format datasets to `../workspace_Data/data/module_controller_intents_/` and patches `../workspace_Data/manifest.llm.json`. Use `--dry-run` to validate without writes. Optional Mimir expansion requires `--sdg-host`, `--sdg-user`, and `--sdg-key`; failed remote expansion leaves local seeds and writes `SDG_ERROR.txt`. diff --git a/manifest.llm.json b/manifest.llm.json index 89f49b5..a13e738 100644 --- a/manifest.llm.json +++ b/manifest.llm.json @@ -27,6 +27,8 @@ "codex_setup": "scripts/codex-setup.sh", "codex_logger_smoke": "scripts/codex-run-logger.sh", "precommit_check": "scripts/precommit-check.sh", + "training_data_dry_run": "python3 scripts/generate_training_data.py --dry-run", + "generate_training_data": "python3 scripts/generate_training_data.py", "install_hooks": "git config core.hooksPath .githooks" }, "environment": { @@ -225,6 +227,17 @@ "dictionary/model.codebook.json", "dictionary/model.examples.jsonl" ], + "sdg_bridge": { + "design_doc": "SDG_BRIDGE_PLAN.md", + "script": "scripts/generate_training_data.py", + "default_output_dir": "../workspace_Data/data/", + "default_manifest": "../workspace_Data/manifest.llm.json", + "dataset_prefix": "module_controller_intents", + "schema_profile": "instruction", + "dry_run": "python3 scripts/generate_training_data.py --dry-run", + "optional_sdg_host": "mimir (100.80.52.47)", + "sdg_engine": "data-designer" + }, "ci": { "codex_environment": ".codex/environments/environment.toml", "gitea_actions": ".gitea/workflows/ci.yml", @@ -239,7 +252,9 @@ "test -s dictionary/actions.index.json", "test -s dictionary/model.codebook.json", "test -s dictionary/model.examples.jsonl", - "test -s dictionary/static-base.json" + "test -s dictionary/static-base.json", + "python3 -m unittest tests/test_generate_training_data.py", + "python3 scripts/generate_training_data.py --dry-run" ] }, "agent_guidance": "See AGENTS.md. Keep src/registry.rs as the source of truth, regenerate dictionary files after action changes, and reject non-exact LLM action outputs." diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..f89f5d6 --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1 @@ +"""Project-local helper scripts.""" diff --git a/scripts/generate_training_data.py b/scripts/generate_training_data.py new file mode 100755 index 0000000..ebb3632 --- /dev/null +++ b/scripts/generate_training_data.py @@ -0,0 +1,527 @@ +#!/usr/bin/env python3 +"""Generate module-controller training data from dictionary artifacts.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from typing import Any + +INSTRUCTION = "Map the following natural language request to the correct action and payload." +DATASET_PREFIX = "module_controller_intents" +DEFAULT_MIN_RECORDS_PER_ACTION = 20 + + +def main() -> int: + args = parse_args() + try: + summary = run_bridge(args) + except Exception as exc: # noqa: BLE001 - CLI should report any bridge failure clearly. + print(f"error: {exc}", file=sys.stderr) + return 1 + + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dictionary-dir", default="dictionary/") + parser.add_argument("--output-dir", default="../workspace_Data/data/") + parser.add_argument("--manifest", default="../workspace_Data/manifest.llm.json") + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--sdg-host") + parser.add_argument("--sdg-user") + parser.add_argument("--sdg-key") + parser.add_argument("--num-records", type=int, default=1000) + parser.add_argument( + "--timestamp", + type=int, + help="Override generation timestamp. Intended for deterministic tests.", + ) + return parser.parse_args() + + +def run_bridge(args: argparse.Namespace) -> dict[str, Any]: + dictionary_dir = Path(args.dictionary_dir) + output_root = Path(args.output_dir) + manifest_path = Path(args.manifest) + generated_at = args.timestamp or int(time.time()) + dataset_key = f"{DATASET_PREFIX}_{generated_at}" + + artifacts = load_dictionary_artifacts(dictionary_dir) + records = build_seed_records(artifacts, generated_at) + train_records, validation_records = split_records_by_action(records) + + summary = { + "dataset_key": dataset_key, + "generated_at": generated_at, + "dry_run": bool(args.dry_run), + "action_count": len(artifacts["actions"]), + "train_records": len(train_records), + "validation_records": len(validation_records), + "output_dir": str(output_root / dataset_key), + "manifest": str(manifest_path), + "sdg_requested": has_sdg_args(args), + } + + if args.dry_run: + return summary + + dataset_dir = output_root / dataset_key + dataset_dir.mkdir(parents=True, exist_ok=False) + write_jsonl(dataset_dir / "train.jsonl", train_records) + write_jsonl(dataset_dir / "validation.jsonl", validation_records) + write_dataset_readme(dataset_dir / "README.md", summary, artifacts) + + expanded_path = None + sdg_error = None + if has_sdg_args(args): + try: + expanded_path = run_mimir_expansion( + args=args, + dataset_dir=dataset_dir, + dataset_key=dataset_key, + generated_at=generated_at, + seed_records=train_records + validation_records, + action_ids=sorted(artifacts["actions"].keys()), + ) + except Exception as exc: # noqa: BLE001 - keep local seed output and report remote failure. + sdg_error = str(exc) + (dataset_dir / "SDG_ERROR.txt").write_text( + f"Mimir/Data Designer expansion failed after local seed generation.\n\n{sdg_error}\n" + ) + + patch_manifest( + manifest_path=manifest_path, + dataset_key=dataset_key, + dataset_dir=dataset_dir, + generated_at=generated_at, + artifacts=artifacts, + expanded_path=expanded_path, + sdg_requested=has_sdg_args(args), + sdg_error=sdg_error, + ) + summary["expanded_path"] = str(expanded_path) if expanded_path else None + summary["sdg_error"] = sdg_error + if sdg_error: + raise RuntimeError(f"local seed dataset was written, but Mimir expansion failed: {sdg_error}") + return summary + + +def load_dictionary_artifacts(dictionary_dir: Path) -> dict[str, Any]: + actions_index = read_json(dictionary_dir / "actions.index.json") + codebook = read_json(dictionary_dir / "model.codebook.json") + examples = read_jsonl(dictionary_dir / "model.examples.jsonl") + actions = actions_index.get("actions", {}) + if not actions: + raise ValueError("actions.index.json contains no actions") + return { + "actions_index": actions_index, + "codebook": codebook, + "examples": examples, + "actions": actions, + "action_to_code": invert_unique(codebook["adaptive"]["actions"], "action codes"), + "field_to_code": invert_unique(codebook["adaptive"]["fields"], "field codes"), + } + + +def build_seed_records(artifacts: dict[str, Any], generated_at: int) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + for action_id in sorted(artifacts["actions"]): + action = artifacts["actions"][action_id] + action_records: list[dict[str, Any]] = [] + sample_payload = sample_payload_for_schema(action.get("payload_schema", {})) + + for phrase in action.get("intent_examples", []): + action_records.append( + build_record(artifacts, action_id, phrase, sample_payload, "seed_intent_example", generated_at) + ) + + for alias in action.get("aliases", []): + phrase = f"{alias} {payload_phrase(action_id, sample_payload)}".strip() + action_records.append( + build_record(artifacts, action_id, phrase, sample_payload, "alias_variant", generated_at) + ) + + for phrase, payload in field_variants(action_id, action.get("payload_schema", {})): + action_records.append( + build_record(artifacts, action_id, phrase, payload, "field_variant", generated_at) + ) + + for phrase, payload in template_variants(action_id, sample_payload, artifacts["examples"]): + action_records.append( + build_record(artifacts, action_id, phrase, payload, "template_expansion", generated_at) + ) + + while len(action_records) < DEFAULT_MIN_RECORDS_PER_ACTION: + ordinal = len(action_records) + 1 + phrase = f"please run {action_id} example {ordinal} with {payload_phrase(action_id, sample_payload)}" + action_records.append( + build_record(artifacts, action_id, phrase, sample_payload, "template_expansion", generated_at) + ) + + records.extend(dedupe_records(action_records)[:DEFAULT_MIN_RECORDS_PER_ACTION]) + + return records + + +def build_record( + artifacts: dict[str, Any], + action_id: str, + input_text: str, + payload: dict[str, Any], + source: str, + generated_at: int, +) -> dict[str, Any]: + verbose = {"action_id": action_id, "payload": payload} + compact = compact_target(artifacts, action_id, payload) + return { + "instruction": INSTRUCTION, + "input": input_text, + "output": json.dumps(verbose, sort_keys=True), + "metadata": { + "action_id": action_id, + "compact": json.dumps(compact, separators=(",", ":"), sort_keys=True), + "source": source, + "split": "", + "generated_at": generated_at, + }, + } + + +def compact_target(artifacts: dict[str, Any], action_id: str, payload: dict[str, Any]) -> dict[str, Any]: + codebook = artifacts["codebook"] + action_to_code = artifacts["action_to_code"] + field_to_code = artifacts["field_to_code"] + if action_id not in action_to_code: + raise ValueError(f"missing compact action code for {action_id}") + + compact_payload = {} + for field, value in sorted(payload.items()): + if field not in field_to_code: + raise ValueError(f"missing compact field code for {field}") + compact_payload[field_to_code[field]] = value + + return { + "v": codebook["dictionary_version"], + "c": codebook["codebook_checksum"], + "a": action_to_code[action_id], + "p": compact_payload, + } + + +def split_records_by_action(records: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + grouped: dict[str, list[dict[str, Any]]] = {} + for record in records: + grouped.setdefault(record["metadata"]["action_id"], []).append(record) + + train: list[dict[str, Any]] = [] + validation: list[dict[str, Any]] = [] + for action_id in sorted(grouped): + action_records = grouped[action_id] + validation_count = max(1, round(len(action_records) * 0.1)) if len(action_records) > 1 else 0 + split_at = len(action_records) - validation_count + for record in action_records[:split_at]: + train.append(with_split(record, "train")) + for record in action_records[split_at:]: + validation.append(with_split(record, "validation")) + return train, validation + + +def with_split(record: dict[str, Any], split: str) -> dict[str, Any]: + cloned = json.loads(json.dumps(record)) + cloned["metadata"]["split"] = split + return cloned + + +def sample_payload_for_schema(schema: dict[str, Any]) -> dict[str, Any]: + properties = schema.get("properties", {}) + return {field: sample_value(field, field_schema) for field, field_schema in sorted(properties.items())} + + +def sample_value(field: str, schema: dict[str, Any]) -> Any: + if "enum" in schema: + return schema["enum"][0] + if field == "expression": + return "2 + 2 * 3" + if field == "value": + return 100 + if field == "from": + return "km" + if field == "to": + return "miles" + if field == "min": + return 1 + if field == "max": + return 100 + if field == "max_chars": + return 100 + if field == "filename": + return "hello.txt" + if field == "content": + return "hello from codex" + if field in {"text", "message"}: + return "hello from codex" + if schema.get("type") in {"number", "integer"}: + return 1 + return f"{field} value" + + +def field_variants(action_id: str, schema: dict[str, Any]) -> list[tuple[str, dict[str, Any]]]: + properties = schema.get("properties", {}) + if not properties: + return [(f"{action_id} with empty payload", {})] + + variants: list[tuple[str, dict[str, Any]]] = [] + base = sample_payload_for_schema(schema) + for field, field_schema in sorted(properties.items()): + if "enum" in field_schema: + for enum_value in field_schema["enum"]: + payload = dict(base) + payload[field] = enum_value + variants.append((f"{action_id} where {field} is {enum_value}", payload)) + elif field_schema.get("type") in {"number", "integer"}: + for value in [0, sample_value(field, field_schema), 999]: + payload = dict(base) + payload[field] = value + variants.append((f"{action_id} with {field} {value}", payload)) + else: + payload = dict(base) + payload[field] = sample_value(field, field_schema) + variants.append((f"{action_id} using {field} {payload[field]}", payload)) + return variants + + +def template_variants( + action_id: str, + sample_payload: dict[str, Any], + examples: list[dict[str, Any]], +) -> list[tuple[str, dict[str, Any]]]: + template_count = max(1, sum(1 for example in examples if example.get("kind") == "synthetic_template")) + variants = [] + for index in range(1, max(5, template_count) + 1): + variants.append((f"template request {index} for {action_id}: {payload_phrase(action_id, sample_payload)}", sample_payload)) + return variants + + +def payload_phrase(action_id: str, payload: dict[str, Any]) -> str: + if not payload: + return "with empty payload" + parts = [f"{key} {value}" for key, value in sorted(payload.items())] + return f"{action_id} with " + ", ".join(parts) + + +def dedupe_records(records: list[dict[str, Any]]) -> list[dict[str, Any]]: + seen = set() + unique = [] + for record in records: + key = (record["input"], record["output"]) + if key in seen: + continue + seen.add(key) + unique.append(record) + return unique + + +def run_mimir_expansion( + args: argparse.Namespace, + dataset_dir: Path, + dataset_key: str, + generated_at: int, + seed_records: list[dict[str, Any]], + action_ids: list[str], +) -> Path: + require_sdg_args(args) + remote_base = "/mnt/storage/data-designer/managed-assets" + remote_seed = f"{remote_base}/module_controller_seed_{generated_at}.jsonl" + remote_config = f"{remote_base}/module_controller_{generated_at}.yaml" + expanded_path = dataset_dir / "expanded.jsonl" + + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + local_seed = tmp_path / "seed.jsonl" + local_config = tmp_path / "module_controller.yaml" + write_jsonl(local_seed, seed_records) + local_config.write_text(data_designer_config(dataset_key, generated_at, remote_seed, action_ids)) + + target = f"{args.sdg_user}@{args.sdg_host}" + scp_base = ["scp", "-i", str(Path(args.sdg_key).expanduser())] + ssh_base = ["ssh", "-i", str(Path(args.sdg_key).expanduser()), target] + subprocess.run([*scp_base, str(local_seed), f"{target}:{remote_seed}"], check=True) + subprocess.run([*scp_base, str(local_config), f"{target}:{remote_config}"], check=True) + subprocess.run( + [ + *ssh_base, + "/home/aaron-pressey/.venvs/data-designer/bin/data-designer", + "create", + remote_config, + "--num-records", + str(args.num_records), + "--dataset-name", + dataset_key, + ], + check=True, + ) + subprocess.run([*scp_base, f"{target}:{remote_base}/{dataset_key}.jsonl", str(expanded_path)], check=True) + return expanded_path + + +def data_designer_config(dataset_key: str, generated_at: int, remote_seed: str, action_ids: list[str]) -> str: + action_ids_json = json.dumps(action_ids) + return f"""# auto-generated by generate_training_data.py - do not edit by hand +# generated_at: {generated_at} + +model_config_path: /mnt/storage/data-designer/model_configs.yaml +model_providers_path: /mnt/storage/data-designer/model_providers.yaml + +dataset: + name: {dataset_key} + schema_profile: instruction + seed_file: {remote_seed} + +columns: + - name: input + type: seed_passthrough + + - name: output + type: llm_text + model_alias: nvidia-text + prompt: | + You are generating training data for a local action dispatcher. + Given the natural language request below, produce valid JSON with + exactly two fields: "action_id" (string) and "payload" (object). + The action_id must be one of: {action_ids_json}. + Vary the phrasing of the input naturally but keep the output schema strict. + + Request: {{{{input}}}} + output_schema: + type: object + required: [action_id, payload] + additionalProperties: false + properties: + action_id: + type: string + enum: {action_ids_json} + payload: + type: object +""" + + +def has_sdg_args(args: argparse.Namespace) -> bool: + return bool(args.sdg_host or args.sdg_user or args.sdg_key) + + +def require_sdg_args(args: argparse.Namespace) -> None: + missing = [name for name in ["sdg_host", "sdg_user", "sdg_key"] if not getattr(args, name)] + if missing: + raise ValueError(f"Mimir expansion requires all SDG flags; missing {', '.join('--' + name.replace('_', '-') for name in missing)}") + + +def patch_manifest( + manifest_path: Path, + dataset_key: str, + dataset_dir: Path, + generated_at: int, + artifacts: dict[str, Any], + expanded_path: Path | None, + sdg_requested: bool, + sdg_error: str | None = None, +) -> None: + manifest = read_json(manifest_path) + datasets = manifest.setdefault("datasets", {}) + if dataset_key in datasets: + raise ValueError(f"manifest already contains dataset {dataset_key}") + + codebook = artifacts["codebook"] + actions_index = artifacts["actions_index"] + datasets[dataset_key] = { + "name": "Module Controller Intents", + "category": "synthetic_sft", + "schema_profile": "instruction", + "source_url": None, + "acquisition_method": "local_synthetic_generator", + "license_spdx": "UNLICENSED", + "redistribution_allowed": False, + "estimated_tokens": None, + "status": "generated_local_remote_failed" if sdg_error else "generated_local", + "local_path": relative_data_path(dataset_dir), + "local_format": "jsonl", + "generator": "mp-ai-module-controller/scripts/generate_training_data.py", + "sdg_engine": "data-designer" if sdg_requested else None, + "sdg_host": "mimir (100.80.52.47)" if sdg_requested else None, + "sdg_model": "nvidia/nemotron-3-nano-30b-a3b" if sdg_requested else None, + "expanded_path": str(expanded_path) if expanded_path else None, + "sdg_error": sdg_error, + "codebook_version": codebook["dictionary_version"], + "codebook_checksum": codebook["codebook_checksum"], + "registry_checksum": actions_index["registry_checksum"], + "generated_at": generated_at, + "doc": f"{relative_data_path(dataset_dir)}/README.md", + } + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=False) + "\n") + + +def relative_data_path(dataset_dir: Path) -> str: + parts = dataset_dir.parts + if "data" in parts: + index = len(parts) - 1 - list(reversed(parts)).index("data") + return "/".join(parts[index:]) + return str(dataset_dir) + + +def write_dataset_readme(path: Path, summary: dict[str, Any], artifacts: dict[str, Any]) -> None: + codebook = artifacts["codebook"] + path.write_text( + "\n".join( + [ + f"# {summary['dataset_key']}", + "", + "Instruction-format seed dataset for module-controller action dispatch.", + "", + f"- Generated at: `{summary['generated_at']}`", + f"- Actions: `{summary['action_count']}`", + f"- Train records: `{summary['train_records']}`", + f"- Validation records: `{summary['validation_records']}`", + f"- Codebook version: `{codebook['dictionary_version']}`", + f"- Codebook checksum: `{codebook['codebook_checksum']}`", + "", + ] + ) + ) + + +def read_json(path: Path) -> Any: + return json.loads(path.read_text()) + + +def read_jsonl(path: Path) -> list[dict[str, Any]]: + rows = [] + for line in path.read_text().splitlines(): + if line.strip(): + rows.append(json.loads(line)) + return rows + + +def write_jsonl(path: Path, records: list[dict[str, Any]]) -> None: + with path.open("w") as file: + for record in records: + file.write(json.dumps(record, sort_keys=True) + "\n") + + +def invert_unique(mapping: dict[str, str], label: str) -> dict[str, str]: + inverted = {} + for code, value in mapping.items(): + if value in inverted: + raise ValueError(f"duplicate {label} value {value}") + inverted[value] = code + return inverted + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/precommit-check.sh b/scripts/precommit-check.sh index 1df8382..ff6eeb8 100755 --- a/scripts/precommit-check.sh +++ b/scripts/precommit-check.sh @@ -30,4 +30,7 @@ for path in ["dictionary/actions.jsonl", "dictionary/model.examples.jsonl"]: json.loads(line) PY +python3 -m unittest tests/test_generate_training_data.py +python3 scripts/generate_training_data.py --dry-run + echo "precommit check complete" diff --git a/tests/test_generate_training_data.py b/tests/test_generate_training_data.py new file mode 100644 index 0000000..1b38b21 --- /dev/null +++ b/tests/test_generate_training_data.py @@ -0,0 +1,101 @@ +import argparse +import json +import shutil +import tempfile +import unittest +from pathlib import Path + +from scripts import generate_training_data as bridge + + +class GenerateTrainingDataTests(unittest.TestCase): + def setUp(self): + self.repo = Path(__file__).resolve().parents[1] + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name) + self.dictionary_dir = self.root / "dictionary" + shutil.copytree(self.repo / "dictionary", self.dictionary_dir) + self.output_dir = self.root / "workspace_Data" / "data" + self.output_dir.mkdir(parents=True) + self.manifest = self.root / "workspace_Data" / "manifest.llm.json" + self.manifest.write_text(json.dumps({"manifest_version": "0.0.0", "datasets": {}})) + + def tearDown(self): + self.temp.cleanup() + + def args(self, **overrides): + values = { + "dictionary_dir": str(self.dictionary_dir), + "output_dir": str(self.output_dir), + "manifest": str(self.manifest), + "dry_run": False, + "sdg_host": None, + "sdg_user": None, + "sdg_key": None, + "num_records": 1000, + "timestamp": 1234567890, + } + values.update(overrides) + return argparse.Namespace(**values) + + def test_seed_generation_covers_current_dictionary(self): + artifacts = bridge.load_dictionary_artifacts(self.dictionary_dir) + records = bridge.build_seed_records(artifacts, generated_at=123) + actions = {record["metadata"]["action_id"] for record in records} + self.assertEqual(len(actions), 13) + self.assertIn("calculate", actions) + self.assertIn("delete_file", actions) + self.assertEqual(len(records), 260) + + def test_compact_output_uses_codebook_codes(self): + artifacts = bridge.load_dictionary_artifacts(self.dictionary_dir) + compact = bridge.compact_target( + artifacts, + "hash_string", + {"text": "hello", "algorithm": "sha256"}, + ) + self.assertEqual(compact["a"], artifacts["action_to_code"]["hash_string"]) + self.assertIn(artifacts["field_to_code"]["text"], compact["p"]) + self.assertIn(artifacts["field_to_code"]["algorithm"], compact["p"]) + + def test_empty_payload_action(self): + artifacts = bridge.load_dictionary_artifacts(self.dictionary_dir) + records = bridge.build_seed_records(artifacts, generated_at=123) + uuid_records = [ + record for record in records if record["metadata"]["action_id"] == "generate_uuid" + ] + self.assertTrue(uuid_records) + output = json.loads(uuid_records[0]["output"]) + compact = json.loads(uuid_records[0]["metadata"]["compact"]) + self.assertEqual(output["payload"], {}) + self.assertEqual(compact["p"], {}) + + def test_split_includes_every_action(self): + artifacts = bridge.load_dictionary_artifacts(self.dictionary_dir) + records = bridge.build_seed_records(artifacts, generated_at=123) + train, validation = bridge.split_records_by_action(records) + train_actions = {record["metadata"]["action_id"] for record in train} + validation_actions = {record["metadata"]["action_id"] for record in validation} + self.assertEqual(train_actions, validation_actions) + self.assertEqual(len(train_actions), 13) + + def test_dry_run_writes_nothing(self): + summary = bridge.run_bridge(self.args(dry_run=True)) + self.assertTrue(summary["dry_run"]) + self.assertFalse((self.output_dir / "module_controller_intents_1234567890").exists()) + manifest = json.loads(self.manifest.read_text()) + self.assertEqual(manifest["datasets"], {}) + + def test_manifest_patch_adds_dataset_without_damaging_existing_keys(self): + bridge.run_bridge(self.args()) + manifest = json.loads(self.manifest.read_text()) + self.assertEqual(manifest["manifest_version"], "0.0.0") + dataset = manifest["datasets"]["module_controller_intents_1234567890"] + self.assertEqual(dataset["schema_profile"], "instruction") + self.assertEqual(dataset["codebook_version"], "1.0.0") + self.assertIn("codebook_checksum", dataset) + self.assertTrue((self.output_dir / "module_controller_intents_1234567890" / "train.jsonl").exists()) + + +if __name__ == "__main__": + unittest.main()