feat: deploy training data to zoe root

scripts/deploy_training_data.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+cd "$(dirname "$0")/.."
+
+export COPYFILE_DISABLE=1
+
+CUSTOM_LOCAL_LLM_ROOT="${CUSTOM_LOCAL_LLM_ROOT:-/Volumes/Zoe/custom-local-llm}"
+TRAINING_DATA_DIR="${TRAINING_DATA_DIR:-$CUSTOM_LOCAL_LLM_ROOT/training-data}"
+TRAINING_DATA_MANIFEST="${TRAINING_DATA_MANIFEST:-$CUSTOM_LOCAL_LLM_ROOT/manifest.llm.json}"
+
+if [[ ! -d "$CUSTOM_LOCAL_LLM_ROOT" ]]; then
+  echo "missing custom local LLM root: $CUSTOM_LOCAL_LLM_ROOT" >&2
+  exit 1
+fi
+
+mkdir -p "$TRAINING_DATA_DIR" "$CUSTOM_LOCAL_LLM_ROOT/logs" "$CUSTOM_LOCAL_LLM_ROOT/models" "$CUSTOM_LOCAL_LLM_ROOT/benchmarks"
+
+if [[ ! -s "$TRAINING_DATA_MANIFEST" ]]; then
+  echo "missing training data manifest: $TRAINING_DATA_MANIFEST" >&2
+  exit 1
+fi
+
+cargo run -- dictionary generate
+python3 scripts/generate_training_data.py \
+  --dictionary-dir dictionary/ \
+  --output-dir "$TRAINING_DATA_DIR" \
+  --manifest "$TRAINING_DATA_MANIFEST"
+
+python3 - <<'PY'
+import os
+from pathlib import Path
+
+root = Path(os.environ.get("CUSTOM_LOCAL_LLM_ROOT", "/Volumes/Zoe/custom-local-llm"))
+for path in root.rglob("._*"):
+    if path.is_file():
+        path.unlink()
+PY

scripts/generate_training_data.py

@@ -15,6 +15,9 @@ from typing import Any
 INSTRUCTION = "Map the following natural language request to the correct action and payload."
 DATASET_PREFIX = "module_controller_intents"
 DEFAULT_MIN_RECORDS_PER_ACTION = 20
+CUSTOM_LOCAL_LLM_ROOT = Path("/Volumes/Zoe/custom-local-llm")
+DEFAULT_OUTPUT_DIR = CUSTOM_LOCAL_LLM_ROOT / "training-data"
+DEFAULT_MANIFEST = CUSTOM_LOCAL_LLM_ROOT / "manifest.llm.json"
 
 
 def main() -> int:
@@ -32,8 +35,8 @@ def main() -> int:
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--dictionary-dir", default="dictionary/")
-    parser.add_argument("--output-dir", default="../workspace_Data/data/")
-    parser.add_argument("--manifest", default="../workspace_Data/manifest.llm.json")
+    parser.add_argument("--output-dir", default=str(DEFAULT_OUTPUT_DIR))
+    parser.add_argument("--manifest", default=str(DEFAULT_MANIFEST))
     parser.add_argument("--dry-run", action="store_true")
     parser.add_argument("--sdg-host")
     parser.add_argument("--sdg-user")
@@ -469,6 +472,9 @@ def patch_manifest(
 
 def relative_data_path(dataset_dir: Path) -> str:
     parts = dataset_dir.parts
+    if "training-data" in parts:
+        index = len(parts) - 1 - list(reversed(parts)).index("training-data")
+        return "/".join(parts[index:])
     if "data" in parts:
         index = len(parts) - 1 - list(reversed(parts)).index("data")
         return "/".join(parts[index:])

scripts/precommit-check.sh

@@ -32,5 +32,6 @@ PY
 
 python3 -m unittest tests/test_generate_training_data.py
 python3 scripts/generate_training_data.py --dry-run
+test -x scripts/deploy_training_data.sh
 
 echo "precommit check complete"