feat: enrich training deploy via mimir sdg

scripts/deploy_training_data.sh

@@ -5,9 +5,22 @@ cd "$(dirname "$0")/.."
 
 export COPYFILE_DISABLE=1
 
+if [[ -f .env ]]; then
+  set -a
+  # shellcheck disable=SC1091
+  source .env
+  set +a
+fi
+
 CUSTOM_LOCAL_LLM_ROOT="${CUSTOM_LOCAL_LLM_ROOT:-/Volumes/Zoe/custom-local-llm}"
 TRAINING_DATA_DIR="${TRAINING_DATA_DIR:-$CUSTOM_LOCAL_LLM_ROOT/training-data}"
 TRAINING_DATA_MANIFEST="${TRAINING_DATA_MANIFEST:-$CUSTOM_LOCAL_LLM_ROOT/manifest.llm.json}"
+ENABLE_MIMIR_SDG="${ENABLE_MIMIR_SDG:-0}"
+MIMIR_SDG_HOST="${MIMIR_SDG_HOST:-100.80.52.47}"
+MIMIR_SDG_USER="${MIMIR_SDG_USER:-aaron-pressey}"
+MIMIR_SDG_KEY="${MIMIR_SDG_KEY:-$HOME/.ssh/silma_orson_ed25519}"
+MIMIR_SDG_MODEL_ALIAS="${MIMIR_SDG_MODEL_ALIAS:-nvidia-text}"
+MIMIR_SDG_NUM_RECORDS="${MIMIR_SDG_NUM_RECORDS:-1000}"
 
 if [[ ! -d "$CUSTOM_LOCAL_LLM_ROOT" ]]; then
   echo "missing custom local LLM root: $CUSTOM_LOCAL_LLM_ROOT" >&2
@@ -22,10 +35,28 @@ if [[ ! -s "$TRAINING_DATA_MANIFEST" ]]; then
 fi
 
 cargo run -- dictionary generate
-python3 scripts/generate_training_data.py \
+bridge_args=(
+  scripts/generate_training_data.py
   --dictionary-dir dictionary/ \
   --output-dir "$TRAINING_DATA_DIR" \
   --manifest "$TRAINING_DATA_MANIFEST"
+)
+
+if [[ "$ENABLE_MIMIR_SDG" = "1" ]]; then
+  if [[ ! -r "$MIMIR_SDG_KEY" ]]; then
+    echo "missing Mimir SDG SSH key: $MIMIR_SDG_KEY" >&2
+    exit 1
+  fi
+  bridge_args+=(
+    --sdg-host "$MIMIR_SDG_HOST"
+    --sdg-user "$MIMIR_SDG_USER"
+    --sdg-key "$MIMIR_SDG_KEY"
+    --sdg-model-alias "$MIMIR_SDG_MODEL_ALIAS"
+    --num-records "$MIMIR_SDG_NUM_RECORDS"
+  )
+fi
+
+python3 "${bridge_args[@]}"
 
 python3 - <<'PY'
 import os