39 lines
1.1 KiB
Bash
39 lines
1.1 KiB
Bash
|
#!/usr/bin/env bash
|
||
|
|
set -euo pipefail
|
||
|
|
|
||
|
|
cd "$(dirname "$0")/.."
|
||
|
|
|
||
|
|
export COPYFILE_DISABLE=1
|
||
|
|
|
||
|
|
CUSTOM_LOCAL_LLM_ROOT="${CUSTOM_LOCAL_LLM_ROOT:-/Volumes/Zoe/custom-local-llm}"
|
||
|
|
TRAINING_DATA_DIR="${TRAINING_DATA_DIR:-$CUSTOM_LOCAL_LLM_ROOT/training-data}"
|
||
|
|
TRAINING_DATA_MANIFEST="${TRAINING_DATA_MANIFEST:-$CUSTOM_LOCAL_LLM_ROOT/manifest.llm.json}"
|
||
|
|
|
||
|
|
if [[ ! -d "$CUSTOM_LOCAL_LLM_ROOT" ]]; then
|
||
|
|
echo "missing custom local LLM root: $CUSTOM_LOCAL_LLM_ROOT" >&2
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
mkdir -p "$TRAINING_DATA_DIR" "$CUSTOM_LOCAL_LLM_ROOT/logs" "$CUSTOM_LOCAL_LLM_ROOT/models" "$CUSTOM_LOCAL_LLM_ROOT/benchmarks"
|
||
|
|
|
||
|
|
if [[ ! -s "$TRAINING_DATA_MANIFEST" ]]; then
|
||
|
|
echo "missing training data manifest: $TRAINING_DATA_MANIFEST" >&2
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
cargo run -- dictionary generate
|
||
|
|
python3 scripts/generate_training_data.py \
|
||
|
|
--dictionary-dir dictionary/ \
|
||
|
|
--output-dir "$TRAINING_DATA_DIR" \
|
||
|
|
--manifest "$TRAINING_DATA_MANIFEST"
|
||
|
|
|
||
|
|
python3 - <<'PY'
|
||
|
|
import os
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
root = Path(os.environ.get("CUSTOM_LOCAL_LLM_ROOT", "/Volumes/Zoe/custom-local-llm"))
|
||
|
|
for path in root.rglob("._*"):
|
||
|
|
if path.is_file():
|
||
|
|
path.unlink()
|
||
|
|
PY
|