diff options
Diffstat (limited to 'prepare_0920.sh')
| -rwxr-xr-x | prepare_0920.sh | 102 |
1 files changed, 102 insertions, 0 deletions
diff --git a/prepare_0920.sh b/prepare_0920.sh new file mode 100755 index 0000000..768c7fa --- /dev/null +++ b/prepare_0920.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo=/home/joseph/boris-2-D112M-n15M +data=/media/joseph/Media/boris-2-data +run=/media/joseph/Media/Boris-2-0920 +old_tok="$data/tokenizer" +new_tok="$run/tokenizer" +log="$run/prepare_0920.log" + +exec >>"$log" 2>&1 +echo "[$(date -Is)] Boris-2-0920 preparation started" + +sources=( + climbmix climbmix_s1 climbmix_s2 climbmix_s3 + climbmix_s4 climbmix_s5 climbmix_s6 climbmix_s7 + climbmix_s8 climbmix_s9 climbmix_s10 climbmix_s11 + climbmix_s12 climbmix_s13 climbmix_s14 climbmix_s15 + finemath finemath_s1 +) + +convert_one() { + local name=$1 out="$run/data/$1" + if [[ -f "$out/retokenize.json" ]]; then + echo "[$(date -Is)] $name already converted" + return + fi + if [[ -e "$out/tokens.bin" || -e "$out/docs.idx" ]]; then + echo "[$(date -Is)] incomplete output for $name; refusing to overwrite" >&2 + return 2 + fi + echo "[$(date -Is)] converting $name" + RAYON_NUM_THREADS=4 nice -n 10 ionice -c2 -n7 \ + "$repo/.venv/bin/python" "$repo/retokenize.py" \ + "$data/$name" "$old_tok" "$new_tok" --out "$out" --batch-docs 2048 +} + +promote_one() { + local name=$1 src="$data/$1" out="$run/data/$1" + [[ -f "$out/retokenize.json" ]] || { echo "missing manifest for $name" >&2; return 2; } + [[ -f "$out/tokens.bin" && -f "$out/docs.idx" ]] || { echo "missing output for $name" >&2; return 2; } + echo "[$(date -Is)] promoting $name" + mv "$src/tokens.bin" "$src/tokens.bin.0917" + mv "$src/docs.idx" "$src/docs.idx.0917" + mv "$out/tokens.bin" "$src/tokens.bin" + mv "$out/docs.idx" "$src/docs.idx" + if ! "$repo/.venv/bin/python" - "$src" <<'PY' +import sys +from pathlib import Path +import numpy as np +p=Path(sys.argv[1]); off=np.memmap(p/'docs.idx',dtype=np.uint64,mode='r') +assert off.size >= 2 and int(off[0]) == 0 +assert int(off[-1])*2 == (p/'tokens.bin').stat().st_size +assert np.all(off[1:] > off[:-1]) +PY + then + mv "$src/tokens.bin" "$out/tokens.bin.bad" + mv "$src/docs.idx" "$out/docs.idx.bad" + mv "$src/tokens.bin.0917" "$src/tokens.bin" + mv "$src/docs.idx.0917" "$src/docs.idx" + echo "validation failed and $name was rolled back" >&2 + return 2 + fi + cp "$out/retokenize.json" "$src/retokenize_0920.json" + rm "$src/tokens.bin.0917" "$src/docs.idx.0917" + echo "[$(date -Is)] promoted and removed superseded token files for $name" +} + +for ((i=0; i<${#sources[@]}; i+=4)); do + pids=(); names=() + for name in "${sources[@]:i:4}"; do + convert_one "$name" & pids+=("$!"); names+=("$name") + done + for pid in "${pids[@]}"; do wait "$pid"; done + for name in "${names[@]}"; do promote_one "$name"; done +done + +# The validation split was started separately before this orchestrator. +while pgrep -f "retokenize.py $data/climbmix_val " >/dev/null; do sleep 20; done +promote_one climbmix_val + +if [[ ! -d "$data/tokenizer-0917" ]]; then + mv "$data/tokenizer" "$data/tokenizer-0917" +fi +cp -a "$new_tok" "$data/tokenizer" + +echo "[$(date -Is)] building 90/10 n-gram vocabulary" +nice -n 10 ionice -c2 -n7 "$repo/.venv/bin/python" "$repo/ngram.py" build \ + --config "$run/config.json" --out "$run/ngram_vocab.bin" + +echo "[$(date -Is)] final verification" +"$repo/.venv/bin/python" "$repo/verify.py" --quick --device cuda:1 + +echo "[$(date -Is)] V100 micro-batch confirmation" +"$repo/.venv/bin/python" "$repo/preflight.py" --device cuda:0 \ + --block 2048 --sweep 8,16 --iters 4 + +setsid "$repo/.venv/bin/python" "$repo/eval.py" --watch --device cuda:1 \ + >"$run/eval_watch.log" 2>&1 < /dev/null & + +echo "[$(date -Is)] launching Boris-2-0920" +exec "$repo/.venv/bin/python" "$repo/train.py" --config "$run/config.json" |
