#!/usr/bin/env bash set -euo pipefail repo=/home/joseph/boris-2-D112M-n15M data=/media/joseph/Media/boris-2-data run=/media/joseph/Media/Boris-2-0920 old_tok="$data/tokenizer" new_tok="$run/tokenizer" log="$run/prepare_0920.log" exec >>"$log" 2>&1 echo "[$(date -Is)] Boris-2-0920 preparation started" sources=( climbmix climbmix_s1 climbmix_s2 climbmix_s3 climbmix_s4 climbmix_s5 climbmix_s6 climbmix_s7 climbmix_s8 climbmix_s9 climbmix_s10 climbmix_s11 climbmix_s12 climbmix_s13 climbmix_s14 climbmix_s15 finemath finemath_s1 ) convert_one() { local name=$1 out="$run/data/$1" if [[ -f "$out/retokenize.json" ]]; then echo "[$(date -Is)] $name already converted" return fi if [[ -e "$out/tokens.bin" || -e "$out/docs.idx" ]]; then echo "[$(date -Is)] incomplete output for $name; refusing to overwrite" >&2 return 2 fi echo "[$(date -Is)] converting $name" RAYON_NUM_THREADS=4 nice -n 10 ionice -c2 -n7 \ "$repo/.venv/bin/python" "$repo/retokenize.py" \ "$data/$name" "$old_tok" "$new_tok" --out "$out" --batch-docs 2048 } promote_one() { local name=$1 src="$data/$1" out="$run/data/$1" [[ -f "$out/retokenize.json" ]] || { echo "missing manifest for $name" >&2; return 2; } [[ -f "$out/tokens.bin" && -f "$out/docs.idx" ]] || { echo "missing output for $name" >&2; return 2; } echo "[$(date -Is)] promoting $name" mv "$src/tokens.bin" "$src/tokens.bin.0917" mv "$src/docs.idx" "$src/docs.idx.0917" mv "$out/tokens.bin" "$src/tokens.bin" mv "$out/docs.idx" "$src/docs.idx" if ! "$repo/.venv/bin/python" - "$src" <<'PY' import sys from pathlib import Path import numpy as np p=Path(sys.argv[1]); off=np.memmap(p/'docs.idx',dtype=np.uint64,mode='r') assert off.size >= 2 and int(off[0]) == 0 assert int(off[-1])*2 == (p/'tokens.bin').stat().st_size assert np.all(off[1:] > off[:-1]) PY then mv "$src/tokens.bin" "$out/tokens.bin.bad" mv "$src/docs.idx" "$out/docs.idx.bad" mv "$src/tokens.bin.0917" "$src/tokens.bin" mv "$src/docs.idx.0917" "$src/docs.idx" echo "validation failed and $name was rolled back" >&2 return 2 fi cp "$out/retokenize.json" "$src/retokenize_0920.json" rm "$src/tokens.bin.0917" "$src/docs.idx.0917" echo "[$(date -Is)] promoted and removed superseded token files for $name" } for ((i=0; i<${#sources[@]}; i+=4)); do pids=(); names=() for name in "${sources[@]:i:4}"; do convert_one "$name" & pids+=("$!"); names+=("$name") done for pid in "${pids[@]}"; do wait "$pid"; done for name in "${names[@]}"; do promote_one "$name"; done done # The validation split was started separately before this orchestrator. while pgrep -f "retokenize.py $data/climbmix_val " >/dev/null; do sleep 20; done promote_one climbmix_val if [[ ! -d "$data/tokenizer-0917" ]]; then mv "$data/tokenizer" "$data/tokenizer-0917" fi cp -a "$new_tok" "$data/tokenizer" echo "[$(date -Is)] building 90/10 n-gram vocabulary" nice -n 10 ionice -c2 -n7 "$repo/.venv/bin/python" "$repo/ngram.py" build \ --config "$run/config.json" --out "$run/ngram_vocab.bin" echo "[$(date -Is)] final verification" "$repo/.venv/bin/python" "$repo/verify.py" --quick --device cuda:1 echo "[$(date -Is)] V100 micro-batch confirmation" "$repo/.venv/bin/python" "$repo/preflight.py" --device cuda:0 \ --block 2048 --sweep 8,16 --iters 4 setsid "$repo/.venv/bin/python" "$repo/eval.py" --watch --device cuda:1 \ >"$run/eval_watch.log" 2>&1 < /dev/null & echo "[$(date -Is)] launching Boris-2-0920" exec "$repo/.venv/bin/python" "$repo/train.py" --config "$run/config.json"