blob: 768c7fa441f0f84ee8d25cadb05ff882154548dc (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
|
#!/usr/bin/env bash
set -euo pipefail
repo=/home/joseph/boris-2-D112M-n15M
data=/media/joseph/Media/boris-2-data
run=/media/joseph/Media/Boris-2-0920
old_tok="$data/tokenizer"
new_tok="$run/tokenizer"
log="$run/prepare_0920.log"
exec >>"$log" 2>&1
echo "[$(date -Is)] Boris-2-0920 preparation started"
sources=(
climbmix climbmix_s1 climbmix_s2 climbmix_s3
climbmix_s4 climbmix_s5 climbmix_s6 climbmix_s7
climbmix_s8 climbmix_s9 climbmix_s10 climbmix_s11
climbmix_s12 climbmix_s13 climbmix_s14 climbmix_s15
finemath finemath_s1
)
convert_one() {
local name=$1 out="$run/data/$1"
if [[ -f "$out/retokenize.json" ]]; then
echo "[$(date -Is)] $name already converted"
return
fi
if [[ -e "$out/tokens.bin" || -e "$out/docs.idx" ]]; then
echo "[$(date -Is)] incomplete output for $name; refusing to overwrite" >&2
return 2
fi
echo "[$(date -Is)] converting $name"
RAYON_NUM_THREADS=4 nice -n 10 ionice -c2 -n7 \
"$repo/.venv/bin/python" "$repo/retokenize.py" \
"$data/$name" "$old_tok" "$new_tok" --out "$out" --batch-docs 2048
}
promote_one() {
local name=$1 src="$data/$1" out="$run/data/$1"
[[ -f "$out/retokenize.json" ]] || { echo "missing manifest for $name" >&2; return 2; }
[[ -f "$out/tokens.bin" && -f "$out/docs.idx" ]] || { echo "missing output for $name" >&2; return 2; }
echo "[$(date -Is)] promoting $name"
mv "$src/tokens.bin" "$src/tokens.bin.0917"
mv "$src/docs.idx" "$src/docs.idx.0917"
mv "$out/tokens.bin" "$src/tokens.bin"
mv "$out/docs.idx" "$src/docs.idx"
if ! "$repo/.venv/bin/python" - "$src" <<'PY'
import sys
from pathlib import Path
import numpy as np
p=Path(sys.argv[1]); off=np.memmap(p/'docs.idx',dtype=np.uint64,mode='r')
assert off.size >= 2 and int(off[0]) == 0
assert int(off[-1])*2 == (p/'tokens.bin').stat().st_size
assert np.all(off[1:] > off[:-1])
PY
then
mv "$src/tokens.bin" "$out/tokens.bin.bad"
mv "$src/docs.idx" "$out/docs.idx.bad"
mv "$src/tokens.bin.0917" "$src/tokens.bin"
mv "$src/docs.idx.0917" "$src/docs.idx"
echo "validation failed and $name was rolled back" >&2
return 2
fi
cp "$out/retokenize.json" "$src/retokenize_0920.json"
rm "$src/tokens.bin.0917" "$src/docs.idx.0917"
echo "[$(date -Is)] promoted and removed superseded token files for $name"
}
for ((i=0; i<${#sources[@]}; i+=4)); do
pids=(); names=()
for name in "${sources[@]:i:4}"; do
convert_one "$name" & pids+=("$!"); names+=("$name")
done
for pid in "${pids[@]}"; do wait "$pid"; done
for name in "${names[@]}"; do promote_one "$name"; done
done
# The validation split was started separately before this orchestrator.
while pgrep -f "retokenize.py $data/climbmix_val " >/dev/null; do sleep 20; done
promote_one climbmix_val
if [[ ! -d "$data/tokenizer-0917" ]]; then
mv "$data/tokenizer" "$data/tokenizer-0917"
fi
cp -a "$new_tok" "$data/tokenizer"
echo "[$(date -Is)] building 90/10 n-gram vocabulary"
nice -n 10 ionice -c2 -n7 "$repo/.venv/bin/python" "$repo/ngram.py" build \
--config "$run/config.json" --out "$run/ngram_vocab.bin"
echo "[$(date -Is)] final verification"
"$repo/.venv/bin/python" "$repo/verify.py" --quick --device cuda:1
echo "[$(date -Is)] V100 micro-batch confirmation"
"$repo/.venv/bin/python" "$repo/preflight.py" --device cuda:0 \
--block 2048 --sweep 8,16 --iters 4
setsid "$repo/.venv/bin/python" "$repo/eval.py" --watch --device cuda:1 \
>"$run/eval_watch.log" 2>&1 < /dev/null &
echo "[$(date -Is)] launching Boris-2-0920"
exec "$repo/.venv/bin/python" "$repo/train.py" --config "$run/config.json"
|