summaryrefslogtreecommitdiff
path: root/prepare_0920.sh
blob: 768c7fa441f0f84ee8d25cadb05ff882154548dc (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
#!/usr/bin/env bash
set -euo pipefail

repo=/home/joseph/boris-2-D112M-n15M
data=/media/joseph/Media/boris-2-data
run=/media/joseph/Media/Boris-2-0920
old_tok="$data/tokenizer"
new_tok="$run/tokenizer"
log="$run/prepare_0920.log"

exec >>"$log" 2>&1
echo "[$(date -Is)] Boris-2-0920 preparation started"

sources=(
  climbmix climbmix_s1 climbmix_s2 climbmix_s3
  climbmix_s4 climbmix_s5 climbmix_s6 climbmix_s7
  climbmix_s8 climbmix_s9 climbmix_s10 climbmix_s11
  climbmix_s12 climbmix_s13 climbmix_s14 climbmix_s15
  finemath finemath_s1
)

convert_one() {
  local name=$1 out="$run/data/$1"
  if [[ -f "$out/retokenize.json" ]]; then
    echo "[$(date -Is)] $name already converted"
    return
  fi
  if [[ -e "$out/tokens.bin" || -e "$out/docs.idx" ]]; then
    echo "[$(date -Is)] incomplete output for $name; refusing to overwrite" >&2
    return 2
  fi
  echo "[$(date -Is)] converting $name"
  RAYON_NUM_THREADS=4 nice -n 10 ionice -c2 -n7 \
    "$repo/.venv/bin/python" "$repo/retokenize.py" \
    "$data/$name" "$old_tok" "$new_tok" --out "$out" --batch-docs 2048
}

promote_one() {
  local name=$1 src="$data/$1" out="$run/data/$1"
  [[ -f "$out/retokenize.json" ]] || { echo "missing manifest for $name" >&2; return 2; }
  [[ -f "$out/tokens.bin" && -f "$out/docs.idx" ]] || { echo "missing output for $name" >&2; return 2; }
  echo "[$(date -Is)] promoting $name"
  mv "$src/tokens.bin" "$src/tokens.bin.0917"
  mv "$src/docs.idx" "$src/docs.idx.0917"
  mv "$out/tokens.bin" "$src/tokens.bin"
  mv "$out/docs.idx" "$src/docs.idx"
  if ! "$repo/.venv/bin/python" - "$src" <<'PY'
import sys
from pathlib import Path
import numpy as np
p=Path(sys.argv[1]); off=np.memmap(p/'docs.idx',dtype=np.uint64,mode='r')
assert off.size >= 2 and int(off[0]) == 0
assert int(off[-1])*2 == (p/'tokens.bin').stat().st_size
assert np.all(off[1:] > off[:-1])
PY
  then
    mv "$src/tokens.bin" "$out/tokens.bin.bad"
    mv "$src/docs.idx" "$out/docs.idx.bad"
    mv "$src/tokens.bin.0917" "$src/tokens.bin"
    mv "$src/docs.idx.0917" "$src/docs.idx"
    echo "validation failed and $name was rolled back" >&2
    return 2
  fi
  cp "$out/retokenize.json" "$src/retokenize_0920.json"
  rm "$src/tokens.bin.0917" "$src/docs.idx.0917"
  echo "[$(date -Is)] promoted and removed superseded token files for $name"
}

for ((i=0; i<${#sources[@]}; i+=4)); do
  pids=(); names=()
  for name in "${sources[@]:i:4}"; do
    convert_one "$name" & pids+=("$!"); names+=("$name")
  done
  for pid in "${pids[@]}"; do wait "$pid"; done
  for name in "${names[@]}"; do promote_one "$name"; done
done

# The validation split was started separately before this orchestrator.
while pgrep -f "retokenize.py $data/climbmix_val " >/dev/null; do sleep 20; done
promote_one climbmix_val

if [[ ! -d "$data/tokenizer-0917" ]]; then
  mv "$data/tokenizer" "$data/tokenizer-0917"
fi
cp -a "$new_tok" "$data/tokenizer"

echo "[$(date -Is)] building 90/10 n-gram vocabulary"
nice -n 10 ionice -c2 -n7 "$repo/.venv/bin/python" "$repo/ngram.py" build \
  --config "$run/config.json" --out "$run/ngram_vocab.bin"

echo "[$(date -Is)] final verification"
"$repo/.venv/bin/python" "$repo/verify.py" --quick --device cuda:1

echo "[$(date -Is)] V100 micro-batch confirmation"
"$repo/.venv/bin/python" "$repo/preflight.py" --device cuda:0 \
  --block 2048 --sweep 8,16 --iters 4

setsid "$repo/.venv/bin/python" "$repo/eval.py" --watch --device cuda:1 \
  >"$run/eval_watch.log" 2>&1 < /dev/null &

echo "[$(date -Is)] launching Boris-2-0920"
exec "$repo/.venv/bin/python" "$repo/train.py" --config "$run/config.json"