1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
|
"""Standalone evaluation, on the second GPU, from a weights-only export.
python eval.py --watch # poll the export manifest forever
python eval.py --ckpt checkpoints/exports/export_0010B.pt
python eval.py --ckpt ... --tasks hellaswag,piqa
Runs in its own process on cuda:1 and never touches the training GPU. Training
writes an export every 10B tokens and appends a manifest line; this picks it up,
scores it, and appends one row per checkpoint to eval_out/results.jsonl keyed by
tokens_seen -- so the whole run plots directly against training tokens.
All five benchmarks are multiple-choice log-likelihood, ArithMark-3 included
(its records are HellaSwag-shaped: a context and four endings). Both the raw
sum and the byte-length-normalised score are reported: acc_norm is the number
the Open SLM Leaderboard shows for HellaSwag, ARC and PIQA, and raw accuracy is
the more meaningful one for ArithMark, where all endings are nearly the same
length and normalising only adds noise.
"""
from __future__ import annotations
import argparse
import json
import math
import os
import re
import time
from pathlib import Path
import numpy as np
import torch
import torch.nn.functional as F
from config import Config
from model import Boris2
from ngram import NGramVocab
from tokenizer import BorisTokenizer
HF_CACHE = Path.home() / ".cache" / "huggingface"
# --------------------------------------------------------------------------- #
# Task loading
# --------------------------------------------------------------------------- #
def _hf(name, *args, **kw):
from datasets import load_dataset
prev = os.environ.get("HF_HUB_OFFLINE")
os.environ.setdefault("HF_HUB_OFFLINE", "1") # never hit the network mid-run
try:
return load_dataset(name, *args, **kw)
finally:
if prev is None:
os.environ.pop("HF_HUB_OFFLINE", None)
def load_hellaswag():
d = _hf("Rowan/hellaswag", split="validation")
def clean(s):
s = s.strip().replace(" [title]", ". ")
s = re.sub(r"\[.*?\]", "", s).replace(" ", " ")
return s
out = []
for r in d:
if str(r["label"]).strip() == "":
continue
# Match lm-evaluation-harness/Open SLM Leaderboard preprocessing.
ctx = r["ctx_a"] + " " + r["ctx_b"].capitalize()
out.append({"ctx": clean(r["activity_label"] + ": " + ctx),
"choices": [" " + clean(x) for x in r["endings"]],
"gold": int(r["label"])})
return out
def load_arc(which):
d = _hf("allenai/ai2_arc", which, split="test")
out = []
for r in d:
ch = r["choices"]
labels = list(ch["label"])
if r["answerKey"] not in labels:
continue
out.append({"ctx": "Question: " + r["question"] + "\nAnswer:",
"choices": [" " + t for t in ch["text"]],
"gold": labels.index(r["answerKey"])})
return out
def load_piqa():
# baber/piqa, not ybisk/piqa: the latter ships a loading script, and
# datasets 5.0 refuses those outright. Same 1,838 validation items.
d = _hf("baber/piqa", split="validation")
return [{"ctx": "Question: " + r["goal"] + "\nAnswer:",
"choices": [" " + r["sol1"], " " + r["sol2"]],
"gold": int(r["label"])} for r in d]
def load_arithmark3():
root = HF_CACHE / "hub" / "datasets--AxiomicLabs--ArithMark-3.0" / "snapshots"
files = sorted(root.rglob("*.jsonl"))
if not files:
raise FileNotFoundError("ArithMark-3.0 is not in the HF cache")
out = []
with open(files[0]) as f:
for line in f:
r = json.loads(line)
out.append({"ctx": r["ctx"], "choices": list(r["endings"]),
"gold": int(r["label"])})
return out
TASKS = {
"hellaswag": load_hellaswag,
"arc_easy": lambda: load_arc("ARC-Easy"),
"arc_challenge": lambda: load_arc("ARC-Challenge"),
"piqa": load_piqa,
"arithmark3": load_arithmark3,
}
# --------------------------------------------------------------------------- #
# Scoring
# --------------------------------------------------------------------------- #
@torch.no_grad()
def score_batch(model, tok, items, dev, batch_size=16, max_len=1024):
"""Summed log-probability of each continuation given its context.
Sequences are right-padded, which is safe under causal attention: a padded
position can only affect positions after it, and none of those are scored.
Only the positions that predict continuation tokens are projected through
the LM head, so the (B, T, V) logit tensor is never built.
"""
flat = []
for i, it in enumerate(items):
c = tok.encode(it["ctx"])
for j, ch in enumerate(it["choices"]):
full, offsets = tok.encode_with_offsets(it["ctx"] + ch)
# Include a token that straddles the text boundary. Its probability
# is the first one that depends on continuation characters.
cut = next((q for q, (_, end) in enumerate(offsets)
if end > len(it["ctx"])), len(full))
left = max(0, len(full) - max_len)
seq = full[left:]
cont_start = max(0, cut - left)
n_cont = max(0, len(seq) - max(cont_start, 1))
# The leading target delimiter is scored but lm-eval excludes it
# from the character count used by acc_norm.
norm_chars = len(ch[1:]) if ch.startswith(" ") else len(ch)
flat.append((i, j, seq, n_cont, norm_chars))
order = sorted(range(len(flat)), key=lambda i: -len(flat[i][2]))
scores: dict[tuple, tuple] = {}
W = model.head_weight()
for a in range(0, len(order), batch_size):
chunk = [flat[i] for i in order[a:a + batch_size]]
T = max(len(s) for _, _, s, _, _ in chunk)
B = len(chunk)
x = torch.zeros(B, T, dtype=torch.int64)
pos = torch.zeros(B, T, dtype=torch.int64)
bnd = torch.zeros(B, T, dtype=torch.bool)
bnd[:, 0] = True
for b, (_, _, s, _, _) in enumerate(chunk):
x[b, :len(s)] = torch.tensor(s)
pos[b] = torch.arange(T)
off = torch.arange(B + 1, dtype=torch.int64) * T
x, pos, bnd, off = x.to(dev), pos.to(dev), bnd.to(dev), off.to(dev)
with torch.autocast("cuda", dtype=torch.float16):
h = model(x, pos, off, bnd, None, model._eval_theta)
for b, (i, j, s, n_cont, nch) in enumerate(chunk):
end = len(s) - 1
start = end - n_cont
sel = h[b, start:end].float() # positions predicting cont
logits = F.linear(sel, W.float())
lp = torch.log_softmax(logits, -1)
tgt = torch.tensor(s[start + 1:end + 1], device=dev)
tot = float(lp.gather(1, tgt[:, None]).sum())
scores[(i, j)] = (tot, nch)
n_hit = n_hit_norm = 0
for i, it in enumerate(items):
raw = [scores[(i, j)][0] for j in range(len(it["choices"]))]
# lm-evaluation-harness acc_norm divides by continuation characters.
norm = [scores[(i, j)][0] / max(scores[(i, j)][1], 1)
for j in range(len(it["choices"]))]
n_hit += int(int(np.argmax(raw)) == it["gold"])
n_hit_norm += int(int(np.argmax(norm)) == it["gold"])
n = len(items)
return {"acc": n_hit / n, "acc_norm": n_hit_norm / n, "n": n}
@torch.no_grad()
def val_loss(model, cfg, dev, max_tokens=10_000_000, block=2048, batch=8):
"""Held-out loss on a fixed split the packer never draws from."""
from data import open_validation, VarlenPacker
vdir = Path(cfg.data.root) / cfg.data.val_source
if not (vdir / "tokens.bin").exists():
return None
st = open_validation(cfg)
c2 = Config.from_json(cfg.to_json())
c2.enforce_budget = False
c2.curriculum.enabled = False
p = VarlenPacker(c2, {"val": st}, 999, split="val")
tot, n, seen = 0.0, 0, 0
while seen < max_tokens:
xs, ys, pos, bnd, bounds, stats = p.batch(batch, block, 0.0)
t = lambda z: torch.from_numpy(z).to(dev)
with torch.autocast("cuda", dtype=torch.float16):
loss = model(t(xs), t(pos), t(bounds), t(bnd), t(ys), model._eval_theta)
valid = int((ys != -100).sum())
tot += float(loss) * valid; n += valid; seen += stats["tokens"]
return tot / max(n, 1)
# --------------------------------------------------------------------------- #
def load_export(path: Path, dev: str):
# The allocator statistics and empty_cache() act on the CURRENT device, not
# on the device the tensors live on. eval runs on cuda:1 while cuda:0 is
# busy training, so this is not optional.
torch.cuda.set_device(dev)
blob = torch.load(path, map_location="cpu", weights_only=False)
cfg = Config.from_json(blob["config"])
# The training head chunk (8192 on the V100) makes the 3060's stock head
# materialize 1 GB fp32 logit chunks; the chunk only sets memory, not the loss.
cfg.optim.lm_head_chunk = min(cfg.optim.lm_head_chunk, 2048)
model = Boris2(cfg).to(dev)
sd = {k: v.to(torch.float32) for k, v in blob["model"].items()}
model.load_state_dict(sd)
if model.ngram is not None:
vocab = NGramVocab.load(cfg.ngram.vocab_path)
got = vocab.fingerprint()
want = blob.get("ngram_fingerprint")
if want is not None and want not in (got, vocab.legacy_fingerprint()):
raise RuntimeError(f"checkpoint expects n-gram vocab {want}, loaded {got}")
model.ngram.load_vocab(vocab, dev)
model.eval()
progress = blob["tokens_seen"] / cfg.train.total_tokens
model._eval_theta = cfg.block_size_at(progress)[1]
return model, cfg, blob
def evaluate(path: Path, dev: str, tasks, out: Path, val_tokens: int, batch: int,
no_ngram: bool = False):
t0 = time.time()
model, cfg, blob = load_export(path, dev)
if no_ngram:
model.ngram_enabled = False
tok = BorisTokenizer.load(Path(cfg.data.root) / "tokenizer")
tok_fp = BorisTokenizer.fingerprint(Path(cfg.data.root) / "tokenizer")
if tok_fp != cfg.model.tokenizer_sha256:
raise RuntimeError(
f"tokenizer fingerprint {tok_fp} does not match checkpoint config "
f"{cfg.model.tokenizer_sha256}")
row = {"checkpoint": path.name, "step": blob["step"],
"tokens_seen": blob["tokens_seen"],
"tokens_b": blob["tokens_seen"] / 1e9, "wall": time.time(),
"ngram": (model.ngram is not None) and not no_ngram}
for name in tasks:
if name not in TASKS:
print(f" unknown task {name}"); continue
try:
items = TASKS[name]()
except Exception as e:
print(f" {name}: unavailable ({type(e).__name__}: {str(e)[:90]})")
continue
r = score_batch(model, tok, items, dev, batch_size=batch)
# The Open SLM Leaderboard uses length-normalized accuracy for all
# five tasks, including ArithMark-3.
row[name] = r["acc_norm"]
row[name + "_acc"] = r["acc"]
row[name + "_acc_norm"] = r["acc_norm"]
row[name + "_n"] = r["n"]
print(f" {name:<15} acc {r['acc']:.4f} acc_norm {r['acc_norm']:.4f} "
f"n={r['n']}")
# Compile only now: val batches are fixed-shape (8 x 2048), so the regions
# compile once; the benchmark batches above vary in length and would
# recompile per shape. Measured on the 3060: val 101 s -> 60 s per 2M
# tokens, loss equal to 4 decimals.
model.maybe_compile()
vl = val_loss(model, cfg, dev, val_tokens)
if vl is not None:
row["val_loss"] = vl
row["val_ppl"] = math.exp(min(vl, 20))
print(f" {'val':<15} loss {vl:.4f} ppl {math.exp(min(vl,20)):.3f}")
keys = [k for k in tasks if k in row]
if keys:
row["mean"] = sum(row[k] for k in keys) / len(keys)
print(f" {'MEAN':<15} {row['mean']:.4f}")
row["eval_seconds"] = time.time() - t0
out.parent.mkdir(parents=True, exist_ok=True)
with open(out, "a") as f:
f.write(json.dumps(row) + "\n")
del model
torch.cuda.empty_cache()
return row
def watch(cfg: Config, dev, tasks, out, val_tokens, batch, poll):
done = set()
if out.exists():
for line in out.read_text().splitlines():
try:
done.add(json.loads(line)["checkpoint"])
except Exception:
pass
print(f" watching {cfg.eval.manifest} ({len(done)} already scored)")
while True:
pending = []
if cfg.eval.manifest.exists():
for line in cfg.eval.manifest.read_text().splitlines():
try:
m = json.loads(line)
except Exception:
continue
p = Path(m["path"])
if p.name not in done and p.exists():
pending.append(p)
for p in sorted(set(pending)):
print(f"\n === {p.name} ===")
try:
evaluate(p, dev, tasks, out, val_tokens, batch)
done.add(p.name)
except Exception as e:
print(f" failed on {p.name}: {type(e).__name__}: {e}")
time.sleep(poll)
def main():
cfg = Config()
ap = argparse.ArgumentParser()
ap.add_argument("--ckpt", type=Path, default=None)
ap.add_argument("--watch", action="store_true")
ap.add_argument("--device", default=cfg.eval.device)
ap.add_argument("--tasks", default=",".join(cfg.eval.tasks))
ap.add_argument("--out", type=Path, default=cfg.eval.out_path)
ap.add_argument("--val-tokens", type=int, default=cfg.eval.val_tokens)
ap.add_argument("--batch", type=int, default=cfg.eval.batch_size)
ap.add_argument("--no-ngram", action="store_true",
help="score with the table switched off -- reports how much "
"the trained model DEPENDS on it, not what it gained")
a = ap.parse_args()
tasks = [t.strip() for t in a.tasks.split(",") if t.strip()]
if a.watch:
watch(cfg, a.device, tasks, a.out, a.val_tokens, a.batch, cfg.eval.poll_seconds)
elif a.ckpt:
evaluate(a.ckpt, a.device, tasks, a.out, a.val_tokens, a.batch,
no_ngram=a.no_ngram)
else:
raise SystemExit("pass --ckpt PATH or --watch")
if __name__ == "__main__":
main()
|