diff options
| author | jdj <jdj@kunix.org> | 2026-11-18 14:47:33 -0800 |
|---|---|---|
| committer | jdj <jdj@kunix.org> | 2026-11-18 14:47:33 -0800 |
| commit | 5d2e4252e695644c34be605b44a14bd80a98a0a6 (patch) | |
| tree | 311a72eda4459af7547b9cd05c0956eafb235f2d /eval.py | |
| download | boris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.tar.gz boris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.zip | |
Boris-2 training code
The code, configuration and data filters used to train Boris-2 (144M,
200B tokens, single V100): model, training loop, data preparation,
n-gram table, V100 kernels, final stages and the exact run configs.
Diffstat (limited to 'eval.py')
| -rw-r--r-- | eval.py | 358 |
1 files changed, 358 insertions, 0 deletions
@@ -0,0 +1,358 @@ +"""Standalone evaluation, on the second GPU, from a weights-only export. + + python eval.py --watch # poll the export manifest forever + python eval.py --ckpt checkpoints/exports/export_0010B.pt + python eval.py --ckpt ... --tasks hellaswag,piqa + +Runs in its own process on cuda:1 and never touches the training GPU. Training +writes an export every 10B tokens and appends a manifest line; this picks it up, +scores it, and appends one row per checkpoint to eval_out/results.jsonl keyed by +tokens_seen -- so the whole run plots directly against training tokens. + +All five benchmarks are multiple-choice log-likelihood, ArithMark-3 included +(its records are HellaSwag-shaped: a context and four endings). Both the raw +sum and the byte-length-normalised score are reported: acc_norm is the number +the Open SLM Leaderboard shows for HellaSwag, ARC and PIQA, and raw accuracy is +the more meaningful one for ArithMark, where all endings are nearly the same +length and normalising only adds noise. +""" +from __future__ import annotations + +import argparse +import json +import math +import os +import re +import time +from pathlib import Path + +import numpy as np +import torch +import torch.nn.functional as F + +from config import Config +from model import Boris2 +from ngram import NGramVocab +from tokenizer import BorisTokenizer + +HF_CACHE = Path.home() / ".cache" / "huggingface" + + +# --------------------------------------------------------------------------- # +# Task loading +# --------------------------------------------------------------------------- # +def _hf(name, *args, **kw): + from datasets import load_dataset + prev = os.environ.get("HF_HUB_OFFLINE") + os.environ.setdefault("HF_HUB_OFFLINE", "1") # never hit the network mid-run + try: + return load_dataset(name, *args, **kw) + finally: + if prev is None: + os.environ.pop("HF_HUB_OFFLINE", None) + + +def load_hellaswag(): + d = _hf("Rowan/hellaswag", split="validation") + def clean(s): + s = s.strip().replace(" [title]", ". ") + s = re.sub(r"\[.*?\]", "", s).replace(" ", " ") + return s + out = [] + for r in d: + if str(r["label"]).strip() == "": + continue + # Match lm-evaluation-harness/Open SLM Leaderboard preprocessing. + ctx = r["ctx_a"] + " " + r["ctx_b"].capitalize() + out.append({"ctx": clean(r["activity_label"] + ": " + ctx), + "choices": [" " + clean(x) for x in r["endings"]], + "gold": int(r["label"])}) + return out + + +def load_arc(which): + d = _hf("allenai/ai2_arc", which, split="test") + out = [] + for r in d: + ch = r["choices"] + labels = list(ch["label"]) + if r["answerKey"] not in labels: + continue + out.append({"ctx": "Question: " + r["question"] + "\nAnswer:", + "choices": [" " + t for t in ch["text"]], + "gold": labels.index(r["answerKey"])}) + return out + + +def load_piqa(): + # baber/piqa, not ybisk/piqa: the latter ships a loading script, and + # datasets 5.0 refuses those outright. Same 1,838 validation items. + d = _hf("baber/piqa", split="validation") + return [{"ctx": "Question: " + r["goal"] + "\nAnswer:", + "choices": [" " + r["sol1"], " " + r["sol2"]], + "gold": int(r["label"])} for r in d] + + +def load_arithmark3(): + root = HF_CACHE / "hub" / "datasets--AxiomicLabs--ArithMark-3.0" / "snapshots" + files = sorted(root.rglob("*.jsonl")) + if not files: + raise FileNotFoundError("ArithMark-3.0 is not in the HF cache") + out = [] + with open(files[0]) as f: + for line in f: + r = json.loads(line) + out.append({"ctx": r["ctx"], "choices": list(r["endings"]), + "gold": int(r["label"])}) + return out + + +TASKS = { + "hellaswag": load_hellaswag, + "arc_easy": lambda: load_arc("ARC-Easy"), + "arc_challenge": lambda: load_arc("ARC-Challenge"), + "piqa": load_piqa, + "arithmark3": load_arithmark3, +} + + +# --------------------------------------------------------------------------- # +# Scoring +# --------------------------------------------------------------------------- # +@torch.no_grad() +def score_batch(model, tok, items, dev, batch_size=16, max_len=1024): + """Summed log-probability of each continuation given its context. + + Sequences are right-padded, which is safe under causal attention: a padded + position can only affect positions after it, and none of those are scored. + Only the positions that predict continuation tokens are projected through + the LM head, so the (B, T, V) logit tensor is never built. + """ + flat = [] + for i, it in enumerate(items): + c = tok.encode(it["ctx"]) + for j, ch in enumerate(it["choices"]): + full, offsets = tok.encode_with_offsets(it["ctx"] + ch) + # Include a token that straddles the text boundary. Its probability + # is the first one that depends on continuation characters. + cut = next((q for q, (_, end) in enumerate(offsets) + if end > len(it["ctx"])), len(full)) + left = max(0, len(full) - max_len) + seq = full[left:] + cont_start = max(0, cut - left) + n_cont = max(0, len(seq) - max(cont_start, 1)) + # The leading target delimiter is scored but lm-eval excludes it + # from the character count used by acc_norm. + norm_chars = len(ch[1:]) if ch.startswith(" ") else len(ch) + flat.append((i, j, seq, n_cont, norm_chars)) + + order = sorted(range(len(flat)), key=lambda i: -len(flat[i][2])) + scores: dict[tuple, tuple] = {} + W = model.head_weight() + + for a in range(0, len(order), batch_size): + chunk = [flat[i] for i in order[a:a + batch_size]] + T = max(len(s) for _, _, s, _, _ in chunk) + B = len(chunk) + x = torch.zeros(B, T, dtype=torch.int64) + pos = torch.zeros(B, T, dtype=torch.int64) + bnd = torch.zeros(B, T, dtype=torch.bool) + bnd[:, 0] = True + for b, (_, _, s, _, _) in enumerate(chunk): + x[b, :len(s)] = torch.tensor(s) + pos[b] = torch.arange(T) + off = torch.arange(B + 1, dtype=torch.int64) * T + x, pos, bnd, off = x.to(dev), pos.to(dev), bnd.to(dev), off.to(dev) + + with torch.autocast("cuda", dtype=torch.float16): + h = model(x, pos, off, bnd, None, model._eval_theta) + + for b, (i, j, s, n_cont, nch) in enumerate(chunk): + end = len(s) - 1 + start = end - n_cont + sel = h[b, start:end].float() # positions predicting cont + logits = F.linear(sel, W.float()) + lp = torch.log_softmax(logits, -1) + tgt = torch.tensor(s[start + 1:end + 1], device=dev) + tot = float(lp.gather(1, tgt[:, None]).sum()) + scores[(i, j)] = (tot, nch) + + n_hit = n_hit_norm = 0 + for i, it in enumerate(items): + raw = [scores[(i, j)][0] for j in range(len(it["choices"]))] + # lm-evaluation-harness acc_norm divides by continuation characters. + norm = [scores[(i, j)][0] / max(scores[(i, j)][1], 1) + for j in range(len(it["choices"]))] + n_hit += int(int(np.argmax(raw)) == it["gold"]) + n_hit_norm += int(int(np.argmax(norm)) == it["gold"]) + n = len(items) + return {"acc": n_hit / n, "acc_norm": n_hit_norm / n, "n": n} + + +@torch.no_grad() +def val_loss(model, cfg, dev, max_tokens=10_000_000, block=2048, batch=8): + """Held-out loss on a fixed split the packer never draws from.""" + from data import open_validation, VarlenPacker + vdir = Path(cfg.data.root) / cfg.data.val_source + if not (vdir / "tokens.bin").exists(): + return None + st = open_validation(cfg) + c2 = Config.from_json(cfg.to_json()) + c2.enforce_budget = False + c2.curriculum.enabled = False + p = VarlenPacker(c2, {"val": st}, 999, split="val") + tot, n, seen = 0.0, 0, 0 + while seen < max_tokens: + xs, ys, pos, bnd, bounds, stats = p.batch(batch, block, 0.0) + t = lambda z: torch.from_numpy(z).to(dev) + with torch.autocast("cuda", dtype=torch.float16): + loss = model(t(xs), t(pos), t(bounds), t(bnd), t(ys), model._eval_theta) + valid = int((ys != -100).sum()) + tot += float(loss) * valid; n += valid; seen += stats["tokens"] + return tot / max(n, 1) + + +# --------------------------------------------------------------------------- # +def load_export(path: Path, dev: str): + # The allocator statistics and empty_cache() act on the CURRENT device, not + # on the device the tensors live on. eval runs on cuda:1 while cuda:0 is + # busy training, so this is not optional. + torch.cuda.set_device(dev) + blob = torch.load(path, map_location="cpu", weights_only=False) + cfg = Config.from_json(blob["config"]) + # The training head chunk (8192 on the V100) makes the 3060's stock head + # materialize 1 GB fp32 logit chunks; the chunk only sets memory, not the loss. + cfg.optim.lm_head_chunk = min(cfg.optim.lm_head_chunk, 2048) + model = Boris2(cfg).to(dev) + sd = {k: v.to(torch.float32) for k, v in blob["model"].items()} + model.load_state_dict(sd) + if model.ngram is not None: + vocab = NGramVocab.load(cfg.ngram.vocab_path) + got = vocab.fingerprint() + want = blob.get("ngram_fingerprint") + if want is not None and want not in (got, vocab.legacy_fingerprint()): + raise RuntimeError(f"checkpoint expects n-gram vocab {want}, loaded {got}") + model.ngram.load_vocab(vocab, dev) + model.eval() + progress = blob["tokens_seen"] / cfg.train.total_tokens + model._eval_theta = cfg.block_size_at(progress)[1] + return model, cfg, blob + + +def evaluate(path: Path, dev: str, tasks, out: Path, val_tokens: int, batch: int, + no_ngram: bool = False): + t0 = time.time() + model, cfg, blob = load_export(path, dev) + if no_ngram: + model.ngram_enabled = False + tok = BorisTokenizer.load(Path(cfg.data.root) / "tokenizer") + tok_fp = BorisTokenizer.fingerprint(Path(cfg.data.root) / "tokenizer") + if tok_fp != cfg.model.tokenizer_sha256: + raise RuntimeError( + f"tokenizer fingerprint {tok_fp} does not match checkpoint config " + f"{cfg.model.tokenizer_sha256}") + row = {"checkpoint": path.name, "step": blob["step"], + "tokens_seen": blob["tokens_seen"], + "tokens_b": blob["tokens_seen"] / 1e9, "wall": time.time(), + "ngram": (model.ngram is not None) and not no_ngram} + + for name in tasks: + if name not in TASKS: + print(f" unknown task {name}"); continue + try: + items = TASKS[name]() + except Exception as e: + print(f" {name}: unavailable ({type(e).__name__}: {str(e)[:90]})") + continue + r = score_batch(model, tok, items, dev, batch_size=batch) + # The Open SLM Leaderboard uses length-normalized accuracy for all + # five tasks, including ArithMark-3. + row[name] = r["acc_norm"] + row[name + "_acc"] = r["acc"] + row[name + "_acc_norm"] = r["acc_norm"] + row[name + "_n"] = r["n"] + print(f" {name:<15} acc {r['acc']:.4f} acc_norm {r['acc_norm']:.4f} " + f"n={r['n']}") + + # Compile only now: val batches are fixed-shape (8 x 2048), so the regions + # compile once; the benchmark batches above vary in length and would + # recompile per shape. Measured on the 3060: val 101 s -> 60 s per 2M + # tokens, loss equal to 4 decimals. + model.maybe_compile() + vl = val_loss(model, cfg, dev, val_tokens) + if vl is not None: + row["val_loss"] = vl + row["val_ppl"] = math.exp(min(vl, 20)) + print(f" {'val':<15} loss {vl:.4f} ppl {math.exp(min(vl,20)):.3f}") + + keys = [k for k in tasks if k in row] + if keys: + row["mean"] = sum(row[k] for k in keys) / len(keys) + print(f" {'MEAN':<15} {row['mean']:.4f}") + row["eval_seconds"] = time.time() - t0 + + out.parent.mkdir(parents=True, exist_ok=True) + with open(out, "a") as f: + f.write(json.dumps(row) + "\n") + del model + torch.cuda.empty_cache() + return row + + +def watch(cfg: Config, dev, tasks, out, val_tokens, batch, poll): + done = set() + if out.exists(): + for line in out.read_text().splitlines(): + try: + done.add(json.loads(line)["checkpoint"]) + except Exception: + pass + print(f" watching {cfg.eval.manifest} ({len(done)} already scored)") + while True: + pending = [] + if cfg.eval.manifest.exists(): + for line in cfg.eval.manifest.read_text().splitlines(): + try: + m = json.loads(line) + except Exception: + continue + p = Path(m["path"]) + if p.name not in done and p.exists(): + pending.append(p) + for p in sorted(set(pending)): + print(f"\n === {p.name} ===") + try: + evaluate(p, dev, tasks, out, val_tokens, batch) + done.add(p.name) + except Exception as e: + print(f" failed on {p.name}: {type(e).__name__}: {e}") + time.sleep(poll) + + +def main(): + cfg = Config() + ap = argparse.ArgumentParser() + ap.add_argument("--ckpt", type=Path, default=None) + ap.add_argument("--watch", action="store_true") + ap.add_argument("--device", default=cfg.eval.device) + ap.add_argument("--tasks", default=",".join(cfg.eval.tasks)) + ap.add_argument("--out", type=Path, default=cfg.eval.out_path) + ap.add_argument("--val-tokens", type=int, default=cfg.eval.val_tokens) + ap.add_argument("--batch", type=int, default=cfg.eval.batch_size) + ap.add_argument("--no-ngram", action="store_true", + help="score with the table switched off -- reports how much " + "the trained model DEPENDS on it, not what it gained") + a = ap.parse_args() + tasks = [t.strip() for t in a.tasks.split(",") if t.strip()] + + if a.watch: + watch(cfg, a.device, tasks, a.out, a.val_tokens, a.batch, cfg.eval.poll_seconds) + elif a.ckpt: + evaluate(a.ckpt, a.device, tasks, a.out, a.val_tokens, a.batch, + no_ngram=a.no_ngram) + else: + raise SystemExit("pass --ckpt PATH or --watch") + + +if __name__ == "__main__": + main() |
