"""Standalone evaluation, on the second GPU, from a weights-only export. python eval.py --watch # poll the export manifest forever python eval.py --ckpt checkpoints/exports/export_0010B.pt python eval.py --ckpt ... --tasks hellaswag,piqa Runs in its own process on cuda:1 and never touches the training GPU. Training writes an export every 10B tokens and appends a manifest line; this picks it up, scores it, and appends one row per checkpoint to eval_out/results.jsonl keyed by tokens_seen -- so the whole run plots directly against training tokens. All five benchmarks are multiple-choice log-likelihood, ArithMark-3 included (its records are HellaSwag-shaped: a context and four endings). Both the raw sum and the byte-length-normalised score are reported: acc_norm is the number the Open SLM Leaderboard shows for HellaSwag, ARC and PIQA, and raw accuracy is the more meaningful one for ArithMark, where all endings are nearly the same length and normalising only adds noise. """ from __future__ import annotations import argparse import json import math import os import re import time from pathlib import Path import numpy as np import torch import torch.nn.functional as F from config import Config from model import Boris2 from ngram import NGramVocab from tokenizer import BorisTokenizer HF_CACHE = Path.home() / ".cache" / "huggingface" # --------------------------------------------------------------------------- # # Task loading # --------------------------------------------------------------------------- # def _hf(name, *args, **kw): from datasets import load_dataset prev = os.environ.get("HF_HUB_OFFLINE") os.environ.setdefault("HF_HUB_OFFLINE", "1") # never hit the network mid-run try: return load_dataset(name, *args, **kw) finally: if prev is None: os.environ.pop("HF_HUB_OFFLINE", None) def load_hellaswag(): d = _hf("Rowan/hellaswag", split="validation") def clean(s): s = s.strip().replace(" [title]", ". ") s = re.sub(r"\[.*?\]", "", s).replace(" ", " ") return s out = [] for r in d: if str(r["label"]).strip() == "": continue # Match lm-evaluation-harness/Open SLM Leaderboard preprocessing. ctx = r["ctx_a"] + " " + r["ctx_b"].capitalize() out.append({"ctx": clean(r["activity_label"] + ": " + ctx), "choices": [" " + clean(x) for x in r["endings"]], "gold": int(r["label"])}) return out def load_arc(which): d = _hf("allenai/ai2_arc", which, split="test") out = [] for r in d: ch = r["choices"] labels = list(ch["label"]) if r["answerKey"] not in labels: continue out.append({"ctx": "Question: " + r["question"] + "\nAnswer:", "choices": [" " + t for t in ch["text"]], "gold": labels.index(r["answerKey"])}) return out def load_piqa(): # baber/piqa, not ybisk/piqa: the latter ships a loading script, and # datasets 5.0 refuses those outright. Same 1,838 validation items. d = _hf("baber/piqa", split="validation") return [{"ctx": "Question: " + r["goal"] + "\nAnswer:", "choices": [" " + r["sol1"], " " + r["sol2"]], "gold": int(r["label"])} for r in d] def load_arithmark3(): root = HF_CACHE / "hub" / "datasets--AxiomicLabs--ArithMark-3.0" / "snapshots" files = sorted(root.rglob("*.jsonl")) if not files: raise FileNotFoundError("ArithMark-3.0 is not in the HF cache") out = [] with open(files[0]) as f: for line in f: r = json.loads(line) out.append({"ctx": r["ctx"], "choices": list(r["endings"]), "gold": int(r["label"])}) return out TASKS = { "hellaswag": load_hellaswag, "arc_easy": lambda: load_arc("ARC-Easy"), "arc_challenge": lambda: load_arc("ARC-Challenge"), "piqa": load_piqa, "arithmark3": load_arithmark3, } # --------------------------------------------------------------------------- # # Scoring # --------------------------------------------------------------------------- # @torch.no_grad() def score_batch(model, tok, items, dev, batch_size=16, max_len=1024): """Summed log-probability of each continuation given its context. Sequences are right-padded, which is safe under causal attention: a padded position can only affect positions after it, and none of those are scored. Only the positions that predict continuation tokens are projected through the LM head, so the (B, T, V) logit tensor is never built. """ flat = [] for i, it in enumerate(items): c = tok.encode(it["ctx"]) for j, ch in enumerate(it["choices"]): full, offsets = tok.encode_with_offsets(it["ctx"] + ch) # Include a token that straddles the text boundary. Its probability # is the first one that depends on continuation characters. cut = next((q for q, (_, end) in enumerate(offsets) if end > len(it["ctx"])), len(full)) left = max(0, len(full) - max_len) seq = full[left:] cont_start = max(0, cut - left) n_cont = max(0, len(seq) - max(cont_start, 1)) # The leading target delimiter is scored but lm-eval excludes it # from the character count used by acc_norm. norm_chars = len(ch[1:]) if ch.startswith(" ") else len(ch) flat.append((i, j, seq, n_cont, norm_chars)) order = sorted(range(len(flat)), key=lambda i: -len(flat[i][2])) scores: dict[tuple, tuple] = {} W = model.head_weight() for a in range(0, len(order), batch_size): chunk = [flat[i] for i in order[a:a + batch_size]] T = max(len(s) for _, _, s, _, _ in chunk) B = len(chunk) x = torch.zeros(B, T, dtype=torch.int64) pos = torch.zeros(B, T, dtype=torch.int64) bnd = torch.zeros(B, T, dtype=torch.bool) bnd[:, 0] = True for b, (_, _, s, _, _) in enumerate(chunk): x[b, :len(s)] = torch.tensor(s) pos[b] = torch.arange(T) off = torch.arange(B + 1, dtype=torch.int64) * T x, pos, bnd, off = x.to(dev), pos.to(dev), bnd.to(dev), off.to(dev) with torch.autocast("cuda", dtype=torch.float16): h = model(x, pos, off, bnd, None, model._eval_theta) for b, (i, j, s, n_cont, nch) in enumerate(chunk): end = len(s) - 1 start = end - n_cont sel = h[b, start:end].float() # positions predicting cont logits = F.linear(sel, W.float()) lp = torch.log_softmax(logits, -1) tgt = torch.tensor(s[start + 1:end + 1], device=dev) tot = float(lp.gather(1, tgt[:, None]).sum()) scores[(i, j)] = (tot, nch) n_hit = n_hit_norm = 0 for i, it in enumerate(items): raw = [scores[(i, j)][0] for j in range(len(it["choices"]))] # lm-evaluation-harness acc_norm divides by continuation characters. norm = [scores[(i, j)][0] / max(scores[(i, j)][1], 1) for j in range(len(it["choices"]))] n_hit += int(int(np.argmax(raw)) == it["gold"]) n_hit_norm += int(int(np.argmax(norm)) == it["gold"]) n = len(items) return {"acc": n_hit / n, "acc_norm": n_hit_norm / n, "n": n} @torch.no_grad() def val_loss(model, cfg, dev, max_tokens=10_000_000, block=2048, batch=8): """Held-out loss on a fixed split the packer never draws from.""" from data import open_validation, VarlenPacker vdir = Path(cfg.data.root) / cfg.data.val_source if not (vdir / "tokens.bin").exists(): return None st = open_validation(cfg) c2 = Config.from_json(cfg.to_json()) c2.enforce_budget = False c2.curriculum.enabled = False p = VarlenPacker(c2, {"val": st}, 999, split="val") tot, n, seen = 0.0, 0, 0 while seen < max_tokens: xs, ys, pos, bnd, bounds, stats = p.batch(batch, block, 0.0) t = lambda z: torch.from_numpy(z).to(dev) with torch.autocast("cuda", dtype=torch.float16): loss = model(t(xs), t(pos), t(bounds), t(bnd), t(ys), model._eval_theta) valid = int((ys != -100).sum()) tot += float(loss) * valid; n += valid; seen += stats["tokens"] return tot / max(n, 1) # --------------------------------------------------------------------------- # def load_export(path: Path, dev: str): # The allocator statistics and empty_cache() act on the CURRENT device, not # on the device the tensors live on. eval runs on cuda:1 while cuda:0 is # busy training, so this is not optional. torch.cuda.set_device(dev) blob = torch.load(path, map_location="cpu", weights_only=False) cfg = Config.from_json(blob["config"]) # The training head chunk (8192 on the V100) makes the 3060's stock head # materialize 1 GB fp32 logit chunks; the chunk only sets memory, not the loss. cfg.optim.lm_head_chunk = min(cfg.optim.lm_head_chunk, 2048) model = Boris2(cfg).to(dev) sd = {k: v.to(torch.float32) for k, v in blob["model"].items()} model.load_state_dict(sd) if model.ngram is not None: vocab = NGramVocab.load(cfg.ngram.vocab_path) got = vocab.fingerprint() want = blob.get("ngram_fingerprint") if want is not None and want not in (got, vocab.legacy_fingerprint()): raise RuntimeError(f"checkpoint expects n-gram vocab {want}, loaded {got}") model.ngram.load_vocab(vocab, dev) model.eval() progress = blob["tokens_seen"] / cfg.train.total_tokens model._eval_theta = cfg.block_size_at(progress)[1] return model, cfg, blob def evaluate(path: Path, dev: str, tasks, out: Path, val_tokens: int, batch: int, no_ngram: bool = False): t0 = time.time() model, cfg, blob = load_export(path, dev) if no_ngram: model.ngram_enabled = False tok = BorisTokenizer.load(Path(cfg.data.root) / "tokenizer") tok_fp = BorisTokenizer.fingerprint(Path(cfg.data.root) / "tokenizer") if tok_fp != cfg.model.tokenizer_sha256: raise RuntimeError( f"tokenizer fingerprint {tok_fp} does not match checkpoint config " f"{cfg.model.tokenizer_sha256}") row = {"checkpoint": path.name, "step": blob["step"], "tokens_seen": blob["tokens_seen"], "tokens_b": blob["tokens_seen"] / 1e9, "wall": time.time(), "ngram": (model.ngram is not None) and not no_ngram} for name in tasks: if name not in TASKS: print(f" unknown task {name}"); continue try: items = TASKS[name]() except Exception as e: print(f" {name}: unavailable ({type(e).__name__}: {str(e)[:90]})") continue r = score_batch(model, tok, items, dev, batch_size=batch) # The Open SLM Leaderboard uses length-normalized accuracy for all # five tasks, including ArithMark-3. row[name] = r["acc_norm"] row[name + "_acc"] = r["acc"] row[name + "_acc_norm"] = r["acc_norm"] row[name + "_n"] = r["n"] print(f" {name:<15} acc {r['acc']:.4f} acc_norm {r['acc_norm']:.4f} " f"n={r['n']}") # Compile only now: val batches are fixed-shape (8 x 2048), so the regions # compile once; the benchmark batches above vary in length and would # recompile per shape. Measured on the 3060: val 101 s -> 60 s per 2M # tokens, loss equal to 4 decimals. model.maybe_compile() vl = val_loss(model, cfg, dev, val_tokens) if vl is not None: row["val_loss"] = vl row["val_ppl"] = math.exp(min(vl, 20)) print(f" {'val':<15} loss {vl:.4f} ppl {math.exp(min(vl,20)):.3f}") keys = [k for k in tasks if k in row] if keys: row["mean"] = sum(row[k] for k in keys) / len(keys) print(f" {'MEAN':<15} {row['mean']:.4f}") row["eval_seconds"] = time.time() - t0 out.parent.mkdir(parents=True, exist_ok=True) with open(out, "a") as f: f.write(json.dumps(row) + "\n") del model torch.cuda.empty_cache() return row def watch(cfg: Config, dev, tasks, out, val_tokens, batch, poll): done = set() if out.exists(): for line in out.read_text().splitlines(): try: done.add(json.loads(line)["checkpoint"]) except Exception: pass print(f" watching {cfg.eval.manifest} ({len(done)} already scored)") while True: pending = [] if cfg.eval.manifest.exists(): for line in cfg.eval.manifest.read_text().splitlines(): try: m = json.loads(line) except Exception: continue p = Path(m["path"]) if p.name not in done and p.exists(): pending.append(p) for p in sorted(set(pending)): print(f"\n === {p.name} ===") try: evaluate(p, dev, tasks, out, val_tokens, batch) done.add(p.name) except Exception as e: print(f" failed on {p.name}: {type(e).__name__}: {e}") time.sleep(poll) def main(): cfg = Config() ap = argparse.ArgumentParser() ap.add_argument("--ckpt", type=Path, default=None) ap.add_argument("--watch", action="store_true") ap.add_argument("--device", default=cfg.eval.device) ap.add_argument("--tasks", default=",".join(cfg.eval.tasks)) ap.add_argument("--out", type=Path, default=cfg.eval.out_path) ap.add_argument("--val-tokens", type=int, default=cfg.eval.val_tokens) ap.add_argument("--batch", type=int, default=cfg.eval.batch_size) ap.add_argument("--no-ngram", action="store_true", help="score with the table switched off -- reports how much " "the trained model DEPENDS on it, not what it gained") a = ap.parse_args() tasks = [t.strip() for t in a.tasks.split(",") if t.strip()] if a.watch: watch(cfg, a.device, tasks, a.out, a.val_tokens, a.batch, cfg.eval.poll_seconds) elif a.ckpt: evaluate(a.ckpt, a.device, tasks, a.out, a.val_tokens, a.batch, no_ngram=a.no_ngram) else: raise SystemExit("pass --ckpt PATH or --watch") if __name__ == "__main__": main()