summaryrefslogtreecommitdiff
path: root/eval.py
diff options
context:
space:
mode:
Diffstat (limited to 'eval.py')
-rw-r--r--eval.py358
1 files changed, 358 insertions, 0 deletions
diff --git a/eval.py b/eval.py
new file mode 100644
index 0000000..88f692a
--- /dev/null
+++ b/eval.py
@@ -0,0 +1,358 @@
+"""Standalone evaluation, on the second GPU, from a weights-only export.
+
+ python eval.py --watch # poll the export manifest forever
+ python eval.py --ckpt checkpoints/exports/export_0010B.pt
+ python eval.py --ckpt ... --tasks hellaswag,piqa
+
+Runs in its own process on cuda:1 and never touches the training GPU. Training
+writes an export every 10B tokens and appends a manifest line; this picks it up,
+scores it, and appends one row per checkpoint to eval_out/results.jsonl keyed by
+tokens_seen -- so the whole run plots directly against training tokens.
+
+All five benchmarks are multiple-choice log-likelihood, ArithMark-3 included
+(its records are HellaSwag-shaped: a context and four endings). Both the raw
+sum and the byte-length-normalised score are reported: acc_norm is the number
+the Open SLM Leaderboard shows for HellaSwag, ARC and PIQA, and raw accuracy is
+the more meaningful one for ArithMark, where all endings are nearly the same
+length and normalising only adds noise.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import os
+import re
+import time
+from pathlib import Path
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+
+from config import Config
+from model import Boris2
+from ngram import NGramVocab
+from tokenizer import BorisTokenizer
+
+HF_CACHE = Path.home() / ".cache" / "huggingface"
+
+
+# --------------------------------------------------------------------------- #
+# Task loading
+# --------------------------------------------------------------------------- #
+def _hf(name, *args, **kw):
+ from datasets import load_dataset
+ prev = os.environ.get("HF_HUB_OFFLINE")
+ os.environ.setdefault("HF_HUB_OFFLINE", "1") # never hit the network mid-run
+ try:
+ return load_dataset(name, *args, **kw)
+ finally:
+ if prev is None:
+ os.environ.pop("HF_HUB_OFFLINE", None)
+
+
+def load_hellaswag():
+ d = _hf("Rowan/hellaswag", split="validation")
+ def clean(s):
+ s = s.strip().replace(" [title]", ". ")
+ s = re.sub(r"\[.*?\]", "", s).replace(" ", " ")
+ return s
+ out = []
+ for r in d:
+ if str(r["label"]).strip() == "":
+ continue
+ # Match lm-evaluation-harness/Open SLM Leaderboard preprocessing.
+ ctx = r["ctx_a"] + " " + r["ctx_b"].capitalize()
+ out.append({"ctx": clean(r["activity_label"] + ": " + ctx),
+ "choices": [" " + clean(x) for x in r["endings"]],
+ "gold": int(r["label"])})
+ return out
+
+
+def load_arc(which):
+ d = _hf("allenai/ai2_arc", which, split="test")
+ out = []
+ for r in d:
+ ch = r["choices"]
+ labels = list(ch["label"])
+ if r["answerKey"] not in labels:
+ continue
+ out.append({"ctx": "Question: " + r["question"] + "\nAnswer:",
+ "choices": [" " + t for t in ch["text"]],
+ "gold": labels.index(r["answerKey"])})
+ return out
+
+
+def load_piqa():
+ # baber/piqa, not ybisk/piqa: the latter ships a loading script, and
+ # datasets 5.0 refuses those outright. Same 1,838 validation items.
+ d = _hf("baber/piqa", split="validation")
+ return [{"ctx": "Question: " + r["goal"] + "\nAnswer:",
+ "choices": [" " + r["sol1"], " " + r["sol2"]],
+ "gold": int(r["label"])} for r in d]
+
+
+def load_arithmark3():
+ root = HF_CACHE / "hub" / "datasets--AxiomicLabs--ArithMark-3.0" / "snapshots"
+ files = sorted(root.rglob("*.jsonl"))
+ if not files:
+ raise FileNotFoundError("ArithMark-3.0 is not in the HF cache")
+ out = []
+ with open(files[0]) as f:
+ for line in f:
+ r = json.loads(line)
+ out.append({"ctx": r["ctx"], "choices": list(r["endings"]),
+ "gold": int(r["label"])})
+ return out
+
+
+TASKS = {
+ "hellaswag": load_hellaswag,
+ "arc_easy": lambda: load_arc("ARC-Easy"),
+ "arc_challenge": lambda: load_arc("ARC-Challenge"),
+ "piqa": load_piqa,
+ "arithmark3": load_arithmark3,
+}
+
+
+# --------------------------------------------------------------------------- #
+# Scoring
+# --------------------------------------------------------------------------- #
+@torch.no_grad()
+def score_batch(model, tok, items, dev, batch_size=16, max_len=1024):
+ """Summed log-probability of each continuation given its context.
+
+ Sequences are right-padded, which is safe under causal attention: a padded
+ position can only affect positions after it, and none of those are scored.
+ Only the positions that predict continuation tokens are projected through
+ the LM head, so the (B, T, V) logit tensor is never built.
+ """
+ flat = []
+ for i, it in enumerate(items):
+ c = tok.encode(it["ctx"])
+ for j, ch in enumerate(it["choices"]):
+ full, offsets = tok.encode_with_offsets(it["ctx"] + ch)
+ # Include a token that straddles the text boundary. Its probability
+ # is the first one that depends on continuation characters.
+ cut = next((q for q, (_, end) in enumerate(offsets)
+ if end > len(it["ctx"])), len(full))
+ left = max(0, len(full) - max_len)
+ seq = full[left:]
+ cont_start = max(0, cut - left)
+ n_cont = max(0, len(seq) - max(cont_start, 1))
+ # The leading target delimiter is scored but lm-eval excludes it
+ # from the character count used by acc_norm.
+ norm_chars = len(ch[1:]) if ch.startswith(" ") else len(ch)
+ flat.append((i, j, seq, n_cont, norm_chars))
+
+ order = sorted(range(len(flat)), key=lambda i: -len(flat[i][2]))
+ scores: dict[tuple, tuple] = {}
+ W = model.head_weight()
+
+ for a in range(0, len(order), batch_size):
+ chunk = [flat[i] for i in order[a:a + batch_size]]
+ T = max(len(s) for _, _, s, _, _ in chunk)
+ B = len(chunk)
+ x = torch.zeros(B, T, dtype=torch.int64)
+ pos = torch.zeros(B, T, dtype=torch.int64)
+ bnd = torch.zeros(B, T, dtype=torch.bool)
+ bnd[:, 0] = True
+ for b, (_, _, s, _, _) in enumerate(chunk):
+ x[b, :len(s)] = torch.tensor(s)
+ pos[b] = torch.arange(T)
+ off = torch.arange(B + 1, dtype=torch.int64) * T
+ x, pos, bnd, off = x.to(dev), pos.to(dev), bnd.to(dev), off.to(dev)
+
+ with torch.autocast("cuda", dtype=torch.float16):
+ h = model(x, pos, off, bnd, None, model._eval_theta)
+
+ for b, (i, j, s, n_cont, nch) in enumerate(chunk):
+ end = len(s) - 1
+ start = end - n_cont
+ sel = h[b, start:end].float() # positions predicting cont
+ logits = F.linear(sel, W.float())
+ lp = torch.log_softmax(logits, -1)
+ tgt = torch.tensor(s[start + 1:end + 1], device=dev)
+ tot = float(lp.gather(1, tgt[:, None]).sum())
+ scores[(i, j)] = (tot, nch)
+
+ n_hit = n_hit_norm = 0
+ for i, it in enumerate(items):
+ raw = [scores[(i, j)][0] for j in range(len(it["choices"]))]
+ # lm-evaluation-harness acc_norm divides by continuation characters.
+ norm = [scores[(i, j)][0] / max(scores[(i, j)][1], 1)
+ for j in range(len(it["choices"]))]
+ n_hit += int(int(np.argmax(raw)) == it["gold"])
+ n_hit_norm += int(int(np.argmax(norm)) == it["gold"])
+ n = len(items)
+ return {"acc": n_hit / n, "acc_norm": n_hit_norm / n, "n": n}
+
+
+@torch.no_grad()
+def val_loss(model, cfg, dev, max_tokens=10_000_000, block=2048, batch=8):
+ """Held-out loss on a fixed split the packer never draws from."""
+ from data import open_validation, VarlenPacker
+ vdir = Path(cfg.data.root) / cfg.data.val_source
+ if not (vdir / "tokens.bin").exists():
+ return None
+ st = open_validation(cfg)
+ c2 = Config.from_json(cfg.to_json())
+ c2.enforce_budget = False
+ c2.curriculum.enabled = False
+ p = VarlenPacker(c2, {"val": st}, 999, split="val")
+ tot, n, seen = 0.0, 0, 0
+ while seen < max_tokens:
+ xs, ys, pos, bnd, bounds, stats = p.batch(batch, block, 0.0)
+ t = lambda z: torch.from_numpy(z).to(dev)
+ with torch.autocast("cuda", dtype=torch.float16):
+ loss = model(t(xs), t(pos), t(bounds), t(bnd), t(ys), model._eval_theta)
+ valid = int((ys != -100).sum())
+ tot += float(loss) * valid; n += valid; seen += stats["tokens"]
+ return tot / max(n, 1)
+
+
+# --------------------------------------------------------------------------- #
+def load_export(path: Path, dev: str):
+ # The allocator statistics and empty_cache() act on the CURRENT device, not
+ # on the device the tensors live on. eval runs on cuda:1 while cuda:0 is
+ # busy training, so this is not optional.
+ torch.cuda.set_device(dev)
+ blob = torch.load(path, map_location="cpu", weights_only=False)
+ cfg = Config.from_json(blob["config"])
+ # The training head chunk (8192 on the V100) makes the 3060's stock head
+ # materialize 1 GB fp32 logit chunks; the chunk only sets memory, not the loss.
+ cfg.optim.lm_head_chunk = min(cfg.optim.lm_head_chunk, 2048)
+ model = Boris2(cfg).to(dev)
+ sd = {k: v.to(torch.float32) for k, v in blob["model"].items()}
+ model.load_state_dict(sd)
+ if model.ngram is not None:
+ vocab = NGramVocab.load(cfg.ngram.vocab_path)
+ got = vocab.fingerprint()
+ want = blob.get("ngram_fingerprint")
+ if want is not None and want not in (got, vocab.legacy_fingerprint()):
+ raise RuntimeError(f"checkpoint expects n-gram vocab {want}, loaded {got}")
+ model.ngram.load_vocab(vocab, dev)
+ model.eval()
+ progress = blob["tokens_seen"] / cfg.train.total_tokens
+ model._eval_theta = cfg.block_size_at(progress)[1]
+ return model, cfg, blob
+
+
+def evaluate(path: Path, dev: str, tasks, out: Path, val_tokens: int, batch: int,
+ no_ngram: bool = False):
+ t0 = time.time()
+ model, cfg, blob = load_export(path, dev)
+ if no_ngram:
+ model.ngram_enabled = False
+ tok = BorisTokenizer.load(Path(cfg.data.root) / "tokenizer")
+ tok_fp = BorisTokenizer.fingerprint(Path(cfg.data.root) / "tokenizer")
+ if tok_fp != cfg.model.tokenizer_sha256:
+ raise RuntimeError(
+ f"tokenizer fingerprint {tok_fp} does not match checkpoint config "
+ f"{cfg.model.tokenizer_sha256}")
+ row = {"checkpoint": path.name, "step": blob["step"],
+ "tokens_seen": blob["tokens_seen"],
+ "tokens_b": blob["tokens_seen"] / 1e9, "wall": time.time(),
+ "ngram": (model.ngram is not None) and not no_ngram}
+
+ for name in tasks:
+ if name not in TASKS:
+ print(f" unknown task {name}"); continue
+ try:
+ items = TASKS[name]()
+ except Exception as e:
+ print(f" {name}: unavailable ({type(e).__name__}: {str(e)[:90]})")
+ continue
+ r = score_batch(model, tok, items, dev, batch_size=batch)
+ # The Open SLM Leaderboard uses length-normalized accuracy for all
+ # five tasks, including ArithMark-3.
+ row[name] = r["acc_norm"]
+ row[name + "_acc"] = r["acc"]
+ row[name + "_acc_norm"] = r["acc_norm"]
+ row[name + "_n"] = r["n"]
+ print(f" {name:<15} acc {r['acc']:.4f} acc_norm {r['acc_norm']:.4f} "
+ f"n={r['n']}")
+
+ # Compile only now: val batches are fixed-shape (8 x 2048), so the regions
+ # compile once; the benchmark batches above vary in length and would
+ # recompile per shape. Measured on the 3060: val 101 s -> 60 s per 2M
+ # tokens, loss equal to 4 decimals.
+ model.maybe_compile()
+ vl = val_loss(model, cfg, dev, val_tokens)
+ if vl is not None:
+ row["val_loss"] = vl
+ row["val_ppl"] = math.exp(min(vl, 20))
+ print(f" {'val':<15} loss {vl:.4f} ppl {math.exp(min(vl,20)):.3f}")
+
+ keys = [k for k in tasks if k in row]
+ if keys:
+ row["mean"] = sum(row[k] for k in keys) / len(keys)
+ print(f" {'MEAN':<15} {row['mean']:.4f}")
+ row["eval_seconds"] = time.time() - t0
+
+ out.parent.mkdir(parents=True, exist_ok=True)
+ with open(out, "a") as f:
+ f.write(json.dumps(row) + "\n")
+ del model
+ torch.cuda.empty_cache()
+ return row
+
+
+def watch(cfg: Config, dev, tasks, out, val_tokens, batch, poll):
+ done = set()
+ if out.exists():
+ for line in out.read_text().splitlines():
+ try:
+ done.add(json.loads(line)["checkpoint"])
+ except Exception:
+ pass
+ print(f" watching {cfg.eval.manifest} ({len(done)} already scored)")
+ while True:
+ pending = []
+ if cfg.eval.manifest.exists():
+ for line in cfg.eval.manifest.read_text().splitlines():
+ try:
+ m = json.loads(line)
+ except Exception:
+ continue
+ p = Path(m["path"])
+ if p.name not in done and p.exists():
+ pending.append(p)
+ for p in sorted(set(pending)):
+ print(f"\n === {p.name} ===")
+ try:
+ evaluate(p, dev, tasks, out, val_tokens, batch)
+ done.add(p.name)
+ except Exception as e:
+ print(f" failed on {p.name}: {type(e).__name__}: {e}")
+ time.sleep(poll)
+
+
+def main():
+ cfg = Config()
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--ckpt", type=Path, default=None)
+ ap.add_argument("--watch", action="store_true")
+ ap.add_argument("--device", default=cfg.eval.device)
+ ap.add_argument("--tasks", default=",".join(cfg.eval.tasks))
+ ap.add_argument("--out", type=Path, default=cfg.eval.out_path)
+ ap.add_argument("--val-tokens", type=int, default=cfg.eval.val_tokens)
+ ap.add_argument("--batch", type=int, default=cfg.eval.batch_size)
+ ap.add_argument("--no-ngram", action="store_true",
+ help="score with the table switched off -- reports how much "
+ "the trained model DEPENDS on it, not what it gained")
+ a = ap.parse_args()
+ tasks = [t.strip() for t in a.tasks.split(",") if t.strip()]
+
+ if a.watch:
+ watch(cfg, a.device, tasks, a.out, a.val_tokens, a.batch, cfg.eval.poll_seconds)
+ elif a.ckpt:
+ evaluate(a.ckpt, a.device, tasks, a.out, a.val_tokens, a.batch,
+ no_ngram=a.no_ngram)
+ else:
+ raise SystemExit("pass --ckpt PATH or --watch")
+
+
+if __name__ == "__main__":
+ main()