From 5d2e4252e695644c34be605b44a14bd80a98a0a6 Mon Sep 17 00:00:00 2001 From: jdj Date: Wed, 18 Nov 2026 14:47:33 -0800 Subject: Boris-2 training code The code, configuration and data filters used to train Boris-2 (144M, 200B tokens, single V100): model, training loop, data preparation, n-gram table, V100 kernels, final stages and the exact run configs. --- verify.py | 1164 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1164 insertions(+) create mode 100644 verify.py (limited to 'verify.py') diff --git a/verify.py b/verify.py new file mode 100644 index 0000000..9418e07 --- /dev/null +++ b/verify.py @@ -0,0 +1,1164 @@ +"""Correctness checks. Run before starting a run, and after touching a kernel. + + python verify.py [--device cuda:0] [--quick] + +The checks that matter most are the isolation ones. Boris-2 packs several +documents into every row, and three separate mechanisms have to keep them apart: +varlen attention, the Canon boundary mask, and the n-gram history mask. A leak +in any of them is invisible in the loss curve -- the model simply learns from +context it will never have at inference. + +Note how check_document_isolation de-zeroes the Canon kernels first. They are +zero-initialised, so at init the convolution contributes exactly nothing and the +test passes no matter how badly it leaks. The 75M run shipped that bug for weeks +behind a test that looked green. +""" +from __future__ import annotations + +import argparse +import math +import os +import sys +import atexit +import shutil +import tempfile +from pathlib import Path + +import numpy as np +import torch +import torch.nn.functional as F + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from config import Config +from model import Boris2, IGNORE, fused_lm_loss, _pre_attn +from ngram import NGramVocab +from optim import Muon, build_optimizers, newton_schulz +from schedule import lr_scale, mixture_at, context_at + +PASS, FAIL = [], [] + + +_TMPDIRS = [] + + +def _tmpdir(): + """mkdtemp that actually cleans up. + + verify.py used bare tempfile.mkdtemp() and never removed anything. /tmp here + is a 16 GB tmpfs -- RAM -- and each run leaves ~500 MB of packer fixtures + behind, so 89 runs had filled it completely. That is not just a disk + problem: it was eating the machine's memory and getting background jobs + OOM-killed. + """ + d = tempfile.mkdtemp() + _TMPDIRS.append(d) + return d + + +@atexit.register +def _cleanup_tmpdirs(): + for d in _TMPDIRS: + shutil.rmtree(d, ignore_errors=True) + + +def check(name, cond, detail=""): + (PASS if cond else FAIL).append(name) + mark = "ok " if cond else "FAIL" + print(f" [{mark}] {name}" + (f" {detail}" if detail else "")) + return cond + + +# --------------------------------------------------------------------------- # +def synth_vocab(cfg, seed=0): + V = cfg.model.vocab_size + rng = np.random.default_rng(seed) + bk = np.unique(rng.integers(0, V * V, size=cfg.ngram.bigram_rows).astype(np.int64)) + tk = np.unique(rng.integers(0, V * V * V, size=cfg.ngram.trigram_rows).astype(np.int64)) + return NGramVocab(V, bk, np.arange(bk.size) + cfg.ngram.bigram_base, + tk, np.arange(tk.size) + cfg.ngram.trigram_base) + + +def synth_batch(cfg, dev, B, T, lens_per_row, seed=0): + g = torch.Generator().manual_seed(seed) + V = cfg.model.vocab_size + x = torch.randint(0, V, (B, T), generator=g).to(dev) + pos = torch.zeros(B, T, dtype=torch.int64) + bnd = torch.zeros(B, T, dtype=torch.bool) + bounds = [] + for b, lens in enumerate(lens_per_row): + p = 0 + for L in lens: + bounds.append(b * T + p) + bnd[b, p] = True + pos[b, p:p + L] = torch.arange(L) + p += L + assert p == T, "row must be exactly full" + bounds.append(B * T) + return (x, pos.to(dev), torch.tensor(bounds, dtype=torch.int64).to(dev), bnd.to(dev)) + + +def build(cfg, dev, compile_regions=False): + cfg.train.compile_regions = compile_regions + m = Boris2(cfg).to(dev) + if m.ngram is not None: + m.ngram.load_vocab(synth_vocab(cfg), dev) + if compile_regions: + m.maybe_compile() + return m + + +# --------------------------------------------------------------------------- # +def check_fused_head(cfg, dev): + """The fused head must equal an ordinary cross-entropy, loss and gradient.""" + torch.manual_seed(0) + N, d, V = 1024, cfg.model.d_model, cfg.model.vocab_size + x = torch.randn(N, d, device=dev, dtype=torch.float16, requires_grad=True) + W = (torch.randn(V, d, device=dev) * 0.02).requires_grad_(True) + t = torch.randint(0, V, (N,), device=dev) + t[::7] = IGNORE + + loss = fused_lm_loss(x, W, t, chunk=256) + loss.backward() + gx, gW = x.grad.clone(), W.grad.clone() + + # Reference in fp32. Comparing against the fp16 unfused head would compare + # two approximations: its grad_logits reach the GEMM at ~3e-8, far below + # what fp16 can represent, so it is the less accurate of the two. + x2 = x.detach().float().clone().requires_grad_(True) + W2 = W.detach().float().clone().requires_grad_(True) + ref = F.cross_entropy(F.linear(x2, W2), t, ignore_index=IGNORE) + ref.backward() + + dl = abs(float(loss) - float(ref)) + rx = (gx.float() - x2.grad).norm() / (x2.grad.norm() + 1e-9) + rw = (gW - W2.grad).norm() / (W2.grad.norm() + 1e-9) + check("fused LM head: loss matches fp32 cross_entropy", dl < 5e-3, f"|d| = {dl:.2e}") + check("fused LM head: dx matches fp32", float(rx) < 5e-3, f"rel-L2 {float(rx):.2e}") + check("fused LM head: dW matches fp32", float(rw) < 5e-3, f"rel-L2 {float(rw):.2e}") + # The point of the fusion: never hold (N, V) logits. Measured against the + # unfused head on the same inputs, at a realistic token count. + def peak_of(fn, N2=8192): + xx = torch.randn(N2, d, device=dev, dtype=torch.float16, requires_grad=True) + WW = (torch.randn(V, d, device=dev) * 0.02).requires_grad_(True) + tt = torch.randint(0, V, (N2,), device=dev) + torch.cuda.synchronize(); torch.cuda.empty_cache() + base = torch.cuda.memory_allocated(dev) + torch.cuda.reset_peak_memory_stats(dev) + fn(xx, WW, tt).backward() + torch.cuda.synchronize() + out = torch.cuda.max_memory_allocated(dev) - base + del xx, WW, tt + torch.cuda.empty_cache() + return out + + fused_peak = peak_of(lambda a, b, c: fused_lm_loss(a, b, c)) + plain_peak = peak_of(lambda a, b, c: F.cross_entropy( + F.linear(a, b.to(a.dtype)).float(), c, ignore_index=IGNORE)) + check("fused LM head: uses far less memory than the unfused head", + fused_peak < plain_peak / 3, + f"{fused_peak/2**20:.0f} MB vs {plain_peak/2**20:.0f} MB " + f"({plain_peak/max(fused_peak,1):.1f}x)") + + +def check_repeated_target_head(cfg, dev): + """Exercise the production Function under autocast, including its rescale.""" + torch.manual_seed(91) + for dtype in (torch.float32, torch.float16): + x = torch.full((1024, 32), 16.0, device=dev, dtype=dtype, requires_grad=True) + w = (torch.randn(256, 32, device=dev) * .01).requires_grad_() + target = torch.zeros(1024, device=dev, dtype=torch.long) + with torch.autocast("cuda", dtype=torch.float16): + loss = fused_lm_loss(x, w, target, chunk=1024) + loss.backward() + xr = x.detach().float().requires_grad_() + wr = w.detach().clone().requires_grad_() + ref = F.cross_entropy(F.linear(xr, wr), target) + ref.backward() + rel = (w.grad - wr.grad).norm() / wr.grad.norm() + check(f"production head repeated-target dW {dtype}", + bool(torch.isfinite(w.grad).all()) and float(rel) < .005, + f"relative error {float(rel):.3g}") + + +def check_canon_causality(cfg, dev): + """A change at position t must never move an output at t' < t.""" + torch.manual_seed(0) + C, K = cfg.model.canon_channels, cfg.model.canon_kernel + B, T = 1, 64 + x = torch.randn(B, T, cfg.model.d_model, device=dev, dtype=torch.float16) + w_qkv = torch.randn(C, cfg.model.d_model, device=dev, dtype=torch.float16) * 0.02 + canon = torch.randn(C, K, device=dev) * 0.5 + m1 = torch.ones(B, T, 1, device=dev, dtype=torch.float16) + m2 = torch.ones(B, T, 1, device=dev, dtype=torch.float16) + hd = cfg.model.head_dim + cos = torch.ones(B, T, 1, hd // 2, device=dev) + sin = torch.zeros(B, T, 1, hd // 2, device=dev) + nw = torch.ones(cfg.model.d_model, device=dev) + args = (nw, 1e-5, w_qkv, canon, m1, m2, cos, sin, cfg.model.n_head, + cfg.model.n_kv_head, hd, True) + q0 = _pre_attn(x, *args)[0] + x2 = x.clone(); x2[0, 32] += 10.0 + q1 = _pre_attn(x2, *args)[0] + check("Canon is strictly causal", torch.equal(q0[0, :32], q1[0, :32])) + moved = int((q0[0, 32:35] != q1[0, 32:35]).any(-1).any(-1).sum()) + check("Canon reaches exactly kernel-1 tokens forward", moved == 3, + f"{moved} of the next 3 positions moved") + + +def check_document_isolation(cfg, dev): + """End-to-end: a change in one document must not move another. + + Canon kernels are randomised first. Zero-initialised, they contribute + nothing and this test would pass on a model that leaks badly. + """ + torch.manual_seed(0) + small = Config.from_json(cfg.to_json()) + small.enforce_budget = False + small.model.n_layer = 6 + m = build(small, dev) + for blk in m.blocks: # de-zero: the whole point + torch.nn.init.normal_(blk.attn.canon, std=0.5) + + B, T = 1, 512 + lens = [[200, 312]] + x, pos, bounds, bnd = synth_batch(small, dev, B, T, lens, seed=3) + with torch.autocast("cuda", dtype=torch.float16): + h0 = m(x, pos, bounds, bnd, None, 10_000.0) + x2 = x.clone(); x2[0, :200] = torch.randint(0, small.model.vocab_size, + (200,), device=dev) + h1 = m(x2, pos, bounds, bnd, None, 10_000.0) + drift = float((h0[0, 200:] - h1[0, 200:]).abs().max()) + check("document isolation: rewriting doc A never moves doc B", drift == 0.0, + f"max logit drift {drift:.3e}") + + same = float((h0[0, :200] - h1[0, :200]).abs().max()) + check("the probe actually perturbs doc A", same > 0.0, f"max change {same:.3e}") + + +def check_attention_matches_loop(cfg, dev): + """Nested varlen attention must equal a per-document causal SDPA.""" + torch.manual_seed(0) + H, KV, D = cfg.model.n_head, cfg.model.n_kv_head, cfg.model.head_dim + lens = [300, 212, 512] + N = sum(lens) + off = torch.tensor([0] + list(np.cumsum(lens)), device=dev) + q = torch.randn(N, H, D, device=dev, dtype=torch.float16) + k = torch.randn(N, H, D, device=dev, dtype=torch.float16) + v = torch.randn(N, H, D, device=dev, dtype=torch.float16) + nj = lambda t: torch.nested.nested_tensor_from_jagged(t, off).transpose(1, 2) + o = F.scaled_dot_product_attention(nj(q), nj(k), nj(v), is_causal=True) + o = o.transpose(1, 2).values() + ref = torch.empty_like(q) + p = 0 + for L in lens: + s = F.scaled_dot_product_attention( + q[p:p+L].transpose(0, 1)[None], k[p:p+L].transpose(0, 1)[None], + v[p:p+L].transpose(0, 1)[None], is_causal=True) + ref[p:p+L] = s[0].transpose(0, 1) + p += L + d = float((o.float() - ref.float()).abs().max()) + check("varlen attention equals a per-document loop", d == 0.0, f"max diff {d:.1e}") + + +def check_volta_kernels(cfg, dev): + """volta_kernels/: the sm_70 attention and LM head, held to the stock paths. + + Runs whenever the device is sm_70, whether or not BORIS_VOLTA_KERNELS is set, + so the kernels are proven before anyone opts in. The bar is the stock path's + own error against an fp64 reference, measured here on the same inputs -- + "same precision" as a number, not an adjective. + """ + if torch.cuda.get_device_capability(dev) != (7, 0): + print(" [skip] volta kernels: device is not sm_70") + return + import model as M + import volta_kernels as VK + from volta_kernels.head import fused_lm_loss as volta_lm_loss + torch.manual_seed(0) + H, D = cfg.model.n_head, cfg.model.head_dim + # Edge lengths around the 64-row tile, a length-1 document, and one long one. + lens = [1, 63, 64, 65, 127, 128, 300, 1300] + N = sum(lens) + off = torch.tensor([0] + list(np.cumsum(lens)), device=dev) + # v as a strided view of a fused qkv, exactly as the model hands it over. + qkv = (torch.randn(N, 3 * H * D, device=dev) * 1.5).half() + q = qkv[:, :H * D].reshape(N, H, D).contiguous().requires_grad_() + k = qkv[:, H * D:2 * H * D].reshape(N, H, D).contiguous().requires_grad_() + vb = qkv.clone().requires_grad_() + v = vb[:, 2 * H * D:].view(N, H, D) + go = torch.randn(N, H, D, device=dev).half() + + def run(fn): + for t in (q, k, vb): + t.grad = None + o = fn() + o.backward(go) + return [o.detach().double(), q.grad.double(), k.grad.double(), + vb.grad[:, 2 * H * D:].reshape(N, H, D).double()] + + def stock(): + nj = lambda t: torch.nested.nested_tensor_from_jagged(t, off).transpose(1, 2) + return F.scaled_dot_product_attention(nj(q), nj(k), nj(v), is_causal=True + ).transpose(1, 2).values() + + qq, kk, vv = (t.detach().double().requires_grad_() for t in (q, k, v)) + outs, p = [], 0 + for L in lens: + sl = slice(p, p + L) + outs.append(F.scaled_dot_product_attention( + qq[sl].transpose(0, 1), kk[sl].transpose(0, 1), vv[sl].transpose(0, 1), + is_causal=True).transpose(0, 1)) + p += L + ref_o = torch.cat(outs) + ref_o.backward(go.double()) + ref = [ref_o.detach(), qq.grad, kk.grad, vv.grad] + S = run(stock) + V = run(lambda: VK.varlen_causal_attention(q, k, v, off)) + rel = lambda a, b: float((a - b).norm() / b.norm()) + for name, i in (("output", 0), ("dq", 1), ("dk", 2), ("dv", 3)): + es, ev = rel(S[i], ref[i]), rel(V[i], ref[i]) + check(f"volta attention: {name} as accurate as memory-efficient SDPA", + ev <= 1.1 * es and math.isfinite(ev), f"rel-L2 vs fp64 {ev:.2e} (stock {es:.2e})") + + # Isolation and causality, exactly: perturb one document / one late token. + def fwd_only(qx, kx, vx): + return VK.varlen_causal_attention(qx, kx, vx, off.clone()).float() + q0, k0, v0 = (t.detach().clone() for t in (q, k, v)) + o0 = fwd_only(q0, k0, v0) + a, b = int(off[6]), int(off[7]) # the 300-token document + q1, k1, v1 = q0.clone(), k0.clone(), v0.clone() + for t in (q1, k1, v1): + t[a:b] = torch.randn_like(t[a:b]) + o1 = fwd_only(q1, k1, v1) + outside = torch.cat([o1[:a] - o0[:a], o1[b:] - o0[b:]]).abs().max().item() + check("volta attention: rewriting one document never moves another", + outside == 0.0 and (o1[a:b] - o0[a:b]).abs().max().item() > 0, + f"max drift outside {outside:.1e}") + t_ = a + 150 + k2, v2 = k0.clone(), v0.clone() + k2[t_] = torch.randn_like(k2[t_]); v2[t_] = torch.randn_like(v2[t_]) + o2 = fwd_only(q0, k2, v2) + before = (o2[a:t_] - o0[a:t_]).abs().max().item() + at = (o2[t_] - o0[t_]).abs().max().item() + check("volta attention: causal (a key never reaches an earlier query)", + before == 0.0 and at > 0, f"drift before {before:.1e}, at {at:.1e}") + + # The per-offsets tile table must not be served stale after an in-place edit. + o3 = off.clone() + _, t_before, _ = VK.tile_table(o3) # doc 0 has 1 token: 1 tile + o3[1] = 65 # in place: now 65 tokens, 2 tiles + cu, tcum, perm = VK.tile_table(o3) + fresh = (o3[1:] - o3[:-1] + 63) // 64 # what the table must now say + check("volta attention: tile table follows in-place edits of the offsets", + int(cu[1]) == 65 and int(tcum[-1]) == int(fresh.clamp_min(0).sum()) + and int(fresh[int(perm[0])]) == int(fresh.max())) + lens_ = (cu[1:] - cu[:-1] + 63) // 64 + check("volta attention: tile table launches the longest documents first", + bool((lens_[perm.long()][:-1] >= lens_[perm.long()][1:]).all()) + and sorted(perm.tolist()) == list(range(len(lens)))) + + # LM head: the one-pass Triton softmax against the production head. + torch.manual_seed(1) + Vv, d = cfg.model.vocab_size, cfg.model.d_model + Nn = 2048 + W = (torch.randn(Vv, d, device=dev) * 0.05) + x = torch.randn(Nn, d, device=dev) * 3.0 + y = torch.randint(0, Vv, (Nn,), device=dev) + y[::5] = IGNORE + + def head(fn, xx, yy): + xx = xx.clone().requires_grad_(); ww = W.clone().requires_grad_() + with torch.autocast("cuda", dtype=torch.float16): + loss = fn(xx, ww, yy, 1024) + loss.backward() + return loss.detach().double(), xx.grad.double(), ww.grad.double() + + was = M.VOLTA_KERNELS + M.VOLTA_KERNELS = False + try: + P = head(M.fused_lm_loss, x, y) + finally: + M.VOLTA_KERNELS = was + Q = head(volta_lm_loss, x, y) + xr = x.double().requires_grad_(); wr = W.double().requires_grad_() + lr_ = F.cross_entropy(xr @ wr.T, y, ignore_index=IGNORE) + lr_.backward() + check("volta head: loss equals the production head", + abs(float(Q[0] - P[0])) <= 1e-6 * abs(float(P[0])), f"{float(Q[0]):.7f} vs {float(P[0]):.7f}") + for name, i, r in (("dx", 1, xr.grad), ("dW", 2, wr.grad)): + ep, eq = rel(P[i], r), rel(Q[i], r) + check(f"volta head: {name} as accurate as the production head", + eq <= 1.02 * ep, f"rel-L2 vs fp64 {eq:.3e} (production {ep:.3e})") + # The dW overflow guard survives: one token repeated through a whole chunk. + xs = x[:1].repeat(1024, 1) * 5.0 + ys = y.clone()[:1024]; ys[:] = 7 + _, dxs, dws = head(volta_lm_loss, xs, ys) + check("volta head: a chunk of one repeated token stays finite", + bool(torch.isfinite(dws).all() and torch.isfinite(dxs).all())) + + # Stage two (80% of the run) packs documents up to 8192 tokens. fp64 would + # need a 4 GB score matrix, so the reference is fp32 memory-efficient SDPA. + torch.manual_seed(3) + lens8 = [8192, 3000, 5000] + N8 = sum(lens8) + off8 = torch.tensor([0] + list(np.cumsum(lens8)), device=dev) + q8, k8, v8, g8 = (torch.randn(N8, H, D, device=dev).half() for _ in range(4)) + + def attn_grads(fn, dtype): + # clone: for fp16, .to() returns the SAME tensor, and the two runs would + # accumulate into each other's .grad + ts = [t.detach().to(dtype).clone().requires_grad_() for t in (q8, k8, v8)] + o = fn(*ts) + o.backward(g8.to(dtype)) + return [o.detach().double()] + [t.grad.double() for t in ts] + + def nested(a, b, c): + nj = lambda t: torch.nested.nested_tensor_from_jagged(t, off8).transpose(1, 2) + return F.scaled_dot_product_attention(nj(a), nj(b), nj(c), is_causal=True + ).transpose(1, 2).values() + ref8 = attn_grads(nested, torch.float32) + S8 = attn_grads(nested, torch.float16) + V8 = attn_grads(lambda a, b, c: VK.varlen_causal_attention(a, b, c, off8), torch.float16) + worst = max(rel(V8[i], ref8[i]) / rel(S8[i], ref8[i]) for i in range(4)) + check("volta attention: an 8192-token document as accurate as stock SDPA", + worst <= 1.1, f"worst error ratio vs stock {worst:.3f} (o, dq, dk, dv vs fp32)") + + # A head call whose row count is not a multiple of the 1024-row chunk. + xp, yp = x[:1024 + 300], y[:1024 + 300] + M.VOLTA_KERNELS = False + try: + Pp = head(M.fused_lm_loss, xp, yp) + finally: + M.VOLTA_KERNELS = was + Qp = head(volta_lm_loss, xp, yp) + check("volta head: a partial final chunk matches the production head", + abs(float(Qp[0] - Pp[0])) <= 1e-6 * abs(float(Pp[0])) + and rel(Qp[1], Pp[1]) < 1e-3 and rel(Qp[2], Pp[2]) < 1e-3, + f"dx {rel(Qp[1], Pp[1]):.1e}, dW {rel(Qp[2], Pp[2]):.1e} apart") + + # Fused Canon + RoPE: precision against fp64 next to the eager region, and + # document isolation with non-zero Canon weights (zero-init would hide a leak). + from volta_kernels.canon_rope_cuda import canon_rope # what the model uses + torch.manual_seed(4) + Bc, Tc = 2, 256 + bndc = torch.zeros(Bc, Tc, dtype=torch.bool, device=dev) + bndc[:, 0] = True; bndc[0, 100] = True; bndc[1, 7] = True; bndc[1, 8] = True + b2c = bndc | F.pad(bndc, (1, 0))[:, :-1] + m1c = (~bndc).float()[:, :, None]; m2c = (~b2c).float()[:, :, None] + posc = torch.zeros(Bc, Tc, dtype=torch.long, device=dev) + for b in range(Bc): + p0 = 0 + for t_ in range(Tc): + if bndc[b, t_]: p0 = t_ + posc[b, t_] = t_ - p0 + invc = 1.0 / (10000.0 ** (torch.arange(0, 64, 2, device=dev).float() / 64)) + fc = torch.outer(torch.arange(Tc, device=dev).float(), invc) + cosc, sinc = fc.cos()[posc].unsqueeze(2), fc.sin()[posc].unsqueeze(2) + qkvc = (torch.randn(Bc, Tc, 1536, device=dev) * 2).half() + cwc = torch.randn(1536, 3, device=dev) * 0.3 + gsc = [torch.randn(Bc, Tc, 8, 64, device=dev).half() for _ in range(3)] + + def region(a, w_, dt): + ww = w_.to(torch.float16).to(dt) + s1 = F.pad(a, (0, 0, 1, 0))[:, :-1] * m1c.to(dt) + s2 = F.pad(a, (0, 0, 2, 0))[:, :-2] * m2c.to(dt) + y_ = a + a * ww[:, 0] + s1 * ww[:, 1] + s2 * ww[:, 2] + c_, s_ = cosc.to(dt), sinc.to(dt) + def rope(t_): + a_, b_ = t_.chunk(2, dim=-1) + return torch.cat([a_ * c_ - b_ * s_, a_ * s_ + b_ * c_], -1) + v_ = lambda i: y_[..., 512 * i:512 * (i + 1)].reshape(Bc, Tc, 8, 64) + return rope(v_(0)), rope(v_(1)), v_(2) + + def cr_run(fn, dt): + a = qkvc.detach().to(dt).clone().requires_grad_() + w_ = cwc.detach().clone().requires_grad_() + outs = fn(a, w_) + torch.autograd.backward(outs, [g.to(outs[0].dtype) for g in gsc]) + return [o.detach().double() for o in outs] + [a.grad.double()] + ref_c = cr_run(lambda a, w_: region(a, w_, torch.float64), torch.float64) + eag_c = cr_run(lambda a, w_: tuple(o.to(torch.float16) for o in + region(a.float(), w_, torch.float32)), torch.float16) + fus_c = cr_run(lambda a, w_: canon_rope(a, w_, m1c, m2c, cosc, sinc), torch.float16) + worst = max(rel(fus_c[i], ref_c[i]) / rel(eag_c[i], ref_c[i]) for i in range(4)) + check("volta canon+rope: q, k, v, dqkv as accurate as the eager region", + worst <= 1.1, f"worst error ratio {worst:.3f}") + q0_, k0_, v0_ = canon_rope(qkvc, cwc, m1c, m2c, cosc, sinc) + pert = qkvc.clone(); pert[0, :100] = torch.randn_like(pert[0, :100]).half() + q1_, k1_, v1_ = canon_rope(pert, cwc, m1c, m2c, cosc, sinc) + leak = max(float((a_[0, 100:].float() - b_[0, 100:].float()).abs().max()) + for a_, b_ in ((q0_, q1_), (k0_, k1_), (v0_, v1_))) + moved = float((v1_[0, :100].float() - v0_[0, :100].float()).abs().max()) + check("volta canon+rope: a document boundary stops the Canon taps exactly", + leak == 0.0 and moved > 0, f"drift past boundary {leak:.1e}") + + # Whole model, both paths, against an fp32 (no autocast) reference, at the + # run's loss scale. Canon de-zeroed so every mechanism is live. + small = Config.from_json(cfg.to_json()) + small.enforce_budget = False + small.model.n_layer = 4 + torch.manual_seed(2) + m = build(small, dev) + for blk in m.blocks: + torch.nn.init.normal_(blk.attn.canon, std=0.1) + lens2 = [[700, 64, 1, 1283], [2048]] + xb, pb, ob, bb = synth_batch(small, dev, 2, 2048, lens2, seed=5) + yb = torch.roll(xb, -1, 1) + + def grads(volta, amp): + M.VOLTA_KERNELS = volta + try: + for prm in m.parameters(): + prm.grad = None + with torch.autocast("cuda", dtype=torch.float16, enabled=amp): + loss = m(xb, pb, ob.clone(), bb, yb, 10_000.0) + (loss * 32768.0).backward() + finally: + M.VOLTA_KERNELS = was + return {n: prm.grad.double() / 32768.0 for n, prm in m.named_parameters() + if prm.grad is not None} + gr, gs, gv = grads(False, False), grads(False, True), grads(True, True) + # The production path with checkpointing OFF merges each block's FFN half + # with the next block's norm+QKV into one region (model._forward_merged). + small.train.grad_checkpoint = False + gmerged = grads(True, True) + m._no_merge = True + gunmerged = grads(True, True) + m._no_merge = False + small.train.grad_checkpoint = True + em = sorted(rel(gmerged[n], gr[n]) for n in gr) + eu = sorted(rel(gunmerged[n], gr[n]) for n in gr) + check("volta merged regions: gradients as close to fp32 as per-block regions", + em[len(em) // 2] <= 1.05 * eu[len(eu) // 2] and max(em) <= 1.05 * max(eu), + f"median {em[len(em) // 2]:.3e} (per-block {eu[len(eu) // 2]:.3e})") + es = sorted(rel(gs[n], gr[n]) for n in gr) + ev = sorted(rel(gv[n], gr[n]) for n in gr) + med_s, med_v = es[len(es) // 2], ev[len(ev) // 2] + check("volta model: gradients as close to fp32 as the stock fp16 path", + med_v <= 1.1 * med_s and max(ev) <= 1.25 * max(es), + f"median rel-L2 {med_v:.3e} (stock {med_s:.3e}), worst {max(ev):.3e} (stock {max(es):.3e})") + + +def check_ngram_isolation(cfg, dev): + """The n-gram key must not span a document boundary.""" + m = build(cfg, dev) + B, T = 1, 32 + x = torch.arange(T, device=dev)[None] % cfg.model.vocab_size + bnd = torch.zeros(B, T, dtype=torch.bool, device=dev) + bnd[0, 0] = True; bnd[0, 16] = True + out = m.ngram(x, bnd, torch.float32) + x2 = x.clone(); x2[0, 14:16] = (x2[0, 14:16] + 7) % cfg.model.vocab_size + out2 = m.ngram(x2, bnd, torch.float32) + d = float((out[0, 16:] - out2[0, 16:]).abs().max()) + check("n-gram history stops at a document boundary", d == 0.0, f"max diff {d:.1e}") + + +def check_muon(cfg, dev): + """Batched Newton-Schulz must agree with the unbatched one, and orthogonalise.""" + torch.manual_seed(0) + G = torch.randn(4, 512, 2368, device=dev) + batched = newton_schulz(G, 5) + single = torch.stack([newton_schulz(G[i:i+1], 5)[0] for i in range(4)]) + r = float((batched - single).norm() / single.norm()) + check("batched Newton-Schulz equals the unbatched one", r < 1e-5, + f"rel-L2 {r:.2e}") + s = torch.linalg.svdvals(batched[0].float()) + check("Newton-Schulz orthogonalises (singular values near 1)", + 0.7 < float(s.mean()) < 1.3, f"mean sv {float(s.mean()):.3f}, " + f"min {float(s.min()):.3f}, max {float(s.max()):.3f}") + + # Fusing q/k/v into one Parameter must not change the update. This is the + # whole justification for storing them fused, so it is asserted, not assumed. + torch.manual_seed(1) + fused = torch.nn.Parameter(torch.randn(768, 512, device=dev)) + fused._boris_role = "matmul" + fused._boris_blocks = ((0, 512), (512, 640), (640, 768)) + grad = torch.randn(768, 512, device=dev) + fused.grad = grad.clone() + parts = [] + for a, b in fused._boris_blocks: + q = torch.nn.Parameter(fused.data[a:b].clone()) + q._boris_role = "matmul" + q.grad = grad[a:b].clone() + parts.append(q) + kw = dict(lr=0.02, momentum=0.95, nesterov=True, ns_steps=5, + weight_decay=0.1, batched=True) + Muon([fused], **kw).step() + Muon(parts, **kw).step() + sep = torch.cat([q.data for q in parts], 0) + d = float((fused.data - sep).abs().max()) + check("a block-fused Parameter updates exactly like separate ones", d == 0.0, + f"max diff {d:.1e}") + + # A (768, 3) Canon kernel must never reach Muon. + m = build(cfg, dev) + muon, adamw = build_optimizers(m, cfg, log=lambda *a: None) + canon_ids = {id(b.attn.canon) for b in m.blocks} + in_muon = any(id(p) in canon_ids for g in muon.param_groups for p in g["params"]) + check("Canon kernels are in AdamW, not Muon", not in_muon) + tbl = id(m.ngram.table) + in_muon_tbl = any(id(p) == tbl for g in muon.param_groups for p in g["params"]) + check("the n-gram table is in AdamW, not Muon", not in_muon_tbl) + + +def check_ngram_ablation(cfg, dev): + """Both arms must build, and the runtime toggle must actually do something.""" + off = Config.from_json(cfg.to_json()) + off.model.use_ngram = False + b_on, b_off = cfg.param_budget()["total"], off.param_budget()["total"] + check("use_ngram leaves the transformer untouched", + cfg.param_budget()["trunk"] == off.param_budget()["trunk"] + and b_on - b_off == cfg.ngram.table_params(cfg.model.d_model), + f"{b_on:,} with table, {b_off:,} without, same {b_off - cfg.model.embed_params():,} trunk") + off.model.n_layer = 4 + off.enforce_budget = False + m = build(off, dev) + check("use_ngram=False builds no table and no projection", + m.ngram is None and not any("ngram" in n for n, _ in m.named_parameters())) + build_optimizers(m, off, log=lambda *a: None) + del m + torch.cuda.empty_cache() + + on = Config.from_json(cfg.to_json()) + on.model.n_layer = 4 + on.enforce_budget = False + m = build(on, dev) + x, pos, bounds, bnd = synth_batch(on, dev, 1, 256, [[128, 128]], seed=5) + with torch.autocast("cuda", dtype=torch.float16): + a = m(x, pos, bounds, bnd, None, 10_000.0) + m.ngram_enabled = False + b = m(x, pos, bounds, bnd, None, 10_000.0) + m.ngram_enabled = True + c = m(x, pos, bounds, bnd, None, 10_000.0) + check("the runtime toggle changes the output", not torch.equal(a, b), + f"max diff {float((a.float()-b.float()).abs().max()):.3e}") + check("the runtime toggle is reversible", torch.equal(a, c)) + del m + torch.cuda.empty_cache() + + +def check_schedule(cfg): + s = cfg.schedule + w, st = s.warmup_frac, s.stable_frac + ok = (abs(lr_scale(0.0, cfg)) < 1e-9 and abs(lr_scale(w, cfg) - 1) < 1e-9 + and abs(lr_scale((w + st) / 2, cfg) - 1) < 1e-9 + and abs(lr_scale(1.0, cfg) - s.final_lr_frac) < 1e-9) + check("WSD: 0 at start, 1 across the stable phase, floor at the end", ok) + mono = all(lr_scale(p, cfg) >= lr_scale(p + 0.005, cfg) - 1e-12 + for p in np.arange(st, 1.0, 0.005)) + check("WSD: the decay never goes back up", mono) + + for p in (0.0, 0.5, 0.86, 0.94, 1.0): + w_ = mixture_at(p, cfg) + assert abs(sum(w_.values()) - 1) < 1e-9 + end = mixture_at(1.0, cfg) + # 2026-09-17, Boris-2-0917. The four checks here used to assert the old + # hand-weighted decay mixture (raw web to zero, curated > 0.7). That corpus + # is gone. ClimbMix's weights are a SEARCH RESULT over the whole mixture, so + # the property worth protecting is the opposite one: that we do not + # re-weight it by phase and quietly undo the optimisation we adopted it for. + flat = all(abs(mixture_at(p, cfg)["climbmix"] - end["climbmix"]) < 1e-9 + for p in (0.0, 0.25, 0.5, 0.87, 0.95)) + check("ClimbMix weights are flat across the run, not re-weighted by phase", + flat, f"climbmix held at {end['climbmix']:.2f} throughout") + check("ClimbMix is the overwhelming majority of the mixture", + end["climbmix"] >= 0.90, f"{end['climbmix']:.2f} of the mixture") + # finemath is the only source carried over from 0907. ArithMark-3 was the + # one benchmark 0907 was competitive on (37.40 vs GPT-X2.5's 38.40) and + # ClimbMix was optimised against PIQA/ARC-E/HellaSwag, not arithmetic. + check("finemath is retained, so ArithMark-3 is not abandoned", + 0.02 <= end.get("finemath", 0.0) <= 0.10, + f"{end.get('finemath', 0.0):.2f} of the mixture") + check("the mixture names only sources that exist on disk", + set(cfg.data.sources) == set(end), + f"{sorted(cfg.data.sources)}") + + b0 = context_at(0.0, cfg) + b1 = context_at(0.99, cfg) + check("context extends later in the run", b1[0] > b0[0], + f"{b0[0]} -> {b1[0]} at theta {b0[1]:,.0f} -> {b1[1]:,.0f}") + check("the step size is identical across context stages", + b0[2] * b0[0] * b0[3] == b1[2] * b1[0] * b1[3] == cfg.train.tokens_per_step, + f"{b0[2]}x{b0[0]}x{b0[3]} = {b0[2]*b0[0]*b0[3]:,}") + + +def check_packer(cfg, dev): + """Targets are next-token within a document, never across one.""" + from data import VarlenPacker, SourceStore + d = Path(_tmpdir()) / "src" + d.mkdir(parents=True) + rng = np.random.default_rng(0) + lens = rng.integers(20, 400, size=400) + toks = rng.integers(0, cfg.model.vocab_size, size=int(lens.sum())).astype(np.uint16) + toks.tofile(d / "tokens.bin") + np.concatenate([[0], np.cumsum(lens)]).astype(np.uint64).tofile(d / "docs.idx") + st = SourceStore.open(d.parent, "src") + cfg2 = Config.from_json(cfg.to_json()) + cfg2.data.sources = ("src",) + cfg2.curriculum.enabled = False + p = VarlenPacker(cfg2, {"src": st}, 7) + xs, ys, pos, bnd, bounds, stats = p.batch(2, 256, 0.0) + + ok_bounds = (bounds[0] == 0 and bounds[-1] == 2 * 256 + and np.all(np.diff(bounds) > 0)) + check("packer: offsets are increasing and cover the batch exactly", ok_bounds) + flat_b = np.zeros(2 * 256, dtype=bool) + flat_b[bounds[:-1]] = True + check("packer: boundary mask agrees with the offsets", + np.array_equal(flat_b, bnd.reshape(-1))) + fx, fy = xs.reshape(-1), ys.reshape(-1) + bad = 0 + for i in range(len(bounds) - 1): + a, b_ = bounds[i], bounds[i + 1] + if b_ - a > 1 and not np.array_equal(fy[a:b_ - 1], fx[a + 1:b_]): + bad += 1 + if fy[b_ - 1] != IGNORE and i + 1 < len(bounds) - 1: + # only legal when the document was carried into the next row + pass + check("packer: targets are next-token inside each document", bad == 0, + f"{bad} documents mismatched") + check("packer: positions restart at every document", + all(pos.reshape(-1)[bounds[i]] == 0 + for i in range(len(bounds) - 1))) + + # The RoPE table is exactly max_block_size long and is indexed by pos, so a + # position past its end is an out-of-bounds gather -- a device-side assert + # thousands of steps in, reported asynchronously at whatever kernel runs + # next. Carried chunks of a long document used to keep counting, and with + # max_doc_tokens at 65,536 they ran off the end. Sample where long-document + # pressure is highest, at both context stages. + lim = cfg.model.max_block_size + worst, worst_at = 0, None + for prog, blk in ((0.0, 256), (0.5, 256), (0.85, 512), (0.95, 512)): + for _ in range(12): + _x, _y, po, _b, bo, _st = p.batch(4, blk, prog) + m = int(po.max()) + if m > worst: + worst, worst_at = m, (prog, blk) + if not all(po.reshape(-1)[bo[i]] == 0 for i in range(len(bo) - 1)): + worst, worst_at = lim + 1, (prog, blk) # force a failure + check("packer: positions stay inside the RoPE table", worst < lim, + f"max pos {worst:,} of {lim:,}" + + (f" at progress {worst_at[0]}, block {worst_at[1]}" if worst_at else "")) + + # Resumability: the same state must replay the same batch. + s = p.state_dict() + a1 = p.batch(2, 256, 0.0)[0] + p.load_state_dict(s) + a2 = p.batch(2, 256, 0.0)[0] + check("packer: a restored state replays the identical batch", + np.array_equal(a1, a2)) + + from data import Prefetcher, validate_tokenization, open_validation + pref = Prefetcher(p, cfg2, dev) + pref.configure(2, 256, 0.0) + pref.start() + try: + pref.next(2, 256) + committed = pref.state_dict() + reference = VarlenPacker(cfg2, {"src": st}, 7) + reference.load_state_dict(committed) + expected = reference.batch(2, 512)[0] + pref.configure(2, 512, 0.0) + actual = pref.next(2, 512)[0].cpu().numpy() + check("prefetch: shape transition replays committed state", + np.array_equal(actual, expected)) + finally: + pref.stop() + + def broken_batch(*args): + raise ValueError("deliberate producer failure") + p.batch = broken_batch + pref = Prefetcher(p, cfg2, dev) + pref.configure(2, 256, 0.0) + pref.start() + propagated = False + try: + pref.next(2, 256) + except RuntimeError as exc: + propagated = isinstance(exc.__cause__, ValueError) + finally: + pref.stop() + check("prefetch: producer failure reaches training instead of hanging", propagated) + rejected = False + try: + validate_tokenization(d, cfg) + except ValueError: + rejected = True + check("data: rejects shards without matching tokenizer provenance", rejected) + check("validation: current tokenizer provenance verified", open_validation(cfg).n_tokens > 0) + + +def check_multistore(cfg): + """Sharded sources must stitch back into one correctly indexed source. + + A source too large for one prepare process is split across workers by data + file range; data.py presents the shard directories as one logical source + rather than concatenating 200GB on disk. If the global-to-shard index + mapping is wrong, training silently reads the wrong documents. + """ + import tempfile + from data import MultiStore, SourceStore + root = Path(_tmpdir()) + rng = np.random.default_rng(0) + truth, parts = [], [] + for sh in range(3): + d = root / f"src_s{sh}"; d.mkdir() + lens = rng.integers(5, 50, size=7) + docs = [rng.integers(1, 1000, size=L).astype(np.uint16) for L in lens] + truth += docs + np.concatenate(docs).tofile(d / "tokens.bin") + np.concatenate([[0], np.cumsum(lens)]).astype(np.uint64).tofile(d / "docs.idx") + parts.append(SourceStore("src", d / "tokens.bin", d / "docs.idx")) + m = MultiStore("src", parts) + check("sharded source: document count and token count add up", + m.n_docs == len(truth) and m.n_tokens == sum(len(t) for t in truth)) + check("sharded source: every global index maps to the right document", + all(np.array_equal(m.doc(i), truth[i]) for i in range(m.n_docs))) + exp = np.flatnonzero(np.array([len(t) for t in truth]) >= 30) + check("sharded source: long-document selection spans shards", + np.array_equal(m.long_docs(30), exp)) + + +def check_checkpoint(cfg, dev): + import checkpoint as C + small = Config.from_json(cfg.to_json()) + small.enforce_budget = False + small.model.n_layer = 4 + m = build(small, dev) + muon, adamw = build_optimizers(m, small, log=lambda *a: None) + for p in m.parameters(): + p.grad = torch.randn_like(p) + muon.step(); adamw.step() + scaler = torch.amp.GradScaler("cuda") + tmp = Path(_tmpdir()) / "c.pt" + C.save_full(tmp, model=m, muon=muon, adamw=adamw, scaler=scaler, packer=None, + step=5, tokens_seen=123, cfg=small, ngram_fingerprint="deadbeef", + background=False) + before = {k: v.clone() for k, v in m.state_dict().items()} + for p in m.parameters(): + p.data.add_(torch.randn_like(p)) + blob = C.load_full(tmp, model=m, muon=muon, adamw=adamw, scaler=scaler, + map_location=dev, expect_fingerprint=small.fingerprint(), + expect_ngram="deadbeef") + same = all(torch.equal(before[k].to(dev), v) for k, v in m.state_dict().items()) + check("checkpoint: weights round-trip exactly", same) + check("checkpoint: step and token count survive", + blob["step"] == 5 and blob["tokens_seen"] == 123) + mom = next(iter(muon.state.values())).get("momentum_buffer") + check("checkpoint: Muon momentum buffers are restored", mom is not None) + + # The packer round-trip, on the real device. This check used to pass + # packer=None, so the resume path was never actually exercised: the first + # real --resume died on it. map_location= moves the packer's saved RNG + # state onto the GPU, and a CPU Generator's set_state rejects a CUDA tensor. + import data as D + srcs = D.open_sources(cfg, log=lambda *a, **k: None) + pk = D.VarlenPacker(cfg, srcs, seed=0) + for _ in range(3): + pk.batch(2, 256, 0.0) + tmp2 = Path(_tmpdir()) / "p.pt" + C.save_full(tmp2, model=m, muon=muon, adamw=adamw, scaler=scaler, packer=pk, + step=7, tokens_seen=99, cfg=small, ngram_fingerprint="deadbeef", + background=False) + want = pk.batch(2, 256, 0.0)[0] + pk2 = D.VarlenPacker(cfg, srcs, seed=0) + ok_resume, err = True, "" + try: + C.load_full(tmp2, model=m, muon=muon, adamw=adamw, scaler=scaler, + packer=pk2, map_location=dev, + expect_fingerprint=small.fingerprint(), + expect_ngram="deadbeef") + except Exception as e: + ok_resume, err = False, f"{type(e).__name__}: {e}" + check("checkpoint: the packer resumes from a GPU map_location", + ok_resume, err) + if ok_resume: + got = pk2.batch(2, 256, 0.0)[0] + check("checkpoint: a resumed packer replays the identical next batch", + np.array_equal(want, got), + "same tokens" if np.array_equal(want, got) else "DIVERGED") + try: + C.load_full(tmp, expect_fingerprint="0000000000000000") + check("checkpoint: a config change is refused, not silently accepted", False) + except SystemExit: + check("checkpoint: a config change is refused, not silently accepted", True) + + +def check_tensor_cores(cfg, dev): + """Prove fp16 autocast reaches a tensor-core kernel rather than sgemm.""" + from torch.profiler import ProfilerActivity, profile + a = torch.randn(4096, 512, device=dev) + b = torch.randn(512, 2368, device=dev) + with profile(activities=[ProfilerActivity.CUDA]) as pr: + with torch.autocast("cuda", dtype=torch.float16): + for _ in range(3): + (a @ b).sum() + torch.cuda.synchronize() + names = " ".join(e.key for e in pr.key_averages()).lower() + tc = any(s in names for s in ("h884", "884gemm", "hgemm", "s16816", "16816", + "tensor", "cutlass")) + check("fp16 GEMMs dispatch to a tensor-core kernel", tc, + [e.key for e in pr.key_averages() if "gemm" in e.key.lower()][:1]) + + +def check_down_proj_headroom(cfg, dev): + """The FFN down projection's fp16 output range -- failure #2, step 4,520. + + Channel 228 of the residual is a massive activation, 173x the RMS (78,115 + against 451 at block 16). The FFN writes into it, so this GEMM's fp16 OUTPUT + crossed 65,504 and became inf, which norm_f turned into NaN. Inputs and + weights were all finite; only the range ran out. DOWN_SHRINK pre-scales + w_down by a power of two, which is exact and keeps the GEMM on tensor cores. + """ + import model as M + peak = 65_504.0 / M.DOWN_SHRINK # what the output may now reach + check("down proj: DOWN_SHRINK is a power of two (so the scaling is exact)", + M.DOWN_SHRINK > 0 and + abs(math.log2(M.DOWN_SHRINK) - round(math.log2(M.DOWN_SHRINK))) < 1e-12, + f"{M.DOWN_SHRINK} = 2^{round(math.log2(M.DOWN_SHRINK))}") + check("down proj: headroom covers the measured step-4,520 peak", + peak > 4 * 65_504.0, + f"overflows at {peak:,.0f} now, was 65,504 ({1/M.DOWN_SHRINK:.0f}x)") + + # the scaling must not change the result: same GEMM, both ways, in fp32 + torch.manual_seed(0) + x = (torch.randn(4096, 2368, device=dev, dtype=torch.float16) * 40) + w = (torch.randn(512, 2368, device=dev, dtype=torch.float16) * 0.5) + plain = F.linear(x.float(), w.float()) + scaled = F.linear(x, w * M.DOWN_SHRINK).float() * (1.0 / M.DOWN_SHRINK) + rel = float((scaled - plain).norm() / plain.norm()) + check("down proj: pre-scaling the weight matches the unscaled GEMM", + rel < 2e-3, f"rel-L2 {rel:.2e}") + + # And it must rescue the real failure shape: an output that lands just past + # 65,504, the way step 4,520's did -- not an absurd one. Headroom is 8x, not + # infinite, so the test is calibrated to the measured overshoot. + torch.manual_seed(1) + big = torch.randn(2048, 2368, device=dev, dtype=torch.float16) + wb = torch.randn(512, 2368, device=dev, dtype=torch.float16) + ref = F.linear(big.float(), wb.float()).abs().amax() + big = (big * (1.6 * 65_504.0 / float(ref)) ** 0.5) # target ~1.6x over + wb = (wb * (1.6 * 65_504.0 / float(ref)) ** 0.5) + true_peak = float(F.linear(big.float(), wb.float()).abs().amax()) + unscaled_bad = not torch.isfinite(F.linear(big, wb)).all() + scaled = F.linear(big, wb * M.DOWN_SHRINK).float() * (1.0 / M.DOWN_SHRINK) + scaled_ok = bool(torch.isfinite(scaled).all()) + check("down proj: an output just past 65,504 overflows unscaled, survives scaled", + bool(unscaled_bad) and scaled_ok, + f"true peak {true_peak:,.0f}: unscaled overflows={bool(unscaled_bad)}, " + f"scaled finite={scaled_ok}") + + +def check_residual_range(cfg, dev): + """The overflow that killed two runs, reproduced from the measured value. + + The residual stream grows with depth. At step 3,456 -- 1% into the budget -- + it reached 65,617 at block 25 against fp16's maximum of 65,504. It saturates + to inf, and _rmsnorm's rsqrt(inf) = 0 turns that into inf * 0 = NaN. No loss + scale can help: the overflow is in the forward pass, so GradScaler halves its + scale forever against a cause halving cannot touch. + """ + from model import _rmsnorm + peak = 65_617.3 # measured, block 25, step 3,456 + w = torch.ones(512, device=dev) + x = torch.full((4, 512), peak, device=dev) + + h16 = x.to(torch.float16) + check("residual: the measured peak really does overflow fp16", + not torch.isfinite(h16).all(), + f"{peak:,.0f} vs fp16 max 65,504") + out16 = _rmsnorm(h16, w, cfg.model.norm_eps) + check("residual: an overflowed fp16 residual makes RMSNorm produce NaN", + bool(torch.isnan(out16).any()), + "rsqrt(inf)=0 then inf*0=nan -- the exact failure") + + out32 = _rmsnorm(x, w, cfg.model.norm_eps) + check("residual: the same value in fp32 stays finite", + bool(torch.isfinite(out32).all()), + f"max |out| {float(out32.abs().amax()):.3f}") + check("residual: fp16_residual is off", + cfg.train.fp16_residual is False, + "fp32 residual stream (costs ~7.6% throughput)") + + +def check_loss_scale_floor(cfg, dev): + """The scale collapse that ended the step-3,810 run, reproduced and fenced. + + GradScaler has no floor. Once the scale falls below ~1e-30, unscale_ computes + grad/scale, overflows fp32 to inf, sets found_inf, and halves the scale + again -- self-reinforcing, no exit, every step skipped from then on. The run + died that way with its weights still perfectly finite. This drives a scaler + into that spiral and asserts the clamp in train.py stops it. + """ + w = torch.zeros(4, device=dev, requires_grad=True) + sc = torch.amp.GradScaler("cuda", init_scale=cfg.train.init_scale) + opt = torch.optim.SGD([w], lr=0.0) + + # Unclamped: 80 consecutive overflows. + bare = torch.amp.GradScaler("cuda", init_scale=cfg.train.init_scale) + for _ in range(80): + bare.scale(torch.zeros((), device=dev)) # lazy-inits _scale + w.grad = torch.full_like(w, float("inf")) + bare.unscale_(opt); bare.step(opt); bare.update() + collapsed = bare.get_scale() + + floored_steps = 0 + for _ in range(80): + sc.scale(torch.zeros((), device=dev)) + w.grad = torch.full_like(w, float("inf")) + sc.unscale_(opt); sc.step(opt); sc.update() + if sc.get_scale() < cfg.train.min_loss_scale: + sc.update(cfg.train.min_loss_scale); floored_steps += 1 + held = sc.get_scale() + + check("loss scale: unclamped GradScaler does collapse (the bug)", + collapsed < 1e-12, + f"80 overflows drive it to {collapsed:.3g}") + check("loss scale: the clamp holds it at the floor", + held == cfg.train.min_loss_scale and math.isfinite(held), + f"held at {held:g}, floored on {floored_steps} steps") + # And the floor must be high enough that unscale_ cannot overflow fp32. + safe = cfg.train.min_loss_scale > 1e-30 + check("loss scale: the floor is above the fp32 unscale_ overflow point", + safe, f"floor {cfg.train.min_loss_scale:g} vs ~1e-30") + check("loss scale: the ceiling is at or below 2^15", + cfg.train.max_loss_scale <= 2.0 ** 15, + f"max {cfg.train.max_loss_scale:,.0f}") + + +def check_head_range(cfg, dev): + """The fused head's dW GEMM must not overflow fp16 on realistic text. + + g = S*(p - onehot): the true token carries -S, and dW sums it over every + token in the chunk with the same target. Synthetic benchmark data is + uniform over the vocabulary and never stacks a target often enough to + matter; real text does. At GRAD_SCALE 4096 this saturated to Inf on step 1, + Inf-Inf put NaN in the tied embedding gradient, and the run died on step 2. + Assert on a batch of the actual corpus, and assert margin, not just + finiteness -- |x| grows over a long run and this must not sit on the edge. + """ + import torch.nn.functional as F + import data as D, ngram as N, model as M + from pathlib import Path + srcs = D.open_sources(cfg, log=lambda *a, **k: None) + pk = D.VarlenPacker(cfg, srcs, seed=0) + m = M.Boris2(cfg).to(dev) + if m.ngram is not None: + m.ngram.load_vocab(N.NGramVocab.load(cfg.ngram.vocab_path), dev) + m.train() + x, y, pos, bnd, bounds, st = pk.batch(8, 2048, 0.0) + t = lambda a, dt: torch.as_tensor(a, dtype=dt, device=dev) + with torch.no_grad(), torch.autocast("cuda", dtype=torch.float16): + h = m(t(x, torch.long), t(pos, torch.long), t(bounds, torch.long), + t(bnd, torch.bool), None, 10_000.0, st["min_seqlen"], st["max_seqlen"]) + xf = h.reshape(-1, h.shape[-1]) + tg = t(y, torch.long).reshape(-1) + # Test the actual custom Function, not a separately reimplemented GEMM. + # The old test bypassed its broken fp16 rescale and therefore passed while + # production overflowed. Compare full gradients with fp32 cross-entropy. + c = cfg.optim.lm_head_chunk + for magnitude, repeated in ((1.0, False), (64.0, True)): + xx = (xf[:c].detach() * magnitude).requires_grad_() + ww = m.head_weight().detach().clone().requires_grad_() + tt = tg[:c].clone() + if repeated: + tt.fill_(int(tg[tg != IGNORE][0])) + with torch.autocast("cuda", dtype=torch.float16): + loss = M.fused_lm_loss(xx, ww, tt, chunk=c) + loss.backward() + xr = xx.detach().float().requires_grad_() + wr = ww.detach().clone().requires_grad_() + ref = F.cross_entropy(F.linear(xr, wr), tt, ignore_index=IGNORE) + ref.backward() + rel = (ww.grad - wr.grad).norm() / wr.grad.norm().clamp_min(1e-12) + check(f"production head real text: magnitude={magnitude:g}, repeated={repeated}", + bool(torch.isfinite(ww.grad).all()) and float(rel) < .01, + f"dW relative error {float(rel):.3g}") + del xx, ww, xr, wr, loss, ref + + del m + torch.cuda.empty_cache() + + +def check_compile(cfg, dev): + """The compiled regions must produce the same numbers as eager.""" + small = Config.from_json(cfg.to_json()) + small.enforce_budget = False + small.model.n_layer = 4 + torch.manual_seed(0) + me = build(small, dev, compile_regions=False) + sd = me.state_dict() + torch.manual_seed(0) + mc = build(small, dev, compile_regions=True) + mc.load_state_dict(sd) + for m_ in (me, mc): + for blk in m_.blocks: + torch.nn.init.normal_(blk.attn.canon, std=0.3) + mc.load_state_dict(me.state_dict()) + x, pos, bounds, bnd = synth_batch(small, dev, 1, 512, [[200, 312]], seed=1) + with torch.autocast("cuda", dtype=torch.float16): + a = me(x, pos, bounds, bnd, None, 10_000.0) + b = mc(x, pos, bounds, bnd, None, 10_000.0) + r = float((a.float() - b.float()).norm() / (a.float().norm() + 1e-9)) + check("compiled regions match eager", r < 5e-3, f"rel-L2 {r:.2e}") + + +# --------------------------------------------------------------------------- # +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--device", default="cuda:0") + ap.add_argument("--quick", action="store_true") + a = ap.parse_args() + dev = a.device + cfg = Config() + torch.cuda.set_device(dev) + torch.backends.cuda.matmul.allow_tf32 = False + + print(f"\n verify: {cfg.run_name} on {dev}") + print(" " + "=" * 74) + print("\n budget and optimizer split") + m = build(cfg, dev) + build_optimizers(m, cfg) + del m + torch.cuda.empty_cache() + + print(" numerics") + check_fused_head(cfg, dev) + check_repeated_target_head(cfg, dev) + check_head_range(cfg, dev) + check_loss_scale_floor(cfg, dev) + check_residual_range(cfg, dev) + check_down_proj_headroom(cfg, dev) + check_muon(cfg, dev) + torch.cuda.empty_cache() + + print("\n isolation") + check_canon_causality(cfg, dev) + check_attention_matches_loop(cfg, dev) + check_volta_kernels(cfg, dev) + check_ngram_isolation(cfg, dev) + check_ngram_ablation(cfg, dev) + check_document_isolation(cfg, dev) + torch.cuda.empty_cache() + + print("\n schedule and data") + check_schedule(cfg) + check_packer(cfg, dev) + check_multistore(cfg) + + print("\n plumbing") + check_tensor_cores(cfg, dev) + check_checkpoint(cfg, dev) + if not a.quick: + check_compile(cfg, dev) + + print("\n " + "=" * 74) + print(f" {len(PASS)} passed, {len(FAIL)} failed") + for f in FAIL: + print(f" FAILED: {f}") + sys.exit(1 if FAIL else 0) + + +if __name__ == "__main__": + main() -- cgit v1.2.3