"""Correctness checks. Run before starting a run, and after touching a kernel. python verify.py [--device cuda:0] [--quick] The checks that matter most are the isolation ones. Boris-2 packs several documents into every row, and three separate mechanisms have to keep them apart: varlen attention, the Canon boundary mask, and the n-gram history mask. A leak in any of them is invisible in the loss curve -- the model simply learns from context it will never have at inference. Note how check_document_isolation de-zeroes the Canon kernels first. They are zero-initialised, so at init the convolution contributes exactly nothing and the test passes no matter how badly it leaks. The 75M run shipped that bug for weeks behind a test that looked green. """ from __future__ import annotations import argparse import math import os import sys import atexit import shutil import tempfile from pathlib import Path import numpy as np import torch import torch.nn.functional as F sys.path.insert(0, str(Path(__file__).resolve().parent)) from config import Config from model import Boris2, IGNORE, fused_lm_loss, _pre_attn from ngram import NGramVocab from optim import Muon, build_optimizers, newton_schulz from schedule import lr_scale, mixture_at, context_at PASS, FAIL = [], [] _TMPDIRS = [] def _tmpdir(): """mkdtemp that actually cleans up. verify.py used bare tempfile.mkdtemp() and never removed anything. /tmp here is a 16 GB tmpfs -- RAM -- and each run leaves ~500 MB of packer fixtures behind, so 89 runs had filled it completely. That is not just a disk problem: it was eating the machine's memory and getting background jobs OOM-killed. """ d = tempfile.mkdtemp() _TMPDIRS.append(d) return d @atexit.register def _cleanup_tmpdirs(): for d in _TMPDIRS: shutil.rmtree(d, ignore_errors=True) def check(name, cond, detail=""): (PASS if cond else FAIL).append(name) mark = "ok " if cond else "FAIL" print(f" [{mark}] {name}" + (f" {detail}" if detail else "")) return cond # --------------------------------------------------------------------------- # def synth_vocab(cfg, seed=0): V = cfg.model.vocab_size rng = np.random.default_rng(seed) bk = np.unique(rng.integers(0, V * V, size=cfg.ngram.bigram_rows).astype(np.int64)) tk = np.unique(rng.integers(0, V * V * V, size=cfg.ngram.trigram_rows).astype(np.int64)) return NGramVocab(V, bk, np.arange(bk.size) + cfg.ngram.bigram_base, tk, np.arange(tk.size) + cfg.ngram.trigram_base) def synth_batch(cfg, dev, B, T, lens_per_row, seed=0): g = torch.Generator().manual_seed(seed) V = cfg.model.vocab_size x = torch.randint(0, V, (B, T), generator=g).to(dev) pos = torch.zeros(B, T, dtype=torch.int64) bnd = torch.zeros(B, T, dtype=torch.bool) bounds = [] for b, lens in enumerate(lens_per_row): p = 0 for L in lens: bounds.append(b * T + p) bnd[b, p] = True pos[b, p:p + L] = torch.arange(L) p += L assert p == T, "row must be exactly full" bounds.append(B * T) return (x, pos.to(dev), torch.tensor(bounds, dtype=torch.int64).to(dev), bnd.to(dev)) def build(cfg, dev, compile_regions=False): cfg.train.compile_regions = compile_regions m = Boris2(cfg).to(dev) if m.ngram is not None: m.ngram.load_vocab(synth_vocab(cfg), dev) if compile_regions: m.maybe_compile() return m # --------------------------------------------------------------------------- # def check_fused_head(cfg, dev): """The fused head must equal an ordinary cross-entropy, loss and gradient.""" torch.manual_seed(0) N, d, V = 1024, cfg.model.d_model, cfg.model.vocab_size x = torch.randn(N, d, device=dev, dtype=torch.float16, requires_grad=True) W = (torch.randn(V, d, device=dev) * 0.02).requires_grad_(True) t = torch.randint(0, V, (N,), device=dev) t[::7] = IGNORE loss = fused_lm_loss(x, W, t, chunk=256) loss.backward() gx, gW = x.grad.clone(), W.grad.clone() # Reference in fp32. Comparing against the fp16 unfused head would compare # two approximations: its grad_logits reach the GEMM at ~3e-8, far below # what fp16 can represent, so it is the less accurate of the two. x2 = x.detach().float().clone().requires_grad_(True) W2 = W.detach().float().clone().requires_grad_(True) ref = F.cross_entropy(F.linear(x2, W2), t, ignore_index=IGNORE) ref.backward() dl = abs(float(loss) - float(ref)) rx = (gx.float() - x2.grad).norm() / (x2.grad.norm() + 1e-9) rw = (gW - W2.grad).norm() / (W2.grad.norm() + 1e-9) check("fused LM head: loss matches fp32 cross_entropy", dl < 5e-3, f"|d| = {dl:.2e}") check("fused LM head: dx matches fp32", float(rx) < 5e-3, f"rel-L2 {float(rx):.2e}") check("fused LM head: dW matches fp32", float(rw) < 5e-3, f"rel-L2 {float(rw):.2e}") # The point of the fusion: never hold (N, V) logits. Measured against the # unfused head on the same inputs, at a realistic token count. def peak_of(fn, N2=8192): xx = torch.randn(N2, d, device=dev, dtype=torch.float16, requires_grad=True) WW = (torch.randn(V, d, device=dev) * 0.02).requires_grad_(True) tt = torch.randint(0, V, (N2,), device=dev) torch.cuda.synchronize(); torch.cuda.empty_cache() base = torch.cuda.memory_allocated(dev) torch.cuda.reset_peak_memory_stats(dev) fn(xx, WW, tt).backward() torch.cuda.synchronize() out = torch.cuda.max_memory_allocated(dev) - base del xx, WW, tt torch.cuda.empty_cache() return out fused_peak = peak_of(lambda a, b, c: fused_lm_loss(a, b, c)) plain_peak = peak_of(lambda a, b, c: F.cross_entropy( F.linear(a, b.to(a.dtype)).float(), c, ignore_index=IGNORE)) check("fused LM head: uses far less memory than the unfused head", fused_peak < plain_peak / 3, f"{fused_peak/2**20:.0f} MB vs {plain_peak/2**20:.0f} MB " f"({plain_peak/max(fused_peak,1):.1f}x)") def check_repeated_target_head(cfg, dev): """Exercise the production Function under autocast, including its rescale.""" torch.manual_seed(91) for dtype in (torch.float32, torch.float16): x = torch.full((1024, 32), 16.0, device=dev, dtype=dtype, requires_grad=True) w = (torch.randn(256, 32, device=dev) * .01).requires_grad_() target = torch.zeros(1024, device=dev, dtype=torch.long) with torch.autocast("cuda", dtype=torch.float16): loss = fused_lm_loss(x, w, target, chunk=1024) loss.backward() xr = x.detach().float().requires_grad_() wr = w.detach().clone().requires_grad_() ref = F.cross_entropy(F.linear(xr, wr), target) ref.backward() rel = (w.grad - wr.grad).norm() / wr.grad.norm() check(f"production head repeated-target dW {dtype}", bool(torch.isfinite(w.grad).all()) and float(rel) < .005, f"relative error {float(rel):.3g}") def check_canon_causality(cfg, dev): """A change at position t must never move an output at t' < t.""" torch.manual_seed(0) C, K = cfg.model.canon_channels, cfg.model.canon_kernel B, T = 1, 64 x = torch.randn(B, T, cfg.model.d_model, device=dev, dtype=torch.float16) w_qkv = torch.randn(C, cfg.model.d_model, device=dev, dtype=torch.float16) * 0.02 canon = torch.randn(C, K, device=dev) * 0.5 m1 = torch.ones(B, T, 1, device=dev, dtype=torch.float16) m2 = torch.ones(B, T, 1, device=dev, dtype=torch.float16) hd = cfg.model.head_dim cos = torch.ones(B, T, 1, hd // 2, device=dev) sin = torch.zeros(B, T, 1, hd // 2, device=dev) nw = torch.ones(cfg.model.d_model, device=dev) args = (nw, 1e-5, w_qkv, canon, m1, m2, cos, sin, cfg.model.n_head, cfg.model.n_kv_head, hd, True) q0 = _pre_attn(x, *args)[0] x2 = x.clone(); x2[0, 32] += 10.0 q1 = _pre_attn(x2, *args)[0] check("Canon is strictly causal", torch.equal(q0[0, :32], q1[0, :32])) moved = int((q0[0, 32:35] != q1[0, 32:35]).any(-1).any(-1).sum()) check("Canon reaches exactly kernel-1 tokens forward", moved == 3, f"{moved} of the next 3 positions moved") def check_document_isolation(cfg, dev): """End-to-end: a change in one document must not move another. Canon kernels are randomised first. Zero-initialised, they contribute nothing and this test would pass on a model that leaks badly. """ torch.manual_seed(0) small = Config.from_json(cfg.to_json()) small.enforce_budget = False small.model.n_layer = 6 m = build(small, dev) for blk in m.blocks: # de-zero: the whole point torch.nn.init.normal_(blk.attn.canon, std=0.5) B, T = 1, 512 lens = [[200, 312]] x, pos, bounds, bnd = synth_batch(small, dev, B, T, lens, seed=3) with torch.autocast("cuda", dtype=torch.float16): h0 = m(x, pos, bounds, bnd, None, 10_000.0) x2 = x.clone(); x2[0, :200] = torch.randint(0, small.model.vocab_size, (200,), device=dev) h1 = m(x2, pos, bounds, bnd, None, 10_000.0) drift = float((h0[0, 200:] - h1[0, 200:]).abs().max()) check("document isolation: rewriting doc A never moves doc B", drift == 0.0, f"max logit drift {drift:.3e}") same = float((h0[0, :200] - h1[0, :200]).abs().max()) check("the probe actually perturbs doc A", same > 0.0, f"max change {same:.3e}") def check_attention_matches_loop(cfg, dev): """Nested varlen attention must equal a per-document causal SDPA.""" torch.manual_seed(0) H, KV, D = cfg.model.n_head, cfg.model.n_kv_head, cfg.model.head_dim lens = [300, 212, 512] N = sum(lens) off = torch.tensor([0] + list(np.cumsum(lens)), device=dev) q = torch.randn(N, H, D, device=dev, dtype=torch.float16) k = torch.randn(N, H, D, device=dev, dtype=torch.float16) v = torch.randn(N, H, D, device=dev, dtype=torch.float16) nj = lambda t: torch.nested.nested_tensor_from_jagged(t, off).transpose(1, 2) o = F.scaled_dot_product_attention(nj(q), nj(k), nj(v), is_causal=True) o = o.transpose(1, 2).values() ref = torch.empty_like(q) p = 0 for L in lens: s = F.scaled_dot_product_attention( q[p:p+L].transpose(0, 1)[None], k[p:p+L].transpose(0, 1)[None], v[p:p+L].transpose(0, 1)[None], is_causal=True) ref[p:p+L] = s[0].transpose(0, 1) p += L d = float((o.float() - ref.float()).abs().max()) check("varlen attention equals a per-document loop", d == 0.0, f"max diff {d:.1e}") def check_volta_kernels(cfg, dev): """volta_kernels/: the sm_70 attention and LM head, held to the stock paths. Runs whenever the device is sm_70, whether or not BORIS_VOLTA_KERNELS is set, so the kernels are proven before anyone opts in. The bar is the stock path's own error against an fp64 reference, measured here on the same inputs -- "same precision" as a number, not an adjective. """ if torch.cuda.get_device_capability(dev) != (7, 0): print(" [skip] volta kernels: device is not sm_70") return import model as M import volta_kernels as VK from volta_kernels.head import fused_lm_loss as volta_lm_loss torch.manual_seed(0) H, D = cfg.model.n_head, cfg.model.head_dim # Edge lengths around the 64-row tile, a length-1 document, and one long one. lens = [1, 63, 64, 65, 127, 128, 300, 1300] N = sum(lens) off = torch.tensor([0] + list(np.cumsum(lens)), device=dev) # v as a strided view of a fused qkv, exactly as the model hands it over. qkv = (torch.randn(N, 3 * H * D, device=dev) * 1.5).half() q = qkv[:, :H * D].reshape(N, H, D).contiguous().requires_grad_() k = qkv[:, H * D:2 * H * D].reshape(N, H, D).contiguous().requires_grad_() vb = qkv.clone().requires_grad_() v = vb[:, 2 * H * D:].view(N, H, D) go = torch.randn(N, H, D, device=dev).half() def run(fn): for t in (q, k, vb): t.grad = None o = fn() o.backward(go) return [o.detach().double(), q.grad.double(), k.grad.double(), vb.grad[:, 2 * H * D:].reshape(N, H, D).double()] def stock(): nj = lambda t: torch.nested.nested_tensor_from_jagged(t, off).transpose(1, 2) return F.scaled_dot_product_attention(nj(q), nj(k), nj(v), is_causal=True ).transpose(1, 2).values() qq, kk, vv = (t.detach().double().requires_grad_() for t in (q, k, v)) outs, p = [], 0 for L in lens: sl = slice(p, p + L) outs.append(F.scaled_dot_product_attention( qq[sl].transpose(0, 1), kk[sl].transpose(0, 1), vv[sl].transpose(0, 1), is_causal=True).transpose(0, 1)) p += L ref_o = torch.cat(outs) ref_o.backward(go.double()) ref = [ref_o.detach(), qq.grad, kk.grad, vv.grad] S = run(stock) V = run(lambda: VK.varlen_causal_attention(q, k, v, off)) rel = lambda a, b: float((a - b).norm() / b.norm()) for name, i in (("output", 0), ("dq", 1), ("dk", 2), ("dv", 3)): es, ev = rel(S[i], ref[i]), rel(V[i], ref[i]) check(f"volta attention: {name} as accurate as memory-efficient SDPA", ev <= 1.1 * es and math.isfinite(ev), f"rel-L2 vs fp64 {ev:.2e} (stock {es:.2e})") # Isolation and causality, exactly: perturb one document / one late token. def fwd_only(qx, kx, vx): return VK.varlen_causal_attention(qx, kx, vx, off.clone()).float() q0, k0, v0 = (t.detach().clone() for t in (q, k, v)) o0 = fwd_only(q0, k0, v0) a, b = int(off[6]), int(off[7]) # the 300-token document q1, k1, v1 = q0.clone(), k0.clone(), v0.clone() for t in (q1, k1, v1): t[a:b] = torch.randn_like(t[a:b]) o1 = fwd_only(q1, k1, v1) outside = torch.cat([o1[:a] - o0[:a], o1[b:] - o0[b:]]).abs().max().item() check("volta attention: rewriting one document never moves another", outside == 0.0 and (o1[a:b] - o0[a:b]).abs().max().item() > 0, f"max drift outside {outside:.1e}") t_ = a + 150 k2, v2 = k0.clone(), v0.clone() k2[t_] = torch.randn_like(k2[t_]); v2[t_] = torch.randn_like(v2[t_]) o2 = fwd_only(q0, k2, v2) before = (o2[a:t_] - o0[a:t_]).abs().max().item() at = (o2[t_] - o0[t_]).abs().max().item() check("volta attention: causal (a key never reaches an earlier query)", before == 0.0 and at > 0, f"drift before {before:.1e}, at {at:.1e}") # The per-offsets tile table must not be served stale after an in-place edit. o3 = off.clone() _, t_before, _ = VK.tile_table(o3) # doc 0 has 1 token: 1 tile o3[1] = 65 # in place: now 65 tokens, 2 tiles cu, tcum, perm = VK.tile_table(o3) fresh = (o3[1:] - o3[:-1] + 63) // 64 # what the table must now say check("volta attention: tile table follows in-place edits of the offsets", int(cu[1]) == 65 and int(tcum[-1]) == int(fresh.clamp_min(0).sum()) and int(fresh[int(perm[0])]) == int(fresh.max())) lens_ = (cu[1:] - cu[:-1] + 63) // 64 check("volta attention: tile table launches the longest documents first", bool((lens_[perm.long()][:-1] >= lens_[perm.long()][1:]).all()) and sorted(perm.tolist()) == list(range(len(lens)))) # LM head: the one-pass Triton softmax against the production head. torch.manual_seed(1) Vv, d = cfg.model.vocab_size, cfg.model.d_model Nn = 2048 W = (torch.randn(Vv, d, device=dev) * 0.05) x = torch.randn(Nn, d, device=dev) * 3.0 y = torch.randint(0, Vv, (Nn,), device=dev) y[::5] = IGNORE def head(fn, xx, yy): xx = xx.clone().requires_grad_(); ww = W.clone().requires_grad_() with torch.autocast("cuda", dtype=torch.float16): loss = fn(xx, ww, yy, 1024) loss.backward() return loss.detach().double(), xx.grad.double(), ww.grad.double() was = M.VOLTA_KERNELS M.VOLTA_KERNELS = False try: P = head(M.fused_lm_loss, x, y) finally: M.VOLTA_KERNELS = was Q = head(volta_lm_loss, x, y) xr = x.double().requires_grad_(); wr = W.double().requires_grad_() lr_ = F.cross_entropy(xr @ wr.T, y, ignore_index=IGNORE) lr_.backward() check("volta head: loss equals the production head", abs(float(Q[0] - P[0])) <= 1e-6 * abs(float(P[0])), f"{float(Q[0]):.7f} vs {float(P[0]):.7f}") for name, i, r in (("dx", 1, xr.grad), ("dW", 2, wr.grad)): ep, eq = rel(P[i], r), rel(Q[i], r) check(f"volta head: {name} as accurate as the production head", eq <= 1.02 * ep, f"rel-L2 vs fp64 {eq:.3e} (production {ep:.3e})") # The dW overflow guard survives: one token repeated through a whole chunk. xs = x[:1].repeat(1024, 1) * 5.0 ys = y.clone()[:1024]; ys[:] = 7 _, dxs, dws = head(volta_lm_loss, xs, ys) check("volta head: a chunk of one repeated token stays finite", bool(torch.isfinite(dws).all() and torch.isfinite(dxs).all())) # Stage two (80% of the run) packs documents up to 8192 tokens. fp64 would # need a 4 GB score matrix, so the reference is fp32 memory-efficient SDPA. torch.manual_seed(3) lens8 = [8192, 3000, 5000] N8 = sum(lens8) off8 = torch.tensor([0] + list(np.cumsum(lens8)), device=dev) q8, k8, v8, g8 = (torch.randn(N8, H, D, device=dev).half() for _ in range(4)) def attn_grads(fn, dtype): # clone: for fp16, .to() returns the SAME tensor, and the two runs would # accumulate into each other's .grad ts = [t.detach().to(dtype).clone().requires_grad_() for t in (q8, k8, v8)] o = fn(*ts) o.backward(g8.to(dtype)) return [o.detach().double()] + [t.grad.double() for t in ts] def nested(a, b, c): nj = lambda t: torch.nested.nested_tensor_from_jagged(t, off8).transpose(1, 2) return F.scaled_dot_product_attention(nj(a), nj(b), nj(c), is_causal=True ).transpose(1, 2).values() ref8 = attn_grads(nested, torch.float32) S8 = attn_grads(nested, torch.float16) V8 = attn_grads(lambda a, b, c: VK.varlen_causal_attention(a, b, c, off8), torch.float16) worst = max(rel(V8[i], ref8[i]) / rel(S8[i], ref8[i]) for i in range(4)) check("volta attention: an 8192-token document as accurate as stock SDPA", worst <= 1.1, f"worst error ratio vs stock {worst:.3f} (o, dq, dk, dv vs fp32)") # A head call whose row count is not a multiple of the 1024-row chunk. xp, yp = x[:1024 + 300], y[:1024 + 300] M.VOLTA_KERNELS = False try: Pp = head(M.fused_lm_loss, xp, yp) finally: M.VOLTA_KERNELS = was Qp = head(volta_lm_loss, xp, yp) check("volta head: a partial final chunk matches the production head", abs(float(Qp[0] - Pp[0])) <= 1e-6 * abs(float(Pp[0])) and rel(Qp[1], Pp[1]) < 1e-3 and rel(Qp[2], Pp[2]) < 1e-3, f"dx {rel(Qp[1], Pp[1]):.1e}, dW {rel(Qp[2], Pp[2]):.1e} apart") # Fused Canon + RoPE: precision against fp64 next to the eager region, and # document isolation with non-zero Canon weights (zero-init would hide a leak). from volta_kernels.canon_rope_cuda import canon_rope # what the model uses torch.manual_seed(4) Bc, Tc = 2, 256 bndc = torch.zeros(Bc, Tc, dtype=torch.bool, device=dev) bndc[:, 0] = True; bndc[0, 100] = True; bndc[1, 7] = True; bndc[1, 8] = True b2c = bndc | F.pad(bndc, (1, 0))[:, :-1] m1c = (~bndc).float()[:, :, None]; m2c = (~b2c).float()[:, :, None] posc = torch.zeros(Bc, Tc, dtype=torch.long, device=dev) for b in range(Bc): p0 = 0 for t_ in range(Tc): if bndc[b, t_]: p0 = t_ posc[b, t_] = t_ - p0 invc = 1.0 / (10000.0 ** (torch.arange(0, 64, 2, device=dev).float() / 64)) fc = torch.outer(torch.arange(Tc, device=dev).float(), invc) cosc, sinc = fc.cos()[posc].unsqueeze(2), fc.sin()[posc].unsqueeze(2) qkvc = (torch.randn(Bc, Tc, 1536, device=dev) * 2).half() cwc = torch.randn(1536, 3, device=dev) * 0.3 gsc = [torch.randn(Bc, Tc, 8, 64, device=dev).half() for _ in range(3)] def region(a, w_, dt): ww = w_.to(torch.float16).to(dt) s1 = F.pad(a, (0, 0, 1, 0))[:, :-1] * m1c.to(dt) s2 = F.pad(a, (0, 0, 2, 0))[:, :-2] * m2c.to(dt) y_ = a + a * ww[:, 0] + s1 * ww[:, 1] + s2 * ww[:, 2] c_, s_ = cosc.to(dt), sinc.to(dt) def rope(t_): a_, b_ = t_.chunk(2, dim=-1) return torch.cat([a_ * c_ - b_ * s_, a_ * s_ + b_ * c_], -1) v_ = lambda i: y_[..., 512 * i:512 * (i + 1)].reshape(Bc, Tc, 8, 64) return rope(v_(0)), rope(v_(1)), v_(2) def cr_run(fn, dt): a = qkvc.detach().to(dt).clone().requires_grad_() w_ = cwc.detach().clone().requires_grad_() outs = fn(a, w_) torch.autograd.backward(outs, [g.to(outs[0].dtype) for g in gsc]) return [o.detach().double() for o in outs] + [a.grad.double()] ref_c = cr_run(lambda a, w_: region(a, w_, torch.float64), torch.float64) eag_c = cr_run(lambda a, w_: tuple(o.to(torch.float16) for o in region(a.float(), w_, torch.float32)), torch.float16) fus_c = cr_run(lambda a, w_: canon_rope(a, w_, m1c, m2c, cosc, sinc), torch.float16) worst = max(rel(fus_c[i], ref_c[i]) / rel(eag_c[i], ref_c[i]) for i in range(4)) check("volta canon+rope: q, k, v, dqkv as accurate as the eager region", worst <= 1.1, f"worst error ratio {worst:.3f}") q0_, k0_, v0_ = canon_rope(qkvc, cwc, m1c, m2c, cosc, sinc) pert = qkvc.clone(); pert[0, :100] = torch.randn_like(pert[0, :100]).half() q1_, k1_, v1_ = canon_rope(pert, cwc, m1c, m2c, cosc, sinc) leak = max(float((a_[0, 100:].float() - b_[0, 100:].float()).abs().max()) for a_, b_ in ((q0_, q1_), (k0_, k1_), (v0_, v1_))) moved = float((v1_[0, :100].float() - v0_[0, :100].float()).abs().max()) check("volta canon+rope: a document boundary stops the Canon taps exactly", leak == 0.0 and moved > 0, f"drift past boundary {leak:.1e}") # Whole model, both paths, against an fp32 (no autocast) reference, at the # run's loss scale. Canon de-zeroed so every mechanism is live. small = Config.from_json(cfg.to_json()) small.enforce_budget = False small.model.n_layer = 4 torch.manual_seed(2) m = build(small, dev) for blk in m.blocks: torch.nn.init.normal_(blk.attn.canon, std=0.1) lens2 = [[700, 64, 1, 1283], [2048]] xb, pb, ob, bb = synth_batch(small, dev, 2, 2048, lens2, seed=5) yb = torch.roll(xb, -1, 1) def grads(volta, amp): M.VOLTA_KERNELS = volta try: for prm in m.parameters(): prm.grad = None with torch.autocast("cuda", dtype=torch.float16, enabled=amp): loss = m(xb, pb, ob.clone(), bb, yb, 10_000.0) (loss * 32768.0).backward() finally: M.VOLTA_KERNELS = was return {n: prm.grad.double() / 32768.0 for n, prm in m.named_parameters() if prm.grad is not None} gr, gs, gv = grads(False, False), grads(False, True), grads(True, True) # The production path with checkpointing OFF merges each block's FFN half # with the next block's norm+QKV into one region (model._forward_merged). small.train.grad_checkpoint = False gmerged = grads(True, True) m._no_merge = True gunmerged = grads(True, True) m._no_merge = False small.train.grad_checkpoint = True em = sorted(rel(gmerged[n], gr[n]) for n in gr) eu = sorted(rel(gunmerged[n], gr[n]) for n in gr) check("volta merged regions: gradients as close to fp32 as per-block regions", em[len(em) // 2] <= 1.05 * eu[len(eu) // 2] and max(em) <= 1.05 * max(eu), f"median {em[len(em) // 2]:.3e} (per-block {eu[len(eu) // 2]:.3e})") es = sorted(rel(gs[n], gr[n]) for n in gr) ev = sorted(rel(gv[n], gr[n]) for n in gr) med_s, med_v = es[len(es) // 2], ev[len(ev) // 2] check("volta model: gradients as close to fp32 as the stock fp16 path", med_v <= 1.1 * med_s and max(ev) <= 1.25 * max(es), f"median rel-L2 {med_v:.3e} (stock {med_s:.3e}), worst {max(ev):.3e} (stock {max(es):.3e})") def check_ngram_isolation(cfg, dev): """The n-gram key must not span a document boundary.""" m = build(cfg, dev) B, T = 1, 32 x = torch.arange(T, device=dev)[None] % cfg.model.vocab_size bnd = torch.zeros(B, T, dtype=torch.bool, device=dev) bnd[0, 0] = True; bnd[0, 16] = True out = m.ngram(x, bnd, torch.float32) x2 = x.clone(); x2[0, 14:16] = (x2[0, 14:16] + 7) % cfg.model.vocab_size out2 = m.ngram(x2, bnd, torch.float32) d = float((out[0, 16:] - out2[0, 16:]).abs().max()) check("n-gram history stops at a document boundary", d == 0.0, f"max diff {d:.1e}") def check_muon(cfg, dev): """Batched Newton-Schulz must agree with the unbatched one, and orthogonalise.""" torch.manual_seed(0) G = torch.randn(4, 512, 2368, device=dev) batched = newton_schulz(G, 5) single = torch.stack([newton_schulz(G[i:i+1], 5)[0] for i in range(4)]) r = float((batched - single).norm() / single.norm()) check("batched Newton-Schulz equals the unbatched one", r < 1e-5, f"rel-L2 {r:.2e}") s = torch.linalg.svdvals(batched[0].float()) check("Newton-Schulz orthogonalises (singular values near 1)", 0.7 < float(s.mean()) < 1.3, f"mean sv {float(s.mean()):.3f}, " f"min {float(s.min()):.3f}, max {float(s.max()):.3f}") # Fusing q/k/v into one Parameter must not change the update. This is the # whole justification for storing them fused, so it is asserted, not assumed. torch.manual_seed(1) fused = torch.nn.Parameter(torch.randn(768, 512, device=dev)) fused._boris_role = "matmul" fused._boris_blocks = ((0, 512), (512, 640), (640, 768)) grad = torch.randn(768, 512, device=dev) fused.grad = grad.clone() parts = [] for a, b in fused._boris_blocks: q = torch.nn.Parameter(fused.data[a:b].clone()) q._boris_role = "matmul" q.grad = grad[a:b].clone() parts.append(q) kw = dict(lr=0.02, momentum=0.95, nesterov=True, ns_steps=5, weight_decay=0.1, batched=True) Muon([fused], **kw).step() Muon(parts, **kw).step() sep = torch.cat([q.data for q in parts], 0) d = float((fused.data - sep).abs().max()) check("a block-fused Parameter updates exactly like separate ones", d == 0.0, f"max diff {d:.1e}") # A (768, 3) Canon kernel must never reach Muon. m = build(cfg, dev) muon, adamw = build_optimizers(m, cfg, log=lambda *a: None) canon_ids = {id(b.attn.canon) for b in m.blocks} in_muon = any(id(p) in canon_ids for g in muon.param_groups for p in g["params"]) check("Canon kernels are in AdamW, not Muon", not in_muon) tbl = id(m.ngram.table) in_muon_tbl = any(id(p) == tbl for g in muon.param_groups for p in g["params"]) check("the n-gram table is in AdamW, not Muon", not in_muon_tbl) def check_ngram_ablation(cfg, dev): """Both arms must build, and the runtime toggle must actually do something.""" off = Config.from_json(cfg.to_json()) off.model.use_ngram = False b_on, b_off = cfg.param_budget()["total"], off.param_budget()["total"] check("use_ngram leaves the transformer untouched", cfg.param_budget()["trunk"] == off.param_budget()["trunk"] and b_on - b_off == cfg.ngram.table_params(cfg.model.d_model), f"{b_on:,} with table, {b_off:,} without, same {b_off - cfg.model.embed_params():,} trunk") off.model.n_layer = 4 off.enforce_budget = False m = build(off, dev) check("use_ngram=False builds no table and no projection", m.ngram is None and not any("ngram" in n for n, _ in m.named_parameters())) build_optimizers(m, off, log=lambda *a: None) del m torch.cuda.empty_cache() on = Config.from_json(cfg.to_json()) on.model.n_layer = 4 on.enforce_budget = False m = build(on, dev) x, pos, bounds, bnd = synth_batch(on, dev, 1, 256, [[128, 128]], seed=5) with torch.autocast("cuda", dtype=torch.float16): a = m(x, pos, bounds, bnd, None, 10_000.0) m.ngram_enabled = False b = m(x, pos, bounds, bnd, None, 10_000.0) m.ngram_enabled = True c = m(x, pos, bounds, bnd, None, 10_000.0) check("the runtime toggle changes the output", not torch.equal(a, b), f"max diff {float((a.float()-b.float()).abs().max()):.3e}") check("the runtime toggle is reversible", torch.equal(a, c)) del m torch.cuda.empty_cache() def check_schedule(cfg): s = cfg.schedule w, st = s.warmup_frac, s.stable_frac ok = (abs(lr_scale(0.0, cfg)) < 1e-9 and abs(lr_scale(w, cfg) - 1) < 1e-9 and abs(lr_scale((w + st) / 2, cfg) - 1) < 1e-9 and abs(lr_scale(1.0, cfg) - s.final_lr_frac) < 1e-9) check("WSD: 0 at start, 1 across the stable phase, floor at the end", ok) mono = all(lr_scale(p, cfg) >= lr_scale(p + 0.005, cfg) - 1e-12 for p in np.arange(st, 1.0, 0.005)) check("WSD: the decay never goes back up", mono) for p in (0.0, 0.5, 0.86, 0.94, 1.0): w_ = mixture_at(p, cfg) assert abs(sum(w_.values()) - 1) < 1e-9 end = mixture_at(1.0, cfg) # 2026-09-17, Boris-2-0917. The four checks here used to assert the old # hand-weighted decay mixture (raw web to zero, curated > 0.7). That corpus # is gone. ClimbMix's weights are a SEARCH RESULT over the whole mixture, so # the property worth protecting is the opposite one: that we do not # re-weight it by phase and quietly undo the optimisation we adopted it for. flat = all(abs(mixture_at(p, cfg)["climbmix"] - end["climbmix"]) < 1e-9 for p in (0.0, 0.25, 0.5, 0.87, 0.95)) check("ClimbMix weights are flat across the run, not re-weighted by phase", flat, f"climbmix held at {end['climbmix']:.2f} throughout") check("ClimbMix is the overwhelming majority of the mixture", end["climbmix"] >= 0.90, f"{end['climbmix']:.2f} of the mixture") # finemath is the only source carried over from 0907. ArithMark-3 was the # one benchmark 0907 was competitive on (37.40 vs GPT-X2.5's 38.40) and # ClimbMix was optimised against PIQA/ARC-E/HellaSwag, not arithmetic. check("finemath is retained, so ArithMark-3 is not abandoned", 0.02 <= end.get("finemath", 0.0) <= 0.10, f"{end.get('finemath', 0.0):.2f} of the mixture") check("the mixture names only sources that exist on disk", set(cfg.data.sources) == set(end), f"{sorted(cfg.data.sources)}") b0 = context_at(0.0, cfg) b1 = context_at(0.99, cfg) check("context extends later in the run", b1[0] > b0[0], f"{b0[0]} -> {b1[0]} at theta {b0[1]:,.0f} -> {b1[1]:,.0f}") check("the step size is identical across context stages", b0[2] * b0[0] * b0[3] == b1[2] * b1[0] * b1[3] == cfg.train.tokens_per_step, f"{b0[2]}x{b0[0]}x{b0[3]} = {b0[2]*b0[0]*b0[3]:,}") def check_packer(cfg, dev): """Targets are next-token within a document, never across one.""" from data import VarlenPacker, SourceStore d = Path(_tmpdir()) / "src" d.mkdir(parents=True) rng = np.random.default_rng(0) lens = rng.integers(20, 400, size=400) toks = rng.integers(0, cfg.model.vocab_size, size=int(lens.sum())).astype(np.uint16) toks.tofile(d / "tokens.bin") np.concatenate([[0], np.cumsum(lens)]).astype(np.uint64).tofile(d / "docs.idx") st = SourceStore.open(d.parent, "src") cfg2 = Config.from_json(cfg.to_json()) cfg2.data.sources = ("src",) cfg2.curriculum.enabled = False p = VarlenPacker(cfg2, {"src": st}, 7) xs, ys, pos, bnd, bounds, stats = p.batch(2, 256, 0.0) ok_bounds = (bounds[0] == 0 and bounds[-1] == 2 * 256 and np.all(np.diff(bounds) > 0)) check("packer: offsets are increasing and cover the batch exactly", ok_bounds) flat_b = np.zeros(2 * 256, dtype=bool) flat_b[bounds[:-1]] = True check("packer: boundary mask agrees with the offsets", np.array_equal(flat_b, bnd.reshape(-1))) fx, fy = xs.reshape(-1), ys.reshape(-1) bad = 0 for i in range(len(bounds) - 1): a, b_ = bounds[i], bounds[i + 1] if b_ - a > 1 and not np.array_equal(fy[a:b_ - 1], fx[a + 1:b_]): bad += 1 if fy[b_ - 1] != IGNORE and i + 1 < len(bounds) - 1: # only legal when the document was carried into the next row pass check("packer: targets are next-token inside each document", bad == 0, f"{bad} documents mismatched") check("packer: positions restart at every document", all(pos.reshape(-1)[bounds[i]] == 0 for i in range(len(bounds) - 1))) # The RoPE table is exactly max_block_size long and is indexed by pos, so a # position past its end is an out-of-bounds gather -- a device-side assert # thousands of steps in, reported asynchronously at whatever kernel runs # next. Carried chunks of a long document used to keep counting, and with # max_doc_tokens at 65,536 they ran off the end. Sample where long-document # pressure is highest, at both context stages. lim = cfg.model.max_block_size worst, worst_at = 0, None for prog, blk in ((0.0, 256), (0.5, 256), (0.85, 512), (0.95, 512)): for _ in range(12): _x, _y, po, _b, bo, _st = p.batch(4, blk, prog) m = int(po.max()) if m > worst: worst, worst_at = m, (prog, blk) if not all(po.reshape(-1)[bo[i]] == 0 for i in range(len(bo) - 1)): worst, worst_at = lim + 1, (prog, blk) # force a failure check("packer: positions stay inside the RoPE table", worst < lim, f"max pos {worst:,} of {lim:,}" + (f" at progress {worst_at[0]}, block {worst_at[1]}" if worst_at else "")) # Resumability: the same state must replay the same batch. s = p.state_dict() a1 = p.batch(2, 256, 0.0)[0] p.load_state_dict(s) a2 = p.batch(2, 256, 0.0)[0] check("packer: a restored state replays the identical batch", np.array_equal(a1, a2)) from data import Prefetcher, validate_tokenization, open_validation pref = Prefetcher(p, cfg2, dev) pref.configure(2, 256, 0.0) pref.start() try: pref.next(2, 256) committed = pref.state_dict() reference = VarlenPacker(cfg2, {"src": st}, 7) reference.load_state_dict(committed) expected = reference.batch(2, 512)[0] pref.configure(2, 512, 0.0) actual = pref.next(2, 512)[0].cpu().numpy() check("prefetch: shape transition replays committed state", np.array_equal(actual, expected)) finally: pref.stop() def broken_batch(*args): raise ValueError("deliberate producer failure") p.batch = broken_batch pref = Prefetcher(p, cfg2, dev) pref.configure(2, 256, 0.0) pref.start() propagated = False try: pref.next(2, 256) except RuntimeError as exc: propagated = isinstance(exc.__cause__, ValueError) finally: pref.stop() check("prefetch: producer failure reaches training instead of hanging", propagated) rejected = False try: validate_tokenization(d, cfg) except ValueError: rejected = True check("data: rejects shards without matching tokenizer provenance", rejected) check("validation: current tokenizer provenance verified", open_validation(cfg).n_tokens > 0) def check_multistore(cfg): """Sharded sources must stitch back into one correctly indexed source. A source too large for one prepare process is split across workers by data file range; data.py presents the shard directories as one logical source rather than concatenating 200GB on disk. If the global-to-shard index mapping is wrong, training silently reads the wrong documents. """ import tempfile from data import MultiStore, SourceStore root = Path(_tmpdir()) rng = np.random.default_rng(0) truth, parts = [], [] for sh in range(3): d = root / f"src_s{sh}"; d.mkdir() lens = rng.integers(5, 50, size=7) docs = [rng.integers(1, 1000, size=L).astype(np.uint16) for L in lens] truth += docs np.concatenate(docs).tofile(d / "tokens.bin") np.concatenate([[0], np.cumsum(lens)]).astype(np.uint64).tofile(d / "docs.idx") parts.append(SourceStore("src", d / "tokens.bin", d / "docs.idx")) m = MultiStore("src", parts) check("sharded source: document count and token count add up", m.n_docs == len(truth) and m.n_tokens == sum(len(t) for t in truth)) check("sharded source: every global index maps to the right document", all(np.array_equal(m.doc(i), truth[i]) for i in range(m.n_docs))) exp = np.flatnonzero(np.array([len(t) for t in truth]) >= 30) check("sharded source: long-document selection spans shards", np.array_equal(m.long_docs(30), exp)) def check_checkpoint(cfg, dev): import checkpoint as C small = Config.from_json(cfg.to_json()) small.enforce_budget = False small.model.n_layer = 4 m = build(small, dev) muon, adamw = build_optimizers(m, small, log=lambda *a: None) for p in m.parameters(): p.grad = torch.randn_like(p) muon.step(); adamw.step() scaler = torch.amp.GradScaler("cuda") tmp = Path(_tmpdir()) / "c.pt" C.save_full(tmp, model=m, muon=muon, adamw=adamw, scaler=scaler, packer=None, step=5, tokens_seen=123, cfg=small, ngram_fingerprint="deadbeef", background=False) before = {k: v.clone() for k, v in m.state_dict().items()} for p in m.parameters(): p.data.add_(torch.randn_like(p)) blob = C.load_full(tmp, model=m, muon=muon, adamw=adamw, scaler=scaler, map_location=dev, expect_fingerprint=small.fingerprint(), expect_ngram="deadbeef") same = all(torch.equal(before[k].to(dev), v) for k, v in m.state_dict().items()) check("checkpoint: weights round-trip exactly", same) check("checkpoint: step and token count survive", blob["step"] == 5 and blob["tokens_seen"] == 123) mom = next(iter(muon.state.values())).get("momentum_buffer") check("checkpoint: Muon momentum buffers are restored", mom is not None) # The packer round-trip, on the real device. This check used to pass # packer=None, so the resume path was never actually exercised: the first # real --resume died on it. map_location= moves the packer's saved RNG # state onto the GPU, and a CPU Generator's set_state rejects a CUDA tensor. import data as D srcs = D.open_sources(cfg, log=lambda *a, **k: None) pk = D.VarlenPacker(cfg, srcs, seed=0) for _ in range(3): pk.batch(2, 256, 0.0) tmp2 = Path(_tmpdir()) / "p.pt" C.save_full(tmp2, model=m, muon=muon, adamw=adamw, scaler=scaler, packer=pk, step=7, tokens_seen=99, cfg=small, ngram_fingerprint="deadbeef", background=False) want = pk.batch(2, 256, 0.0)[0] pk2 = D.VarlenPacker(cfg, srcs, seed=0) ok_resume, err = True, "" try: C.load_full(tmp2, model=m, muon=muon, adamw=adamw, scaler=scaler, packer=pk2, map_location=dev, expect_fingerprint=small.fingerprint(), expect_ngram="deadbeef") except Exception as e: ok_resume, err = False, f"{type(e).__name__}: {e}" check("checkpoint: the packer resumes from a GPU map_location", ok_resume, err) if ok_resume: got = pk2.batch(2, 256, 0.0)[0] check("checkpoint: a resumed packer replays the identical next batch", np.array_equal(want, got), "same tokens" if np.array_equal(want, got) else "DIVERGED") try: C.load_full(tmp, expect_fingerprint="0000000000000000") check("checkpoint: a config change is refused, not silently accepted", False) except SystemExit: check("checkpoint: a config change is refused, not silently accepted", True) def check_tensor_cores(cfg, dev): """Prove fp16 autocast reaches a tensor-core kernel rather than sgemm.""" from torch.profiler import ProfilerActivity, profile a = torch.randn(4096, 512, device=dev) b = torch.randn(512, 2368, device=dev) with profile(activities=[ProfilerActivity.CUDA]) as pr: with torch.autocast("cuda", dtype=torch.float16): for _ in range(3): (a @ b).sum() torch.cuda.synchronize() names = " ".join(e.key for e in pr.key_averages()).lower() tc = any(s in names for s in ("h884", "884gemm", "hgemm", "s16816", "16816", "tensor", "cutlass")) check("fp16 GEMMs dispatch to a tensor-core kernel", tc, [e.key for e in pr.key_averages() if "gemm" in e.key.lower()][:1]) def check_down_proj_headroom(cfg, dev): """The FFN down projection's fp16 output range -- failure #2, step 4,520. Channel 228 of the residual is a massive activation, 173x the RMS (78,115 against 451 at block 16). The FFN writes into it, so this GEMM's fp16 OUTPUT crossed 65,504 and became inf, which norm_f turned into NaN. Inputs and weights were all finite; only the range ran out. DOWN_SHRINK pre-scales w_down by a power of two, which is exact and keeps the GEMM on tensor cores. """ import model as M peak = 65_504.0 / M.DOWN_SHRINK # what the output may now reach check("down proj: DOWN_SHRINK is a power of two (so the scaling is exact)", M.DOWN_SHRINK > 0 and abs(math.log2(M.DOWN_SHRINK) - round(math.log2(M.DOWN_SHRINK))) < 1e-12, f"{M.DOWN_SHRINK} = 2^{round(math.log2(M.DOWN_SHRINK))}") check("down proj: headroom covers the measured step-4,520 peak", peak > 4 * 65_504.0, f"overflows at {peak:,.0f} now, was 65,504 ({1/M.DOWN_SHRINK:.0f}x)") # the scaling must not change the result: same GEMM, both ways, in fp32 torch.manual_seed(0) x = (torch.randn(4096, 2368, device=dev, dtype=torch.float16) * 40) w = (torch.randn(512, 2368, device=dev, dtype=torch.float16) * 0.5) plain = F.linear(x.float(), w.float()) scaled = F.linear(x, w * M.DOWN_SHRINK).float() * (1.0 / M.DOWN_SHRINK) rel = float((scaled - plain).norm() / plain.norm()) check("down proj: pre-scaling the weight matches the unscaled GEMM", rel < 2e-3, f"rel-L2 {rel:.2e}") # And it must rescue the real failure shape: an output that lands just past # 65,504, the way step 4,520's did -- not an absurd one. Headroom is 8x, not # infinite, so the test is calibrated to the measured overshoot. torch.manual_seed(1) big = torch.randn(2048, 2368, device=dev, dtype=torch.float16) wb = torch.randn(512, 2368, device=dev, dtype=torch.float16) ref = F.linear(big.float(), wb.float()).abs().amax() big = (big * (1.6 * 65_504.0 / float(ref)) ** 0.5) # target ~1.6x over wb = (wb * (1.6 * 65_504.0 / float(ref)) ** 0.5) true_peak = float(F.linear(big.float(), wb.float()).abs().amax()) unscaled_bad = not torch.isfinite(F.linear(big, wb)).all() scaled = F.linear(big, wb * M.DOWN_SHRINK).float() * (1.0 / M.DOWN_SHRINK) scaled_ok = bool(torch.isfinite(scaled).all()) check("down proj: an output just past 65,504 overflows unscaled, survives scaled", bool(unscaled_bad) and scaled_ok, f"true peak {true_peak:,.0f}: unscaled overflows={bool(unscaled_bad)}, " f"scaled finite={scaled_ok}") def check_residual_range(cfg, dev): """The overflow that killed two runs, reproduced from the measured value. The residual stream grows with depth. At step 3,456 -- 1% into the budget -- it reached 65,617 at block 25 against fp16's maximum of 65,504. It saturates to inf, and _rmsnorm's rsqrt(inf) = 0 turns that into inf * 0 = NaN. No loss scale can help: the overflow is in the forward pass, so GradScaler halves its scale forever against a cause halving cannot touch. """ from model import _rmsnorm peak = 65_617.3 # measured, block 25, step 3,456 w = torch.ones(512, device=dev) x = torch.full((4, 512), peak, device=dev) h16 = x.to(torch.float16) check("residual: the measured peak really does overflow fp16", not torch.isfinite(h16).all(), f"{peak:,.0f} vs fp16 max 65,504") out16 = _rmsnorm(h16, w, cfg.model.norm_eps) check("residual: an overflowed fp16 residual makes RMSNorm produce NaN", bool(torch.isnan(out16).any()), "rsqrt(inf)=0 then inf*0=nan -- the exact failure") out32 = _rmsnorm(x, w, cfg.model.norm_eps) check("residual: the same value in fp32 stays finite", bool(torch.isfinite(out32).all()), f"max |out| {float(out32.abs().amax()):.3f}") check("residual: fp16_residual is off", cfg.train.fp16_residual is False, "fp32 residual stream (costs ~7.6% throughput)") def check_loss_scale_floor(cfg, dev): """The scale collapse that ended the step-3,810 run, reproduced and fenced. GradScaler has no floor. Once the scale falls below ~1e-30, unscale_ computes grad/scale, overflows fp32 to inf, sets found_inf, and halves the scale again -- self-reinforcing, no exit, every step skipped from then on. The run died that way with its weights still perfectly finite. This drives a scaler into that spiral and asserts the clamp in train.py stops it. """ w = torch.zeros(4, device=dev, requires_grad=True) sc = torch.amp.GradScaler("cuda", init_scale=cfg.train.init_scale) opt = torch.optim.SGD([w], lr=0.0) # Unclamped: 80 consecutive overflows. bare = torch.amp.GradScaler("cuda", init_scale=cfg.train.init_scale) for _ in range(80): bare.scale(torch.zeros((), device=dev)) # lazy-inits _scale w.grad = torch.full_like(w, float("inf")) bare.unscale_(opt); bare.step(opt); bare.update() collapsed = bare.get_scale() floored_steps = 0 for _ in range(80): sc.scale(torch.zeros((), device=dev)) w.grad = torch.full_like(w, float("inf")) sc.unscale_(opt); sc.step(opt); sc.update() if sc.get_scale() < cfg.train.min_loss_scale: sc.update(cfg.train.min_loss_scale); floored_steps += 1 held = sc.get_scale() check("loss scale: unclamped GradScaler does collapse (the bug)", collapsed < 1e-12, f"80 overflows drive it to {collapsed:.3g}") check("loss scale: the clamp holds it at the floor", held == cfg.train.min_loss_scale and math.isfinite(held), f"held at {held:g}, floored on {floored_steps} steps") # And the floor must be high enough that unscale_ cannot overflow fp32. safe = cfg.train.min_loss_scale > 1e-30 check("loss scale: the floor is above the fp32 unscale_ overflow point", safe, f"floor {cfg.train.min_loss_scale:g} vs ~1e-30") check("loss scale: the ceiling is at or below 2^15", cfg.train.max_loss_scale <= 2.0 ** 15, f"max {cfg.train.max_loss_scale:,.0f}") def check_head_range(cfg, dev): """The fused head's dW GEMM must not overflow fp16 on realistic text. g = S*(p - onehot): the true token carries -S, and dW sums it over every token in the chunk with the same target. Synthetic benchmark data is uniform over the vocabulary and never stacks a target often enough to matter; real text does. At GRAD_SCALE 4096 this saturated to Inf on step 1, Inf-Inf put NaN in the tied embedding gradient, and the run died on step 2. Assert on a batch of the actual corpus, and assert margin, not just finiteness -- |x| grows over a long run and this must not sit on the edge. """ import torch.nn.functional as F import data as D, ngram as N, model as M from pathlib import Path srcs = D.open_sources(cfg, log=lambda *a, **k: None) pk = D.VarlenPacker(cfg, srcs, seed=0) m = M.Boris2(cfg).to(dev) if m.ngram is not None: m.ngram.load_vocab(N.NGramVocab.load(cfg.ngram.vocab_path), dev) m.train() x, y, pos, bnd, bounds, st = pk.batch(8, 2048, 0.0) t = lambda a, dt: torch.as_tensor(a, dtype=dt, device=dev) with torch.no_grad(), torch.autocast("cuda", dtype=torch.float16): h = m(t(x, torch.long), t(pos, torch.long), t(bounds, torch.long), t(bnd, torch.bool), None, 10_000.0, st["min_seqlen"], st["max_seqlen"]) xf = h.reshape(-1, h.shape[-1]) tg = t(y, torch.long).reshape(-1) # Test the actual custom Function, not a separately reimplemented GEMM. # The old test bypassed its broken fp16 rescale and therefore passed while # production overflowed. Compare full gradients with fp32 cross-entropy. c = cfg.optim.lm_head_chunk for magnitude, repeated in ((1.0, False), (64.0, True)): xx = (xf[:c].detach() * magnitude).requires_grad_() ww = m.head_weight().detach().clone().requires_grad_() tt = tg[:c].clone() if repeated: tt.fill_(int(tg[tg != IGNORE][0])) with torch.autocast("cuda", dtype=torch.float16): loss = M.fused_lm_loss(xx, ww, tt, chunk=c) loss.backward() xr = xx.detach().float().requires_grad_() wr = ww.detach().clone().requires_grad_() ref = F.cross_entropy(F.linear(xr, wr), tt, ignore_index=IGNORE) ref.backward() rel = (ww.grad - wr.grad).norm() / wr.grad.norm().clamp_min(1e-12) check(f"production head real text: magnitude={magnitude:g}, repeated={repeated}", bool(torch.isfinite(ww.grad).all()) and float(rel) < .01, f"dW relative error {float(rel):.3g}") del xx, ww, xr, wr, loss, ref del m torch.cuda.empty_cache() def check_compile(cfg, dev): """The compiled regions must produce the same numbers as eager.""" small = Config.from_json(cfg.to_json()) small.enforce_budget = False small.model.n_layer = 4 torch.manual_seed(0) me = build(small, dev, compile_regions=False) sd = me.state_dict() torch.manual_seed(0) mc = build(small, dev, compile_regions=True) mc.load_state_dict(sd) for m_ in (me, mc): for blk in m_.blocks: torch.nn.init.normal_(blk.attn.canon, std=0.3) mc.load_state_dict(me.state_dict()) x, pos, bounds, bnd = synth_batch(small, dev, 1, 512, [[200, 312]], seed=1) with torch.autocast("cuda", dtype=torch.float16): a = me(x, pos, bounds, bnd, None, 10_000.0) b = mc(x, pos, bounds, bnd, None, 10_000.0) r = float((a.float() - b.float()).norm() / (a.float().norm() + 1e-9)) check("compiled regions match eager", r < 5e-3, f"rel-L2 {r:.2e}") # --------------------------------------------------------------------------- # def main(): ap = argparse.ArgumentParser() ap.add_argument("--device", default="cuda:0") ap.add_argument("--quick", action="store_true") a = ap.parse_args() dev = a.device cfg = Config() torch.cuda.set_device(dev) torch.backends.cuda.matmul.allow_tf32 = False print(f"\n verify: {cfg.run_name} on {dev}") print(" " + "=" * 74) print("\n budget and optimizer split") m = build(cfg, dev) build_optimizers(m, cfg) del m torch.cuda.empty_cache() print(" numerics") check_fused_head(cfg, dev) check_repeated_target_head(cfg, dev) check_head_range(cfg, dev) check_loss_scale_floor(cfg, dev) check_residual_range(cfg, dev) check_down_proj_headroom(cfg, dev) check_muon(cfg, dev) torch.cuda.empty_cache() print("\n isolation") check_canon_causality(cfg, dev) check_attention_matches_loop(cfg, dev) check_volta_kernels(cfg, dev) check_ngram_isolation(cfg, dev) check_ngram_ablation(cfg, dev) check_document_isolation(cfg, dev) torch.cuda.empty_cache() print("\n schedule and data") check_schedule(cfg) check_packer(cfg, dev) check_multistore(cfg) print("\n plumbing") check_tensor_cores(cfg, dev) check_checkpoint(cfg, dev) if not a.quick: check_compile(cfg, dev) print("\n " + "=" * 74) print(f" {len(PASS)} passed, {len(FAIL)} failed") for f in FAIL: print(f" FAILED: {f}") sys.exit(1 if FAIL else 0) if __name__ == "__main__": main()