summaryrefslogtreecommitdiff
path: root/verify.py
diff options
context:
space:
mode:
authorjdj <jdj@kunix.org>2026-11-18 14:47:33 -0800
committerjdj <jdj@kunix.org>2026-11-18 14:47:33 -0800
commit5d2e4252e695644c34be605b44a14bd80a98a0a6 (patch)
tree311a72eda4459af7547b9cd05c0956eafb235f2d /verify.py
downloadboris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.tar.gz
boris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.zip
Boris-2 training code
The code, configuration and data filters used to train Boris-2 (144M, 200B tokens, single V100): model, training loop, data preparation, n-gram table, V100 kernels, final stages and the exact run configs.
Diffstat (limited to 'verify.py')
-rw-r--r--verify.py1164
1 files changed, 1164 insertions, 0 deletions
diff --git a/verify.py b/verify.py
new file mode 100644
index 0000000..9418e07
--- /dev/null
+++ b/verify.py
@@ -0,0 +1,1164 @@
+"""Correctness checks. Run before starting a run, and after touching a kernel.
+
+ python verify.py [--device cuda:0] [--quick]
+
+The checks that matter most are the isolation ones. Boris-2 packs several
+documents into every row, and three separate mechanisms have to keep them apart:
+varlen attention, the Canon boundary mask, and the n-gram history mask. A leak
+in any of them is invisible in the loss curve -- the model simply learns from
+context it will never have at inference.
+
+Note how check_document_isolation de-zeroes the Canon kernels first. They are
+zero-initialised, so at init the convolution contributes exactly nothing and the
+test passes no matter how badly it leaks. The 75M run shipped that bug for weeks
+behind a test that looked green.
+"""
+from __future__ import annotations
+
+import argparse
+import math
+import os
+import sys
+import atexit
+import shutil
+import tempfile
+from pathlib import Path
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from config import Config
+from model import Boris2, IGNORE, fused_lm_loss, _pre_attn
+from ngram import NGramVocab
+from optim import Muon, build_optimizers, newton_schulz
+from schedule import lr_scale, mixture_at, context_at
+
+PASS, FAIL = [], []
+
+
+_TMPDIRS = []
+
+
+def _tmpdir():
+ """mkdtemp that actually cleans up.
+
+ verify.py used bare tempfile.mkdtemp() and never removed anything. /tmp here
+ is a 16 GB tmpfs -- RAM -- and each run leaves ~500 MB of packer fixtures
+ behind, so 89 runs had filled it completely. That is not just a disk
+ problem: it was eating the machine's memory and getting background jobs
+ OOM-killed.
+ """
+ d = tempfile.mkdtemp()
+ _TMPDIRS.append(d)
+ return d
+
+
+@atexit.register
+def _cleanup_tmpdirs():
+ for d in _TMPDIRS:
+ shutil.rmtree(d, ignore_errors=True)
+
+
+def check(name, cond, detail=""):
+ (PASS if cond else FAIL).append(name)
+ mark = "ok " if cond else "FAIL"
+ print(f" [{mark}] {name}" + (f" {detail}" if detail else ""))
+ return cond
+
+
+# --------------------------------------------------------------------------- #
+def synth_vocab(cfg, seed=0):
+ V = cfg.model.vocab_size
+ rng = np.random.default_rng(seed)
+ bk = np.unique(rng.integers(0, V * V, size=cfg.ngram.bigram_rows).astype(np.int64))
+ tk = np.unique(rng.integers(0, V * V * V, size=cfg.ngram.trigram_rows).astype(np.int64))
+ return NGramVocab(V, bk, np.arange(bk.size) + cfg.ngram.bigram_base,
+ tk, np.arange(tk.size) + cfg.ngram.trigram_base)
+
+
+def synth_batch(cfg, dev, B, T, lens_per_row, seed=0):
+ g = torch.Generator().manual_seed(seed)
+ V = cfg.model.vocab_size
+ x = torch.randint(0, V, (B, T), generator=g).to(dev)
+ pos = torch.zeros(B, T, dtype=torch.int64)
+ bnd = torch.zeros(B, T, dtype=torch.bool)
+ bounds = []
+ for b, lens in enumerate(lens_per_row):
+ p = 0
+ for L in lens:
+ bounds.append(b * T + p)
+ bnd[b, p] = True
+ pos[b, p:p + L] = torch.arange(L)
+ p += L
+ assert p == T, "row must be exactly full"
+ bounds.append(B * T)
+ return (x, pos.to(dev), torch.tensor(bounds, dtype=torch.int64).to(dev), bnd.to(dev))
+
+
+def build(cfg, dev, compile_regions=False):
+ cfg.train.compile_regions = compile_regions
+ m = Boris2(cfg).to(dev)
+ if m.ngram is not None:
+ m.ngram.load_vocab(synth_vocab(cfg), dev)
+ if compile_regions:
+ m.maybe_compile()
+ return m
+
+
+# --------------------------------------------------------------------------- #
+def check_fused_head(cfg, dev):
+ """The fused head must equal an ordinary cross-entropy, loss and gradient."""
+ torch.manual_seed(0)
+ N, d, V = 1024, cfg.model.d_model, cfg.model.vocab_size
+ x = torch.randn(N, d, device=dev, dtype=torch.float16, requires_grad=True)
+ W = (torch.randn(V, d, device=dev) * 0.02).requires_grad_(True)
+ t = torch.randint(0, V, (N,), device=dev)
+ t[::7] = IGNORE
+
+ loss = fused_lm_loss(x, W, t, chunk=256)
+ loss.backward()
+ gx, gW = x.grad.clone(), W.grad.clone()
+
+ # Reference in fp32. Comparing against the fp16 unfused head would compare
+ # two approximations: its grad_logits reach the GEMM at ~3e-8, far below
+ # what fp16 can represent, so it is the less accurate of the two.
+ x2 = x.detach().float().clone().requires_grad_(True)
+ W2 = W.detach().float().clone().requires_grad_(True)
+ ref = F.cross_entropy(F.linear(x2, W2), t, ignore_index=IGNORE)
+ ref.backward()
+
+ dl = abs(float(loss) - float(ref))
+ rx = (gx.float() - x2.grad).norm() / (x2.grad.norm() + 1e-9)
+ rw = (gW - W2.grad).norm() / (W2.grad.norm() + 1e-9)
+ check("fused LM head: loss matches fp32 cross_entropy", dl < 5e-3, f"|d| = {dl:.2e}")
+ check("fused LM head: dx matches fp32", float(rx) < 5e-3, f"rel-L2 {float(rx):.2e}")
+ check("fused LM head: dW matches fp32", float(rw) < 5e-3, f"rel-L2 {float(rw):.2e}")
+ # The point of the fusion: never hold (N, V) logits. Measured against the
+ # unfused head on the same inputs, at a realistic token count.
+ def peak_of(fn, N2=8192):
+ xx = torch.randn(N2, d, device=dev, dtype=torch.float16, requires_grad=True)
+ WW = (torch.randn(V, d, device=dev) * 0.02).requires_grad_(True)
+ tt = torch.randint(0, V, (N2,), device=dev)
+ torch.cuda.synchronize(); torch.cuda.empty_cache()
+ base = torch.cuda.memory_allocated(dev)
+ torch.cuda.reset_peak_memory_stats(dev)
+ fn(xx, WW, tt).backward()
+ torch.cuda.synchronize()
+ out = torch.cuda.max_memory_allocated(dev) - base
+ del xx, WW, tt
+ torch.cuda.empty_cache()
+ return out
+
+ fused_peak = peak_of(lambda a, b, c: fused_lm_loss(a, b, c))
+ plain_peak = peak_of(lambda a, b, c: F.cross_entropy(
+ F.linear(a, b.to(a.dtype)).float(), c, ignore_index=IGNORE))
+ check("fused LM head: uses far less memory than the unfused head",
+ fused_peak < plain_peak / 3,
+ f"{fused_peak/2**20:.0f} MB vs {plain_peak/2**20:.0f} MB "
+ f"({plain_peak/max(fused_peak,1):.1f}x)")
+
+
+def check_repeated_target_head(cfg, dev):
+ """Exercise the production Function under autocast, including its rescale."""
+ torch.manual_seed(91)
+ for dtype in (torch.float32, torch.float16):
+ x = torch.full((1024, 32), 16.0, device=dev, dtype=dtype, requires_grad=True)
+ w = (torch.randn(256, 32, device=dev) * .01).requires_grad_()
+ target = torch.zeros(1024, device=dev, dtype=torch.long)
+ with torch.autocast("cuda", dtype=torch.float16):
+ loss = fused_lm_loss(x, w, target, chunk=1024)
+ loss.backward()
+ xr = x.detach().float().requires_grad_()
+ wr = w.detach().clone().requires_grad_()
+ ref = F.cross_entropy(F.linear(xr, wr), target)
+ ref.backward()
+ rel = (w.grad - wr.grad).norm() / wr.grad.norm()
+ check(f"production head repeated-target dW {dtype}",
+ bool(torch.isfinite(w.grad).all()) and float(rel) < .005,
+ f"relative error {float(rel):.3g}")
+
+
+def check_canon_causality(cfg, dev):
+ """A change at position t must never move an output at t' < t."""
+ torch.manual_seed(0)
+ C, K = cfg.model.canon_channels, cfg.model.canon_kernel
+ B, T = 1, 64
+ x = torch.randn(B, T, cfg.model.d_model, device=dev, dtype=torch.float16)
+ w_qkv = torch.randn(C, cfg.model.d_model, device=dev, dtype=torch.float16) * 0.02
+ canon = torch.randn(C, K, device=dev) * 0.5
+ m1 = torch.ones(B, T, 1, device=dev, dtype=torch.float16)
+ m2 = torch.ones(B, T, 1, device=dev, dtype=torch.float16)
+ hd = cfg.model.head_dim
+ cos = torch.ones(B, T, 1, hd // 2, device=dev)
+ sin = torch.zeros(B, T, 1, hd // 2, device=dev)
+ nw = torch.ones(cfg.model.d_model, device=dev)
+ args = (nw, 1e-5, w_qkv, canon, m1, m2, cos, sin, cfg.model.n_head,
+ cfg.model.n_kv_head, hd, True)
+ q0 = _pre_attn(x, *args)[0]
+ x2 = x.clone(); x2[0, 32] += 10.0
+ q1 = _pre_attn(x2, *args)[0]
+ check("Canon is strictly causal", torch.equal(q0[0, :32], q1[0, :32]))
+ moved = int((q0[0, 32:35] != q1[0, 32:35]).any(-1).any(-1).sum())
+ check("Canon reaches exactly kernel-1 tokens forward", moved == 3,
+ f"{moved} of the next 3 positions moved")
+
+
+def check_document_isolation(cfg, dev):
+ """End-to-end: a change in one document must not move another.
+
+ Canon kernels are randomised first. Zero-initialised, they contribute
+ nothing and this test would pass on a model that leaks badly.
+ """
+ torch.manual_seed(0)
+ small = Config.from_json(cfg.to_json())
+ small.enforce_budget = False
+ small.model.n_layer = 6
+ m = build(small, dev)
+ for blk in m.blocks: # de-zero: the whole point
+ torch.nn.init.normal_(blk.attn.canon, std=0.5)
+
+ B, T = 1, 512
+ lens = [[200, 312]]
+ x, pos, bounds, bnd = synth_batch(small, dev, B, T, lens, seed=3)
+ with torch.autocast("cuda", dtype=torch.float16):
+ h0 = m(x, pos, bounds, bnd, None, 10_000.0)
+ x2 = x.clone(); x2[0, :200] = torch.randint(0, small.model.vocab_size,
+ (200,), device=dev)
+ h1 = m(x2, pos, bounds, bnd, None, 10_000.0)
+ drift = float((h0[0, 200:] - h1[0, 200:]).abs().max())
+ check("document isolation: rewriting doc A never moves doc B", drift == 0.0,
+ f"max logit drift {drift:.3e}")
+
+ same = float((h0[0, :200] - h1[0, :200]).abs().max())
+ check("the probe actually perturbs doc A", same > 0.0, f"max change {same:.3e}")
+
+
+def check_attention_matches_loop(cfg, dev):
+ """Nested varlen attention must equal a per-document causal SDPA."""
+ torch.manual_seed(0)
+ H, KV, D = cfg.model.n_head, cfg.model.n_kv_head, cfg.model.head_dim
+ lens = [300, 212, 512]
+ N = sum(lens)
+ off = torch.tensor([0] + list(np.cumsum(lens)), device=dev)
+ q = torch.randn(N, H, D, device=dev, dtype=torch.float16)
+ k = torch.randn(N, H, D, device=dev, dtype=torch.float16)
+ v = torch.randn(N, H, D, device=dev, dtype=torch.float16)
+ nj = lambda t: torch.nested.nested_tensor_from_jagged(t, off).transpose(1, 2)
+ o = F.scaled_dot_product_attention(nj(q), nj(k), nj(v), is_causal=True)
+ o = o.transpose(1, 2).values()
+ ref = torch.empty_like(q)
+ p = 0
+ for L in lens:
+ s = F.scaled_dot_product_attention(
+ q[p:p+L].transpose(0, 1)[None], k[p:p+L].transpose(0, 1)[None],
+ v[p:p+L].transpose(0, 1)[None], is_causal=True)
+ ref[p:p+L] = s[0].transpose(0, 1)
+ p += L
+ d = float((o.float() - ref.float()).abs().max())
+ check("varlen attention equals a per-document loop", d == 0.0, f"max diff {d:.1e}")
+
+
+def check_volta_kernels(cfg, dev):
+ """volta_kernels/: the sm_70 attention and LM head, held to the stock paths.
+
+ Runs whenever the device is sm_70, whether or not BORIS_VOLTA_KERNELS is set,
+ so the kernels are proven before anyone opts in. The bar is the stock path's
+ own error against an fp64 reference, measured here on the same inputs --
+ "same precision" as a number, not an adjective.
+ """
+ if torch.cuda.get_device_capability(dev) != (7, 0):
+ print(" [skip] volta kernels: device is not sm_70")
+ return
+ import model as M
+ import volta_kernels as VK
+ from volta_kernels.head import fused_lm_loss as volta_lm_loss
+ torch.manual_seed(0)
+ H, D = cfg.model.n_head, cfg.model.head_dim
+ # Edge lengths around the 64-row tile, a length-1 document, and one long one.
+ lens = [1, 63, 64, 65, 127, 128, 300, 1300]
+ N = sum(lens)
+ off = torch.tensor([0] + list(np.cumsum(lens)), device=dev)
+ # v as a strided view of a fused qkv, exactly as the model hands it over.
+ qkv = (torch.randn(N, 3 * H * D, device=dev) * 1.5).half()
+ q = qkv[:, :H * D].reshape(N, H, D).contiguous().requires_grad_()
+ k = qkv[:, H * D:2 * H * D].reshape(N, H, D).contiguous().requires_grad_()
+ vb = qkv.clone().requires_grad_()
+ v = vb[:, 2 * H * D:].view(N, H, D)
+ go = torch.randn(N, H, D, device=dev).half()
+
+ def run(fn):
+ for t in (q, k, vb):
+ t.grad = None
+ o = fn()
+ o.backward(go)
+ return [o.detach().double(), q.grad.double(), k.grad.double(),
+ vb.grad[:, 2 * H * D:].reshape(N, H, D).double()]
+
+ def stock():
+ nj = lambda t: torch.nested.nested_tensor_from_jagged(t, off).transpose(1, 2)
+ return F.scaled_dot_product_attention(nj(q), nj(k), nj(v), is_causal=True
+ ).transpose(1, 2).values()
+
+ qq, kk, vv = (t.detach().double().requires_grad_() for t in (q, k, v))
+ outs, p = [], 0
+ for L in lens:
+ sl = slice(p, p + L)
+ outs.append(F.scaled_dot_product_attention(
+ qq[sl].transpose(0, 1), kk[sl].transpose(0, 1), vv[sl].transpose(0, 1),
+ is_causal=True).transpose(0, 1))
+ p += L
+ ref_o = torch.cat(outs)
+ ref_o.backward(go.double())
+ ref = [ref_o.detach(), qq.grad, kk.grad, vv.grad]
+ S = run(stock)
+ V = run(lambda: VK.varlen_causal_attention(q, k, v, off))
+ rel = lambda a, b: float((a - b).norm() / b.norm())
+ for name, i in (("output", 0), ("dq", 1), ("dk", 2), ("dv", 3)):
+ es, ev = rel(S[i], ref[i]), rel(V[i], ref[i])
+ check(f"volta attention: {name} as accurate as memory-efficient SDPA",
+ ev <= 1.1 * es and math.isfinite(ev), f"rel-L2 vs fp64 {ev:.2e} (stock {es:.2e})")
+
+ # Isolation and causality, exactly: perturb one document / one late token.
+ def fwd_only(qx, kx, vx):
+ return VK.varlen_causal_attention(qx, kx, vx, off.clone()).float()
+ q0, k0, v0 = (t.detach().clone() for t in (q, k, v))
+ o0 = fwd_only(q0, k0, v0)
+ a, b = int(off[6]), int(off[7]) # the 300-token document
+ q1, k1, v1 = q0.clone(), k0.clone(), v0.clone()
+ for t in (q1, k1, v1):
+ t[a:b] = torch.randn_like(t[a:b])
+ o1 = fwd_only(q1, k1, v1)
+ outside = torch.cat([o1[:a] - o0[:a], o1[b:] - o0[b:]]).abs().max().item()
+ check("volta attention: rewriting one document never moves another",
+ outside == 0.0 and (o1[a:b] - o0[a:b]).abs().max().item() > 0,
+ f"max drift outside {outside:.1e}")
+ t_ = a + 150
+ k2, v2 = k0.clone(), v0.clone()
+ k2[t_] = torch.randn_like(k2[t_]); v2[t_] = torch.randn_like(v2[t_])
+ o2 = fwd_only(q0, k2, v2)
+ before = (o2[a:t_] - o0[a:t_]).abs().max().item()
+ at = (o2[t_] - o0[t_]).abs().max().item()
+ check("volta attention: causal (a key never reaches an earlier query)",
+ before == 0.0 and at > 0, f"drift before {before:.1e}, at {at:.1e}")
+
+ # The per-offsets tile table must not be served stale after an in-place edit.
+ o3 = off.clone()
+ _, t_before, _ = VK.tile_table(o3) # doc 0 has 1 token: 1 tile
+ o3[1] = 65 # in place: now 65 tokens, 2 tiles
+ cu, tcum, perm = VK.tile_table(o3)
+ fresh = (o3[1:] - o3[:-1] + 63) // 64 # what the table must now say
+ check("volta attention: tile table follows in-place edits of the offsets",
+ int(cu[1]) == 65 and int(tcum[-1]) == int(fresh.clamp_min(0).sum())
+ and int(fresh[int(perm[0])]) == int(fresh.max()))
+ lens_ = (cu[1:] - cu[:-1] + 63) // 64
+ check("volta attention: tile table launches the longest documents first",
+ bool((lens_[perm.long()][:-1] >= lens_[perm.long()][1:]).all())
+ and sorted(perm.tolist()) == list(range(len(lens))))
+
+ # LM head: the one-pass Triton softmax against the production head.
+ torch.manual_seed(1)
+ Vv, d = cfg.model.vocab_size, cfg.model.d_model
+ Nn = 2048
+ W = (torch.randn(Vv, d, device=dev) * 0.05)
+ x = torch.randn(Nn, d, device=dev) * 3.0
+ y = torch.randint(0, Vv, (Nn,), device=dev)
+ y[::5] = IGNORE
+
+ def head(fn, xx, yy):
+ xx = xx.clone().requires_grad_(); ww = W.clone().requires_grad_()
+ with torch.autocast("cuda", dtype=torch.float16):
+ loss = fn(xx, ww, yy, 1024)
+ loss.backward()
+ return loss.detach().double(), xx.grad.double(), ww.grad.double()
+
+ was = M.VOLTA_KERNELS
+ M.VOLTA_KERNELS = False
+ try:
+ P = head(M.fused_lm_loss, x, y)
+ finally:
+ M.VOLTA_KERNELS = was
+ Q = head(volta_lm_loss, x, y)
+ xr = x.double().requires_grad_(); wr = W.double().requires_grad_()
+ lr_ = F.cross_entropy(xr @ wr.T, y, ignore_index=IGNORE)
+ lr_.backward()
+ check("volta head: loss equals the production head",
+ abs(float(Q[0] - P[0])) <= 1e-6 * abs(float(P[0])), f"{float(Q[0]):.7f} vs {float(P[0]):.7f}")
+ for name, i, r in (("dx", 1, xr.grad), ("dW", 2, wr.grad)):
+ ep, eq = rel(P[i], r), rel(Q[i], r)
+ check(f"volta head: {name} as accurate as the production head",
+ eq <= 1.02 * ep, f"rel-L2 vs fp64 {eq:.3e} (production {ep:.3e})")
+ # The dW overflow guard survives: one token repeated through a whole chunk.
+ xs = x[:1].repeat(1024, 1) * 5.0
+ ys = y.clone()[:1024]; ys[:] = 7
+ _, dxs, dws = head(volta_lm_loss, xs, ys)
+ check("volta head: a chunk of one repeated token stays finite",
+ bool(torch.isfinite(dws).all() and torch.isfinite(dxs).all()))
+
+ # Stage two (80% of the run) packs documents up to 8192 tokens. fp64 would
+ # need a 4 GB score matrix, so the reference is fp32 memory-efficient SDPA.
+ torch.manual_seed(3)
+ lens8 = [8192, 3000, 5000]
+ N8 = sum(lens8)
+ off8 = torch.tensor([0] + list(np.cumsum(lens8)), device=dev)
+ q8, k8, v8, g8 = (torch.randn(N8, H, D, device=dev).half() for _ in range(4))
+
+ def attn_grads(fn, dtype):
+ # clone: for fp16, .to() returns the SAME tensor, and the two runs would
+ # accumulate into each other's .grad
+ ts = [t.detach().to(dtype).clone().requires_grad_() for t in (q8, k8, v8)]
+ o = fn(*ts)
+ o.backward(g8.to(dtype))
+ return [o.detach().double()] + [t.grad.double() for t in ts]
+
+ def nested(a, b, c):
+ nj = lambda t: torch.nested.nested_tensor_from_jagged(t, off8).transpose(1, 2)
+ return F.scaled_dot_product_attention(nj(a), nj(b), nj(c), is_causal=True
+ ).transpose(1, 2).values()
+ ref8 = attn_grads(nested, torch.float32)
+ S8 = attn_grads(nested, torch.float16)
+ V8 = attn_grads(lambda a, b, c: VK.varlen_causal_attention(a, b, c, off8), torch.float16)
+ worst = max(rel(V8[i], ref8[i]) / rel(S8[i], ref8[i]) for i in range(4))
+ check("volta attention: an 8192-token document as accurate as stock SDPA",
+ worst <= 1.1, f"worst error ratio vs stock {worst:.3f} (o, dq, dk, dv vs fp32)")
+
+ # A head call whose row count is not a multiple of the 1024-row chunk.
+ xp, yp = x[:1024 + 300], y[:1024 + 300]
+ M.VOLTA_KERNELS = False
+ try:
+ Pp = head(M.fused_lm_loss, xp, yp)
+ finally:
+ M.VOLTA_KERNELS = was
+ Qp = head(volta_lm_loss, xp, yp)
+ check("volta head: a partial final chunk matches the production head",
+ abs(float(Qp[0] - Pp[0])) <= 1e-6 * abs(float(Pp[0]))
+ and rel(Qp[1], Pp[1]) < 1e-3 and rel(Qp[2], Pp[2]) < 1e-3,
+ f"dx {rel(Qp[1], Pp[1]):.1e}, dW {rel(Qp[2], Pp[2]):.1e} apart")
+
+ # Fused Canon + RoPE: precision against fp64 next to the eager region, and
+ # document isolation with non-zero Canon weights (zero-init would hide a leak).
+ from volta_kernels.canon_rope_cuda import canon_rope # what the model uses
+ torch.manual_seed(4)
+ Bc, Tc = 2, 256
+ bndc = torch.zeros(Bc, Tc, dtype=torch.bool, device=dev)
+ bndc[:, 0] = True; bndc[0, 100] = True; bndc[1, 7] = True; bndc[1, 8] = True
+ b2c = bndc | F.pad(bndc, (1, 0))[:, :-1]
+ m1c = (~bndc).float()[:, :, None]; m2c = (~b2c).float()[:, :, None]
+ posc = torch.zeros(Bc, Tc, dtype=torch.long, device=dev)
+ for b in range(Bc):
+ p0 = 0
+ for t_ in range(Tc):
+ if bndc[b, t_]: p0 = t_
+ posc[b, t_] = t_ - p0
+ invc = 1.0 / (10000.0 ** (torch.arange(0, 64, 2, device=dev).float() / 64))
+ fc = torch.outer(torch.arange(Tc, device=dev).float(), invc)
+ cosc, sinc = fc.cos()[posc].unsqueeze(2), fc.sin()[posc].unsqueeze(2)
+ qkvc = (torch.randn(Bc, Tc, 1536, device=dev) * 2).half()
+ cwc = torch.randn(1536, 3, device=dev) * 0.3
+ gsc = [torch.randn(Bc, Tc, 8, 64, device=dev).half() for _ in range(3)]
+
+ def region(a, w_, dt):
+ ww = w_.to(torch.float16).to(dt)
+ s1 = F.pad(a, (0, 0, 1, 0))[:, :-1] * m1c.to(dt)
+ s2 = F.pad(a, (0, 0, 2, 0))[:, :-2] * m2c.to(dt)
+ y_ = a + a * ww[:, 0] + s1 * ww[:, 1] + s2 * ww[:, 2]
+ c_, s_ = cosc.to(dt), sinc.to(dt)
+ def rope(t_):
+ a_, b_ = t_.chunk(2, dim=-1)
+ return torch.cat([a_ * c_ - b_ * s_, a_ * s_ + b_ * c_], -1)
+ v_ = lambda i: y_[..., 512 * i:512 * (i + 1)].reshape(Bc, Tc, 8, 64)
+ return rope(v_(0)), rope(v_(1)), v_(2)
+
+ def cr_run(fn, dt):
+ a = qkvc.detach().to(dt).clone().requires_grad_()
+ w_ = cwc.detach().clone().requires_grad_()
+ outs = fn(a, w_)
+ torch.autograd.backward(outs, [g.to(outs[0].dtype) for g in gsc])
+ return [o.detach().double() for o in outs] + [a.grad.double()]
+ ref_c = cr_run(lambda a, w_: region(a, w_, torch.float64), torch.float64)
+ eag_c = cr_run(lambda a, w_: tuple(o.to(torch.float16) for o in
+ region(a.float(), w_, torch.float32)), torch.float16)
+ fus_c = cr_run(lambda a, w_: canon_rope(a, w_, m1c, m2c, cosc, sinc), torch.float16)
+ worst = max(rel(fus_c[i], ref_c[i]) / rel(eag_c[i], ref_c[i]) for i in range(4))
+ check("volta canon+rope: q, k, v, dqkv as accurate as the eager region",
+ worst <= 1.1, f"worst error ratio {worst:.3f}")
+ q0_, k0_, v0_ = canon_rope(qkvc, cwc, m1c, m2c, cosc, sinc)
+ pert = qkvc.clone(); pert[0, :100] = torch.randn_like(pert[0, :100]).half()
+ q1_, k1_, v1_ = canon_rope(pert, cwc, m1c, m2c, cosc, sinc)
+ leak = max(float((a_[0, 100:].float() - b_[0, 100:].float()).abs().max())
+ for a_, b_ in ((q0_, q1_), (k0_, k1_), (v0_, v1_)))
+ moved = float((v1_[0, :100].float() - v0_[0, :100].float()).abs().max())
+ check("volta canon+rope: a document boundary stops the Canon taps exactly",
+ leak == 0.0 and moved > 0, f"drift past boundary {leak:.1e}")
+
+ # Whole model, both paths, against an fp32 (no autocast) reference, at the
+ # run's loss scale. Canon de-zeroed so every mechanism is live.
+ small = Config.from_json(cfg.to_json())
+ small.enforce_budget = False
+ small.model.n_layer = 4
+ torch.manual_seed(2)
+ m = build(small, dev)
+ for blk in m.blocks:
+ torch.nn.init.normal_(blk.attn.canon, std=0.1)
+ lens2 = [[700, 64, 1, 1283], [2048]]
+ xb, pb, ob, bb = synth_batch(small, dev, 2, 2048, lens2, seed=5)
+ yb = torch.roll(xb, -1, 1)
+
+ def grads(volta, amp):
+ M.VOLTA_KERNELS = volta
+ try:
+ for prm in m.parameters():
+ prm.grad = None
+ with torch.autocast("cuda", dtype=torch.float16, enabled=amp):
+ loss = m(xb, pb, ob.clone(), bb, yb, 10_000.0)
+ (loss * 32768.0).backward()
+ finally:
+ M.VOLTA_KERNELS = was
+ return {n: prm.grad.double() / 32768.0 for n, prm in m.named_parameters()
+ if prm.grad is not None}
+ gr, gs, gv = grads(False, False), grads(False, True), grads(True, True)
+ # The production path with checkpointing OFF merges each block's FFN half
+ # with the next block's norm+QKV into one region (model._forward_merged).
+ small.train.grad_checkpoint = False
+ gmerged = grads(True, True)
+ m._no_merge = True
+ gunmerged = grads(True, True)
+ m._no_merge = False
+ small.train.grad_checkpoint = True
+ em = sorted(rel(gmerged[n], gr[n]) for n in gr)
+ eu = sorted(rel(gunmerged[n], gr[n]) for n in gr)
+ check("volta merged regions: gradients as close to fp32 as per-block regions",
+ em[len(em) // 2] <= 1.05 * eu[len(eu) // 2] and max(em) <= 1.05 * max(eu),
+ f"median {em[len(em) // 2]:.3e} (per-block {eu[len(eu) // 2]:.3e})")
+ es = sorted(rel(gs[n], gr[n]) for n in gr)
+ ev = sorted(rel(gv[n], gr[n]) for n in gr)
+ med_s, med_v = es[len(es) // 2], ev[len(ev) // 2]
+ check("volta model: gradients as close to fp32 as the stock fp16 path",
+ med_v <= 1.1 * med_s and max(ev) <= 1.25 * max(es),
+ f"median rel-L2 {med_v:.3e} (stock {med_s:.3e}), worst {max(ev):.3e} (stock {max(es):.3e})")
+
+
+def check_ngram_isolation(cfg, dev):
+ """The n-gram key must not span a document boundary."""
+ m = build(cfg, dev)
+ B, T = 1, 32
+ x = torch.arange(T, device=dev)[None] % cfg.model.vocab_size
+ bnd = torch.zeros(B, T, dtype=torch.bool, device=dev)
+ bnd[0, 0] = True; bnd[0, 16] = True
+ out = m.ngram(x, bnd, torch.float32)
+ x2 = x.clone(); x2[0, 14:16] = (x2[0, 14:16] + 7) % cfg.model.vocab_size
+ out2 = m.ngram(x2, bnd, torch.float32)
+ d = float((out[0, 16:] - out2[0, 16:]).abs().max())
+ check("n-gram history stops at a document boundary", d == 0.0, f"max diff {d:.1e}")
+
+
+def check_muon(cfg, dev):
+ """Batched Newton-Schulz must agree with the unbatched one, and orthogonalise."""
+ torch.manual_seed(0)
+ G = torch.randn(4, 512, 2368, device=dev)
+ batched = newton_schulz(G, 5)
+ single = torch.stack([newton_schulz(G[i:i+1], 5)[0] for i in range(4)])
+ r = float((batched - single).norm() / single.norm())
+ check("batched Newton-Schulz equals the unbatched one", r < 1e-5,
+ f"rel-L2 {r:.2e}")
+ s = torch.linalg.svdvals(batched[0].float())
+ check("Newton-Schulz orthogonalises (singular values near 1)",
+ 0.7 < float(s.mean()) < 1.3, f"mean sv {float(s.mean()):.3f}, "
+ f"min {float(s.min()):.3f}, max {float(s.max()):.3f}")
+
+ # Fusing q/k/v into one Parameter must not change the update. This is the
+ # whole justification for storing them fused, so it is asserted, not assumed.
+ torch.manual_seed(1)
+ fused = torch.nn.Parameter(torch.randn(768, 512, device=dev))
+ fused._boris_role = "matmul"
+ fused._boris_blocks = ((0, 512), (512, 640), (640, 768))
+ grad = torch.randn(768, 512, device=dev)
+ fused.grad = grad.clone()
+ parts = []
+ for a, b in fused._boris_blocks:
+ q = torch.nn.Parameter(fused.data[a:b].clone())
+ q._boris_role = "matmul"
+ q.grad = grad[a:b].clone()
+ parts.append(q)
+ kw = dict(lr=0.02, momentum=0.95, nesterov=True, ns_steps=5,
+ weight_decay=0.1, batched=True)
+ Muon([fused], **kw).step()
+ Muon(parts, **kw).step()
+ sep = torch.cat([q.data for q in parts], 0)
+ d = float((fused.data - sep).abs().max())
+ check("a block-fused Parameter updates exactly like separate ones", d == 0.0,
+ f"max diff {d:.1e}")
+
+ # A (768, 3) Canon kernel must never reach Muon.
+ m = build(cfg, dev)
+ muon, adamw = build_optimizers(m, cfg, log=lambda *a: None)
+ canon_ids = {id(b.attn.canon) for b in m.blocks}
+ in_muon = any(id(p) in canon_ids for g in muon.param_groups for p in g["params"])
+ check("Canon kernels are in AdamW, not Muon", not in_muon)
+ tbl = id(m.ngram.table)
+ in_muon_tbl = any(id(p) == tbl for g in muon.param_groups for p in g["params"])
+ check("the n-gram table is in AdamW, not Muon", not in_muon_tbl)
+
+
+def check_ngram_ablation(cfg, dev):
+ """Both arms must build, and the runtime toggle must actually do something."""
+ off = Config.from_json(cfg.to_json())
+ off.model.use_ngram = False
+ b_on, b_off = cfg.param_budget()["total"], off.param_budget()["total"]
+ check("use_ngram leaves the transformer untouched",
+ cfg.param_budget()["trunk"] == off.param_budget()["trunk"]
+ and b_on - b_off == cfg.ngram.table_params(cfg.model.d_model),
+ f"{b_on:,} with table, {b_off:,} without, same {b_off - cfg.model.embed_params():,} trunk")
+ off.model.n_layer = 4
+ off.enforce_budget = False
+ m = build(off, dev)
+ check("use_ngram=False builds no table and no projection",
+ m.ngram is None and not any("ngram" in n for n, _ in m.named_parameters()))
+ build_optimizers(m, off, log=lambda *a: None)
+ del m
+ torch.cuda.empty_cache()
+
+ on = Config.from_json(cfg.to_json())
+ on.model.n_layer = 4
+ on.enforce_budget = False
+ m = build(on, dev)
+ x, pos, bounds, bnd = synth_batch(on, dev, 1, 256, [[128, 128]], seed=5)
+ with torch.autocast("cuda", dtype=torch.float16):
+ a = m(x, pos, bounds, bnd, None, 10_000.0)
+ m.ngram_enabled = False
+ b = m(x, pos, bounds, bnd, None, 10_000.0)
+ m.ngram_enabled = True
+ c = m(x, pos, bounds, bnd, None, 10_000.0)
+ check("the runtime toggle changes the output", not torch.equal(a, b),
+ f"max diff {float((a.float()-b.float()).abs().max()):.3e}")
+ check("the runtime toggle is reversible", torch.equal(a, c))
+ del m
+ torch.cuda.empty_cache()
+
+
+def check_schedule(cfg):
+ s = cfg.schedule
+ w, st = s.warmup_frac, s.stable_frac
+ ok = (abs(lr_scale(0.0, cfg)) < 1e-9 and abs(lr_scale(w, cfg) - 1) < 1e-9
+ and abs(lr_scale((w + st) / 2, cfg) - 1) < 1e-9
+ and abs(lr_scale(1.0, cfg) - s.final_lr_frac) < 1e-9)
+ check("WSD: 0 at start, 1 across the stable phase, floor at the end", ok)
+ mono = all(lr_scale(p, cfg) >= lr_scale(p + 0.005, cfg) - 1e-12
+ for p in np.arange(st, 1.0, 0.005))
+ check("WSD: the decay never goes back up", mono)
+
+ for p in (0.0, 0.5, 0.86, 0.94, 1.0):
+ w_ = mixture_at(p, cfg)
+ assert abs(sum(w_.values()) - 1) < 1e-9
+ end = mixture_at(1.0, cfg)
+ # 2026-09-17, Boris-2-0917. The four checks here used to assert the old
+ # hand-weighted decay mixture (raw web to zero, curated > 0.7). That corpus
+ # is gone. ClimbMix's weights are a SEARCH RESULT over the whole mixture, so
+ # the property worth protecting is the opposite one: that we do not
+ # re-weight it by phase and quietly undo the optimisation we adopted it for.
+ flat = all(abs(mixture_at(p, cfg)["climbmix"] - end["climbmix"]) < 1e-9
+ for p in (0.0, 0.25, 0.5, 0.87, 0.95))
+ check("ClimbMix weights are flat across the run, not re-weighted by phase",
+ flat, f"climbmix held at {end['climbmix']:.2f} throughout")
+ check("ClimbMix is the overwhelming majority of the mixture",
+ end["climbmix"] >= 0.90, f"{end['climbmix']:.2f} of the mixture")
+ # finemath is the only source carried over from 0907. ArithMark-3 was the
+ # one benchmark 0907 was competitive on (37.40 vs GPT-X2.5's 38.40) and
+ # ClimbMix was optimised against PIQA/ARC-E/HellaSwag, not arithmetic.
+ check("finemath is retained, so ArithMark-3 is not abandoned",
+ 0.02 <= end.get("finemath", 0.0) <= 0.10,
+ f"{end.get('finemath', 0.0):.2f} of the mixture")
+ check("the mixture names only sources that exist on disk",
+ set(cfg.data.sources) == set(end),
+ f"{sorted(cfg.data.sources)}")
+
+ b0 = context_at(0.0, cfg)
+ b1 = context_at(0.99, cfg)
+ check("context extends later in the run", b1[0] > b0[0],
+ f"{b0[0]} -> {b1[0]} at theta {b0[1]:,.0f} -> {b1[1]:,.0f}")
+ check("the step size is identical across context stages",
+ b0[2] * b0[0] * b0[3] == b1[2] * b1[0] * b1[3] == cfg.train.tokens_per_step,
+ f"{b0[2]}x{b0[0]}x{b0[3]} = {b0[2]*b0[0]*b0[3]:,}")
+
+
+def check_packer(cfg, dev):
+ """Targets are next-token within a document, never across one."""
+ from data import VarlenPacker, SourceStore
+ d = Path(_tmpdir()) / "src"
+ d.mkdir(parents=True)
+ rng = np.random.default_rng(0)
+ lens = rng.integers(20, 400, size=400)
+ toks = rng.integers(0, cfg.model.vocab_size, size=int(lens.sum())).astype(np.uint16)
+ toks.tofile(d / "tokens.bin")
+ np.concatenate([[0], np.cumsum(lens)]).astype(np.uint64).tofile(d / "docs.idx")
+ st = SourceStore.open(d.parent, "src")
+ cfg2 = Config.from_json(cfg.to_json())
+ cfg2.data.sources = ("src",)
+ cfg2.curriculum.enabled = False
+ p = VarlenPacker(cfg2, {"src": st}, 7)
+ xs, ys, pos, bnd, bounds, stats = p.batch(2, 256, 0.0)
+
+ ok_bounds = (bounds[0] == 0 and bounds[-1] == 2 * 256
+ and np.all(np.diff(bounds) > 0))
+ check("packer: offsets are increasing and cover the batch exactly", ok_bounds)
+ flat_b = np.zeros(2 * 256, dtype=bool)
+ flat_b[bounds[:-1]] = True
+ check("packer: boundary mask agrees with the offsets",
+ np.array_equal(flat_b, bnd.reshape(-1)))
+ fx, fy = xs.reshape(-1), ys.reshape(-1)
+ bad = 0
+ for i in range(len(bounds) - 1):
+ a, b_ = bounds[i], bounds[i + 1]
+ if b_ - a > 1 and not np.array_equal(fy[a:b_ - 1], fx[a + 1:b_]):
+ bad += 1
+ if fy[b_ - 1] != IGNORE and i + 1 < len(bounds) - 1:
+ # only legal when the document was carried into the next row
+ pass
+ check("packer: targets are next-token inside each document", bad == 0,
+ f"{bad} documents mismatched")
+ check("packer: positions restart at every document",
+ all(pos.reshape(-1)[bounds[i]] == 0
+ for i in range(len(bounds) - 1)))
+
+ # The RoPE table is exactly max_block_size long and is indexed by pos, so a
+ # position past its end is an out-of-bounds gather -- a device-side assert
+ # thousands of steps in, reported asynchronously at whatever kernel runs
+ # next. Carried chunks of a long document used to keep counting, and with
+ # max_doc_tokens at 65,536 they ran off the end. Sample where long-document
+ # pressure is highest, at both context stages.
+ lim = cfg.model.max_block_size
+ worst, worst_at = 0, None
+ for prog, blk in ((0.0, 256), (0.5, 256), (0.85, 512), (0.95, 512)):
+ for _ in range(12):
+ _x, _y, po, _b, bo, _st = p.batch(4, blk, prog)
+ m = int(po.max())
+ if m > worst:
+ worst, worst_at = m, (prog, blk)
+ if not all(po.reshape(-1)[bo[i]] == 0 for i in range(len(bo) - 1)):
+ worst, worst_at = lim + 1, (prog, blk) # force a failure
+ check("packer: positions stay inside the RoPE table", worst < lim,
+ f"max pos {worst:,} of {lim:,}"
+ + (f" at progress {worst_at[0]}, block {worst_at[1]}" if worst_at else ""))
+
+ # Resumability: the same state must replay the same batch.
+ s = p.state_dict()
+ a1 = p.batch(2, 256, 0.0)[0]
+ p.load_state_dict(s)
+ a2 = p.batch(2, 256, 0.0)[0]
+ check("packer: a restored state replays the identical batch",
+ np.array_equal(a1, a2))
+
+ from data import Prefetcher, validate_tokenization, open_validation
+ pref = Prefetcher(p, cfg2, dev)
+ pref.configure(2, 256, 0.0)
+ pref.start()
+ try:
+ pref.next(2, 256)
+ committed = pref.state_dict()
+ reference = VarlenPacker(cfg2, {"src": st}, 7)
+ reference.load_state_dict(committed)
+ expected = reference.batch(2, 512)[0]
+ pref.configure(2, 512, 0.0)
+ actual = pref.next(2, 512)[0].cpu().numpy()
+ check("prefetch: shape transition replays committed state",
+ np.array_equal(actual, expected))
+ finally:
+ pref.stop()
+
+ def broken_batch(*args):
+ raise ValueError("deliberate producer failure")
+ p.batch = broken_batch
+ pref = Prefetcher(p, cfg2, dev)
+ pref.configure(2, 256, 0.0)
+ pref.start()
+ propagated = False
+ try:
+ pref.next(2, 256)
+ except RuntimeError as exc:
+ propagated = isinstance(exc.__cause__, ValueError)
+ finally:
+ pref.stop()
+ check("prefetch: producer failure reaches training instead of hanging", propagated)
+ rejected = False
+ try:
+ validate_tokenization(d, cfg)
+ except ValueError:
+ rejected = True
+ check("data: rejects shards without matching tokenizer provenance", rejected)
+ check("validation: current tokenizer provenance verified", open_validation(cfg).n_tokens > 0)
+
+
+def check_multistore(cfg):
+ """Sharded sources must stitch back into one correctly indexed source.
+
+ A source too large for one prepare process is split across workers by data
+ file range; data.py presents the shard directories as one logical source
+ rather than concatenating 200GB on disk. If the global-to-shard index
+ mapping is wrong, training silently reads the wrong documents.
+ """
+ import tempfile
+ from data import MultiStore, SourceStore
+ root = Path(_tmpdir())
+ rng = np.random.default_rng(0)
+ truth, parts = [], []
+ for sh in range(3):
+ d = root / f"src_s{sh}"; d.mkdir()
+ lens = rng.integers(5, 50, size=7)
+ docs = [rng.integers(1, 1000, size=L).astype(np.uint16) for L in lens]
+ truth += docs
+ np.concatenate(docs).tofile(d / "tokens.bin")
+ np.concatenate([[0], np.cumsum(lens)]).astype(np.uint64).tofile(d / "docs.idx")
+ parts.append(SourceStore("src", d / "tokens.bin", d / "docs.idx"))
+ m = MultiStore("src", parts)
+ check("sharded source: document count and token count add up",
+ m.n_docs == len(truth) and m.n_tokens == sum(len(t) for t in truth))
+ check("sharded source: every global index maps to the right document",
+ all(np.array_equal(m.doc(i), truth[i]) for i in range(m.n_docs)))
+ exp = np.flatnonzero(np.array([len(t) for t in truth]) >= 30)
+ check("sharded source: long-document selection spans shards",
+ np.array_equal(m.long_docs(30), exp))
+
+
+def check_checkpoint(cfg, dev):
+ import checkpoint as C
+ small = Config.from_json(cfg.to_json())
+ small.enforce_budget = False
+ small.model.n_layer = 4
+ m = build(small, dev)
+ muon, adamw = build_optimizers(m, small, log=lambda *a: None)
+ for p in m.parameters():
+ p.grad = torch.randn_like(p)
+ muon.step(); adamw.step()
+ scaler = torch.amp.GradScaler("cuda")
+ tmp = Path(_tmpdir()) / "c.pt"
+ C.save_full(tmp, model=m, muon=muon, adamw=adamw, scaler=scaler, packer=None,
+ step=5, tokens_seen=123, cfg=small, ngram_fingerprint="deadbeef",
+ background=False)
+ before = {k: v.clone() for k, v in m.state_dict().items()}
+ for p in m.parameters():
+ p.data.add_(torch.randn_like(p))
+ blob = C.load_full(tmp, model=m, muon=muon, adamw=adamw, scaler=scaler,
+ map_location=dev, expect_fingerprint=small.fingerprint(),
+ expect_ngram="deadbeef")
+ same = all(torch.equal(before[k].to(dev), v) for k, v in m.state_dict().items())
+ check("checkpoint: weights round-trip exactly", same)
+ check("checkpoint: step and token count survive",
+ blob["step"] == 5 and blob["tokens_seen"] == 123)
+ mom = next(iter(muon.state.values())).get("momentum_buffer")
+ check("checkpoint: Muon momentum buffers are restored", mom is not None)
+
+ # The packer round-trip, on the real device. This check used to pass
+ # packer=None, so the resume path was never actually exercised: the first
+ # real --resume died on it. map_location=<cuda> moves the packer's saved RNG
+ # state onto the GPU, and a CPU Generator's set_state rejects a CUDA tensor.
+ import data as D
+ srcs = D.open_sources(cfg, log=lambda *a, **k: None)
+ pk = D.VarlenPacker(cfg, srcs, seed=0)
+ for _ in range(3):
+ pk.batch(2, 256, 0.0)
+ tmp2 = Path(_tmpdir()) / "p.pt"
+ C.save_full(tmp2, model=m, muon=muon, adamw=adamw, scaler=scaler, packer=pk,
+ step=7, tokens_seen=99, cfg=small, ngram_fingerprint="deadbeef",
+ background=False)
+ want = pk.batch(2, 256, 0.0)[0]
+ pk2 = D.VarlenPacker(cfg, srcs, seed=0)
+ ok_resume, err = True, ""
+ try:
+ C.load_full(tmp2, model=m, muon=muon, adamw=adamw, scaler=scaler,
+ packer=pk2, map_location=dev,
+ expect_fingerprint=small.fingerprint(),
+ expect_ngram="deadbeef")
+ except Exception as e:
+ ok_resume, err = False, f"{type(e).__name__}: {e}"
+ check("checkpoint: the packer resumes from a GPU map_location",
+ ok_resume, err)
+ if ok_resume:
+ got = pk2.batch(2, 256, 0.0)[0]
+ check("checkpoint: a resumed packer replays the identical next batch",
+ np.array_equal(want, got),
+ "same tokens" if np.array_equal(want, got) else "DIVERGED")
+ try:
+ C.load_full(tmp, expect_fingerprint="0000000000000000")
+ check("checkpoint: a config change is refused, not silently accepted", False)
+ except SystemExit:
+ check("checkpoint: a config change is refused, not silently accepted", True)
+
+
+def check_tensor_cores(cfg, dev):
+ """Prove fp16 autocast reaches a tensor-core kernel rather than sgemm."""
+ from torch.profiler import ProfilerActivity, profile
+ a = torch.randn(4096, 512, device=dev)
+ b = torch.randn(512, 2368, device=dev)
+ with profile(activities=[ProfilerActivity.CUDA]) as pr:
+ with torch.autocast("cuda", dtype=torch.float16):
+ for _ in range(3):
+ (a @ b).sum()
+ torch.cuda.synchronize()
+ names = " ".join(e.key for e in pr.key_averages()).lower()
+ tc = any(s in names for s in ("h884", "884gemm", "hgemm", "s16816", "16816",
+ "tensor", "cutlass"))
+ check("fp16 GEMMs dispatch to a tensor-core kernel", tc,
+ [e.key for e in pr.key_averages() if "gemm" in e.key.lower()][:1])
+
+
+def check_down_proj_headroom(cfg, dev):
+ """The FFN down projection's fp16 output range -- failure #2, step 4,520.
+
+ Channel 228 of the residual is a massive activation, 173x the RMS (78,115
+ against 451 at block 16). The FFN writes into it, so this GEMM's fp16 OUTPUT
+ crossed 65,504 and became inf, which norm_f turned into NaN. Inputs and
+ weights were all finite; only the range ran out. DOWN_SHRINK pre-scales
+ w_down by a power of two, which is exact and keeps the GEMM on tensor cores.
+ """
+ import model as M
+ peak = 65_504.0 / M.DOWN_SHRINK # what the output may now reach
+ check("down proj: DOWN_SHRINK is a power of two (so the scaling is exact)",
+ M.DOWN_SHRINK > 0 and
+ abs(math.log2(M.DOWN_SHRINK) - round(math.log2(M.DOWN_SHRINK))) < 1e-12,
+ f"{M.DOWN_SHRINK} = 2^{round(math.log2(M.DOWN_SHRINK))}")
+ check("down proj: headroom covers the measured step-4,520 peak",
+ peak > 4 * 65_504.0,
+ f"overflows at {peak:,.0f} now, was 65,504 ({1/M.DOWN_SHRINK:.0f}x)")
+
+ # the scaling must not change the result: same GEMM, both ways, in fp32
+ torch.manual_seed(0)
+ x = (torch.randn(4096, 2368, device=dev, dtype=torch.float16) * 40)
+ w = (torch.randn(512, 2368, device=dev, dtype=torch.float16) * 0.5)
+ plain = F.linear(x.float(), w.float())
+ scaled = F.linear(x, w * M.DOWN_SHRINK).float() * (1.0 / M.DOWN_SHRINK)
+ rel = float((scaled - plain).norm() / plain.norm())
+ check("down proj: pre-scaling the weight matches the unscaled GEMM",
+ rel < 2e-3, f"rel-L2 {rel:.2e}")
+
+ # And it must rescue the real failure shape: an output that lands just past
+ # 65,504, the way step 4,520's did -- not an absurd one. Headroom is 8x, not
+ # infinite, so the test is calibrated to the measured overshoot.
+ torch.manual_seed(1)
+ big = torch.randn(2048, 2368, device=dev, dtype=torch.float16)
+ wb = torch.randn(512, 2368, device=dev, dtype=torch.float16)
+ ref = F.linear(big.float(), wb.float()).abs().amax()
+ big = (big * (1.6 * 65_504.0 / float(ref)) ** 0.5) # target ~1.6x over
+ wb = (wb * (1.6 * 65_504.0 / float(ref)) ** 0.5)
+ true_peak = float(F.linear(big.float(), wb.float()).abs().amax())
+ unscaled_bad = not torch.isfinite(F.linear(big, wb)).all()
+ scaled = F.linear(big, wb * M.DOWN_SHRINK).float() * (1.0 / M.DOWN_SHRINK)
+ scaled_ok = bool(torch.isfinite(scaled).all())
+ check("down proj: an output just past 65,504 overflows unscaled, survives scaled",
+ bool(unscaled_bad) and scaled_ok,
+ f"true peak {true_peak:,.0f}: unscaled overflows={bool(unscaled_bad)}, "
+ f"scaled finite={scaled_ok}")
+
+
+def check_residual_range(cfg, dev):
+ """The overflow that killed two runs, reproduced from the measured value.
+
+ The residual stream grows with depth. At step 3,456 -- 1% into the budget --
+ it reached 65,617 at block 25 against fp16's maximum of 65,504. It saturates
+ to inf, and _rmsnorm's rsqrt(inf) = 0 turns that into inf * 0 = NaN. No loss
+ scale can help: the overflow is in the forward pass, so GradScaler halves its
+ scale forever against a cause halving cannot touch.
+ """
+ from model import _rmsnorm
+ peak = 65_617.3 # measured, block 25, step 3,456
+ w = torch.ones(512, device=dev)
+ x = torch.full((4, 512), peak, device=dev)
+
+ h16 = x.to(torch.float16)
+ check("residual: the measured peak really does overflow fp16",
+ not torch.isfinite(h16).all(),
+ f"{peak:,.0f} vs fp16 max 65,504")
+ out16 = _rmsnorm(h16, w, cfg.model.norm_eps)
+ check("residual: an overflowed fp16 residual makes RMSNorm produce NaN",
+ bool(torch.isnan(out16).any()),
+ "rsqrt(inf)=0 then inf*0=nan -- the exact failure")
+
+ out32 = _rmsnorm(x, w, cfg.model.norm_eps)
+ check("residual: the same value in fp32 stays finite",
+ bool(torch.isfinite(out32).all()),
+ f"max |out| {float(out32.abs().amax()):.3f}")
+ check("residual: fp16_residual is off",
+ cfg.train.fp16_residual is False,
+ "fp32 residual stream (costs ~7.6% throughput)")
+
+
+def check_loss_scale_floor(cfg, dev):
+ """The scale collapse that ended the step-3,810 run, reproduced and fenced.
+
+ GradScaler has no floor. Once the scale falls below ~1e-30, unscale_ computes
+ grad/scale, overflows fp32 to inf, sets found_inf, and halves the scale
+ again -- self-reinforcing, no exit, every step skipped from then on. The run
+ died that way with its weights still perfectly finite. This drives a scaler
+ into that spiral and asserts the clamp in train.py stops it.
+ """
+ w = torch.zeros(4, device=dev, requires_grad=True)
+ sc = torch.amp.GradScaler("cuda", init_scale=cfg.train.init_scale)
+ opt = torch.optim.SGD([w], lr=0.0)
+
+ # Unclamped: 80 consecutive overflows.
+ bare = torch.amp.GradScaler("cuda", init_scale=cfg.train.init_scale)
+ for _ in range(80):
+ bare.scale(torch.zeros((), device=dev)) # lazy-inits _scale
+ w.grad = torch.full_like(w, float("inf"))
+ bare.unscale_(opt); bare.step(opt); bare.update()
+ collapsed = bare.get_scale()
+
+ floored_steps = 0
+ for _ in range(80):
+ sc.scale(torch.zeros((), device=dev))
+ w.grad = torch.full_like(w, float("inf"))
+ sc.unscale_(opt); sc.step(opt); sc.update()
+ if sc.get_scale() < cfg.train.min_loss_scale:
+ sc.update(cfg.train.min_loss_scale); floored_steps += 1
+ held = sc.get_scale()
+
+ check("loss scale: unclamped GradScaler does collapse (the bug)",
+ collapsed < 1e-12,
+ f"80 overflows drive it to {collapsed:.3g}")
+ check("loss scale: the clamp holds it at the floor",
+ held == cfg.train.min_loss_scale and math.isfinite(held),
+ f"held at {held:g}, floored on {floored_steps} steps")
+ # And the floor must be high enough that unscale_ cannot overflow fp32.
+ safe = cfg.train.min_loss_scale > 1e-30
+ check("loss scale: the floor is above the fp32 unscale_ overflow point",
+ safe, f"floor {cfg.train.min_loss_scale:g} vs ~1e-30")
+ check("loss scale: the ceiling is at or below 2^15",
+ cfg.train.max_loss_scale <= 2.0 ** 15,
+ f"max {cfg.train.max_loss_scale:,.0f}")
+
+
+def check_head_range(cfg, dev):
+ """The fused head's dW GEMM must not overflow fp16 on realistic text.
+
+ g = S*(p - onehot): the true token carries -S, and dW sums it over every
+ token in the chunk with the same target. Synthetic benchmark data is
+ uniform over the vocabulary and never stacks a target often enough to
+ matter; real text does. At GRAD_SCALE 4096 this saturated to Inf on step 1,
+ Inf-Inf put NaN in the tied embedding gradient, and the run died on step 2.
+ Assert on a batch of the actual corpus, and assert margin, not just
+ finiteness -- |x| grows over a long run and this must not sit on the edge.
+ """
+ import torch.nn.functional as F
+ import data as D, ngram as N, model as M
+ from pathlib import Path
+ srcs = D.open_sources(cfg, log=lambda *a, **k: None)
+ pk = D.VarlenPacker(cfg, srcs, seed=0)
+ m = M.Boris2(cfg).to(dev)
+ if m.ngram is not None:
+ m.ngram.load_vocab(N.NGramVocab.load(cfg.ngram.vocab_path), dev)
+ m.train()
+ x, y, pos, bnd, bounds, st = pk.batch(8, 2048, 0.0)
+ t = lambda a, dt: torch.as_tensor(a, dtype=dt, device=dev)
+ with torch.no_grad(), torch.autocast("cuda", dtype=torch.float16):
+ h = m(t(x, torch.long), t(pos, torch.long), t(bounds, torch.long),
+ t(bnd, torch.bool), None, 10_000.0, st["min_seqlen"], st["max_seqlen"])
+ xf = h.reshape(-1, h.shape[-1])
+ tg = t(y, torch.long).reshape(-1)
+ # Test the actual custom Function, not a separately reimplemented GEMM.
+ # The old test bypassed its broken fp16 rescale and therefore passed while
+ # production overflowed. Compare full gradients with fp32 cross-entropy.
+ c = cfg.optim.lm_head_chunk
+ for magnitude, repeated in ((1.0, False), (64.0, True)):
+ xx = (xf[:c].detach() * magnitude).requires_grad_()
+ ww = m.head_weight().detach().clone().requires_grad_()
+ tt = tg[:c].clone()
+ if repeated:
+ tt.fill_(int(tg[tg != IGNORE][0]))
+ with torch.autocast("cuda", dtype=torch.float16):
+ loss = M.fused_lm_loss(xx, ww, tt, chunk=c)
+ loss.backward()
+ xr = xx.detach().float().requires_grad_()
+ wr = ww.detach().clone().requires_grad_()
+ ref = F.cross_entropy(F.linear(xr, wr), tt, ignore_index=IGNORE)
+ ref.backward()
+ rel = (ww.grad - wr.grad).norm() / wr.grad.norm().clamp_min(1e-12)
+ check(f"production head real text: magnitude={magnitude:g}, repeated={repeated}",
+ bool(torch.isfinite(ww.grad).all()) and float(rel) < .01,
+ f"dW relative error {float(rel):.3g}")
+ del xx, ww, xr, wr, loss, ref
+
+ del m
+ torch.cuda.empty_cache()
+
+
+def check_compile(cfg, dev):
+ """The compiled regions must produce the same numbers as eager."""
+ small = Config.from_json(cfg.to_json())
+ small.enforce_budget = False
+ small.model.n_layer = 4
+ torch.manual_seed(0)
+ me = build(small, dev, compile_regions=False)
+ sd = me.state_dict()
+ torch.manual_seed(0)
+ mc = build(small, dev, compile_regions=True)
+ mc.load_state_dict(sd)
+ for m_ in (me, mc):
+ for blk in m_.blocks:
+ torch.nn.init.normal_(blk.attn.canon, std=0.3)
+ mc.load_state_dict(me.state_dict())
+ x, pos, bounds, bnd = synth_batch(small, dev, 1, 512, [[200, 312]], seed=1)
+ with torch.autocast("cuda", dtype=torch.float16):
+ a = me(x, pos, bounds, bnd, None, 10_000.0)
+ b = mc(x, pos, bounds, bnd, None, 10_000.0)
+ r = float((a.float() - b.float()).norm() / (a.float().norm() + 1e-9))
+ check("compiled regions match eager", r < 5e-3, f"rel-L2 {r:.2e}")
+
+
+# --------------------------------------------------------------------------- #
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--device", default="cuda:0")
+ ap.add_argument("--quick", action="store_true")
+ a = ap.parse_args()
+ dev = a.device
+ cfg = Config()
+ torch.cuda.set_device(dev)
+ torch.backends.cuda.matmul.allow_tf32 = False
+
+ print(f"\n verify: {cfg.run_name} on {dev}")
+ print(" " + "=" * 74)
+ print("\n budget and optimizer split")
+ m = build(cfg, dev)
+ build_optimizers(m, cfg)
+ del m
+ torch.cuda.empty_cache()
+
+ print(" numerics")
+ check_fused_head(cfg, dev)
+ check_repeated_target_head(cfg, dev)
+ check_head_range(cfg, dev)
+ check_loss_scale_floor(cfg, dev)
+ check_residual_range(cfg, dev)
+ check_down_proj_headroom(cfg, dev)
+ check_muon(cfg, dev)
+ torch.cuda.empty_cache()
+
+ print("\n isolation")
+ check_canon_causality(cfg, dev)
+ check_attention_matches_loop(cfg, dev)
+ check_volta_kernels(cfg, dev)
+ check_ngram_isolation(cfg, dev)
+ check_ngram_ablation(cfg, dev)
+ check_document_isolation(cfg, dev)
+ torch.cuda.empty_cache()
+
+ print("\n schedule and data")
+ check_schedule(cfg)
+ check_packer(cfg, dev)
+ check_multistore(cfg)
+
+ print("\n plumbing")
+ check_tensor_cores(cfg, dev)
+ check_checkpoint(cfg, dev)
+ if not a.quick:
+ check_compile(cfg, dev)
+
+ print("\n " + "=" * 74)
+ print(f" {len(PASS)} passed, {len(FAIL)} failed")
+ for f in FAIL:
+ print(f" FAILED: {f}")
+ sys.exit(1 if FAIL else 0)
+
+
+if __name__ == "__main__":
+ main()