summaryrefslogtreecommitdiff
path: root/model.py
diff options
context:
space:
mode:
authorjdj <jdj@kunix.org>2026-11-18 14:47:33 -0800
committerjdj <jdj@kunix.org>2026-11-18 14:47:33 -0800
commit5d2e4252e695644c34be605b44a14bd80a98a0a6 (patch)
tree311a72eda4459af7547b9cd05c0956eafb235f2d /model.py
downloadboris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.tar.gz
boris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.zip
Boris-2 training code
The code, configuration and data filters used to train Boris-2 (144M, 200B tokens, single V100): model, training loop, data preparation, n-gram table, V100 kernels, final stages and the exact run configs.
Diffstat (limited to 'model.py')
-rw-r--r--model.py833
1 files changed, 833 insertions, 0 deletions
diff --git a/model.py b/model.py
new file mode 100644
index 0000000..69d0db5
--- /dev/null
+++ b/model.py
@@ -0,0 +1,833 @@
+"""Boris-2: a d512 x 31L decoder-only transformer with fused Canon layers.
+
+Shape of a block:
+
+ h = h + o_proj( attn( canon( qkv_proj( norm(h) ) ) ) )
+ h = h + down( silu(gate) * up )( norm(h) )
+
+Four things here are not the textbook spelling, and each is a measured win on
+sm_70 rather than a preference:
+
+1. Q, K and V are one GEMM. The weights stay three separate Parameters and are
+ concatenated at call time, so Muon still orthogonalises each independently --
+ fusing the Parameters themselves would change what the optimizer operates on
+ and would not be lossless. The concatenated weight is ~1.2MB against a
+ 16,384-row activation.
+
+2. Canon is one kernel over the concatenated Q|K|V block. Depthwise convolution
+ is per-channel, so a single 768-channel convolution equals three separate
+ ones on the Q/K/V channel ranges: 31 kernels per forward instead of 93. It is
+ spelled as three shifted multiply-adds on the native (B,T,C) layout rather
+ than F.conv1d(groups=C), which needs a (B,T,C)->(B,C,T) transpose whose
+ following reshape materialises a copy. Measured on this V100, fwd+bwd over
+ the real shape: 4.30 ms for conv1d, 1.10 ms for the compiled shift form --
+ substantially faster than the transposing grouped-convolution spelling.
+
+3. Canon is document-aware. Kernel 3 stacked 31 deep reaches 62 tokens back, and
+ rows hold several packed documents, so without masking every block pulls the
+ previous document's tail across the boundary. The mask costs 0.10 ms/layer.
+ This is the bug the 75M run found the hard way; verify.py probes for it with
+ the output projections de-zeroed, because at init a zero-init projection
+ makes the test pass no matter how badly the model leaks.
+
+4. Attention is varlen over a jagged nested tensor: one SDPA launch for the
+ whole batch, cost sum(len_i^2) rather than T^2. Measured against the two
+ alternatives at ctx 2048 with realistic document lengths, fwd+bwd:
+ nested 11.09 ms, dense block-diagonal mask 26.97 ms, per-document loop
+ 45.84 ms. Nested is bit-identical to the loop (max diff 0.0).
+
+The residual stream is fp32 on the V100 because measured massive activations can
+exceed fp16's finite range. Matmul inputs are cast by the compiled regions under
+autocast, preserving tensor-core execution without risking a forward overflow.
+"""
+from __future__ import annotations
+
+import math
+import os
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from config import Config, ModelConfig
+from ngram import NGramTable
+from volta_kernels.integ_ops import AddNorm, LinAcc, TiedEmbedding
+
+IGNORE = -100
+
+# Hand-written sm_70 kernels (volta_kernels/): varlen FlashAttention with mma.sync
+# and a one-pass LM-head softmax. Opt-in, because they are adopted by a running
+# 200B-token job at a resume: BORIS_VOLTA_KERNELS=1. Same operand dtypes and fp32
+# accumulation as the paths they replace; verify.py checks both against fp64.
+# They only engage on an sm_70 device -- elsewhere (eval on the 3060) the stock
+# paths run regardless.
+VOLTA_KERNELS = os.environ.get("BORIS_VOLTA_KERNELS", "0") == "1"
+
+
+def _volta(t: torch.Tensor) -> bool:
+ return VOLTA_KERNELS and t.is_cuda and torch.cuda.get_device_capability(t.device) == (7, 0)
+
+
+# --------------------------------------------------------------------------- #
+# Compiled fixed-shape regions
+# --------------------------------------------------------------------------- #
+# Whole-model compile is not used: the 75M run measured it 28% slower in
+# realistic conditions because document counts vary and Inductor recompiles
+# mid-run. These regions never see a ragged shape -- only (B, T, C) with B and T
+# fixed for the whole context stage -- so each compiles once per stage.
+
+def _rmsnorm(x, w, eps: float):
+ f = x.float()
+ f = f * torch.rsqrt(f.pow(2).mean(-1, keepdim=True) + eps)
+ return (f * w.float()).to(x.dtype)
+
+
+def _pre_attn(h, w_norm, eps: float, w_qkv, canon_w, m1, m2, cos, sin,
+ n_head: int, n_kv_head: int, head_dim: int, use_canon: bool,
+ out16: bool = False):
+ """Norm, one GEMM, one depthwise convolution, one rotation -- one region.
+
+ The RMSNorm is inside the region on purpose. Written as its own eager module
+ it upcasts to fp32, and that materialises a (B, T, 512) fp32 tensor for
+ every one of the 53 norms in the model -- 25% of forward+backward was
+ elementwise traffic before these regions were widened, and this was most of
+ it. Inductor keeps the reduction in fp32 without ever writing the fp32
+ tensor out.
+
+ canon_w is (C, 3) with column 0 the weight on token t, column 1 on t-1 and
+ column 2 on t-2. m1 masks the t-1 tap and m2 the t-2 tap at document
+ boundaries; both are (B, T, 1) and are computed once per forward, not per
+ layer.
+ """
+ f = h.float()
+ x = (f * torch.rsqrt(f.pow(2).mean(-1, keepdim=True) + eps)
+ * w_norm.float()).to(h.dtype)
+ qkv = F.linear(x, w_qkv) # (B, T, C)
+ if use_canon:
+ w = canon_w.to(qkv.dtype)
+ s1 = F.pad(qkv, (0, 0, 1, 0))[:, :-1] * m1
+ s2 = F.pad(qkv, (0, 0, 2, 0))[:, :-2] * m2
+ qkv = qkv + qkv * w[:, 0] + s1 * w[:, 1] + s2 * w[:, 2]
+
+ B, T, _ = qkv.shape
+ qd, kd = n_head * head_dim, n_kv_head * head_dim
+ q = qkv[..., :qd].view(B, T, n_head, head_dim)
+ k = qkv[..., qd:qd + kd].view(B, T, n_kv_head, head_dim)
+ v = qkv[..., qd + kd:].view(B, T, n_kv_head, head_dim)
+
+ def rope(t):
+ f = t.float()
+ a, b = f.chunk(2, dim=-1)
+ return torch.cat([a * cos - b * sin, a * sin + b * cos], -1).to(t.dtype)
+
+ if out16:
+ # The Volta attention takes fp16. Casting here rounds the same fp32
+ # values autocast would round, but inside the region's epilogue, so the
+ # fp32 q/k/v are never written out and read back by separate casts.
+ h16 = torch.float16
+ return rope(q).to(h16), rope(k).to(h16), v.to(h16)
+ return rope(q), rope(k), v
+
+
+def _pre_attn_qkv(h, w_norm, eps: float, w_qkv):
+ """Norm and the fused QKV GEMM only -- the Volta path's region.
+
+ Canon and RoPE then run as one fused Triton pass (volta_kernels/canon_rope.py)
+ instead of Inductor's pad-copy / combine / rotate kernels.
+ """
+ f = h.float()
+ x = (f * torch.rsqrt(f.pow(2).mean(-1, keepdim=True) + eps)
+ * w_norm.float()).to(h.dtype)
+ return F.linear(x, w_qkv)
+
+
+def _post_then_pre(h, o, w_o, w_norm2, eps2, w_gu, w_down, w_norm1n, eps1n, w_qkvn,
+ extra):
+ """Block i's FFN half and block i+1's norm + QKV GEMM as ONE region.
+
+ As separate regions the residual h crossed a graph boundary between two
+ consumers, so its two backward gradient streams were summed by an eager
+ fp32 add (~192 MB of traffic per layer) and h was written and read back.
+ Here Inductor fuses that sum into the region's backward. `extra` is the
+ n-gram residual when block i+1 is where it is inserted (it depends only on
+ the token ids), else None -- each value gets its own compiled graph.
+ """
+ h = _post_attn(h, o, w_o, w_norm2, eps2, w_gu, w_down)
+ if extra is not None:
+ h = h + extra
+ return h, _pre_attn_qkv(h, w_norm1n, eps1n, w_qkvn)
+
+
+def _post_then_final_norm(h, o, w_o, w_norm2, eps2, w_gu, w_down, w_normf, epsf):
+ """The last block's FFN half and the final RMSNorm as one region."""
+ h = _post_attn(h, o, w_o, w_norm2, eps2, w_gu, w_down)
+ return _rmsnorm(h, w_normf, epsf)
+
+
+# Power-of-two headroom for the FFN down projection's fp16 output. 1/8 puts the
+# measured step-4,520 peak (~65.5k) at ~8.2k, leaving 8x before it overflows
+# again, and costs only inputs below 4.9e-4 -- 1.6e-8 of the peak, which
+# contribute nothing to a sum dominated by terms 6e7 times larger.
+DOWN_SHRINK = 0.125
+
+
+def _post_attn(h, o, w_o, w_norm, eps: float, w_gu, w_down):
+ """Output projection, residual, norm, SwiGLU, residual -- one region.
+
+ Everything in a block except the attention itself. The two residual adds and
+ the second norm were separate eager kernels over the widest tensors in the
+ model; inside the region Inductor folds them into the GEMM epilogues, and
+ the fp32 norm intermediate is never written out.
+ """
+ h = h + F.linear(o, w_o)
+ f = h.float()
+ x = (f * torch.rsqrt(f.pow(2).mean(-1, keepdim=True) + eps)
+ * w_norm.float()).to(h.dtype)
+ gu = F.linear(x, w_gu)
+ g, u = gu.chunk(2, dim=-1)
+ # The down projection is the one GEMM here whose fp16 OUTPUT can overflow.
+ # Its inputs are finite and so is the true result -- it is the fp16 range
+ # that runs out. Channel 228 of the residual is a "massive activation",
+ # 173x the RMS (78,115 against an RMS of 451 at block 16), and the FFN has
+ # to write into it, so at step 4,520 this output crossed 65,504 and became
+ # inf; norm_f then turned that into NaN via rsqrt(inf)=0, inf*0.
+ # Pre-scaling w_down is exact (a power of two), keeps the GEMM on tensor
+ # cores, costs one pass over a 1.2M-element weight rather than the 77M-
+ # element activation, and is divided back out in fp32 where there is room.
+ # Every partial sum is scaled too, so this is safe even with cuBLAS's
+ # reduced-precision fp16 reductions enabled.
+ return h + F.linear(F.silu(g) * u,
+ w_down * DOWN_SHRINK).float() * (1.0 / DOWN_SHRINK)
+
+
+# --------------------------------------------------------------------------- #
+# Merged-path regions with the swarm additions (volta_kernels/integ_ops.py)
+# --------------------------------------------------------------------------- #
+# ft = (wacc, norm) -- which additions a compiled region uses:
+# wacc: weight-gradient GEMMs accumulate into the fp32 .grad (accs given)
+# norm: residual-add + RMSNorm backward and norm-weight grad in one Triton pass
+# Weights arrive either as the fp32 Parameters (cast here, as autocast would) or
+# as the cached, pre-scaled fp16 copies (Boris2._w16); both give the same fp16
+# operand, so the forward is bitwise the live one either way.
+FT_ALL = (True, True)
+
+
+def _w16(w, scale: float = 1.0):
+ if w.dtype == torch.float16:
+ return w # cached copy, already scaled
+ return (w * scale).to(torch.float16) if scale != 1.0 else w.to(torch.float16)
+
+
+def _lin_x(x, w, acc, scale: float = 1.0):
+ if acc is None:
+ if w.dtype == torch.float16:
+ return F.linear(x, w)
+ return F.linear(x, w * scale if scale != 1.0 else w)
+ return LinAcc.apply(x.to(torch.float16), _w16(w, scale), acc, scale)
+
+
+def _norm_x(h, w, eps: float):
+ f = h.float()
+ return (f * torch.rsqrt(f.pow(2).mean(-1, keepdim=True) + eps)
+ * w.float()).to(h.dtype)
+
+
+def _ffn_x(h, o, w_o, w_norm, eps: float, w_gu, w_down, a_o, a_gu, a_down, ft):
+ """Block FFN half up to (but excluding) the down projection's residual add:
+ returns h1 = h + o_proj(o) and the fp16 down output (pre-scaled by
+ DOWN_SHRINK), so the next norm can fold the add into its backward."""
+ wacc, norm = ft
+ a = _lin_x(o, w_o, a_o)
+ if norm:
+ h1, x = AddNorm.apply(h, a, w_norm, eps, 1.0, False)
+ else:
+ h1 = h + a
+ x = _norm_x(h1, w_norm, eps)
+ gu = _lin_x(x, w_gu, a_gu)
+ g, u = gu.chunk(2, dim=-1)
+ return h1, _lin_x(F.silu(g) * u, w_down, a_down, DOWN_SHRINK)
+
+
+def _post_then_pre_x(h, o, w_o, w_norm2, eps2, w_gu, w_down, w_norm1n, eps1n, w_qkvn,
+ extra, accs, ft):
+ a_o, a_gu, a_down, a_qkvn = accs if accs is not None else (None,) * 4
+ h1, dn = _ffn_x(h, o, w_o, w_norm2, eps2, w_gu, w_down, a_o, a_gu, a_down, ft)
+ if ft[1] and extra is None:
+ h2, x = AddNorm.apply(h1, dn, w_norm1n, eps1n, 1.0 / DOWN_SHRINK, True)
+ else:
+ h2 = h1 + dn.float() * (1.0 / DOWN_SHRINK)
+ if extra is not None:
+ h2 = h2 + extra
+ x = _norm_x(h2, w_norm1n, eps1n)
+ return h2, _lin_x(x, w_qkvn, a_qkvn)
+
+
+def _post_then_final_norm_x(h, o, w_o, w_norm2, eps2, w_gu, w_down, w_normf, epsf,
+ accs, ft):
+ a_o, a_gu, a_down = accs if accs is not None else (None,) * 3
+ h1, dn = _ffn_x(h, o, w_o, w_norm2, eps2, w_gu, w_down, a_o, a_gu, a_down, ft)
+ return _rmsnorm(h1 + dn.float() * (1.0 / DOWN_SHRINK), w_normf, epsf)
+
+
+def _pre_attn_qkv_x(h, w_norm, eps: float, w_qkv, acc):
+ return _lin_x(_norm_x(h, w_norm, eps), w_qkv, acc)
+
+
+# --------------------------------------------------------------------------- #
+# Fused linear cross-entropy
+# --------------------------------------------------------------------------- #
+def _head_chunk(logits, safe, vc, scale: float):
+ """logsumexp, loss, and the scaled softmax, as one fused pass.
+
+ Eager, this chain is five separate passes over a (chunk, 32768) fp32 tensor
+ -- at micro_batch 16 that is 4.3 GB read and written five times per step.
+ Inductor folds it to two.
+ """
+ lse = torch.logsumexp(logits, dim=-1)
+ picked = logits.gather(1, safe[:, None]).squeeze(1)
+ loss = torch.where(vc, lse - picked, torch.zeros_like(lse)).sum()
+ p = (logits - lse[:, None]).exp() * (vc[:, None].to(logits.dtype) * scale)
+ return loss, p
+
+
+_head_chunk_c = None
+
+
+class _FusedLMHead(torch.autograd.Function):
+ """Cross-entropy over the LM head without ever holding (N, V) logits.
+
+ The naive head materialises the logits three times -- fp16 from the
+ projection, fp32 inside cross_entropy, and again for the backward -- which
+ at N=16,384 and V=32,768 is 2.1 GB a copy and sets the micro-batch.
+
+ The head's gradient depends on nothing downstream except a scalar, so this
+ computes dx and dW during the forward walk over vocab-sized chunks and
+ keeps only those. No recomputation in the backward: total GEMM work is the
+ same as the unfused head, and peak memory is one chunk of logits.
+
+ Gradients are held at sum scale (not divided by the token count) and lifted
+ by a further constant: a softmax over 32,768 classes has typical entries
+ near 3e-5, which is SUBNORMAL in fp16 (the smallest normal is 6.1e-5), so
+ the natural scaling would compute the whole dx GEMM on values with a few
+ bits of mantissa. GRAD_SCALE lifts them clear of that. The factor, the 1/N
+ and the loss scale are all divided out together in the backward.
+
+ The constant is bounded from ABOVE by the dW GEMM, which was missed until
+ real data hit it. g = S*(p - onehot), so the true token carries -S, and
+ dW sums that over every token in the chunk sharing a vocab row. Measured on
+ a real batch: the most frequent target appears 89 times in a 2048-token
+ chunk and |x| reaches 5.4, so at S=4096 the largest |dW| entry is 504,389
+ -- 7.7x past fp16's 65,504. It saturated to Inf, Inf-Inf gave NaN in the
+ tied embedding gradient, and the run died on step 2 with loss nan.
+
+ Measured max |dW| against S, same batch:
+
+ S=4096 504,389 OVERFLOW S=256 31,524 (2.1x margin)
+ S=1024 126,097 OVERFLOW S=64 7,881 (8.3x margin)
+
+ 64 is chosen over a value that splits the difference because the two risks
+ are not symmetric. Both grow over a long run -- |x| rises as the residual
+ stream grows, and a confident softmax makes small probabilities smaller --
+ but an overflow is a NaN that ends the run, while an underflow silently
+ drops terms that were already negligible. At S=64 the smallest nonzero
+ |p*S| is 1.48e-4, still 2.4x above fp16's min normal.
+ """
+
+ GRAD_SCALE = 64.0
+ # PERF.md's last unapplied win: accumulate the head's dW in fp16 rather than
+ # fp32. Their +10.1% came from an fp32 accumulate forcing a non-tensor-core
+ # sgemm; ours is already a tensor-core fp16 GEMM with only the accumulator
+ # buffer in fp32, so the mechanism differs. Measured either way.
+ FP16_DW = False
+
+ # Shrinks g for the dW GEMM only (a power of two, so it is exact and is
+ # divided back out losslessly in fp32). dx keeps the full GRAD_SCALE.
+ DW_SHRINK = 0.25
+
+ @staticmethod
+ def forward(ctx, x, W, targets, chunk: int):
+ N, d = x.shape
+ V = W.shape[0]
+ # Cast once, explicitly, rather than relying on autocast to do it: this
+ # runs inside a custom Function and is called from verify.py outside any
+ # autocast region.
+ Wc = W.to(x.dtype)
+ valid = targets != IGNORE
+ # Keep this scalar on-device. int(valid.sum()) was one forced CPU/GPU
+ # synchronization per micro-batch: sixteen pipeline stalls per step.
+ n_valid = valid.sum().clamp_min(1)
+ loss_sum = x.new_zeros((), dtype=torch.float32)
+ dx = torch.empty_like(x)
+ dW = torch.zeros_like(W, dtype=(torch.float16 if _FusedLMHead.FP16_DW
+ else torch.float32))
+
+ for a in range(0, N, chunk):
+ b = min(a + chunk, N)
+ xc, tc = x[a:b], targets[a:b]
+ logits = F.linear(xc, Wc).float() # (c, V)
+ safe = tc.clamp_min(0)
+ vc = (tc != IGNORE)
+ S = _FusedLMHead.GRAD_SCALE
+ loss, p = _head_chunk_c(logits, safe, vc, S)
+ loss_sum += loss
+ del logits
+ # The one-hot subtraction touches c elements, not c*V, so it stays
+ # outside the fused pass.
+ p.scatter_add_(1, safe[:, None], (-S * vc).to(p.dtype)[:, None])
+ g = p.to(x.dtype)
+ del p
+ dx[a:b] = F.linear(g, Wc.t())
+ # dW is the one GEMM here whose fp16 OUTPUT can overflow, and no
+ # loss scale can save it: g and xc are built inside this Function,
+ # so GradScaler halving its scale changes nothing while found_inf
+ # skips AdamW forever. That froze the embeddings, head, norms, Canon
+ # and n-gram table at step ~230 while Muon kept training the trunk
+ # against them; loss bottomed at 6.48 and climbed to nan by 350.
+ # Bound it by construction instead of by choosing a constant:
+ # rescale xc by a power of two so |xn| <= 1 (exact, no rounding),
+ # and shrink g for this GEMM only. |sum| <= S*DW_SHRINK*chunk = 32k
+ # then holds for ANY data, including a chunk of one repeated token,
+ # and both factors are divided back out in fp32.
+ r2 = torch.exp2(torch.ceil(torch.log2(
+ xc.abs().amax().float().clamp_min(1e-30))))
+ xn = xc / r2.to(xc.dtype)
+ gd = g * _FusedLMHead.DW_SHRINK
+ back = r2 / _FusedLMHead.DW_SHRINK
+ dw = gd.t() @ xn
+ # Restore magnitude AFTER widening. Multiplying in fp16 undoes
+ # the overflow protection before the fp32 accumulator sees it.
+ dW += (dw.float() * back).to(dW.dtype)
+
+ ctx.w_dtype = W.dtype
+ ctx.save_for_backward(dx, dW)
+ ctx.n_valid = n_valid
+ return loss_sum / n_valid
+
+ @staticmethod
+ def backward(ctx, grad_out):
+ dx, dW = ctx.saved_tensors
+ s = grad_out / (ctx.n_valid * _FusedLMHead.GRAD_SCALE)
+ # In fp32, then cast. s is ~2.8e-7 without a loss scale, which is
+ # subnormal in fp16 and would carry about two bits of mantissa -- it
+ # cost 7e-02 relative on dx before this was float. Under GradScaler s
+ # lands near 1e-2 and the difference is invisible, which is exactly why
+ # it would never have shown up in the loss curve.
+ return (dx.float() * s).to(dx.dtype), (dW.float() * s).to(ctx.w_dtype), None, None
+
+
+def fused_lm_loss(x, W, targets, chunk: int = 2048, stash: bool = False):
+ if _volta(x):
+ from volta_kernels.head import fused_lm_loss as volta_lm_loss
+ return volta_lm_loss(x, W, targets, chunk, stash)
+ assert not stash
+ global _head_chunk_c
+ if _head_chunk_c is None:
+ _head_chunk_c = torch.compile(_head_chunk, dynamic=False)
+ return _FusedLMHead.apply(x, W, targets, chunk)
+
+
+# --------------------------------------------------------------------------- #
+# Modules
+# --------------------------------------------------------------------------- #
+class RMSNorm(nn.Module):
+ def __init__(self, dim: int, eps: float):
+ super().__init__()
+ self.weight = nn.Parameter(torch.ones(dim))
+ self.eps = eps
+ self.weight._boris_role = "norm"
+
+ def forward(self, x):
+ return _rmsnorm(x, self.weight, self.eps)
+
+
+class Attention(nn.Module):
+ """Causal GQA (8 query heads over 2 KV heads) with fused Canon, varlen."""
+
+ def __init__(self, cfg: ModelConfig):
+ super().__init__()
+ self.cfg = cfg
+ d, hd = cfg.d_model, cfg.head_dim
+ self.n_head, self.n_kv_head, self.head_dim = cfg.n_head, cfg.n_kv_head, hd
+ self.rep = cfg.n_head // cfg.n_kv_head
+
+ # Q, K and V are ONE Parameter, and Muon orthogonalises its three row
+ # blocks independently -- which is exactly what it would do to three
+ # separate Parameters, because the quintic iteration acts on each matrix
+ # alone. Storing them fused removes a torch.cat per layer per
+ # micro-step (2.4% of forward+backward) and its backward slice, without
+ # changing what the optimizer operates on. verify.py asserts the two
+ # spellings produce identical updates.
+ self.qkv_proj = nn.Parameter(torch.empty(cfg.q_dim + 2 * cfg.kv_dim, d))
+ self.qkv_proj._boris_role = "matmul"
+ self.qkv_proj._boris_blocks = ((0, cfg.q_dim),
+ (cfg.q_dim, cfg.q_dim + cfg.kv_dim),
+ (cfg.q_dim + cfg.kv_dim, cfg.q_dim + 2 * cfg.kv_dim))
+ self.o_proj = nn.Parameter(torch.empty(d, cfg.q_dim))
+ self.o_proj._boris_role = "matmul"
+
+ if cfg.use_canon:
+ # (C, 3), zero-init: the layer is exactly the identity at step 0, so
+ # a Canon run and a no-Canon run start from the same function. The
+ # gradient w.r.t. the kernel is dL/dy * x and is non-zero, so it
+ # trains from the first step regardless.
+ self.canon = nn.Parameter(torch.zeros(cfg.canon_channels, cfg.canon_kernel))
+ self.canon._boris_role = "canon"
+ else:
+ self.register_parameter("canon", None)
+
+ def forward(self, h, norm_w, eps, ctx):
+ volta = self.head_dim == 64 and _volta(h)
+ if volta and self.canon is not None and self.rep == 1:
+ from volta_kernels.canon_rope_cuda import canon_rope
+ qkv = ctx.pre_attn_qkv(h, norm_w, eps, self.qkv_proj)
+ q, k, v = canon_rope(qkv, self.canon, ctx.m1, ctx.m2, ctx.cos, ctx.sin)
+ else:
+ q, k, v = ctx.pre_attn(h, norm_w, eps, self.qkv_proj, self.canon,
+ ctx.m1, ctx.m2, ctx.cos, ctx.sin,
+ self.n_head, self.n_kv_head, self.head_dim,
+ self.canon is not None, volta)
+ B, T = q.shape[0], q.shape[1]
+ q = q.reshape(B * T, self.n_head, self.head_dim)
+ # rep == 1 under full MHA: repeat_interleave(1) is a no-op that still
+ # allocates and copies a (B*T, n_head, head_dim) tensor -- 33 MB per
+ # call per layer at micro_batch 16, twice (k and v), on every block in
+ # forward AND backward. Skip it.
+ k = k.reshape(B * T, self.n_kv_head, self.head_dim)
+ v = v.reshape(B * T, self.n_kv_head, self.head_dim)
+ if self.rep > 1:
+ k = k.repeat_interleave(self.rep, 1)
+ v = v.repeat_interleave(self.rep, 1)
+
+ off = ctx.offsets
+ if volta:
+ from volta_kernels import varlen_causal_attention
+ # q/k/v arrive fp16 from the region (out16): the cast autocast
+ # would apply inside SDPA, since the Canon masks keep them fp32.
+ o = varlen_causal_attention(q, k, v, off)
+ return o.reshape(B, T, self.n_head * self.head_dim)
+ nj = lambda t: torch.nested.nested_tensor_from_jagged(
+ t, off, min_seqlen=ctx.min_seqlen, max_seqlen=ctx.max_seqlen)
+ nq = nj(q).transpose(1, 2)
+ nk = nj(k).transpose(1, 2)
+ nv = nj(v).transpose(1, 2)
+ o = F.scaled_dot_product_attention(nq, nk, nv, is_causal=True)
+ return o.transpose(1, 2).values().reshape(B, T, self.n_head * self.head_dim)
+
+
+class FFN(nn.Module):
+ """SwiGLU, inner width 2368. gate and up are one GEMM at call time."""
+
+ def __init__(self, cfg: ModelConfig):
+ super().__init__()
+ h, d = cfg.ffn_hidden, cfg.d_model
+ # gate and up fused for the same reason q/k/v are; Muon still sees two
+ # independent (2368, 512) matrices.
+ self.gate_up_proj = nn.Parameter(torch.empty(2 * h, d))
+ self.gate_up_proj._boris_role = "matmul"
+ self.gate_up_proj._boris_blocks = ((0, h), (h, 2 * h))
+ self.down_proj = nn.Parameter(torch.empty(d, h))
+ self.down_proj._boris_role = "matmul"
+
+
+class Block(nn.Module):
+ def __init__(self, cfg: ModelConfig):
+ super().__init__()
+ self.norm1 = RMSNorm(cfg.d_model, cfg.norm_eps)
+ self.attn = Attention(cfg)
+ self.norm2 = RMSNorm(cfg.d_model, cfg.norm_eps)
+ self.ffn = FFN(cfg)
+
+ def forward(self, h, ctx):
+ o = self.attn(h, self.norm1.weight, self.norm1.eps, ctx)
+ return ctx.post_attn(h, o, self.attn.o_proj, self.norm2.weight,
+ self.norm2.eps, self.ffn.gate_up_proj,
+ self.ffn.down_proj)
+
+
+class RunContext:
+ """Everything shared across all blocks for one forward.
+
+ RoPE tables are gathered once here rather than once per attention layer,
+ and the Canon boundary masks once rather than once per Canon. Holding the
+ compiled callables here keeps a single compiled graph per stage shared by
+ every block.
+ """
+
+ def __init__(self, cos, sin, m1, m2, offsets, pre_attn, post_attn,
+ min_seqlen=None, max_seqlen=None, pre_attn_qkv=None):
+ self.pre_attn_qkv = pre_attn_qkv
+ self.cos, self.sin = cos, sin
+ self.m1, self.m2 = m1, m2
+ self.offsets = offsets
+ self.pre_attn, self.post_attn = pre_attn, post_attn
+ # Without these, every SDPA call derives them from the offsets with a
+ # .max().item() -- a host sync, 131 of them per forward, each stalling
+ # CPU run-ahead. The packer knows both while it is laying documents
+ # down, so they cost nothing to carry.
+ self.min_seqlen, self.max_seqlen = min_seqlen, max_seqlen
+
+
+class Boris2(nn.Module):
+ def __init__(self, cfg: Config):
+ super().__init__()
+ self.cfg = cfg
+ m = cfg.model
+ self.embed = nn.Embedding(m.vocab_size, m.d_model)
+ self.embed.weight._boris_role = "embedding"
+ self.blocks = nn.ModuleList([Block(m) for _ in range(m.n_layer)])
+ self.norm_f = RMSNorm(m.d_model, m.norm_eps)
+ self.ngram = NGramTable(cfg.ngram, m.d_model) if m.use_ngram else None
+ # Runtime toggle, for scoring one trained checkpoint both ways. Note
+ # what it measures: after training, the trunk has co-adapted to the
+ # table, so switching it off reports how much the model DEPENDS on it,
+ # not how much it GAINED from it. The gain needs the use_ngram=False
+ # training arm.
+ self.ngram_enabled = True
+ if not m.tie_embeddings:
+ self.lm_head = nn.Parameter(torch.empty(m.vocab_size, m.d_model))
+ self.lm_head._boris_role = "embedding"
+
+ self._rope_cache: dict[tuple, tuple] = {}
+ self._pre_attn = _pre_attn
+ self._pre_attn_qkv = _pre_attn_qkv
+ self._post_attn = _post_attn
+ self._post_then_pre = _post_then_pre
+ self._post_then_final_norm = _post_then_final_norm
+ self._pre_attn_qkv_x = _pre_attn_qkv_x
+ self._post_then_pre_x = _post_then_pre_x
+ self._post_then_final_norm_x = _post_then_final_norm_x
+ # Swarm additions on the merged path (see volta_kernels/integ_ops.py):
+ # (wgrad_acc, fused norm backward) and the tied embedding's fused grad
+ # accumulate. w16_cache (fp16 weight copies cast once per optimizer
+ # step) is OFF: ~0.15% at mb 4, within noise, and it is only correct
+ # while every step starts with zero_grad(set_to_none=True).
+ self.ft = FT_ALL
+ self.tied_acc = True
+ self.w16_cache = False
+ self._w16c: dict = {}
+ self._compiled = False
+ self.apply_init()
+
+ # -- init ------------------------------------------------------------- #
+ def apply_init(self):
+ m = self.cfg.model
+ nn.init.normal_(self.embed.weight, std=0.02)
+ if not m.tie_embeddings:
+ nn.init.normal_(self.lm_head, std=0.02)
+ # Residual-path projections are scaled down so the residual variance
+ # does not grow with depth -- at 31 layers this matters more than it
+ # does at 10.
+ res_scale = 1.0 / math.sqrt(2 * m.n_layer)
+ for blk in self.blocks:
+ for p in (blk.attn.qkv_proj, blk.ffn.gate_up_proj):
+ nn.init.normal_(p, std=0.02)
+ for p in (blk.attn.o_proj, blk.ffn.down_proj):
+ nn.init.normal_(p, std=0.02 * res_scale)
+ if blk.attn.canon is not None:
+ nn.init.zeros_(blk.attn.canon)
+
+ def maybe_compile(self):
+ if self._compiled or not self.cfg.train.compile_regions:
+ return
+ self._pre_attn = torch.compile(_pre_attn, dynamic=False)
+ self._pre_attn_qkv = torch.compile(_pre_attn_qkv, dynamic=False)
+ self._post_then_pre = torch.compile(_post_then_pre, dynamic=False)
+ self._post_then_final_norm = torch.compile(_post_then_final_norm, dynamic=False)
+ self._post_attn = torch.compile(_post_attn, dynamic=False)
+ self._pre_attn_qkv_x = torch.compile(_pre_attn_qkv_x, dynamic=False)
+ self._post_then_pre_x = torch.compile(_post_then_pre_x, dynamic=False)
+ self._post_then_final_norm_x = torch.compile(_post_then_final_norm_x, dynamic=False)
+ self._compiled = True
+
+ def head_weight(self):
+ return self.embed.weight if self.cfg.model.tie_embeddings else self.lm_head
+
+ # -- rope ------------------------------------------------------------- #
+ def rope_tables(self, pos, theta: float, device, dtype=torch.float32):
+ """cos/sin for (B, T) per-document positions, gathered once per forward.
+
+ Positions restart at every document, so a document packed at row offset
+ 1500 is rotated as though it began at 0. RoPE is relative, so this is
+ not strictly required inside a document -- but it keeps the absolute
+ range a document sees equal to its own length, which is what the
+ context stage is defined in terms of.
+ """
+ hd = self.cfg.model.head_dim
+ key = (theta, hd, device)
+ if key not in self._rope_cache:
+ inv = 1.0 / (theta ** (torch.arange(0, hd, 2, device=device).float() / hd))
+ t = torch.arange(self.cfg.model.max_block_size, device=device).float()
+ f = torch.outer(t, inv)
+ self._rope_cache[key] = (f.cos().to(dtype), f.sin().to(dtype))
+ cos, sin = self._rope_cache[key]
+ return cos[pos].unsqueeze(2), sin[pos].unsqueeze(2) # (B, T, 1, hd/2)
+
+ # -- forward ---------------------------------------------------------- #
+ def forward(self, idx, pos, offsets, boundary, targets=None,
+ theta: float = 10_000.0, min_seqlen=None, max_seqlen=None):
+ """idx/pos/boundary are (B, T); offsets indexes the flattened B*T stream."""
+ cos, sin = self.rope_tables(pos, theta, idx.device)
+ # m1: is t-1 in this document? m2: is t-2? Once per forward, not per layer.
+ b1 = boundary
+ b2 = boundary | F.pad(boundary, (1, 0))[:, :-1]
+ dt = torch.float16 if self.cfg.train.fp16_residual else torch.float32
+ m1 = (~b1).to(dt)[:, :, None]
+ m2 = (~b2).to(dt)[:, :, None]
+ ctx = RunContext(cos, sin, m1, m2, offsets, self._pre_attn, self._post_attn,
+ min_seqlen, max_seqlen, self._pre_attn_qkv)
+
+ gc = self.cfg.train.grad_checkpoint
+ every = max(1, self.cfg.train.grad_checkpoint_every)
+ a0 = self.blocks[0].attn
+ merged = (_volta(idx) and a0.canon is not None and a0.rep == 1 and a0.head_dim == 64
+ and not (gc and self.training) and not self.cfg.train.fp16_residual
+ and not getattr(self, "_no_merge", False))
+ # Tied-embedding fused accumulate: only when the head's backward is
+ # certain to run first and stash its dW (training forward with targets).
+ stash = (merged and self.tied_acc and targets is not None
+ and torch.is_grad_enabled() and self.cfg.model.tie_embeddings
+ and self.embed.weight.requires_grad)
+ if stash:
+ h = TiedEmbedding.apply(idx, self.embed.weight)
+ else:
+ h = self.embed(idx)
+ if self.cfg.train.fp16_residual:
+ h = h.to(torch.float16)
+
+ if merged:
+ h = self._forward_merged(h, idx, boundary, ctx)
+ if targets is None:
+ return h
+ return fused_lm_loss(h.reshape(-1, h.shape[-1]), self.head_weight(),
+ targets.reshape(-1), self.cfg.optim.lm_head_chunk,
+ stash)
+ for i, blk in enumerate(self.blocks):
+ if (self.ngram is not None and self.ngram_enabled
+ and i == self.cfg.ngram.insert_after):
+ h = h + self.ngram(idx, boundary, h.dtype)
+ if gc and self.training and (i % every == 0):
+ h = torch.utils.checkpoint.checkpoint(
+ blk, h, ctx, use_reentrant=False)
+ else:
+ h = blk(h, ctx)
+
+ h = self.norm_f(h)
+ if targets is None:
+ return h
+ loss = fused_lm_loss(h.reshape(-1, h.shape[-1]), self.head_weight(),
+ targets.reshape(-1), self.cfg.optim.lm_head_chunk)
+ return loss
+
+ @staticmethod
+ def _acc(p):
+ """p.grad as the fp32 accumulator the weight-gradient GEMMs add into;
+ created (zeroed) on the first micro-step after zero_grad(set_to_none)."""
+ if not p.requires_grad:
+ return None
+ if p.grad is None:
+ p.grad = torch.zeros_like(p, memory_format=torch.contiguous_format)
+ return p.grad
+
+ def _wq(self, p, acc, scale: float = 1.0):
+ """The weight as the region should receive it: the fp32 Parameter
+ (cast inside the region), or -- with w16_cache -- an fp16 copy cast once
+ per optimizer step. The copy is keyed on the identity of p.grad: every
+ step starts with zero_grad(set_to_none=True), so a new accumulator means
+ the weights may have moved. (p._version can NOT be used: Muon updates
+ p.data in place, which does not bump the version counter.)"""
+ if not (self.w16_cache and acc is not None):
+ return p
+ import weakref
+ e = self._w16c.get(id(p))
+ if e is not None and e[1]() is acc:
+ return e[0]
+ with torch.no_grad():
+ w = (p * scale if scale != 1.0 else p).to(torch.float16)
+ self._w16c[id(p)] = (w, weakref.ref(acc))
+ return w
+
+ def _forward_merged(self, h, idx, boundary, ctx):
+ """Blocks with each residual boundary inside one compiled region
+ (Volta path, no gradient checkpointing). Same operations, same order,
+ as the per-block path; returns norm_f(h)."""
+ if torch.is_grad_enabled() and any(self.ft):
+ return self._forward_merged_x(h, idx, boundary, ctx)
+ return self._forward_merged_stock(h, idx, boundary, ctx)
+
+ def _forward_merged_x(self, h, idx, boundary, ctx):
+ """_forward_merged with the swarm additions (self.ft, self.w16_cache)."""
+ from volta_kernels import canon_rope_attention
+ ft = self.ft
+ wacc = ft[0]
+ A = self._acc if wacc else (lambda p: None)
+ W = self._wq
+ blocks = self.blocks
+ L = len(blocks)
+ ins = self.cfg.ngram.insert_after if (self.ngram is not None
+ and self.ngram_enabled) else -1
+ ng = self.ngram(idx, boundary, h.dtype) if ins >= 0 else None
+ if ins == 0:
+ h = h + ng
+ B, T = h.shape[0], h.shape[1]
+ b0 = blocks[0]
+ a0 = A(b0.attn.qkv_proj)
+ qkv = self._pre_attn_qkv_x(h, b0.norm1.weight, b0.norm1.eps,
+ W(b0.attn.qkv_proj, a0), a0)
+ for i, blk in enumerate(blocks):
+ a = blk.attn
+ o = canon_rope_attention(qkv, a.canon, ctx.m1, ctx.m2, ctx.cos, ctx.sin,
+ ctx.offsets).reshape(B, T, a.n_head * a.head_dim)
+ f = blk.ffn
+ ao, agu, adn = A(a.o_proj), A(f.gate_up_proj), A(f.down_proj)
+ wo, wgu, wdn = W(a.o_proj, ao), W(f.gate_up_proj, agu), W(f.down_proj, adn, DOWN_SHRINK)
+ if i + 1 < L:
+ nb = blocks[i + 1]
+ aq = A(nb.attn.qkv_proj)
+ h, qkv = self._post_then_pre_x(
+ h, o, wo, blk.norm2.weight, blk.norm2.eps, wgu, wdn,
+ nb.norm1.weight, nb.norm1.eps, W(nb.attn.qkv_proj, aq),
+ ng if i + 1 == ins else None,
+ (ao, agu, adn, aq) if wacc else None, ft)
+ else:
+ h = self._post_then_final_norm_x(
+ h, o, wo, blk.norm2.weight, blk.norm2.eps, wgu, wdn,
+ self.norm_f.weight, self.norm_f.eps,
+ (ao, agu, adn) if wacc else None, ft)
+ return h
+
+ def _forward_merged_stock(self, h, idx, boundary, ctx):
+ from volta_kernels import canon_rope_attention
+ blocks = self.blocks
+ L = len(blocks)
+ ins = self.cfg.ngram.insert_after if (self.ngram is not None
+ and self.ngram_enabled) else -1
+ ng = self.ngram(idx, boundary, h.dtype) if ins >= 0 else None
+ if ins == 0:
+ h = h + ng
+ B, T = h.shape[0], h.shape[1]
+ b0 = blocks[0]
+ qkv = self._pre_attn_qkv(h, b0.norm1.weight, b0.norm1.eps, b0.attn.qkv_proj)
+ for i, blk in enumerate(blocks):
+ a = blk.attn
+ # Canon+RoPE+attention as one autograd Function: dQ reaches the
+ # Canon backward in fp32 (see volta_kernels._CanonRopeAttn).
+ o = canon_rope_attention(qkv, a.canon, ctx.m1, ctx.m2, ctx.cos, ctx.sin,
+ ctx.offsets).reshape(B, T, a.n_head * a.head_dim)
+ if i + 1 < L:
+ nb = blocks[i + 1]
+ h, qkv = self._post_then_pre(
+ h, o, a.o_proj, blk.norm2.weight, blk.norm2.eps,
+ blk.ffn.gate_up_proj, blk.ffn.down_proj,
+ nb.norm1.weight, nb.norm1.eps, nb.attn.qkv_proj,
+ ng if i + 1 == ins else None)
+ else:
+ h = self._post_then_final_norm(
+ h, o, a.o_proj, blk.norm2.weight, blk.norm2.eps,
+ blk.ffn.gate_up_proj, blk.ffn.down_proj,
+ self.norm_f.weight, self.norm_f.eps)
+ return h
+
+ @torch.no_grad()
+ def logits_for_eval(self, idx, pos, offsets, boundary, theta: float = 10_000.0):
+ h = self.forward(idx, pos, offsets, boundary, None, theta)
+ return F.linear(h, self.head_weight())