From 5d2e4252e695644c34be605b44a14bd80a98a0a6 Mon Sep 17 00:00:00 2001 From: jdj Date: Wed, 18 Nov 2026 14:47:33 -0800 Subject: Boris-2 training code The code, configuration and data filters used to train Boris-2 (144M, 200B tokens, single V100): model, training loop, data preparation, n-gram table, V100 kernels, final stages and the exact run configs. --- model.py | 833 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 833 insertions(+) create mode 100644 model.py (limited to 'model.py') diff --git a/model.py b/model.py new file mode 100644 index 0000000..69d0db5 --- /dev/null +++ b/model.py @@ -0,0 +1,833 @@ +"""Boris-2: a d512 x 31L decoder-only transformer with fused Canon layers. + +Shape of a block: + + h = h + o_proj( attn( canon( qkv_proj( norm(h) ) ) ) ) + h = h + down( silu(gate) * up )( norm(h) ) + +Four things here are not the textbook spelling, and each is a measured win on +sm_70 rather than a preference: + +1. Q, K and V are one GEMM. The weights stay three separate Parameters and are + concatenated at call time, so Muon still orthogonalises each independently -- + fusing the Parameters themselves would change what the optimizer operates on + and would not be lossless. The concatenated weight is ~1.2MB against a + 16,384-row activation. + +2. Canon is one kernel over the concatenated Q|K|V block. Depthwise convolution + is per-channel, so a single 768-channel convolution equals three separate + ones on the Q/K/V channel ranges: 31 kernels per forward instead of 93. It is + spelled as three shifted multiply-adds on the native (B,T,C) layout rather + than F.conv1d(groups=C), which needs a (B,T,C)->(B,C,T) transpose whose + following reshape materialises a copy. Measured on this V100, fwd+bwd over + the real shape: 4.30 ms for conv1d, 1.10 ms for the compiled shift form -- + substantially faster than the transposing grouped-convolution spelling. + +3. Canon is document-aware. Kernel 3 stacked 31 deep reaches 62 tokens back, and + rows hold several packed documents, so without masking every block pulls the + previous document's tail across the boundary. The mask costs 0.10 ms/layer. + This is the bug the 75M run found the hard way; verify.py probes for it with + the output projections de-zeroed, because at init a zero-init projection + makes the test pass no matter how badly the model leaks. + +4. Attention is varlen over a jagged nested tensor: one SDPA launch for the + whole batch, cost sum(len_i^2) rather than T^2. Measured against the two + alternatives at ctx 2048 with realistic document lengths, fwd+bwd: + nested 11.09 ms, dense block-diagonal mask 26.97 ms, per-document loop + 45.84 ms. Nested is bit-identical to the loop (max diff 0.0). + +The residual stream is fp32 on the V100 because measured massive activations can +exceed fp16's finite range. Matmul inputs are cast by the compiled regions under +autocast, preserving tensor-core execution without risking a forward overflow. +""" +from __future__ import annotations + +import math +import os + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from config import Config, ModelConfig +from ngram import NGramTable +from volta_kernels.integ_ops import AddNorm, LinAcc, TiedEmbedding + +IGNORE = -100 + +# Hand-written sm_70 kernels (volta_kernels/): varlen FlashAttention with mma.sync +# and a one-pass LM-head softmax. Opt-in, because they are adopted by a running +# 200B-token job at a resume: BORIS_VOLTA_KERNELS=1. Same operand dtypes and fp32 +# accumulation as the paths they replace; verify.py checks both against fp64. +# They only engage on an sm_70 device -- elsewhere (eval on the 3060) the stock +# paths run regardless. +VOLTA_KERNELS = os.environ.get("BORIS_VOLTA_KERNELS", "0") == "1" + + +def _volta(t: torch.Tensor) -> bool: + return VOLTA_KERNELS and t.is_cuda and torch.cuda.get_device_capability(t.device) == (7, 0) + + +# --------------------------------------------------------------------------- # +# Compiled fixed-shape regions +# --------------------------------------------------------------------------- # +# Whole-model compile is not used: the 75M run measured it 28% slower in +# realistic conditions because document counts vary and Inductor recompiles +# mid-run. These regions never see a ragged shape -- only (B, T, C) with B and T +# fixed for the whole context stage -- so each compiles once per stage. + +def _rmsnorm(x, w, eps: float): + f = x.float() + f = f * torch.rsqrt(f.pow(2).mean(-1, keepdim=True) + eps) + return (f * w.float()).to(x.dtype) + + +def _pre_attn(h, w_norm, eps: float, w_qkv, canon_w, m1, m2, cos, sin, + n_head: int, n_kv_head: int, head_dim: int, use_canon: bool, + out16: bool = False): + """Norm, one GEMM, one depthwise convolution, one rotation -- one region. + + The RMSNorm is inside the region on purpose. Written as its own eager module + it upcasts to fp32, and that materialises a (B, T, 512) fp32 tensor for + every one of the 53 norms in the model -- 25% of forward+backward was + elementwise traffic before these regions were widened, and this was most of + it. Inductor keeps the reduction in fp32 without ever writing the fp32 + tensor out. + + canon_w is (C, 3) with column 0 the weight on token t, column 1 on t-1 and + column 2 on t-2. m1 masks the t-1 tap and m2 the t-2 tap at document + boundaries; both are (B, T, 1) and are computed once per forward, not per + layer. + """ + f = h.float() + x = (f * torch.rsqrt(f.pow(2).mean(-1, keepdim=True) + eps) + * w_norm.float()).to(h.dtype) + qkv = F.linear(x, w_qkv) # (B, T, C) + if use_canon: + w = canon_w.to(qkv.dtype) + s1 = F.pad(qkv, (0, 0, 1, 0))[:, :-1] * m1 + s2 = F.pad(qkv, (0, 0, 2, 0))[:, :-2] * m2 + qkv = qkv + qkv * w[:, 0] + s1 * w[:, 1] + s2 * w[:, 2] + + B, T, _ = qkv.shape + qd, kd = n_head * head_dim, n_kv_head * head_dim + q = qkv[..., :qd].view(B, T, n_head, head_dim) + k = qkv[..., qd:qd + kd].view(B, T, n_kv_head, head_dim) + v = qkv[..., qd + kd:].view(B, T, n_kv_head, head_dim) + + def rope(t): + f = t.float() + a, b = f.chunk(2, dim=-1) + return torch.cat([a * cos - b * sin, a * sin + b * cos], -1).to(t.dtype) + + if out16: + # The Volta attention takes fp16. Casting here rounds the same fp32 + # values autocast would round, but inside the region's epilogue, so the + # fp32 q/k/v are never written out and read back by separate casts. + h16 = torch.float16 + return rope(q).to(h16), rope(k).to(h16), v.to(h16) + return rope(q), rope(k), v + + +def _pre_attn_qkv(h, w_norm, eps: float, w_qkv): + """Norm and the fused QKV GEMM only -- the Volta path's region. + + Canon and RoPE then run as one fused Triton pass (volta_kernels/canon_rope.py) + instead of Inductor's pad-copy / combine / rotate kernels. + """ + f = h.float() + x = (f * torch.rsqrt(f.pow(2).mean(-1, keepdim=True) + eps) + * w_norm.float()).to(h.dtype) + return F.linear(x, w_qkv) + + +def _post_then_pre(h, o, w_o, w_norm2, eps2, w_gu, w_down, w_norm1n, eps1n, w_qkvn, + extra): + """Block i's FFN half and block i+1's norm + QKV GEMM as ONE region. + + As separate regions the residual h crossed a graph boundary between two + consumers, so its two backward gradient streams were summed by an eager + fp32 add (~192 MB of traffic per layer) and h was written and read back. + Here Inductor fuses that sum into the region's backward. `extra` is the + n-gram residual when block i+1 is where it is inserted (it depends only on + the token ids), else None -- each value gets its own compiled graph. + """ + h = _post_attn(h, o, w_o, w_norm2, eps2, w_gu, w_down) + if extra is not None: + h = h + extra + return h, _pre_attn_qkv(h, w_norm1n, eps1n, w_qkvn) + + +def _post_then_final_norm(h, o, w_o, w_norm2, eps2, w_gu, w_down, w_normf, epsf): + """The last block's FFN half and the final RMSNorm as one region.""" + h = _post_attn(h, o, w_o, w_norm2, eps2, w_gu, w_down) + return _rmsnorm(h, w_normf, epsf) + + +# Power-of-two headroom for the FFN down projection's fp16 output. 1/8 puts the +# measured step-4,520 peak (~65.5k) at ~8.2k, leaving 8x before it overflows +# again, and costs only inputs below 4.9e-4 -- 1.6e-8 of the peak, which +# contribute nothing to a sum dominated by terms 6e7 times larger. +DOWN_SHRINK = 0.125 + + +def _post_attn(h, o, w_o, w_norm, eps: float, w_gu, w_down): + """Output projection, residual, norm, SwiGLU, residual -- one region. + + Everything in a block except the attention itself. The two residual adds and + the second norm were separate eager kernels over the widest tensors in the + model; inside the region Inductor folds them into the GEMM epilogues, and + the fp32 norm intermediate is never written out. + """ + h = h + F.linear(o, w_o) + f = h.float() + x = (f * torch.rsqrt(f.pow(2).mean(-1, keepdim=True) + eps) + * w_norm.float()).to(h.dtype) + gu = F.linear(x, w_gu) + g, u = gu.chunk(2, dim=-1) + # The down projection is the one GEMM here whose fp16 OUTPUT can overflow. + # Its inputs are finite and so is the true result -- it is the fp16 range + # that runs out. Channel 228 of the residual is a "massive activation", + # 173x the RMS (78,115 against an RMS of 451 at block 16), and the FFN has + # to write into it, so at step 4,520 this output crossed 65,504 and became + # inf; norm_f then turned that into NaN via rsqrt(inf)=0, inf*0. + # Pre-scaling w_down is exact (a power of two), keeps the GEMM on tensor + # cores, costs one pass over a 1.2M-element weight rather than the 77M- + # element activation, and is divided back out in fp32 where there is room. + # Every partial sum is scaled too, so this is safe even with cuBLAS's + # reduced-precision fp16 reductions enabled. + return h + F.linear(F.silu(g) * u, + w_down * DOWN_SHRINK).float() * (1.0 / DOWN_SHRINK) + + +# --------------------------------------------------------------------------- # +# Merged-path regions with the swarm additions (volta_kernels/integ_ops.py) +# --------------------------------------------------------------------------- # +# ft = (wacc, norm) -- which additions a compiled region uses: +# wacc: weight-gradient GEMMs accumulate into the fp32 .grad (accs given) +# norm: residual-add + RMSNorm backward and norm-weight grad in one Triton pass +# Weights arrive either as the fp32 Parameters (cast here, as autocast would) or +# as the cached, pre-scaled fp16 copies (Boris2._w16); both give the same fp16 +# operand, so the forward is bitwise the live one either way. +FT_ALL = (True, True) + + +def _w16(w, scale: float = 1.0): + if w.dtype == torch.float16: + return w # cached copy, already scaled + return (w * scale).to(torch.float16) if scale != 1.0 else w.to(torch.float16) + + +def _lin_x(x, w, acc, scale: float = 1.0): + if acc is None: + if w.dtype == torch.float16: + return F.linear(x, w) + return F.linear(x, w * scale if scale != 1.0 else w) + return LinAcc.apply(x.to(torch.float16), _w16(w, scale), acc, scale) + + +def _norm_x(h, w, eps: float): + f = h.float() + return (f * torch.rsqrt(f.pow(2).mean(-1, keepdim=True) + eps) + * w.float()).to(h.dtype) + + +def _ffn_x(h, o, w_o, w_norm, eps: float, w_gu, w_down, a_o, a_gu, a_down, ft): + """Block FFN half up to (but excluding) the down projection's residual add: + returns h1 = h + o_proj(o) and the fp16 down output (pre-scaled by + DOWN_SHRINK), so the next norm can fold the add into its backward.""" + wacc, norm = ft + a = _lin_x(o, w_o, a_o) + if norm: + h1, x = AddNorm.apply(h, a, w_norm, eps, 1.0, False) + else: + h1 = h + a + x = _norm_x(h1, w_norm, eps) + gu = _lin_x(x, w_gu, a_gu) + g, u = gu.chunk(2, dim=-1) + return h1, _lin_x(F.silu(g) * u, w_down, a_down, DOWN_SHRINK) + + +def _post_then_pre_x(h, o, w_o, w_norm2, eps2, w_gu, w_down, w_norm1n, eps1n, w_qkvn, + extra, accs, ft): + a_o, a_gu, a_down, a_qkvn = accs if accs is not None else (None,) * 4 + h1, dn = _ffn_x(h, o, w_o, w_norm2, eps2, w_gu, w_down, a_o, a_gu, a_down, ft) + if ft[1] and extra is None: + h2, x = AddNorm.apply(h1, dn, w_norm1n, eps1n, 1.0 / DOWN_SHRINK, True) + else: + h2 = h1 + dn.float() * (1.0 / DOWN_SHRINK) + if extra is not None: + h2 = h2 + extra + x = _norm_x(h2, w_norm1n, eps1n) + return h2, _lin_x(x, w_qkvn, a_qkvn) + + +def _post_then_final_norm_x(h, o, w_o, w_norm2, eps2, w_gu, w_down, w_normf, epsf, + accs, ft): + a_o, a_gu, a_down = accs if accs is not None else (None,) * 3 + h1, dn = _ffn_x(h, o, w_o, w_norm2, eps2, w_gu, w_down, a_o, a_gu, a_down, ft) + return _rmsnorm(h1 + dn.float() * (1.0 / DOWN_SHRINK), w_normf, epsf) + + +def _pre_attn_qkv_x(h, w_norm, eps: float, w_qkv, acc): + return _lin_x(_norm_x(h, w_norm, eps), w_qkv, acc) + + +# --------------------------------------------------------------------------- # +# Fused linear cross-entropy +# --------------------------------------------------------------------------- # +def _head_chunk(logits, safe, vc, scale: float): + """logsumexp, loss, and the scaled softmax, as one fused pass. + + Eager, this chain is five separate passes over a (chunk, 32768) fp32 tensor + -- at micro_batch 16 that is 4.3 GB read and written five times per step. + Inductor folds it to two. + """ + lse = torch.logsumexp(logits, dim=-1) + picked = logits.gather(1, safe[:, None]).squeeze(1) + loss = torch.where(vc, lse - picked, torch.zeros_like(lse)).sum() + p = (logits - lse[:, None]).exp() * (vc[:, None].to(logits.dtype) * scale) + return loss, p + + +_head_chunk_c = None + + +class _FusedLMHead(torch.autograd.Function): + """Cross-entropy over the LM head without ever holding (N, V) logits. + + The naive head materialises the logits three times -- fp16 from the + projection, fp32 inside cross_entropy, and again for the backward -- which + at N=16,384 and V=32,768 is 2.1 GB a copy and sets the micro-batch. + + The head's gradient depends on nothing downstream except a scalar, so this + computes dx and dW during the forward walk over vocab-sized chunks and + keeps only those. No recomputation in the backward: total GEMM work is the + same as the unfused head, and peak memory is one chunk of logits. + + Gradients are held at sum scale (not divided by the token count) and lifted + by a further constant: a softmax over 32,768 classes has typical entries + near 3e-5, which is SUBNORMAL in fp16 (the smallest normal is 6.1e-5), so + the natural scaling would compute the whole dx GEMM on values with a few + bits of mantissa. GRAD_SCALE lifts them clear of that. The factor, the 1/N + and the loss scale are all divided out together in the backward. + + The constant is bounded from ABOVE by the dW GEMM, which was missed until + real data hit it. g = S*(p - onehot), so the true token carries -S, and + dW sums that over every token in the chunk sharing a vocab row. Measured on + a real batch: the most frequent target appears 89 times in a 2048-token + chunk and |x| reaches 5.4, so at S=4096 the largest |dW| entry is 504,389 + -- 7.7x past fp16's 65,504. It saturated to Inf, Inf-Inf gave NaN in the + tied embedding gradient, and the run died on step 2 with loss nan. + + Measured max |dW| against S, same batch: + + S=4096 504,389 OVERFLOW S=256 31,524 (2.1x margin) + S=1024 126,097 OVERFLOW S=64 7,881 (8.3x margin) + + 64 is chosen over a value that splits the difference because the two risks + are not symmetric. Both grow over a long run -- |x| rises as the residual + stream grows, and a confident softmax makes small probabilities smaller -- + but an overflow is a NaN that ends the run, while an underflow silently + drops terms that were already negligible. At S=64 the smallest nonzero + |p*S| is 1.48e-4, still 2.4x above fp16's min normal. + """ + + GRAD_SCALE = 64.0 + # PERF.md's last unapplied win: accumulate the head's dW in fp16 rather than + # fp32. Their +10.1% came from an fp32 accumulate forcing a non-tensor-core + # sgemm; ours is already a tensor-core fp16 GEMM with only the accumulator + # buffer in fp32, so the mechanism differs. Measured either way. + FP16_DW = False + + # Shrinks g for the dW GEMM only (a power of two, so it is exact and is + # divided back out losslessly in fp32). dx keeps the full GRAD_SCALE. + DW_SHRINK = 0.25 + + @staticmethod + def forward(ctx, x, W, targets, chunk: int): + N, d = x.shape + V = W.shape[0] + # Cast once, explicitly, rather than relying on autocast to do it: this + # runs inside a custom Function and is called from verify.py outside any + # autocast region. + Wc = W.to(x.dtype) + valid = targets != IGNORE + # Keep this scalar on-device. int(valid.sum()) was one forced CPU/GPU + # synchronization per micro-batch: sixteen pipeline stalls per step. + n_valid = valid.sum().clamp_min(1) + loss_sum = x.new_zeros((), dtype=torch.float32) + dx = torch.empty_like(x) + dW = torch.zeros_like(W, dtype=(torch.float16 if _FusedLMHead.FP16_DW + else torch.float32)) + + for a in range(0, N, chunk): + b = min(a + chunk, N) + xc, tc = x[a:b], targets[a:b] + logits = F.linear(xc, Wc).float() # (c, V) + safe = tc.clamp_min(0) + vc = (tc != IGNORE) + S = _FusedLMHead.GRAD_SCALE + loss, p = _head_chunk_c(logits, safe, vc, S) + loss_sum += loss + del logits + # The one-hot subtraction touches c elements, not c*V, so it stays + # outside the fused pass. + p.scatter_add_(1, safe[:, None], (-S * vc).to(p.dtype)[:, None]) + g = p.to(x.dtype) + del p + dx[a:b] = F.linear(g, Wc.t()) + # dW is the one GEMM here whose fp16 OUTPUT can overflow, and no + # loss scale can save it: g and xc are built inside this Function, + # so GradScaler halving its scale changes nothing while found_inf + # skips AdamW forever. That froze the embeddings, head, norms, Canon + # and n-gram table at step ~230 while Muon kept training the trunk + # against them; loss bottomed at 6.48 and climbed to nan by 350. + # Bound it by construction instead of by choosing a constant: + # rescale xc by a power of two so |xn| <= 1 (exact, no rounding), + # and shrink g for this GEMM only. |sum| <= S*DW_SHRINK*chunk = 32k + # then holds for ANY data, including a chunk of one repeated token, + # and both factors are divided back out in fp32. + r2 = torch.exp2(torch.ceil(torch.log2( + xc.abs().amax().float().clamp_min(1e-30)))) + xn = xc / r2.to(xc.dtype) + gd = g * _FusedLMHead.DW_SHRINK + back = r2 / _FusedLMHead.DW_SHRINK + dw = gd.t() @ xn + # Restore magnitude AFTER widening. Multiplying in fp16 undoes + # the overflow protection before the fp32 accumulator sees it. + dW += (dw.float() * back).to(dW.dtype) + + ctx.w_dtype = W.dtype + ctx.save_for_backward(dx, dW) + ctx.n_valid = n_valid + return loss_sum / n_valid + + @staticmethod + def backward(ctx, grad_out): + dx, dW = ctx.saved_tensors + s = grad_out / (ctx.n_valid * _FusedLMHead.GRAD_SCALE) + # In fp32, then cast. s is ~2.8e-7 without a loss scale, which is + # subnormal in fp16 and would carry about two bits of mantissa -- it + # cost 7e-02 relative on dx before this was float. Under GradScaler s + # lands near 1e-2 and the difference is invisible, which is exactly why + # it would never have shown up in the loss curve. + return (dx.float() * s).to(dx.dtype), (dW.float() * s).to(ctx.w_dtype), None, None + + +def fused_lm_loss(x, W, targets, chunk: int = 2048, stash: bool = False): + if _volta(x): + from volta_kernels.head import fused_lm_loss as volta_lm_loss + return volta_lm_loss(x, W, targets, chunk, stash) + assert not stash + global _head_chunk_c + if _head_chunk_c is None: + _head_chunk_c = torch.compile(_head_chunk, dynamic=False) + return _FusedLMHead.apply(x, W, targets, chunk) + + +# --------------------------------------------------------------------------- # +# Modules +# --------------------------------------------------------------------------- # +class RMSNorm(nn.Module): + def __init__(self, dim: int, eps: float): + super().__init__() + self.weight = nn.Parameter(torch.ones(dim)) + self.eps = eps + self.weight._boris_role = "norm" + + def forward(self, x): + return _rmsnorm(x, self.weight, self.eps) + + +class Attention(nn.Module): + """Causal GQA (8 query heads over 2 KV heads) with fused Canon, varlen.""" + + def __init__(self, cfg: ModelConfig): + super().__init__() + self.cfg = cfg + d, hd = cfg.d_model, cfg.head_dim + self.n_head, self.n_kv_head, self.head_dim = cfg.n_head, cfg.n_kv_head, hd + self.rep = cfg.n_head // cfg.n_kv_head + + # Q, K and V are ONE Parameter, and Muon orthogonalises its three row + # blocks independently -- which is exactly what it would do to three + # separate Parameters, because the quintic iteration acts on each matrix + # alone. Storing them fused removes a torch.cat per layer per + # micro-step (2.4% of forward+backward) and its backward slice, without + # changing what the optimizer operates on. verify.py asserts the two + # spellings produce identical updates. + self.qkv_proj = nn.Parameter(torch.empty(cfg.q_dim + 2 * cfg.kv_dim, d)) + self.qkv_proj._boris_role = "matmul" + self.qkv_proj._boris_blocks = ((0, cfg.q_dim), + (cfg.q_dim, cfg.q_dim + cfg.kv_dim), + (cfg.q_dim + cfg.kv_dim, cfg.q_dim + 2 * cfg.kv_dim)) + self.o_proj = nn.Parameter(torch.empty(d, cfg.q_dim)) + self.o_proj._boris_role = "matmul" + + if cfg.use_canon: + # (C, 3), zero-init: the layer is exactly the identity at step 0, so + # a Canon run and a no-Canon run start from the same function. The + # gradient w.r.t. the kernel is dL/dy * x and is non-zero, so it + # trains from the first step regardless. + self.canon = nn.Parameter(torch.zeros(cfg.canon_channels, cfg.canon_kernel)) + self.canon._boris_role = "canon" + else: + self.register_parameter("canon", None) + + def forward(self, h, norm_w, eps, ctx): + volta = self.head_dim == 64 and _volta(h) + if volta and self.canon is not None and self.rep == 1: + from volta_kernels.canon_rope_cuda import canon_rope + qkv = ctx.pre_attn_qkv(h, norm_w, eps, self.qkv_proj) + q, k, v = canon_rope(qkv, self.canon, ctx.m1, ctx.m2, ctx.cos, ctx.sin) + else: + q, k, v = ctx.pre_attn(h, norm_w, eps, self.qkv_proj, self.canon, + ctx.m1, ctx.m2, ctx.cos, ctx.sin, + self.n_head, self.n_kv_head, self.head_dim, + self.canon is not None, volta) + B, T = q.shape[0], q.shape[1] + q = q.reshape(B * T, self.n_head, self.head_dim) + # rep == 1 under full MHA: repeat_interleave(1) is a no-op that still + # allocates and copies a (B*T, n_head, head_dim) tensor -- 33 MB per + # call per layer at micro_batch 16, twice (k and v), on every block in + # forward AND backward. Skip it. + k = k.reshape(B * T, self.n_kv_head, self.head_dim) + v = v.reshape(B * T, self.n_kv_head, self.head_dim) + if self.rep > 1: + k = k.repeat_interleave(self.rep, 1) + v = v.repeat_interleave(self.rep, 1) + + off = ctx.offsets + if volta: + from volta_kernels import varlen_causal_attention + # q/k/v arrive fp16 from the region (out16): the cast autocast + # would apply inside SDPA, since the Canon masks keep them fp32. + o = varlen_causal_attention(q, k, v, off) + return o.reshape(B, T, self.n_head * self.head_dim) + nj = lambda t: torch.nested.nested_tensor_from_jagged( + t, off, min_seqlen=ctx.min_seqlen, max_seqlen=ctx.max_seqlen) + nq = nj(q).transpose(1, 2) + nk = nj(k).transpose(1, 2) + nv = nj(v).transpose(1, 2) + o = F.scaled_dot_product_attention(nq, nk, nv, is_causal=True) + return o.transpose(1, 2).values().reshape(B, T, self.n_head * self.head_dim) + + +class FFN(nn.Module): + """SwiGLU, inner width 2368. gate and up are one GEMM at call time.""" + + def __init__(self, cfg: ModelConfig): + super().__init__() + h, d = cfg.ffn_hidden, cfg.d_model + # gate and up fused for the same reason q/k/v are; Muon still sees two + # independent (2368, 512) matrices. + self.gate_up_proj = nn.Parameter(torch.empty(2 * h, d)) + self.gate_up_proj._boris_role = "matmul" + self.gate_up_proj._boris_blocks = ((0, h), (h, 2 * h)) + self.down_proj = nn.Parameter(torch.empty(d, h)) + self.down_proj._boris_role = "matmul" + + +class Block(nn.Module): + def __init__(self, cfg: ModelConfig): + super().__init__() + self.norm1 = RMSNorm(cfg.d_model, cfg.norm_eps) + self.attn = Attention(cfg) + self.norm2 = RMSNorm(cfg.d_model, cfg.norm_eps) + self.ffn = FFN(cfg) + + def forward(self, h, ctx): + o = self.attn(h, self.norm1.weight, self.norm1.eps, ctx) + return ctx.post_attn(h, o, self.attn.o_proj, self.norm2.weight, + self.norm2.eps, self.ffn.gate_up_proj, + self.ffn.down_proj) + + +class RunContext: + """Everything shared across all blocks for one forward. + + RoPE tables are gathered once here rather than once per attention layer, + and the Canon boundary masks once rather than once per Canon. Holding the + compiled callables here keeps a single compiled graph per stage shared by + every block. + """ + + def __init__(self, cos, sin, m1, m2, offsets, pre_attn, post_attn, + min_seqlen=None, max_seqlen=None, pre_attn_qkv=None): + self.pre_attn_qkv = pre_attn_qkv + self.cos, self.sin = cos, sin + self.m1, self.m2 = m1, m2 + self.offsets = offsets + self.pre_attn, self.post_attn = pre_attn, post_attn + # Without these, every SDPA call derives them from the offsets with a + # .max().item() -- a host sync, 131 of them per forward, each stalling + # CPU run-ahead. The packer knows both while it is laying documents + # down, so they cost nothing to carry. + self.min_seqlen, self.max_seqlen = min_seqlen, max_seqlen + + +class Boris2(nn.Module): + def __init__(self, cfg: Config): + super().__init__() + self.cfg = cfg + m = cfg.model + self.embed = nn.Embedding(m.vocab_size, m.d_model) + self.embed.weight._boris_role = "embedding" + self.blocks = nn.ModuleList([Block(m) for _ in range(m.n_layer)]) + self.norm_f = RMSNorm(m.d_model, m.norm_eps) + self.ngram = NGramTable(cfg.ngram, m.d_model) if m.use_ngram else None + # Runtime toggle, for scoring one trained checkpoint both ways. Note + # what it measures: after training, the trunk has co-adapted to the + # table, so switching it off reports how much the model DEPENDS on it, + # not how much it GAINED from it. The gain needs the use_ngram=False + # training arm. + self.ngram_enabled = True + if not m.tie_embeddings: + self.lm_head = nn.Parameter(torch.empty(m.vocab_size, m.d_model)) + self.lm_head._boris_role = "embedding" + + self._rope_cache: dict[tuple, tuple] = {} + self._pre_attn = _pre_attn + self._pre_attn_qkv = _pre_attn_qkv + self._post_attn = _post_attn + self._post_then_pre = _post_then_pre + self._post_then_final_norm = _post_then_final_norm + self._pre_attn_qkv_x = _pre_attn_qkv_x + self._post_then_pre_x = _post_then_pre_x + self._post_then_final_norm_x = _post_then_final_norm_x + # Swarm additions on the merged path (see volta_kernels/integ_ops.py): + # (wgrad_acc, fused norm backward) and the tied embedding's fused grad + # accumulate. w16_cache (fp16 weight copies cast once per optimizer + # step) is OFF: ~0.15% at mb 4, within noise, and it is only correct + # while every step starts with zero_grad(set_to_none=True). + self.ft = FT_ALL + self.tied_acc = True + self.w16_cache = False + self._w16c: dict = {} + self._compiled = False + self.apply_init() + + # -- init ------------------------------------------------------------- # + def apply_init(self): + m = self.cfg.model + nn.init.normal_(self.embed.weight, std=0.02) + if not m.tie_embeddings: + nn.init.normal_(self.lm_head, std=0.02) + # Residual-path projections are scaled down so the residual variance + # does not grow with depth -- at 31 layers this matters more than it + # does at 10. + res_scale = 1.0 / math.sqrt(2 * m.n_layer) + for blk in self.blocks: + for p in (blk.attn.qkv_proj, blk.ffn.gate_up_proj): + nn.init.normal_(p, std=0.02) + for p in (blk.attn.o_proj, blk.ffn.down_proj): + nn.init.normal_(p, std=0.02 * res_scale) + if blk.attn.canon is not None: + nn.init.zeros_(blk.attn.canon) + + def maybe_compile(self): + if self._compiled or not self.cfg.train.compile_regions: + return + self._pre_attn = torch.compile(_pre_attn, dynamic=False) + self._pre_attn_qkv = torch.compile(_pre_attn_qkv, dynamic=False) + self._post_then_pre = torch.compile(_post_then_pre, dynamic=False) + self._post_then_final_norm = torch.compile(_post_then_final_norm, dynamic=False) + self._post_attn = torch.compile(_post_attn, dynamic=False) + self._pre_attn_qkv_x = torch.compile(_pre_attn_qkv_x, dynamic=False) + self._post_then_pre_x = torch.compile(_post_then_pre_x, dynamic=False) + self._post_then_final_norm_x = torch.compile(_post_then_final_norm_x, dynamic=False) + self._compiled = True + + def head_weight(self): + return self.embed.weight if self.cfg.model.tie_embeddings else self.lm_head + + # -- rope ------------------------------------------------------------- # + def rope_tables(self, pos, theta: float, device, dtype=torch.float32): + """cos/sin for (B, T) per-document positions, gathered once per forward. + + Positions restart at every document, so a document packed at row offset + 1500 is rotated as though it began at 0. RoPE is relative, so this is + not strictly required inside a document -- but it keeps the absolute + range a document sees equal to its own length, which is what the + context stage is defined in terms of. + """ + hd = self.cfg.model.head_dim + key = (theta, hd, device) + if key not in self._rope_cache: + inv = 1.0 / (theta ** (torch.arange(0, hd, 2, device=device).float() / hd)) + t = torch.arange(self.cfg.model.max_block_size, device=device).float() + f = torch.outer(t, inv) + self._rope_cache[key] = (f.cos().to(dtype), f.sin().to(dtype)) + cos, sin = self._rope_cache[key] + return cos[pos].unsqueeze(2), sin[pos].unsqueeze(2) # (B, T, 1, hd/2) + + # -- forward ---------------------------------------------------------- # + def forward(self, idx, pos, offsets, boundary, targets=None, + theta: float = 10_000.0, min_seqlen=None, max_seqlen=None): + """idx/pos/boundary are (B, T); offsets indexes the flattened B*T stream.""" + cos, sin = self.rope_tables(pos, theta, idx.device) + # m1: is t-1 in this document? m2: is t-2? Once per forward, not per layer. + b1 = boundary + b2 = boundary | F.pad(boundary, (1, 0))[:, :-1] + dt = torch.float16 if self.cfg.train.fp16_residual else torch.float32 + m1 = (~b1).to(dt)[:, :, None] + m2 = (~b2).to(dt)[:, :, None] + ctx = RunContext(cos, sin, m1, m2, offsets, self._pre_attn, self._post_attn, + min_seqlen, max_seqlen, self._pre_attn_qkv) + + gc = self.cfg.train.grad_checkpoint + every = max(1, self.cfg.train.grad_checkpoint_every) + a0 = self.blocks[0].attn + merged = (_volta(idx) and a0.canon is not None and a0.rep == 1 and a0.head_dim == 64 + and not (gc and self.training) and not self.cfg.train.fp16_residual + and not getattr(self, "_no_merge", False)) + # Tied-embedding fused accumulate: only when the head's backward is + # certain to run first and stash its dW (training forward with targets). + stash = (merged and self.tied_acc and targets is not None + and torch.is_grad_enabled() and self.cfg.model.tie_embeddings + and self.embed.weight.requires_grad) + if stash: + h = TiedEmbedding.apply(idx, self.embed.weight) + else: + h = self.embed(idx) + if self.cfg.train.fp16_residual: + h = h.to(torch.float16) + + if merged: + h = self._forward_merged(h, idx, boundary, ctx) + if targets is None: + return h + return fused_lm_loss(h.reshape(-1, h.shape[-1]), self.head_weight(), + targets.reshape(-1), self.cfg.optim.lm_head_chunk, + stash) + for i, blk in enumerate(self.blocks): + if (self.ngram is not None and self.ngram_enabled + and i == self.cfg.ngram.insert_after): + h = h + self.ngram(idx, boundary, h.dtype) + if gc and self.training and (i % every == 0): + h = torch.utils.checkpoint.checkpoint( + blk, h, ctx, use_reentrant=False) + else: + h = blk(h, ctx) + + h = self.norm_f(h) + if targets is None: + return h + loss = fused_lm_loss(h.reshape(-1, h.shape[-1]), self.head_weight(), + targets.reshape(-1), self.cfg.optim.lm_head_chunk) + return loss + + @staticmethod + def _acc(p): + """p.grad as the fp32 accumulator the weight-gradient GEMMs add into; + created (zeroed) on the first micro-step after zero_grad(set_to_none).""" + if not p.requires_grad: + return None + if p.grad is None: + p.grad = torch.zeros_like(p, memory_format=torch.contiguous_format) + return p.grad + + def _wq(self, p, acc, scale: float = 1.0): + """The weight as the region should receive it: the fp32 Parameter + (cast inside the region), or -- with w16_cache -- an fp16 copy cast once + per optimizer step. The copy is keyed on the identity of p.grad: every + step starts with zero_grad(set_to_none=True), so a new accumulator means + the weights may have moved. (p._version can NOT be used: Muon updates + p.data in place, which does not bump the version counter.)""" + if not (self.w16_cache and acc is not None): + return p + import weakref + e = self._w16c.get(id(p)) + if e is not None and e[1]() is acc: + return e[0] + with torch.no_grad(): + w = (p * scale if scale != 1.0 else p).to(torch.float16) + self._w16c[id(p)] = (w, weakref.ref(acc)) + return w + + def _forward_merged(self, h, idx, boundary, ctx): + """Blocks with each residual boundary inside one compiled region + (Volta path, no gradient checkpointing). Same operations, same order, + as the per-block path; returns norm_f(h).""" + if torch.is_grad_enabled() and any(self.ft): + return self._forward_merged_x(h, idx, boundary, ctx) + return self._forward_merged_stock(h, idx, boundary, ctx) + + def _forward_merged_x(self, h, idx, boundary, ctx): + """_forward_merged with the swarm additions (self.ft, self.w16_cache).""" + from volta_kernels import canon_rope_attention + ft = self.ft + wacc = ft[0] + A = self._acc if wacc else (lambda p: None) + W = self._wq + blocks = self.blocks + L = len(blocks) + ins = self.cfg.ngram.insert_after if (self.ngram is not None + and self.ngram_enabled) else -1 + ng = self.ngram(idx, boundary, h.dtype) if ins >= 0 else None + if ins == 0: + h = h + ng + B, T = h.shape[0], h.shape[1] + b0 = blocks[0] + a0 = A(b0.attn.qkv_proj) + qkv = self._pre_attn_qkv_x(h, b0.norm1.weight, b0.norm1.eps, + W(b0.attn.qkv_proj, a0), a0) + for i, blk in enumerate(blocks): + a = blk.attn + o = canon_rope_attention(qkv, a.canon, ctx.m1, ctx.m2, ctx.cos, ctx.sin, + ctx.offsets).reshape(B, T, a.n_head * a.head_dim) + f = blk.ffn + ao, agu, adn = A(a.o_proj), A(f.gate_up_proj), A(f.down_proj) + wo, wgu, wdn = W(a.o_proj, ao), W(f.gate_up_proj, agu), W(f.down_proj, adn, DOWN_SHRINK) + if i + 1 < L: + nb = blocks[i + 1] + aq = A(nb.attn.qkv_proj) + h, qkv = self._post_then_pre_x( + h, o, wo, blk.norm2.weight, blk.norm2.eps, wgu, wdn, + nb.norm1.weight, nb.norm1.eps, W(nb.attn.qkv_proj, aq), + ng if i + 1 == ins else None, + (ao, agu, adn, aq) if wacc else None, ft) + else: + h = self._post_then_final_norm_x( + h, o, wo, blk.norm2.weight, blk.norm2.eps, wgu, wdn, + self.norm_f.weight, self.norm_f.eps, + (ao, agu, adn) if wacc else None, ft) + return h + + def _forward_merged_stock(self, h, idx, boundary, ctx): + from volta_kernels import canon_rope_attention + blocks = self.blocks + L = len(blocks) + ins = self.cfg.ngram.insert_after if (self.ngram is not None + and self.ngram_enabled) else -1 + ng = self.ngram(idx, boundary, h.dtype) if ins >= 0 else None + if ins == 0: + h = h + ng + B, T = h.shape[0], h.shape[1] + b0 = blocks[0] + qkv = self._pre_attn_qkv(h, b0.norm1.weight, b0.norm1.eps, b0.attn.qkv_proj) + for i, blk in enumerate(blocks): + a = blk.attn + # Canon+RoPE+attention as one autograd Function: dQ reaches the + # Canon backward in fp32 (see volta_kernels._CanonRopeAttn). + o = canon_rope_attention(qkv, a.canon, ctx.m1, ctx.m2, ctx.cos, ctx.sin, + ctx.offsets).reshape(B, T, a.n_head * a.head_dim) + if i + 1 < L: + nb = blocks[i + 1] + h, qkv = self._post_then_pre( + h, o, a.o_proj, blk.norm2.weight, blk.norm2.eps, + blk.ffn.gate_up_proj, blk.ffn.down_proj, + nb.norm1.weight, nb.norm1.eps, nb.attn.qkv_proj, + ng if i + 1 == ins else None) + else: + h = self._post_then_final_norm( + h, o, a.o_proj, blk.norm2.weight, blk.norm2.eps, + blk.ffn.gate_up_proj, blk.ffn.down_proj, + self.norm_f.weight, self.norm_f.eps) + return h + + @torch.no_grad() + def logits_for_eval(self, idx, pos, offsets, boundary, theta: float = 10_000.0): + h = self.forward(idx, pos, offsets, boundary, None, theta) + return F.linear(h, self.head_weight()) -- cgit v1.2.3