"""Boris-2: a d512 x 31L decoder-only transformer with fused Canon layers. Shape of a block: h = h + o_proj( attn( canon( qkv_proj( norm(h) ) ) ) ) h = h + down( silu(gate) * up )( norm(h) ) Four things here are not the textbook spelling, and each is a measured win on sm_70 rather than a preference: 1. Q, K and V are one GEMM. The weights stay three separate Parameters and are concatenated at call time, so Muon still orthogonalises each independently -- fusing the Parameters themselves would change what the optimizer operates on and would not be lossless. The concatenated weight is ~1.2MB against a 16,384-row activation. 2. Canon is one kernel over the concatenated Q|K|V block. Depthwise convolution is per-channel, so a single 768-channel convolution equals three separate ones on the Q/K/V channel ranges: 31 kernels per forward instead of 93. It is spelled as three shifted multiply-adds on the native (B,T,C) layout rather than F.conv1d(groups=C), which needs a (B,T,C)->(B,C,T) transpose whose following reshape materialises a copy. Measured on this V100, fwd+bwd over the real shape: 4.30 ms for conv1d, 1.10 ms for the compiled shift form -- substantially faster than the transposing grouped-convolution spelling. 3. Canon is document-aware. Kernel 3 stacked 31 deep reaches 62 tokens back, and rows hold several packed documents, so without masking every block pulls the previous document's tail across the boundary. The mask costs 0.10 ms/layer. This is the bug the 75M run found the hard way; verify.py probes for it with the output projections de-zeroed, because at init a zero-init projection makes the test pass no matter how badly the model leaks. 4. Attention is varlen over a jagged nested tensor: one SDPA launch for the whole batch, cost sum(len_i^2) rather than T^2. Measured against the two alternatives at ctx 2048 with realistic document lengths, fwd+bwd: nested 11.09 ms, dense block-diagonal mask 26.97 ms, per-document loop 45.84 ms. Nested is bit-identical to the loop (max diff 0.0). The residual stream is fp32 on the V100 because measured massive activations can exceed fp16's finite range. Matmul inputs are cast by the compiled regions under autocast, preserving tensor-core execution without risking a forward overflow. """ from __future__ import annotations import math import os import torch import torch.nn as nn import torch.nn.functional as F from config import Config, ModelConfig from ngram import NGramTable from volta_kernels.integ_ops import AddNorm, LinAcc, TiedEmbedding IGNORE = -100 # Hand-written sm_70 kernels (volta_kernels/): varlen FlashAttention with mma.sync # and a one-pass LM-head softmax. Opt-in, because they are adopted by a running # 200B-token job at a resume: BORIS_VOLTA_KERNELS=1. Same operand dtypes and fp32 # accumulation as the paths they replace; verify.py checks both against fp64. # They only engage on an sm_70 device -- elsewhere (eval on the 3060) the stock # paths run regardless. VOLTA_KERNELS = os.environ.get("BORIS_VOLTA_KERNELS", "0") == "1" def _volta(t: torch.Tensor) -> bool: return VOLTA_KERNELS and t.is_cuda and torch.cuda.get_device_capability(t.device) == (7, 0) # --------------------------------------------------------------------------- # # Compiled fixed-shape regions # --------------------------------------------------------------------------- # # Whole-model compile is not used: the 75M run measured it 28% slower in # realistic conditions because document counts vary and Inductor recompiles # mid-run. These regions never see a ragged shape -- only (B, T, C) with B and T # fixed for the whole context stage -- so each compiles once per stage. def _rmsnorm(x, w, eps: float): f = x.float() f = f * torch.rsqrt(f.pow(2).mean(-1, keepdim=True) + eps) return (f * w.float()).to(x.dtype) def _pre_attn(h, w_norm, eps: float, w_qkv, canon_w, m1, m2, cos, sin, n_head: int, n_kv_head: int, head_dim: int, use_canon: bool, out16: bool = False): """Norm, one GEMM, one depthwise convolution, one rotation -- one region. The RMSNorm is inside the region on purpose. Written as its own eager module it upcasts to fp32, and that materialises a (B, T, 512) fp32 tensor for every one of the 53 norms in the model -- 25% of forward+backward was elementwise traffic before these regions were widened, and this was most of it. Inductor keeps the reduction in fp32 without ever writing the fp32 tensor out. canon_w is (C, 3) with column 0 the weight on token t, column 1 on t-1 and column 2 on t-2. m1 masks the t-1 tap and m2 the t-2 tap at document boundaries; both are (B, T, 1) and are computed once per forward, not per layer. """ f = h.float() x = (f * torch.rsqrt(f.pow(2).mean(-1, keepdim=True) + eps) * w_norm.float()).to(h.dtype) qkv = F.linear(x, w_qkv) # (B, T, C) if use_canon: w = canon_w.to(qkv.dtype) s1 = F.pad(qkv, (0, 0, 1, 0))[:, :-1] * m1 s2 = F.pad(qkv, (0, 0, 2, 0))[:, :-2] * m2 qkv = qkv + qkv * w[:, 0] + s1 * w[:, 1] + s2 * w[:, 2] B, T, _ = qkv.shape qd, kd = n_head * head_dim, n_kv_head * head_dim q = qkv[..., :qd].view(B, T, n_head, head_dim) k = qkv[..., qd:qd + kd].view(B, T, n_kv_head, head_dim) v = qkv[..., qd + kd:].view(B, T, n_kv_head, head_dim) def rope(t): f = t.float() a, b = f.chunk(2, dim=-1) return torch.cat([a * cos - b * sin, a * sin + b * cos], -1).to(t.dtype) if out16: # The Volta attention takes fp16. Casting here rounds the same fp32 # values autocast would round, but inside the region's epilogue, so the # fp32 q/k/v are never written out and read back by separate casts. h16 = torch.float16 return rope(q).to(h16), rope(k).to(h16), v.to(h16) return rope(q), rope(k), v def _pre_attn_qkv(h, w_norm, eps: float, w_qkv): """Norm and the fused QKV GEMM only -- the Volta path's region. Canon and RoPE then run as one fused Triton pass (volta_kernels/canon_rope.py) instead of Inductor's pad-copy / combine / rotate kernels. """ f = h.float() x = (f * torch.rsqrt(f.pow(2).mean(-1, keepdim=True) + eps) * w_norm.float()).to(h.dtype) return F.linear(x, w_qkv) def _post_then_pre(h, o, w_o, w_norm2, eps2, w_gu, w_down, w_norm1n, eps1n, w_qkvn, extra): """Block i's FFN half and block i+1's norm + QKV GEMM as ONE region. As separate regions the residual h crossed a graph boundary between two consumers, so its two backward gradient streams were summed by an eager fp32 add (~192 MB of traffic per layer) and h was written and read back. Here Inductor fuses that sum into the region's backward. `extra` is the n-gram residual when block i+1 is where it is inserted (it depends only on the token ids), else None -- each value gets its own compiled graph. """ h = _post_attn(h, o, w_o, w_norm2, eps2, w_gu, w_down) if extra is not None: h = h + extra return h, _pre_attn_qkv(h, w_norm1n, eps1n, w_qkvn) def _post_then_final_norm(h, o, w_o, w_norm2, eps2, w_gu, w_down, w_normf, epsf): """The last block's FFN half and the final RMSNorm as one region.""" h = _post_attn(h, o, w_o, w_norm2, eps2, w_gu, w_down) return _rmsnorm(h, w_normf, epsf) # Power-of-two headroom for the FFN down projection's fp16 output. 1/8 puts the # measured step-4,520 peak (~65.5k) at ~8.2k, leaving 8x before it overflows # again, and costs only inputs below 4.9e-4 -- 1.6e-8 of the peak, which # contribute nothing to a sum dominated by terms 6e7 times larger. DOWN_SHRINK = 0.125 def _post_attn(h, o, w_o, w_norm, eps: float, w_gu, w_down): """Output projection, residual, norm, SwiGLU, residual -- one region. Everything in a block except the attention itself. The two residual adds and the second norm were separate eager kernels over the widest tensors in the model; inside the region Inductor folds them into the GEMM epilogues, and the fp32 norm intermediate is never written out. """ h = h + F.linear(o, w_o) f = h.float() x = (f * torch.rsqrt(f.pow(2).mean(-1, keepdim=True) + eps) * w_norm.float()).to(h.dtype) gu = F.linear(x, w_gu) g, u = gu.chunk(2, dim=-1) # The down projection is the one GEMM here whose fp16 OUTPUT can overflow. # Its inputs are finite and so is the true result -- it is the fp16 range # that runs out. Channel 228 of the residual is a "massive activation", # 173x the RMS (78,115 against an RMS of 451 at block 16), and the FFN has # to write into it, so at step 4,520 this output crossed 65,504 and became # inf; norm_f then turned that into NaN via rsqrt(inf)=0, inf*0. # Pre-scaling w_down is exact (a power of two), keeps the GEMM on tensor # cores, costs one pass over a 1.2M-element weight rather than the 77M- # element activation, and is divided back out in fp32 where there is room. # Every partial sum is scaled too, so this is safe even with cuBLAS's # reduced-precision fp16 reductions enabled. return h + F.linear(F.silu(g) * u, w_down * DOWN_SHRINK).float() * (1.0 / DOWN_SHRINK) # --------------------------------------------------------------------------- # # Merged-path regions with the swarm additions (volta_kernels/integ_ops.py) # --------------------------------------------------------------------------- # # ft = (wacc, norm) -- which additions a compiled region uses: # wacc: weight-gradient GEMMs accumulate into the fp32 .grad (accs given) # norm: residual-add + RMSNorm backward and norm-weight grad in one Triton pass # Weights arrive either as the fp32 Parameters (cast here, as autocast would) or # as the cached, pre-scaled fp16 copies (Boris2._w16); both give the same fp16 # operand, so the forward is bitwise the live one either way. FT_ALL = (True, True) def _w16(w, scale: float = 1.0): if w.dtype == torch.float16: return w # cached copy, already scaled return (w * scale).to(torch.float16) if scale != 1.0 else w.to(torch.float16) def _lin_x(x, w, acc, scale: float = 1.0): if acc is None: if w.dtype == torch.float16: return F.linear(x, w) return F.linear(x, w * scale if scale != 1.0 else w) return LinAcc.apply(x.to(torch.float16), _w16(w, scale), acc, scale) def _norm_x(h, w, eps: float): f = h.float() return (f * torch.rsqrt(f.pow(2).mean(-1, keepdim=True) + eps) * w.float()).to(h.dtype) def _ffn_x(h, o, w_o, w_norm, eps: float, w_gu, w_down, a_o, a_gu, a_down, ft): """Block FFN half up to (but excluding) the down projection's residual add: returns h1 = h + o_proj(o) and the fp16 down output (pre-scaled by DOWN_SHRINK), so the next norm can fold the add into its backward.""" wacc, norm = ft a = _lin_x(o, w_o, a_o) if norm: h1, x = AddNorm.apply(h, a, w_norm, eps, 1.0, False) else: h1 = h + a x = _norm_x(h1, w_norm, eps) gu = _lin_x(x, w_gu, a_gu) g, u = gu.chunk(2, dim=-1) return h1, _lin_x(F.silu(g) * u, w_down, a_down, DOWN_SHRINK) def _post_then_pre_x(h, o, w_o, w_norm2, eps2, w_gu, w_down, w_norm1n, eps1n, w_qkvn, extra, accs, ft): a_o, a_gu, a_down, a_qkvn = accs if accs is not None else (None,) * 4 h1, dn = _ffn_x(h, o, w_o, w_norm2, eps2, w_gu, w_down, a_o, a_gu, a_down, ft) if ft[1] and extra is None: h2, x = AddNorm.apply(h1, dn, w_norm1n, eps1n, 1.0 / DOWN_SHRINK, True) else: h2 = h1 + dn.float() * (1.0 / DOWN_SHRINK) if extra is not None: h2 = h2 + extra x = _norm_x(h2, w_norm1n, eps1n) return h2, _lin_x(x, w_qkvn, a_qkvn) def _post_then_final_norm_x(h, o, w_o, w_norm2, eps2, w_gu, w_down, w_normf, epsf, accs, ft): a_o, a_gu, a_down = accs if accs is not None else (None,) * 3 h1, dn = _ffn_x(h, o, w_o, w_norm2, eps2, w_gu, w_down, a_o, a_gu, a_down, ft) return _rmsnorm(h1 + dn.float() * (1.0 / DOWN_SHRINK), w_normf, epsf) def _pre_attn_qkv_x(h, w_norm, eps: float, w_qkv, acc): return _lin_x(_norm_x(h, w_norm, eps), w_qkv, acc) # --------------------------------------------------------------------------- # # Fused linear cross-entropy # --------------------------------------------------------------------------- # def _head_chunk(logits, safe, vc, scale: float): """logsumexp, loss, and the scaled softmax, as one fused pass. Eager, this chain is five separate passes over a (chunk, 32768) fp32 tensor -- at micro_batch 16 that is 4.3 GB read and written five times per step. Inductor folds it to two. """ lse = torch.logsumexp(logits, dim=-1) picked = logits.gather(1, safe[:, None]).squeeze(1) loss = torch.where(vc, lse - picked, torch.zeros_like(lse)).sum() p = (logits - lse[:, None]).exp() * (vc[:, None].to(logits.dtype) * scale) return loss, p _head_chunk_c = None class _FusedLMHead(torch.autograd.Function): """Cross-entropy over the LM head without ever holding (N, V) logits. The naive head materialises the logits three times -- fp16 from the projection, fp32 inside cross_entropy, and again for the backward -- which at N=16,384 and V=32,768 is 2.1 GB a copy and sets the micro-batch. The head's gradient depends on nothing downstream except a scalar, so this computes dx and dW during the forward walk over vocab-sized chunks and keeps only those. No recomputation in the backward: total GEMM work is the same as the unfused head, and peak memory is one chunk of logits. Gradients are held at sum scale (not divided by the token count) and lifted by a further constant: a softmax over 32,768 classes has typical entries near 3e-5, which is SUBNORMAL in fp16 (the smallest normal is 6.1e-5), so the natural scaling would compute the whole dx GEMM on values with a few bits of mantissa. GRAD_SCALE lifts them clear of that. The factor, the 1/N and the loss scale are all divided out together in the backward. The constant is bounded from ABOVE by the dW GEMM, which was missed until real data hit it. g = S*(p - onehot), so the true token carries -S, and dW sums that over every token in the chunk sharing a vocab row. Measured on a real batch: the most frequent target appears 89 times in a 2048-token chunk and |x| reaches 5.4, so at S=4096 the largest |dW| entry is 504,389 -- 7.7x past fp16's 65,504. It saturated to Inf, Inf-Inf gave NaN in the tied embedding gradient, and the run died on step 2 with loss nan. Measured max |dW| against S, same batch: S=4096 504,389 OVERFLOW S=256 31,524 (2.1x margin) S=1024 126,097 OVERFLOW S=64 7,881 (8.3x margin) 64 is chosen over a value that splits the difference because the two risks are not symmetric. Both grow over a long run -- |x| rises as the residual stream grows, and a confident softmax makes small probabilities smaller -- but an overflow is a NaN that ends the run, while an underflow silently drops terms that were already negligible. At S=64 the smallest nonzero |p*S| is 1.48e-4, still 2.4x above fp16's min normal. """ GRAD_SCALE = 64.0 # PERF.md's last unapplied win: accumulate the head's dW in fp16 rather than # fp32. Their +10.1% came from an fp32 accumulate forcing a non-tensor-core # sgemm; ours is already a tensor-core fp16 GEMM with only the accumulator # buffer in fp32, so the mechanism differs. Measured either way. FP16_DW = False # Shrinks g for the dW GEMM only (a power of two, so it is exact and is # divided back out losslessly in fp32). dx keeps the full GRAD_SCALE. DW_SHRINK = 0.25 @staticmethod def forward(ctx, x, W, targets, chunk: int): N, d = x.shape V = W.shape[0] # Cast once, explicitly, rather than relying on autocast to do it: this # runs inside a custom Function and is called from verify.py outside any # autocast region. Wc = W.to(x.dtype) valid = targets != IGNORE # Keep this scalar on-device. int(valid.sum()) was one forced CPU/GPU # synchronization per micro-batch: sixteen pipeline stalls per step. n_valid = valid.sum().clamp_min(1) loss_sum = x.new_zeros((), dtype=torch.float32) dx = torch.empty_like(x) dW = torch.zeros_like(W, dtype=(torch.float16 if _FusedLMHead.FP16_DW else torch.float32)) for a in range(0, N, chunk): b = min(a + chunk, N) xc, tc = x[a:b], targets[a:b] logits = F.linear(xc, Wc).float() # (c, V) safe = tc.clamp_min(0) vc = (tc != IGNORE) S = _FusedLMHead.GRAD_SCALE loss, p = _head_chunk_c(logits, safe, vc, S) loss_sum += loss del logits # The one-hot subtraction touches c elements, not c*V, so it stays # outside the fused pass. p.scatter_add_(1, safe[:, None], (-S * vc).to(p.dtype)[:, None]) g = p.to(x.dtype) del p dx[a:b] = F.linear(g, Wc.t()) # dW is the one GEMM here whose fp16 OUTPUT can overflow, and no # loss scale can save it: g and xc are built inside this Function, # so GradScaler halving its scale changes nothing while found_inf # skips AdamW forever. That froze the embeddings, head, norms, Canon # and n-gram table at step ~230 while Muon kept training the trunk # against them; loss bottomed at 6.48 and climbed to nan by 350. # Bound it by construction instead of by choosing a constant: # rescale xc by a power of two so |xn| <= 1 (exact, no rounding), # and shrink g for this GEMM only. |sum| <= S*DW_SHRINK*chunk = 32k # then holds for ANY data, including a chunk of one repeated token, # and both factors are divided back out in fp32. r2 = torch.exp2(torch.ceil(torch.log2( xc.abs().amax().float().clamp_min(1e-30)))) xn = xc / r2.to(xc.dtype) gd = g * _FusedLMHead.DW_SHRINK back = r2 / _FusedLMHead.DW_SHRINK dw = gd.t() @ xn # Restore magnitude AFTER widening. Multiplying in fp16 undoes # the overflow protection before the fp32 accumulator sees it. dW += (dw.float() * back).to(dW.dtype) ctx.w_dtype = W.dtype ctx.save_for_backward(dx, dW) ctx.n_valid = n_valid return loss_sum / n_valid @staticmethod def backward(ctx, grad_out): dx, dW = ctx.saved_tensors s = grad_out / (ctx.n_valid * _FusedLMHead.GRAD_SCALE) # In fp32, then cast. s is ~2.8e-7 without a loss scale, which is # subnormal in fp16 and would carry about two bits of mantissa -- it # cost 7e-02 relative on dx before this was float. Under GradScaler s # lands near 1e-2 and the difference is invisible, which is exactly why # it would never have shown up in the loss curve. return (dx.float() * s).to(dx.dtype), (dW.float() * s).to(ctx.w_dtype), None, None def fused_lm_loss(x, W, targets, chunk: int = 2048, stash: bool = False): if _volta(x): from volta_kernels.head import fused_lm_loss as volta_lm_loss return volta_lm_loss(x, W, targets, chunk, stash) assert not stash global _head_chunk_c if _head_chunk_c is None: _head_chunk_c = torch.compile(_head_chunk, dynamic=False) return _FusedLMHead.apply(x, W, targets, chunk) # --------------------------------------------------------------------------- # # Modules # --------------------------------------------------------------------------- # class RMSNorm(nn.Module): def __init__(self, dim: int, eps: float): super().__init__() self.weight = nn.Parameter(torch.ones(dim)) self.eps = eps self.weight._boris_role = "norm" def forward(self, x): return _rmsnorm(x, self.weight, self.eps) class Attention(nn.Module): """Causal GQA (8 query heads over 2 KV heads) with fused Canon, varlen.""" def __init__(self, cfg: ModelConfig): super().__init__() self.cfg = cfg d, hd = cfg.d_model, cfg.head_dim self.n_head, self.n_kv_head, self.head_dim = cfg.n_head, cfg.n_kv_head, hd self.rep = cfg.n_head // cfg.n_kv_head # Q, K and V are ONE Parameter, and Muon orthogonalises its three row # blocks independently -- which is exactly what it would do to three # separate Parameters, because the quintic iteration acts on each matrix # alone. Storing them fused removes a torch.cat per layer per # micro-step (2.4% of forward+backward) and its backward slice, without # changing what the optimizer operates on. verify.py asserts the two # spellings produce identical updates. self.qkv_proj = nn.Parameter(torch.empty(cfg.q_dim + 2 * cfg.kv_dim, d)) self.qkv_proj._boris_role = "matmul" self.qkv_proj._boris_blocks = ((0, cfg.q_dim), (cfg.q_dim, cfg.q_dim + cfg.kv_dim), (cfg.q_dim + cfg.kv_dim, cfg.q_dim + 2 * cfg.kv_dim)) self.o_proj = nn.Parameter(torch.empty(d, cfg.q_dim)) self.o_proj._boris_role = "matmul" if cfg.use_canon: # (C, 3), zero-init: the layer is exactly the identity at step 0, so # a Canon run and a no-Canon run start from the same function. The # gradient w.r.t. the kernel is dL/dy * x and is non-zero, so it # trains from the first step regardless. self.canon = nn.Parameter(torch.zeros(cfg.canon_channels, cfg.canon_kernel)) self.canon._boris_role = "canon" else: self.register_parameter("canon", None) def forward(self, h, norm_w, eps, ctx): volta = self.head_dim == 64 and _volta(h) if volta and self.canon is not None and self.rep == 1: from volta_kernels.canon_rope_cuda import canon_rope qkv = ctx.pre_attn_qkv(h, norm_w, eps, self.qkv_proj) q, k, v = canon_rope(qkv, self.canon, ctx.m1, ctx.m2, ctx.cos, ctx.sin) else: q, k, v = ctx.pre_attn(h, norm_w, eps, self.qkv_proj, self.canon, ctx.m1, ctx.m2, ctx.cos, ctx.sin, self.n_head, self.n_kv_head, self.head_dim, self.canon is not None, volta) B, T = q.shape[0], q.shape[1] q = q.reshape(B * T, self.n_head, self.head_dim) # rep == 1 under full MHA: repeat_interleave(1) is a no-op that still # allocates and copies a (B*T, n_head, head_dim) tensor -- 33 MB per # call per layer at micro_batch 16, twice (k and v), on every block in # forward AND backward. Skip it. k = k.reshape(B * T, self.n_kv_head, self.head_dim) v = v.reshape(B * T, self.n_kv_head, self.head_dim) if self.rep > 1: k = k.repeat_interleave(self.rep, 1) v = v.repeat_interleave(self.rep, 1) off = ctx.offsets if volta: from volta_kernels import varlen_causal_attention # q/k/v arrive fp16 from the region (out16): the cast autocast # would apply inside SDPA, since the Canon masks keep them fp32. o = varlen_causal_attention(q, k, v, off) return o.reshape(B, T, self.n_head * self.head_dim) nj = lambda t: torch.nested.nested_tensor_from_jagged( t, off, min_seqlen=ctx.min_seqlen, max_seqlen=ctx.max_seqlen) nq = nj(q).transpose(1, 2) nk = nj(k).transpose(1, 2) nv = nj(v).transpose(1, 2) o = F.scaled_dot_product_attention(nq, nk, nv, is_causal=True) return o.transpose(1, 2).values().reshape(B, T, self.n_head * self.head_dim) class FFN(nn.Module): """SwiGLU, inner width 2368. gate and up are one GEMM at call time.""" def __init__(self, cfg: ModelConfig): super().__init__() h, d = cfg.ffn_hidden, cfg.d_model # gate and up fused for the same reason q/k/v are; Muon still sees two # independent (2368, 512) matrices. self.gate_up_proj = nn.Parameter(torch.empty(2 * h, d)) self.gate_up_proj._boris_role = "matmul" self.gate_up_proj._boris_blocks = ((0, h), (h, 2 * h)) self.down_proj = nn.Parameter(torch.empty(d, h)) self.down_proj._boris_role = "matmul" class Block(nn.Module): def __init__(self, cfg: ModelConfig): super().__init__() self.norm1 = RMSNorm(cfg.d_model, cfg.norm_eps) self.attn = Attention(cfg) self.norm2 = RMSNorm(cfg.d_model, cfg.norm_eps) self.ffn = FFN(cfg) def forward(self, h, ctx): o = self.attn(h, self.norm1.weight, self.norm1.eps, ctx) return ctx.post_attn(h, o, self.attn.o_proj, self.norm2.weight, self.norm2.eps, self.ffn.gate_up_proj, self.ffn.down_proj) class RunContext: """Everything shared across all blocks for one forward. RoPE tables are gathered once here rather than once per attention layer, and the Canon boundary masks once rather than once per Canon. Holding the compiled callables here keeps a single compiled graph per stage shared by every block. """ def __init__(self, cos, sin, m1, m2, offsets, pre_attn, post_attn, min_seqlen=None, max_seqlen=None, pre_attn_qkv=None): self.pre_attn_qkv = pre_attn_qkv self.cos, self.sin = cos, sin self.m1, self.m2 = m1, m2 self.offsets = offsets self.pre_attn, self.post_attn = pre_attn, post_attn # Without these, every SDPA call derives them from the offsets with a # .max().item() -- a host sync, 131 of them per forward, each stalling # CPU run-ahead. The packer knows both while it is laying documents # down, so they cost nothing to carry. self.min_seqlen, self.max_seqlen = min_seqlen, max_seqlen class Boris2(nn.Module): def __init__(self, cfg: Config): super().__init__() self.cfg = cfg m = cfg.model self.embed = nn.Embedding(m.vocab_size, m.d_model) self.embed.weight._boris_role = "embedding" self.blocks = nn.ModuleList([Block(m) for _ in range(m.n_layer)]) self.norm_f = RMSNorm(m.d_model, m.norm_eps) self.ngram = NGramTable(cfg.ngram, m.d_model) if m.use_ngram else None # Runtime toggle, for scoring one trained checkpoint both ways. Note # what it measures: after training, the trunk has co-adapted to the # table, so switching it off reports how much the model DEPENDS on it, # not how much it GAINED from it. The gain needs the use_ngram=False # training arm. self.ngram_enabled = True if not m.tie_embeddings: self.lm_head = nn.Parameter(torch.empty(m.vocab_size, m.d_model)) self.lm_head._boris_role = "embedding" self._rope_cache: dict[tuple, tuple] = {} self._pre_attn = _pre_attn self._pre_attn_qkv = _pre_attn_qkv self._post_attn = _post_attn self._post_then_pre = _post_then_pre self._post_then_final_norm = _post_then_final_norm self._pre_attn_qkv_x = _pre_attn_qkv_x self._post_then_pre_x = _post_then_pre_x self._post_then_final_norm_x = _post_then_final_norm_x # Swarm additions on the merged path (see volta_kernels/integ_ops.py): # (wgrad_acc, fused norm backward) and the tied embedding's fused grad # accumulate. w16_cache (fp16 weight copies cast once per optimizer # step) is OFF: ~0.15% at mb 4, within noise, and it is only correct # while every step starts with zero_grad(set_to_none=True). self.ft = FT_ALL self.tied_acc = True self.w16_cache = False self._w16c: dict = {} self._compiled = False self.apply_init() # -- init ------------------------------------------------------------- # def apply_init(self): m = self.cfg.model nn.init.normal_(self.embed.weight, std=0.02) if not m.tie_embeddings: nn.init.normal_(self.lm_head, std=0.02) # Residual-path projections are scaled down so the residual variance # does not grow with depth -- at 31 layers this matters more than it # does at 10. res_scale = 1.0 / math.sqrt(2 * m.n_layer) for blk in self.blocks: for p in (blk.attn.qkv_proj, blk.ffn.gate_up_proj): nn.init.normal_(p, std=0.02) for p in (blk.attn.o_proj, blk.ffn.down_proj): nn.init.normal_(p, std=0.02 * res_scale) if blk.attn.canon is not None: nn.init.zeros_(blk.attn.canon) def maybe_compile(self): if self._compiled or not self.cfg.train.compile_regions: return self._pre_attn = torch.compile(_pre_attn, dynamic=False) self._pre_attn_qkv = torch.compile(_pre_attn_qkv, dynamic=False) self._post_then_pre = torch.compile(_post_then_pre, dynamic=False) self._post_then_final_norm = torch.compile(_post_then_final_norm, dynamic=False) self._post_attn = torch.compile(_post_attn, dynamic=False) self._pre_attn_qkv_x = torch.compile(_pre_attn_qkv_x, dynamic=False) self._post_then_pre_x = torch.compile(_post_then_pre_x, dynamic=False) self._post_then_final_norm_x = torch.compile(_post_then_final_norm_x, dynamic=False) self._compiled = True def head_weight(self): return self.embed.weight if self.cfg.model.tie_embeddings else self.lm_head # -- rope ------------------------------------------------------------- # def rope_tables(self, pos, theta: float, device, dtype=torch.float32): """cos/sin for (B, T) per-document positions, gathered once per forward. Positions restart at every document, so a document packed at row offset 1500 is rotated as though it began at 0. RoPE is relative, so this is not strictly required inside a document -- but it keeps the absolute range a document sees equal to its own length, which is what the context stage is defined in terms of. """ hd = self.cfg.model.head_dim key = (theta, hd, device) if key not in self._rope_cache: inv = 1.0 / (theta ** (torch.arange(0, hd, 2, device=device).float() / hd)) t = torch.arange(self.cfg.model.max_block_size, device=device).float() f = torch.outer(t, inv) self._rope_cache[key] = (f.cos().to(dtype), f.sin().to(dtype)) cos, sin = self._rope_cache[key] return cos[pos].unsqueeze(2), sin[pos].unsqueeze(2) # (B, T, 1, hd/2) # -- forward ---------------------------------------------------------- # def forward(self, idx, pos, offsets, boundary, targets=None, theta: float = 10_000.0, min_seqlen=None, max_seqlen=None): """idx/pos/boundary are (B, T); offsets indexes the flattened B*T stream.""" cos, sin = self.rope_tables(pos, theta, idx.device) # m1: is t-1 in this document? m2: is t-2? Once per forward, not per layer. b1 = boundary b2 = boundary | F.pad(boundary, (1, 0))[:, :-1] dt = torch.float16 if self.cfg.train.fp16_residual else torch.float32 m1 = (~b1).to(dt)[:, :, None] m2 = (~b2).to(dt)[:, :, None] ctx = RunContext(cos, sin, m1, m2, offsets, self._pre_attn, self._post_attn, min_seqlen, max_seqlen, self._pre_attn_qkv) gc = self.cfg.train.grad_checkpoint every = max(1, self.cfg.train.grad_checkpoint_every) a0 = self.blocks[0].attn merged = (_volta(idx) and a0.canon is not None and a0.rep == 1 and a0.head_dim == 64 and not (gc and self.training) and not self.cfg.train.fp16_residual and not getattr(self, "_no_merge", False)) # Tied-embedding fused accumulate: only when the head's backward is # certain to run first and stash its dW (training forward with targets). stash = (merged and self.tied_acc and targets is not None and torch.is_grad_enabled() and self.cfg.model.tie_embeddings and self.embed.weight.requires_grad) if stash: h = TiedEmbedding.apply(idx, self.embed.weight) else: h = self.embed(idx) if self.cfg.train.fp16_residual: h = h.to(torch.float16) if merged: h = self._forward_merged(h, idx, boundary, ctx) if targets is None: return h return fused_lm_loss(h.reshape(-1, h.shape[-1]), self.head_weight(), targets.reshape(-1), self.cfg.optim.lm_head_chunk, stash) for i, blk in enumerate(self.blocks): if (self.ngram is not None and self.ngram_enabled and i == self.cfg.ngram.insert_after): h = h + self.ngram(idx, boundary, h.dtype) if gc and self.training and (i % every == 0): h = torch.utils.checkpoint.checkpoint( blk, h, ctx, use_reentrant=False) else: h = blk(h, ctx) h = self.norm_f(h) if targets is None: return h loss = fused_lm_loss(h.reshape(-1, h.shape[-1]), self.head_weight(), targets.reshape(-1), self.cfg.optim.lm_head_chunk) return loss @staticmethod def _acc(p): """p.grad as the fp32 accumulator the weight-gradient GEMMs add into; created (zeroed) on the first micro-step after zero_grad(set_to_none).""" if not p.requires_grad: return None if p.grad is None: p.grad = torch.zeros_like(p, memory_format=torch.contiguous_format) return p.grad def _wq(self, p, acc, scale: float = 1.0): """The weight as the region should receive it: the fp32 Parameter (cast inside the region), or -- with w16_cache -- an fp16 copy cast once per optimizer step. The copy is keyed on the identity of p.grad: every step starts with zero_grad(set_to_none=True), so a new accumulator means the weights may have moved. (p._version can NOT be used: Muon updates p.data in place, which does not bump the version counter.)""" if not (self.w16_cache and acc is not None): return p import weakref e = self._w16c.get(id(p)) if e is not None and e[1]() is acc: return e[0] with torch.no_grad(): w = (p * scale if scale != 1.0 else p).to(torch.float16) self._w16c[id(p)] = (w, weakref.ref(acc)) return w def _forward_merged(self, h, idx, boundary, ctx): """Blocks with each residual boundary inside one compiled region (Volta path, no gradient checkpointing). Same operations, same order, as the per-block path; returns norm_f(h).""" if torch.is_grad_enabled() and any(self.ft): return self._forward_merged_x(h, idx, boundary, ctx) return self._forward_merged_stock(h, idx, boundary, ctx) def _forward_merged_x(self, h, idx, boundary, ctx): """_forward_merged with the swarm additions (self.ft, self.w16_cache).""" from volta_kernels import canon_rope_attention ft = self.ft wacc = ft[0] A = self._acc if wacc else (lambda p: None) W = self._wq blocks = self.blocks L = len(blocks) ins = self.cfg.ngram.insert_after if (self.ngram is not None and self.ngram_enabled) else -1 ng = self.ngram(idx, boundary, h.dtype) if ins >= 0 else None if ins == 0: h = h + ng B, T = h.shape[0], h.shape[1] b0 = blocks[0] a0 = A(b0.attn.qkv_proj) qkv = self._pre_attn_qkv_x(h, b0.norm1.weight, b0.norm1.eps, W(b0.attn.qkv_proj, a0), a0) for i, blk in enumerate(blocks): a = blk.attn o = canon_rope_attention(qkv, a.canon, ctx.m1, ctx.m2, ctx.cos, ctx.sin, ctx.offsets).reshape(B, T, a.n_head * a.head_dim) f = blk.ffn ao, agu, adn = A(a.o_proj), A(f.gate_up_proj), A(f.down_proj) wo, wgu, wdn = W(a.o_proj, ao), W(f.gate_up_proj, agu), W(f.down_proj, adn, DOWN_SHRINK) if i + 1 < L: nb = blocks[i + 1] aq = A(nb.attn.qkv_proj) h, qkv = self._post_then_pre_x( h, o, wo, blk.norm2.weight, blk.norm2.eps, wgu, wdn, nb.norm1.weight, nb.norm1.eps, W(nb.attn.qkv_proj, aq), ng if i + 1 == ins else None, (ao, agu, adn, aq) if wacc else None, ft) else: h = self._post_then_final_norm_x( h, o, wo, blk.norm2.weight, blk.norm2.eps, wgu, wdn, self.norm_f.weight, self.norm_f.eps, (ao, agu, adn) if wacc else None, ft) return h def _forward_merged_stock(self, h, idx, boundary, ctx): from volta_kernels import canon_rope_attention blocks = self.blocks L = len(blocks) ins = self.cfg.ngram.insert_after if (self.ngram is not None and self.ngram_enabled) else -1 ng = self.ngram(idx, boundary, h.dtype) if ins >= 0 else None if ins == 0: h = h + ng B, T = h.shape[0], h.shape[1] b0 = blocks[0] qkv = self._pre_attn_qkv(h, b0.norm1.weight, b0.norm1.eps, b0.attn.qkv_proj) for i, blk in enumerate(blocks): a = blk.attn # Canon+RoPE+attention as one autograd Function: dQ reaches the # Canon backward in fp32 (see volta_kernels._CanonRopeAttn). o = canon_rope_attention(qkv, a.canon, ctx.m1, ctx.m2, ctx.cos, ctx.sin, ctx.offsets).reshape(B, T, a.n_head * a.head_dim) if i + 1 < L: nb = blocks[i + 1] h, qkv = self._post_then_pre( h, o, a.o_proj, blk.norm2.weight, blk.norm2.eps, blk.ffn.gate_up_proj, blk.ffn.down_proj, nb.norm1.weight, nb.norm1.eps, nb.attn.qkv_proj, ng if i + 1 == ins else None) else: h = self._post_then_final_norm( h, o, a.o_proj, blk.norm2.weight, blk.norm2.eps, blk.ffn.gate_up_proj, blk.ffn.down_proj, self.norm_f.weight, self.norm_f.eps) return h @torch.no_grad() def logits_for_eval(self, idx, pos, offsets, boundary, theta: float = 10_000.0): h = self.forward(idx, pos, offsets, boundary, None, theta) return F.linear(h, self.head_weight())