From 5d2e4252e695644c34be605b44a14bd80a98a0a6 Mon Sep 17 00:00:00 2001 From: jdj Date: Wed, 18 Nov 2026 14:47:33 -0800 Subject: Boris-2 training code The code, configuration and data filters used to train Boris-2 (144M, 200B tokens, single V100): model, training loop, data preparation, n-gram table, V100 kernels, final stages and the exact run configs. --- volta_kernels/head.py | 168 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 volta_kernels/head.py (limited to 'volta_kernels/head.py') diff --git a/volta_kernels/head.py b/volta_kernels/head.py new file mode 100644 index 0000000..35c3974 --- /dev/null +++ b/volta_kernels/head.py @@ -0,0 +1,168 @@ +"""Fused LM head (cross-entropy without (N, V) logits), with the elementwise +middle rewritten as one Triton pass. + +Same algorithm, same constants and the same operand precision as +model._FusedLMHead -- read that docstring for why GRAD_SCALE is 64, why the dW +GEMM is shrunk, and why the backward multiplier stays fp32. What changes is only +how many times each chunk's logits cross memory. + +Production spelling, per 1024-row chunk at micro_batch 16 (x fp32 because the +residual stream is fp32): + + F.linear(xc, Wc) autocast casts W (32768x512 fp32) to fp16 -- again + .float() fp16 logits -> fp32 copy 128 MB + _head_chunk_c 3 passes over fp32 logits, writes fp32 p 128 MB + scatter_add_ one-hot, in place + p.to(x.dtype) a no-op: x is fp32, so g is fp32 + F.linear(g, Wc.t()) autocast casts g to fp16 and W to fp16 again + dx[a:b] = ... fp16 -> fp32 copy into dx + g * DW_SHRINK another fp32 pass 128 MB + gd.t() @ xn autocast casts gd^T (strided) to fp16 + +Here W and x are cast once per call, not twice per chunk, and one kernel reads +the fp16 logits once and writes the two fp16 operands the GEMMs were being fed +anyway: fp16(p'), where p' is the scaled softmax minus the one-hot. dW is +accumulated with an fp32-output GEMM, so it is never rounded to fp16 at all -- +the production path's overflow guard (x rescale, DW_SHRINK) has nothing left to +guard. lse differs from Inductor's only in fp32 reduction order. +""" +from __future__ import annotations + +import torch +import torch.nn.functional as F +import triton +import triton.language as tl + +IGNORE = -100 + + +@triton.jit +def _softmax_grad_kernel(logits, tgt, g_out, row_loss, V: tl.constexpr, S: tl.constexpr): + """One program per row: the whole 32k-wide row stays in registers. + + g_out = fp16(S * softmax - S * onehot), the operand both head GEMMs take. + (A blocked two-pass version with online max/sum measured no faster.) + """ + row = tl.program_id(0).to(tl.int64) + cols = tl.arange(0, V) + x = tl.load(logits + row * V + cols).to(tl.float32) + t = tl.load(tgt + row) + valid = t != -100 + safe = tl.where(valid, t, 0) + # Inductor's logsumexp decomposition, including its guard for an + # all-infinite row. + m = tl.max(x, 0) + m = tl.where(tl.abs(m) == float("inf"), 0.0, m) + lse = tl.log(tl.sum(tl.exp(x - m), 0)) + m + vs = valid.to(tl.float32) * S + p = tl.exp(x - lse) * vs + hit = cols == safe + p = tl.where(hit, p + (-S * valid.to(tl.float32)), p) + tl.store(g_out + row * V + cols, p.to(tl.float16)) + picked = tl.sum(tl.where(hit, x, 0.0), 0) + tl.store(row_loss + row, tl.where(valid, lse - picked, 0.0)) + + +class FusedLMHead(torch.autograd.Function): + GRAD_SCALE = 64.0 + + @staticmethod + def forward(ctx, x, W, targets, chunk: int): + N, d = x.shape + V = W.shape[0] + W16 = W.to(torch.float16) + x16 = x.to(torch.float16) + valid = targets != IGNORE + n_valid = valid.sum().clamp_min(1) + # dx is written fp32 straight from the GEMM's fp32 accumulator. With an + # fp16 output cuBLAS may pick split-k kernels that sum partials in fp16 + # (allowed by reduced-precision reduction); fp32 output rules that out. + # Measured on the real fixture: dx error vs fp64 5.31e-4 -> 3.43e-4, + # and 2.9% faster. + dx16 = torch.empty((N, d), device=x.device, dtype=torch.float32) + # zeros, not empty: do not rely on beta=0 never reading the buffer in the + # out_dtype addmm path (0.07 ms per micro-step) + dW = torch.zeros_like(W, dtype=torch.float32) + row_loss = torch.empty(N, device=x.device, dtype=torch.float32) + c = min(chunk, N) + logits = torch.empty((c, V), device=x.device, dtype=torch.float16) + g = torch.empty_like(logits) + S = FusedLMHead.GRAD_SCALE + + for a in range(0, N, chunk): + b = min(a + chunk, N) + n = b - a + lg = logits[:n] + torch.mm(x16[a:b], W16.t(), out=lg) + _softmax_grad_kernel[(n,)](lg, targets[a:b], g[:n], row_loss[a:b], + V=V, S=S, num_warps=16) + torch.mm(g[:n], W16, out_dtype=torch.float32, out=dx16[a:b]) + # dW: fp16 operands, fp32 accumulation AND fp32 output (cuBLAS + # s884gemm + fp32 split-k reduce), accumulated across chunks with + # beta=1. The production head rounds every chunk's dW to fp16 and so + # needs the |x| <= 1 rescale and DW_SHRINK to keep that output from + # overflowing; with an fp32 output there is nothing to overflow. + # Strictly more precise (dW rel-L2 vs fp64 5.24e-4 -> 5.01e-4 on + # the real fixture), 12.6% faster, one fewer 64 MB write per chunk. + torch.addmm(dW, g[:n].t(), x16[a:b], beta=0 if a == 0 else 1, + out_dtype=torch.float32, out=dW) + + ctx.w_dtype = W.dtype + ctx.x_dtype = x.dtype + ctx.W_ref = W + ctx.save_for_backward(dx16, dW) + ctx.n_valid = n_valid + return row_loss.sum() / n_valid + + @staticmethod + def backward(ctx, grad_out): + dx16, dW = ctx.saved_tensors + s = grad_out / (ctx.n_valid * FusedLMHead.GRAD_SCALE) + return ((dx16.float() * s).to(ctx.x_dtype), (dW * s).to(ctx.w_dtype), + None, None) + + +class FusedLMHeadStash(FusedLMHead): + """Same forward. The backward hands (dW, s) to the tied embedding's backward + (integ_ops.TiedEmbedding), which folds dW*s, the embedding segment sums and + the .grad accumulate into one pass, instead of returning dW*s to autograd.""" + + @staticmethod + def backward(ctx, grad_out): + from .integ_ops import stash_head_grad + dx16, dW = ctx.saved_tensors + s = grad_out / (ctx.n_valid * FusedLMHead.GRAD_SCALE) + stash_head_grad(ctx.W_ref, dW, s) + return (dx16.float() * s).to(ctx.x_dtype), None, None, None + + +def _loss_only(x, W, targets, chunk): + N = x.shape[0] + V = W.shape[0] + W16 = W.to(torch.float16) + x16 = x.to(torch.float16) + n_valid = (targets != IGNORE).sum().clamp_min(1) + row_loss = torch.empty(N, device=x.device, dtype=torch.float32) + c = min(chunk, N) + logits = torch.empty((c, V), device=x.device, dtype=torch.float16) + g = torch.empty_like(logits) + for a in range(0, N, chunk): + b = min(a + chunk, N) + n = b - a + torch.mm(x16[a:b], W16.t(), out=logits[:n]) + _softmax_grad_kernel[(n,)](logits[:n], targets[a:b], g[:n], row_loss[a:b], + V=V, S=FusedLMHead.GRAD_SCALE, num_warps=16) + return row_loss.sum() / n_valid + + +def fused_lm_loss(x, W, targets, chunk: int = 1024, stash: bool = False): + if not torch.is_grad_enabled(): + # Validation (quick_val runs under no_grad): no backward will come, so + # skip the dx and dW GEMMs entirely -- 2.25x faster head. Same logits + # GEMM and softmax kernel, so the loss is bitwise the training path's. + # (Checked here, not inside the Function: its forward always runs with + # grad mode off, and the tied weight keeps requires_grad under no_grad.) + return _loss_only(x, W, targets, chunk) + if stash: + return FusedLMHeadStash.apply(x, W, targets, chunk) + return FusedLMHead.apply(x, W, targets, chunk) -- cgit v1.2.3