"""Fused LM head (cross-entropy without (N, V) logits), with the elementwise middle rewritten as one Triton pass. Same algorithm, same constants and the same operand precision as model._FusedLMHead -- read that docstring for why GRAD_SCALE is 64, why the dW GEMM is shrunk, and why the backward multiplier stays fp32. What changes is only how many times each chunk's logits cross memory. Production spelling, per 1024-row chunk at micro_batch 16 (x fp32 because the residual stream is fp32): F.linear(xc, Wc) autocast casts W (32768x512 fp32) to fp16 -- again .float() fp16 logits -> fp32 copy 128 MB _head_chunk_c 3 passes over fp32 logits, writes fp32 p 128 MB scatter_add_ one-hot, in place p.to(x.dtype) a no-op: x is fp32, so g is fp32 F.linear(g, Wc.t()) autocast casts g to fp16 and W to fp16 again dx[a:b] = ... fp16 -> fp32 copy into dx g * DW_SHRINK another fp32 pass 128 MB gd.t() @ xn autocast casts gd^T (strided) to fp16 Here W and x are cast once per call, not twice per chunk, and one kernel reads the fp16 logits once and writes the two fp16 operands the GEMMs were being fed anyway: fp16(p'), where p' is the scaled softmax minus the one-hot. dW is accumulated with an fp32-output GEMM, so it is never rounded to fp16 at all -- the production path's overflow guard (x rescale, DW_SHRINK) has nothing left to guard. lse differs from Inductor's only in fp32 reduction order. """ from __future__ import annotations import torch import torch.nn.functional as F import triton import triton.language as tl IGNORE = -100 @triton.jit def _softmax_grad_kernel(logits, tgt, g_out, row_loss, V: tl.constexpr, S: tl.constexpr): """One program per row: the whole 32k-wide row stays in registers. g_out = fp16(S * softmax - S * onehot), the operand both head GEMMs take. (A blocked two-pass version with online max/sum measured no faster.) """ row = tl.program_id(0).to(tl.int64) cols = tl.arange(0, V) x = tl.load(logits + row * V + cols).to(tl.float32) t = tl.load(tgt + row) valid = t != -100 safe = tl.where(valid, t, 0) # Inductor's logsumexp decomposition, including its guard for an # all-infinite row. m = tl.max(x, 0) m = tl.where(tl.abs(m) == float("inf"), 0.0, m) lse = tl.log(tl.sum(tl.exp(x - m), 0)) + m vs = valid.to(tl.float32) * S p = tl.exp(x - lse) * vs hit = cols == safe p = tl.where(hit, p + (-S * valid.to(tl.float32)), p) tl.store(g_out + row * V + cols, p.to(tl.float16)) picked = tl.sum(tl.where(hit, x, 0.0), 0) tl.store(row_loss + row, tl.where(valid, lse - picked, 0.0)) class FusedLMHead(torch.autograd.Function): GRAD_SCALE = 64.0 @staticmethod def forward(ctx, x, W, targets, chunk: int): N, d = x.shape V = W.shape[0] W16 = W.to(torch.float16) x16 = x.to(torch.float16) valid = targets != IGNORE n_valid = valid.sum().clamp_min(1) # dx is written fp32 straight from the GEMM's fp32 accumulator. With an # fp16 output cuBLAS may pick split-k kernels that sum partials in fp16 # (allowed by reduced-precision reduction); fp32 output rules that out. # Measured on the real fixture: dx error vs fp64 5.31e-4 -> 3.43e-4, # and 2.9% faster. dx16 = torch.empty((N, d), device=x.device, dtype=torch.float32) # zeros, not empty: do not rely on beta=0 never reading the buffer in the # out_dtype addmm path (0.07 ms per micro-step) dW = torch.zeros_like(W, dtype=torch.float32) row_loss = torch.empty(N, device=x.device, dtype=torch.float32) c = min(chunk, N) logits = torch.empty((c, V), device=x.device, dtype=torch.float16) g = torch.empty_like(logits) S = FusedLMHead.GRAD_SCALE for a in range(0, N, chunk): b = min(a + chunk, N) n = b - a lg = logits[:n] torch.mm(x16[a:b], W16.t(), out=lg) _softmax_grad_kernel[(n,)](lg, targets[a:b], g[:n], row_loss[a:b], V=V, S=S, num_warps=16) torch.mm(g[:n], W16, out_dtype=torch.float32, out=dx16[a:b]) # dW: fp16 operands, fp32 accumulation AND fp32 output (cuBLAS # s884gemm + fp32 split-k reduce), accumulated across chunks with # beta=1. The production head rounds every chunk's dW to fp16 and so # needs the |x| <= 1 rescale and DW_SHRINK to keep that output from # overflowing; with an fp32 output there is nothing to overflow. # Strictly more precise (dW rel-L2 vs fp64 5.24e-4 -> 5.01e-4 on # the real fixture), 12.6% faster, one fewer 64 MB write per chunk. torch.addmm(dW, g[:n].t(), x16[a:b], beta=0 if a == 0 else 1, out_dtype=torch.float32, out=dW) ctx.w_dtype = W.dtype ctx.x_dtype = x.dtype ctx.W_ref = W ctx.save_for_backward(dx16, dW) ctx.n_valid = n_valid return row_loss.sum() / n_valid @staticmethod def backward(ctx, grad_out): dx16, dW = ctx.saved_tensors s = grad_out / (ctx.n_valid * FusedLMHead.GRAD_SCALE) return ((dx16.float() * s).to(ctx.x_dtype), (dW * s).to(ctx.w_dtype), None, None) class FusedLMHeadStash(FusedLMHead): """Same forward. The backward hands (dW, s) to the tied embedding's backward (integ_ops.TiedEmbedding), which folds dW*s, the embedding segment sums and the .grad accumulate into one pass, instead of returning dW*s to autograd.""" @staticmethod def backward(ctx, grad_out): from .integ_ops import stash_head_grad dx16, dW = ctx.saved_tensors s = grad_out / (ctx.n_valid * FusedLMHead.GRAD_SCALE) stash_head_grad(ctx.W_ref, dW, s) return (dx16.float() * s).to(ctx.x_dtype), None, None, None def _loss_only(x, W, targets, chunk): N = x.shape[0] V = W.shape[0] W16 = W.to(torch.float16) x16 = x.to(torch.float16) n_valid = (targets != IGNORE).sum().clamp_min(1) row_loss = torch.empty(N, device=x.device, dtype=torch.float32) c = min(chunk, N) logits = torch.empty((c, V), device=x.device, dtype=torch.float16) g = torch.empty_like(logits) for a in range(0, N, chunk): b = min(a + chunk, N) n = b - a torch.mm(x16[a:b], W16.t(), out=logits[:n]) _softmax_grad_kernel[(n,)](logits[:n], targets[a:b], g[:n], row_loss[a:b], V=V, S=FusedLMHead.GRAD_SCALE, num_warps=16) return row_loss.sum() / n_valid def fused_lm_loss(x, W, targets, chunk: int = 1024, stash: bool = False): if not torch.is_grad_enabled(): # Validation (quick_val runs under no_grad): no backward will come, so # skip the dx and dW GEMMs entirely -- 2.25x faster head. Same logits # GEMM and softmax kernel, so the loss is bitwise the training path's. # (Checked here, not inside the Function: its forward always runs with # grad mode off, and the tied weight keeps requires_grad under no_grad.) return _loss_only(x, W, targets, chunk) if stash: return FusedLMHeadStash.apply(x, W, targets, chunk) return FusedLMHead.apply(x, W, targets, chunk)