summaryrefslogtreecommitdiff
path: root/volta_kernels/head.py
diff options
context:
space:
mode:
Diffstat (limited to 'volta_kernels/head.py')
-rw-r--r--volta_kernels/head.py168
1 files changed, 168 insertions, 0 deletions
diff --git a/volta_kernels/head.py b/volta_kernels/head.py
new file mode 100644
index 0000000..35c3974
--- /dev/null
+++ b/volta_kernels/head.py
@@ -0,0 +1,168 @@
+"""Fused LM head (cross-entropy without (N, V) logits), with the elementwise
+middle rewritten as one Triton pass.
+
+Same algorithm, same constants and the same operand precision as
+model._FusedLMHead -- read that docstring for why GRAD_SCALE is 64, why the dW
+GEMM is shrunk, and why the backward multiplier stays fp32. What changes is only
+how many times each chunk's logits cross memory.
+
+Production spelling, per 1024-row chunk at micro_batch 16 (x fp32 because the
+residual stream is fp32):
+
+ F.linear(xc, Wc) autocast casts W (32768x512 fp32) to fp16 -- again
+ .float() fp16 logits -> fp32 copy 128 MB
+ _head_chunk_c 3 passes over fp32 logits, writes fp32 p 128 MB
+ scatter_add_ one-hot, in place
+ p.to(x.dtype) a no-op: x is fp32, so g is fp32
+ F.linear(g, Wc.t()) autocast casts g to fp16 and W to fp16 again
+ dx[a:b] = ... fp16 -> fp32 copy into dx
+ g * DW_SHRINK another fp32 pass 128 MB
+ gd.t() @ xn autocast casts gd^T (strided) to fp16
+
+Here W and x are cast once per call, not twice per chunk, and one kernel reads
+the fp16 logits once and writes the two fp16 operands the GEMMs were being fed
+anyway: fp16(p'), where p' is the scaled softmax minus the one-hot. dW is
+accumulated with an fp32-output GEMM, so it is never rounded to fp16 at all --
+the production path's overflow guard (x rescale, DW_SHRINK) has nothing left to
+guard. lse differs from Inductor's only in fp32 reduction order.
+"""
+from __future__ import annotations
+
+import torch
+import torch.nn.functional as F
+import triton
+import triton.language as tl
+
+IGNORE = -100
+
+
+@triton.jit
+def _softmax_grad_kernel(logits, tgt, g_out, row_loss, V: tl.constexpr, S: tl.constexpr):
+ """One program per row: the whole 32k-wide row stays in registers.
+
+ g_out = fp16(S * softmax - S * onehot), the operand both head GEMMs take.
+ (A blocked two-pass version with online max/sum measured no faster.)
+ """
+ row = tl.program_id(0).to(tl.int64)
+ cols = tl.arange(0, V)
+ x = tl.load(logits + row * V + cols).to(tl.float32)
+ t = tl.load(tgt + row)
+ valid = t != -100
+ safe = tl.where(valid, t, 0)
+ # Inductor's logsumexp decomposition, including its guard for an
+ # all-infinite row.
+ m = tl.max(x, 0)
+ m = tl.where(tl.abs(m) == float("inf"), 0.0, m)
+ lse = tl.log(tl.sum(tl.exp(x - m), 0)) + m
+ vs = valid.to(tl.float32) * S
+ p = tl.exp(x - lse) * vs
+ hit = cols == safe
+ p = tl.where(hit, p + (-S * valid.to(tl.float32)), p)
+ tl.store(g_out + row * V + cols, p.to(tl.float16))
+ picked = tl.sum(tl.where(hit, x, 0.0), 0)
+ tl.store(row_loss + row, tl.where(valid, lse - picked, 0.0))
+
+
+class FusedLMHead(torch.autograd.Function):
+ GRAD_SCALE = 64.0
+
+ @staticmethod
+ def forward(ctx, x, W, targets, chunk: int):
+ N, d = x.shape
+ V = W.shape[0]
+ W16 = W.to(torch.float16)
+ x16 = x.to(torch.float16)
+ valid = targets != IGNORE
+ n_valid = valid.sum().clamp_min(1)
+ # dx is written fp32 straight from the GEMM's fp32 accumulator. With an
+ # fp16 output cuBLAS may pick split-k kernels that sum partials in fp16
+ # (allowed by reduced-precision reduction); fp32 output rules that out.
+ # Measured on the real fixture: dx error vs fp64 5.31e-4 -> 3.43e-4,
+ # and 2.9% faster.
+ dx16 = torch.empty((N, d), device=x.device, dtype=torch.float32)
+ # zeros, not empty: do not rely on beta=0 never reading the buffer in the
+ # out_dtype addmm path (0.07 ms per micro-step)
+ dW = torch.zeros_like(W, dtype=torch.float32)
+ row_loss = torch.empty(N, device=x.device, dtype=torch.float32)
+ c = min(chunk, N)
+ logits = torch.empty((c, V), device=x.device, dtype=torch.float16)
+ g = torch.empty_like(logits)
+ S = FusedLMHead.GRAD_SCALE
+
+ for a in range(0, N, chunk):
+ b = min(a + chunk, N)
+ n = b - a
+ lg = logits[:n]
+ torch.mm(x16[a:b], W16.t(), out=lg)
+ _softmax_grad_kernel[(n,)](lg, targets[a:b], g[:n], row_loss[a:b],
+ V=V, S=S, num_warps=16)
+ torch.mm(g[:n], W16, out_dtype=torch.float32, out=dx16[a:b])
+ # dW: fp16 operands, fp32 accumulation AND fp32 output (cuBLAS
+ # s884gemm + fp32 split-k reduce), accumulated across chunks with
+ # beta=1. The production head rounds every chunk's dW to fp16 and so
+ # needs the |x| <= 1 rescale and DW_SHRINK to keep that output from
+ # overflowing; with an fp32 output there is nothing to overflow.
+ # Strictly more precise (dW rel-L2 vs fp64 5.24e-4 -> 5.01e-4 on
+ # the real fixture), 12.6% faster, one fewer 64 MB write per chunk.
+ torch.addmm(dW, g[:n].t(), x16[a:b], beta=0 if a == 0 else 1,
+ out_dtype=torch.float32, out=dW)
+
+ ctx.w_dtype = W.dtype
+ ctx.x_dtype = x.dtype
+ ctx.W_ref = W
+ ctx.save_for_backward(dx16, dW)
+ ctx.n_valid = n_valid
+ return row_loss.sum() / n_valid
+
+ @staticmethod
+ def backward(ctx, grad_out):
+ dx16, dW = ctx.saved_tensors
+ s = grad_out / (ctx.n_valid * FusedLMHead.GRAD_SCALE)
+ return ((dx16.float() * s).to(ctx.x_dtype), (dW * s).to(ctx.w_dtype),
+ None, None)
+
+
+class FusedLMHeadStash(FusedLMHead):
+ """Same forward. The backward hands (dW, s) to the tied embedding's backward
+ (integ_ops.TiedEmbedding), which folds dW*s, the embedding segment sums and
+ the .grad accumulate into one pass, instead of returning dW*s to autograd."""
+
+ @staticmethod
+ def backward(ctx, grad_out):
+ from .integ_ops import stash_head_grad
+ dx16, dW = ctx.saved_tensors
+ s = grad_out / (ctx.n_valid * FusedLMHead.GRAD_SCALE)
+ stash_head_grad(ctx.W_ref, dW, s)
+ return (dx16.float() * s).to(ctx.x_dtype), None, None, None
+
+
+def _loss_only(x, W, targets, chunk):
+ N = x.shape[0]
+ V = W.shape[0]
+ W16 = W.to(torch.float16)
+ x16 = x.to(torch.float16)
+ n_valid = (targets != IGNORE).sum().clamp_min(1)
+ row_loss = torch.empty(N, device=x.device, dtype=torch.float32)
+ c = min(chunk, N)
+ logits = torch.empty((c, V), device=x.device, dtype=torch.float16)
+ g = torch.empty_like(logits)
+ for a in range(0, N, chunk):
+ b = min(a + chunk, N)
+ n = b - a
+ torch.mm(x16[a:b], W16.t(), out=logits[:n])
+ _softmax_grad_kernel[(n,)](logits[:n], targets[a:b], g[:n], row_loss[a:b],
+ V=V, S=FusedLMHead.GRAD_SCALE, num_warps=16)
+ return row_loss.sum() / n_valid
+
+
+def fused_lm_loss(x, W, targets, chunk: int = 1024, stash: bool = False):
+ if not torch.is_grad_enabled():
+ # Validation (quick_val runs under no_grad): no backward will come, so
+ # skip the dx and dW GEMMs entirely -- 2.25x faster head. Same logits
+ # GEMM and softmax kernel, so the loss is bitwise the training path's.
+ # (Checked here, not inside the Function: its forward always runs with
+ # grad mode off, and the tied weight keeps requires_grad under no_grad.)
+ return _loss_only(x, W, targets, chunk)
+ if stash:
+ return FusedLMHeadStash.apply(x, W, targets, chunk)
+ return FusedLMHead.apply(x, W, targets, chunk)