summaryrefslogtreecommitdiff
path: root/volta_kernels/integ_ops.py
diff options
context:
space:
mode:
Diffstat (limited to 'volta_kernels/integ_ops.py')
-rw-r--r--volta_kernels/integ_ops.py256
1 files changed, 256 insertions, 0 deletions
diff --git a/volta_kernels/integ_ops.py b/volta_kernels/integ_ops.py
new file mode 100644
index 0000000..8f05f01
--- /dev/null
+++ b/volta_kernels/integ_ops.py
@@ -0,0 +1,256 @@
+"""Model-side additions adopted from the kernel swarm, integrated on the merged
+(BORIS_VOLTA_KERNELS=1) path. Everything here is used by model.py's
+`_forward_merged`; nothing changes the forward values (the loss is bitwise the
+live one).
+
+1. wgrad_acc (block_autograd b, = resid_fp32 V1 = precision_plus dw):
+ the weight-gradient GEMM of every trunk matmul accumulates straight into
+ the fp32 p.grad (cuBLAS beta=1, fp32 output). Removes the fp16 rounding of
+ dW (and of its split-k partials), the fp16->fp32 cast and AccumulateGrad's
+ add_. dW error vs fp64 ~100x lower.
+2. norm_bwd (elem_audit): residual-add + RMSNorm backward and the norm-weight
+ gradient in ONE Triton pass (16-row blocks, dw partials in registers)
+ instead of Inductor's row kernel + a split column reduction that re-reads
+ the same 100-134 MB. Row arithmetic replicates Inductor's live kernel, so
+ dh/dh16 (and hence every other gradient) are bitwise the live ones.
+ (3. the CUTLASS SwiGLU epilogue was integrated, measured bitwise but not
+ faster on the whole micro-step, and dropped -- see ../epi_dropped.py.)
+4. TiedEmbedding (ngram_embed A): the head's dW*s, the embedding's segment
+ sums and the .grad accumulate in one pass (bitwise).
+"""
+from __future__ import annotations
+
+import torch
+import torch.nn.functional as F
+import triton
+import triton.language as tl
+from torch.library import triton_op, wrap_triton
+
+F16, F32 = torch.float16, torch.float32
+
+
+# --------------------------------------------------------------------------- #
+# 1. weight gradients accumulated by the GEMM into fp32 .grad
+# --------------------------------------------------------------------------- #
+@torch.library.custom_op("integ::wgrad_acc", mutates_args=("acc",))
+def wgrad_acc(dy: torch.Tensor, x: torch.Tensor, acc: torch.Tensor, alpha: float) -> None:
+ dy2 = dy.reshape(-1, dy.shape[-1])
+ x2 = x.reshape(-1, x.shape[-1])
+ torch.addmm(acc, dy2.t(), x2, beta=1, alpha=alpha, out_dtype=F32, out=acc)
+
+
+@wgrad_acc.register_fake
+def _(dy, x, acc, alpha):
+ return None
+
+
+class LinAcc(torch.autograd.Function):
+ """y = x16 @ w16^T. The weight gradient alpha * dy^T x16 is ADDED into `acc`
+ (the fp32 p.grad) by the GEMM itself and not returned to autograd. w16 is
+ the (pre-scaled) fp16 weight; alpha undoes nothing -- it is the scale the
+ forward applied to the fp32 weight (DOWN_SHRINK), i.e. d(w*s)/dw."""
+
+ @staticmethod
+ def forward(ctx, x16, w16, acc, alpha: float):
+ ctx.save_for_backward(x16, w16, acc)
+ ctx.alpha = alpha
+ return F.linear(x16, w16)
+
+ @staticmethod
+ def backward(ctx, dy):
+ x16, w16, acc = ctx.saved_tensors
+ torch.ops.integ.wgrad_acc(dy, x16, acc, ctx.alpha)
+ return dy @ w16, None, None, None
+
+
+# --------------------------------------------------------------------------- #
+# 2. fused residual-add + RMSNorm backward with the norm-weight gradient
+# --------------------------------------------------------------------------- #
+@triton.jit
+def _norm_bwd_bw_kernel(h, a, rstd, w, gres, gx, dh, dh16, dwp, ROWS: tl.constexpr,
+ D: tl.constexpr, HAS_A: tl.constexpr, OSCALE: tl.constexpr):
+ """Row-for-row the arithmetic of Inductor's live norm-backward row kernel
+ (triton_per_fused__to_copy_add_div_mul_pow_sum_view_*: one 512-wide row as a
+ 1-D block, 4 warps, num_stages 1, the same op DAG and so the same FMA
+ contractions), so dh and dh16 are BITWISE the live ones and every other
+ gradient stays bitwise too. The only change: the norm-weight gradient
+ sum_rows(gx * (h1*rstd)) -- the same per-element products as Inductor's
+ separate reduction kernel -- is accumulated here in fp32 over ROWS rows
+ (partials then summed in fp64), instead of by a second pass that re-reads
+ gx, h and a.
+ d = g_res + (gx*w)*rstd + ((sum(gx*w*h1)*-0.5)*rstd^3/D) * (2*h1)"""
+ pid = tl.program_id(0).to(tl.int64)
+ cols = tl.arange(0, D)
+ wv = tl.load(w + cols, eviction_policy="evict_last")
+ acc = tl.zeros([D], tl.float32)
+ for i in range(ROWS):
+ r = pid * ROWS + i
+ off = r * D + cols
+ g = tl.load(gx + off).to(tl.float32)
+ hv = tl.load(h + off)
+ if HAS_A:
+ hv = hv + tl.load(a + off).to(tl.float32)
+ gr = tl.load(gres + off)
+ rs = tl.load(rstd + r)
+ gw = g * wv
+ sm = tl.sum(gw * hv, 0)
+ t = gr + gw * rs
+ k = ((sm * -0.5) * (rs * rs * rs)) * (1.0 / D)
+ d = t + k * (hv * 2.0)
+ tl.store(dh + off, d)
+ if OSCALE == 1.0:
+ tl.store(dh16 + off, d.to(tl.float16))
+ else:
+ tl.store(dh16 + off, (d * OSCALE).to(tl.float16))
+ acc += g * (hv * rs)
+ tl.store(dwp + pid * D + cols, acc)
+
+
+@triton.jit
+def _colsum64_kernel(p, out, NR, D: tl.constexpr, BC: tl.constexpr, BR: tl.constexpr):
+ """Stage 1 of an fp64 column sum: out[k, c] = sum of rows k*BR..k*BR+BR-1 of p
+ (fp32 in, fp64 accumulate and out). Grid (D // BC, cdiv(NR, BR))."""
+ c = tl.program_id(0) * BC + tl.arange(0, BC)
+ k = tl.program_id(1)
+ r = k * BR + tl.arange(0, BR)
+ x = tl.load(p + r[:, None] * D + c[None, :], mask=r[:, None] < NR, other=0.0)
+ tl.store(out + k * D + c, tl.sum(x.to(tl.float64), 0))
+
+
+def _colsum64(p):
+ """fp32 (NR, D) -> fp32 (D,) column sums with an fp64 accumulator (deterministic)."""
+ NR, D = p.shape
+ BR = 64
+ nk = triton.cdiv(NR, BR)
+ part = torch.empty((nk, D), device=p.device, dtype=torch.float64)
+ wrap_triton(_colsum64_kernel)[(D // 64, nk)](p, part, NR, D=D, BC=64, BR=BR, num_warps=4)
+ return part.sum(0).to(F32)
+
+
+_NB_ROWS = 16 # rows per program: fp32 dw partial per 16 rows, partials summed in fp64
+
+
+def _norm_bwd_launch(h, a, rstd, w, gres, gx, oscale: float, has_a: bool):
+ N, D = h.shape
+ dh = torch.empty_like(h)
+ dh16 = torch.empty((N, D), device=h.device, dtype=F16)
+ dwp = torch.empty((N // _NB_ROWS, D), device=h.device, dtype=F32)
+ wrap_triton(_norm_bwd_bw_kernel)[(N // _NB_ROWS,)](
+ h, a, rstd, w, gres, gx, dh, dh16, dwp, ROWS=_NB_ROWS, D=D,
+ HAS_A=has_a, OSCALE=oscale, num_warps=4, num_stages=1)
+ return dh, dh16, _colsum64(dwp)
+
+
+@triton_op("integ::norm_bwd_a", mutates_args={})
+def norm_bwd_a(h: torch.Tensor, a: torch.Tensor, rstd: torch.Tensor, w: torch.Tensor,
+ gres: torch.Tensor, gx: torch.Tensor, oscale: float
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ return _norm_bwd_launch(h, a, rstd, w, gres, gx, oscale, True)
+
+
+@triton_op("integ::norm_bwd_h", mutates_args={})
+def norm_bwd_h(h: torch.Tensor, rstd: torch.Tensor, w: torch.Tensor,
+ gres: torch.Tensor, gx: torch.Tensor, oscale: float
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ return _norm_bwd_launch(h, h, rstd, w, gres, gx, oscale, False)
+
+
+class AddNorm(torch.autograd.Function):
+ """(h1, x16) = (h + s*a.float(), fp16(rmsnorm(h1) * w)).
+
+ Forward: the same torch ops as the live region, so Inductor emits the same
+ fused kernel (bitwise). Backward: one Triton pass (_norm_bwd_bw_kernel) that
+ reproduces the live row kernel bit for bit -- dh1 (fp32, to the residual)
+ and da16 = fp16(s*dh1) (to the o_proj / down GEMM backward) -- and also
+ accumulates the norm-weight gradient sum_rows(gx*xhat), which the live graph
+ computes in a second kernel re-reading gx, h and a.
+
+ save_out=False saves (h, a) -- the tensors the live graph already keeps for
+ this norm (h is the previous region's output, a the o_proj output).
+ save_out=True saves h1 instead -- for the next block's norm1, where h1 is
+ this region's output and is kept anyway as the next region's input.
+ """
+
+ @staticmethod
+ def forward(ctx, h, a, w, eps: float, s: float, save_out: bool):
+ h1 = h + a.float() if s == 1.0 else h + a.float() * s
+ rstd = torch.rsqrt(h1.pow(2).mean(-1, keepdim=True) + eps)
+ x16 = (h1 * rstd * w.float()).to(F16)
+ if save_out:
+ ctx.save_for_backward(h1, rstd, w)
+ else:
+ ctx.save_for_backward(h, a, rstd, w)
+ ctx.s, ctx.save_out = s, save_out
+ return h1, x16
+
+ @staticmethod
+ def backward(ctx, gh1, gx):
+ D = gx.shape[-1]
+ if ctx.save_out:
+ h1, rstd, w = ctx.saved_tensors
+ shp = h1.shape
+ dh, da, dw = torch.ops.integ.norm_bwd_h(
+ h1.reshape(-1, D), rstd.reshape(-1), w,
+ gh1.reshape(-1, D).contiguous(), gx.reshape(-1, D).contiguous(), ctx.s)
+ else:
+ h, a, rstd, w = ctx.saved_tensors
+ shp = h.shape
+ dh, da, dw = torch.ops.integ.norm_bwd_a(
+ h.reshape(-1, D), a.reshape(-1, D), rstd.reshape(-1), w,
+ gh1.reshape(-1, D).contiguous(), gx.reshape(-1, D).contiguous(), ctx.s)
+ return dh.view(shp), da.view(shp), dw, None, None, None
+
+
+# --------------------------------------------------------------------------- #
+# 4. tied embedding: acc + (head dW * s + embedding segment sums) in one pass
+# --------------------------------------------------------------------------- #
+@triton.jit
+def _tied_accum_dense(acc_ptr, dw_ptr, s_ptr, seg_ptr, out_ptr, n,
+ HAS_ACC: tl.constexpr, BLOCK: tl.constexpr):
+ offs = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)
+ m = offs < n
+ s = tl.load(s_ptr)
+ v = tl.load(dw_ptr + offs, mask=m) * s + tl.load(seg_ptr + offs, mask=m)
+ if HAS_ACC:
+ v = tl.load(acc_ptr + offs, mask=m) + v
+ tl.store(out_ptr + offs, v, mask=m)
+
+
+_STASH: dict = {}
+
+
+def stash_head_grad(W, dW, s):
+ """Called by the head's backward in place of returning dW*s for W."""
+ _STASH[id(W)] = (dW, s)
+
+
+class TiedEmbedding(torch.autograd.Function):
+ """F.embedding(idx, W) for the tied weight. Its backward adds
+ (head dW * s) + segment sums into W.grad in one pass -- the exact rounding
+ sequence of the stock path (dW*s rounded, + seg, + acc; FMA contraction
+ off), in place of: the dW*s pass, autograd's tied-weight sum and
+ AccumulateGrad's add_. Requires the head to stash its dW (stash=True)."""
+
+ @staticmethod
+ def forward(ctx, idx, W):
+ ctx.save_for_backward(idx)
+ ctx.W = W
+ return F.embedding(idx, W)
+
+ @staticmethod
+ def backward(ctx, g):
+ (idx,) = ctx.saved_tensors
+ W = ctx.W
+ dW, s = _STASH.pop(id(W))
+ seg = torch.ops.aten.embedding_dense_backward(
+ g.reshape(-1, g.shape[-1]), idx.reshape(-1), W.shape[0], -1, False)
+ has = W.grad is not None
+ out = W.grad if has else seg # first micro-step: in place into seg
+ n = W.numel()
+ BLOCK = 2048
+ _tied_accum_dense[(triton.cdiv(n, BLOCK),)](
+ out, dW, s.reshape(1).float(), seg, out, n, HAS_ACC=has, BLOCK=BLOCK,
+ num_warps=4, enable_fp_fusion=False)
+ if not has:
+ W.grad = out
+ return None, None