diff options
Diffstat (limited to 'volta_kernels/integ_ops.py')
| -rw-r--r-- | volta_kernels/integ_ops.py | 256 |
1 files changed, 256 insertions, 0 deletions
diff --git a/volta_kernels/integ_ops.py b/volta_kernels/integ_ops.py new file mode 100644 index 0000000..8f05f01 --- /dev/null +++ b/volta_kernels/integ_ops.py @@ -0,0 +1,256 @@ +"""Model-side additions adopted from the kernel swarm, integrated on the merged +(BORIS_VOLTA_KERNELS=1) path. Everything here is used by model.py's +`_forward_merged`; nothing changes the forward values (the loss is bitwise the +live one). + +1. wgrad_acc (block_autograd b, = resid_fp32 V1 = precision_plus dw): + the weight-gradient GEMM of every trunk matmul accumulates straight into + the fp32 p.grad (cuBLAS beta=1, fp32 output). Removes the fp16 rounding of + dW (and of its split-k partials), the fp16->fp32 cast and AccumulateGrad's + add_. dW error vs fp64 ~100x lower. +2. norm_bwd (elem_audit): residual-add + RMSNorm backward and the norm-weight + gradient in ONE Triton pass (16-row blocks, dw partials in registers) + instead of Inductor's row kernel + a split column reduction that re-reads + the same 100-134 MB. Row arithmetic replicates Inductor's live kernel, so + dh/dh16 (and hence every other gradient) are bitwise the live ones. + (3. the CUTLASS SwiGLU epilogue was integrated, measured bitwise but not + faster on the whole micro-step, and dropped -- see ../epi_dropped.py.) +4. TiedEmbedding (ngram_embed A): the head's dW*s, the embedding's segment + sums and the .grad accumulate in one pass (bitwise). +""" +from __future__ import annotations + +import torch +import torch.nn.functional as F +import triton +import triton.language as tl +from torch.library import triton_op, wrap_triton + +F16, F32 = torch.float16, torch.float32 + + +# --------------------------------------------------------------------------- # +# 1. weight gradients accumulated by the GEMM into fp32 .grad +# --------------------------------------------------------------------------- # +@torch.library.custom_op("integ::wgrad_acc", mutates_args=("acc",)) +def wgrad_acc(dy: torch.Tensor, x: torch.Tensor, acc: torch.Tensor, alpha: float) -> None: + dy2 = dy.reshape(-1, dy.shape[-1]) + x2 = x.reshape(-1, x.shape[-1]) + torch.addmm(acc, dy2.t(), x2, beta=1, alpha=alpha, out_dtype=F32, out=acc) + + +@wgrad_acc.register_fake +def _(dy, x, acc, alpha): + return None + + +class LinAcc(torch.autograd.Function): + """y = x16 @ w16^T. The weight gradient alpha * dy^T x16 is ADDED into `acc` + (the fp32 p.grad) by the GEMM itself and not returned to autograd. w16 is + the (pre-scaled) fp16 weight; alpha undoes nothing -- it is the scale the + forward applied to the fp32 weight (DOWN_SHRINK), i.e. d(w*s)/dw.""" + + @staticmethod + def forward(ctx, x16, w16, acc, alpha: float): + ctx.save_for_backward(x16, w16, acc) + ctx.alpha = alpha + return F.linear(x16, w16) + + @staticmethod + def backward(ctx, dy): + x16, w16, acc = ctx.saved_tensors + torch.ops.integ.wgrad_acc(dy, x16, acc, ctx.alpha) + return dy @ w16, None, None, None + + +# --------------------------------------------------------------------------- # +# 2. fused residual-add + RMSNorm backward with the norm-weight gradient +# --------------------------------------------------------------------------- # +@triton.jit +def _norm_bwd_bw_kernel(h, a, rstd, w, gres, gx, dh, dh16, dwp, ROWS: tl.constexpr, + D: tl.constexpr, HAS_A: tl.constexpr, OSCALE: tl.constexpr): + """Row-for-row the arithmetic of Inductor's live norm-backward row kernel + (triton_per_fused__to_copy_add_div_mul_pow_sum_view_*: one 512-wide row as a + 1-D block, 4 warps, num_stages 1, the same op DAG and so the same FMA + contractions), so dh and dh16 are BITWISE the live ones and every other + gradient stays bitwise too. The only change: the norm-weight gradient + sum_rows(gx * (h1*rstd)) -- the same per-element products as Inductor's + separate reduction kernel -- is accumulated here in fp32 over ROWS rows + (partials then summed in fp64), instead of by a second pass that re-reads + gx, h and a. + d = g_res + (gx*w)*rstd + ((sum(gx*w*h1)*-0.5)*rstd^3/D) * (2*h1)""" + pid = tl.program_id(0).to(tl.int64) + cols = tl.arange(0, D) + wv = tl.load(w + cols, eviction_policy="evict_last") + acc = tl.zeros([D], tl.float32) + for i in range(ROWS): + r = pid * ROWS + i + off = r * D + cols + g = tl.load(gx + off).to(tl.float32) + hv = tl.load(h + off) + if HAS_A: + hv = hv + tl.load(a + off).to(tl.float32) + gr = tl.load(gres + off) + rs = tl.load(rstd + r) + gw = g * wv + sm = tl.sum(gw * hv, 0) + t = gr + gw * rs + k = ((sm * -0.5) * (rs * rs * rs)) * (1.0 / D) + d = t + k * (hv * 2.0) + tl.store(dh + off, d) + if OSCALE == 1.0: + tl.store(dh16 + off, d.to(tl.float16)) + else: + tl.store(dh16 + off, (d * OSCALE).to(tl.float16)) + acc += g * (hv * rs) + tl.store(dwp + pid * D + cols, acc) + + +@triton.jit +def _colsum64_kernel(p, out, NR, D: tl.constexpr, BC: tl.constexpr, BR: tl.constexpr): + """Stage 1 of an fp64 column sum: out[k, c] = sum of rows k*BR..k*BR+BR-1 of p + (fp32 in, fp64 accumulate and out). Grid (D // BC, cdiv(NR, BR)).""" + c = tl.program_id(0) * BC + tl.arange(0, BC) + k = tl.program_id(1) + r = k * BR + tl.arange(0, BR) + x = tl.load(p + r[:, None] * D + c[None, :], mask=r[:, None] < NR, other=0.0) + tl.store(out + k * D + c, tl.sum(x.to(tl.float64), 0)) + + +def _colsum64(p): + """fp32 (NR, D) -> fp32 (D,) column sums with an fp64 accumulator (deterministic).""" + NR, D = p.shape + BR = 64 + nk = triton.cdiv(NR, BR) + part = torch.empty((nk, D), device=p.device, dtype=torch.float64) + wrap_triton(_colsum64_kernel)[(D // 64, nk)](p, part, NR, D=D, BC=64, BR=BR, num_warps=4) + return part.sum(0).to(F32) + + +_NB_ROWS = 16 # rows per program: fp32 dw partial per 16 rows, partials summed in fp64 + + +def _norm_bwd_launch(h, a, rstd, w, gres, gx, oscale: float, has_a: bool): + N, D = h.shape + dh = torch.empty_like(h) + dh16 = torch.empty((N, D), device=h.device, dtype=F16) + dwp = torch.empty((N // _NB_ROWS, D), device=h.device, dtype=F32) + wrap_triton(_norm_bwd_bw_kernel)[(N // _NB_ROWS,)]( + h, a, rstd, w, gres, gx, dh, dh16, dwp, ROWS=_NB_ROWS, D=D, + HAS_A=has_a, OSCALE=oscale, num_warps=4, num_stages=1) + return dh, dh16, _colsum64(dwp) + + +@triton_op("integ::norm_bwd_a", mutates_args={}) +def norm_bwd_a(h: torch.Tensor, a: torch.Tensor, rstd: torch.Tensor, w: torch.Tensor, + gres: torch.Tensor, gx: torch.Tensor, oscale: float + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + return _norm_bwd_launch(h, a, rstd, w, gres, gx, oscale, True) + + +@triton_op("integ::norm_bwd_h", mutates_args={}) +def norm_bwd_h(h: torch.Tensor, rstd: torch.Tensor, w: torch.Tensor, + gres: torch.Tensor, gx: torch.Tensor, oscale: float + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + return _norm_bwd_launch(h, h, rstd, w, gres, gx, oscale, False) + + +class AddNorm(torch.autograd.Function): + """(h1, x16) = (h + s*a.float(), fp16(rmsnorm(h1) * w)). + + Forward: the same torch ops as the live region, so Inductor emits the same + fused kernel (bitwise). Backward: one Triton pass (_norm_bwd_bw_kernel) that + reproduces the live row kernel bit for bit -- dh1 (fp32, to the residual) + and da16 = fp16(s*dh1) (to the o_proj / down GEMM backward) -- and also + accumulates the norm-weight gradient sum_rows(gx*xhat), which the live graph + computes in a second kernel re-reading gx, h and a. + + save_out=False saves (h, a) -- the tensors the live graph already keeps for + this norm (h is the previous region's output, a the o_proj output). + save_out=True saves h1 instead -- for the next block's norm1, where h1 is + this region's output and is kept anyway as the next region's input. + """ + + @staticmethod + def forward(ctx, h, a, w, eps: float, s: float, save_out: bool): + h1 = h + a.float() if s == 1.0 else h + a.float() * s + rstd = torch.rsqrt(h1.pow(2).mean(-1, keepdim=True) + eps) + x16 = (h1 * rstd * w.float()).to(F16) + if save_out: + ctx.save_for_backward(h1, rstd, w) + else: + ctx.save_for_backward(h, a, rstd, w) + ctx.s, ctx.save_out = s, save_out + return h1, x16 + + @staticmethod + def backward(ctx, gh1, gx): + D = gx.shape[-1] + if ctx.save_out: + h1, rstd, w = ctx.saved_tensors + shp = h1.shape + dh, da, dw = torch.ops.integ.norm_bwd_h( + h1.reshape(-1, D), rstd.reshape(-1), w, + gh1.reshape(-1, D).contiguous(), gx.reshape(-1, D).contiguous(), ctx.s) + else: + h, a, rstd, w = ctx.saved_tensors + shp = h.shape + dh, da, dw = torch.ops.integ.norm_bwd_a( + h.reshape(-1, D), a.reshape(-1, D), rstd.reshape(-1), w, + gh1.reshape(-1, D).contiguous(), gx.reshape(-1, D).contiguous(), ctx.s) + return dh.view(shp), da.view(shp), dw, None, None, None + + +# --------------------------------------------------------------------------- # +# 4. tied embedding: acc + (head dW * s + embedding segment sums) in one pass +# --------------------------------------------------------------------------- # +@triton.jit +def _tied_accum_dense(acc_ptr, dw_ptr, s_ptr, seg_ptr, out_ptr, n, + HAS_ACC: tl.constexpr, BLOCK: tl.constexpr): + offs = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK) + m = offs < n + s = tl.load(s_ptr) + v = tl.load(dw_ptr + offs, mask=m) * s + tl.load(seg_ptr + offs, mask=m) + if HAS_ACC: + v = tl.load(acc_ptr + offs, mask=m) + v + tl.store(out_ptr + offs, v, mask=m) + + +_STASH: dict = {} + + +def stash_head_grad(W, dW, s): + """Called by the head's backward in place of returning dW*s for W.""" + _STASH[id(W)] = (dW, s) + + +class TiedEmbedding(torch.autograd.Function): + """F.embedding(idx, W) for the tied weight. Its backward adds + (head dW * s) + segment sums into W.grad in one pass -- the exact rounding + sequence of the stock path (dW*s rounded, + seg, + acc; FMA contraction + off), in place of: the dW*s pass, autograd's tied-weight sum and + AccumulateGrad's add_. Requires the head to stash its dW (stash=True).""" + + @staticmethod + def forward(ctx, idx, W): + ctx.save_for_backward(idx) + ctx.W = W + return F.embedding(idx, W) + + @staticmethod + def backward(ctx, g): + (idx,) = ctx.saved_tensors + W = ctx.W + dW, s = _STASH.pop(id(W)) + seg = torch.ops.aten.embedding_dense_backward( + g.reshape(-1, g.shape[-1]), idx.reshape(-1), W.shape[0], -1, False) + has = W.grad is not None + out = W.grad if has else seg # first micro-step: in place into seg + n = W.numel() + BLOCK = 2048 + _tied_accum_dense[(triton.cdiv(n, BLOCK),)]( + out, dW, s.reshape(1).float(), seg, out, n, HAS_ACC=has, BLOCK=BLOCK, + num_warps=4, enable_fp_fusion=False) + if not has: + W.grad = out + return None, None |
