"""Model-side additions adopted from the kernel swarm, integrated on the merged (BORIS_VOLTA_KERNELS=1) path. Everything here is used by model.py's `_forward_merged`; nothing changes the forward values (the loss is bitwise the live one). 1. wgrad_acc (block_autograd b, = resid_fp32 V1 = precision_plus dw): the weight-gradient GEMM of every trunk matmul accumulates straight into the fp32 p.grad (cuBLAS beta=1, fp32 output). Removes the fp16 rounding of dW (and of its split-k partials), the fp16->fp32 cast and AccumulateGrad's add_. dW error vs fp64 ~100x lower. 2. norm_bwd (elem_audit): residual-add + RMSNorm backward and the norm-weight gradient in ONE Triton pass (16-row blocks, dw partials in registers) instead of Inductor's row kernel + a split column reduction that re-reads the same 100-134 MB. Row arithmetic replicates Inductor's live kernel, so dh/dh16 (and hence every other gradient) are bitwise the live ones. (3. the CUTLASS SwiGLU epilogue was integrated, measured bitwise but not faster on the whole micro-step, and dropped -- see ../epi_dropped.py.) 4. TiedEmbedding (ngram_embed A): the head's dW*s, the embedding's segment sums and the .grad accumulate in one pass (bitwise). """ from __future__ import annotations import torch import torch.nn.functional as F import triton import triton.language as tl from torch.library import triton_op, wrap_triton F16, F32 = torch.float16, torch.float32 # --------------------------------------------------------------------------- # # 1. weight gradients accumulated by the GEMM into fp32 .grad # --------------------------------------------------------------------------- # @torch.library.custom_op("integ::wgrad_acc", mutates_args=("acc",)) def wgrad_acc(dy: torch.Tensor, x: torch.Tensor, acc: torch.Tensor, alpha: float) -> None: dy2 = dy.reshape(-1, dy.shape[-1]) x2 = x.reshape(-1, x.shape[-1]) torch.addmm(acc, dy2.t(), x2, beta=1, alpha=alpha, out_dtype=F32, out=acc) @wgrad_acc.register_fake def _(dy, x, acc, alpha): return None class LinAcc(torch.autograd.Function): """y = x16 @ w16^T. The weight gradient alpha * dy^T x16 is ADDED into `acc` (the fp32 p.grad) by the GEMM itself and not returned to autograd. w16 is the (pre-scaled) fp16 weight; alpha undoes nothing -- it is the scale the forward applied to the fp32 weight (DOWN_SHRINK), i.e. d(w*s)/dw.""" @staticmethod def forward(ctx, x16, w16, acc, alpha: float): ctx.save_for_backward(x16, w16, acc) ctx.alpha = alpha return F.linear(x16, w16) @staticmethod def backward(ctx, dy): x16, w16, acc = ctx.saved_tensors torch.ops.integ.wgrad_acc(dy, x16, acc, ctx.alpha) return dy @ w16, None, None, None # --------------------------------------------------------------------------- # # 2. fused residual-add + RMSNorm backward with the norm-weight gradient # --------------------------------------------------------------------------- # @triton.jit def _norm_bwd_bw_kernel(h, a, rstd, w, gres, gx, dh, dh16, dwp, ROWS: tl.constexpr, D: tl.constexpr, HAS_A: tl.constexpr, OSCALE: tl.constexpr): """Row-for-row the arithmetic of Inductor's live norm-backward row kernel (triton_per_fused__to_copy_add_div_mul_pow_sum_view_*: one 512-wide row as a 1-D block, 4 warps, num_stages 1, the same op DAG and so the same FMA contractions), so dh and dh16 are BITWISE the live ones and every other gradient stays bitwise too. The only change: the norm-weight gradient sum_rows(gx * (h1*rstd)) -- the same per-element products as Inductor's separate reduction kernel -- is accumulated here in fp32 over ROWS rows (partials then summed in fp64), instead of by a second pass that re-reads gx, h and a. d = g_res + (gx*w)*rstd + ((sum(gx*w*h1)*-0.5)*rstd^3/D) * (2*h1)""" pid = tl.program_id(0).to(tl.int64) cols = tl.arange(0, D) wv = tl.load(w + cols, eviction_policy="evict_last") acc = tl.zeros([D], tl.float32) for i in range(ROWS): r = pid * ROWS + i off = r * D + cols g = tl.load(gx + off).to(tl.float32) hv = tl.load(h + off) if HAS_A: hv = hv + tl.load(a + off).to(tl.float32) gr = tl.load(gres + off) rs = tl.load(rstd + r) gw = g * wv sm = tl.sum(gw * hv, 0) t = gr + gw * rs k = ((sm * -0.5) * (rs * rs * rs)) * (1.0 / D) d = t + k * (hv * 2.0) tl.store(dh + off, d) if OSCALE == 1.0: tl.store(dh16 + off, d.to(tl.float16)) else: tl.store(dh16 + off, (d * OSCALE).to(tl.float16)) acc += g * (hv * rs) tl.store(dwp + pid * D + cols, acc) @triton.jit def _colsum64_kernel(p, out, NR, D: tl.constexpr, BC: tl.constexpr, BR: tl.constexpr): """Stage 1 of an fp64 column sum: out[k, c] = sum of rows k*BR..k*BR+BR-1 of p (fp32 in, fp64 accumulate and out). Grid (D // BC, cdiv(NR, BR)).""" c = tl.program_id(0) * BC + tl.arange(0, BC) k = tl.program_id(1) r = k * BR + tl.arange(0, BR) x = tl.load(p + r[:, None] * D + c[None, :], mask=r[:, None] < NR, other=0.0) tl.store(out + k * D + c, tl.sum(x.to(tl.float64), 0)) def _colsum64(p): """fp32 (NR, D) -> fp32 (D,) column sums with an fp64 accumulator (deterministic).""" NR, D = p.shape BR = 64 nk = triton.cdiv(NR, BR) part = torch.empty((nk, D), device=p.device, dtype=torch.float64) wrap_triton(_colsum64_kernel)[(D // 64, nk)](p, part, NR, D=D, BC=64, BR=BR, num_warps=4) return part.sum(0).to(F32) _NB_ROWS = 16 # rows per program: fp32 dw partial per 16 rows, partials summed in fp64 def _norm_bwd_launch(h, a, rstd, w, gres, gx, oscale: float, has_a: bool): N, D = h.shape dh = torch.empty_like(h) dh16 = torch.empty((N, D), device=h.device, dtype=F16) dwp = torch.empty((N // _NB_ROWS, D), device=h.device, dtype=F32) wrap_triton(_norm_bwd_bw_kernel)[(N // _NB_ROWS,)]( h, a, rstd, w, gres, gx, dh, dh16, dwp, ROWS=_NB_ROWS, D=D, HAS_A=has_a, OSCALE=oscale, num_warps=4, num_stages=1) return dh, dh16, _colsum64(dwp) @triton_op("integ::norm_bwd_a", mutates_args={}) def norm_bwd_a(h: torch.Tensor, a: torch.Tensor, rstd: torch.Tensor, w: torch.Tensor, gres: torch.Tensor, gx: torch.Tensor, oscale: float ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: return _norm_bwd_launch(h, a, rstd, w, gres, gx, oscale, True) @triton_op("integ::norm_bwd_h", mutates_args={}) def norm_bwd_h(h: torch.Tensor, rstd: torch.Tensor, w: torch.Tensor, gres: torch.Tensor, gx: torch.Tensor, oscale: float ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: return _norm_bwd_launch(h, h, rstd, w, gres, gx, oscale, False) class AddNorm(torch.autograd.Function): """(h1, x16) = (h + s*a.float(), fp16(rmsnorm(h1) * w)). Forward: the same torch ops as the live region, so Inductor emits the same fused kernel (bitwise). Backward: one Triton pass (_norm_bwd_bw_kernel) that reproduces the live row kernel bit for bit -- dh1 (fp32, to the residual) and da16 = fp16(s*dh1) (to the o_proj / down GEMM backward) -- and also accumulates the norm-weight gradient sum_rows(gx*xhat), which the live graph computes in a second kernel re-reading gx, h and a. save_out=False saves (h, a) -- the tensors the live graph already keeps for this norm (h is the previous region's output, a the o_proj output). save_out=True saves h1 instead -- for the next block's norm1, where h1 is this region's output and is kept anyway as the next region's input. """ @staticmethod def forward(ctx, h, a, w, eps: float, s: float, save_out: bool): h1 = h + a.float() if s == 1.0 else h + a.float() * s rstd = torch.rsqrt(h1.pow(2).mean(-1, keepdim=True) + eps) x16 = (h1 * rstd * w.float()).to(F16) if save_out: ctx.save_for_backward(h1, rstd, w) else: ctx.save_for_backward(h, a, rstd, w) ctx.s, ctx.save_out = s, save_out return h1, x16 @staticmethod def backward(ctx, gh1, gx): D = gx.shape[-1] if ctx.save_out: h1, rstd, w = ctx.saved_tensors shp = h1.shape dh, da, dw = torch.ops.integ.norm_bwd_h( h1.reshape(-1, D), rstd.reshape(-1), w, gh1.reshape(-1, D).contiguous(), gx.reshape(-1, D).contiguous(), ctx.s) else: h, a, rstd, w = ctx.saved_tensors shp = h.shape dh, da, dw = torch.ops.integ.norm_bwd_a( h.reshape(-1, D), a.reshape(-1, D), rstd.reshape(-1), w, gh1.reshape(-1, D).contiguous(), gx.reshape(-1, D).contiguous(), ctx.s) return dh.view(shp), da.view(shp), dw, None, None, None # --------------------------------------------------------------------------- # # 4. tied embedding: acc + (head dW * s + embedding segment sums) in one pass # --------------------------------------------------------------------------- # @triton.jit def _tied_accum_dense(acc_ptr, dw_ptr, s_ptr, seg_ptr, out_ptr, n, HAS_ACC: tl.constexpr, BLOCK: tl.constexpr): offs = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK) m = offs < n s = tl.load(s_ptr) v = tl.load(dw_ptr + offs, mask=m) * s + tl.load(seg_ptr + offs, mask=m) if HAS_ACC: v = tl.load(acc_ptr + offs, mask=m) + v tl.store(out_ptr + offs, v, mask=m) _STASH: dict = {} def stash_head_grad(W, dW, s): """Called by the head's backward in place of returning dW*s for W.""" _STASH[id(W)] = (dW, s) class TiedEmbedding(torch.autograd.Function): """F.embedding(idx, W) for the tied weight. Its backward adds (head dW * s) + segment sums into W.grad in one pass -- the exact rounding sequence of the stock path (dW*s rounded, + seg, + acc; FMA contraction off), in place of: the dW*s pass, autograd's tied-weight sum and AccumulateGrad's add_. Requires the head to stash its dW (stash=True).""" @staticmethod def forward(ctx, idx, W): ctx.save_for_backward(idx) ctx.W = W return F.embedding(idx, W) @staticmethod def backward(ctx, g): (idx,) = ctx.saved_tensors W = ctx.W dW, s = _STASH.pop(id(W)) seg = torch.ops.aten.embedding_dense_backward( g.reshape(-1, g.shape[-1]), idx.reshape(-1), W.shape[0], -1, False) has = W.grad is not None out = W.grad if has else seg # first micro-step: in place into seg n = W.numel() BLOCK = 2048 _tied_accum_dense[(triton.cdiv(n, BLOCK),)]( out, dW, s.reshape(1).float(), seg, out, n, HAS_ACC=has, BLOCK=BLOCK, num_warps=4, enable_fp_fusion=False) if not has: W.grad = out return None, None