"""Fused Canon + RoPE for the Volta path: one Triton pass forward, one backward. What it replaces, inside the compiled _pre_attn region (Inductor's choices): forward constant_pad_nd x2 materialise zero-padded fp32 copies of qkv add_mul the 3-tap combine, reading those copies to_copy_cat RoPE on q and k, cast to fp16 backward slice_backward / pad backward / a 1536x3 weight-grad reduction, RoPE backward, as separate passes over (N, 1536) Semantics are the region's, operation for operation: w = fp16(canon_w) (the region casts it to qkv's dtype) y[n] = x[n] + x[n]*w0 + (x[n-1]*m1[n])*w1 + (x[n-2]*m2[n])*w2 fp32 q, k = rope(y) in fp32, then fp16; v = fp16(y) m1/m2 are the per-token document masks (0 where t-1 / t-2 is in another document) and the shifts never cross a packed row. The backward is the exact transpose; the weight gradient is reduced in fp32 and rounded to fp16 before widening, as the gradient of the region's fp16 cast of canon_w is. """ from __future__ import annotations import torch import triton import triton.language as tl C = 1536 # q | k | v channels HALF = 32 # RoPE pairs (d, d+32) within a 64-wide head @triton.jit def _canon_half(qkv, w, rows, ch, mm1, mm2, ok1, ok2): x0 = tl.load(qkv + rows).to(tl.float32) x1 = tl.load(qkv + rows - 1536, mask=ok1, other=0.0).to(tl.float32) x2 = tl.load(qkv + rows - 3072, mask=ok2, other=0.0).to(tl.float32) w0 = tl.load(w + ch * 3).to(tl.float16).to(tl.float32)[None, :] w1 = tl.load(w + ch * 3 + 1).to(tl.float16).to(tl.float32)[None, :] w2 = tl.load(w + ch * 3 + 2).to(tl.float16).to(tl.float32)[None, :] return x0 + x0 * w0 + (x1 * mm1) * w1 + (x2 * mm2) * w2 @triton.jit def _fwd(qkv, w, m1, m2, cos, sin, out, T, GROUP: tl.constexpr, TB: tl.constexpr, HALF: tl.constexpr): """TB tokens x one 64-wide head of one group (0 q, 1 k, 2 v).""" pid_n = tl.program_id(0) h = tl.program_id(1) n = pid_n * TB + tl.arange(0, TB) d = tl.arange(0, HALF) t = n % T ca = GROUP * 512 + h * 64 + d cb = ca + HALF ok1 = (t >= 1)[:, None] ok2 = (t >= 2)[:, None] mm1 = tl.load(m1 + n)[:, None] mm2 = tl.load(m2 + n)[:, None] ya = _canon_half(qkv, w, n[:, None] * 1536 + ca[None, :], ca, mm1, mm2, ok1, ok2) yb = _canon_half(qkv, w, n[:, None] * 1536 + cb[None, :], cb, mm1, mm2, ok1, ok2) orow = n[:, None] * 512 + h * 64 + d[None, :] if GROUP < 2: cs = tl.load(cos + n[:, None] * HALF + d[None, :]) sn = tl.load(sin + n[:, None] * HALF + d[None, :]) oa = ya * cs - yb * sn ob = ya * sn + yb * cs else: oa = ya ob = yb tl.store(out + orow, oa.to(tl.float16)) tl.store(out + orow + HALF, ob.to(tl.float16)) @triton.jit def _dy(src, cos, sin, n, t, S: tl.constexpr, d, ocol, T, ROPE: tl.constexpr, HALF: tl.constexpr): """Gradient w.r.t. the Canon output at token n+S, un-rotated; 0 past the row.""" nn = n + S ok = ((t + S) < T)[:, None] ga = tl.load(src + nn[:, None] * 512 + ocol, mask=ok, other=0.0).to(tl.float32) gb = tl.load(src + nn[:, None] * 512 + ocol + HALF, mask=ok, other=0.0).to(tl.float32) if ROPE: cs = tl.load(cos + nn[:, None] * HALF + d[None, :], mask=ok, other=0.0) sn = tl.load(sin + nn[:, None] * HALF + d[None, :], mask=ok, other=0.0) return ga * cs + gb * sn, gb * cs - ga * sn return ga, gb @triton.jit def _bwd_half(qkv, w, dqkv, dw_part, pid_n, n, ch, dy0, dy1, dy2, mm1, mm2, mm1n, mm2n, p1, p2): w0 = tl.load(w + ch * 3).to(tl.float16).to(tl.float32)[None, :] w1 = tl.load(w + ch * 3 + 1).to(tl.float16).to(tl.float32)[None, :] w2 = tl.load(w + ch * 3 + 2).to(tl.float16).to(tl.float32)[None, :] rows = n[:, None] * 1536 + ch[None, :] # dx[n] = dy[n](1 + w0) + dy[n+1] m1[n+1] w1 + dy[n+2] m2[n+2] w2 dx = dy0 + dy0 * w0 + (dy1 * mm1n) * w1 + (dy2 * mm2n) * w2 tl.store(dqkv + rows, dx.to(tl.float16)) x0 = tl.load(qkv + rows).to(tl.float32) x1 = tl.load(qkv + rows - 1536, mask=p1, other=0.0).to(tl.float32) x2 = tl.load(qkv + rows - 3072, mask=p2, other=0.0).to(tl.float32) base = dw_part + pid_n * (1536 * 3) + ch * 3 tl.store(base, tl.sum(dy0 * x0, 0)) tl.store(base + 1, tl.sum(dy0 * (x1 * mm1), 0)) tl.store(base + 2, tl.sum(dy0 * (x2 * mm2), 0)) @triton.jit def _bwd(qkv, w, m1, m2, cos, sin, src, dqkv, dw_part, T, GROUP: tl.constexpr, TB: tl.constexpr, HALF: tl.constexpr): pid_n = tl.program_id(0) h = tl.program_id(1) n = pid_n * TB + tl.arange(0, TB) d = tl.arange(0, HALF) t = n % T ca = GROUP * 512 + h * 64 + d cb = ca + HALF ocol = h * 64 + d[None, :] ROPE: tl.constexpr = GROUP < 2 dya0, dyb0 = _dy(src, cos, sin, n, t, 0, d, ocol, T, ROPE, HALF) dya1, dyb1 = _dy(src, cos, sin, n, t, 1, d, ocol, T, ROPE, HALF) dya2, dyb2 = _dy(src, cos, sin, n, t, 2, d, ocol, T, ROPE, HALF) mm1n = tl.load(m1 + n + 1, mask=(t + 1) < T, other=0.0)[:, None] mm2n = tl.load(m2 + n + 2, mask=(t + 2) < T, other=0.0)[:, None] mm1 = tl.load(m1 + n)[:, None] mm2 = tl.load(m2 + n)[:, None] p1 = (t >= 1)[:, None] p2 = (t >= 2)[:, None] _bwd_half(qkv, w, dqkv, dw_part, pid_n, n, ca, dya0, dya1, dya2, mm1, mm2, mm1n, mm2n, p1, p2) _bwd_half(qkv, w, dqkv, dw_part, pid_n, n, cb, dyb0, dyb1, dyb2, mm1, mm2, mm1n, mm2n, p1, p2) TB = 32 class CanonRope(torch.autograd.Function): @staticmethod def forward(ctx, qkv, canon_w, m1, m2, cos, sin): B, T, Cc = qkv.shape assert Cc == C and qkv.dtype == torch.float16 and qkv.is_contiguous() N = B * T assert N % TB == 0 q = torch.empty((B, T, 8, 64), device=qkv.device, dtype=torch.float16) k = torch.empty_like(q) v = torch.empty_like(q) m1f = m1.reshape(N).contiguous() m2f = m2.reshape(N).contiguous() cosf = cos.reshape(N, HALF).contiguous() sinf = sin.reshape(N, HALF).contiguous() for g_, out in enumerate((q, k, v)): _fwd[(N // TB, 8)](qkv, canon_w, m1f, m2f, cosf, sinf, out, T, GROUP=g_, TB=TB, HALF=HALF, num_warps=4) ctx.save_for_backward(qkv, canon_w, m1f, m2f, cosf, sinf) ctx.T = T return q, k, v @staticmethod def backward(ctx, gq, gk, gv): qkv, canon_w, m1f, m2f, cosf, sinf = ctx.saved_tensors B, T, _ = qkv.shape N = B * T gq, gk, gv = (g.to(torch.float16).contiguous() for g in (gq, gk, gv)) dqkv = torch.empty_like(qkv) part = torch.empty((N // TB, C, 3), device=qkv.device, dtype=torch.float32) for g_, src in enumerate((gq, gk, gv)): _bwd[(N // TB, 8)](qkv, canon_w, m1f, m2f, cosf, sinf, src, dqkv, part, T, GROUP=g_, TB=TB, HALF=HALF, num_warps=4) # fp32 reduction, then the fp16 rounding the region's cast of canon_w implies dw = part.sum(0).to(torch.float16).to(canon_w.dtype) return dqkv, dw, None, None, None, None def canon_rope(qkv, canon_w, m1, m2, cos, sin): """qkv (B,T,1536) fp16 -> q, k, v (B,T,8,64) fp16, Canon + RoPE applied.""" return CanonRope.apply(qkv, canon_w, m1, m2, cos, sin)