summaryrefslogtreecommitdiff
path: root/volta_kernels/canon_rope.py
diff options
context:
space:
mode:
Diffstat (limited to 'volta_kernels/canon_rope.py')
-rw-r--r--volta_kernels/canon_rope.py171
1 files changed, 171 insertions, 0 deletions
diff --git a/volta_kernels/canon_rope.py b/volta_kernels/canon_rope.py
new file mode 100644
index 0000000..08b6d8c
--- /dev/null
+++ b/volta_kernels/canon_rope.py
@@ -0,0 +1,171 @@
+"""Fused Canon + RoPE for the Volta path: one Triton pass forward, one backward.
+
+What it replaces, inside the compiled _pre_attn region (Inductor's choices):
+ forward constant_pad_nd x2 materialise zero-padded fp32 copies of qkv
+ add_mul the 3-tap combine, reading those copies
+ to_copy_cat RoPE on q and k, cast to fp16
+ backward slice_backward / pad backward / a 1536x3 weight-grad reduction,
+ RoPE backward, as separate passes over (N, 1536)
+
+Semantics are the region's, operation for operation:
+ w = fp16(canon_w) (the region casts it to qkv's dtype)
+ y[n] = x[n] + x[n]*w0 + (x[n-1]*m1[n])*w1 + (x[n-2]*m2[n])*w2 fp32
+ q, k = rope(y) in fp32, then fp16; v = fp16(y)
+m1/m2 are the per-token document masks (0 where t-1 / t-2 is in another
+document) and the shifts never cross a packed row. The backward is the exact
+transpose; the weight gradient is reduced in fp32 and rounded to fp16 before
+widening, as the gradient of the region's fp16 cast of canon_w is.
+"""
+from __future__ import annotations
+
+import torch
+import triton
+import triton.language as tl
+
+C = 1536 # q | k | v channels
+HALF = 32 # RoPE pairs (d, d+32) within a 64-wide head
+
+
+@triton.jit
+def _canon_half(qkv, w, rows, ch, mm1, mm2, ok1, ok2):
+ x0 = tl.load(qkv + rows).to(tl.float32)
+ x1 = tl.load(qkv + rows - 1536, mask=ok1, other=0.0).to(tl.float32)
+ x2 = tl.load(qkv + rows - 3072, mask=ok2, other=0.0).to(tl.float32)
+ w0 = tl.load(w + ch * 3).to(tl.float16).to(tl.float32)[None, :]
+ w1 = tl.load(w + ch * 3 + 1).to(tl.float16).to(tl.float32)[None, :]
+ w2 = tl.load(w + ch * 3 + 2).to(tl.float16).to(tl.float32)[None, :]
+ return x0 + x0 * w0 + (x1 * mm1) * w1 + (x2 * mm2) * w2
+
+
+@triton.jit
+def _fwd(qkv, w, m1, m2, cos, sin, out, T, GROUP: tl.constexpr,
+ TB: tl.constexpr, HALF: tl.constexpr):
+ """TB tokens x one 64-wide head of one group (0 q, 1 k, 2 v)."""
+ pid_n = tl.program_id(0)
+ h = tl.program_id(1)
+ n = pid_n * TB + tl.arange(0, TB)
+ d = tl.arange(0, HALF)
+ t = n % T
+ ca = GROUP * 512 + h * 64 + d
+ cb = ca + HALF
+ ok1 = (t >= 1)[:, None]
+ ok2 = (t >= 2)[:, None]
+ mm1 = tl.load(m1 + n)[:, None]
+ mm2 = tl.load(m2 + n)[:, None]
+ ya = _canon_half(qkv, w, n[:, None] * 1536 + ca[None, :], ca, mm1, mm2, ok1, ok2)
+ yb = _canon_half(qkv, w, n[:, None] * 1536 + cb[None, :], cb, mm1, mm2, ok1, ok2)
+ orow = n[:, None] * 512 + h * 64 + d[None, :]
+ if GROUP < 2:
+ cs = tl.load(cos + n[:, None] * HALF + d[None, :])
+ sn = tl.load(sin + n[:, None] * HALF + d[None, :])
+ oa = ya * cs - yb * sn
+ ob = ya * sn + yb * cs
+ else:
+ oa = ya
+ ob = yb
+ tl.store(out + orow, oa.to(tl.float16))
+ tl.store(out + orow + HALF, ob.to(tl.float16))
+
+
+@triton.jit
+def _dy(src, cos, sin, n, t, S: tl.constexpr, d, ocol, T, ROPE: tl.constexpr,
+ HALF: tl.constexpr):
+ """Gradient w.r.t. the Canon output at token n+S, un-rotated; 0 past the row."""
+ nn = n + S
+ ok = ((t + S) < T)[:, None]
+ ga = tl.load(src + nn[:, None] * 512 + ocol, mask=ok, other=0.0).to(tl.float32)
+ gb = tl.load(src + nn[:, None] * 512 + ocol + HALF, mask=ok, other=0.0).to(tl.float32)
+ if ROPE:
+ cs = tl.load(cos + nn[:, None] * HALF + d[None, :], mask=ok, other=0.0)
+ sn = tl.load(sin + nn[:, None] * HALF + d[None, :], mask=ok, other=0.0)
+ return ga * cs + gb * sn, gb * cs - ga * sn
+ return ga, gb
+
+
+@triton.jit
+def _bwd_half(qkv, w, dqkv, dw_part, pid_n, n, ch, dy0, dy1, dy2,
+ mm1, mm2, mm1n, mm2n, p1, p2):
+ w0 = tl.load(w + ch * 3).to(tl.float16).to(tl.float32)[None, :]
+ w1 = tl.load(w + ch * 3 + 1).to(tl.float16).to(tl.float32)[None, :]
+ w2 = tl.load(w + ch * 3 + 2).to(tl.float16).to(tl.float32)[None, :]
+ rows = n[:, None] * 1536 + ch[None, :]
+ # dx[n] = dy[n](1 + w0) + dy[n+1] m1[n+1] w1 + dy[n+2] m2[n+2] w2
+ dx = dy0 + dy0 * w0 + (dy1 * mm1n) * w1 + (dy2 * mm2n) * w2
+ tl.store(dqkv + rows, dx.to(tl.float16))
+ x0 = tl.load(qkv + rows).to(tl.float32)
+ x1 = tl.load(qkv + rows - 1536, mask=p1, other=0.0).to(tl.float32)
+ x2 = tl.load(qkv + rows - 3072, mask=p2, other=0.0).to(tl.float32)
+ base = dw_part + pid_n * (1536 * 3) + ch * 3
+ tl.store(base, tl.sum(dy0 * x0, 0))
+ tl.store(base + 1, tl.sum(dy0 * (x1 * mm1), 0))
+ tl.store(base + 2, tl.sum(dy0 * (x2 * mm2), 0))
+
+
+@triton.jit
+def _bwd(qkv, w, m1, m2, cos, sin, src, dqkv, dw_part, T, GROUP: tl.constexpr,
+ TB: tl.constexpr, HALF: tl.constexpr):
+ pid_n = tl.program_id(0)
+ h = tl.program_id(1)
+ n = pid_n * TB + tl.arange(0, TB)
+ d = tl.arange(0, HALF)
+ t = n % T
+ ca = GROUP * 512 + h * 64 + d
+ cb = ca + HALF
+ ocol = h * 64 + d[None, :]
+ ROPE: tl.constexpr = GROUP < 2
+ dya0, dyb0 = _dy(src, cos, sin, n, t, 0, d, ocol, T, ROPE, HALF)
+ dya1, dyb1 = _dy(src, cos, sin, n, t, 1, d, ocol, T, ROPE, HALF)
+ dya2, dyb2 = _dy(src, cos, sin, n, t, 2, d, ocol, T, ROPE, HALF)
+ mm1n = tl.load(m1 + n + 1, mask=(t + 1) < T, other=0.0)[:, None]
+ mm2n = tl.load(m2 + n + 2, mask=(t + 2) < T, other=0.0)[:, None]
+ mm1 = tl.load(m1 + n)[:, None]
+ mm2 = tl.load(m2 + n)[:, None]
+ p1 = (t >= 1)[:, None]
+ p2 = (t >= 2)[:, None]
+ _bwd_half(qkv, w, dqkv, dw_part, pid_n, n, ca, dya0, dya1, dya2, mm1, mm2, mm1n, mm2n, p1, p2)
+ _bwd_half(qkv, w, dqkv, dw_part, pid_n, n, cb, dyb0, dyb1, dyb2, mm1, mm2, mm1n, mm2n, p1, p2)
+
+
+TB = 32
+
+
+class CanonRope(torch.autograd.Function):
+ @staticmethod
+ def forward(ctx, qkv, canon_w, m1, m2, cos, sin):
+ B, T, Cc = qkv.shape
+ assert Cc == C and qkv.dtype == torch.float16 and qkv.is_contiguous()
+ N = B * T
+ assert N % TB == 0
+ q = torch.empty((B, T, 8, 64), device=qkv.device, dtype=torch.float16)
+ k = torch.empty_like(q)
+ v = torch.empty_like(q)
+ m1f = m1.reshape(N).contiguous()
+ m2f = m2.reshape(N).contiguous()
+ cosf = cos.reshape(N, HALF).contiguous()
+ sinf = sin.reshape(N, HALF).contiguous()
+ for g_, out in enumerate((q, k, v)):
+ _fwd[(N // TB, 8)](qkv, canon_w, m1f, m2f, cosf, sinf, out, T, GROUP=g_,
+ TB=TB, HALF=HALF, num_warps=4)
+ ctx.save_for_backward(qkv, canon_w, m1f, m2f, cosf, sinf)
+ ctx.T = T
+ return q, k, v
+
+ @staticmethod
+ def backward(ctx, gq, gk, gv):
+ qkv, canon_w, m1f, m2f, cosf, sinf = ctx.saved_tensors
+ B, T, _ = qkv.shape
+ N = B * T
+ gq, gk, gv = (g.to(torch.float16).contiguous() for g in (gq, gk, gv))
+ dqkv = torch.empty_like(qkv)
+ part = torch.empty((N // TB, C, 3), device=qkv.device, dtype=torch.float32)
+ for g_, src in enumerate((gq, gk, gv)):
+ _bwd[(N // TB, 8)](qkv, canon_w, m1f, m2f, cosf, sinf, src, dqkv, part, T,
+ GROUP=g_, TB=TB, HALF=HALF, num_warps=4)
+ # fp32 reduction, then the fp16 rounding the region's cast of canon_w implies
+ dw = part.sum(0).to(torch.float16).to(canon_w.dtype)
+ return dqkv, dw, None, None, None, None
+
+
+def canon_rope(qkv, canon_w, m1, m2, cos, sin):
+ """qkv (B,T,1536) fp16 -> q, k, v (B,T,8,64) fp16, Canon + RoPE applied."""
+ return CanonRope.apply(qkv, canon_w, m1, m2, cos, sin)