"""Canon + RoPE in CUDA: a sliding window down the sequence (drop-in for canon_rope.py). Written by a second Claude Code reviewer on 2026-09-27; adopted after review. Each thread owns 8 channels and walks 64 (fwd) / 128 (bwd, backwards) tokens, keeping the two previous tokens in registers instead of re-reading neighbours as the Triton version does, with 16-byte loads and stores throughout. Measured at full size (31 layers, mb 16, idle V100): fwd 0.61 -> 0.32 ms, bwd 1.20 -> 0.57 ms, whole micro-step +4.1%. q, k, v, dqkv and the loss are bitwise-identical to the Triton kernel (its RoPE rounding order is copied: fmaf(y, c, -(p*s))); the Canon weight gradient differs only in fp32 summation order (same error vs fp64). Requires T % 128 == 0 (2048 and 8192 both are). """ import os, torch from torch.utils.cpp_extension import load_inline SRC = r""" #include #include #include constexpr int FCH = 64; // tokens per block (sliding window down the sequence) __global__ void __launch_bounds__(192) canon_rope_fwd( const half* __restrict__ qkv, const float* __restrict__ w, const float* __restrict__ m1, const float* __restrict__ m2, const float* __restrict__ cs, const float* __restrict__ sn, half* __restrict__ q, half* __restrict__ k, half* __restrict__ v, int T) { const int c8 = threadIdx.x, ch = 8 * c8, g = ch >> 9, col = ch & 511, d0 = col & 63; const long n0 = (long)blockIdx.x * FCH; const int t0 = n0 % T; float w0[8], w1[8], w2[8]; #pragma unroll for (int i = 0; i < 8; ++i) { // the region casts canon_w to fp16 w0[i] = __half2float(__float2half(w[(ch + i) * 3 + 0])); w1[i] = __half2float(__float2half(w[(ch + i) * 3 + 1])); w2[i] = __half2float(__float2half(w[(ch + i) * 3 + 2])); } auto ld = [&](long n, float* x) { uint4 u = *reinterpret_cast(qkv + n * 1536 + ch); const __half2* h = reinterpret_cast(&u); #pragma unroll for (int i = 0; i < 4; ++i) { float2 f = __half22float2(h[i]); x[2*i] = f.x; x[2*i+1] = f.y; } }; float x1[8], x2[8], x0[8]; #pragma unroll for (int i = 0; i < 8; ++i) x1[i] = x2[i] = 0.f; if (t0 >= 1) ld(n0 - 1, x1); if (t0 >= 2) ld(n0 - 2, x2); half* out = g == 0 ? q : (g == 1 ? k : v); #pragma unroll 8 for (int s = 0; s < FCH; ++s) { const long n = n0 + s; ld(n, x0); const float a1 = m1[n], a2 = m2[n]; float y[8]; #pragma unroll for (int i = 0; i < 8; ++i) y[i] = x0[i] + x0[i] * w0[i] + (x1[i] * a1) * w1[i] + (x2[i] * a2) * w2[i]; float o[8]; if (g < 2) { const int d = d0 & 31; const float4* cp = reinterpret_cast(cs + n * 32 + d); const float4* sp = reinterpret_cast(sn + n * 32 + d); float4 c0 = cp[0], c1 = cp[1], s0 = sp[0], s1 = sp[1]; float c[8] = {c0.x, c0.y, c0.z, c0.w, c1.x, c1.y, c1.z, c1.w}; float sv[8] = {s0.x, s0.y, s0.z, s0.w, s1.x, s1.y, s1.z, s1.w}; #pragma unroll for (int i = 0; i < 8; ++i) { float p = __shfl_xor_sync(0xffffffff, y[i], 4); // partner channel d +/- 32 o[i] = d0 < 32 ? fmaf(y[i], c[i], -(p * sv[i])) : fmaf(y[i], c[i], p * sv[i]); } } else { #pragma unroll for (int i = 0; i < 8; ++i) o[i] = y[i]; } uint4 u; __half2* h = reinterpret_cast<__half2*>(&u); #pragma unroll for (int i = 0; i < 4; ++i) h[i] = __floats2half2_rn(o[2*i], o[2*i+1]); *reinterpret_cast(out + n * 512 + col) = u; #pragma unroll for (int i = 0; i < 8; ++i) { x2[i] = x1[i]; x1[i] = x0[i]; } } } std::vector fwd(torch::Tensor qkv, torch::Tensor w, torch::Tensor m1, torch::Tensor m2, torch::Tensor cs, torch::Tensor sn) { const int B = qkv.size(0), T = qkv.size(1); const long N = (long)B * T; TORCH_CHECK(T % FCH == 0); auto q = torch::empty({B, T, 8, 64}, qkv.options()), k = torch::empty_like(q), v = torch::empty_like(q); canon_rope_fwd<<>>( (const half*)qkv.data_ptr(), w.data_ptr(), m1.data_ptr(), m2.data_ptr(), cs.data_ptr(), sn.data_ptr(), (half*)q.data_ptr(), (half*)k.data_ptr(), (half*)v.data_ptr(), T); return {q, k, v}; } constexpr int CH = 128; __device__ __forceinline__ void ld8(const half* p, float* x) { uint4 u = *reinterpret_cast(p); const __half2* h = reinterpret_cast(&u); #pragma unroll for (int i = 0; i < 4; ++i) { float2 f = __half22float2(h[i]); x[2*i] = f.x; x[2*i+1] = f.y; } } // Q32: the q-gradient arrives as the attention backward's raw fp32 dQ // accumulator (times qscale) instead of an fp16 tensor, so it is never rounded. template __global__ void __launch_bounds__(192) canon_rope_bwd( const half* __restrict__ qkv, const float* __restrict__ w, const float* __restrict__ m1, const float* __restrict__ m2, const float* __restrict__ cs, const float* __restrict__ sn, const void* __restrict__ gq_, const half* __restrict__ gk, const half* __restrict__ gv, half* __restrict__ dqkv, float* __restrict__ part, int T, float qscale) { const half* gq = reinterpret_cast(gq_); const float* gq32 = reinterpret_cast(gq_); const int c8 = threadIdx.x, ch = 8 * c8, g = ch >> 9, col = ch & 511, d0 = col & 63; const long n0 = (long)blockIdx.x * CH; const int t0 = n0 % T; const half* G = g == 0 ? gq : (g == 1 ? gk : gv); float w0[8], w1[8], w2[8]; #pragma unroll for (int i = 0; i < 8; ++i) { w0[i] = __half2float(__float2half(w[(ch + i) * 3 + 0])); w1[i] = __half2float(__float2half(w[(ch + i) * 3 + 1])); w2[i] = __half2float(__float2half(w[(ch + i) * 3 + 2])); } // dy(n): gradient w.r.t. the Canon output at token n, un-rotated. auto dy = [&](long n, float* out) { float gg[8]; if (Q32 && g == 0) { const float4* p4 = reinterpret_cast(gq32 + n * 512 + col); float4 a = p4[0], b = p4[1]; gg[0] = a.x * qscale; gg[1] = a.y * qscale; gg[2] = a.z * qscale; gg[3] = a.w * qscale; gg[4] = b.x * qscale; gg[5] = b.y * qscale; gg[6] = b.z * qscale; gg[7] = b.w * qscale; } else { ld8(G + n * 512 + col, gg); } if (g < 2) { const int d = d0 & 31; const float4* cp = reinterpret_cast(cs + n * 32 + d); const float4* sp = reinterpret_cast(sn + n * 32 + d); float4 c0 = cp[0], c1 = cp[1], s0 = sp[0], s1 = sp[1]; float cs_[8] = {c0.x, c0.y, c0.z, c0.w, c1.x, c1.y, c1.z, c1.w}; float sn_[8] = {s0.x, s0.y, s0.z, s0.w, s1.x, s1.y, s1.z, s1.w}; #pragma unroll for (int i = 0; i < 8; ++i) { float p = __shfl_xor_sync(0xffffffff, gg[i], 4); float ga = d0 < 32 ? gg[i] : p, gb = d0 < 32 ? p : gg[i]; out[i] = d0 < 32 ? fmaf(ga, cs_[i], gb * sn_[i]) : fmaf(gb, cs_[i], -(ga * sn_[i])); } } else { #pragma unroll for (int i = 0; i < 8; ++i) out[i] = gg[i]; } }; const long nl = n0 + CH - 1; // walk from the chunk's last token backwards const int tl = t0 + CH - 1; float d0v[8], d1v[8], d2v[8], x0[8], xm1[8], xm2[8]; float mm1n = 0.f, mm2n = 0.f, m2p1 = 0.f; // m2p1 = m2[n+1] #pragma unroll for (int i = 0; i < 8; ++i) d1v[i] = d2v[i] = 0.f; // dy at n+1, n+2 must be computed by every lane (shuffles), masked afterwards. { float tmp[8]; long a = (tl + 1 < T) ? nl + 1 : nl; dy(a, tmp); if (tl + 1 < T) { mm1n = m1[nl + 1]; m2p1 = m2[nl + 1]; #pragma unroll for (int i = 0; i < 8; ++i) d1v[i] = tmp[i]; } long b = (tl + 2 < T) ? nl + 2 : nl; dy(b, tmp); if (tl + 2 < T) { mm2n = m2[nl + 2]; #pragma unroll for (int i = 0; i < 8; ++i) d2v[i] = tmp[i]; } } ld8(qkv + nl * 1536 + ch, x0); ld8(qkv + (nl - 1) * 1536 + ch, xm1); // tl >= 31 so both exist ld8(qkv + (nl - 2) * 1536 + ch, xm2); float a0[8], a1[8], a2[8]; #pragma unroll for (int i = 0; i < 8; ++i) a0[i] = a1[i] = a2[i] = 0.f; #pragma unroll 8 for (int s = CH - 1; s >= 0; --s) { const long n = n0 + s; const int t = t0 + s; dy(n, d0v); const float m1n = m1[n], m2n = m2[n]; float dx[8]; #pragma unroll for (int i = 0; i < 8; ++i) { dx[i] = d0v[i] + d0v[i] * w0[i] + (d1v[i] * mm1n) * w1[i] + (d2v[i] * mm2n) * w2[i]; a0[i] += d0v[i] * x0[i]; a1[i] += d0v[i] * (xm1[i] * m1n); a2[i] += d0v[i] * (xm2[i] * m2n); } uint4 u; __half2* h = reinterpret_cast<__half2*>(&u); #pragma unroll for (int i = 0; i < 4; ++i) h[i] = __floats2half2_rn(dx[2*i], dx[2*i+1]); *reinterpret_cast(dqkv + n * 1536 + ch) = u; mm2n = m2p1; m2p1 = m2n; mm1n = m1n; #pragma unroll for (int i = 0; i < 8; ++i) { d2v[i] = d1v[i]; d1v[i] = d0v[i]; x0[i] = xm1[i]; xm1[i] = xm2[i]; } if (s > 0) { if (t - 3 >= 0) ld8(qkv + (n - 3) * 1536 + ch, xm2); else { #pragma unroll for (int i = 0; i < 8; ++i) xm2[i] = 0.f; } if (t - 1 < 1) { // next token (t-1) is t==0: its t-1 tap is outside the row #pragma unroll for (int i = 0; i < 8; ++i) xm1[i] = 0.f; } } } float* pp = part + (long)blockIdx.x * 1536 * 3 + ch * 3; #pragma unroll for (int i = 0; i < 8; ++i) { pp[3*i] = a0[i]; pp[3*i+1] = a1[i]; pp[3*i+2] = a2[i]; } } std::vector bwd(torch::Tensor qkv, torch::Tensor w, torch::Tensor m1, torch::Tensor m2, torch::Tensor cs, torch::Tensor sn, torch::Tensor gq, torch::Tensor gk, torch::Tensor gv) { const int B = qkv.size(0), T = qkv.size(1); const long N = (long)B * T; auto dqkv = torch::empty_like(qkv); auto part = torch::empty({N / CH, 1536, 3}, qkv.options().dtype(torch::kFloat)); canon_rope_bwd<<>>( (const half*)qkv.data_ptr(), w.data_ptr(), m1.data_ptr(), m2.data_ptr(), cs.data_ptr(), sn.data_ptr(), gq.data_ptr(), (const half*)gk.data_ptr(), (const half*)gv.data_ptr(), (half*)dqkv.data_ptr(), part.data_ptr(), T, 1.f); C10_CUDA_KERNEL_LAUNCH_CHECK(); return {dqkv, part}; } // gq32: (B, T, 8, 64) fp32 raw dQ accumulator; qscale folds in the softmax scale. std::vector bwd_q32(torch::Tensor qkv, torch::Tensor w, torch::Tensor m1, torch::Tensor m2, torch::Tensor cs, torch::Tensor sn, torch::Tensor gq32, torch::Tensor gk, torch::Tensor gv, double qscale) { TORCH_CHECK(gq32.scalar_type() == at::kFloat && gq32.is_contiguous(), "gq32 must be contiguous fp32"); const int B = qkv.size(0), T = qkv.size(1); const long N = (long)B * T; TORCH_CHECK(T % CH == 0, "T must be a multiple of 128"); auto dqkv = torch::empty_like(qkv); auto part = torch::empty({N / CH, 1536, 3}, qkv.options().dtype(torch::kFloat)); canon_rope_bwd<<>>( (const half*)qkv.data_ptr(), w.data_ptr(), m1.data_ptr(), m2.data_ptr(), cs.data_ptr(), sn.data_ptr(), gq32.data_ptr(), (const half*)gk.data_ptr(), (const half*)gv.data_ptr(), (half*)dqkv.data_ptr(), part.data_ptr(), T, (float)qscale); C10_CUDA_KERNEL_LAUNCH_CHECK(); return {dqkv, part}; } """ _here = os.path.dirname(os.path.abspath(__file__)); os.makedirs(os.path.join(_here, "build_canon"), exist_ok=True) os.environ.setdefault("TORCH_CUDA_ARCH_LIST", "7.0") ext = load_inline("boris_volta_canon", cpp_sources= "std::vector fwd(torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor);" "std::vector bwd(torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor);" "std::vector bwd_q32(torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, double);", cuda_sources=SRC, functions=["fwd", "bwd", "bwd_q32"], extra_cuda_cflags=["-O3", "-arch=sm_70"], build_directory=os.path.join(_here, "build_canon")) class CanonRopeFast(torch.autograd.Function): @staticmethod def forward(ctx, qkv, canon_w, m1, m2, cos, sin): B, T, C = qkv.shape; N = B * T assert C == 1536 and qkv.dtype == torch.float16 and qkv.is_contiguous() and T % 128 == 0 m1f, m2f = m1.reshape(N).contiguous(), m2.reshape(N).contiguous() cf, sf = cos.reshape(N, 32).contiguous(), sin.reshape(N, 32).contiguous() q, k, v = ext.fwd(qkv, canon_w.contiguous(), m1f, m2f, cf, sf) ctx.save_for_backward(qkv, canon_w, m1f, m2f, cf, sf) return q, k, v @staticmethod def backward(ctx, gq, gk, gv): qkv, w, m1f, m2f, cf, sf = ctx.saved_tensors gq, gk, gv = (g.to(torch.float16).contiguous() for g in (gq, gk, gv)) dqkv, part = ext.bwd(qkv, w.contiguous(), m1f, m2f, cf, sf, gq, gk, gv) # fp32 straight from the fp32 reduction. Rounding through fp16 (as the # eager region's cast implied) left ~3% of entries subnormal on the live # checkpoint; keeping fp32 is strictly more precise. return dqkv, part.sum(0).to(w.dtype), None, None, None, None def canon_rope(qkv, canon_w, m1, m2, cos, sin): return CanonRopeFast.apply(qkv, canon_w, m1, m2, cos, sin)