diff options
Diffstat (limited to 'volta_kernels/canon_rope_cuda.py')
| -rw-r--r-- | volta_kernels/canon_rope_cuda.py | 266 |
1 files changed, 266 insertions, 0 deletions
diff --git a/volta_kernels/canon_rope_cuda.py b/volta_kernels/canon_rope_cuda.py new file mode 100644 index 0000000..e632212 --- /dev/null +++ b/volta_kernels/canon_rope_cuda.py @@ -0,0 +1,266 @@ +"""Canon + RoPE in CUDA: a sliding window down the sequence (drop-in for canon_rope.py). + +Written by a second Claude Code reviewer on 2026-09-27; adopted after review. +Each thread owns 8 channels and walks 64 (fwd) / 128 (bwd, backwards) tokens, +keeping the two previous tokens in registers instead of re-reading neighbours +as the Triton version does, with 16-byte loads and stores throughout. +Measured at full size (31 layers, mb 16, idle V100): fwd 0.61 -> 0.32 ms, +bwd 1.20 -> 0.57 ms, whole micro-step +4.1%. q, k, v, dqkv and the loss are +bitwise-identical to the Triton kernel (its RoPE rounding order is copied: +fmaf(y, c, -(p*s))); the Canon weight gradient differs only in fp32 summation +order (same error vs fp64). Requires T % 128 == 0 (2048 and 8192 both are). +""" +import os, torch +from torch.utils.cpp_extension import load_inline +SRC = r""" +#include <torch/extension.h> +#include <ATen/cuda/CUDAContext.h> +#include <cuda_fp16.h> +constexpr int FCH = 64; // tokens per block (sliding window down the sequence) +__global__ void __launch_bounds__(192) canon_rope_fwd( + const half* __restrict__ qkv, const float* __restrict__ w, const float* __restrict__ m1, + const float* __restrict__ m2, const float* __restrict__ cs, const float* __restrict__ sn, + half* __restrict__ q, half* __restrict__ k, half* __restrict__ v, int T) { + const int c8 = threadIdx.x, ch = 8 * c8, g = ch >> 9, col = ch & 511, d0 = col & 63; + const long n0 = (long)blockIdx.x * FCH; + const int t0 = n0 % T; + float w0[8], w1[8], w2[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) { // the region casts canon_w to fp16 + w0[i] = __half2float(__float2half(w[(ch + i) * 3 + 0])); + w1[i] = __half2float(__float2half(w[(ch + i) * 3 + 1])); + w2[i] = __half2float(__float2half(w[(ch + i) * 3 + 2])); + } + auto ld = [&](long n, float* x) { + uint4 u = *reinterpret_cast<const uint4*>(qkv + n * 1536 + ch); + const __half2* h = reinterpret_cast<const __half2*>(&u); +#pragma unroll + for (int i = 0; i < 4; ++i) { float2 f = __half22float2(h[i]); x[2*i] = f.x; x[2*i+1] = f.y; } + }; + float x1[8], x2[8], x0[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) x1[i] = x2[i] = 0.f; + if (t0 >= 1) ld(n0 - 1, x1); + if (t0 >= 2) ld(n0 - 2, x2); + half* out = g == 0 ? q : (g == 1 ? k : v); +#pragma unroll 8 + for (int s = 0; s < FCH; ++s) { + const long n = n0 + s; + ld(n, x0); + const float a1 = m1[n], a2 = m2[n]; + float y[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) + y[i] = x0[i] + x0[i] * w0[i] + (x1[i] * a1) * w1[i] + (x2[i] * a2) * w2[i]; + float o[8]; + if (g < 2) { + const int d = d0 & 31; + const float4* cp = reinterpret_cast<const float4*>(cs + n * 32 + d); + const float4* sp = reinterpret_cast<const float4*>(sn + n * 32 + d); + float4 c0 = cp[0], c1 = cp[1], s0 = sp[0], s1 = sp[1]; + float c[8] = {c0.x, c0.y, c0.z, c0.w, c1.x, c1.y, c1.z, c1.w}; + float sv[8] = {s0.x, s0.y, s0.z, s0.w, s1.x, s1.y, s1.z, s1.w}; +#pragma unroll + for (int i = 0; i < 8; ++i) { + float p = __shfl_xor_sync(0xffffffff, y[i], 4); // partner channel d +/- 32 + o[i] = d0 < 32 ? fmaf(y[i], c[i], -(p * sv[i])) : fmaf(y[i], c[i], p * sv[i]); + } + } else { +#pragma unroll + for (int i = 0; i < 8; ++i) o[i] = y[i]; + } + uint4 u; + __half2* h = reinterpret_cast<__half2*>(&u); +#pragma unroll + for (int i = 0; i < 4; ++i) h[i] = __floats2half2_rn(o[2*i], o[2*i+1]); + *reinterpret_cast<uint4*>(out + n * 512 + col) = u; +#pragma unroll + for (int i = 0; i < 8; ++i) { x2[i] = x1[i]; x1[i] = x0[i]; } + } +} +std::vector<torch::Tensor> fwd(torch::Tensor qkv, torch::Tensor w, torch::Tensor m1, torch::Tensor m2, + torch::Tensor cs, torch::Tensor sn) { + const int B = qkv.size(0), T = qkv.size(1); const long N = (long)B * T; + TORCH_CHECK(T % FCH == 0); + auto q = torch::empty({B, T, 8, 64}, qkv.options()), k = torch::empty_like(q), v = torch::empty_like(q); + canon_rope_fwd<<<N / FCH, 192, 0, at::cuda::getCurrentCUDAStream()>>>( + (const half*)qkv.data_ptr(), w.data_ptr<float>(), m1.data_ptr<float>(), m2.data_ptr<float>(), + cs.data_ptr<float>(), sn.data_ptr<float>(), (half*)q.data_ptr(), (half*)k.data_ptr(), + (half*)v.data_ptr(), T); + return {q, k, v}; +} + +constexpr int CH = 128; +__device__ __forceinline__ void ld8(const half* p, float* x) { + uint4 u = *reinterpret_cast<const uint4*>(p); + const __half2* h = reinterpret_cast<const __half2*>(&u); +#pragma unroll + for (int i = 0; i < 4; ++i) { float2 f = __half22float2(h[i]); x[2*i] = f.x; x[2*i+1] = f.y; } +} +// Q32: the q-gradient arrives as the attention backward's raw fp32 dQ +// accumulator (times qscale) instead of an fp16 tensor, so it is never rounded. +template <bool Q32> +__global__ void __launch_bounds__(192) canon_rope_bwd( + const half* __restrict__ qkv, const float* __restrict__ w, const float* __restrict__ m1, + const float* __restrict__ m2, const float* __restrict__ cs, const float* __restrict__ sn, + const void* __restrict__ gq_, const half* __restrict__ gk, const half* __restrict__ gv, + half* __restrict__ dqkv, float* __restrict__ part, int T, float qscale) { + const half* gq = reinterpret_cast<const half*>(gq_); + const float* gq32 = reinterpret_cast<const float*>(gq_); + const int c8 = threadIdx.x, ch = 8 * c8, g = ch >> 9, col = ch & 511, d0 = col & 63; + const long n0 = (long)blockIdx.x * CH; + const int t0 = n0 % T; + const half* G = g == 0 ? gq : (g == 1 ? gk : gv); + float w0[8], w1[8], w2[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) { + w0[i] = __half2float(__float2half(w[(ch + i) * 3 + 0])); + w1[i] = __half2float(__float2half(w[(ch + i) * 3 + 1])); + w2[i] = __half2float(__float2half(w[(ch + i) * 3 + 2])); + } + // dy(n): gradient w.r.t. the Canon output at token n, un-rotated. + auto dy = [&](long n, float* out) { + float gg[8]; + if (Q32 && g == 0) { + const float4* p4 = reinterpret_cast<const float4*>(gq32 + n * 512 + col); + float4 a = p4[0], b = p4[1]; + gg[0] = a.x * qscale; gg[1] = a.y * qscale; gg[2] = a.z * qscale; gg[3] = a.w * qscale; + gg[4] = b.x * qscale; gg[5] = b.y * qscale; gg[6] = b.z * qscale; gg[7] = b.w * qscale; + } else { + ld8(G + n * 512 + col, gg); + } + if (g < 2) { + const int d = d0 & 31; + const float4* cp = reinterpret_cast<const float4*>(cs + n * 32 + d); + const float4* sp = reinterpret_cast<const float4*>(sn + n * 32 + d); + float4 c0 = cp[0], c1 = cp[1], s0 = sp[0], s1 = sp[1]; + float cs_[8] = {c0.x, c0.y, c0.z, c0.w, c1.x, c1.y, c1.z, c1.w}; + float sn_[8] = {s0.x, s0.y, s0.z, s0.w, s1.x, s1.y, s1.z, s1.w}; +#pragma unroll + for (int i = 0; i < 8; ++i) { + float p = __shfl_xor_sync(0xffffffff, gg[i], 4); + float ga = d0 < 32 ? gg[i] : p, gb = d0 < 32 ? p : gg[i]; + out[i] = d0 < 32 ? fmaf(ga, cs_[i], gb * sn_[i]) : fmaf(gb, cs_[i], -(ga * sn_[i])); + } + } else { +#pragma unroll + for (int i = 0; i < 8; ++i) out[i] = gg[i]; + } + }; + const long nl = n0 + CH - 1; // walk from the chunk's last token backwards + const int tl = t0 + CH - 1; + float d0v[8], d1v[8], d2v[8], x0[8], xm1[8], xm2[8]; + float mm1n = 0.f, mm2n = 0.f, m2p1 = 0.f; // m2p1 = m2[n+1] +#pragma unroll + for (int i = 0; i < 8; ++i) d1v[i] = d2v[i] = 0.f; + // dy at n+1, n+2 must be computed by every lane (shuffles), masked afterwards. + { float tmp[8]; + long a = (tl + 1 < T) ? nl + 1 : nl; dy(a, tmp); + if (tl + 1 < T) { mm1n = m1[nl + 1]; m2p1 = m2[nl + 1]; +#pragma unroll + for (int i = 0; i < 8; ++i) d1v[i] = tmp[i]; } + long b = (tl + 2 < T) ? nl + 2 : nl; dy(b, tmp); + if (tl + 2 < T) { mm2n = m2[nl + 2]; +#pragma unroll + for (int i = 0; i < 8; ++i) d2v[i] = tmp[i]; } } + ld8(qkv + nl * 1536 + ch, x0); + ld8(qkv + (nl - 1) * 1536 + ch, xm1); // tl >= 31 so both exist + ld8(qkv + (nl - 2) * 1536 + ch, xm2); + float a0[8], a1[8], a2[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) a0[i] = a1[i] = a2[i] = 0.f; + #pragma unroll 8 + for (int s = CH - 1; s >= 0; --s) { + const long n = n0 + s; const int t = t0 + s; + dy(n, d0v); + const float m1n = m1[n], m2n = m2[n]; + float dx[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) { + dx[i] = d0v[i] + d0v[i] * w0[i] + (d1v[i] * mm1n) * w1[i] + (d2v[i] * mm2n) * w2[i]; + a0[i] += d0v[i] * x0[i]; + a1[i] += d0v[i] * (xm1[i] * m1n); + a2[i] += d0v[i] * (xm2[i] * m2n); + } + uint4 u; __half2* h = reinterpret_cast<__half2*>(&u); +#pragma unroll + for (int i = 0; i < 4; ++i) h[i] = __floats2half2_rn(dx[2*i], dx[2*i+1]); + *reinterpret_cast<uint4*>(dqkv + n * 1536 + ch) = u; + mm2n = m2p1; m2p1 = m2n; mm1n = m1n; +#pragma unroll + for (int i = 0; i < 8; ++i) { d2v[i] = d1v[i]; d1v[i] = d0v[i]; x0[i] = xm1[i]; xm1[i] = xm2[i]; } + if (s > 0) { + if (t - 3 >= 0) ld8(qkv + (n - 3) * 1536 + ch, xm2); + else { +#pragma unroll + for (int i = 0; i < 8; ++i) xm2[i] = 0.f; } + if (t - 1 < 1) { // next token (t-1) is t==0: its t-1 tap is outside the row +#pragma unroll + for (int i = 0; i < 8; ++i) xm1[i] = 0.f; } + } + } + float* pp = part + (long)blockIdx.x * 1536 * 3 + ch * 3; +#pragma unroll + for (int i = 0; i < 8; ++i) { pp[3*i] = a0[i]; pp[3*i+1] = a1[i]; pp[3*i+2] = a2[i]; } +} +std::vector<torch::Tensor> bwd(torch::Tensor qkv, torch::Tensor w, torch::Tensor m1, torch::Tensor m2, + torch::Tensor cs, torch::Tensor sn, torch::Tensor gq, torch::Tensor gk, + torch::Tensor gv) { + const int B = qkv.size(0), T = qkv.size(1); const long N = (long)B * T; + auto dqkv = torch::empty_like(qkv); + auto part = torch::empty({N / CH, 1536, 3}, qkv.options().dtype(torch::kFloat)); + canon_rope_bwd<false><<<N / CH, 192, 0, at::cuda::getCurrentCUDAStream()>>>( + (const half*)qkv.data_ptr(), w.data_ptr<float>(), m1.data_ptr<float>(), m2.data_ptr<float>(), + cs.data_ptr<float>(), sn.data_ptr<float>(), gq.data_ptr(), (const half*)gk.data_ptr(), + (const half*)gv.data_ptr(), (half*)dqkv.data_ptr(), part.data_ptr<float>(), T, 1.f); + C10_CUDA_KERNEL_LAUNCH_CHECK(); + return {dqkv, part}; +} +// gq32: (B, T, 8, 64) fp32 raw dQ accumulator; qscale folds in the softmax scale. +std::vector<torch::Tensor> bwd_q32(torch::Tensor qkv, torch::Tensor w, torch::Tensor m1, torch::Tensor m2, + torch::Tensor cs, torch::Tensor sn, torch::Tensor gq32, torch::Tensor gk, + torch::Tensor gv, double qscale) { + TORCH_CHECK(gq32.scalar_type() == at::kFloat && gq32.is_contiguous(), "gq32 must be contiguous fp32"); + const int B = qkv.size(0), T = qkv.size(1); const long N = (long)B * T; + TORCH_CHECK(T % CH == 0, "T must be a multiple of 128"); + auto dqkv = torch::empty_like(qkv); + auto part = torch::empty({N / CH, 1536, 3}, qkv.options().dtype(torch::kFloat)); + canon_rope_bwd<true><<<N / CH, 192, 0, at::cuda::getCurrentCUDAStream()>>>( + (const half*)qkv.data_ptr(), w.data_ptr<float>(), m1.data_ptr<float>(), m2.data_ptr<float>(), + cs.data_ptr<float>(), sn.data_ptr<float>(), gq32.data_ptr(), (const half*)gk.data_ptr(), + (const half*)gv.data_ptr(), (half*)dqkv.data_ptr(), part.data_ptr<float>(), T, (float)qscale); + C10_CUDA_KERNEL_LAUNCH_CHECK(); + return {dqkv, part}; +} +""" +_here = os.path.dirname(os.path.abspath(__file__)); os.makedirs(os.path.join(_here, "build_canon"), exist_ok=True) +os.environ.setdefault("TORCH_CUDA_ARCH_LIST", "7.0") +ext = load_inline("boris_volta_canon", cpp_sources= + "std::vector<torch::Tensor> fwd(torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor);" + "std::vector<torch::Tensor> bwd(torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor);" + "std::vector<torch::Tensor> bwd_q32(torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, double);", + cuda_sources=SRC, functions=["fwd", "bwd", "bwd_q32"], extra_cuda_cflags=["-O3", "-arch=sm_70"], + build_directory=os.path.join(_here, "build_canon")) + +class CanonRopeFast(torch.autograd.Function): + @staticmethod + def forward(ctx, qkv, canon_w, m1, m2, cos, sin): + B, T, C = qkv.shape; N = B * T + assert C == 1536 and qkv.dtype == torch.float16 and qkv.is_contiguous() and T % 128 == 0 + m1f, m2f = m1.reshape(N).contiguous(), m2.reshape(N).contiguous() + cf, sf = cos.reshape(N, 32).contiguous(), sin.reshape(N, 32).contiguous() + q, k, v = ext.fwd(qkv, canon_w.contiguous(), m1f, m2f, cf, sf) + ctx.save_for_backward(qkv, canon_w, m1f, m2f, cf, sf) + return q, k, v + @staticmethod + def backward(ctx, gq, gk, gv): + qkv, w, m1f, m2f, cf, sf = ctx.saved_tensors + gq, gk, gv = (g.to(torch.float16).contiguous() for g in (gq, gk, gv)) + dqkv, part = ext.bwd(qkv, w.contiguous(), m1f, m2f, cf, sf, gq, gk, gv) + # fp32 straight from the fp32 reduction. Rounding through fp16 (as the + # eager region's cast implied) left ~3% of entries subnormal on the live + # checkpoint; keeping fp32 is strictly more precise. + return dqkv, part.sum(0).to(w.dtype), None, None, None, None + +def canon_rope(qkv, canon_w, m1, m2, cos, sin): + return CanonRopeFast.apply(qkv, canon_w, m1, m2, cos, sin) |
