"""Canon + RoPE in CUDA, bandwidth rewrite (swarm agent canon_bw, 2026-09-27). Same math, same rounding as volta_kernels/canon_rope_cuda.py (q/k/v and dqkv bitwise identical; Canon weight-gradient partials bitwise identical, so the weight gradient is too when reduced the same way). Why the old kernels sit at ~65% of DRAM bandwidth (V100, 72 SMs): fwd 88 regs x 192 thr -> 3 CTAs/SM (18 warps), grid 512 = 2.37 waves (last wave 37% full), one 16-B load in flight per thread. bwd 162 regs x 192 thr -> 2 CTAs/SM (12 warps), grid 256 = 1.78 waves. What changes here: * CPT channels per thread (default 4 instead of 8) and one q/k/v group per CTA (g = blockIdx.x % 3, so the q, k and v CTAs of one token chunk run side by side and share cos/sin and halo rows through L2): ~half the registers, 128-thread CTAs, 3x more CTAs -> high occupancy, small tail. * U-deep register prefetch of the streamed rows (qkv / gradients / masks). * bwd keeps the 128-token chunk and the backward walk order, so the Canon weight-gradient partials are the same fp32 sums in the same order. """ import os, torch from torch.utils.cpp_extension import load_inline SRC = r""" #include #include #include template struct HV; // CPT fp16 values in one vector load template <> struct HV<8> { typedef uint4 T; }; template <> struct HV<4> { typedef uint2 T; }; template <> struct HV<2> { typedef unsigned int T; }; template struct FV; // CPT fp32 values template <> struct FV<8> { typedef struct { float4 a, b; } T; }; template <> struct FV<4> { typedef struct { float4 a; } T; }; template <> struct FV<2> { typedef struct { float2 a; } T; }; template __device__ __forceinline__ typename HV::T ldh(const half* p) { return *reinterpret_cast::T*>(p); } template __device__ __forceinline__ void cvh(const typename HV::T& u, float* x) { const __half2* h = reinterpret_cast(&u); #pragma unroll for (int i = 0; i < CPT / 2; ++i) { float2 f = __half22float2(h[i]); x[2*i] = f.x; x[2*i+1] = f.y; } } template __device__ __forceinline__ void sth(half* p, const float* o) { typename HV::T u; __half2* h = reinterpret_cast<__half2*>(&u); #pragma unroll for (int i = 0; i < CPT / 2; ++i) h[i] = __floats2half2_rn(o[2*i], o[2*i+1]); *reinterpret_cast::T*>(p) = u; } template __device__ __forceinline__ void ldcs(const float* p, float* x) { if constexpr (CPT == 2) { float2 f = *reinterpret_cast(p); x[0] = f.x; x[1] = f.y; } else { const float4* q = reinterpret_cast(p); #pragma unroll for (int j = 0; j < CPT / 4; ++j) { float4 f = q[j]; x[4*j] = f.x; x[4*j+1] = f.y; x[4*j+2] = f.z; x[4*j+3] = f.w; } } } // ---------------------------------------------------------------- forward // grid: (N/FCH) * 3 CTAs of TPB = 512/CPT threads; CTA b handles group g = b % 3 // (q, k, v) of token chunk b / 3. template __global__ void __launch_bounds__(512 / CPT, MINB) crbw_fwd( const half* __restrict__ qkv, const float* __restrict__ w, const float* __restrict__ m1, const float* __restrict__ m2, const float* __restrict__ cs, const float* __restrict__ sn, half* __restrict__ q, half* __restrict__ k, half* __restrict__ v, int T) { constexpr int PX = 32 / CPT; // lane distance of the RoPE partner (d +/- 32) const int g = blockIdx.x % 3; const int col = CPT * threadIdx.x, ch = g * 512 + col, d0 = col & 63; const long n0 = (long)(blockIdx.x / 3) * FCH; const int t0 = n0 % T; float w0[CPT], w1[CPT], w2[CPT]; #pragma unroll for (int i = 0; i < CPT; ++i) { // the region casts canon_w to fp16 w0[i] = __half2float(__float2half(w[(ch + i) * 3 + 0])); w1[i] = __half2float(__float2half(w[(ch + i) * 3 + 1])); w2[i] = __half2float(__float2half(w[(ch + i) * 3 + 2])); } const half* src = qkv + ch; float x1[CPT], x2[CPT], x0[CPT]; #pragma unroll for (int i = 0; i < CPT; ++i) x1[i] = x2[i] = 0.f; if (t0 >= 1) cvh(ldh(src + (n0 - 1) * 1536), x1); if (t0 >= 2) cvh(ldh(src + (n0 - 2) * 1536), x2); half* out = (g == 0 ? q : (g == 1 ? k : v)) + col; const int d = d0 & 31; typename HV::T raw[U]; float A1[U], A2[U]; #pragma unroll for (int u = 0; u < U; ++u) { raw[u] = ldh(src + (n0 + u) * 1536); A1[u] = m1[n0 + u]; A2[u] = m2[n0 + u]; } #pragma unroll 1 for (int s0 = 0; s0 < FCH; s0 += U) { typename HV::T cur[U]; float C1[U], C2[U]; #pragma unroll for (int u = 0; u < U; ++u) { cur[u] = raw[u]; C1[u] = A1[u]; C2[u] = A2[u]; } if (s0 + U < FCH) { #pragma unroll for (int u = 0; u < U; ++u) { const long n = n0 + s0 + U + u; raw[u] = ldh(src + n * 1536); A1[u] = m1[n]; A2[u] = m2[n]; } } #pragma unroll for (int u = 0; u < U; ++u) { const long n = n0 + s0 + u; cvh(cur[u], x0); const float a1 = C1[u], a2 = C2[u]; float y[CPT]; #pragma unroll for (int i = 0; i < CPT; ++i) y[i] = x0[i] + x0[i] * w0[i] + (x1[i] * a1) * w1[i] + (x2[i] * a2) * w2[i]; float o[CPT]; if (g < 2) { float c[CPT], sv[CPT]; ldcs(cs + n * 32 + d, c); ldcs(sn + n * 32 + d, sv); #pragma unroll for (int i = 0; i < CPT; ++i) { float p = __shfl_xor_sync(0xffffffff, y[i], PX); o[i] = d0 < 32 ? fmaf(y[i], c[i], -(p * sv[i])) : fmaf(y[i], c[i], p * sv[i]); } } else { #pragma unroll for (int i = 0; i < CPT; ++i) o[i] = y[i]; } sth(out + n * 512, o); #pragma unroll for (int i = 0; i < CPT; ++i) { x2[i] = x1[i]; x1[i] = x0[i]; } } } } // ---------------------------------------------------------------- backward // Q32: the q-gradient is the attention backward's raw fp32 dQ accumulator, times qscale. // CH must stay 128 for the weight-gradient partials to be the old kernel's sums. template __global__ void __launch_bounds__(512 / CPT, MINB) crbw_bwd( const half* __restrict__ qkv, const float* __restrict__ w, const float* __restrict__ m1, const float* __restrict__ m2, const float* __restrict__ cs, const float* __restrict__ sn, const void* __restrict__ gq_, const half* __restrict__ gk, const half* __restrict__ gv, half* __restrict__ dqkv, float* __restrict__ part, int T, float qscale) { constexpr int PX = 32 / CPT; const int g = blockIdx.x % 3; const int col = CPT * threadIdx.x, ch = g * 512 + col, d0 = col & 63, d = d0 & 31; const long chunk = blockIdx.x / 3, n0 = chunk * CH; const int t0 = n0 % T; const bool f32 = Q32 && g == 0; const half* G = (g == 0 ? reinterpret_cast(gq_) : (g == 1 ? gk : gv)) + col; const float* G32 = reinterpret_cast(gq_) + col; float w0[CPT], w1[CPT], w2[CPT]; #pragma unroll for (int i = 0; i < CPT; ++i) { w0[i] = __half2float(__float2half(w[(ch + i) * 3 + 0])); w1[i] = __half2float(__float2half(w[(ch + i) * 3 + 1])); w2[i] = __half2float(__float2half(w[(ch + i) * 3 + 2])); } // raw gradient row load (fp32 path: CPT floats; fp16 path: CPT halves in .a's bytes) typedef typename FV::T F; auto ldg_ = [&](long n, F& r) { if (f32) r = *reinterpret_cast(G32 + n * 512); else *reinterpret_cast::T*>(&r) = ldh(G + n * 512); }; // dy(n) from a raw gradient row: un-rotated gradient w.r.t. the Canon output. auto dy = [&](long n, const F& r, float* out) { float gg[CPT]; if (f32) { const float* f = reinterpret_cast(&r); #pragma unroll for (int i = 0; i < CPT; ++i) gg[i] = f[i] * qscale; } else { cvh(*reinterpret_cast::T*>(&r), gg); } if (g < 2) { float c[CPT], sv[CPT]; ldcs(cs + n * 32 + d, c); ldcs(sn + n * 32 + d, sv); #pragma unroll for (int i = 0; i < CPT; ++i) { float p = __shfl_xor_sync(0xffffffff, gg[i], PX); float ga = d0 < 32 ? gg[i] : p, gb = d0 < 32 ? p : gg[i]; out[i] = d0 < 32 ? fmaf(ga, c[i], gb * sv[i]) : fmaf(gb, c[i], -(ga * sv[i])); } } else { #pragma unroll for (int i = 0; i < CPT; ++i) out[i] = gg[i]; } }; const half* X = qkv + ch; const long nl = n0 + CH - 1; const int tl = t0 + CH - 1; float d0v[CPT], d1v[CPT], d2v[CPT], x0[CPT], xm1[CPT], xm2[CPT]; float mm1n = 0.f, mm2n = 0.f, m2p1 = 0.f; #pragma unroll for (int i = 0; i < CPT; ++i) d1v[i] = d2v[i] = 0.f; { F r; float tmp[CPT]; long a = (tl + 1 < T) ? nl + 1 : nl; ldg_(a, r); dy(a, r, tmp); if (tl + 1 < T) { mm1n = m1[nl + 1]; m2p1 = m2[nl + 1]; #pragma unroll for (int i = 0; i < CPT; ++i) d1v[i] = tmp[i]; } long b = (tl + 2 < T) ? nl + 2 : nl; ldg_(b, r); dy(b, r, tmp); if (tl + 2 < T) { mm2n = m2[nl + 2]; #pragma unroll for (int i = 0; i < CPT; ++i) d2v[i] = tmp[i]; } } cvh(ldh(X + nl * 1536), x0); cvh(ldh(X + (nl - 1) * 1536), xm1); cvh(ldh(X + (nl - 2) * 1536), xm2); float a0[CPT], a1[CPT], a2[CPT]; #pragma unroll for (int i = 0; i < CPT; ++i) a0[i] = a1[i] = a2[i] = 0.f; // prefetch ring: for step s (token n = n0+s) we need grad row n, masks n, x row n-3. F rg[U]; typename HV::T rx[U]; float R1[U], R2[U]; auto fetch = [&](int s, int u) { const long n = n0 + s; const int t = t0 + s; ldg_(n, rg[u]); R1[u] = m1[n]; R2[u] = m2[n]; if (t - 3 >= 0) rx[u] = ldh(X + (n - 3) * 1536); }; #pragma unroll for (int u = 0; u < U; ++u) fetch(CH - 1 - u, u); #pragma unroll 1 for (int sb = CH - 1; sb >= 0; sb -= U) { F cg[U]; typename HV::T cx[U]; float C1[U], C2[U]; #pragma unroll for (int u = 0; u < U; ++u) { cg[u] = rg[u]; cx[u] = rx[u]; C1[u] = R1[u]; C2[u] = R2[u]; } if (sb - U >= 0) { #pragma unroll for (int u = 0; u < U; ++u) fetch(sb - U - u, u); } #pragma unroll for (int u = 0; u < U; ++u) { const int s = sb - u; const long n = n0 + s; const int t = t0 + s; dy(n, cg[u], d0v); const float m1n = C1[u], m2n = C2[u]; float dx[CPT]; #pragma unroll for (int i = 0; i < CPT; ++i) { dx[i] = d0v[i] + d0v[i] * w0[i] + (d1v[i] * mm1n) * w1[i] + (d2v[i] * mm2n) * w2[i]; a0[i] += d0v[i] * x0[i]; a1[i] += d0v[i] * (xm1[i] * m1n); a2[i] += d0v[i] * (xm2[i] * m2n); } sth(dqkv + n * 1536 + ch, dx); mm2n = m2p1; m2p1 = m2n; mm1n = m1n; #pragma unroll for (int i = 0; i < CPT; ++i) { d2v[i] = d1v[i]; d1v[i] = d0v[i]; x0[i] = xm1[i]; xm1[i] = xm2[i]; } if (s > 0) { if (t - 3 >= 0) cvh(cx[u], xm2); else { #pragma unroll for (int i = 0; i < CPT; ++i) xm2[i] = 0.f; } if (t - 1 < 1) { #pragma unroll for (int i = 0; i < CPT; ++i) xm1[i] = 0.f; } } } } float* pp = part + chunk * 1536 * 3 + ch * 3; #pragma unroll for (int i = 0; i < CPT; ++i) { pp[3*i] = a0[i]; pp[3*i+1] = a1[i]; pp[3*i+2] = a2[i]; } } // Deterministic Canon weight-gradient reduction: sum over chunks in fp64 (more // precise than the fp32 torch sum), one thread per (channel, tap). __global__ void crbw_wsum(const float* __restrict__ part, float* __restrict__ dw, int nchunk) { const int j = blockIdx.x * blockDim.x + threadIdx.x; if (j >= 1536 * 3) return; double s = 0.0; for (int c = 0; c < nchunk; ++c) s += (double)part[(long)c * 4608 + j]; dw[j] = (float)s; } #define FWD_CFG(CPT, FCH, U, MB) \ if (cpt == CPT && fch == FCH && u == U && minb == MB) { \ TORCH_CHECK(T % FCH == 0); \ crbw_fwd<<<(N / FCH) * 3, 512 / CPT, 0, st>>>(P(qkv), w.data_ptr(), m1.data_ptr(), \ m2.data_ptr(), cs.data_ptr(), sn.data_ptr(), P(q), P(k), P(v), T); \ C10_CUDA_KERNEL_LAUNCH_CHECK(); return {q, k, v}; } #define P(t) reinterpret_cast(t.data_ptr()) std::vector fwd(torch::Tensor qkv, torch::Tensor w, torch::Tensor m1, torch::Tensor m2, torch::Tensor cs, torch::Tensor sn, int64_t cpt, int64_t fch, int64_t u, int64_t minb) { const int B = qkv.size(0), T = qkv.size(1); const long N = (long)B * T; auto q = torch::empty({B, T, 8, 64}, qkv.options()), k = torch::empty_like(q), v = torch::empty_like(q); auto st = at::cuda::getCurrentCUDAStream(); FWD_CFG(4, 64, 4, 1) FWD_CFG(4, 64, 4, 6) FWD_CFG(4, 64, 4, 8) FWD_CFG(4, 64, 2, 8) FWD_CFG(2, 64, 4, 4) FWD_CFG(2, 64, 2, 4) FWD_CFG(4, 32, 4, 6) FWD_CFG(4, 64, 1, 8) FWD_CFG(2, 64, 4, 1) TORCH_CHECK(false, "unsupported fwd config"); } #define BWD_CFG(CPT, U, MB) \ if (cpt == CPT && u == U && minb == MB) { \ if (q32) crbw_bwd<<<(N / 128) * 3, 512 / CPT, 0, st>>>(P(qkv), w.data_ptr(), \ m1.data_ptr(), m2.data_ptr(), cs.data_ptr(), sn.data_ptr(), gq.data_ptr(), \ P(gk), P(gv), P(dqkv), part.data_ptr(), T, (float)qscale); \ else crbw_bwd<<<(N / 128) * 3, 512 / CPT, 0, st>>>(P(qkv), w.data_ptr(), \ m1.data_ptr(), m2.data_ptr(), cs.data_ptr(), sn.data_ptr(), gq.data_ptr(), \ P(gk), P(gv), P(dqkv), part.data_ptr(), T, (float)qscale); \ C10_CUDA_KERNEL_LAUNCH_CHECK(); ok = true; } // q32: gq is the (B, T, 8, 64) fp32 raw dQ accumulator; qscale folds in the softmax scale. // wsum: 0 -> return part (N/128, 1536, 3) like the old op; 1 -> fused fp64 reduction, returns dw (1536, 3). std::vector bwd(torch::Tensor qkv, torch::Tensor w, torch::Tensor m1, torch::Tensor m2, torch::Tensor cs, torch::Tensor sn, torch::Tensor gq, torch::Tensor gk, torch::Tensor gv, double qscale, bool q32, int64_t cpt, int64_t u, int64_t minb, bool wsum) { const int B = qkv.size(0), T = qkv.size(1); const long N = (long)B * T; TORCH_CHECK(T % 128 == 0, "T must be a multiple of 128"); TORCH_CHECK(gq.is_contiguous() && gk.is_contiguous() && gv.is_contiguous()); TORCH_CHECK(!q32 || gq.scalar_type() == at::kFloat, "gq32 must be fp32"); auto dqkv = torch::empty_like(qkv); auto part = torch::empty({N / 128, 1536, 3}, qkv.options().dtype(torch::kFloat)); auto st = at::cuda::getCurrentCUDAStream(); bool ok = false; BWD_CFG(4, 2, 1) BWD_CFG(4, 2, 5) BWD_CFG(4, 2, 6) BWD_CFG(4, 1, 6) BWD_CFG(4, 1, 1) BWD_CFG(2, 2, 3) BWD_CFG(2, 1, 3) BWD_CFG(2, 2, 1) BWD_CFG(4, 4, 5) TORCH_CHECK(ok, "unsupported bwd config"); if (!wsum) return {dqkv, part}; auto dw = torch::empty({1536, 3}, part.options()); crbw_wsum<<<(4608 + 127) / 128, 128, 0, st>>>(part.data_ptr(), dw.data_ptr(), (int)(N / 128)); C10_CUDA_KERNEL_LAUNCH_CHECK(); return {dqkv, dw}; } """ _here = os.path.dirname(os.path.abspath(__file__)) _build = os.path.join(_here, "build_canon_bw"); os.makedirs(_build, exist_ok=True) os.environ.setdefault("TORCH_CUDA_ARCH_LIST", "7.0") ext = load_inline("boris_volta_canon_bw", cpp_sources= "std::vector fwd(torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, int64_t, int64_t, int64_t, int64_t);" "std::vector bwd(torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, double, bool, int64_t, int64_t, int64_t, bool);", cuda_sources=SRC, functions=["fwd", "bwd"], extra_cuda_cflags=["-O3", "-arch=sm_70"], build_directory=_build)