summaryrefslogtreecommitdiff
path: root/volta_kernels/canon_rope_cuda.py
diff options
context:
space:
mode:
authorjdj <jdj@kunix.org>2026-11-18 14:47:33 -0800
committerjdj <jdj@kunix.org>2026-11-18 14:47:33 -0800
commit5d2e4252e695644c34be605b44a14bd80a98a0a6 (patch)
tree311a72eda4459af7547b9cd05c0956eafb235f2d /volta_kernels/canon_rope_cuda.py
downloadboris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.tar.gz
boris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.zip
Boris-2 training code
The code, configuration and data filters used to train Boris-2 (144M, 200B tokens, single V100): model, training loop, data preparation, n-gram table, V100 kernels, final stages and the exact run configs.
Diffstat (limited to 'volta_kernels/canon_rope_cuda.py')
-rw-r--r--volta_kernels/canon_rope_cuda.py266
1 files changed, 266 insertions, 0 deletions
diff --git a/volta_kernels/canon_rope_cuda.py b/volta_kernels/canon_rope_cuda.py
new file mode 100644
index 0000000..e632212
--- /dev/null
+++ b/volta_kernels/canon_rope_cuda.py
@@ -0,0 +1,266 @@
+"""Canon + RoPE in CUDA: a sliding window down the sequence (drop-in for canon_rope.py).
+
+Written by a second Claude Code reviewer on 2026-09-27; adopted after review.
+Each thread owns 8 channels and walks 64 (fwd) / 128 (bwd, backwards) tokens,
+keeping the two previous tokens in registers instead of re-reading neighbours
+as the Triton version does, with 16-byte loads and stores throughout.
+Measured at full size (31 layers, mb 16, idle V100): fwd 0.61 -> 0.32 ms,
+bwd 1.20 -> 0.57 ms, whole micro-step +4.1%. q, k, v, dqkv and the loss are
+bitwise-identical to the Triton kernel (its RoPE rounding order is copied:
+fmaf(y, c, -(p*s))); the Canon weight gradient differs only in fp32 summation
+order (same error vs fp64). Requires T % 128 == 0 (2048 and 8192 both are).
+"""
+import os, torch
+from torch.utils.cpp_extension import load_inline
+SRC = r"""
+#include <torch/extension.h>
+#include <ATen/cuda/CUDAContext.h>
+#include <cuda_fp16.h>
+constexpr int FCH = 64; // tokens per block (sliding window down the sequence)
+__global__ void __launch_bounds__(192) canon_rope_fwd(
+ const half* __restrict__ qkv, const float* __restrict__ w, const float* __restrict__ m1,
+ const float* __restrict__ m2, const float* __restrict__ cs, const float* __restrict__ sn,
+ half* __restrict__ q, half* __restrict__ k, half* __restrict__ v, int T) {
+ const int c8 = threadIdx.x, ch = 8 * c8, g = ch >> 9, col = ch & 511, d0 = col & 63;
+ const long n0 = (long)blockIdx.x * FCH;
+ const int t0 = n0 % T;
+ float w0[8], w1[8], w2[8];
+#pragma unroll
+ for (int i = 0; i < 8; ++i) { // the region casts canon_w to fp16
+ w0[i] = __half2float(__float2half(w[(ch + i) * 3 + 0]));
+ w1[i] = __half2float(__float2half(w[(ch + i) * 3 + 1]));
+ w2[i] = __half2float(__float2half(w[(ch + i) * 3 + 2]));
+ }
+ auto ld = [&](long n, float* x) {
+ uint4 u = *reinterpret_cast<const uint4*>(qkv + n * 1536 + ch);
+ const __half2* h = reinterpret_cast<const __half2*>(&u);
+#pragma unroll
+ for (int i = 0; i < 4; ++i) { float2 f = __half22float2(h[i]); x[2*i] = f.x; x[2*i+1] = f.y; }
+ };
+ float x1[8], x2[8], x0[8];
+#pragma unroll
+ for (int i = 0; i < 8; ++i) x1[i] = x2[i] = 0.f;
+ if (t0 >= 1) ld(n0 - 1, x1);
+ if (t0 >= 2) ld(n0 - 2, x2);
+ half* out = g == 0 ? q : (g == 1 ? k : v);
+#pragma unroll 8
+ for (int s = 0; s < FCH; ++s) {
+ const long n = n0 + s;
+ ld(n, x0);
+ const float a1 = m1[n], a2 = m2[n];
+ float y[8];
+#pragma unroll
+ for (int i = 0; i < 8; ++i)
+ y[i] = x0[i] + x0[i] * w0[i] + (x1[i] * a1) * w1[i] + (x2[i] * a2) * w2[i];
+ float o[8];
+ if (g < 2) {
+ const int d = d0 & 31;
+ const float4* cp = reinterpret_cast<const float4*>(cs + n * 32 + d);
+ const float4* sp = reinterpret_cast<const float4*>(sn + n * 32 + d);
+ float4 c0 = cp[0], c1 = cp[1], s0 = sp[0], s1 = sp[1];
+ float c[8] = {c0.x, c0.y, c0.z, c0.w, c1.x, c1.y, c1.z, c1.w};
+ float sv[8] = {s0.x, s0.y, s0.z, s0.w, s1.x, s1.y, s1.z, s1.w};
+#pragma unroll
+ for (int i = 0; i < 8; ++i) {
+ float p = __shfl_xor_sync(0xffffffff, y[i], 4); // partner channel d +/- 32
+ o[i] = d0 < 32 ? fmaf(y[i], c[i], -(p * sv[i])) : fmaf(y[i], c[i], p * sv[i]);
+ }
+ } else {
+#pragma unroll
+ for (int i = 0; i < 8; ++i) o[i] = y[i];
+ }
+ uint4 u;
+ __half2* h = reinterpret_cast<__half2*>(&u);
+#pragma unroll
+ for (int i = 0; i < 4; ++i) h[i] = __floats2half2_rn(o[2*i], o[2*i+1]);
+ *reinterpret_cast<uint4*>(out + n * 512 + col) = u;
+#pragma unroll
+ for (int i = 0; i < 8; ++i) { x2[i] = x1[i]; x1[i] = x0[i]; }
+ }
+}
+std::vector<torch::Tensor> fwd(torch::Tensor qkv, torch::Tensor w, torch::Tensor m1, torch::Tensor m2,
+ torch::Tensor cs, torch::Tensor sn) {
+ const int B = qkv.size(0), T = qkv.size(1); const long N = (long)B * T;
+ TORCH_CHECK(T % FCH == 0);
+ auto q = torch::empty({B, T, 8, 64}, qkv.options()), k = torch::empty_like(q), v = torch::empty_like(q);
+ canon_rope_fwd<<<N / FCH, 192, 0, at::cuda::getCurrentCUDAStream()>>>(
+ (const half*)qkv.data_ptr(), w.data_ptr<float>(), m1.data_ptr<float>(), m2.data_ptr<float>(),
+ cs.data_ptr<float>(), sn.data_ptr<float>(), (half*)q.data_ptr(), (half*)k.data_ptr(),
+ (half*)v.data_ptr(), T);
+ return {q, k, v};
+}
+
+constexpr int CH = 128;
+__device__ __forceinline__ void ld8(const half* p, float* x) {
+ uint4 u = *reinterpret_cast<const uint4*>(p);
+ const __half2* h = reinterpret_cast<const __half2*>(&u);
+#pragma unroll
+ for (int i = 0; i < 4; ++i) { float2 f = __half22float2(h[i]); x[2*i] = f.x; x[2*i+1] = f.y; }
+}
+// Q32: the q-gradient arrives as the attention backward's raw fp32 dQ
+// accumulator (times qscale) instead of an fp16 tensor, so it is never rounded.
+template <bool Q32>
+__global__ void __launch_bounds__(192) canon_rope_bwd(
+ const half* __restrict__ qkv, const float* __restrict__ w, const float* __restrict__ m1,
+ const float* __restrict__ m2, const float* __restrict__ cs, const float* __restrict__ sn,
+ const void* __restrict__ gq_, const half* __restrict__ gk, const half* __restrict__ gv,
+ half* __restrict__ dqkv, float* __restrict__ part, int T, float qscale) {
+ const half* gq = reinterpret_cast<const half*>(gq_);
+ const float* gq32 = reinterpret_cast<const float*>(gq_);
+ const int c8 = threadIdx.x, ch = 8 * c8, g = ch >> 9, col = ch & 511, d0 = col & 63;
+ const long n0 = (long)blockIdx.x * CH;
+ const int t0 = n0 % T;
+ const half* G = g == 0 ? gq : (g == 1 ? gk : gv);
+ float w0[8], w1[8], w2[8];
+#pragma unroll
+ for (int i = 0; i < 8; ++i) {
+ w0[i] = __half2float(__float2half(w[(ch + i) * 3 + 0]));
+ w1[i] = __half2float(__float2half(w[(ch + i) * 3 + 1]));
+ w2[i] = __half2float(__float2half(w[(ch + i) * 3 + 2]));
+ }
+ // dy(n): gradient w.r.t. the Canon output at token n, un-rotated.
+ auto dy = [&](long n, float* out) {
+ float gg[8];
+ if (Q32 && g == 0) {
+ const float4* p4 = reinterpret_cast<const float4*>(gq32 + n * 512 + col);
+ float4 a = p4[0], b = p4[1];
+ gg[0] = a.x * qscale; gg[1] = a.y * qscale; gg[2] = a.z * qscale; gg[3] = a.w * qscale;
+ gg[4] = b.x * qscale; gg[5] = b.y * qscale; gg[6] = b.z * qscale; gg[7] = b.w * qscale;
+ } else {
+ ld8(G + n * 512 + col, gg);
+ }
+ if (g < 2) {
+ const int d = d0 & 31;
+ const float4* cp = reinterpret_cast<const float4*>(cs + n * 32 + d);
+ const float4* sp = reinterpret_cast<const float4*>(sn + n * 32 + d);
+ float4 c0 = cp[0], c1 = cp[1], s0 = sp[0], s1 = sp[1];
+ float cs_[8] = {c0.x, c0.y, c0.z, c0.w, c1.x, c1.y, c1.z, c1.w};
+ float sn_[8] = {s0.x, s0.y, s0.z, s0.w, s1.x, s1.y, s1.z, s1.w};
+#pragma unroll
+ for (int i = 0; i < 8; ++i) {
+ float p = __shfl_xor_sync(0xffffffff, gg[i], 4);
+ float ga = d0 < 32 ? gg[i] : p, gb = d0 < 32 ? p : gg[i];
+ out[i] = d0 < 32 ? fmaf(ga, cs_[i], gb * sn_[i]) : fmaf(gb, cs_[i], -(ga * sn_[i]));
+ }
+ } else {
+#pragma unroll
+ for (int i = 0; i < 8; ++i) out[i] = gg[i];
+ }
+ };
+ const long nl = n0 + CH - 1; // walk from the chunk's last token backwards
+ const int tl = t0 + CH - 1;
+ float d0v[8], d1v[8], d2v[8], x0[8], xm1[8], xm2[8];
+ float mm1n = 0.f, mm2n = 0.f, m2p1 = 0.f; // m2p1 = m2[n+1]
+#pragma unroll
+ for (int i = 0; i < 8; ++i) d1v[i] = d2v[i] = 0.f;
+ // dy at n+1, n+2 must be computed by every lane (shuffles), masked afterwards.
+ { float tmp[8];
+ long a = (tl + 1 < T) ? nl + 1 : nl; dy(a, tmp);
+ if (tl + 1 < T) { mm1n = m1[nl + 1]; m2p1 = m2[nl + 1];
+#pragma unroll
+ for (int i = 0; i < 8; ++i) d1v[i] = tmp[i]; }
+ long b = (tl + 2 < T) ? nl + 2 : nl; dy(b, tmp);
+ if (tl + 2 < T) { mm2n = m2[nl + 2];
+#pragma unroll
+ for (int i = 0; i < 8; ++i) d2v[i] = tmp[i]; } }
+ ld8(qkv + nl * 1536 + ch, x0);
+ ld8(qkv + (nl - 1) * 1536 + ch, xm1); // tl >= 31 so both exist
+ ld8(qkv + (nl - 2) * 1536 + ch, xm2);
+ float a0[8], a1[8], a2[8];
+#pragma unroll
+ for (int i = 0; i < 8; ++i) a0[i] = a1[i] = a2[i] = 0.f;
+ #pragma unroll 8
+ for (int s = CH - 1; s >= 0; --s) {
+ const long n = n0 + s; const int t = t0 + s;
+ dy(n, d0v);
+ const float m1n = m1[n], m2n = m2[n];
+ float dx[8];
+#pragma unroll
+ for (int i = 0; i < 8; ++i) {
+ dx[i] = d0v[i] + d0v[i] * w0[i] + (d1v[i] * mm1n) * w1[i] + (d2v[i] * mm2n) * w2[i];
+ a0[i] += d0v[i] * x0[i];
+ a1[i] += d0v[i] * (xm1[i] * m1n);
+ a2[i] += d0v[i] * (xm2[i] * m2n);
+ }
+ uint4 u; __half2* h = reinterpret_cast<__half2*>(&u);
+#pragma unroll
+ for (int i = 0; i < 4; ++i) h[i] = __floats2half2_rn(dx[2*i], dx[2*i+1]);
+ *reinterpret_cast<uint4*>(dqkv + n * 1536 + ch) = u;
+ mm2n = m2p1; m2p1 = m2n; mm1n = m1n;
+#pragma unroll
+ for (int i = 0; i < 8; ++i) { d2v[i] = d1v[i]; d1v[i] = d0v[i]; x0[i] = xm1[i]; xm1[i] = xm2[i]; }
+ if (s > 0) {
+ if (t - 3 >= 0) ld8(qkv + (n - 3) * 1536 + ch, xm2);
+ else {
+#pragma unroll
+ for (int i = 0; i < 8; ++i) xm2[i] = 0.f; }
+ if (t - 1 < 1) { // next token (t-1) is t==0: its t-1 tap is outside the row
+#pragma unroll
+ for (int i = 0; i < 8; ++i) xm1[i] = 0.f; }
+ }
+ }
+ float* pp = part + (long)blockIdx.x * 1536 * 3 + ch * 3;
+#pragma unroll
+ for (int i = 0; i < 8; ++i) { pp[3*i] = a0[i]; pp[3*i+1] = a1[i]; pp[3*i+2] = a2[i]; }
+}
+std::vector<torch::Tensor> bwd(torch::Tensor qkv, torch::Tensor w, torch::Tensor m1, torch::Tensor m2,
+ torch::Tensor cs, torch::Tensor sn, torch::Tensor gq, torch::Tensor gk,
+ torch::Tensor gv) {
+ const int B = qkv.size(0), T = qkv.size(1); const long N = (long)B * T;
+ auto dqkv = torch::empty_like(qkv);
+ auto part = torch::empty({N / CH, 1536, 3}, qkv.options().dtype(torch::kFloat));
+ canon_rope_bwd<false><<<N / CH, 192, 0, at::cuda::getCurrentCUDAStream()>>>(
+ (const half*)qkv.data_ptr(), w.data_ptr<float>(), m1.data_ptr<float>(), m2.data_ptr<float>(),
+ cs.data_ptr<float>(), sn.data_ptr<float>(), gq.data_ptr(), (const half*)gk.data_ptr(),
+ (const half*)gv.data_ptr(), (half*)dqkv.data_ptr(), part.data_ptr<float>(), T, 1.f);
+ C10_CUDA_KERNEL_LAUNCH_CHECK();
+ return {dqkv, part};
+}
+// gq32: (B, T, 8, 64) fp32 raw dQ accumulator; qscale folds in the softmax scale.
+std::vector<torch::Tensor> bwd_q32(torch::Tensor qkv, torch::Tensor w, torch::Tensor m1, torch::Tensor m2,
+ torch::Tensor cs, torch::Tensor sn, torch::Tensor gq32, torch::Tensor gk,
+ torch::Tensor gv, double qscale) {
+ TORCH_CHECK(gq32.scalar_type() == at::kFloat && gq32.is_contiguous(), "gq32 must be contiguous fp32");
+ const int B = qkv.size(0), T = qkv.size(1); const long N = (long)B * T;
+ TORCH_CHECK(T % CH == 0, "T must be a multiple of 128");
+ auto dqkv = torch::empty_like(qkv);
+ auto part = torch::empty({N / CH, 1536, 3}, qkv.options().dtype(torch::kFloat));
+ canon_rope_bwd<true><<<N / CH, 192, 0, at::cuda::getCurrentCUDAStream()>>>(
+ (const half*)qkv.data_ptr(), w.data_ptr<float>(), m1.data_ptr<float>(), m2.data_ptr<float>(),
+ cs.data_ptr<float>(), sn.data_ptr<float>(), gq32.data_ptr(), (const half*)gk.data_ptr(),
+ (const half*)gv.data_ptr(), (half*)dqkv.data_ptr(), part.data_ptr<float>(), T, (float)qscale);
+ C10_CUDA_KERNEL_LAUNCH_CHECK();
+ return {dqkv, part};
+}
+"""
+_here = os.path.dirname(os.path.abspath(__file__)); os.makedirs(os.path.join(_here, "build_canon"), exist_ok=True)
+os.environ.setdefault("TORCH_CUDA_ARCH_LIST", "7.0")
+ext = load_inline("boris_volta_canon", cpp_sources=
+ "std::vector<torch::Tensor> fwd(torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor);"
+ "std::vector<torch::Tensor> bwd(torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor);"
+ "std::vector<torch::Tensor> bwd_q32(torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, double);",
+ cuda_sources=SRC, functions=["fwd", "bwd", "bwd_q32"], extra_cuda_cflags=["-O3", "-arch=sm_70"],
+ build_directory=os.path.join(_here, "build_canon"))
+
+class CanonRopeFast(torch.autograd.Function):
+ @staticmethod
+ def forward(ctx, qkv, canon_w, m1, m2, cos, sin):
+ B, T, C = qkv.shape; N = B * T
+ assert C == 1536 and qkv.dtype == torch.float16 and qkv.is_contiguous() and T % 128 == 0
+ m1f, m2f = m1.reshape(N).contiguous(), m2.reshape(N).contiguous()
+ cf, sf = cos.reshape(N, 32).contiguous(), sin.reshape(N, 32).contiguous()
+ q, k, v = ext.fwd(qkv, canon_w.contiguous(), m1f, m2f, cf, sf)
+ ctx.save_for_backward(qkv, canon_w, m1f, m2f, cf, sf)
+ return q, k, v
+ @staticmethod
+ def backward(ctx, gq, gk, gv):
+ qkv, w, m1f, m2f, cf, sf = ctx.saved_tensors
+ gq, gk, gv = (g.to(torch.float16).contiguous() for g in (gq, gk, gv))
+ dqkv, part = ext.bwd(qkv, w.contiguous(), m1f, m2f, cf, sf, gq, gk, gv)
+ # fp32 straight from the fp32 reduction. Rounding through fp16 (as the
+ # eager region's cast implied) left ~3% of entries subnormal on the live
+ # checkpoint; keeping fp32 is strictly more precise.
+ return dqkv, part.sum(0).to(w.dtype), None, None, None, None
+
+def canon_rope(qkv, canon_w, m1, m2, cos, sin):
+ return CanonRopeFast.apply(qkv, canon_w, m1, m2, cos, sin)