diff options
| author | jdj <jdj@kunix.org> | 2026-11-18 14:47:33 -0800 |
|---|---|---|
| committer | jdj <jdj@kunix.org> | 2026-11-18 14:47:33 -0800 |
| commit | 5d2e4252e695644c34be605b44a14bd80a98a0a6 (patch) | |
| tree | 311a72eda4459af7547b9cd05c0956eafb235f2d /volta_kernels/canon_rope_bw.py | |
| download | boris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.tar.gz boris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.zip | |
Boris-2 training code
The code, configuration and data filters used to train Boris-2 (144M,
200B tokens, single V100): model, training loop, data preparation,
n-gram table, V100 kernels, final stages and the exact run configs.
Diffstat (limited to 'volta_kernels/canon_rope_bw.py')
| -rw-r--r-- | volta_kernels/canon_rope_bw.py | 327 |
1 files changed, 327 insertions, 0 deletions
diff --git a/volta_kernels/canon_rope_bw.py b/volta_kernels/canon_rope_bw.py new file mode 100644 index 0000000..172de86 --- /dev/null +++ b/volta_kernels/canon_rope_bw.py @@ -0,0 +1,327 @@ +"""Canon + RoPE in CUDA, bandwidth rewrite (swarm agent canon_bw, 2026-09-27). + +Same math, same rounding as volta_kernels/canon_rope_cuda.py (q/k/v and dqkv +bitwise identical; Canon weight-gradient partials bitwise identical, so the +weight gradient is too when reduced the same way). + +Why the old kernels sit at ~65% of DRAM bandwidth (V100, 72 SMs): + fwd 88 regs x 192 thr -> 3 CTAs/SM (18 warps), grid 512 = 2.37 waves + (last wave 37% full), one 16-B load in flight per thread. + bwd 162 regs x 192 thr -> 2 CTAs/SM (12 warps), grid 256 = 1.78 waves. +What changes here: + * CPT channels per thread (default 4 instead of 8) and one q/k/v group per + CTA (g = blockIdx.x % 3, so the q, k and v CTAs of one token chunk run + side by side and share cos/sin and halo rows through L2): ~half the + registers, 128-thread CTAs, 3x more CTAs -> high occupancy, small tail. + * U-deep register prefetch of the streamed rows (qkv / gradients / masks). + * bwd keeps the 128-token chunk and the backward walk order, so the Canon + weight-gradient partials are the same fp32 sums in the same order. +""" +import os, torch +from torch.utils.cpp_extension import load_inline + +SRC = r""" +#include <torch/extension.h> +#include <ATen/cuda/CUDAContext.h> +#include <cuda_fp16.h> + +template <int CPT> struct HV; // CPT fp16 values in one vector load +template <> struct HV<8> { typedef uint4 T; }; +template <> struct HV<4> { typedef uint2 T; }; +template <> struct HV<2> { typedef unsigned int T; }; +template <int CPT> struct FV; // CPT fp32 values +template <> struct FV<8> { typedef struct { float4 a, b; } T; }; +template <> struct FV<4> { typedef struct { float4 a; } T; }; +template <> struct FV<2> { typedef struct { float2 a; } T; }; + +template <int CPT> __device__ __forceinline__ typename HV<CPT>::T ldh(const half* p) { + return *reinterpret_cast<const typename HV<CPT>::T*>(p); } +template <int CPT> __device__ __forceinline__ void cvh(const typename HV<CPT>::T& u, float* x) { + const __half2* h = reinterpret_cast<const __half2*>(&u); +#pragma unroll + for (int i = 0; i < CPT / 2; ++i) { float2 f = __half22float2(h[i]); x[2*i] = f.x; x[2*i+1] = f.y; } +} +template <int CPT> __device__ __forceinline__ void sth(half* p, const float* o) { + typename HV<CPT>::T u; __half2* h = reinterpret_cast<__half2*>(&u); +#pragma unroll + for (int i = 0; i < CPT / 2; ++i) h[i] = __floats2half2_rn(o[2*i], o[2*i+1]); + *reinterpret_cast<typename HV<CPT>::T*>(p) = u; +} +template <int CPT> __device__ __forceinline__ void ldcs(const float* p, float* x) { + if constexpr (CPT == 2) { float2 f = *reinterpret_cast<const float2*>(p); x[0] = f.x; x[1] = f.y; } + else { + const float4* q = reinterpret_cast<const float4*>(p); +#pragma unroll + for (int j = 0; j < CPT / 4; ++j) { float4 f = q[j]; x[4*j] = f.x; x[4*j+1] = f.y; x[4*j+2] = f.z; x[4*j+3] = f.w; } + } +} + +// ---------------------------------------------------------------- forward +// grid: (N/FCH) * 3 CTAs of TPB = 512/CPT threads; CTA b handles group g = b % 3 +// (q, k, v) of token chunk b / 3. +template <int CPT, int FCH, int U, int MINB> +__global__ void __launch_bounds__(512 / CPT, MINB) crbw_fwd( + const half* __restrict__ qkv, const float* __restrict__ w, const float* __restrict__ m1, + const float* __restrict__ m2, const float* __restrict__ cs, const float* __restrict__ sn, + half* __restrict__ q, half* __restrict__ k, half* __restrict__ v, int T) { + constexpr int PX = 32 / CPT; // lane distance of the RoPE partner (d +/- 32) + const int g = blockIdx.x % 3; + const int col = CPT * threadIdx.x, ch = g * 512 + col, d0 = col & 63; + const long n0 = (long)(blockIdx.x / 3) * FCH; + const int t0 = n0 % T; + float w0[CPT], w1[CPT], w2[CPT]; +#pragma unroll + for (int i = 0; i < CPT; ++i) { // the region casts canon_w to fp16 + w0[i] = __half2float(__float2half(w[(ch + i) * 3 + 0])); + w1[i] = __half2float(__float2half(w[(ch + i) * 3 + 1])); + w2[i] = __half2float(__float2half(w[(ch + i) * 3 + 2])); + } + const half* src = qkv + ch; + float x1[CPT], x2[CPT], x0[CPT]; +#pragma unroll + for (int i = 0; i < CPT; ++i) x1[i] = x2[i] = 0.f; + if (t0 >= 1) cvh<CPT>(ldh<CPT>(src + (n0 - 1) * 1536), x1); + if (t0 >= 2) cvh<CPT>(ldh<CPT>(src + (n0 - 2) * 1536), x2); + half* out = (g == 0 ? q : (g == 1 ? k : v)) + col; + const int d = d0 & 31; + typename HV<CPT>::T raw[U]; float A1[U], A2[U]; +#pragma unroll + for (int u = 0; u < U; ++u) { raw[u] = ldh<CPT>(src + (n0 + u) * 1536); A1[u] = m1[n0 + u]; A2[u] = m2[n0 + u]; } +#pragma unroll 1 + for (int s0 = 0; s0 < FCH; s0 += U) { + typename HV<CPT>::T cur[U]; float C1[U], C2[U]; +#pragma unroll + for (int u = 0; u < U; ++u) { cur[u] = raw[u]; C1[u] = A1[u]; C2[u] = A2[u]; } + if (s0 + U < FCH) { +#pragma unroll + for (int u = 0; u < U; ++u) { + const long n = n0 + s0 + U + u; + raw[u] = ldh<CPT>(src + n * 1536); A1[u] = m1[n]; A2[u] = m2[n]; + } + } +#pragma unroll + for (int u = 0; u < U; ++u) { + const long n = n0 + s0 + u; + cvh<CPT>(cur[u], x0); + const float a1 = C1[u], a2 = C2[u]; + float y[CPT]; +#pragma unroll + for (int i = 0; i < CPT; ++i) + y[i] = x0[i] + x0[i] * w0[i] + (x1[i] * a1) * w1[i] + (x2[i] * a2) * w2[i]; + float o[CPT]; + if (g < 2) { + float c[CPT], sv[CPT]; + ldcs<CPT>(cs + n * 32 + d, c); ldcs<CPT>(sn + n * 32 + d, sv); +#pragma unroll + for (int i = 0; i < CPT; ++i) { + float p = __shfl_xor_sync(0xffffffff, y[i], PX); + o[i] = d0 < 32 ? fmaf(y[i], c[i], -(p * sv[i])) : fmaf(y[i], c[i], p * sv[i]); + } + } else { +#pragma unroll + for (int i = 0; i < CPT; ++i) o[i] = y[i]; + } + sth<CPT>(out + n * 512, o); +#pragma unroll + for (int i = 0; i < CPT; ++i) { x2[i] = x1[i]; x1[i] = x0[i]; } + } + } +} + +// ---------------------------------------------------------------- backward +// Q32: the q-gradient is the attention backward's raw fp32 dQ accumulator, times qscale. +// CH must stay 128 for the weight-gradient partials to be the old kernel's sums. +template <int CPT, int CH, int U, bool Q32, int MINB> +__global__ void __launch_bounds__(512 / CPT, MINB) crbw_bwd( + const half* __restrict__ qkv, const float* __restrict__ w, const float* __restrict__ m1, + const float* __restrict__ m2, const float* __restrict__ cs, const float* __restrict__ sn, + const void* __restrict__ gq_, const half* __restrict__ gk, const half* __restrict__ gv, + half* __restrict__ dqkv, float* __restrict__ part, int T, float qscale) { + constexpr int PX = 32 / CPT; + const int g = blockIdx.x % 3; + const int col = CPT * threadIdx.x, ch = g * 512 + col, d0 = col & 63, d = d0 & 31; + const long chunk = blockIdx.x / 3, n0 = chunk * CH; + const int t0 = n0 % T; + const bool f32 = Q32 && g == 0; + const half* G = (g == 0 ? reinterpret_cast<const half*>(gq_) : (g == 1 ? gk : gv)) + col; + const float* G32 = reinterpret_cast<const float*>(gq_) + col; + float w0[CPT], w1[CPT], w2[CPT]; +#pragma unroll + for (int i = 0; i < CPT; ++i) { + w0[i] = __half2float(__float2half(w[(ch + i) * 3 + 0])); + w1[i] = __half2float(__float2half(w[(ch + i) * 3 + 1])); + w2[i] = __half2float(__float2half(w[(ch + i) * 3 + 2])); + } + // raw gradient row load (fp32 path: CPT floats; fp16 path: CPT halves in .a's bytes) + typedef typename FV<CPT>::T F; + auto ldg_ = [&](long n, F& r) { + if (f32) r = *reinterpret_cast<const F*>(G32 + n * 512); + else *reinterpret_cast<typename HV<CPT>::T*>(&r) = ldh<CPT>(G + n * 512); + }; + // dy(n) from a raw gradient row: un-rotated gradient w.r.t. the Canon output. + auto dy = [&](long n, const F& r, float* out) { + float gg[CPT]; + if (f32) { + const float* f = reinterpret_cast<const float*>(&r); +#pragma unroll + for (int i = 0; i < CPT; ++i) gg[i] = f[i] * qscale; + } else { + cvh<CPT>(*reinterpret_cast<const typename HV<CPT>::T*>(&r), gg); + } + if (g < 2) { + float c[CPT], sv[CPT]; + ldcs<CPT>(cs + n * 32 + d, c); ldcs<CPT>(sn + n * 32 + d, sv); +#pragma unroll + for (int i = 0; i < CPT; ++i) { + float p = __shfl_xor_sync(0xffffffff, gg[i], PX); + float ga = d0 < 32 ? gg[i] : p, gb = d0 < 32 ? p : gg[i]; + out[i] = d0 < 32 ? fmaf(ga, c[i], gb * sv[i]) : fmaf(gb, c[i], -(ga * sv[i])); + } + } else { +#pragma unroll + for (int i = 0; i < CPT; ++i) out[i] = gg[i]; + } + }; + const half* X = qkv + ch; + const long nl = n0 + CH - 1; + const int tl = t0 + CH - 1; + float d0v[CPT], d1v[CPT], d2v[CPT], x0[CPT], xm1[CPT], xm2[CPT]; + float mm1n = 0.f, mm2n = 0.f, m2p1 = 0.f; +#pragma unroll + for (int i = 0; i < CPT; ++i) d1v[i] = d2v[i] = 0.f; + { F r; float tmp[CPT]; + long a = (tl + 1 < T) ? nl + 1 : nl; ldg_(a, r); dy(a, r, tmp); + if (tl + 1 < T) { mm1n = m1[nl + 1]; m2p1 = m2[nl + 1]; +#pragma unroll + for (int i = 0; i < CPT; ++i) d1v[i] = tmp[i]; } + long b = (tl + 2 < T) ? nl + 2 : nl; ldg_(b, r); dy(b, r, tmp); + if (tl + 2 < T) { mm2n = m2[nl + 2]; +#pragma unroll + for (int i = 0; i < CPT; ++i) d2v[i] = tmp[i]; } } + cvh<CPT>(ldh<CPT>(X + nl * 1536), x0); + cvh<CPT>(ldh<CPT>(X + (nl - 1) * 1536), xm1); + cvh<CPT>(ldh<CPT>(X + (nl - 2) * 1536), xm2); + float a0[CPT], a1[CPT], a2[CPT]; +#pragma unroll + for (int i = 0; i < CPT; ++i) a0[i] = a1[i] = a2[i] = 0.f; + // prefetch ring: for step s (token n = n0+s) we need grad row n, masks n, x row n-3. + F rg[U]; typename HV<CPT>::T rx[U]; float R1[U], R2[U]; + auto fetch = [&](int s, int u) { + const long n = n0 + s; const int t = t0 + s; + ldg_(n, rg[u]); R1[u] = m1[n]; R2[u] = m2[n]; + if (t - 3 >= 0) rx[u] = ldh<CPT>(X + (n - 3) * 1536); + }; +#pragma unroll + for (int u = 0; u < U; ++u) fetch(CH - 1 - u, u); +#pragma unroll 1 + for (int sb = CH - 1; sb >= 0; sb -= U) { + F cg[U]; typename HV<CPT>::T cx[U]; float C1[U], C2[U]; +#pragma unroll + for (int u = 0; u < U; ++u) { cg[u] = rg[u]; cx[u] = rx[u]; C1[u] = R1[u]; C2[u] = R2[u]; } + if (sb - U >= 0) { +#pragma unroll + for (int u = 0; u < U; ++u) fetch(sb - U - u, u); + } +#pragma unroll + for (int u = 0; u < U; ++u) { + const int s = sb - u; + const long n = n0 + s; const int t = t0 + s; + dy(n, cg[u], d0v); + const float m1n = C1[u], m2n = C2[u]; + float dx[CPT]; +#pragma unroll + for (int i = 0; i < CPT; ++i) { + dx[i] = d0v[i] + d0v[i] * w0[i] + (d1v[i] * mm1n) * w1[i] + (d2v[i] * mm2n) * w2[i]; + a0[i] += d0v[i] * x0[i]; + a1[i] += d0v[i] * (xm1[i] * m1n); + a2[i] += d0v[i] * (xm2[i] * m2n); + } + sth<CPT>(dqkv + n * 1536 + ch, dx); + mm2n = m2p1; m2p1 = m2n; mm1n = m1n; +#pragma unroll + for (int i = 0; i < CPT; ++i) { d2v[i] = d1v[i]; d1v[i] = d0v[i]; x0[i] = xm1[i]; xm1[i] = xm2[i]; } + if (s > 0) { + if (t - 3 >= 0) cvh<CPT>(cx[u], xm2); + else { +#pragma unroll + for (int i = 0; i < CPT; ++i) xm2[i] = 0.f; } + if (t - 1 < 1) { +#pragma unroll + for (int i = 0; i < CPT; ++i) xm1[i] = 0.f; } + } + } + } + float* pp = part + chunk * 1536 * 3 + ch * 3; +#pragma unroll + for (int i = 0; i < CPT; ++i) { pp[3*i] = a0[i]; pp[3*i+1] = a1[i]; pp[3*i+2] = a2[i]; } +} + +// Deterministic Canon weight-gradient reduction: sum over chunks in fp64 (more +// precise than the fp32 torch sum), one thread per (channel, tap). +__global__ void crbw_wsum(const float* __restrict__ part, float* __restrict__ dw, int nchunk) { + const int j = blockIdx.x * blockDim.x + threadIdx.x; + if (j >= 1536 * 3) return; + double s = 0.0; + for (int c = 0; c < nchunk; ++c) s += (double)part[(long)c * 4608 + j]; + dw[j] = (float)s; +} + +#define FWD_CFG(CPT, FCH, U, MB) \ + if (cpt == CPT && fch == FCH && u == U && minb == MB) { \ + TORCH_CHECK(T % FCH == 0); \ + crbw_fwd<CPT, FCH, U, MB><<<(N / FCH) * 3, 512 / CPT, 0, st>>>(P(qkv), w.data_ptr<float>(), m1.data_ptr<float>(), \ + m2.data_ptr<float>(), cs.data_ptr<float>(), sn.data_ptr<float>(), P(q), P(k), P(v), T); \ + C10_CUDA_KERNEL_LAUNCH_CHECK(); return {q, k, v}; } +#define P(t) reinterpret_cast<half*>(t.data_ptr()) + +std::vector<torch::Tensor> fwd(torch::Tensor qkv, torch::Tensor w, torch::Tensor m1, torch::Tensor m2, + torch::Tensor cs, torch::Tensor sn, int64_t cpt, int64_t fch, int64_t u, int64_t minb) { + const int B = qkv.size(0), T = qkv.size(1); const long N = (long)B * T; + auto q = torch::empty({B, T, 8, 64}, qkv.options()), k = torch::empty_like(q), v = torch::empty_like(q); + auto st = at::cuda::getCurrentCUDAStream(); + FWD_CFG(4, 64, 4, 1) FWD_CFG(4, 64, 4, 6) FWD_CFG(4, 64, 4, 8) FWD_CFG(4, 64, 2, 8) FWD_CFG(2, 64, 4, 4) + FWD_CFG(2, 64, 2, 4) FWD_CFG(4, 32, 4, 6) FWD_CFG(4, 64, 1, 8) FWD_CFG(2, 64, 4, 1) + TORCH_CHECK(false, "unsupported fwd config"); +} + +#define BWD_CFG(CPT, U, MB) \ + if (cpt == CPT && u == U && minb == MB) { \ + if (q32) crbw_bwd<CPT, 128, U, true, MB><<<(N / 128) * 3, 512 / CPT, 0, st>>>(P(qkv), w.data_ptr<float>(), \ + m1.data_ptr<float>(), m2.data_ptr<float>(), cs.data_ptr<float>(), sn.data_ptr<float>(), gq.data_ptr(), \ + P(gk), P(gv), P(dqkv), part.data_ptr<float>(), T, (float)qscale); \ + else crbw_bwd<CPT, 128, U, false, MB><<<(N / 128) * 3, 512 / CPT, 0, st>>>(P(qkv), w.data_ptr<float>(), \ + m1.data_ptr<float>(), m2.data_ptr<float>(), cs.data_ptr<float>(), sn.data_ptr<float>(), gq.data_ptr(), \ + P(gk), P(gv), P(dqkv), part.data_ptr<float>(), T, (float)qscale); \ + C10_CUDA_KERNEL_LAUNCH_CHECK(); ok = true; } + +// q32: gq is the (B, T, 8, 64) fp32 raw dQ accumulator; qscale folds in the softmax scale. +// wsum: 0 -> return part (N/128, 1536, 3) like the old op; 1 -> fused fp64 reduction, returns dw (1536, 3). +std::vector<torch::Tensor> bwd(torch::Tensor qkv, torch::Tensor w, torch::Tensor m1, torch::Tensor m2, + torch::Tensor cs, torch::Tensor sn, torch::Tensor gq, torch::Tensor gk, + torch::Tensor gv, double qscale, bool q32, int64_t cpt, int64_t u, int64_t minb, bool wsum) { + const int B = qkv.size(0), T = qkv.size(1); const long N = (long)B * T; + TORCH_CHECK(T % 128 == 0, "T must be a multiple of 128"); + TORCH_CHECK(gq.is_contiguous() && gk.is_contiguous() && gv.is_contiguous()); + TORCH_CHECK(!q32 || gq.scalar_type() == at::kFloat, "gq32 must be fp32"); + auto dqkv = torch::empty_like(qkv); + auto part = torch::empty({N / 128, 1536, 3}, qkv.options().dtype(torch::kFloat)); + auto st = at::cuda::getCurrentCUDAStream(); + bool ok = false; + BWD_CFG(4, 2, 1) BWD_CFG(4, 2, 5) BWD_CFG(4, 2, 6) BWD_CFG(4, 1, 6) BWD_CFG(4, 1, 1) BWD_CFG(2, 2, 3) + BWD_CFG(2, 1, 3) BWD_CFG(2, 2, 1) BWD_CFG(4, 4, 5) + TORCH_CHECK(ok, "unsupported bwd config"); + if (!wsum) return {dqkv, part}; + auto dw = torch::empty({1536, 3}, part.options()); + crbw_wsum<<<(4608 + 127) / 128, 128, 0, st>>>(part.data_ptr<float>(), dw.data_ptr<float>(), (int)(N / 128)); + C10_CUDA_KERNEL_LAUNCH_CHECK(); + return {dqkv, dw}; +} +""" +_here = os.path.dirname(os.path.abspath(__file__)) +_build = os.path.join(_here, "build_canon_bw"); os.makedirs(_build, exist_ok=True) +os.environ.setdefault("TORCH_CUDA_ARCH_LIST", "7.0") +ext = load_inline("boris_volta_canon_bw", cpp_sources= + "std::vector<torch::Tensor> fwd(torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, int64_t, int64_t, int64_t, int64_t);" + "std::vector<torch::Tensor> bwd(torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, double, bool, int64_t, int64_t, int64_t, bool);", + cuda_sources=SRC, functions=["fwd", "bwd"], extra_cuda_cflags=["-O3", "-arch=sm_70"], + build_directory=_build) |
