1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
|
"""Canon + RoPE in CUDA, bandwidth rewrite (swarm agent canon_bw, 2026-09-27).
Same math, same rounding as volta_kernels/canon_rope_cuda.py (q/k/v and dqkv
bitwise identical; Canon weight-gradient partials bitwise identical, so the
weight gradient is too when reduced the same way).
Why the old kernels sit at ~65% of DRAM bandwidth (V100, 72 SMs):
fwd 88 regs x 192 thr -> 3 CTAs/SM (18 warps), grid 512 = 2.37 waves
(last wave 37% full), one 16-B load in flight per thread.
bwd 162 regs x 192 thr -> 2 CTAs/SM (12 warps), grid 256 = 1.78 waves.
What changes here:
* CPT channels per thread (default 4 instead of 8) and one q/k/v group per
CTA (g = blockIdx.x % 3, so the q, k and v CTAs of one token chunk run
side by side and share cos/sin and halo rows through L2): ~half the
registers, 128-thread CTAs, 3x more CTAs -> high occupancy, small tail.
* U-deep register prefetch of the streamed rows (qkv / gradients / masks).
* bwd keeps the 128-token chunk and the backward walk order, so the Canon
weight-gradient partials are the same fp32 sums in the same order.
"""
import os, torch
from torch.utils.cpp_extension import load_inline
SRC = r"""
#include <torch/extension.h>
#include <ATen/cuda/CUDAContext.h>
#include <cuda_fp16.h>
template <int CPT> struct HV; // CPT fp16 values in one vector load
template <> struct HV<8> { typedef uint4 T; };
template <> struct HV<4> { typedef uint2 T; };
template <> struct HV<2> { typedef unsigned int T; };
template <int CPT> struct FV; // CPT fp32 values
template <> struct FV<8> { typedef struct { float4 a, b; } T; };
template <> struct FV<4> { typedef struct { float4 a; } T; };
template <> struct FV<2> { typedef struct { float2 a; } T; };
template <int CPT> __device__ __forceinline__ typename HV<CPT>::T ldh(const half* p) {
return *reinterpret_cast<const typename HV<CPT>::T*>(p); }
template <int CPT> __device__ __forceinline__ void cvh(const typename HV<CPT>::T& u, float* x) {
const __half2* h = reinterpret_cast<const __half2*>(&u);
#pragma unroll
for (int i = 0; i < CPT / 2; ++i) { float2 f = __half22float2(h[i]); x[2*i] = f.x; x[2*i+1] = f.y; }
}
template <int CPT> __device__ __forceinline__ void sth(half* p, const float* o) {
typename HV<CPT>::T u; __half2* h = reinterpret_cast<__half2*>(&u);
#pragma unroll
for (int i = 0; i < CPT / 2; ++i) h[i] = __floats2half2_rn(o[2*i], o[2*i+1]);
*reinterpret_cast<typename HV<CPT>::T*>(p) = u;
}
template <int CPT> __device__ __forceinline__ void ldcs(const float* p, float* x) {
if constexpr (CPT == 2) { float2 f = *reinterpret_cast<const float2*>(p); x[0] = f.x; x[1] = f.y; }
else {
const float4* q = reinterpret_cast<const float4*>(p);
#pragma unroll
for (int j = 0; j < CPT / 4; ++j) { float4 f = q[j]; x[4*j] = f.x; x[4*j+1] = f.y; x[4*j+2] = f.z; x[4*j+3] = f.w; }
}
}
// ---------------------------------------------------------------- forward
// grid: (N/FCH) * 3 CTAs of TPB = 512/CPT threads; CTA b handles group g = b % 3
// (q, k, v) of token chunk b / 3.
template <int CPT, int FCH, int U, int MINB>
__global__ void __launch_bounds__(512 / CPT, MINB) crbw_fwd(
const half* __restrict__ qkv, const float* __restrict__ w, const float* __restrict__ m1,
const float* __restrict__ m2, const float* __restrict__ cs, const float* __restrict__ sn,
half* __restrict__ q, half* __restrict__ k, half* __restrict__ v, int T) {
constexpr int PX = 32 / CPT; // lane distance of the RoPE partner (d +/- 32)
const int g = blockIdx.x % 3;
const int col = CPT * threadIdx.x, ch = g * 512 + col, d0 = col & 63;
const long n0 = (long)(blockIdx.x / 3) * FCH;
const int t0 = n0 % T;
float w0[CPT], w1[CPT], w2[CPT];
#pragma unroll
for (int i = 0; i < CPT; ++i) { // the region casts canon_w to fp16
w0[i] = __half2float(__float2half(w[(ch + i) * 3 + 0]));
w1[i] = __half2float(__float2half(w[(ch + i) * 3 + 1]));
w2[i] = __half2float(__float2half(w[(ch + i) * 3 + 2]));
}
const half* src = qkv + ch;
float x1[CPT], x2[CPT], x0[CPT];
#pragma unroll
for (int i = 0; i < CPT; ++i) x1[i] = x2[i] = 0.f;
if (t0 >= 1) cvh<CPT>(ldh<CPT>(src + (n0 - 1) * 1536), x1);
if (t0 >= 2) cvh<CPT>(ldh<CPT>(src + (n0 - 2) * 1536), x2);
half* out = (g == 0 ? q : (g == 1 ? k : v)) + col;
const int d = d0 & 31;
typename HV<CPT>::T raw[U]; float A1[U], A2[U];
#pragma unroll
for (int u = 0; u < U; ++u) { raw[u] = ldh<CPT>(src + (n0 + u) * 1536); A1[u] = m1[n0 + u]; A2[u] = m2[n0 + u]; }
#pragma unroll 1
for (int s0 = 0; s0 < FCH; s0 += U) {
typename HV<CPT>::T cur[U]; float C1[U], C2[U];
#pragma unroll
for (int u = 0; u < U; ++u) { cur[u] = raw[u]; C1[u] = A1[u]; C2[u] = A2[u]; }
if (s0 + U < FCH) {
#pragma unroll
for (int u = 0; u < U; ++u) {
const long n = n0 + s0 + U + u;
raw[u] = ldh<CPT>(src + n * 1536); A1[u] = m1[n]; A2[u] = m2[n];
}
}
#pragma unroll
for (int u = 0; u < U; ++u) {
const long n = n0 + s0 + u;
cvh<CPT>(cur[u], x0);
const float a1 = C1[u], a2 = C2[u];
float y[CPT];
#pragma unroll
for (int i = 0; i < CPT; ++i)
y[i] = x0[i] + x0[i] * w0[i] + (x1[i] * a1) * w1[i] + (x2[i] * a2) * w2[i];
float o[CPT];
if (g < 2) {
float c[CPT], sv[CPT];
ldcs<CPT>(cs + n * 32 + d, c); ldcs<CPT>(sn + n * 32 + d, sv);
#pragma unroll
for (int i = 0; i < CPT; ++i) {
float p = __shfl_xor_sync(0xffffffff, y[i], PX);
o[i] = d0 < 32 ? fmaf(y[i], c[i], -(p * sv[i])) : fmaf(y[i], c[i], p * sv[i]);
}
} else {
#pragma unroll
for (int i = 0; i < CPT; ++i) o[i] = y[i];
}
sth<CPT>(out + n * 512, o);
#pragma unroll
for (int i = 0; i < CPT; ++i) { x2[i] = x1[i]; x1[i] = x0[i]; }
}
}
}
// ---------------------------------------------------------------- backward
// Q32: the q-gradient is the attention backward's raw fp32 dQ accumulator, times qscale.
// CH must stay 128 for the weight-gradient partials to be the old kernel's sums.
template <int CPT, int CH, int U, bool Q32, int MINB>
__global__ void __launch_bounds__(512 / CPT, MINB) crbw_bwd(
const half* __restrict__ qkv, const float* __restrict__ w, const float* __restrict__ m1,
const float* __restrict__ m2, const float* __restrict__ cs, const float* __restrict__ sn,
const void* __restrict__ gq_, const half* __restrict__ gk, const half* __restrict__ gv,
half* __restrict__ dqkv, float* __restrict__ part, int T, float qscale) {
constexpr int PX = 32 / CPT;
const int g = blockIdx.x % 3;
const int col = CPT * threadIdx.x, ch = g * 512 + col, d0 = col & 63, d = d0 & 31;
const long chunk = blockIdx.x / 3, n0 = chunk * CH;
const int t0 = n0 % T;
const bool f32 = Q32 && g == 0;
const half* G = (g == 0 ? reinterpret_cast<const half*>(gq_) : (g == 1 ? gk : gv)) + col;
const float* G32 = reinterpret_cast<const float*>(gq_) + col;
float w0[CPT], w1[CPT], w2[CPT];
#pragma unroll
for (int i = 0; i < CPT; ++i) {
w0[i] = __half2float(__float2half(w[(ch + i) * 3 + 0]));
w1[i] = __half2float(__float2half(w[(ch + i) * 3 + 1]));
w2[i] = __half2float(__float2half(w[(ch + i) * 3 + 2]));
}
// raw gradient row load (fp32 path: CPT floats; fp16 path: CPT halves in .a's bytes)
typedef typename FV<CPT>::T F;
auto ldg_ = [&](long n, F& r) {
if (f32) r = *reinterpret_cast<const F*>(G32 + n * 512);
else *reinterpret_cast<typename HV<CPT>::T*>(&r) = ldh<CPT>(G + n * 512);
};
// dy(n) from a raw gradient row: un-rotated gradient w.r.t. the Canon output.
auto dy = [&](long n, const F& r, float* out) {
float gg[CPT];
if (f32) {
const float* f = reinterpret_cast<const float*>(&r);
#pragma unroll
for (int i = 0; i < CPT; ++i) gg[i] = f[i] * qscale;
} else {
cvh<CPT>(*reinterpret_cast<const typename HV<CPT>::T*>(&r), gg);
}
if (g < 2) {
float c[CPT], sv[CPT];
ldcs<CPT>(cs + n * 32 + d, c); ldcs<CPT>(sn + n * 32 + d, sv);
#pragma unroll
for (int i = 0; i < CPT; ++i) {
float p = __shfl_xor_sync(0xffffffff, gg[i], PX);
float ga = d0 < 32 ? gg[i] : p, gb = d0 < 32 ? p : gg[i];
out[i] = d0 < 32 ? fmaf(ga, c[i], gb * sv[i]) : fmaf(gb, c[i], -(ga * sv[i]));
}
} else {
#pragma unroll
for (int i = 0; i < CPT; ++i) out[i] = gg[i];
}
};
const half* X = qkv + ch;
const long nl = n0 + CH - 1;
const int tl = t0 + CH - 1;
float d0v[CPT], d1v[CPT], d2v[CPT], x0[CPT], xm1[CPT], xm2[CPT];
float mm1n = 0.f, mm2n = 0.f, m2p1 = 0.f;
#pragma unroll
for (int i = 0; i < CPT; ++i) d1v[i] = d2v[i] = 0.f;
{ F r; float tmp[CPT];
long a = (tl + 1 < T) ? nl + 1 : nl; ldg_(a, r); dy(a, r, tmp);
if (tl + 1 < T) { mm1n = m1[nl + 1]; m2p1 = m2[nl + 1];
#pragma unroll
for (int i = 0; i < CPT; ++i) d1v[i] = tmp[i]; }
long b = (tl + 2 < T) ? nl + 2 : nl; ldg_(b, r); dy(b, r, tmp);
if (tl + 2 < T) { mm2n = m2[nl + 2];
#pragma unroll
for (int i = 0; i < CPT; ++i) d2v[i] = tmp[i]; } }
cvh<CPT>(ldh<CPT>(X + nl * 1536), x0);
cvh<CPT>(ldh<CPT>(X + (nl - 1) * 1536), xm1);
cvh<CPT>(ldh<CPT>(X + (nl - 2) * 1536), xm2);
float a0[CPT], a1[CPT], a2[CPT];
#pragma unroll
for (int i = 0; i < CPT; ++i) a0[i] = a1[i] = a2[i] = 0.f;
// prefetch ring: for step s (token n = n0+s) we need grad row n, masks n, x row n-3.
F rg[U]; typename HV<CPT>::T rx[U]; float R1[U], R2[U];
auto fetch = [&](int s, int u) {
const long n = n0 + s; const int t = t0 + s;
ldg_(n, rg[u]); R1[u] = m1[n]; R2[u] = m2[n];
if (t - 3 >= 0) rx[u] = ldh<CPT>(X + (n - 3) * 1536);
};
#pragma unroll
for (int u = 0; u < U; ++u) fetch(CH - 1 - u, u);
#pragma unroll 1
for (int sb = CH - 1; sb >= 0; sb -= U) {
F cg[U]; typename HV<CPT>::T cx[U]; float C1[U], C2[U];
#pragma unroll
for (int u = 0; u < U; ++u) { cg[u] = rg[u]; cx[u] = rx[u]; C1[u] = R1[u]; C2[u] = R2[u]; }
if (sb - U >= 0) {
#pragma unroll
for (int u = 0; u < U; ++u) fetch(sb - U - u, u);
}
#pragma unroll
for (int u = 0; u < U; ++u) {
const int s = sb - u;
const long n = n0 + s; const int t = t0 + s;
dy(n, cg[u], d0v);
const float m1n = C1[u], m2n = C2[u];
float dx[CPT];
#pragma unroll
for (int i = 0; i < CPT; ++i) {
dx[i] = d0v[i] + d0v[i] * w0[i] + (d1v[i] * mm1n) * w1[i] + (d2v[i] * mm2n) * w2[i];
a0[i] += d0v[i] * x0[i];
a1[i] += d0v[i] * (xm1[i] * m1n);
a2[i] += d0v[i] * (xm2[i] * m2n);
}
sth<CPT>(dqkv + n * 1536 + ch, dx);
mm2n = m2p1; m2p1 = m2n; mm1n = m1n;
#pragma unroll
for (int i = 0; i < CPT; ++i) { d2v[i] = d1v[i]; d1v[i] = d0v[i]; x0[i] = xm1[i]; xm1[i] = xm2[i]; }
if (s > 0) {
if (t - 3 >= 0) cvh<CPT>(cx[u], xm2);
else {
#pragma unroll
for (int i = 0; i < CPT; ++i) xm2[i] = 0.f; }
if (t - 1 < 1) {
#pragma unroll
for (int i = 0; i < CPT; ++i) xm1[i] = 0.f; }
}
}
}
float* pp = part + chunk * 1536 * 3 + ch * 3;
#pragma unroll
for (int i = 0; i < CPT; ++i) { pp[3*i] = a0[i]; pp[3*i+1] = a1[i]; pp[3*i+2] = a2[i]; }
}
// Deterministic Canon weight-gradient reduction: sum over chunks in fp64 (more
// precise than the fp32 torch sum), one thread per (channel, tap).
__global__ void crbw_wsum(const float* __restrict__ part, float* __restrict__ dw, int nchunk) {
const int j = blockIdx.x * blockDim.x + threadIdx.x;
if (j >= 1536 * 3) return;
double s = 0.0;
for (int c = 0; c < nchunk; ++c) s += (double)part[(long)c * 4608 + j];
dw[j] = (float)s;
}
#define FWD_CFG(CPT, FCH, U, MB) \
if (cpt == CPT && fch == FCH && u == U && minb == MB) { \
TORCH_CHECK(T % FCH == 0); \
crbw_fwd<CPT, FCH, U, MB><<<(N / FCH) * 3, 512 / CPT, 0, st>>>(P(qkv), w.data_ptr<float>(), m1.data_ptr<float>(), \
m2.data_ptr<float>(), cs.data_ptr<float>(), sn.data_ptr<float>(), P(q), P(k), P(v), T); \
C10_CUDA_KERNEL_LAUNCH_CHECK(); return {q, k, v}; }
#define P(t) reinterpret_cast<half*>(t.data_ptr())
std::vector<torch::Tensor> fwd(torch::Tensor qkv, torch::Tensor w, torch::Tensor m1, torch::Tensor m2,
torch::Tensor cs, torch::Tensor sn, int64_t cpt, int64_t fch, int64_t u, int64_t minb) {
const int B = qkv.size(0), T = qkv.size(1); const long N = (long)B * T;
auto q = torch::empty({B, T, 8, 64}, qkv.options()), k = torch::empty_like(q), v = torch::empty_like(q);
auto st = at::cuda::getCurrentCUDAStream();
FWD_CFG(4, 64, 4, 1) FWD_CFG(4, 64, 4, 6) FWD_CFG(4, 64, 4, 8) FWD_CFG(4, 64, 2, 8) FWD_CFG(2, 64, 4, 4)
FWD_CFG(2, 64, 2, 4) FWD_CFG(4, 32, 4, 6) FWD_CFG(4, 64, 1, 8) FWD_CFG(2, 64, 4, 1)
TORCH_CHECK(false, "unsupported fwd config");
}
#define BWD_CFG(CPT, U, MB) \
if (cpt == CPT && u == U && minb == MB) { \
if (q32) crbw_bwd<CPT, 128, U, true, MB><<<(N / 128) * 3, 512 / CPT, 0, st>>>(P(qkv), w.data_ptr<float>(), \
m1.data_ptr<float>(), m2.data_ptr<float>(), cs.data_ptr<float>(), sn.data_ptr<float>(), gq.data_ptr(), \
P(gk), P(gv), P(dqkv), part.data_ptr<float>(), T, (float)qscale); \
else crbw_bwd<CPT, 128, U, false, MB><<<(N / 128) * 3, 512 / CPT, 0, st>>>(P(qkv), w.data_ptr<float>(), \
m1.data_ptr<float>(), m2.data_ptr<float>(), cs.data_ptr<float>(), sn.data_ptr<float>(), gq.data_ptr(), \
P(gk), P(gv), P(dqkv), part.data_ptr<float>(), T, (float)qscale); \
C10_CUDA_KERNEL_LAUNCH_CHECK(); ok = true; }
// q32: gq is the (B, T, 8, 64) fp32 raw dQ accumulator; qscale folds in the softmax scale.
// wsum: 0 -> return part (N/128, 1536, 3) like the old op; 1 -> fused fp64 reduction, returns dw (1536, 3).
std::vector<torch::Tensor> bwd(torch::Tensor qkv, torch::Tensor w, torch::Tensor m1, torch::Tensor m2,
torch::Tensor cs, torch::Tensor sn, torch::Tensor gq, torch::Tensor gk,
torch::Tensor gv, double qscale, bool q32, int64_t cpt, int64_t u, int64_t minb, bool wsum) {
const int B = qkv.size(0), T = qkv.size(1); const long N = (long)B * T;
TORCH_CHECK(T % 128 == 0, "T must be a multiple of 128");
TORCH_CHECK(gq.is_contiguous() && gk.is_contiguous() && gv.is_contiguous());
TORCH_CHECK(!q32 || gq.scalar_type() == at::kFloat, "gq32 must be fp32");
auto dqkv = torch::empty_like(qkv);
auto part = torch::empty({N / 128, 1536, 3}, qkv.options().dtype(torch::kFloat));
auto st = at::cuda::getCurrentCUDAStream();
bool ok = false;
BWD_CFG(4, 2, 1) BWD_CFG(4, 2, 5) BWD_CFG(4, 2, 6) BWD_CFG(4, 1, 6) BWD_CFG(4, 1, 1) BWD_CFG(2, 2, 3)
BWD_CFG(2, 1, 3) BWD_CFG(2, 2, 1) BWD_CFG(4, 4, 5)
TORCH_CHECK(ok, "unsupported bwd config");
if (!wsum) return {dqkv, part};
auto dw = torch::empty({1536, 3}, part.options());
crbw_wsum<<<(4608 + 127) / 128, 128, 0, st>>>(part.data_ptr<float>(), dw.data_ptr<float>(), (int)(N / 128));
C10_CUDA_KERNEL_LAUNCH_CHECK();
return {dqkv, dw};
}
"""
_here = os.path.dirname(os.path.abspath(__file__))
_build = os.path.join(_here, "build_canon_bw"); os.makedirs(_build, exist_ok=True)
os.environ.setdefault("TORCH_CUDA_ARCH_LIST", "7.0")
ext = load_inline("boris_volta_canon_bw", cpp_sources=
"std::vector<torch::Tensor> fwd(torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, int64_t, int64_t, int64_t, int64_t);"
"std::vector<torch::Tensor> bwd(torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, double, bool, int64_t, int64_t, int64_t, bool);",
cuda_sources=SRC, functions=["fwd", "bwd"], extra_cuda_cflags=["-O3", "-arch=sm_70"],
build_directory=_build)
|