1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
|
"""Canon + RoPE in CUDA: a sliding window down the sequence (drop-in for canon_rope.py).
Written by a second Claude Code reviewer on 2026-09-27; adopted after review.
Each thread owns 8 channels and walks 64 (fwd) / 128 (bwd, backwards) tokens,
keeping the two previous tokens in registers instead of re-reading neighbours
as the Triton version does, with 16-byte loads and stores throughout.
Measured at full size (31 layers, mb 16, idle V100): fwd 0.61 -> 0.32 ms,
bwd 1.20 -> 0.57 ms, whole micro-step +4.1%. q, k, v, dqkv and the loss are
bitwise-identical to the Triton kernel (its RoPE rounding order is copied:
fmaf(y, c, -(p*s))); the Canon weight gradient differs only in fp32 summation
order (same error vs fp64). Requires T % 128 == 0 (2048 and 8192 both are).
"""
import os, torch
from torch.utils.cpp_extension import load_inline
SRC = r"""
#include <torch/extension.h>
#include <ATen/cuda/CUDAContext.h>
#include <cuda_fp16.h>
constexpr int FCH = 64; // tokens per block (sliding window down the sequence)
__global__ void __launch_bounds__(192) canon_rope_fwd(
const half* __restrict__ qkv, const float* __restrict__ w, const float* __restrict__ m1,
const float* __restrict__ m2, const float* __restrict__ cs, const float* __restrict__ sn,
half* __restrict__ q, half* __restrict__ k, half* __restrict__ v, int T) {
const int c8 = threadIdx.x, ch = 8 * c8, g = ch >> 9, col = ch & 511, d0 = col & 63;
const long n0 = (long)blockIdx.x * FCH;
const int t0 = n0 % T;
float w0[8], w1[8], w2[8];
#pragma unroll
for (int i = 0; i < 8; ++i) { // the region casts canon_w to fp16
w0[i] = __half2float(__float2half(w[(ch + i) * 3 + 0]));
w1[i] = __half2float(__float2half(w[(ch + i) * 3 + 1]));
w2[i] = __half2float(__float2half(w[(ch + i) * 3 + 2]));
}
auto ld = [&](long n, float* x) {
uint4 u = *reinterpret_cast<const uint4*>(qkv + n * 1536 + ch);
const __half2* h = reinterpret_cast<const __half2*>(&u);
#pragma unroll
for (int i = 0; i < 4; ++i) { float2 f = __half22float2(h[i]); x[2*i] = f.x; x[2*i+1] = f.y; }
};
float x1[8], x2[8], x0[8];
#pragma unroll
for (int i = 0; i < 8; ++i) x1[i] = x2[i] = 0.f;
if (t0 >= 1) ld(n0 - 1, x1);
if (t0 >= 2) ld(n0 - 2, x2);
half* out = g == 0 ? q : (g == 1 ? k : v);
#pragma unroll 8
for (int s = 0; s < FCH; ++s) {
const long n = n0 + s;
ld(n, x0);
const float a1 = m1[n], a2 = m2[n];
float y[8];
#pragma unroll
for (int i = 0; i < 8; ++i)
y[i] = x0[i] + x0[i] * w0[i] + (x1[i] * a1) * w1[i] + (x2[i] * a2) * w2[i];
float o[8];
if (g < 2) {
const int d = d0 & 31;
const float4* cp = reinterpret_cast<const float4*>(cs + n * 32 + d);
const float4* sp = reinterpret_cast<const float4*>(sn + n * 32 + d);
float4 c0 = cp[0], c1 = cp[1], s0 = sp[0], s1 = sp[1];
float c[8] = {c0.x, c0.y, c0.z, c0.w, c1.x, c1.y, c1.z, c1.w};
float sv[8] = {s0.x, s0.y, s0.z, s0.w, s1.x, s1.y, s1.z, s1.w};
#pragma unroll
for (int i = 0; i < 8; ++i) {
float p = __shfl_xor_sync(0xffffffff, y[i], 4); // partner channel d +/- 32
o[i] = d0 < 32 ? fmaf(y[i], c[i], -(p * sv[i])) : fmaf(y[i], c[i], p * sv[i]);
}
} else {
#pragma unroll
for (int i = 0; i < 8; ++i) o[i] = y[i];
}
uint4 u;
__half2* h = reinterpret_cast<__half2*>(&u);
#pragma unroll
for (int i = 0; i < 4; ++i) h[i] = __floats2half2_rn(o[2*i], o[2*i+1]);
*reinterpret_cast<uint4*>(out + n * 512 + col) = u;
#pragma unroll
for (int i = 0; i < 8; ++i) { x2[i] = x1[i]; x1[i] = x0[i]; }
}
}
std::vector<torch::Tensor> fwd(torch::Tensor qkv, torch::Tensor w, torch::Tensor m1, torch::Tensor m2,
torch::Tensor cs, torch::Tensor sn) {
const int B = qkv.size(0), T = qkv.size(1); const long N = (long)B * T;
TORCH_CHECK(T % FCH == 0);
auto q = torch::empty({B, T, 8, 64}, qkv.options()), k = torch::empty_like(q), v = torch::empty_like(q);
canon_rope_fwd<<<N / FCH, 192, 0, at::cuda::getCurrentCUDAStream()>>>(
(const half*)qkv.data_ptr(), w.data_ptr<float>(), m1.data_ptr<float>(), m2.data_ptr<float>(),
cs.data_ptr<float>(), sn.data_ptr<float>(), (half*)q.data_ptr(), (half*)k.data_ptr(),
(half*)v.data_ptr(), T);
return {q, k, v};
}
constexpr int CH = 128;
__device__ __forceinline__ void ld8(const half* p, float* x) {
uint4 u = *reinterpret_cast<const uint4*>(p);
const __half2* h = reinterpret_cast<const __half2*>(&u);
#pragma unroll
for (int i = 0; i < 4; ++i) { float2 f = __half22float2(h[i]); x[2*i] = f.x; x[2*i+1] = f.y; }
}
// Q32: the q-gradient arrives as the attention backward's raw fp32 dQ
// accumulator (times qscale) instead of an fp16 tensor, so it is never rounded.
template <bool Q32>
__global__ void __launch_bounds__(192) canon_rope_bwd(
const half* __restrict__ qkv, const float* __restrict__ w, const float* __restrict__ m1,
const float* __restrict__ m2, const float* __restrict__ cs, const float* __restrict__ sn,
const void* __restrict__ gq_, const half* __restrict__ gk, const half* __restrict__ gv,
half* __restrict__ dqkv, float* __restrict__ part, int T, float qscale) {
const half* gq = reinterpret_cast<const half*>(gq_);
const float* gq32 = reinterpret_cast<const float*>(gq_);
const int c8 = threadIdx.x, ch = 8 * c8, g = ch >> 9, col = ch & 511, d0 = col & 63;
const long n0 = (long)blockIdx.x * CH;
const int t0 = n0 % T;
const half* G = g == 0 ? gq : (g == 1 ? gk : gv);
float w0[8], w1[8], w2[8];
#pragma unroll
for (int i = 0; i < 8; ++i) {
w0[i] = __half2float(__float2half(w[(ch + i) * 3 + 0]));
w1[i] = __half2float(__float2half(w[(ch + i) * 3 + 1]));
w2[i] = __half2float(__float2half(w[(ch + i) * 3 + 2]));
}
// dy(n): gradient w.r.t. the Canon output at token n, un-rotated.
auto dy = [&](long n, float* out) {
float gg[8];
if (Q32 && g == 0) {
const float4* p4 = reinterpret_cast<const float4*>(gq32 + n * 512 + col);
float4 a = p4[0], b = p4[1];
gg[0] = a.x * qscale; gg[1] = a.y * qscale; gg[2] = a.z * qscale; gg[3] = a.w * qscale;
gg[4] = b.x * qscale; gg[5] = b.y * qscale; gg[6] = b.z * qscale; gg[7] = b.w * qscale;
} else {
ld8(G + n * 512 + col, gg);
}
if (g < 2) {
const int d = d0 & 31;
const float4* cp = reinterpret_cast<const float4*>(cs + n * 32 + d);
const float4* sp = reinterpret_cast<const float4*>(sn + n * 32 + d);
float4 c0 = cp[0], c1 = cp[1], s0 = sp[0], s1 = sp[1];
float cs_[8] = {c0.x, c0.y, c0.z, c0.w, c1.x, c1.y, c1.z, c1.w};
float sn_[8] = {s0.x, s0.y, s0.z, s0.w, s1.x, s1.y, s1.z, s1.w};
#pragma unroll
for (int i = 0; i < 8; ++i) {
float p = __shfl_xor_sync(0xffffffff, gg[i], 4);
float ga = d0 < 32 ? gg[i] : p, gb = d0 < 32 ? p : gg[i];
out[i] = d0 < 32 ? fmaf(ga, cs_[i], gb * sn_[i]) : fmaf(gb, cs_[i], -(ga * sn_[i]));
}
} else {
#pragma unroll
for (int i = 0; i < 8; ++i) out[i] = gg[i];
}
};
const long nl = n0 + CH - 1; // walk from the chunk's last token backwards
const int tl = t0 + CH - 1;
float d0v[8], d1v[8], d2v[8], x0[8], xm1[8], xm2[8];
float mm1n = 0.f, mm2n = 0.f, m2p1 = 0.f; // m2p1 = m2[n+1]
#pragma unroll
for (int i = 0; i < 8; ++i) d1v[i] = d2v[i] = 0.f;
// dy at n+1, n+2 must be computed by every lane (shuffles), masked afterwards.
{ float tmp[8];
long a = (tl + 1 < T) ? nl + 1 : nl; dy(a, tmp);
if (tl + 1 < T) { mm1n = m1[nl + 1]; m2p1 = m2[nl + 1];
#pragma unroll
for (int i = 0; i < 8; ++i) d1v[i] = tmp[i]; }
long b = (tl + 2 < T) ? nl + 2 : nl; dy(b, tmp);
if (tl + 2 < T) { mm2n = m2[nl + 2];
#pragma unroll
for (int i = 0; i < 8; ++i) d2v[i] = tmp[i]; } }
ld8(qkv + nl * 1536 + ch, x0);
ld8(qkv + (nl - 1) * 1536 + ch, xm1); // tl >= 31 so both exist
ld8(qkv + (nl - 2) * 1536 + ch, xm2);
float a0[8], a1[8], a2[8];
#pragma unroll
for (int i = 0; i < 8; ++i) a0[i] = a1[i] = a2[i] = 0.f;
#pragma unroll 8
for (int s = CH - 1; s >= 0; --s) {
const long n = n0 + s; const int t = t0 + s;
dy(n, d0v);
const float m1n = m1[n], m2n = m2[n];
float dx[8];
#pragma unroll
for (int i = 0; i < 8; ++i) {
dx[i] = d0v[i] + d0v[i] * w0[i] + (d1v[i] * mm1n) * w1[i] + (d2v[i] * mm2n) * w2[i];
a0[i] += d0v[i] * x0[i];
a1[i] += d0v[i] * (xm1[i] * m1n);
a2[i] += d0v[i] * (xm2[i] * m2n);
}
uint4 u; __half2* h = reinterpret_cast<__half2*>(&u);
#pragma unroll
for (int i = 0; i < 4; ++i) h[i] = __floats2half2_rn(dx[2*i], dx[2*i+1]);
*reinterpret_cast<uint4*>(dqkv + n * 1536 + ch) = u;
mm2n = m2p1; m2p1 = m2n; mm1n = m1n;
#pragma unroll
for (int i = 0; i < 8; ++i) { d2v[i] = d1v[i]; d1v[i] = d0v[i]; x0[i] = xm1[i]; xm1[i] = xm2[i]; }
if (s > 0) {
if (t - 3 >= 0) ld8(qkv + (n - 3) * 1536 + ch, xm2);
else {
#pragma unroll
for (int i = 0; i < 8; ++i) xm2[i] = 0.f; }
if (t - 1 < 1) { // next token (t-1) is t==0: its t-1 tap is outside the row
#pragma unroll
for (int i = 0; i < 8; ++i) xm1[i] = 0.f; }
}
}
float* pp = part + (long)blockIdx.x * 1536 * 3 + ch * 3;
#pragma unroll
for (int i = 0; i < 8; ++i) { pp[3*i] = a0[i]; pp[3*i+1] = a1[i]; pp[3*i+2] = a2[i]; }
}
std::vector<torch::Tensor> bwd(torch::Tensor qkv, torch::Tensor w, torch::Tensor m1, torch::Tensor m2,
torch::Tensor cs, torch::Tensor sn, torch::Tensor gq, torch::Tensor gk,
torch::Tensor gv) {
const int B = qkv.size(0), T = qkv.size(1); const long N = (long)B * T;
auto dqkv = torch::empty_like(qkv);
auto part = torch::empty({N / CH, 1536, 3}, qkv.options().dtype(torch::kFloat));
canon_rope_bwd<false><<<N / CH, 192, 0, at::cuda::getCurrentCUDAStream()>>>(
(const half*)qkv.data_ptr(), w.data_ptr<float>(), m1.data_ptr<float>(), m2.data_ptr<float>(),
cs.data_ptr<float>(), sn.data_ptr<float>(), gq.data_ptr(), (const half*)gk.data_ptr(),
(const half*)gv.data_ptr(), (half*)dqkv.data_ptr(), part.data_ptr<float>(), T, 1.f);
C10_CUDA_KERNEL_LAUNCH_CHECK();
return {dqkv, part};
}
// gq32: (B, T, 8, 64) fp32 raw dQ accumulator; qscale folds in the softmax scale.
std::vector<torch::Tensor> bwd_q32(torch::Tensor qkv, torch::Tensor w, torch::Tensor m1, torch::Tensor m2,
torch::Tensor cs, torch::Tensor sn, torch::Tensor gq32, torch::Tensor gk,
torch::Tensor gv, double qscale) {
TORCH_CHECK(gq32.scalar_type() == at::kFloat && gq32.is_contiguous(), "gq32 must be contiguous fp32");
const int B = qkv.size(0), T = qkv.size(1); const long N = (long)B * T;
TORCH_CHECK(T % CH == 0, "T must be a multiple of 128");
auto dqkv = torch::empty_like(qkv);
auto part = torch::empty({N / CH, 1536, 3}, qkv.options().dtype(torch::kFloat));
canon_rope_bwd<true><<<N / CH, 192, 0, at::cuda::getCurrentCUDAStream()>>>(
(const half*)qkv.data_ptr(), w.data_ptr<float>(), m1.data_ptr<float>(), m2.data_ptr<float>(),
cs.data_ptr<float>(), sn.data_ptr<float>(), gq32.data_ptr(), (const half*)gk.data_ptr(),
(const half*)gv.data_ptr(), (half*)dqkv.data_ptr(), part.data_ptr<float>(), T, (float)qscale);
C10_CUDA_KERNEL_LAUNCH_CHECK();
return {dqkv, part};
}
"""
_here = os.path.dirname(os.path.abspath(__file__)); os.makedirs(os.path.join(_here, "build_canon"), exist_ok=True)
os.environ.setdefault("TORCH_CUDA_ARCH_LIST", "7.0")
ext = load_inline("boris_volta_canon", cpp_sources=
"std::vector<torch::Tensor> fwd(torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor);"
"std::vector<torch::Tensor> bwd(torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor);"
"std::vector<torch::Tensor> bwd_q32(torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, double);",
cuda_sources=SRC, functions=["fwd", "bwd", "bwd_q32"], extra_cuda_cflags=["-O3", "-arch=sm_70"],
build_directory=os.path.join(_here, "build_canon"))
class CanonRopeFast(torch.autograd.Function):
@staticmethod
def forward(ctx, qkv, canon_w, m1, m2, cos, sin):
B, T, C = qkv.shape; N = B * T
assert C == 1536 and qkv.dtype == torch.float16 and qkv.is_contiguous() and T % 128 == 0
m1f, m2f = m1.reshape(N).contiguous(), m2.reshape(N).contiguous()
cf, sf = cos.reshape(N, 32).contiguous(), sin.reshape(N, 32).contiguous()
q, k, v = ext.fwd(qkv, canon_w.contiguous(), m1f, m2f, cf, sf)
ctx.save_for_backward(qkv, canon_w, m1f, m2f, cf, sf)
return q, k, v
@staticmethod
def backward(ctx, gq, gk, gv):
qkv, w, m1f, m2f, cf, sf = ctx.saved_tensors
gq, gk, gv = (g.to(torch.float16).contiguous() for g in (gq, gk, gv))
dqkv, part = ext.bwd(qkv, w.contiguous(), m1f, m2f, cf, sf, gq, gk, gv)
# fp32 straight from the fp32 reduction. Rounding through fp16 (as the
# eager region's cast implied) left ~3% of entries subnormal on the live
# checkpoint; keeping fp32 is strictly more precise.
return dqkv, part.sum(0).to(w.dtype), None, None, None, None
def canon_rope(qkv, canon_w, m1, m2, cos, sin):
return CanonRopeFast.apply(qkv, canon_w, m1, m2, cos, sin)
|