1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
|
"""Model-side additions adopted from the kernel swarm, integrated on the merged
(BORIS_VOLTA_KERNELS=1) path. Everything here is used by model.py's
`_forward_merged`; nothing changes the forward values (the loss is bitwise the
live one).
1. wgrad_acc (block_autograd b, = resid_fp32 V1 = precision_plus dw):
the weight-gradient GEMM of every trunk matmul accumulates straight into
the fp32 p.grad (cuBLAS beta=1, fp32 output). Removes the fp16 rounding of
dW (and of its split-k partials), the fp16->fp32 cast and AccumulateGrad's
add_. dW error vs fp64 ~100x lower.
2. norm_bwd (elem_audit): residual-add + RMSNorm backward and the norm-weight
gradient in ONE Triton pass (16-row blocks, dw partials in registers)
instead of Inductor's row kernel + a split column reduction that re-reads
the same 100-134 MB. Row arithmetic replicates Inductor's live kernel, so
dh/dh16 (and hence every other gradient) are bitwise the live ones.
(3. the CUTLASS SwiGLU epilogue was integrated, measured bitwise but not
faster on the whole micro-step, and dropped -- see ../epi_dropped.py.)
4. TiedEmbedding (ngram_embed A): the head's dW*s, the embedding's segment
sums and the .grad accumulate in one pass (bitwise).
"""
from __future__ import annotations
import torch
import torch.nn.functional as F
import triton
import triton.language as tl
from torch.library import triton_op, wrap_triton
F16, F32 = torch.float16, torch.float32
# --------------------------------------------------------------------------- #
# 1. weight gradients accumulated by the GEMM into fp32 .grad
# --------------------------------------------------------------------------- #
@torch.library.custom_op("integ::wgrad_acc", mutates_args=("acc",))
def wgrad_acc(dy: torch.Tensor, x: torch.Tensor, acc: torch.Tensor, alpha: float) -> None:
dy2 = dy.reshape(-1, dy.shape[-1])
x2 = x.reshape(-1, x.shape[-1])
torch.addmm(acc, dy2.t(), x2, beta=1, alpha=alpha, out_dtype=F32, out=acc)
@wgrad_acc.register_fake
def _(dy, x, acc, alpha):
return None
class LinAcc(torch.autograd.Function):
"""y = x16 @ w16^T. The weight gradient alpha * dy^T x16 is ADDED into `acc`
(the fp32 p.grad) by the GEMM itself and not returned to autograd. w16 is
the (pre-scaled) fp16 weight; alpha undoes nothing -- it is the scale the
forward applied to the fp32 weight (DOWN_SHRINK), i.e. d(w*s)/dw."""
@staticmethod
def forward(ctx, x16, w16, acc, alpha: float):
ctx.save_for_backward(x16, w16, acc)
ctx.alpha = alpha
return F.linear(x16, w16)
@staticmethod
def backward(ctx, dy):
x16, w16, acc = ctx.saved_tensors
torch.ops.integ.wgrad_acc(dy, x16, acc, ctx.alpha)
return dy @ w16, None, None, None
# --------------------------------------------------------------------------- #
# 2. fused residual-add + RMSNorm backward with the norm-weight gradient
# --------------------------------------------------------------------------- #
@triton.jit
def _norm_bwd_bw_kernel(h, a, rstd, w, gres, gx, dh, dh16, dwp, ROWS: tl.constexpr,
D: tl.constexpr, HAS_A: tl.constexpr, OSCALE: tl.constexpr):
"""Row-for-row the arithmetic of Inductor's live norm-backward row kernel
(triton_per_fused__to_copy_add_div_mul_pow_sum_view_*: one 512-wide row as a
1-D block, 4 warps, num_stages 1, the same op DAG and so the same FMA
contractions), so dh and dh16 are BITWISE the live ones and every other
gradient stays bitwise too. The only change: the norm-weight gradient
sum_rows(gx * (h1*rstd)) -- the same per-element products as Inductor's
separate reduction kernel -- is accumulated here in fp32 over ROWS rows
(partials then summed in fp64), instead of by a second pass that re-reads
gx, h and a.
d = g_res + (gx*w)*rstd + ((sum(gx*w*h1)*-0.5)*rstd^3/D) * (2*h1)"""
pid = tl.program_id(0).to(tl.int64)
cols = tl.arange(0, D)
wv = tl.load(w + cols, eviction_policy="evict_last")
acc = tl.zeros([D], tl.float32)
for i in range(ROWS):
r = pid * ROWS + i
off = r * D + cols
g = tl.load(gx + off).to(tl.float32)
hv = tl.load(h + off)
if HAS_A:
hv = hv + tl.load(a + off).to(tl.float32)
gr = tl.load(gres + off)
rs = tl.load(rstd + r)
gw = g * wv
sm = tl.sum(gw * hv, 0)
t = gr + gw * rs
k = ((sm * -0.5) * (rs * rs * rs)) * (1.0 / D)
d = t + k * (hv * 2.0)
tl.store(dh + off, d)
if OSCALE == 1.0:
tl.store(dh16 + off, d.to(tl.float16))
else:
tl.store(dh16 + off, (d * OSCALE).to(tl.float16))
acc += g * (hv * rs)
tl.store(dwp + pid * D + cols, acc)
@triton.jit
def _colsum64_kernel(p, out, NR, D: tl.constexpr, BC: tl.constexpr, BR: tl.constexpr):
"""Stage 1 of an fp64 column sum: out[k, c] = sum of rows k*BR..k*BR+BR-1 of p
(fp32 in, fp64 accumulate and out). Grid (D // BC, cdiv(NR, BR))."""
c = tl.program_id(0) * BC + tl.arange(0, BC)
k = tl.program_id(1)
r = k * BR + tl.arange(0, BR)
x = tl.load(p + r[:, None] * D + c[None, :], mask=r[:, None] < NR, other=0.0)
tl.store(out + k * D + c, tl.sum(x.to(tl.float64), 0))
def _colsum64(p):
"""fp32 (NR, D) -> fp32 (D,) column sums with an fp64 accumulator (deterministic)."""
NR, D = p.shape
BR = 64
nk = triton.cdiv(NR, BR)
part = torch.empty((nk, D), device=p.device, dtype=torch.float64)
wrap_triton(_colsum64_kernel)[(D // 64, nk)](p, part, NR, D=D, BC=64, BR=BR, num_warps=4)
return part.sum(0).to(F32)
_NB_ROWS = 16 # rows per program: fp32 dw partial per 16 rows, partials summed in fp64
def _norm_bwd_launch(h, a, rstd, w, gres, gx, oscale: float, has_a: bool):
N, D = h.shape
dh = torch.empty_like(h)
dh16 = torch.empty((N, D), device=h.device, dtype=F16)
dwp = torch.empty((N // _NB_ROWS, D), device=h.device, dtype=F32)
wrap_triton(_norm_bwd_bw_kernel)[(N // _NB_ROWS,)](
h, a, rstd, w, gres, gx, dh, dh16, dwp, ROWS=_NB_ROWS, D=D,
HAS_A=has_a, OSCALE=oscale, num_warps=4, num_stages=1)
return dh, dh16, _colsum64(dwp)
@triton_op("integ::norm_bwd_a", mutates_args={})
def norm_bwd_a(h: torch.Tensor, a: torch.Tensor, rstd: torch.Tensor, w: torch.Tensor,
gres: torch.Tensor, gx: torch.Tensor, oscale: float
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
return _norm_bwd_launch(h, a, rstd, w, gres, gx, oscale, True)
@triton_op("integ::norm_bwd_h", mutates_args={})
def norm_bwd_h(h: torch.Tensor, rstd: torch.Tensor, w: torch.Tensor,
gres: torch.Tensor, gx: torch.Tensor, oscale: float
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
return _norm_bwd_launch(h, h, rstd, w, gres, gx, oscale, False)
class AddNorm(torch.autograd.Function):
"""(h1, x16) = (h + s*a.float(), fp16(rmsnorm(h1) * w)).
Forward: the same torch ops as the live region, so Inductor emits the same
fused kernel (bitwise). Backward: one Triton pass (_norm_bwd_bw_kernel) that
reproduces the live row kernel bit for bit -- dh1 (fp32, to the residual)
and da16 = fp16(s*dh1) (to the o_proj / down GEMM backward) -- and also
accumulates the norm-weight gradient sum_rows(gx*xhat), which the live graph
computes in a second kernel re-reading gx, h and a.
save_out=False saves (h, a) -- the tensors the live graph already keeps for
this norm (h is the previous region's output, a the o_proj output).
save_out=True saves h1 instead -- for the next block's norm1, where h1 is
this region's output and is kept anyway as the next region's input.
"""
@staticmethod
def forward(ctx, h, a, w, eps: float, s: float, save_out: bool):
h1 = h + a.float() if s == 1.0 else h + a.float() * s
rstd = torch.rsqrt(h1.pow(2).mean(-1, keepdim=True) + eps)
x16 = (h1 * rstd * w.float()).to(F16)
if save_out:
ctx.save_for_backward(h1, rstd, w)
else:
ctx.save_for_backward(h, a, rstd, w)
ctx.s, ctx.save_out = s, save_out
return h1, x16
@staticmethod
def backward(ctx, gh1, gx):
D = gx.shape[-1]
if ctx.save_out:
h1, rstd, w = ctx.saved_tensors
shp = h1.shape
dh, da, dw = torch.ops.integ.norm_bwd_h(
h1.reshape(-1, D), rstd.reshape(-1), w,
gh1.reshape(-1, D).contiguous(), gx.reshape(-1, D).contiguous(), ctx.s)
else:
h, a, rstd, w = ctx.saved_tensors
shp = h.shape
dh, da, dw = torch.ops.integ.norm_bwd_a(
h.reshape(-1, D), a.reshape(-1, D), rstd.reshape(-1), w,
gh1.reshape(-1, D).contiguous(), gx.reshape(-1, D).contiguous(), ctx.s)
return dh.view(shp), da.view(shp), dw, None, None, None
# --------------------------------------------------------------------------- #
# 4. tied embedding: acc + (head dW * s + embedding segment sums) in one pass
# --------------------------------------------------------------------------- #
@triton.jit
def _tied_accum_dense(acc_ptr, dw_ptr, s_ptr, seg_ptr, out_ptr, n,
HAS_ACC: tl.constexpr, BLOCK: tl.constexpr):
offs = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)
m = offs < n
s = tl.load(s_ptr)
v = tl.load(dw_ptr + offs, mask=m) * s + tl.load(seg_ptr + offs, mask=m)
if HAS_ACC:
v = tl.load(acc_ptr + offs, mask=m) + v
tl.store(out_ptr + offs, v, mask=m)
_STASH: dict = {}
def stash_head_grad(W, dW, s):
"""Called by the head's backward in place of returning dW*s for W."""
_STASH[id(W)] = (dW, s)
class TiedEmbedding(torch.autograd.Function):
"""F.embedding(idx, W) for the tied weight. Its backward adds
(head dW * s) + segment sums into W.grad in one pass -- the exact rounding
sequence of the stock path (dW*s rounded, + seg, + acc; FMA contraction
off), in place of: the dW*s pass, autograd's tied-weight sum and
AccumulateGrad's add_. Requires the head to stash its dW (stash=True)."""
@staticmethod
def forward(ctx, idx, W):
ctx.save_for_backward(idx)
ctx.W = W
return F.embedding(idx, W)
@staticmethod
def backward(ctx, g):
(idx,) = ctx.saved_tensors
W = ctx.W
dW, s = _STASH.pop(id(W))
seg = torch.ops.aten.embedding_dense_backward(
g.reshape(-1, g.shape[-1]), idx.reshape(-1), W.shape[0], -1, False)
has = W.grad is not None
out = W.grad if has else seg # first micro-step: in place into seg
n = W.numel()
BLOCK = 2048
_tied_accum_dense[(triton.cdiv(n, BLOCK),)](
out, dW, s.reshape(1).float(), seg, out, n, HAS_ACC=has, BLOCK=BLOCK,
num_warps=4, enable_fp_fusion=False)
if not has:
W.grad = out
return None, None
|