1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
|
"""Fused LM head (cross-entropy without (N, V) logits), with the elementwise
middle rewritten as one Triton pass.
Same algorithm, same constants and the same operand precision as
model._FusedLMHead -- read that docstring for why GRAD_SCALE is 64, why the dW
GEMM is shrunk, and why the backward multiplier stays fp32. What changes is only
how many times each chunk's logits cross memory.
Production spelling, per 1024-row chunk at micro_batch 16 (x fp32 because the
residual stream is fp32):
F.linear(xc, Wc) autocast casts W (32768x512 fp32) to fp16 -- again
.float() fp16 logits -> fp32 copy 128 MB
_head_chunk_c 3 passes over fp32 logits, writes fp32 p 128 MB
scatter_add_ one-hot, in place
p.to(x.dtype) a no-op: x is fp32, so g is fp32
F.linear(g, Wc.t()) autocast casts g to fp16 and W to fp16 again
dx[a:b] = ... fp16 -> fp32 copy into dx
g * DW_SHRINK another fp32 pass 128 MB
gd.t() @ xn autocast casts gd^T (strided) to fp16
Here W and x are cast once per call, not twice per chunk, and one kernel reads
the fp16 logits once and writes the two fp16 operands the GEMMs were being fed
anyway: fp16(p'), where p' is the scaled softmax minus the one-hot. dW is
accumulated with an fp32-output GEMM, so it is never rounded to fp16 at all --
the production path's overflow guard (x rescale, DW_SHRINK) has nothing left to
guard. lse differs from Inductor's only in fp32 reduction order.
"""
from __future__ import annotations
import torch
import torch.nn.functional as F
import triton
import triton.language as tl
IGNORE = -100
@triton.jit
def _softmax_grad_kernel(logits, tgt, g_out, row_loss, V: tl.constexpr, S: tl.constexpr):
"""One program per row: the whole 32k-wide row stays in registers.
g_out = fp16(S * softmax - S * onehot), the operand both head GEMMs take.
(A blocked two-pass version with online max/sum measured no faster.)
"""
row = tl.program_id(0).to(tl.int64)
cols = tl.arange(0, V)
x = tl.load(logits + row * V + cols).to(tl.float32)
t = tl.load(tgt + row)
valid = t != -100
safe = tl.where(valid, t, 0)
# Inductor's logsumexp decomposition, including its guard for an
# all-infinite row.
m = tl.max(x, 0)
m = tl.where(tl.abs(m) == float("inf"), 0.0, m)
lse = tl.log(tl.sum(tl.exp(x - m), 0)) + m
vs = valid.to(tl.float32) * S
p = tl.exp(x - lse) * vs
hit = cols == safe
p = tl.where(hit, p + (-S * valid.to(tl.float32)), p)
tl.store(g_out + row * V + cols, p.to(tl.float16))
picked = tl.sum(tl.where(hit, x, 0.0), 0)
tl.store(row_loss + row, tl.where(valid, lse - picked, 0.0))
class FusedLMHead(torch.autograd.Function):
GRAD_SCALE = 64.0
@staticmethod
def forward(ctx, x, W, targets, chunk: int):
N, d = x.shape
V = W.shape[0]
W16 = W.to(torch.float16)
x16 = x.to(torch.float16)
valid = targets != IGNORE
n_valid = valid.sum().clamp_min(1)
# dx is written fp32 straight from the GEMM's fp32 accumulator. With an
# fp16 output cuBLAS may pick split-k kernels that sum partials in fp16
# (allowed by reduced-precision reduction); fp32 output rules that out.
# Measured on the real fixture: dx error vs fp64 5.31e-4 -> 3.43e-4,
# and 2.9% faster.
dx16 = torch.empty((N, d), device=x.device, dtype=torch.float32)
# zeros, not empty: do not rely on beta=0 never reading the buffer in the
# out_dtype addmm path (0.07 ms per micro-step)
dW = torch.zeros_like(W, dtype=torch.float32)
row_loss = torch.empty(N, device=x.device, dtype=torch.float32)
c = min(chunk, N)
logits = torch.empty((c, V), device=x.device, dtype=torch.float16)
g = torch.empty_like(logits)
S = FusedLMHead.GRAD_SCALE
for a in range(0, N, chunk):
b = min(a + chunk, N)
n = b - a
lg = logits[:n]
torch.mm(x16[a:b], W16.t(), out=lg)
_softmax_grad_kernel[(n,)](lg, targets[a:b], g[:n], row_loss[a:b],
V=V, S=S, num_warps=16)
torch.mm(g[:n], W16, out_dtype=torch.float32, out=dx16[a:b])
# dW: fp16 operands, fp32 accumulation AND fp32 output (cuBLAS
# s884gemm + fp32 split-k reduce), accumulated across chunks with
# beta=1. The production head rounds every chunk's dW to fp16 and so
# needs the |x| <= 1 rescale and DW_SHRINK to keep that output from
# overflowing; with an fp32 output there is nothing to overflow.
# Strictly more precise (dW rel-L2 vs fp64 5.24e-4 -> 5.01e-4 on
# the real fixture), 12.6% faster, one fewer 64 MB write per chunk.
torch.addmm(dW, g[:n].t(), x16[a:b], beta=0 if a == 0 else 1,
out_dtype=torch.float32, out=dW)
ctx.w_dtype = W.dtype
ctx.x_dtype = x.dtype
ctx.W_ref = W
ctx.save_for_backward(dx16, dW)
ctx.n_valid = n_valid
return row_loss.sum() / n_valid
@staticmethod
def backward(ctx, grad_out):
dx16, dW = ctx.saved_tensors
s = grad_out / (ctx.n_valid * FusedLMHead.GRAD_SCALE)
return ((dx16.float() * s).to(ctx.x_dtype), (dW * s).to(ctx.w_dtype),
None, None)
class FusedLMHeadStash(FusedLMHead):
"""Same forward. The backward hands (dW, s) to the tied embedding's backward
(integ_ops.TiedEmbedding), which folds dW*s, the embedding segment sums and
the .grad accumulate into one pass, instead of returning dW*s to autograd."""
@staticmethod
def backward(ctx, grad_out):
from .integ_ops import stash_head_grad
dx16, dW = ctx.saved_tensors
s = grad_out / (ctx.n_valid * FusedLMHead.GRAD_SCALE)
stash_head_grad(ctx.W_ref, dW, s)
return (dx16.float() * s).to(ctx.x_dtype), None, None, None
def _loss_only(x, W, targets, chunk):
N = x.shape[0]
V = W.shape[0]
W16 = W.to(torch.float16)
x16 = x.to(torch.float16)
n_valid = (targets != IGNORE).sum().clamp_min(1)
row_loss = torch.empty(N, device=x.device, dtype=torch.float32)
c = min(chunk, N)
logits = torch.empty((c, V), device=x.device, dtype=torch.float16)
g = torch.empty_like(logits)
for a in range(0, N, chunk):
b = min(a + chunk, N)
n = b - a
torch.mm(x16[a:b], W16.t(), out=logits[:n])
_softmax_grad_kernel[(n,)](logits[:n], targets[a:b], g[:n], row_loss[a:b],
V=V, S=FusedLMHead.GRAD_SCALE, num_warps=16)
return row_loss.sum() / n_valid
def fused_lm_loss(x, W, targets, chunk: int = 1024, stash: bool = False):
if not torch.is_grad_enabled():
# Validation (quick_val runs under no_grad): no backward will come, so
# skip the dx and dW GEMMs entirely -- 2.25x faster head. Same logits
# GEMM and softmax kernel, so the loss is bitwise the training path's.
# (Checked here, not inside the Function: its forward always runs with
# grad mode off, and the tied weight keeps requires_grad under no_grad.)
return _loss_only(x, W, targets, chunk)
if stash:
return FusedLMHeadStash.apply(x, W, targets, chunk)
return FusedLMHead.apply(x, W, targets, chunk)
|