summaryrefslogtreecommitdiff
path: root/volta_kernels/integ_ops.py
blob: 8f05f01d6ee302f66b759dc344d5ea705f7c0f9a (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
"""Model-side additions adopted from the kernel swarm, integrated on the merged
(BORIS_VOLTA_KERNELS=1) path. Everything here is used by model.py's
`_forward_merged`; nothing changes the forward values (the loss is bitwise the
live one).

1. wgrad_acc (block_autograd b, = resid_fp32 V1 = precision_plus dw):
   the weight-gradient GEMM of every trunk matmul accumulates straight into
   the fp32 p.grad (cuBLAS beta=1, fp32 output). Removes the fp16 rounding of
   dW (and of its split-k partials), the fp16->fp32 cast and AccumulateGrad's
   add_. dW error vs fp64 ~100x lower.
2. norm_bwd (elem_audit): residual-add + RMSNorm backward and the norm-weight
   gradient in ONE Triton pass (16-row blocks, dw partials in registers)
   instead of Inductor's row kernel + a split column reduction that re-reads
   the same 100-134 MB. Row arithmetic replicates Inductor's live kernel, so
   dh/dh16 (and hence every other gradient) are bitwise the live ones.
   (3. the CUTLASS SwiGLU epilogue was integrated, measured bitwise but not
   faster on the whole micro-step, and dropped -- see ../epi_dropped.py.)
4. TiedEmbedding (ngram_embed A): the head's dW*s, the embedding's segment
   sums and the .grad accumulate in one pass (bitwise).
"""
from __future__ import annotations

import torch
import torch.nn.functional as F
import triton
import triton.language as tl
from torch.library import triton_op, wrap_triton

F16, F32 = torch.float16, torch.float32


# --------------------------------------------------------------------------- #
# 1. weight gradients accumulated by the GEMM into fp32 .grad
# --------------------------------------------------------------------------- #
@torch.library.custom_op("integ::wgrad_acc", mutates_args=("acc",))
def wgrad_acc(dy: torch.Tensor, x: torch.Tensor, acc: torch.Tensor, alpha: float) -> None:
    dy2 = dy.reshape(-1, dy.shape[-1])
    x2 = x.reshape(-1, x.shape[-1])
    torch.addmm(acc, dy2.t(), x2, beta=1, alpha=alpha, out_dtype=F32, out=acc)


@wgrad_acc.register_fake
def _(dy, x, acc, alpha):
    return None


class LinAcc(torch.autograd.Function):
    """y = x16 @ w16^T. The weight gradient alpha * dy^T x16 is ADDED into `acc`
    (the fp32 p.grad) by the GEMM itself and not returned to autograd. w16 is
    the (pre-scaled) fp16 weight; alpha undoes nothing -- it is the scale the
    forward applied to the fp32 weight (DOWN_SHRINK), i.e. d(w*s)/dw."""

    @staticmethod
    def forward(ctx, x16, w16, acc, alpha: float):
        ctx.save_for_backward(x16, w16, acc)
        ctx.alpha = alpha
        return F.linear(x16, w16)

    @staticmethod
    def backward(ctx, dy):
        x16, w16, acc = ctx.saved_tensors
        torch.ops.integ.wgrad_acc(dy, x16, acc, ctx.alpha)
        return dy @ w16, None, None, None


# --------------------------------------------------------------------------- #
# 2. fused residual-add + RMSNorm backward with the norm-weight gradient
# --------------------------------------------------------------------------- #
@triton.jit
def _norm_bwd_bw_kernel(h, a, rstd, w, gres, gx, dh, dh16, dwp, ROWS: tl.constexpr,
                        D: tl.constexpr, HAS_A: tl.constexpr, OSCALE: tl.constexpr):
    """Row-for-row the arithmetic of Inductor's live norm-backward row kernel
    (triton_per_fused__to_copy_add_div_mul_pow_sum_view_*: one 512-wide row as a
    1-D block, 4 warps, num_stages 1, the same op DAG and so the same FMA
    contractions), so dh and dh16 are BITWISE the live ones and every other
    gradient stays bitwise too. The only change: the norm-weight gradient
    sum_rows(gx * (h1*rstd)) -- the same per-element products as Inductor's
    separate reduction kernel -- is accumulated here in fp32 over ROWS rows
    (partials then summed in fp64), instead of by a second pass that re-reads
    gx, h and a.
        d = g_res + (gx*w)*rstd + ((sum(gx*w*h1)*-0.5)*rstd^3/D) * (2*h1)"""
    pid = tl.program_id(0).to(tl.int64)
    cols = tl.arange(0, D)
    wv = tl.load(w + cols, eviction_policy="evict_last")
    acc = tl.zeros([D], tl.float32)
    for i in range(ROWS):
        r = pid * ROWS + i
        off = r * D + cols
        g = tl.load(gx + off).to(tl.float32)
        hv = tl.load(h + off)
        if HAS_A:
            hv = hv + tl.load(a + off).to(tl.float32)
        gr = tl.load(gres + off)
        rs = tl.load(rstd + r)
        gw = g * wv
        sm = tl.sum(gw * hv, 0)
        t = gr + gw * rs
        k = ((sm * -0.5) * (rs * rs * rs)) * (1.0 / D)
        d = t + k * (hv * 2.0)
        tl.store(dh + off, d)
        if OSCALE == 1.0:
            tl.store(dh16 + off, d.to(tl.float16))
        else:
            tl.store(dh16 + off, (d * OSCALE).to(tl.float16))
        acc += g * (hv * rs)
    tl.store(dwp + pid * D + cols, acc)


@triton.jit
def _colsum64_kernel(p, out, NR, D: tl.constexpr, BC: tl.constexpr, BR: tl.constexpr):
    """Stage 1 of an fp64 column sum: out[k, c] = sum of rows k*BR..k*BR+BR-1 of p
    (fp32 in, fp64 accumulate and out). Grid (D // BC, cdiv(NR, BR))."""
    c = tl.program_id(0) * BC + tl.arange(0, BC)
    k = tl.program_id(1)
    r = k * BR + tl.arange(0, BR)
    x = tl.load(p + r[:, None] * D + c[None, :], mask=r[:, None] < NR, other=0.0)
    tl.store(out + k * D + c, tl.sum(x.to(tl.float64), 0))


def _colsum64(p):
    """fp32 (NR, D) -> fp32 (D,) column sums with an fp64 accumulator (deterministic)."""
    NR, D = p.shape
    BR = 64
    nk = triton.cdiv(NR, BR)
    part = torch.empty((nk, D), device=p.device, dtype=torch.float64)
    wrap_triton(_colsum64_kernel)[(D // 64, nk)](p, part, NR, D=D, BC=64, BR=BR, num_warps=4)
    return part.sum(0).to(F32)


_NB_ROWS = 16   # rows per program: fp32 dw partial per 16 rows, partials summed in fp64


def _norm_bwd_launch(h, a, rstd, w, gres, gx, oscale: float, has_a: bool):
    N, D = h.shape
    dh = torch.empty_like(h)
    dh16 = torch.empty((N, D), device=h.device, dtype=F16)
    dwp = torch.empty((N // _NB_ROWS, D), device=h.device, dtype=F32)
    wrap_triton(_norm_bwd_bw_kernel)[(N // _NB_ROWS,)](
        h, a, rstd, w, gres, gx, dh, dh16, dwp, ROWS=_NB_ROWS, D=D,
        HAS_A=has_a, OSCALE=oscale, num_warps=4, num_stages=1)
    return dh, dh16, _colsum64(dwp)


@triton_op("integ::norm_bwd_a", mutates_args={})
def norm_bwd_a(h: torch.Tensor, a: torch.Tensor, rstd: torch.Tensor, w: torch.Tensor,
               gres: torch.Tensor, gx: torch.Tensor, oscale: float
               ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
    return _norm_bwd_launch(h, a, rstd, w, gres, gx, oscale, True)


@triton_op("integ::norm_bwd_h", mutates_args={})
def norm_bwd_h(h: torch.Tensor, rstd: torch.Tensor, w: torch.Tensor,
               gres: torch.Tensor, gx: torch.Tensor, oscale: float
               ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
    return _norm_bwd_launch(h, h, rstd, w, gres, gx, oscale, False)


class AddNorm(torch.autograd.Function):
    """(h1, x16) = (h + s*a.float(),  fp16(rmsnorm(h1) * w)).

    Forward: the same torch ops as the live region, so Inductor emits the same
    fused kernel (bitwise). Backward: one Triton pass (_norm_bwd_bw_kernel) that
    reproduces the live row kernel bit for bit -- dh1 (fp32, to the residual)
    and da16 = fp16(s*dh1) (to the o_proj / down GEMM backward) -- and also
    accumulates the norm-weight gradient sum_rows(gx*xhat), which the live graph
    computes in a second kernel re-reading gx, h and a.

    save_out=False saves (h, a) -- the tensors the live graph already keeps for
    this norm (h is the previous region's output, a the o_proj output).
    save_out=True saves h1 instead -- for the next block's norm1, where h1 is
    this region's output and is kept anyway as the next region's input.
    """

    @staticmethod
    def forward(ctx, h, a, w, eps: float, s: float, save_out: bool):
        h1 = h + a.float() if s == 1.0 else h + a.float() * s
        rstd = torch.rsqrt(h1.pow(2).mean(-1, keepdim=True) + eps)
        x16 = (h1 * rstd * w.float()).to(F16)
        if save_out:
            ctx.save_for_backward(h1, rstd, w)
        else:
            ctx.save_for_backward(h, a, rstd, w)
        ctx.s, ctx.save_out = s, save_out
        return h1, x16

    @staticmethod
    def backward(ctx, gh1, gx):
        D = gx.shape[-1]
        if ctx.save_out:
            h1, rstd, w = ctx.saved_tensors
            shp = h1.shape
            dh, da, dw = torch.ops.integ.norm_bwd_h(
                h1.reshape(-1, D), rstd.reshape(-1), w,
                gh1.reshape(-1, D).contiguous(), gx.reshape(-1, D).contiguous(), ctx.s)
        else:
            h, a, rstd, w = ctx.saved_tensors
            shp = h.shape
            dh, da, dw = torch.ops.integ.norm_bwd_a(
                h.reshape(-1, D), a.reshape(-1, D), rstd.reshape(-1), w,
                gh1.reshape(-1, D).contiguous(), gx.reshape(-1, D).contiguous(), ctx.s)
        return dh.view(shp), da.view(shp), dw, None, None, None


# --------------------------------------------------------------------------- #
# 4. tied embedding: acc + (head dW * s + embedding segment sums) in one pass
# --------------------------------------------------------------------------- #
@triton.jit
def _tied_accum_dense(acc_ptr, dw_ptr, s_ptr, seg_ptr, out_ptr, n,
                      HAS_ACC: tl.constexpr, BLOCK: tl.constexpr):
    offs = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)
    m = offs < n
    s = tl.load(s_ptr)
    v = tl.load(dw_ptr + offs, mask=m) * s + tl.load(seg_ptr + offs, mask=m)
    if HAS_ACC:
        v = tl.load(acc_ptr + offs, mask=m) + v
    tl.store(out_ptr + offs, v, mask=m)


_STASH: dict = {}


def stash_head_grad(W, dW, s):
    """Called by the head's backward in place of returning dW*s for W."""
    _STASH[id(W)] = (dW, s)


class TiedEmbedding(torch.autograd.Function):
    """F.embedding(idx, W) for the tied weight. Its backward adds
    (head dW * s) + segment sums into W.grad in one pass -- the exact rounding
    sequence of the stock path (dW*s rounded, + seg, + acc; FMA contraction
    off), in place of: the dW*s pass, autograd's tied-weight sum and
    AccumulateGrad's add_. Requires the head to stash its dW (stash=True)."""

    @staticmethod
    def forward(ctx, idx, W):
        ctx.save_for_backward(idx)
        ctx.W = W
        return F.embedding(idx, W)

    @staticmethod
    def backward(ctx, g):
        (idx,) = ctx.saved_tensors
        W = ctx.W
        dW, s = _STASH.pop(id(W))
        seg = torch.ops.aten.embedding_dense_backward(
            g.reshape(-1, g.shape[-1]), idx.reshape(-1), W.shape[0], -1, False)
        has = W.grad is not None
        out = W.grad if has else seg          # first micro-step: in place into seg
        n = W.numel()
        BLOCK = 2048
        _tied_accum_dense[(triton.cdiv(n, BLOCK),)](
            out, dW, s.reshape(1).float(), seg, out, n, HAS_ACC=has, BLOCK=BLOCK,
            num_warps=4, enable_fp_fusion=False)
        if not has:
            W.grad = out
        return None, None