summaryrefslogtreecommitdiff
path: root/volta_kernels/__init__.py
blob: 06b9547e6cc5bf03ff5490f2d026d99fb61e467e (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
"""Hand-written sm_70 kernels for Boris-2. See README.md in this directory.

Built on first use with torch.utils.cpp_extension (nvcc, -arch=sm_70) into
volta_kernels/build/, which is kept between runs so a resume does not recompile.
"""
from __future__ import annotations

import os
from pathlib import Path

import torch

_HERE = Path(__file__).resolve().parent
_ext = None


def _load():
    global _ext
    if _ext is None:
        from torch.utils.cpp_extension import load
        build = _HERE / "build_v4"   # new dir: never relink the .so a running trainer has mapped
        build.mkdir(exist_ok=True)
        os.environ.setdefault("TORCH_CUDA_ARCH_LIST", "7.0")
        _ext = load(name="boris_volta_attn_v4", sources=[str(_HERE / "attn.cu")],
                    extra_cuda_cflags=["-O3", "-arch=sm_70", "-lineinfo"],
                    build_directory=str(build), verbose=False)
    return _ext


def tile_table(offsets: torch.Tensor):
    """(cu, tcum, perm) int32 for a document-offset tensor, cached on the tensor.

    tcum is the cumulative 64-tile count over documents sorted LONGEST FIRST and
    perm maps that order back to document indices, so the heaviest CTAs launch
    first and short documents fill the tail of the grid. Launch order only: the
    results are bitwise-identical to packer order; measured fwd -6.5%, bwd -5.6%.

    One forward calls attention 31 times (plus recomputation) with the same
    offsets, so the table is built once. Pure GPU ops -- no host sync.
    """
    tab = getattr(offsets, "_volta_tab", None)
    # Keyed on the version counter too: an in-place edit of the offsets must
    # never be served a stale table.
    if tab is None or tab[3] != offsets._version:
        cu = offsets.to(torch.int32)
        tiles = (cu[1:] - cu[:-1] + 63) // 64
        order = torch.argsort(tiles, descending=True, stable=True)
        tcum = torch.zeros_like(cu)
        tcum[1:] = torch.cumsum(tiles[order], 0)
        tab = (cu, tcum, order.to(torch.int32), offsets._version)
        offsets._volta_tab = tab
    return tab[0], tab[1], tab[2]


class _VarlenAttn(torch.autograd.Function):
    @staticmethod
    def forward(ctx, q, k, v, cu, tcum, perm, scale):
        ext = _load()
        o, lse = ext.fwd(q, k, v, cu, tcum, perm, scale)
        ctx.save_for_backward(q, k, v, o, lse, cu, tcum, perm)
        ctx.scale = scale
        return o

    @staticmethod
    def backward(ctx, do):
        q, k, v, o, lse, cu, tcum, perm = ctx.saved_tensors
        dq, dk, dv = _load().bwd(do, q, k, v, o, lse, cu, tcum, perm, ctx.scale)
        return dq, dk, dv, None, None, None, None


def varlen_causal_attention(q, k, v, offsets, scale: float | None = None):
    """Causal attention within each document of a packed (N, H, 64) fp16 stream.

    Equivalent to SDPA(is_causal=True) over a jagged nested tensor built from
    `offsets` (the packer's document boundaries into the flattened stream).
    """
    if scale is None:
        scale = q.shape[-1] ** -0.5
    cu, tcum, perm = tile_table(offsets)
    return _VarlenAttn.apply(q, k, v, cu, tcum, perm, scale)


class _CanonRopeAttn(torch.autograd.Function):
    """Canon+RoPE and varlen attention as ONE autograd Function.

    Kept separate, the attention backward rounded dQ to fp16 in its own pass
    (bwd_post_kernel) and the Canon backward read it back: ~100 MB of traffic
    per layer and an fp16 rounding of every dQ. Merged, the Canon backward
    reads the raw fp32 dQ accumulator and applies the (power-of-two) softmax
    scale itself. Measured: dqkv error vs fp64 1.2038e-3 -> 1.1955e-3, region
    -1.0% at micro_batch 16.
    """

    @staticmethod
    def forward(ctx, qkv, w, m1, m2, cos, sin, offsets, scale):
        from .canon_rope_bw import ext as canon_ext
        B, T, _ = qkv.shape
        N = B * T
        m1f, m2f = m1.reshape(N).contiguous(), m2.reshape(N).contiguous()
        cf, sf = cos.reshape(N, 32).contiguous(), sin.reshape(N, 32).contiguous()
        # canon_rope_bw (swarm canon_bw): bitwise-identical q/k/v, fwd -33% / bwd -17%
        q, k, v = canon_ext.fwd(qkv, w.contiguous(), m1f, m2f, cf, sf, 4, 64, 4, 1)
        cu, tcum, perm = tile_table(offsets)
        qv, kv, vv = q.view(N, 8, 64), k.view(N, 8, 64), v.view(N, 8, 64)
        o, lse = _load().fwd(qv, kv, vv, cu, tcum, perm, scale)
        ctx.save_for_backward(qkv, w, m1f, m2f, cf, sf, qv, kv, vv, o, lse, cu, tcum, perm)
        ctx.scale = scale
        return o

    @staticmethod
    def backward(ctx, do):
        from .canon_rope_bw import ext as canon_ext
        qkv, w, m1f, m2f, cf, sf, q, k, v, o, lse, cu, tcum, perm = ctx.saved_tensors
        B, T, _ = qkv.shape
        dqacc, dk, dv = _load().bwd_raw(do, q, k, v, o, lse, cu, tcum, perm, ctx.scale)
        dqkv, part = canon_ext.bwd(qkv, w.contiguous(), m1f, m2f, cf, sf,
                                   dqacc.view(B, T, 8, 64), dk.view(B, T, 8, 64),
                                   dv.view(B, T, 8, 64), ctx.scale, True, 4, 2, 1, False)
        return dqkv, part.sum(0).to(w.dtype), None, None, None, None, None, None


def canon_rope_attention(qkv, canon_w, m1, m2, cos, sin, offsets):
    """qkv (B, T, 1536) fp16 -> Canon + RoPE -> causal varlen attention -> (N, 8, 64)."""
    return _CanonRopeAttn.apply(qkv, canon_w, m1, m2, cos, sin, offsets, 0.125)