"""Hand-written sm_70 kernels for Boris-2. See README.md in this directory. Built on first use with torch.utils.cpp_extension (nvcc, -arch=sm_70) into volta_kernels/build/, which is kept between runs so a resume does not recompile. """ from __future__ import annotations import os from pathlib import Path import torch _HERE = Path(__file__).resolve().parent _ext = None def _load(): global _ext if _ext is None: from torch.utils.cpp_extension import load build = _HERE / "build_v4" # new dir: never relink the .so a running trainer has mapped build.mkdir(exist_ok=True) os.environ.setdefault("TORCH_CUDA_ARCH_LIST", "7.0") _ext = load(name="boris_volta_attn_v4", sources=[str(_HERE / "attn.cu")], extra_cuda_cflags=["-O3", "-arch=sm_70", "-lineinfo"], build_directory=str(build), verbose=False) return _ext def tile_table(offsets: torch.Tensor): """(cu, tcum, perm) int32 for a document-offset tensor, cached on the tensor. tcum is the cumulative 64-tile count over documents sorted LONGEST FIRST and perm maps that order back to document indices, so the heaviest CTAs launch first and short documents fill the tail of the grid. Launch order only: the results are bitwise-identical to packer order; measured fwd -6.5%, bwd -5.6%. One forward calls attention 31 times (plus recomputation) with the same offsets, so the table is built once. Pure GPU ops -- no host sync. """ tab = getattr(offsets, "_volta_tab", None) # Keyed on the version counter too: an in-place edit of the offsets must # never be served a stale table. if tab is None or tab[3] != offsets._version: cu = offsets.to(torch.int32) tiles = (cu[1:] - cu[:-1] + 63) // 64 order = torch.argsort(tiles, descending=True, stable=True) tcum = torch.zeros_like(cu) tcum[1:] = torch.cumsum(tiles[order], 0) tab = (cu, tcum, order.to(torch.int32), offsets._version) offsets._volta_tab = tab return tab[0], tab[1], tab[2] class _VarlenAttn(torch.autograd.Function): @staticmethod def forward(ctx, q, k, v, cu, tcum, perm, scale): ext = _load() o, lse = ext.fwd(q, k, v, cu, tcum, perm, scale) ctx.save_for_backward(q, k, v, o, lse, cu, tcum, perm) ctx.scale = scale return o @staticmethod def backward(ctx, do): q, k, v, o, lse, cu, tcum, perm = ctx.saved_tensors dq, dk, dv = _load().bwd(do, q, k, v, o, lse, cu, tcum, perm, ctx.scale) return dq, dk, dv, None, None, None, None def varlen_causal_attention(q, k, v, offsets, scale: float | None = None): """Causal attention within each document of a packed (N, H, 64) fp16 stream. Equivalent to SDPA(is_causal=True) over a jagged nested tensor built from `offsets` (the packer's document boundaries into the flattened stream). """ if scale is None: scale = q.shape[-1] ** -0.5 cu, tcum, perm = tile_table(offsets) return _VarlenAttn.apply(q, k, v, cu, tcum, perm, scale) class _CanonRopeAttn(torch.autograd.Function): """Canon+RoPE and varlen attention as ONE autograd Function. Kept separate, the attention backward rounded dQ to fp16 in its own pass (bwd_post_kernel) and the Canon backward read it back: ~100 MB of traffic per layer and an fp16 rounding of every dQ. Merged, the Canon backward reads the raw fp32 dQ accumulator and applies the (power-of-two) softmax scale itself. Measured: dqkv error vs fp64 1.2038e-3 -> 1.1955e-3, region -1.0% at micro_batch 16. """ @staticmethod def forward(ctx, qkv, w, m1, m2, cos, sin, offsets, scale): from .canon_rope_bw import ext as canon_ext B, T, _ = qkv.shape N = B * T m1f, m2f = m1.reshape(N).contiguous(), m2.reshape(N).contiguous() cf, sf = cos.reshape(N, 32).contiguous(), sin.reshape(N, 32).contiguous() # canon_rope_bw (swarm canon_bw): bitwise-identical q/k/v, fwd -33% / bwd -17% q, k, v = canon_ext.fwd(qkv, w.contiguous(), m1f, m2f, cf, sf, 4, 64, 4, 1) cu, tcum, perm = tile_table(offsets) qv, kv, vv = q.view(N, 8, 64), k.view(N, 8, 64), v.view(N, 8, 64) o, lse = _load().fwd(qv, kv, vv, cu, tcum, perm, scale) ctx.save_for_backward(qkv, w, m1f, m2f, cf, sf, qv, kv, vv, o, lse, cu, tcum, perm) ctx.scale = scale return o @staticmethod def backward(ctx, do): from .canon_rope_bw import ext as canon_ext qkv, w, m1f, m2f, cf, sf, q, k, v, o, lse, cu, tcum, perm = ctx.saved_tensors B, T, _ = qkv.shape dqacc, dk, dv = _load().bwd_raw(do, q, k, v, o, lse, cu, tcum, perm, ctx.scale) dqkv, part = canon_ext.bwd(qkv, w.contiguous(), m1f, m2f, cf, sf, dqacc.view(B, T, 8, 64), dk.view(B, T, 8, 64), dv.view(B, T, 8, 64), ctx.scale, True, 4, 2, 1, False) return dqkv, part.sum(0).to(w.dtype), None, None, None, None, None, None def canon_rope_attention(qkv, canon_w, m1, m2, cos, sin, offsets): """qkv (B, T, 1536) fp16 -> Canon + RoPE -> causal varlen attention -> (N, 8, 64).""" return _CanonRopeAttn.apply(qkv, canon_w, m1, m2, cos, sin, offsets, 0.125)