summaryrefslogtreecommitdiff
path: root/volta_kernels/__init__.py
diff options
context:
space:
mode:
authorjdj <jdj@kunix.org>2026-11-18 14:47:33 -0800
committerjdj <jdj@kunix.org>2026-11-18 14:47:33 -0800
commit5d2e4252e695644c34be605b44a14bd80a98a0a6 (patch)
tree311a72eda4459af7547b9cd05c0956eafb235f2d /volta_kernels/__init__.py
downloadboris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.tar.gz
boris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.zip
Boris-2 training code
The code, configuration and data filters used to train Boris-2 (144M, 200B tokens, single V100): model, training loop, data preparation, n-gram table, V100 kernels, final stages and the exact run configs.
Diffstat (limited to 'volta_kernels/__init__.py')
-rw-r--r--volta_kernels/__init__.py124
1 files changed, 124 insertions, 0 deletions
diff --git a/volta_kernels/__init__.py b/volta_kernels/__init__.py
new file mode 100644
index 0000000..06b9547
--- /dev/null
+++ b/volta_kernels/__init__.py
@@ -0,0 +1,124 @@
+"""Hand-written sm_70 kernels for Boris-2. See README.md in this directory.
+
+Built on first use with torch.utils.cpp_extension (nvcc, -arch=sm_70) into
+volta_kernels/build/, which is kept between runs so a resume does not recompile.
+"""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+import torch
+
+_HERE = Path(__file__).resolve().parent
+_ext = None
+
+
+def _load():
+ global _ext
+ if _ext is None:
+ from torch.utils.cpp_extension import load
+ build = _HERE / "build_v4" # new dir: never relink the .so a running trainer has mapped
+ build.mkdir(exist_ok=True)
+ os.environ.setdefault("TORCH_CUDA_ARCH_LIST", "7.0")
+ _ext = load(name="boris_volta_attn_v4", sources=[str(_HERE / "attn.cu")],
+ extra_cuda_cflags=["-O3", "-arch=sm_70", "-lineinfo"],
+ build_directory=str(build), verbose=False)
+ return _ext
+
+
+def tile_table(offsets: torch.Tensor):
+ """(cu, tcum, perm) int32 for a document-offset tensor, cached on the tensor.
+
+ tcum is the cumulative 64-tile count over documents sorted LONGEST FIRST and
+ perm maps that order back to document indices, so the heaviest CTAs launch
+ first and short documents fill the tail of the grid. Launch order only: the
+ results are bitwise-identical to packer order; measured fwd -6.5%, bwd -5.6%.
+
+ One forward calls attention 31 times (plus recomputation) with the same
+ offsets, so the table is built once. Pure GPU ops -- no host sync.
+ """
+ tab = getattr(offsets, "_volta_tab", None)
+ # Keyed on the version counter too: an in-place edit of the offsets must
+ # never be served a stale table.
+ if tab is None or tab[3] != offsets._version:
+ cu = offsets.to(torch.int32)
+ tiles = (cu[1:] - cu[:-1] + 63) // 64
+ order = torch.argsort(tiles, descending=True, stable=True)
+ tcum = torch.zeros_like(cu)
+ tcum[1:] = torch.cumsum(tiles[order], 0)
+ tab = (cu, tcum, order.to(torch.int32), offsets._version)
+ offsets._volta_tab = tab
+ return tab[0], tab[1], tab[2]
+
+
+class _VarlenAttn(torch.autograd.Function):
+ @staticmethod
+ def forward(ctx, q, k, v, cu, tcum, perm, scale):
+ ext = _load()
+ o, lse = ext.fwd(q, k, v, cu, tcum, perm, scale)
+ ctx.save_for_backward(q, k, v, o, lse, cu, tcum, perm)
+ ctx.scale = scale
+ return o
+
+ @staticmethod
+ def backward(ctx, do):
+ q, k, v, o, lse, cu, tcum, perm = ctx.saved_tensors
+ dq, dk, dv = _load().bwd(do, q, k, v, o, lse, cu, tcum, perm, ctx.scale)
+ return dq, dk, dv, None, None, None, None
+
+
+def varlen_causal_attention(q, k, v, offsets, scale: float | None = None):
+ """Causal attention within each document of a packed (N, H, 64) fp16 stream.
+
+ Equivalent to SDPA(is_causal=True) over a jagged nested tensor built from
+ `offsets` (the packer's document boundaries into the flattened stream).
+ """
+ if scale is None:
+ scale = q.shape[-1] ** -0.5
+ cu, tcum, perm = tile_table(offsets)
+ return _VarlenAttn.apply(q, k, v, cu, tcum, perm, scale)
+
+
+class _CanonRopeAttn(torch.autograd.Function):
+ """Canon+RoPE and varlen attention as ONE autograd Function.
+
+ Kept separate, the attention backward rounded dQ to fp16 in its own pass
+ (bwd_post_kernel) and the Canon backward read it back: ~100 MB of traffic
+ per layer and an fp16 rounding of every dQ. Merged, the Canon backward
+ reads the raw fp32 dQ accumulator and applies the (power-of-two) softmax
+ scale itself. Measured: dqkv error vs fp64 1.2038e-3 -> 1.1955e-3, region
+ -1.0% at micro_batch 16.
+ """
+
+ @staticmethod
+ def forward(ctx, qkv, w, m1, m2, cos, sin, offsets, scale):
+ from .canon_rope_bw import ext as canon_ext
+ B, T, _ = qkv.shape
+ N = B * T
+ m1f, m2f = m1.reshape(N).contiguous(), m2.reshape(N).contiguous()
+ cf, sf = cos.reshape(N, 32).contiguous(), sin.reshape(N, 32).contiguous()
+ # canon_rope_bw (swarm canon_bw): bitwise-identical q/k/v, fwd -33% / bwd -17%
+ q, k, v = canon_ext.fwd(qkv, w.contiguous(), m1f, m2f, cf, sf, 4, 64, 4, 1)
+ cu, tcum, perm = tile_table(offsets)
+ qv, kv, vv = q.view(N, 8, 64), k.view(N, 8, 64), v.view(N, 8, 64)
+ o, lse = _load().fwd(qv, kv, vv, cu, tcum, perm, scale)
+ ctx.save_for_backward(qkv, w, m1f, m2f, cf, sf, qv, kv, vv, o, lse, cu, tcum, perm)
+ ctx.scale = scale
+ return o
+
+ @staticmethod
+ def backward(ctx, do):
+ from .canon_rope_bw import ext as canon_ext
+ qkv, w, m1f, m2f, cf, sf, q, k, v, o, lse, cu, tcum, perm = ctx.saved_tensors
+ B, T, _ = qkv.shape
+ dqacc, dk, dv = _load().bwd_raw(do, q, k, v, o, lse, cu, tcum, perm, ctx.scale)
+ dqkv, part = canon_ext.bwd(qkv, w.contiguous(), m1f, m2f, cf, sf,
+ dqacc.view(B, T, 8, 64), dk.view(B, T, 8, 64),
+ dv.view(B, T, 8, 64), ctx.scale, True, 4, 2, 1, False)
+ return dqkv, part.sum(0).to(w.dtype), None, None, None, None, None, None
+
+
+def canon_rope_attention(qkv, canon_w, m1, m2, cos, sin, offsets):
+ """qkv (B, T, 1536) fp16 -> Canon + RoPE -> causal varlen attention -> (N, 8, 64)."""
+ return _CanonRopeAttn.apply(qkv, canon_w, m1, m2, cos, sin, offsets, 0.125)