From 5d2e4252e695644c34be605b44a14bd80a98a0a6 Mon Sep 17 00:00:00 2001 From: jdj Date: Wed, 18 Nov 2026 14:47:33 -0800 Subject: Boris-2 training code The code, configuration and data filters used to train Boris-2 (144M, 200B tokens, single V100): model, training loop, data preparation, n-gram table, V100 kernels, final stages and the exact run configs. --- volta_kernels/__init__.py | 124 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 volta_kernels/__init__.py (limited to 'volta_kernels/__init__.py') diff --git a/volta_kernels/__init__.py b/volta_kernels/__init__.py new file mode 100644 index 0000000..06b9547 --- /dev/null +++ b/volta_kernels/__init__.py @@ -0,0 +1,124 @@ +"""Hand-written sm_70 kernels for Boris-2. See README.md in this directory. + +Built on first use with torch.utils.cpp_extension (nvcc, -arch=sm_70) into +volta_kernels/build/, which is kept between runs so a resume does not recompile. +""" +from __future__ import annotations + +import os +from pathlib import Path + +import torch + +_HERE = Path(__file__).resolve().parent +_ext = None + + +def _load(): + global _ext + if _ext is None: + from torch.utils.cpp_extension import load + build = _HERE / "build_v4" # new dir: never relink the .so a running trainer has mapped + build.mkdir(exist_ok=True) + os.environ.setdefault("TORCH_CUDA_ARCH_LIST", "7.0") + _ext = load(name="boris_volta_attn_v4", sources=[str(_HERE / "attn.cu")], + extra_cuda_cflags=["-O3", "-arch=sm_70", "-lineinfo"], + build_directory=str(build), verbose=False) + return _ext + + +def tile_table(offsets: torch.Tensor): + """(cu, tcum, perm) int32 for a document-offset tensor, cached on the tensor. + + tcum is the cumulative 64-tile count over documents sorted LONGEST FIRST and + perm maps that order back to document indices, so the heaviest CTAs launch + first and short documents fill the tail of the grid. Launch order only: the + results are bitwise-identical to packer order; measured fwd -6.5%, bwd -5.6%. + + One forward calls attention 31 times (plus recomputation) with the same + offsets, so the table is built once. Pure GPU ops -- no host sync. + """ + tab = getattr(offsets, "_volta_tab", None) + # Keyed on the version counter too: an in-place edit of the offsets must + # never be served a stale table. + if tab is None or tab[3] != offsets._version: + cu = offsets.to(torch.int32) + tiles = (cu[1:] - cu[:-1] + 63) // 64 + order = torch.argsort(tiles, descending=True, stable=True) + tcum = torch.zeros_like(cu) + tcum[1:] = torch.cumsum(tiles[order], 0) + tab = (cu, tcum, order.to(torch.int32), offsets._version) + offsets._volta_tab = tab + return tab[0], tab[1], tab[2] + + +class _VarlenAttn(torch.autograd.Function): + @staticmethod + def forward(ctx, q, k, v, cu, tcum, perm, scale): + ext = _load() + o, lse = ext.fwd(q, k, v, cu, tcum, perm, scale) + ctx.save_for_backward(q, k, v, o, lse, cu, tcum, perm) + ctx.scale = scale + return o + + @staticmethod + def backward(ctx, do): + q, k, v, o, lse, cu, tcum, perm = ctx.saved_tensors + dq, dk, dv = _load().bwd(do, q, k, v, o, lse, cu, tcum, perm, ctx.scale) + return dq, dk, dv, None, None, None, None + + +def varlen_causal_attention(q, k, v, offsets, scale: float | None = None): + """Causal attention within each document of a packed (N, H, 64) fp16 stream. + + Equivalent to SDPA(is_causal=True) over a jagged nested tensor built from + `offsets` (the packer's document boundaries into the flattened stream). + """ + if scale is None: + scale = q.shape[-1] ** -0.5 + cu, tcum, perm = tile_table(offsets) + return _VarlenAttn.apply(q, k, v, cu, tcum, perm, scale) + + +class _CanonRopeAttn(torch.autograd.Function): + """Canon+RoPE and varlen attention as ONE autograd Function. + + Kept separate, the attention backward rounded dQ to fp16 in its own pass + (bwd_post_kernel) and the Canon backward read it back: ~100 MB of traffic + per layer and an fp16 rounding of every dQ. Merged, the Canon backward + reads the raw fp32 dQ accumulator and applies the (power-of-two) softmax + scale itself. Measured: dqkv error vs fp64 1.2038e-3 -> 1.1955e-3, region + -1.0% at micro_batch 16. + """ + + @staticmethod + def forward(ctx, qkv, w, m1, m2, cos, sin, offsets, scale): + from .canon_rope_bw import ext as canon_ext + B, T, _ = qkv.shape + N = B * T + m1f, m2f = m1.reshape(N).contiguous(), m2.reshape(N).contiguous() + cf, sf = cos.reshape(N, 32).contiguous(), sin.reshape(N, 32).contiguous() + # canon_rope_bw (swarm canon_bw): bitwise-identical q/k/v, fwd -33% / bwd -17% + q, k, v = canon_ext.fwd(qkv, w.contiguous(), m1f, m2f, cf, sf, 4, 64, 4, 1) + cu, tcum, perm = tile_table(offsets) + qv, kv, vv = q.view(N, 8, 64), k.view(N, 8, 64), v.view(N, 8, 64) + o, lse = _load().fwd(qv, kv, vv, cu, tcum, perm, scale) + ctx.save_for_backward(qkv, w, m1f, m2f, cf, sf, qv, kv, vv, o, lse, cu, tcum, perm) + ctx.scale = scale + return o + + @staticmethod + def backward(ctx, do): + from .canon_rope_bw import ext as canon_ext + qkv, w, m1f, m2f, cf, sf, q, k, v, o, lse, cu, tcum, perm = ctx.saved_tensors + B, T, _ = qkv.shape + dqacc, dk, dv = _load().bwd_raw(do, q, k, v, o, lse, cu, tcum, perm, ctx.scale) + dqkv, part = canon_ext.bwd(qkv, w.contiguous(), m1f, m2f, cf, sf, + dqacc.view(B, T, 8, 64), dk.view(B, T, 8, 64), + dv.view(B, T, 8, 64), ctx.scale, True, 4, 2, 1, False) + return dqkv, part.sum(0).to(w.dtype), None, None, None, None, None, None + + +def canon_rope_attention(qkv, canon_w, m1, m2, cos, sin, offsets): + """qkv (B, T, 1536) fp16 -> Canon + RoPE -> causal varlen attention -> (N, 8, 64).""" + return _CanonRopeAttn.apply(qkv, canon_w, m1, m2, cos, sin, offsets, 0.125) -- cgit v1.2.3