From 5d2e4252e695644c34be605b44a14bd80a98a0a6 Mon Sep 17 00:00:00 2001 From: jdj Date: Wed, 18 Nov 2026 14:47:33 -0800 Subject: Boris-2 training code The code, configuration and data filters used to train Boris-2 (144M, 200B tokens, single V100): model, training loop, data preparation, n-gram table, V100 kernels, final stages and the exact run configs. --- volta_kernels/README.md | 165 +++++++++ volta_kernels/__init__.py | 124 +++++++ volta_kernels/attn.cu | 761 +++++++++++++++++++++++++++++++++++++++ volta_kernels/canon_rope.py | 171 +++++++++ volta_kernels/canon_rope_bw.py | 327 +++++++++++++++++ volta_kernels/canon_rope_cuda.py | 266 ++++++++++++++ volta_kernels/head.py | 168 +++++++++ volta_kernels/integ_ops.py | 256 +++++++++++++ 8 files changed, 2238 insertions(+) create mode 100644 volta_kernels/README.md create mode 100644 volta_kernels/__init__.py create mode 100644 volta_kernels/attn.cu create mode 100644 volta_kernels/canon_rope.py create mode 100644 volta_kernels/canon_rope_bw.py create mode 100644 volta_kernels/canon_rope_cuda.py create mode 100644 volta_kernels/head.py create mode 100644 volta_kernels/integ_ops.py (limited to 'volta_kernels') diff --git a/volta_kernels/README.md b/volta_kernels/README.md new file mode 100644 index 0000000..dd80018 --- /dev/null +++ b/volta_kernels/README.md @@ -0,0 +1,165 @@ +# volta_kernels — hand-written sm_70 kernels for Boris-2 + +Opt-in: `BORIS_VOLTA_KERNELS=1`. Engages only on an sm_70 device (the V100); +eval on the 3060 always runs the stock paths. Unset, nothing changes. + + BORIS_VOLTA_KERNELS=1 bash logs/launch_chain.sh --resume + +First use JIT-builds `attn.cu` with nvcc into `build/` (~1 min, then cached). + +## What is here + +| file | replaces | measured (V100, beside the live run) | +|---|---|---| +| `attn.cu` fwd | CUTLASS mem-efficient SDPA fwd (nested/jagged) | ~2x | +| `attn.cu` bwd | mem-efficient SDPA bwd (was ~4x its own fwd) | ~2x | +| `head.py` | `_FusedLMHead` elementwise middle + per-chunk casts | 1.8x, -25% peak mem | +| `model._pre_attn(out16)` | separate fp32->fp16 casts of q/k/v | fused into the region | + +End to end, 8-layer slice of the live config, full micro-step: **1.41x** +contended (stable over 3 repeats), 2.05x in one uncontended window. The +31-layer mb-16 number is only observable on the run itself. + +## Why CUDA and not Triton + +On sm_70, Triton's `tl.dot` compiles to plain FMA -- no tensor cores (checked: +2,048 `fma.rn` in the PTX, no `mma`). The attention uses raw +`mma.sync.aligned.m8n8k4` (HMMA.884, what CUTLASS sm70 uses). Fragment layouts, +verified empirically on this card: + + A.row: row = j + 4hi, k = 0..3 A.col: k = j, row = 4hi + 0..3 + B.col: n = j + 4hi, k = 0..3 B.row: k = j, n = 4hi + 0..3 + D f32: row = (lane&1) + (i&2) + 4hi, col = (i&4) + (lane&2) + (i&1) + j = lane%4, hi = lane>=16; quad-pair qp = (lane>>2)&3 owns one 8x8 tile + +The head's softmax is Triton (elementwise/reduction, no dot), which is fine. + +## Precision + +Same operand dtypes and fp32 accumulation as what they replace. `verify.py` +`check_volta_kernels` (always runs on sm_70) holds every output to the stock +path's own error against fp64 on the same inputs: + + attention out 2.22e-4 (= stock) dq 3.66e-4 (stock 4.23e-4) + dk 3.51e-4 (stock 4.05e-4) dv 2.88e-4 (= stock) + head dx, dW identical error to production; loss equal to 9 digits; + ~0.02% of dx elements differ by 1 ulp (lse reduction order) + model median grad rel-L2 vs fp32 1.29e-3 (stock 1.30-1.32e-3), at the + run's 2^15 loss scale. Without a loss scale the two fp16 paths + differ ~6% on early layers -- subnormal noise, not a bug. + +Document isolation and causality are exact (zero drift). dQ uses fp32 atomics, +so attention backward is not bitwise deterministic run to run (fp32 order only). + +## Gotchas found on the way + +- q/k/v leave `_pre_attn` fp32, not fp16: the Canon masks m1/m2 are fp32 and + promote. Stock SDPA casts them via autocast; the Volta path does the same cast + inside the compiled region (`out16`). +- In production the head's `g` is fp32 (x is fp32), and autocast re-casts W to + fp16 twice per chunk. The rewrite casts once per call. +- `dW.addcmul_(fp16, fp32)` takes torch's slow unvectorized path (0.9 ms/chunk); + replaced by a Triton FMA. Exact, because `back` is a power of two. +- `__launch_bounds__(128, 3)` makes both kernels slower (spills): 248/216 regs, + 2 CTAs/SM is the best measured point. +- Remaining bwd split (ablation): dQ phase ~26%, tile loads ~6%, atomics ~4%. + +## Live result and later experiments (2026-09-27) + +Adopted at step 28,239: **35.5k -> 45.2k tok/s, 30.5 -> 38.8 TF, MFU 24.4% -> +31.0%**, ETA 60 -> 47.5 days. `logs/launch_chain.sh` exports +`BORIS_VOLTA_KERNELS=1` by default. + +Measured afterwards, so nobody re-tries them blind: + +- **Megakernel / launch overhead.** Whole micro-step captured as one CUDA graph + vs eager, identical kernels: 3.3% faster at micro_batch 2 (8-layer slice), + i.e. ~0.5% at production mb 16. That is the ceiling for anything whose win is + removing launches; the GEMMs (~55% of the step) are cuBLAS. +- **Skipping fully-masked MMA blocks on diagonal tiles** (bitwise-identical + results): fwd 4.5% and bwd 18.6% SLOWER. Warp-dependent loop bounds break the + unrolled MMA schedule, and the CTA waits at the barrier for its busiest warp + anyway. Dropped. +- **Adopted, no speed effect:** gd written over the logits buffer (-64 MB); + shape/device/metadata guards in the extension. +- verify.py now also covers an 8192-token document (stage two; fp32 reference, + error ratio vs stock 1.000) and a head call with a partial final chunk. +- Test gotcha: `t.to(torch.float16)` on an fp16 tensor returns the same tensor, + so two gradient runs "on copies" accumulate into one .grad. Clone. +- **Megakernel, stage 1** (`experiments/mega_ffn2.cu`): the whole FFN half of a + block forward (RMSNorm -> gate/up -> SiLU*up -> down -> residual) in one + kernel, intermediate never leaving shared memory. Precision identical to stock + (FFN-delta error vs fp64 6.454e-4 both). Speed at mb 16: 23.7 ms first build, + 16.1 ms with register-prefetched weights, vs **6.5 ms** Inductor+cuBLAS -- 2.5x + SLOWER. Structural: holding a full 512-wide x tile costs 64 KB of shared + memory, so a CTA can own only 64 rows, each weight byte is reused by only 64 + rows (cuBLAS reuses across 128x128 tiles), and one CTA per SM leaves nothing + to hide the barriers. The fusion's best case is avoiding the gate/up write, + and training's backward needs gate/up saved anyway. Not pursued further. + +## Round 2 (2026-09-27 evening) -- reviewed with GPT-6 Sol and Muse Spark + +Adopted (dormant until the next relaunch; verify 14/14): +- **Head dW via fp32-output GEMM** (GPT's idea): `torch.addmm(dW, g^T, x16, + beta=0|1, out_dtype=float32)` selects `volta_s884gemm_fp16` + fp32 split-k + reduce. dW is never rounded to fp16, so the x-rescale / DW_SHRINK overflow + guard and the second fp16 operand (gd, 64 MB write per chunk) are gone. + **Head 12.6% faster; dW error vs fp64 5.24e-4 -> 5.01e-4 (fixture).** +- **Forward K/V register prefetch**: next tile's global loads issued before the + current tile's MMAs (fwd has register headroom: 216). **fwd 4.5% faster, + bitwise-identical output.** + +Measured and rejected: +- Megakernel FFN at 128 rows (column split, gate/up recomputed): 24.3 ms vs + 6.5 ms stock (and 16.0 ms at 64 rows). 1.67x FLOPs plus spills. +- Blocked two-pass head softmax with online max/sum + evict_first (Muse): no + gain over the one-pass kernel. +- Rejected on inspection: aliasing P into Ks in fwd (race -- other warps still + read Ks for QK^T), folding scale*log2e into fp16 Q (changes rounding), Q/dO + register prefetch in bwd (248 regs, no headroom). +- **Fused Canon + RoPE** (`canon_rope.py`, Triton, fwd + bwd): replaces + Inductor's zero-padded fp32 copies of qkv, the 3-tap combine, RoPE and their + backward passes. 1.96x on that region; same error as the region for q/k/v and + dqkv, weight gradient closer to fp64. Isolation exact at document boundaries. + Whole 8-layer slice: 1.41x -> **1.538x** vs stock (contended), i.e. ~+9% over + the kernels live since step 28,239. verify.py: 80/80. +- **Canon + RoPE in CUDA** (`canon_rope_cuda.py`, from a second Claude Code + reviewer): sliding window down the sequence, 8 channels per thread, 16-byte + I/O. Bitwise-identical q/k/v/dqkv to the Triton version (checked + independently), 3.55x the compiled region at mb 16, **+4.1% whole micro-step + at full size**. The model now uses this; `canon_rope.py` (Triton) is kept as + the reference. verify.py 80/80. +- **Longest-document-first CTA order** (GPT-6 Sol's research lead): the tile + table cumsums over documents sorted by tile count and passes the permutation + to both kernels. Bitwise-identical; idle V100, mb 16: fwd 1.010 -> 0.945 ms + (-6.5%), bwd 2.712 -> 2.559 ms (-5.6%); ctx 8192 about -6% both. verify 81/81. +- **GEMM algorithm sweep** (TunableOp, idle V100): cuBLAS "Default" won every + block shape. The GEMMs already run at 54-59% of the 125 TF peak (QKV 73.5 TF, + gate/up 71, down 74, o_proj 67). There is little left to take from them. +- Attention, by contrast, runs at ~24-27% of peak (fwd ~30 TF, bwd ~27 TF + counted). That gap is the largest remaining one. +- **Masks only where needed** (after an ablation showed softmax math is ~19% + of the forward): a CTA-uniform branch skips the per-element integer mask on + every tile except the diagonal and a document's last tile. Bitwise-identical + (dQ differs only by its usual atomic-order noise, measured equal to + old-vs-old). Idle V100: fwd -4.9%, bwd -2.1% at both 2048 and 8192. + Tried and dropped: skipping the output rescale when no max moved (no gain), + folding the scale into ex2's FMA (+1.2% more but not bitwise). +- Volta attention literature (2026-09): SparkAttention's backward gains need + fp16 accumulation (excluded); its register-shuffle C->A layout trick targets + the P shared-memory round trip, which the ablation shows costs nothing here. + No public fp32-accumulation varlen training kernel for sm_70 beats this one. +- **Head dx written fp32 from the GEMM** (`out_dtype=float32`): cuBLAS can no + longer pick split-k kernels that reduce partials in fp16. dx error vs fp64 + 5.31e-4 -> 3.43e-4 and 2.9% faster; also makes a larger lm_head_chunk safe + (chunk 4096: head -11.6%, dx 3.37e-4; with fp16 dx the same chunk raised dx + error 41%). verify 81/81, model median grad error 1.208e-3 (stock 1.293e-3). +- **Hardware correction:** this PG500-216 has **72 SMs** (not 80) and HBM at + 1107 MHz. fp16 tensor peak = 72 x 8 x 128 x 1.38 GHz = **101.7 TF**, not 125; + every MFU quoted against 125 understates by 1.23x. Measured device copy + 956 GB/s. So the GEMMs run at ~70-73% of real peak, and the ~680 GB/s + elementwise kernels have ~30% headroom. +- Tried and reverted: one-thread-per-row bwd_pre_kernel (3.6x SLOWER -- the + warp-per-row version was already coalesced at ~680 GB/s). The SwiGLU backward + pairing lead was dropped on reading Inductor's kernel: the "redundant" reads + hit cache. diff --git a/volta_kernels/__init__.py b/volta_kernels/__init__.py new file mode 100644 index 0000000..06b9547 --- /dev/null +++ b/volta_kernels/__init__.py @@ -0,0 +1,124 @@ +"""Hand-written sm_70 kernels for Boris-2. See README.md in this directory. + +Built on first use with torch.utils.cpp_extension (nvcc, -arch=sm_70) into +volta_kernels/build/, which is kept between runs so a resume does not recompile. +""" +from __future__ import annotations + +import os +from pathlib import Path + +import torch + +_HERE = Path(__file__).resolve().parent +_ext = None + + +def _load(): + global _ext + if _ext is None: + from torch.utils.cpp_extension import load + build = _HERE / "build_v4" # new dir: never relink the .so a running trainer has mapped + build.mkdir(exist_ok=True) + os.environ.setdefault("TORCH_CUDA_ARCH_LIST", "7.0") + _ext = load(name="boris_volta_attn_v4", sources=[str(_HERE / "attn.cu")], + extra_cuda_cflags=["-O3", "-arch=sm_70", "-lineinfo"], + build_directory=str(build), verbose=False) + return _ext + + +def tile_table(offsets: torch.Tensor): + """(cu, tcum, perm) int32 for a document-offset tensor, cached on the tensor. + + tcum is the cumulative 64-tile count over documents sorted LONGEST FIRST and + perm maps that order back to document indices, so the heaviest CTAs launch + first and short documents fill the tail of the grid. Launch order only: the + results are bitwise-identical to packer order; measured fwd -6.5%, bwd -5.6%. + + One forward calls attention 31 times (plus recomputation) with the same + offsets, so the table is built once. Pure GPU ops -- no host sync. + """ + tab = getattr(offsets, "_volta_tab", None) + # Keyed on the version counter too: an in-place edit of the offsets must + # never be served a stale table. + if tab is None or tab[3] != offsets._version: + cu = offsets.to(torch.int32) + tiles = (cu[1:] - cu[:-1] + 63) // 64 + order = torch.argsort(tiles, descending=True, stable=True) + tcum = torch.zeros_like(cu) + tcum[1:] = torch.cumsum(tiles[order], 0) + tab = (cu, tcum, order.to(torch.int32), offsets._version) + offsets._volta_tab = tab + return tab[0], tab[1], tab[2] + + +class _VarlenAttn(torch.autograd.Function): + @staticmethod + def forward(ctx, q, k, v, cu, tcum, perm, scale): + ext = _load() + o, lse = ext.fwd(q, k, v, cu, tcum, perm, scale) + ctx.save_for_backward(q, k, v, o, lse, cu, tcum, perm) + ctx.scale = scale + return o + + @staticmethod + def backward(ctx, do): + q, k, v, o, lse, cu, tcum, perm = ctx.saved_tensors + dq, dk, dv = _load().bwd(do, q, k, v, o, lse, cu, tcum, perm, ctx.scale) + return dq, dk, dv, None, None, None, None + + +def varlen_causal_attention(q, k, v, offsets, scale: float | None = None): + """Causal attention within each document of a packed (N, H, 64) fp16 stream. + + Equivalent to SDPA(is_causal=True) over a jagged nested tensor built from + `offsets` (the packer's document boundaries into the flattened stream). + """ + if scale is None: + scale = q.shape[-1] ** -0.5 + cu, tcum, perm = tile_table(offsets) + return _VarlenAttn.apply(q, k, v, cu, tcum, perm, scale) + + +class _CanonRopeAttn(torch.autograd.Function): + """Canon+RoPE and varlen attention as ONE autograd Function. + + Kept separate, the attention backward rounded dQ to fp16 in its own pass + (bwd_post_kernel) and the Canon backward read it back: ~100 MB of traffic + per layer and an fp16 rounding of every dQ. Merged, the Canon backward + reads the raw fp32 dQ accumulator and applies the (power-of-two) softmax + scale itself. Measured: dqkv error vs fp64 1.2038e-3 -> 1.1955e-3, region + -1.0% at micro_batch 16. + """ + + @staticmethod + def forward(ctx, qkv, w, m1, m2, cos, sin, offsets, scale): + from .canon_rope_bw import ext as canon_ext + B, T, _ = qkv.shape + N = B * T + m1f, m2f = m1.reshape(N).contiguous(), m2.reshape(N).contiguous() + cf, sf = cos.reshape(N, 32).contiguous(), sin.reshape(N, 32).contiguous() + # canon_rope_bw (swarm canon_bw): bitwise-identical q/k/v, fwd -33% / bwd -17% + q, k, v = canon_ext.fwd(qkv, w.contiguous(), m1f, m2f, cf, sf, 4, 64, 4, 1) + cu, tcum, perm = tile_table(offsets) + qv, kv, vv = q.view(N, 8, 64), k.view(N, 8, 64), v.view(N, 8, 64) + o, lse = _load().fwd(qv, kv, vv, cu, tcum, perm, scale) + ctx.save_for_backward(qkv, w, m1f, m2f, cf, sf, qv, kv, vv, o, lse, cu, tcum, perm) + ctx.scale = scale + return o + + @staticmethod + def backward(ctx, do): + from .canon_rope_bw import ext as canon_ext + qkv, w, m1f, m2f, cf, sf, q, k, v, o, lse, cu, tcum, perm = ctx.saved_tensors + B, T, _ = qkv.shape + dqacc, dk, dv = _load().bwd_raw(do, q, k, v, o, lse, cu, tcum, perm, ctx.scale) + dqkv, part = canon_ext.bwd(qkv, w.contiguous(), m1f, m2f, cf, sf, + dqacc.view(B, T, 8, 64), dk.view(B, T, 8, 64), + dv.view(B, T, 8, 64), ctx.scale, True, 4, 2, 1, False) + return dqkv, part.sum(0).to(w.dtype), None, None, None, None, None, None + + +def canon_rope_attention(qkv, canon_w, m1, m2, cos, sin, offsets): + """qkv (B, T, 1536) fp16 -> Canon + RoPE -> causal varlen attention -> (N, 8, 64).""" + return _CanonRopeAttn.apply(qkv, canon_w, m1, m2, cos, sin, offsets, 0.125) diff --git a/volta_kernels/attn.cu b/volta_kernels/attn.cu new file mode 100644 index 0000000..1725723 --- /dev/null +++ b/volta_kernels/attn.cu @@ -0,0 +1,761 @@ +// swarm/integ_attn: merged attention kernel = live attn.cu + +// attn_softmax padded smem (row stride 84 halves), fwd software pipelining (double-buffered K, +// next QK^T overlapping softmax), unrolled full-tile dQ atomic loop; +// bwd ILP kept as a knob, default off (see BWD_ILP) (polynomial exp NOT included) +// attn_long 1-D grid, heads interleaved in the global longest-first launch order +// attn_bwd mode 6: next query tile's Q, dO, lse, D prefetched into registers (BWD_PF) +// o, lse, dk, dv bitwise identical to the live kernel; dq differs only by fp32 atomic order. +// Varlen causal FlashAttention for sm_70 (V100), head_dim 64, fp16 in/out. +// +// Why this exists: PyTorch's only SDPA backend that trains on sm_70 is the +// CUTLASS memory-efficient kernel, whose backward measured ~4x its own forward +// on Boris-2's shapes (FlashAttention-2 does ~2.5x). Triton cannot help here: +// on sm_70 tl.dot lowers to plain FMA (no tensor cores), so this is CUDA with +// raw mma.sync.m8n8k4 -- the same HMMA.884 instruction CUTLASS's sm70 path uses. +// +// Precision is the memory-efficient kernel's precision class, deliberately: +// * every GEMM takes fp16 operands and accumulates in fp32 +// * softmax statistics (max, sum, logsumexp) are fp32 +// * P and dS are rounded to fp16 only as GEMM operands, as in any fp16 kernel +// * dQ is accumulated in an fp32 buffer across key tiles and rounded once +// +// Fragment layouts of m8n8k4 (verified empirically on this card, see +// volta_kernels/README.md). j = lane%4, hi = lane>=16, quad-pair qp = (lane>>2)&3: +// A.row : row = j + 4hi, k = 0..3 A.col : k = j, row = 4hi + 0..3 +// B.col : n = j + 4hi, k = 0..3 B.row : k = j, n = 4hi + 0..3 +// D f32 : row = (lane&1) + (i&2) + 4hi, col = (i&4) + (lane&2) + (i&1) +// Each quad-pair computes its own 8x8 tile; the four are arranged 2x2 into a +// 16x16 warp tile: m offset 8*(qp&1), n offset 8*(qp>>1). +#include +#include +#include +#include + +namespace { + +constexpr int HD = 64; // head dim +constexpr int TL = 64; // tile edge (queries or keys) +constexpr int NT = 128; // 4 warps, 16 tile rows each + +// 64x64 fp16 tiles live unpadded in shared memory with the 8-byte segments of +// each row XOR-swizzled. The mapping makes every fragment access pattern used +// below (A.row, B.col: 8 rows x one segment; B.row, A.col: 4 rows x two +// segments) hit distinct banks within each half-warp. +__device__ __forceinline__ int swz(int r) { return ((r & 3) << 2) | ((r >> 2) & 3); } +__device__ __forceinline__ int so(int r, int c) { + return r * 64 + ((((c >> 2) ^ swz(r))) << 2) + (c & 3); +} + +// Compile-time knobs (defaults = the adopted combination). +// BWD_ILP (attn_softmax) and BWD_PF (attn_bwd mode 6) combine without spills (255 regs, 2 CTAs/SM), +// but with the prefetch on, ILP adds nothing measurable (bwd vs live: PF only 0.944/0.945 at +// 16x2048, 0.900 at 4x8192; PF+ILP 0.946/0.947 and 0.897). PF alone wins at stage 1 and leaves +// 17 registers of headroom (238), so ILP is off by default; set -DBWD_ILP=1 to re-test. +#ifndef BWD_ILP +#define BWD_ILP 0 // bwd: S^T MMAs first, then exp(S^T) and the dP^T MMAs in one basic block +#endif +#ifndef BWD_PF +#define BWD_PF 1 // bwd: next query tile's Q/dO/lse/D prefetched into registers +#endif + +// Padded layout: row stride 84 halves = 42 words. Every fragment pattern used +// here (8 rows {0..3,8..11} x 8B; 4 rows x 2 segments 16B apart; P's 32-bit +// stores; commit's 2 rows x 8 segments) hits distinct banks per half-warp, and +// all per-ks / per-nb offsets become compile-time immediates. +constexpr int LDP = 84; +template __device__ __forceinline__ int lo(int r, int c) { + if constexpr (P) return r * LDP + c; else return so(r, c); +} + +__device__ __forceinline__ uint2 lds2(const half* t, int off) { + return *reinterpret_cast(t + off); +} + +#define DEF_MMA(LA, LB) \ + __device__ __forceinline__ void mma_##LA##_##LB(float* d, uint2 a, uint2 b) { \ + asm("mma.sync.aligned.m8n8k4." #LA "." #LB ".f32.f16.f16.f32 " \ + "{%0,%1,%2,%3,%4,%5,%6,%7}, {%8,%9}, {%10,%11}, {%0,%1,%2,%3,%4,%5,%6,%7};" \ + : "+f"(d[0]), "+f"(d[1]), "+f"(d[2]), "+f"(d[3]), \ + "+f"(d[4]), "+f"(d[5]), "+f"(d[6]), "+f"(d[7]) \ + : "r"(a.x), "r"(a.y), "r"(b.x), "r"(b.y)); \ + } +DEF_MMA(row, col) +DEF_MMA(row, row) +DEF_MMA(col, row) + +__device__ __forceinline__ float ex2(float x) { + float y; + asm("ex2.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} + +__device__ __forceinline__ unsigned pack_h2(float a, float b) { + __half2 h = __floats2half2_rn(a, b); + return *reinterpret_cast(&h); +} + +// Copy rows [row0, row0+64) of one head (64 contiguous halves at token stride +// `ts`) into a swizzled tile. Rows at or past `nrows` are zero-filled. +template +__device__ __forceinline__ void load_tile(half* t, const half* g, int ts, int row0, + int nrows, int tid) { +#pragma unroll + for (int it = 0; it < (TL * 8) / NT; ++it) { + int idx = tid + it * NT; + int r = idx >> 3, c = (idx & 7) * 8; + uint4 v = make_uint4(0, 0, 0, 0); + if (row0 + r < nrows) + v = *reinterpret_cast(g + (long)(row0 + r) * ts + c); + *reinterpret_cast(t + lo(r, c)) = make_uint2(v.x, v.y); + *reinterpret_cast(t + lo(r, c + 4)) = make_uint2(v.z, v.w); + } +} + +// Register-staged copy of one 64x64 tile: global loads issued early, the +// swizzled shared-memory stores later, so the load latency overlaps compute. +struct TileRegs { uint4 v[(TL * 8) / NT]; }; +__device__ __forceinline__ void fetch_tile(TileRegs& R, const half* g, int ts, int row0, int nrows, int tid) { +#pragma unroll + for (int it = 0; it < (TL * 8) / NT; ++it) { + int idx = tid + it * NT, r = idx >> 3, c = (idx & 7) * 8; + R.v[it] = (row0 + r < nrows) ? *reinterpret_cast(g + (long)(row0 + r) * ts + c) + : make_uint4(0, 0, 0, 0); + } +} +template +__device__ __forceinline__ void commit_tile(half* t, const TileRegs& R, int tid) { +#pragma unroll + for (int it = 0; it < (TL * 8) / NT; ++it) { + int idx = tid + it * NT, r = idx >> 3, c = (idx & 7) * 8; + *reinterpret_cast(t + lo(r, c)) = make_uint2(R.v[it].x, R.v[it].y); + *reinterpret_cast(t + lo(r, c + 4)) = make_uint2(R.v[it].z, R.v[it].w); + } +} + +// Map a flat block index onto (document, tile). tcum[d] is the number of tiles +// in documents before d; tcum[ndocs] the total. +__device__ __forceinline__ bool find_tile(const int* tcum, const int* perm, int ndocs, int b, + int& doc, int& local, int& ntiles) { + if (b >= tcum[ndocs]) return false; + int lo = 0, hi = ndocs - 1; + while (lo < hi) { + int mid = (lo + hi + 1) >> 1; + if (tcum[mid] <= b) lo = mid; else hi = mid - 1; + } + doc = perm[lo]; // tcum is over documents sorted longest-first + local = b - tcum[lo]; + ntiles = tcum[lo + 1] - tcum[lo]; + return true; +} + +// -------------------------------------------------------------------------- +// Forward. One CTA = one 64-query tile of one document, one head. +// -------------------------------------------------------------------------- +__global__ void __launch_bounds__(NT) fwd_kernel( + const half* __restrict__ Q, const half* __restrict__ K, const half* __restrict__ V, + half* __restrict__ O, float* __restrict__ LSE, const int* __restrict__ cu, + const int* __restrict__ tcum, const int* __restrict__ perm, int ndocs, int N, int H, + int sq, int sk, int sv, int s_o, float scale_log2) { + // Padded tiles (row stride LDP); K double-buffered for the software pipeline. + __shared__ __align__(16) half Ks2[2][TL * LDP]; + __shared__ __align__(16) half Vs[TL * LDP]; + __shared__ __align__(16) half Ps[4][16 * LDP]; + + // 1-D grid, heads interleaved: the longest-first order holds across heads. + const int h = blockIdx.x % H; + int doc, local, ntiles; + if (!find_tile(tcum, perm, ndocs, blockIdx.x / H, doc, local, ntiles)) return; + const int t = ntiles - 1 - local; // longest-running tiles launch first + const int s0 = cu[doc], L = cu[doc + 1] - s0; + + const int tid = threadIdx.x, w = tid >> 5, lane = tid & 31; + const int j = lane & 3, hi = lane >> 4, qp = (lane >> 2) & 3; + const int mo = 8 * (qp & 1), no = 8 * (qp >> 1); + + uint2 qf[16]; + { + int qrow = t * TL + 16 * w + mo + j + 4 * hi; + const half* g = Q + (long)(s0 + qrow) * sq + h * HD; +#pragma unroll + for (int ks = 0; ks < 16; ++ks) + qf[ks] = qrow < L ? *reinterpret_cast(g + 4 * ks) : make_uint2(0, 0); + } + const int qr0 = t * TL + 16 * w + mo + (lane & 1) + 4 * hi; // i&2 == 0 + float m[2] = {-INFINITY, -INFINITY}, l[2] = {0.f, 0.f}; + float o[4][8]; +#pragma unroll + for (int a = 0; a < 4; ++a) +#pragma unroll + for (int i = 0; i < 8; ++i) o[a][i] = 0.f; + + half* P = Ps[w]; + const half* Kg = K + (long)s0 * sk + h * HD; + const half* Vg = V + (long)s0 * sv + h * HD; + + float s[4][8]; + TileRegs kr, vr; + // Prologue: K_0 committed, S_0 computed; the loop body computes S_{kt+1} + // while doing softmax(S_kt). Ks2[kt&1] holds K_kt. + fetch_tile(kr, Kg, sk, 0, L, tid); + fetch_tile(vr, Vg, sv, 0, L, tid); + commit_tile(Ks2[0], kr, tid); + if (t > 0) fetch_tile(kr, Kg, sk, TL, L, tid); + __syncthreads(); +#pragma unroll + for (int a = 0; a < 4; ++a) +#pragma unroll + for (int i = 0; i < 8; ++i) s[a][i] = 0.f; +#pragma unroll + for (int ks = 0; ks < 16; ++ks) +#pragma unroll + for (int nb = 0; nb < 4; ++nb) + mma_row_col(s[nb], qf[ks], lds2(Ks2[0], lo(nb * 16 + no + j + 4 * hi, 4 * ks))); + + for (int kt = 0; kt <= t; ++kt) { + __syncthreads(); // everyone done with Vs (PV of kt-1) and Ks2[(kt+1)&1] (S_kt's MMAs) + commit_tile(Vs, vr, tid); + if (kt < t) commit_tile(Ks2[(kt + 1) & 1], kr, tid); + if (kt < t) { + fetch_tile(vr, Vg, sv, (kt + 1) * TL, L, tid); + if (kt + 1 < t) fetch_tile(kr, Kg, sk, (kt + 2) * TL, L, tid); + } + __syncthreads(); + + const bool diag = (kt == t); + const bool masked = diag || kt * TL + TL > L; + float mx[2] = {m[0], m[1]}; + // Mask by select only; the max, the next tile's MMAs and the exp share one + // basic block so ptxas can interleave HMMA with the softmax FP work. + if (masked) { +#pragma unroll + for (int nb = 0; nb < 4; ++nb) +#pragma unroll + for (int i = 0; i < 8; ++i) { + int key = kt * TL + nb * 16 + no + (i & 4) + (lane & 2) + (i & 1); + int qr = qr0 + (i & 2); + bool ok = key < L && (!diag || key <= qr); + s[nb][i] = ok ? s[nb][i] : -INFINITY; + } + } + float sn[4][8]; + { + const half* Kn = Ks2[(kt + 1) & 1]; // stale on the last tile; result unused +#pragma unroll + for (int a = 0; a < 4; ++a) +#pragma unroll + for (int i = 0; i < 8; ++i) sn[a][i] = 0.f; +#pragma unroll + for (int ks = 0; ks < 16; ++ks) +#pragma unroll + for (int nb = 0; nb < 4; ++nb) + mma_row_col(sn[nb], qf[ks], lds2(Kn, lo(nb * 16 + no + j + 4 * hi, 4 * ks))); + } +#pragma unroll + for (int nb = 0; nb < 4; ++nb) +#pragma unroll + for (int i = 0; i < 8; ++i) mx[(i >> 1) & 1] = fmaxf(mx[(i >> 1) & 1], s[nb][i]); +#pragma unroll + for (int r = 0; r < 2; ++r) { + mx[r] = fmaxf(mx[r], __shfl_xor_sync(0xffffffff, mx[r], 2)); + mx[r] = fmaxf(mx[r], __shfl_xor_sync(0xffffffff, mx[r], 8)); + } + float alpha[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + alpha[r] = ex2((m[r] - mx[r]) * scale_log2); // exp2(-inf) = 0 on the first tile + m[r] = mx[r]; + l[r] *= alpha[r]; + } +#pragma unroll + for (int nb = 0; nb < 4; ++nb) +#pragma unroll + for (int i = 0; i < 8; ++i) { + int r = (i >> 1) & 1; + float p = ex2(fmaf(s[nb][i], scale_log2, -m[r] * scale_log2)); + s[nb][i] = p; + l[r] += p; + o[nb][i] *= alpha[r]; + } + + __syncwarp(); +#pragma unroll + for (int nb = 0; nb < 4; ++nb) +#pragma unroll + for (int i = 0; i < 8; i += 2) { + int row = mo + (lane & 1) + (i & 2) + 4 * hi; + int col = nb * 16 + no + (i & 4) + (lane & 2); + *reinterpret_cast(P + lo(row, col)) = pack_h2(s[nb][i], s[nb][i + 1]); + } + __syncwarp(); +#pragma unroll + for (int ks = 0; ks < 16; ++ks) { + uint2 a = lds2(P, lo(mo + j + 4 * hi, 4 * ks)); +#pragma unroll + for (int nb = 0; nb < 4; ++nb) + mma_row_row(o[nb], a, lds2(Vs, lo(4 * ks + j, nb * 16 + no + 4 * hi))); + } +#pragma unroll + for (int a = 0; a < 4; ++a) +#pragma unroll + for (int i = 0; i < 8; ++i) s[a][i] = sn[a][i]; + } + +#pragma unroll + for (int r = 0; r < 2; ++r) { + l[r] += __shfl_xor_sync(0xffffffff, l[r], 2); + l[r] += __shfl_xor_sync(0xffffffff, l[r], 8); + } +#pragma unroll + for (int r = 0; r < 2; ++r) { + int qr = qr0 + 2 * r; + if (qr >= L) continue; + float inv = 1.f / l[r]; + half* og = O + (long)(s0 + qr) * s_o + h * HD; +#pragma unroll + for (int nb = 0; nb < 4; ++nb) +#pragma unroll + for (int i = 2 * r; i < 8; i += 4) { + int col = nb * 16 + no + (i & 4) + (lane & 2); + *reinterpret_cast(og + col) = pack_h2(o[nb][i] * inv, o[nb][i + 1] * inv); + } + if ((lane & 2) == 0 && no == 0) LSE[(long)h * N + s0 + qr] = m[r] * scale_log2 + __log2f(l[r]); + } +} + +// -------------------------------------------------------------------------- +// Backward. One CTA = one 64-key tile of one document, one head; it walks the +// query tiles at and after it. dK/dV stay in registers; dQ is accumulated in +// an fp32 buffer with atomics (same as FlashAttention-2). +// -------------------------------------------------------------------------- +__global__ void __launch_bounds__(NT) bwd_kernel( + const half* __restrict__ Q, const half* __restrict__ K, const half* __restrict__ V, + const half* __restrict__ dO, const float* __restrict__ LSE, + const float* __restrict__ Dv, float* __restrict__ dQacc, half* __restrict__ dK, + half* __restrict__ dV, const int* __restrict__ cu, const int* __restrict__ tcum, + const int* __restrict__ perm, + int ndocs, int N, int H, int sq, int sk, int sv, int sdo, int sdk, int sdv, + float scale_log2, float scale) { + __shared__ __align__(16) half Ks[TL * HD]; + __shared__ __align__(16) half Vs[TL * HD]; + // Q and dO for the current query tile (padded, row stride LDP). Once dK/dV + // are done with them they are dead for the iteration, and the same bytes + // stage dQ in fp32. + __shared__ __align__(16) half QdOs[2 * TL * LDP]; + half* Qs = QdOs; + half* dOs = QdOs + TL * LDP; + float* dQs = reinterpret_cast(QdOs); + __shared__ __align__(16) half St[TL * TL]; // P^T, then dS^T (key rows, query cols) + __shared__ float lse_s[TL], d_s[TL]; + + // 1-D grid, heads interleaved: the longest-first order holds across heads. + const int h = blockIdx.x % H; + int doc, kt, ntiles; + if (!find_tile(tcum, perm, ndocs, blockIdx.x / H, doc, kt, ntiles)) return; + const int s0 = cu[doc], L = cu[doc + 1] - s0; + + const int tid = threadIdx.x, w = tid >> 5, lane = tid & 31; + const int j = lane & 3, hi = lane >> 4, qp = (lane >> 2) & 3; + const int mo = 8 * (qp & 1), no = 8 * (qp >> 1); + + load_tile(Ks, K + (long)s0 * sk + h * HD, sk, kt * TL, L, tid); + load_tile(Vs, V + (long)s0 * sv + h * HD, sv, kt * TL, L, tid); + + float dk[4][8], dv[4][8]; +#pragma unroll + for (int a = 0; a < 4; ++a) +#pragma unroll + for (int i = 0; i < 8; ++i) dk[a][i] = dv[a][i] = 0.f; + + const int kr0 = kt * TL + 16 * w + mo + (lane & 1) + 4 * hi; // this lane's key rows (i&2==0) + half* Sw = St + 16 * w * 64; // this warp's 16 key rows + const half* Qg = Q + (long)s0 * sq + h * HD; + const half* dOg = dO + (long)s0 * sdo + h * HD; +#if BWD_PF + // The next query tile's Q/dO (and lse/D) are fetched into registers while + // the current tile computes, like the forward's K/V prefetch. + TileRegs qreg, dreg; + float lse_n = 0.f, d_n = 0.f; + fetch_tile(qreg, Qg, sq, kt * TL, L, tid); + fetch_tile(dreg, dOg, sdo, kt * TL, L, tid); + if (tid < TL) { + int qr = kt * TL + tid; + if (qr < L) { lse_n = LSE[(long)h * N + s0 + qr]; d_n = Dv[(long)h * N + s0 + qr]; } + } +#endif + + for (int qt = kt; qt < ntiles; ++qt) { + __syncthreads(); +#if BWD_PF + commit_tile(Qs, qreg, tid); + commit_tile(dOs, dreg, tid); + if (tid < TL) { lse_s[tid] = lse_n; d_s[tid] = d_n; } + if (qt + 1 < ntiles) { + fetch_tile(qreg, Qg, sq, (qt + 1) * TL, L, tid); + fetch_tile(dreg, dOg, sdo, (qt + 1) * TL, L, tid); + if (tid < TL) { + int qr = (qt + 1) * TL + tid; + bool ok = qr < L; + lse_n = ok ? LSE[(long)h * N + s0 + qr] : 0.f; + d_n = ok ? Dv[(long)h * N + s0 + qr] : 0.f; + } + } +#else + load_tile(Qs, Qg, sq, qt * TL, L, tid); + load_tile(dOs, dOg, sdo, qt * TL, L, tid); + if (tid < TL) { + int qr = qt * TL + tid; + bool ok = qr < L; + lse_s[tid] = ok ? LSE[(long)h * N + s0 + qr] : 0.f; + d_s[tid] = ok ? Dv[(long)h * N + s0 + qr] : 0.f; + } +#endif + __syncthreads(); + + // S^T = K_w Q^T and dP^T = V_w dO^T, both (16 keys x 64 queries). + float st[4][8], dp[4][8]; +#pragma unroll + for (int a = 0; a < 4; ++a) +#pragma unroll + for (int i = 0; i < 8; ++i) st[a][i] = dp[a][i] = 0.f; +#if BWD_ILP + // S^T first; the dP^T MMAs are issued in the same basic block as exp(S^T) + // so ptxas interleaves HMMA with the MUFU/FFMA work. +#pragma unroll + for (int ks = 0; ks < 16; ++ks) { + uint2 ak = lds2(Ks, so(16 * w + mo + j + 4 * hi, 4 * ks)); +#pragma unroll + for (int nb = 0; nb < 4; ++nb) + mma_row_col(st[nb], ak, lds2(Qs, lo(nb * 16 + no + j + 4 * hi, 4 * ks))); + } +#define DP_MMAS \ + _Pragma("unroll") for (int ks = 0; ks < 16; ++ks) { \ + uint2 av = lds2(Vs, so(16 * w + mo + j + 4 * hi, 4 * ks)); \ + _Pragma("unroll") for (int nb = 0; nb < 4; ++nb) \ + mma_row_col(dp[nb], av, lds2(dOs, lo(nb * 16 + no + j + 4 * hi, 4 * ks))); \ + } +#define DS_FROM_P \ + _Pragma("unroll") for (int nb = 0; nb < 4; ++nb) \ + _Pragma("unroll") for (int i = 0; i < 8; ++i) { \ + int c = nb * 16 + no + (i & 4) + (lane & 2) + (i & 1); \ + dp[nb][i] = st[nb][i] * (dp[nb][i] - d_s[c]); \ + } +#else +#pragma unroll + for (int ks = 0; ks < 16; ++ks) { + uint2 ak = lds2(Ks, so(16 * w + mo + j + 4 * hi, 4 * ks)); + uint2 av = lds2(Vs, so(16 * w + mo + j + 4 * hi, 4 * ks)); +#pragma unroll + for (int nb = 0; nb < 4; ++nb) { + int n = nb * 16 + no + j + 4 * hi; + mma_row_col(st[nb], ak, lds2(Qs, lo(n, 4 * ks))); + mma_row_col(dp[nb], av, lds2(dOs, lo(n, 4 * ks))); + } + } +#endif + + const bool diag = (qt == kt); + // Masks are needed only on the diagonal tile and a document's last query + // tile (a partial key tile is always diagonal); uniform across the CTA. + if (diag || qt * TL + TL > L) { +#pragma unroll + for (int nb = 0; nb < 4; ++nb) +#pragma unroll + for (int i = 0; i < 8; ++i) { + int c = nb * 16 + no + (i & 4) + (lane & 2) + (i & 1); + int qr = qt * TL + c; + int kr = kr0 + (i & 2); + bool ok = qr < L && kr < L && (!diag || kr <= qr); + float p = ok ? ex2(fmaf(st[nb][i], scale_log2, -lse_s[c])) : 0.f; + st[nb][i] = p; +#if !BWD_ILP + dp[nb][i] = p * (dp[nb][i] - d_s[c]); +#endif + } +#if BWD_ILP + DP_MMAS + DS_FROM_P +#endif + } else { +#pragma unroll + for (int nb = 0; nb < 4; ++nb) +#pragma unroll + for (int i = 0; i < 8; ++i) { + int c = nb * 16 + no + (i & 4) + (lane & 2) + (i & 1); + float p = ex2(fmaf(st[nb][i], scale_log2, -lse_s[c])); + st[nb][i] = p; +#if !BWD_ILP + dp[nb][i] = p * (dp[nb][i] - d_s[c]); +#endif + } +#if BWD_ILP + DP_MMAS + DS_FROM_P +#endif + } +#undef DP_MMAS +#undef DS_FROM_P + + // dV_w += P^T_w dO + __syncwarp(); +#pragma unroll + for (int nb = 0; nb < 4; ++nb) +#pragma unroll + for (int i = 0; i < 8; i += 2) { + int row = mo + (lane & 1) + (i & 2) + 4 * hi; + int col = nb * 16 + no + (i & 4) + (lane & 2); + *reinterpret_cast(Sw + so(row, col)) = pack_h2(st[nb][i], st[nb][i + 1]); + } + __syncwarp(); +#pragma unroll + for (int ks = 0; ks < 16; ++ks) { + uint2 a = lds2(Sw, so(mo + j + 4 * hi, 4 * ks)); +#pragma unroll + for (int nb = 0; nb < 4; ++nb) + mma_row_row(dv[nb], a, lds2(dOs, lo(4 * ks + j, nb * 16 + no + 4 * hi))); + } + + // dK_w += dS^T_w Q + __syncwarp(); +#pragma unroll + for (int nb = 0; nb < 4; ++nb) +#pragma unroll + for (int i = 0; i < 8; i += 2) { + int row = mo + (lane & 1) + (i & 2) + 4 * hi; + int col = nb * 16 + no + (i & 4) + (lane & 2); + *reinterpret_cast(Sw + so(row, col)) = pack_h2(dp[nb][i], dp[nb][i + 1]); + } + __syncwarp(); +#pragma unroll + for (int ks = 0; ks < 16; ++ks) { + uint2 a = lds2(Sw, so(mo + j + 4 * hi, 4 * ks)); +#pragma unroll + for (int nb = 0; nb < 4; ++nb) + mma_row_row(dk[nb], a, lds2(Qs, lo(4 * ks + j, nb * 16 + no + 4 * hi))); + } + + // dQ (64 queries x 64 dims) += dS K; warp w takes queries 16w..16w+15. + // Its A operand is dS = (dS^T)^T, read straight from St as A.col. + __syncthreads(); + float dq[4][8]; +#pragma unroll + for (int a = 0; a < 4; ++a) +#pragma unroll + for (int i = 0; i < 8; ++i) dq[a][i] = 0.f; +#pragma unroll + for (int ks = 0; ks < 16; ++ks) { + uint2 a = lds2(St, so(4 * ks + j, 16 * w + mo + 4 * hi)); +#pragma unroll + for (int nb = 0; nb < 4; ++nb) + mma_col_row(dq[nb], a, lds2(Ks, so(4 * ks + j, nb * 16 + no + 4 * hi))); + } + // Stage the warp's 16x64 dQ slice in shared memory and issue the atomics + // row-contiguous: each warp-wide RED then covers 128 consecutive bytes + // (4 sectors) instead of scattering over 8 rows 2KB apart. Row stride is + // 64 floats; float2 slots are XOR-swizzled by row so both the fragment- + // layout writes and the row reads are bank-conflict free. + float* dQw = dQs + 16 * w * 64; +#pragma unroll + for (int nb = 0; nb < 4; ++nb) +#pragma unroll + for (int i = 0; i < 8; i += 2) { + int row = mo + (lane & 1) + (i & 2) + 4 * hi; + int c2 = (nb * 16 + no + (i & 4) + (lane & 2)) >> 1; + int f = ((row & 1) << 1) | (row & 8); + *reinterpret_cast(dQw + row * 64 + 2 * (c2 ^ f)) = + make_float2(dq[nb][i], dq[nb][i + 1]); + } + __syncwarp(); + { + const int qbase = qt * TL + 16 * w; + const int nrow = min(16, L - qbase); + if (nrow == 16) { + // full 16-row slice: unrolled, smem offsets are 4 lane bases + immediates, + // global offsets one 64-bit pointer + row * (H*HD) increments. + float* gp = dQacc + ((long)(s0 + qbase) * H + h) * HD + lane; + const long rs = (long)H * HD; + const int x0 = lane >> 1, b0 = lane & 1; +#pragma unroll + for (int row = 0; row < 16; ++row) { + const int f = ((row & 1) << 1) | (row & 8); + const float* sr = dQw + row * 64 + 2 * (x0 ^ f) + b0; + atomicAdd(gp, sr[0]); + atomicAdd(gp + 32, sr[32]); + gp += rs; + } + } else + for (int it = 0; it < 2 * nrow; ++it) { + int row = it >> 1, col = (it & 1) * 32 + lane; + int f = ((row & 1) << 1) | (row & 8); + float v = dQw[row * 64 + 2 * ((col >> 1) ^ f) + (col & 1)]; + atomicAdd(dQacc + ((long)(s0 + qbase + row) * H + h) * HD + col, v); + } + } + } + +#pragma unroll + for (int r = 0; r < 2; ++r) { + int kr = kr0 + 2 * r; + if (kr >= L) continue; + half* gk = dK + (long)(s0 + kr) * sdk + h * HD; + half* gv = dV + (long)(s0 + kr) * sdv + h * HD; +#pragma unroll + for (int nb = 0; nb < 4; ++nb) +#pragma unroll + for (int i = 2 * r; i < 8; i += 4) { + int col = nb * 16 + no + (i & 4) + (lane & 2); + *reinterpret_cast(gk + col) = pack_h2(dk[nb][i] * scale, dk[nb][i + 1] * scale); + *reinterpret_cast(gv + col) = pack_h2(dv[nb][i], dv[nb][i + 1]); + } + } +} + +// D[h][n] = sum_d dO[n,h,d] * O[n,h,d] in fp32. One warp per (n, h) row pair. +__global__ void bwd_pre_kernel(const half* __restrict__ dO, const half* __restrict__ O, + float* __restrict__ Dv, int N, int H, int sdo, int s_o) { + int gw = (blockIdx.x * blockDim.x + threadIdx.x) >> 5, lane = threadIdx.x & 31; + if (gw >= N * H) return; + int n = gw / H, h = gw % H; + __half2 a = reinterpret_cast(dO + (long)n * sdo + h * HD)[lane]; + __half2 b = reinterpret_cast(O + (long)n * s_o + h * HD)[lane]; + float2 fa = __half22float2(a), fb = __half22float2(b); + float acc = fa.x * fb.x + fa.y * fb.y; +#pragma unroll + for (int o = 16; o; o >>= 1) acc += __shfl_xor_sync(0xffffffff, acc, o); + if (lane == 0) Dv[(long)h * N + n] = acc; +} + +// dq = fp16(dQacc * scale), four elements per thread. +__global__ void bwd_post_kernel(const float4* __restrict__ acc, uint2* __restrict__ dq, + long n4, float scale) { + long i = blockIdx.x * (long)blockDim.x + threadIdx.x; + if (i >= n4) return; + float4 v = acc[i]; + dq[i] = make_uint2(pack_h2(v.x * scale, v.y * scale), pack_h2(v.z * scale, v.w * scale)); +} + +void check_qkv(const torch::Tensor& t, const char* name) { + TORCH_CHECK(t.is_cuda() && t.scalar_type() == at::kHalf, name, " must be cuda fp16"); + TORCH_CHECK(t.dim() == 3 && t.size(2) == HD && t.stride(2) == 1 && t.stride(1) == HD, + name, " must be (N, H, 64) with contiguous heads"); + TORCH_CHECK(t.stride(0) % 8 == 0, name, " token stride must be a multiple of 8"); +} + +void check_meta(const torch::Tensor& q, const torch::Tensor& k, const torch::Tensor& v, + const torch::Tensor& cu, const torch::Tensor& tcum) { + TORCH_CHECK(k.sizes() == q.sizes() && v.sizes() == q.sizes(), + "q, k, v must have the same (N, H, 64) shape"); + TORCH_CHECK(k.device() == q.device() && v.device() == q.device() && + cu.device() == q.device() && tcum.device() == q.device(), + "q, k, v, cu, tcum must be on one device"); + TORCH_CHECK(cu.scalar_type() == at::kInt && tcum.scalar_type() == at::kInt && + cu.dim() == 1 && tcum.dim() == 1 && cu.is_contiguous() && tcum.is_contiguous(), + "cu and tcum must be contiguous 1-D int32"); + TORCH_CHECK(cu.numel() >= 2 && tcum.numel() == cu.numel(), + "cu and tcum must both have ndocs+1 entries"); +} + +} // namespace + +// q, k, v: (N, H, 64) fp16. cu: (ndocs+1,) int32 offsets. tcum: (ndocs+1,) int32 +// cumulative 64-tile counts. Returns (o, lse) with lse (H, N) fp32 in log2 units. +std::vector fwd(torch::Tensor q, torch::Tensor k, torch::Tensor v, + torch::Tensor cu, torch::Tensor tcum, torch::Tensor perm, double scale) { + check_qkv(q, "q"); check_qkv(k, "k"); check_qkv(v, "v"); + check_meta(q, k, v, cu, tcum); + const int N = q.size(0), H = q.size(1), ndocs = cu.numel() - 1; + auto o = torch::empty({N, H, HD}, q.options()); + auto lse = torch::empty({H, N}, q.options().dtype(torch::kFloat)); + // 1-D grid, heads interleaved (b = blockIdx.x / H); >= H * sum of ceil(len/64) + dim3 grid(((N + TL - 1) / TL + ndocs) * H); + fwd_kernel<<>>( + (const half*)q.data_ptr(), (const half*)k.data_ptr(), (const half*)v.data_ptr(), + (half*)o.data_ptr(), lse.data_ptr(), cu.data_ptr(), tcum.data_ptr(), perm.data_ptr(), + ndocs, N, H, q.stride(0), k.stride(0), v.stride(0), o.stride(0), + (float)(scale * 1.4426950408889634)); + C10_CUDA_KERNEL_LAUNCH_CHECK(); + return {o, lse}; +} + +// Returns (dq, dk, dv), each (N, H, 64) fp16. +std::vector bwd_raw(torch::Tensor dout, torch::Tensor q, torch::Tensor k, + torch::Tensor v, torch::Tensor o, torch::Tensor lse, + torch::Tensor cu, torch::Tensor tcum, torch::Tensor perm, double scale) { + check_qkv(q, "q"); check_qkv(k, "k"); check_qkv(v, "v"); + check_meta(q, k, v, cu, tcum); + dout = dout.contiguous(); + check_qkv(dout, "dout"); + TORCH_CHECK(dout.sizes() == q.sizes() && o.sizes() == q.sizes() && o.stride(2) == 1 && + o.stride(1) == HD && dout.device() == q.device() && o.device() == q.device(), + "dout and o must match q"); + TORCH_CHECK(lse.scalar_type() == at::kFloat && lse.is_contiguous() && lse.dim() == 2 && + lse.size(0) == q.size(1) && lse.size(1) == q.size(0), "lse must be (H, N) fp32"); + const int N = q.size(0), H = q.size(1), ndocs = cu.numel() - 1; + // D_i = rowsum(dO * O), fp32, laid out (H, N) like lse. + auto stream = at::cuda::getCurrentCUDAStream(); + auto Dv = torch::empty({H, N}, q.options().dtype(torch::kFloat)); + bwd_pre_kernel<<<(N * H * 32 + 255) / 256, 256, 0, stream>>>( + (const half*)dout.data_ptr(), (const half*)o.data_ptr(), Dv.data_ptr(), N, H, + dout.stride(0), o.stride(0)); + auto dqacc = torch::zeros({N, H, HD}, q.options().dtype(torch::kFloat)); + auto dk = torch::empty({N, H, HD}, q.options()); + auto dv = torch::empty({N, H, HD}, q.options()); + // 1-D grid, heads interleaved (b = blockIdx.x / H); >= H * sum of ceil(len/64) + dim3 grid(((N + TL - 1) / TL + ndocs) * H); + bwd_kernel<<>>( + (const half*)q.data_ptr(), (const half*)k.data_ptr(), (const half*)v.data_ptr(), + (const half*)dout.data_ptr(), lse.data_ptr(), Dv.data_ptr(), + dqacc.data_ptr(), (half*)dk.data_ptr(), (half*)dv.data_ptr(), + cu.data_ptr(), tcum.data_ptr(), perm.data_ptr(), ndocs, N, H, q.stride(0), k.stride(0), + v.stride(0), dout.stride(0), dk.stride(0), dv.stride(0), + (float)(scale * 1.4426950408889634), (float)scale); + C10_CUDA_KERNEL_LAUNCH_CHECK(); + // raw fp32 dQ accumulator, NOT yet multiplied by the softmax scale + return {dqacc, dk, dv}; +} + +std::vector bwd(torch::Tensor dout, torch::Tensor q, torch::Tensor k, + torch::Tensor v, torch::Tensor o, torch::Tensor lse, + torch::Tensor cu, torch::Tensor tcum, torch::Tensor perm, double scale) { + check_qkv(q, "q"); check_qkv(k, "k"); check_qkv(v, "v"); + check_meta(q, k, v, cu, tcum); + dout = dout.contiguous(); + check_qkv(dout, "dout"); + TORCH_CHECK(dout.sizes() == q.sizes() && o.sizes() == q.sizes() && o.stride(2) == 1 && + o.stride(1) == HD && dout.device() == q.device() && o.device() == q.device(), + "dout and o must match q"); + TORCH_CHECK(lse.scalar_type() == at::kFloat && lse.is_contiguous() && lse.dim() == 2 && + lse.size(0) == q.size(1) && lse.size(1) == q.size(0), "lse must be (H, N) fp32"); + const int N = q.size(0), H = q.size(1), ndocs = cu.numel() - 1; + // D_i = rowsum(dO * O), fp32, laid out (H, N) like lse. + auto stream = at::cuda::getCurrentCUDAStream(); + auto Dv = torch::empty({H, N}, q.options().dtype(torch::kFloat)); + bwd_pre_kernel<<<(N * H * 32 + 255) / 256, 256, 0, stream>>>( + (const half*)dout.data_ptr(), (const half*)o.data_ptr(), Dv.data_ptr(), N, H, + dout.stride(0), o.stride(0)); + auto dqacc = torch::zeros({N, H, HD}, q.options().dtype(torch::kFloat)); + auto dk = torch::empty({N, H, HD}, q.options()); + auto dv = torch::empty({N, H, HD}, q.options()); + // 1-D grid, heads interleaved (b = blockIdx.x / H); >= H * sum of ceil(len/64) + dim3 grid(((N + TL - 1) / TL + ndocs) * H); + bwd_kernel<<>>( + (const half*)q.data_ptr(), (const half*)k.data_ptr(), (const half*)v.data_ptr(), + (const half*)dout.data_ptr(), lse.data_ptr(), Dv.data_ptr(), + dqacc.data_ptr(), (half*)dk.data_ptr(), (half*)dv.data_ptr(), + cu.data_ptr(), tcum.data_ptr(), perm.data_ptr(), ndocs, N, H, q.stride(0), k.stride(0), + v.stride(0), dout.stride(0), dk.stride(0), dv.stride(0), + (float)(scale * 1.4426950408889634), (float)scale); + C10_CUDA_KERNEL_LAUNCH_CHECK(); + auto dq = torch::empty({N, H, HD}, q.options()); + long n4 = (long)N * H * HD / 4; + bwd_post_kernel<<<(n4 + 255) / 256, 256, 0, stream>>>( + reinterpret_cast(dqacc.data_ptr()), + reinterpret_cast(dq.data_ptr()), n4, (float)scale); + C10_CUDA_KERNEL_LAUNCH_CHECK(); + return {dq, dk, dv}; +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("fwd", &fwd); + m.def("bwd", &bwd); + m.def("bwd_raw", &bwd_raw); +} diff --git a/volta_kernels/canon_rope.py b/volta_kernels/canon_rope.py new file mode 100644 index 0000000..08b6d8c --- /dev/null +++ b/volta_kernels/canon_rope.py @@ -0,0 +1,171 @@ +"""Fused Canon + RoPE for the Volta path: one Triton pass forward, one backward. + +What it replaces, inside the compiled _pre_attn region (Inductor's choices): + forward constant_pad_nd x2 materialise zero-padded fp32 copies of qkv + add_mul the 3-tap combine, reading those copies + to_copy_cat RoPE on q and k, cast to fp16 + backward slice_backward / pad backward / a 1536x3 weight-grad reduction, + RoPE backward, as separate passes over (N, 1536) + +Semantics are the region's, operation for operation: + w = fp16(canon_w) (the region casts it to qkv's dtype) + y[n] = x[n] + x[n]*w0 + (x[n-1]*m1[n])*w1 + (x[n-2]*m2[n])*w2 fp32 + q, k = rope(y) in fp32, then fp16; v = fp16(y) +m1/m2 are the per-token document masks (0 where t-1 / t-2 is in another +document) and the shifts never cross a packed row. The backward is the exact +transpose; the weight gradient is reduced in fp32 and rounded to fp16 before +widening, as the gradient of the region's fp16 cast of canon_w is. +""" +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +C = 1536 # q | k | v channels +HALF = 32 # RoPE pairs (d, d+32) within a 64-wide head + + +@triton.jit +def _canon_half(qkv, w, rows, ch, mm1, mm2, ok1, ok2): + x0 = tl.load(qkv + rows).to(tl.float32) + x1 = tl.load(qkv + rows - 1536, mask=ok1, other=0.0).to(tl.float32) + x2 = tl.load(qkv + rows - 3072, mask=ok2, other=0.0).to(tl.float32) + w0 = tl.load(w + ch * 3).to(tl.float16).to(tl.float32)[None, :] + w1 = tl.load(w + ch * 3 + 1).to(tl.float16).to(tl.float32)[None, :] + w2 = tl.load(w + ch * 3 + 2).to(tl.float16).to(tl.float32)[None, :] + return x0 + x0 * w0 + (x1 * mm1) * w1 + (x2 * mm2) * w2 + + +@triton.jit +def _fwd(qkv, w, m1, m2, cos, sin, out, T, GROUP: tl.constexpr, + TB: tl.constexpr, HALF: tl.constexpr): + """TB tokens x one 64-wide head of one group (0 q, 1 k, 2 v).""" + pid_n = tl.program_id(0) + h = tl.program_id(1) + n = pid_n * TB + tl.arange(0, TB) + d = tl.arange(0, HALF) + t = n % T + ca = GROUP * 512 + h * 64 + d + cb = ca + HALF + ok1 = (t >= 1)[:, None] + ok2 = (t >= 2)[:, None] + mm1 = tl.load(m1 + n)[:, None] + mm2 = tl.load(m2 + n)[:, None] + ya = _canon_half(qkv, w, n[:, None] * 1536 + ca[None, :], ca, mm1, mm2, ok1, ok2) + yb = _canon_half(qkv, w, n[:, None] * 1536 + cb[None, :], cb, mm1, mm2, ok1, ok2) + orow = n[:, None] * 512 + h * 64 + d[None, :] + if GROUP < 2: + cs = tl.load(cos + n[:, None] * HALF + d[None, :]) + sn = tl.load(sin + n[:, None] * HALF + d[None, :]) + oa = ya * cs - yb * sn + ob = ya * sn + yb * cs + else: + oa = ya + ob = yb + tl.store(out + orow, oa.to(tl.float16)) + tl.store(out + orow + HALF, ob.to(tl.float16)) + + +@triton.jit +def _dy(src, cos, sin, n, t, S: tl.constexpr, d, ocol, T, ROPE: tl.constexpr, + HALF: tl.constexpr): + """Gradient w.r.t. the Canon output at token n+S, un-rotated; 0 past the row.""" + nn = n + S + ok = ((t + S) < T)[:, None] + ga = tl.load(src + nn[:, None] * 512 + ocol, mask=ok, other=0.0).to(tl.float32) + gb = tl.load(src + nn[:, None] * 512 + ocol + HALF, mask=ok, other=0.0).to(tl.float32) + if ROPE: + cs = tl.load(cos + nn[:, None] * HALF + d[None, :], mask=ok, other=0.0) + sn = tl.load(sin + nn[:, None] * HALF + d[None, :], mask=ok, other=0.0) + return ga * cs + gb * sn, gb * cs - ga * sn + return ga, gb + + +@triton.jit +def _bwd_half(qkv, w, dqkv, dw_part, pid_n, n, ch, dy0, dy1, dy2, + mm1, mm2, mm1n, mm2n, p1, p2): + w0 = tl.load(w + ch * 3).to(tl.float16).to(tl.float32)[None, :] + w1 = tl.load(w + ch * 3 + 1).to(tl.float16).to(tl.float32)[None, :] + w2 = tl.load(w + ch * 3 + 2).to(tl.float16).to(tl.float32)[None, :] + rows = n[:, None] * 1536 + ch[None, :] + # dx[n] = dy[n](1 + w0) + dy[n+1] m1[n+1] w1 + dy[n+2] m2[n+2] w2 + dx = dy0 + dy0 * w0 + (dy1 * mm1n) * w1 + (dy2 * mm2n) * w2 + tl.store(dqkv + rows, dx.to(tl.float16)) + x0 = tl.load(qkv + rows).to(tl.float32) + x1 = tl.load(qkv + rows - 1536, mask=p1, other=0.0).to(tl.float32) + x2 = tl.load(qkv + rows - 3072, mask=p2, other=0.0).to(tl.float32) + base = dw_part + pid_n * (1536 * 3) + ch * 3 + tl.store(base, tl.sum(dy0 * x0, 0)) + tl.store(base + 1, tl.sum(dy0 * (x1 * mm1), 0)) + tl.store(base + 2, tl.sum(dy0 * (x2 * mm2), 0)) + + +@triton.jit +def _bwd(qkv, w, m1, m2, cos, sin, src, dqkv, dw_part, T, GROUP: tl.constexpr, + TB: tl.constexpr, HALF: tl.constexpr): + pid_n = tl.program_id(0) + h = tl.program_id(1) + n = pid_n * TB + tl.arange(0, TB) + d = tl.arange(0, HALF) + t = n % T + ca = GROUP * 512 + h * 64 + d + cb = ca + HALF + ocol = h * 64 + d[None, :] + ROPE: tl.constexpr = GROUP < 2 + dya0, dyb0 = _dy(src, cos, sin, n, t, 0, d, ocol, T, ROPE, HALF) + dya1, dyb1 = _dy(src, cos, sin, n, t, 1, d, ocol, T, ROPE, HALF) + dya2, dyb2 = _dy(src, cos, sin, n, t, 2, d, ocol, T, ROPE, HALF) + mm1n = tl.load(m1 + n + 1, mask=(t + 1) < T, other=0.0)[:, None] + mm2n = tl.load(m2 + n + 2, mask=(t + 2) < T, other=0.0)[:, None] + mm1 = tl.load(m1 + n)[:, None] + mm2 = tl.load(m2 + n)[:, None] + p1 = (t >= 1)[:, None] + p2 = (t >= 2)[:, None] + _bwd_half(qkv, w, dqkv, dw_part, pid_n, n, ca, dya0, dya1, dya2, mm1, mm2, mm1n, mm2n, p1, p2) + _bwd_half(qkv, w, dqkv, dw_part, pid_n, n, cb, dyb0, dyb1, dyb2, mm1, mm2, mm1n, mm2n, p1, p2) + + +TB = 32 + + +class CanonRope(torch.autograd.Function): + @staticmethod + def forward(ctx, qkv, canon_w, m1, m2, cos, sin): + B, T, Cc = qkv.shape + assert Cc == C and qkv.dtype == torch.float16 and qkv.is_contiguous() + N = B * T + assert N % TB == 0 + q = torch.empty((B, T, 8, 64), device=qkv.device, dtype=torch.float16) + k = torch.empty_like(q) + v = torch.empty_like(q) + m1f = m1.reshape(N).contiguous() + m2f = m2.reshape(N).contiguous() + cosf = cos.reshape(N, HALF).contiguous() + sinf = sin.reshape(N, HALF).contiguous() + for g_, out in enumerate((q, k, v)): + _fwd[(N // TB, 8)](qkv, canon_w, m1f, m2f, cosf, sinf, out, T, GROUP=g_, + TB=TB, HALF=HALF, num_warps=4) + ctx.save_for_backward(qkv, canon_w, m1f, m2f, cosf, sinf) + ctx.T = T + return q, k, v + + @staticmethod + def backward(ctx, gq, gk, gv): + qkv, canon_w, m1f, m2f, cosf, sinf = ctx.saved_tensors + B, T, _ = qkv.shape + N = B * T + gq, gk, gv = (g.to(torch.float16).contiguous() for g in (gq, gk, gv)) + dqkv = torch.empty_like(qkv) + part = torch.empty((N // TB, C, 3), device=qkv.device, dtype=torch.float32) + for g_, src in enumerate((gq, gk, gv)): + _bwd[(N // TB, 8)](qkv, canon_w, m1f, m2f, cosf, sinf, src, dqkv, part, T, + GROUP=g_, TB=TB, HALF=HALF, num_warps=4) + # fp32 reduction, then the fp16 rounding the region's cast of canon_w implies + dw = part.sum(0).to(torch.float16).to(canon_w.dtype) + return dqkv, dw, None, None, None, None + + +def canon_rope(qkv, canon_w, m1, m2, cos, sin): + """qkv (B,T,1536) fp16 -> q, k, v (B,T,8,64) fp16, Canon + RoPE applied.""" + return CanonRope.apply(qkv, canon_w, m1, m2, cos, sin) diff --git a/volta_kernels/canon_rope_bw.py b/volta_kernels/canon_rope_bw.py new file mode 100644 index 0000000..172de86 --- /dev/null +++ b/volta_kernels/canon_rope_bw.py @@ -0,0 +1,327 @@ +"""Canon + RoPE in CUDA, bandwidth rewrite (swarm agent canon_bw, 2026-09-27). + +Same math, same rounding as volta_kernels/canon_rope_cuda.py (q/k/v and dqkv +bitwise identical; Canon weight-gradient partials bitwise identical, so the +weight gradient is too when reduced the same way). + +Why the old kernels sit at ~65% of DRAM bandwidth (V100, 72 SMs): + fwd 88 regs x 192 thr -> 3 CTAs/SM (18 warps), grid 512 = 2.37 waves + (last wave 37% full), one 16-B load in flight per thread. + bwd 162 regs x 192 thr -> 2 CTAs/SM (12 warps), grid 256 = 1.78 waves. +What changes here: + * CPT channels per thread (default 4 instead of 8) and one q/k/v group per + CTA (g = blockIdx.x % 3, so the q, k and v CTAs of one token chunk run + side by side and share cos/sin and halo rows through L2): ~half the + registers, 128-thread CTAs, 3x more CTAs -> high occupancy, small tail. + * U-deep register prefetch of the streamed rows (qkv / gradients / masks). + * bwd keeps the 128-token chunk and the backward walk order, so the Canon + weight-gradient partials are the same fp32 sums in the same order. +""" +import os, torch +from torch.utils.cpp_extension import load_inline + +SRC = r""" +#include +#include +#include + +template struct HV; // CPT fp16 values in one vector load +template <> struct HV<8> { typedef uint4 T; }; +template <> struct HV<4> { typedef uint2 T; }; +template <> struct HV<2> { typedef unsigned int T; }; +template struct FV; // CPT fp32 values +template <> struct FV<8> { typedef struct { float4 a, b; } T; }; +template <> struct FV<4> { typedef struct { float4 a; } T; }; +template <> struct FV<2> { typedef struct { float2 a; } T; }; + +template __device__ __forceinline__ typename HV::T ldh(const half* p) { + return *reinterpret_cast::T*>(p); } +template __device__ __forceinline__ void cvh(const typename HV::T& u, float* x) { + const __half2* h = reinterpret_cast(&u); +#pragma unroll + for (int i = 0; i < CPT / 2; ++i) { float2 f = __half22float2(h[i]); x[2*i] = f.x; x[2*i+1] = f.y; } +} +template __device__ __forceinline__ void sth(half* p, const float* o) { + typename HV::T u; __half2* h = reinterpret_cast<__half2*>(&u); +#pragma unroll + for (int i = 0; i < CPT / 2; ++i) h[i] = __floats2half2_rn(o[2*i], o[2*i+1]); + *reinterpret_cast::T*>(p) = u; +} +template __device__ __forceinline__ void ldcs(const float* p, float* x) { + if constexpr (CPT == 2) { float2 f = *reinterpret_cast(p); x[0] = f.x; x[1] = f.y; } + else { + const float4* q = reinterpret_cast(p); +#pragma unroll + for (int j = 0; j < CPT / 4; ++j) { float4 f = q[j]; x[4*j] = f.x; x[4*j+1] = f.y; x[4*j+2] = f.z; x[4*j+3] = f.w; } + } +} + +// ---------------------------------------------------------------- forward +// grid: (N/FCH) * 3 CTAs of TPB = 512/CPT threads; CTA b handles group g = b % 3 +// (q, k, v) of token chunk b / 3. +template +__global__ void __launch_bounds__(512 / CPT, MINB) crbw_fwd( + const half* __restrict__ qkv, const float* __restrict__ w, const float* __restrict__ m1, + const float* __restrict__ m2, const float* __restrict__ cs, const float* __restrict__ sn, + half* __restrict__ q, half* __restrict__ k, half* __restrict__ v, int T) { + constexpr int PX = 32 / CPT; // lane distance of the RoPE partner (d +/- 32) + const int g = blockIdx.x % 3; + const int col = CPT * threadIdx.x, ch = g * 512 + col, d0 = col & 63; + const long n0 = (long)(blockIdx.x / 3) * FCH; + const int t0 = n0 % T; + float w0[CPT], w1[CPT], w2[CPT]; +#pragma unroll + for (int i = 0; i < CPT; ++i) { // the region casts canon_w to fp16 + w0[i] = __half2float(__float2half(w[(ch + i) * 3 + 0])); + w1[i] = __half2float(__float2half(w[(ch + i) * 3 + 1])); + w2[i] = __half2float(__float2half(w[(ch + i) * 3 + 2])); + } + const half* src = qkv + ch; + float x1[CPT], x2[CPT], x0[CPT]; +#pragma unroll + for (int i = 0; i < CPT; ++i) x1[i] = x2[i] = 0.f; + if (t0 >= 1) cvh(ldh(src + (n0 - 1) * 1536), x1); + if (t0 >= 2) cvh(ldh(src + (n0 - 2) * 1536), x2); + half* out = (g == 0 ? q : (g == 1 ? k : v)) + col; + const int d = d0 & 31; + typename HV::T raw[U]; float A1[U], A2[U]; +#pragma unroll + for (int u = 0; u < U; ++u) { raw[u] = ldh(src + (n0 + u) * 1536); A1[u] = m1[n0 + u]; A2[u] = m2[n0 + u]; } +#pragma unroll 1 + for (int s0 = 0; s0 < FCH; s0 += U) { + typename HV::T cur[U]; float C1[U], C2[U]; +#pragma unroll + for (int u = 0; u < U; ++u) { cur[u] = raw[u]; C1[u] = A1[u]; C2[u] = A2[u]; } + if (s0 + U < FCH) { +#pragma unroll + for (int u = 0; u < U; ++u) { + const long n = n0 + s0 + U + u; + raw[u] = ldh(src + n * 1536); A1[u] = m1[n]; A2[u] = m2[n]; + } + } +#pragma unroll + for (int u = 0; u < U; ++u) { + const long n = n0 + s0 + u; + cvh(cur[u], x0); + const float a1 = C1[u], a2 = C2[u]; + float y[CPT]; +#pragma unroll + for (int i = 0; i < CPT; ++i) + y[i] = x0[i] + x0[i] * w0[i] + (x1[i] * a1) * w1[i] + (x2[i] * a2) * w2[i]; + float o[CPT]; + if (g < 2) { + float c[CPT], sv[CPT]; + ldcs(cs + n * 32 + d, c); ldcs(sn + n * 32 + d, sv); +#pragma unroll + for (int i = 0; i < CPT; ++i) { + float p = __shfl_xor_sync(0xffffffff, y[i], PX); + o[i] = d0 < 32 ? fmaf(y[i], c[i], -(p * sv[i])) : fmaf(y[i], c[i], p * sv[i]); + } + } else { +#pragma unroll + for (int i = 0; i < CPT; ++i) o[i] = y[i]; + } + sth(out + n * 512, o); +#pragma unroll + for (int i = 0; i < CPT; ++i) { x2[i] = x1[i]; x1[i] = x0[i]; } + } + } +} + +// ---------------------------------------------------------------- backward +// Q32: the q-gradient is the attention backward's raw fp32 dQ accumulator, times qscale. +// CH must stay 128 for the weight-gradient partials to be the old kernel's sums. +template +__global__ void __launch_bounds__(512 / CPT, MINB) crbw_bwd( + const half* __restrict__ qkv, const float* __restrict__ w, const float* __restrict__ m1, + const float* __restrict__ m2, const float* __restrict__ cs, const float* __restrict__ sn, + const void* __restrict__ gq_, const half* __restrict__ gk, const half* __restrict__ gv, + half* __restrict__ dqkv, float* __restrict__ part, int T, float qscale) { + constexpr int PX = 32 / CPT; + const int g = blockIdx.x % 3; + const int col = CPT * threadIdx.x, ch = g * 512 + col, d0 = col & 63, d = d0 & 31; + const long chunk = blockIdx.x / 3, n0 = chunk * CH; + const int t0 = n0 % T; + const bool f32 = Q32 && g == 0; + const half* G = (g == 0 ? reinterpret_cast(gq_) : (g == 1 ? gk : gv)) + col; + const float* G32 = reinterpret_cast(gq_) + col; + float w0[CPT], w1[CPT], w2[CPT]; +#pragma unroll + for (int i = 0; i < CPT; ++i) { + w0[i] = __half2float(__float2half(w[(ch + i) * 3 + 0])); + w1[i] = __half2float(__float2half(w[(ch + i) * 3 + 1])); + w2[i] = __half2float(__float2half(w[(ch + i) * 3 + 2])); + } + // raw gradient row load (fp32 path: CPT floats; fp16 path: CPT halves in .a's bytes) + typedef typename FV::T F; + auto ldg_ = [&](long n, F& r) { + if (f32) r = *reinterpret_cast(G32 + n * 512); + else *reinterpret_cast::T*>(&r) = ldh(G + n * 512); + }; + // dy(n) from a raw gradient row: un-rotated gradient w.r.t. the Canon output. + auto dy = [&](long n, const F& r, float* out) { + float gg[CPT]; + if (f32) { + const float* f = reinterpret_cast(&r); +#pragma unroll + for (int i = 0; i < CPT; ++i) gg[i] = f[i] * qscale; + } else { + cvh(*reinterpret_cast::T*>(&r), gg); + } + if (g < 2) { + float c[CPT], sv[CPT]; + ldcs(cs + n * 32 + d, c); ldcs(sn + n * 32 + d, sv); +#pragma unroll + for (int i = 0; i < CPT; ++i) { + float p = __shfl_xor_sync(0xffffffff, gg[i], PX); + float ga = d0 < 32 ? gg[i] : p, gb = d0 < 32 ? p : gg[i]; + out[i] = d0 < 32 ? fmaf(ga, c[i], gb * sv[i]) : fmaf(gb, c[i], -(ga * sv[i])); + } + } else { +#pragma unroll + for (int i = 0; i < CPT; ++i) out[i] = gg[i]; + } + }; + const half* X = qkv + ch; + const long nl = n0 + CH - 1; + const int tl = t0 + CH - 1; + float d0v[CPT], d1v[CPT], d2v[CPT], x0[CPT], xm1[CPT], xm2[CPT]; + float mm1n = 0.f, mm2n = 0.f, m2p1 = 0.f; +#pragma unroll + for (int i = 0; i < CPT; ++i) d1v[i] = d2v[i] = 0.f; + { F r; float tmp[CPT]; + long a = (tl + 1 < T) ? nl + 1 : nl; ldg_(a, r); dy(a, r, tmp); + if (tl + 1 < T) { mm1n = m1[nl + 1]; m2p1 = m2[nl + 1]; +#pragma unroll + for (int i = 0; i < CPT; ++i) d1v[i] = tmp[i]; } + long b = (tl + 2 < T) ? nl + 2 : nl; ldg_(b, r); dy(b, r, tmp); + if (tl + 2 < T) { mm2n = m2[nl + 2]; +#pragma unroll + for (int i = 0; i < CPT; ++i) d2v[i] = tmp[i]; } } + cvh(ldh(X + nl * 1536), x0); + cvh(ldh(X + (nl - 1) * 1536), xm1); + cvh(ldh(X + (nl - 2) * 1536), xm2); + float a0[CPT], a1[CPT], a2[CPT]; +#pragma unroll + for (int i = 0; i < CPT; ++i) a0[i] = a1[i] = a2[i] = 0.f; + // prefetch ring: for step s (token n = n0+s) we need grad row n, masks n, x row n-3. + F rg[U]; typename HV::T rx[U]; float R1[U], R2[U]; + auto fetch = [&](int s, int u) { + const long n = n0 + s; const int t = t0 + s; + ldg_(n, rg[u]); R1[u] = m1[n]; R2[u] = m2[n]; + if (t - 3 >= 0) rx[u] = ldh(X + (n - 3) * 1536); + }; +#pragma unroll + for (int u = 0; u < U; ++u) fetch(CH - 1 - u, u); +#pragma unroll 1 + for (int sb = CH - 1; sb >= 0; sb -= U) { + F cg[U]; typename HV::T cx[U]; float C1[U], C2[U]; +#pragma unroll + for (int u = 0; u < U; ++u) { cg[u] = rg[u]; cx[u] = rx[u]; C1[u] = R1[u]; C2[u] = R2[u]; } + if (sb - U >= 0) { +#pragma unroll + for (int u = 0; u < U; ++u) fetch(sb - U - u, u); + } +#pragma unroll + for (int u = 0; u < U; ++u) { + const int s = sb - u; + const long n = n0 + s; const int t = t0 + s; + dy(n, cg[u], d0v); + const float m1n = C1[u], m2n = C2[u]; + float dx[CPT]; +#pragma unroll + for (int i = 0; i < CPT; ++i) { + dx[i] = d0v[i] + d0v[i] * w0[i] + (d1v[i] * mm1n) * w1[i] + (d2v[i] * mm2n) * w2[i]; + a0[i] += d0v[i] * x0[i]; + a1[i] += d0v[i] * (xm1[i] * m1n); + a2[i] += d0v[i] * (xm2[i] * m2n); + } + sth(dqkv + n * 1536 + ch, dx); + mm2n = m2p1; m2p1 = m2n; mm1n = m1n; +#pragma unroll + for (int i = 0; i < CPT; ++i) { d2v[i] = d1v[i]; d1v[i] = d0v[i]; x0[i] = xm1[i]; xm1[i] = xm2[i]; } + if (s > 0) { + if (t - 3 >= 0) cvh(cx[u], xm2); + else { +#pragma unroll + for (int i = 0; i < CPT; ++i) xm2[i] = 0.f; } + if (t - 1 < 1) { +#pragma unroll + for (int i = 0; i < CPT; ++i) xm1[i] = 0.f; } + } + } + } + float* pp = part + chunk * 1536 * 3 + ch * 3; +#pragma unroll + for (int i = 0; i < CPT; ++i) { pp[3*i] = a0[i]; pp[3*i+1] = a1[i]; pp[3*i+2] = a2[i]; } +} + +// Deterministic Canon weight-gradient reduction: sum over chunks in fp64 (more +// precise than the fp32 torch sum), one thread per (channel, tap). +__global__ void crbw_wsum(const float* __restrict__ part, float* __restrict__ dw, int nchunk) { + const int j = blockIdx.x * blockDim.x + threadIdx.x; + if (j >= 1536 * 3) return; + double s = 0.0; + for (int c = 0; c < nchunk; ++c) s += (double)part[(long)c * 4608 + j]; + dw[j] = (float)s; +} + +#define FWD_CFG(CPT, FCH, U, MB) \ + if (cpt == CPT && fch == FCH && u == U && minb == MB) { \ + TORCH_CHECK(T % FCH == 0); \ + crbw_fwd<<<(N / FCH) * 3, 512 / CPT, 0, st>>>(P(qkv), w.data_ptr(), m1.data_ptr(), \ + m2.data_ptr(), cs.data_ptr(), sn.data_ptr(), P(q), P(k), P(v), T); \ + C10_CUDA_KERNEL_LAUNCH_CHECK(); return {q, k, v}; } +#define P(t) reinterpret_cast(t.data_ptr()) + +std::vector fwd(torch::Tensor qkv, torch::Tensor w, torch::Tensor m1, torch::Tensor m2, + torch::Tensor cs, torch::Tensor sn, int64_t cpt, int64_t fch, int64_t u, int64_t minb) { + const int B = qkv.size(0), T = qkv.size(1); const long N = (long)B * T; + auto q = torch::empty({B, T, 8, 64}, qkv.options()), k = torch::empty_like(q), v = torch::empty_like(q); + auto st = at::cuda::getCurrentCUDAStream(); + FWD_CFG(4, 64, 4, 1) FWD_CFG(4, 64, 4, 6) FWD_CFG(4, 64, 4, 8) FWD_CFG(4, 64, 2, 8) FWD_CFG(2, 64, 4, 4) + FWD_CFG(2, 64, 2, 4) FWD_CFG(4, 32, 4, 6) FWD_CFG(4, 64, 1, 8) FWD_CFG(2, 64, 4, 1) + TORCH_CHECK(false, "unsupported fwd config"); +} + +#define BWD_CFG(CPT, U, MB) \ + if (cpt == CPT && u == U && minb == MB) { \ + if (q32) crbw_bwd<<<(N / 128) * 3, 512 / CPT, 0, st>>>(P(qkv), w.data_ptr(), \ + m1.data_ptr(), m2.data_ptr(), cs.data_ptr(), sn.data_ptr(), gq.data_ptr(), \ + P(gk), P(gv), P(dqkv), part.data_ptr(), T, (float)qscale); \ + else crbw_bwd<<<(N / 128) * 3, 512 / CPT, 0, st>>>(P(qkv), w.data_ptr(), \ + m1.data_ptr(), m2.data_ptr(), cs.data_ptr(), sn.data_ptr(), gq.data_ptr(), \ + P(gk), P(gv), P(dqkv), part.data_ptr(), T, (float)qscale); \ + C10_CUDA_KERNEL_LAUNCH_CHECK(); ok = true; } + +// q32: gq is the (B, T, 8, 64) fp32 raw dQ accumulator; qscale folds in the softmax scale. +// wsum: 0 -> return part (N/128, 1536, 3) like the old op; 1 -> fused fp64 reduction, returns dw (1536, 3). +std::vector bwd(torch::Tensor qkv, torch::Tensor w, torch::Tensor m1, torch::Tensor m2, + torch::Tensor cs, torch::Tensor sn, torch::Tensor gq, torch::Tensor gk, + torch::Tensor gv, double qscale, bool q32, int64_t cpt, int64_t u, int64_t minb, bool wsum) { + const int B = qkv.size(0), T = qkv.size(1); const long N = (long)B * T; + TORCH_CHECK(T % 128 == 0, "T must be a multiple of 128"); + TORCH_CHECK(gq.is_contiguous() && gk.is_contiguous() && gv.is_contiguous()); + TORCH_CHECK(!q32 || gq.scalar_type() == at::kFloat, "gq32 must be fp32"); + auto dqkv = torch::empty_like(qkv); + auto part = torch::empty({N / 128, 1536, 3}, qkv.options().dtype(torch::kFloat)); + auto st = at::cuda::getCurrentCUDAStream(); + bool ok = false; + BWD_CFG(4, 2, 1) BWD_CFG(4, 2, 5) BWD_CFG(4, 2, 6) BWD_CFG(4, 1, 6) BWD_CFG(4, 1, 1) BWD_CFG(2, 2, 3) + BWD_CFG(2, 1, 3) BWD_CFG(2, 2, 1) BWD_CFG(4, 4, 5) + TORCH_CHECK(ok, "unsupported bwd config"); + if (!wsum) return {dqkv, part}; + auto dw = torch::empty({1536, 3}, part.options()); + crbw_wsum<<<(4608 + 127) / 128, 128, 0, st>>>(part.data_ptr(), dw.data_ptr(), (int)(N / 128)); + C10_CUDA_KERNEL_LAUNCH_CHECK(); + return {dqkv, dw}; +} +""" +_here = os.path.dirname(os.path.abspath(__file__)) +_build = os.path.join(_here, "build_canon_bw"); os.makedirs(_build, exist_ok=True) +os.environ.setdefault("TORCH_CUDA_ARCH_LIST", "7.0") +ext = load_inline("boris_volta_canon_bw", cpp_sources= + "std::vector fwd(torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, int64_t, int64_t, int64_t, int64_t);" + "std::vector bwd(torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, double, bool, int64_t, int64_t, int64_t, bool);", + cuda_sources=SRC, functions=["fwd", "bwd"], extra_cuda_cflags=["-O3", "-arch=sm_70"], + build_directory=_build) diff --git a/volta_kernels/canon_rope_cuda.py b/volta_kernels/canon_rope_cuda.py new file mode 100644 index 0000000..e632212 --- /dev/null +++ b/volta_kernels/canon_rope_cuda.py @@ -0,0 +1,266 @@ +"""Canon + RoPE in CUDA: a sliding window down the sequence (drop-in for canon_rope.py). + +Written by a second Claude Code reviewer on 2026-09-27; adopted after review. +Each thread owns 8 channels and walks 64 (fwd) / 128 (bwd, backwards) tokens, +keeping the two previous tokens in registers instead of re-reading neighbours +as the Triton version does, with 16-byte loads and stores throughout. +Measured at full size (31 layers, mb 16, idle V100): fwd 0.61 -> 0.32 ms, +bwd 1.20 -> 0.57 ms, whole micro-step +4.1%. q, k, v, dqkv and the loss are +bitwise-identical to the Triton kernel (its RoPE rounding order is copied: +fmaf(y, c, -(p*s))); the Canon weight gradient differs only in fp32 summation +order (same error vs fp64). Requires T % 128 == 0 (2048 and 8192 both are). +""" +import os, torch +from torch.utils.cpp_extension import load_inline +SRC = r""" +#include +#include +#include +constexpr int FCH = 64; // tokens per block (sliding window down the sequence) +__global__ void __launch_bounds__(192) canon_rope_fwd( + const half* __restrict__ qkv, const float* __restrict__ w, const float* __restrict__ m1, + const float* __restrict__ m2, const float* __restrict__ cs, const float* __restrict__ sn, + half* __restrict__ q, half* __restrict__ k, half* __restrict__ v, int T) { + const int c8 = threadIdx.x, ch = 8 * c8, g = ch >> 9, col = ch & 511, d0 = col & 63; + const long n0 = (long)blockIdx.x * FCH; + const int t0 = n0 % T; + float w0[8], w1[8], w2[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) { // the region casts canon_w to fp16 + w0[i] = __half2float(__float2half(w[(ch + i) * 3 + 0])); + w1[i] = __half2float(__float2half(w[(ch + i) * 3 + 1])); + w2[i] = __half2float(__float2half(w[(ch + i) * 3 + 2])); + } + auto ld = [&](long n, float* x) { + uint4 u = *reinterpret_cast(qkv + n * 1536 + ch); + const __half2* h = reinterpret_cast(&u); +#pragma unroll + for (int i = 0; i < 4; ++i) { float2 f = __half22float2(h[i]); x[2*i] = f.x; x[2*i+1] = f.y; } + }; + float x1[8], x2[8], x0[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) x1[i] = x2[i] = 0.f; + if (t0 >= 1) ld(n0 - 1, x1); + if (t0 >= 2) ld(n0 - 2, x2); + half* out = g == 0 ? q : (g == 1 ? k : v); +#pragma unroll 8 + for (int s = 0; s < FCH; ++s) { + const long n = n0 + s; + ld(n, x0); + const float a1 = m1[n], a2 = m2[n]; + float y[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) + y[i] = x0[i] + x0[i] * w0[i] + (x1[i] * a1) * w1[i] + (x2[i] * a2) * w2[i]; + float o[8]; + if (g < 2) { + const int d = d0 & 31; + const float4* cp = reinterpret_cast(cs + n * 32 + d); + const float4* sp = reinterpret_cast(sn + n * 32 + d); + float4 c0 = cp[0], c1 = cp[1], s0 = sp[0], s1 = sp[1]; + float c[8] = {c0.x, c0.y, c0.z, c0.w, c1.x, c1.y, c1.z, c1.w}; + float sv[8] = {s0.x, s0.y, s0.z, s0.w, s1.x, s1.y, s1.z, s1.w}; +#pragma unroll + for (int i = 0; i < 8; ++i) { + float p = __shfl_xor_sync(0xffffffff, y[i], 4); // partner channel d +/- 32 + o[i] = d0 < 32 ? fmaf(y[i], c[i], -(p * sv[i])) : fmaf(y[i], c[i], p * sv[i]); + } + } else { +#pragma unroll + for (int i = 0; i < 8; ++i) o[i] = y[i]; + } + uint4 u; + __half2* h = reinterpret_cast<__half2*>(&u); +#pragma unroll + for (int i = 0; i < 4; ++i) h[i] = __floats2half2_rn(o[2*i], o[2*i+1]); + *reinterpret_cast(out + n * 512 + col) = u; +#pragma unroll + for (int i = 0; i < 8; ++i) { x2[i] = x1[i]; x1[i] = x0[i]; } + } +} +std::vector fwd(torch::Tensor qkv, torch::Tensor w, torch::Tensor m1, torch::Tensor m2, + torch::Tensor cs, torch::Tensor sn) { + const int B = qkv.size(0), T = qkv.size(1); const long N = (long)B * T; + TORCH_CHECK(T % FCH == 0); + auto q = torch::empty({B, T, 8, 64}, qkv.options()), k = torch::empty_like(q), v = torch::empty_like(q); + canon_rope_fwd<<>>( + (const half*)qkv.data_ptr(), w.data_ptr(), m1.data_ptr(), m2.data_ptr(), + cs.data_ptr(), sn.data_ptr(), (half*)q.data_ptr(), (half*)k.data_ptr(), + (half*)v.data_ptr(), T); + return {q, k, v}; +} + +constexpr int CH = 128; +__device__ __forceinline__ void ld8(const half* p, float* x) { + uint4 u = *reinterpret_cast(p); + const __half2* h = reinterpret_cast(&u); +#pragma unroll + for (int i = 0; i < 4; ++i) { float2 f = __half22float2(h[i]); x[2*i] = f.x; x[2*i+1] = f.y; } +} +// Q32: the q-gradient arrives as the attention backward's raw fp32 dQ +// accumulator (times qscale) instead of an fp16 tensor, so it is never rounded. +template +__global__ void __launch_bounds__(192) canon_rope_bwd( + const half* __restrict__ qkv, const float* __restrict__ w, const float* __restrict__ m1, + const float* __restrict__ m2, const float* __restrict__ cs, const float* __restrict__ sn, + const void* __restrict__ gq_, const half* __restrict__ gk, const half* __restrict__ gv, + half* __restrict__ dqkv, float* __restrict__ part, int T, float qscale) { + const half* gq = reinterpret_cast(gq_); + const float* gq32 = reinterpret_cast(gq_); + const int c8 = threadIdx.x, ch = 8 * c8, g = ch >> 9, col = ch & 511, d0 = col & 63; + const long n0 = (long)blockIdx.x * CH; + const int t0 = n0 % T; + const half* G = g == 0 ? gq : (g == 1 ? gk : gv); + float w0[8], w1[8], w2[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) { + w0[i] = __half2float(__float2half(w[(ch + i) * 3 + 0])); + w1[i] = __half2float(__float2half(w[(ch + i) * 3 + 1])); + w2[i] = __half2float(__float2half(w[(ch + i) * 3 + 2])); + } + // dy(n): gradient w.r.t. the Canon output at token n, un-rotated. + auto dy = [&](long n, float* out) { + float gg[8]; + if (Q32 && g == 0) { + const float4* p4 = reinterpret_cast(gq32 + n * 512 + col); + float4 a = p4[0], b = p4[1]; + gg[0] = a.x * qscale; gg[1] = a.y * qscale; gg[2] = a.z * qscale; gg[3] = a.w * qscale; + gg[4] = b.x * qscale; gg[5] = b.y * qscale; gg[6] = b.z * qscale; gg[7] = b.w * qscale; + } else { + ld8(G + n * 512 + col, gg); + } + if (g < 2) { + const int d = d0 & 31; + const float4* cp = reinterpret_cast(cs + n * 32 + d); + const float4* sp = reinterpret_cast(sn + n * 32 + d); + float4 c0 = cp[0], c1 = cp[1], s0 = sp[0], s1 = sp[1]; + float cs_[8] = {c0.x, c0.y, c0.z, c0.w, c1.x, c1.y, c1.z, c1.w}; + float sn_[8] = {s0.x, s0.y, s0.z, s0.w, s1.x, s1.y, s1.z, s1.w}; +#pragma unroll + for (int i = 0; i < 8; ++i) { + float p = __shfl_xor_sync(0xffffffff, gg[i], 4); + float ga = d0 < 32 ? gg[i] : p, gb = d0 < 32 ? p : gg[i]; + out[i] = d0 < 32 ? fmaf(ga, cs_[i], gb * sn_[i]) : fmaf(gb, cs_[i], -(ga * sn_[i])); + } + } else { +#pragma unroll + for (int i = 0; i < 8; ++i) out[i] = gg[i]; + } + }; + const long nl = n0 + CH - 1; // walk from the chunk's last token backwards + const int tl = t0 + CH - 1; + float d0v[8], d1v[8], d2v[8], x0[8], xm1[8], xm2[8]; + float mm1n = 0.f, mm2n = 0.f, m2p1 = 0.f; // m2p1 = m2[n+1] +#pragma unroll + for (int i = 0; i < 8; ++i) d1v[i] = d2v[i] = 0.f; + // dy at n+1, n+2 must be computed by every lane (shuffles), masked afterwards. + { float tmp[8]; + long a = (tl + 1 < T) ? nl + 1 : nl; dy(a, tmp); + if (tl + 1 < T) { mm1n = m1[nl + 1]; m2p1 = m2[nl + 1]; +#pragma unroll + for (int i = 0; i < 8; ++i) d1v[i] = tmp[i]; } + long b = (tl + 2 < T) ? nl + 2 : nl; dy(b, tmp); + if (tl + 2 < T) { mm2n = m2[nl + 2]; +#pragma unroll + for (int i = 0; i < 8; ++i) d2v[i] = tmp[i]; } } + ld8(qkv + nl * 1536 + ch, x0); + ld8(qkv + (nl - 1) * 1536 + ch, xm1); // tl >= 31 so both exist + ld8(qkv + (nl - 2) * 1536 + ch, xm2); + float a0[8], a1[8], a2[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) a0[i] = a1[i] = a2[i] = 0.f; + #pragma unroll 8 + for (int s = CH - 1; s >= 0; --s) { + const long n = n0 + s; const int t = t0 + s; + dy(n, d0v); + const float m1n = m1[n], m2n = m2[n]; + float dx[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) { + dx[i] = d0v[i] + d0v[i] * w0[i] + (d1v[i] * mm1n) * w1[i] + (d2v[i] * mm2n) * w2[i]; + a0[i] += d0v[i] * x0[i]; + a1[i] += d0v[i] * (xm1[i] * m1n); + a2[i] += d0v[i] * (xm2[i] * m2n); + } + uint4 u; __half2* h = reinterpret_cast<__half2*>(&u); +#pragma unroll + for (int i = 0; i < 4; ++i) h[i] = __floats2half2_rn(dx[2*i], dx[2*i+1]); + *reinterpret_cast(dqkv + n * 1536 + ch) = u; + mm2n = m2p1; m2p1 = m2n; mm1n = m1n; +#pragma unroll + for (int i = 0; i < 8; ++i) { d2v[i] = d1v[i]; d1v[i] = d0v[i]; x0[i] = xm1[i]; xm1[i] = xm2[i]; } + if (s > 0) { + if (t - 3 >= 0) ld8(qkv + (n - 3) * 1536 + ch, xm2); + else { +#pragma unroll + for (int i = 0; i < 8; ++i) xm2[i] = 0.f; } + if (t - 1 < 1) { // next token (t-1) is t==0: its t-1 tap is outside the row +#pragma unroll + for (int i = 0; i < 8; ++i) xm1[i] = 0.f; } + } + } + float* pp = part + (long)blockIdx.x * 1536 * 3 + ch * 3; +#pragma unroll + for (int i = 0; i < 8; ++i) { pp[3*i] = a0[i]; pp[3*i+1] = a1[i]; pp[3*i+2] = a2[i]; } +} +std::vector bwd(torch::Tensor qkv, torch::Tensor w, torch::Tensor m1, torch::Tensor m2, + torch::Tensor cs, torch::Tensor sn, torch::Tensor gq, torch::Tensor gk, + torch::Tensor gv) { + const int B = qkv.size(0), T = qkv.size(1); const long N = (long)B * T; + auto dqkv = torch::empty_like(qkv); + auto part = torch::empty({N / CH, 1536, 3}, qkv.options().dtype(torch::kFloat)); + canon_rope_bwd<<>>( + (const half*)qkv.data_ptr(), w.data_ptr(), m1.data_ptr(), m2.data_ptr(), + cs.data_ptr(), sn.data_ptr(), gq.data_ptr(), (const half*)gk.data_ptr(), + (const half*)gv.data_ptr(), (half*)dqkv.data_ptr(), part.data_ptr(), T, 1.f); + C10_CUDA_KERNEL_LAUNCH_CHECK(); + return {dqkv, part}; +} +// gq32: (B, T, 8, 64) fp32 raw dQ accumulator; qscale folds in the softmax scale. +std::vector bwd_q32(torch::Tensor qkv, torch::Tensor w, torch::Tensor m1, torch::Tensor m2, + torch::Tensor cs, torch::Tensor sn, torch::Tensor gq32, torch::Tensor gk, + torch::Tensor gv, double qscale) { + TORCH_CHECK(gq32.scalar_type() == at::kFloat && gq32.is_contiguous(), "gq32 must be contiguous fp32"); + const int B = qkv.size(0), T = qkv.size(1); const long N = (long)B * T; + TORCH_CHECK(T % CH == 0, "T must be a multiple of 128"); + auto dqkv = torch::empty_like(qkv); + auto part = torch::empty({N / CH, 1536, 3}, qkv.options().dtype(torch::kFloat)); + canon_rope_bwd<<>>( + (const half*)qkv.data_ptr(), w.data_ptr(), m1.data_ptr(), m2.data_ptr(), + cs.data_ptr(), sn.data_ptr(), gq32.data_ptr(), (const half*)gk.data_ptr(), + (const half*)gv.data_ptr(), (half*)dqkv.data_ptr(), part.data_ptr(), T, (float)qscale); + C10_CUDA_KERNEL_LAUNCH_CHECK(); + return {dqkv, part}; +} +""" +_here = os.path.dirname(os.path.abspath(__file__)); os.makedirs(os.path.join(_here, "build_canon"), exist_ok=True) +os.environ.setdefault("TORCH_CUDA_ARCH_LIST", "7.0") +ext = load_inline("boris_volta_canon", cpp_sources= + "std::vector fwd(torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor);" + "std::vector bwd(torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor);" + "std::vector bwd_q32(torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, double);", + cuda_sources=SRC, functions=["fwd", "bwd", "bwd_q32"], extra_cuda_cflags=["-O3", "-arch=sm_70"], + build_directory=os.path.join(_here, "build_canon")) + +class CanonRopeFast(torch.autograd.Function): + @staticmethod + def forward(ctx, qkv, canon_w, m1, m2, cos, sin): + B, T, C = qkv.shape; N = B * T + assert C == 1536 and qkv.dtype == torch.float16 and qkv.is_contiguous() and T % 128 == 0 + m1f, m2f = m1.reshape(N).contiguous(), m2.reshape(N).contiguous() + cf, sf = cos.reshape(N, 32).contiguous(), sin.reshape(N, 32).contiguous() + q, k, v = ext.fwd(qkv, canon_w.contiguous(), m1f, m2f, cf, sf) + ctx.save_for_backward(qkv, canon_w, m1f, m2f, cf, sf) + return q, k, v + @staticmethod + def backward(ctx, gq, gk, gv): + qkv, w, m1f, m2f, cf, sf = ctx.saved_tensors + gq, gk, gv = (g.to(torch.float16).contiguous() for g in (gq, gk, gv)) + dqkv, part = ext.bwd(qkv, w.contiguous(), m1f, m2f, cf, sf, gq, gk, gv) + # fp32 straight from the fp32 reduction. Rounding through fp16 (as the + # eager region's cast implied) left ~3% of entries subnormal on the live + # checkpoint; keeping fp32 is strictly more precise. + return dqkv, part.sum(0).to(w.dtype), None, None, None, None + +def canon_rope(qkv, canon_w, m1, m2, cos, sin): + return CanonRopeFast.apply(qkv, canon_w, m1, m2, cos, sin) diff --git a/volta_kernels/head.py b/volta_kernels/head.py new file mode 100644 index 0000000..35c3974 --- /dev/null +++ b/volta_kernels/head.py @@ -0,0 +1,168 @@ +"""Fused LM head (cross-entropy without (N, V) logits), with the elementwise +middle rewritten as one Triton pass. + +Same algorithm, same constants and the same operand precision as +model._FusedLMHead -- read that docstring for why GRAD_SCALE is 64, why the dW +GEMM is shrunk, and why the backward multiplier stays fp32. What changes is only +how many times each chunk's logits cross memory. + +Production spelling, per 1024-row chunk at micro_batch 16 (x fp32 because the +residual stream is fp32): + + F.linear(xc, Wc) autocast casts W (32768x512 fp32) to fp16 -- again + .float() fp16 logits -> fp32 copy 128 MB + _head_chunk_c 3 passes over fp32 logits, writes fp32 p 128 MB + scatter_add_ one-hot, in place + p.to(x.dtype) a no-op: x is fp32, so g is fp32 + F.linear(g, Wc.t()) autocast casts g to fp16 and W to fp16 again + dx[a:b] = ... fp16 -> fp32 copy into dx + g * DW_SHRINK another fp32 pass 128 MB + gd.t() @ xn autocast casts gd^T (strided) to fp16 + +Here W and x are cast once per call, not twice per chunk, and one kernel reads +the fp16 logits once and writes the two fp16 operands the GEMMs were being fed +anyway: fp16(p'), where p' is the scaled softmax minus the one-hot. dW is +accumulated with an fp32-output GEMM, so it is never rounded to fp16 at all -- +the production path's overflow guard (x rescale, DW_SHRINK) has nothing left to +guard. lse differs from Inductor's only in fp32 reduction order. +""" +from __future__ import annotations + +import torch +import torch.nn.functional as F +import triton +import triton.language as tl + +IGNORE = -100 + + +@triton.jit +def _softmax_grad_kernel(logits, tgt, g_out, row_loss, V: tl.constexpr, S: tl.constexpr): + """One program per row: the whole 32k-wide row stays in registers. + + g_out = fp16(S * softmax - S * onehot), the operand both head GEMMs take. + (A blocked two-pass version with online max/sum measured no faster.) + """ + row = tl.program_id(0).to(tl.int64) + cols = tl.arange(0, V) + x = tl.load(logits + row * V + cols).to(tl.float32) + t = tl.load(tgt + row) + valid = t != -100 + safe = tl.where(valid, t, 0) + # Inductor's logsumexp decomposition, including its guard for an + # all-infinite row. + m = tl.max(x, 0) + m = tl.where(tl.abs(m) == float("inf"), 0.0, m) + lse = tl.log(tl.sum(tl.exp(x - m), 0)) + m + vs = valid.to(tl.float32) * S + p = tl.exp(x - lse) * vs + hit = cols == safe + p = tl.where(hit, p + (-S * valid.to(tl.float32)), p) + tl.store(g_out + row * V + cols, p.to(tl.float16)) + picked = tl.sum(tl.where(hit, x, 0.0), 0) + tl.store(row_loss + row, tl.where(valid, lse - picked, 0.0)) + + +class FusedLMHead(torch.autograd.Function): + GRAD_SCALE = 64.0 + + @staticmethod + def forward(ctx, x, W, targets, chunk: int): + N, d = x.shape + V = W.shape[0] + W16 = W.to(torch.float16) + x16 = x.to(torch.float16) + valid = targets != IGNORE + n_valid = valid.sum().clamp_min(1) + # dx is written fp32 straight from the GEMM's fp32 accumulator. With an + # fp16 output cuBLAS may pick split-k kernels that sum partials in fp16 + # (allowed by reduced-precision reduction); fp32 output rules that out. + # Measured on the real fixture: dx error vs fp64 5.31e-4 -> 3.43e-4, + # and 2.9% faster. + dx16 = torch.empty((N, d), device=x.device, dtype=torch.float32) + # zeros, not empty: do not rely on beta=0 never reading the buffer in the + # out_dtype addmm path (0.07 ms per micro-step) + dW = torch.zeros_like(W, dtype=torch.float32) + row_loss = torch.empty(N, device=x.device, dtype=torch.float32) + c = min(chunk, N) + logits = torch.empty((c, V), device=x.device, dtype=torch.float16) + g = torch.empty_like(logits) + S = FusedLMHead.GRAD_SCALE + + for a in range(0, N, chunk): + b = min(a + chunk, N) + n = b - a + lg = logits[:n] + torch.mm(x16[a:b], W16.t(), out=lg) + _softmax_grad_kernel[(n,)](lg, targets[a:b], g[:n], row_loss[a:b], + V=V, S=S, num_warps=16) + torch.mm(g[:n], W16, out_dtype=torch.float32, out=dx16[a:b]) + # dW: fp16 operands, fp32 accumulation AND fp32 output (cuBLAS + # s884gemm + fp32 split-k reduce), accumulated across chunks with + # beta=1. The production head rounds every chunk's dW to fp16 and so + # needs the |x| <= 1 rescale and DW_SHRINK to keep that output from + # overflowing; with an fp32 output there is nothing to overflow. + # Strictly more precise (dW rel-L2 vs fp64 5.24e-4 -> 5.01e-4 on + # the real fixture), 12.6% faster, one fewer 64 MB write per chunk. + torch.addmm(dW, g[:n].t(), x16[a:b], beta=0 if a == 0 else 1, + out_dtype=torch.float32, out=dW) + + ctx.w_dtype = W.dtype + ctx.x_dtype = x.dtype + ctx.W_ref = W + ctx.save_for_backward(dx16, dW) + ctx.n_valid = n_valid + return row_loss.sum() / n_valid + + @staticmethod + def backward(ctx, grad_out): + dx16, dW = ctx.saved_tensors + s = grad_out / (ctx.n_valid * FusedLMHead.GRAD_SCALE) + return ((dx16.float() * s).to(ctx.x_dtype), (dW * s).to(ctx.w_dtype), + None, None) + + +class FusedLMHeadStash(FusedLMHead): + """Same forward. The backward hands (dW, s) to the tied embedding's backward + (integ_ops.TiedEmbedding), which folds dW*s, the embedding segment sums and + the .grad accumulate into one pass, instead of returning dW*s to autograd.""" + + @staticmethod + def backward(ctx, grad_out): + from .integ_ops import stash_head_grad + dx16, dW = ctx.saved_tensors + s = grad_out / (ctx.n_valid * FusedLMHead.GRAD_SCALE) + stash_head_grad(ctx.W_ref, dW, s) + return (dx16.float() * s).to(ctx.x_dtype), None, None, None + + +def _loss_only(x, W, targets, chunk): + N = x.shape[0] + V = W.shape[0] + W16 = W.to(torch.float16) + x16 = x.to(torch.float16) + n_valid = (targets != IGNORE).sum().clamp_min(1) + row_loss = torch.empty(N, device=x.device, dtype=torch.float32) + c = min(chunk, N) + logits = torch.empty((c, V), device=x.device, dtype=torch.float16) + g = torch.empty_like(logits) + for a in range(0, N, chunk): + b = min(a + chunk, N) + n = b - a + torch.mm(x16[a:b], W16.t(), out=logits[:n]) + _softmax_grad_kernel[(n,)](logits[:n], targets[a:b], g[:n], row_loss[a:b], + V=V, S=FusedLMHead.GRAD_SCALE, num_warps=16) + return row_loss.sum() / n_valid + + +def fused_lm_loss(x, W, targets, chunk: int = 1024, stash: bool = False): + if not torch.is_grad_enabled(): + # Validation (quick_val runs under no_grad): no backward will come, so + # skip the dx and dW GEMMs entirely -- 2.25x faster head. Same logits + # GEMM and softmax kernel, so the loss is bitwise the training path's. + # (Checked here, not inside the Function: its forward always runs with + # grad mode off, and the tied weight keeps requires_grad under no_grad.) + return _loss_only(x, W, targets, chunk) + if stash: + return FusedLMHeadStash.apply(x, W, targets, chunk) + return FusedLMHead.apply(x, W, targets, chunk) diff --git a/volta_kernels/integ_ops.py b/volta_kernels/integ_ops.py new file mode 100644 index 0000000..8f05f01 --- /dev/null +++ b/volta_kernels/integ_ops.py @@ -0,0 +1,256 @@ +"""Model-side additions adopted from the kernel swarm, integrated on the merged +(BORIS_VOLTA_KERNELS=1) path. Everything here is used by model.py's +`_forward_merged`; nothing changes the forward values (the loss is bitwise the +live one). + +1. wgrad_acc (block_autograd b, = resid_fp32 V1 = precision_plus dw): + the weight-gradient GEMM of every trunk matmul accumulates straight into + the fp32 p.grad (cuBLAS beta=1, fp32 output). Removes the fp16 rounding of + dW (and of its split-k partials), the fp16->fp32 cast and AccumulateGrad's + add_. dW error vs fp64 ~100x lower. +2. norm_bwd (elem_audit): residual-add + RMSNorm backward and the norm-weight + gradient in ONE Triton pass (16-row blocks, dw partials in registers) + instead of Inductor's row kernel + a split column reduction that re-reads + the same 100-134 MB. Row arithmetic replicates Inductor's live kernel, so + dh/dh16 (and hence every other gradient) are bitwise the live ones. + (3. the CUTLASS SwiGLU epilogue was integrated, measured bitwise but not + faster on the whole micro-step, and dropped -- see ../epi_dropped.py.) +4. TiedEmbedding (ngram_embed A): the head's dW*s, the embedding's segment + sums and the .grad accumulate in one pass (bitwise). +""" +from __future__ import annotations + +import torch +import torch.nn.functional as F +import triton +import triton.language as tl +from torch.library import triton_op, wrap_triton + +F16, F32 = torch.float16, torch.float32 + + +# --------------------------------------------------------------------------- # +# 1. weight gradients accumulated by the GEMM into fp32 .grad +# --------------------------------------------------------------------------- # +@torch.library.custom_op("integ::wgrad_acc", mutates_args=("acc",)) +def wgrad_acc(dy: torch.Tensor, x: torch.Tensor, acc: torch.Tensor, alpha: float) -> None: + dy2 = dy.reshape(-1, dy.shape[-1]) + x2 = x.reshape(-1, x.shape[-1]) + torch.addmm(acc, dy2.t(), x2, beta=1, alpha=alpha, out_dtype=F32, out=acc) + + +@wgrad_acc.register_fake +def _(dy, x, acc, alpha): + return None + + +class LinAcc(torch.autograd.Function): + """y = x16 @ w16^T. The weight gradient alpha * dy^T x16 is ADDED into `acc` + (the fp32 p.grad) by the GEMM itself and not returned to autograd. w16 is + the (pre-scaled) fp16 weight; alpha undoes nothing -- it is the scale the + forward applied to the fp32 weight (DOWN_SHRINK), i.e. d(w*s)/dw.""" + + @staticmethod + def forward(ctx, x16, w16, acc, alpha: float): + ctx.save_for_backward(x16, w16, acc) + ctx.alpha = alpha + return F.linear(x16, w16) + + @staticmethod + def backward(ctx, dy): + x16, w16, acc = ctx.saved_tensors + torch.ops.integ.wgrad_acc(dy, x16, acc, ctx.alpha) + return dy @ w16, None, None, None + + +# --------------------------------------------------------------------------- # +# 2. fused residual-add + RMSNorm backward with the norm-weight gradient +# --------------------------------------------------------------------------- # +@triton.jit +def _norm_bwd_bw_kernel(h, a, rstd, w, gres, gx, dh, dh16, dwp, ROWS: tl.constexpr, + D: tl.constexpr, HAS_A: tl.constexpr, OSCALE: tl.constexpr): + """Row-for-row the arithmetic of Inductor's live norm-backward row kernel + (triton_per_fused__to_copy_add_div_mul_pow_sum_view_*: one 512-wide row as a + 1-D block, 4 warps, num_stages 1, the same op DAG and so the same FMA + contractions), so dh and dh16 are BITWISE the live ones and every other + gradient stays bitwise too. The only change: the norm-weight gradient + sum_rows(gx * (h1*rstd)) -- the same per-element products as Inductor's + separate reduction kernel -- is accumulated here in fp32 over ROWS rows + (partials then summed in fp64), instead of by a second pass that re-reads + gx, h and a. + d = g_res + (gx*w)*rstd + ((sum(gx*w*h1)*-0.5)*rstd^3/D) * (2*h1)""" + pid = tl.program_id(0).to(tl.int64) + cols = tl.arange(0, D) + wv = tl.load(w + cols, eviction_policy="evict_last") + acc = tl.zeros([D], tl.float32) + for i in range(ROWS): + r = pid * ROWS + i + off = r * D + cols + g = tl.load(gx + off).to(tl.float32) + hv = tl.load(h + off) + if HAS_A: + hv = hv + tl.load(a + off).to(tl.float32) + gr = tl.load(gres + off) + rs = tl.load(rstd + r) + gw = g * wv + sm = tl.sum(gw * hv, 0) + t = gr + gw * rs + k = ((sm * -0.5) * (rs * rs * rs)) * (1.0 / D) + d = t + k * (hv * 2.0) + tl.store(dh + off, d) + if OSCALE == 1.0: + tl.store(dh16 + off, d.to(tl.float16)) + else: + tl.store(dh16 + off, (d * OSCALE).to(tl.float16)) + acc += g * (hv * rs) + tl.store(dwp + pid * D + cols, acc) + + +@triton.jit +def _colsum64_kernel(p, out, NR, D: tl.constexpr, BC: tl.constexpr, BR: tl.constexpr): + """Stage 1 of an fp64 column sum: out[k, c] = sum of rows k*BR..k*BR+BR-1 of p + (fp32 in, fp64 accumulate and out). Grid (D // BC, cdiv(NR, BR)).""" + c = tl.program_id(0) * BC + tl.arange(0, BC) + k = tl.program_id(1) + r = k * BR + tl.arange(0, BR) + x = tl.load(p + r[:, None] * D + c[None, :], mask=r[:, None] < NR, other=0.0) + tl.store(out + k * D + c, tl.sum(x.to(tl.float64), 0)) + + +def _colsum64(p): + """fp32 (NR, D) -> fp32 (D,) column sums with an fp64 accumulator (deterministic).""" + NR, D = p.shape + BR = 64 + nk = triton.cdiv(NR, BR) + part = torch.empty((nk, D), device=p.device, dtype=torch.float64) + wrap_triton(_colsum64_kernel)[(D // 64, nk)](p, part, NR, D=D, BC=64, BR=BR, num_warps=4) + return part.sum(0).to(F32) + + +_NB_ROWS = 16 # rows per program: fp32 dw partial per 16 rows, partials summed in fp64 + + +def _norm_bwd_launch(h, a, rstd, w, gres, gx, oscale: float, has_a: bool): + N, D = h.shape + dh = torch.empty_like(h) + dh16 = torch.empty((N, D), device=h.device, dtype=F16) + dwp = torch.empty((N // _NB_ROWS, D), device=h.device, dtype=F32) + wrap_triton(_norm_bwd_bw_kernel)[(N // _NB_ROWS,)]( + h, a, rstd, w, gres, gx, dh, dh16, dwp, ROWS=_NB_ROWS, D=D, + HAS_A=has_a, OSCALE=oscale, num_warps=4, num_stages=1) + return dh, dh16, _colsum64(dwp) + + +@triton_op("integ::norm_bwd_a", mutates_args={}) +def norm_bwd_a(h: torch.Tensor, a: torch.Tensor, rstd: torch.Tensor, w: torch.Tensor, + gres: torch.Tensor, gx: torch.Tensor, oscale: float + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + return _norm_bwd_launch(h, a, rstd, w, gres, gx, oscale, True) + + +@triton_op("integ::norm_bwd_h", mutates_args={}) +def norm_bwd_h(h: torch.Tensor, rstd: torch.Tensor, w: torch.Tensor, + gres: torch.Tensor, gx: torch.Tensor, oscale: float + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + return _norm_bwd_launch(h, h, rstd, w, gres, gx, oscale, False) + + +class AddNorm(torch.autograd.Function): + """(h1, x16) = (h + s*a.float(), fp16(rmsnorm(h1) * w)). + + Forward: the same torch ops as the live region, so Inductor emits the same + fused kernel (bitwise). Backward: one Triton pass (_norm_bwd_bw_kernel) that + reproduces the live row kernel bit for bit -- dh1 (fp32, to the residual) + and da16 = fp16(s*dh1) (to the o_proj / down GEMM backward) -- and also + accumulates the norm-weight gradient sum_rows(gx*xhat), which the live graph + computes in a second kernel re-reading gx, h and a. + + save_out=False saves (h, a) -- the tensors the live graph already keeps for + this norm (h is the previous region's output, a the o_proj output). + save_out=True saves h1 instead -- for the next block's norm1, where h1 is + this region's output and is kept anyway as the next region's input. + """ + + @staticmethod + def forward(ctx, h, a, w, eps: float, s: float, save_out: bool): + h1 = h + a.float() if s == 1.0 else h + a.float() * s + rstd = torch.rsqrt(h1.pow(2).mean(-1, keepdim=True) + eps) + x16 = (h1 * rstd * w.float()).to(F16) + if save_out: + ctx.save_for_backward(h1, rstd, w) + else: + ctx.save_for_backward(h, a, rstd, w) + ctx.s, ctx.save_out = s, save_out + return h1, x16 + + @staticmethod + def backward(ctx, gh1, gx): + D = gx.shape[-1] + if ctx.save_out: + h1, rstd, w = ctx.saved_tensors + shp = h1.shape + dh, da, dw = torch.ops.integ.norm_bwd_h( + h1.reshape(-1, D), rstd.reshape(-1), w, + gh1.reshape(-1, D).contiguous(), gx.reshape(-1, D).contiguous(), ctx.s) + else: + h, a, rstd, w = ctx.saved_tensors + shp = h.shape + dh, da, dw = torch.ops.integ.norm_bwd_a( + h.reshape(-1, D), a.reshape(-1, D), rstd.reshape(-1), w, + gh1.reshape(-1, D).contiguous(), gx.reshape(-1, D).contiguous(), ctx.s) + return dh.view(shp), da.view(shp), dw, None, None, None + + +# --------------------------------------------------------------------------- # +# 4. tied embedding: acc + (head dW * s + embedding segment sums) in one pass +# --------------------------------------------------------------------------- # +@triton.jit +def _tied_accum_dense(acc_ptr, dw_ptr, s_ptr, seg_ptr, out_ptr, n, + HAS_ACC: tl.constexpr, BLOCK: tl.constexpr): + offs = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK) + m = offs < n + s = tl.load(s_ptr) + v = tl.load(dw_ptr + offs, mask=m) * s + tl.load(seg_ptr + offs, mask=m) + if HAS_ACC: + v = tl.load(acc_ptr + offs, mask=m) + v + tl.store(out_ptr + offs, v, mask=m) + + +_STASH: dict = {} + + +def stash_head_grad(W, dW, s): + """Called by the head's backward in place of returning dW*s for W.""" + _STASH[id(W)] = (dW, s) + + +class TiedEmbedding(torch.autograd.Function): + """F.embedding(idx, W) for the tied weight. Its backward adds + (head dW * s) + segment sums into W.grad in one pass -- the exact rounding + sequence of the stock path (dW*s rounded, + seg, + acc; FMA contraction + off), in place of: the dW*s pass, autograd's tied-weight sum and + AccumulateGrad's add_. Requires the head to stash its dW (stash=True).""" + + @staticmethod + def forward(ctx, idx, W): + ctx.save_for_backward(idx) + ctx.W = W + return F.embedding(idx, W) + + @staticmethod + def backward(ctx, g): + (idx,) = ctx.saved_tensors + W = ctx.W + dW, s = _STASH.pop(id(W)) + seg = torch.ops.aten.embedding_dense_backward( + g.reshape(-1, g.shape[-1]), idx.reshape(-1), W.shape[0], -1, False) + has = W.grad is not None + out = W.grad if has else seg # first micro-step: in place into seg + n = W.numel() + BLOCK = 2048 + _tied_accum_dense[(triton.cdiv(n, BLOCK),)]( + out, dW, s.reshape(1).float(), seg, out, n, HAS_ACC=has, BLOCK=BLOCK, + num_warps=4, enable_fp_fusion=False) + if not has: + W.grad = out + return None, None -- cgit v1.2.3