From 5d2e4252e695644c34be605b44a14bd80a98a0a6 Mon Sep 17 00:00:00 2001 From: jdj Date: Wed, 18 Nov 2026 14:47:33 -0800 Subject: Boris-2 training code The code, configuration and data filters used to train Boris-2 (144M, 200B tokens, single V100): model, training loop, data preparation, n-gram table, V100 kernels, final stages and the exact run configs. --- volta_kernels/canon_rope.py | 171 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 volta_kernels/canon_rope.py (limited to 'volta_kernels/canon_rope.py') diff --git a/volta_kernels/canon_rope.py b/volta_kernels/canon_rope.py new file mode 100644 index 0000000..08b6d8c --- /dev/null +++ b/volta_kernels/canon_rope.py @@ -0,0 +1,171 @@ +"""Fused Canon + RoPE for the Volta path: one Triton pass forward, one backward. + +What it replaces, inside the compiled _pre_attn region (Inductor's choices): + forward constant_pad_nd x2 materialise zero-padded fp32 copies of qkv + add_mul the 3-tap combine, reading those copies + to_copy_cat RoPE on q and k, cast to fp16 + backward slice_backward / pad backward / a 1536x3 weight-grad reduction, + RoPE backward, as separate passes over (N, 1536) + +Semantics are the region's, operation for operation: + w = fp16(canon_w) (the region casts it to qkv's dtype) + y[n] = x[n] + x[n]*w0 + (x[n-1]*m1[n])*w1 + (x[n-2]*m2[n])*w2 fp32 + q, k = rope(y) in fp32, then fp16; v = fp16(y) +m1/m2 are the per-token document masks (0 where t-1 / t-2 is in another +document) and the shifts never cross a packed row. The backward is the exact +transpose; the weight gradient is reduced in fp32 and rounded to fp16 before +widening, as the gradient of the region's fp16 cast of canon_w is. +""" +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +C = 1536 # q | k | v channels +HALF = 32 # RoPE pairs (d, d+32) within a 64-wide head + + +@triton.jit +def _canon_half(qkv, w, rows, ch, mm1, mm2, ok1, ok2): + x0 = tl.load(qkv + rows).to(tl.float32) + x1 = tl.load(qkv + rows - 1536, mask=ok1, other=0.0).to(tl.float32) + x2 = tl.load(qkv + rows - 3072, mask=ok2, other=0.0).to(tl.float32) + w0 = tl.load(w + ch * 3).to(tl.float16).to(tl.float32)[None, :] + w1 = tl.load(w + ch * 3 + 1).to(tl.float16).to(tl.float32)[None, :] + w2 = tl.load(w + ch * 3 + 2).to(tl.float16).to(tl.float32)[None, :] + return x0 + x0 * w0 + (x1 * mm1) * w1 + (x2 * mm2) * w2 + + +@triton.jit +def _fwd(qkv, w, m1, m2, cos, sin, out, T, GROUP: tl.constexpr, + TB: tl.constexpr, HALF: tl.constexpr): + """TB tokens x one 64-wide head of one group (0 q, 1 k, 2 v).""" + pid_n = tl.program_id(0) + h = tl.program_id(1) + n = pid_n * TB + tl.arange(0, TB) + d = tl.arange(0, HALF) + t = n % T + ca = GROUP * 512 + h * 64 + d + cb = ca + HALF + ok1 = (t >= 1)[:, None] + ok2 = (t >= 2)[:, None] + mm1 = tl.load(m1 + n)[:, None] + mm2 = tl.load(m2 + n)[:, None] + ya = _canon_half(qkv, w, n[:, None] * 1536 + ca[None, :], ca, mm1, mm2, ok1, ok2) + yb = _canon_half(qkv, w, n[:, None] * 1536 + cb[None, :], cb, mm1, mm2, ok1, ok2) + orow = n[:, None] * 512 + h * 64 + d[None, :] + if GROUP < 2: + cs = tl.load(cos + n[:, None] * HALF + d[None, :]) + sn = tl.load(sin + n[:, None] * HALF + d[None, :]) + oa = ya * cs - yb * sn + ob = ya * sn + yb * cs + else: + oa = ya + ob = yb + tl.store(out + orow, oa.to(tl.float16)) + tl.store(out + orow + HALF, ob.to(tl.float16)) + + +@triton.jit +def _dy(src, cos, sin, n, t, S: tl.constexpr, d, ocol, T, ROPE: tl.constexpr, + HALF: tl.constexpr): + """Gradient w.r.t. the Canon output at token n+S, un-rotated; 0 past the row.""" + nn = n + S + ok = ((t + S) < T)[:, None] + ga = tl.load(src + nn[:, None] * 512 + ocol, mask=ok, other=0.0).to(tl.float32) + gb = tl.load(src + nn[:, None] * 512 + ocol + HALF, mask=ok, other=0.0).to(tl.float32) + if ROPE: + cs = tl.load(cos + nn[:, None] * HALF + d[None, :], mask=ok, other=0.0) + sn = tl.load(sin + nn[:, None] * HALF + d[None, :], mask=ok, other=0.0) + return ga * cs + gb * sn, gb * cs - ga * sn + return ga, gb + + +@triton.jit +def _bwd_half(qkv, w, dqkv, dw_part, pid_n, n, ch, dy0, dy1, dy2, + mm1, mm2, mm1n, mm2n, p1, p2): + w0 = tl.load(w + ch * 3).to(tl.float16).to(tl.float32)[None, :] + w1 = tl.load(w + ch * 3 + 1).to(tl.float16).to(tl.float32)[None, :] + w2 = tl.load(w + ch * 3 + 2).to(tl.float16).to(tl.float32)[None, :] + rows = n[:, None] * 1536 + ch[None, :] + # dx[n] = dy[n](1 + w0) + dy[n+1] m1[n+1] w1 + dy[n+2] m2[n+2] w2 + dx = dy0 + dy0 * w0 + (dy1 * mm1n) * w1 + (dy2 * mm2n) * w2 + tl.store(dqkv + rows, dx.to(tl.float16)) + x0 = tl.load(qkv + rows).to(tl.float32) + x1 = tl.load(qkv + rows - 1536, mask=p1, other=0.0).to(tl.float32) + x2 = tl.load(qkv + rows - 3072, mask=p2, other=0.0).to(tl.float32) + base = dw_part + pid_n * (1536 * 3) + ch * 3 + tl.store(base, tl.sum(dy0 * x0, 0)) + tl.store(base + 1, tl.sum(dy0 * (x1 * mm1), 0)) + tl.store(base + 2, tl.sum(dy0 * (x2 * mm2), 0)) + + +@triton.jit +def _bwd(qkv, w, m1, m2, cos, sin, src, dqkv, dw_part, T, GROUP: tl.constexpr, + TB: tl.constexpr, HALF: tl.constexpr): + pid_n = tl.program_id(0) + h = tl.program_id(1) + n = pid_n * TB + tl.arange(0, TB) + d = tl.arange(0, HALF) + t = n % T + ca = GROUP * 512 + h * 64 + d + cb = ca + HALF + ocol = h * 64 + d[None, :] + ROPE: tl.constexpr = GROUP < 2 + dya0, dyb0 = _dy(src, cos, sin, n, t, 0, d, ocol, T, ROPE, HALF) + dya1, dyb1 = _dy(src, cos, sin, n, t, 1, d, ocol, T, ROPE, HALF) + dya2, dyb2 = _dy(src, cos, sin, n, t, 2, d, ocol, T, ROPE, HALF) + mm1n = tl.load(m1 + n + 1, mask=(t + 1) < T, other=0.0)[:, None] + mm2n = tl.load(m2 + n + 2, mask=(t + 2) < T, other=0.0)[:, None] + mm1 = tl.load(m1 + n)[:, None] + mm2 = tl.load(m2 + n)[:, None] + p1 = (t >= 1)[:, None] + p2 = (t >= 2)[:, None] + _bwd_half(qkv, w, dqkv, dw_part, pid_n, n, ca, dya0, dya1, dya2, mm1, mm2, mm1n, mm2n, p1, p2) + _bwd_half(qkv, w, dqkv, dw_part, pid_n, n, cb, dyb0, dyb1, dyb2, mm1, mm2, mm1n, mm2n, p1, p2) + + +TB = 32 + + +class CanonRope(torch.autograd.Function): + @staticmethod + def forward(ctx, qkv, canon_w, m1, m2, cos, sin): + B, T, Cc = qkv.shape + assert Cc == C and qkv.dtype == torch.float16 and qkv.is_contiguous() + N = B * T + assert N % TB == 0 + q = torch.empty((B, T, 8, 64), device=qkv.device, dtype=torch.float16) + k = torch.empty_like(q) + v = torch.empty_like(q) + m1f = m1.reshape(N).contiguous() + m2f = m2.reshape(N).contiguous() + cosf = cos.reshape(N, HALF).contiguous() + sinf = sin.reshape(N, HALF).contiguous() + for g_, out in enumerate((q, k, v)): + _fwd[(N // TB, 8)](qkv, canon_w, m1f, m2f, cosf, sinf, out, T, GROUP=g_, + TB=TB, HALF=HALF, num_warps=4) + ctx.save_for_backward(qkv, canon_w, m1f, m2f, cosf, sinf) + ctx.T = T + return q, k, v + + @staticmethod + def backward(ctx, gq, gk, gv): + qkv, canon_w, m1f, m2f, cosf, sinf = ctx.saved_tensors + B, T, _ = qkv.shape + N = B * T + gq, gk, gv = (g.to(torch.float16).contiguous() for g in (gq, gk, gv)) + dqkv = torch.empty_like(qkv) + part = torch.empty((N // TB, C, 3), device=qkv.device, dtype=torch.float32) + for g_, src in enumerate((gq, gk, gv)): + _bwd[(N // TB, 8)](qkv, canon_w, m1f, m2f, cosf, sinf, src, dqkv, part, T, + GROUP=g_, TB=TB, HALF=HALF, num_warps=4) + # fp32 reduction, then the fp16 rounding the region's cast of canon_w implies + dw = part.sum(0).to(torch.float16).to(canon_w.dtype) + return dqkv, dw, None, None, None, None + + +def canon_rope(qkv, canon_w, m1, m2, cos, sin): + """qkv (B,T,1536) fp16 -> q, k, v (B,T,8,64) fp16, Canon + RoPE applied.""" + return CanonRope.apply(qkv, canon_w, m1, m2, cos, sin) -- cgit v1.2.3