1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
|
"""Boris-2: a d512 x 31L decoder-only transformer with fused Canon layers.
Shape of a block:
h = h + o_proj( attn( canon( qkv_proj( norm(h) ) ) ) )
h = h + down( silu(gate) * up )( norm(h) )
Four things here are not the textbook spelling, and each is a measured win on
sm_70 rather than a preference:
1. Q, K and V are one GEMM. The weights stay three separate Parameters and are
concatenated at call time, so Muon still orthogonalises each independently --
fusing the Parameters themselves would change what the optimizer operates on
and would not be lossless. The concatenated weight is ~1.2MB against a
16,384-row activation.
2. Canon is one kernel over the concatenated Q|K|V block. Depthwise convolution
is per-channel, so a single 768-channel convolution equals three separate
ones on the Q/K/V channel ranges: 31 kernels per forward instead of 93. It is
spelled as three shifted multiply-adds on the native (B,T,C) layout rather
than F.conv1d(groups=C), which needs a (B,T,C)->(B,C,T) transpose whose
following reshape materialises a copy. Measured on this V100, fwd+bwd over
the real shape: 4.30 ms for conv1d, 1.10 ms for the compiled shift form --
substantially faster than the transposing grouped-convolution spelling.
3. Canon is document-aware. Kernel 3 stacked 31 deep reaches 62 tokens back, and
rows hold several packed documents, so without masking every block pulls the
previous document's tail across the boundary. The mask costs 0.10 ms/layer.
This is the bug the 75M run found the hard way; verify.py probes for it with
the output projections de-zeroed, because at init a zero-init projection
makes the test pass no matter how badly the model leaks.
4. Attention is varlen over a jagged nested tensor: one SDPA launch for the
whole batch, cost sum(len_i^2) rather than T^2. Measured against the two
alternatives at ctx 2048 with realistic document lengths, fwd+bwd:
nested 11.09 ms, dense block-diagonal mask 26.97 ms, per-document loop
45.84 ms. Nested is bit-identical to the loop (max diff 0.0).
The residual stream is fp32 on the V100 because measured massive activations can
exceed fp16's finite range. Matmul inputs are cast by the compiled regions under
autocast, preserving tensor-core execution without risking a forward overflow.
"""
from __future__ import annotations
import math
import os
import torch
import torch.nn as nn
import torch.nn.functional as F
from config import Config, ModelConfig
from ngram import NGramTable
from volta_kernels.integ_ops import AddNorm, LinAcc, TiedEmbedding
IGNORE = -100
# Hand-written sm_70 kernels (volta_kernels/): varlen FlashAttention with mma.sync
# and a one-pass LM-head softmax. Opt-in, because they are adopted by a running
# 200B-token job at a resume: BORIS_VOLTA_KERNELS=1. Same operand dtypes and fp32
# accumulation as the paths they replace; verify.py checks both against fp64.
# They only engage on an sm_70 device -- elsewhere (eval on the 3060) the stock
# paths run regardless.
VOLTA_KERNELS = os.environ.get("BORIS_VOLTA_KERNELS", "0") == "1"
def _volta(t: torch.Tensor) -> bool:
return VOLTA_KERNELS and t.is_cuda and torch.cuda.get_device_capability(t.device) == (7, 0)
# --------------------------------------------------------------------------- #
# Compiled fixed-shape regions
# --------------------------------------------------------------------------- #
# Whole-model compile is not used: the 75M run measured it 28% slower in
# realistic conditions because document counts vary and Inductor recompiles
# mid-run. These regions never see a ragged shape -- only (B, T, C) with B and T
# fixed for the whole context stage -- so each compiles once per stage.
def _rmsnorm(x, w, eps: float):
f = x.float()
f = f * torch.rsqrt(f.pow(2).mean(-1, keepdim=True) + eps)
return (f * w.float()).to(x.dtype)
def _pre_attn(h, w_norm, eps: float, w_qkv, canon_w, m1, m2, cos, sin,
n_head: int, n_kv_head: int, head_dim: int, use_canon: bool,
out16: bool = False):
"""Norm, one GEMM, one depthwise convolution, one rotation -- one region.
The RMSNorm is inside the region on purpose. Written as its own eager module
it upcasts to fp32, and that materialises a (B, T, 512) fp32 tensor for
every one of the 53 norms in the model -- 25% of forward+backward was
elementwise traffic before these regions were widened, and this was most of
it. Inductor keeps the reduction in fp32 without ever writing the fp32
tensor out.
canon_w is (C, 3) with column 0 the weight on token t, column 1 on t-1 and
column 2 on t-2. m1 masks the t-1 tap and m2 the t-2 tap at document
boundaries; both are (B, T, 1) and are computed once per forward, not per
layer.
"""
f = h.float()
x = (f * torch.rsqrt(f.pow(2).mean(-1, keepdim=True) + eps)
* w_norm.float()).to(h.dtype)
qkv = F.linear(x, w_qkv) # (B, T, C)
if use_canon:
w = canon_w.to(qkv.dtype)
s1 = F.pad(qkv, (0, 0, 1, 0))[:, :-1] * m1
s2 = F.pad(qkv, (0, 0, 2, 0))[:, :-2] * m2
qkv = qkv + qkv * w[:, 0] + s1 * w[:, 1] + s2 * w[:, 2]
B, T, _ = qkv.shape
qd, kd = n_head * head_dim, n_kv_head * head_dim
q = qkv[..., :qd].view(B, T, n_head, head_dim)
k = qkv[..., qd:qd + kd].view(B, T, n_kv_head, head_dim)
v = qkv[..., qd + kd:].view(B, T, n_kv_head, head_dim)
def rope(t):
f = t.float()
a, b = f.chunk(2, dim=-1)
return torch.cat([a * cos - b * sin, a * sin + b * cos], -1).to(t.dtype)
if out16:
# The Volta attention takes fp16. Casting here rounds the same fp32
# values autocast would round, but inside the region's epilogue, so the
# fp32 q/k/v are never written out and read back by separate casts.
h16 = torch.float16
return rope(q).to(h16), rope(k).to(h16), v.to(h16)
return rope(q), rope(k), v
def _pre_attn_qkv(h, w_norm, eps: float, w_qkv):
"""Norm and the fused QKV GEMM only -- the Volta path's region.
Canon and RoPE then run as one fused Triton pass (volta_kernels/canon_rope.py)
instead of Inductor's pad-copy / combine / rotate kernels.
"""
f = h.float()
x = (f * torch.rsqrt(f.pow(2).mean(-1, keepdim=True) + eps)
* w_norm.float()).to(h.dtype)
return F.linear(x, w_qkv)
def _post_then_pre(h, o, w_o, w_norm2, eps2, w_gu, w_down, w_norm1n, eps1n, w_qkvn,
extra):
"""Block i's FFN half and block i+1's norm + QKV GEMM as ONE region.
As separate regions the residual h crossed a graph boundary between two
consumers, so its two backward gradient streams were summed by an eager
fp32 add (~192 MB of traffic per layer) and h was written and read back.
Here Inductor fuses that sum into the region's backward. `extra` is the
n-gram residual when block i+1 is where it is inserted (it depends only on
the token ids), else None -- each value gets its own compiled graph.
"""
h = _post_attn(h, o, w_o, w_norm2, eps2, w_gu, w_down)
if extra is not None:
h = h + extra
return h, _pre_attn_qkv(h, w_norm1n, eps1n, w_qkvn)
def _post_then_final_norm(h, o, w_o, w_norm2, eps2, w_gu, w_down, w_normf, epsf):
"""The last block's FFN half and the final RMSNorm as one region."""
h = _post_attn(h, o, w_o, w_norm2, eps2, w_gu, w_down)
return _rmsnorm(h, w_normf, epsf)
# Power-of-two headroom for the FFN down projection's fp16 output. 1/8 puts the
# measured step-4,520 peak (~65.5k) at ~8.2k, leaving 8x before it overflows
# again, and costs only inputs below 4.9e-4 -- 1.6e-8 of the peak, which
# contribute nothing to a sum dominated by terms 6e7 times larger.
DOWN_SHRINK = 0.125
def _post_attn(h, o, w_o, w_norm, eps: float, w_gu, w_down):
"""Output projection, residual, norm, SwiGLU, residual -- one region.
Everything in a block except the attention itself. The two residual adds and
the second norm were separate eager kernels over the widest tensors in the
model; inside the region Inductor folds them into the GEMM epilogues, and
the fp32 norm intermediate is never written out.
"""
h = h + F.linear(o, w_o)
f = h.float()
x = (f * torch.rsqrt(f.pow(2).mean(-1, keepdim=True) + eps)
* w_norm.float()).to(h.dtype)
gu = F.linear(x, w_gu)
g, u = gu.chunk(2, dim=-1)
# The down projection is the one GEMM here whose fp16 OUTPUT can overflow.
# Its inputs are finite and so is the true result -- it is the fp16 range
# that runs out. Channel 228 of the residual is a "massive activation",
# 173x the RMS (78,115 against an RMS of 451 at block 16), and the FFN has
# to write into it, so at step 4,520 this output crossed 65,504 and became
# inf; norm_f then turned that into NaN via rsqrt(inf)=0, inf*0.
# Pre-scaling w_down is exact (a power of two), keeps the GEMM on tensor
# cores, costs one pass over a 1.2M-element weight rather than the 77M-
# element activation, and is divided back out in fp32 where there is room.
# Every partial sum is scaled too, so this is safe even with cuBLAS's
# reduced-precision fp16 reductions enabled.
return h + F.linear(F.silu(g) * u,
w_down * DOWN_SHRINK).float() * (1.0 / DOWN_SHRINK)
# --------------------------------------------------------------------------- #
# Merged-path regions with the swarm additions (volta_kernels/integ_ops.py)
# --------------------------------------------------------------------------- #
# ft = (wacc, norm) -- which additions a compiled region uses:
# wacc: weight-gradient GEMMs accumulate into the fp32 .grad (accs given)
# norm: residual-add + RMSNorm backward and norm-weight grad in one Triton pass
# Weights arrive either as the fp32 Parameters (cast here, as autocast would) or
# as the cached, pre-scaled fp16 copies (Boris2._w16); both give the same fp16
# operand, so the forward is bitwise the live one either way.
FT_ALL = (True, True)
def _w16(w, scale: float = 1.0):
if w.dtype == torch.float16:
return w # cached copy, already scaled
return (w * scale).to(torch.float16) if scale != 1.0 else w.to(torch.float16)
def _lin_x(x, w, acc, scale: float = 1.0):
if acc is None:
if w.dtype == torch.float16:
return F.linear(x, w)
return F.linear(x, w * scale if scale != 1.0 else w)
return LinAcc.apply(x.to(torch.float16), _w16(w, scale), acc, scale)
def _norm_x(h, w, eps: float):
f = h.float()
return (f * torch.rsqrt(f.pow(2).mean(-1, keepdim=True) + eps)
* w.float()).to(h.dtype)
def _ffn_x(h, o, w_o, w_norm, eps: float, w_gu, w_down, a_o, a_gu, a_down, ft):
"""Block FFN half up to (but excluding) the down projection's residual add:
returns h1 = h + o_proj(o) and the fp16 down output (pre-scaled by
DOWN_SHRINK), so the next norm can fold the add into its backward."""
wacc, norm = ft
a = _lin_x(o, w_o, a_o)
if norm:
h1, x = AddNorm.apply(h, a, w_norm, eps, 1.0, False)
else:
h1 = h + a
x = _norm_x(h1, w_norm, eps)
gu = _lin_x(x, w_gu, a_gu)
g, u = gu.chunk(2, dim=-1)
return h1, _lin_x(F.silu(g) * u, w_down, a_down, DOWN_SHRINK)
def _post_then_pre_x(h, o, w_o, w_norm2, eps2, w_gu, w_down, w_norm1n, eps1n, w_qkvn,
extra, accs, ft):
a_o, a_gu, a_down, a_qkvn = accs if accs is not None else (None,) * 4
h1, dn = _ffn_x(h, o, w_o, w_norm2, eps2, w_gu, w_down, a_o, a_gu, a_down, ft)
if ft[1] and extra is None:
h2, x = AddNorm.apply(h1, dn, w_norm1n, eps1n, 1.0 / DOWN_SHRINK, True)
else:
h2 = h1 + dn.float() * (1.0 / DOWN_SHRINK)
if extra is not None:
h2 = h2 + extra
x = _norm_x(h2, w_norm1n, eps1n)
return h2, _lin_x(x, w_qkvn, a_qkvn)
def _post_then_final_norm_x(h, o, w_o, w_norm2, eps2, w_gu, w_down, w_normf, epsf,
accs, ft):
a_o, a_gu, a_down = accs if accs is not None else (None,) * 3
h1, dn = _ffn_x(h, o, w_o, w_norm2, eps2, w_gu, w_down, a_o, a_gu, a_down, ft)
return _rmsnorm(h1 + dn.float() * (1.0 / DOWN_SHRINK), w_normf, epsf)
def _pre_attn_qkv_x(h, w_norm, eps: float, w_qkv, acc):
return _lin_x(_norm_x(h, w_norm, eps), w_qkv, acc)
# --------------------------------------------------------------------------- #
# Fused linear cross-entropy
# --------------------------------------------------------------------------- #
def _head_chunk(logits, safe, vc, scale: float):
"""logsumexp, loss, and the scaled softmax, as one fused pass.
Eager, this chain is five separate passes over a (chunk, 32768) fp32 tensor
-- at micro_batch 16 that is 4.3 GB read and written five times per step.
Inductor folds it to two.
"""
lse = torch.logsumexp(logits, dim=-1)
picked = logits.gather(1, safe[:, None]).squeeze(1)
loss = torch.where(vc, lse - picked, torch.zeros_like(lse)).sum()
p = (logits - lse[:, None]).exp() * (vc[:, None].to(logits.dtype) * scale)
return loss, p
_head_chunk_c = None
class _FusedLMHead(torch.autograd.Function):
"""Cross-entropy over the LM head without ever holding (N, V) logits.
The naive head materialises the logits three times -- fp16 from the
projection, fp32 inside cross_entropy, and again for the backward -- which
at N=16,384 and V=32,768 is 2.1 GB a copy and sets the micro-batch.
The head's gradient depends on nothing downstream except a scalar, so this
computes dx and dW during the forward walk over vocab-sized chunks and
keeps only those. No recomputation in the backward: total GEMM work is the
same as the unfused head, and peak memory is one chunk of logits.
Gradients are held at sum scale (not divided by the token count) and lifted
by a further constant: a softmax over 32,768 classes has typical entries
near 3e-5, which is SUBNORMAL in fp16 (the smallest normal is 6.1e-5), so
the natural scaling would compute the whole dx GEMM on values with a few
bits of mantissa. GRAD_SCALE lifts them clear of that. The factor, the 1/N
and the loss scale are all divided out together in the backward.
The constant is bounded from ABOVE by the dW GEMM, which was missed until
real data hit it. g = S*(p - onehot), so the true token carries -S, and
dW sums that over every token in the chunk sharing a vocab row. Measured on
a real batch: the most frequent target appears 89 times in a 2048-token
chunk and |x| reaches 5.4, so at S=4096 the largest |dW| entry is 504,389
-- 7.7x past fp16's 65,504. It saturated to Inf, Inf-Inf gave NaN in the
tied embedding gradient, and the run died on step 2 with loss nan.
Measured max |dW| against S, same batch:
S=4096 504,389 OVERFLOW S=256 31,524 (2.1x margin)
S=1024 126,097 OVERFLOW S=64 7,881 (8.3x margin)
64 is chosen over a value that splits the difference because the two risks
are not symmetric. Both grow over a long run -- |x| rises as the residual
stream grows, and a confident softmax makes small probabilities smaller --
but an overflow is a NaN that ends the run, while an underflow silently
drops terms that were already negligible. At S=64 the smallest nonzero
|p*S| is 1.48e-4, still 2.4x above fp16's min normal.
"""
GRAD_SCALE = 64.0
# PERF.md's last unapplied win: accumulate the head's dW in fp16 rather than
# fp32. Their +10.1% came from an fp32 accumulate forcing a non-tensor-core
# sgemm; ours is already a tensor-core fp16 GEMM with only the accumulator
# buffer in fp32, so the mechanism differs. Measured either way.
FP16_DW = False
# Shrinks g for the dW GEMM only (a power of two, so it is exact and is
# divided back out losslessly in fp32). dx keeps the full GRAD_SCALE.
DW_SHRINK = 0.25
@staticmethod
def forward(ctx, x, W, targets, chunk: int):
N, d = x.shape
V = W.shape[0]
# Cast once, explicitly, rather than relying on autocast to do it: this
# runs inside a custom Function and is called from verify.py outside any
# autocast region.
Wc = W.to(x.dtype)
valid = targets != IGNORE
# Keep this scalar on-device. int(valid.sum()) was one forced CPU/GPU
# synchronization per micro-batch: sixteen pipeline stalls per step.
n_valid = valid.sum().clamp_min(1)
loss_sum = x.new_zeros((), dtype=torch.float32)
dx = torch.empty_like(x)
dW = torch.zeros_like(W, dtype=(torch.float16 if _FusedLMHead.FP16_DW
else torch.float32))
for a in range(0, N, chunk):
b = min(a + chunk, N)
xc, tc = x[a:b], targets[a:b]
logits = F.linear(xc, Wc).float() # (c, V)
safe = tc.clamp_min(0)
vc = (tc != IGNORE)
S = _FusedLMHead.GRAD_SCALE
loss, p = _head_chunk_c(logits, safe, vc, S)
loss_sum += loss
del logits
# The one-hot subtraction touches c elements, not c*V, so it stays
# outside the fused pass.
p.scatter_add_(1, safe[:, None], (-S * vc).to(p.dtype)[:, None])
g = p.to(x.dtype)
del p
dx[a:b] = F.linear(g, Wc.t())
# dW is the one GEMM here whose fp16 OUTPUT can overflow, and no
# loss scale can save it: g and xc are built inside this Function,
# so GradScaler halving its scale changes nothing while found_inf
# skips AdamW forever. That froze the embeddings, head, norms, Canon
# and n-gram table at step ~230 while Muon kept training the trunk
# against them; loss bottomed at 6.48 and climbed to nan by 350.
# Bound it by construction instead of by choosing a constant:
# rescale xc by a power of two so |xn| <= 1 (exact, no rounding),
# and shrink g for this GEMM only. |sum| <= S*DW_SHRINK*chunk = 32k
# then holds for ANY data, including a chunk of one repeated token,
# and both factors are divided back out in fp32.
r2 = torch.exp2(torch.ceil(torch.log2(
xc.abs().amax().float().clamp_min(1e-30))))
xn = xc / r2.to(xc.dtype)
gd = g * _FusedLMHead.DW_SHRINK
back = r2 / _FusedLMHead.DW_SHRINK
dw = gd.t() @ xn
# Restore magnitude AFTER widening. Multiplying in fp16 undoes
# the overflow protection before the fp32 accumulator sees it.
dW += (dw.float() * back).to(dW.dtype)
ctx.w_dtype = W.dtype
ctx.save_for_backward(dx, dW)
ctx.n_valid = n_valid
return loss_sum / n_valid
@staticmethod
def backward(ctx, grad_out):
dx, dW = ctx.saved_tensors
s = grad_out / (ctx.n_valid * _FusedLMHead.GRAD_SCALE)
# In fp32, then cast. s is ~2.8e-7 without a loss scale, which is
# subnormal in fp16 and would carry about two bits of mantissa -- it
# cost 7e-02 relative on dx before this was float. Under GradScaler s
# lands near 1e-2 and the difference is invisible, which is exactly why
# it would never have shown up in the loss curve.
return (dx.float() * s).to(dx.dtype), (dW.float() * s).to(ctx.w_dtype), None, None
def fused_lm_loss(x, W, targets, chunk: int = 2048, stash: bool = False):
if _volta(x):
from volta_kernels.head import fused_lm_loss as volta_lm_loss
return volta_lm_loss(x, W, targets, chunk, stash)
assert not stash
global _head_chunk_c
if _head_chunk_c is None:
_head_chunk_c = torch.compile(_head_chunk, dynamic=False)
return _FusedLMHead.apply(x, W, targets, chunk)
# --------------------------------------------------------------------------- #
# Modules
# --------------------------------------------------------------------------- #
class RMSNorm(nn.Module):
def __init__(self, dim: int, eps: float):
super().__init__()
self.weight = nn.Parameter(torch.ones(dim))
self.eps = eps
self.weight._boris_role = "norm"
def forward(self, x):
return _rmsnorm(x, self.weight, self.eps)
class Attention(nn.Module):
"""Causal GQA (8 query heads over 2 KV heads) with fused Canon, varlen."""
def __init__(self, cfg: ModelConfig):
super().__init__()
self.cfg = cfg
d, hd = cfg.d_model, cfg.head_dim
self.n_head, self.n_kv_head, self.head_dim = cfg.n_head, cfg.n_kv_head, hd
self.rep = cfg.n_head // cfg.n_kv_head
# Q, K and V are ONE Parameter, and Muon orthogonalises its three row
# blocks independently -- which is exactly what it would do to three
# separate Parameters, because the quintic iteration acts on each matrix
# alone. Storing them fused removes a torch.cat per layer per
# micro-step (2.4% of forward+backward) and its backward slice, without
# changing what the optimizer operates on. verify.py asserts the two
# spellings produce identical updates.
self.qkv_proj = nn.Parameter(torch.empty(cfg.q_dim + 2 * cfg.kv_dim, d))
self.qkv_proj._boris_role = "matmul"
self.qkv_proj._boris_blocks = ((0, cfg.q_dim),
(cfg.q_dim, cfg.q_dim + cfg.kv_dim),
(cfg.q_dim + cfg.kv_dim, cfg.q_dim + 2 * cfg.kv_dim))
self.o_proj = nn.Parameter(torch.empty(d, cfg.q_dim))
self.o_proj._boris_role = "matmul"
if cfg.use_canon:
# (C, 3), zero-init: the layer is exactly the identity at step 0, so
# a Canon run and a no-Canon run start from the same function. The
# gradient w.r.t. the kernel is dL/dy * x and is non-zero, so it
# trains from the first step regardless.
self.canon = nn.Parameter(torch.zeros(cfg.canon_channels, cfg.canon_kernel))
self.canon._boris_role = "canon"
else:
self.register_parameter("canon", None)
def forward(self, h, norm_w, eps, ctx):
volta = self.head_dim == 64 and _volta(h)
if volta and self.canon is not None and self.rep == 1:
from volta_kernels.canon_rope_cuda import canon_rope
qkv = ctx.pre_attn_qkv(h, norm_w, eps, self.qkv_proj)
q, k, v = canon_rope(qkv, self.canon, ctx.m1, ctx.m2, ctx.cos, ctx.sin)
else:
q, k, v = ctx.pre_attn(h, norm_w, eps, self.qkv_proj, self.canon,
ctx.m1, ctx.m2, ctx.cos, ctx.sin,
self.n_head, self.n_kv_head, self.head_dim,
self.canon is not None, volta)
B, T = q.shape[0], q.shape[1]
q = q.reshape(B * T, self.n_head, self.head_dim)
# rep == 1 under full MHA: repeat_interleave(1) is a no-op that still
# allocates and copies a (B*T, n_head, head_dim) tensor -- 33 MB per
# call per layer at micro_batch 16, twice (k and v), on every block in
# forward AND backward. Skip it.
k = k.reshape(B * T, self.n_kv_head, self.head_dim)
v = v.reshape(B * T, self.n_kv_head, self.head_dim)
if self.rep > 1:
k = k.repeat_interleave(self.rep, 1)
v = v.repeat_interleave(self.rep, 1)
off = ctx.offsets
if volta:
from volta_kernels import varlen_causal_attention
# q/k/v arrive fp16 from the region (out16): the cast autocast
# would apply inside SDPA, since the Canon masks keep them fp32.
o = varlen_causal_attention(q, k, v, off)
return o.reshape(B, T, self.n_head * self.head_dim)
nj = lambda t: torch.nested.nested_tensor_from_jagged(
t, off, min_seqlen=ctx.min_seqlen, max_seqlen=ctx.max_seqlen)
nq = nj(q).transpose(1, 2)
nk = nj(k).transpose(1, 2)
nv = nj(v).transpose(1, 2)
o = F.scaled_dot_product_attention(nq, nk, nv, is_causal=True)
return o.transpose(1, 2).values().reshape(B, T, self.n_head * self.head_dim)
class FFN(nn.Module):
"""SwiGLU, inner width 2368. gate and up are one GEMM at call time."""
def __init__(self, cfg: ModelConfig):
super().__init__()
h, d = cfg.ffn_hidden, cfg.d_model
# gate and up fused for the same reason q/k/v are; Muon still sees two
# independent (2368, 512) matrices.
self.gate_up_proj = nn.Parameter(torch.empty(2 * h, d))
self.gate_up_proj._boris_role = "matmul"
self.gate_up_proj._boris_blocks = ((0, h), (h, 2 * h))
self.down_proj = nn.Parameter(torch.empty(d, h))
self.down_proj._boris_role = "matmul"
class Block(nn.Module):
def __init__(self, cfg: ModelConfig):
super().__init__()
self.norm1 = RMSNorm(cfg.d_model, cfg.norm_eps)
self.attn = Attention(cfg)
self.norm2 = RMSNorm(cfg.d_model, cfg.norm_eps)
self.ffn = FFN(cfg)
def forward(self, h, ctx):
o = self.attn(h, self.norm1.weight, self.norm1.eps, ctx)
return ctx.post_attn(h, o, self.attn.o_proj, self.norm2.weight,
self.norm2.eps, self.ffn.gate_up_proj,
self.ffn.down_proj)
class RunContext:
"""Everything shared across all blocks for one forward.
RoPE tables are gathered once here rather than once per attention layer,
and the Canon boundary masks once rather than once per Canon. Holding the
compiled callables here keeps a single compiled graph per stage shared by
every block.
"""
def __init__(self, cos, sin, m1, m2, offsets, pre_attn, post_attn,
min_seqlen=None, max_seqlen=None, pre_attn_qkv=None):
self.pre_attn_qkv = pre_attn_qkv
self.cos, self.sin = cos, sin
self.m1, self.m2 = m1, m2
self.offsets = offsets
self.pre_attn, self.post_attn = pre_attn, post_attn
# Without these, every SDPA call derives them from the offsets with a
# .max().item() -- a host sync, 131 of them per forward, each stalling
# CPU run-ahead. The packer knows both while it is laying documents
# down, so they cost nothing to carry.
self.min_seqlen, self.max_seqlen = min_seqlen, max_seqlen
class Boris2(nn.Module):
def __init__(self, cfg: Config):
super().__init__()
self.cfg = cfg
m = cfg.model
self.embed = nn.Embedding(m.vocab_size, m.d_model)
self.embed.weight._boris_role = "embedding"
self.blocks = nn.ModuleList([Block(m) for _ in range(m.n_layer)])
self.norm_f = RMSNorm(m.d_model, m.norm_eps)
self.ngram = NGramTable(cfg.ngram, m.d_model) if m.use_ngram else None
# Runtime toggle, for scoring one trained checkpoint both ways. Note
# what it measures: after training, the trunk has co-adapted to the
# table, so switching it off reports how much the model DEPENDS on it,
# not how much it GAINED from it. The gain needs the use_ngram=False
# training arm.
self.ngram_enabled = True
if not m.tie_embeddings:
self.lm_head = nn.Parameter(torch.empty(m.vocab_size, m.d_model))
self.lm_head._boris_role = "embedding"
self._rope_cache: dict[tuple, tuple] = {}
self._pre_attn = _pre_attn
self._pre_attn_qkv = _pre_attn_qkv
self._post_attn = _post_attn
self._post_then_pre = _post_then_pre
self._post_then_final_norm = _post_then_final_norm
self._pre_attn_qkv_x = _pre_attn_qkv_x
self._post_then_pre_x = _post_then_pre_x
self._post_then_final_norm_x = _post_then_final_norm_x
# Swarm additions on the merged path (see volta_kernels/integ_ops.py):
# (wgrad_acc, fused norm backward) and the tied embedding's fused grad
# accumulate. w16_cache (fp16 weight copies cast once per optimizer
# step) is OFF: ~0.15% at mb 4, within noise, and it is only correct
# while every step starts with zero_grad(set_to_none=True).
self.ft = FT_ALL
self.tied_acc = True
self.w16_cache = False
self._w16c: dict = {}
self._compiled = False
self.apply_init()
# -- init ------------------------------------------------------------- #
def apply_init(self):
m = self.cfg.model
nn.init.normal_(self.embed.weight, std=0.02)
if not m.tie_embeddings:
nn.init.normal_(self.lm_head, std=0.02)
# Residual-path projections are scaled down so the residual variance
# does not grow with depth -- at 31 layers this matters more than it
# does at 10.
res_scale = 1.0 / math.sqrt(2 * m.n_layer)
for blk in self.blocks:
for p in (blk.attn.qkv_proj, blk.ffn.gate_up_proj):
nn.init.normal_(p, std=0.02)
for p in (blk.attn.o_proj, blk.ffn.down_proj):
nn.init.normal_(p, std=0.02 * res_scale)
if blk.attn.canon is not None:
nn.init.zeros_(blk.attn.canon)
def maybe_compile(self):
if self._compiled or not self.cfg.train.compile_regions:
return
self._pre_attn = torch.compile(_pre_attn, dynamic=False)
self._pre_attn_qkv = torch.compile(_pre_attn_qkv, dynamic=False)
self._post_then_pre = torch.compile(_post_then_pre, dynamic=False)
self._post_then_final_norm = torch.compile(_post_then_final_norm, dynamic=False)
self._post_attn = torch.compile(_post_attn, dynamic=False)
self._pre_attn_qkv_x = torch.compile(_pre_attn_qkv_x, dynamic=False)
self._post_then_pre_x = torch.compile(_post_then_pre_x, dynamic=False)
self._post_then_final_norm_x = torch.compile(_post_then_final_norm_x, dynamic=False)
self._compiled = True
def head_weight(self):
return self.embed.weight if self.cfg.model.tie_embeddings else self.lm_head
# -- rope ------------------------------------------------------------- #
def rope_tables(self, pos, theta: float, device, dtype=torch.float32):
"""cos/sin for (B, T) per-document positions, gathered once per forward.
Positions restart at every document, so a document packed at row offset
1500 is rotated as though it began at 0. RoPE is relative, so this is
not strictly required inside a document -- but it keeps the absolute
range a document sees equal to its own length, which is what the
context stage is defined in terms of.
"""
hd = self.cfg.model.head_dim
key = (theta, hd, device)
if key not in self._rope_cache:
inv = 1.0 / (theta ** (torch.arange(0, hd, 2, device=device).float() / hd))
t = torch.arange(self.cfg.model.max_block_size, device=device).float()
f = torch.outer(t, inv)
self._rope_cache[key] = (f.cos().to(dtype), f.sin().to(dtype))
cos, sin = self._rope_cache[key]
return cos[pos].unsqueeze(2), sin[pos].unsqueeze(2) # (B, T, 1, hd/2)
# -- forward ---------------------------------------------------------- #
def forward(self, idx, pos, offsets, boundary, targets=None,
theta: float = 10_000.0, min_seqlen=None, max_seqlen=None):
"""idx/pos/boundary are (B, T); offsets indexes the flattened B*T stream."""
cos, sin = self.rope_tables(pos, theta, idx.device)
# m1: is t-1 in this document? m2: is t-2? Once per forward, not per layer.
b1 = boundary
b2 = boundary | F.pad(boundary, (1, 0))[:, :-1]
dt = torch.float16 if self.cfg.train.fp16_residual else torch.float32
m1 = (~b1).to(dt)[:, :, None]
m2 = (~b2).to(dt)[:, :, None]
ctx = RunContext(cos, sin, m1, m2, offsets, self._pre_attn, self._post_attn,
min_seqlen, max_seqlen, self._pre_attn_qkv)
gc = self.cfg.train.grad_checkpoint
every = max(1, self.cfg.train.grad_checkpoint_every)
a0 = self.blocks[0].attn
merged = (_volta(idx) and a0.canon is not None and a0.rep == 1 and a0.head_dim == 64
and not (gc and self.training) and not self.cfg.train.fp16_residual
and not getattr(self, "_no_merge", False))
# Tied-embedding fused accumulate: only when the head's backward is
# certain to run first and stash its dW (training forward with targets).
stash = (merged and self.tied_acc and targets is not None
and torch.is_grad_enabled() and self.cfg.model.tie_embeddings
and self.embed.weight.requires_grad)
if stash:
h = TiedEmbedding.apply(idx, self.embed.weight)
else:
h = self.embed(idx)
if self.cfg.train.fp16_residual:
h = h.to(torch.float16)
if merged:
h = self._forward_merged(h, idx, boundary, ctx)
if targets is None:
return h
return fused_lm_loss(h.reshape(-1, h.shape[-1]), self.head_weight(),
targets.reshape(-1), self.cfg.optim.lm_head_chunk,
stash)
for i, blk in enumerate(self.blocks):
if (self.ngram is not None and self.ngram_enabled
and i == self.cfg.ngram.insert_after):
h = h + self.ngram(idx, boundary, h.dtype)
if gc and self.training and (i % every == 0):
h = torch.utils.checkpoint.checkpoint(
blk, h, ctx, use_reentrant=False)
else:
h = blk(h, ctx)
h = self.norm_f(h)
if targets is None:
return h
loss = fused_lm_loss(h.reshape(-1, h.shape[-1]), self.head_weight(),
targets.reshape(-1), self.cfg.optim.lm_head_chunk)
return loss
@staticmethod
def _acc(p):
"""p.grad as the fp32 accumulator the weight-gradient GEMMs add into;
created (zeroed) on the first micro-step after zero_grad(set_to_none)."""
if not p.requires_grad:
return None
if p.grad is None:
p.grad = torch.zeros_like(p, memory_format=torch.contiguous_format)
return p.grad
def _wq(self, p, acc, scale: float = 1.0):
"""The weight as the region should receive it: the fp32 Parameter
(cast inside the region), or -- with w16_cache -- an fp16 copy cast once
per optimizer step. The copy is keyed on the identity of p.grad: every
step starts with zero_grad(set_to_none=True), so a new accumulator means
the weights may have moved. (p._version can NOT be used: Muon updates
p.data in place, which does not bump the version counter.)"""
if not (self.w16_cache and acc is not None):
return p
import weakref
e = self._w16c.get(id(p))
if e is not None and e[1]() is acc:
return e[0]
with torch.no_grad():
w = (p * scale if scale != 1.0 else p).to(torch.float16)
self._w16c[id(p)] = (w, weakref.ref(acc))
return w
def _forward_merged(self, h, idx, boundary, ctx):
"""Blocks with each residual boundary inside one compiled region
(Volta path, no gradient checkpointing). Same operations, same order,
as the per-block path; returns norm_f(h)."""
if torch.is_grad_enabled() and any(self.ft):
return self._forward_merged_x(h, idx, boundary, ctx)
return self._forward_merged_stock(h, idx, boundary, ctx)
def _forward_merged_x(self, h, idx, boundary, ctx):
"""_forward_merged with the swarm additions (self.ft, self.w16_cache)."""
from volta_kernels import canon_rope_attention
ft = self.ft
wacc = ft[0]
A = self._acc if wacc else (lambda p: None)
W = self._wq
blocks = self.blocks
L = len(blocks)
ins = self.cfg.ngram.insert_after if (self.ngram is not None
and self.ngram_enabled) else -1
ng = self.ngram(idx, boundary, h.dtype) if ins >= 0 else None
if ins == 0:
h = h + ng
B, T = h.shape[0], h.shape[1]
b0 = blocks[0]
a0 = A(b0.attn.qkv_proj)
qkv = self._pre_attn_qkv_x(h, b0.norm1.weight, b0.norm1.eps,
W(b0.attn.qkv_proj, a0), a0)
for i, blk in enumerate(blocks):
a = blk.attn
o = canon_rope_attention(qkv, a.canon, ctx.m1, ctx.m2, ctx.cos, ctx.sin,
ctx.offsets).reshape(B, T, a.n_head * a.head_dim)
f = blk.ffn
ao, agu, adn = A(a.o_proj), A(f.gate_up_proj), A(f.down_proj)
wo, wgu, wdn = W(a.o_proj, ao), W(f.gate_up_proj, agu), W(f.down_proj, adn, DOWN_SHRINK)
if i + 1 < L:
nb = blocks[i + 1]
aq = A(nb.attn.qkv_proj)
h, qkv = self._post_then_pre_x(
h, o, wo, blk.norm2.weight, blk.norm2.eps, wgu, wdn,
nb.norm1.weight, nb.norm1.eps, W(nb.attn.qkv_proj, aq),
ng if i + 1 == ins else None,
(ao, agu, adn, aq) if wacc else None, ft)
else:
h = self._post_then_final_norm_x(
h, o, wo, blk.norm2.weight, blk.norm2.eps, wgu, wdn,
self.norm_f.weight, self.norm_f.eps,
(ao, agu, adn) if wacc else None, ft)
return h
def _forward_merged_stock(self, h, idx, boundary, ctx):
from volta_kernels import canon_rope_attention
blocks = self.blocks
L = len(blocks)
ins = self.cfg.ngram.insert_after if (self.ngram is not None
and self.ngram_enabled) else -1
ng = self.ngram(idx, boundary, h.dtype) if ins >= 0 else None
if ins == 0:
h = h + ng
B, T = h.shape[0], h.shape[1]
b0 = blocks[0]
qkv = self._pre_attn_qkv(h, b0.norm1.weight, b0.norm1.eps, b0.attn.qkv_proj)
for i, blk in enumerate(blocks):
a = blk.attn
# Canon+RoPE+attention as one autograd Function: dQ reaches the
# Canon backward in fp32 (see volta_kernels._CanonRopeAttn).
o = canon_rope_attention(qkv, a.canon, ctx.m1, ctx.m2, ctx.cos, ctx.sin,
ctx.offsets).reshape(B, T, a.n_head * a.head_dim)
if i + 1 < L:
nb = blocks[i + 1]
h, qkv = self._post_then_pre(
h, o, a.o_proj, blk.norm2.weight, blk.norm2.eps,
blk.ffn.gate_up_proj, blk.ffn.down_proj,
nb.norm1.weight, nb.norm1.eps, nb.attn.qkv_proj,
ng if i + 1 == ins else None)
else:
h = self._post_then_final_norm(
h, o, a.o_proj, blk.norm2.weight, blk.norm2.eps,
blk.ffn.gate_up_proj, blk.ffn.down_proj,
self.norm_f.weight, self.norm_f.eps)
return h
@torch.no_grad()
def logits_for_eval(self, idx, pos, offsets, boundary, theta: float = 10_000.0):
h = self.forward(idx, pos, offsets, boundary, None, theta)
return F.linear(h, self.head_weight())
|