diff options
| author | jdj <jdj@kunix.org> | 2026-11-18 14:47:33 -0800 |
|---|---|---|
| committer | jdj <jdj@kunix.org> | 2026-11-18 14:47:33 -0800 |
| commit | 5d2e4252e695644c34be605b44a14bd80a98a0a6 (patch) | |
| tree | 311a72eda4459af7547b9cd05c0956eafb235f2d /optim.py | |
| download | boris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.tar.gz boris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.zip | |
Boris-2 training code
The code, configuration and data filters used to train Boris-2 (144M,
200B tokens, single V100): model, training loop, data preparation,
n-gram table, V100 kernels, final stages and the exact run configs.
Diffstat (limited to 'optim.py')
| -rw-r--r-- | optim.py | 293 |
1 files changed, 293 insertions, 0 deletions
diff --git a/optim.py b/optim.py new file mode 100644 index 0000000..7d87abb --- /dev/null +++ b/optim.py @@ -0,0 +1,293 @@ +"""Muon on the 2-D matmul weights, AdamW on everything else. + +The split is by role, tagged on each Parameter at construction, and never by +sniffing ndim. Canon kernels are (768, 3) tensors: two-dimensional, and +emphatically not matrices to orthogonalise. A shape rule would hand all 26 of +them to Muon and nothing would ever complain -- the run would simply be a little +worse for several weeks. `build_optimizers` prints the full assignment and +asserts it against config.param_budget() before the first step. + +Newton-Schulz is batched across same-shaped matrices: the quintic iteration +becomes a handful of bmm chains instead of 183 independent ones, ~60 kernel +launches per optimizer step in place of ~2,300. It runs in fp32 -- 1.8 measured +a bf16 iteration diverging 5.4e-02 relative from the fp32 path, because five +steps amplify rounding well past bf16 epsilon, and sm_70 has no bf16 tensor +cores to gain from anyway. + +Some Parameters hold several logical matrices. Q, K and V live in one (768, 512) +tensor and gate/up in one (4736, 512), because storing them fused removes a +torch.cat per layer per micro-step from the forward. Those Parameters carry a +`_boris_blocks` tag giving the row range of each logical matrix, and Muon +orthogonalises each range independently -- which is precisely what it would do +to separate Parameters, since the iteration acts on one matrix at a time. +Momentum is elementwise and so is unaffected by the split. +""" +from __future__ import annotations + +import math + +import torch +from torch.optim import AdamW + +MUON_ROLES = {"matmul"} +ADAMW_ROLES = {"embedding", "ngram_table", "canon", "norm"} +# Roles whose SCALE is a free parameter that nothing else restrains. +DECAY_ROLES = {"embedding", "ngram_table"} + +# Quintic coefficients from Jordan et al.; tuned so the iteration converges on +# the singular values of a Frobenius-normalised matrix in five steps. +_NS_ABC = (3.4445, -4.7750, 2.0315) + + +@torch.no_grad() +def newton_schulz(X: torch.Tensor, steps: int = 5, eps: float = 1e-7) -> torch.Tensor: + """Orthogonalise a batch of matrices. X is (G, m, n) with m <= n. + + Frobenius-normalised per matrix, so every matrix in the batch converges on + its own scale rather than the batch's. + """ + a, b, c = _NS_ABC + X = X / (X.norm(dim=(1, 2), keepdim=True) + eps) + for _ in range(steps): + # Same polynomial, with the scalar terms folded into the GEMM epilogues + # (swarm muon_ns "fused32"): B = b*A + c*A@A and X = a*X + B@X are one + # baddbmm each instead of a bmm plus two elementwise passes. Error vs + # fp64 equal or lower (4.185e-6 -> 4.143e-6 median), ~7% faster. + A = torch.bmm(X, X.transpose(1, 2)) + B = torch.baddbmm(A, A, A, beta=b, alpha=c) + X = torch.baddbmm(X, B, X, beta=a) + return X + + +class Muon(torch.optim.Optimizer): + """Momentum, then an orthogonalised update, on 2-D matmul weights only.""" + + def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True, ns_steps=5, + weight_decay=0.1, batched=True, ns_dtype=torch.float32, + rms_scale=True): + super().__init__(list(params), dict(lr=lr, momentum=momentum, + nesterov=nesterov, ns_steps=ns_steps, + weight_decay=weight_decay)) + self.batched = batched + self.ns_dtype = ns_dtype + self.rms_scale = rms_scale + self._plan: list | None = None + + @staticmethod + def _units(group): + """(parameter, row range or None, matrix shape) for every logical matrix.""" + out = [] + for p in group["params"]: + if p.grad is None: + continue + blocks = getattr(p, "_boris_blocks", None) + if blocks is None: + out.append((p, None, tuple(p.shape))) + else: + for a, b in blocks: + out.append((p, (int(a), int(b)), (int(b) - int(a), p.shape[1]))) + return out + + def _build_plan(self, group): + """Bucket logical matrices by canonical shape once; shapes never change. + + Orthogonalisation commutes with transpose, so a (2368, 512) matrix and a + (512, 2368) one share a bucket -- the wide orientation is canonical and + the narrow ones are transposed in and back out. + """ + buckets: dict[tuple, list] = {} + for p, rng, shape in self._units(group): + m, n = shape + buckets.setdefault((min(m, n), max(m, n)), []).append((p, rng, shape, m > n)) + return list(buckets.items()) + + @torch.no_grad() + def step(self, closure=None): + loss = closure() if closure is not None else None + for group in self.param_groups: + lr, mom = group["lr"], group["momentum"] + nesterov, wd = group["nesterov"], group["weight_decay"] + steps = group["ns_steps"] + + # momentum, in place, on every parameter + bufs = [] + for p in group["params"]: + if p.grad is None: + continue + st = self.state[p] + if "momentum_buffer" not in st: + st["momentum_buffer"] = torch.zeros_like(p) + buf = st["momentum_buffer"] + buf.mul_(mom).add_(p.grad) + g = p.grad.add(buf, alpha=mom) if nesterov else buf + bufs.append((p, g)) + + gmap = {id(p): g for p, g in bufs} + + if not self.batched: + for p, rng, shape in self._units(group): + g = gmap[id(p)] + g = g if rng is None else g[rng[0]:rng[1]] + tr = shape[0] > shape[1] + x = (g.t() if tr else g).to(self.ns_dtype)[None] + o = newton_schulz(x, steps)[0] + self._apply(p, rng, shape, o.t() if tr else o, lr, wd) + continue + + if self._plan is None: + self._plan = self._build_plan(group) + for _key, members in self._plan: + stack = torch.stack([ + ((gmap[id(p)] if rng is None else gmap[id(p)][rng[0]:rng[1]]).t() + if tr else + (gmap[id(p)] if rng is None else gmap[id(p)][rng[0]:rng[1]]) + ).to(self.ns_dtype) + for p, rng, shape, tr in members]) + out = newton_schulz(stack, steps) + for i, (p, rng, shape, tr) in enumerate(members): + o = out[i] + self._apply(p, rng, shape, o.t() if tr else o, lr, wd) + return loss + + def _apply(self, p, rng, shape, o, lr, wd): + # 0.2*sqrt(max(m,n)) makes the update RMS the same for a 512x512 matrix + # and a 512x2368 one, so a single LR is meaningful across a 4.63x FFN. + # Computed from the LOGICAL matrix shape, not the storage tensor. + s = 0.2 * math.sqrt(max(shape)) if self.rms_scale else 1.0 + t = p.data if rng is None else p.data[rng[0]:rng[1]] + if wd: + t.mul_(1 - lr * wd) + t.add_(o.to(t.dtype), alpha=-lr * s) + + +# --------------------------------------------------------------------------- # +# The split +# --------------------------------------------------------------------------- # +def _role(p, name: str) -> str: + r = getattr(p, "_boris_role", None) + if r is None: + raise AssertionError( + f"parameter '{name}' has no _boris_role tag. Every parameter must be " + f"assigned to an optimizer explicitly at construction -- an untagged " + f"parameter is a misconfiguration, not a default.") + return r + + +def build_optimizers(model, cfg, log=print): + muon_params, adamw_params = [], [] + adamw_roles = [] + rows = [] + for name, p in model.named_parameters(): + if not p.requires_grad: + continue + r = _role(p, name) + if r in MUON_ROLES: + muon_params.append(p); which = "muon" + elif r in ADAMW_ROLES: + adamw_params.append(p); adamw_roles.append((p, r)); which = "adamw" + else: + raise AssertionError(f"parameter '{name}' has unknown role '{r}'") + rows.append((name, tuple(p.shape), p.numel(), r, which)) + + o = cfg.optim + muon = Muon(muon_params, lr=o.muon_lr, momentum=o.muon_momentum, + nesterov=o.muon_nesterov, ns_steps=o.muon_ns_steps, + weight_decay=o.muon_weight_decay, batched=o.muon_batched, + ns_dtype=getattr(torch, o.muon_ns_dtype), rms_scale=o.muon_rms_scale) + # 2026-09-17: AdamW used ONE weight_decay for every role, and it was 0.0. + # Muon decays its matmuls at 0.1 and they are flat to within 1% over 9.3B + # tokens; the undecayed AdamW tensors are not. Measured 20.8B -> 30.1B: + # embed.weight 1879 -> 2275 (+21.1%) wd 0.0 + # ngram.table 2747 -> 3326 (+21.1%) wd 0.0 + # qkv_proj 167 -> 168 ( +0.4%) wd 0.1 + # down_proj 327 -> 324 ( -1.0%) wd 0.1 + # Nothing restrains the two largest tensors, so they inflate, norm_f shrinks + # to hold logit scale roughly constant (8.05 -> 7.12), and 66% of the total + # gradient power ends up in that one 512-vector fighting the embedding. + # AdamW's step is ~lr regardless of scale, so a growing ||w|| means the + # RELATIVE update lr/||w|| decays -- embedding directions calcify. Row norms + # grow ~+1.07 per 5B (linear), which extrapolates to ~4x by 200B. + # Decay the two big tensors; leave norms and canon alone, as is conventional + # -- norm gains are a scale, not a representation, and canon is near-flat + # already (+0.8% to +5.7%). + # + # 2026-10-07: canon is no longer near-flat. At the corrected Muon lr (0929) it + # grew ~+20% per 10B with nothing restraining it, and the norm1 gains shrank + # to compensate. It gets its own group (adamw_canon_weight_decay); the group + # ORDER is [decay, norms, canon] and checkpoint._migrate_adamw_groups remaps + # older 1- and 2-group states onto it by parameter identity. + decay = [p for p, r in adamw_roles if r in DECAY_ROLES] + norms = [p for p, r in adamw_roles if r not in DECAY_ROLES and r != "canon"] + canon = [p for p, r in adamw_roles if r == "canon"] + adamw = AdamW([{"params": decay, "weight_decay": o.adamw_weight_decay}, + {"params": norms, "weight_decay": o.adamw_norm_weight_decay}, + {"params": canon, "weight_decay": o.adamw_canon_weight_decay}], + lr=o.adamw_lr, betas=tuple(o.adamw_betas), + eps=o.adamw_eps, fused=torch.cuda.is_available()) + n = lambda ps: f"{len(ps)} tensors ({sum(p.numel() for p in ps):,} params)" + log(f"\n adamw decay groups: {n(decay)} at wd {o.adamw_weight_decay}, " + f"{n(norms)} at wd {o.adamw_norm_weight_decay}, " + f"canon {n(canon)} at wd {o.adamw_canon_weight_decay}") + + _report_and_assert(rows, muon_params, adamw_params, model, cfg, log) + return muon, adamw + + +def _report_and_assert(rows, muon_params, adamw_params, model, cfg, log): + by_role: dict[str, list] = {} + for name, shape, n, role, which in rows: + by_role.setdefault(role, []).append((name, shape, n, which)) + + log("\n parameter groups") + log(" " + "-" * 74) + log(f" {'role':<13}{'optimizer':<10}{'tensors':>9}{'parameters':>15} example shape") + total = 0 + for role in ("matmul", "embedding", "ngram_table", "canon", "norm"): + if role not in by_role: + continue + e = by_role[role] + n = sum(x[2] for x in e) + total += n + log(f" {role:<13}{e[0][3]:<10}{len(e):>9}{n:>15,} {e[0][1]}") + log(" " + "-" * 74) + n_muon = sum(p.numel() for p in muon_params) + n_adamw = sum(p.numel() for p in adamw_params) + n_mat = sum(len(getattr(p, "_boris_blocks", (None,))) for p in muon_params) + log(f" {'MUON':<23}{len(muon_params):>9}{n_muon:>15,} " + f"{n_mat} logical matrices") + log(f" {'ADAMW':<23}{len(adamw_params):>9}{n_adamw:>15,}") + log(f" {'TOTAL':<23}{len(rows):>9}{total:>15,}") + + ids_m = {id(p) for p in muon_params} + ids_a = {id(p) for p in adamw_params} + assert not (ids_m & ids_a), "a parameter landed in both optimizers" + all_ids = {id(p) for p in model.parameters() if p.requires_grad} + assert ids_m | ids_a == all_ids, ( + f"{len(all_ids - (ids_m | ids_a))} trainable parameters are in no optimizer") + + for p in muon_params: + assert p.ndim == 2 and min(p.shape) > 3, ( + f"a non-matrix of shape {tuple(p.shape)} reached Muon") + blocks = getattr(p, "_boris_blocks", None) + if blocks is not None: + assert blocks[0][0] == 0 and blocks[-1][1] == p.shape[0], ( + f"_boris_blocks {blocks} do not tile the {tuple(p.shape)} tensor") + for (a1, b1), (a2, _) in zip(blocks, blocks[1:]): + assert b1 == a2, f"_boris_blocks {blocks} overlap or leave a gap" + b = cfg.param_budget() + assert n_muon == b["matmul"], ( + f"Muon holds {n_muon:,} parameters, the budget says {b['matmul']:,}") + assert total == b["total"], ( + f"model has {total:,} parameters, the budget says {b['total']:,}") + cfg.assert_budget() + log(f" budget assertions pass; fingerprint {cfg.fingerprint()}\n") + + +def set_lr(muon, adamw, cfg, scale: float) -> tuple[float, float]: + ml = cfg.optim.muon_lr * scale + al = cfg.optim.adamw_lr * scale + for g in muon.param_groups: + g["lr"] = ml + for g in adamw.param_groups: + g["lr"] = al + return ml, al |
