"""Muon on the 2-D matmul weights, AdamW on everything else. The split is by role, tagged on each Parameter at construction, and never by sniffing ndim. Canon kernels are (768, 3) tensors: two-dimensional, and emphatically not matrices to orthogonalise. A shape rule would hand all 26 of them to Muon and nothing would ever complain -- the run would simply be a little worse for several weeks. `build_optimizers` prints the full assignment and asserts it against config.param_budget() before the first step. Newton-Schulz is batched across same-shaped matrices: the quintic iteration becomes a handful of bmm chains instead of 183 independent ones, ~60 kernel launches per optimizer step in place of ~2,300. It runs in fp32 -- 1.8 measured a bf16 iteration diverging 5.4e-02 relative from the fp32 path, because five steps amplify rounding well past bf16 epsilon, and sm_70 has no bf16 tensor cores to gain from anyway. Some Parameters hold several logical matrices. Q, K and V live in one (768, 512) tensor and gate/up in one (4736, 512), because storing them fused removes a torch.cat per layer per micro-step from the forward. Those Parameters carry a `_boris_blocks` tag giving the row range of each logical matrix, and Muon orthogonalises each range independently -- which is precisely what it would do to separate Parameters, since the iteration acts on one matrix at a time. Momentum is elementwise and so is unaffected by the split. """ from __future__ import annotations import math import torch from torch.optim import AdamW MUON_ROLES = {"matmul"} ADAMW_ROLES = {"embedding", "ngram_table", "canon", "norm"} # Roles whose SCALE is a free parameter that nothing else restrains. DECAY_ROLES = {"embedding", "ngram_table"} # Quintic coefficients from Jordan et al.; tuned so the iteration converges on # the singular values of a Frobenius-normalised matrix in five steps. _NS_ABC = (3.4445, -4.7750, 2.0315) @torch.no_grad() def newton_schulz(X: torch.Tensor, steps: int = 5, eps: float = 1e-7) -> torch.Tensor: """Orthogonalise a batch of matrices. X is (G, m, n) with m <= n. Frobenius-normalised per matrix, so every matrix in the batch converges on its own scale rather than the batch's. """ a, b, c = _NS_ABC X = X / (X.norm(dim=(1, 2), keepdim=True) + eps) for _ in range(steps): # Same polynomial, with the scalar terms folded into the GEMM epilogues # (swarm muon_ns "fused32"): B = b*A + c*A@A and X = a*X + B@X are one # baddbmm each instead of a bmm plus two elementwise passes. Error vs # fp64 equal or lower (4.185e-6 -> 4.143e-6 median), ~7% faster. A = torch.bmm(X, X.transpose(1, 2)) B = torch.baddbmm(A, A, A, beta=b, alpha=c) X = torch.baddbmm(X, B, X, beta=a) return X class Muon(torch.optim.Optimizer): """Momentum, then an orthogonalised update, on 2-D matmul weights only.""" def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True, ns_steps=5, weight_decay=0.1, batched=True, ns_dtype=torch.float32, rms_scale=True): super().__init__(list(params), dict(lr=lr, momentum=momentum, nesterov=nesterov, ns_steps=ns_steps, weight_decay=weight_decay)) self.batched = batched self.ns_dtype = ns_dtype self.rms_scale = rms_scale self._plan: list | None = None @staticmethod def _units(group): """(parameter, row range or None, matrix shape) for every logical matrix.""" out = [] for p in group["params"]: if p.grad is None: continue blocks = getattr(p, "_boris_blocks", None) if blocks is None: out.append((p, None, tuple(p.shape))) else: for a, b in blocks: out.append((p, (int(a), int(b)), (int(b) - int(a), p.shape[1]))) return out def _build_plan(self, group): """Bucket logical matrices by canonical shape once; shapes never change. Orthogonalisation commutes with transpose, so a (2368, 512) matrix and a (512, 2368) one share a bucket -- the wide orientation is canonical and the narrow ones are transposed in and back out. """ buckets: dict[tuple, list] = {} for p, rng, shape in self._units(group): m, n = shape buckets.setdefault((min(m, n), max(m, n)), []).append((p, rng, shape, m > n)) return list(buckets.items()) @torch.no_grad() def step(self, closure=None): loss = closure() if closure is not None else None for group in self.param_groups: lr, mom = group["lr"], group["momentum"] nesterov, wd = group["nesterov"], group["weight_decay"] steps = group["ns_steps"] # momentum, in place, on every parameter bufs = [] for p in group["params"]: if p.grad is None: continue st = self.state[p] if "momentum_buffer" not in st: st["momentum_buffer"] = torch.zeros_like(p) buf = st["momentum_buffer"] buf.mul_(mom).add_(p.grad) g = p.grad.add(buf, alpha=mom) if nesterov else buf bufs.append((p, g)) gmap = {id(p): g for p, g in bufs} if not self.batched: for p, rng, shape in self._units(group): g = gmap[id(p)] g = g if rng is None else g[rng[0]:rng[1]] tr = shape[0] > shape[1] x = (g.t() if tr else g).to(self.ns_dtype)[None] o = newton_schulz(x, steps)[0] self._apply(p, rng, shape, o.t() if tr else o, lr, wd) continue if self._plan is None: self._plan = self._build_plan(group) for _key, members in self._plan: stack = torch.stack([ ((gmap[id(p)] if rng is None else gmap[id(p)][rng[0]:rng[1]]).t() if tr else (gmap[id(p)] if rng is None else gmap[id(p)][rng[0]:rng[1]]) ).to(self.ns_dtype) for p, rng, shape, tr in members]) out = newton_schulz(stack, steps) for i, (p, rng, shape, tr) in enumerate(members): o = out[i] self._apply(p, rng, shape, o.t() if tr else o, lr, wd) return loss def _apply(self, p, rng, shape, o, lr, wd): # 0.2*sqrt(max(m,n)) makes the update RMS the same for a 512x512 matrix # and a 512x2368 one, so a single LR is meaningful across a 4.63x FFN. # Computed from the LOGICAL matrix shape, not the storage tensor. s = 0.2 * math.sqrt(max(shape)) if self.rms_scale else 1.0 t = p.data if rng is None else p.data[rng[0]:rng[1]] if wd: t.mul_(1 - lr * wd) t.add_(o.to(t.dtype), alpha=-lr * s) # --------------------------------------------------------------------------- # # The split # --------------------------------------------------------------------------- # def _role(p, name: str) -> str: r = getattr(p, "_boris_role", None) if r is None: raise AssertionError( f"parameter '{name}' has no _boris_role tag. Every parameter must be " f"assigned to an optimizer explicitly at construction -- an untagged " f"parameter is a misconfiguration, not a default.") return r def build_optimizers(model, cfg, log=print): muon_params, adamw_params = [], [] adamw_roles = [] rows = [] for name, p in model.named_parameters(): if not p.requires_grad: continue r = _role(p, name) if r in MUON_ROLES: muon_params.append(p); which = "muon" elif r in ADAMW_ROLES: adamw_params.append(p); adamw_roles.append((p, r)); which = "adamw" else: raise AssertionError(f"parameter '{name}' has unknown role '{r}'") rows.append((name, tuple(p.shape), p.numel(), r, which)) o = cfg.optim muon = Muon(muon_params, lr=o.muon_lr, momentum=o.muon_momentum, nesterov=o.muon_nesterov, ns_steps=o.muon_ns_steps, weight_decay=o.muon_weight_decay, batched=o.muon_batched, ns_dtype=getattr(torch, o.muon_ns_dtype), rms_scale=o.muon_rms_scale) # 2026-09-17: AdamW used ONE weight_decay for every role, and it was 0.0. # Muon decays its matmuls at 0.1 and they are flat to within 1% over 9.3B # tokens; the undecayed AdamW tensors are not. Measured 20.8B -> 30.1B: # embed.weight 1879 -> 2275 (+21.1%) wd 0.0 # ngram.table 2747 -> 3326 (+21.1%) wd 0.0 # qkv_proj 167 -> 168 ( +0.4%) wd 0.1 # down_proj 327 -> 324 ( -1.0%) wd 0.1 # Nothing restrains the two largest tensors, so they inflate, norm_f shrinks # to hold logit scale roughly constant (8.05 -> 7.12), and 66% of the total # gradient power ends up in that one 512-vector fighting the embedding. # AdamW's step is ~lr regardless of scale, so a growing ||w|| means the # RELATIVE update lr/||w|| decays -- embedding directions calcify. Row norms # grow ~+1.07 per 5B (linear), which extrapolates to ~4x by 200B. # Decay the two big tensors; leave norms and canon alone, as is conventional # -- norm gains are a scale, not a representation, and canon is near-flat # already (+0.8% to +5.7%). # # 2026-10-07: canon is no longer near-flat. At the corrected Muon lr (0929) it # grew ~+20% per 10B with nothing restraining it, and the norm1 gains shrank # to compensate. It gets its own group (adamw_canon_weight_decay); the group # ORDER is [decay, norms, canon] and checkpoint._migrate_adamw_groups remaps # older 1- and 2-group states onto it by parameter identity. decay = [p for p, r in adamw_roles if r in DECAY_ROLES] norms = [p for p, r in adamw_roles if r not in DECAY_ROLES and r != "canon"] canon = [p for p, r in adamw_roles if r == "canon"] adamw = AdamW([{"params": decay, "weight_decay": o.adamw_weight_decay}, {"params": norms, "weight_decay": o.adamw_norm_weight_decay}, {"params": canon, "weight_decay": o.adamw_canon_weight_decay}], lr=o.adamw_lr, betas=tuple(o.adamw_betas), eps=o.adamw_eps, fused=torch.cuda.is_available()) n = lambda ps: f"{len(ps)} tensors ({sum(p.numel() for p in ps):,} params)" log(f"\n adamw decay groups: {n(decay)} at wd {o.adamw_weight_decay}, " f"{n(norms)} at wd {o.adamw_norm_weight_decay}, " f"canon {n(canon)} at wd {o.adamw_canon_weight_decay}") _report_and_assert(rows, muon_params, adamw_params, model, cfg, log) return muon, adamw def _report_and_assert(rows, muon_params, adamw_params, model, cfg, log): by_role: dict[str, list] = {} for name, shape, n, role, which in rows: by_role.setdefault(role, []).append((name, shape, n, which)) log("\n parameter groups") log(" " + "-" * 74) log(f" {'role':<13}{'optimizer':<10}{'tensors':>9}{'parameters':>15} example shape") total = 0 for role in ("matmul", "embedding", "ngram_table", "canon", "norm"): if role not in by_role: continue e = by_role[role] n = sum(x[2] for x in e) total += n log(f" {role:<13}{e[0][3]:<10}{len(e):>9}{n:>15,} {e[0][1]}") log(" " + "-" * 74) n_muon = sum(p.numel() for p in muon_params) n_adamw = sum(p.numel() for p in adamw_params) n_mat = sum(len(getattr(p, "_boris_blocks", (None,))) for p in muon_params) log(f" {'MUON':<23}{len(muon_params):>9}{n_muon:>15,} " f"{n_mat} logical matrices") log(f" {'ADAMW':<23}{len(adamw_params):>9}{n_adamw:>15,}") log(f" {'TOTAL':<23}{len(rows):>9}{total:>15,}") ids_m = {id(p) for p in muon_params} ids_a = {id(p) for p in adamw_params} assert not (ids_m & ids_a), "a parameter landed in both optimizers" all_ids = {id(p) for p in model.parameters() if p.requires_grad} assert ids_m | ids_a == all_ids, ( f"{len(all_ids - (ids_m | ids_a))} trainable parameters are in no optimizer") for p in muon_params: assert p.ndim == 2 and min(p.shape) > 3, ( f"a non-matrix of shape {tuple(p.shape)} reached Muon") blocks = getattr(p, "_boris_blocks", None) if blocks is not None: assert blocks[0][0] == 0 and blocks[-1][1] == p.shape[0], ( f"_boris_blocks {blocks} do not tile the {tuple(p.shape)} tensor") for (a1, b1), (a2, _) in zip(blocks, blocks[1:]): assert b1 == a2, f"_boris_blocks {blocks} overlap or leave a gap" b = cfg.param_budget() assert n_muon == b["matmul"], ( f"Muon holds {n_muon:,} parameters, the budget says {b['matmul']:,}") assert total == b["total"], ( f"model has {total:,} parameters, the budget says {b['total']:,}") cfg.assert_budget() log(f" budget assertions pass; fingerprint {cfg.fingerprint()}\n") def set_lr(muon, adamw, cfg, scale: float) -> tuple[float, float]: ml = cfg.optim.muon_lr * scale al = cfg.optim.adamw_lr * scale for g in muon.param_groups: g["lr"] = ml for g in adamw.param_groups: g["lr"] = al return ml, al