summaryrefslogtreecommitdiff
path: root/optim.py
diff options
context:
space:
mode:
Diffstat (limited to 'optim.py')
-rw-r--r--optim.py293
1 files changed, 293 insertions, 0 deletions
diff --git a/optim.py b/optim.py
new file mode 100644
index 0000000..7d87abb
--- /dev/null
+++ b/optim.py
@@ -0,0 +1,293 @@
+"""Muon on the 2-D matmul weights, AdamW on everything else.
+
+The split is by role, tagged on each Parameter at construction, and never by
+sniffing ndim. Canon kernels are (768, 3) tensors: two-dimensional, and
+emphatically not matrices to orthogonalise. A shape rule would hand all 26 of
+them to Muon and nothing would ever complain -- the run would simply be a little
+worse for several weeks. `build_optimizers` prints the full assignment and
+asserts it against config.param_budget() before the first step.
+
+Newton-Schulz is batched across same-shaped matrices: the quintic iteration
+becomes a handful of bmm chains instead of 183 independent ones, ~60 kernel
+launches per optimizer step in place of ~2,300. It runs in fp32 -- 1.8 measured
+a bf16 iteration diverging 5.4e-02 relative from the fp32 path, because five
+steps amplify rounding well past bf16 epsilon, and sm_70 has no bf16 tensor
+cores to gain from anyway.
+
+Some Parameters hold several logical matrices. Q, K and V live in one (768, 512)
+tensor and gate/up in one (4736, 512), because storing them fused removes a
+torch.cat per layer per micro-step from the forward. Those Parameters carry a
+`_boris_blocks` tag giving the row range of each logical matrix, and Muon
+orthogonalises each range independently -- which is precisely what it would do
+to separate Parameters, since the iteration acts on one matrix at a time.
+Momentum is elementwise and so is unaffected by the split.
+"""
+from __future__ import annotations
+
+import math
+
+import torch
+from torch.optim import AdamW
+
+MUON_ROLES = {"matmul"}
+ADAMW_ROLES = {"embedding", "ngram_table", "canon", "norm"}
+# Roles whose SCALE is a free parameter that nothing else restrains.
+DECAY_ROLES = {"embedding", "ngram_table"}
+
+# Quintic coefficients from Jordan et al.; tuned so the iteration converges on
+# the singular values of a Frobenius-normalised matrix in five steps.
+_NS_ABC = (3.4445, -4.7750, 2.0315)
+
+
+@torch.no_grad()
+def newton_schulz(X: torch.Tensor, steps: int = 5, eps: float = 1e-7) -> torch.Tensor:
+ """Orthogonalise a batch of matrices. X is (G, m, n) with m <= n.
+
+ Frobenius-normalised per matrix, so every matrix in the batch converges on
+ its own scale rather than the batch's.
+ """
+ a, b, c = _NS_ABC
+ X = X / (X.norm(dim=(1, 2), keepdim=True) + eps)
+ for _ in range(steps):
+ # Same polynomial, with the scalar terms folded into the GEMM epilogues
+ # (swarm muon_ns "fused32"): B = b*A + c*A@A and X = a*X + B@X are one
+ # baddbmm each instead of a bmm plus two elementwise passes. Error vs
+ # fp64 equal or lower (4.185e-6 -> 4.143e-6 median), ~7% faster.
+ A = torch.bmm(X, X.transpose(1, 2))
+ B = torch.baddbmm(A, A, A, beta=b, alpha=c)
+ X = torch.baddbmm(X, B, X, beta=a)
+ return X
+
+
+class Muon(torch.optim.Optimizer):
+ """Momentum, then an orthogonalised update, on 2-D matmul weights only."""
+
+ def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True, ns_steps=5,
+ weight_decay=0.1, batched=True, ns_dtype=torch.float32,
+ rms_scale=True):
+ super().__init__(list(params), dict(lr=lr, momentum=momentum,
+ nesterov=nesterov, ns_steps=ns_steps,
+ weight_decay=weight_decay))
+ self.batched = batched
+ self.ns_dtype = ns_dtype
+ self.rms_scale = rms_scale
+ self._plan: list | None = None
+
+ @staticmethod
+ def _units(group):
+ """(parameter, row range or None, matrix shape) for every logical matrix."""
+ out = []
+ for p in group["params"]:
+ if p.grad is None:
+ continue
+ blocks = getattr(p, "_boris_blocks", None)
+ if blocks is None:
+ out.append((p, None, tuple(p.shape)))
+ else:
+ for a, b in blocks:
+ out.append((p, (int(a), int(b)), (int(b) - int(a), p.shape[1])))
+ return out
+
+ def _build_plan(self, group):
+ """Bucket logical matrices by canonical shape once; shapes never change.
+
+ Orthogonalisation commutes with transpose, so a (2368, 512) matrix and a
+ (512, 2368) one share a bucket -- the wide orientation is canonical and
+ the narrow ones are transposed in and back out.
+ """
+ buckets: dict[tuple, list] = {}
+ for p, rng, shape in self._units(group):
+ m, n = shape
+ buckets.setdefault((min(m, n), max(m, n)), []).append((p, rng, shape, m > n))
+ return list(buckets.items())
+
+ @torch.no_grad()
+ def step(self, closure=None):
+ loss = closure() if closure is not None else None
+ for group in self.param_groups:
+ lr, mom = group["lr"], group["momentum"]
+ nesterov, wd = group["nesterov"], group["weight_decay"]
+ steps = group["ns_steps"]
+
+ # momentum, in place, on every parameter
+ bufs = []
+ for p in group["params"]:
+ if p.grad is None:
+ continue
+ st = self.state[p]
+ if "momentum_buffer" not in st:
+ st["momentum_buffer"] = torch.zeros_like(p)
+ buf = st["momentum_buffer"]
+ buf.mul_(mom).add_(p.grad)
+ g = p.grad.add(buf, alpha=mom) if nesterov else buf
+ bufs.append((p, g))
+
+ gmap = {id(p): g for p, g in bufs}
+
+ if not self.batched:
+ for p, rng, shape in self._units(group):
+ g = gmap[id(p)]
+ g = g if rng is None else g[rng[0]:rng[1]]
+ tr = shape[0] > shape[1]
+ x = (g.t() if tr else g).to(self.ns_dtype)[None]
+ o = newton_schulz(x, steps)[0]
+ self._apply(p, rng, shape, o.t() if tr else o, lr, wd)
+ continue
+
+ if self._plan is None:
+ self._plan = self._build_plan(group)
+ for _key, members in self._plan:
+ stack = torch.stack([
+ ((gmap[id(p)] if rng is None else gmap[id(p)][rng[0]:rng[1]]).t()
+ if tr else
+ (gmap[id(p)] if rng is None else gmap[id(p)][rng[0]:rng[1]])
+ ).to(self.ns_dtype)
+ for p, rng, shape, tr in members])
+ out = newton_schulz(stack, steps)
+ for i, (p, rng, shape, tr) in enumerate(members):
+ o = out[i]
+ self._apply(p, rng, shape, o.t() if tr else o, lr, wd)
+ return loss
+
+ def _apply(self, p, rng, shape, o, lr, wd):
+ # 0.2*sqrt(max(m,n)) makes the update RMS the same for a 512x512 matrix
+ # and a 512x2368 one, so a single LR is meaningful across a 4.63x FFN.
+ # Computed from the LOGICAL matrix shape, not the storage tensor.
+ s = 0.2 * math.sqrt(max(shape)) if self.rms_scale else 1.0
+ t = p.data if rng is None else p.data[rng[0]:rng[1]]
+ if wd:
+ t.mul_(1 - lr * wd)
+ t.add_(o.to(t.dtype), alpha=-lr * s)
+
+
+# --------------------------------------------------------------------------- #
+# The split
+# --------------------------------------------------------------------------- #
+def _role(p, name: str) -> str:
+ r = getattr(p, "_boris_role", None)
+ if r is None:
+ raise AssertionError(
+ f"parameter '{name}' has no _boris_role tag. Every parameter must be "
+ f"assigned to an optimizer explicitly at construction -- an untagged "
+ f"parameter is a misconfiguration, not a default.")
+ return r
+
+
+def build_optimizers(model, cfg, log=print):
+ muon_params, adamw_params = [], []
+ adamw_roles = []
+ rows = []
+ for name, p in model.named_parameters():
+ if not p.requires_grad:
+ continue
+ r = _role(p, name)
+ if r in MUON_ROLES:
+ muon_params.append(p); which = "muon"
+ elif r in ADAMW_ROLES:
+ adamw_params.append(p); adamw_roles.append((p, r)); which = "adamw"
+ else:
+ raise AssertionError(f"parameter '{name}' has unknown role '{r}'")
+ rows.append((name, tuple(p.shape), p.numel(), r, which))
+
+ o = cfg.optim
+ muon = Muon(muon_params, lr=o.muon_lr, momentum=o.muon_momentum,
+ nesterov=o.muon_nesterov, ns_steps=o.muon_ns_steps,
+ weight_decay=o.muon_weight_decay, batched=o.muon_batched,
+ ns_dtype=getattr(torch, o.muon_ns_dtype), rms_scale=o.muon_rms_scale)
+ # 2026-09-17: AdamW used ONE weight_decay for every role, and it was 0.0.
+ # Muon decays its matmuls at 0.1 and they are flat to within 1% over 9.3B
+ # tokens; the undecayed AdamW tensors are not. Measured 20.8B -> 30.1B:
+ # embed.weight 1879 -> 2275 (+21.1%) wd 0.0
+ # ngram.table 2747 -> 3326 (+21.1%) wd 0.0
+ # qkv_proj 167 -> 168 ( +0.4%) wd 0.1
+ # down_proj 327 -> 324 ( -1.0%) wd 0.1
+ # Nothing restrains the two largest tensors, so they inflate, norm_f shrinks
+ # to hold logit scale roughly constant (8.05 -> 7.12), and 66% of the total
+ # gradient power ends up in that one 512-vector fighting the embedding.
+ # AdamW's step is ~lr regardless of scale, so a growing ||w|| means the
+ # RELATIVE update lr/||w|| decays -- embedding directions calcify. Row norms
+ # grow ~+1.07 per 5B (linear), which extrapolates to ~4x by 200B.
+ # Decay the two big tensors; leave norms and canon alone, as is conventional
+ # -- norm gains are a scale, not a representation, and canon is near-flat
+ # already (+0.8% to +5.7%).
+ #
+ # 2026-10-07: canon is no longer near-flat. At the corrected Muon lr (0929) it
+ # grew ~+20% per 10B with nothing restraining it, and the norm1 gains shrank
+ # to compensate. It gets its own group (adamw_canon_weight_decay); the group
+ # ORDER is [decay, norms, canon] and checkpoint._migrate_adamw_groups remaps
+ # older 1- and 2-group states onto it by parameter identity.
+ decay = [p for p, r in adamw_roles if r in DECAY_ROLES]
+ norms = [p for p, r in adamw_roles if r not in DECAY_ROLES and r != "canon"]
+ canon = [p for p, r in adamw_roles if r == "canon"]
+ adamw = AdamW([{"params": decay, "weight_decay": o.adamw_weight_decay},
+ {"params": norms, "weight_decay": o.adamw_norm_weight_decay},
+ {"params": canon, "weight_decay": o.adamw_canon_weight_decay}],
+ lr=o.adamw_lr, betas=tuple(o.adamw_betas),
+ eps=o.adamw_eps, fused=torch.cuda.is_available())
+ n = lambda ps: f"{len(ps)} tensors ({sum(p.numel() for p in ps):,} params)"
+ log(f"\n adamw decay groups: {n(decay)} at wd {o.adamw_weight_decay}, "
+ f"{n(norms)} at wd {o.adamw_norm_weight_decay}, "
+ f"canon {n(canon)} at wd {o.adamw_canon_weight_decay}")
+
+ _report_and_assert(rows, muon_params, adamw_params, model, cfg, log)
+ return muon, adamw
+
+
+def _report_and_assert(rows, muon_params, adamw_params, model, cfg, log):
+ by_role: dict[str, list] = {}
+ for name, shape, n, role, which in rows:
+ by_role.setdefault(role, []).append((name, shape, n, which))
+
+ log("\n parameter groups")
+ log(" " + "-" * 74)
+ log(f" {'role':<13}{'optimizer':<10}{'tensors':>9}{'parameters':>15} example shape")
+ total = 0
+ for role in ("matmul", "embedding", "ngram_table", "canon", "norm"):
+ if role not in by_role:
+ continue
+ e = by_role[role]
+ n = sum(x[2] for x in e)
+ total += n
+ log(f" {role:<13}{e[0][3]:<10}{len(e):>9}{n:>15,} {e[0][1]}")
+ log(" " + "-" * 74)
+ n_muon = sum(p.numel() for p in muon_params)
+ n_adamw = sum(p.numel() for p in adamw_params)
+ n_mat = sum(len(getattr(p, "_boris_blocks", (None,))) for p in muon_params)
+ log(f" {'MUON':<23}{len(muon_params):>9}{n_muon:>15,} "
+ f"{n_mat} logical matrices")
+ log(f" {'ADAMW':<23}{len(adamw_params):>9}{n_adamw:>15,}")
+ log(f" {'TOTAL':<23}{len(rows):>9}{total:>15,}")
+
+ ids_m = {id(p) for p in muon_params}
+ ids_a = {id(p) for p in adamw_params}
+ assert not (ids_m & ids_a), "a parameter landed in both optimizers"
+ all_ids = {id(p) for p in model.parameters() if p.requires_grad}
+ assert ids_m | ids_a == all_ids, (
+ f"{len(all_ids - (ids_m | ids_a))} trainable parameters are in no optimizer")
+
+ for p in muon_params:
+ assert p.ndim == 2 and min(p.shape) > 3, (
+ f"a non-matrix of shape {tuple(p.shape)} reached Muon")
+ blocks = getattr(p, "_boris_blocks", None)
+ if blocks is not None:
+ assert blocks[0][0] == 0 and blocks[-1][1] == p.shape[0], (
+ f"_boris_blocks {blocks} do not tile the {tuple(p.shape)} tensor")
+ for (a1, b1), (a2, _) in zip(blocks, blocks[1:]):
+ assert b1 == a2, f"_boris_blocks {blocks} overlap or leave a gap"
+ b = cfg.param_budget()
+ assert n_muon == b["matmul"], (
+ f"Muon holds {n_muon:,} parameters, the budget says {b['matmul']:,}")
+ assert total == b["total"], (
+ f"model has {total:,} parameters, the budget says {b['total']:,}")
+ cfg.assert_budget()
+ log(f" budget assertions pass; fingerprint {cfg.fingerprint()}\n")
+
+
+def set_lr(muon, adamw, cfg, scale: float) -> tuple[float, float]:
+ ml = cfg.optim.muon_lr * scale
+ al = cfg.optim.adamw_lr * scale
+ for g in muon.param_groups:
+ g["lr"] = ml
+ for g in adamw.param_groups:
+ g["lr"] = al
+ return ml, al