summaryrefslogtreecommitdiff
path: root/optim.py
blob: 7d87abbd4db640d5de3dfd95910767ec36eef132 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
"""Muon on the 2-D matmul weights, AdamW on everything else.

The split is by role, tagged on each Parameter at construction, and never by
sniffing ndim. Canon kernels are (768, 3) tensors: two-dimensional, and
emphatically not matrices to orthogonalise. A shape rule would hand all 26 of
them to Muon and nothing would ever complain -- the run would simply be a little
worse for several weeks. `build_optimizers` prints the full assignment and
asserts it against config.param_budget() before the first step.

Newton-Schulz is batched across same-shaped matrices: the quintic iteration
becomes a handful of bmm chains instead of 183 independent ones, ~60 kernel
launches per optimizer step in place of ~2,300. It runs in fp32 -- 1.8 measured
a bf16 iteration diverging 5.4e-02 relative from the fp32 path, because five
steps amplify rounding well past bf16 epsilon, and sm_70 has no bf16 tensor
cores to gain from anyway.

Some Parameters hold several logical matrices. Q, K and V live in one (768, 512)
tensor and gate/up in one (4736, 512), because storing them fused removes a
torch.cat per layer per micro-step from the forward. Those Parameters carry a
`_boris_blocks` tag giving the row range of each logical matrix, and Muon
orthogonalises each range independently -- which is precisely what it would do
to separate Parameters, since the iteration acts on one matrix at a time.
Momentum is elementwise and so is unaffected by the split.
"""
from __future__ import annotations

import math

import torch
from torch.optim import AdamW

MUON_ROLES = {"matmul"}
ADAMW_ROLES = {"embedding", "ngram_table", "canon", "norm"}
# Roles whose SCALE is a free parameter that nothing else restrains.
DECAY_ROLES = {"embedding", "ngram_table"}

# Quintic coefficients from Jordan et al.; tuned so the iteration converges on
# the singular values of a Frobenius-normalised matrix in five steps.
_NS_ABC = (3.4445, -4.7750, 2.0315)


@torch.no_grad()
def newton_schulz(X: torch.Tensor, steps: int = 5, eps: float = 1e-7) -> torch.Tensor:
    """Orthogonalise a batch of matrices. X is (G, m, n) with m <= n.

    Frobenius-normalised per matrix, so every matrix in the batch converges on
    its own scale rather than the batch's.
    """
    a, b, c = _NS_ABC
    X = X / (X.norm(dim=(1, 2), keepdim=True) + eps)
    for _ in range(steps):
        # Same polynomial, with the scalar terms folded into the GEMM epilogues
        # (swarm muon_ns "fused32"): B = b*A + c*A@A and X = a*X + B@X are one
        # baddbmm each instead of a bmm plus two elementwise passes. Error vs
        # fp64 equal or lower (4.185e-6 -> 4.143e-6 median), ~7% faster.
        A = torch.bmm(X, X.transpose(1, 2))
        B = torch.baddbmm(A, A, A, beta=b, alpha=c)
        X = torch.baddbmm(X, B, X, beta=a)
    return X


class Muon(torch.optim.Optimizer):
    """Momentum, then an orthogonalised update, on 2-D matmul weights only."""

    def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True, ns_steps=5,
                 weight_decay=0.1, batched=True, ns_dtype=torch.float32,
                 rms_scale=True):
        super().__init__(list(params), dict(lr=lr, momentum=momentum,
                                            nesterov=nesterov, ns_steps=ns_steps,
                                            weight_decay=weight_decay))
        self.batched = batched
        self.ns_dtype = ns_dtype
        self.rms_scale = rms_scale
        self._plan: list | None = None

    @staticmethod
    def _units(group):
        """(parameter, row range or None, matrix shape) for every logical matrix."""
        out = []
        for p in group["params"]:
            if p.grad is None:
                continue
            blocks = getattr(p, "_boris_blocks", None)
            if blocks is None:
                out.append((p, None, tuple(p.shape)))
            else:
                for a, b in blocks:
                    out.append((p, (int(a), int(b)), (int(b) - int(a), p.shape[1])))
        return out

    def _build_plan(self, group):
        """Bucket logical matrices by canonical shape once; shapes never change.

        Orthogonalisation commutes with transpose, so a (2368, 512) matrix and a
        (512, 2368) one share a bucket -- the wide orientation is canonical and
        the narrow ones are transposed in and back out.
        """
        buckets: dict[tuple, list] = {}
        for p, rng, shape in self._units(group):
            m, n = shape
            buckets.setdefault((min(m, n), max(m, n)), []).append((p, rng, shape, m > n))
        return list(buckets.items())

    @torch.no_grad()
    def step(self, closure=None):
        loss = closure() if closure is not None else None
        for group in self.param_groups:
            lr, mom = group["lr"], group["momentum"]
            nesterov, wd = group["nesterov"], group["weight_decay"]
            steps = group["ns_steps"]

            # momentum, in place, on every parameter
            bufs = []
            for p in group["params"]:
                if p.grad is None:
                    continue
                st = self.state[p]
                if "momentum_buffer" not in st:
                    st["momentum_buffer"] = torch.zeros_like(p)
                buf = st["momentum_buffer"]
                buf.mul_(mom).add_(p.grad)
                g = p.grad.add(buf, alpha=mom) if nesterov else buf
                bufs.append((p, g))

            gmap = {id(p): g for p, g in bufs}

            if not self.batched:
                for p, rng, shape in self._units(group):
                    g = gmap[id(p)]
                    g = g if rng is None else g[rng[0]:rng[1]]
                    tr = shape[0] > shape[1]
                    x = (g.t() if tr else g).to(self.ns_dtype)[None]
                    o = newton_schulz(x, steps)[0]
                    self._apply(p, rng, shape, o.t() if tr else o, lr, wd)
                continue

            if self._plan is None:
                self._plan = self._build_plan(group)
            for _key, members in self._plan:
                stack = torch.stack([
                    ((gmap[id(p)] if rng is None else gmap[id(p)][rng[0]:rng[1]]).t()
                     if tr else
                     (gmap[id(p)] if rng is None else gmap[id(p)][rng[0]:rng[1]])
                     ).to(self.ns_dtype)
                    for p, rng, shape, tr in members])
                out = newton_schulz(stack, steps)
                for i, (p, rng, shape, tr) in enumerate(members):
                    o = out[i]
                    self._apply(p, rng, shape, o.t() if tr else o, lr, wd)
        return loss

    def _apply(self, p, rng, shape, o, lr, wd):
        # 0.2*sqrt(max(m,n)) makes the update RMS the same for a 512x512 matrix
        # and a 512x2368 one, so a single LR is meaningful across a 4.63x FFN.
        # Computed from the LOGICAL matrix shape, not the storage tensor.
        s = 0.2 * math.sqrt(max(shape)) if self.rms_scale else 1.0
        t = p.data if rng is None else p.data[rng[0]:rng[1]]
        if wd:
            t.mul_(1 - lr * wd)
        t.add_(o.to(t.dtype), alpha=-lr * s)


# --------------------------------------------------------------------------- #
# The split
# --------------------------------------------------------------------------- #
def _role(p, name: str) -> str:
    r = getattr(p, "_boris_role", None)
    if r is None:
        raise AssertionError(
            f"parameter '{name}' has no _boris_role tag. Every parameter must be "
            f"assigned to an optimizer explicitly at construction -- an untagged "
            f"parameter is a misconfiguration, not a default.")
    return r


def build_optimizers(model, cfg, log=print):
    muon_params, adamw_params = [], []
    adamw_roles = []
    rows = []
    for name, p in model.named_parameters():
        if not p.requires_grad:
            continue
        r = _role(p, name)
        if r in MUON_ROLES:
            muon_params.append(p); which = "muon"
        elif r in ADAMW_ROLES:
            adamw_params.append(p); adamw_roles.append((p, r)); which = "adamw"
        else:
            raise AssertionError(f"parameter '{name}' has unknown role '{r}'")
        rows.append((name, tuple(p.shape), p.numel(), r, which))

    o = cfg.optim
    muon = Muon(muon_params, lr=o.muon_lr, momentum=o.muon_momentum,
                nesterov=o.muon_nesterov, ns_steps=o.muon_ns_steps,
                weight_decay=o.muon_weight_decay, batched=o.muon_batched,
                ns_dtype=getattr(torch, o.muon_ns_dtype), rms_scale=o.muon_rms_scale)
    # 2026-09-17: AdamW used ONE weight_decay for every role, and it was 0.0.
    # Muon decays its matmuls at 0.1 and they are flat to within 1% over 9.3B
    # tokens; the undecayed AdamW tensors are not. Measured 20.8B -> 30.1B:
    #     embed.weight  1879 -> 2275  (+21.1%)   wd 0.0
    #     ngram.table   2747 -> 3326  (+21.1%)   wd 0.0
    #     qkv_proj       167 ->  168  ( +0.4%)   wd 0.1
    #     down_proj      327 ->  324  ( -1.0%)   wd 0.1
    # Nothing restrains the two largest tensors, so they inflate, norm_f shrinks
    # to hold logit scale roughly constant (8.05 -> 7.12), and 66% of the total
    # gradient power ends up in that one 512-vector fighting the embedding.
    # AdamW's step is ~lr regardless of scale, so a growing ||w|| means the
    # RELATIVE update lr/||w|| decays -- embedding directions calcify. Row norms
    # grow ~+1.07 per 5B (linear), which extrapolates to ~4x by 200B.
    # Decay the two big tensors; leave norms and canon alone, as is conventional
    # -- norm gains are a scale, not a representation, and canon is near-flat
    # already (+0.8% to +5.7%).
    #
    # 2026-10-07: canon is no longer near-flat. At the corrected Muon lr (0929) it
    # grew ~+20% per 10B with nothing restraining it, and the norm1 gains shrank
    # to compensate. It gets its own group (adamw_canon_weight_decay); the group
    # ORDER is [decay, norms, canon] and checkpoint._migrate_adamw_groups remaps
    # older 1- and 2-group states onto it by parameter identity.
    decay = [p for p, r in adamw_roles if r in DECAY_ROLES]
    norms = [p for p, r in adamw_roles if r not in DECAY_ROLES and r != "canon"]
    canon = [p for p, r in adamw_roles if r == "canon"]
    adamw = AdamW([{"params": decay, "weight_decay": o.adamw_weight_decay},
                   {"params": norms, "weight_decay": o.adamw_norm_weight_decay},
                   {"params": canon, "weight_decay": o.adamw_canon_weight_decay}],
                  lr=o.adamw_lr, betas=tuple(o.adamw_betas),
                  eps=o.adamw_eps, fused=torch.cuda.is_available())
    n = lambda ps: f"{len(ps)} tensors ({sum(p.numel() for p in ps):,} params)"
    log(f"\n  adamw decay groups: {n(decay)} at wd {o.adamw_weight_decay}, "
        f"{n(norms)} at wd {o.adamw_norm_weight_decay}, "
        f"canon {n(canon)} at wd {o.adamw_canon_weight_decay}")

    _report_and_assert(rows, muon_params, adamw_params, model, cfg, log)
    return muon, adamw


def _report_and_assert(rows, muon_params, adamw_params, model, cfg, log):
    by_role: dict[str, list] = {}
    for name, shape, n, role, which in rows:
        by_role.setdefault(role, []).append((name, shape, n, which))

    log("\n  parameter groups")
    log("  " + "-" * 74)
    log(f"  {'role':<13}{'optimizer':<10}{'tensors':>9}{'parameters':>15}  example shape")
    total = 0
    for role in ("matmul", "embedding", "ngram_table", "canon", "norm"):
        if role not in by_role:
            continue
        e = by_role[role]
        n = sum(x[2] for x in e)
        total += n
        log(f"  {role:<13}{e[0][3]:<10}{len(e):>9}{n:>15,}  {e[0][1]}")
    log("  " + "-" * 74)
    n_muon = sum(p.numel() for p in muon_params)
    n_adamw = sum(p.numel() for p in adamw_params)
    n_mat = sum(len(getattr(p, "_boris_blocks", (None,))) for p in muon_params)
    log(f"  {'MUON':<23}{len(muon_params):>9}{n_muon:>15,}  "
        f"{n_mat} logical matrices")
    log(f"  {'ADAMW':<23}{len(adamw_params):>9}{n_adamw:>15,}")
    log(f"  {'TOTAL':<23}{len(rows):>9}{total:>15,}")

    ids_m = {id(p) for p in muon_params}
    ids_a = {id(p) for p in adamw_params}
    assert not (ids_m & ids_a), "a parameter landed in both optimizers"
    all_ids = {id(p) for p in model.parameters() if p.requires_grad}
    assert ids_m | ids_a == all_ids, (
        f"{len(all_ids - (ids_m | ids_a))} trainable parameters are in no optimizer")

    for p in muon_params:
        assert p.ndim == 2 and min(p.shape) > 3, (
            f"a non-matrix of shape {tuple(p.shape)} reached Muon")
        blocks = getattr(p, "_boris_blocks", None)
        if blocks is not None:
            assert blocks[0][0] == 0 and blocks[-1][1] == p.shape[0], (
                f"_boris_blocks {blocks} do not tile the {tuple(p.shape)} tensor")
            for (a1, b1), (a2, _) in zip(blocks, blocks[1:]):
                assert b1 == a2, f"_boris_blocks {blocks} overlap or leave a gap"
    b = cfg.param_budget()
    assert n_muon == b["matmul"], (
        f"Muon holds {n_muon:,} parameters, the budget says {b['matmul']:,}")
    assert total == b["total"], (
        f"model has {total:,} parameters, the budget says {b['total']:,}")
    cfg.assert_budget()
    log(f"  budget assertions pass; fingerprint {cfg.fingerprint()}\n")


def set_lr(muon, adamw, cfg, scale: float) -> tuple[float, float]:
    ml = cfg.optim.muon_lr * scale
    al = cfg.optim.adamw_lr * scale
    for g in muon.param_groups:
        g["lr"] = ml
    for g in adamw.param_groups:
        g["lr"] = al
    return ml, al