1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
|
"""Checkpoint save, load, rotate, and the weights-only export for eval.
Three kinds, and they are not interchangeable:
rolling Every 45 minutes, two kept, rotated. This is what a run actually
resumes from. The 50B milestones are 25+ days apart; a run that is
expected to be interrupted cannot use them as its resume point.
full Every 50B tokens, permanent. Weights, Muon momentum buffers, AdamW
moments, the loss scale, the packer position and every RNG stream.
The end-of-stable one is kept forever and never rotated: variants
branch from it as a tree, and re-warming from a decayed model costs
a loss spike and a slow recovery.
export Every 10B tokens, weights only, fp16, ~287MB. Written to a watched
directory with a manifest line; eval.py on the second GPU picks it
up. Training never waits for eval and eval never touches GPU 0.
Writes are atomic (tmp file, fsync, rename), so a checkpoint interrupted mid-
write cannot leave a torn `latest.pt` behind. The GPU->CPU copy is synchronous
and takes about a second; the disk write happens on a background thread, so the
training loop stalls for the copy and not for the SATA SSD.
"""
from __future__ import annotations
import json
import os
import random
import re
import threading
import time
from pathlib import Path
import numpy as np
import torch
from config import Config
_writer_lock = threading.Lock()
def _atomic_write(blob, path: Path):
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_suffix(path.suffix + ".tmp")
with open(tmp, "wb") as f:
torch.save(blob, f)
f.flush()
os.fsync(f.fileno())
os.replace(tmp, path)
def _cpu_state(sd: dict) -> dict:
return {k: (v.detach().to("cpu", copy=True) if torch.is_tensor(v) else v)
for k, v in sd.items()}
def _deep_cpu(obj):
if torch.is_tensor(obj):
return obj.detach().to("cpu", copy=True)
if isinstance(obj, dict):
return {k: _deep_cpu(v) for k, v in obj.items()}
if isinstance(obj, (list, tuple)):
return type(obj)(_deep_cpu(v) for v in obj)
return obj
def save_full(path: Path, *, model, muon, adamw, scaler, packer, step: int,
tokens_seen: int, cfg: Config, ngram_fingerprint: str,
best_val: float = float("inf"), extra: dict | None = None,
background: bool = True, rotate_pattern: str | None = None,
rotate_keep: int | None = None):
raw = getattr(model, "_orig_mod", model)
blob = {
"format": 1,
"model": _cpu_state(raw.state_dict()),
"muon": _deep_cpu(muon.state_dict()),
"adamw": _deep_cpu(adamw.state_dict()),
"scaler": scaler.state_dict() if scaler is not None else None,
"packer": packer.state_dict() if packer is not None else None,
"step": int(step),
"tokens_seen": int(tokens_seen),
"best_val": float(best_val),
"config": cfg.to_json(),
"fingerprint": cfg.fingerprint(),
"ngram_fingerprint": ngram_fingerprint,
"rng": {"python": random.getstate(),
"numpy": np.random.get_state(),
"torch": torch.get_rng_state(),
"cuda": torch.cuda.get_rng_state_all() if torch.cuda.is_available() else None},
"extra": extra or {},
"wall": time.time(),
}
def _write():
with _writer_lock:
_atomic_write(blob, path)
if rotate_pattern is not None and rotate_keep is not None:
rotate(path.parent, rotate_pattern, rotate_keep)
if background:
t = threading.Thread(target=_write, daemon=False)
t.start()
return t
_write()
return None
def save_export(path: Path, *, model, cfg: Config, step: int, tokens_seen: int,
manifest: Path, ngram_fingerprint: str, background: bool = True):
"""Weights only, fp16, no optimizer state -- the eval handoff."""
raw = getattr(model, "_orig_mod", model)
sd = {k: v.detach().to("cpu", torch.float16, copy=True)
for k, v in raw.state_dict().items()}
blob = {"format": 1, "model": sd, "config": cfg.to_json(),
"step": int(step), "tokens_seen": int(tokens_seen),
"fingerprint": cfg.fingerprint(),
"ngram_fingerprint": ngram_fingerprint, "wall": time.time()}
def _write():
with _writer_lock:
_atomic_write(blob, path)
manifest.parent.mkdir(parents=True, exist_ok=True)
line = json.dumps({"path": str(path), "step": int(step),
"tokens_seen": int(tokens_seen),
"wall": time.time()})
with open(manifest, "a") as f:
f.write(line + "\n")
f.flush()
os.fsync(f.fileno())
if background:
t = threading.Thread(target=_write, daemon=False)
t.start()
return t
_write()
return None
def _migrate_adamw_groups(blob, model, adamw):
"""Re-key an older AdamW state onto the current param_groups by identity.
state_dict indexes parameters by position across all groups, so ANY change in
grouping -- 1 group (before 2026-09-17), 2 groups [embedding+ngram | norms+canon]
(before 2026-10-07), now 3 groups [embedding+ngram | norms | canon] -- moves
indices. load_state_dict refuses a group-count mismatch, and if counts lined up
it would pair a parameter with another one's exp_avg/exp_avg_sq. Rebuild the
saved order from the layout that wrote it and permute by parameter identity;
every moved moment tensor must match its parameter's shape.
"""
from optim import _role, ADAMW_ROLES
saved = blob["adamw"]
live = [p for g in adamw.param_groups for p in g["params"]]
named = [(p, _role(p, n)) for n, p in model.named_parameters()
if p.requires_grad and _role(p, n) in ADAMW_ROLES]
layouts = {1: [{"embedding", "ngram_table", "norm", "canon"}],
2: [{"embedding", "ngram_table"}, {"norm", "canon"}],
3: [{"embedding", "ngram_table"}, {"norm"}, {"canon"}]}
k = len(saved["param_groups"])
sizes = [len(g["params"]) for g in saved["param_groups"]]
old_order = [p for roles in layouts[k] for p, r in named if r in roles]
if [sum(r in roles for _, r in named) for roles in layouts[k]] != sizes:
raise AssertionError(f"adamw migration: saved group sizes {sizes} do not "
f"match the {k}-group layout -- refusing to guess")
if [id(p) for p in old_order] == [id(p) for p in live] and k == len(adamw.param_groups):
return saved
if len(old_order) != len(live):
raise AssertionError(
f"adamw migration: {len(old_order)} saved tensors vs "
f"{len(live)} live tensors -- refusing to guess")
pos = {id(p): i for i, p in enumerate(old_order)}
perm = [pos[id(p)] for p in live]
old_state = saved["state"]
new_state = {j: old_state[o] for j, o in enumerate(perm) if o in old_state}
for j, st in new_state.items():
for key in ("exp_avg", "exp_avg_sq"):
if key in st:
assert st[key].shape == live[j].shape, (
f"adamw migration: {key} {tuple(st[key].shape)} landed on a "
f"{tuple(live[j].shape)} parameter")
tmpl = {kk: v for kk, v in saved["param_groups"][0].items() if kk != "params"}
groups, j = [], 0
for g in adamw.param_groups:
n = len(g["params"])
groups.append({**tmpl, "weight_decay": g["weight_decay"],
"params": list(range(j, j + n))})
j += n
return {"state": new_state, "param_groups": groups}
def load_full(path: Path, *, model=None, muon=None, adamw=None, scaler=None,
packer=None, map_location="cpu", expect_fingerprint: str | None = None,
expect_ngram: str | None = None, strict: bool = True) -> dict:
blob = torch.load(Path(path), map_location=map_location, weights_only=False)
if expect_fingerprint is not None and blob.get("fingerprint") != expect_fingerprint:
raise SystemExit(
f"{path} was written by a run whose config fingerprint is "
f"{blob.get('fingerprint')}, this run is {expect_fingerprint}. The "
f"model, optimizer or schedule has changed; resuming would silently "
f"train a different thing. Pass --force-config if that is intended.")
expected_ngrams = ((expect_ngram,) if isinstance(expect_ngram, str)
else tuple(expect_ngram or ()))
if expected_ngrams and blob.get("ngram_fingerprint") not in (None, *expected_ngrams):
raise SystemExit(
f"{path} was trained against ngram_vocab.bin "
f"{blob.get('ngram_fingerprint')}, this run loaded {expected_ngrams}. "
f"Every row index in the table would mean a different n-gram.")
if model is not None:
raw = getattr(model, "_orig_mod", model)
raw.load_state_dict(blob["model"], strict=strict)
if muon is not None and blob.get("muon"):
muon.load_state_dict(blob["muon"])
muon._plan = None # rebuilt against the live Parameters
if adamw is not None and blob.get("adamw"):
adamw.load_state_dict(_migrate_adamw_groups(blob, raw, adamw))
if scaler is not None and blob.get("scaler"):
scaler.load_state_dict(blob["scaler"])
if packer is not None and blob.get("packer"):
packer.load_state_dict(blob["packer"])
r = blob.get("rng") or {}
if r:
random.setstate(r["python"])
np.random.set_state(r["numpy"])
torch.set_rng_state(r["torch"].cpu() if torch.is_tensor(r["torch"]) else r["torch"])
if r.get("cuda") is not None and torch.cuda.is_available():
try:
torch.cuda.set_rng_state_all([s.cpu() for s in r["cuda"]])
except Exception:
pass
return blob
def rotate(directory: Path, pattern: str, keep: int) -> None:
d = Path(directory)
if not d.exists():
return
rx = re.compile(pattern)
files = sorted((p for p in d.iterdir() if rx.match(p.name)),
key=lambda p: p.stat().st_mtime)
for p in files[:-keep] if keep > 0 else files:
try:
p.unlink()
except OSError:
pass
def latest_rolling(directory: Path) -> Path | None:
d = Path(directory)
if not d.exists():
return None
files = [p for p in d.iterdir() if re.match(r"rolling_\d+\.pt$", p.name)]
if not files:
return None
return max(files, key=lambda p: p.stat().st_mtime)
|