"""Checkpoint save, load, rotate, and the weights-only export for eval. Three kinds, and they are not interchangeable: rolling Every 45 minutes, two kept, rotated. This is what a run actually resumes from. The 50B milestones are 25+ days apart; a run that is expected to be interrupted cannot use them as its resume point. full Every 50B tokens, permanent. Weights, Muon momentum buffers, AdamW moments, the loss scale, the packer position and every RNG stream. The end-of-stable one is kept forever and never rotated: variants branch from it as a tree, and re-warming from a decayed model costs a loss spike and a slow recovery. export Every 10B tokens, weights only, fp16, ~287MB. Written to a watched directory with a manifest line; eval.py on the second GPU picks it up. Training never waits for eval and eval never touches GPU 0. Writes are atomic (tmp file, fsync, rename), so a checkpoint interrupted mid- write cannot leave a torn `latest.pt` behind. The GPU->CPU copy is synchronous and takes about a second; the disk write happens on a background thread, so the training loop stalls for the copy and not for the SATA SSD. """ from __future__ import annotations import json import os import random import re import threading import time from pathlib import Path import numpy as np import torch from config import Config _writer_lock = threading.Lock() def _atomic_write(blob, path: Path): path = Path(path) path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_suffix(path.suffix + ".tmp") with open(tmp, "wb") as f: torch.save(blob, f) f.flush() os.fsync(f.fileno()) os.replace(tmp, path) def _cpu_state(sd: dict) -> dict: return {k: (v.detach().to("cpu", copy=True) if torch.is_tensor(v) else v) for k, v in sd.items()} def _deep_cpu(obj): if torch.is_tensor(obj): return obj.detach().to("cpu", copy=True) if isinstance(obj, dict): return {k: _deep_cpu(v) for k, v in obj.items()} if isinstance(obj, (list, tuple)): return type(obj)(_deep_cpu(v) for v in obj) return obj def save_full(path: Path, *, model, muon, adamw, scaler, packer, step: int, tokens_seen: int, cfg: Config, ngram_fingerprint: str, best_val: float = float("inf"), extra: dict | None = None, background: bool = True, rotate_pattern: str | None = None, rotate_keep: int | None = None): raw = getattr(model, "_orig_mod", model) blob = { "format": 1, "model": _cpu_state(raw.state_dict()), "muon": _deep_cpu(muon.state_dict()), "adamw": _deep_cpu(adamw.state_dict()), "scaler": scaler.state_dict() if scaler is not None else None, "packer": packer.state_dict() if packer is not None else None, "step": int(step), "tokens_seen": int(tokens_seen), "best_val": float(best_val), "config": cfg.to_json(), "fingerprint": cfg.fingerprint(), "ngram_fingerprint": ngram_fingerprint, "rng": {"python": random.getstate(), "numpy": np.random.get_state(), "torch": torch.get_rng_state(), "cuda": torch.cuda.get_rng_state_all() if torch.cuda.is_available() else None}, "extra": extra or {}, "wall": time.time(), } def _write(): with _writer_lock: _atomic_write(blob, path) if rotate_pattern is not None and rotate_keep is not None: rotate(path.parent, rotate_pattern, rotate_keep) if background: t = threading.Thread(target=_write, daemon=False) t.start() return t _write() return None def save_export(path: Path, *, model, cfg: Config, step: int, tokens_seen: int, manifest: Path, ngram_fingerprint: str, background: bool = True): """Weights only, fp16, no optimizer state -- the eval handoff.""" raw = getattr(model, "_orig_mod", model) sd = {k: v.detach().to("cpu", torch.float16, copy=True) for k, v in raw.state_dict().items()} blob = {"format": 1, "model": sd, "config": cfg.to_json(), "step": int(step), "tokens_seen": int(tokens_seen), "fingerprint": cfg.fingerprint(), "ngram_fingerprint": ngram_fingerprint, "wall": time.time()} def _write(): with _writer_lock: _atomic_write(blob, path) manifest.parent.mkdir(parents=True, exist_ok=True) line = json.dumps({"path": str(path), "step": int(step), "tokens_seen": int(tokens_seen), "wall": time.time()}) with open(manifest, "a") as f: f.write(line + "\n") f.flush() os.fsync(f.fileno()) if background: t = threading.Thread(target=_write, daemon=False) t.start() return t _write() return None def _migrate_adamw_groups(blob, model, adamw): """Re-key an older AdamW state onto the current param_groups by identity. state_dict indexes parameters by position across all groups, so ANY change in grouping -- 1 group (before 2026-09-17), 2 groups [embedding+ngram | norms+canon] (before 2026-10-07), now 3 groups [embedding+ngram | norms | canon] -- moves indices. load_state_dict refuses a group-count mismatch, and if counts lined up it would pair a parameter with another one's exp_avg/exp_avg_sq. Rebuild the saved order from the layout that wrote it and permute by parameter identity; every moved moment tensor must match its parameter's shape. """ from optim import _role, ADAMW_ROLES saved = blob["adamw"] live = [p for g in adamw.param_groups for p in g["params"]] named = [(p, _role(p, n)) for n, p in model.named_parameters() if p.requires_grad and _role(p, n) in ADAMW_ROLES] layouts = {1: [{"embedding", "ngram_table", "norm", "canon"}], 2: [{"embedding", "ngram_table"}, {"norm", "canon"}], 3: [{"embedding", "ngram_table"}, {"norm"}, {"canon"}]} k = len(saved["param_groups"]) sizes = [len(g["params"]) for g in saved["param_groups"]] old_order = [p for roles in layouts[k] for p, r in named if r in roles] if [sum(r in roles for _, r in named) for roles in layouts[k]] != sizes: raise AssertionError(f"adamw migration: saved group sizes {sizes} do not " f"match the {k}-group layout -- refusing to guess") if [id(p) for p in old_order] == [id(p) for p in live] and k == len(adamw.param_groups): return saved if len(old_order) != len(live): raise AssertionError( f"adamw migration: {len(old_order)} saved tensors vs " f"{len(live)} live tensors -- refusing to guess") pos = {id(p): i for i, p in enumerate(old_order)} perm = [pos[id(p)] for p in live] old_state = saved["state"] new_state = {j: old_state[o] for j, o in enumerate(perm) if o in old_state} for j, st in new_state.items(): for key in ("exp_avg", "exp_avg_sq"): if key in st: assert st[key].shape == live[j].shape, ( f"adamw migration: {key} {tuple(st[key].shape)} landed on a " f"{tuple(live[j].shape)} parameter") tmpl = {kk: v for kk, v in saved["param_groups"][0].items() if kk != "params"} groups, j = [], 0 for g in adamw.param_groups: n = len(g["params"]) groups.append({**tmpl, "weight_decay": g["weight_decay"], "params": list(range(j, j + n))}) j += n return {"state": new_state, "param_groups": groups} def load_full(path: Path, *, model=None, muon=None, adamw=None, scaler=None, packer=None, map_location="cpu", expect_fingerprint: str | None = None, expect_ngram: str | None = None, strict: bool = True) -> dict: blob = torch.load(Path(path), map_location=map_location, weights_only=False) if expect_fingerprint is not None and blob.get("fingerprint") != expect_fingerprint: raise SystemExit( f"{path} was written by a run whose config fingerprint is " f"{blob.get('fingerprint')}, this run is {expect_fingerprint}. The " f"model, optimizer or schedule has changed; resuming would silently " f"train a different thing. Pass --force-config if that is intended.") expected_ngrams = ((expect_ngram,) if isinstance(expect_ngram, str) else tuple(expect_ngram or ())) if expected_ngrams and blob.get("ngram_fingerprint") not in (None, *expected_ngrams): raise SystemExit( f"{path} was trained against ngram_vocab.bin " f"{blob.get('ngram_fingerprint')}, this run loaded {expected_ngrams}. " f"Every row index in the table would mean a different n-gram.") if model is not None: raw = getattr(model, "_orig_mod", model) raw.load_state_dict(blob["model"], strict=strict) if muon is not None and blob.get("muon"): muon.load_state_dict(blob["muon"]) muon._plan = None # rebuilt against the live Parameters if adamw is not None and blob.get("adamw"): adamw.load_state_dict(_migrate_adamw_groups(blob, raw, adamw)) if scaler is not None and blob.get("scaler"): scaler.load_state_dict(blob["scaler"]) if packer is not None and blob.get("packer"): packer.load_state_dict(blob["packer"]) r = blob.get("rng") or {} if r: random.setstate(r["python"]) np.random.set_state(r["numpy"]) torch.set_rng_state(r["torch"].cpu() if torch.is_tensor(r["torch"]) else r["torch"]) if r.get("cuda") is not None and torch.cuda.is_available(): try: torch.cuda.set_rng_state_all([s.cpu() for s in r["cuda"]]) except Exception: pass return blob def rotate(directory: Path, pattern: str, keep: int) -> None: d = Path(directory) if not d.exists(): return rx = re.compile(pattern) files = sorted((p for p in d.iterdir() if rx.match(p.name)), key=lambda p: p.stat().st_mtime) for p in files[:-keep] if keep > 0 else files: try: p.unlink() except OSError: pass def latest_rolling(directory: Path) -> Path | None: d = Path(directory) if not d.exists(): return None files = [p for p in d.iterdir() if re.match(r"rolling_\d+\.pt$", p.name)] if not files: return None return max(files, key=lambda p: p.stat().st_mtime)