summaryrefslogtreecommitdiff
path: root/checkpoint.py
diff options
context:
space:
mode:
Diffstat (limited to 'checkpoint.py')
-rw-r--r--checkpoint.py254
1 files changed, 254 insertions, 0 deletions
diff --git a/checkpoint.py b/checkpoint.py
new file mode 100644
index 0000000..5da5921
--- /dev/null
+++ b/checkpoint.py
@@ -0,0 +1,254 @@
+"""Checkpoint save, load, rotate, and the weights-only export for eval.
+
+Three kinds, and they are not interchangeable:
+
+ rolling Every 45 minutes, two kept, rotated. This is what a run actually
+ resumes from. The 50B milestones are 25+ days apart; a run that is
+ expected to be interrupted cannot use them as its resume point.
+ full Every 50B tokens, permanent. Weights, Muon momentum buffers, AdamW
+ moments, the loss scale, the packer position and every RNG stream.
+ The end-of-stable one is kept forever and never rotated: variants
+ branch from it as a tree, and re-warming from a decayed model costs
+ a loss spike and a slow recovery.
+ export Every 10B tokens, weights only, fp16, ~287MB. Written to a watched
+ directory with a manifest line; eval.py on the second GPU picks it
+ up. Training never waits for eval and eval never touches GPU 0.
+
+Writes are atomic (tmp file, fsync, rename), so a checkpoint interrupted mid-
+write cannot leave a torn `latest.pt` behind. The GPU->CPU copy is synchronous
+and takes about a second; the disk write happens on a background thread, so the
+training loop stalls for the copy and not for the SATA SSD.
+"""
+from __future__ import annotations
+
+import json
+import os
+import random
+import re
+import threading
+import time
+from pathlib import Path
+
+import numpy as np
+import torch
+
+from config import Config
+
+_writer_lock = threading.Lock()
+
+
+def _atomic_write(blob, path: Path):
+ path = Path(path)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ tmp = path.with_suffix(path.suffix + ".tmp")
+ with open(tmp, "wb") as f:
+ torch.save(blob, f)
+ f.flush()
+ os.fsync(f.fileno())
+ os.replace(tmp, path)
+
+
+def _cpu_state(sd: dict) -> dict:
+ return {k: (v.detach().to("cpu", copy=True) if torch.is_tensor(v) else v)
+ for k, v in sd.items()}
+
+
+def _deep_cpu(obj):
+ if torch.is_tensor(obj):
+ return obj.detach().to("cpu", copy=True)
+ if isinstance(obj, dict):
+ return {k: _deep_cpu(v) for k, v in obj.items()}
+ if isinstance(obj, (list, tuple)):
+ return type(obj)(_deep_cpu(v) for v in obj)
+ return obj
+
+
+def save_full(path: Path, *, model, muon, adamw, scaler, packer, step: int,
+ tokens_seen: int, cfg: Config, ngram_fingerprint: str,
+ best_val: float = float("inf"), extra: dict | None = None,
+ background: bool = True, rotate_pattern: str | None = None,
+ rotate_keep: int | None = None):
+ raw = getattr(model, "_orig_mod", model)
+ blob = {
+ "format": 1,
+ "model": _cpu_state(raw.state_dict()),
+ "muon": _deep_cpu(muon.state_dict()),
+ "adamw": _deep_cpu(adamw.state_dict()),
+ "scaler": scaler.state_dict() if scaler is not None else None,
+ "packer": packer.state_dict() if packer is not None else None,
+ "step": int(step),
+ "tokens_seen": int(tokens_seen),
+ "best_val": float(best_val),
+ "config": cfg.to_json(),
+ "fingerprint": cfg.fingerprint(),
+ "ngram_fingerprint": ngram_fingerprint,
+ "rng": {"python": random.getstate(),
+ "numpy": np.random.get_state(),
+ "torch": torch.get_rng_state(),
+ "cuda": torch.cuda.get_rng_state_all() if torch.cuda.is_available() else None},
+ "extra": extra or {},
+ "wall": time.time(),
+ }
+
+ def _write():
+ with _writer_lock:
+ _atomic_write(blob, path)
+ if rotate_pattern is not None and rotate_keep is not None:
+ rotate(path.parent, rotate_pattern, rotate_keep)
+
+ if background:
+ t = threading.Thread(target=_write, daemon=False)
+ t.start()
+ return t
+ _write()
+ return None
+
+
+def save_export(path: Path, *, model, cfg: Config, step: int, tokens_seen: int,
+ manifest: Path, ngram_fingerprint: str, background: bool = True):
+ """Weights only, fp16, no optimizer state -- the eval handoff."""
+ raw = getattr(model, "_orig_mod", model)
+ sd = {k: v.detach().to("cpu", torch.float16, copy=True)
+ for k, v in raw.state_dict().items()}
+ blob = {"format": 1, "model": sd, "config": cfg.to_json(),
+ "step": int(step), "tokens_seen": int(tokens_seen),
+ "fingerprint": cfg.fingerprint(),
+ "ngram_fingerprint": ngram_fingerprint, "wall": time.time()}
+
+ def _write():
+ with _writer_lock:
+ _atomic_write(blob, path)
+ manifest.parent.mkdir(parents=True, exist_ok=True)
+ line = json.dumps({"path": str(path), "step": int(step),
+ "tokens_seen": int(tokens_seen),
+ "wall": time.time()})
+ with open(manifest, "a") as f:
+ f.write(line + "\n")
+ f.flush()
+ os.fsync(f.fileno())
+
+ if background:
+ t = threading.Thread(target=_write, daemon=False)
+ t.start()
+ return t
+ _write()
+ return None
+
+
+def _migrate_adamw_groups(blob, model, adamw):
+ """Re-key an older AdamW state onto the current param_groups by identity.
+
+ state_dict indexes parameters by position across all groups, so ANY change in
+ grouping -- 1 group (before 2026-09-17), 2 groups [embedding+ngram | norms+canon]
+ (before 2026-10-07), now 3 groups [embedding+ngram | norms | canon] -- moves
+ indices. load_state_dict refuses a group-count mismatch, and if counts lined up
+ it would pair a parameter with another one's exp_avg/exp_avg_sq. Rebuild the
+ saved order from the layout that wrote it and permute by parameter identity;
+ every moved moment tensor must match its parameter's shape.
+ """
+ from optim import _role, ADAMW_ROLES
+ saved = blob["adamw"]
+ live = [p for g in adamw.param_groups for p in g["params"]]
+ named = [(p, _role(p, n)) for n, p in model.named_parameters()
+ if p.requires_grad and _role(p, n) in ADAMW_ROLES]
+ layouts = {1: [{"embedding", "ngram_table", "norm", "canon"}],
+ 2: [{"embedding", "ngram_table"}, {"norm", "canon"}],
+ 3: [{"embedding", "ngram_table"}, {"norm"}, {"canon"}]}
+ k = len(saved["param_groups"])
+ sizes = [len(g["params"]) for g in saved["param_groups"]]
+ old_order = [p for roles in layouts[k] for p, r in named if r in roles]
+ if [sum(r in roles for _, r in named) for roles in layouts[k]] != sizes:
+ raise AssertionError(f"adamw migration: saved group sizes {sizes} do not "
+ f"match the {k}-group layout -- refusing to guess")
+ if [id(p) for p in old_order] == [id(p) for p in live] and k == len(adamw.param_groups):
+ return saved
+ if len(old_order) != len(live):
+ raise AssertionError(
+ f"adamw migration: {len(old_order)} saved tensors vs "
+ f"{len(live)} live tensors -- refusing to guess")
+ pos = {id(p): i for i, p in enumerate(old_order)}
+ perm = [pos[id(p)] for p in live]
+ old_state = saved["state"]
+ new_state = {j: old_state[o] for j, o in enumerate(perm) if o in old_state}
+ for j, st in new_state.items():
+ for key in ("exp_avg", "exp_avg_sq"):
+ if key in st:
+ assert st[key].shape == live[j].shape, (
+ f"adamw migration: {key} {tuple(st[key].shape)} landed on a "
+ f"{tuple(live[j].shape)} parameter")
+ tmpl = {kk: v for kk, v in saved["param_groups"][0].items() if kk != "params"}
+ groups, j = [], 0
+ for g in adamw.param_groups:
+ n = len(g["params"])
+ groups.append({**tmpl, "weight_decay": g["weight_decay"],
+ "params": list(range(j, j + n))})
+ j += n
+ return {"state": new_state, "param_groups": groups}
+
+
+def load_full(path: Path, *, model=None, muon=None, adamw=None, scaler=None,
+ packer=None, map_location="cpu", expect_fingerprint: str | None = None,
+ expect_ngram: str | None = None, strict: bool = True) -> dict:
+ blob = torch.load(Path(path), map_location=map_location, weights_only=False)
+
+ if expect_fingerprint is not None and blob.get("fingerprint") != expect_fingerprint:
+ raise SystemExit(
+ f"{path} was written by a run whose config fingerprint is "
+ f"{blob.get('fingerprint')}, this run is {expect_fingerprint}. The "
+ f"model, optimizer or schedule has changed; resuming would silently "
+ f"train a different thing. Pass --force-config if that is intended.")
+ expected_ngrams = ((expect_ngram,) if isinstance(expect_ngram, str)
+ else tuple(expect_ngram or ()))
+ if expected_ngrams and blob.get("ngram_fingerprint") not in (None, *expected_ngrams):
+ raise SystemExit(
+ f"{path} was trained against ngram_vocab.bin "
+ f"{blob.get('ngram_fingerprint')}, this run loaded {expected_ngrams}. "
+ f"Every row index in the table would mean a different n-gram.")
+
+ if model is not None:
+ raw = getattr(model, "_orig_mod", model)
+ raw.load_state_dict(blob["model"], strict=strict)
+ if muon is not None and blob.get("muon"):
+ muon.load_state_dict(blob["muon"])
+ muon._plan = None # rebuilt against the live Parameters
+ if adamw is not None and blob.get("adamw"):
+ adamw.load_state_dict(_migrate_adamw_groups(blob, raw, adamw))
+ if scaler is not None and blob.get("scaler"):
+ scaler.load_state_dict(blob["scaler"])
+ if packer is not None and blob.get("packer"):
+ packer.load_state_dict(blob["packer"])
+ r = blob.get("rng") or {}
+ if r:
+ random.setstate(r["python"])
+ np.random.set_state(r["numpy"])
+ torch.set_rng_state(r["torch"].cpu() if torch.is_tensor(r["torch"]) else r["torch"])
+ if r.get("cuda") is not None and torch.cuda.is_available():
+ try:
+ torch.cuda.set_rng_state_all([s.cpu() for s in r["cuda"]])
+ except Exception:
+ pass
+ return blob
+
+
+def rotate(directory: Path, pattern: str, keep: int) -> None:
+ d = Path(directory)
+ if not d.exists():
+ return
+ rx = re.compile(pattern)
+ files = sorted((p for p in d.iterdir() if rx.match(p.name)),
+ key=lambda p: p.stat().st_mtime)
+ for p in files[:-keep] if keep > 0 else files:
+ try:
+ p.unlink()
+ except OSError:
+ pass
+
+
+def latest_rolling(directory: Path) -> Path | None:
+ d = Path(directory)
+ if not d.exists():
+ return None
+ files = [p for p in d.iterdir() if re.match(r"rolling_\d+\.pt$", p.name)]
+ if not files:
+ return None
+ return max(files, key=lambda p: p.stat().st_mtime)