diff options
Diffstat (limited to 'checkpoint.py')
| -rw-r--r-- | checkpoint.py | 254 |
1 files changed, 254 insertions, 0 deletions
diff --git a/checkpoint.py b/checkpoint.py new file mode 100644 index 0000000..5da5921 --- /dev/null +++ b/checkpoint.py @@ -0,0 +1,254 @@ +"""Checkpoint save, load, rotate, and the weights-only export for eval. + +Three kinds, and they are not interchangeable: + + rolling Every 45 minutes, two kept, rotated. This is what a run actually + resumes from. The 50B milestones are 25+ days apart; a run that is + expected to be interrupted cannot use them as its resume point. + full Every 50B tokens, permanent. Weights, Muon momentum buffers, AdamW + moments, the loss scale, the packer position and every RNG stream. + The end-of-stable one is kept forever and never rotated: variants + branch from it as a tree, and re-warming from a decayed model costs + a loss spike and a slow recovery. + export Every 10B tokens, weights only, fp16, ~287MB. Written to a watched + directory with a manifest line; eval.py on the second GPU picks it + up. Training never waits for eval and eval never touches GPU 0. + +Writes are atomic (tmp file, fsync, rename), so a checkpoint interrupted mid- +write cannot leave a torn `latest.pt` behind. The GPU->CPU copy is synchronous +and takes about a second; the disk write happens on a background thread, so the +training loop stalls for the copy and not for the SATA SSD. +""" +from __future__ import annotations + +import json +import os +import random +import re +import threading +import time +from pathlib import Path + +import numpy as np +import torch + +from config import Config + +_writer_lock = threading.Lock() + + +def _atomic_write(blob, path: Path): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(path.suffix + ".tmp") + with open(tmp, "wb") as f: + torch.save(blob, f) + f.flush() + os.fsync(f.fileno()) + os.replace(tmp, path) + + +def _cpu_state(sd: dict) -> dict: + return {k: (v.detach().to("cpu", copy=True) if torch.is_tensor(v) else v) + for k, v in sd.items()} + + +def _deep_cpu(obj): + if torch.is_tensor(obj): + return obj.detach().to("cpu", copy=True) + if isinstance(obj, dict): + return {k: _deep_cpu(v) for k, v in obj.items()} + if isinstance(obj, (list, tuple)): + return type(obj)(_deep_cpu(v) for v in obj) + return obj + + +def save_full(path: Path, *, model, muon, adamw, scaler, packer, step: int, + tokens_seen: int, cfg: Config, ngram_fingerprint: str, + best_val: float = float("inf"), extra: dict | None = None, + background: bool = True, rotate_pattern: str | None = None, + rotate_keep: int | None = None): + raw = getattr(model, "_orig_mod", model) + blob = { + "format": 1, + "model": _cpu_state(raw.state_dict()), + "muon": _deep_cpu(muon.state_dict()), + "adamw": _deep_cpu(adamw.state_dict()), + "scaler": scaler.state_dict() if scaler is not None else None, + "packer": packer.state_dict() if packer is not None else None, + "step": int(step), + "tokens_seen": int(tokens_seen), + "best_val": float(best_val), + "config": cfg.to_json(), + "fingerprint": cfg.fingerprint(), + "ngram_fingerprint": ngram_fingerprint, + "rng": {"python": random.getstate(), + "numpy": np.random.get_state(), + "torch": torch.get_rng_state(), + "cuda": torch.cuda.get_rng_state_all() if torch.cuda.is_available() else None}, + "extra": extra or {}, + "wall": time.time(), + } + + def _write(): + with _writer_lock: + _atomic_write(blob, path) + if rotate_pattern is not None and rotate_keep is not None: + rotate(path.parent, rotate_pattern, rotate_keep) + + if background: + t = threading.Thread(target=_write, daemon=False) + t.start() + return t + _write() + return None + + +def save_export(path: Path, *, model, cfg: Config, step: int, tokens_seen: int, + manifest: Path, ngram_fingerprint: str, background: bool = True): + """Weights only, fp16, no optimizer state -- the eval handoff.""" + raw = getattr(model, "_orig_mod", model) + sd = {k: v.detach().to("cpu", torch.float16, copy=True) + for k, v in raw.state_dict().items()} + blob = {"format": 1, "model": sd, "config": cfg.to_json(), + "step": int(step), "tokens_seen": int(tokens_seen), + "fingerprint": cfg.fingerprint(), + "ngram_fingerprint": ngram_fingerprint, "wall": time.time()} + + def _write(): + with _writer_lock: + _atomic_write(blob, path) + manifest.parent.mkdir(parents=True, exist_ok=True) + line = json.dumps({"path": str(path), "step": int(step), + "tokens_seen": int(tokens_seen), + "wall": time.time()}) + with open(manifest, "a") as f: + f.write(line + "\n") + f.flush() + os.fsync(f.fileno()) + + if background: + t = threading.Thread(target=_write, daemon=False) + t.start() + return t + _write() + return None + + +def _migrate_adamw_groups(blob, model, adamw): + """Re-key an older AdamW state onto the current param_groups by identity. + + state_dict indexes parameters by position across all groups, so ANY change in + grouping -- 1 group (before 2026-09-17), 2 groups [embedding+ngram | norms+canon] + (before 2026-10-07), now 3 groups [embedding+ngram | norms | canon] -- moves + indices. load_state_dict refuses a group-count mismatch, and if counts lined up + it would pair a parameter with another one's exp_avg/exp_avg_sq. Rebuild the + saved order from the layout that wrote it and permute by parameter identity; + every moved moment tensor must match its parameter's shape. + """ + from optim import _role, ADAMW_ROLES + saved = blob["adamw"] + live = [p for g in adamw.param_groups for p in g["params"]] + named = [(p, _role(p, n)) for n, p in model.named_parameters() + if p.requires_grad and _role(p, n) in ADAMW_ROLES] + layouts = {1: [{"embedding", "ngram_table", "norm", "canon"}], + 2: [{"embedding", "ngram_table"}, {"norm", "canon"}], + 3: [{"embedding", "ngram_table"}, {"norm"}, {"canon"}]} + k = len(saved["param_groups"]) + sizes = [len(g["params"]) for g in saved["param_groups"]] + old_order = [p for roles in layouts[k] for p, r in named if r in roles] + if [sum(r in roles for _, r in named) for roles in layouts[k]] != sizes: + raise AssertionError(f"adamw migration: saved group sizes {sizes} do not " + f"match the {k}-group layout -- refusing to guess") + if [id(p) for p in old_order] == [id(p) for p in live] and k == len(adamw.param_groups): + return saved + if len(old_order) != len(live): + raise AssertionError( + f"adamw migration: {len(old_order)} saved tensors vs " + f"{len(live)} live tensors -- refusing to guess") + pos = {id(p): i for i, p in enumerate(old_order)} + perm = [pos[id(p)] for p in live] + old_state = saved["state"] + new_state = {j: old_state[o] for j, o in enumerate(perm) if o in old_state} + for j, st in new_state.items(): + for key in ("exp_avg", "exp_avg_sq"): + if key in st: + assert st[key].shape == live[j].shape, ( + f"adamw migration: {key} {tuple(st[key].shape)} landed on a " + f"{tuple(live[j].shape)} parameter") + tmpl = {kk: v for kk, v in saved["param_groups"][0].items() if kk != "params"} + groups, j = [], 0 + for g in adamw.param_groups: + n = len(g["params"]) + groups.append({**tmpl, "weight_decay": g["weight_decay"], + "params": list(range(j, j + n))}) + j += n + return {"state": new_state, "param_groups": groups} + + +def load_full(path: Path, *, model=None, muon=None, adamw=None, scaler=None, + packer=None, map_location="cpu", expect_fingerprint: str | None = None, + expect_ngram: str | None = None, strict: bool = True) -> dict: + blob = torch.load(Path(path), map_location=map_location, weights_only=False) + + if expect_fingerprint is not None and blob.get("fingerprint") != expect_fingerprint: + raise SystemExit( + f"{path} was written by a run whose config fingerprint is " + f"{blob.get('fingerprint')}, this run is {expect_fingerprint}. The " + f"model, optimizer or schedule has changed; resuming would silently " + f"train a different thing. Pass --force-config if that is intended.") + expected_ngrams = ((expect_ngram,) if isinstance(expect_ngram, str) + else tuple(expect_ngram or ())) + if expected_ngrams and blob.get("ngram_fingerprint") not in (None, *expected_ngrams): + raise SystemExit( + f"{path} was trained against ngram_vocab.bin " + f"{blob.get('ngram_fingerprint')}, this run loaded {expected_ngrams}. " + f"Every row index in the table would mean a different n-gram.") + + if model is not None: + raw = getattr(model, "_orig_mod", model) + raw.load_state_dict(blob["model"], strict=strict) + if muon is not None and blob.get("muon"): + muon.load_state_dict(blob["muon"]) + muon._plan = None # rebuilt against the live Parameters + if adamw is not None and blob.get("adamw"): + adamw.load_state_dict(_migrate_adamw_groups(blob, raw, adamw)) + if scaler is not None and blob.get("scaler"): + scaler.load_state_dict(blob["scaler"]) + if packer is not None and blob.get("packer"): + packer.load_state_dict(blob["packer"]) + r = blob.get("rng") or {} + if r: + random.setstate(r["python"]) + np.random.set_state(r["numpy"]) + torch.set_rng_state(r["torch"].cpu() if torch.is_tensor(r["torch"]) else r["torch"]) + if r.get("cuda") is not None and torch.cuda.is_available(): + try: + torch.cuda.set_rng_state_all([s.cpu() for s in r["cuda"]]) + except Exception: + pass + return blob + + +def rotate(directory: Path, pattern: str, keep: int) -> None: + d = Path(directory) + if not d.exists(): + return + rx = re.compile(pattern) + files = sorted((p for p in d.iterdir() if rx.match(p.name)), + key=lambda p: p.stat().st_mtime) + for p in files[:-keep] if keep > 0 else files: + try: + p.unlink() + except OSError: + pass + + +def latest_rolling(directory: Path) -> Path | None: + d = Path(directory) + if not d.exists(): + return None + files = [p for p in d.iterdir() if re.match(r"rolling_\d+\.pt$", p.name)] + if not files: + return None + return max(files, key=lambda p: p.stat().st_mtime) |
