diff options
Diffstat (limited to 'train.py')
| -rw-r--r-- | train.py | 596 |
1 files changed, 596 insertions, 0 deletions
diff --git a/train.py b/train.py new file mode 100644 index 0000000..113b873 --- /dev/null +++ b/train.py @@ -0,0 +1,596 @@ +"""Boris-2 pretraining loop: 200B tokens, fp16, single V100. + +Run: + python train.py # fresh + python train.py --resume # from the newest rolling checkpoint + python train.py --resume path.pt # from a specific one + +The loop assumes it will be interrupted. Everything needed to continue -- the +weights, both optimizer states, the loss scale, the packer's RNG and its carried +document tail -- goes into a rolling checkpoint every 45 minutes, and a resume +replays the identical token stream from there. +""" +from __future__ import annotations + +import os + +os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") + +import argparse +import json +import math +import subprocess +import sys +import time +from datetime import datetime, timezone +from pathlib import Path + +import torch + +from config import LOG_DIR, Config +import checkpoint as ckpt +from data import Prefetcher, VarlenPacker, open_sources, open_validation +from model import Boris2 +from ngram import NGramVocab +from optim import build_optimizers, set_lr +from schedule import (context_at, lr_scale, mixture_at, phase_at, + stable_end_tokens) +from tokenizer import BorisTokenizer + + +# --------------------------------------------------------------------------- # +def assert_environment(cfg: Config, log): + """Catch a silently-wrong precision setup in seconds rather than on day 30.""" + if not torch.cuda.is_available(): + raise SystemExit("no CUDA device") + dev = torch.device(cfg.train.device) + p = torch.cuda.get_device_properties(dev) + log(f" device {p.name} sm_{p.major}{p.minor} " + f"{p.total_memory/2**30:.1f} GB {p.multi_processor_count} SMs") + if p.major != 7: + log(f" WARNING: this run is tuned for sm_70; found sm_{p.major}{p.minor}") + + assert cfg.train.amp_dtype == "float16", ( + "sm_70 tensor cores are fp16-in/fp32-accumulate and have no bf16 path; " + "bf16 would run in software and TF32 does not exist before sm_80") + torch.backends.cuda.matmul.allow_tf32 = False + torch.backends.cudnn.allow_tf32 = False + torch.backends.cudnn.benchmark = True + # Measured +7.4% end to end on this card. It lets cuBLAS reduce fp16 GEMM + # split-k partials in fp16; checked against fp32 references in verify.py, + # where the LM head gradients are unaffected at 9e-04 / 2e-04. It is the + # default, and it is set explicitly so a future torch cannot flip it + # silently and cost a week. + torch.backends.cuda.matmul.allow_fp16_reduced_precision_reduction = True + + # Prove autocast is actually engaged and producing fp16, rather than the + # loop quietly running fp32 for weeks at a quarter of the throughput. + a = torch.randn(256, 256, device=dev) + with torch.autocast("cuda", dtype=torch.float16): + out = a @ a + assert out.dtype == torch.float16, ( + f"autocast produced {out.dtype}: fp16 is not engaged") + log(f" precision fp16 autocast + GradScaler, TF32 off, bf16 unused") + + +_NVML = None + + +def _nvml_temp(index: int) -> float: + """Temperature through libnvidia-ml directly: ~18 us, versus ~14 ms to fork + nvidia-smi (whose indices are NVML's, so the numbering is the same).""" + global _NVML + import ctypes + if _NVML is None: + lib = ctypes.CDLL("libnvidia-ml.so.1") + if lib.nvmlInit_v2() != 0: + raise RuntimeError("nvmlInit failed") + _NVML = (lib, {}) + lib, handles = _NVML + if index not in handles: + h = ctypes.c_void_p() + if lib.nvmlDeviceGetHandleByIndex_v2(index, ctypes.byref(h)) != 0: + raise RuntimeError("nvml handle failed") + handles[index] = h + t = ctypes.c_uint() + if lib.nvmlDeviceGetTemperature(handles[index], 0, ctypes.byref(t)) != 0: + raise RuntimeError("nvml temperature read failed") + return float(t.value) + + +def gpu_temp(index: int = 0) -> float: + try: + return _nvml_temp(index) + except Exception: + pass + try: + r = subprocess.run(["nvidia-smi", f"--id={index}", + "--query-gpu=temperature.gpu", "--format=csv,noheader"], + capture_output=True, text=True, timeout=4) + return float(r.stdout.strip().splitlines()[0]) + except Exception: + return float("nan") + + +def device_index(device: str | torch.device) -> int: + """Physical CUDA index used by diagnostics; never assume cuda:0.""" + d = torch.device(device) + return torch.cuda.current_device() if d.index is None else d.index + + +class Logger: + def __init__(self, path: Path): + path.parent.mkdir(parents=True, exist_ok=True) + self.f = open(path, "a") + + def line(self, s: str): + print(s, flush=True) + self.f.write(json.dumps({"msg": s, "timestamp": datetime.now(timezone.utc).isoformat()}) + "\n") + self.f.flush() + + def row(self, d: dict): + d = {**d, "timestamp": datetime.now(timezone.utc).isoformat()} + self.f.write(json.dumps(d) + "\n") + self.f.flush() + + +def step_flops(cfg: Config, tokens: int, sumsq: int) -> float: + """Exact fwd+bwd FLOPs for a step, attention included at what it really did. + + Attention is varlen, so its cost is sum(len_i^2) over the documents actually + packed -- not block_size^2. Reporting the dense figure would overstate the + model's utilisation by roughly the block sparsity, which at ctx 2048 is 57%. + """ + m = cfg.model + b = cfg.param_budget() + matmul = 2 * b["matmul"] * tokens + head = 2 * m.vocab_size * m.d_model * tokens + attn = m.n_layer * 2 * (m.n_head * m.head_dim) * sumsq + return 3.0 * (matmul + head + attn) + + +# --------------------------------------------------------------------------- # +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--resume", nargs="?", const="auto", default=None) + ap.add_argument("--config", type=Path, default=None) + ap.add_argument("--max-steps", type=int, default=None) + ap.add_argument("--force-config", action="store_true") + ap.add_argument("--no-compile", action="store_true") + a = ap.parse_args() + + cfg = Config.from_json(a.config.read_text()) if a.config else Config() + ckpt_dir = Path(cfg.checkpoint_dir) + if not a.resume and ckpt_dir.exists() and any(ckpt_dir.glob("*.pt")): + raise SystemExit("checkpoint directory is nonempty: use --resume or a new directory") + if a.no_compile: + cfg.train.compile_regions = False + log = Logger(LOG_DIR / f"{cfg.run_name}.jsonl") + log.line(f"\n {cfg.run_name}") + log.line(" " + "=" * 74) + assert_environment(cfg, log.line) + + torch.manual_seed(cfg.train.seed) + dev = cfg.train.device + torch.cuda.set_device(dev) + + tok_dir = Path(cfg.data.root) / "tokenizer" + tok_fp = BorisTokenizer.fingerprint(tok_dir) + if tok_fp != cfg.model.tokenizer_sha256: + raise SystemExit( + f"tokenizer fingerprint {tok_fp} at {tok_dir} does not match " + f"config {cfg.model.tokenizer_sha256}; token ids and embedding rows " + "would disagree") + + # -- data ---------------------------------------------------------- # + stores = open_sources(cfg, log.line) + packer = VarlenPacker(cfg, stores, cfg.train.seed, split="train") + # The held-out split is just another prepared source directory, and it is + # deliberately not in cfg.data.sources, so the training packer can never + # draw from it. + val_packer = None + if (Path(cfg.data.root) / cfg.data.val_source / "tokens.bin").exists(): + vcfg = Config.from_json(cfg.to_json()) + vcfg.curriculum.enabled = False + val_packer = VarlenPacker(vcfg, {"val": open_validation(cfg)}, + 12345, split="val") + log.line(f" val split " + f"{val_packer.stores['val'].n_tokens/1e6:.1f}M tokens held out") + + # -- model --------------------------------------------------------- # + model = Boris2(cfg).to(dev) + if model.ngram is not None: + vocab = NGramVocab.load(cfg.ngram.vocab_path) + assert vocab.vocab_size == cfg.model.vocab_size, ( + f"ngram_vocab.bin was built for vocab {vocab.vocab_size}, model is " + f"{cfg.model.vocab_size}") + model.ngram.load_vocab(vocab, dev) + ng_fp = vocab.fingerprint() + ng_compatible = (ng_fp, vocab.legacy_fingerprint()) + log.line(f" ngram vocab {vocab.bigram_keys.size:,} bigram + " + f"{vocab.trigram_keys.size:,} trigram rows, fingerprint {ng_fp}") + else: + ng_fp = "disabled" + ng_compatible = (ng_fp,) + log.line(" ngram table OFF (use_ngram=False): dense control arm") + if not a.no_compile: + model.maybe_compile() + + muon, adamw = build_optimizers(model, cfg, log.line) + scaler = torch.amp.GradScaler("cuda", init_scale=cfg.train.init_scale) + + # -- resume -------------------------------------------------------- # + step, tokens_seen, best_val = 0, 0, float("inf") + bad_grad = bad_loss = floored = 0 # consecutive non-finite log intervals + if a.resume: + path = (ckpt.latest_rolling(ckpt_dir) if a.resume == "auto" + else Path(a.resume)) + if path is None: + raise SystemExit("--resume requested but no rolling checkpoint exists") + else: + blob = ckpt.load_full( + path, model=model, muon=muon, adamw=adamw, scaler=scaler, + packer=packer, map_location=dev, + expect_fingerprint=None if a.force_config else cfg.fingerprint(), + expect_ngram=None if a.force_config else ng_compatible) + step, tokens_seen = blob["step"], blob["tokens_seen"] + best_val = blob.get("best_val", float("inf")) + log.line(f" resumed {path.name} at step {step:,}, " + f"{tokens_seen/1e9:.2f}B tokens") + + # -- run ------------------------------------------------------------ # + # The run ends when the token budget is spent, not at a fixed step count: tokens_per_step may change at a + # resume (2026-11-05: doubled for the final decay). Step numbers keep counting up, because rolling + # checkpoints are named and pruned by step. For an unchanged batch this equals cfg.total_steps. + total_steps = a.max_steps or (step + max(0, cfg.train.total_tokens - tokens_seen) // cfg.train.tokens_per_step) + progress = tokens_seen / cfg.train.total_tokens + block, theta, mb, accum = context_at(progress, cfg) + pre = Prefetcher(packer, cfg, dev) + pre.configure(mb, block, progress) + pre.start() + + log.line(f" budget {cfg.train.total_tokens/1e9:.0f}B tokens over " + f"{cfg.total_steps:,} steps of {cfg.train.tokens_per_step:,}") + log.line(f" context block {block}, theta {theta:,.0f}, " + f"micro_batch {mb} x {accum} accum") + log.line(" " + "=" * 74 + "\n") + + t_run = time.time() + t_ckpt = time.time() + next_export = ((tokens_seen // cfg.train.export_every_tokens) + 1) * cfg.train.export_every_tokens + next_full = ((tokens_seen // cfg.train.full_ckpt_every_tokens) + 1) * cfg.train.full_ckpt_every_tokens + next_val = ((tokens_seen // cfg.train.val_every_tokens) + 1) * cfg.train.val_every_tokens + stable_end = stable_end_tokens(cfg) + stable_saved = tokens_seen >= stable_end + attempt = step + win_t, win_tok, win_loss, win_flops = time.time(), 0, 0.0, 0.0 + win_steps = 0 + retry_batches = [] + consecutive_skips = 0 + + temp_poll = 0 + bad_temp_reads = 0 + temperature = gpu_temp(device_index(dev)) + while step < total_steps: + # Leave headroom below the user's 80 C cap without changing power limits. + # gpu_temp() reads NVML directly (~18 us), so every step is free again; + # it falls back to forking nvidia-smi (~14 ms), where every 5th step + # keeps the idle GPU time small. + temp_poll += 1 + if _NVML is not None or temp_poll % 5 == 1: + temperature = gpu_temp(device_index(dev)) + # One failed nvidia-smi read used to end the run; tolerate a few in a + # row (still refusing to train unmonitored for long). + bad_temp_reads = 0 if math.isfinite(temperature) else bad_temp_reads + 1 + if bad_temp_reads >= 3: + raise RuntimeError("cannot read GPU temperature 3 times running; " + "refusing unmonitored training") + if math.isfinite(temperature) and temperature >= 78: + # The sensor flickers 77 <-> 78 at steady load, so a single reading + # of 78 paused the run for 5 s of idle GPU. Confirm it a second + # later; the thresholds (pause at 78, resume at 75, cap 80) are + # unchanged and a real climb cannot gain 2 C in a second. + time.sleep(1) + temperature = gpu_temp(device_index(dev)) + if math.isfinite(temperature) and temperature >= 78: + log.line(f"thermal pause at {temperature:.0f} C; waiting for <=75 C") + fails = 0 + while not (math.isfinite(temperature) and temperature <= 75): + time.sleep(1) + temperature = gpu_temp(device_index(dev)) + fails = 0 if math.isfinite(temperature) else fails + 1 + if fails >= 3: + raise RuntimeError("GPU temperature unavailable during cooldown") + progress = tokens_seen / cfg.train.total_tokens + nb, ntheta, nmb, naccum = context_at(progress, cfg) + if (nb, nmb) != (block, mb): + log.line(f"\n context stage block {block} -> {nb}, " + f"theta {theta:,.0f} -> {ntheta:,.0f}, " + f"micro_batch {mb} -> {nmb} (one recompile expected)\n") + block, theta, mb, accum = nb, ntheta, nmb, naccum + pre.configure(mb, block, progress) + torch.cuda.empty_cache() + else: + theta = ntheta + pre.configure(mb, block, progress) + + scale = lr_scale(progress, cfg) + mlr, alr = set_lr(muon, adamw, cfg, scale) + + # Accumulated on the GPU. Reading it per micro-step with float() forces + # a host sync 16 times a step and destroys CPU run-ahead; it is read + # once, at log time, well after the fact. + loss_acc = torch.zeros((), device=dev) + tok_acc, sumsq_acc = 0, 0 + for micro in range(accum): + if micro == len(retry_batches): + retry_batches.append(pre.next(mb, block)) + x, y, pos, bnd, bounds, stats = retry_batches[micro] + with torch.autocast("cuda", dtype=torch.float16): + loss = model(x, pos, bounds, bnd, y, theta, + stats["min_seqlen"], stats["max_seqlen"]) + scaler.scale(loss / accum).backward() + loss_acc += loss.detach() / accum + tok_acc += stats["tokens"] + sumsq_acc += stats["sumsq"] + + grad_list = [p.grad for p in model.parameters() if p.grad is not None] + scaler.unscale_(muon) + scaler.unscale_(adamw) + # Not clip_grad_norm_: it scales every gradient by a coefficient derived + # from the norm over ALL parameters, so a single Inf anywhere -- routine + # on the first steps, where init_scale is 2^16 and GradScaler is meant + # to overflow, back off and skip -- makes the coefficient non-finite and + # writes NaN into gradients that were fine. found_inf is recorded per + # optimizer at unscale_ time, so Muon would skip while AdamW stepped on + # the freshly-poisoned grads, and 0 * NaN = NaN defeats even the lr-0 + # warmup step. That is exactly how this run died: step 1 loss 10.4739, + # step 2 loss nan. Leave the gradients untouched when the norm is not + # finite and let GradScaler do its job. + norms = torch._foreach_norm(grad_list) + gnorm = torch.linalg.vector_norm(torch.stack(norms)) + coef = torch.where(torch.isfinite(gnorm), + (cfg.optim.grad_clip / (gnorm + 1e-6)).clamp(max=1.0), + torch.ones_like(gnorm)) + # A tensor coefficient, and torch.where rather than a Python branch: + # reading gnorm here would be a host sync on every step. + pre_scale = scaler.get_scale() + # GradScaler makes its skip decision per optimizer. That allowed a head + # overflow to skip AdamW while Muon still changed the trunk: a partial + # update that desynchronised the model. Both optimizers see unscaled + # gradients here, so make one global finite decision and step both or + # neither. The scalar reads replace the loss read already performed + # below; they do not add another synchronization point. + loss_value = float(loss_acc) + gnorm_value = float(gnorm) + # coef is exactly 1.0 unless the norm reaches the clip (it never has in + # this run), and x * 1.0 == x, so skipping the multiply is bitwise the + # same update. It is applied, as before, whenever clipping engages. + if math.isfinite(gnorm_value) and gnorm_value + 1e-6 >= cfg.optim.grad_clip: + torch._foreach_mul_(grad_list, coef) + did_step = math.isfinite(loss_value) and math.isfinite(gnorm_value) + if did_step: + muon.step() + adamw.step() + scaler.update() + # A non-finite LOSS is a forward overflow, and halving the loss scale + # cannot touch it -- the forward does not depend on the scale at all. + # GradScaler halves anyway, every time, and that ratchet is what walked + # two runs down to the floor. Restore the scale on those steps: the step + # is still skipped (found_inf), so the weights are safe, and a rare + # forward NaN costs exactly one step instead of compounding. Genuine + # gradient overflow, where the loss is finite, still halves normally. + # One host sync per step, not the sixteen a per-micro-batch check costs. + if not math.isfinite(loss_value): + scaler.update(pre_scale) + # Clamp the loss scale at both ends. GradScaler will otherwise run the + # scale down without limit, and below ~1e-30 unscale_ (grad/scale) + # overflows fp32 to inf, sets found_inf, and halves it again -- a + # self-reinforcing collapse with no exit, which is what ended the run at + # step 3,810 with the weights still perfectly finite. The ceiling is the + # other half: the scale had grown to 131,072 before the first overflow. + _s = scaler.get_scale() + if _s < cfg.train.min_loss_scale: + scaler.update(cfg.train.min_loss_scale) + floored += 1 + elif _s > cfg.train.max_loss_scale: + scaler.update(cfg.train.max_loss_scale) + floored = 0 + else: + floored = 0 + muon.zero_grad(set_to_none=True) + adamw.zero_grad(set_to_none=True) + + if did_step: + step += 1 + tokens_seen += tok_acc + retry_batches.clear() + consecutive_skips = 0 + else: + consecutive_skips += 1 + # A non-finite LOSS is a forward overflow: deterministic, so retrying + # the identical batch only repeats it until the 20-skip abort. Move + # on to fresh data. Gradient overflow (finite loss) still retries the + # same batch, where the halved loss scale changes the outcome. + if not math.isfinite(loss_value): + retry_batches.clear() + log.row({"event": "skipped_update", "step": step, + "attempt": attempt + 1, "loss": loss_value, + "grad_norm": gnorm_value, "scale": scaler.get_scale(), + "consecutive_skips": consecutive_skips}) + if consecutive_skips >= 20: + log.line(f"ABORT: 20 consecutive failed updates at step {step}; same batch retried") + ckpt.save_full(ckpt_dir / f"abort_{step:09d}.pt", model=model, + muon=muon, adamw=adamw, scaler=scaler, + packer=pre, step=step, tokens_seen=tokens_seen, + cfg=cfg, ngram_fingerprint=ng_fp, + best_val=best_val, background=False) + raise SystemExit(3) + attempt += 1 + win_tok += tok_acc + win_loss += loss_acc + win_steps += 1 + win_flops += step_flops(cfg, tok_acc, sumsq_acc) + + # -- logging ---------------------------------------------------- # + if attempt % cfg.train.log_every_steps == 0: + dt = time.time() - win_t + tps = win_tok / max(dt, 1e-9) + tflops = win_flops / max(dt, 1e-9) / 1e12 + eta = (cfg.train.total_tokens - tokens_seen) / max(tps, 1e-9) + temp = gpu_temp(device_index(dev)) + mem = torch.cuda.max_memory_allocated() / 2**30 + log.row({"step": step, "attempt": attempt, "tokens": tokens_seen, + "loss": float(win_loss) / max(win_steps, 1), + "lr_muon": mlr, "lr_adamw": alr, "grad_norm": gnorm_value, + "scale": scaler.get_scale(), "tok_s": tps, + "tflops": tflops, "block": block, "temp": temp, + "vram_gb": mem, "phase": phase_at(progress, cfg), + "wall": time.time() - t_run}) + print(f" step {step:>7,}/{total_steps:,} {progress:>6.2%} " + f"loss {float(win_loss)/max(win_steps, 1):6.4f} " + f"lr {mlr:.2e} gn {gnorm_value:5.2f} " + f"s{math.log2(max(scaler.get_scale(), 1e-45)):+03.0f} " + f"{tps/1e3:6.1f}k tok/s {tflops:5.1f} TF " + f"{temp:.0f}C {mem:4.1f}G " + f"eta {eta/86400:5.1f}d", flush=True) + + # -- health ------------------------------------------------- # + # A non-finite grad norm is normal in ones and twos: GradScaler + # probes for its loss scale by overflowing on purpose. PERSISTENT + # non-finite is a different animal -- it means found_inf fires every + # step and an optimizer is being skipped every step. That is what + # happened at step 230 of the first launch: the head's dW GEMM + # overflowed fp16 independently of the loss scale, so AdamW never + # stepped again, the embeddings and head froze while Muon kept + # training the trunk, and the loss climbed from 6.48 to nan over + # 180 steps and then burned 3,300 more. Neither the temperature nor + # the throughput showed anything wrong. Stop instead of cooking. + bad_grad = 0 if math.isfinite(gnorm_value) else bad_grad + 1 + # Divide by the steps ACTUALLY in this window, not the nominal + # window size. A resume lands mid-window (step 57,340 with + # log_every_steps=10), so the first line after every resume + # under-reported the loss by up to 10x -- 0.337 for a model + # whose real loss was 3.37. + win_mean = float(win_loss) / max(win_steps, 1) + bad_loss = 0 if math.isfinite(win_mean) else bad_loss + 1 + if bad_loss * cfg.train.log_every_steps >= cfg.train.abort_after_bad: + print(f"\n ABORT: loss has been non-finite for " + f"{bad_loss * cfg.train.log_every_steps} steps.\n", + flush=True) + ckpt.save_full(ckpt_dir / f"abort_{step:09d}.pt", model=model, + muon=muon, adamw=adamw, scaler=scaler, + packer=pre, step=step, tokens_seen=tokens_seen, + cfg=cfg, ngram_fingerprint=ng_fp, + best_val=best_val, background=False) + raise SystemExit(3) + if floored >= cfg.train.abort_after_floored: + print(f"\n ABORT: the loss scale has been pinned at its floor " + f"({cfg.train.min_loss_scale}) for {floored} steps -- " + f"every step is being skipped and nothing is training.\n", + flush=True) + ckpt.save_full(ckpt_dir / f"abort_{step:09d}.pt", model=model, + muon=muon, adamw=adamw, scaler=scaler, + packer=pre, step=step, tokens_seen=tokens_seen, + cfg=cfg, ngram_fingerprint=ng_fp, + best_val=best_val, background=False) + raise SystemExit(3) + if bad_grad * cfg.train.log_every_steps >= cfg.train.abort_after_bad: + print(f"\n ABORT: the gradient norm has been non-finite for " + f"{bad_grad * cfg.train.log_every_steps} steps -- an " + f"optimizer is being skipped every step.\n", flush=True) + ckpt.save_full(ckpt_dir / f"abort_{step:09d}.pt", model=model, + muon=muon, adamw=adamw, scaler=scaler, + packer=pre, step=step, tokens_seen=tokens_seen, + cfg=cfg, ngram_fingerprint=ng_fp, + best_val=best_val, background=False) + raise SystemExit(3) + + win_t, win_tok, win_loss, win_flops = time.time(), 0, 0.0, 0.0 + win_steps = 0 + + # -- validation ------------------------------------------------- # + if val_packer is not None and tokens_seen >= next_val: + next_val += cfg.train.val_every_tokens + vl = quick_val(model, val_packer, cfg, dev, block, theta, progress) + best_val = min(best_val, vl) + log.row({"step": step, "tokens": tokens_seen, "val_loss": vl, + "val_ppl": math.exp(min(vl, 20))}) + print(f" val {vl:.4f} ppl {math.exp(min(vl,20)):.2f}", flush=True) + + # -- checkpoints ------------------------------------------------- # + if (time.time() - t_ckpt) / 60 >= cfg.train.rolling_ckpt_minutes: + t_ckpt = time.time() + p = ckpt_dir / f"rolling_{step:09d}.pt" + ckpt.save_full(p, model=model, muon=muon, adamw=adamw, scaler=scaler, + packer=pre, step=step, tokens_seen=tokens_seen, + cfg=cfg, ngram_fingerprint=ng_fp, best_val=best_val, + rotate_pattern=r"rolling_\d+\.pt$", + rotate_keep=cfg.train.rolling_ckpt_keep) + log.line(f" rolling checkpoint {p.name}") + + if tokens_seen >= next_export: + next_export += cfg.train.export_every_tokens + d = cfg.eval.export_dir + p = d / f"export_{tokens_seen//10**9:04d}B.pt" + ckpt.save_export(p, model=model, cfg=cfg, step=step, + tokens_seen=tokens_seen, manifest=cfg.eval.manifest, + ngram_fingerprint=ng_fp) + log.line(f" export {p.name} -> eval queue") + + if tokens_seen >= next_full: + next_full += cfg.train.full_ckpt_every_tokens + p = ckpt_dir / f"full_{tokens_seen//10**9:04d}B.pt" + ckpt.save_full(p, model=model, muon=muon, adamw=adamw, scaler=scaler, + packer=pre, step=step, tokens_seen=tokens_seen, + cfg=cfg, ngram_fingerprint=ng_fp, best_val=best_val) + log.line(f" full checkpoint {p.name}") + + if not stable_saved and tokens_seen >= stable_end: + stable_saved = True + p = ckpt_dir / "stable.pt" + ckpt.save_full(p, model=model, muon=muon, adamw=adamw, scaler=scaler, + packer=pre, step=step, tokens_seen=tokens_seen, + cfg=cfg, ngram_fingerprint=ng_fp, best_val=best_val, + background=False) + log.line(f" END OF STABLE PHASE -> stable.pt (permanent, never rotated)") + + pre.stop() + p = ckpt_dir / "final.pt" + ckpt.save_full(p, model=model, muon=muon, adamw=adamw, scaler=scaler, + packer=pre, step=step, tokens_seen=tokens_seen, cfg=cfg, + ngram_fingerprint=ng_fp, best_val=best_val, background=False) + log.line(f"\n done: {tokens_seen/1e9:.2f}B tokens in " + f"{(time.time()-t_run)/86400:.2f} days -> {p}") + + +@torch.no_grad() +def quick_val(model, val_packer, cfg, dev, block, theta, progress, + batches: int = 16): + """A short in-loop validation. The full suite runs on the second GPU.""" + model.eval() + tot, n = 0.0, 0 + # Score the SAME held-out batches every time. The packer's RNG used to run + # on from call to call (and restart at its seed on every resume), so each + # validation drew a fresh sample and consecutive values wandered by about + # +/-0.05 -- larger than the effects being tracked. Snapshot the state on + # the first call and rewind to it on every call. + if not hasattr(val_packer, "_boris_val_state0"): + val_packer._boris_val_state0 = val_packer.state_dict() + val_packer.load_state_dict(val_packer._boris_val_state0) + # Stage-appropriate micro-batch: using the 2048-token value at block 8192 + # would quadruple the validation activation footprint and OOM late in run. + mb = cfg.micro_batch_at(progress) + for _ in range(batches): + xs, ys, pos, bnd, bounds, _ = val_packer.batch(mb, block, 0.0) + t = lambda z: torch.from_numpy(z).to(dev) + with torch.autocast("cuda", dtype=torch.float16): + loss = model(t(xs), t(pos), t(bounds), t(bnd), t(ys), theta) + valid = int((ys != -100).sum()) + tot += float(loss) * valid; n += valid + model.train() + return tot / max(n, 1) + + +if __name__ == "__main__": + main() |
