summaryrefslogtreecommitdiff
path: root/train.py
diff options
context:
space:
mode:
authorjdj <jdj@kunix.org>2026-11-18 14:47:33 -0800
committerjdj <jdj@kunix.org>2026-11-18 14:47:33 -0800
commit5d2e4252e695644c34be605b44a14bd80a98a0a6 (patch)
tree311a72eda4459af7547b9cd05c0956eafb235f2d /train.py
downloadboris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.tar.gz
boris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.zip
Boris-2 training code
The code, configuration and data filters used to train Boris-2 (144M, 200B tokens, single V100): model, training loop, data preparation, n-gram table, V100 kernels, final stages and the exact run configs.
Diffstat (limited to 'train.py')
-rw-r--r--train.py596
1 files changed, 596 insertions, 0 deletions
diff --git a/train.py b/train.py
new file mode 100644
index 0000000..113b873
--- /dev/null
+++ b/train.py
@@ -0,0 +1,596 @@
+"""Boris-2 pretraining loop: 200B tokens, fp16, single V100.
+
+Run:
+ python train.py # fresh
+ python train.py --resume # from the newest rolling checkpoint
+ python train.py --resume path.pt # from a specific one
+
+The loop assumes it will be interrupted. Everything needed to continue -- the
+weights, both optimizer states, the loss scale, the packer's RNG and its carried
+document tail -- goes into a rolling checkpoint every 45 minutes, and a resume
+replays the identical token stream from there.
+"""
+from __future__ import annotations
+
+import os
+
+os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
+
+import argparse
+import json
+import math
+import subprocess
+import sys
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+
+import torch
+
+from config import LOG_DIR, Config
+import checkpoint as ckpt
+from data import Prefetcher, VarlenPacker, open_sources, open_validation
+from model import Boris2
+from ngram import NGramVocab
+from optim import build_optimizers, set_lr
+from schedule import (context_at, lr_scale, mixture_at, phase_at,
+ stable_end_tokens)
+from tokenizer import BorisTokenizer
+
+
+# --------------------------------------------------------------------------- #
+def assert_environment(cfg: Config, log):
+ """Catch a silently-wrong precision setup in seconds rather than on day 30."""
+ if not torch.cuda.is_available():
+ raise SystemExit("no CUDA device")
+ dev = torch.device(cfg.train.device)
+ p = torch.cuda.get_device_properties(dev)
+ log(f" device {p.name} sm_{p.major}{p.minor} "
+ f"{p.total_memory/2**30:.1f} GB {p.multi_processor_count} SMs")
+ if p.major != 7:
+ log(f" WARNING: this run is tuned for sm_70; found sm_{p.major}{p.minor}")
+
+ assert cfg.train.amp_dtype == "float16", (
+ "sm_70 tensor cores are fp16-in/fp32-accumulate and have no bf16 path; "
+ "bf16 would run in software and TF32 does not exist before sm_80")
+ torch.backends.cuda.matmul.allow_tf32 = False
+ torch.backends.cudnn.allow_tf32 = False
+ torch.backends.cudnn.benchmark = True
+ # Measured +7.4% end to end on this card. It lets cuBLAS reduce fp16 GEMM
+ # split-k partials in fp16; checked against fp32 references in verify.py,
+ # where the LM head gradients are unaffected at 9e-04 / 2e-04. It is the
+ # default, and it is set explicitly so a future torch cannot flip it
+ # silently and cost a week.
+ torch.backends.cuda.matmul.allow_fp16_reduced_precision_reduction = True
+
+ # Prove autocast is actually engaged and producing fp16, rather than the
+ # loop quietly running fp32 for weeks at a quarter of the throughput.
+ a = torch.randn(256, 256, device=dev)
+ with torch.autocast("cuda", dtype=torch.float16):
+ out = a @ a
+ assert out.dtype == torch.float16, (
+ f"autocast produced {out.dtype}: fp16 is not engaged")
+ log(f" precision fp16 autocast + GradScaler, TF32 off, bf16 unused")
+
+
+_NVML = None
+
+
+def _nvml_temp(index: int) -> float:
+ """Temperature through libnvidia-ml directly: ~18 us, versus ~14 ms to fork
+ nvidia-smi (whose indices are NVML's, so the numbering is the same)."""
+ global _NVML
+ import ctypes
+ if _NVML is None:
+ lib = ctypes.CDLL("libnvidia-ml.so.1")
+ if lib.nvmlInit_v2() != 0:
+ raise RuntimeError("nvmlInit failed")
+ _NVML = (lib, {})
+ lib, handles = _NVML
+ if index not in handles:
+ h = ctypes.c_void_p()
+ if lib.nvmlDeviceGetHandleByIndex_v2(index, ctypes.byref(h)) != 0:
+ raise RuntimeError("nvml handle failed")
+ handles[index] = h
+ t = ctypes.c_uint()
+ if lib.nvmlDeviceGetTemperature(handles[index], 0, ctypes.byref(t)) != 0:
+ raise RuntimeError("nvml temperature read failed")
+ return float(t.value)
+
+
+def gpu_temp(index: int = 0) -> float:
+ try:
+ return _nvml_temp(index)
+ except Exception:
+ pass
+ try:
+ r = subprocess.run(["nvidia-smi", f"--id={index}",
+ "--query-gpu=temperature.gpu", "--format=csv,noheader"],
+ capture_output=True, text=True, timeout=4)
+ return float(r.stdout.strip().splitlines()[0])
+ except Exception:
+ return float("nan")
+
+
+def device_index(device: str | torch.device) -> int:
+ """Physical CUDA index used by diagnostics; never assume cuda:0."""
+ d = torch.device(device)
+ return torch.cuda.current_device() if d.index is None else d.index
+
+
+class Logger:
+ def __init__(self, path: Path):
+ path.parent.mkdir(parents=True, exist_ok=True)
+ self.f = open(path, "a")
+
+ def line(self, s: str):
+ print(s, flush=True)
+ self.f.write(json.dumps({"msg": s, "timestamp": datetime.now(timezone.utc).isoformat()}) + "\n")
+ self.f.flush()
+
+ def row(self, d: dict):
+ d = {**d, "timestamp": datetime.now(timezone.utc).isoformat()}
+ self.f.write(json.dumps(d) + "\n")
+ self.f.flush()
+
+
+def step_flops(cfg: Config, tokens: int, sumsq: int) -> float:
+ """Exact fwd+bwd FLOPs for a step, attention included at what it really did.
+
+ Attention is varlen, so its cost is sum(len_i^2) over the documents actually
+ packed -- not block_size^2. Reporting the dense figure would overstate the
+ model's utilisation by roughly the block sparsity, which at ctx 2048 is 57%.
+ """
+ m = cfg.model
+ b = cfg.param_budget()
+ matmul = 2 * b["matmul"] * tokens
+ head = 2 * m.vocab_size * m.d_model * tokens
+ attn = m.n_layer * 2 * (m.n_head * m.head_dim) * sumsq
+ return 3.0 * (matmul + head + attn)
+
+
+# --------------------------------------------------------------------------- #
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--resume", nargs="?", const="auto", default=None)
+ ap.add_argument("--config", type=Path, default=None)
+ ap.add_argument("--max-steps", type=int, default=None)
+ ap.add_argument("--force-config", action="store_true")
+ ap.add_argument("--no-compile", action="store_true")
+ a = ap.parse_args()
+
+ cfg = Config.from_json(a.config.read_text()) if a.config else Config()
+ ckpt_dir = Path(cfg.checkpoint_dir)
+ if not a.resume and ckpt_dir.exists() and any(ckpt_dir.glob("*.pt")):
+ raise SystemExit("checkpoint directory is nonempty: use --resume or a new directory")
+ if a.no_compile:
+ cfg.train.compile_regions = False
+ log = Logger(LOG_DIR / f"{cfg.run_name}.jsonl")
+ log.line(f"\n {cfg.run_name}")
+ log.line(" " + "=" * 74)
+ assert_environment(cfg, log.line)
+
+ torch.manual_seed(cfg.train.seed)
+ dev = cfg.train.device
+ torch.cuda.set_device(dev)
+
+ tok_dir = Path(cfg.data.root) / "tokenizer"
+ tok_fp = BorisTokenizer.fingerprint(tok_dir)
+ if tok_fp != cfg.model.tokenizer_sha256:
+ raise SystemExit(
+ f"tokenizer fingerprint {tok_fp} at {tok_dir} does not match "
+ f"config {cfg.model.tokenizer_sha256}; token ids and embedding rows "
+ "would disagree")
+
+ # -- data ---------------------------------------------------------- #
+ stores = open_sources(cfg, log.line)
+ packer = VarlenPacker(cfg, stores, cfg.train.seed, split="train")
+ # The held-out split is just another prepared source directory, and it is
+ # deliberately not in cfg.data.sources, so the training packer can never
+ # draw from it.
+ val_packer = None
+ if (Path(cfg.data.root) / cfg.data.val_source / "tokens.bin").exists():
+ vcfg = Config.from_json(cfg.to_json())
+ vcfg.curriculum.enabled = False
+ val_packer = VarlenPacker(vcfg, {"val": open_validation(cfg)},
+ 12345, split="val")
+ log.line(f" val split "
+ f"{val_packer.stores['val'].n_tokens/1e6:.1f}M tokens held out")
+
+ # -- model --------------------------------------------------------- #
+ model = Boris2(cfg).to(dev)
+ if model.ngram is not None:
+ vocab = NGramVocab.load(cfg.ngram.vocab_path)
+ assert vocab.vocab_size == cfg.model.vocab_size, (
+ f"ngram_vocab.bin was built for vocab {vocab.vocab_size}, model is "
+ f"{cfg.model.vocab_size}")
+ model.ngram.load_vocab(vocab, dev)
+ ng_fp = vocab.fingerprint()
+ ng_compatible = (ng_fp, vocab.legacy_fingerprint())
+ log.line(f" ngram vocab {vocab.bigram_keys.size:,} bigram + "
+ f"{vocab.trigram_keys.size:,} trigram rows, fingerprint {ng_fp}")
+ else:
+ ng_fp = "disabled"
+ ng_compatible = (ng_fp,)
+ log.line(" ngram table OFF (use_ngram=False): dense control arm")
+ if not a.no_compile:
+ model.maybe_compile()
+
+ muon, adamw = build_optimizers(model, cfg, log.line)
+ scaler = torch.amp.GradScaler("cuda", init_scale=cfg.train.init_scale)
+
+ # -- resume -------------------------------------------------------- #
+ step, tokens_seen, best_val = 0, 0, float("inf")
+ bad_grad = bad_loss = floored = 0 # consecutive non-finite log intervals
+ if a.resume:
+ path = (ckpt.latest_rolling(ckpt_dir) if a.resume == "auto"
+ else Path(a.resume))
+ if path is None:
+ raise SystemExit("--resume requested but no rolling checkpoint exists")
+ else:
+ blob = ckpt.load_full(
+ path, model=model, muon=muon, adamw=adamw, scaler=scaler,
+ packer=packer, map_location=dev,
+ expect_fingerprint=None if a.force_config else cfg.fingerprint(),
+ expect_ngram=None if a.force_config else ng_compatible)
+ step, tokens_seen = blob["step"], blob["tokens_seen"]
+ best_val = blob.get("best_val", float("inf"))
+ log.line(f" resumed {path.name} at step {step:,}, "
+ f"{tokens_seen/1e9:.2f}B tokens")
+
+ # -- run ------------------------------------------------------------ #
+ # The run ends when the token budget is spent, not at a fixed step count: tokens_per_step may change at a
+ # resume (2026-11-05: doubled for the final decay). Step numbers keep counting up, because rolling
+ # checkpoints are named and pruned by step. For an unchanged batch this equals cfg.total_steps.
+ total_steps = a.max_steps or (step + max(0, cfg.train.total_tokens - tokens_seen) // cfg.train.tokens_per_step)
+ progress = tokens_seen / cfg.train.total_tokens
+ block, theta, mb, accum = context_at(progress, cfg)
+ pre = Prefetcher(packer, cfg, dev)
+ pre.configure(mb, block, progress)
+ pre.start()
+
+ log.line(f" budget {cfg.train.total_tokens/1e9:.0f}B tokens over "
+ f"{cfg.total_steps:,} steps of {cfg.train.tokens_per_step:,}")
+ log.line(f" context block {block}, theta {theta:,.0f}, "
+ f"micro_batch {mb} x {accum} accum")
+ log.line(" " + "=" * 74 + "\n")
+
+ t_run = time.time()
+ t_ckpt = time.time()
+ next_export = ((tokens_seen // cfg.train.export_every_tokens) + 1) * cfg.train.export_every_tokens
+ next_full = ((tokens_seen // cfg.train.full_ckpt_every_tokens) + 1) * cfg.train.full_ckpt_every_tokens
+ next_val = ((tokens_seen // cfg.train.val_every_tokens) + 1) * cfg.train.val_every_tokens
+ stable_end = stable_end_tokens(cfg)
+ stable_saved = tokens_seen >= stable_end
+ attempt = step
+ win_t, win_tok, win_loss, win_flops = time.time(), 0, 0.0, 0.0
+ win_steps = 0
+ retry_batches = []
+ consecutive_skips = 0
+
+ temp_poll = 0
+ bad_temp_reads = 0
+ temperature = gpu_temp(device_index(dev))
+ while step < total_steps:
+ # Leave headroom below the user's 80 C cap without changing power limits.
+ # gpu_temp() reads NVML directly (~18 us), so every step is free again;
+ # it falls back to forking nvidia-smi (~14 ms), where every 5th step
+ # keeps the idle GPU time small.
+ temp_poll += 1
+ if _NVML is not None or temp_poll % 5 == 1:
+ temperature = gpu_temp(device_index(dev))
+ # One failed nvidia-smi read used to end the run; tolerate a few in a
+ # row (still refusing to train unmonitored for long).
+ bad_temp_reads = 0 if math.isfinite(temperature) else bad_temp_reads + 1
+ if bad_temp_reads >= 3:
+ raise RuntimeError("cannot read GPU temperature 3 times running; "
+ "refusing unmonitored training")
+ if math.isfinite(temperature) and temperature >= 78:
+ # The sensor flickers 77 <-> 78 at steady load, so a single reading
+ # of 78 paused the run for 5 s of idle GPU. Confirm it a second
+ # later; the thresholds (pause at 78, resume at 75, cap 80) are
+ # unchanged and a real climb cannot gain 2 C in a second.
+ time.sleep(1)
+ temperature = gpu_temp(device_index(dev))
+ if math.isfinite(temperature) and temperature >= 78:
+ log.line(f"thermal pause at {temperature:.0f} C; waiting for <=75 C")
+ fails = 0
+ while not (math.isfinite(temperature) and temperature <= 75):
+ time.sleep(1)
+ temperature = gpu_temp(device_index(dev))
+ fails = 0 if math.isfinite(temperature) else fails + 1
+ if fails >= 3:
+ raise RuntimeError("GPU temperature unavailable during cooldown")
+ progress = tokens_seen / cfg.train.total_tokens
+ nb, ntheta, nmb, naccum = context_at(progress, cfg)
+ if (nb, nmb) != (block, mb):
+ log.line(f"\n context stage block {block} -> {nb}, "
+ f"theta {theta:,.0f} -> {ntheta:,.0f}, "
+ f"micro_batch {mb} -> {nmb} (one recompile expected)\n")
+ block, theta, mb, accum = nb, ntheta, nmb, naccum
+ pre.configure(mb, block, progress)
+ torch.cuda.empty_cache()
+ else:
+ theta = ntheta
+ pre.configure(mb, block, progress)
+
+ scale = lr_scale(progress, cfg)
+ mlr, alr = set_lr(muon, adamw, cfg, scale)
+
+ # Accumulated on the GPU. Reading it per micro-step with float() forces
+ # a host sync 16 times a step and destroys CPU run-ahead; it is read
+ # once, at log time, well after the fact.
+ loss_acc = torch.zeros((), device=dev)
+ tok_acc, sumsq_acc = 0, 0
+ for micro in range(accum):
+ if micro == len(retry_batches):
+ retry_batches.append(pre.next(mb, block))
+ x, y, pos, bnd, bounds, stats = retry_batches[micro]
+ with torch.autocast("cuda", dtype=torch.float16):
+ loss = model(x, pos, bounds, bnd, y, theta,
+ stats["min_seqlen"], stats["max_seqlen"])
+ scaler.scale(loss / accum).backward()
+ loss_acc += loss.detach() / accum
+ tok_acc += stats["tokens"]
+ sumsq_acc += stats["sumsq"]
+
+ grad_list = [p.grad for p in model.parameters() if p.grad is not None]
+ scaler.unscale_(muon)
+ scaler.unscale_(adamw)
+ # Not clip_grad_norm_: it scales every gradient by a coefficient derived
+ # from the norm over ALL parameters, so a single Inf anywhere -- routine
+ # on the first steps, where init_scale is 2^16 and GradScaler is meant
+ # to overflow, back off and skip -- makes the coefficient non-finite and
+ # writes NaN into gradients that were fine. found_inf is recorded per
+ # optimizer at unscale_ time, so Muon would skip while AdamW stepped on
+ # the freshly-poisoned grads, and 0 * NaN = NaN defeats even the lr-0
+ # warmup step. That is exactly how this run died: step 1 loss 10.4739,
+ # step 2 loss nan. Leave the gradients untouched when the norm is not
+ # finite and let GradScaler do its job.
+ norms = torch._foreach_norm(grad_list)
+ gnorm = torch.linalg.vector_norm(torch.stack(norms))
+ coef = torch.where(torch.isfinite(gnorm),
+ (cfg.optim.grad_clip / (gnorm + 1e-6)).clamp(max=1.0),
+ torch.ones_like(gnorm))
+ # A tensor coefficient, and torch.where rather than a Python branch:
+ # reading gnorm here would be a host sync on every step.
+ pre_scale = scaler.get_scale()
+ # GradScaler makes its skip decision per optimizer. That allowed a head
+ # overflow to skip AdamW while Muon still changed the trunk: a partial
+ # update that desynchronised the model. Both optimizers see unscaled
+ # gradients here, so make one global finite decision and step both or
+ # neither. The scalar reads replace the loss read already performed
+ # below; they do not add another synchronization point.
+ loss_value = float(loss_acc)
+ gnorm_value = float(gnorm)
+ # coef is exactly 1.0 unless the norm reaches the clip (it never has in
+ # this run), and x * 1.0 == x, so skipping the multiply is bitwise the
+ # same update. It is applied, as before, whenever clipping engages.
+ if math.isfinite(gnorm_value) and gnorm_value + 1e-6 >= cfg.optim.grad_clip:
+ torch._foreach_mul_(grad_list, coef)
+ did_step = math.isfinite(loss_value) and math.isfinite(gnorm_value)
+ if did_step:
+ muon.step()
+ adamw.step()
+ scaler.update()
+ # A non-finite LOSS is a forward overflow, and halving the loss scale
+ # cannot touch it -- the forward does not depend on the scale at all.
+ # GradScaler halves anyway, every time, and that ratchet is what walked
+ # two runs down to the floor. Restore the scale on those steps: the step
+ # is still skipped (found_inf), so the weights are safe, and a rare
+ # forward NaN costs exactly one step instead of compounding. Genuine
+ # gradient overflow, where the loss is finite, still halves normally.
+ # One host sync per step, not the sixteen a per-micro-batch check costs.
+ if not math.isfinite(loss_value):
+ scaler.update(pre_scale)
+ # Clamp the loss scale at both ends. GradScaler will otherwise run the
+ # scale down without limit, and below ~1e-30 unscale_ (grad/scale)
+ # overflows fp32 to inf, sets found_inf, and halves it again -- a
+ # self-reinforcing collapse with no exit, which is what ended the run at
+ # step 3,810 with the weights still perfectly finite. The ceiling is the
+ # other half: the scale had grown to 131,072 before the first overflow.
+ _s = scaler.get_scale()
+ if _s < cfg.train.min_loss_scale:
+ scaler.update(cfg.train.min_loss_scale)
+ floored += 1
+ elif _s > cfg.train.max_loss_scale:
+ scaler.update(cfg.train.max_loss_scale)
+ floored = 0
+ else:
+ floored = 0
+ muon.zero_grad(set_to_none=True)
+ adamw.zero_grad(set_to_none=True)
+
+ if did_step:
+ step += 1
+ tokens_seen += tok_acc
+ retry_batches.clear()
+ consecutive_skips = 0
+ else:
+ consecutive_skips += 1
+ # A non-finite LOSS is a forward overflow: deterministic, so retrying
+ # the identical batch only repeats it until the 20-skip abort. Move
+ # on to fresh data. Gradient overflow (finite loss) still retries the
+ # same batch, where the halved loss scale changes the outcome.
+ if not math.isfinite(loss_value):
+ retry_batches.clear()
+ log.row({"event": "skipped_update", "step": step,
+ "attempt": attempt + 1, "loss": loss_value,
+ "grad_norm": gnorm_value, "scale": scaler.get_scale(),
+ "consecutive_skips": consecutive_skips})
+ if consecutive_skips >= 20:
+ log.line(f"ABORT: 20 consecutive failed updates at step {step}; same batch retried")
+ ckpt.save_full(ckpt_dir / f"abort_{step:09d}.pt", model=model,
+ muon=muon, adamw=adamw, scaler=scaler,
+ packer=pre, step=step, tokens_seen=tokens_seen,
+ cfg=cfg, ngram_fingerprint=ng_fp,
+ best_val=best_val, background=False)
+ raise SystemExit(3)
+ attempt += 1
+ win_tok += tok_acc
+ win_loss += loss_acc
+ win_steps += 1
+ win_flops += step_flops(cfg, tok_acc, sumsq_acc)
+
+ # -- logging ---------------------------------------------------- #
+ if attempt % cfg.train.log_every_steps == 0:
+ dt = time.time() - win_t
+ tps = win_tok / max(dt, 1e-9)
+ tflops = win_flops / max(dt, 1e-9) / 1e12
+ eta = (cfg.train.total_tokens - tokens_seen) / max(tps, 1e-9)
+ temp = gpu_temp(device_index(dev))
+ mem = torch.cuda.max_memory_allocated() / 2**30
+ log.row({"step": step, "attempt": attempt, "tokens": tokens_seen,
+ "loss": float(win_loss) / max(win_steps, 1),
+ "lr_muon": mlr, "lr_adamw": alr, "grad_norm": gnorm_value,
+ "scale": scaler.get_scale(), "tok_s": tps,
+ "tflops": tflops, "block": block, "temp": temp,
+ "vram_gb": mem, "phase": phase_at(progress, cfg),
+ "wall": time.time() - t_run})
+ print(f" step {step:>7,}/{total_steps:,} {progress:>6.2%} "
+ f"loss {float(win_loss)/max(win_steps, 1):6.4f} "
+ f"lr {mlr:.2e} gn {gnorm_value:5.2f} "
+ f"s{math.log2(max(scaler.get_scale(), 1e-45)):+03.0f} "
+ f"{tps/1e3:6.1f}k tok/s {tflops:5.1f} TF "
+ f"{temp:.0f}C {mem:4.1f}G "
+ f"eta {eta/86400:5.1f}d", flush=True)
+
+ # -- health ------------------------------------------------- #
+ # A non-finite grad norm is normal in ones and twos: GradScaler
+ # probes for its loss scale by overflowing on purpose. PERSISTENT
+ # non-finite is a different animal -- it means found_inf fires every
+ # step and an optimizer is being skipped every step. That is what
+ # happened at step 230 of the first launch: the head's dW GEMM
+ # overflowed fp16 independently of the loss scale, so AdamW never
+ # stepped again, the embeddings and head froze while Muon kept
+ # training the trunk, and the loss climbed from 6.48 to nan over
+ # 180 steps and then burned 3,300 more. Neither the temperature nor
+ # the throughput showed anything wrong. Stop instead of cooking.
+ bad_grad = 0 if math.isfinite(gnorm_value) else bad_grad + 1
+ # Divide by the steps ACTUALLY in this window, not the nominal
+ # window size. A resume lands mid-window (step 57,340 with
+ # log_every_steps=10), so the first line after every resume
+ # under-reported the loss by up to 10x -- 0.337 for a model
+ # whose real loss was 3.37.
+ win_mean = float(win_loss) / max(win_steps, 1)
+ bad_loss = 0 if math.isfinite(win_mean) else bad_loss + 1
+ if bad_loss * cfg.train.log_every_steps >= cfg.train.abort_after_bad:
+ print(f"\n ABORT: loss has been non-finite for "
+ f"{bad_loss * cfg.train.log_every_steps} steps.\n",
+ flush=True)
+ ckpt.save_full(ckpt_dir / f"abort_{step:09d}.pt", model=model,
+ muon=muon, adamw=adamw, scaler=scaler,
+ packer=pre, step=step, tokens_seen=tokens_seen,
+ cfg=cfg, ngram_fingerprint=ng_fp,
+ best_val=best_val, background=False)
+ raise SystemExit(3)
+ if floored >= cfg.train.abort_after_floored:
+ print(f"\n ABORT: the loss scale has been pinned at its floor "
+ f"({cfg.train.min_loss_scale}) for {floored} steps -- "
+ f"every step is being skipped and nothing is training.\n",
+ flush=True)
+ ckpt.save_full(ckpt_dir / f"abort_{step:09d}.pt", model=model,
+ muon=muon, adamw=adamw, scaler=scaler,
+ packer=pre, step=step, tokens_seen=tokens_seen,
+ cfg=cfg, ngram_fingerprint=ng_fp,
+ best_val=best_val, background=False)
+ raise SystemExit(3)
+ if bad_grad * cfg.train.log_every_steps >= cfg.train.abort_after_bad:
+ print(f"\n ABORT: the gradient norm has been non-finite for "
+ f"{bad_grad * cfg.train.log_every_steps} steps -- an "
+ f"optimizer is being skipped every step.\n", flush=True)
+ ckpt.save_full(ckpt_dir / f"abort_{step:09d}.pt", model=model,
+ muon=muon, adamw=adamw, scaler=scaler,
+ packer=pre, step=step, tokens_seen=tokens_seen,
+ cfg=cfg, ngram_fingerprint=ng_fp,
+ best_val=best_val, background=False)
+ raise SystemExit(3)
+
+ win_t, win_tok, win_loss, win_flops = time.time(), 0, 0.0, 0.0
+ win_steps = 0
+
+ # -- validation ------------------------------------------------- #
+ if val_packer is not None and tokens_seen >= next_val:
+ next_val += cfg.train.val_every_tokens
+ vl = quick_val(model, val_packer, cfg, dev, block, theta, progress)
+ best_val = min(best_val, vl)
+ log.row({"step": step, "tokens": tokens_seen, "val_loss": vl,
+ "val_ppl": math.exp(min(vl, 20))})
+ print(f" val {vl:.4f} ppl {math.exp(min(vl,20)):.2f}", flush=True)
+
+ # -- checkpoints ------------------------------------------------- #
+ if (time.time() - t_ckpt) / 60 >= cfg.train.rolling_ckpt_minutes:
+ t_ckpt = time.time()
+ p = ckpt_dir / f"rolling_{step:09d}.pt"
+ ckpt.save_full(p, model=model, muon=muon, adamw=adamw, scaler=scaler,
+ packer=pre, step=step, tokens_seen=tokens_seen,
+ cfg=cfg, ngram_fingerprint=ng_fp, best_val=best_val,
+ rotate_pattern=r"rolling_\d+\.pt$",
+ rotate_keep=cfg.train.rolling_ckpt_keep)
+ log.line(f" rolling checkpoint {p.name}")
+
+ if tokens_seen >= next_export:
+ next_export += cfg.train.export_every_tokens
+ d = cfg.eval.export_dir
+ p = d / f"export_{tokens_seen//10**9:04d}B.pt"
+ ckpt.save_export(p, model=model, cfg=cfg, step=step,
+ tokens_seen=tokens_seen, manifest=cfg.eval.manifest,
+ ngram_fingerprint=ng_fp)
+ log.line(f" export {p.name} -> eval queue")
+
+ if tokens_seen >= next_full:
+ next_full += cfg.train.full_ckpt_every_tokens
+ p = ckpt_dir / f"full_{tokens_seen//10**9:04d}B.pt"
+ ckpt.save_full(p, model=model, muon=muon, adamw=adamw, scaler=scaler,
+ packer=pre, step=step, tokens_seen=tokens_seen,
+ cfg=cfg, ngram_fingerprint=ng_fp, best_val=best_val)
+ log.line(f" full checkpoint {p.name}")
+
+ if not stable_saved and tokens_seen >= stable_end:
+ stable_saved = True
+ p = ckpt_dir / "stable.pt"
+ ckpt.save_full(p, model=model, muon=muon, adamw=adamw, scaler=scaler,
+ packer=pre, step=step, tokens_seen=tokens_seen,
+ cfg=cfg, ngram_fingerprint=ng_fp, best_val=best_val,
+ background=False)
+ log.line(f" END OF STABLE PHASE -> stable.pt (permanent, never rotated)")
+
+ pre.stop()
+ p = ckpt_dir / "final.pt"
+ ckpt.save_full(p, model=model, muon=muon, adamw=adamw, scaler=scaler,
+ packer=pre, step=step, tokens_seen=tokens_seen, cfg=cfg,
+ ngram_fingerprint=ng_fp, best_val=best_val, background=False)
+ log.line(f"\n done: {tokens_seen/1e9:.2f}B tokens in "
+ f"{(time.time()-t_run)/86400:.2f} days -> {p}")
+
+
+@torch.no_grad()
+def quick_val(model, val_packer, cfg, dev, block, theta, progress,
+ batches: int = 16):
+ """A short in-loop validation. The full suite runs on the second GPU."""
+ model.eval()
+ tot, n = 0.0, 0
+ # Score the SAME held-out batches every time. The packer's RNG used to run
+ # on from call to call (and restart at its seed on every resume), so each
+ # validation drew a fresh sample and consecutive values wandered by about
+ # +/-0.05 -- larger than the effects being tracked. Snapshot the state on
+ # the first call and rewind to it on every call.
+ if not hasattr(val_packer, "_boris_val_state0"):
+ val_packer._boris_val_state0 = val_packer.state_dict()
+ val_packer.load_state_dict(val_packer._boris_val_state0)
+ # Stage-appropriate micro-batch: using the 2048-token value at block 8192
+ # would quadruple the validation activation footprint and OOM late in run.
+ mb = cfg.micro_batch_at(progress)
+ for _ in range(batches):
+ xs, ys, pos, bnd, bounds, _ = val_packer.batch(mb, block, 0.0)
+ t = lambda z: torch.from_numpy(z).to(dev)
+ with torch.autocast("cuda", dtype=torch.float16):
+ loss = model(t(xs), t(pos), t(bounds), t(bnd), t(ys), theta)
+ valid = int((ys != -100).sum())
+ tot += float(loss) * valid; n += valid
+ model.train()
+ return tot / max(n, 1)
+
+
+if __name__ == "__main__":
+ main()