"""Boris-2 pretraining loop: 200B tokens, fp16, single V100. Run: python train.py # fresh python train.py --resume # from the newest rolling checkpoint python train.py --resume path.pt # from a specific one The loop assumes it will be interrupted. Everything needed to continue -- the weights, both optimizer states, the loss scale, the packer's RNG and its carried document tail -- goes into a rolling checkpoint every 45 minutes, and a resume replays the identical token stream from there. """ from __future__ import annotations import os os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") import argparse import json import math import subprocess import sys import time from datetime import datetime, timezone from pathlib import Path import torch from config import LOG_DIR, Config import checkpoint as ckpt from data import Prefetcher, VarlenPacker, open_sources, open_validation from model import Boris2 from ngram import NGramVocab from optim import build_optimizers, set_lr from schedule import (context_at, lr_scale, mixture_at, phase_at, stable_end_tokens) from tokenizer import BorisTokenizer # --------------------------------------------------------------------------- # def assert_environment(cfg: Config, log): """Catch a silently-wrong precision setup in seconds rather than on day 30.""" if not torch.cuda.is_available(): raise SystemExit("no CUDA device") dev = torch.device(cfg.train.device) p = torch.cuda.get_device_properties(dev) log(f" device {p.name} sm_{p.major}{p.minor} " f"{p.total_memory/2**30:.1f} GB {p.multi_processor_count} SMs") if p.major != 7: log(f" WARNING: this run is tuned for sm_70; found sm_{p.major}{p.minor}") assert cfg.train.amp_dtype == "float16", ( "sm_70 tensor cores are fp16-in/fp32-accumulate and have no bf16 path; " "bf16 would run in software and TF32 does not exist before sm_80") torch.backends.cuda.matmul.allow_tf32 = False torch.backends.cudnn.allow_tf32 = False torch.backends.cudnn.benchmark = True # Measured +7.4% end to end on this card. It lets cuBLAS reduce fp16 GEMM # split-k partials in fp16; checked against fp32 references in verify.py, # where the LM head gradients are unaffected at 9e-04 / 2e-04. It is the # default, and it is set explicitly so a future torch cannot flip it # silently and cost a week. torch.backends.cuda.matmul.allow_fp16_reduced_precision_reduction = True # Prove autocast is actually engaged and producing fp16, rather than the # loop quietly running fp32 for weeks at a quarter of the throughput. a = torch.randn(256, 256, device=dev) with torch.autocast("cuda", dtype=torch.float16): out = a @ a assert out.dtype == torch.float16, ( f"autocast produced {out.dtype}: fp16 is not engaged") log(f" precision fp16 autocast + GradScaler, TF32 off, bf16 unused") _NVML = None def _nvml_temp(index: int) -> float: """Temperature through libnvidia-ml directly: ~18 us, versus ~14 ms to fork nvidia-smi (whose indices are NVML's, so the numbering is the same).""" global _NVML import ctypes if _NVML is None: lib = ctypes.CDLL("libnvidia-ml.so.1") if lib.nvmlInit_v2() != 0: raise RuntimeError("nvmlInit failed") _NVML = (lib, {}) lib, handles = _NVML if index not in handles: h = ctypes.c_void_p() if lib.nvmlDeviceGetHandleByIndex_v2(index, ctypes.byref(h)) != 0: raise RuntimeError("nvml handle failed") handles[index] = h t = ctypes.c_uint() if lib.nvmlDeviceGetTemperature(handles[index], 0, ctypes.byref(t)) != 0: raise RuntimeError("nvml temperature read failed") return float(t.value) def gpu_temp(index: int = 0) -> float: try: return _nvml_temp(index) except Exception: pass try: r = subprocess.run(["nvidia-smi", f"--id={index}", "--query-gpu=temperature.gpu", "--format=csv,noheader"], capture_output=True, text=True, timeout=4) return float(r.stdout.strip().splitlines()[0]) except Exception: return float("nan") def device_index(device: str | torch.device) -> int: """Physical CUDA index used by diagnostics; never assume cuda:0.""" d = torch.device(device) return torch.cuda.current_device() if d.index is None else d.index class Logger: def __init__(self, path: Path): path.parent.mkdir(parents=True, exist_ok=True) self.f = open(path, "a") def line(self, s: str): print(s, flush=True) self.f.write(json.dumps({"msg": s, "timestamp": datetime.now(timezone.utc).isoformat()}) + "\n") self.f.flush() def row(self, d: dict): d = {**d, "timestamp": datetime.now(timezone.utc).isoformat()} self.f.write(json.dumps(d) + "\n") self.f.flush() def step_flops(cfg: Config, tokens: int, sumsq: int) -> float: """Exact fwd+bwd FLOPs for a step, attention included at what it really did. Attention is varlen, so its cost is sum(len_i^2) over the documents actually packed -- not block_size^2. Reporting the dense figure would overstate the model's utilisation by roughly the block sparsity, which at ctx 2048 is 57%. """ m = cfg.model b = cfg.param_budget() matmul = 2 * b["matmul"] * tokens head = 2 * m.vocab_size * m.d_model * tokens attn = m.n_layer * 2 * (m.n_head * m.head_dim) * sumsq return 3.0 * (matmul + head + attn) # --------------------------------------------------------------------------- # def main(): ap = argparse.ArgumentParser() ap.add_argument("--resume", nargs="?", const="auto", default=None) ap.add_argument("--config", type=Path, default=None) ap.add_argument("--max-steps", type=int, default=None) ap.add_argument("--force-config", action="store_true") ap.add_argument("--no-compile", action="store_true") a = ap.parse_args() cfg = Config.from_json(a.config.read_text()) if a.config else Config() ckpt_dir = Path(cfg.checkpoint_dir) if not a.resume and ckpt_dir.exists() and any(ckpt_dir.glob("*.pt")): raise SystemExit("checkpoint directory is nonempty: use --resume or a new directory") if a.no_compile: cfg.train.compile_regions = False log = Logger(LOG_DIR / f"{cfg.run_name}.jsonl") log.line(f"\n {cfg.run_name}") log.line(" " + "=" * 74) assert_environment(cfg, log.line) torch.manual_seed(cfg.train.seed) dev = cfg.train.device torch.cuda.set_device(dev) tok_dir = Path(cfg.data.root) / "tokenizer" tok_fp = BorisTokenizer.fingerprint(tok_dir) if tok_fp != cfg.model.tokenizer_sha256: raise SystemExit( f"tokenizer fingerprint {tok_fp} at {tok_dir} does not match " f"config {cfg.model.tokenizer_sha256}; token ids and embedding rows " "would disagree") # -- data ---------------------------------------------------------- # stores = open_sources(cfg, log.line) packer = VarlenPacker(cfg, stores, cfg.train.seed, split="train") # The held-out split is just another prepared source directory, and it is # deliberately not in cfg.data.sources, so the training packer can never # draw from it. val_packer = None if (Path(cfg.data.root) / cfg.data.val_source / "tokens.bin").exists(): vcfg = Config.from_json(cfg.to_json()) vcfg.curriculum.enabled = False val_packer = VarlenPacker(vcfg, {"val": open_validation(cfg)}, 12345, split="val") log.line(f" val split " f"{val_packer.stores['val'].n_tokens/1e6:.1f}M tokens held out") # -- model --------------------------------------------------------- # model = Boris2(cfg).to(dev) if model.ngram is not None: vocab = NGramVocab.load(cfg.ngram.vocab_path) assert vocab.vocab_size == cfg.model.vocab_size, ( f"ngram_vocab.bin was built for vocab {vocab.vocab_size}, model is " f"{cfg.model.vocab_size}") model.ngram.load_vocab(vocab, dev) ng_fp = vocab.fingerprint() ng_compatible = (ng_fp, vocab.legacy_fingerprint()) log.line(f" ngram vocab {vocab.bigram_keys.size:,} bigram + " f"{vocab.trigram_keys.size:,} trigram rows, fingerprint {ng_fp}") else: ng_fp = "disabled" ng_compatible = (ng_fp,) log.line(" ngram table OFF (use_ngram=False): dense control arm") if not a.no_compile: model.maybe_compile() muon, adamw = build_optimizers(model, cfg, log.line) scaler = torch.amp.GradScaler("cuda", init_scale=cfg.train.init_scale) # -- resume -------------------------------------------------------- # step, tokens_seen, best_val = 0, 0, float("inf") bad_grad = bad_loss = floored = 0 # consecutive non-finite log intervals if a.resume: path = (ckpt.latest_rolling(ckpt_dir) if a.resume == "auto" else Path(a.resume)) if path is None: raise SystemExit("--resume requested but no rolling checkpoint exists") else: blob = ckpt.load_full( path, model=model, muon=muon, adamw=adamw, scaler=scaler, packer=packer, map_location=dev, expect_fingerprint=None if a.force_config else cfg.fingerprint(), expect_ngram=None if a.force_config else ng_compatible) step, tokens_seen = blob["step"], blob["tokens_seen"] best_val = blob.get("best_val", float("inf")) log.line(f" resumed {path.name} at step {step:,}, " f"{tokens_seen/1e9:.2f}B tokens") # -- run ------------------------------------------------------------ # # The run ends when the token budget is spent, not at a fixed step count: tokens_per_step may change at a # resume (2026-11-05: doubled for the final decay). Step numbers keep counting up, because rolling # checkpoints are named and pruned by step. For an unchanged batch this equals cfg.total_steps. total_steps = a.max_steps or (step + max(0, cfg.train.total_tokens - tokens_seen) // cfg.train.tokens_per_step) progress = tokens_seen / cfg.train.total_tokens block, theta, mb, accum = context_at(progress, cfg) pre = Prefetcher(packer, cfg, dev) pre.configure(mb, block, progress) pre.start() log.line(f" budget {cfg.train.total_tokens/1e9:.0f}B tokens over " f"{cfg.total_steps:,} steps of {cfg.train.tokens_per_step:,}") log.line(f" context block {block}, theta {theta:,.0f}, " f"micro_batch {mb} x {accum} accum") log.line(" " + "=" * 74 + "\n") t_run = time.time() t_ckpt = time.time() next_export = ((tokens_seen // cfg.train.export_every_tokens) + 1) * cfg.train.export_every_tokens next_full = ((tokens_seen // cfg.train.full_ckpt_every_tokens) + 1) * cfg.train.full_ckpt_every_tokens next_val = ((tokens_seen // cfg.train.val_every_tokens) + 1) * cfg.train.val_every_tokens stable_end = stable_end_tokens(cfg) stable_saved = tokens_seen >= stable_end attempt = step win_t, win_tok, win_loss, win_flops = time.time(), 0, 0.0, 0.0 win_steps = 0 retry_batches = [] consecutive_skips = 0 temp_poll = 0 bad_temp_reads = 0 temperature = gpu_temp(device_index(dev)) while step < total_steps: # Leave headroom below the user's 80 C cap without changing power limits. # gpu_temp() reads NVML directly (~18 us), so every step is free again; # it falls back to forking nvidia-smi (~14 ms), where every 5th step # keeps the idle GPU time small. temp_poll += 1 if _NVML is not None or temp_poll % 5 == 1: temperature = gpu_temp(device_index(dev)) # One failed nvidia-smi read used to end the run; tolerate a few in a # row (still refusing to train unmonitored for long). bad_temp_reads = 0 if math.isfinite(temperature) else bad_temp_reads + 1 if bad_temp_reads >= 3: raise RuntimeError("cannot read GPU temperature 3 times running; " "refusing unmonitored training") if math.isfinite(temperature) and temperature >= 78: # The sensor flickers 77 <-> 78 at steady load, so a single reading # of 78 paused the run for 5 s of idle GPU. Confirm it a second # later; the thresholds (pause at 78, resume at 75, cap 80) are # unchanged and a real climb cannot gain 2 C in a second. time.sleep(1) temperature = gpu_temp(device_index(dev)) if math.isfinite(temperature) and temperature >= 78: log.line(f"thermal pause at {temperature:.0f} C; waiting for <=75 C") fails = 0 while not (math.isfinite(temperature) and temperature <= 75): time.sleep(1) temperature = gpu_temp(device_index(dev)) fails = 0 if math.isfinite(temperature) else fails + 1 if fails >= 3: raise RuntimeError("GPU temperature unavailable during cooldown") progress = tokens_seen / cfg.train.total_tokens nb, ntheta, nmb, naccum = context_at(progress, cfg) if (nb, nmb) != (block, mb): log.line(f"\n context stage block {block} -> {nb}, " f"theta {theta:,.0f} -> {ntheta:,.0f}, " f"micro_batch {mb} -> {nmb} (one recompile expected)\n") block, theta, mb, accum = nb, ntheta, nmb, naccum pre.configure(mb, block, progress) torch.cuda.empty_cache() else: theta = ntheta pre.configure(mb, block, progress) scale = lr_scale(progress, cfg) mlr, alr = set_lr(muon, adamw, cfg, scale) # Accumulated on the GPU. Reading it per micro-step with float() forces # a host sync 16 times a step and destroys CPU run-ahead; it is read # once, at log time, well after the fact. loss_acc = torch.zeros((), device=dev) tok_acc, sumsq_acc = 0, 0 for micro in range(accum): if micro == len(retry_batches): retry_batches.append(pre.next(mb, block)) x, y, pos, bnd, bounds, stats = retry_batches[micro] with torch.autocast("cuda", dtype=torch.float16): loss = model(x, pos, bounds, bnd, y, theta, stats["min_seqlen"], stats["max_seqlen"]) scaler.scale(loss / accum).backward() loss_acc += loss.detach() / accum tok_acc += stats["tokens"] sumsq_acc += stats["sumsq"] grad_list = [p.grad for p in model.parameters() if p.grad is not None] scaler.unscale_(muon) scaler.unscale_(adamw) # Not clip_grad_norm_: it scales every gradient by a coefficient derived # from the norm over ALL parameters, so a single Inf anywhere -- routine # on the first steps, where init_scale is 2^16 and GradScaler is meant # to overflow, back off and skip -- makes the coefficient non-finite and # writes NaN into gradients that were fine. found_inf is recorded per # optimizer at unscale_ time, so Muon would skip while AdamW stepped on # the freshly-poisoned grads, and 0 * NaN = NaN defeats even the lr-0 # warmup step. That is exactly how this run died: step 1 loss 10.4739, # step 2 loss nan. Leave the gradients untouched when the norm is not # finite and let GradScaler do its job. norms = torch._foreach_norm(grad_list) gnorm = torch.linalg.vector_norm(torch.stack(norms)) coef = torch.where(torch.isfinite(gnorm), (cfg.optim.grad_clip / (gnorm + 1e-6)).clamp(max=1.0), torch.ones_like(gnorm)) # A tensor coefficient, and torch.where rather than a Python branch: # reading gnorm here would be a host sync on every step. pre_scale = scaler.get_scale() # GradScaler makes its skip decision per optimizer. That allowed a head # overflow to skip AdamW while Muon still changed the trunk: a partial # update that desynchronised the model. Both optimizers see unscaled # gradients here, so make one global finite decision and step both or # neither. The scalar reads replace the loss read already performed # below; they do not add another synchronization point. loss_value = float(loss_acc) gnorm_value = float(gnorm) # coef is exactly 1.0 unless the norm reaches the clip (it never has in # this run), and x * 1.0 == x, so skipping the multiply is bitwise the # same update. It is applied, as before, whenever clipping engages. if math.isfinite(gnorm_value) and gnorm_value + 1e-6 >= cfg.optim.grad_clip: torch._foreach_mul_(grad_list, coef) did_step = math.isfinite(loss_value) and math.isfinite(gnorm_value) if did_step: muon.step() adamw.step() scaler.update() # A non-finite LOSS is a forward overflow, and halving the loss scale # cannot touch it -- the forward does not depend on the scale at all. # GradScaler halves anyway, every time, and that ratchet is what walked # two runs down to the floor. Restore the scale on those steps: the step # is still skipped (found_inf), so the weights are safe, and a rare # forward NaN costs exactly one step instead of compounding. Genuine # gradient overflow, where the loss is finite, still halves normally. # One host sync per step, not the sixteen a per-micro-batch check costs. if not math.isfinite(loss_value): scaler.update(pre_scale) # Clamp the loss scale at both ends. GradScaler will otherwise run the # scale down without limit, and below ~1e-30 unscale_ (grad/scale) # overflows fp32 to inf, sets found_inf, and halves it again -- a # self-reinforcing collapse with no exit, which is what ended the run at # step 3,810 with the weights still perfectly finite. The ceiling is the # other half: the scale had grown to 131,072 before the first overflow. _s = scaler.get_scale() if _s < cfg.train.min_loss_scale: scaler.update(cfg.train.min_loss_scale) floored += 1 elif _s > cfg.train.max_loss_scale: scaler.update(cfg.train.max_loss_scale) floored = 0 else: floored = 0 muon.zero_grad(set_to_none=True) adamw.zero_grad(set_to_none=True) if did_step: step += 1 tokens_seen += tok_acc retry_batches.clear() consecutive_skips = 0 else: consecutive_skips += 1 # A non-finite LOSS is a forward overflow: deterministic, so retrying # the identical batch only repeats it until the 20-skip abort. Move # on to fresh data. Gradient overflow (finite loss) still retries the # same batch, where the halved loss scale changes the outcome. if not math.isfinite(loss_value): retry_batches.clear() log.row({"event": "skipped_update", "step": step, "attempt": attempt + 1, "loss": loss_value, "grad_norm": gnorm_value, "scale": scaler.get_scale(), "consecutive_skips": consecutive_skips}) if consecutive_skips >= 20: log.line(f"ABORT: 20 consecutive failed updates at step {step}; same batch retried") ckpt.save_full(ckpt_dir / f"abort_{step:09d}.pt", model=model, muon=muon, adamw=adamw, scaler=scaler, packer=pre, step=step, tokens_seen=tokens_seen, cfg=cfg, ngram_fingerprint=ng_fp, best_val=best_val, background=False) raise SystemExit(3) attempt += 1 win_tok += tok_acc win_loss += loss_acc win_steps += 1 win_flops += step_flops(cfg, tok_acc, sumsq_acc) # -- logging ---------------------------------------------------- # if attempt % cfg.train.log_every_steps == 0: dt = time.time() - win_t tps = win_tok / max(dt, 1e-9) tflops = win_flops / max(dt, 1e-9) / 1e12 eta = (cfg.train.total_tokens - tokens_seen) / max(tps, 1e-9) temp = gpu_temp(device_index(dev)) mem = torch.cuda.max_memory_allocated() / 2**30 log.row({"step": step, "attempt": attempt, "tokens": tokens_seen, "loss": float(win_loss) / max(win_steps, 1), "lr_muon": mlr, "lr_adamw": alr, "grad_norm": gnorm_value, "scale": scaler.get_scale(), "tok_s": tps, "tflops": tflops, "block": block, "temp": temp, "vram_gb": mem, "phase": phase_at(progress, cfg), "wall": time.time() - t_run}) print(f" step {step:>7,}/{total_steps:,} {progress:>6.2%} " f"loss {float(win_loss)/max(win_steps, 1):6.4f} " f"lr {mlr:.2e} gn {gnorm_value:5.2f} " f"s{math.log2(max(scaler.get_scale(), 1e-45)):+03.0f} " f"{tps/1e3:6.1f}k tok/s {tflops:5.1f} TF " f"{temp:.0f}C {mem:4.1f}G " f"eta {eta/86400:5.1f}d", flush=True) # -- health ------------------------------------------------- # # A non-finite grad norm is normal in ones and twos: GradScaler # probes for its loss scale by overflowing on purpose. PERSISTENT # non-finite is a different animal -- it means found_inf fires every # step and an optimizer is being skipped every step. That is what # happened at step 230 of the first launch: the head's dW GEMM # overflowed fp16 independently of the loss scale, so AdamW never # stepped again, the embeddings and head froze while Muon kept # training the trunk, and the loss climbed from 6.48 to nan over # 180 steps and then burned 3,300 more. Neither the temperature nor # the throughput showed anything wrong. Stop instead of cooking. bad_grad = 0 if math.isfinite(gnorm_value) else bad_grad + 1 # Divide by the steps ACTUALLY in this window, not the nominal # window size. A resume lands mid-window (step 57,340 with # log_every_steps=10), so the first line after every resume # under-reported the loss by up to 10x -- 0.337 for a model # whose real loss was 3.37. win_mean = float(win_loss) / max(win_steps, 1) bad_loss = 0 if math.isfinite(win_mean) else bad_loss + 1 if bad_loss * cfg.train.log_every_steps >= cfg.train.abort_after_bad: print(f"\n ABORT: loss has been non-finite for " f"{bad_loss * cfg.train.log_every_steps} steps.\n", flush=True) ckpt.save_full(ckpt_dir / f"abort_{step:09d}.pt", model=model, muon=muon, adamw=adamw, scaler=scaler, packer=pre, step=step, tokens_seen=tokens_seen, cfg=cfg, ngram_fingerprint=ng_fp, best_val=best_val, background=False) raise SystemExit(3) if floored >= cfg.train.abort_after_floored: print(f"\n ABORT: the loss scale has been pinned at its floor " f"({cfg.train.min_loss_scale}) for {floored} steps -- " f"every step is being skipped and nothing is training.\n", flush=True) ckpt.save_full(ckpt_dir / f"abort_{step:09d}.pt", model=model, muon=muon, adamw=adamw, scaler=scaler, packer=pre, step=step, tokens_seen=tokens_seen, cfg=cfg, ngram_fingerprint=ng_fp, best_val=best_val, background=False) raise SystemExit(3) if bad_grad * cfg.train.log_every_steps >= cfg.train.abort_after_bad: print(f"\n ABORT: the gradient norm has been non-finite for " f"{bad_grad * cfg.train.log_every_steps} steps -- an " f"optimizer is being skipped every step.\n", flush=True) ckpt.save_full(ckpt_dir / f"abort_{step:09d}.pt", model=model, muon=muon, adamw=adamw, scaler=scaler, packer=pre, step=step, tokens_seen=tokens_seen, cfg=cfg, ngram_fingerprint=ng_fp, best_val=best_val, background=False) raise SystemExit(3) win_t, win_tok, win_loss, win_flops = time.time(), 0, 0.0, 0.0 win_steps = 0 # -- validation ------------------------------------------------- # if val_packer is not None and tokens_seen >= next_val: next_val += cfg.train.val_every_tokens vl = quick_val(model, val_packer, cfg, dev, block, theta, progress) best_val = min(best_val, vl) log.row({"step": step, "tokens": tokens_seen, "val_loss": vl, "val_ppl": math.exp(min(vl, 20))}) print(f" val {vl:.4f} ppl {math.exp(min(vl,20)):.2f}", flush=True) # -- checkpoints ------------------------------------------------- # if (time.time() - t_ckpt) / 60 >= cfg.train.rolling_ckpt_minutes: t_ckpt = time.time() p = ckpt_dir / f"rolling_{step:09d}.pt" ckpt.save_full(p, model=model, muon=muon, adamw=adamw, scaler=scaler, packer=pre, step=step, tokens_seen=tokens_seen, cfg=cfg, ngram_fingerprint=ng_fp, best_val=best_val, rotate_pattern=r"rolling_\d+\.pt$", rotate_keep=cfg.train.rolling_ckpt_keep) log.line(f" rolling checkpoint {p.name}") if tokens_seen >= next_export: next_export += cfg.train.export_every_tokens d = cfg.eval.export_dir p = d / f"export_{tokens_seen//10**9:04d}B.pt" ckpt.save_export(p, model=model, cfg=cfg, step=step, tokens_seen=tokens_seen, manifest=cfg.eval.manifest, ngram_fingerprint=ng_fp) log.line(f" export {p.name} -> eval queue") if tokens_seen >= next_full: next_full += cfg.train.full_ckpt_every_tokens p = ckpt_dir / f"full_{tokens_seen//10**9:04d}B.pt" ckpt.save_full(p, model=model, muon=muon, adamw=adamw, scaler=scaler, packer=pre, step=step, tokens_seen=tokens_seen, cfg=cfg, ngram_fingerprint=ng_fp, best_val=best_val) log.line(f" full checkpoint {p.name}") if not stable_saved and tokens_seen >= stable_end: stable_saved = True p = ckpt_dir / "stable.pt" ckpt.save_full(p, model=model, muon=muon, adamw=adamw, scaler=scaler, packer=pre, step=step, tokens_seen=tokens_seen, cfg=cfg, ngram_fingerprint=ng_fp, best_val=best_val, background=False) log.line(f" END OF STABLE PHASE -> stable.pt (permanent, never rotated)") pre.stop() p = ckpt_dir / "final.pt" ckpt.save_full(p, model=model, muon=muon, adamw=adamw, scaler=scaler, packer=pre, step=step, tokens_seen=tokens_seen, cfg=cfg, ngram_fingerprint=ng_fp, best_val=best_val, background=False) log.line(f"\n done: {tokens_seen/1e9:.2f}B tokens in " f"{(time.time()-t_run)/86400:.2f} days -> {p}") @torch.no_grad() def quick_val(model, val_packer, cfg, dev, block, theta, progress, batches: int = 16): """A short in-loop validation. The full suite runs on the second GPU.""" model.eval() tot, n = 0.0, 0 # Score the SAME held-out batches every time. The packer's RNG used to run # on from call to call (and restart at its seed on every resume), so each # validation drew a fresh sample and consecutive values wandered by about # +/-0.05 -- larger than the effects being tracked. Snapshot the state on # the first call and rewind to it on every call. if not hasattr(val_packer, "_boris_val_state0"): val_packer._boris_val_state0 = val_packer.state_dict() val_packer.load_state_dict(val_packer._boris_val_state0) # Stage-appropriate micro-batch: using the 2048-token value at block 8192 # would quadruple the validation activation footprint and OOM late in run. mb = cfg.micro_batch_at(progress) for _ in range(batches): xs, ys, pos, bnd, bounds, _ = val_packer.batch(mb, block, 0.0) t = lambda z: torch.from_numpy(z).to(dev) with torch.autocast("cuda", dtype=torch.float16): loss = model(t(xs), t(pos), t(bounds), t(bnd), t(ys), theta) valid = int((ys != -100).sum()) tot += float(loss) * valid; n += valid model.train() return tot / max(n, 1) if __name__ == "__main__": main()