"""Measure throughput before committing weeks to a setting. python preflight.py # measure the configured setting python preflight.py --sweep 4,8,16 # micro-batch sweep python preflight.py --ablate # what each optimisation is worth python preflight.py --block 8192 --sweep 1,2,4 Every number in the plan is an estimate until this has run on the actual card with the actual shapes. Re-run it after any change to the model or the kernels: the optimum micro-batch moves as bottlenecks fall, so an early measurement expires. """ from __future__ import annotations import argparse import math import os os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") import time from pathlib import Path import numpy as np import torch from config import Config from model import Boris2 from ngram import NGramVocab from optim import build_optimizers, set_lr from train import step_flops def synth_vocab(cfg, seed=0): V = cfg.model.vocab_size rng = np.random.default_rng(seed) bk = np.unique(rng.integers(0, V * V, size=cfg.ngram.bigram_rows).astype(np.int64)) tk = np.unique(rng.integers(0, V * V * V, size=cfg.ngram.trigram_rows).astype(np.int64)) return NGramVocab(V, bk, np.arange(bk.size) + cfg.ngram.bigram_base, tk, np.arange(tk.size) + cfg.ngram.trigram_base) def make_batch(cfg, dev, B, T, seed=0, worst_case=False): """Realistic packed rows: lognormal document lengths, mean ~560 tokens. worst_case makes every row a single document of the full block size. That is the largest attention any batch can produce -- cost is sum(len_i^2), so one 2048-token document costs what fourteen 560-token ones do -- and it is the case the chosen micro_batch has to survive, not the average one. """ g = torch.Generator().manual_seed(seed) bounds, pos, bnd = [], np.zeros((B, T), np.int64), np.zeros((B, T), bool) for b in range(B): p = 0 while p < T: L = T if worst_case else int(min(T - p, max(32, torch.empty(1).log_normal_(6.4, 0.9, generator=g).item()))) bounds.append(b * T + p) bnd[b, p] = True pos[b, p:p + L] = np.arange(L) p += L bounds.append(B * T) lens = np.diff(np.asarray(bounds)) V = cfg.model.vocab_size t = lambda a, d=torch.int64: torch.from_numpy(np.ascontiguousarray(a)).to(dev, d) return (torch.randint(0, V, (B, T), device=dev), torch.randint(0, V, (B, T), device=dev), t(pos), t(bnd, torch.bool), t(np.asarray(bounds)), int((lens * lens).sum()), int(lens.min()), int(lens.max())) def build(cfg, dev, compile_regions): cfg.train.compile_regions = compile_regions m = Boris2(cfg).to(dev) if m.ngram is not None: m.ngram.load_vocab(synth_vocab(cfg), dev) if compile_regions: m.maybe_compile() return m def measure(cfg, dev, B, T, iters=6, warmup=3, compile_regions=True, log=print, worst_case=False): torch.cuda.set_device(dev) torch.cuda.empty_cache() torch.cuda.reset_peak_memory_stats(dev) m = build(cfg, dev, compile_regions) muon, adamw = build_optimizers(m, cfg, log=lambda *a: None) scaler = torch.amp.GradScaler("cuda", init_scale=2. ** 16) batches = [make_batch(cfg, dev, B, T, seed=i, worst_case=worst_case) for i in range(3)] def one(i): x, y, pos, bnd, off, sumsq, lo, hi = batches[i % len(batches)] with torch.autocast("cuda", dtype=torch.float16): loss = m(x, pos, off, bnd, y, 10_000.0, lo, hi) scaler.scale(loss).backward() scaler.unscale_(muon); scaler.unscale_(adamw) # Match train.py. clip_grad_norm_ plus two scaler.step calls benchmarked # a different synchronization and overflow path than the real run, so # its micro-batch ranking could be wrong by the margin we care about. grads = [p.grad for p in m.parameters() if p.grad is not None] norms = torch._foreach_norm(grads) gnorm = torch.linalg.vector_norm(torch.stack(norms)) coef = torch.where(torch.isfinite(gnorm), (cfg.optim.grad_clip / (gnorm + 1e-6)).clamp(max=1.0), torch.ones_like(gnorm)) torch._foreach_mul_(grads, coef) if math.isfinite(float(loss.detach())) and math.isfinite(float(gnorm)): muon.step(); adamw.step() scaler.update() muon.zero_grad(set_to_none=True); adamw.zero_grad(set_to_none=True) return sumsq for i in range(warmup): one(i) torch.cuda.synchronize() t0 = time.perf_counter() ss = 0 for i in range(iters): ss += one(i) torch.cuda.synchronize() dt = time.perf_counter() - t0 tok = B * T * iters tps = tok / dt fl = step_flops(cfg, tok, ss) / dt / 1e12 peak = torch.cuda.max_memory_allocated(dev) / 2 ** 30 del m, muon, adamw torch.cuda.empty_cache() return tps, fl, peak def report_row(label, tps, fl, peak, total_tokens, peak_tflops): days = total_tokens / tps / 86400 print(f" {label:<22}{tps/1e3:>9.2f}k tok/s{fl:>9.1f} TF" f"{100*fl/peak_tflops:>7.1f}%{peak:>8.1f} GB{days:>9.1f} d") def main(): cfg = Config() ap = argparse.ArgumentParser() ap.add_argument("--device", default=cfg.train.device) ap.add_argument("--block", type=int, default=None) ap.add_argument("--sweep", default=None) ap.add_argument("--ablate", action="store_true") ap.add_argument("--worst-case", action="store_true", help="one full-block document per row: the largest attention " "any batch can produce") ap.add_argument("--iters", type=int, default=6) a = ap.parse_args() dev = a.device # Set the current device before anything allocates. torch.cuda.empty_cache() # and the allocator statistics act on the CURRENT device, not on the device # the tensors are on -- leaving it at cuda:0 while measuring cuda:1 mixes # two allocators and surfaces as an illegal memory access, not as an error # that names the cause. torch.cuda.set_device(dev) torch.backends.cuda.matmul.allow_tf32 = False torch.backends.cuda.matmul.allow_fp16_reduced_precision_reduction = True torch.backends.cudnn.benchmark = True p = torch.cuda.get_device_properties(dev) # fp16 tensor-core peak: V100 SXM2 125, PCIe 112, and GeForce halves the # fp32-accumulate rate. Only used to turn TFLOPS into a percentage. peak_tflops = 125.0 if p.major == 7 else (2 * p.multi_processor_count * 128 * 2 * 1.7e9 / 1e12) T = a.block or cfg.model.context_stages[0][1] print(f"\n preflight: {cfg.run_name} on {p.name} (sm_{p.major}{p.minor}, " f"{p.total_memory/2**30:.1f} GB)") print(f" block {T}, step {cfg.train.tokens_per_step:,} tokens, " f"budget {cfg.train.total_tokens/1e9:.0f}B") print(f" assumed fp16 peak {peak_tflops:.0f} TFLOPS\n") print(f" {'setting':<22}{'throughput':>17}{'':>9}{'MFU':>7}{'peak':>8}{'200B in':>9}") print(" " + "-" * 72) if a.sweep: for mb in [int(x) for x in a.sweep.split(",")]: if cfg.train.tokens_per_step % (mb * T): print(f" micro_batch {mb:<10}skipped: {mb}x{T} does not divide " f"the {cfg.train.tokens_per_step:,}-token step") continue try: tps, fl, peak = measure(cfg, dev, mb, T, iters=a.iters, worst_case=a.worst_case) report_row(f"micro_batch {mb}" + (" worst-case" if a.worst_case else ""), tps, fl, peak, cfg.train.total_tokens, peak_tflops) except torch.OutOfMemoryError: print(f" micro_batch {mb:<10}OOM") torch.cuda.empty_cache() elif a.ablate: mb = cfg.train.micro_batch[T] variants = [("all optimisations", dict()), ("no compiled regions", dict(compile=False)), ("no fused Canon", dict(canon=False)), ("fp32 residual", dict(fp16res=False)), ("unbatched Newton-Schulz", dict(batched=False))] for label, v in variants: c = Config.from_json(cfg.to_json()) if v.get("canon") is False: c.model.use_canon = False if v.get("fp16res") is False: c.train.fp16_residual = False if v.get("batched") is False: c.optim.muon_batched = False try: tps, fl, peak = measure(c, dev, mb, T, iters=a.iters, compile_regions=v.get("compile", True)) report_row(label, tps, fl, peak, cfg.train.total_tokens, peak_tflops) except torch.OutOfMemoryError: print(f" {label:<22}OOM"); torch.cuda.empty_cache() else: mb = cfg.train.micro_batch[T] tps, fl, peak = measure(cfg, dev, mb, T, iters=a.iters, worst_case=a.worst_case) report_row(f"micro_batch {mb}" + (" worst-case" if a.worst_case else ""), tps, fl, peak, cfg.train.total_tokens, peak_tflops) print(" " + "-" * 72) print(" The '200B in' column extrapolates ONE stage across the whole budget,") print(" which no real run does. The honest projection is two stages:") frac = cfg.model.context_stages[-1][0] t1 = cfg.train.total_tokens * frac t2 = cfg.train.total_tokens - t1 print(f" 0-{frac:.0%} {t1/1e9:.0f}B at ctx {cfg.model.context_stages[0][1]} " f"(micro_batch {cfg.train.micro_batch[cfg.model.context_stages[0][1]]})") print(f" {frac:.0%}-100% {t2/1e9:.0f}B at ctx {cfg.model.context_stages[-1][1]} " f"(micro_batch {cfg.train.micro_batch[cfg.model.context_stages[-1][1]]})") print(" and the second stage is much slower per token, because attention") print(" costs sum(len_i^2) and the curriculum deliberately feeds it long") print(" documents once the window opens. Run --block 8192 to measure it.\n") print(" Peak VRAM is measured on lognormal document lengths. Re-run with") print(" --worst-case (one full-block document per row) before committing a") print(" micro_batch: that is the largest attention any batch can produce.\n") if __name__ == "__main__": main()