diff options
Diffstat (limited to 'preflight.py')
| -rw-r--r-- | preflight.py | 238 |
1 files changed, 238 insertions, 0 deletions
diff --git a/preflight.py b/preflight.py new file mode 100644 index 0000000..4481b8b --- /dev/null +++ b/preflight.py @@ -0,0 +1,238 @@ +"""Measure throughput before committing weeks to a setting. + + python preflight.py # measure the configured setting + python preflight.py --sweep 4,8,16 # micro-batch sweep + python preflight.py --ablate # what each optimisation is worth + python preflight.py --block 8192 --sweep 1,2,4 + +Every number in the plan is an estimate until this has run on the actual card +with the actual shapes. Re-run it after any change to the model or the kernels: +the optimum micro-batch moves as bottlenecks fall, so an early measurement +expires. +""" +from __future__ import annotations + +import argparse +import math +import os + +os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") + +import time +from pathlib import Path + +import numpy as np +import torch + +from config import Config +from model import Boris2 +from ngram import NGramVocab +from optim import build_optimizers, set_lr +from train import step_flops + + +def synth_vocab(cfg, seed=0): + V = cfg.model.vocab_size + rng = np.random.default_rng(seed) + bk = np.unique(rng.integers(0, V * V, size=cfg.ngram.bigram_rows).astype(np.int64)) + tk = np.unique(rng.integers(0, V * V * V, size=cfg.ngram.trigram_rows).astype(np.int64)) + return NGramVocab(V, bk, np.arange(bk.size) + cfg.ngram.bigram_base, + tk, np.arange(tk.size) + cfg.ngram.trigram_base) + + +def make_batch(cfg, dev, B, T, seed=0, worst_case=False): + """Realistic packed rows: lognormal document lengths, mean ~560 tokens. + + worst_case makes every row a single document of the full block size. That is + the largest attention any batch can produce -- cost is sum(len_i^2), so one + 2048-token document costs what fourteen 560-token ones do -- and it is the + case the chosen micro_batch has to survive, not the average one. + """ + g = torch.Generator().manual_seed(seed) + bounds, pos, bnd = [], np.zeros((B, T), np.int64), np.zeros((B, T), bool) + for b in range(B): + p = 0 + while p < T: + L = T if worst_case else int(min(T - p, max(32, torch.empty(1).log_normal_(6.4, 0.9, generator=g).item()))) + bounds.append(b * T + p) + bnd[b, p] = True + pos[b, p:p + L] = np.arange(L) + p += L + bounds.append(B * T) + lens = np.diff(np.asarray(bounds)) + V = cfg.model.vocab_size + t = lambda a, d=torch.int64: torch.from_numpy(np.ascontiguousarray(a)).to(dev, d) + return (torch.randint(0, V, (B, T), device=dev), + torch.randint(0, V, (B, T), device=dev), + t(pos), t(bnd, torch.bool), t(np.asarray(bounds)), + int((lens * lens).sum()), int(lens.min()), int(lens.max())) + + +def build(cfg, dev, compile_regions): + cfg.train.compile_regions = compile_regions + m = Boris2(cfg).to(dev) + if m.ngram is not None: + m.ngram.load_vocab(synth_vocab(cfg), dev) + if compile_regions: + m.maybe_compile() + return m + + +def measure(cfg, dev, B, T, iters=6, warmup=3, compile_regions=True, log=print, + worst_case=False): + torch.cuda.set_device(dev) + torch.cuda.empty_cache() + torch.cuda.reset_peak_memory_stats(dev) + m = build(cfg, dev, compile_regions) + muon, adamw = build_optimizers(m, cfg, log=lambda *a: None) + scaler = torch.amp.GradScaler("cuda", init_scale=2. ** 16) + batches = [make_batch(cfg, dev, B, T, seed=i, worst_case=worst_case) + for i in range(3)] + + def one(i): + x, y, pos, bnd, off, sumsq, lo, hi = batches[i % len(batches)] + with torch.autocast("cuda", dtype=torch.float16): + loss = m(x, pos, off, bnd, y, 10_000.0, lo, hi) + scaler.scale(loss).backward() + scaler.unscale_(muon); scaler.unscale_(adamw) + # Match train.py. clip_grad_norm_ plus two scaler.step calls benchmarked + # a different synchronization and overflow path than the real run, so + # its micro-batch ranking could be wrong by the margin we care about. + grads = [p.grad for p in m.parameters() if p.grad is not None] + norms = torch._foreach_norm(grads) + gnorm = torch.linalg.vector_norm(torch.stack(norms)) + coef = torch.where(torch.isfinite(gnorm), + (cfg.optim.grad_clip / (gnorm + 1e-6)).clamp(max=1.0), + torch.ones_like(gnorm)) + torch._foreach_mul_(grads, coef) + if math.isfinite(float(loss.detach())) and math.isfinite(float(gnorm)): + muon.step(); adamw.step() + scaler.update() + muon.zero_grad(set_to_none=True); adamw.zero_grad(set_to_none=True) + return sumsq + + for i in range(warmup): + one(i) + torch.cuda.synchronize() + t0 = time.perf_counter() + ss = 0 + for i in range(iters): + ss += one(i) + torch.cuda.synchronize() + dt = time.perf_counter() - t0 + + tok = B * T * iters + tps = tok / dt + fl = step_flops(cfg, tok, ss) / dt / 1e12 + peak = torch.cuda.max_memory_allocated(dev) / 2 ** 30 + del m, muon, adamw + torch.cuda.empty_cache() + return tps, fl, peak + + +def report_row(label, tps, fl, peak, total_tokens, peak_tflops): + days = total_tokens / tps / 86400 + print(f" {label:<22}{tps/1e3:>9.2f}k tok/s{fl:>9.1f} TF" + f"{100*fl/peak_tflops:>7.1f}%{peak:>8.1f} GB{days:>9.1f} d") + + +def main(): + cfg = Config() + ap = argparse.ArgumentParser() + ap.add_argument("--device", default=cfg.train.device) + ap.add_argument("--block", type=int, default=None) + ap.add_argument("--sweep", default=None) + ap.add_argument("--ablate", action="store_true") + ap.add_argument("--worst-case", action="store_true", + help="one full-block document per row: the largest attention " + "any batch can produce") + ap.add_argument("--iters", type=int, default=6) + a = ap.parse_args() + + dev = a.device + # Set the current device before anything allocates. torch.cuda.empty_cache() + # and the allocator statistics act on the CURRENT device, not on the device + # the tensors are on -- leaving it at cuda:0 while measuring cuda:1 mixes + # two allocators and surfaces as an illegal memory access, not as an error + # that names the cause. + torch.cuda.set_device(dev) + torch.backends.cuda.matmul.allow_tf32 = False + torch.backends.cuda.matmul.allow_fp16_reduced_precision_reduction = True + torch.backends.cudnn.benchmark = True + p = torch.cuda.get_device_properties(dev) + # fp16 tensor-core peak: V100 SXM2 125, PCIe 112, and GeForce halves the + # fp32-accumulate rate. Only used to turn TFLOPS into a percentage. + peak_tflops = 125.0 if p.major == 7 else (2 * p.multi_processor_count * 128 * 2 + * 1.7e9 / 1e12) + T = a.block or cfg.model.context_stages[0][1] + + print(f"\n preflight: {cfg.run_name} on {p.name} (sm_{p.major}{p.minor}, " + f"{p.total_memory/2**30:.1f} GB)") + print(f" block {T}, step {cfg.train.tokens_per_step:,} tokens, " + f"budget {cfg.train.total_tokens/1e9:.0f}B") + print(f" assumed fp16 peak {peak_tflops:.0f} TFLOPS\n") + print(f" {'setting':<22}{'throughput':>17}{'':>9}{'MFU':>7}{'peak':>8}{'200B in':>9}") + print(" " + "-" * 72) + + if a.sweep: + for mb in [int(x) for x in a.sweep.split(",")]: + if cfg.train.tokens_per_step % (mb * T): + print(f" micro_batch {mb:<10}skipped: {mb}x{T} does not divide " + f"the {cfg.train.tokens_per_step:,}-token step") + continue + try: + tps, fl, peak = measure(cfg, dev, mb, T, iters=a.iters, + worst_case=a.worst_case) + report_row(f"micro_batch {mb}" + (" worst-case" if a.worst_case else ""), + tps, fl, peak, cfg.train.total_tokens, peak_tflops) + except torch.OutOfMemoryError: + print(f" micro_batch {mb:<10}OOM") + torch.cuda.empty_cache() + elif a.ablate: + mb = cfg.train.micro_batch[T] + variants = [("all optimisations", dict()), + ("no compiled regions", dict(compile=False)), + ("no fused Canon", dict(canon=False)), + ("fp32 residual", dict(fp16res=False)), + ("unbatched Newton-Schulz", dict(batched=False))] + for label, v in variants: + c = Config.from_json(cfg.to_json()) + if v.get("canon") is False: + c.model.use_canon = False + if v.get("fp16res") is False: + c.train.fp16_residual = False + if v.get("batched") is False: + c.optim.muon_batched = False + try: + tps, fl, peak = measure(c, dev, mb, T, iters=a.iters, + compile_regions=v.get("compile", True)) + report_row(label, tps, fl, peak, cfg.train.total_tokens, peak_tflops) + except torch.OutOfMemoryError: + print(f" {label:<22}OOM"); torch.cuda.empty_cache() + else: + mb = cfg.train.micro_batch[T] + tps, fl, peak = measure(cfg, dev, mb, T, iters=a.iters, + worst_case=a.worst_case) + report_row(f"micro_batch {mb}" + (" worst-case" if a.worst_case else ""), + tps, fl, peak, cfg.train.total_tokens, peak_tflops) + + print(" " + "-" * 72) + print(" The '200B in' column extrapolates ONE stage across the whole budget,") + print(" which no real run does. The honest projection is two stages:") + frac = cfg.model.context_stages[-1][0] + t1 = cfg.train.total_tokens * frac + t2 = cfg.train.total_tokens - t1 + print(f" 0-{frac:.0%} {t1/1e9:.0f}B at ctx {cfg.model.context_stages[0][1]} " + f"(micro_batch {cfg.train.micro_batch[cfg.model.context_stages[0][1]]})") + print(f" {frac:.0%}-100% {t2/1e9:.0f}B at ctx {cfg.model.context_stages[-1][1]} " + f"(micro_batch {cfg.train.micro_batch[cfg.model.context_stages[-1][1]]})") + print(" and the second stage is much slower per token, because attention") + print(" costs sum(len_i^2) and the curriculum deliberately feeds it long") + print(" documents once the window opens. Run --block 8192 to measure it.\n") + print(" Peak VRAM is measured on lognormal document lengths. Re-run with") + print(" --worst-case (one full-block document per row) before committing a") + print(" micro_batch: that is the largest attention any batch can produce.\n") + + +if __name__ == "__main__": + main() |
