summaryrefslogtreecommitdiff
path: root/preflight.py
diff options
context:
space:
mode:
Diffstat (limited to 'preflight.py')
-rw-r--r--preflight.py238
1 files changed, 238 insertions, 0 deletions
diff --git a/preflight.py b/preflight.py
new file mode 100644
index 0000000..4481b8b
--- /dev/null
+++ b/preflight.py
@@ -0,0 +1,238 @@
+"""Measure throughput before committing weeks to a setting.
+
+ python preflight.py # measure the configured setting
+ python preflight.py --sweep 4,8,16 # micro-batch sweep
+ python preflight.py --ablate # what each optimisation is worth
+ python preflight.py --block 8192 --sweep 1,2,4
+
+Every number in the plan is an estimate until this has run on the actual card
+with the actual shapes. Re-run it after any change to the model or the kernels:
+the optimum micro-batch moves as bottlenecks fall, so an early measurement
+expires.
+"""
+from __future__ import annotations
+
+import argparse
+import math
+import os
+
+os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
+
+import time
+from pathlib import Path
+
+import numpy as np
+import torch
+
+from config import Config
+from model import Boris2
+from ngram import NGramVocab
+from optim import build_optimizers, set_lr
+from train import step_flops
+
+
+def synth_vocab(cfg, seed=0):
+ V = cfg.model.vocab_size
+ rng = np.random.default_rng(seed)
+ bk = np.unique(rng.integers(0, V * V, size=cfg.ngram.bigram_rows).astype(np.int64))
+ tk = np.unique(rng.integers(0, V * V * V, size=cfg.ngram.trigram_rows).astype(np.int64))
+ return NGramVocab(V, bk, np.arange(bk.size) + cfg.ngram.bigram_base,
+ tk, np.arange(tk.size) + cfg.ngram.trigram_base)
+
+
+def make_batch(cfg, dev, B, T, seed=0, worst_case=False):
+ """Realistic packed rows: lognormal document lengths, mean ~560 tokens.
+
+ worst_case makes every row a single document of the full block size. That is
+ the largest attention any batch can produce -- cost is sum(len_i^2), so one
+ 2048-token document costs what fourteen 560-token ones do -- and it is the
+ case the chosen micro_batch has to survive, not the average one.
+ """
+ g = torch.Generator().manual_seed(seed)
+ bounds, pos, bnd = [], np.zeros((B, T), np.int64), np.zeros((B, T), bool)
+ for b in range(B):
+ p = 0
+ while p < T:
+ L = T if worst_case else int(min(T - p, max(32, torch.empty(1).log_normal_(6.4, 0.9, generator=g).item())))
+ bounds.append(b * T + p)
+ bnd[b, p] = True
+ pos[b, p:p + L] = np.arange(L)
+ p += L
+ bounds.append(B * T)
+ lens = np.diff(np.asarray(bounds))
+ V = cfg.model.vocab_size
+ t = lambda a, d=torch.int64: torch.from_numpy(np.ascontiguousarray(a)).to(dev, d)
+ return (torch.randint(0, V, (B, T), device=dev),
+ torch.randint(0, V, (B, T), device=dev),
+ t(pos), t(bnd, torch.bool), t(np.asarray(bounds)),
+ int((lens * lens).sum()), int(lens.min()), int(lens.max()))
+
+
+def build(cfg, dev, compile_regions):
+ cfg.train.compile_regions = compile_regions
+ m = Boris2(cfg).to(dev)
+ if m.ngram is not None:
+ m.ngram.load_vocab(synth_vocab(cfg), dev)
+ if compile_regions:
+ m.maybe_compile()
+ return m
+
+
+def measure(cfg, dev, B, T, iters=6, warmup=3, compile_regions=True, log=print,
+ worst_case=False):
+ torch.cuda.set_device(dev)
+ torch.cuda.empty_cache()
+ torch.cuda.reset_peak_memory_stats(dev)
+ m = build(cfg, dev, compile_regions)
+ muon, adamw = build_optimizers(m, cfg, log=lambda *a: None)
+ scaler = torch.amp.GradScaler("cuda", init_scale=2. ** 16)
+ batches = [make_batch(cfg, dev, B, T, seed=i, worst_case=worst_case)
+ for i in range(3)]
+
+ def one(i):
+ x, y, pos, bnd, off, sumsq, lo, hi = batches[i % len(batches)]
+ with torch.autocast("cuda", dtype=torch.float16):
+ loss = m(x, pos, off, bnd, y, 10_000.0, lo, hi)
+ scaler.scale(loss).backward()
+ scaler.unscale_(muon); scaler.unscale_(adamw)
+ # Match train.py. clip_grad_norm_ plus two scaler.step calls benchmarked
+ # a different synchronization and overflow path than the real run, so
+ # its micro-batch ranking could be wrong by the margin we care about.
+ grads = [p.grad for p in m.parameters() if p.grad is not None]
+ norms = torch._foreach_norm(grads)
+ gnorm = torch.linalg.vector_norm(torch.stack(norms))
+ coef = torch.where(torch.isfinite(gnorm),
+ (cfg.optim.grad_clip / (gnorm + 1e-6)).clamp(max=1.0),
+ torch.ones_like(gnorm))
+ torch._foreach_mul_(grads, coef)
+ if math.isfinite(float(loss.detach())) and math.isfinite(float(gnorm)):
+ muon.step(); adamw.step()
+ scaler.update()
+ muon.zero_grad(set_to_none=True); adamw.zero_grad(set_to_none=True)
+ return sumsq
+
+ for i in range(warmup):
+ one(i)
+ torch.cuda.synchronize()
+ t0 = time.perf_counter()
+ ss = 0
+ for i in range(iters):
+ ss += one(i)
+ torch.cuda.synchronize()
+ dt = time.perf_counter() - t0
+
+ tok = B * T * iters
+ tps = tok / dt
+ fl = step_flops(cfg, tok, ss) / dt / 1e12
+ peak = torch.cuda.max_memory_allocated(dev) / 2 ** 30
+ del m, muon, adamw
+ torch.cuda.empty_cache()
+ return tps, fl, peak
+
+
+def report_row(label, tps, fl, peak, total_tokens, peak_tflops):
+ days = total_tokens / tps / 86400
+ print(f" {label:<22}{tps/1e3:>9.2f}k tok/s{fl:>9.1f} TF"
+ f"{100*fl/peak_tflops:>7.1f}%{peak:>8.1f} GB{days:>9.1f} d")
+
+
+def main():
+ cfg = Config()
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--device", default=cfg.train.device)
+ ap.add_argument("--block", type=int, default=None)
+ ap.add_argument("--sweep", default=None)
+ ap.add_argument("--ablate", action="store_true")
+ ap.add_argument("--worst-case", action="store_true",
+ help="one full-block document per row: the largest attention "
+ "any batch can produce")
+ ap.add_argument("--iters", type=int, default=6)
+ a = ap.parse_args()
+
+ dev = a.device
+ # Set the current device before anything allocates. torch.cuda.empty_cache()
+ # and the allocator statistics act on the CURRENT device, not on the device
+ # the tensors are on -- leaving it at cuda:0 while measuring cuda:1 mixes
+ # two allocators and surfaces as an illegal memory access, not as an error
+ # that names the cause.
+ torch.cuda.set_device(dev)
+ torch.backends.cuda.matmul.allow_tf32 = False
+ torch.backends.cuda.matmul.allow_fp16_reduced_precision_reduction = True
+ torch.backends.cudnn.benchmark = True
+ p = torch.cuda.get_device_properties(dev)
+ # fp16 tensor-core peak: V100 SXM2 125, PCIe 112, and GeForce halves the
+ # fp32-accumulate rate. Only used to turn TFLOPS into a percentage.
+ peak_tflops = 125.0 if p.major == 7 else (2 * p.multi_processor_count * 128 * 2
+ * 1.7e9 / 1e12)
+ T = a.block or cfg.model.context_stages[0][1]
+
+ print(f"\n preflight: {cfg.run_name} on {p.name} (sm_{p.major}{p.minor}, "
+ f"{p.total_memory/2**30:.1f} GB)")
+ print(f" block {T}, step {cfg.train.tokens_per_step:,} tokens, "
+ f"budget {cfg.train.total_tokens/1e9:.0f}B")
+ print(f" assumed fp16 peak {peak_tflops:.0f} TFLOPS\n")
+ print(f" {'setting':<22}{'throughput':>17}{'':>9}{'MFU':>7}{'peak':>8}{'200B in':>9}")
+ print(" " + "-" * 72)
+
+ if a.sweep:
+ for mb in [int(x) for x in a.sweep.split(",")]:
+ if cfg.train.tokens_per_step % (mb * T):
+ print(f" micro_batch {mb:<10}skipped: {mb}x{T} does not divide "
+ f"the {cfg.train.tokens_per_step:,}-token step")
+ continue
+ try:
+ tps, fl, peak = measure(cfg, dev, mb, T, iters=a.iters,
+ worst_case=a.worst_case)
+ report_row(f"micro_batch {mb}" + (" worst-case" if a.worst_case else ""),
+ tps, fl, peak, cfg.train.total_tokens, peak_tflops)
+ except torch.OutOfMemoryError:
+ print(f" micro_batch {mb:<10}OOM")
+ torch.cuda.empty_cache()
+ elif a.ablate:
+ mb = cfg.train.micro_batch[T]
+ variants = [("all optimisations", dict()),
+ ("no compiled regions", dict(compile=False)),
+ ("no fused Canon", dict(canon=False)),
+ ("fp32 residual", dict(fp16res=False)),
+ ("unbatched Newton-Schulz", dict(batched=False))]
+ for label, v in variants:
+ c = Config.from_json(cfg.to_json())
+ if v.get("canon") is False:
+ c.model.use_canon = False
+ if v.get("fp16res") is False:
+ c.train.fp16_residual = False
+ if v.get("batched") is False:
+ c.optim.muon_batched = False
+ try:
+ tps, fl, peak = measure(c, dev, mb, T, iters=a.iters,
+ compile_regions=v.get("compile", True))
+ report_row(label, tps, fl, peak, cfg.train.total_tokens, peak_tflops)
+ except torch.OutOfMemoryError:
+ print(f" {label:<22}OOM"); torch.cuda.empty_cache()
+ else:
+ mb = cfg.train.micro_batch[T]
+ tps, fl, peak = measure(cfg, dev, mb, T, iters=a.iters,
+ worst_case=a.worst_case)
+ report_row(f"micro_batch {mb}" + (" worst-case" if a.worst_case else ""),
+ tps, fl, peak, cfg.train.total_tokens, peak_tflops)
+
+ print(" " + "-" * 72)
+ print(" The '200B in' column extrapolates ONE stage across the whole budget,")
+ print(" which no real run does. The honest projection is two stages:")
+ frac = cfg.model.context_stages[-1][0]
+ t1 = cfg.train.total_tokens * frac
+ t2 = cfg.train.total_tokens - t1
+ print(f" 0-{frac:.0%} {t1/1e9:.0f}B at ctx {cfg.model.context_stages[0][1]} "
+ f"(micro_batch {cfg.train.micro_batch[cfg.model.context_stages[0][1]]})")
+ print(f" {frac:.0%}-100% {t2/1e9:.0f}B at ctx {cfg.model.context_stages[-1][1]} "
+ f"(micro_batch {cfg.train.micro_batch[cfg.model.context_stages[-1][1]]})")
+ print(" and the second stage is much slower per token, because attention")
+ print(" costs sum(len_i^2) and the curriculum deliberately feeds it long")
+ print(" documents once the window opens. Run --block 8192 to measure it.\n")
+ print(" Peak VRAM is measured on lognormal document lengths. Re-run with")
+ print(" --worst-case (one full-block document per row) before committing a")
+ print(" micro_batch: that is the largest attention any batch can produce.\n")
+
+
+if __name__ == "__main__":
+ main()