From 5d2e4252e695644c34be605b44a14bd80a98a0a6 Mon Sep 17 00:00:00 2001 From: jdj Date: Wed, 18 Nov 2026 14:47:33 -0800 Subject: Boris-2 training code The code, configuration and data filters used to train Boris-2 (144M, 200B tokens, single V100): model, training loop, data preparation, n-gram table, V100 kernels, final stages and the exact run configs. --- scripts/pilot_train.py | 191 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100644 scripts/pilot_train.py (limited to 'scripts/pilot_train.py') diff --git a/scripts/pilot_train.py b/scripts/pilot_train.py new file mode 100644 index 0000000..5d7e85f --- /dev/null +++ b/scripts/pilot_train.py @@ -0,0 +1,191 @@ +"""Pilot branches from a real Boris checkpoint (2026-10-10). + +Generalises branch_decay.py: either GPU, the Volta kernels on the V100, the main +run's batch, a constant or 1-sqrt-decaying LR (optionally to a non-zero floor), +and an optional output-superposition phase (TST, Peng et al. 2026, section 5.2): +inputs stay ordinary tokens, but for the first `--bag-tokens` tokens each position +is trained to predict the bag of its next s tokens -- uniform 1/s multi-hot cross +entropy on the same LM head -- then training reverts to next-token prediction. +Bag targets never cross a document boundary (the packer's IGNORE marks a document's +last position; a target is valid only if every step to it is valid). + +Every arm with the same --seed sees the same token stream. + + python probe/pilot_train.py NAME --tokens 3e8 --mix climbmix:0.95,finemath:0.05 \ + [--dev cuda:0 --mb 16 --tps 524288] [--lr const|decay --final-frac 0.0] \ + [--bag 4 --bag-tokens 1.5e8] --ckpt START.pt --out DIR +""" +from __future__ import annotations + +import argparse, json, math, sys, time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +import torch +import torch.nn.functional as F +from torch.utils.checkpoint import checkpoint + +from config import Config +from data import Prefetcher, VarlenPacker, open_sources +from model import Boris2 +from ngram import NGramVocab +from optim import build_optimizers, set_lr +import checkpoint as ckpt_mod + +IGNORE = -100 + + +def bag_targets(y, s): + """(B,T) next-token targets -> (B,T,s) targets for t+1..t+s and their validity.""" + B, T = y.shape + tg = torch.full((B, T, s), IGNORE, dtype=y.dtype, device=y.device) + ok = torch.zeros(B, T, s, dtype=torch.bool, device=y.device) + run = torch.ones(B, T, dtype=torch.bool, device=y.device) + for j in range(s): + yj = torch.full_like(y, IGNORE) + yj[:, :T - j] = y[:, j:] + run = run & (yj != IGNORE) + tg[:, :, j] = torch.where(run, yj, torch.full_like(yj, IGNORE)) + ok[:, :, j] = run + return tg, ok + + +def _bag_chunk(h, W, tg, ok): + lg = F.linear(h.to(torch.float16), W.to(torch.float16)).float() + lp = lg.gather(1, tg.clamp_min(0)) - torch.logsumexp(lg, -1, keepdim=True) + n = ok.sum(1) + per = (lp * ok).sum(1) / n.clamp_min(1) + return -(per * (n > 0)).sum(), (n > 0).sum() + + +def bag_loss(h, W, y, s, chunk=2048): + tg, ok = bag_targets(y, s) + h, tg, ok = h.reshape(-1, h.shape[-1]), tg.reshape(-1, s), ok.reshape(-1, s) + tot, cnt = 0.0, 0 + for a in range(0, h.shape[0], chunk): + l, c = checkpoint(_bag_chunk, h[a:a + chunk], W, tg[a:a + chunk], ok[a:a + chunk], + use_reentrant=False) + tot = tot + l; cnt = cnt + c + return tot / max(int(cnt), 1) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("name"); ap.add_argument("--tokens", type=float, required=True) + ap.add_argument("--mix", required=True, help="per-document draw weights src:w,...") + ap.add_argument("--dev", default="cuda:1"); ap.add_argument("--mb", type=int, default=2) + ap.add_argument("--tps", type=int, default=131072) + ap.add_argument("--lr", choices=["const", "decay", "wsd"], default="decay") + ap.add_argument("--decay-tokens", type=float, default=0) + ap.add_argument("--final-frac", type=float, default=0.0) + ap.add_argument("--bag", type=int, default=0); ap.add_argument("--bag-tokens", type=float, default=0) + ap.add_argument("--ckpt", required=True); ap.add_argument("--out", required=True) + ap.add_argument("--seed", type=int, default=1234) + ap.add_argument("--peak", type=float, default=1.0, help="multiplier on the config's peak LRs (0.1 = a tenth)") + ap.add_argument("--compile", action="store_true") + ap.add_argument("--max-steps", type=int, default=None) + ap.add_argument("--block", type=int, default=2048, help="sequence length (Boris-2-Nano: 1024)") + ap.add_argument("--mix-before", default=None, help="mix used before --switch-tokens; --mix after it") + ap.add_argument("--switch-tokens", type=float, default=0) + a = ap.parse_args() + dev = a.dev; torch.cuda.set_device(dev) + parse = lambda m: {k: float(v) for k, v in (kv.split(":") for kv in m.split(","))} + mix = parse(a.mix); mix0 = parse(a.mix_before) if a.mix_before else None + + blob = torch.load(a.ckpt, map_location="cpu", weights_only=False) + cfg = Config.from_json(blob["config"]) + t0_tokens, ngfp = blob["tokens_seen"], blob.get("ngram_fingerprint") + allsrc = dict.fromkeys(list(mix) + list(mix0 or {})) + cfg.data.sources = tuple(allsrc) + full = lambda m: {k: m.get(k, 0.0) for k in allsrc} + cfg.curriculum.phases = [["pilot", 1.0, full(mix0 or mix)]] + cfg.curriculum.smooth = False + cfg.optim.lm_head_chunk = min(cfg.optim.lm_head_chunk, 8192 if "cuda:0" in dev else 2048) + cfg.train.device = dev + cfg.train.tokens_per_step = a.tps + cfg.train.micro_batch = {a.block: a.mb, 8192: max(1, a.mb // 4)} + torch.manual_seed(a.seed) + + stores = open_sources(cfg, lambda *x: None) + packer = VarlenPacker(cfg, stores, a.seed, split="train") + model = Boris2(cfg).to(dev) + model.load_state_dict({k: v.float() for k, v in blob["model"].items()}) + if model.ngram is not None: + model.ngram.load_vocab(NGramVocab.load(cfg.ngram.vocab_path), dev) + if a.compile: + model.maybe_compile() + muon, adamw = build_optimizers(model, cfg, lambda *x: None) + if "muon" in blob: # model-only exports start with fresh optimizer state + muon.load_state_dict(blob["muon"]) + adamw.load_state_dict(ckpt_mod._migrate_adamw_groups(blob, model, adamw)) + scaler = torch.amp.GradScaler("cuda", init_scale=2.0 ** 15) + theta = float(cfg.block_size_at(t0_tokens / cfg.train.total_tokens)[1]) + del blob + + accum = a.tps // (a.mb * a.block) + steps = int(a.tokens) // a.tps + if a.max_steps: steps = min(steps, a.max_steps) + bag_steps = int(a.bag_tokens) // a.tps if a.bag else 0 + pre = Prefetcher(packer, cfg, dev); pre.configure(a.mb, a.block, 0.0); pre.start() + from train import gpu_temp, device_index + di = device_index(dev) + W = model.head_weight() + out = Path(a.out); out.mkdir(parents=True, exist_ok=True) + log = open(out / f"{a.name}.log", "a") + t0 = time.time(); tokens = 0 + switch_step = int(a.switch_tokens) // a.tps if mix0 else -1 + for step in range(steps): + if step == switch_step: # change the data mix in place (same packer state otherwise) + pre.stop() + cfg.curriculum.phases = [["pilot", 1.0, full(mix)]] + packer = VarlenPacker(cfg, stores, a.seed + 1, split="train") + pre = Prefetcher(packer, cfg, dev); pre.configure(a.mb, a.block, 0.0); pre.start() + print(f"{a.name} step {step}: switched data mix", flush=True); log.write(f"switched mix at step {step}\n") + if step % 5 == 0 and gpu_temp(di) >= 79: + while gpu_temp(di) > 76: + time.sleep(2) + frac = step / steps + if a.lr == "wsd": + dsteps = max(1, int(a.decay_tokens) // a.tps); d0 = steps - dsteps + scale = 1.0 if step < d0 else a.final_frac + (1 - a.final_frac) * (1.0 - math.sqrt((step - d0) / dsteps)) + else: + scale = 1.0 if a.lr == "const" else a.final_frac + (1 - a.final_frac) * (1.0 - math.sqrt(frac)) + set_lr(muon, adamw, cfg, scale * a.peak) + in_bag = step < bag_steps + acc = torch.zeros((), device=dev) + for _ in range(accum): + x, y, pos, bnd, bounds, st = pre.next(a.mb, a.block) + with torch.autocast("cuda", dtype=torch.float16): + if in_bag: + h = model(x, pos, bounds, bnd, None, theta, st["min_seqlen"], st["max_seqlen"]) + loss = bag_loss(h, W, y, a.bag) + else: + loss = model(x, pos, bounds, bnd, y, theta, st["min_seqlen"], st["max_seqlen"]) + scaler.scale(loss / accum).backward() + acc += loss.detach() / accum; tokens += st["tokens"] + scaler.unscale_(muon); scaler.unscale_(adamw) + gl = [p.grad for p in model.parameters() if p.grad is not None] + gn = float(torch.linalg.vector_norm(torch.stack(torch._foreach_norm(gl)))) + if math.isfinite(gn): + if gn >= cfg.optim.grad_clip: + torch._foreach_mul_(gl, cfg.optim.grad_clip / (gn + 1e-6)) + muon.step(); adamw.step() + scaler.update() + if scaler.get_scale() > 2.0 ** 15: + scaler.update(2.0 ** 15) + muon.zero_grad(set_to_none=True); adamw.zero_grad(set_to_none=True) + if step % 10 == 0 or step == steps - 1: + msg = (f"{a.name} step {step}/{steps} {'BAG' if in_bag else 'ntp'} loss {float(acc):.4f} " + f"gn {gn:.3f} lr_scale {scale:.3f} {tokens/(time.time()-t0)/1e3:.1f}k tok/s " + f"temp {gpu_temp(di)}") + print(msg, flush=True); log.write(msg + "\n"); log.flush() + pre.stop() + exp = out / f"pilot_{a.name}.pt" + torch.save({"model": model.state_dict(), "config": cfg.to_json(), "step": steps, + "tokens_seen": t0_tokens + tokens, "ngram_fingerprint": ngfp}, exp) + print(json.dumps({"name": a.name, "tokens": tokens, "minutes": (time.time() - t0) / 60, + "bag": a.bag, "bag_steps": bag_steps, "lr": a.lr, "export": str(exp)}), flush=True) + + +if __name__ == "__main__": + main() -- cgit v1.2.3