1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
|
"""Pilot branches from a real Boris checkpoint (2026-10-10).
Generalises branch_decay.py: either GPU, the Volta kernels on the V100, the main
run's batch, a constant or 1-sqrt-decaying LR (optionally to a non-zero floor),
and an optional output-superposition phase (TST, Peng et al. 2026, section 5.2):
inputs stay ordinary tokens, but for the first `--bag-tokens` tokens each position
is trained to predict the bag of its next s tokens -- uniform 1/s multi-hot cross
entropy on the same LM head -- then training reverts to next-token prediction.
Bag targets never cross a document boundary (the packer's IGNORE marks a document's
last position; a target is valid only if every step to it is valid).
Every arm with the same --seed sees the same token stream.
python probe/pilot_train.py NAME --tokens 3e8 --mix climbmix:0.95,finemath:0.05 \
[--dev cuda:0 --mb 16 --tps 524288] [--lr const|decay --final-frac 0.0] \
[--bag 4 --bag-tokens 1.5e8] --ckpt START.pt --out DIR
"""
from __future__ import annotations
import argparse, json, math, sys, time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
import torch
import torch.nn.functional as F
from torch.utils.checkpoint import checkpoint
from config import Config
from data import Prefetcher, VarlenPacker, open_sources
from model import Boris2
from ngram import NGramVocab
from optim import build_optimizers, set_lr
import checkpoint as ckpt_mod
IGNORE = -100
def bag_targets(y, s):
"""(B,T) next-token targets -> (B,T,s) targets for t+1..t+s and their validity."""
B, T = y.shape
tg = torch.full((B, T, s), IGNORE, dtype=y.dtype, device=y.device)
ok = torch.zeros(B, T, s, dtype=torch.bool, device=y.device)
run = torch.ones(B, T, dtype=torch.bool, device=y.device)
for j in range(s):
yj = torch.full_like(y, IGNORE)
yj[:, :T - j] = y[:, j:]
run = run & (yj != IGNORE)
tg[:, :, j] = torch.where(run, yj, torch.full_like(yj, IGNORE))
ok[:, :, j] = run
return tg, ok
def _bag_chunk(h, W, tg, ok):
lg = F.linear(h.to(torch.float16), W.to(torch.float16)).float()
lp = lg.gather(1, tg.clamp_min(0)) - torch.logsumexp(lg, -1, keepdim=True)
n = ok.sum(1)
per = (lp * ok).sum(1) / n.clamp_min(1)
return -(per * (n > 0)).sum(), (n > 0).sum()
def bag_loss(h, W, y, s, chunk=2048):
tg, ok = bag_targets(y, s)
h, tg, ok = h.reshape(-1, h.shape[-1]), tg.reshape(-1, s), ok.reshape(-1, s)
tot, cnt = 0.0, 0
for a in range(0, h.shape[0], chunk):
l, c = checkpoint(_bag_chunk, h[a:a + chunk], W, tg[a:a + chunk], ok[a:a + chunk],
use_reentrant=False)
tot = tot + l; cnt = cnt + c
return tot / max(int(cnt), 1)
def main():
ap = argparse.ArgumentParser()
ap.add_argument("name"); ap.add_argument("--tokens", type=float, required=True)
ap.add_argument("--mix", required=True, help="per-document draw weights src:w,...")
ap.add_argument("--dev", default="cuda:1"); ap.add_argument("--mb", type=int, default=2)
ap.add_argument("--tps", type=int, default=131072)
ap.add_argument("--lr", choices=["const", "decay", "wsd"], default="decay")
ap.add_argument("--decay-tokens", type=float, default=0)
ap.add_argument("--final-frac", type=float, default=0.0)
ap.add_argument("--bag", type=int, default=0); ap.add_argument("--bag-tokens", type=float, default=0)
ap.add_argument("--ckpt", required=True); ap.add_argument("--out", required=True)
ap.add_argument("--seed", type=int, default=1234)
ap.add_argument("--peak", type=float, default=1.0, help="multiplier on the config's peak LRs (0.1 = a tenth)")
ap.add_argument("--compile", action="store_true")
ap.add_argument("--max-steps", type=int, default=None)
ap.add_argument("--block", type=int, default=2048, help="sequence length (Boris-2-Nano: 1024)")
ap.add_argument("--mix-before", default=None, help="mix used before --switch-tokens; --mix after it")
ap.add_argument("--switch-tokens", type=float, default=0)
a = ap.parse_args()
dev = a.dev; torch.cuda.set_device(dev)
parse = lambda m: {k: float(v) for k, v in (kv.split(":") for kv in m.split(","))}
mix = parse(a.mix); mix0 = parse(a.mix_before) if a.mix_before else None
blob = torch.load(a.ckpt, map_location="cpu", weights_only=False)
cfg = Config.from_json(blob["config"])
t0_tokens, ngfp = blob["tokens_seen"], blob.get("ngram_fingerprint")
allsrc = dict.fromkeys(list(mix) + list(mix0 or {}))
cfg.data.sources = tuple(allsrc)
full = lambda m: {k: m.get(k, 0.0) for k in allsrc}
cfg.curriculum.phases = [["pilot", 1.0, full(mix0 or mix)]]
cfg.curriculum.smooth = False
cfg.optim.lm_head_chunk = min(cfg.optim.lm_head_chunk, 8192 if "cuda:0" in dev else 2048)
cfg.train.device = dev
cfg.train.tokens_per_step = a.tps
cfg.train.micro_batch = {a.block: a.mb, 8192: max(1, a.mb // 4)}
torch.manual_seed(a.seed)
stores = open_sources(cfg, lambda *x: None)
packer = VarlenPacker(cfg, stores, a.seed, split="train")
model = Boris2(cfg).to(dev)
model.load_state_dict({k: v.float() for k, v in blob["model"].items()})
if model.ngram is not None:
model.ngram.load_vocab(NGramVocab.load(cfg.ngram.vocab_path), dev)
if a.compile:
model.maybe_compile()
muon, adamw = build_optimizers(model, cfg, lambda *x: None)
if "muon" in blob: # model-only exports start with fresh optimizer state
muon.load_state_dict(blob["muon"])
adamw.load_state_dict(ckpt_mod._migrate_adamw_groups(blob, model, adamw))
scaler = torch.amp.GradScaler("cuda", init_scale=2.0 ** 15)
theta = float(cfg.block_size_at(t0_tokens / cfg.train.total_tokens)[1])
del blob
accum = a.tps // (a.mb * a.block)
steps = int(a.tokens) // a.tps
if a.max_steps: steps = min(steps, a.max_steps)
bag_steps = int(a.bag_tokens) // a.tps if a.bag else 0
pre = Prefetcher(packer, cfg, dev); pre.configure(a.mb, a.block, 0.0); pre.start()
from train import gpu_temp, device_index
di = device_index(dev)
W = model.head_weight()
out = Path(a.out); out.mkdir(parents=True, exist_ok=True)
log = open(out / f"{a.name}.log", "a")
t0 = time.time(); tokens = 0
switch_step = int(a.switch_tokens) // a.tps if mix0 else -1
for step in range(steps):
if step == switch_step: # change the data mix in place (same packer state otherwise)
pre.stop()
cfg.curriculum.phases = [["pilot", 1.0, full(mix)]]
packer = VarlenPacker(cfg, stores, a.seed + 1, split="train")
pre = Prefetcher(packer, cfg, dev); pre.configure(a.mb, a.block, 0.0); pre.start()
print(f"{a.name} step {step}: switched data mix", flush=True); log.write(f"switched mix at step {step}\n")
if step % 5 == 0 and gpu_temp(di) >= 79:
while gpu_temp(di) > 76:
time.sleep(2)
frac = step / steps
if a.lr == "wsd":
dsteps = max(1, int(a.decay_tokens) // a.tps); d0 = steps - dsteps
scale = 1.0 if step < d0 else a.final_frac + (1 - a.final_frac) * (1.0 - math.sqrt((step - d0) / dsteps))
else:
scale = 1.0 if a.lr == "const" else a.final_frac + (1 - a.final_frac) * (1.0 - math.sqrt(frac))
set_lr(muon, adamw, cfg, scale * a.peak)
in_bag = step < bag_steps
acc = torch.zeros((), device=dev)
for _ in range(accum):
x, y, pos, bnd, bounds, st = pre.next(a.mb, a.block)
with torch.autocast("cuda", dtype=torch.float16):
if in_bag:
h = model(x, pos, bounds, bnd, None, theta, st["min_seqlen"], st["max_seqlen"])
loss = bag_loss(h, W, y, a.bag)
else:
loss = model(x, pos, bounds, bnd, y, theta, st["min_seqlen"], st["max_seqlen"])
scaler.scale(loss / accum).backward()
acc += loss.detach() / accum; tokens += st["tokens"]
scaler.unscale_(muon); scaler.unscale_(adamw)
gl = [p.grad for p in model.parameters() if p.grad is not None]
gn = float(torch.linalg.vector_norm(torch.stack(torch._foreach_norm(gl))))
if math.isfinite(gn):
if gn >= cfg.optim.grad_clip:
torch._foreach_mul_(gl, cfg.optim.grad_clip / (gn + 1e-6))
muon.step(); adamw.step()
scaler.update()
if scaler.get_scale() > 2.0 ** 15:
scaler.update(2.0 ** 15)
muon.zero_grad(set_to_none=True); adamw.zero_grad(set_to_none=True)
if step % 10 == 0 or step == steps - 1:
msg = (f"{a.name} step {step}/{steps} {'BAG' if in_bag else 'ntp'} loss {float(acc):.4f} "
f"gn {gn:.3f} lr_scale {scale:.3f} {tokens/(time.time()-t0)/1e3:.1f}k tok/s "
f"temp {gpu_temp(di)}")
print(msg, flush=True); log.write(msg + "\n"); log.flush()
pre.stop()
exp = out / f"pilot_{a.name}.pt"
torch.save({"model": model.state_dict(), "config": cfg.to_json(), "step": steps,
"tokens_seen": t0_tokens + tokens, "ngram_fingerprint": ngfp}, exp)
print(json.dumps({"name": a.name, "tokens": tokens, "minutes": (time.time() - t0) / 60,
"bag": a.bag, "bag_steps": bag_steps, "lr": a.lr, "export": str(exp)}), flush=True)
if __name__ == "__main__":
main()
|