summaryrefslogtreecommitdiff
path: root/scripts/pilot_train.py
blob: 5d7e85f9d8b6e88f9d99287ca0f2dfa24e7a9adb (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
"""Pilot branches from a real Boris checkpoint (2026-10-10).

Generalises branch_decay.py: either GPU, the Volta kernels on the V100, the main
run's batch, a constant or 1-sqrt-decaying LR (optionally to a non-zero floor),
and an optional output-superposition phase (TST, Peng et al. 2026, section 5.2):
inputs stay ordinary tokens, but for the first `--bag-tokens` tokens each position
is trained to predict the bag of its next s tokens -- uniform 1/s multi-hot cross
entropy on the same LM head -- then training reverts to next-token prediction.
Bag targets never cross a document boundary (the packer's IGNORE marks a document's
last position; a target is valid only if every step to it is valid).

Every arm with the same --seed sees the same token stream.

    python probe/pilot_train.py NAME --tokens 3e8 --mix climbmix:0.95,finemath:0.05 \
        [--dev cuda:0 --mb 16 --tps 524288] [--lr const|decay --final-frac 0.0] \
        [--bag 4 --bag-tokens 1.5e8] --ckpt START.pt --out DIR
"""
from __future__ import annotations

import argparse, json, math, sys, time
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
import torch
import torch.nn.functional as F
from torch.utils.checkpoint import checkpoint

from config import Config
from data import Prefetcher, VarlenPacker, open_sources
from model import Boris2
from ngram import NGramVocab
from optim import build_optimizers, set_lr
import checkpoint as ckpt_mod

IGNORE = -100


def bag_targets(y, s):
    """(B,T) next-token targets -> (B,T,s) targets for t+1..t+s and their validity."""
    B, T = y.shape
    tg = torch.full((B, T, s), IGNORE, dtype=y.dtype, device=y.device)
    ok = torch.zeros(B, T, s, dtype=torch.bool, device=y.device)
    run = torch.ones(B, T, dtype=torch.bool, device=y.device)
    for j in range(s):
        yj = torch.full_like(y, IGNORE)
        yj[:, :T - j] = y[:, j:]
        run = run & (yj != IGNORE)
        tg[:, :, j] = torch.where(run, yj, torch.full_like(yj, IGNORE))
        ok[:, :, j] = run
    return tg, ok


def _bag_chunk(h, W, tg, ok):
    lg = F.linear(h.to(torch.float16), W.to(torch.float16)).float()
    lp = lg.gather(1, tg.clamp_min(0)) - torch.logsumexp(lg, -1, keepdim=True)
    n = ok.sum(1)
    per = (lp * ok).sum(1) / n.clamp_min(1)
    return -(per * (n > 0)).sum(), (n > 0).sum()


def bag_loss(h, W, y, s, chunk=2048):
    tg, ok = bag_targets(y, s)
    h, tg, ok = h.reshape(-1, h.shape[-1]), tg.reshape(-1, s), ok.reshape(-1, s)
    tot, cnt = 0.0, 0
    for a in range(0, h.shape[0], chunk):
        l, c = checkpoint(_bag_chunk, h[a:a + chunk], W, tg[a:a + chunk], ok[a:a + chunk],
                          use_reentrant=False)
        tot = tot + l; cnt = cnt + c
    return tot / max(int(cnt), 1)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("name"); ap.add_argument("--tokens", type=float, required=True)
    ap.add_argument("--mix", required=True, help="per-document draw weights src:w,...")
    ap.add_argument("--dev", default="cuda:1"); ap.add_argument("--mb", type=int, default=2)
    ap.add_argument("--tps", type=int, default=131072)
    ap.add_argument("--lr", choices=["const", "decay", "wsd"], default="decay")
    ap.add_argument("--decay-tokens", type=float, default=0)
    ap.add_argument("--final-frac", type=float, default=0.0)
    ap.add_argument("--bag", type=int, default=0); ap.add_argument("--bag-tokens", type=float, default=0)
    ap.add_argument("--ckpt", required=True); ap.add_argument("--out", required=True)
    ap.add_argument("--seed", type=int, default=1234)
    ap.add_argument("--peak", type=float, default=1.0, help="multiplier on the config's peak LRs (0.1 = a tenth)")
    ap.add_argument("--compile", action="store_true")
    ap.add_argument("--max-steps", type=int, default=None)
    ap.add_argument("--block", type=int, default=2048, help="sequence length (Boris-2-Nano: 1024)")
    ap.add_argument("--mix-before", default=None, help="mix used before --switch-tokens; --mix after it")
    ap.add_argument("--switch-tokens", type=float, default=0)
    a = ap.parse_args()
    dev = a.dev; torch.cuda.set_device(dev)
    parse = lambda m: {k: float(v) for k, v in (kv.split(":") for kv in m.split(","))}
    mix = parse(a.mix); mix0 = parse(a.mix_before) if a.mix_before else None

    blob = torch.load(a.ckpt, map_location="cpu", weights_only=False)
    cfg = Config.from_json(blob["config"])
    t0_tokens, ngfp = blob["tokens_seen"], blob.get("ngram_fingerprint")
    allsrc = dict.fromkeys(list(mix) + list(mix0 or {}))
    cfg.data.sources = tuple(allsrc)
    full = lambda m: {k: m.get(k, 0.0) for k in allsrc}
    cfg.curriculum.phases = [["pilot", 1.0, full(mix0 or mix)]]
    cfg.curriculum.smooth = False
    cfg.optim.lm_head_chunk = min(cfg.optim.lm_head_chunk, 8192 if "cuda:0" in dev else 2048)
    cfg.train.device = dev
    cfg.train.tokens_per_step = a.tps
    cfg.train.micro_batch = {a.block: a.mb, 8192: max(1, a.mb // 4)}
    torch.manual_seed(a.seed)

    stores = open_sources(cfg, lambda *x: None)
    packer = VarlenPacker(cfg, stores, a.seed, split="train")
    model = Boris2(cfg).to(dev)
    model.load_state_dict({k: v.float() for k, v in blob["model"].items()})
    if model.ngram is not None:
        model.ngram.load_vocab(NGramVocab.load(cfg.ngram.vocab_path), dev)
    if a.compile:
        model.maybe_compile()
    muon, adamw = build_optimizers(model, cfg, lambda *x: None)
    if "muon" in blob:                               # model-only exports start with fresh optimizer state
        muon.load_state_dict(blob["muon"])
        adamw.load_state_dict(ckpt_mod._migrate_adamw_groups(blob, model, adamw))
    scaler = torch.amp.GradScaler("cuda", init_scale=2.0 ** 15)
    theta = float(cfg.block_size_at(t0_tokens / cfg.train.total_tokens)[1])
    del blob

    accum = a.tps // (a.mb * a.block)
    steps = int(a.tokens) // a.tps
    if a.max_steps: steps = min(steps, a.max_steps)
    bag_steps = int(a.bag_tokens) // a.tps if a.bag else 0
    pre = Prefetcher(packer, cfg, dev); pre.configure(a.mb, a.block, 0.0); pre.start()
    from train import gpu_temp, device_index
    di = device_index(dev)
    W = model.head_weight()
    out = Path(a.out); out.mkdir(parents=True, exist_ok=True)
    log = open(out / f"{a.name}.log", "a")
    t0 = time.time(); tokens = 0
    switch_step = int(a.switch_tokens) // a.tps if mix0 else -1
    for step in range(steps):
        if step == switch_step:                      # change the data mix in place (same packer state otherwise)
            pre.stop()
            cfg.curriculum.phases = [["pilot", 1.0, full(mix)]]
            packer = VarlenPacker(cfg, stores, a.seed + 1, split="train")
            pre = Prefetcher(packer, cfg, dev); pre.configure(a.mb, a.block, 0.0); pre.start()
            print(f"{a.name} step {step}: switched data mix", flush=True); log.write(f"switched mix at step {step}\n")
        if step % 5 == 0 and gpu_temp(di) >= 79:
            while gpu_temp(di) > 76:
                time.sleep(2)
        frac = step / steps
        if a.lr == "wsd":
            dsteps = max(1, int(a.decay_tokens) // a.tps); d0 = steps - dsteps
            scale = 1.0 if step < d0 else a.final_frac + (1 - a.final_frac) * (1.0 - math.sqrt((step - d0) / dsteps))
        else:
            scale = 1.0 if a.lr == "const" else a.final_frac + (1 - a.final_frac) * (1.0 - math.sqrt(frac))
        set_lr(muon, adamw, cfg, scale * a.peak)
        in_bag = step < bag_steps
        acc = torch.zeros((), device=dev)
        for _ in range(accum):
            x, y, pos, bnd, bounds, st = pre.next(a.mb, a.block)
            with torch.autocast("cuda", dtype=torch.float16):
                if in_bag:
                    h = model(x, pos, bounds, bnd, None, theta, st["min_seqlen"], st["max_seqlen"])
                    loss = bag_loss(h, W, y, a.bag)
                else:
                    loss = model(x, pos, bounds, bnd, y, theta, st["min_seqlen"], st["max_seqlen"])
            scaler.scale(loss / accum).backward()
            acc += loss.detach() / accum; tokens += st["tokens"]
        scaler.unscale_(muon); scaler.unscale_(adamw)
        gl = [p.grad for p in model.parameters() if p.grad is not None]
        gn = float(torch.linalg.vector_norm(torch.stack(torch._foreach_norm(gl))))
        if math.isfinite(gn):
            if gn >= cfg.optim.grad_clip:
                torch._foreach_mul_(gl, cfg.optim.grad_clip / (gn + 1e-6))
            muon.step(); adamw.step()
        scaler.update()
        if scaler.get_scale() > 2.0 ** 15:
            scaler.update(2.0 ** 15)
        muon.zero_grad(set_to_none=True); adamw.zero_grad(set_to_none=True)
        if step % 10 == 0 or step == steps - 1:
            msg = (f"{a.name} step {step}/{steps} {'BAG' if in_bag else 'ntp'} loss {float(acc):.4f} "
                   f"gn {gn:.3f} lr_scale {scale:.3f} {tokens/(time.time()-t0)/1e3:.1f}k tok/s "
                   f"temp {gpu_temp(di)}")
            print(msg, flush=True); log.write(msg + "\n"); log.flush()
    pre.stop()
    exp = out / f"pilot_{a.name}.pt"
    torch.save({"model": model.state_dict(), "config": cfg.to_json(), "step": steps,
                "tokens_seen": t0_tokens + tokens, "ngram_fingerprint": ngfp}, exp)
    print(json.dumps({"name": a.name, "tokens": tokens, "minutes": (time.time() - t0) / 60,
                      "bag": a.bag, "bag_steps": bag_steps, "lr": a.lr, "export": str(exp)}), flush=True)


if __name__ == "__main__":
    main()