summaryrefslogtreecommitdiff
path: root/sft_chat.py
blob: e258e28aefc79bef7160ea15efd4c04c3d5016e5 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
"""Chat SFT for Boris-2 on smol-smoltalk (2026-11-11) -> Boris-2-Instruct.

ChatML-style turns written as plain text (no new tokens, so the 32,768 vocabulary, the n-gram table and the
export path are unchanged):
    <|im_start|>user\n{content}<|im_end|>\n<|im_start|>assistant\n{content}<|im_end|>\n ... <|endoftext|>
Loss only on assistant contents and their closing "<|im_end|>". Conversations are packed as separate documents
into 2048-token rows (attention, Canon and the n-gram lookup reset at each one). Holdout = smol-smoltalk test.

    BORIS_VOLTA_KERNELS=1 python sft_chat.py START.pt OUT.pt [--lr 1e-4 --epochs 1]
"""
import argparse, math, os, random, sys, time
from array import array
from pathlib import Path
import numpy as np, torch
sys.path.insert(0, str(Path(__file__).resolve().parent))
import eval as E
from tokenizer import BorisTokenizer

IGNORE, T = -100, 2048


def encode_conv(msgs, tok, eos):
    """-> (ids, sup): sup[t] = 1 if token t is a training target (assistant content + its <|im_end|>\n)."""
    ids, sup = [], []
    for m in msgs:
        head = tok.encode(f"<|im_start|>{m['role']}\n")
        body = tok.encode(m["content"].strip() + "<|im_end|>") + tok.encode("\n")
        ids += head + body; sup += [0] * len(head) + [int(m["role"] == "assistant")] * len(body)
    ids.append(eos); sup.append(0)
    return ids[:T], sup[:T]


def tokenize(split, tok, eos, cache, limit=None):
    """Flat uint16 tokens + uint8 target mask + int64 starts, cached as .npy (Python int lists took ~25 GB)."""
    if (cache / "ids.npy").exists():
        return tuple(np.load(cache / f"{k}.npy") for k in ("ids", "sup", "start"))
    ids, sup, start = array("H"), array("B"), array("q", [0])
    for i, row in enumerate(split):
        if limit and i >= limit: break
        a, b = encode_conv(row["messages"], tok, eos); ids.extend(a); sup.extend(b); start.append(len(ids))
    out = (np.frombuffer(ids, np.uint16).copy(), np.frombuffer(sup, np.uint8).copy(), np.frombuffer(start, np.int64).copy())
    cache.mkdir(parents=True, exist_ok=True)
    for k, v in zip(("ids", "sup", "start"), out): np.save(cache / f"{k}.npy", v)
    return out


def rows(lens, shuffle, seed):
    order = list(range(len(lens)))
    if shuffle: random.Random(seed).shuffle(order)
    out, cur, n = [], [], 0
    for e in order:
        L = int(lens[e])
        if n + L > T: out.append(cur); cur, n = [], 0
        cur.append(e); n += L
    if cur: out.append(cur)
    return out


def batch(rl, data, eos, dev):
    ids, sup, start = data
    B = len(rl); idx = torch.full((B, T), eos, dtype=torch.long); tgt = torch.full((B, T), IGNORE, dtype=torch.long)
    pos = torch.zeros((B, T), dtype=torch.long); bnd = torch.zeros((B, T), dtype=torch.bool); offs, lens = [0], []
    for b, row in enumerate(rl):
        t = 0
        for e in row:
            s0, s1 = int(start[e]), int(start[e + 1]); L = s1 - s0
            x = torch.from_numpy(ids[s0:s1].astype(np.int64)); m = torch.from_numpy(sup[s0:s1].astype(bool))
            lab = torch.full((L,), IGNORE, dtype=torch.long); lab[:-1] = torch.where(m[1:], x[1:], IGNORE)
            idx[b, t:t + L] = x; tgt[b, t:t + L] = lab                # position t predicts token t+1
            bnd[b, t] = True; pos[b, t:t + L] = torch.arange(L); lens.append(L); t += L; offs.append(b * T + t)
        if t < T:
            bnd[b, t] = True; pos[b, t:] = torch.arange(T - t); lens.append(T - t); offs.append(b * T + T)
    o = torch.tensor(offs, dtype=torch.int64)
    return idx.to(dev), pos.to(dev), o.to(dev), bnd.to(dev), tgt.to(dev), min(lens), max(lens)


def openthink(path, fmt):
    """OpenThink rows -> {"train", "test"} lists of chats. plain: the trace as-is ("... Final answer: X").
    think: "<think>\\n{reasoning}\\n</think>\\n\\nFinal answer: X" (empty think block for answer-only rows).
    Holdout = 800 rows chosen with a fixed seed (same split for every format and seed)."""
    import json
    recs = [json.loads(l) for l in open(path)]
    def reply(t):
        t = t.strip()
        if fmt == "plain": return t
        head, _, ans = t.rpartition("Final answer:")
        return f"<think>\n{head.strip()}\n</think>\n\nFinal answer:{ans}"
    chats = [{"messages": [{"role": "user", "content": r["question"]}, {"role": "assistant", "content": reply(r["trace"])}]} for r in recs]
    order = list(range(len(chats))); random.Random(1234).shuffle(order); hold = set(order[:800])
    return {"train": [c for i, c in enumerate(chats) if i not in hold], "test": [c for i, c in enumerate(chats) if i in hold]}


def main():
    ap = argparse.ArgumentParser(); ap.add_argument("start"); ap.add_argument("out")
    ap.add_argument("--lr", type=float, default=1e-4); ap.add_argument("--epochs", type=int, default=1)
    ap.add_argument("--mb", type=int, default=16); ap.add_argument("--accum", type=int, default=4)
    ap.add_argument("--dev", default="cuda:0"); ap.add_argument("--max-steps", type=int, default=0)
    ap.add_argument("--data", default=None, help="OpenThink jsonl (question/trace) instead of smol-smoltalk")
    ap.add_argument("--fmt", default="plain", choices=("plain", "think")); ap.add_argument("--seed", type=int, default=0)
    ap.add_argument("--cache", default="/media/joseph/Media/instruct/tok")
    ap.add_argument("--mix", type=int, default=0, help="with --data: smol-smoltalk + N copies of the OpenThink train split; "
                    "smol-smoltalk replies get an empty think block when --fmt think")
    a = ap.parse_args(); dev = a.dev; torch.manual_seed(a.seed)
    os.environ.setdefault("HF_DATASETS_CACHE", "/media/joseph/Media/hf-cache/datasets")
    from datasets import load_dataset
    ds = openthink(a.data, a.fmt) if a.data else load_dataset("HuggingFaceTB/smol-smoltalk")
    if a.data and a.mix:
        import itertools
        st = load_dataset("HuggingFaceTB/smol-smoltalk")
        def wrap(c):
            if a.fmt != "think": return c
            return {"messages": [dict(m, content=f"<think>\n\n</think>\n\n{m['content'].strip()}") if m["role"] == "assistant" else m
                                 for m in c["messages"]]}
        ds = {"train": itertools.chain((wrap(c) for c in st["train"]), *[ds["train"]] * a.mix),
              "test": [wrap(c) for c in itertools.islice(st["test"], 800)] + ds["test"]}
    model, cfg, blob = E.load_export(Path(a.start), dev)
    tok = BorisTokenizer.load(Path(cfg.data.root) / "tokenizer"); eos = cfg.data.eos_id
    t0 = time.time()
    C = Path(a.cache)
    tr = tokenize(ds["train"], tok, eos, C / "train"); va = tokenize(ds["test"], tok, eos, C / "test", None if a.mix else 800)
    print(f"tokenized {len(tr[2]) - 1} train conversations ({tr[0].size/1e6:.0f}M tokens, {int(tr[1].sum())/1e6:.0f}M supervised) in {time.time()-t0:.0f}s", flush=True)
    theta = model._eval_theta
    opt = torch.optim.AdamW(model.parameters(), lr=a.lr, betas=(0.9, 0.95), weight_decay=0.0, fused=True)
    scaler = torch.amp.GradScaler("cuda", init_scale=2.0 ** 15)
    R0 = rows(np.diff(tr[2]), False, 0); steps = math.ceil(len(R0) / (a.mb * a.accum)) * a.epochs
    if a.max_steps: steps = min(steps, a.max_steps)
    warm = max(1, steps // 30); VR = rows(np.diff(va[2]), False, 0)

    def val():
        model.eval(); tot = n = 0.0
        with torch.no_grad():
            for i in range(0, len(VR), a.mb):
                x, p, o, bd, y, mn, mx = batch(VR[i:i + a.mb], va, eos, dev)
                with torch.autocast("cuda", dtype=torch.float16):
                    l = model(x, p, o, bd, y, theta, mn, mx)
                k = int((y != IGNORE).sum()); tot += float(l) * k; n += k
        model.train(); return tot / max(n, 1)

    print(f"{steps} steps of {a.mb * a.accum} rows; holdout loss before {val():.4f}", flush=True)
    model.train(); step = 0; t0 = time.time(); seen = 0
    for ep in range(a.epochs):
        R = rows(np.diff(tr[2]), True, 100 + ep + 1000 * a.seed)
        for i in range(0, len(R), a.mb * a.accum):
            if step >= steps: break
            lr = a.lr * min(1, (step + 1) / warm) * 0.5 * (1 + math.cos(math.pi * step / steps))
            for g in opt.param_groups: g["lr"] = lr
            chunk = R[i:i + a.mb * a.accum]
            for j in range(0, len(chunk), a.mb):
                x, p, o, bd, y, mn, mx = batch(chunk[j:j + a.mb], tr, eos, dev)
                with torch.autocast("cuda", dtype=torch.float16):
                    loss = model(x, p, o, bd, y, theta, mn, mx)
                scaler.scale(loss * len(chunk[j:j + a.mb]) / len(chunk)).backward(); seen += x.numel()
            scaler.unscale_(opt); torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
            scaler.step(opt); scaler.update(); opt.zero_grad(set_to_none=True); step += 1
            if step % 50 == 0 or step == steps:
                print(f"step {step}/{steps} loss {float(loss):.4f} lr {lr:.2e} {seen/(time.time()-t0)/1e3:.1f}k tok/s", flush=True)
            if step % 500 == 0: print(f"  holdout loss {val():.4f}", flush=True)
    print(f"final holdout loss {val():.4f}", flush=True)
    keep = set(blob["model"]); sd = model.state_dict()
    blob["model"] = {k: sd[k].detach().float().cpu() for k in keep}
    torch.save(blob, a.out); print("saved", a.out, flush=True)


if __name__ == "__main__":
    main()