"""Chat SFT for Boris-2 on smol-smoltalk (2026-11-11) -> Boris-2-Instruct. ChatML-style turns written as plain text (no new tokens, so the 32,768 vocabulary, the n-gram table and the export path are unchanged): <|im_start|>user\n{content}<|im_end|>\n<|im_start|>assistant\n{content}<|im_end|>\n ... <|endoftext|> Loss only on assistant contents and their closing "<|im_end|>". Conversations are packed as separate documents into 2048-token rows (attention, Canon and the n-gram lookup reset at each one). Holdout = smol-smoltalk test. BORIS_VOLTA_KERNELS=1 python sft_chat.py START.pt OUT.pt [--lr 1e-4 --epochs 1] """ import argparse, math, os, random, sys, time from array import array from pathlib import Path import numpy as np, torch sys.path.insert(0, str(Path(__file__).resolve().parent)) import eval as E from tokenizer import BorisTokenizer IGNORE, T = -100, 2048 def encode_conv(msgs, tok, eos): """-> (ids, sup): sup[t] = 1 if token t is a training target (assistant content + its <|im_end|>\n).""" ids, sup = [], [] for m in msgs: head = tok.encode(f"<|im_start|>{m['role']}\n") body = tok.encode(m["content"].strip() + "<|im_end|>") + tok.encode("\n") ids += head + body; sup += [0] * len(head) + [int(m["role"] == "assistant")] * len(body) ids.append(eos); sup.append(0) return ids[:T], sup[:T] def tokenize(split, tok, eos, cache, limit=None): """Flat uint16 tokens + uint8 target mask + int64 starts, cached as .npy (Python int lists took ~25 GB).""" if (cache / "ids.npy").exists(): return tuple(np.load(cache / f"{k}.npy") for k in ("ids", "sup", "start")) ids, sup, start = array("H"), array("B"), array("q", [0]) for i, row in enumerate(split): if limit and i >= limit: break a, b = encode_conv(row["messages"], tok, eos); ids.extend(a); sup.extend(b); start.append(len(ids)) out = (np.frombuffer(ids, np.uint16).copy(), np.frombuffer(sup, np.uint8).copy(), np.frombuffer(start, np.int64).copy()) cache.mkdir(parents=True, exist_ok=True) for k, v in zip(("ids", "sup", "start"), out): np.save(cache / f"{k}.npy", v) return out def rows(lens, shuffle, seed): order = list(range(len(lens))) if shuffle: random.Random(seed).shuffle(order) out, cur, n = [], [], 0 for e in order: L = int(lens[e]) if n + L > T: out.append(cur); cur, n = [], 0 cur.append(e); n += L if cur: out.append(cur) return out def batch(rl, data, eos, dev): ids, sup, start = data B = len(rl); idx = torch.full((B, T), eos, dtype=torch.long); tgt = torch.full((B, T), IGNORE, dtype=torch.long) pos = torch.zeros((B, T), dtype=torch.long); bnd = torch.zeros((B, T), dtype=torch.bool); offs, lens = [0], [] for b, row in enumerate(rl): t = 0 for e in row: s0, s1 = int(start[e]), int(start[e + 1]); L = s1 - s0 x = torch.from_numpy(ids[s0:s1].astype(np.int64)); m = torch.from_numpy(sup[s0:s1].astype(bool)) lab = torch.full((L,), IGNORE, dtype=torch.long); lab[:-1] = torch.where(m[1:], x[1:], IGNORE) idx[b, t:t + L] = x; tgt[b, t:t + L] = lab # position t predicts token t+1 bnd[b, t] = True; pos[b, t:t + L] = torch.arange(L); lens.append(L); t += L; offs.append(b * T + t) if t < T: bnd[b, t] = True; pos[b, t:] = torch.arange(T - t); lens.append(T - t); offs.append(b * T + T) o = torch.tensor(offs, dtype=torch.int64) return idx.to(dev), pos.to(dev), o.to(dev), bnd.to(dev), tgt.to(dev), min(lens), max(lens) def openthink(path, fmt): """OpenThink rows -> {"train", "test"} lists of chats. plain: the trace as-is ("... Final answer: X"). think: "\\n{reasoning}\\n\\n\\nFinal answer: X" (empty think block for answer-only rows). Holdout = 800 rows chosen with a fixed seed (same split for every format and seed).""" import json recs = [json.loads(l) for l in open(path)] def reply(t): t = t.strip() if fmt == "plain": return t head, _, ans = t.rpartition("Final answer:") return f"\n{head.strip()}\n\n\nFinal answer:{ans}" chats = [{"messages": [{"role": "user", "content": r["question"]}, {"role": "assistant", "content": reply(r["trace"])}]} for r in recs] order = list(range(len(chats))); random.Random(1234).shuffle(order); hold = set(order[:800]) return {"train": [c for i, c in enumerate(chats) if i not in hold], "test": [c for i, c in enumerate(chats) if i in hold]} def main(): ap = argparse.ArgumentParser(); ap.add_argument("start"); ap.add_argument("out") ap.add_argument("--lr", type=float, default=1e-4); ap.add_argument("--epochs", type=int, default=1) ap.add_argument("--mb", type=int, default=16); ap.add_argument("--accum", type=int, default=4) ap.add_argument("--dev", default="cuda:0"); ap.add_argument("--max-steps", type=int, default=0) ap.add_argument("--data", default=None, help="OpenThink jsonl (question/trace) instead of smol-smoltalk") ap.add_argument("--fmt", default="plain", choices=("plain", "think")); ap.add_argument("--seed", type=int, default=0) ap.add_argument("--cache", default="/media/joseph/Media/instruct/tok") ap.add_argument("--mix", type=int, default=0, help="with --data: smol-smoltalk + N copies of the OpenThink train split; " "smol-smoltalk replies get an empty think block when --fmt think") a = ap.parse_args(); dev = a.dev; torch.manual_seed(a.seed) os.environ.setdefault("HF_DATASETS_CACHE", "/media/joseph/Media/hf-cache/datasets") from datasets import load_dataset ds = openthink(a.data, a.fmt) if a.data else load_dataset("HuggingFaceTB/smol-smoltalk") if a.data and a.mix: import itertools st = load_dataset("HuggingFaceTB/smol-smoltalk") def wrap(c): if a.fmt != "think": return c return {"messages": [dict(m, content=f"\n\n\n\n{m['content'].strip()}") if m["role"] == "assistant" else m for m in c["messages"]]} ds = {"train": itertools.chain((wrap(c) for c in st["train"]), *[ds["train"]] * a.mix), "test": [wrap(c) for c in itertools.islice(st["test"], 800)] + ds["test"]} model, cfg, blob = E.load_export(Path(a.start), dev) tok = BorisTokenizer.load(Path(cfg.data.root) / "tokenizer"); eos = cfg.data.eos_id t0 = time.time() C = Path(a.cache) tr = tokenize(ds["train"], tok, eos, C / "train"); va = tokenize(ds["test"], tok, eos, C / "test", None if a.mix else 800) print(f"tokenized {len(tr[2]) - 1} train conversations ({tr[0].size/1e6:.0f}M tokens, {int(tr[1].sum())/1e6:.0f}M supervised) in {time.time()-t0:.0f}s", flush=True) theta = model._eval_theta opt = torch.optim.AdamW(model.parameters(), lr=a.lr, betas=(0.9, 0.95), weight_decay=0.0, fused=True) scaler = torch.amp.GradScaler("cuda", init_scale=2.0 ** 15) R0 = rows(np.diff(tr[2]), False, 0); steps = math.ceil(len(R0) / (a.mb * a.accum)) * a.epochs if a.max_steps: steps = min(steps, a.max_steps) warm = max(1, steps // 30); VR = rows(np.diff(va[2]), False, 0) def val(): model.eval(); tot = n = 0.0 with torch.no_grad(): for i in range(0, len(VR), a.mb): x, p, o, bd, y, mn, mx = batch(VR[i:i + a.mb], va, eos, dev) with torch.autocast("cuda", dtype=torch.float16): l = model(x, p, o, bd, y, theta, mn, mx) k = int((y != IGNORE).sum()); tot += float(l) * k; n += k model.train(); return tot / max(n, 1) print(f"{steps} steps of {a.mb * a.accum} rows; holdout loss before {val():.4f}", flush=True) model.train(); step = 0; t0 = time.time(); seen = 0 for ep in range(a.epochs): R = rows(np.diff(tr[2]), True, 100 + ep + 1000 * a.seed) for i in range(0, len(R), a.mb * a.accum): if step >= steps: break lr = a.lr * min(1, (step + 1) / warm) * 0.5 * (1 + math.cos(math.pi * step / steps)) for g in opt.param_groups: g["lr"] = lr chunk = R[i:i + a.mb * a.accum] for j in range(0, len(chunk), a.mb): x, p, o, bd, y, mn, mx = batch(chunk[j:j + a.mb], tr, eos, dev) with torch.autocast("cuda", dtype=torch.float16): loss = model(x, p, o, bd, y, theta, mn, mx) scaler.scale(loss * len(chunk[j:j + a.mb]) / len(chunk)).backward(); seen += x.numel() scaler.unscale_(opt); torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) scaler.step(opt); scaler.update(); opt.zero_grad(set_to_none=True); step += 1 if step % 50 == 0 or step == steps: print(f"step {step}/{steps} loss {float(loss):.4f} lr {lr:.2e} {seen/(time.time()-t0)/1e3:.1f}k tok/s", flush=True) if step % 500 == 0: print(f" holdout loss {val():.4f}", flush=True) print(f"final holdout loss {val():.4f}", flush=True) keep = set(blob["model"]); sd = model.state_dict() blob["model"] = {k: sd[k].detach().float().cpu() for k in keep} torch.save(blob, a.out); print("saved", a.out, flush=True) if __name__ == "__main__": main()