From 5d2e4252e695644c34be605b44a14bd80a98a0a6 Mon Sep 17 00:00:00 2001 From: jdj Date: Wed, 18 Nov 2026 14:47:33 -0800 Subject: Boris-2 training code The code, configuration and data filters used to train Boris-2 (144M, 200B tokens, single V100): model, training loop, data preparation, n-gram table, V100 kernels, final stages and the exact run configs. --- sft_chat.py | 165 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 sft_chat.py (limited to 'sft_chat.py') diff --git a/sft_chat.py b/sft_chat.py new file mode 100644 index 0000000..e258e28 --- /dev/null +++ b/sft_chat.py @@ -0,0 +1,165 @@ +"""Chat SFT for Boris-2 on smol-smoltalk (2026-11-11) -> Boris-2-Instruct. + +ChatML-style turns written as plain text (no new tokens, so the 32,768 vocabulary, the n-gram table and the +export path are unchanged): + <|im_start|>user\n{content}<|im_end|>\n<|im_start|>assistant\n{content}<|im_end|>\n ... <|endoftext|> +Loss only on assistant contents and their closing "<|im_end|>". Conversations are packed as separate documents +into 2048-token rows (attention, Canon and the n-gram lookup reset at each one). Holdout = smol-smoltalk test. + + BORIS_VOLTA_KERNELS=1 python sft_chat.py START.pt OUT.pt [--lr 1e-4 --epochs 1] +""" +import argparse, math, os, random, sys, time +from array import array +from pathlib import Path +import numpy as np, torch +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import eval as E +from tokenizer import BorisTokenizer + +IGNORE, T = -100, 2048 + + +def encode_conv(msgs, tok, eos): + """-> (ids, sup): sup[t] = 1 if token t is a training target (assistant content + its <|im_end|>\n).""" + ids, sup = [], [] + for m in msgs: + head = tok.encode(f"<|im_start|>{m['role']}\n") + body = tok.encode(m["content"].strip() + "<|im_end|>") + tok.encode("\n") + ids += head + body; sup += [0] * len(head) + [int(m["role"] == "assistant")] * len(body) + ids.append(eos); sup.append(0) + return ids[:T], sup[:T] + + +def tokenize(split, tok, eos, cache, limit=None): + """Flat uint16 tokens + uint8 target mask + int64 starts, cached as .npy (Python int lists took ~25 GB).""" + if (cache / "ids.npy").exists(): + return tuple(np.load(cache / f"{k}.npy") for k in ("ids", "sup", "start")) + ids, sup, start = array("H"), array("B"), array("q", [0]) + for i, row in enumerate(split): + if limit and i >= limit: break + a, b = encode_conv(row["messages"], tok, eos); ids.extend(a); sup.extend(b); start.append(len(ids)) + out = (np.frombuffer(ids, np.uint16).copy(), np.frombuffer(sup, np.uint8).copy(), np.frombuffer(start, np.int64).copy()) + cache.mkdir(parents=True, exist_ok=True) + for k, v in zip(("ids", "sup", "start"), out): np.save(cache / f"{k}.npy", v) + return out + + +def rows(lens, shuffle, seed): + order = list(range(len(lens))) + if shuffle: random.Random(seed).shuffle(order) + out, cur, n = [], [], 0 + for e in order: + L = int(lens[e]) + if n + L > T: out.append(cur); cur, n = [], 0 + cur.append(e); n += L + if cur: out.append(cur) + return out + + +def batch(rl, data, eos, dev): + ids, sup, start = data + B = len(rl); idx = torch.full((B, T), eos, dtype=torch.long); tgt = torch.full((B, T), IGNORE, dtype=torch.long) + pos = torch.zeros((B, T), dtype=torch.long); bnd = torch.zeros((B, T), dtype=torch.bool); offs, lens = [0], [] + for b, row in enumerate(rl): + t = 0 + for e in row: + s0, s1 = int(start[e]), int(start[e + 1]); L = s1 - s0 + x = torch.from_numpy(ids[s0:s1].astype(np.int64)); m = torch.from_numpy(sup[s0:s1].astype(bool)) + lab = torch.full((L,), IGNORE, dtype=torch.long); lab[:-1] = torch.where(m[1:], x[1:], IGNORE) + idx[b, t:t + L] = x; tgt[b, t:t + L] = lab # position t predicts token t+1 + bnd[b, t] = True; pos[b, t:t + L] = torch.arange(L); lens.append(L); t += L; offs.append(b * T + t) + if t < T: + bnd[b, t] = True; pos[b, t:] = torch.arange(T - t); lens.append(T - t); offs.append(b * T + T) + o = torch.tensor(offs, dtype=torch.int64) + return idx.to(dev), pos.to(dev), o.to(dev), bnd.to(dev), tgt.to(dev), min(lens), max(lens) + + +def openthink(path, fmt): + """OpenThink rows -> {"train", "test"} lists of chats. plain: the trace as-is ("... Final answer: X"). + think: "\\n{reasoning}\\n\\n\\nFinal answer: X" (empty think block for answer-only rows). + Holdout = 800 rows chosen with a fixed seed (same split for every format and seed).""" + import json + recs = [json.loads(l) for l in open(path)] + def reply(t): + t = t.strip() + if fmt == "plain": return t + head, _, ans = t.rpartition("Final answer:") + return f"\n{head.strip()}\n\n\nFinal answer:{ans}" + chats = [{"messages": [{"role": "user", "content": r["question"]}, {"role": "assistant", "content": reply(r["trace"])}]} for r in recs] + order = list(range(len(chats))); random.Random(1234).shuffle(order); hold = set(order[:800]) + return {"train": [c for i, c in enumerate(chats) if i not in hold], "test": [c for i, c in enumerate(chats) if i in hold]} + + +def main(): + ap = argparse.ArgumentParser(); ap.add_argument("start"); ap.add_argument("out") + ap.add_argument("--lr", type=float, default=1e-4); ap.add_argument("--epochs", type=int, default=1) + ap.add_argument("--mb", type=int, default=16); ap.add_argument("--accum", type=int, default=4) + ap.add_argument("--dev", default="cuda:0"); ap.add_argument("--max-steps", type=int, default=0) + ap.add_argument("--data", default=None, help="OpenThink jsonl (question/trace) instead of smol-smoltalk") + ap.add_argument("--fmt", default="plain", choices=("plain", "think")); ap.add_argument("--seed", type=int, default=0) + ap.add_argument("--cache", default="/media/joseph/Media/instruct/tok") + ap.add_argument("--mix", type=int, default=0, help="with --data: smol-smoltalk + N copies of the OpenThink train split; " + "smol-smoltalk replies get an empty think block when --fmt think") + a = ap.parse_args(); dev = a.dev; torch.manual_seed(a.seed) + os.environ.setdefault("HF_DATASETS_CACHE", "/media/joseph/Media/hf-cache/datasets") + from datasets import load_dataset + ds = openthink(a.data, a.fmt) if a.data else load_dataset("HuggingFaceTB/smol-smoltalk") + if a.data and a.mix: + import itertools + st = load_dataset("HuggingFaceTB/smol-smoltalk") + def wrap(c): + if a.fmt != "think": return c + return {"messages": [dict(m, content=f"\n\n\n\n{m['content'].strip()}") if m["role"] == "assistant" else m + for m in c["messages"]]} + ds = {"train": itertools.chain((wrap(c) for c in st["train"]), *[ds["train"]] * a.mix), + "test": [wrap(c) for c in itertools.islice(st["test"], 800)] + ds["test"]} + model, cfg, blob = E.load_export(Path(a.start), dev) + tok = BorisTokenizer.load(Path(cfg.data.root) / "tokenizer"); eos = cfg.data.eos_id + t0 = time.time() + C = Path(a.cache) + tr = tokenize(ds["train"], tok, eos, C / "train"); va = tokenize(ds["test"], tok, eos, C / "test", None if a.mix else 800) + print(f"tokenized {len(tr[2]) - 1} train conversations ({tr[0].size/1e6:.0f}M tokens, {int(tr[1].sum())/1e6:.0f}M supervised) in {time.time()-t0:.0f}s", flush=True) + theta = model._eval_theta + opt = torch.optim.AdamW(model.parameters(), lr=a.lr, betas=(0.9, 0.95), weight_decay=0.0, fused=True) + scaler = torch.amp.GradScaler("cuda", init_scale=2.0 ** 15) + R0 = rows(np.diff(tr[2]), False, 0); steps = math.ceil(len(R0) / (a.mb * a.accum)) * a.epochs + if a.max_steps: steps = min(steps, a.max_steps) + warm = max(1, steps // 30); VR = rows(np.diff(va[2]), False, 0) + + def val(): + model.eval(); tot = n = 0.0 + with torch.no_grad(): + for i in range(0, len(VR), a.mb): + x, p, o, bd, y, mn, mx = batch(VR[i:i + a.mb], va, eos, dev) + with torch.autocast("cuda", dtype=torch.float16): + l = model(x, p, o, bd, y, theta, mn, mx) + k = int((y != IGNORE).sum()); tot += float(l) * k; n += k + model.train(); return tot / max(n, 1) + + print(f"{steps} steps of {a.mb * a.accum} rows; holdout loss before {val():.4f}", flush=True) + model.train(); step = 0; t0 = time.time(); seen = 0 + for ep in range(a.epochs): + R = rows(np.diff(tr[2]), True, 100 + ep + 1000 * a.seed) + for i in range(0, len(R), a.mb * a.accum): + if step >= steps: break + lr = a.lr * min(1, (step + 1) / warm) * 0.5 * (1 + math.cos(math.pi * step / steps)) + for g in opt.param_groups: g["lr"] = lr + chunk = R[i:i + a.mb * a.accum] + for j in range(0, len(chunk), a.mb): + x, p, o, bd, y, mn, mx = batch(chunk[j:j + a.mb], tr, eos, dev) + with torch.autocast("cuda", dtype=torch.float16): + loss = model(x, p, o, bd, y, theta, mn, mx) + scaler.scale(loss * len(chunk[j:j + a.mb]) / len(chunk)).backward(); seen += x.numel() + scaler.unscale_(opt); torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) + scaler.step(opt); scaler.update(); opt.zero_grad(set_to_none=True); step += 1 + if step % 50 == 0 or step == steps: + print(f"step {step}/{steps} loss {float(loss):.4f} lr {lr:.2e} {seen/(time.time()-t0)/1e3:.1f}k tok/s", flush=True) + if step % 500 == 0: print(f" holdout loss {val():.4f}", flush=True) + print(f"final holdout loss {val():.4f}", flush=True) + keep = set(blob["model"]); sd = model.state_dict() + blob["model"] = {k: sd[k].detach().float().cpu() for k in keep} + torch.save(blob, a.out); print("saved", a.out, flush=True) + + +if __name__ == "__main__": + main() -- cgit v1.2.3