summaryrefslogtreecommitdiff
path: root/sft_chat.py
diff options
context:
space:
mode:
Diffstat (limited to 'sft_chat.py')
-rw-r--r--sft_chat.py165
1 files changed, 165 insertions, 0 deletions
diff --git a/sft_chat.py b/sft_chat.py
new file mode 100644
index 0000000..e258e28
--- /dev/null
+++ b/sft_chat.py
@@ -0,0 +1,165 @@
+"""Chat SFT for Boris-2 on smol-smoltalk (2026-11-11) -> Boris-2-Instruct.
+
+ChatML-style turns written as plain text (no new tokens, so the 32,768 vocabulary, the n-gram table and the
+export path are unchanged):
+ <|im_start|>user\n{content}<|im_end|>\n<|im_start|>assistant\n{content}<|im_end|>\n ... <|endoftext|>
+Loss only on assistant contents and their closing "<|im_end|>". Conversations are packed as separate documents
+into 2048-token rows (attention, Canon and the n-gram lookup reset at each one). Holdout = smol-smoltalk test.
+
+ BORIS_VOLTA_KERNELS=1 python sft_chat.py START.pt OUT.pt [--lr 1e-4 --epochs 1]
+"""
+import argparse, math, os, random, sys, time
+from array import array
+from pathlib import Path
+import numpy as np, torch
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+import eval as E
+from tokenizer import BorisTokenizer
+
+IGNORE, T = -100, 2048
+
+
+def encode_conv(msgs, tok, eos):
+ """-> (ids, sup): sup[t] = 1 if token t is a training target (assistant content + its <|im_end|>\n)."""
+ ids, sup = [], []
+ for m in msgs:
+ head = tok.encode(f"<|im_start|>{m['role']}\n")
+ body = tok.encode(m["content"].strip() + "<|im_end|>") + tok.encode("\n")
+ ids += head + body; sup += [0] * len(head) + [int(m["role"] == "assistant")] * len(body)
+ ids.append(eos); sup.append(0)
+ return ids[:T], sup[:T]
+
+
+def tokenize(split, tok, eos, cache, limit=None):
+ """Flat uint16 tokens + uint8 target mask + int64 starts, cached as .npy (Python int lists took ~25 GB)."""
+ if (cache / "ids.npy").exists():
+ return tuple(np.load(cache / f"{k}.npy") for k in ("ids", "sup", "start"))
+ ids, sup, start = array("H"), array("B"), array("q", [0])
+ for i, row in enumerate(split):
+ if limit and i >= limit: break
+ a, b = encode_conv(row["messages"], tok, eos); ids.extend(a); sup.extend(b); start.append(len(ids))
+ out = (np.frombuffer(ids, np.uint16).copy(), np.frombuffer(sup, np.uint8).copy(), np.frombuffer(start, np.int64).copy())
+ cache.mkdir(parents=True, exist_ok=True)
+ for k, v in zip(("ids", "sup", "start"), out): np.save(cache / f"{k}.npy", v)
+ return out
+
+
+def rows(lens, shuffle, seed):
+ order = list(range(len(lens)))
+ if shuffle: random.Random(seed).shuffle(order)
+ out, cur, n = [], [], 0
+ for e in order:
+ L = int(lens[e])
+ if n + L > T: out.append(cur); cur, n = [], 0
+ cur.append(e); n += L
+ if cur: out.append(cur)
+ return out
+
+
+def batch(rl, data, eos, dev):
+ ids, sup, start = data
+ B = len(rl); idx = torch.full((B, T), eos, dtype=torch.long); tgt = torch.full((B, T), IGNORE, dtype=torch.long)
+ pos = torch.zeros((B, T), dtype=torch.long); bnd = torch.zeros((B, T), dtype=torch.bool); offs, lens = [0], []
+ for b, row in enumerate(rl):
+ t = 0
+ for e in row:
+ s0, s1 = int(start[e]), int(start[e + 1]); L = s1 - s0
+ x = torch.from_numpy(ids[s0:s1].astype(np.int64)); m = torch.from_numpy(sup[s0:s1].astype(bool))
+ lab = torch.full((L,), IGNORE, dtype=torch.long); lab[:-1] = torch.where(m[1:], x[1:], IGNORE)
+ idx[b, t:t + L] = x; tgt[b, t:t + L] = lab # position t predicts token t+1
+ bnd[b, t] = True; pos[b, t:t + L] = torch.arange(L); lens.append(L); t += L; offs.append(b * T + t)
+ if t < T:
+ bnd[b, t] = True; pos[b, t:] = torch.arange(T - t); lens.append(T - t); offs.append(b * T + T)
+ o = torch.tensor(offs, dtype=torch.int64)
+ return idx.to(dev), pos.to(dev), o.to(dev), bnd.to(dev), tgt.to(dev), min(lens), max(lens)
+
+
+def openthink(path, fmt):
+ """OpenThink rows -> {"train", "test"} lists of chats. plain: the trace as-is ("... Final answer: X").
+ think: "<think>\\n{reasoning}\\n</think>\\n\\nFinal answer: X" (empty think block for answer-only rows).
+ Holdout = 800 rows chosen with a fixed seed (same split for every format and seed)."""
+ import json
+ recs = [json.loads(l) for l in open(path)]
+ def reply(t):
+ t = t.strip()
+ if fmt == "plain": return t
+ head, _, ans = t.rpartition("Final answer:")
+ return f"<think>\n{head.strip()}\n</think>\n\nFinal answer:{ans}"
+ chats = [{"messages": [{"role": "user", "content": r["question"]}, {"role": "assistant", "content": reply(r["trace"])}]} for r in recs]
+ order = list(range(len(chats))); random.Random(1234).shuffle(order); hold = set(order[:800])
+ return {"train": [c for i, c in enumerate(chats) if i not in hold], "test": [c for i, c in enumerate(chats) if i in hold]}
+
+
+def main():
+ ap = argparse.ArgumentParser(); ap.add_argument("start"); ap.add_argument("out")
+ ap.add_argument("--lr", type=float, default=1e-4); ap.add_argument("--epochs", type=int, default=1)
+ ap.add_argument("--mb", type=int, default=16); ap.add_argument("--accum", type=int, default=4)
+ ap.add_argument("--dev", default="cuda:0"); ap.add_argument("--max-steps", type=int, default=0)
+ ap.add_argument("--data", default=None, help="OpenThink jsonl (question/trace) instead of smol-smoltalk")
+ ap.add_argument("--fmt", default="plain", choices=("plain", "think")); ap.add_argument("--seed", type=int, default=0)
+ ap.add_argument("--cache", default="/media/joseph/Media/instruct/tok")
+ ap.add_argument("--mix", type=int, default=0, help="with --data: smol-smoltalk + N copies of the OpenThink train split; "
+ "smol-smoltalk replies get an empty think block when --fmt think")
+ a = ap.parse_args(); dev = a.dev; torch.manual_seed(a.seed)
+ os.environ.setdefault("HF_DATASETS_CACHE", "/media/joseph/Media/hf-cache/datasets")
+ from datasets import load_dataset
+ ds = openthink(a.data, a.fmt) if a.data else load_dataset("HuggingFaceTB/smol-smoltalk")
+ if a.data and a.mix:
+ import itertools
+ st = load_dataset("HuggingFaceTB/smol-smoltalk")
+ def wrap(c):
+ if a.fmt != "think": return c
+ return {"messages": [dict(m, content=f"<think>\n\n</think>\n\n{m['content'].strip()}") if m["role"] == "assistant" else m
+ for m in c["messages"]]}
+ ds = {"train": itertools.chain((wrap(c) for c in st["train"]), *[ds["train"]] * a.mix),
+ "test": [wrap(c) for c in itertools.islice(st["test"], 800)] + ds["test"]}
+ model, cfg, blob = E.load_export(Path(a.start), dev)
+ tok = BorisTokenizer.load(Path(cfg.data.root) / "tokenizer"); eos = cfg.data.eos_id
+ t0 = time.time()
+ C = Path(a.cache)
+ tr = tokenize(ds["train"], tok, eos, C / "train"); va = tokenize(ds["test"], tok, eos, C / "test", None if a.mix else 800)
+ print(f"tokenized {len(tr[2]) - 1} train conversations ({tr[0].size/1e6:.0f}M tokens, {int(tr[1].sum())/1e6:.0f}M supervised) in {time.time()-t0:.0f}s", flush=True)
+ theta = model._eval_theta
+ opt = torch.optim.AdamW(model.parameters(), lr=a.lr, betas=(0.9, 0.95), weight_decay=0.0, fused=True)
+ scaler = torch.amp.GradScaler("cuda", init_scale=2.0 ** 15)
+ R0 = rows(np.diff(tr[2]), False, 0); steps = math.ceil(len(R0) / (a.mb * a.accum)) * a.epochs
+ if a.max_steps: steps = min(steps, a.max_steps)
+ warm = max(1, steps // 30); VR = rows(np.diff(va[2]), False, 0)
+
+ def val():
+ model.eval(); tot = n = 0.0
+ with torch.no_grad():
+ for i in range(0, len(VR), a.mb):
+ x, p, o, bd, y, mn, mx = batch(VR[i:i + a.mb], va, eos, dev)
+ with torch.autocast("cuda", dtype=torch.float16):
+ l = model(x, p, o, bd, y, theta, mn, mx)
+ k = int((y != IGNORE).sum()); tot += float(l) * k; n += k
+ model.train(); return tot / max(n, 1)
+
+ print(f"{steps} steps of {a.mb * a.accum} rows; holdout loss before {val():.4f}", flush=True)
+ model.train(); step = 0; t0 = time.time(); seen = 0
+ for ep in range(a.epochs):
+ R = rows(np.diff(tr[2]), True, 100 + ep + 1000 * a.seed)
+ for i in range(0, len(R), a.mb * a.accum):
+ if step >= steps: break
+ lr = a.lr * min(1, (step + 1) / warm) * 0.5 * (1 + math.cos(math.pi * step / steps))
+ for g in opt.param_groups: g["lr"] = lr
+ chunk = R[i:i + a.mb * a.accum]
+ for j in range(0, len(chunk), a.mb):
+ x, p, o, bd, y, mn, mx = batch(chunk[j:j + a.mb], tr, eos, dev)
+ with torch.autocast("cuda", dtype=torch.float16):
+ loss = model(x, p, o, bd, y, theta, mn, mx)
+ scaler.scale(loss * len(chunk[j:j + a.mb]) / len(chunk)).backward(); seen += x.numel()
+ scaler.unscale_(opt); torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
+ scaler.step(opt); scaler.update(); opt.zero_grad(set_to_none=True); step += 1
+ if step % 50 == 0 or step == steps:
+ print(f"step {step}/{steps} loss {float(loss):.4f} lr {lr:.2e} {seen/(time.time()-t0)/1e3:.1f}k tok/s", flush=True)
+ if step % 500 == 0: print(f" holdout loss {val():.4f}", flush=True)
+ print(f"final holdout loss {val():.4f}", flush=True)
+ keep = set(blob["model"]); sd = model.state_dict()
+ blob["model"] = {k: sd[k].detach().float().cpu() for k in keep}
+ torch.save(blob, a.out); print("saved", a.out, flush=True)
+
+
+if __name__ == "__main__":
+ main()