diff options
| author | jdj <jdj@kunix.org> | 2026-11-18 14:47:33 -0800 |
|---|---|---|
| committer | jdj <jdj@kunix.org> | 2026-11-18 14:47:33 -0800 |
| commit | 5d2e4252e695644c34be605b44a14bd80a98a0a6 (patch) | |
| tree | 311a72eda4459af7547b9cd05c0956eafb235f2d /scripts | |
| download | boris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.tar.gz boris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.zip | |
Boris-2 training code
The code, configuration and data filters used to train Boris-2 (144M,
200B tokens, single V100): model, training loop, data preparation,
n-gram table, V100 kernels, final stages and the exact run configs.
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/hellaswag_title_split.py | 67 | ||||
| -rw-r--r-- | scripts/pilot_train.py | 191 | ||||
| -rw-r--r-- | scripts/prepare_orca.py | 23 | ||||
| -rw-r--r-- | scripts/prepare_wordprob.py | 187 | ||||
| -rw-r--r-- | scripts/sample.py | 47 | ||||
| -rw-r--r-- | scripts/scan_decay_contam.py | 17 |
6 files changed, 532 insertions, 0 deletions
diff --git a/scripts/hellaswag_title_split.py b/scripts/hellaswag_title_split.py new file mode 100644 index 0000000..da83229 --- /dev/null +++ b/scripts/hellaswag_title_split.py @@ -0,0 +1,67 @@ +"""HellaSwag gain split by WikiHow-title overlap with the decay's how-to data +(the check cagliostro-v3.5's card ran). Items: HellaSwag validation, in eval.py's order. +Groups: WikiHow items whose article title is also a Cosmopedia-wikihow tutorial title, +WikiHow items whose title is not, and ActivityNet items (no WikiHow link at all). +Per-item acc_norm before/after the decay; a real skill gain shows up about equally in +the first two groups, memorised topics would show up only in the first. + + python probe/hellaswag_title_split.py BEFORE.pt AFTER.pt OUT.json +""" +import json, re, sys, inspect +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +import numpy as np, torch, pyarrow.parquet as pq +from huggingface_hub import hf_hub_download +from datasets import load_dataset +import eval as E +from tokenizer import BorisTokenizer + +def norm(t): + t = t.lower().strip() + t = re.sub(r"^how to\s+", "", t) + return re.sub(r"[^a-z0-9 ]+", "", t).strip() + +titles = set() +for k in range(2): + p = hf_hub_download("HuggingFaceTB/cosmopedia", f"data/wikihow/train-0000{k}-of-00002.parquet", repo_type="dataset") + for pr in pq.read_table(p, columns=["prompt"]).column("prompt").to_pylist(): + m = re.search(r'tutorial on "(.+?)"', pr) + if m: titles.add(norm(m.group(1))) + +raw = [r for r in load_dataset("Rowan/hellaswag", split="validation") if str(r["label"]).strip() != ""] +group = [] +for r in raw: + if r["source_id"].startswith("activitynet"): + group.append("activitynet"); continue + m = re.search(r"\[header\]\s*(.+?)\s*\[title\]", r["ctx_a"]) + group.append("wikihow_title_in_data" if m and norm(m.group(1)) in titles else + "wikihow_title_not_in_data" if m else "wikihow_no_title") +items = E.TASKS["hellaswag"]() +assert len(items) == len(group) + +src = inspect.getsource(E.score_batch).replace(" n_hit = n_hit_norm = 0", " return scores\n n_hit = 0") +ns = dict(vars(E)); exec(src, ns); score = ns["score_batch"] +dev = "cuda:1"; torch.cuda.set_device(dev) +right = {} +for p in sys.argv[1:3]: + model, cfg, blob = E.load_export(Path(p), dev) + tok = BorisTokenizer.load(Path(cfg.data.root) / "tokenizer") + with torch.no_grad(): + s = score(model, tok, items, dev, batch_size=256) + right[p] = np.array([int(np.argmax([s[(i, j)][0] / max(s[(i, j)][1], 1) for j in range(len(it["choices"]))]) == it["gold"]) + for i, it in enumerate(items)]) + del model; torch.cuda.empty_cache() +a, b = right[sys.argv[1]], right[sys.argv[2]] +g = np.array(group); out = {"cosmopedia_wikihow_titles": len(titles)} +wk = np.isin(g, ["wikihow_title_in_data", "wikihow_title_not_in_data"]) +out["title_overlap_share_of_titled_wikihow_items"] = float((g == "wikihow_title_in_data").sum() / max(wk.sum(), 1)) +print(f"Cosmopedia wikihow titles: {len(titles):,}; titled WikiHow items whose title is in the data: " + f"{100*out['title_overlap_share_of_titled_wikihow_items']:.1f}%") +for name in ["wikihow_title_in_data", "wikihow_title_not_in_data", "wikihow_no_title", "activitynet", "all"]: + m = np.ones(len(g), bool) if name == "all" else (g == name) + if not m.any(): continue + d = (b[m] - a[m]).astype(float); se = d.std(ddof=1) / np.sqrt(m.sum()) if m.sum() > 1 else 0.0 + out[name] = {"n": int(m.sum()), "before": float(a[m].mean() * 100), "after": float(b[m].mean() * 100), + "gain": float(d.mean() * 100), "gain_95ci": float(1.96 * se * 100)} + print(f" {name:28s} n={m.sum():5d} {a[m].mean()*100:6.2f} -> {b[m].mean()*100:6.2f} gain {d.mean()*100:+5.2f} ± {1.96*se*100:.2f}") +json.dump(out, open(sys.argv[3], "w"), indent=2) diff --git a/scripts/pilot_train.py b/scripts/pilot_train.py new file mode 100644 index 0000000..5d7e85f --- /dev/null +++ b/scripts/pilot_train.py @@ -0,0 +1,191 @@ +"""Pilot branches from a real Boris checkpoint (2026-10-10). + +Generalises branch_decay.py: either GPU, the Volta kernels on the V100, the main +run's batch, a constant or 1-sqrt-decaying LR (optionally to a non-zero floor), +and an optional output-superposition phase (TST, Peng et al. 2026, section 5.2): +inputs stay ordinary tokens, but for the first `--bag-tokens` tokens each position +is trained to predict the bag of its next s tokens -- uniform 1/s multi-hot cross +entropy on the same LM head -- then training reverts to next-token prediction. +Bag targets never cross a document boundary (the packer's IGNORE marks a document's +last position; a target is valid only if every step to it is valid). + +Every arm with the same --seed sees the same token stream. + + python probe/pilot_train.py NAME --tokens 3e8 --mix climbmix:0.95,finemath:0.05 \ + [--dev cuda:0 --mb 16 --tps 524288] [--lr const|decay --final-frac 0.0] \ + [--bag 4 --bag-tokens 1.5e8] --ckpt START.pt --out DIR +""" +from __future__ import annotations + +import argparse, json, math, sys, time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +import torch +import torch.nn.functional as F +from torch.utils.checkpoint import checkpoint + +from config import Config +from data import Prefetcher, VarlenPacker, open_sources +from model import Boris2 +from ngram import NGramVocab +from optim import build_optimizers, set_lr +import checkpoint as ckpt_mod + +IGNORE = -100 + + +def bag_targets(y, s): + """(B,T) next-token targets -> (B,T,s) targets for t+1..t+s and their validity.""" + B, T = y.shape + tg = torch.full((B, T, s), IGNORE, dtype=y.dtype, device=y.device) + ok = torch.zeros(B, T, s, dtype=torch.bool, device=y.device) + run = torch.ones(B, T, dtype=torch.bool, device=y.device) + for j in range(s): + yj = torch.full_like(y, IGNORE) + yj[:, :T - j] = y[:, j:] + run = run & (yj != IGNORE) + tg[:, :, j] = torch.where(run, yj, torch.full_like(yj, IGNORE)) + ok[:, :, j] = run + return tg, ok + + +def _bag_chunk(h, W, tg, ok): + lg = F.linear(h.to(torch.float16), W.to(torch.float16)).float() + lp = lg.gather(1, tg.clamp_min(0)) - torch.logsumexp(lg, -1, keepdim=True) + n = ok.sum(1) + per = (lp * ok).sum(1) / n.clamp_min(1) + return -(per * (n > 0)).sum(), (n > 0).sum() + + +def bag_loss(h, W, y, s, chunk=2048): + tg, ok = bag_targets(y, s) + h, tg, ok = h.reshape(-1, h.shape[-1]), tg.reshape(-1, s), ok.reshape(-1, s) + tot, cnt = 0.0, 0 + for a in range(0, h.shape[0], chunk): + l, c = checkpoint(_bag_chunk, h[a:a + chunk], W, tg[a:a + chunk], ok[a:a + chunk], + use_reentrant=False) + tot = tot + l; cnt = cnt + c + return tot / max(int(cnt), 1) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("name"); ap.add_argument("--tokens", type=float, required=True) + ap.add_argument("--mix", required=True, help="per-document draw weights src:w,...") + ap.add_argument("--dev", default="cuda:1"); ap.add_argument("--mb", type=int, default=2) + ap.add_argument("--tps", type=int, default=131072) + ap.add_argument("--lr", choices=["const", "decay", "wsd"], default="decay") + ap.add_argument("--decay-tokens", type=float, default=0) + ap.add_argument("--final-frac", type=float, default=0.0) + ap.add_argument("--bag", type=int, default=0); ap.add_argument("--bag-tokens", type=float, default=0) + ap.add_argument("--ckpt", required=True); ap.add_argument("--out", required=True) + ap.add_argument("--seed", type=int, default=1234) + ap.add_argument("--peak", type=float, default=1.0, help="multiplier on the config's peak LRs (0.1 = a tenth)") + ap.add_argument("--compile", action="store_true") + ap.add_argument("--max-steps", type=int, default=None) + ap.add_argument("--block", type=int, default=2048, help="sequence length (Boris-2-Nano: 1024)") + ap.add_argument("--mix-before", default=None, help="mix used before --switch-tokens; --mix after it") + ap.add_argument("--switch-tokens", type=float, default=0) + a = ap.parse_args() + dev = a.dev; torch.cuda.set_device(dev) + parse = lambda m: {k: float(v) for k, v in (kv.split(":") for kv in m.split(","))} + mix = parse(a.mix); mix0 = parse(a.mix_before) if a.mix_before else None + + blob = torch.load(a.ckpt, map_location="cpu", weights_only=False) + cfg = Config.from_json(blob["config"]) + t0_tokens, ngfp = blob["tokens_seen"], blob.get("ngram_fingerprint") + allsrc = dict.fromkeys(list(mix) + list(mix0 or {})) + cfg.data.sources = tuple(allsrc) + full = lambda m: {k: m.get(k, 0.0) for k in allsrc} + cfg.curriculum.phases = [["pilot", 1.0, full(mix0 or mix)]] + cfg.curriculum.smooth = False + cfg.optim.lm_head_chunk = min(cfg.optim.lm_head_chunk, 8192 if "cuda:0" in dev else 2048) + cfg.train.device = dev + cfg.train.tokens_per_step = a.tps + cfg.train.micro_batch = {a.block: a.mb, 8192: max(1, a.mb // 4)} + torch.manual_seed(a.seed) + + stores = open_sources(cfg, lambda *x: None) + packer = VarlenPacker(cfg, stores, a.seed, split="train") + model = Boris2(cfg).to(dev) + model.load_state_dict({k: v.float() for k, v in blob["model"].items()}) + if model.ngram is not None: + model.ngram.load_vocab(NGramVocab.load(cfg.ngram.vocab_path), dev) + if a.compile: + model.maybe_compile() + muon, adamw = build_optimizers(model, cfg, lambda *x: None) + if "muon" in blob: # model-only exports start with fresh optimizer state + muon.load_state_dict(blob["muon"]) + adamw.load_state_dict(ckpt_mod._migrate_adamw_groups(blob, model, adamw)) + scaler = torch.amp.GradScaler("cuda", init_scale=2.0 ** 15) + theta = float(cfg.block_size_at(t0_tokens / cfg.train.total_tokens)[1]) + del blob + + accum = a.tps // (a.mb * a.block) + steps = int(a.tokens) // a.tps + if a.max_steps: steps = min(steps, a.max_steps) + bag_steps = int(a.bag_tokens) // a.tps if a.bag else 0 + pre = Prefetcher(packer, cfg, dev); pre.configure(a.mb, a.block, 0.0); pre.start() + from train import gpu_temp, device_index + di = device_index(dev) + W = model.head_weight() + out = Path(a.out); out.mkdir(parents=True, exist_ok=True) + log = open(out / f"{a.name}.log", "a") + t0 = time.time(); tokens = 0 + switch_step = int(a.switch_tokens) // a.tps if mix0 else -1 + for step in range(steps): + if step == switch_step: # change the data mix in place (same packer state otherwise) + pre.stop() + cfg.curriculum.phases = [["pilot", 1.0, full(mix)]] + packer = VarlenPacker(cfg, stores, a.seed + 1, split="train") + pre = Prefetcher(packer, cfg, dev); pre.configure(a.mb, a.block, 0.0); pre.start() + print(f"{a.name} step {step}: switched data mix", flush=True); log.write(f"switched mix at step {step}\n") + if step % 5 == 0 and gpu_temp(di) >= 79: + while gpu_temp(di) > 76: + time.sleep(2) + frac = step / steps + if a.lr == "wsd": + dsteps = max(1, int(a.decay_tokens) // a.tps); d0 = steps - dsteps + scale = 1.0 if step < d0 else a.final_frac + (1 - a.final_frac) * (1.0 - math.sqrt((step - d0) / dsteps)) + else: + scale = 1.0 if a.lr == "const" else a.final_frac + (1 - a.final_frac) * (1.0 - math.sqrt(frac)) + set_lr(muon, adamw, cfg, scale * a.peak) + in_bag = step < bag_steps + acc = torch.zeros((), device=dev) + for _ in range(accum): + x, y, pos, bnd, bounds, st = pre.next(a.mb, a.block) + with torch.autocast("cuda", dtype=torch.float16): + if in_bag: + h = model(x, pos, bounds, bnd, None, theta, st["min_seqlen"], st["max_seqlen"]) + loss = bag_loss(h, W, y, a.bag) + else: + loss = model(x, pos, bounds, bnd, y, theta, st["min_seqlen"], st["max_seqlen"]) + scaler.scale(loss / accum).backward() + acc += loss.detach() / accum; tokens += st["tokens"] + scaler.unscale_(muon); scaler.unscale_(adamw) + gl = [p.grad for p in model.parameters() if p.grad is not None] + gn = float(torch.linalg.vector_norm(torch.stack(torch._foreach_norm(gl)))) + if math.isfinite(gn): + if gn >= cfg.optim.grad_clip: + torch._foreach_mul_(gl, cfg.optim.grad_clip / (gn + 1e-6)) + muon.step(); adamw.step() + scaler.update() + if scaler.get_scale() > 2.0 ** 15: + scaler.update(2.0 ** 15) + muon.zero_grad(set_to_none=True); adamw.zero_grad(set_to_none=True) + if step % 10 == 0 or step == steps - 1: + msg = (f"{a.name} step {step}/{steps} {'BAG' if in_bag else 'ntp'} loss {float(acc):.4f} " + f"gn {gn:.3f} lr_scale {scale:.3f} {tokens/(time.time()-t0)/1e3:.1f}k tok/s " + f"temp {gpu_temp(di)}") + print(msg, flush=True); log.write(msg + "\n"); log.flush() + pre.stop() + exp = out / f"pilot_{a.name}.pt" + torch.save({"model": model.state_dict(), "config": cfg.to_json(), "step": steps, + "tokens_seen": t0_tokens + tokens, "ngram_fingerprint": ngfp}, exp) + print(json.dumps({"name": a.name, "tokens": tokens, "minutes": (time.time() - t0) / 60, + "bag": a.bag, "bag_steps": bag_steps, "lr": a.lr, "export": str(exp)}), flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/prepare_orca.py b/scripts/prepare_orca.py new file mode 100644 index 0000000..37f27b4 --- /dev/null +++ b/scripts/prepare_orca.py @@ -0,0 +1,23 @@ +"""microsoft/orca-math-word-problems-200k -> data/orca_math (question + answer as one +document), 13-gram decontaminated against the eval suite like every other source.""" +import sys, json +sys.path.insert(0, "/home/joseph/boris-2-D112M-n15M") +from datasets import load_dataset +import prepare +from tokenizer import BorisTokenizer +tok = BorisTokenizer.load(prepare.DATA_DIR / "tokenizer"); ban = prepare.eval_ngrams() +w = prepare.ShardWriter(prepare.DATA_DIR / "orca_math", tok.eos_id) +assert w.n_tokens == 0, "data/orca_math already has tokens" +ds = load_dataset("microsoft/orca-math-word-problems-200k", split="train") +kept = dropped = toks = 0; batch = [] +for r in ds: + t = r["question"].strip() + "\n\n" + r["answer"].strip() + if prepare.contaminated(t, ban): dropped += 1; continue + batch.append(t) + if len(batch) == 1000: + toks += w.write(tok.encode_batch(batch)); kept += len(batch); batch = [] +if batch: toks += w.write(tok.encode_batch(batch)); kept += len(batch) +w.close() +json.dump({"source": "microsoft/orca-math-word-problems-200k", "documents": kept, "tokens": toks, + "dropped_contaminated": dropped}, open(prepare.DATA_DIR / "orca_math" / "prepare_meta.json", "w"), indent=2) +print("orca_math:", kept, "docs", toks, "tokens", dropped, "dropped as contaminated") diff --git a/scripts/prepare_wordprob.py b/scripts/prepare_wordprob.py new file mode 100644 index 0000000..cfa0cd6 --- /dev/null +++ b/scripts/prepare_wordprob.py @@ -0,0 +1,187 @@ +"""Public word-problem datasets -> Boris token shards (2026-10-10). + + python probe/prepare_wordprob.py tinygsm --tokens 1.5e9 + python probe/prepare_wordprob.py elem + +tinygsm: TinyGSM/TinyGSM (MIT) -- question, then its Python solution. +elem: MU-NLPC Calc-ape210k and Calc-mawps, training splits only (MIT) -- question, then + "Answer: <equation> = <result>". +Every document goes through the same 13-gram filter as the rest of the corpus (the ban +set covers HellaSwag, ARC, PIQA and every ArithMark-3 item). Writes data/<name>/ with a +provenance file. +""" +import argparse, hashlib, json, sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from datasets import load_dataset +import prepare +from tokenizer import BorisTokenizer + + +def tinygsm(): + for r in load_dataset("TinyGSM/TinyGSM", split="train", streaming=True): + yield r["question"].strip() + "\n\n" + r["code"].strip() + + +def elem(): + # Training splits only: ASDiv-A and SVAMP exist only as test sets and are left out. + for name in ("MU-NLPC/Calc-ape210k", "MU-NLPC/Calc-mawps"): + for split in ("train",): + for r in load_dataset(name, split=split): + q = r["question"].strip() + eq = (r.get("equation") or "").replace("x =", "").strip() + res = str(r.get("result") or "").strip() + yield f"{q}\nAnswer: {eq} = {res}" if eq and res and eq != res else f"{q}\nAnswer: {res}" + + +def _raw(repo, fn): + from huggingface_hub import hf_hub_download + return json.load(open(hf_hub_download(repo, fn, repo_type="dataset"))) + + +def metamath_gsm(): + # GSM8K-derived rows only (rephrasings, self-verification, FOBAR); the MATH rows are competition level. + for r in _raw("meta-math/MetaMathQA", "MetaMathQA-395K.json"): + if r["type"].startswith("GSM"): + yield r["query"].strip() + "\n" + r["response"].strip() + + +def gsm8k_aug(): + import re + # Training file only; the repo also ships gsm8k_test.json, which is never read. + d = _raw("zen-E/GSM8k-Aug", "gsm8k_aug_train.json") # columnar: {question: [...], cot: [...], answer: [...]} + for r in (dict(zip(d, vals)) for vals in zip(*d.values())): + steps = re.findall(r"<<([^=<>]+)=([^<>]+)>>", r.get("cot", "")) + work = "\n".join(f"{a} = {b}" for a, b in steps) + yield r["question"].strip() + ("\n" + work if work else "") + f"\nAnswer: {str(r['answer']).strip()}" + + +def grade_lessons(): + for r in _raw("ajibawa-2023/Maths-Grade-School", "Maths-Grade-School-0-Final.json"): + t = (r.get("output") or "").strip() + if t: + yield t + + +def omi2_gsm(): + # Full OpenMathInstruct-2 (CC-BY-4.0), grade-school rows only: gsm8k + augmented_gsm8k (~2.6M problems). + from huggingface_hub import hf_hub_download + import pyarrow.parquet as pq + for k in range(32): + t = pq.read_table(hf_hub_download("nvidia/OpenMathInstruct-2", f"data/train-{k:05d}-of-00032.parquet", repo_type="dataset"), + columns=["problem", "generated_solution", "problem_source"]) + for pr, so, src in zip(*(t.column(c).to_pylist() for c in ("problem", "generated_solution", "problem_source"))): + if "gsm" in src: + yield pr.strip() + "\n" + so.strip() + + +def libremath(): + # LibreMath v1 (K2-Horizon, Apache-2.0): single-part and two-part items generated so far. + import glob + for d in ("v1", "v1_pairs", "v1_pairs_key2"): + for f in sorted(glob.glob(f"/media/joseph/Media/libremath/{d}/items-*.jsonl")): + for l in open(f): + r = json.loads(l) + yield r["problem"].strip() + "\n" + r["solution"].strip() + + +def libremath_tpl(): + # LibreMath v2: problems filled from K2-Horizon-written templates (both keys), every value computed in code. + import glob + for d in ("v2_templates_key1", "v2_templates_key2"): + for f in sorted(glob.glob(f"/media/joseph/Media/libremath/{d}/items-*.jsonl")): + for l in open(f): + r = json.loads(l) + yield r["problem"].strip() + "\n" + r["solution"].strip() + + +def _hs_banned_titles(): + import re + norm = lambda t: re.sub(r"[^a-z0-9 ]", "", t.lower().replace("how to", "")).split() + banned = set() + for sp in ("validation", "test"): + for r in load_dataset("Rowan/hellaswag", split=sp): + m = re.match(r"\[header\] (.*?) \[", r["ctx_a"] + " [") + if r["source_id"].startswith("wikihow") and m: banned.add(" ".join(norm(m.group(1)))) + return banned, norm + + +def cosmo1_how_tf(): + # Cosmopedia v1 "wikihow" (the set used in Preview-1/2), now also dropping any article whose title + # matches a HellaSwag WikiHow item, as for cosmo2_how. + import re + from huggingface_hub import hf_hub_download + import pyarrow.parquet as pq + banned, norm = _hs_banned_titles(); dropped = 0 + for k in range(2): + t = pq.read_table(hf_hub_download("HuggingFaceTB/cosmopedia", f"data/wikihow/train-0000{k}-of-00002.parquet", repo_type="dataset"), + columns=["prompt", "text"]) + for pr, tx in zip(t.column("prompt").to_pylist(), t.column("text").to_pylist()): + m = re.search(r'tutorial on "(.+?)"', pr) + if m and " ".join(norm(m.group(1))) in banned: + dropped += 1; continue + yield tx.strip() + print(f" {dropped:,} dropped for a HellaSwag title", flush=True) + + +def cosmo2_how(): + # Cosmopedia v2 (SmolLM-Corpus, ODC-BY; Mixtral-8x7B generations), "wikihow" format only. + # HellaSwag draws on WikiHow, so every document goes through the 13-gram eval filter below. + # Stricter than the 13-gram filter: some articles were prompted with real WikiHow titles, so any article + # whose title matches the title of a HellaSwag WikiHow item (validation or test) is dropped outright. + import re + norm = lambda t: re.sub(r"[^a-z0-9 ]", "", t.lower().replace("how to", "")).split() + banned = set() + for sp in ("validation", "test"): + for r in load_dataset("Rowan/hellaswag", split=sp): + m = re.match(r"\[header\] (.*?) \[", r["ctx_a"] + " [") + if r["source_id"].startswith("wikihow") and m: banned.add(" ".join(norm(m.group(1)))) + print(f" {len(banned):,} HellaSwag WikiHow titles banned", flush=True) + dropped = 0 + import os + start = int(os.environ.get("COSMO2_START_SHARD", "0")) # resume after a network failure: shards are read in order + files = [f"hf://datasets/HuggingFaceTB/smollm-corpus/cosmopedia-v2/train-{k:05d}-of-00104.parquet" for k in range(start, 104)] + for r in load_dataset("parquet", data_files=files, split="train", streaming=True): + if r["format"] != "wikihow": continue + m = re.search(r'"([^"]+)" in the style of WikiHow', r["prompt"]) or re.match(r"\s*Title:\s*(.*)", r["text"]) + if m and " ".join(norm(m.group(1))) in banned: + dropped += 1 + if dropped % 1000 == 0: print(f" {dropped:,} dropped for a HellaSwag title", flush=True) + continue + yield r["text"].strip() + + +GEN = {"tinygsm": (tinygsm, "tinygsm"), "elem": (elem, "elem_wordprob"), "metamath_gsm": (metamath_gsm, "metamath_gsm"), + "gsm8k_aug": (gsm8k_aug, "gsm8k_aug"), "grade_lessons": (grade_lessons, "grade_lessons"), + "omi2_gsm": (omi2_gsm, "omi2_gsm"), "libremath": (libremath, "libremath_v1"), "cosmo2_how": (cosmo2_how, "cosmo2_how"), "libremath_tpl": (libremath_tpl, "libremath_tpl"), "cosmo1_how_tf": (cosmo1_how_tf, "cosmo_how_v1_tf")} + + +def main(): + ap = argparse.ArgumentParser(); ap.add_argument("name", choices=list(GEN)) + ap.add_argument("--tokens", type=float, default=1e12) + a = ap.parse_args() + fn, outname = GEN[a.name] + out = prepare.DATA_DIR / outname + tok = BorisTokenizer.load(prepare.DATA_DIR / "tokenizer"); ban = prepare.eval_ngrams() + import os + w = prepare.ShardWriter(out, tok.eos_id); assert w.n_tokens == 0 or os.environ.get("COSMO2_START_SHARD"), f"{out} not empty" + kept = dropped = toks = 0; batch = [] + gen = fn() + for t in gen: + if prepare.contaminated(t, ban): + dropped += 1; continue + batch.append(t) + if len(batch) == 1000: + toks += w.write(tok.encode_batch(batch)); kept += len(batch); batch = [] + if kept % 200000 == 0: print(f" {kept:,} docs {toks/1e6:.0f}M tokens {dropped} dropped", flush=True) + if toks >= a.tokens: break + if batch: toks += w.write(tok.encode_batch(batch)); kept += len(batch) + w.close() + sha = hashlib.sha256(open(prepare.DATA_DIR / "tokenizer" / "tokenizer.json", "rb").read()).hexdigest() + json.dump({"source": a.name, "documents": kept, "new_tokens": toks, "dropped_contaminated": dropped, + "new_tokenizer_sha256": sha}, open(out / "retokenize_0920.json", "w"), indent=2) + print(f"{out.name}: {kept:,} docs, {toks:,} tokens, {dropped} dropped as contaminated") + + +if __name__ == "__main__": + main() diff --git a/scripts/sample.py b/scripts/sample.py new file mode 100644 index 0000000..b19a600 --- /dev/null +++ b/scripts/sample.py @@ -0,0 +1,47 @@ +"""Sample text from the newest Boris-2-0929 rolling checkpoint on the 3060 (no KV cache; +recomputes the full prefix each token, fine for short samples).""" +import glob, sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +import torch, torch.nn.functional as F +import eval as E +from tokenizer import BorisTokenizer + +dev = "cuda:1" +ck = sorted(glob.glob("/media/joseph/Media/Boris-2-0929/checkpoints/rolling_*.pt"))[-1] +b = torch.load(ck, map_location="cpu", weights_only=False) +tmp = "/home/joseph/boris-2-D112M-n15M/probe/_sample_export.pt" +torch.save({k: b[k] for k in ("model", "config", "step", "tokens_seen", "ngram_fingerprint")}, tmp) +del b +model, cfg, blob = E.load_export(Path(tmp), dev) +Path(tmp).unlink() +tok = BorisTokenizer.load(Path(cfg.data.root) / "tokenizer") +W = model.head_weight().float() +print(f"checkpoint step {blob['step']} tokens {blob['tokens_seen']/1e9:.2f}B\n") + +prompts = sys.argv[1:] or [ + "The capital of France is", + "Once upon a time, in a small village by the sea,", + "Photosynthesis is the process by which", + "To make a cup of tea, first", + "Question: If I have 12 apples and give away 5, how many do I have left?\nAnswer:", + "The most important invention of the 20th century was", +] +torch.manual_seed(0) +for p in prompts: + ids = tok.encode(p) + for _ in range(60): + T = len(ids) + x = torch.tensor([ids], device=dev) + pos = torch.arange(T, device=dev)[None] + bnd = torch.zeros(1, T, dtype=torch.bool, device=dev); bnd[0, 0] = True + off = torch.tensor([0, T], device=dev) + with torch.no_grad(), torch.autocast("cuda", dtype=torch.float16): + h = model(x, pos, off, bnd, None, model._eval_theta) + logits = F.linear(h[0, -1].float(), W) / 0.7 + top = torch.topk(logits, 40) + nxt = int(top.indices[torch.multinomial(F.softmax(top.values, -1), 1)]) + if nxt == tok.eos_id: + break + ids.append(nxt) + print("=" * 70); print(tok.decode(ids)); print() diff --git a/scripts/scan_decay_contam.py b/scripts/scan_decay_contam.py new file mode 100644 index 0000000..2577a05 --- /dev/null +++ b/scripts/scan_decay_contam.py @@ -0,0 +1,17 @@ +"""13-gram contamination scan of the decay sources (same ban set as prepare.py).""" +import sys, json +from pathlib import Path +sys.path.insert(0, "/home/joseph/boris-2-D112M-n15M") +import numpy as np +import prepare +from tokenizer import BorisTokenizer +ROOT = Path("/home/joseph/boris-2-D112M-n15M/data") +tok = BorisTokenizer.load(ROOT / "tokenizer"); ban = prepare.eval_ngrams() +for name in sys.argv[1:]: + ix = np.memmap(ROOT / name / "docs.idx", dtype=np.uint64, mode="r") + tt = np.memmap(ROOT / name / "tokens.bin", dtype=np.uint16, mode="r") + bad = [] + for i in range(len(ix) - 1): + t = tok.tk.decode(tt[int(ix[i]):int(ix[i + 1])].tolist(), skip_special_tokens=True) + if prepare.contaminated(t, ban): bad.append(i) + print(json.dumps({"source": name, "docs": len(ix) - 1, "contaminated": len(bad), "ids": bad}), flush=True) |
