summaryrefslogtreecommitdiff
path: root/scripts/prepare_wordprob.py
diff options
context:
space:
mode:
authorjdj <jdj@kunix.org>2026-11-18 14:47:33 -0800
committerjdj <jdj@kunix.org>2026-11-18 14:47:33 -0800
commit5d2e4252e695644c34be605b44a14bd80a98a0a6 (patch)
tree311a72eda4459af7547b9cd05c0956eafb235f2d /scripts/prepare_wordprob.py
downloadboris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.tar.gz
boris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.zip
Boris-2 training code
The code, configuration and data filters used to train Boris-2 (144M, 200B tokens, single V100): model, training loop, data preparation, n-gram table, V100 kernels, final stages and the exact run configs.
Diffstat (limited to 'scripts/prepare_wordprob.py')
-rw-r--r--scripts/prepare_wordprob.py187
1 files changed, 187 insertions, 0 deletions
diff --git a/scripts/prepare_wordprob.py b/scripts/prepare_wordprob.py
new file mode 100644
index 0000000..cfa0cd6
--- /dev/null
+++ b/scripts/prepare_wordprob.py
@@ -0,0 +1,187 @@
+"""Public word-problem datasets -> Boris token shards (2026-10-10).
+
+ python probe/prepare_wordprob.py tinygsm --tokens 1.5e9
+ python probe/prepare_wordprob.py elem
+
+tinygsm: TinyGSM/TinyGSM (MIT) -- question, then its Python solution.
+elem: MU-NLPC Calc-ape210k and Calc-mawps, training splits only (MIT) -- question, then
+ "Answer: <equation> = <result>".
+Every document goes through the same 13-gram filter as the rest of the corpus (the ban
+set covers HellaSwag, ARC, PIQA and every ArithMark-3 item). Writes data/<name>/ with a
+provenance file.
+"""
+import argparse, hashlib, json, sys
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+from datasets import load_dataset
+import prepare
+from tokenizer import BorisTokenizer
+
+
+def tinygsm():
+ for r in load_dataset("TinyGSM/TinyGSM", split="train", streaming=True):
+ yield r["question"].strip() + "\n\n" + r["code"].strip()
+
+
+def elem():
+ # Training splits only: ASDiv-A and SVAMP exist only as test sets and are left out.
+ for name in ("MU-NLPC/Calc-ape210k", "MU-NLPC/Calc-mawps"):
+ for split in ("train",):
+ for r in load_dataset(name, split=split):
+ q = r["question"].strip()
+ eq = (r.get("equation") or "").replace("x =", "").strip()
+ res = str(r.get("result") or "").strip()
+ yield f"{q}\nAnswer: {eq} = {res}" if eq and res and eq != res else f"{q}\nAnswer: {res}"
+
+
+def _raw(repo, fn):
+ from huggingface_hub import hf_hub_download
+ return json.load(open(hf_hub_download(repo, fn, repo_type="dataset")))
+
+
+def metamath_gsm():
+ # GSM8K-derived rows only (rephrasings, self-verification, FOBAR); the MATH rows are competition level.
+ for r in _raw("meta-math/MetaMathQA", "MetaMathQA-395K.json"):
+ if r["type"].startswith("GSM"):
+ yield r["query"].strip() + "\n" + r["response"].strip()
+
+
+def gsm8k_aug():
+ import re
+ # Training file only; the repo also ships gsm8k_test.json, which is never read.
+ d = _raw("zen-E/GSM8k-Aug", "gsm8k_aug_train.json") # columnar: {question: [...], cot: [...], answer: [...]}
+ for r in (dict(zip(d, vals)) for vals in zip(*d.values())):
+ steps = re.findall(r"<<([^=<>]+)=([^<>]+)>>", r.get("cot", ""))
+ work = "\n".join(f"{a} = {b}" for a, b in steps)
+ yield r["question"].strip() + ("\n" + work if work else "") + f"\nAnswer: {str(r['answer']).strip()}"
+
+
+def grade_lessons():
+ for r in _raw("ajibawa-2023/Maths-Grade-School", "Maths-Grade-School-0-Final.json"):
+ t = (r.get("output") or "").strip()
+ if t:
+ yield t
+
+
+def omi2_gsm():
+ # Full OpenMathInstruct-2 (CC-BY-4.0), grade-school rows only: gsm8k + augmented_gsm8k (~2.6M problems).
+ from huggingface_hub import hf_hub_download
+ import pyarrow.parquet as pq
+ for k in range(32):
+ t = pq.read_table(hf_hub_download("nvidia/OpenMathInstruct-2", f"data/train-{k:05d}-of-00032.parquet", repo_type="dataset"),
+ columns=["problem", "generated_solution", "problem_source"])
+ for pr, so, src in zip(*(t.column(c).to_pylist() for c in ("problem", "generated_solution", "problem_source"))):
+ if "gsm" in src:
+ yield pr.strip() + "\n" + so.strip()
+
+
+def libremath():
+ # LibreMath v1 (K2-Horizon, Apache-2.0): single-part and two-part items generated so far.
+ import glob
+ for d in ("v1", "v1_pairs", "v1_pairs_key2"):
+ for f in sorted(glob.glob(f"/media/joseph/Media/libremath/{d}/items-*.jsonl")):
+ for l in open(f):
+ r = json.loads(l)
+ yield r["problem"].strip() + "\n" + r["solution"].strip()
+
+
+def libremath_tpl():
+ # LibreMath v2: problems filled from K2-Horizon-written templates (both keys), every value computed in code.
+ import glob
+ for d in ("v2_templates_key1", "v2_templates_key2"):
+ for f in sorted(glob.glob(f"/media/joseph/Media/libremath/{d}/items-*.jsonl")):
+ for l in open(f):
+ r = json.loads(l)
+ yield r["problem"].strip() + "\n" + r["solution"].strip()
+
+
+def _hs_banned_titles():
+ import re
+ norm = lambda t: re.sub(r"[^a-z0-9 ]", "", t.lower().replace("how to", "")).split()
+ banned = set()
+ for sp in ("validation", "test"):
+ for r in load_dataset("Rowan/hellaswag", split=sp):
+ m = re.match(r"\[header\] (.*?) \[", r["ctx_a"] + " [")
+ if r["source_id"].startswith("wikihow") and m: banned.add(" ".join(norm(m.group(1))))
+ return banned, norm
+
+
+def cosmo1_how_tf():
+ # Cosmopedia v1 "wikihow" (the set used in Preview-1/2), now also dropping any article whose title
+ # matches a HellaSwag WikiHow item, as for cosmo2_how.
+ import re
+ from huggingface_hub import hf_hub_download
+ import pyarrow.parquet as pq
+ banned, norm = _hs_banned_titles(); dropped = 0
+ for k in range(2):
+ t = pq.read_table(hf_hub_download("HuggingFaceTB/cosmopedia", f"data/wikihow/train-0000{k}-of-00002.parquet", repo_type="dataset"),
+ columns=["prompt", "text"])
+ for pr, tx in zip(t.column("prompt").to_pylist(), t.column("text").to_pylist()):
+ m = re.search(r'tutorial on "(.+?)"', pr)
+ if m and " ".join(norm(m.group(1))) in banned:
+ dropped += 1; continue
+ yield tx.strip()
+ print(f" {dropped:,} dropped for a HellaSwag title", flush=True)
+
+
+def cosmo2_how():
+ # Cosmopedia v2 (SmolLM-Corpus, ODC-BY; Mixtral-8x7B generations), "wikihow" format only.
+ # HellaSwag draws on WikiHow, so every document goes through the 13-gram eval filter below.
+ # Stricter than the 13-gram filter: some articles were prompted with real WikiHow titles, so any article
+ # whose title matches the title of a HellaSwag WikiHow item (validation or test) is dropped outright.
+ import re
+ norm = lambda t: re.sub(r"[^a-z0-9 ]", "", t.lower().replace("how to", "")).split()
+ banned = set()
+ for sp in ("validation", "test"):
+ for r in load_dataset("Rowan/hellaswag", split=sp):
+ m = re.match(r"\[header\] (.*?) \[", r["ctx_a"] + " [")
+ if r["source_id"].startswith("wikihow") and m: banned.add(" ".join(norm(m.group(1))))
+ print(f" {len(banned):,} HellaSwag WikiHow titles banned", flush=True)
+ dropped = 0
+ import os
+ start = int(os.environ.get("COSMO2_START_SHARD", "0")) # resume after a network failure: shards are read in order
+ files = [f"hf://datasets/HuggingFaceTB/smollm-corpus/cosmopedia-v2/train-{k:05d}-of-00104.parquet" for k in range(start, 104)]
+ for r in load_dataset("parquet", data_files=files, split="train", streaming=True):
+ if r["format"] != "wikihow": continue
+ m = re.search(r'"([^"]+)" in the style of WikiHow', r["prompt"]) or re.match(r"\s*Title:\s*(.*)", r["text"])
+ if m and " ".join(norm(m.group(1))) in banned:
+ dropped += 1
+ if dropped % 1000 == 0: print(f" {dropped:,} dropped for a HellaSwag title", flush=True)
+ continue
+ yield r["text"].strip()
+
+
+GEN = {"tinygsm": (tinygsm, "tinygsm"), "elem": (elem, "elem_wordprob"), "metamath_gsm": (metamath_gsm, "metamath_gsm"),
+ "gsm8k_aug": (gsm8k_aug, "gsm8k_aug"), "grade_lessons": (grade_lessons, "grade_lessons"),
+ "omi2_gsm": (omi2_gsm, "omi2_gsm"), "libremath": (libremath, "libremath_v1"), "cosmo2_how": (cosmo2_how, "cosmo2_how"), "libremath_tpl": (libremath_tpl, "libremath_tpl"), "cosmo1_how_tf": (cosmo1_how_tf, "cosmo_how_v1_tf")}
+
+
+def main():
+ ap = argparse.ArgumentParser(); ap.add_argument("name", choices=list(GEN))
+ ap.add_argument("--tokens", type=float, default=1e12)
+ a = ap.parse_args()
+ fn, outname = GEN[a.name]
+ out = prepare.DATA_DIR / outname
+ tok = BorisTokenizer.load(prepare.DATA_DIR / "tokenizer"); ban = prepare.eval_ngrams()
+ import os
+ w = prepare.ShardWriter(out, tok.eos_id); assert w.n_tokens == 0 or os.environ.get("COSMO2_START_SHARD"), f"{out} not empty"
+ kept = dropped = toks = 0; batch = []
+ gen = fn()
+ for t in gen:
+ if prepare.contaminated(t, ban):
+ dropped += 1; continue
+ batch.append(t)
+ if len(batch) == 1000:
+ toks += w.write(tok.encode_batch(batch)); kept += len(batch); batch = []
+ if kept % 200000 == 0: print(f" {kept:,} docs {toks/1e6:.0f}M tokens {dropped} dropped", flush=True)
+ if toks >= a.tokens: break
+ if batch: toks += w.write(tok.encode_batch(batch)); kept += len(batch)
+ w.close()
+ sha = hashlib.sha256(open(prepare.DATA_DIR / "tokenizer" / "tokenizer.json", "rb").read()).hexdigest()
+ json.dump({"source": a.name, "documents": kept, "new_tokens": toks, "dropped_contaminated": dropped,
+ "new_tokenizer_sha256": sha}, open(out / "retokenize_0920.json", "w"), indent=2)
+ print(f"{out.name}: {kept:,} docs, {toks:,} tokens, {dropped} dropped as contaminated")
+
+
+if __name__ == "__main__":
+ main()