summaryrefslogtreecommitdiff
path: root/scripts/prepare_orca.py
diff options
context:
space:
mode:
authorjdj <jdj@kunix.org>2026-11-18 14:47:33 -0800
committerjdj <jdj@kunix.org>2026-11-18 14:47:33 -0800
commit5d2e4252e695644c34be605b44a14bd80a98a0a6 (patch)
tree311a72eda4459af7547b9cd05c0956eafb235f2d /scripts/prepare_orca.py
downloadboris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.tar.gz
boris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.zip
Boris-2 training code
The code, configuration and data filters used to train Boris-2 (144M, 200B tokens, single V100): model, training loop, data preparation, n-gram table, V100 kernels, final stages and the exact run configs.
Diffstat (limited to 'scripts/prepare_orca.py')
-rw-r--r--scripts/prepare_orca.py23
1 files changed, 23 insertions, 0 deletions
diff --git a/scripts/prepare_orca.py b/scripts/prepare_orca.py
new file mode 100644
index 0000000..37f27b4
--- /dev/null
+++ b/scripts/prepare_orca.py
@@ -0,0 +1,23 @@
+"""microsoft/orca-math-word-problems-200k -> data/orca_math (question + answer as one
+document), 13-gram decontaminated against the eval suite like every other source."""
+import sys, json
+sys.path.insert(0, "/home/joseph/boris-2-D112M-n15M")
+from datasets import load_dataset
+import prepare
+from tokenizer import BorisTokenizer
+tok = BorisTokenizer.load(prepare.DATA_DIR / "tokenizer"); ban = prepare.eval_ngrams()
+w = prepare.ShardWriter(prepare.DATA_DIR / "orca_math", tok.eos_id)
+assert w.n_tokens == 0, "data/orca_math already has tokens"
+ds = load_dataset("microsoft/orca-math-word-problems-200k", split="train")
+kept = dropped = toks = 0; batch = []
+for r in ds:
+ t = r["question"].strip() + "\n\n" + r["answer"].strip()
+ if prepare.contaminated(t, ban): dropped += 1; continue
+ batch.append(t)
+ if len(batch) == 1000:
+ toks += w.write(tok.encode_batch(batch)); kept += len(batch); batch = []
+if batch: toks += w.write(tok.encode_batch(batch)); kept += len(batch)
+w.close()
+json.dump({"source": "microsoft/orca-math-word-problems-200k", "documents": kept, "tokens": toks,
+ "dropped_contaminated": dropped}, open(prepare.DATA_DIR / "orca_math" / "prepare_meta.json", "w"), indent=2)
+print("orca_math:", kept, "docs", toks, "tokens", dropped, "dropped as contaminated")