"""microsoft/orca-math-word-problems-200k -> data/orca_math (question + answer as one document), 13-gram decontaminated against the eval suite like every other source.""" import sys, json sys.path.insert(0, "/home/joseph/boris-2-D112M-n15M") from datasets import load_dataset import prepare from tokenizer import BorisTokenizer tok = BorisTokenizer.load(prepare.DATA_DIR / "tokenizer"); ban = prepare.eval_ngrams() w = prepare.ShardWriter(prepare.DATA_DIR / "orca_math", tok.eos_id) assert w.n_tokens == 0, "data/orca_math already has tokens" ds = load_dataset("microsoft/orca-math-word-problems-200k", split="train") kept = dropped = toks = 0; batch = [] for r in ds: t = r["question"].strip() + "\n\n" + r["answer"].strip() if prepare.contaminated(t, ban): dropped += 1; continue batch.append(t) if len(batch) == 1000: toks += w.write(tok.encode_batch(batch)); kept += len(batch); batch = [] if batch: toks += w.write(tok.encode_batch(batch)); kept += len(batch) w.close() json.dump({"source": "microsoft/orca-math-word-problems-200k", "documents": kept, "tokens": toks, "dropped_contaminated": dropped}, open(prepare.DATA_DIR / "orca_math" / "prepare_meta.json", "w"), indent=2) print("orca_math:", kept, "docs", toks, "tokens", dropped, "dropped as contaminated")