summaryrefslogtreecommitdiff
path: root/scripts/prepare_orca.py
blob: 37f27b4b6d4914cab08a618ebce6771d23fbb4d3 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
"""microsoft/orca-math-word-problems-200k -> data/orca_math (question + answer as one
document), 13-gram decontaminated against the eval suite like every other source."""
import sys, json
sys.path.insert(0, "/home/joseph/boris-2-D112M-n15M")
from datasets import load_dataset
import prepare
from tokenizer import BorisTokenizer
tok = BorisTokenizer.load(prepare.DATA_DIR / "tokenizer"); ban = prepare.eval_ngrams()
w = prepare.ShardWriter(prepare.DATA_DIR / "orca_math", tok.eos_id)
assert w.n_tokens == 0, "data/orca_math already has tokens"
ds = load_dataset("microsoft/orca-math-word-problems-200k", split="train")
kept = dropped = toks = 0; batch = []
for r in ds:
    t = r["question"].strip() + "\n\n" + r["answer"].strip()
    if prepare.contaminated(t, ban): dropped += 1; continue
    batch.append(t)
    if len(batch) == 1000:
        toks += w.write(tok.encode_batch(batch)); kept += len(batch); batch = []
if batch: toks += w.write(tok.encode_batch(batch)); kept += len(batch)
w.close()
json.dump({"source": "microsoft/orca-math-word-problems-200k", "documents": kept, "tokens": toks,
           "dropped_contaminated": dropped}, open(prepare.DATA_DIR / "orca_math" / "prepare_meta.json", "w"), indent=2)
print("orca_math:", kept, "docs", toks, "tokens", dropped, "dropped as contaminated")