diff options
Diffstat (limited to 'scripts/prepare_orca.py')
| -rw-r--r-- | scripts/prepare_orca.py | 23 |
1 files changed, 23 insertions, 0 deletions
diff --git a/scripts/prepare_orca.py b/scripts/prepare_orca.py new file mode 100644 index 0000000..37f27b4 --- /dev/null +++ b/scripts/prepare_orca.py @@ -0,0 +1,23 @@ +"""microsoft/orca-math-word-problems-200k -> data/orca_math (question + answer as one +document), 13-gram decontaminated against the eval suite like every other source.""" +import sys, json +sys.path.insert(0, "/home/joseph/boris-2-D112M-n15M") +from datasets import load_dataset +import prepare +from tokenizer import BorisTokenizer +tok = BorisTokenizer.load(prepare.DATA_DIR / "tokenizer"); ban = prepare.eval_ngrams() +w = prepare.ShardWriter(prepare.DATA_DIR / "orca_math", tok.eos_id) +assert w.n_tokens == 0, "data/orca_math already has tokens" +ds = load_dataset("microsoft/orca-math-word-problems-200k", split="train") +kept = dropped = toks = 0; batch = [] +for r in ds: + t = r["question"].strip() + "\n\n" + r["answer"].strip() + if prepare.contaminated(t, ban): dropped += 1; continue + batch.append(t) + if len(batch) == 1000: + toks += w.write(tok.encode_batch(batch)); kept += len(batch); batch = [] +if batch: toks += w.write(tok.encode_batch(batch)); kept += len(batch) +w.close() +json.dump({"source": "microsoft/orca-math-word-problems-200k", "documents": kept, "tokens": toks, + "dropped_contaminated": dropped}, open(prepare.DATA_DIR / "orca_math" / "prepare_meta.json", "w"), indent=2) +print("orca_math:", kept, "docs", toks, "tokens", dropped, "dropped as contaminated") |
