diff options
| author | jdj <jdj@kunix.org> | 2026-11-18 14:47:33 -0800 |
|---|---|---|
| committer | jdj <jdj@kunix.org> | 2026-11-18 14:47:33 -0800 |
| commit | 5d2e4252e695644c34be605b44a14bd80a98a0a6 (patch) | |
| tree | 311a72eda4459af7547b9cd05c0956eafb235f2d /retokenize.py | |
| download | boris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.tar.gz boris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.zip | |
Boris-2 training code
The code, configuration and data filters used to train Boris-2 (144M,
200B tokens, single V100): model, training loop, data preparation,
n-gram table, V100 kernels, final stages and the exact run configs.
Diffstat (limited to 'retokenize.py')
| -rw-r--r-- | retokenize.py | 103 |
1 files changed, 103 insertions, 0 deletions
diff --git a/retokenize.py b/retokenize.py new file mode 100644 index 0000000..6e9b88d --- /dev/null +++ b/retokenize.py @@ -0,0 +1,103 @@ +"""Transactionally convert prepared uint16 shards to a different Boris BPE. + +The source format preserves document boundaries, so decoding each old document +and encoding it with the new byte-level BPE is lossless. Conversion writes a +sibling directory and never mutates the source shard. A separate, explicit +swap step is required after training has stopped and validation has passed. + +Example: + python retokenize.py data/climbmix data/tokenizer data/tokenizer-0920 \ + --out data/climbmix.retokenized-0920 +""" +from __future__ import annotations + +import argparse +import json +import os +import time +from pathlib import Path + +import numpy as np + +from tokenizer import BorisTokenizer + + +def convert(source: Path, old_dir: Path, new_dir: Path, out: Path, + batch_docs: int = 2048) -> None: + source, out = Path(source), Path(out) + if source.resolve() == out.resolve(): + raise SystemExit("output must differ from source; in-place conversion is unsafe") + out.mkdir(parents=True, exist_ok=True) + out_tokens, out_idx = out / "tokens.bin", out / "docs.idx" + if out_tokens.exists() or out_idx.exists(): + raise SystemExit(f"{out}: output files already exist; refusing to append or overwrite") + + old, new = BorisTokenizer.load(old_dir), BorisTokenizer.load(new_dir) + tokens = np.memmap(source / "tokens.bin", dtype=np.uint16, mode="r") + offsets = np.memmap(source / "docs.idx", dtype=np.uint64, mode="r") + if offsets.size < 2 or int(offsets[0]) != 0 or int(offsets[-1]) != tokens.size: + raise SystemExit(f"{source}: inconsistent token/index files") + + t0 = time.time() + written = 0 + with open(out_tokens, "wb") as ft, open(out_idx, "wb") as fi: + np.asarray([0], dtype=np.uint64).tofile(fi) + for start in range(0, offsets.size - 1, batch_docs): + stop = min(start + batch_docs, offsets.size - 1) + texts = [] + for i in range(start, stop): + a, b = int(offsets[i]), int(offsets[i + 1]) + ids = tokens[a:b] + if ids.size and int(ids[-1]) == old.eos_id: + ids = ids[:-1] + texts.append(old.decode(ids)) + encoded = new.encode_batch(texts) + chunks, ends = [], [] + for text, ids in zip(texts, encoded): + # Both tokenizers are byte-level; this makes corruption a hard + # failure before the source shard is ever considered for swap. + if new.decode(ids) != text: + raise RuntimeError(f"round-trip failed at document {start + len(ends)}") + chunk = np.asarray(ids + [new.eos_id], dtype=np.uint16) + chunks.append(chunk) + written += chunk.size + ends.append(written) + if chunks: + np.concatenate(chunks).tofile(ft) + np.asarray(ends, dtype=np.uint64).tofile(fi) + if start == 0 or (start // batch_docs) % 100 == 0: + rate = written / max(time.time() - t0, 1e-9) + print(f" {stop:,}/{offsets.size-1:,} docs {written/1e9:.3f}B tokens " + f"{rate/1e6:.2f}M tok/s", flush=True) + ft.flush(); os.fsync(ft.fileno()) + fi.flush(); os.fsync(fi.fileno()) + + check_idx = np.memmap(out_idx, dtype=np.uint64, mode="r") + expected_bytes = int(check_idx[-1]) * np.dtype(np.uint16).itemsize + if check_idx.size != offsets.size or out_tokens.stat().st_size != expected_bytes: + raise RuntimeError("converted shard failed final size/index validation") + manifest = { + "source": str(source.resolve()), + "documents": int(offsets.size - 1), + "old_tokens": int(tokens.size), + "new_tokens": int(check_idx[-1]), + "old_tokenizer_sha256": BorisTokenizer.fingerprint(old_dir), + "new_tokenizer_sha256": BorisTokenizer.fingerprint(new_dir), + } + (out / "retokenize.json").write_text(json.dumps(manifest, indent=2) + "\n") + print(json.dumps(manifest, indent=2)) + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("source", type=Path) + ap.add_argument("old_tokenizer", type=Path) + ap.add_argument("new_tokenizer", type=Path) + ap.add_argument("--out", type=Path, required=True) + ap.add_argument("--batch-docs", type=int, default=2048) + a = ap.parse_args() + convert(a.source, a.old_tokenizer, a.new_tokenizer, a.out, a.batch_docs) + + +if __name__ == "__main__": + main() |
