From 5d2e4252e695644c34be605b44a14bd80a98a0a6 Mon Sep 17 00:00:00 2001 From: jdj Date: Wed, 18 Nov 2026 14:47:33 -0800 Subject: Boris-2 training code The code, configuration and data filters used to train Boris-2 (144M, 200B tokens, single V100): model, training loop, data preparation, n-gram table, V100 kernels, final stages and the exact run configs. --- retokenize.py | 103 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 retokenize.py (limited to 'retokenize.py') diff --git a/retokenize.py b/retokenize.py new file mode 100644 index 0000000..6e9b88d --- /dev/null +++ b/retokenize.py @@ -0,0 +1,103 @@ +"""Transactionally convert prepared uint16 shards to a different Boris BPE. + +The source format preserves document boundaries, so decoding each old document +and encoding it with the new byte-level BPE is lossless. Conversion writes a +sibling directory and never mutates the source shard. A separate, explicit +swap step is required after training has stopped and validation has passed. + +Example: + python retokenize.py data/climbmix data/tokenizer data/tokenizer-0920 \ + --out data/climbmix.retokenized-0920 +""" +from __future__ import annotations + +import argparse +import json +import os +import time +from pathlib import Path + +import numpy as np + +from tokenizer import BorisTokenizer + + +def convert(source: Path, old_dir: Path, new_dir: Path, out: Path, + batch_docs: int = 2048) -> None: + source, out = Path(source), Path(out) + if source.resolve() == out.resolve(): + raise SystemExit("output must differ from source; in-place conversion is unsafe") + out.mkdir(parents=True, exist_ok=True) + out_tokens, out_idx = out / "tokens.bin", out / "docs.idx" + if out_tokens.exists() or out_idx.exists(): + raise SystemExit(f"{out}: output files already exist; refusing to append or overwrite") + + old, new = BorisTokenizer.load(old_dir), BorisTokenizer.load(new_dir) + tokens = np.memmap(source / "tokens.bin", dtype=np.uint16, mode="r") + offsets = np.memmap(source / "docs.idx", dtype=np.uint64, mode="r") + if offsets.size < 2 or int(offsets[0]) != 0 or int(offsets[-1]) != tokens.size: + raise SystemExit(f"{source}: inconsistent token/index files") + + t0 = time.time() + written = 0 + with open(out_tokens, "wb") as ft, open(out_idx, "wb") as fi: + np.asarray([0], dtype=np.uint64).tofile(fi) + for start in range(0, offsets.size - 1, batch_docs): + stop = min(start + batch_docs, offsets.size - 1) + texts = [] + for i in range(start, stop): + a, b = int(offsets[i]), int(offsets[i + 1]) + ids = tokens[a:b] + if ids.size and int(ids[-1]) == old.eos_id: + ids = ids[:-1] + texts.append(old.decode(ids)) + encoded = new.encode_batch(texts) + chunks, ends = [], [] + for text, ids in zip(texts, encoded): + # Both tokenizers are byte-level; this makes corruption a hard + # failure before the source shard is ever considered for swap. + if new.decode(ids) != text: + raise RuntimeError(f"round-trip failed at document {start + len(ends)}") + chunk = np.asarray(ids + [new.eos_id], dtype=np.uint16) + chunks.append(chunk) + written += chunk.size + ends.append(written) + if chunks: + np.concatenate(chunks).tofile(ft) + np.asarray(ends, dtype=np.uint64).tofile(fi) + if start == 0 or (start // batch_docs) % 100 == 0: + rate = written / max(time.time() - t0, 1e-9) + print(f" {stop:,}/{offsets.size-1:,} docs {written/1e9:.3f}B tokens " + f"{rate/1e6:.2f}M tok/s", flush=True) + ft.flush(); os.fsync(ft.fileno()) + fi.flush(); os.fsync(fi.fileno()) + + check_idx = np.memmap(out_idx, dtype=np.uint64, mode="r") + expected_bytes = int(check_idx[-1]) * np.dtype(np.uint16).itemsize + if check_idx.size != offsets.size or out_tokens.stat().st_size != expected_bytes: + raise RuntimeError("converted shard failed final size/index validation") + manifest = { + "source": str(source.resolve()), + "documents": int(offsets.size - 1), + "old_tokens": int(tokens.size), + "new_tokens": int(check_idx[-1]), + "old_tokenizer_sha256": BorisTokenizer.fingerprint(old_dir), + "new_tokenizer_sha256": BorisTokenizer.fingerprint(new_dir), + } + (out / "retokenize.json").write_text(json.dumps(manifest, indent=2) + "\n") + print(json.dumps(manifest, indent=2)) + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("source", type=Path) + ap.add_argument("old_tokenizer", type=Path) + ap.add_argument("new_tokenizer", type=Path) + ap.add_argument("--out", type=Path, required=True) + ap.add_argument("--batch-docs", type=int, default=2048) + a = ap.parse_args() + convert(a.source, a.old_tokenizer, a.new_tokenizer, a.out, a.batch_docs) + + +if __name__ == "__main__": + main() -- cgit v1.2.3