"""Transactionally convert prepared uint16 shards to a different Boris BPE. The source format preserves document boundaries, so decoding each old document and encoding it with the new byte-level BPE is lossless. Conversion writes a sibling directory and never mutates the source shard. A separate, explicit swap step is required after training has stopped and validation has passed. Example: python retokenize.py data/climbmix data/tokenizer data/tokenizer-0920 \ --out data/climbmix.retokenized-0920 """ from __future__ import annotations import argparse import json import os import time from pathlib import Path import numpy as np from tokenizer import BorisTokenizer def convert(source: Path, old_dir: Path, new_dir: Path, out: Path, batch_docs: int = 2048) -> None: source, out = Path(source), Path(out) if source.resolve() == out.resolve(): raise SystemExit("output must differ from source; in-place conversion is unsafe") out.mkdir(parents=True, exist_ok=True) out_tokens, out_idx = out / "tokens.bin", out / "docs.idx" if out_tokens.exists() or out_idx.exists(): raise SystemExit(f"{out}: output files already exist; refusing to append or overwrite") old, new = BorisTokenizer.load(old_dir), BorisTokenizer.load(new_dir) tokens = np.memmap(source / "tokens.bin", dtype=np.uint16, mode="r") offsets = np.memmap(source / "docs.idx", dtype=np.uint64, mode="r") if offsets.size < 2 or int(offsets[0]) != 0 or int(offsets[-1]) != tokens.size: raise SystemExit(f"{source}: inconsistent token/index files") t0 = time.time() written = 0 with open(out_tokens, "wb") as ft, open(out_idx, "wb") as fi: np.asarray([0], dtype=np.uint64).tofile(fi) for start in range(0, offsets.size - 1, batch_docs): stop = min(start + batch_docs, offsets.size - 1) texts = [] for i in range(start, stop): a, b = int(offsets[i]), int(offsets[i + 1]) ids = tokens[a:b] if ids.size and int(ids[-1]) == old.eos_id: ids = ids[:-1] texts.append(old.decode(ids)) encoded = new.encode_batch(texts) chunks, ends = [], [] for text, ids in zip(texts, encoded): # Both tokenizers are byte-level; this makes corruption a hard # failure before the source shard is ever considered for swap. if new.decode(ids) != text: raise RuntimeError(f"round-trip failed at document {start + len(ends)}") chunk = np.asarray(ids + [new.eos_id], dtype=np.uint16) chunks.append(chunk) written += chunk.size ends.append(written) if chunks: np.concatenate(chunks).tofile(ft) np.asarray(ends, dtype=np.uint64).tofile(fi) if start == 0 or (start // batch_docs) % 100 == 0: rate = written / max(time.time() - t0, 1e-9) print(f" {stop:,}/{offsets.size-1:,} docs {written/1e9:.3f}B tokens " f"{rate/1e6:.2f}M tok/s", flush=True) ft.flush(); os.fsync(ft.fileno()) fi.flush(); os.fsync(fi.fileno()) check_idx = np.memmap(out_idx, dtype=np.uint64, mode="r") expected_bytes = int(check_idx[-1]) * np.dtype(np.uint16).itemsize if check_idx.size != offsets.size or out_tokens.stat().st_size != expected_bytes: raise RuntimeError("converted shard failed final size/index validation") manifest = { "source": str(source.resolve()), "documents": int(offsets.size - 1), "old_tokens": int(tokens.size), "new_tokens": int(check_idx[-1]), "old_tokenizer_sha256": BorisTokenizer.fingerprint(old_dir), "new_tokenizer_sha256": BorisTokenizer.fingerprint(new_dir), } (out / "retokenize.json").write_text(json.dumps(manifest, indent=2) + "\n") print(json.dumps(manifest, indent=2)) def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("source", type=Path) ap.add_argument("old_tokenizer", type=Path) ap.add_argument("new_tokenizer", type=Path) ap.add_argument("--out", type=Path, required=True) ap.add_argument("--batch-docs", type=int, default=2048) a = ap.parse_args() convert(a.source, a.old_tokenizer, a.new_tokenizer, a.out, a.batch_docs) if __name__ == "__main__": main()