summaryrefslogtreecommitdiff
path: root/retokenize.py
diff options
context:
space:
mode:
Diffstat (limited to 'retokenize.py')
-rw-r--r--retokenize.py103
1 files changed, 103 insertions, 0 deletions
diff --git a/retokenize.py b/retokenize.py
new file mode 100644
index 0000000..6e9b88d
--- /dev/null
+++ b/retokenize.py
@@ -0,0 +1,103 @@
+"""Transactionally convert prepared uint16 shards to a different Boris BPE.
+
+The source format preserves document boundaries, so decoding each old document
+and encoding it with the new byte-level BPE is lossless. Conversion writes a
+sibling directory and never mutates the source shard. A separate, explicit
+swap step is required after training has stopped and validation has passed.
+
+Example:
+ python retokenize.py data/climbmix data/tokenizer data/tokenizer-0920 \
+ --out data/climbmix.retokenized-0920
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import time
+from pathlib import Path
+
+import numpy as np
+
+from tokenizer import BorisTokenizer
+
+
+def convert(source: Path, old_dir: Path, new_dir: Path, out: Path,
+ batch_docs: int = 2048) -> None:
+ source, out = Path(source), Path(out)
+ if source.resolve() == out.resolve():
+ raise SystemExit("output must differ from source; in-place conversion is unsafe")
+ out.mkdir(parents=True, exist_ok=True)
+ out_tokens, out_idx = out / "tokens.bin", out / "docs.idx"
+ if out_tokens.exists() or out_idx.exists():
+ raise SystemExit(f"{out}: output files already exist; refusing to append or overwrite")
+
+ old, new = BorisTokenizer.load(old_dir), BorisTokenizer.load(new_dir)
+ tokens = np.memmap(source / "tokens.bin", dtype=np.uint16, mode="r")
+ offsets = np.memmap(source / "docs.idx", dtype=np.uint64, mode="r")
+ if offsets.size < 2 or int(offsets[0]) != 0 or int(offsets[-1]) != tokens.size:
+ raise SystemExit(f"{source}: inconsistent token/index files")
+
+ t0 = time.time()
+ written = 0
+ with open(out_tokens, "wb") as ft, open(out_idx, "wb") as fi:
+ np.asarray([0], dtype=np.uint64).tofile(fi)
+ for start in range(0, offsets.size - 1, batch_docs):
+ stop = min(start + batch_docs, offsets.size - 1)
+ texts = []
+ for i in range(start, stop):
+ a, b = int(offsets[i]), int(offsets[i + 1])
+ ids = tokens[a:b]
+ if ids.size and int(ids[-1]) == old.eos_id:
+ ids = ids[:-1]
+ texts.append(old.decode(ids))
+ encoded = new.encode_batch(texts)
+ chunks, ends = [], []
+ for text, ids in zip(texts, encoded):
+ # Both tokenizers are byte-level; this makes corruption a hard
+ # failure before the source shard is ever considered for swap.
+ if new.decode(ids) != text:
+ raise RuntimeError(f"round-trip failed at document {start + len(ends)}")
+ chunk = np.asarray(ids + [new.eos_id], dtype=np.uint16)
+ chunks.append(chunk)
+ written += chunk.size
+ ends.append(written)
+ if chunks:
+ np.concatenate(chunks).tofile(ft)
+ np.asarray(ends, dtype=np.uint64).tofile(fi)
+ if start == 0 or (start // batch_docs) % 100 == 0:
+ rate = written / max(time.time() - t0, 1e-9)
+ print(f" {stop:,}/{offsets.size-1:,} docs {written/1e9:.3f}B tokens "
+ f"{rate/1e6:.2f}M tok/s", flush=True)
+ ft.flush(); os.fsync(ft.fileno())
+ fi.flush(); os.fsync(fi.fileno())
+
+ check_idx = np.memmap(out_idx, dtype=np.uint64, mode="r")
+ expected_bytes = int(check_idx[-1]) * np.dtype(np.uint16).itemsize
+ if check_idx.size != offsets.size or out_tokens.stat().st_size != expected_bytes:
+ raise RuntimeError("converted shard failed final size/index validation")
+ manifest = {
+ "source": str(source.resolve()),
+ "documents": int(offsets.size - 1),
+ "old_tokens": int(tokens.size),
+ "new_tokens": int(check_idx[-1]),
+ "old_tokenizer_sha256": BorisTokenizer.fingerprint(old_dir),
+ "new_tokenizer_sha256": BorisTokenizer.fingerprint(new_dir),
+ }
+ (out / "retokenize.json").write_text(json.dumps(manifest, indent=2) + "\n")
+ print(json.dumps(manifest, indent=2))
+
+
+def main() -> None:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("source", type=Path)
+ ap.add_argument("old_tokenizer", type=Path)
+ ap.add_argument("new_tokenizer", type=Path)
+ ap.add_argument("--out", type=Path, required=True)
+ ap.add_argument("--batch-docs", type=int, default=2048)
+ a = ap.parse_args()
+ convert(a.source, a.old_tokenizer, a.new_tokenizer, a.out, a.batch_docs)
+
+
+if __name__ == "__main__":
+ main()