1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
|
"""Transactionally convert prepared uint16 shards to a different Boris BPE.
The source format preserves document boundaries, so decoding each old document
and encoding it with the new byte-level BPE is lossless. Conversion writes a
sibling directory and never mutates the source shard. A separate, explicit
swap step is required after training has stopped and validation has passed.
Example:
python retokenize.py data/climbmix data/tokenizer data/tokenizer-0920 \
--out data/climbmix.retokenized-0920
"""
from __future__ import annotations
import argparse
import json
import os
import time
from pathlib import Path
import numpy as np
from tokenizer import BorisTokenizer
def convert(source: Path, old_dir: Path, new_dir: Path, out: Path,
batch_docs: int = 2048) -> None:
source, out = Path(source), Path(out)
if source.resolve() == out.resolve():
raise SystemExit("output must differ from source; in-place conversion is unsafe")
out.mkdir(parents=True, exist_ok=True)
out_tokens, out_idx = out / "tokens.bin", out / "docs.idx"
if out_tokens.exists() or out_idx.exists():
raise SystemExit(f"{out}: output files already exist; refusing to append or overwrite")
old, new = BorisTokenizer.load(old_dir), BorisTokenizer.load(new_dir)
tokens = np.memmap(source / "tokens.bin", dtype=np.uint16, mode="r")
offsets = np.memmap(source / "docs.idx", dtype=np.uint64, mode="r")
if offsets.size < 2 or int(offsets[0]) != 0 or int(offsets[-1]) != tokens.size:
raise SystemExit(f"{source}: inconsistent token/index files")
t0 = time.time()
written = 0
with open(out_tokens, "wb") as ft, open(out_idx, "wb") as fi:
np.asarray([0], dtype=np.uint64).tofile(fi)
for start in range(0, offsets.size - 1, batch_docs):
stop = min(start + batch_docs, offsets.size - 1)
texts = []
for i in range(start, stop):
a, b = int(offsets[i]), int(offsets[i + 1])
ids = tokens[a:b]
if ids.size and int(ids[-1]) == old.eos_id:
ids = ids[:-1]
texts.append(old.decode(ids))
encoded = new.encode_batch(texts)
chunks, ends = [], []
for text, ids in zip(texts, encoded):
# Both tokenizers are byte-level; this makes corruption a hard
# failure before the source shard is ever considered for swap.
if new.decode(ids) != text:
raise RuntimeError(f"round-trip failed at document {start + len(ends)}")
chunk = np.asarray(ids + [new.eos_id], dtype=np.uint16)
chunks.append(chunk)
written += chunk.size
ends.append(written)
if chunks:
np.concatenate(chunks).tofile(ft)
np.asarray(ends, dtype=np.uint64).tofile(fi)
if start == 0 or (start // batch_docs) % 100 == 0:
rate = written / max(time.time() - t0, 1e-9)
print(f" {stop:,}/{offsets.size-1:,} docs {written/1e9:.3f}B tokens "
f"{rate/1e6:.2f}M tok/s", flush=True)
ft.flush(); os.fsync(ft.fileno())
fi.flush(); os.fsync(fi.fileno())
check_idx = np.memmap(out_idx, dtype=np.uint64, mode="r")
expected_bytes = int(check_idx[-1]) * np.dtype(np.uint16).itemsize
if check_idx.size != offsets.size or out_tokens.stat().st_size != expected_bytes:
raise RuntimeError("converted shard failed final size/index validation")
manifest = {
"source": str(source.resolve()),
"documents": int(offsets.size - 1),
"old_tokens": int(tokens.size),
"new_tokens": int(check_idx[-1]),
"old_tokenizer_sha256": BorisTokenizer.fingerprint(old_dir),
"new_tokenizer_sha256": BorisTokenizer.fingerprint(new_dir),
}
(out / "retokenize.json").write_text(json.dumps(manifest, indent=2) + "\n")
print(json.dumps(manifest, indent=2))
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("source", type=Path)
ap.add_argument("old_tokenizer", type=Path)
ap.add_argument("new_tokenizer", type=Path)
ap.add_argument("--out", type=Path, required=True)
ap.add_argument("--batch-docs", type=int, default=2048)
a = ap.parse_args()
convert(a.source, a.old_tokenizer, a.new_tokenizer, a.out, a.batch_docs)
if __name__ == "__main__":
main()
|