summaryrefslogtreecommitdiff
path: root/retokenize.py
blob: 6e9b88d634c97d8a91d3fb833c16e6582f58eae4 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
"""Transactionally convert prepared uint16 shards to a different Boris BPE.

The source format preserves document boundaries, so decoding each old document
and encoding it with the new byte-level BPE is lossless. Conversion writes a
sibling directory and never mutates the source shard. A separate, explicit
swap step is required after training has stopped and validation has passed.

Example:
    python retokenize.py data/climbmix data/tokenizer data/tokenizer-0920 \
        --out data/climbmix.retokenized-0920
"""
from __future__ import annotations

import argparse
import json
import os
import time
from pathlib import Path

import numpy as np

from tokenizer import BorisTokenizer


def convert(source: Path, old_dir: Path, new_dir: Path, out: Path,
            batch_docs: int = 2048) -> None:
    source, out = Path(source), Path(out)
    if source.resolve() == out.resolve():
        raise SystemExit("output must differ from source; in-place conversion is unsafe")
    out.mkdir(parents=True, exist_ok=True)
    out_tokens, out_idx = out / "tokens.bin", out / "docs.idx"
    if out_tokens.exists() or out_idx.exists():
        raise SystemExit(f"{out}: output files already exist; refusing to append or overwrite")

    old, new = BorisTokenizer.load(old_dir), BorisTokenizer.load(new_dir)
    tokens = np.memmap(source / "tokens.bin", dtype=np.uint16, mode="r")
    offsets = np.memmap(source / "docs.idx", dtype=np.uint64, mode="r")
    if offsets.size < 2 or int(offsets[0]) != 0 or int(offsets[-1]) != tokens.size:
        raise SystemExit(f"{source}: inconsistent token/index files")

    t0 = time.time()
    written = 0
    with open(out_tokens, "wb") as ft, open(out_idx, "wb") as fi:
        np.asarray([0], dtype=np.uint64).tofile(fi)
        for start in range(0, offsets.size - 1, batch_docs):
            stop = min(start + batch_docs, offsets.size - 1)
            texts = []
            for i in range(start, stop):
                a, b = int(offsets[i]), int(offsets[i + 1])
                ids = tokens[a:b]
                if ids.size and int(ids[-1]) == old.eos_id:
                    ids = ids[:-1]
                texts.append(old.decode(ids))
            encoded = new.encode_batch(texts)
            chunks, ends = [], []
            for text, ids in zip(texts, encoded):
                # Both tokenizers are byte-level; this makes corruption a hard
                # failure before the source shard is ever considered for swap.
                if new.decode(ids) != text:
                    raise RuntimeError(f"round-trip failed at document {start + len(ends)}")
                chunk = np.asarray(ids + [new.eos_id], dtype=np.uint16)
                chunks.append(chunk)
                written += chunk.size
                ends.append(written)
            if chunks:
                np.concatenate(chunks).tofile(ft)
                np.asarray(ends, dtype=np.uint64).tofile(fi)
            if start == 0 or (start // batch_docs) % 100 == 0:
                rate = written / max(time.time() - t0, 1e-9)
                print(f"  {stop:,}/{offsets.size-1:,} docs  {written/1e9:.3f}B tokens  "
                      f"{rate/1e6:.2f}M tok/s", flush=True)
        ft.flush(); os.fsync(ft.fileno())
        fi.flush(); os.fsync(fi.fileno())

    check_idx = np.memmap(out_idx, dtype=np.uint64, mode="r")
    expected_bytes = int(check_idx[-1]) * np.dtype(np.uint16).itemsize
    if check_idx.size != offsets.size or out_tokens.stat().st_size != expected_bytes:
        raise RuntimeError("converted shard failed final size/index validation")
    manifest = {
        "source": str(source.resolve()),
        "documents": int(offsets.size - 1),
        "old_tokens": int(tokens.size),
        "new_tokens": int(check_idx[-1]),
        "old_tokenizer_sha256": BorisTokenizer.fingerprint(old_dir),
        "new_tokenizer_sha256": BorisTokenizer.fingerprint(new_dir),
    }
    (out / "retokenize.json").write_text(json.dumps(manifest, indent=2) + "\n")
    print(json.dumps(manifest, indent=2))


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("source", type=Path)
    ap.add_argument("old_tokenizer", type=Path)
    ap.add_argument("new_tokenizer", type=Path)
    ap.add_argument("--out", type=Path, required=True)
    ap.add_argument("--batch-docs", type=int, default=2048)
    a = ap.parse_args()
    convert(a.source, a.old_tokenizer, a.new_tokenizer, a.out, a.batch_docs)


if __name__ == "__main__":
    main()