summaryrefslogtreecommitdiff
path: root/tokenizer.py
blob: 74ea73258631a2ef082c2c80346acc471688d818 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
"""Our own byte-level BPE, trained on this corpus.

32,768 tokens. A power of two needs no padding for the tensor cores, stores in
uint16, and costs 16.8M tied embedding parameters against 25.8M for a 50k
vocabulary -- at a 143.5M budget that difference is most of a layer's worth of
FFN. Byte-level, so every input round-trips and there is no unknown token.

    python tokenizer.py train --sample data/tokenizer_sample.txt
    python tokenizer.py inspect
"""
from __future__ import annotations

import argparse
import hashlib
import json
from pathlib import Path

EOS = "<|endoftext|>"


class BorisTokenizer:
    def __init__(self, tk, eos_id: int):
        self.tk = tk
        self.eos_id = eos_id

    @property
    def vocab_size(self) -> int:
        return self.tk.get_vocab_size()

    def encode(self, text: str) -> list[int]:
        return self.tk.encode(text, add_special_tokens=False).ids

    def encode_with_offsets(self, text: str) -> tuple[list[int], list[tuple[int, int]]]:
        """Token ids and offsets into the original string.

        Evaluation needs the offsets because BPE is not prefix-stable: the last
        context token can merge with the first continuation characters. Splitting
        ``encode(context + choice)`` at ``len(encode(context))`` can therefore
        score the wrong tokens.
        """
        e = self.tk.encode(text, add_special_tokens=False)
        return e.ids, e.offsets

    def encode_batch(self, texts: list[str]) -> list[list[int]]:
        return [e.ids for e in self.tk.encode_batch(texts, add_special_tokens=False)]

    def decode(self, ids) -> str:
        return self.tk.decode(list(ids), skip_special_tokens=False)

    def save(self, d: Path):
        d = Path(d); d.mkdir(parents=True, exist_ok=True)
        self.tk.save(str(d / "tokenizer.json"))
        (d / "meta.json").write_text(json.dumps(
            {"eos_id": self.eos_id, "vocab_size": self.vocab_size}, indent=2))

    @staticmethod
    def fingerprint(d: Path) -> str:
        """Full tokenizer identity used by checkpoint/config compatibility."""
        h = hashlib.sha256()
        with open(Path(d) / "tokenizer.json", "rb") as f:
            for chunk in iter(lambda: f.read(1 << 20), b""):
                h.update(chunk)
        return h.hexdigest()

    @staticmethod
    def load(d: Path) -> "BorisTokenizer":
        from tokenizers import Tokenizer
        d = Path(d)
        import os
        if os.environ.get("BORIS_DIGIT_SPLIT") == "1" and (d.parent / "tokenizer_digits").exists():
            tk = Tokenizer.from_file(str(d.parent / "tokenizer_digits" / "tokenizer.json"))
        else:
            tk = Tokenizer.from_file(str(d / "tokenizer.json"))
        meta = json.loads((d / "meta.json").read_text())
        return BorisTokenizer(tk, meta["eos_id"])

    @staticmethod
    def train(files: list[Path], vocab_size: int, out: Path) -> "BorisTokenizer":
        from tokenizers import Tokenizer, decoders, models, pre_tokenizers, trainers
        tk = Tokenizer(models.BPE(unk_token=None))
        # ByteLevel with no initial-space prefix: documents are packed rather
        # than sentence-fed, so a leading space would be wrong at every
        # document start.
        tk.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
        tk.decoder = decoders.ByteLevel()
        trainer = trainers.BpeTrainer(
            vocab_size=vocab_size, special_tokens=[EOS], show_progress=True,
            initial_alphabet=pre_tokenizers.ByteLevel.alphabet())
        tk.train([str(f) for f in files], trainer)
        t = BorisTokenizer(tk, tk.token_to_id(EOS))
        assert t.eos_id == 0, f"{EOS} must be id 0, got {t.eos_id}"
        assert t.vocab_size == vocab_size, (
            f"trained {t.vocab_size} tokens, wanted {vocab_size}: the sample is "
            f"probably too small to reach the target vocabulary")
        t.save(out)
        return t


if __name__ == "__main__":
    from config import DATA_DIR, Config
    ap = argparse.ArgumentParser()
    ap.add_argument("cmd", choices=["train", "inspect"])
    ap.add_argument("--sample", type=Path, nargs="+",
                    default=[DATA_DIR / "tokenizer_sample.txt"])
    ap.add_argument("--out", type=Path, default=DATA_DIR / "tokenizer")
    a = ap.parse_args()
    cfg = Config()
    if a.cmd == "train":
        t = BorisTokenizer.train(a.sample, cfg.model.vocab_size, a.out)
        print(f"  trained {t.vocab_size} tokens -> {a.out}")
    else:
        t = BorisTokenizer.load(a.out)
        s = "The quick brown fox jumps over the lazy dog. 3 + 19 = 22."
        ids = t.encode(s)
        print(f"  vocab {t.vocab_size}  eos {t.eos_id}")
        print(f"  {len(ids)} tokens for {len(s)} chars ({len(s)/len(ids):.2f} ch/tok)")
        assert t.decode(ids) == s, "byte-level BPE must round-trip exactly"
        print("  round-trip ok")