1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
|
"""Our own byte-level BPE, trained on this corpus.
32,768 tokens. A power of two needs no padding for the tensor cores, stores in
uint16, and costs 16.8M tied embedding parameters against 25.8M for a 50k
vocabulary -- at a 143.5M budget that difference is most of a layer's worth of
FFN. Byte-level, so every input round-trips and there is no unknown token.
python tokenizer.py train --sample data/tokenizer_sample.txt
python tokenizer.py inspect
"""
from __future__ import annotations
import argparse
import hashlib
import json
from pathlib import Path
EOS = "<|endoftext|>"
class BorisTokenizer:
def __init__(self, tk, eos_id: int):
self.tk = tk
self.eos_id = eos_id
@property
def vocab_size(self) -> int:
return self.tk.get_vocab_size()
def encode(self, text: str) -> list[int]:
return self.tk.encode(text, add_special_tokens=False).ids
def encode_with_offsets(self, text: str) -> tuple[list[int], list[tuple[int, int]]]:
"""Token ids and offsets into the original string.
Evaluation needs the offsets because BPE is not prefix-stable: the last
context token can merge with the first continuation characters. Splitting
``encode(context + choice)`` at ``len(encode(context))`` can therefore
score the wrong tokens.
"""
e = self.tk.encode(text, add_special_tokens=False)
return e.ids, e.offsets
def encode_batch(self, texts: list[str]) -> list[list[int]]:
return [e.ids for e in self.tk.encode_batch(texts, add_special_tokens=False)]
def decode(self, ids) -> str:
return self.tk.decode(list(ids), skip_special_tokens=False)
def save(self, d: Path):
d = Path(d); d.mkdir(parents=True, exist_ok=True)
self.tk.save(str(d / "tokenizer.json"))
(d / "meta.json").write_text(json.dumps(
{"eos_id": self.eos_id, "vocab_size": self.vocab_size}, indent=2))
@staticmethod
def fingerprint(d: Path) -> str:
"""Full tokenizer identity used by checkpoint/config compatibility."""
h = hashlib.sha256()
with open(Path(d) / "tokenizer.json", "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
h.update(chunk)
return h.hexdigest()
@staticmethod
def load(d: Path) -> "BorisTokenizer":
from tokenizers import Tokenizer
d = Path(d)
import os
if os.environ.get("BORIS_DIGIT_SPLIT") == "1" and (d.parent / "tokenizer_digits").exists():
tk = Tokenizer.from_file(str(d.parent / "tokenizer_digits" / "tokenizer.json"))
else:
tk = Tokenizer.from_file(str(d / "tokenizer.json"))
meta = json.loads((d / "meta.json").read_text())
return BorisTokenizer(tk, meta["eos_id"])
@staticmethod
def train(files: list[Path], vocab_size: int, out: Path) -> "BorisTokenizer":
from tokenizers import Tokenizer, decoders, models, pre_tokenizers, trainers
tk = Tokenizer(models.BPE(unk_token=None))
# ByteLevel with no initial-space prefix: documents are packed rather
# than sentence-fed, so a leading space would be wrong at every
# document start.
tk.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
tk.decoder = decoders.ByteLevel()
trainer = trainers.BpeTrainer(
vocab_size=vocab_size, special_tokens=[EOS], show_progress=True,
initial_alphabet=pre_tokenizers.ByteLevel.alphabet())
tk.train([str(f) for f in files], trainer)
t = BorisTokenizer(tk, tk.token_to_id(EOS))
assert t.eos_id == 0, f"{EOS} must be id 0, got {t.eos_id}"
assert t.vocab_size == vocab_size, (
f"trained {t.vocab_size} tokens, wanted {vocab_size}: the sample is "
f"probably too small to reach the target vocabulary")
t.save(out)
return t
if __name__ == "__main__":
from config import DATA_DIR, Config
ap = argparse.ArgumentParser()
ap.add_argument("cmd", choices=["train", "inspect"])
ap.add_argument("--sample", type=Path, nargs="+",
default=[DATA_DIR / "tokenizer_sample.txt"])
ap.add_argument("--out", type=Path, default=DATA_DIR / "tokenizer")
a = ap.parse_args()
cfg = Config()
if a.cmd == "train":
t = BorisTokenizer.train(a.sample, cfg.model.vocab_size, a.out)
print(f" trained {t.vocab_size} tokens -> {a.out}")
else:
t = BorisTokenizer.load(a.out)
s = "The quick brown fox jumps over the lazy dog. 3 + 19 = 22."
ids = t.encode(s)
print(f" vocab {t.vocab_size} eos {t.eos_id}")
print(f" {len(ids)} tokens for {len(s)} chars ({len(s)/len(ids):.2f} ch/tok)")
assert t.decode(ids) == s, "byte-level BPE must round-trip exactly"
print(" round-trip ok")
|