"""Our own byte-level BPE, trained on this corpus. 32,768 tokens. A power of two needs no padding for the tensor cores, stores in uint16, and costs 16.8M tied embedding parameters against 25.8M for a 50k vocabulary -- at a 143.5M budget that difference is most of a layer's worth of FFN. Byte-level, so every input round-trips and there is no unknown token. python tokenizer.py train --sample data/tokenizer_sample.txt python tokenizer.py inspect """ from __future__ import annotations import argparse import hashlib import json from pathlib import Path EOS = "<|endoftext|>" class BorisTokenizer: def __init__(self, tk, eos_id: int): self.tk = tk self.eos_id = eos_id @property def vocab_size(self) -> int: return self.tk.get_vocab_size() def encode(self, text: str) -> list[int]: return self.tk.encode(text, add_special_tokens=False).ids def encode_with_offsets(self, text: str) -> tuple[list[int], list[tuple[int, int]]]: """Token ids and offsets into the original string. Evaluation needs the offsets because BPE is not prefix-stable: the last context token can merge with the first continuation characters. Splitting ``encode(context + choice)`` at ``len(encode(context))`` can therefore score the wrong tokens. """ e = self.tk.encode(text, add_special_tokens=False) return e.ids, e.offsets def encode_batch(self, texts: list[str]) -> list[list[int]]: return [e.ids for e in self.tk.encode_batch(texts, add_special_tokens=False)] def decode(self, ids) -> str: return self.tk.decode(list(ids), skip_special_tokens=False) def save(self, d: Path): d = Path(d); d.mkdir(parents=True, exist_ok=True) self.tk.save(str(d / "tokenizer.json")) (d / "meta.json").write_text(json.dumps( {"eos_id": self.eos_id, "vocab_size": self.vocab_size}, indent=2)) @staticmethod def fingerprint(d: Path) -> str: """Full tokenizer identity used by checkpoint/config compatibility.""" h = hashlib.sha256() with open(Path(d) / "tokenizer.json", "rb") as f: for chunk in iter(lambda: f.read(1 << 20), b""): h.update(chunk) return h.hexdigest() @staticmethod def load(d: Path) -> "BorisTokenizer": from tokenizers import Tokenizer d = Path(d) import os if os.environ.get("BORIS_DIGIT_SPLIT") == "1" and (d.parent / "tokenizer_digits").exists(): tk = Tokenizer.from_file(str(d.parent / "tokenizer_digits" / "tokenizer.json")) else: tk = Tokenizer.from_file(str(d / "tokenizer.json")) meta = json.loads((d / "meta.json").read_text()) return BorisTokenizer(tk, meta["eos_id"]) @staticmethod def train(files: list[Path], vocab_size: int, out: Path) -> "BorisTokenizer": from tokenizers import Tokenizer, decoders, models, pre_tokenizers, trainers tk = Tokenizer(models.BPE(unk_token=None)) # ByteLevel with no initial-space prefix: documents are packed rather # than sentence-fed, so a leading space would be wrong at every # document start. tk.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False) tk.decoder = decoders.ByteLevel() trainer = trainers.BpeTrainer( vocab_size=vocab_size, special_tokens=[EOS], show_progress=True, initial_alphabet=pre_tokenizers.ByteLevel.alphabet()) tk.train([str(f) for f in files], trainer) t = BorisTokenizer(tk, tk.token_to_id(EOS)) assert t.eos_id == 0, f"{EOS} must be id 0, got {t.eos_id}" assert t.vocab_size == vocab_size, ( f"trained {t.vocab_size} tokens, wanted {vocab_size}: the sample is " f"probably too small to reach the target vocabulary") t.save(out) return t if __name__ == "__main__": from config import DATA_DIR, Config ap = argparse.ArgumentParser() ap.add_argument("cmd", choices=["train", "inspect"]) ap.add_argument("--sample", type=Path, nargs="+", default=[DATA_DIR / "tokenizer_sample.txt"]) ap.add_argument("--out", type=Path, default=DATA_DIR / "tokenizer") a = ap.parse_args() cfg = Config() if a.cmd == "train": t = BorisTokenizer.train(a.sample, cfg.model.vocab_size, a.out) print(f" trained {t.vocab_size} tokens -> {a.out}") else: t = BorisTokenizer.load(a.out) s = "The quick brown fox jumps over the lazy dog. 3 + 19 = 22." ids = t.encode(s) print(f" vocab {t.vocab_size} eos {t.eos_id}") print(f" {len(ids)} tokens for {len(s)} chars ({len(s)/len(ids):.2f} ch/tok)") assert t.decode(ids) == s, "byte-level BPE must round-trip exactly" print(" round-trip ok")