From 5d2e4252e695644c34be605b44a14bd80a98a0a6 Mon Sep 17 00:00:00 2001 From: jdj Date: Wed, 18 Nov 2026 14:47:33 -0800 Subject: Boris-2 training code The code, configuration and data filters used to train Boris-2 (144M, 200B tokens, single V100): model, training loop, data preparation, n-gram table, V100 kernels, final stages and the exact run configs. --- tokenizer.py | 118 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 tokenizer.py (limited to 'tokenizer.py') diff --git a/tokenizer.py b/tokenizer.py new file mode 100644 index 0000000..74ea732 --- /dev/null +++ b/tokenizer.py @@ -0,0 +1,118 @@ +"""Our own byte-level BPE, trained on this corpus. + +32,768 tokens. A power of two needs no padding for the tensor cores, stores in +uint16, and costs 16.8M tied embedding parameters against 25.8M for a 50k +vocabulary -- at a 143.5M budget that difference is most of a layer's worth of +FFN. Byte-level, so every input round-trips and there is no unknown token. + + python tokenizer.py train --sample data/tokenizer_sample.txt + python tokenizer.py inspect +""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +EOS = "<|endoftext|>" + + +class BorisTokenizer: + def __init__(self, tk, eos_id: int): + self.tk = tk + self.eos_id = eos_id + + @property + def vocab_size(self) -> int: + return self.tk.get_vocab_size() + + def encode(self, text: str) -> list[int]: + return self.tk.encode(text, add_special_tokens=False).ids + + def encode_with_offsets(self, text: str) -> tuple[list[int], list[tuple[int, int]]]: + """Token ids and offsets into the original string. + + Evaluation needs the offsets because BPE is not prefix-stable: the last + context token can merge with the first continuation characters. Splitting + ``encode(context + choice)`` at ``len(encode(context))`` can therefore + score the wrong tokens. + """ + e = self.tk.encode(text, add_special_tokens=False) + return e.ids, e.offsets + + def encode_batch(self, texts: list[str]) -> list[list[int]]: + return [e.ids for e in self.tk.encode_batch(texts, add_special_tokens=False)] + + def decode(self, ids) -> str: + return self.tk.decode(list(ids), skip_special_tokens=False) + + def save(self, d: Path): + d = Path(d); d.mkdir(parents=True, exist_ok=True) + self.tk.save(str(d / "tokenizer.json")) + (d / "meta.json").write_text(json.dumps( + {"eos_id": self.eos_id, "vocab_size": self.vocab_size}, indent=2)) + + @staticmethod + def fingerprint(d: Path) -> str: + """Full tokenizer identity used by checkpoint/config compatibility.""" + h = hashlib.sha256() + with open(Path(d) / "tokenizer.json", "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + @staticmethod + def load(d: Path) -> "BorisTokenizer": + from tokenizers import Tokenizer + d = Path(d) + import os + if os.environ.get("BORIS_DIGIT_SPLIT") == "1" and (d.parent / "tokenizer_digits").exists(): + tk = Tokenizer.from_file(str(d.parent / "tokenizer_digits" / "tokenizer.json")) + else: + tk = Tokenizer.from_file(str(d / "tokenizer.json")) + meta = json.loads((d / "meta.json").read_text()) + return BorisTokenizer(tk, meta["eos_id"]) + + @staticmethod + def train(files: list[Path], vocab_size: int, out: Path) -> "BorisTokenizer": + from tokenizers import Tokenizer, decoders, models, pre_tokenizers, trainers + tk = Tokenizer(models.BPE(unk_token=None)) + # ByteLevel with no initial-space prefix: documents are packed rather + # than sentence-fed, so a leading space would be wrong at every + # document start. + tk.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False) + tk.decoder = decoders.ByteLevel() + trainer = trainers.BpeTrainer( + vocab_size=vocab_size, special_tokens=[EOS], show_progress=True, + initial_alphabet=pre_tokenizers.ByteLevel.alphabet()) + tk.train([str(f) for f in files], trainer) + t = BorisTokenizer(tk, tk.token_to_id(EOS)) + assert t.eos_id == 0, f"{EOS} must be id 0, got {t.eos_id}" + assert t.vocab_size == vocab_size, ( + f"trained {t.vocab_size} tokens, wanted {vocab_size}: the sample is " + f"probably too small to reach the target vocabulary") + t.save(out) + return t + + +if __name__ == "__main__": + from config import DATA_DIR, Config + ap = argparse.ArgumentParser() + ap.add_argument("cmd", choices=["train", "inspect"]) + ap.add_argument("--sample", type=Path, nargs="+", + default=[DATA_DIR / "tokenizer_sample.txt"]) + ap.add_argument("--out", type=Path, default=DATA_DIR / "tokenizer") + a = ap.parse_args() + cfg = Config() + if a.cmd == "train": + t = BorisTokenizer.train(a.sample, cfg.model.vocab_size, a.out) + print(f" trained {t.vocab_size} tokens -> {a.out}") + else: + t = BorisTokenizer.load(a.out) + s = "The quick brown fox jumps over the lazy dog. 3 + 19 = 22." + ids = t.encode(s) + print(f" vocab {t.vocab_size} eos {t.eos_id}") + print(f" {len(ids)} tokens for {len(s)} chars ({len(s)/len(ids):.2f} ch/tok)") + assert t.decode(ids) == s, "byte-level BPE must round-trip exactly" + print(" round-trip ok") -- cgit v1.2.3