summaryrefslogtreecommitdiff
path: root/tokenizer.py
diff options
context:
space:
mode:
authorjdj <jdj@kunix.org>2026-11-18 14:47:33 -0800
committerjdj <jdj@kunix.org>2026-11-18 14:47:33 -0800
commit5d2e4252e695644c34be605b44a14bd80a98a0a6 (patch)
tree311a72eda4459af7547b9cd05c0956eafb235f2d /tokenizer.py
downloadboris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.tar.gz
boris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.zip
Boris-2 training code
The code, configuration and data filters used to train Boris-2 (144M, 200B tokens, single V100): model, training loop, data preparation, n-gram table, V100 kernels, final stages and the exact run configs.
Diffstat (limited to 'tokenizer.py')
-rw-r--r--tokenizer.py118
1 files changed, 118 insertions, 0 deletions
diff --git a/tokenizer.py b/tokenizer.py
new file mode 100644
index 0000000..74ea732
--- /dev/null
+++ b/tokenizer.py
@@ -0,0 +1,118 @@
+"""Our own byte-level BPE, trained on this corpus.
+
+32,768 tokens. A power of two needs no padding for the tensor cores, stores in
+uint16, and costs 16.8M tied embedding parameters against 25.8M for a 50k
+vocabulary -- at a 143.5M budget that difference is most of a layer's worth of
+FFN. Byte-level, so every input round-trips and there is no unknown token.
+
+ python tokenizer.py train --sample data/tokenizer_sample.txt
+ python tokenizer.py inspect
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+from pathlib import Path
+
+EOS = "<|endoftext|>"
+
+
+class BorisTokenizer:
+ def __init__(self, tk, eos_id: int):
+ self.tk = tk
+ self.eos_id = eos_id
+
+ @property
+ def vocab_size(self) -> int:
+ return self.tk.get_vocab_size()
+
+ def encode(self, text: str) -> list[int]:
+ return self.tk.encode(text, add_special_tokens=False).ids
+
+ def encode_with_offsets(self, text: str) -> tuple[list[int], list[tuple[int, int]]]:
+ """Token ids and offsets into the original string.
+
+ Evaluation needs the offsets because BPE is not prefix-stable: the last
+ context token can merge with the first continuation characters. Splitting
+ ``encode(context + choice)`` at ``len(encode(context))`` can therefore
+ score the wrong tokens.
+ """
+ e = self.tk.encode(text, add_special_tokens=False)
+ return e.ids, e.offsets
+
+ def encode_batch(self, texts: list[str]) -> list[list[int]]:
+ return [e.ids for e in self.tk.encode_batch(texts, add_special_tokens=False)]
+
+ def decode(self, ids) -> str:
+ return self.tk.decode(list(ids), skip_special_tokens=False)
+
+ def save(self, d: Path):
+ d = Path(d); d.mkdir(parents=True, exist_ok=True)
+ self.tk.save(str(d / "tokenizer.json"))
+ (d / "meta.json").write_text(json.dumps(
+ {"eos_id": self.eos_id, "vocab_size": self.vocab_size}, indent=2))
+
+ @staticmethod
+ def fingerprint(d: Path) -> str:
+ """Full tokenizer identity used by checkpoint/config compatibility."""
+ h = hashlib.sha256()
+ with open(Path(d) / "tokenizer.json", "rb") as f:
+ for chunk in iter(lambda: f.read(1 << 20), b""):
+ h.update(chunk)
+ return h.hexdigest()
+
+ @staticmethod
+ def load(d: Path) -> "BorisTokenizer":
+ from tokenizers import Tokenizer
+ d = Path(d)
+ import os
+ if os.environ.get("BORIS_DIGIT_SPLIT") == "1" and (d.parent / "tokenizer_digits").exists():
+ tk = Tokenizer.from_file(str(d.parent / "tokenizer_digits" / "tokenizer.json"))
+ else:
+ tk = Tokenizer.from_file(str(d / "tokenizer.json"))
+ meta = json.loads((d / "meta.json").read_text())
+ return BorisTokenizer(tk, meta["eos_id"])
+
+ @staticmethod
+ def train(files: list[Path], vocab_size: int, out: Path) -> "BorisTokenizer":
+ from tokenizers import Tokenizer, decoders, models, pre_tokenizers, trainers
+ tk = Tokenizer(models.BPE(unk_token=None))
+ # ByteLevel with no initial-space prefix: documents are packed rather
+ # than sentence-fed, so a leading space would be wrong at every
+ # document start.
+ tk.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
+ tk.decoder = decoders.ByteLevel()
+ trainer = trainers.BpeTrainer(
+ vocab_size=vocab_size, special_tokens=[EOS], show_progress=True,
+ initial_alphabet=pre_tokenizers.ByteLevel.alphabet())
+ tk.train([str(f) for f in files], trainer)
+ t = BorisTokenizer(tk, tk.token_to_id(EOS))
+ assert t.eos_id == 0, f"{EOS} must be id 0, got {t.eos_id}"
+ assert t.vocab_size == vocab_size, (
+ f"trained {t.vocab_size} tokens, wanted {vocab_size}: the sample is "
+ f"probably too small to reach the target vocabulary")
+ t.save(out)
+ return t
+
+
+if __name__ == "__main__":
+ from config import DATA_DIR, Config
+ ap = argparse.ArgumentParser()
+ ap.add_argument("cmd", choices=["train", "inspect"])
+ ap.add_argument("--sample", type=Path, nargs="+",
+ default=[DATA_DIR / "tokenizer_sample.txt"])
+ ap.add_argument("--out", type=Path, default=DATA_DIR / "tokenizer")
+ a = ap.parse_args()
+ cfg = Config()
+ if a.cmd == "train":
+ t = BorisTokenizer.train(a.sample, cfg.model.vocab_size, a.out)
+ print(f" trained {t.vocab_size} tokens -> {a.out}")
+ else:
+ t = BorisTokenizer.load(a.out)
+ s = "The quick brown fox jumps over the lazy dog. 3 + 19 = 22."
+ ids = t.encode(s)
+ print(f" vocab {t.vocab_size} eos {t.eos_id}")
+ print(f" {len(ids)} tokens for {len(s)} chars ({len(s)/len(ids):.2f} ch/tok)")
+ assert t.decode(ids) == s, "byte-level BPE must round-trip exactly"
+ print(" round-trip ok")