From 5d2e4252e695644c34be605b44a14bd80a98a0a6 Mon Sep 17 00:00:00 2001 From: jdj Date: Wed, 18 Nov 2026 14:47:33 -0800 Subject: Boris-2 training code The code, configuration and data filters used to train Boris-2 (144M, 200B tokens, single V100): model, training loop, data preparation, n-gram table, V100 kernels, final stages and the exact run configs. --- data.py | 488 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 488 insertions(+) create mode 100644 data.py (limited to 'data.py') diff --git a/data.py b/data.py new file mode 100644 index 0000000..0d71175 --- /dev/null +++ b/data.py @@ -0,0 +1,488 @@ +"""Varlen-packed data pipeline. + +Each source is a memory-mapped uint16 token file plus a uint64 index of document +start offsets. Batches are packed, not EOS-concatenated: documents are laid end +to end into rows of `block_size`, and the packer emits the document boundaries +directly. It knows them for free while laying documents down, so nothing later +has to re-derive them with a GPU scan and a host sync on every forward. + +Four tensors come out of the packer besides the tokens, and every one of them +exists to stop a document seeing another document: + + offsets document boundaries into the flattened (B*T) stream. Feeds the + jagged nested tensor, so attention is per-document. + boundary (B, T) bool, True where a document starts. Masks the Canon taps + and the n-gram history. + pos (B, T) int32, position within the document. RoPE restarts at each + document rather than running on for the whole row. + targets next token within the document; the last position of a document + predicts nothing and is IGNORE. + +Everything stochastic runs on a torch.Generator whose state is checkpointed, +together with the partially consumed document carried between batches, so a +resumed run replays the exact same token stream. +""" +from __future__ import annotations + +import queue +import threading +from pathlib import Path +import json + +import numpy as np +import torch + +from config import Config +from schedule import long_doc_pressure, mixture_at + +IGNORE = -100 + + +_DIGITS = None + + +def _digit_split(doc): + """BORIS_DIGIT_SPLIT=1 (pilot only): rewrite every number token as single digits, + e.g. "Ġ999" -> "Ġ" "9" "9" "9", exactly as data/tokenizer_digits/tokenizer.json encodes + text (verified on 4,000 ClimbMix documents: 3,997 identical). Off by default.""" + global _DIGITS + import os + if os.environ.get("BORIS_DIGIT_SPLIT") != "1": + return doc + if _DIGITS is None: + z = np.load(Path(__file__).resolve().parent / "data/tokenizer_digits/expand.npz") + _DIGITS = (z["flat"], z["starts"], z["lens"]) + flat, starts, lens = _DIGITS + L = lens[doc] + if (L == 1).all(): + return doc + idx = np.repeat(starts[doc], L) + (np.arange(int(L.sum())) - np.repeat(np.cumsum(L) - L, L)) + return flat[idx] + + +def validate_tokenization(directory, cfg): + """Fail closed if a prepared 0920 shard uses different token IDs.""" + from config import LEGACY_TOKENIZER_SHA256 + if cfg.model.tokenizer_sha256 == LEGACY_TOKENIZER_SHA256: + return + path = Path(directory) / "retokenize_0920.json" + if not path.exists(): + raise ValueError(f"missing tokenizer provenance: {path}") + got = json.loads(path.read_text()).get("new_tokenizer_sha256") + if got != cfg.model.tokenizer_sha256: + raise ValueError(f"tokenizer mismatch in {directory}: {got}") + + +def open_validation(cfg): + directory = Path(cfg.data.root) / cfg.data.val_source + validate_tokenization(directory, cfg) + return SourceStore.open(cfg.data.root, cfg.data.val_source) + + +class SourceStore: + """One tokenized source: tokens.bin (uint16) + docs.idx (uint64).""" + + def __init__(self, name: str, bin_path: Path, idx_path: Path): + self.name = name + self.tokens = np.memmap(bin_path, dtype=np.uint16, mode="r") + # Keep the multi-GB document index file mapped instead of copying it + # into anonymous RAM. Random lookup only needs two adjacent entries. + self.offsets = np.memmap(idx_path, dtype=np.uint64, mode="r") + if self.offsets.size < 2: + raise ValueError(f"{idx_path}: need at least two offsets") + self.n_docs = self.offsets.size - 1 + raw_lengths = np.diff(self.offsets) + if raw_lengths.size and raw_lengths.max() > np.iinfo(np.uint32).max: + raise ValueError(f"{idx_path}: a document exceeds uint32 token length") + self.lengths = raw_lengths.astype(np.uint32) + self._long_cache: dict[int, np.ndarray] = {} + + @property + def n_tokens(self) -> int: + return int(self.offsets[-1]) + + def doc(self, i: int) -> np.ndarray: + a, b = int(self.offsets[i]), int(self.offsets[i + 1]) + return self.tokens[a:b] + + def long_docs(self, floor: int) -> np.ndarray: + """Indices of documents at least `floor` tokens long, computed once. + + Built from the length array, which is already in memory -- no pass over + the token file, which for a 200GB source would be its own job. + """ + if floor not in self._long_cache: + self._long_cache[floor] = np.flatnonzero(self.lengths >= floor) + return self._long_cache[floor] + + @staticmethod + def open(root: Path, name: str) -> "SourceStore": + d = Path(root) / name + return SourceStore(name, d / "tokens.bin", d / "docs.idx") + + +class MultiStore: + """Several shard directories presented as one logical source. + + A source too large for one prepare process is split across workers by data + file range, each writing its own directory. Stitching them here means the + shards are never concatenated on disk -- copying 200GB to merge them would + cost as much as downloading them and need the free space twice over. + + Document indices are global: doc i belongs to whichever shard's cumulative + range contains it. + """ + + def __init__(self, name: str, stores: list[SourceStore]): + self.name = name + self.stores = stores + counts = np.array([s.n_docs for s in stores], dtype=np.int64) + self._bounds = np.concatenate([[0], np.cumsum(counts)]) + self.n_docs = int(self._bounds[-1]) + self.lengths = np.concatenate([s.lengths for s in stores]) + # MultiStore owns the combined length index; retaining every per-shard + # copy doubled resident memory (about 2.5 GB for ClimbMix). + for s in stores: + s.lengths = np.empty(0, dtype=np.uint32) + self._long_cache: dict[int, np.ndarray] = {} + + @property + def n_tokens(self) -> int: + return sum(s.n_tokens for s in self.stores) + + def doc(self, i: int) -> np.ndarray: + j = int(np.searchsorted(self._bounds, i, side="right")) - 1 + return self.stores[j].doc(i - int(self._bounds[j])) + + def long_docs(self, floor: int) -> np.ndarray: + if floor not in self._long_cache: + self._long_cache[floor] = np.flatnonzero(self.lengths >= floor) + return self._long_cache[floor] + + +class VarlenPacker: + """Draws documents per the curriculum and packs them into fixed-width rows.""" + + def __init__(self, cfg: Config, stores: dict[str, SourceStore], seed: int, + split: str = "train"): + self.cfg = cfg + self.stores = stores + self.split = split + self.gen = torch.Generator().manual_seed(seed) + self._carry: np.ndarray | None = None + self._carry_name: str | None = None + self._carry_pos = 0 # position within the carried document + self.docs_seen = 0 + self.source_tokens = {k: 0 for k in stores} + + # -- sampling ------------------------------------------------------- # + def _pick_source(self, weights) -> str: + names = [n for n in weights if n in self.stores and weights[n] > 0] + if not names: + names = sorted(self.stores) + probs = torch.ones(len(names)) + else: + probs = torch.tensor([weights[n] for n in names], dtype=torch.float) + i = int(torch.multinomial(probs, 1, generator=self.gen).item()) + return names[i] + + def _long_draw_prob(self, name: str, st, floor: int, token_frac: float) -> float: + """Per-draw probability of taking a long document so that `token_frac` of + this source's TOKENS come from documents >= floor. + + long_doc_target_frac is a share of tokens (config.py; the stage-2 plan was + "20B of 40B tokens from docs >= 4096"). It used to be applied as a per-draw + probability, and since long documents average ~13x the mean length, 0.5 of + draws was ~94% of stage-2 tokens. With q the long-pool probability, a + uniform draw otherwise, L the mean length, Ll the long-pool mean and + Eu = E[len * 1(len >= floor)] under a uniform draw: + tokens_long / tokens = (q Ll + (1-q) Eu) / (q Ll + (1-q) L) = f + => q = (f L - Eu) / ((1-f) Ll - Eu + f L), clamped to [0, 1]. + """ + key = (name, floor, token_frac) + cache = self.__dict__.setdefault("_long_prob_cache", {}) + if key not in cache: + lens = st.lengths.astype(np.float64) + L = lens.mean() if lens.size else 0.0 + longm = lens >= floor + Ll = lens[longm].mean() if longm.any() else 0.0 + Eu = (lens * longm).mean() if lens.size else 0.0 + den = (1.0 - token_frac) * Ll - Eu + token_frac * L + q = (token_frac * L - Eu) / den if den > 0 else 0.0 + cache[key] = float(min(max(q, 0.0), 1.0)) + return cache[key] + + def _next_doc(self, weights, floor: int, long_frac: float): + name = self._pick_source(weights) + st = self.stores[name] + q = self._long_draw_prob(name, st, floor, long_frac) if floor > 0 else 0.0 + want_long = floor > 0 and float(torch.rand(1, generator=self.gen)) < q + pool = st.long_docs(floor) if want_long else None + if pool is not None and pool.size: + j = int(torch.randint(pool.size, (1,), generator=self.gen)) + i = int(pool[j]) + else: + i = int(torch.randint(st.n_docs, (1,), generator=self.gen)) + self.docs_seen += 1 + return name, _digit_split(np.asarray(st.doc(i), dtype=np.int64)) + + # -- packing -------------------------------------------------------- # + def batch(self, micro_batch: int, block: int, progress: float = 0.0): + weights = (mixture_at(progress, self.cfg) if self.split == "train" + else {n: 1.0 for n in self.stores}) + floor, long_frac = (long_doc_pressure(progress, self.cfg) + if self.split == "train" else (0, 0.0)) + B, T = micro_batch, block + xs = np.zeros((B, T), dtype=np.int64) + ys = np.full((B, T), IGNORE, dtype=np.int64) + pos = np.zeros((B, T), dtype=np.int64) + bnd = np.zeros((B, T), dtype=bool) + bounds: list[int] = [] + used: dict[str, int] = {} + + for b in range(B): + p = 0 + while p < T: + if self._carry is not None and self._carry.size: + # Positions restart on a carried chunk. The chunk opens a + # new attention segment (its own bounds entry, its own bnd + # marker, Canon and the n-gram table both masked across it), + # so it cannot attend to the part of the document already + # emitted -- continuing the count described a context the + # chunk does not have, and RoPE is relative, so restarting + # is identical for every pair it can actually see. It also + # keeps pos inside the RoPE table: max_doc_tokens is 65,536 + # against a table of max_block_size, and a carried document + # ran pos to 33,789 and indexed off the end of it. + name, d, start = self._carry_name, self._carry, 0 + self._carry = None + else: + name, d = self._next_doc(weights, floor, long_frac) + start = 0 + # Long documents are emitted over as many row segments as + # necessary. The old slice discarded every token after + # max_doc_tokens despite the configuration promising that + # documents were split, making 882.5M stored corpus tokens + # unreachable. + take = min(d.size, T - p) + bounds.append(b * T + p) + bnd[b, p] = True + xs[b, p:p + take] = d[:take] + pos[b, p:p + take] = np.arange(start, start + take) + # y = next token inside this document; the final slot of a + # document predicts nothing and stays IGNORE. + if take > 1: + ys[b, p:p + take - 1] = d[1:take] + if take < d.size: + # The document continues in the next row: carry the tail, + # and the last kept token does have a next token. + ys[b, p + take - 1] = d[take] + self._carry = d[take:] + self._carry_name = name + self._carry_pos = 0 + used[name] = used.get(name, 0) + take + p += take + + for k, v in used.items(): + self.source_tokens[k] = self.source_tokens.get(k, 0) + v + bounds.append(B * T) + bounds = np.asarray(bounds, dtype=np.int64) + # sum(len^2) is the exact attention cost of this batch. Computed here, + # where the lengths are already known, so the MFU line reports what + # attention actually did rather than the T^2 a dense mask would have. + lens = np.diff(bounds) + stats = {"used": used, "sumsq": int((lens * lens).sum()), + "tokens": B * T, "docs": int(lens.size), + # Carried so the nested-tensor SDPA path never has to derive + # them from the offsets with a host sync. + "min_seqlen": int(lens.min()), "max_seqlen": int(lens.max())} + return (xs, ys, pos, bnd, bounds, stats) + + # -- resumability ---------------------------------------------------- # + def state_dict(self) -> dict: + return {"gen": self.gen.get_state(), + "carry": None if self._carry is None else self._carry.copy(), + "carry_name": self._carry_name, + "carry_pos": self._carry_pos, + "docs_seen": self.docs_seen, + "source_tokens": dict(self.source_tokens)} + + def load_state_dict(self, st: dict) -> None: + if st is None: + return + # .cpu(): checkpoints are loaded with map_location=, which + # moves this RNG state onto the GPU too, and a CPU Generator's set_state + # demands a CPU ByteTensor ("RNG state must be a torch.ByteTensor"). + # Resume had never actually been run, so this went unnoticed until the + # first real one. + g = st["gen"] + self.gen.set_state(g.cpu().to(torch.uint8) if torch.is_tensor(g) + else torch.as_tensor(g, dtype=torch.uint8)) + c = st.get("carry") + name = st.get("carry_name") + # A checkpoint carries the half-consumed document it was mid-way + # through, tagged with its source name. Resuming that checkpoint onto a + # DIFFERENT corpus -- an anneal probe, or 0907 weights on ClimbMix -- + # leaves that name pointing at a source that no longer exists, and + # every later _pick_source lookup would KeyError. Drop the carry: one + # truncated document at the resume point costs nothing. + if name is not None and name not in self.stores: + c, name = None, None + self._carry = None if c is None else np.asarray(c) + self._carry_name = name + self._carry_pos = 0 if name is None else int(st.get("carry_pos", 0)) + self.docs_seen = int(st.get("docs_seen", 0)) + self.source_tokens.update(st.get("source_tokens", {})) + + +# --------------------------------------------------------------------------- # +# Prefetch +# --------------------------------------------------------------------------- # +class Prefetcher: + """One producer thread, pinned staging buffers, non-blocking H2D. + + A single producer, not a pool: the packer's RNG stream is the run's + reproducibility guarantee, and several workers drawing from it would make + the token order depend on thread scheduling. Packing is numpy slicing over a + memmap and costs far less than a step, so one thread stays well ahead. + """ + + def __init__(self, packer: VarlenPacker, cfg: Config, device: str): + self.packer = packer + self.cfg = cfg + self.device = device + self.q: queue.Queue = queue.Queue(maxsize=cfg.data.prefetch_depth) + self._shape = None # (micro_batch, block, progress) + self._stop = threading.Event() + self._t = threading.Thread(target=self._run, daemon=True) + self._lock = threading.Lock() + # The producer advances packer ahead of training. Checkpoints must store + # the state after the last CONSUMED batch, not its speculative state. + self._committed_state = packer.state_dict() + self._generation = 0 + self._error = None + + def configure(self, micro_batch: int, block: int, progress: float): + with self._lock: + if (self._shape is not None and + self._shape[:2] != (micro_batch, block)): + # Rewind speculative producer work before a shape transition. + # Otherwise every queued 2048-token batch discarded at the 8192 + # stage silently disappears from the reproducible token stream. + try: + while True: + self.q.get_nowait() + except queue.Empty: + pass + self.packer.load_state_dict(self._committed_state) + self._generation += 1 + self._shape = (micro_batch, block, progress) + + def start(self): + self._t.start() + return self + + def stop(self): + self._stop.set() + try: + while True: + self.q.get_nowait() + except queue.Empty: + pass + self._t.join(timeout=5) + + def state_dict(self): + with self._lock: + return self._committed_state + + def _run(self): + try: + self._produce() + except BaseException as exc: + self._error = exc + self._stop.set() + + def _produce(self): + while not self._stop.is_set(): + with self._lock: + shape = self._shape + generation = self._generation + if shape is not None: + mb, block, progress = shape + out = self.packer.batch(mb, block, progress) + state = self.packer.state_dict() + if shape is None: + self._stop.wait(0.01) + continue + xs, ys, pos, bnd, bounds, stats = out + item = (torch.from_numpy(xs).pin_memory(), + torch.from_numpy(ys).pin_memory(), + torch.from_numpy(pos).pin_memory(), + torch.from_numpy(bnd).pin_memory(), + torch.from_numpy(bounds).pin_memory(), + stats, (mb, block), state, generation) + while not self._stop.is_set(): + try: + self.q.put(item, timeout=0.25) + break + except queue.Full: + continue + + def next(self, micro_batch: int, block: int): + """Blocks only if the producer has fallen behind; drops stale shapes + after a context stage change.""" + while True: + if self._error is not None: + raise RuntimeError("data prefetch failed") from self._error + try: + xs, ys, pos, bnd, bounds, stats, shape, state, generation = self.q.get(timeout=1) + except queue.Empty: + if self._stop.is_set(): + raise RuntimeError("data prefetch stopped") from self._error + continue + if shape == (micro_batch, block) and generation == self._generation: + with self._lock: + self._committed_state = state + break + d = self.device + return (xs.to(d, non_blocking=True), ys.to(d, non_blocking=True), + pos.to(d, non_blocking=True), bnd.to(d, non_blocking=True), + bounds.to(d, non_blocking=True), stats) + + +def open_sources(cfg: Config, log=print) -> dict[str, SourceStore]: + stores = {} + missing = [] + for name in cfg.data.sources: + root = Path(cfg.data.root) + # A shard that has not written anything yet has an empty tokens.bin, + # which cannot be mmapped. Skip it: it contributes nothing until it + # does, and it will be picked up next time the run starts. + dirs = [d for d in sorted(root.glob(f"{name}")) + + sorted(root.glob(f"{name}_s*")) + if (d / "tokens.bin").exists() and (d / "docs.idx").exists() + and (d / "tokens.bin").stat().st_size > 0 + and (d / "docs.idx").stat().st_size > 8] + if not dirs: + missing.append(name) + continue + for d in dirs: + validate_tokenization(d, cfg) + parts = [SourceStore(name, d / "tokens.bin", d / "docs.idx") for d in dirs] + stores[name] = parts[0] if len(parts) == 1 else MultiStore(name, parts) + if len(parts) > 1: + log(f" {name}: {len(parts)} shards stitched " + f"({', '.join(d.name for d in dirs)})") + if not stores: + raise SystemExit( + f"no prepared sources under {cfg.data.root}. Run prepare.py first.") + if missing: + log(f" WARNING: sources not prepared and excluded from the mixture: " + f"{', '.join(missing)}") + tot = sum(s.n_tokens for s in stores.values()) + log(f" {len(stores)} sources, {tot/1e9:.1f}B tokens, " + f"{sum(s.n_docs for s in stores.values())/1e6:.1f}M documents") + for n, s in sorted(stores.items()): + log(f" {n:<16}{s.n_tokens/1e9:>8.2f}B {s.n_docs/1e6:>7.2f}M docs " + f"mean len {s.n_tokens/max(s.n_docs,1):>6.0f}") + return stores -- cgit v1.2.3