"""Varlen-packed data pipeline. Each source is a memory-mapped uint16 token file plus a uint64 index of document start offsets. Batches are packed, not EOS-concatenated: documents are laid end to end into rows of `block_size`, and the packer emits the document boundaries directly. It knows them for free while laying documents down, so nothing later has to re-derive them with a GPU scan and a host sync on every forward. Four tensors come out of the packer besides the tokens, and every one of them exists to stop a document seeing another document: offsets document boundaries into the flattened (B*T) stream. Feeds the jagged nested tensor, so attention is per-document. boundary (B, T) bool, True where a document starts. Masks the Canon taps and the n-gram history. pos (B, T) int32, position within the document. RoPE restarts at each document rather than running on for the whole row. targets next token within the document; the last position of a document predicts nothing and is IGNORE. Everything stochastic runs on a torch.Generator whose state is checkpointed, together with the partially consumed document carried between batches, so a resumed run replays the exact same token stream. """ from __future__ import annotations import queue import threading from pathlib import Path import json import numpy as np import torch from config import Config from schedule import long_doc_pressure, mixture_at IGNORE = -100 _DIGITS = None def _digit_split(doc): """BORIS_DIGIT_SPLIT=1 (pilot only): rewrite every number token as single digits, e.g. "Ġ999" -> "Ġ" "9" "9" "9", exactly as data/tokenizer_digits/tokenizer.json encodes text (verified on 4,000 ClimbMix documents: 3,997 identical). Off by default.""" global _DIGITS import os if os.environ.get("BORIS_DIGIT_SPLIT") != "1": return doc if _DIGITS is None: z = np.load(Path(__file__).resolve().parent / "data/tokenizer_digits/expand.npz") _DIGITS = (z["flat"], z["starts"], z["lens"]) flat, starts, lens = _DIGITS L = lens[doc] if (L == 1).all(): return doc idx = np.repeat(starts[doc], L) + (np.arange(int(L.sum())) - np.repeat(np.cumsum(L) - L, L)) return flat[idx] def validate_tokenization(directory, cfg): """Fail closed if a prepared 0920 shard uses different token IDs.""" from config import LEGACY_TOKENIZER_SHA256 if cfg.model.tokenizer_sha256 == LEGACY_TOKENIZER_SHA256: return path = Path(directory) / "retokenize_0920.json" if not path.exists(): raise ValueError(f"missing tokenizer provenance: {path}") got = json.loads(path.read_text()).get("new_tokenizer_sha256") if got != cfg.model.tokenizer_sha256: raise ValueError(f"tokenizer mismatch in {directory}: {got}") def open_validation(cfg): directory = Path(cfg.data.root) / cfg.data.val_source validate_tokenization(directory, cfg) return SourceStore.open(cfg.data.root, cfg.data.val_source) class SourceStore: """One tokenized source: tokens.bin (uint16) + docs.idx (uint64).""" def __init__(self, name: str, bin_path: Path, idx_path: Path): self.name = name self.tokens = np.memmap(bin_path, dtype=np.uint16, mode="r") # Keep the multi-GB document index file mapped instead of copying it # into anonymous RAM. Random lookup only needs two adjacent entries. self.offsets = np.memmap(idx_path, dtype=np.uint64, mode="r") if self.offsets.size < 2: raise ValueError(f"{idx_path}: need at least two offsets") self.n_docs = self.offsets.size - 1 raw_lengths = np.diff(self.offsets) if raw_lengths.size and raw_lengths.max() > np.iinfo(np.uint32).max: raise ValueError(f"{idx_path}: a document exceeds uint32 token length") self.lengths = raw_lengths.astype(np.uint32) self._long_cache: dict[int, np.ndarray] = {} @property def n_tokens(self) -> int: return int(self.offsets[-1]) def doc(self, i: int) -> np.ndarray: a, b = int(self.offsets[i]), int(self.offsets[i + 1]) return self.tokens[a:b] def long_docs(self, floor: int) -> np.ndarray: """Indices of documents at least `floor` tokens long, computed once. Built from the length array, which is already in memory -- no pass over the token file, which for a 200GB source would be its own job. """ if floor not in self._long_cache: self._long_cache[floor] = np.flatnonzero(self.lengths >= floor) return self._long_cache[floor] @staticmethod def open(root: Path, name: str) -> "SourceStore": d = Path(root) / name return SourceStore(name, d / "tokens.bin", d / "docs.idx") class MultiStore: """Several shard directories presented as one logical source. A source too large for one prepare process is split across workers by data file range, each writing its own directory. Stitching them here means the shards are never concatenated on disk -- copying 200GB to merge them would cost as much as downloading them and need the free space twice over. Document indices are global: doc i belongs to whichever shard's cumulative range contains it. """ def __init__(self, name: str, stores: list[SourceStore]): self.name = name self.stores = stores counts = np.array([s.n_docs for s in stores], dtype=np.int64) self._bounds = np.concatenate([[0], np.cumsum(counts)]) self.n_docs = int(self._bounds[-1]) self.lengths = np.concatenate([s.lengths for s in stores]) # MultiStore owns the combined length index; retaining every per-shard # copy doubled resident memory (about 2.5 GB for ClimbMix). for s in stores: s.lengths = np.empty(0, dtype=np.uint32) self._long_cache: dict[int, np.ndarray] = {} @property def n_tokens(self) -> int: return sum(s.n_tokens for s in self.stores) def doc(self, i: int) -> np.ndarray: j = int(np.searchsorted(self._bounds, i, side="right")) - 1 return self.stores[j].doc(i - int(self._bounds[j])) def long_docs(self, floor: int) -> np.ndarray: if floor not in self._long_cache: self._long_cache[floor] = np.flatnonzero(self.lengths >= floor) return self._long_cache[floor] class VarlenPacker: """Draws documents per the curriculum and packs them into fixed-width rows.""" def __init__(self, cfg: Config, stores: dict[str, SourceStore], seed: int, split: str = "train"): self.cfg = cfg self.stores = stores self.split = split self.gen = torch.Generator().manual_seed(seed) self._carry: np.ndarray | None = None self._carry_name: str | None = None self._carry_pos = 0 # position within the carried document self.docs_seen = 0 self.source_tokens = {k: 0 for k in stores} # -- sampling ------------------------------------------------------- # def _pick_source(self, weights) -> str: names = [n for n in weights if n in self.stores and weights[n] > 0] if not names: names = sorted(self.stores) probs = torch.ones(len(names)) else: probs = torch.tensor([weights[n] for n in names], dtype=torch.float) i = int(torch.multinomial(probs, 1, generator=self.gen).item()) return names[i] def _long_draw_prob(self, name: str, st, floor: int, token_frac: float) -> float: """Per-draw probability of taking a long document so that `token_frac` of this source's TOKENS come from documents >= floor. long_doc_target_frac is a share of tokens (config.py; the stage-2 plan was "20B of 40B tokens from docs >= 4096"). It used to be applied as a per-draw probability, and since long documents average ~13x the mean length, 0.5 of draws was ~94% of stage-2 tokens. With q the long-pool probability, a uniform draw otherwise, L the mean length, Ll the long-pool mean and Eu = E[len * 1(len >= floor)] under a uniform draw: tokens_long / tokens = (q Ll + (1-q) Eu) / (q Ll + (1-q) L) = f => q = (f L - Eu) / ((1-f) Ll - Eu + f L), clamped to [0, 1]. """ key = (name, floor, token_frac) cache = self.__dict__.setdefault("_long_prob_cache", {}) if key not in cache: lens = st.lengths.astype(np.float64) L = lens.mean() if lens.size else 0.0 longm = lens >= floor Ll = lens[longm].mean() if longm.any() else 0.0 Eu = (lens * longm).mean() if lens.size else 0.0 den = (1.0 - token_frac) * Ll - Eu + token_frac * L q = (token_frac * L - Eu) / den if den > 0 else 0.0 cache[key] = float(min(max(q, 0.0), 1.0)) return cache[key] def _next_doc(self, weights, floor: int, long_frac: float): name = self._pick_source(weights) st = self.stores[name] q = self._long_draw_prob(name, st, floor, long_frac) if floor > 0 else 0.0 want_long = floor > 0 and float(torch.rand(1, generator=self.gen)) < q pool = st.long_docs(floor) if want_long else None if pool is not None and pool.size: j = int(torch.randint(pool.size, (1,), generator=self.gen)) i = int(pool[j]) else: i = int(torch.randint(st.n_docs, (1,), generator=self.gen)) self.docs_seen += 1 return name, _digit_split(np.asarray(st.doc(i), dtype=np.int64)) # -- packing -------------------------------------------------------- # def batch(self, micro_batch: int, block: int, progress: float = 0.0): weights = (mixture_at(progress, self.cfg) if self.split == "train" else {n: 1.0 for n in self.stores}) floor, long_frac = (long_doc_pressure(progress, self.cfg) if self.split == "train" else (0, 0.0)) B, T = micro_batch, block xs = np.zeros((B, T), dtype=np.int64) ys = np.full((B, T), IGNORE, dtype=np.int64) pos = np.zeros((B, T), dtype=np.int64) bnd = np.zeros((B, T), dtype=bool) bounds: list[int] = [] used: dict[str, int] = {} for b in range(B): p = 0 while p < T: if self._carry is not None and self._carry.size: # Positions restart on a carried chunk. The chunk opens a # new attention segment (its own bounds entry, its own bnd # marker, Canon and the n-gram table both masked across it), # so it cannot attend to the part of the document already # emitted -- continuing the count described a context the # chunk does not have, and RoPE is relative, so restarting # is identical for every pair it can actually see. It also # keeps pos inside the RoPE table: max_doc_tokens is 65,536 # against a table of max_block_size, and a carried document # ran pos to 33,789 and indexed off the end of it. name, d, start = self._carry_name, self._carry, 0 self._carry = None else: name, d = self._next_doc(weights, floor, long_frac) start = 0 # Long documents are emitted over as many row segments as # necessary. The old slice discarded every token after # max_doc_tokens despite the configuration promising that # documents were split, making 882.5M stored corpus tokens # unreachable. take = min(d.size, T - p) bounds.append(b * T + p) bnd[b, p] = True xs[b, p:p + take] = d[:take] pos[b, p:p + take] = np.arange(start, start + take) # y = next token inside this document; the final slot of a # document predicts nothing and stays IGNORE. if take > 1: ys[b, p:p + take - 1] = d[1:take] if take < d.size: # The document continues in the next row: carry the tail, # and the last kept token does have a next token. ys[b, p + take - 1] = d[take] self._carry = d[take:] self._carry_name = name self._carry_pos = 0 used[name] = used.get(name, 0) + take p += take for k, v in used.items(): self.source_tokens[k] = self.source_tokens.get(k, 0) + v bounds.append(B * T) bounds = np.asarray(bounds, dtype=np.int64) # sum(len^2) is the exact attention cost of this batch. Computed here, # where the lengths are already known, so the MFU line reports what # attention actually did rather than the T^2 a dense mask would have. lens = np.diff(bounds) stats = {"used": used, "sumsq": int((lens * lens).sum()), "tokens": B * T, "docs": int(lens.size), # Carried so the nested-tensor SDPA path never has to derive # them from the offsets with a host sync. "min_seqlen": int(lens.min()), "max_seqlen": int(lens.max())} return (xs, ys, pos, bnd, bounds, stats) # -- resumability ---------------------------------------------------- # def state_dict(self) -> dict: return {"gen": self.gen.get_state(), "carry": None if self._carry is None else self._carry.copy(), "carry_name": self._carry_name, "carry_pos": self._carry_pos, "docs_seen": self.docs_seen, "source_tokens": dict(self.source_tokens)} def load_state_dict(self, st: dict) -> None: if st is None: return # .cpu(): checkpoints are loaded with map_location=, which # moves this RNG state onto the GPU too, and a CPU Generator's set_state # demands a CPU ByteTensor ("RNG state must be a torch.ByteTensor"). # Resume had never actually been run, so this went unnoticed until the # first real one. g = st["gen"] self.gen.set_state(g.cpu().to(torch.uint8) if torch.is_tensor(g) else torch.as_tensor(g, dtype=torch.uint8)) c = st.get("carry") name = st.get("carry_name") # A checkpoint carries the half-consumed document it was mid-way # through, tagged with its source name. Resuming that checkpoint onto a # DIFFERENT corpus -- an anneal probe, or 0907 weights on ClimbMix -- # leaves that name pointing at a source that no longer exists, and # every later _pick_source lookup would KeyError. Drop the carry: one # truncated document at the resume point costs nothing. if name is not None and name not in self.stores: c, name = None, None self._carry = None if c is None else np.asarray(c) self._carry_name = name self._carry_pos = 0 if name is None else int(st.get("carry_pos", 0)) self.docs_seen = int(st.get("docs_seen", 0)) self.source_tokens.update(st.get("source_tokens", {})) # --------------------------------------------------------------------------- # # Prefetch # --------------------------------------------------------------------------- # class Prefetcher: """One producer thread, pinned staging buffers, non-blocking H2D. A single producer, not a pool: the packer's RNG stream is the run's reproducibility guarantee, and several workers drawing from it would make the token order depend on thread scheduling. Packing is numpy slicing over a memmap and costs far less than a step, so one thread stays well ahead. """ def __init__(self, packer: VarlenPacker, cfg: Config, device: str): self.packer = packer self.cfg = cfg self.device = device self.q: queue.Queue = queue.Queue(maxsize=cfg.data.prefetch_depth) self._shape = None # (micro_batch, block, progress) self._stop = threading.Event() self._t = threading.Thread(target=self._run, daemon=True) self._lock = threading.Lock() # The producer advances packer ahead of training. Checkpoints must store # the state after the last CONSUMED batch, not its speculative state. self._committed_state = packer.state_dict() self._generation = 0 self._error = None def configure(self, micro_batch: int, block: int, progress: float): with self._lock: if (self._shape is not None and self._shape[:2] != (micro_batch, block)): # Rewind speculative producer work before a shape transition. # Otherwise every queued 2048-token batch discarded at the 8192 # stage silently disappears from the reproducible token stream. try: while True: self.q.get_nowait() except queue.Empty: pass self.packer.load_state_dict(self._committed_state) self._generation += 1 self._shape = (micro_batch, block, progress) def start(self): self._t.start() return self def stop(self): self._stop.set() try: while True: self.q.get_nowait() except queue.Empty: pass self._t.join(timeout=5) def state_dict(self): with self._lock: return self._committed_state def _run(self): try: self._produce() except BaseException as exc: self._error = exc self._stop.set() def _produce(self): while not self._stop.is_set(): with self._lock: shape = self._shape generation = self._generation if shape is not None: mb, block, progress = shape out = self.packer.batch(mb, block, progress) state = self.packer.state_dict() if shape is None: self._stop.wait(0.01) continue xs, ys, pos, bnd, bounds, stats = out item = (torch.from_numpy(xs).pin_memory(), torch.from_numpy(ys).pin_memory(), torch.from_numpy(pos).pin_memory(), torch.from_numpy(bnd).pin_memory(), torch.from_numpy(bounds).pin_memory(), stats, (mb, block), state, generation) while not self._stop.is_set(): try: self.q.put(item, timeout=0.25) break except queue.Full: continue def next(self, micro_batch: int, block: int): """Blocks only if the producer has fallen behind; drops stale shapes after a context stage change.""" while True: if self._error is not None: raise RuntimeError("data prefetch failed") from self._error try: xs, ys, pos, bnd, bounds, stats, shape, state, generation = self.q.get(timeout=1) except queue.Empty: if self._stop.is_set(): raise RuntimeError("data prefetch stopped") from self._error continue if shape == (micro_batch, block) and generation == self._generation: with self._lock: self._committed_state = state break d = self.device return (xs.to(d, non_blocking=True), ys.to(d, non_blocking=True), pos.to(d, non_blocking=True), bnd.to(d, non_blocking=True), bounds.to(d, non_blocking=True), stats) def open_sources(cfg: Config, log=print) -> dict[str, SourceStore]: stores = {} missing = [] for name in cfg.data.sources: root = Path(cfg.data.root) # A shard that has not written anything yet has an empty tokens.bin, # which cannot be mmapped. Skip it: it contributes nothing until it # does, and it will be picked up next time the run starts. dirs = [d for d in sorted(root.glob(f"{name}")) + sorted(root.glob(f"{name}_s*")) if (d / "tokens.bin").exists() and (d / "docs.idx").exists() and (d / "tokens.bin").stat().st_size > 0 and (d / "docs.idx").stat().st_size > 8] if not dirs: missing.append(name) continue for d in dirs: validate_tokenization(d, cfg) parts = [SourceStore(name, d / "tokens.bin", d / "docs.idx") for d in dirs] stores[name] = parts[0] if len(parts) == 1 else MultiStore(name, parts) if len(parts) > 1: log(f" {name}: {len(parts)} shards stitched " f"({', '.join(d.name for d in dirs)})") if not stores: raise SystemExit( f"no prepared sources under {cfg.data.root}. Run prepare.py first.") if missing: log(f" WARNING: sources not prepared and excluded from the mixture: " f"{', '.join(missing)}") tot = sum(s.n_tokens for s in stores.values()) log(f" {len(stores)} sources, {tot/1e9:.1f}B tokens, " f"{sum(s.n_docs for s in stores.values())/1e6:.1f}M documents") for n, s in sorted(stores.items()): log(f" {n:<16}{s.n_tokens/1e9:>8.2f}B {s.n_docs/1e6:>7.2f}M docs " f"mean len {s.n_tokens/max(s.n_docs,1):>6.0f}") return stores