summaryrefslogtreecommitdiff
path: root/data.py
diff options
context:
space:
mode:
Diffstat (limited to 'data.py')
-rw-r--r--data.py488
1 files changed, 488 insertions, 0 deletions
diff --git a/data.py b/data.py
new file mode 100644
index 0000000..0d71175
--- /dev/null
+++ b/data.py
@@ -0,0 +1,488 @@
+"""Varlen-packed data pipeline.
+
+Each source is a memory-mapped uint16 token file plus a uint64 index of document
+start offsets. Batches are packed, not EOS-concatenated: documents are laid end
+to end into rows of `block_size`, and the packer emits the document boundaries
+directly. It knows them for free while laying documents down, so nothing later
+has to re-derive them with a GPU scan and a host sync on every forward.
+
+Four tensors come out of the packer besides the tokens, and every one of them
+exists to stop a document seeing another document:
+
+ offsets document boundaries into the flattened (B*T) stream. Feeds the
+ jagged nested tensor, so attention is per-document.
+ boundary (B, T) bool, True where a document starts. Masks the Canon taps
+ and the n-gram history.
+ pos (B, T) int32, position within the document. RoPE restarts at each
+ document rather than running on for the whole row.
+ targets next token within the document; the last position of a document
+ predicts nothing and is IGNORE.
+
+Everything stochastic runs on a torch.Generator whose state is checkpointed,
+together with the partially consumed document carried between batches, so a
+resumed run replays the exact same token stream.
+"""
+from __future__ import annotations
+
+import queue
+import threading
+from pathlib import Path
+import json
+
+import numpy as np
+import torch
+
+from config import Config
+from schedule import long_doc_pressure, mixture_at
+
+IGNORE = -100
+
+
+_DIGITS = None
+
+
+def _digit_split(doc):
+ """BORIS_DIGIT_SPLIT=1 (pilot only): rewrite every number token as single digits,
+ e.g. "Ġ999" -> "Ġ" "9" "9" "9", exactly as data/tokenizer_digits/tokenizer.json encodes
+ text (verified on 4,000 ClimbMix documents: 3,997 identical). Off by default."""
+ global _DIGITS
+ import os
+ if os.environ.get("BORIS_DIGIT_SPLIT") != "1":
+ return doc
+ if _DIGITS is None:
+ z = np.load(Path(__file__).resolve().parent / "data/tokenizer_digits/expand.npz")
+ _DIGITS = (z["flat"], z["starts"], z["lens"])
+ flat, starts, lens = _DIGITS
+ L = lens[doc]
+ if (L == 1).all():
+ return doc
+ idx = np.repeat(starts[doc], L) + (np.arange(int(L.sum())) - np.repeat(np.cumsum(L) - L, L))
+ return flat[idx]
+
+
+def validate_tokenization(directory, cfg):
+ """Fail closed if a prepared 0920 shard uses different token IDs."""
+ from config import LEGACY_TOKENIZER_SHA256
+ if cfg.model.tokenizer_sha256 == LEGACY_TOKENIZER_SHA256:
+ return
+ path = Path(directory) / "retokenize_0920.json"
+ if not path.exists():
+ raise ValueError(f"missing tokenizer provenance: {path}")
+ got = json.loads(path.read_text()).get("new_tokenizer_sha256")
+ if got != cfg.model.tokenizer_sha256:
+ raise ValueError(f"tokenizer mismatch in {directory}: {got}")
+
+
+def open_validation(cfg):
+ directory = Path(cfg.data.root) / cfg.data.val_source
+ validate_tokenization(directory, cfg)
+ return SourceStore.open(cfg.data.root, cfg.data.val_source)
+
+
+class SourceStore:
+ """One tokenized source: tokens.bin (uint16) + docs.idx (uint64)."""
+
+ def __init__(self, name: str, bin_path: Path, idx_path: Path):
+ self.name = name
+ self.tokens = np.memmap(bin_path, dtype=np.uint16, mode="r")
+ # Keep the multi-GB document index file mapped instead of copying it
+ # into anonymous RAM. Random lookup only needs two adjacent entries.
+ self.offsets = np.memmap(idx_path, dtype=np.uint64, mode="r")
+ if self.offsets.size < 2:
+ raise ValueError(f"{idx_path}: need at least two offsets")
+ self.n_docs = self.offsets.size - 1
+ raw_lengths = np.diff(self.offsets)
+ if raw_lengths.size and raw_lengths.max() > np.iinfo(np.uint32).max:
+ raise ValueError(f"{idx_path}: a document exceeds uint32 token length")
+ self.lengths = raw_lengths.astype(np.uint32)
+ self._long_cache: dict[int, np.ndarray] = {}
+
+ @property
+ def n_tokens(self) -> int:
+ return int(self.offsets[-1])
+
+ def doc(self, i: int) -> np.ndarray:
+ a, b = int(self.offsets[i]), int(self.offsets[i + 1])
+ return self.tokens[a:b]
+
+ def long_docs(self, floor: int) -> np.ndarray:
+ """Indices of documents at least `floor` tokens long, computed once.
+
+ Built from the length array, which is already in memory -- no pass over
+ the token file, which for a 200GB source would be its own job.
+ """
+ if floor not in self._long_cache:
+ self._long_cache[floor] = np.flatnonzero(self.lengths >= floor)
+ return self._long_cache[floor]
+
+ @staticmethod
+ def open(root: Path, name: str) -> "SourceStore":
+ d = Path(root) / name
+ return SourceStore(name, d / "tokens.bin", d / "docs.idx")
+
+
+class MultiStore:
+ """Several shard directories presented as one logical source.
+
+ A source too large for one prepare process is split across workers by data
+ file range, each writing its own directory. Stitching them here means the
+ shards are never concatenated on disk -- copying 200GB to merge them would
+ cost as much as downloading them and need the free space twice over.
+
+ Document indices are global: doc i belongs to whichever shard's cumulative
+ range contains it.
+ """
+
+ def __init__(self, name: str, stores: list[SourceStore]):
+ self.name = name
+ self.stores = stores
+ counts = np.array([s.n_docs for s in stores], dtype=np.int64)
+ self._bounds = np.concatenate([[0], np.cumsum(counts)])
+ self.n_docs = int(self._bounds[-1])
+ self.lengths = np.concatenate([s.lengths for s in stores])
+ # MultiStore owns the combined length index; retaining every per-shard
+ # copy doubled resident memory (about 2.5 GB for ClimbMix).
+ for s in stores:
+ s.lengths = np.empty(0, dtype=np.uint32)
+ self._long_cache: dict[int, np.ndarray] = {}
+
+ @property
+ def n_tokens(self) -> int:
+ return sum(s.n_tokens for s in self.stores)
+
+ def doc(self, i: int) -> np.ndarray:
+ j = int(np.searchsorted(self._bounds, i, side="right")) - 1
+ return self.stores[j].doc(i - int(self._bounds[j]))
+
+ def long_docs(self, floor: int) -> np.ndarray:
+ if floor not in self._long_cache:
+ self._long_cache[floor] = np.flatnonzero(self.lengths >= floor)
+ return self._long_cache[floor]
+
+
+class VarlenPacker:
+ """Draws documents per the curriculum and packs them into fixed-width rows."""
+
+ def __init__(self, cfg: Config, stores: dict[str, SourceStore], seed: int,
+ split: str = "train"):
+ self.cfg = cfg
+ self.stores = stores
+ self.split = split
+ self.gen = torch.Generator().manual_seed(seed)
+ self._carry: np.ndarray | None = None
+ self._carry_name: str | None = None
+ self._carry_pos = 0 # position within the carried document
+ self.docs_seen = 0
+ self.source_tokens = {k: 0 for k in stores}
+
+ # -- sampling ------------------------------------------------------- #
+ def _pick_source(self, weights) -> str:
+ names = [n for n in weights if n in self.stores and weights[n] > 0]
+ if not names:
+ names = sorted(self.stores)
+ probs = torch.ones(len(names))
+ else:
+ probs = torch.tensor([weights[n] for n in names], dtype=torch.float)
+ i = int(torch.multinomial(probs, 1, generator=self.gen).item())
+ return names[i]
+
+ def _long_draw_prob(self, name: str, st, floor: int, token_frac: float) -> float:
+ """Per-draw probability of taking a long document so that `token_frac` of
+ this source's TOKENS come from documents >= floor.
+
+ long_doc_target_frac is a share of tokens (config.py; the stage-2 plan was
+ "20B of 40B tokens from docs >= 4096"). It used to be applied as a per-draw
+ probability, and since long documents average ~13x the mean length, 0.5 of
+ draws was ~94% of stage-2 tokens. With q the long-pool probability, a
+ uniform draw otherwise, L the mean length, Ll the long-pool mean and
+ Eu = E[len * 1(len >= floor)] under a uniform draw:
+ tokens_long / tokens = (q Ll + (1-q) Eu) / (q Ll + (1-q) L) = f
+ => q = (f L - Eu) / ((1-f) Ll - Eu + f L), clamped to [0, 1].
+ """
+ key = (name, floor, token_frac)
+ cache = self.__dict__.setdefault("_long_prob_cache", {})
+ if key not in cache:
+ lens = st.lengths.astype(np.float64)
+ L = lens.mean() if lens.size else 0.0
+ longm = lens >= floor
+ Ll = lens[longm].mean() if longm.any() else 0.0
+ Eu = (lens * longm).mean() if lens.size else 0.0
+ den = (1.0 - token_frac) * Ll - Eu + token_frac * L
+ q = (token_frac * L - Eu) / den if den > 0 else 0.0
+ cache[key] = float(min(max(q, 0.0), 1.0))
+ return cache[key]
+
+ def _next_doc(self, weights, floor: int, long_frac: float):
+ name = self._pick_source(weights)
+ st = self.stores[name]
+ q = self._long_draw_prob(name, st, floor, long_frac) if floor > 0 else 0.0
+ want_long = floor > 0 and float(torch.rand(1, generator=self.gen)) < q
+ pool = st.long_docs(floor) if want_long else None
+ if pool is not None and pool.size:
+ j = int(torch.randint(pool.size, (1,), generator=self.gen))
+ i = int(pool[j])
+ else:
+ i = int(torch.randint(st.n_docs, (1,), generator=self.gen))
+ self.docs_seen += 1
+ return name, _digit_split(np.asarray(st.doc(i), dtype=np.int64))
+
+ # -- packing -------------------------------------------------------- #
+ def batch(self, micro_batch: int, block: int, progress: float = 0.0):
+ weights = (mixture_at(progress, self.cfg) if self.split == "train"
+ else {n: 1.0 for n in self.stores})
+ floor, long_frac = (long_doc_pressure(progress, self.cfg)
+ if self.split == "train" else (0, 0.0))
+ B, T = micro_batch, block
+ xs = np.zeros((B, T), dtype=np.int64)
+ ys = np.full((B, T), IGNORE, dtype=np.int64)
+ pos = np.zeros((B, T), dtype=np.int64)
+ bnd = np.zeros((B, T), dtype=bool)
+ bounds: list[int] = []
+ used: dict[str, int] = {}
+
+ for b in range(B):
+ p = 0
+ while p < T:
+ if self._carry is not None and self._carry.size:
+ # Positions restart on a carried chunk. The chunk opens a
+ # new attention segment (its own bounds entry, its own bnd
+ # marker, Canon and the n-gram table both masked across it),
+ # so it cannot attend to the part of the document already
+ # emitted -- continuing the count described a context the
+ # chunk does not have, and RoPE is relative, so restarting
+ # is identical for every pair it can actually see. It also
+ # keeps pos inside the RoPE table: max_doc_tokens is 65,536
+ # against a table of max_block_size, and a carried document
+ # ran pos to 33,789 and indexed off the end of it.
+ name, d, start = self._carry_name, self._carry, 0
+ self._carry = None
+ else:
+ name, d = self._next_doc(weights, floor, long_frac)
+ start = 0
+ # Long documents are emitted over as many row segments as
+ # necessary. The old slice discarded every token after
+ # max_doc_tokens despite the configuration promising that
+ # documents were split, making 882.5M stored corpus tokens
+ # unreachable.
+ take = min(d.size, T - p)
+ bounds.append(b * T + p)
+ bnd[b, p] = True
+ xs[b, p:p + take] = d[:take]
+ pos[b, p:p + take] = np.arange(start, start + take)
+ # y = next token inside this document; the final slot of a
+ # document predicts nothing and stays IGNORE.
+ if take > 1:
+ ys[b, p:p + take - 1] = d[1:take]
+ if take < d.size:
+ # The document continues in the next row: carry the tail,
+ # and the last kept token does have a next token.
+ ys[b, p + take - 1] = d[take]
+ self._carry = d[take:]
+ self._carry_name = name
+ self._carry_pos = 0
+ used[name] = used.get(name, 0) + take
+ p += take
+
+ for k, v in used.items():
+ self.source_tokens[k] = self.source_tokens.get(k, 0) + v
+ bounds.append(B * T)
+ bounds = np.asarray(bounds, dtype=np.int64)
+ # sum(len^2) is the exact attention cost of this batch. Computed here,
+ # where the lengths are already known, so the MFU line reports what
+ # attention actually did rather than the T^2 a dense mask would have.
+ lens = np.diff(bounds)
+ stats = {"used": used, "sumsq": int((lens * lens).sum()),
+ "tokens": B * T, "docs": int(lens.size),
+ # Carried so the nested-tensor SDPA path never has to derive
+ # them from the offsets with a host sync.
+ "min_seqlen": int(lens.min()), "max_seqlen": int(lens.max())}
+ return (xs, ys, pos, bnd, bounds, stats)
+
+ # -- resumability ---------------------------------------------------- #
+ def state_dict(self) -> dict:
+ return {"gen": self.gen.get_state(),
+ "carry": None if self._carry is None else self._carry.copy(),
+ "carry_name": self._carry_name,
+ "carry_pos": self._carry_pos,
+ "docs_seen": self.docs_seen,
+ "source_tokens": dict(self.source_tokens)}
+
+ def load_state_dict(self, st: dict) -> None:
+ if st is None:
+ return
+ # .cpu(): checkpoints are loaded with map_location=<cuda device>, which
+ # moves this RNG state onto the GPU too, and a CPU Generator's set_state
+ # demands a CPU ByteTensor ("RNG state must be a torch.ByteTensor").
+ # Resume had never actually been run, so this went unnoticed until the
+ # first real one.
+ g = st["gen"]
+ self.gen.set_state(g.cpu().to(torch.uint8) if torch.is_tensor(g)
+ else torch.as_tensor(g, dtype=torch.uint8))
+ c = st.get("carry")
+ name = st.get("carry_name")
+ # A checkpoint carries the half-consumed document it was mid-way
+ # through, tagged with its source name. Resuming that checkpoint onto a
+ # DIFFERENT corpus -- an anneal probe, or 0907 weights on ClimbMix --
+ # leaves that name pointing at a source that no longer exists, and
+ # every later _pick_source lookup would KeyError. Drop the carry: one
+ # truncated document at the resume point costs nothing.
+ if name is not None and name not in self.stores:
+ c, name = None, None
+ self._carry = None if c is None else np.asarray(c)
+ self._carry_name = name
+ self._carry_pos = 0 if name is None else int(st.get("carry_pos", 0))
+ self.docs_seen = int(st.get("docs_seen", 0))
+ self.source_tokens.update(st.get("source_tokens", {}))
+
+
+# --------------------------------------------------------------------------- #
+# Prefetch
+# --------------------------------------------------------------------------- #
+class Prefetcher:
+ """One producer thread, pinned staging buffers, non-blocking H2D.
+
+ A single producer, not a pool: the packer's RNG stream is the run's
+ reproducibility guarantee, and several workers drawing from it would make
+ the token order depend on thread scheduling. Packing is numpy slicing over a
+ memmap and costs far less than a step, so one thread stays well ahead.
+ """
+
+ def __init__(self, packer: VarlenPacker, cfg: Config, device: str):
+ self.packer = packer
+ self.cfg = cfg
+ self.device = device
+ self.q: queue.Queue = queue.Queue(maxsize=cfg.data.prefetch_depth)
+ self._shape = None # (micro_batch, block, progress)
+ self._stop = threading.Event()
+ self._t = threading.Thread(target=self._run, daemon=True)
+ self._lock = threading.Lock()
+ # The producer advances packer ahead of training. Checkpoints must store
+ # the state after the last CONSUMED batch, not its speculative state.
+ self._committed_state = packer.state_dict()
+ self._generation = 0
+ self._error = None
+
+ def configure(self, micro_batch: int, block: int, progress: float):
+ with self._lock:
+ if (self._shape is not None and
+ self._shape[:2] != (micro_batch, block)):
+ # Rewind speculative producer work before a shape transition.
+ # Otherwise every queued 2048-token batch discarded at the 8192
+ # stage silently disappears from the reproducible token stream.
+ try:
+ while True:
+ self.q.get_nowait()
+ except queue.Empty:
+ pass
+ self.packer.load_state_dict(self._committed_state)
+ self._generation += 1
+ self._shape = (micro_batch, block, progress)
+
+ def start(self):
+ self._t.start()
+ return self
+
+ def stop(self):
+ self._stop.set()
+ try:
+ while True:
+ self.q.get_nowait()
+ except queue.Empty:
+ pass
+ self._t.join(timeout=5)
+
+ def state_dict(self):
+ with self._lock:
+ return self._committed_state
+
+ def _run(self):
+ try:
+ self._produce()
+ except BaseException as exc:
+ self._error = exc
+ self._stop.set()
+
+ def _produce(self):
+ while not self._stop.is_set():
+ with self._lock:
+ shape = self._shape
+ generation = self._generation
+ if shape is not None:
+ mb, block, progress = shape
+ out = self.packer.batch(mb, block, progress)
+ state = self.packer.state_dict()
+ if shape is None:
+ self._stop.wait(0.01)
+ continue
+ xs, ys, pos, bnd, bounds, stats = out
+ item = (torch.from_numpy(xs).pin_memory(),
+ torch.from_numpy(ys).pin_memory(),
+ torch.from_numpy(pos).pin_memory(),
+ torch.from_numpy(bnd).pin_memory(),
+ torch.from_numpy(bounds).pin_memory(),
+ stats, (mb, block), state, generation)
+ while not self._stop.is_set():
+ try:
+ self.q.put(item, timeout=0.25)
+ break
+ except queue.Full:
+ continue
+
+ def next(self, micro_batch: int, block: int):
+ """Blocks only if the producer has fallen behind; drops stale shapes
+ after a context stage change."""
+ while True:
+ if self._error is not None:
+ raise RuntimeError("data prefetch failed") from self._error
+ try:
+ xs, ys, pos, bnd, bounds, stats, shape, state, generation = self.q.get(timeout=1)
+ except queue.Empty:
+ if self._stop.is_set():
+ raise RuntimeError("data prefetch stopped") from self._error
+ continue
+ if shape == (micro_batch, block) and generation == self._generation:
+ with self._lock:
+ self._committed_state = state
+ break
+ d = self.device
+ return (xs.to(d, non_blocking=True), ys.to(d, non_blocking=True),
+ pos.to(d, non_blocking=True), bnd.to(d, non_blocking=True),
+ bounds.to(d, non_blocking=True), stats)
+
+
+def open_sources(cfg: Config, log=print) -> dict[str, SourceStore]:
+ stores = {}
+ missing = []
+ for name in cfg.data.sources:
+ root = Path(cfg.data.root)
+ # A shard that has not written anything yet has an empty tokens.bin,
+ # which cannot be mmapped. Skip it: it contributes nothing until it
+ # does, and it will be picked up next time the run starts.
+ dirs = [d for d in sorted(root.glob(f"{name}"))
+ + sorted(root.glob(f"{name}_s*"))
+ if (d / "tokens.bin").exists() and (d / "docs.idx").exists()
+ and (d / "tokens.bin").stat().st_size > 0
+ and (d / "docs.idx").stat().st_size > 8]
+ if not dirs:
+ missing.append(name)
+ continue
+ for d in dirs:
+ validate_tokenization(d, cfg)
+ parts = [SourceStore(name, d / "tokens.bin", d / "docs.idx") for d in dirs]
+ stores[name] = parts[0] if len(parts) == 1 else MultiStore(name, parts)
+ if len(parts) > 1:
+ log(f" {name}: {len(parts)} shards stitched "
+ f"({', '.join(d.name for d in dirs)})")
+ if not stores:
+ raise SystemExit(
+ f"no prepared sources under {cfg.data.root}. Run prepare.py first.")
+ if missing:
+ log(f" WARNING: sources not prepared and excluded from the mixture: "
+ f"{', '.join(missing)}")
+ tot = sum(s.n_tokens for s in stores.values())
+ log(f" {len(stores)} sources, {tot/1e9:.1f}B tokens, "
+ f"{sum(s.n_docs for s in stores.values())/1e6:.1f}M documents")
+ for n, s in sorted(stores.items()):
+ log(f" {n:<16}{s.n_tokens/1e9:>8.2f}B {s.n_docs/1e6:>7.2f}M docs "
+ f"mean len {s.n_tokens/max(s.n_docs,1):>6.0f}")
+ return stores