summaryrefslogtreecommitdiff
path: root/data.py
blob: 0d7117509d7b14493bdbbe756bc3419065a09f45 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
"""Varlen-packed data pipeline.

Each source is a memory-mapped uint16 token file plus a uint64 index of document
start offsets. Batches are packed, not EOS-concatenated: documents are laid end
to end into rows of `block_size`, and the packer emits the document boundaries
directly. It knows them for free while laying documents down, so nothing later
has to re-derive them with a GPU scan and a host sync on every forward.

Four tensors come out of the packer besides the tokens, and every one of them
exists to stop a document seeing another document:

    offsets   document boundaries into the flattened (B*T) stream. Feeds the
              jagged nested tensor, so attention is per-document.
    boundary  (B, T) bool, True where a document starts. Masks the Canon taps
              and the n-gram history.
    pos       (B, T) int32, position within the document. RoPE restarts at each
              document rather than running on for the whole row.
    targets   next token within the document; the last position of a document
              predicts nothing and is IGNORE.

Everything stochastic runs on a torch.Generator whose state is checkpointed,
together with the partially consumed document carried between batches, so a
resumed run replays the exact same token stream.
"""
from __future__ import annotations

import queue
import threading
from pathlib import Path
import json

import numpy as np
import torch

from config import Config
from schedule import long_doc_pressure, mixture_at

IGNORE = -100


_DIGITS = None


def _digit_split(doc):
    """BORIS_DIGIT_SPLIT=1 (pilot only): rewrite every number token as single digits,
    e.g. "Ġ999" -> "Ġ" "9" "9" "9", exactly as data/tokenizer_digits/tokenizer.json encodes
    text (verified on 4,000 ClimbMix documents: 3,997 identical). Off by default."""
    global _DIGITS
    import os
    if os.environ.get("BORIS_DIGIT_SPLIT") != "1":
        return doc
    if _DIGITS is None:
        z = np.load(Path(__file__).resolve().parent / "data/tokenizer_digits/expand.npz")
        _DIGITS = (z["flat"], z["starts"], z["lens"])
    flat, starts, lens = _DIGITS
    L = lens[doc]
    if (L == 1).all():
        return doc
    idx = np.repeat(starts[doc], L) + (np.arange(int(L.sum())) - np.repeat(np.cumsum(L) - L, L))
    return flat[idx]


def validate_tokenization(directory, cfg):
    """Fail closed if a prepared 0920 shard uses different token IDs."""
    from config import LEGACY_TOKENIZER_SHA256
    if cfg.model.tokenizer_sha256 == LEGACY_TOKENIZER_SHA256:
        return
    path = Path(directory) / "retokenize_0920.json"
    if not path.exists():
        raise ValueError(f"missing tokenizer provenance: {path}")
    got = json.loads(path.read_text()).get("new_tokenizer_sha256")
    if got != cfg.model.tokenizer_sha256:
        raise ValueError(f"tokenizer mismatch in {directory}: {got}")


def open_validation(cfg):
    directory = Path(cfg.data.root) / cfg.data.val_source
    validate_tokenization(directory, cfg)
    return SourceStore.open(cfg.data.root, cfg.data.val_source)


class SourceStore:
    """One tokenized source: tokens.bin (uint16) + docs.idx (uint64)."""

    def __init__(self, name: str, bin_path: Path, idx_path: Path):
        self.name = name
        self.tokens = np.memmap(bin_path, dtype=np.uint16, mode="r")
        # Keep the multi-GB document index file mapped instead of copying it
        # into anonymous RAM. Random lookup only needs two adjacent entries.
        self.offsets = np.memmap(idx_path, dtype=np.uint64, mode="r")
        if self.offsets.size < 2:
            raise ValueError(f"{idx_path}: need at least two offsets")
        self.n_docs = self.offsets.size - 1
        raw_lengths = np.diff(self.offsets)
        if raw_lengths.size and raw_lengths.max() > np.iinfo(np.uint32).max:
            raise ValueError(f"{idx_path}: a document exceeds uint32 token length")
        self.lengths = raw_lengths.astype(np.uint32)
        self._long_cache: dict[int, np.ndarray] = {}

    @property
    def n_tokens(self) -> int:
        return int(self.offsets[-1])

    def doc(self, i: int) -> np.ndarray:
        a, b = int(self.offsets[i]), int(self.offsets[i + 1])
        return self.tokens[a:b]

    def long_docs(self, floor: int) -> np.ndarray:
        """Indices of documents at least `floor` tokens long, computed once.

        Built from the length array, which is already in memory -- no pass over
        the token file, which for a 200GB source would be its own job.
        """
        if floor not in self._long_cache:
            self._long_cache[floor] = np.flatnonzero(self.lengths >= floor)
        return self._long_cache[floor]

    @staticmethod
    def open(root: Path, name: str) -> "SourceStore":
        d = Path(root) / name
        return SourceStore(name, d / "tokens.bin", d / "docs.idx")


class MultiStore:
    """Several shard directories presented as one logical source.

    A source too large for one prepare process is split across workers by data
    file range, each writing its own directory. Stitching them here means the
    shards are never concatenated on disk -- copying 200GB to merge them would
    cost as much as downloading them and need the free space twice over.

    Document indices are global: doc i belongs to whichever shard's cumulative
    range contains it.
    """

    def __init__(self, name: str, stores: list[SourceStore]):
        self.name = name
        self.stores = stores
        counts = np.array([s.n_docs for s in stores], dtype=np.int64)
        self._bounds = np.concatenate([[0], np.cumsum(counts)])
        self.n_docs = int(self._bounds[-1])
        self.lengths = np.concatenate([s.lengths for s in stores])
        # MultiStore owns the combined length index; retaining every per-shard
        # copy doubled resident memory (about 2.5 GB for ClimbMix).
        for s in stores:
            s.lengths = np.empty(0, dtype=np.uint32)
        self._long_cache: dict[int, np.ndarray] = {}

    @property
    def n_tokens(self) -> int:
        return sum(s.n_tokens for s in self.stores)

    def doc(self, i: int) -> np.ndarray:
        j = int(np.searchsorted(self._bounds, i, side="right")) - 1
        return self.stores[j].doc(i - int(self._bounds[j]))

    def long_docs(self, floor: int) -> np.ndarray:
        if floor not in self._long_cache:
            self._long_cache[floor] = np.flatnonzero(self.lengths >= floor)
        return self._long_cache[floor]


class VarlenPacker:
    """Draws documents per the curriculum and packs them into fixed-width rows."""

    def __init__(self, cfg: Config, stores: dict[str, SourceStore], seed: int,
                 split: str = "train"):
        self.cfg = cfg
        self.stores = stores
        self.split = split
        self.gen = torch.Generator().manual_seed(seed)
        self._carry: np.ndarray | None = None
        self._carry_name: str | None = None
        self._carry_pos = 0                 # position within the carried document
        self.docs_seen = 0
        self.source_tokens = {k: 0 for k in stores}

    # -- sampling ------------------------------------------------------- #
    def _pick_source(self, weights) -> str:
        names = [n for n in weights if n in self.stores and weights[n] > 0]
        if not names:
            names = sorted(self.stores)
            probs = torch.ones(len(names))
        else:
            probs = torch.tensor([weights[n] for n in names], dtype=torch.float)
        i = int(torch.multinomial(probs, 1, generator=self.gen).item())
        return names[i]

    def _long_draw_prob(self, name: str, st, floor: int, token_frac: float) -> float:
        """Per-draw probability of taking a long document so that `token_frac` of
        this source's TOKENS come from documents >= floor.

        long_doc_target_frac is a share of tokens (config.py; the stage-2 plan was
        "20B of 40B tokens from docs >= 4096"). It used to be applied as a per-draw
        probability, and since long documents average ~13x the mean length, 0.5 of
        draws was ~94% of stage-2 tokens. With q the long-pool probability, a
        uniform draw otherwise, L the mean length, Ll the long-pool mean and
        Eu = E[len * 1(len >= floor)] under a uniform draw:
            tokens_long / tokens = (q Ll + (1-q) Eu) / (q Ll + (1-q) L) = f
            =>  q = (f L - Eu) / ((1-f) Ll - Eu + f L),  clamped to [0, 1].
        """
        key = (name, floor, token_frac)
        cache = self.__dict__.setdefault("_long_prob_cache", {})
        if key not in cache:
            lens = st.lengths.astype(np.float64)
            L = lens.mean() if lens.size else 0.0
            longm = lens >= floor
            Ll = lens[longm].mean() if longm.any() else 0.0
            Eu = (lens * longm).mean() if lens.size else 0.0
            den = (1.0 - token_frac) * Ll - Eu + token_frac * L
            q = (token_frac * L - Eu) / den if den > 0 else 0.0
            cache[key] = float(min(max(q, 0.0), 1.0))
        return cache[key]

    def _next_doc(self, weights, floor: int, long_frac: float):
        name = self._pick_source(weights)
        st = self.stores[name]
        q = self._long_draw_prob(name, st, floor, long_frac) if floor > 0 else 0.0
        want_long = floor > 0 and float(torch.rand(1, generator=self.gen)) < q
        pool = st.long_docs(floor) if want_long else None
        if pool is not None and pool.size:
            j = int(torch.randint(pool.size, (1,), generator=self.gen))
            i = int(pool[j])
        else:
            i = int(torch.randint(st.n_docs, (1,), generator=self.gen))
        self.docs_seen += 1
        return name, _digit_split(np.asarray(st.doc(i), dtype=np.int64))

    # -- packing -------------------------------------------------------- #
    def batch(self, micro_batch: int, block: int, progress: float = 0.0):
        weights = (mixture_at(progress, self.cfg) if self.split == "train"
                   else {n: 1.0 for n in self.stores})
        floor, long_frac = (long_doc_pressure(progress, self.cfg)
                            if self.split == "train" else (0, 0.0))
        B, T = micro_batch, block
        xs = np.zeros((B, T), dtype=np.int64)
        ys = np.full((B, T), IGNORE, dtype=np.int64)
        pos = np.zeros((B, T), dtype=np.int64)
        bnd = np.zeros((B, T), dtype=bool)
        bounds: list[int] = []
        used: dict[str, int] = {}

        for b in range(B):
            p = 0
            while p < T:
                if self._carry is not None and self._carry.size:
                    # Positions restart on a carried chunk. The chunk opens a
                    # new attention segment (its own bounds entry, its own bnd
                    # marker, Canon and the n-gram table both masked across it),
                    # so it cannot attend to the part of the document already
                    # emitted -- continuing the count described a context the
                    # chunk does not have, and RoPE is relative, so restarting
                    # is identical for every pair it can actually see. It also
                    # keeps pos inside the RoPE table: max_doc_tokens is 65,536
                    # against a table of max_block_size, and a carried document
                    # ran pos to 33,789 and indexed off the end of it.
                    name, d, start = self._carry_name, self._carry, 0
                    self._carry = None
                else:
                    name, d = self._next_doc(weights, floor, long_frac)
                    start = 0
                    # Long documents are emitted over as many row segments as
                    # necessary. The old slice discarded every token after
                    # max_doc_tokens despite the configuration promising that
                    # documents were split, making 882.5M stored corpus tokens
                    # unreachable.
                take = min(d.size, T - p)
                bounds.append(b * T + p)
                bnd[b, p] = True
                xs[b, p:p + take] = d[:take]
                pos[b, p:p + take] = np.arange(start, start + take)
                # y = next token inside this document; the final slot of a
                # document predicts nothing and stays IGNORE.
                if take > 1:
                    ys[b, p:p + take - 1] = d[1:take]
                if take < d.size:
                    # The document continues in the next row: carry the tail,
                    # and the last kept token does have a next token.
                    ys[b, p + take - 1] = d[take]
                    self._carry = d[take:]
                    self._carry_name = name
                    self._carry_pos = 0
                used[name] = used.get(name, 0) + take
                p += take

        for k, v in used.items():
            self.source_tokens[k] = self.source_tokens.get(k, 0) + v
        bounds.append(B * T)
        bounds = np.asarray(bounds, dtype=np.int64)
        # sum(len^2) is the exact attention cost of this batch. Computed here,
        # where the lengths are already known, so the MFU line reports what
        # attention actually did rather than the T^2 a dense mask would have.
        lens = np.diff(bounds)
        stats = {"used": used, "sumsq": int((lens * lens).sum()),
                 "tokens": B * T, "docs": int(lens.size),
                 # Carried so the nested-tensor SDPA path never has to derive
                 # them from the offsets with a host sync.
                 "min_seqlen": int(lens.min()), "max_seqlen": int(lens.max())}
        return (xs, ys, pos, bnd, bounds, stats)

    # -- resumability ---------------------------------------------------- #
    def state_dict(self) -> dict:
        return {"gen": self.gen.get_state(),
                "carry": None if self._carry is None else self._carry.copy(),
                "carry_name": self._carry_name,
                "carry_pos": self._carry_pos,
                "docs_seen": self.docs_seen,
                "source_tokens": dict(self.source_tokens)}

    def load_state_dict(self, st: dict) -> None:
        if st is None:
            return
        # .cpu(): checkpoints are loaded with map_location=<cuda device>, which
        # moves this RNG state onto the GPU too, and a CPU Generator's set_state
        # demands a CPU ByteTensor ("RNG state must be a torch.ByteTensor").
        # Resume had never actually been run, so this went unnoticed until the
        # first real one.
        g = st["gen"]
        self.gen.set_state(g.cpu().to(torch.uint8) if torch.is_tensor(g)
                           else torch.as_tensor(g, dtype=torch.uint8))
        c = st.get("carry")
        name = st.get("carry_name")
        # A checkpoint carries the half-consumed document it was mid-way
        # through, tagged with its source name. Resuming that checkpoint onto a
        # DIFFERENT corpus -- an anneal probe, or 0907 weights on ClimbMix --
        # leaves that name pointing at a source that no longer exists, and
        # every later _pick_source lookup would KeyError. Drop the carry: one
        # truncated document at the resume point costs nothing.
        if name is not None and name not in self.stores:
            c, name = None, None
        self._carry = None if c is None else np.asarray(c)
        self._carry_name = name
        self._carry_pos = 0 if name is None else int(st.get("carry_pos", 0))
        self.docs_seen = int(st.get("docs_seen", 0))
        self.source_tokens.update(st.get("source_tokens", {}))


# --------------------------------------------------------------------------- #
# Prefetch
# --------------------------------------------------------------------------- #
class Prefetcher:
    """One producer thread, pinned staging buffers, non-blocking H2D.

    A single producer, not a pool: the packer's RNG stream is the run's
    reproducibility guarantee, and several workers drawing from it would make
    the token order depend on thread scheduling. Packing is numpy slicing over a
    memmap and costs far less than a step, so one thread stays well ahead.
    """

    def __init__(self, packer: VarlenPacker, cfg: Config, device: str):
        self.packer = packer
        self.cfg = cfg
        self.device = device
        self.q: queue.Queue = queue.Queue(maxsize=cfg.data.prefetch_depth)
        self._shape = None                      # (micro_batch, block, progress)
        self._stop = threading.Event()
        self._t = threading.Thread(target=self._run, daemon=True)
        self._lock = threading.Lock()
        # The producer advances packer ahead of training. Checkpoints must store
        # the state after the last CONSUMED batch, not its speculative state.
        self._committed_state = packer.state_dict()
        self._generation = 0
        self._error = None

    def configure(self, micro_batch: int, block: int, progress: float):
        with self._lock:
            if (self._shape is not None and
                    self._shape[:2] != (micro_batch, block)):
                # Rewind speculative producer work before a shape transition.
                # Otherwise every queued 2048-token batch discarded at the 8192
                # stage silently disappears from the reproducible token stream.
                try:
                    while True:
                        self.q.get_nowait()
                except queue.Empty:
                    pass
                self.packer.load_state_dict(self._committed_state)
                self._generation += 1
            self._shape = (micro_batch, block, progress)

    def start(self):
        self._t.start()
        return self

    def stop(self):
        self._stop.set()
        try:
            while True:
                self.q.get_nowait()
        except queue.Empty:
            pass
        self._t.join(timeout=5)

    def state_dict(self):
        with self._lock:
            return self._committed_state

    def _run(self):
        try:
            self._produce()
        except BaseException as exc:
            self._error = exc
            self._stop.set()

    def _produce(self):
        while not self._stop.is_set():
            with self._lock:
                shape = self._shape
                generation = self._generation
                if shape is not None:
                    mb, block, progress = shape
                    out = self.packer.batch(mb, block, progress)
                    state = self.packer.state_dict()
            if shape is None:
                self._stop.wait(0.01)
                continue
            xs, ys, pos, bnd, bounds, stats = out
            item = (torch.from_numpy(xs).pin_memory(),
                    torch.from_numpy(ys).pin_memory(),
                    torch.from_numpy(pos).pin_memory(),
                    torch.from_numpy(bnd).pin_memory(),
                    torch.from_numpy(bounds).pin_memory(),
                    stats, (mb, block), state, generation)
            while not self._stop.is_set():
                try:
                    self.q.put(item, timeout=0.25)
                    break
                except queue.Full:
                    continue

    def next(self, micro_batch: int, block: int):
        """Blocks only if the producer has fallen behind; drops stale shapes
        after a context stage change."""
        while True:
            if self._error is not None:
                raise RuntimeError("data prefetch failed") from self._error
            try:
                xs, ys, pos, bnd, bounds, stats, shape, state, generation = self.q.get(timeout=1)
            except queue.Empty:
                if self._stop.is_set():
                    raise RuntimeError("data prefetch stopped") from self._error
                continue
            if shape == (micro_batch, block) and generation == self._generation:
                with self._lock:
                    self._committed_state = state
                break
        d = self.device
        return (xs.to(d, non_blocking=True), ys.to(d, non_blocking=True),
                pos.to(d, non_blocking=True), bnd.to(d, non_blocking=True),
                bounds.to(d, non_blocking=True), stats)


def open_sources(cfg: Config, log=print) -> dict[str, SourceStore]:
    stores = {}
    missing = []
    for name in cfg.data.sources:
        root = Path(cfg.data.root)
        # A shard that has not written anything yet has an empty tokens.bin,
        # which cannot be mmapped. Skip it: it contributes nothing until it
        # does, and it will be picked up next time the run starts.
        dirs = [d for d in sorted(root.glob(f"{name}"))
                + sorted(root.glob(f"{name}_s*"))
                if (d / "tokens.bin").exists() and (d / "docs.idx").exists()
                and (d / "tokens.bin").stat().st_size > 0
                and (d / "docs.idx").stat().st_size > 8]
        if not dirs:
            missing.append(name)
            continue
        for d in dirs:
            validate_tokenization(d, cfg)
        parts = [SourceStore(name, d / "tokens.bin", d / "docs.idx") for d in dirs]
        stores[name] = parts[0] if len(parts) == 1 else MultiStore(name, parts)
        if len(parts) > 1:
            log(f"    {name}: {len(parts)} shards stitched "
                f"({', '.join(d.name for d in dirs)})")
    if not stores:
        raise SystemExit(
            f"no prepared sources under {cfg.data.root}. Run prepare.py first.")
    if missing:
        log(f"  WARNING: sources not prepared and excluded from the mixture: "
            f"{', '.join(missing)}")
    tot = sum(s.n_tokens for s in stores.values())
    log(f"  {len(stores)} sources, {tot/1e9:.1f}B tokens, "
        f"{sum(s.n_docs for s in stores.values())/1e6:.1f}M documents")
    for n, s in sorted(stores.items()):
        log(f"    {n:<16}{s.n_tokens/1e9:>8.2f}B  {s.n_docs/1e6:>7.2f}M docs  "
            f"mean len {s.n_tokens/max(s.n_docs,1):>6.0f}")
    return stores