summaryrefslogtreecommitdiff
path: root/prepare.py
diff options
context:
space:
mode:
Diffstat (limited to 'prepare.py')
-rw-r--r--prepare.py645
1 files changed, 645 insertions, 0 deletions
diff --git a/prepare.py b/prepare.py
new file mode 100644
index 0000000..d4f0188
--- /dev/null
+++ b/prepare.py
@@ -0,0 +1,645 @@
+"""Corpus preparation: stream, decontaminate, tokenize, shard.
+
+ python prepare.py plan # token targets per source
+ python prepare.py sample --mb 2000 # text sample for the tokenizer
+ python prepare.py source --name ultra_fineweb --tokens 120e9
+ python prepare.py val --tokens 20e6
+ python prepare.py status
+
+Sources are streamed from the Hub and written straight out as uint16 tokens.
+Nothing keeps the raw text: 200B tokens is ~400GB stored, and the text it came
+from is more than twice that -- there is 664GB free on the Media SSD and 20GB on
+/home, so the token files live on Media and the text never lands at all.
+
+Each source becomes data/<name>/tokens.bin (uint16) plus docs.idx (uint64
+document start offsets). Both are appended to incrementally and a progress file
+records how far the stream got, so preparation is resumable and training can
+start on the first sources while the rest are still arriving.
+
+Decontamination runs inline: any document sharing a 13-gram with an evaluation
+item is dropped before it is ever written. Filtering afterwards would mean
+rewriting a 200GB file.
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import re
+import os
+import sys
+import time
+from pathlib import Path
+
+import numpy as np
+
+from config import DATA_DIR, Config
+from tokenizer import BorisTokenizer
+
+os.environ.setdefault("TOKENIZERS_PARALLELISM", "true")
+
+# name -> (repo, configs, split, text field). Resolved against the Hub on
+# 2026-09-06 with probe/sources_probe.py, not from memory -- Ultra-FineWeb
+# splits by language rather than "train", and finephrase has five configs, so
+# both of those failed when they were guessed. Run `prepare.py check` after any
+# edit here; it opens every source and pulls one record.
+#
+# `configs` may be a tuple: the source is consumed in that order until its token
+# target is met. Cosmopedia needs this -- no single config holds enough.
+SOURCES = {
+ "ultra_fineweb": ("openbmb/Ultra-FineWeb", ("default",), "en", "content"),
+ "cosmopedia": ("HuggingFaceTB/cosmopedia",
+ ("web_samples_v2", "web_samples_v1", "auto_math_text",
+ "stories", "openstax", "stanford", "wikihow", "khanacademy"),
+ "train", "text"),
+ "finephrase": ("HuggingFaceFW/finephrase", ("all",), "train", "text"),
+ "fineweb_edu": ("HuggingFaceFW/fineweb-edu", ("sample-100BT",), "train", "text"),
+ "dclm": ("mlfoundations/dclm-baseline-1.0-parquet", ("default",),
+ "train", "text"),
+ # L3 of openbmb's L0-L4 tiering: Ultra-FineWeb (the L2 selected layer) put
+ # through Q&A generation and multi-style rewriting with MiniCPM4/Qwen3.
+ # Their own decay-phase data for MiniCPM5-1B, which is exactly what we use
+ # it for -- it carries the decay weight that would otherwise go to L2
+ # general web text.
+ # Math. The eval suite includes ArithMark-3 (arithmetic reasoning) and the
+ # rest of this corpus has no dedicated mathematical content -- an omission,
+ # not a choice. finemath-4plus is the high-quality tier: ~39GB of text
+ # filtered from CommonCrawl by a classifier trained on Llama-3.1-70B
+ # annotations, roughly 8B tokens under our BPE.
+ # 4plus first (the high-quality tier, ~8B tokens under our BPE), then
+ # 3plus as the fallback -- the curriculum draws 10.7B, more than 4plus holds.
+ "finemath": ("HuggingFaceTB/finemath",
+ ("finemath-4plus", "finemath-3plus"), "train", "text"),
+ # 2026-10-10: elementary math facts. Preview-1 sat at chance on ArithMark
+ # division/patterns/multiplication where GPT-X3 (10% UltraData-Math L3 for
+ # its whole run) scores 64-92%. L3 is rewritten math in four styles; one
+ # worker per config so the four stay balanced.
+ "ultra_math_l3": ("openbmb/UltraData-Math",
+ ("UltraData-Math-L3-QA-Synthetic",
+ "UltraData-Math-L3-Textbook-Exercise-Synthetic",
+ "UltraData-Math-L3-Multi-Style-Synthetic",
+ "UltraData-Math-L3-Conversation-Synthetic"),
+ "train", "content"),
+ "ultra_fineweb_l3": ("openbmb/Ultra-FineWeb-L3",
+ ("Ultra-FineWeb-L3-en-QA-Synthetic",
+ "Ultra-FineWeb-L3-en-Multi-Style-Synthetic"),
+ "train", "content"),
+}
+
+NGRAM_N = 13
+BAN_CACHE = DATA_DIR / "decontam_ngrams.npy"
+
+
+# --------------------------------------------------------------------------- #
+_NON_ALNUM = re.compile(r"[^0-9a-z]+")
+
+
+def _norm(s: str) -> list[str]:
+ """Lowercase, split on anything non-alphanumeric.
+
+ A per-character Python generator here ran once over every document in the
+ corpus -- hundreds of billions of interpreted operations. The regex does the
+ same work in C.
+ """
+ return _NON_ALNUM.sub(" ", s.lower()).split()
+
+
+def _h(s: str) -> int:
+ """Stable 64-bit hash.
+
+ Not builtin hash(): PYTHONHASHSEED randomises string hashing per process, so
+ a set built in one process is meaningless in another -- and the ban set is
+ built once and shared by five. 64 bits keeps false positives negligible
+ against ~900k banned n-grams; a 32-bit hash would falsely drop ~0.8% of all
+ documents.
+ """
+ return int.from_bytes(hashlib.blake2b(s.encode("utf-8", "ignore"),
+ digest_size=8).digest(), "big",
+ signed=True)
+
+
+def eval_ngrams(log=print, use_cache: bool = True):
+ """Sorted array of banned 13-gram hashes, over every evaluation item.
+
+ Built once and cached: five prepare processes run in parallel and each was
+ rebuilding this from scratch.
+ """
+ import numpy as _np
+ if use_cache and BAN_CACHE.exists():
+ a = _np.load(BAN_CACHE)
+ log(f" decontamination: {a.size:,} banned n-grams (cached)")
+ return a
+
+ import eval as E
+ out = set()
+ missing = []
+ for name, loader in E.TASKS.items():
+ try:
+ items = loader()
+ except Exception as e:
+ missing.append(f"{name} ({type(e).__name__}: {str(e)[:60]})")
+ continue
+ n0 = len(out)
+ for it in items:
+ for text in [it["ctx"]] + list(it["choices"]):
+ w = _norm(text)
+ for i in range(max(0, len(w) - NGRAM_N + 1)):
+ out.add(_h(" ".join(w[i:i + NGRAM_N])))
+ log(f" {name}: {len(items)} items, +{len(out)-n0:,} n-grams")
+ if missing:
+ # Never a warning. A benchmark that silently skips decontamination
+ # produces a score that looks like a result and is not one.
+ raise SystemExit(
+ " decontamination could not load: " + "; ".join(missing) +
+ "\n Refusing to write training data that has not been checked "
+ "against every evaluation set. Fix the loader in eval.py TASKS, or "
+ "pass --no-decontam if you accept scoring on contaminated data.")
+ a = _np.sort(_np.fromiter(out, dtype=_np.int64, count=len(out)))
+ BAN_CACHE.parent.mkdir(parents=True, exist_ok=True)
+ _np.save(BAN_CACHE, a)
+ log(f" decontamination: {a.size:,} banned n-grams -> {BAN_CACHE.name}")
+ return a
+
+
+def contaminated(text: str, ban) -> bool:
+ """True if the document shares a 13-gram window with an evaluation item.
+
+ The document side steps by NGRAM_N rather than by 1. The evaluation side
+ holds every overlapping window, so any shared span of 2*NGRAM_N-1 = 25 words
+ or more still contains one aligned document window and is always caught --
+ at a thirteenth of the work. Scanning every overlapping window here was
+ ~160 billion hashes over the full corpus, which does not finish.
+ """
+ w = _norm(text)
+ if len(w) < NGRAM_N or ban.size == 0:
+ return False
+ keys = np.fromiter(
+ (_h(" ".join(w[i:i + NGRAM_N]))
+ for i in range(0, len(w) - NGRAM_N + 1, NGRAM_N)),
+ dtype=np.int64)
+ if keys.size == 0:
+ return False
+ i = np.searchsorted(ban, keys)
+ i[i >= ban.size] = ban.size - 1
+ return bool((ban[i] == keys).any())
+
+
+# --------------------------------------------------------------------------- #
+class ShardWriter:
+ """Appends uint16 tokens and uint64 document offsets, crash-safely.
+
+ The offsets file is the authority: on resume the token file is truncated
+ back to the last complete document, so a kill mid-write cannot leave a torn
+ final document that the packer would read as real text.
+ """
+
+ def __init__(self, d: Path, eos_id: int):
+ self.d = Path(d); self.d.mkdir(parents=True, exist_ok=True)
+ self.bin = self.d / "tokens.bin"
+ self.idx = self.d / "docs.idx"
+ self.eos = eos_id
+ if not self.idx.exists():
+ np.array([0], dtype=np.uint64).tofile(self.idx)
+ off = np.fromfile(self.idx, dtype=np.uint64)
+ self.n_tokens = int(off[-1])
+ self.n_docs = off.size - 1
+ if self.bin.exists() and self.bin.stat().st_size != self.n_tokens * 2:
+ with open(self.bin, "r+b") as f:
+ f.truncate(self.n_tokens * 2)
+ self.fb = open(self.bin, "ab")
+ self.fi = open(self.idx, "ab")
+
+ def write(self, docs: list[list[int]]):
+ offs = []
+ buf = []
+ n = self.n_tokens
+ for ids in docs:
+ if not ids:
+ continue
+ ids = ids + [self.eos]
+ buf.append(np.asarray(ids, dtype=np.uint16))
+ n += len(ids)
+ offs.append(n)
+ if not offs:
+ return 0
+ np.concatenate(buf).tofile(self.fb)
+ np.asarray(offs, dtype=np.uint64).tofile(self.fi)
+ added = n - self.n_tokens
+ self.n_tokens = n
+ self.n_docs += len(offs)
+ return added
+
+ def flush(self):
+ self.fb.flush(); os.fsync(self.fb.fileno())
+ self.fi.flush(); os.fsync(self.fi.fileno())
+
+ def close(self):
+ self.flush(); self.fb.close(); self.fi.close()
+
+
+def progress_path(name: str) -> Path:
+ return DATA_DIR / name / "progress.json"
+
+
+def load_progress(name: str) -> dict:
+ p = progress_path(name)
+ return json.loads(p.read_text()) if p.exists() else {"seen": 0, "kept": 0,
+ "dropped": 0, "tokens": 0}
+
+
+def save_progress(name: str, d: dict):
+ p = progress_path(name)
+ p.parent.mkdir(parents=True, exist_ok=True)
+ p.write_text(json.dumps(d, indent=2))
+
+
+# --------------------------------------------------------------------------- #
+def resolve_files(repo: str, conf: str, split: str) -> list[str]:
+ """The concrete data files behind one config/split, in order.
+
+ Streaming a whole config and skipping consumed records on resume re-fetches
+ and re-decodes everything already done -- the cost of resuming grows with
+ how far you got, which is the wrong shape for a job that runs for a day.
+ Iterating file by file makes resume O(1): finished files are never touched
+ again.
+ """
+ from datasets import load_dataset_builder
+ b = load_dataset_builder(repo, conf)
+ df = b.config.data_files
+ files = df[split] if split in df else list(df.values())[0]
+ return [str(f) for f in files]
+
+
+def _stream_file(path: str, field: str):
+ """One data file, one column."""
+ from datasets import load_dataset
+ fmt = "json" if ".json" in path else "parquet"
+ ds = load_dataset(fmt, data_files=[path], split="train", streaming=True)
+ try:
+ ds = ds.select_columns([field]) # decode one column, not twelve
+ except Exception:
+ pass
+ return ds
+
+
+def prepare_source(name: str, target_tokens: int, cfg: Config, decontam=True,
+ batch=1000, log=print, file_start: int = 0,
+ file_end: int | None = None, out: str | None = None,
+ configs: tuple | None = None):
+ """Tokenize one source into data/<out or name>/.
+
+ file_start/file_end bound which data files this worker owns. A source whose
+ files are large enough is otherwise one serial process and sets the finish
+ time for the whole job: Ultra-FineWeb needs 106B tokens and was projected at
+ 48h alone while the link and CPU both had headroom. Disjoint file ranges let
+ several workers share it, and because resume is already file-indexed the
+ bookkeeping is just a bound on the loop.
+
+ data.py stitches data/<name>/ and data/<name>_s*/ back into one logical
+ source, so the shards are never copied or merged.
+ """
+ if name not in SOURCES:
+ raise SystemExit(f"unknown source '{name}'. Known: {', '.join(SOURCES)}")
+ repo, confs, split, field = SOURCES[name]
+ if configs:
+ confs = tuple(configs)
+ tok = BorisTokenizer.load(DATA_DIR / "tokenizer")
+ ban = eval_ngrams(log) if decontam else np.zeros(0, dtype=np.int64)
+ out = out or name
+ w = ShardWriter(DATA_DIR / out, tok.eos_id)
+ prog = load_progress(out)
+ # tokens.bin is appended immediately; progress.json is only checkpointed at
+ # file boundaries and every 200M tokens. A worker killed mid-file therefore
+ # has more on disk than its progress claims, and resuming at the recorded
+ # file would write those documents twice. Trust the writer, and skip the
+ # partially-consumed file -- losing the rest of one file is free, duplicate
+ # documents in the corpus are not.
+ if prog.get("tokens", 0) != w.n_tokens:
+ log(f" progress lagged the writer ({prog.get('tokens',0)/1e9:.3f}B "
+ f"recorded vs {w.n_tokens/1e9:.3f}B on disk); resuming past the "
+ f"partial file")
+ prog["tokens"] = w.n_tokens
+ if w.n_tokens:
+ prog["file_idx"] = prog.get("file_idx", file_start) + 1
+ log(f" {name}: {repo} {list(confs)}")
+ log(f" already {prog['tokens']/1e9:.2f}B tokens in {w.n_docs:,} documents")
+ if w.n_tokens >= target_tokens:
+ log(" target already met"); return
+
+ t0 = time.time()
+ texts, n_since = [], 0
+ try:
+ while prog.get("cfg_idx", 0) < len(confs) and prog["tokens"] < target_tokens:
+ conf = confs[prog.get("cfg_idx", 0)]
+ files = resolve_files(repo, conf, split)
+ hi = len(files) if file_end is None else min(file_end, len(files))
+ if prog.get("file_idx", 0) < file_start:
+ prog["file_idx"] = file_start
+ log(f" config {conf}: {len(files)} files, this worker owns "
+ f"[{file_start}, {hi}), resuming at file {prog.get('file_idx', 0)}")
+ while (prog.get("file_idx", 0) < hi
+ and prog["tokens"] < target_tokens):
+ fi = prog.get("file_idx", 0)
+ # A transient network error must not be fatal. Five of eight workers
+ # died overnight on a single httpx "client has been closed" -- one
+ # WiFi blip -- and sat idle for hours with the job half done. Retry
+ # the file; on persistent failure skip it, because every source here
+ # has far more files than the token budget needs.
+ attempt, rec_iter = 0, None
+ while attempt < 5:
+ try:
+ rec_iter = _stream_file(files[fi], field)
+ break
+ except Exception as e:
+ attempt += 1
+ log(f" file {fi}: open failed ({type(e).__name__}), "
+ f"retry {attempt}/5 in {2**attempt}s")
+ time.sleep(2 ** attempt)
+ if rec_iter is None:
+ log(f" file {fi}: unreadable after 5 attempts, skipping")
+ prog["file_idx"] = fi + 1
+ save_progress(out, prog)
+ continue
+ try:
+ for rec in rec_iter:
+ t = rec.get(field) or ""
+ if not t:
+ continue
+ if ban.size and contaminated(t, ban):
+ prog["dropped"] += 1
+ continue
+ texts.append(t)
+ if len(texts) >= batch:
+ added = w.write(tok.encode_batch(texts))
+ prog["kept"] += len(texts); prog["tokens"] += added
+ texts = []; n_since += added
+ if n_since > 200_000_000:
+ w.flush(); save_progress(out, prog); n_since = 0
+ r = prog["tokens"] / max(time.time() - t0, 1e-9)
+ log(f" {prog['tokens']/1e9:7.2f}B / {target_tokens/1e9:.0f}B "
+ f"file {fi+1}/{len(files)} {w.n_docs/1e6:6.2f}M docs "
+ f"{r/1e6:5.2f}M tok/s dropped {prog['dropped']:,} "
+ f"eta {(target_tokens-prog['tokens'])/max(r,1)/3600:5.1f}h")
+ if prog["tokens"] >= target_tokens:
+ break
+ except Exception as e:
+ # Mid-file failure: the documents already written are safe and
+ # the reconciliation on the next start skips the partial file.
+ log(f" file {fi}: stream failed mid-way ({type(e).__name__}: "
+ f"{str(e)[:80]}); moving on")
+ # This file is fully consumed; resume will never fetch it again.
+ prog["file_idx"] = fi + 1
+ if texts:
+ added = w.write(tok.encode_batch(texts))
+ prog["kept"] += len(texts); prog["tokens"] += added
+ texts = []; n_since += added
+ w.flush(); save_progress(out, prog)
+ if prog["tokens"] < target_tokens:
+ log(f" config {conf} exhausted at {prog['tokens']/1e9:.2f}B")
+ prog["cfg_idx"] = prog.get("cfg_idx", 0) + 1
+ prog["file_idx"] = file_start
+ save_progress(out, prog)
+ finally:
+ if texts:
+ prog["tokens"] += w.write(tok.encode_batch(texts))
+ prog["kept"] += len(texts)
+ w.close(); save_progress(out, prog)
+ log(f" done: {prog['tokens']/1e9:.2f}B tokens, {w.n_docs:,} documents, "
+ f"{prog['dropped']:,} dropped as contaminated")
+
+
+def prepare_sample(mb: int, cfg: Config, log=print):
+ """A stratified text sample for training the tokenizer."""
+ from datasets import load_dataset
+ out = DATA_DIR / "tokenizer_sample.txt"
+ out.parent.mkdir(parents=True, exist_ok=True)
+ per = mb * 1024 * 1024 // len(SOURCES)
+ with open(out, "w") as f:
+ for name, (repo, confs, split, field) in SOURCES.items():
+ n = 0
+ # Not skipped on failure. The tokenizer is the one artifact that
+ # cannot be changed later without invalidating every token file, the
+ # n-gram vocab and every checkpoint, so training it on four of five
+ # sources because one quietly failed is not an acceptable outcome.
+ ds = load_dataset(repo, confs[0], split=split, streaming=True)
+ for rec in ds:
+ t = rec.get(field) or ""
+ if not t:
+ continue
+ f.write(t.replace("\x00", "") + "\n")
+ n += len(t)
+ if n >= per:
+ break
+ log(f" {name}: {n/1e6:.0f} MB")
+ log(f" wrote {out} ({out.stat().st_size/1e6:.0f} MB)")
+
+
+def _val_start(name: str, confs: tuple, log=print):
+ """The (config, file index) where this source's untouched files begin.
+
+ The old implementation streamed a config from record zero and called next()
+ five million times to get past what training consumed. That re-downloads and
+ re-decodes tens of gigabytes to produce three megabytes of tokens, and with
+ every shard's "seen" counter at 0 it wasn't even reliably disjoint. The
+ download already recorded, per shard, which config and file it stopped at,
+ so the disjoint region is known exactly and reaching it costs nothing.
+ """
+ shards = [name] + sorted(
+ p.parent.name for p in DATA_DIR.glob(f"{name}_s*/progress.json"))
+ progs = [load_progress(s) for s in shards
+ if progress_path(s).exists()]
+ if not progs:
+ return confs[0], 0
+ max_cfg = max(p.get("cfg_idx", 0) for p in progs)
+ if max_cfg + 1 < len(confs):
+ return confs[max_cfg + 1], 0 # a whole config training never opened
+ hi = max(p.get("file_idx", 0) for p in progs
+ if p.get("cfg_idx", 0) == max_cfg)
+ return confs[max_cfg], hi + 2 # +2: skip the partially-read file
+
+
+def prepare_val(tokens: int, cfg: Config, log=print):
+ """A fixed held-out split, drawn from every source, that the training packer
+ never opens -- data.open_sources only ever loads cfg.data.sources.
+
+ Resumable per source: data/val/progress.json records which sources are
+ finished, so an interrupted run does not redo them.
+ """
+ tok = BorisTokenizer.load(DATA_DIR / "tokenizer")
+ w = ShardWriter(DATA_DIR / "val", tok.eos_id)
+ prog = load_progress("val")
+ done = set(prog.get("done", []))
+ per = tokens // len(SOURCES)
+ for name, (repo, confs, split, field) in SOURCES.items():
+ if name in done:
+ log(f" {name}: already in val split"); continue
+ try:
+ conf, fi = _val_start(name, confs, log)
+ files = resolve_files(repo, conf, split)
+ except Exception as e:
+ log(f" {name}: skipped ({type(e).__name__}: {e})"); continue
+ if fi >= len(files):
+ log(f" {name}: no untouched files left in {conf}; skipped")
+ continue
+ got, texts = 0, []
+ t0 = time.time()
+ while got < per and fi < len(files):
+ try:
+ recs = _stream_file(files[fi], field)
+ except Exception as e:
+ log(f" {name} file {fi}: {type(e).__name__}, skipping")
+ fi += 1; continue
+ for rec in recs:
+ t = rec.get(field) or ""
+ if not t:
+ continue
+ texts.append(t)
+ if len(texts) >= 500:
+ got += w.write(tok.encode_batch(texts)); texts = []
+ if got >= per:
+ break
+ fi += 1
+ if texts:
+ got += w.write(tok.encode_batch(texts))
+ log(f" {name}: {got/1e6:.1f}M val tokens from {conf} "
+ f"(files from {fi}) in {time.time()-t0:.0f}s")
+ done.add(name)
+ save_progress("val", {"done": sorted(done), "tokens": w.n_tokens})
+ w.close()
+ log(f" val split: {w.n_tokens/1e6:.1f}M tokens, {w.n_docs:,} documents")
+
+
+def check(cfg: Config, log=print):
+ """Open every source and pull one record. Run this before downloading.
+
+ Every field in SOURCES is a guess until something has actually read from the
+ repo -- two of the five were wrong the first time, and one of them only
+ surfaced after the sample step had already run.
+ """
+ from datasets import load_dataset
+ bad = 0
+ for name, (repo, confs, split, field) in SOURCES.items():
+ for conf in confs:
+ try:
+ ds = load_dataset(repo, conf, split=split, streaming=True)
+ rec = next(iter(ds))
+ t = rec.get(field)
+ if not isinstance(t, str) or not t:
+ raise ValueError(
+ f"field '{field}' is {type(t).__name__}, not text; "
+ f"available: {list(rec.keys())}")
+ log(f" ok {name:<15}{repo} [{conf}] / {split} / {field} "
+ f"first record {len(t):,} chars")
+ except Exception as e:
+ bad += 1
+ log(f" FAIL {name:<15}{repo} [{conf}] / {split} / {field}")
+ log(f" {type(e).__name__}: {str(e)[:160]}")
+ if bad:
+ raise SystemExit(f"\n {bad} source/config pairs unreachable -- fix SOURCES "
+ f"before downloading 400GB against a broken registry")
+ log("\n all sources reachable")
+
+
+def plan(cfg: Config, log=print):
+ """Token target per source: what the curriculum will actually draw."""
+ from schedule import mixture_at
+ total = cfg.train.total_tokens
+ steps = 200
+ draw = {n: 0.0 for n in cfg.data.sources}
+ means = {}
+ for n in cfg.data.sources:
+ dirs = [d for d in sorted(DATA_DIR.glob(n)) +
+ sorted(DATA_DIR.glob(f"{n}_s*")) if (d / "docs.idx").exists()]
+ nt = nd = 0
+ for d in dirs:
+ o = np.memmap(d / "docs.idx", dtype=np.uint64, mode="r")
+ nt += int(o[-1]); nd += o.size - 1
+ means[n] = nt / max(nd, 1)
+ for i in range(steps):
+ w = mixture_at((i + 0.5) / steps, cfg)
+ # The packer samples sources per document, so turn document draw
+ # probabilities into expected token shares before reporting capacity.
+ z = sum(w[n] * means[n] for n in w) or 1.0
+ for k, v in w.items():
+ draw[k] += (v * means[k] / z) * total / steps
+ log(f" {cfg.train.total_tokens/1e9:.0f}B token budget, "
+ f"{cfg.total_steps:,} steps\n")
+ log(f" {'source':<16}{'draws':>10}{'unique needed':>16}{'on disk':>12}")
+ log(" " + "-" * 56)
+ for n in cfg.data.sources:
+ have = 0
+ p = DATA_DIR / n / "docs.idx"
+ if p.exists():
+ have = int(np.fromfile(p, dtype=np.uint64)[-1])
+ log(f" {n:<16}{draw[n]/1e9:>9.1f}B{draw[n]/1e9:>15.1f}B"
+ f"{have/1e9:>11.2f}B")
+ log(" " + "-" * 56)
+ log(f" {'TOTAL':<16}{sum(draw.values())/1e9:>9.1f}B"
+ f"{sum(draw.values())/1e9:>15.1f}B")
+ log(f"\n stored as uint16: {sum(draw.values())*2/1e12:.2f} TB at one epoch.")
+ log(f" Repeating a smaller unique corpus is the intended fallback -- set a")
+ log(f" lower --tokens per source and the packer will resample it.")
+
+
+def status(cfg: Config, log=print):
+ log(f" {'source':<16}{'tokens':>12}{'docs':>12}{'dropped':>10}{'GB':>8}")
+ log(" " + "-" * 58)
+ for n in list(cfg.data.sources) + ["val"]:
+ dirs = [d for d in sorted(DATA_DIR.glob(n)) + sorted(DATA_DIR.glob(f"{n}_s*"))
+ if (d / "docs.idx").exists()]
+ if not dirs:
+ log(f" {n:<16}{'-':>12}"); continue
+ tot = docs = dropped = 0
+ for d in dirs:
+ off = np.fromfile(d / "docs.idx", dtype=np.uint64)
+ tot += int(off[-1]); docs += off.size - 1
+ dropped += load_progress(d.name).get("dropped", 0)
+ tag = f"{n} ({len(dirs)}sh)" if len(dirs) > 1 else n
+ log(f" {tag:<16}{tot/1e9:>11.2f}B{docs/1e6:>11.2f}M"
+ f"{dropped:>10,}{tot*2/2**30:>8.1f}")
+
+
+def main():
+ cfg = Config()
+ ap = argparse.ArgumentParser()
+ sub = ap.add_subparsers(dest="cmd", required=True)
+ sub.add_parser("plan"); sub.add_parser("status"); sub.add_parser("check")
+ s = sub.add_parser("sample"); s.add_argument("--mb", type=int, default=2000)
+ s = sub.add_parser("source")
+ s.add_argument("--name", required=True)
+ s.add_argument("--tokens", type=float, required=True)
+ s.add_argument("--no-decontam", action="store_true")
+ s.add_argument("--configs", default=None,
+ help="comma-separated config override, so two workers can "
+ "split a multi-config source between them")
+ s.add_argument("--file-start", type=int, default=0)
+ s.add_argument("--file-end", type=int, default=None)
+ s.add_argument("--out", default=None,
+ help="output directory name; use <name>_sN for a shard")
+ s.add_argument("--batch", type=int, default=250,
+ help="documents per encode_batch call. Each HF Encoding "
+ "carries ids, offsets, attention and word-id arrays, so "
+ "this sets the transient memory peak per worker; 1000 "
+ "across eight workers drove the box 16GB into swap.")
+ s = sub.add_parser("val"); s.add_argument("--tokens", type=float, default=2e7)
+ a = ap.parse_args()
+
+ if a.cmd == "check":
+ check(cfg)
+ elif a.cmd == "plan":
+ plan(cfg)
+ elif a.cmd == "status":
+ status(cfg)
+ elif a.cmd == "sample":
+ prepare_sample(a.mb, cfg)
+ elif a.cmd == "source":
+ prepare_source(a.name, int(a.tokens), cfg, decontam=not a.no_decontam,
+ file_start=a.file_start, file_end=a.file_end, out=a.out,
+ batch=a.batch,
+ configs=tuple(a.configs.split(",")) if a.configs else None)
+ elif a.cmd == "val":
+ prepare_val(int(a.tokens), cfg)
+
+
+if __name__ == "__main__":
+ main()