diff options
Diffstat (limited to 'prepare.py')
| -rw-r--r-- | prepare.py | 645 |
1 files changed, 645 insertions, 0 deletions
diff --git a/prepare.py b/prepare.py new file mode 100644 index 0000000..d4f0188 --- /dev/null +++ b/prepare.py @@ -0,0 +1,645 @@ +"""Corpus preparation: stream, decontaminate, tokenize, shard. + + python prepare.py plan # token targets per source + python prepare.py sample --mb 2000 # text sample for the tokenizer + python prepare.py source --name ultra_fineweb --tokens 120e9 + python prepare.py val --tokens 20e6 + python prepare.py status + +Sources are streamed from the Hub and written straight out as uint16 tokens. +Nothing keeps the raw text: 200B tokens is ~400GB stored, and the text it came +from is more than twice that -- there is 664GB free on the Media SSD and 20GB on +/home, so the token files live on Media and the text never lands at all. + +Each source becomes data/<name>/tokens.bin (uint16) plus docs.idx (uint64 +document start offsets). Both are appended to incrementally and a progress file +records how far the stream got, so preparation is resumable and training can +start on the first sources while the rest are still arriving. + +Decontamination runs inline: any document sharing a 13-gram with an evaluation +item is dropped before it is ever written. Filtering afterwards would mean +rewriting a 200GB file. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import os +import sys +import time +from pathlib import Path + +import numpy as np + +from config import DATA_DIR, Config +from tokenizer import BorisTokenizer + +os.environ.setdefault("TOKENIZERS_PARALLELISM", "true") + +# name -> (repo, configs, split, text field). Resolved against the Hub on +# 2026-09-06 with probe/sources_probe.py, not from memory -- Ultra-FineWeb +# splits by language rather than "train", and finephrase has five configs, so +# both of those failed when they were guessed. Run `prepare.py check` after any +# edit here; it opens every source and pulls one record. +# +# `configs` may be a tuple: the source is consumed in that order until its token +# target is met. Cosmopedia needs this -- no single config holds enough. +SOURCES = { + "ultra_fineweb": ("openbmb/Ultra-FineWeb", ("default",), "en", "content"), + "cosmopedia": ("HuggingFaceTB/cosmopedia", + ("web_samples_v2", "web_samples_v1", "auto_math_text", + "stories", "openstax", "stanford", "wikihow", "khanacademy"), + "train", "text"), + "finephrase": ("HuggingFaceFW/finephrase", ("all",), "train", "text"), + "fineweb_edu": ("HuggingFaceFW/fineweb-edu", ("sample-100BT",), "train", "text"), + "dclm": ("mlfoundations/dclm-baseline-1.0-parquet", ("default",), + "train", "text"), + # L3 of openbmb's L0-L4 tiering: Ultra-FineWeb (the L2 selected layer) put + # through Q&A generation and multi-style rewriting with MiniCPM4/Qwen3. + # Their own decay-phase data for MiniCPM5-1B, which is exactly what we use + # it for -- it carries the decay weight that would otherwise go to L2 + # general web text. + # Math. The eval suite includes ArithMark-3 (arithmetic reasoning) and the + # rest of this corpus has no dedicated mathematical content -- an omission, + # not a choice. finemath-4plus is the high-quality tier: ~39GB of text + # filtered from CommonCrawl by a classifier trained on Llama-3.1-70B + # annotations, roughly 8B tokens under our BPE. + # 4plus first (the high-quality tier, ~8B tokens under our BPE), then + # 3plus as the fallback -- the curriculum draws 10.7B, more than 4plus holds. + "finemath": ("HuggingFaceTB/finemath", + ("finemath-4plus", "finemath-3plus"), "train", "text"), + # 2026-10-10: elementary math facts. Preview-1 sat at chance on ArithMark + # division/patterns/multiplication where GPT-X3 (10% UltraData-Math L3 for + # its whole run) scores 64-92%. L3 is rewritten math in four styles; one + # worker per config so the four stay balanced. + "ultra_math_l3": ("openbmb/UltraData-Math", + ("UltraData-Math-L3-QA-Synthetic", + "UltraData-Math-L3-Textbook-Exercise-Synthetic", + "UltraData-Math-L3-Multi-Style-Synthetic", + "UltraData-Math-L3-Conversation-Synthetic"), + "train", "content"), + "ultra_fineweb_l3": ("openbmb/Ultra-FineWeb-L3", + ("Ultra-FineWeb-L3-en-QA-Synthetic", + "Ultra-FineWeb-L3-en-Multi-Style-Synthetic"), + "train", "content"), +} + +NGRAM_N = 13 +BAN_CACHE = DATA_DIR / "decontam_ngrams.npy" + + +# --------------------------------------------------------------------------- # +_NON_ALNUM = re.compile(r"[^0-9a-z]+") + + +def _norm(s: str) -> list[str]: + """Lowercase, split on anything non-alphanumeric. + + A per-character Python generator here ran once over every document in the + corpus -- hundreds of billions of interpreted operations. The regex does the + same work in C. + """ + return _NON_ALNUM.sub(" ", s.lower()).split() + + +def _h(s: str) -> int: + """Stable 64-bit hash. + + Not builtin hash(): PYTHONHASHSEED randomises string hashing per process, so + a set built in one process is meaningless in another -- and the ban set is + built once and shared by five. 64 bits keeps false positives negligible + against ~900k banned n-grams; a 32-bit hash would falsely drop ~0.8% of all + documents. + """ + return int.from_bytes(hashlib.blake2b(s.encode("utf-8", "ignore"), + digest_size=8).digest(), "big", + signed=True) + + +def eval_ngrams(log=print, use_cache: bool = True): + """Sorted array of banned 13-gram hashes, over every evaluation item. + + Built once and cached: five prepare processes run in parallel and each was + rebuilding this from scratch. + """ + import numpy as _np + if use_cache and BAN_CACHE.exists(): + a = _np.load(BAN_CACHE) + log(f" decontamination: {a.size:,} banned n-grams (cached)") + return a + + import eval as E + out = set() + missing = [] + for name, loader in E.TASKS.items(): + try: + items = loader() + except Exception as e: + missing.append(f"{name} ({type(e).__name__}: {str(e)[:60]})") + continue + n0 = len(out) + for it in items: + for text in [it["ctx"]] + list(it["choices"]): + w = _norm(text) + for i in range(max(0, len(w) - NGRAM_N + 1)): + out.add(_h(" ".join(w[i:i + NGRAM_N]))) + log(f" {name}: {len(items)} items, +{len(out)-n0:,} n-grams") + if missing: + # Never a warning. A benchmark that silently skips decontamination + # produces a score that looks like a result and is not one. + raise SystemExit( + " decontamination could not load: " + "; ".join(missing) + + "\n Refusing to write training data that has not been checked " + "against every evaluation set. Fix the loader in eval.py TASKS, or " + "pass --no-decontam if you accept scoring on contaminated data.") + a = _np.sort(_np.fromiter(out, dtype=_np.int64, count=len(out))) + BAN_CACHE.parent.mkdir(parents=True, exist_ok=True) + _np.save(BAN_CACHE, a) + log(f" decontamination: {a.size:,} banned n-grams -> {BAN_CACHE.name}") + return a + + +def contaminated(text: str, ban) -> bool: + """True if the document shares a 13-gram window with an evaluation item. + + The document side steps by NGRAM_N rather than by 1. The evaluation side + holds every overlapping window, so any shared span of 2*NGRAM_N-1 = 25 words + or more still contains one aligned document window and is always caught -- + at a thirteenth of the work. Scanning every overlapping window here was + ~160 billion hashes over the full corpus, which does not finish. + """ + w = _norm(text) + if len(w) < NGRAM_N or ban.size == 0: + return False + keys = np.fromiter( + (_h(" ".join(w[i:i + NGRAM_N])) + for i in range(0, len(w) - NGRAM_N + 1, NGRAM_N)), + dtype=np.int64) + if keys.size == 0: + return False + i = np.searchsorted(ban, keys) + i[i >= ban.size] = ban.size - 1 + return bool((ban[i] == keys).any()) + + +# --------------------------------------------------------------------------- # +class ShardWriter: + """Appends uint16 tokens and uint64 document offsets, crash-safely. + + The offsets file is the authority: on resume the token file is truncated + back to the last complete document, so a kill mid-write cannot leave a torn + final document that the packer would read as real text. + """ + + def __init__(self, d: Path, eos_id: int): + self.d = Path(d); self.d.mkdir(parents=True, exist_ok=True) + self.bin = self.d / "tokens.bin" + self.idx = self.d / "docs.idx" + self.eos = eos_id + if not self.idx.exists(): + np.array([0], dtype=np.uint64).tofile(self.idx) + off = np.fromfile(self.idx, dtype=np.uint64) + self.n_tokens = int(off[-1]) + self.n_docs = off.size - 1 + if self.bin.exists() and self.bin.stat().st_size != self.n_tokens * 2: + with open(self.bin, "r+b") as f: + f.truncate(self.n_tokens * 2) + self.fb = open(self.bin, "ab") + self.fi = open(self.idx, "ab") + + def write(self, docs: list[list[int]]): + offs = [] + buf = [] + n = self.n_tokens + for ids in docs: + if not ids: + continue + ids = ids + [self.eos] + buf.append(np.asarray(ids, dtype=np.uint16)) + n += len(ids) + offs.append(n) + if not offs: + return 0 + np.concatenate(buf).tofile(self.fb) + np.asarray(offs, dtype=np.uint64).tofile(self.fi) + added = n - self.n_tokens + self.n_tokens = n + self.n_docs += len(offs) + return added + + def flush(self): + self.fb.flush(); os.fsync(self.fb.fileno()) + self.fi.flush(); os.fsync(self.fi.fileno()) + + def close(self): + self.flush(); self.fb.close(); self.fi.close() + + +def progress_path(name: str) -> Path: + return DATA_DIR / name / "progress.json" + + +def load_progress(name: str) -> dict: + p = progress_path(name) + return json.loads(p.read_text()) if p.exists() else {"seen": 0, "kept": 0, + "dropped": 0, "tokens": 0} + + +def save_progress(name: str, d: dict): + p = progress_path(name) + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(json.dumps(d, indent=2)) + + +# --------------------------------------------------------------------------- # +def resolve_files(repo: str, conf: str, split: str) -> list[str]: + """The concrete data files behind one config/split, in order. + + Streaming a whole config and skipping consumed records on resume re-fetches + and re-decodes everything already done -- the cost of resuming grows with + how far you got, which is the wrong shape for a job that runs for a day. + Iterating file by file makes resume O(1): finished files are never touched + again. + """ + from datasets import load_dataset_builder + b = load_dataset_builder(repo, conf) + df = b.config.data_files + files = df[split] if split in df else list(df.values())[0] + return [str(f) for f in files] + + +def _stream_file(path: str, field: str): + """One data file, one column.""" + from datasets import load_dataset + fmt = "json" if ".json" in path else "parquet" + ds = load_dataset(fmt, data_files=[path], split="train", streaming=True) + try: + ds = ds.select_columns([field]) # decode one column, not twelve + except Exception: + pass + return ds + + +def prepare_source(name: str, target_tokens: int, cfg: Config, decontam=True, + batch=1000, log=print, file_start: int = 0, + file_end: int | None = None, out: str | None = None, + configs: tuple | None = None): + """Tokenize one source into data/<out or name>/. + + file_start/file_end bound which data files this worker owns. A source whose + files are large enough is otherwise one serial process and sets the finish + time for the whole job: Ultra-FineWeb needs 106B tokens and was projected at + 48h alone while the link and CPU both had headroom. Disjoint file ranges let + several workers share it, and because resume is already file-indexed the + bookkeeping is just a bound on the loop. + + data.py stitches data/<name>/ and data/<name>_s*/ back into one logical + source, so the shards are never copied or merged. + """ + if name not in SOURCES: + raise SystemExit(f"unknown source '{name}'. Known: {', '.join(SOURCES)}") + repo, confs, split, field = SOURCES[name] + if configs: + confs = tuple(configs) + tok = BorisTokenizer.load(DATA_DIR / "tokenizer") + ban = eval_ngrams(log) if decontam else np.zeros(0, dtype=np.int64) + out = out or name + w = ShardWriter(DATA_DIR / out, tok.eos_id) + prog = load_progress(out) + # tokens.bin is appended immediately; progress.json is only checkpointed at + # file boundaries and every 200M tokens. A worker killed mid-file therefore + # has more on disk than its progress claims, and resuming at the recorded + # file would write those documents twice. Trust the writer, and skip the + # partially-consumed file -- losing the rest of one file is free, duplicate + # documents in the corpus are not. + if prog.get("tokens", 0) != w.n_tokens: + log(f" progress lagged the writer ({prog.get('tokens',0)/1e9:.3f}B " + f"recorded vs {w.n_tokens/1e9:.3f}B on disk); resuming past the " + f"partial file") + prog["tokens"] = w.n_tokens + if w.n_tokens: + prog["file_idx"] = prog.get("file_idx", file_start) + 1 + log(f" {name}: {repo} {list(confs)}") + log(f" already {prog['tokens']/1e9:.2f}B tokens in {w.n_docs:,} documents") + if w.n_tokens >= target_tokens: + log(" target already met"); return + + t0 = time.time() + texts, n_since = [], 0 + try: + while prog.get("cfg_idx", 0) < len(confs) and prog["tokens"] < target_tokens: + conf = confs[prog.get("cfg_idx", 0)] + files = resolve_files(repo, conf, split) + hi = len(files) if file_end is None else min(file_end, len(files)) + if prog.get("file_idx", 0) < file_start: + prog["file_idx"] = file_start + log(f" config {conf}: {len(files)} files, this worker owns " + f"[{file_start}, {hi}), resuming at file {prog.get('file_idx', 0)}") + while (prog.get("file_idx", 0) < hi + and prog["tokens"] < target_tokens): + fi = prog.get("file_idx", 0) + # A transient network error must not be fatal. Five of eight workers + # died overnight on a single httpx "client has been closed" -- one + # WiFi blip -- and sat idle for hours with the job half done. Retry + # the file; on persistent failure skip it, because every source here + # has far more files than the token budget needs. + attempt, rec_iter = 0, None + while attempt < 5: + try: + rec_iter = _stream_file(files[fi], field) + break + except Exception as e: + attempt += 1 + log(f" file {fi}: open failed ({type(e).__name__}), " + f"retry {attempt}/5 in {2**attempt}s") + time.sleep(2 ** attempt) + if rec_iter is None: + log(f" file {fi}: unreadable after 5 attempts, skipping") + prog["file_idx"] = fi + 1 + save_progress(out, prog) + continue + try: + for rec in rec_iter: + t = rec.get(field) or "" + if not t: + continue + if ban.size and contaminated(t, ban): + prog["dropped"] += 1 + continue + texts.append(t) + if len(texts) >= batch: + added = w.write(tok.encode_batch(texts)) + prog["kept"] += len(texts); prog["tokens"] += added + texts = []; n_since += added + if n_since > 200_000_000: + w.flush(); save_progress(out, prog); n_since = 0 + r = prog["tokens"] / max(time.time() - t0, 1e-9) + log(f" {prog['tokens']/1e9:7.2f}B / {target_tokens/1e9:.0f}B " + f"file {fi+1}/{len(files)} {w.n_docs/1e6:6.2f}M docs " + f"{r/1e6:5.2f}M tok/s dropped {prog['dropped']:,} " + f"eta {(target_tokens-prog['tokens'])/max(r,1)/3600:5.1f}h") + if prog["tokens"] >= target_tokens: + break + except Exception as e: + # Mid-file failure: the documents already written are safe and + # the reconciliation on the next start skips the partial file. + log(f" file {fi}: stream failed mid-way ({type(e).__name__}: " + f"{str(e)[:80]}); moving on") + # This file is fully consumed; resume will never fetch it again. + prog["file_idx"] = fi + 1 + if texts: + added = w.write(tok.encode_batch(texts)) + prog["kept"] += len(texts); prog["tokens"] += added + texts = []; n_since += added + w.flush(); save_progress(out, prog) + if prog["tokens"] < target_tokens: + log(f" config {conf} exhausted at {prog['tokens']/1e9:.2f}B") + prog["cfg_idx"] = prog.get("cfg_idx", 0) + 1 + prog["file_idx"] = file_start + save_progress(out, prog) + finally: + if texts: + prog["tokens"] += w.write(tok.encode_batch(texts)) + prog["kept"] += len(texts) + w.close(); save_progress(out, prog) + log(f" done: {prog['tokens']/1e9:.2f}B tokens, {w.n_docs:,} documents, " + f"{prog['dropped']:,} dropped as contaminated") + + +def prepare_sample(mb: int, cfg: Config, log=print): + """A stratified text sample for training the tokenizer.""" + from datasets import load_dataset + out = DATA_DIR / "tokenizer_sample.txt" + out.parent.mkdir(parents=True, exist_ok=True) + per = mb * 1024 * 1024 // len(SOURCES) + with open(out, "w") as f: + for name, (repo, confs, split, field) in SOURCES.items(): + n = 0 + # Not skipped on failure. The tokenizer is the one artifact that + # cannot be changed later without invalidating every token file, the + # n-gram vocab and every checkpoint, so training it on four of five + # sources because one quietly failed is not an acceptable outcome. + ds = load_dataset(repo, confs[0], split=split, streaming=True) + for rec in ds: + t = rec.get(field) or "" + if not t: + continue + f.write(t.replace("\x00", "") + "\n") + n += len(t) + if n >= per: + break + log(f" {name}: {n/1e6:.0f} MB") + log(f" wrote {out} ({out.stat().st_size/1e6:.0f} MB)") + + +def _val_start(name: str, confs: tuple, log=print): + """The (config, file index) where this source's untouched files begin. + + The old implementation streamed a config from record zero and called next() + five million times to get past what training consumed. That re-downloads and + re-decodes tens of gigabytes to produce three megabytes of tokens, and with + every shard's "seen" counter at 0 it wasn't even reliably disjoint. The + download already recorded, per shard, which config and file it stopped at, + so the disjoint region is known exactly and reaching it costs nothing. + """ + shards = [name] + sorted( + p.parent.name for p in DATA_DIR.glob(f"{name}_s*/progress.json")) + progs = [load_progress(s) for s in shards + if progress_path(s).exists()] + if not progs: + return confs[0], 0 + max_cfg = max(p.get("cfg_idx", 0) for p in progs) + if max_cfg + 1 < len(confs): + return confs[max_cfg + 1], 0 # a whole config training never opened + hi = max(p.get("file_idx", 0) for p in progs + if p.get("cfg_idx", 0) == max_cfg) + return confs[max_cfg], hi + 2 # +2: skip the partially-read file + + +def prepare_val(tokens: int, cfg: Config, log=print): + """A fixed held-out split, drawn from every source, that the training packer + never opens -- data.open_sources only ever loads cfg.data.sources. + + Resumable per source: data/val/progress.json records which sources are + finished, so an interrupted run does not redo them. + """ + tok = BorisTokenizer.load(DATA_DIR / "tokenizer") + w = ShardWriter(DATA_DIR / "val", tok.eos_id) + prog = load_progress("val") + done = set(prog.get("done", [])) + per = tokens // len(SOURCES) + for name, (repo, confs, split, field) in SOURCES.items(): + if name in done: + log(f" {name}: already in val split"); continue + try: + conf, fi = _val_start(name, confs, log) + files = resolve_files(repo, conf, split) + except Exception as e: + log(f" {name}: skipped ({type(e).__name__}: {e})"); continue + if fi >= len(files): + log(f" {name}: no untouched files left in {conf}; skipped") + continue + got, texts = 0, [] + t0 = time.time() + while got < per and fi < len(files): + try: + recs = _stream_file(files[fi], field) + except Exception as e: + log(f" {name} file {fi}: {type(e).__name__}, skipping") + fi += 1; continue + for rec in recs: + t = rec.get(field) or "" + if not t: + continue + texts.append(t) + if len(texts) >= 500: + got += w.write(tok.encode_batch(texts)); texts = [] + if got >= per: + break + fi += 1 + if texts: + got += w.write(tok.encode_batch(texts)) + log(f" {name}: {got/1e6:.1f}M val tokens from {conf} " + f"(files from {fi}) in {time.time()-t0:.0f}s") + done.add(name) + save_progress("val", {"done": sorted(done), "tokens": w.n_tokens}) + w.close() + log(f" val split: {w.n_tokens/1e6:.1f}M tokens, {w.n_docs:,} documents") + + +def check(cfg: Config, log=print): + """Open every source and pull one record. Run this before downloading. + + Every field in SOURCES is a guess until something has actually read from the + repo -- two of the five were wrong the first time, and one of them only + surfaced after the sample step had already run. + """ + from datasets import load_dataset + bad = 0 + for name, (repo, confs, split, field) in SOURCES.items(): + for conf in confs: + try: + ds = load_dataset(repo, conf, split=split, streaming=True) + rec = next(iter(ds)) + t = rec.get(field) + if not isinstance(t, str) or not t: + raise ValueError( + f"field '{field}' is {type(t).__name__}, not text; " + f"available: {list(rec.keys())}") + log(f" ok {name:<15}{repo} [{conf}] / {split} / {field} " + f"first record {len(t):,} chars") + except Exception as e: + bad += 1 + log(f" FAIL {name:<15}{repo} [{conf}] / {split} / {field}") + log(f" {type(e).__name__}: {str(e)[:160]}") + if bad: + raise SystemExit(f"\n {bad} source/config pairs unreachable -- fix SOURCES " + f"before downloading 400GB against a broken registry") + log("\n all sources reachable") + + +def plan(cfg: Config, log=print): + """Token target per source: what the curriculum will actually draw.""" + from schedule import mixture_at + total = cfg.train.total_tokens + steps = 200 + draw = {n: 0.0 for n in cfg.data.sources} + means = {} + for n in cfg.data.sources: + dirs = [d for d in sorted(DATA_DIR.glob(n)) + + sorted(DATA_DIR.glob(f"{n}_s*")) if (d / "docs.idx").exists()] + nt = nd = 0 + for d in dirs: + o = np.memmap(d / "docs.idx", dtype=np.uint64, mode="r") + nt += int(o[-1]); nd += o.size - 1 + means[n] = nt / max(nd, 1) + for i in range(steps): + w = mixture_at((i + 0.5) / steps, cfg) + # The packer samples sources per document, so turn document draw + # probabilities into expected token shares before reporting capacity. + z = sum(w[n] * means[n] for n in w) or 1.0 + for k, v in w.items(): + draw[k] += (v * means[k] / z) * total / steps + log(f" {cfg.train.total_tokens/1e9:.0f}B token budget, " + f"{cfg.total_steps:,} steps\n") + log(f" {'source':<16}{'draws':>10}{'unique needed':>16}{'on disk':>12}") + log(" " + "-" * 56) + for n in cfg.data.sources: + have = 0 + p = DATA_DIR / n / "docs.idx" + if p.exists(): + have = int(np.fromfile(p, dtype=np.uint64)[-1]) + log(f" {n:<16}{draw[n]/1e9:>9.1f}B{draw[n]/1e9:>15.1f}B" + f"{have/1e9:>11.2f}B") + log(" " + "-" * 56) + log(f" {'TOTAL':<16}{sum(draw.values())/1e9:>9.1f}B" + f"{sum(draw.values())/1e9:>15.1f}B") + log(f"\n stored as uint16: {sum(draw.values())*2/1e12:.2f} TB at one epoch.") + log(f" Repeating a smaller unique corpus is the intended fallback -- set a") + log(f" lower --tokens per source and the packer will resample it.") + + +def status(cfg: Config, log=print): + log(f" {'source':<16}{'tokens':>12}{'docs':>12}{'dropped':>10}{'GB':>8}") + log(" " + "-" * 58) + for n in list(cfg.data.sources) + ["val"]: + dirs = [d for d in sorted(DATA_DIR.glob(n)) + sorted(DATA_DIR.glob(f"{n}_s*")) + if (d / "docs.idx").exists()] + if not dirs: + log(f" {n:<16}{'-':>12}"); continue + tot = docs = dropped = 0 + for d in dirs: + off = np.fromfile(d / "docs.idx", dtype=np.uint64) + tot += int(off[-1]); docs += off.size - 1 + dropped += load_progress(d.name).get("dropped", 0) + tag = f"{n} ({len(dirs)}sh)" if len(dirs) > 1 else n + log(f" {tag:<16}{tot/1e9:>11.2f}B{docs/1e6:>11.2f}M" + f"{dropped:>10,}{tot*2/2**30:>8.1f}") + + +def main(): + cfg = Config() + ap = argparse.ArgumentParser() + sub = ap.add_subparsers(dest="cmd", required=True) + sub.add_parser("plan"); sub.add_parser("status"); sub.add_parser("check") + s = sub.add_parser("sample"); s.add_argument("--mb", type=int, default=2000) + s = sub.add_parser("source") + s.add_argument("--name", required=True) + s.add_argument("--tokens", type=float, required=True) + s.add_argument("--no-decontam", action="store_true") + s.add_argument("--configs", default=None, + help="comma-separated config override, so two workers can " + "split a multi-config source between them") + s.add_argument("--file-start", type=int, default=0) + s.add_argument("--file-end", type=int, default=None) + s.add_argument("--out", default=None, + help="output directory name; use <name>_sN for a shard") + s.add_argument("--batch", type=int, default=250, + help="documents per encode_batch call. Each HF Encoding " + "carries ids, offsets, attention and word-id arrays, so " + "this sets the transient memory peak per worker; 1000 " + "across eight workers drove the box 16GB into swap.") + s = sub.add_parser("val"); s.add_argument("--tokens", type=float, default=2e7) + a = ap.parse_args() + + if a.cmd == "check": + check(cfg) + elif a.cmd == "plan": + plan(cfg) + elif a.cmd == "status": + status(cfg) + elif a.cmd == "sample": + prepare_sample(a.mb, cfg) + elif a.cmd == "source": + prepare_source(a.name, int(a.tokens), cfg, decontam=not a.no_decontam, + file_start=a.file_start, file_end=a.file_end, out=a.out, + batch=a.batch, + configs=tuple(a.configs.split(",")) if a.configs else None) + elif a.cmd == "val": + prepare_val(int(a.tokens), cfg) + + +if __name__ == "__main__": + main() |
