From 5d2e4252e695644c34be605b44a14bd80a98a0a6 Mon Sep 17 00:00:00 2001 From: jdj Date: Wed, 18 Nov 2026 14:47:33 -0800 Subject: Boris-2 training code The code, configuration and data filters used to train Boris-2 (144M, 200B tokens, single V100): model, training loop, data preparation, n-gram table, V100 kernels, final stages and the exact run configs. --- scripts/scan_decay_contam.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 scripts/scan_decay_contam.py (limited to 'scripts/scan_decay_contam.py') diff --git a/scripts/scan_decay_contam.py b/scripts/scan_decay_contam.py new file mode 100644 index 0000000..2577a05 --- /dev/null +++ b/scripts/scan_decay_contam.py @@ -0,0 +1,17 @@ +"""13-gram contamination scan of the decay sources (same ban set as prepare.py).""" +import sys, json +from pathlib import Path +sys.path.insert(0, "/home/joseph/boris-2-D112M-n15M") +import numpy as np +import prepare +from tokenizer import BorisTokenizer +ROOT = Path("/home/joseph/boris-2-D112M-n15M/data") +tok = BorisTokenizer.load(ROOT / "tokenizer"); ban = prepare.eval_ngrams() +for name in sys.argv[1:]: + ix = np.memmap(ROOT / name / "docs.idx", dtype=np.uint64, mode="r") + tt = np.memmap(ROOT / name / "tokens.bin", dtype=np.uint16, mode="r") + bad = [] + for i in range(len(ix) - 1): + t = tok.tk.decode(tt[int(ix[i]):int(ix[i + 1])].tolist(), skip_special_tokens=True) + if prepare.contaminated(t, ban): bad.append(i) + print(json.dumps({"source": name, "docs": len(ix) - 1, "contaminated": len(bad), "ids": bad}), flush=True) -- cgit v1.2.3