diff options
| author | jdj <jdj@kunix.org> | 2026-11-18 14:47:33 -0800 |
|---|---|---|
| committer | jdj <jdj@kunix.org> | 2026-11-18 14:47:33 -0800 |
| commit | 5d2e4252e695644c34be605b44a14bd80a98a0a6 (patch) | |
| tree | 311a72eda4459af7547b9cd05c0956eafb235f2d /scripts/scan_decay_contam.py | |
| download | boris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.tar.gz boris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.zip | |
Boris-2 training code
The code, configuration and data filters used to train Boris-2 (144M,
200B tokens, single V100): model, training loop, data preparation,
n-gram table, V100 kernels, final stages and the exact run configs.
Diffstat (limited to 'scripts/scan_decay_contam.py')
| -rw-r--r-- | scripts/scan_decay_contam.py | 17 |
1 files changed, 17 insertions, 0 deletions
diff --git a/scripts/scan_decay_contam.py b/scripts/scan_decay_contam.py new file mode 100644 index 0000000..2577a05 --- /dev/null +++ b/scripts/scan_decay_contam.py @@ -0,0 +1,17 @@ +"""13-gram contamination scan of the decay sources (same ban set as prepare.py).""" +import sys, json +from pathlib import Path +sys.path.insert(0, "/home/joseph/boris-2-D112M-n15M") +import numpy as np +import prepare +from tokenizer import BorisTokenizer +ROOT = Path("/home/joseph/boris-2-D112M-n15M/data") +tok = BorisTokenizer.load(ROOT / "tokenizer"); ban = prepare.eval_ngrams() +for name in sys.argv[1:]: + ix = np.memmap(ROOT / name / "docs.idx", dtype=np.uint64, mode="r") + tt = np.memmap(ROOT / name / "tokens.bin", dtype=np.uint16, mode="r") + bad = [] + for i in range(len(ix) - 1): + t = tok.tk.decode(tt[int(ix[i]):int(ix[i + 1])].tolist(), skip_special_tokens=True) + if prepare.contaminated(t, ban): bad.append(i) + print(json.dumps({"source": name, "docs": len(ix) - 1, "contaminated": len(bad), "ids": bad}), flush=True) |
