summaryrefslogtreecommitdiff
path: root/scripts/scan_decay_contam.py
diff options
context:
space:
mode:
authorjdj <jdj@kunix.org>2026-11-18 14:47:33 -0800
committerjdj <jdj@kunix.org>2026-11-18 14:47:33 -0800
commit5d2e4252e695644c34be605b44a14bd80a98a0a6 (patch)
tree311a72eda4459af7547b9cd05c0956eafb235f2d /scripts/scan_decay_contam.py
downloadboris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.tar.gz
boris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.zip
Boris-2 training code
The code, configuration and data filters used to train Boris-2 (144M, 200B tokens, single V100): model, training loop, data preparation, n-gram table, V100 kernels, final stages and the exact run configs.
Diffstat (limited to 'scripts/scan_decay_contam.py')
-rw-r--r--scripts/scan_decay_contam.py17
1 files changed, 17 insertions, 0 deletions
diff --git a/scripts/scan_decay_contam.py b/scripts/scan_decay_contam.py
new file mode 100644
index 0000000..2577a05
--- /dev/null
+++ b/scripts/scan_decay_contam.py
@@ -0,0 +1,17 @@
+"""13-gram contamination scan of the decay sources (same ban set as prepare.py)."""
+import sys, json
+from pathlib import Path
+sys.path.insert(0, "/home/joseph/boris-2-D112M-n15M")
+import numpy as np
+import prepare
+from tokenizer import BorisTokenizer
+ROOT = Path("/home/joseph/boris-2-D112M-n15M/data")
+tok = BorisTokenizer.load(ROOT / "tokenizer"); ban = prepare.eval_ngrams()
+for name in sys.argv[1:]:
+ ix = np.memmap(ROOT / name / "docs.idx", dtype=np.uint64, mode="r")
+ tt = np.memmap(ROOT / name / "tokens.bin", dtype=np.uint16, mode="r")
+ bad = []
+ for i in range(len(ix) - 1):
+ t = tok.tk.decode(tt[int(ix[i]):int(ix[i + 1])].tolist(), skip_special_tokens=True)
+ if prepare.contaminated(t, ban): bad.append(i)
+ print(json.dumps({"source": name, "docs": len(ix) - 1, "contaminated": len(bad), "ids": bad}), flush=True)