diff options
Diffstat (limited to 'scripts/scan_decay_contam.py')
| -rw-r--r-- | scripts/scan_decay_contam.py | 17 |
1 files changed, 17 insertions, 0 deletions
diff --git a/scripts/scan_decay_contam.py b/scripts/scan_decay_contam.py new file mode 100644 index 0000000..2577a05 --- /dev/null +++ b/scripts/scan_decay_contam.py @@ -0,0 +1,17 @@ +"""13-gram contamination scan of the decay sources (same ban set as prepare.py).""" +import sys, json +from pathlib import Path +sys.path.insert(0, "/home/joseph/boris-2-D112M-n15M") +import numpy as np +import prepare +from tokenizer import BorisTokenizer +ROOT = Path("/home/joseph/boris-2-D112M-n15M/data") +tok = BorisTokenizer.load(ROOT / "tokenizer"); ban = prepare.eval_ngrams() +for name in sys.argv[1:]: + ix = np.memmap(ROOT / name / "docs.idx", dtype=np.uint64, mode="r") + tt = np.memmap(ROOT / name / "tokens.bin", dtype=np.uint16, mode="r") + bad = [] + for i in range(len(ix) - 1): + t = tok.tk.decode(tt[int(ix[i]):int(ix[i + 1])].tolist(), skip_special_tokens=True) + if prepare.contaminated(t, ban): bad.append(i) + print(json.dumps({"source": name, "docs": len(ix) - 1, "contaminated": len(bad), "ids": bad}), flush=True) |
