blob: 2577a05052ee34e531f0230ed7aaea55e954fafb (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
|
"""13-gram contamination scan of the decay sources (same ban set as prepare.py)."""
import sys, json
from pathlib import Path
sys.path.insert(0, "/home/joseph/boris-2-D112M-n15M")
import numpy as np
import prepare
from tokenizer import BorisTokenizer
ROOT = Path("/home/joseph/boris-2-D112M-n15M/data")
tok = BorisTokenizer.load(ROOT / "tokenizer"); ban = prepare.eval_ngrams()
for name in sys.argv[1:]:
ix = np.memmap(ROOT / name / "docs.idx", dtype=np.uint64, mode="r")
tt = np.memmap(ROOT / name / "tokens.bin", dtype=np.uint16, mode="r")
bad = []
for i in range(len(ix) - 1):
t = tok.tk.decode(tt[int(ix[i]):int(ix[i + 1])].tolist(), skip_special_tokens=True)
if prepare.contaminated(t, ban): bad.append(i)
print(json.dumps({"source": name, "docs": len(ix) - 1, "contaminated": len(bad), "ids": bad}), flush=True)
|