"""13-gram contamination scan of the decay sources (same ban set as prepare.py).""" import sys, json from pathlib import Path sys.path.insert(0, "/home/joseph/boris-2-D112M-n15M") import numpy as np import prepare from tokenizer import BorisTokenizer ROOT = Path("/home/joseph/boris-2-D112M-n15M/data") tok = BorisTokenizer.load(ROOT / "tokenizer"); ban = prepare.eval_ngrams() for name in sys.argv[1:]: ix = np.memmap(ROOT / name / "docs.idx", dtype=np.uint64, mode="r") tt = np.memmap(ROOT / name / "tokens.bin", dtype=np.uint16, mode="r") bad = [] for i in range(len(ix) - 1): t = tok.tk.decode(tt[int(ix[i]):int(ix[i + 1])].tolist(), skip_special_tokens=True) if prepare.contaminated(t, ban): bad.append(i) print(json.dumps({"source": name, "docs": len(ix) - 1, "contaminated": len(bad), "ids": bad}), flush=True)