1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
|
"""HellaSwag gain split by WikiHow-title overlap with the decay's how-to data
(the check cagliostro-v3.5's card ran). Items: HellaSwag validation, in eval.py's order.
Groups: WikiHow items whose article title is also a Cosmopedia-wikihow tutorial title,
WikiHow items whose title is not, and ActivityNet items (no WikiHow link at all).
Per-item acc_norm before/after the decay; a real skill gain shows up about equally in
the first two groups, memorised topics would show up only in the first.
python probe/hellaswag_title_split.py BEFORE.pt AFTER.pt OUT.json
"""
import json, re, sys, inspect
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
import numpy as np, torch, pyarrow.parquet as pq
from huggingface_hub import hf_hub_download
from datasets import load_dataset
import eval as E
from tokenizer import BorisTokenizer
def norm(t):
t = t.lower().strip()
t = re.sub(r"^how to\s+", "", t)
return re.sub(r"[^a-z0-9 ]+", "", t).strip()
titles = set()
for k in range(2):
p = hf_hub_download("HuggingFaceTB/cosmopedia", f"data/wikihow/train-0000{k}-of-00002.parquet", repo_type="dataset")
for pr in pq.read_table(p, columns=["prompt"]).column("prompt").to_pylist():
m = re.search(r'tutorial on "(.+?)"', pr)
if m: titles.add(norm(m.group(1)))
raw = [r for r in load_dataset("Rowan/hellaswag", split="validation") if str(r["label"]).strip() != ""]
group = []
for r in raw:
if r["source_id"].startswith("activitynet"):
group.append("activitynet"); continue
m = re.search(r"\[header\]\s*(.+?)\s*\[title\]", r["ctx_a"])
group.append("wikihow_title_in_data" if m and norm(m.group(1)) in titles else
"wikihow_title_not_in_data" if m else "wikihow_no_title")
items = E.TASKS["hellaswag"]()
assert len(items) == len(group)
src = inspect.getsource(E.score_batch).replace(" n_hit = n_hit_norm = 0", " return scores\n n_hit = 0")
ns = dict(vars(E)); exec(src, ns); score = ns["score_batch"]
dev = "cuda:1"; torch.cuda.set_device(dev)
right = {}
for p in sys.argv[1:3]:
model, cfg, blob = E.load_export(Path(p), dev)
tok = BorisTokenizer.load(Path(cfg.data.root) / "tokenizer")
with torch.no_grad():
s = score(model, tok, items, dev, batch_size=256)
right[p] = np.array([int(np.argmax([s[(i, j)][0] / max(s[(i, j)][1], 1) for j in range(len(it["choices"]))]) == it["gold"])
for i, it in enumerate(items)])
del model; torch.cuda.empty_cache()
a, b = right[sys.argv[1]], right[sys.argv[2]]
g = np.array(group); out = {"cosmopedia_wikihow_titles": len(titles)}
wk = np.isin(g, ["wikihow_title_in_data", "wikihow_title_not_in_data"])
out["title_overlap_share_of_titled_wikihow_items"] = float((g == "wikihow_title_in_data").sum() / max(wk.sum(), 1))
print(f"Cosmopedia wikihow titles: {len(titles):,}; titled WikiHow items whose title is in the data: "
f"{100*out['title_overlap_share_of_titled_wikihow_items']:.1f}%")
for name in ["wikihow_title_in_data", "wikihow_title_not_in_data", "wikihow_no_title", "activitynet", "all"]:
m = np.ones(len(g), bool) if name == "all" else (g == name)
if not m.any(): continue
d = (b[m] - a[m]).astype(float); se = d.std(ddof=1) / np.sqrt(m.sum()) if m.sum() > 1 else 0.0
out[name] = {"n": int(m.sum()), "before": float(a[m].mean() * 100), "after": float(b[m].mean() * 100),
"gain": float(d.mean() * 100), "gain_95ci": float(1.96 * se * 100)}
print(f" {name:28s} n={m.sum():5d} {a[m].mean()*100:6.2f} -> {b[m].mean()*100:6.2f} gain {d.mean()*100:+5.2f} ± {1.96*se*100:.2f}")
json.dump(out, open(sys.argv[3], "w"), indent=2)
|