summaryrefslogtreecommitdiff
path: root/scripts/hellaswag_title_split.py
blob: da832297471d6c88bc1373a96356cb9468372921 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
"""HellaSwag gain split by WikiHow-title overlap with the decay's how-to data
(the check cagliostro-v3.5's card ran). Items: HellaSwag validation, in eval.py's order.
Groups: WikiHow items whose article title is also a Cosmopedia-wikihow tutorial title,
WikiHow items whose title is not, and ActivityNet items (no WikiHow link at all).
Per-item acc_norm before/after the decay; a real skill gain shows up about equally in
the first two groups, memorised topics would show up only in the first.

    python probe/hellaswag_title_split.py BEFORE.pt AFTER.pt OUT.json
"""
import json, re, sys, inspect
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
import numpy as np, torch, pyarrow.parquet as pq
from huggingface_hub import hf_hub_download
from datasets import load_dataset
import eval as E
from tokenizer import BorisTokenizer

def norm(t):
    t = t.lower().strip()
    t = re.sub(r"^how to\s+", "", t)
    return re.sub(r"[^a-z0-9 ]+", "", t).strip()

titles = set()
for k in range(2):
    p = hf_hub_download("HuggingFaceTB/cosmopedia", f"data/wikihow/train-0000{k}-of-00002.parquet", repo_type="dataset")
    for pr in pq.read_table(p, columns=["prompt"]).column("prompt").to_pylist():
        m = re.search(r'tutorial on "(.+?)"', pr)
        if m: titles.add(norm(m.group(1)))

raw = [r for r in load_dataset("Rowan/hellaswag", split="validation") if str(r["label"]).strip() != ""]
group = []
for r in raw:
    if r["source_id"].startswith("activitynet"):
        group.append("activitynet"); continue
    m = re.search(r"\[header\]\s*(.+?)\s*\[title\]", r["ctx_a"])
    group.append("wikihow_title_in_data" if m and norm(m.group(1)) in titles else
                 "wikihow_title_not_in_data" if m else "wikihow_no_title")
items = E.TASKS["hellaswag"]()
assert len(items) == len(group)

src = inspect.getsource(E.score_batch).replace("    n_hit = n_hit_norm = 0", "    return scores\n    n_hit = 0")
ns = dict(vars(E)); exec(src, ns); score = ns["score_batch"]
dev = "cuda:1"; torch.cuda.set_device(dev)
right = {}
for p in sys.argv[1:3]:
    model, cfg, blob = E.load_export(Path(p), dev)
    tok = BorisTokenizer.load(Path(cfg.data.root) / "tokenizer")
    with torch.no_grad():
        s = score(model, tok, items, dev, batch_size=256)
    right[p] = np.array([int(np.argmax([s[(i, j)][0] / max(s[(i, j)][1], 1) for j in range(len(it["choices"]))]) == it["gold"])
                         for i, it in enumerate(items)])
    del model; torch.cuda.empty_cache()
a, b = right[sys.argv[1]], right[sys.argv[2]]
g = np.array(group); out = {"cosmopedia_wikihow_titles": len(titles)}
wk = np.isin(g, ["wikihow_title_in_data", "wikihow_title_not_in_data"])
out["title_overlap_share_of_titled_wikihow_items"] = float((g == "wikihow_title_in_data").sum() / max(wk.sum(), 1))
print(f"Cosmopedia wikihow titles: {len(titles):,}; titled WikiHow items whose title is in the data: "
      f"{100*out['title_overlap_share_of_titled_wikihow_items']:.1f}%")
for name in ["wikihow_title_in_data", "wikihow_title_not_in_data", "wikihow_no_title", "activitynet", "all"]:
    m = np.ones(len(g), bool) if name == "all" else (g == name)
    if not m.any(): continue
    d = (b[m] - a[m]).astype(float); se = d.std(ddof=1) / np.sqrt(m.sum()) if m.sum() > 1 else 0.0
    out[name] = {"n": int(m.sum()), "before": float(a[m].mean() * 100), "after": float(b[m].mean() * 100),
                 "gain": float(d.mean() * 100), "gain_95ci": float(1.96 * se * 100)}
    print(f"  {name:28s} n={m.sum():5d}  {a[m].mean()*100:6.2f} -> {b[m].mean()*100:6.2f}  gain {d.mean()*100:+5.2f} ± {1.96*se*100:.2f}")
json.dump(out, open(sys.argv[3], "w"), indent=2)