summaryrefslogtreecommitdiff
path: root/scripts/hellaswag_title_split.py
diff options
context:
space:
mode:
Diffstat (limited to 'scripts/hellaswag_title_split.py')
-rw-r--r--scripts/hellaswag_title_split.py67
1 files changed, 67 insertions, 0 deletions
diff --git a/scripts/hellaswag_title_split.py b/scripts/hellaswag_title_split.py
new file mode 100644
index 0000000..da83229
--- /dev/null
+++ b/scripts/hellaswag_title_split.py
@@ -0,0 +1,67 @@
+"""HellaSwag gain split by WikiHow-title overlap with the decay's how-to data
+(the check cagliostro-v3.5's card ran). Items: HellaSwag validation, in eval.py's order.
+Groups: WikiHow items whose article title is also a Cosmopedia-wikihow tutorial title,
+WikiHow items whose title is not, and ActivityNet items (no WikiHow link at all).
+Per-item acc_norm before/after the decay; a real skill gain shows up about equally in
+the first two groups, memorised topics would show up only in the first.
+
+ python probe/hellaswag_title_split.py BEFORE.pt AFTER.pt OUT.json
+"""
+import json, re, sys, inspect
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+import numpy as np, torch, pyarrow.parquet as pq
+from huggingface_hub import hf_hub_download
+from datasets import load_dataset
+import eval as E
+from tokenizer import BorisTokenizer
+
+def norm(t):
+ t = t.lower().strip()
+ t = re.sub(r"^how to\s+", "", t)
+ return re.sub(r"[^a-z0-9 ]+", "", t).strip()
+
+titles = set()
+for k in range(2):
+ p = hf_hub_download("HuggingFaceTB/cosmopedia", f"data/wikihow/train-0000{k}-of-00002.parquet", repo_type="dataset")
+ for pr in pq.read_table(p, columns=["prompt"]).column("prompt").to_pylist():
+ m = re.search(r'tutorial on "(.+?)"', pr)
+ if m: titles.add(norm(m.group(1)))
+
+raw = [r for r in load_dataset("Rowan/hellaswag", split="validation") if str(r["label"]).strip() != ""]
+group = []
+for r in raw:
+ if r["source_id"].startswith("activitynet"):
+ group.append("activitynet"); continue
+ m = re.search(r"\[header\]\s*(.+?)\s*\[title\]", r["ctx_a"])
+ group.append("wikihow_title_in_data" if m and norm(m.group(1)) in titles else
+ "wikihow_title_not_in_data" if m else "wikihow_no_title")
+items = E.TASKS["hellaswag"]()
+assert len(items) == len(group)
+
+src = inspect.getsource(E.score_batch).replace(" n_hit = n_hit_norm = 0", " return scores\n n_hit = 0")
+ns = dict(vars(E)); exec(src, ns); score = ns["score_batch"]
+dev = "cuda:1"; torch.cuda.set_device(dev)
+right = {}
+for p in sys.argv[1:3]:
+ model, cfg, blob = E.load_export(Path(p), dev)
+ tok = BorisTokenizer.load(Path(cfg.data.root) / "tokenizer")
+ with torch.no_grad():
+ s = score(model, tok, items, dev, batch_size=256)
+ right[p] = np.array([int(np.argmax([s[(i, j)][0] / max(s[(i, j)][1], 1) for j in range(len(it["choices"]))]) == it["gold"])
+ for i, it in enumerate(items)])
+ del model; torch.cuda.empty_cache()
+a, b = right[sys.argv[1]], right[sys.argv[2]]
+g = np.array(group); out = {"cosmopedia_wikihow_titles": len(titles)}
+wk = np.isin(g, ["wikihow_title_in_data", "wikihow_title_not_in_data"])
+out["title_overlap_share_of_titled_wikihow_items"] = float((g == "wikihow_title_in_data").sum() / max(wk.sum(), 1))
+print(f"Cosmopedia wikihow titles: {len(titles):,}; titled WikiHow items whose title is in the data: "
+ f"{100*out['title_overlap_share_of_titled_wikihow_items']:.1f}%")
+for name in ["wikihow_title_in_data", "wikihow_title_not_in_data", "wikihow_no_title", "activitynet", "all"]:
+ m = np.ones(len(g), bool) if name == "all" else (g == name)
+ if not m.any(): continue
+ d = (b[m] - a[m]).astype(float); se = d.std(ddof=1) / np.sqrt(m.sum()) if m.sum() > 1 else 0.0
+ out[name] = {"n": int(m.sum()), "before": float(a[m].mean() * 100), "after": float(b[m].mean() * 100),
+ "gain": float(d.mean() * 100), "gain_95ci": float(1.96 * se * 100)}
+ print(f" {name:28s} n={m.sum():5d} {a[m].mean()*100:6.2f} -> {b[m].mean()*100:6.2f} gain {d.mean()*100:+5.2f} ± {1.96*se*100:.2f}")
+json.dump(out, open(sys.argv[3], "w"), indent=2)