"""HellaSwag gain split by WikiHow-title overlap with the decay's how-to data (the check cagliostro-v3.5's card ran). Items: HellaSwag validation, in eval.py's order. Groups: WikiHow items whose article title is also a Cosmopedia-wikihow tutorial title, WikiHow items whose title is not, and ActivityNet items (no WikiHow link at all). Per-item acc_norm before/after the decay; a real skill gain shows up about equally in the first two groups, memorised topics would show up only in the first. python probe/hellaswag_title_split.py BEFORE.pt AFTER.pt OUT.json """ import json, re, sys, inspect from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) import numpy as np, torch, pyarrow.parquet as pq from huggingface_hub import hf_hub_download from datasets import load_dataset import eval as E from tokenizer import BorisTokenizer def norm(t): t = t.lower().strip() t = re.sub(r"^how to\s+", "", t) return re.sub(r"[^a-z0-9 ]+", "", t).strip() titles = set() for k in range(2): p = hf_hub_download("HuggingFaceTB/cosmopedia", f"data/wikihow/train-0000{k}-of-00002.parquet", repo_type="dataset") for pr in pq.read_table(p, columns=["prompt"]).column("prompt").to_pylist(): m = re.search(r'tutorial on "(.+?)"', pr) if m: titles.add(norm(m.group(1))) raw = [r for r in load_dataset("Rowan/hellaswag", split="validation") if str(r["label"]).strip() != ""] group = [] for r in raw: if r["source_id"].startswith("activitynet"): group.append("activitynet"); continue m = re.search(r"\[header\]\s*(.+?)\s*\[title\]", r["ctx_a"]) group.append("wikihow_title_in_data" if m and norm(m.group(1)) in titles else "wikihow_title_not_in_data" if m else "wikihow_no_title") items = E.TASKS["hellaswag"]() assert len(items) == len(group) src = inspect.getsource(E.score_batch).replace(" n_hit = n_hit_norm = 0", " return scores\n n_hit = 0") ns = dict(vars(E)); exec(src, ns); score = ns["score_batch"] dev = "cuda:1"; torch.cuda.set_device(dev) right = {} for p in sys.argv[1:3]: model, cfg, blob = E.load_export(Path(p), dev) tok = BorisTokenizer.load(Path(cfg.data.root) / "tokenizer") with torch.no_grad(): s = score(model, tok, items, dev, batch_size=256) right[p] = np.array([int(np.argmax([s[(i, j)][0] / max(s[(i, j)][1], 1) for j in range(len(it["choices"]))]) == it["gold"]) for i, it in enumerate(items)]) del model; torch.cuda.empty_cache() a, b = right[sys.argv[1]], right[sys.argv[2]] g = np.array(group); out = {"cosmopedia_wikihow_titles": len(titles)} wk = np.isin(g, ["wikihow_title_in_data", "wikihow_title_not_in_data"]) out["title_overlap_share_of_titled_wikihow_items"] = float((g == "wikihow_title_in_data").sum() / max(wk.sum(), 1)) print(f"Cosmopedia wikihow titles: {len(titles):,}; titled WikiHow items whose title is in the data: " f"{100*out['title_overlap_share_of_titled_wikihow_items']:.1f}%") for name in ["wikihow_title_in_data", "wikihow_title_not_in_data", "wikihow_no_title", "activitynet", "all"]: m = np.ones(len(g), bool) if name == "all" else (g == name) if not m.any(): continue d = (b[m] - a[m]).astype(float); se = d.std(ddof=1) / np.sqrt(m.sum()) if m.sum() > 1 else 0.0 out[name] = {"n": int(m.sum()), "before": float(a[m].mean() * 100), "after": float(b[m].mean() * 100), "gain": float(d.mean() * 100), "gain_95ci": float(1.96 * se * 100)} print(f" {name:28s} n={m.sum():5d} {a[m].mean()*100:6.2f} -> {b[m].mean()*100:6.2f} gain {d.mean()*100:+5.2f} ± {1.96*se*100:.2f}") json.dump(out, open(sys.argv[3], "w"), indent=2)