diff options
| author | jdj <jdj@kunix.org> | 2026-11-18 14:47:33 -0800 |
|---|---|---|
| committer | jdj <jdj@kunix.org> | 2026-11-18 14:47:33 -0800 |
| commit | 5d2e4252e695644c34be605b44a14bd80a98a0a6 (patch) | |
| tree | 311a72eda4459af7547b9cd05c0956eafb235f2d /scripts/hellaswag_title_split.py | |
| download | boris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.tar.gz boris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.zip | |
Boris-2 training code
The code, configuration and data filters used to train Boris-2 (144M,
200B tokens, single V100): model, training loop, data preparation,
n-gram table, V100 kernels, final stages and the exact run configs.
Diffstat (limited to 'scripts/hellaswag_title_split.py')
| -rw-r--r-- | scripts/hellaswag_title_split.py | 67 |
1 files changed, 67 insertions, 0 deletions
diff --git a/scripts/hellaswag_title_split.py b/scripts/hellaswag_title_split.py new file mode 100644 index 0000000..da83229 --- /dev/null +++ b/scripts/hellaswag_title_split.py @@ -0,0 +1,67 @@ +"""HellaSwag gain split by WikiHow-title overlap with the decay's how-to data +(the check cagliostro-v3.5's card ran). Items: HellaSwag validation, in eval.py's order. +Groups: WikiHow items whose article title is also a Cosmopedia-wikihow tutorial title, +WikiHow items whose title is not, and ActivityNet items (no WikiHow link at all). +Per-item acc_norm before/after the decay; a real skill gain shows up about equally in +the first two groups, memorised topics would show up only in the first. + + python probe/hellaswag_title_split.py BEFORE.pt AFTER.pt OUT.json +""" +import json, re, sys, inspect +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +import numpy as np, torch, pyarrow.parquet as pq +from huggingface_hub import hf_hub_download +from datasets import load_dataset +import eval as E +from tokenizer import BorisTokenizer + +def norm(t): + t = t.lower().strip() + t = re.sub(r"^how to\s+", "", t) + return re.sub(r"[^a-z0-9 ]+", "", t).strip() + +titles = set() +for k in range(2): + p = hf_hub_download("HuggingFaceTB/cosmopedia", f"data/wikihow/train-0000{k}-of-00002.parquet", repo_type="dataset") + for pr in pq.read_table(p, columns=["prompt"]).column("prompt").to_pylist(): + m = re.search(r'tutorial on "(.+?)"', pr) + if m: titles.add(norm(m.group(1))) + +raw = [r for r in load_dataset("Rowan/hellaswag", split="validation") if str(r["label"]).strip() != ""] +group = [] +for r in raw: + if r["source_id"].startswith("activitynet"): + group.append("activitynet"); continue + m = re.search(r"\[header\]\s*(.+?)\s*\[title\]", r["ctx_a"]) + group.append("wikihow_title_in_data" if m and norm(m.group(1)) in titles else + "wikihow_title_not_in_data" if m else "wikihow_no_title") +items = E.TASKS["hellaswag"]() +assert len(items) == len(group) + +src = inspect.getsource(E.score_batch).replace(" n_hit = n_hit_norm = 0", " return scores\n n_hit = 0") +ns = dict(vars(E)); exec(src, ns); score = ns["score_batch"] +dev = "cuda:1"; torch.cuda.set_device(dev) +right = {} +for p in sys.argv[1:3]: + model, cfg, blob = E.load_export(Path(p), dev) + tok = BorisTokenizer.load(Path(cfg.data.root) / "tokenizer") + with torch.no_grad(): + s = score(model, tok, items, dev, batch_size=256) + right[p] = np.array([int(np.argmax([s[(i, j)][0] / max(s[(i, j)][1], 1) for j in range(len(it["choices"]))]) == it["gold"]) + for i, it in enumerate(items)]) + del model; torch.cuda.empty_cache() +a, b = right[sys.argv[1]], right[sys.argv[2]] +g = np.array(group); out = {"cosmopedia_wikihow_titles": len(titles)} +wk = np.isin(g, ["wikihow_title_in_data", "wikihow_title_not_in_data"]) +out["title_overlap_share_of_titled_wikihow_items"] = float((g == "wikihow_title_in_data").sum() / max(wk.sum(), 1)) +print(f"Cosmopedia wikihow titles: {len(titles):,}; titled WikiHow items whose title is in the data: " + f"{100*out['title_overlap_share_of_titled_wikihow_items']:.1f}%") +for name in ["wikihow_title_in_data", "wikihow_title_not_in_data", "wikihow_no_title", "activitynet", "all"]: + m = np.ones(len(g), bool) if name == "all" else (g == name) + if not m.any(): continue + d = (b[m] - a[m]).astype(float); se = d.std(ddof=1) / np.sqrt(m.sum()) if m.sum() > 1 else 0.0 + out[name] = {"n": int(m.sum()), "before": float(a[m].mean() * 100), "after": float(b[m].mean() * 100), + "gain": float(d.mean() * 100), "gain_95ci": float(1.96 * se * 100)} + print(f" {name:28s} n={m.sum():5d} {a[m].mean()*100:6.2f} -> {b[m].mean()*100:6.2f} gain {d.mean()*100:+5.2f} ± {1.96*se*100:.2f}") +json.dump(out, open(sys.argv[3], "w"), indent=2) |
