"""LibreMath-1.0 generator: K2-Horizon-375B writes problems around specs.py's facts. python libremath/generate.py OUT_DIR --items 200000 [--per-request 20] [--rps 2.8] [--seed N] Every request carries `per-request` specs, each with its own randomly chosen format, setting and names. The reply is a JSON list; each item is kept only if * its final answer equals the computed answer, * the problem ends with a question mark (ArithMark items are unfinished sentences), * it uses none of ArithMark's 53 character names, * it shares no 8-word span with any ArithMark, SVAMP or ASDiv test item, * it passes the corpus-wide 13-gram ban (HellaSwag, ARC, PIQA, ArithMark), * it is not a duplicate of an earlier problem. Kept items go to OUT_DIR/items-XXXXX.jsonl; counters and rejects to OUT_DIR/stats.json. Resumable: shards already written are counted and the seed advances past them. """ import argparse, asyncio, hashlib, json, random, re, sys, time from pathlib import Path import aiohttp ROOT = Path(__file__).resolve().parent sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT.parent)) import specs from arithcheck import bad_steps import os API = os.environ.get("LIBREMATH_API", "https://api.ifm.ai/v1/chat/completions") MODEL = os.environ.get("LIBREMATH_MODEL", "IFM/K2-Horizon-375B-A23B") TEMPLATE_KW = json.loads(os.environ.get("LIBREMATH_TEMPLATE_KW", '{"reasoning_effort": "low"}')) import os KEY = open(Path(os.environ.get("LIBREMATH_KEY_FILE", Path.home() / ".config/libremath/api_key"))).read().strip() FORMATS = { "word_problem": "a word problem that ends with a question, then a step-by-step solution", "textbook_example": "a textbook-style worked example: state the problem as a question, then explain each step as a teacher would", "dialogue": "a short dialogue in which one person asks the question and another works it out aloud", "find_the_mistake": "a question, a student's incorrect attempt, and a correction that explains the error and reaches the right answer", "concept_then_example": "two or three sentences explaining the math idea, then an example question and its solution", "estimate_then_solve": "a question, a quick estimate, then the exact solution and a check that it is reasonable", "two_methods": "a question solved two different ways that reach the same answer", "real_world_planning": "a planning question someone faces in daily life, solved step by step", } SETTINGS = ("bakery|soccer tournament|public library|aquarium|science fair|road trip|community garden|recycling drive|school band|" "train station|camping trip|school store|wildlife park|robotics club|dairy farm|history museum|theater rehearsal|" "bicycle shop|kitchen|craft fair|weather station|space camp|swimming meet|bookbinding workshop|food bank|ferry|" "orchard|chess club|pottery studio|hiking trail|bus route|lemonade stand|carpentry project|marathon|zoo nursery|" "art class|recording studio|coral reef survey|ski lodge|post office|pet shelter|harbor|tea shop|rugby practice|" "greenhouse|bird count|dance recital|basketball league|volcano exhibit|street market|ice rink|bowling alley|" "beekeeping|newspaper delivery|chemistry lab|planetarium|tailor shop|fishing trip|quilting circle|apple cider mill|" "school election|cricket match|map making|kite festival|summer reading program|hospital supply room|airport|" "film club|vegetable stand|furniture store|scout troop|tree planting|cooking contest|science museum").split("|") NAMES = ("Aarav Adaeze Aiko Akira Alejandro Amara Anaya Andrei Annika Arjun Astrid Ayaan Aziz Bea Benedikt Bianca Bilal " "Carmen Chidi Chloe Darius Dmitri Elif Emeka Esme Farah Fatima Felix Freya Gabriel Gustavo Hana Hiroshi Ines Ingrid " "Isaac Jamal Jasper Javier Joaquin Jonah Kai Kalani Keanu Kenji Kofi Laila Lars Leila Lorenzo Lucia Magnus Malik " "Marisol Matteo Miguel Mila Minh Nadia Naomi Niamh Nikhil Oluwaseun Oscar Pablo Paloma Quentin Rafael Rania Rohan " "Rosa Saanvi Santiago Selin Seo-yeon Siddharth Silas Siobhan Sunita Tariq Teodor Thandiwe Tomas Valentina Wren " "Xiomara Yara Yusuf Zainab Zara Ziad").split() def norm_words(s): return re.sub(r"[^0-9a-z]+", " ", s.lower()).split() def ngrams(words, n): return {" ".join(words[i:i + n]) for i in range(len(words) - n + 1)} def load_bans(): am = [json.loads(l) for l in open(Path.home() / ".cache/huggingface/hub/datasets--AxiomicLabs--ArithMark-3.0/snapshots/6f6e59dd9b7e2c63455f7af7f838f9ecc3d0a746/arithmark-3.jsonl")] am_names = {re.match(r"[A-Z][a-z]+", r["ctx"]).group(0) for r in am if re.match(r"[A-Z][a-z]+", r["ctx"])} am_names -= {"At", "In", "The", "A", "An", "On", "For", "During", "After", "Each", "Every", "Last"} texts = [r["ctx"] for r in am] from datasets import load_dataset texts += [r["Body"] + " " + r["Question"] for r in load_dataset("ChilleD/SVAMP", split="test")] texts += [r["body"] + " " + r["question"] for r in load_dataset("EleutherAI/asdiv", split="validation")] ban8 = set().union(*(ngrams(norm_words(t), 8) for t in texts)) import prepare return am_names, ban8, prepare.eval_ngrams(lambda *a: None), prepare.contaminated def same_answer(got, want): """Compare the model's final answer with the computed one, tolerating units, $ signs, commas and words around the number ("56 meters", "$11.71", "3/4 of the pizza").""" from fractions import Fraction g, w = str(got).strip(), str(want) if re.fullmatch(r"\d{1,2}:\d{2} [AP]M", w): m = re.search(r"(\d{1,2}:\d{2})\s*([AaPp])\.?\s*[Mm]", g) return bool(m) and f"{m.group(1)} {m.group(2).upper()}M" == w g = g.replace(",", "") if "/" in w: m = re.findall(r"-?\d+\s*/\s*\d+", g) try: return bool(m) and Fraction(m[-1].replace(" ", "")) == Fraction(w) except Exception: return False nums = re.findall(r"-?\d+(?:\.\d+)?", g) try: return bool(nums) and abs(float(nums[-1]) - float(w)) < 1e-9 except Exception: return False def build_prompt(batch, rng): """batch: list of (spec_a, spec_b). Each item is one scenario with two questions.""" lines = [] for i, (sa, sb) in enumerate(batch): fmt = rng.choice(list(FORMATS)); setting = rng.choice(SETTINGS); names = rng.sample(NAMES, 2) for x in (sa, sb): x["format"], x["setting"], x["names"] = fmt, setting, names lines.append(f'{i}. format: {FORMATS[fmt]}. Setting: {setting}. People (use only if needed): {names[0]}, {names[1]}.\n' f' Question A: grade {sa["grade"]}, topic "{sa["topic"]}", facts: {sa["facts"]}. Correct answer: {sa["answer"]}.\n' f' Question B: grade {sb["grade"]}, topic "{sb["topic"]}", facts: {sb["facts"]}. Correct answer: {sb["answer"]}.') return ("You are writing items for an open educational math dataset. For each spec below, write one original item: a short " "scenario in the given setting followed by two questions, (a) and (b), that use the given facts. Vary wording, sentence " "structure and length widely between items. Each question must be a direct question ending with a question mark, and the " "item's text must end with question (b). Use realistic quantities and units. The solution must work through (a) and then " "(b) step by step, explaining why each step is done, and finish each part with a one-sentence check that the answer is " "reasonable. Answers must equal the given correct answers. Write plain text only: no markdown, bold, headings or bullet symbols.\n\n" + "\n".join(lines) + '\n\nReply with only a JSON array, one object per spec in order: ' '[{"id": 0, "problem": "...", "solution": "...", "answer_a": "...", "answer_b": "..."}, ...]') class Gen: def __init__(a, out, args): a.out, a.args = out, args; a.stats = {"requests": 0, "http_errors": 0, "kept": 0, "tokens_out": 0, "reject": {}} a.am_names, a.ban8, a.ban13, a.contaminated = load_bans() a.seen = set(); a.buf = []; a.shard = len(list(out.glob("items-*.jsonl"))) a.rng = random.Random(args.seed + 1_000_003 * a.shard) def rej(a, why): a.stats["reject"][why] = a.stats["reject"].get(why, 0) + 1 def answer_of(a, it, keys, sol): fa = next((it[k] for k in keys if it.get(k) not in (None, "")), None) return fa def check(a, pair, it): s, s2 = pair p, sol = (re.sub(r"\*\*|__|^#+\s*", "", str(it.get(k, "")), flags=re.M).strip() for k in ("problem", "solution")) if not p or not sol: return a.rej("empty") fa = a.answer_of(it, ("answer_a", "a", "final_answer_a"), sol) fb = a.answer_of(it, ("answer_b", "b", "final_answer_b"), sol) if fa is None or fb is None: return a.rej("missing_answer") if not same_answer(fb, s2["answer"]): return a.rej("wrong_answer") if not same_answer(fa, s["answer"]): if a.stats["reject"].get("wrong_answer", 0) < 200: with open(a.out / "rejects_wrong_answer.jsonl", "a") as f: f.write(json.dumps({"spec": {k: s[k] for k in ("topic", "facts", "answer")}, "got": fa, "problem": p[:300]}) + "\n") return a.rej("wrong_answer") if not p.rstrip().endswith("?"): return a.rej("not_a_question") if s.get("format") != "find_the_mistake" and bad_steps(sol): return a.rej("bad_arithmetic_step") text = p + "\n" + sol; words = norm_words(text) if set(re.findall(r"[A-Z][a-z]+", text)) & a.am_names: return a.rej("arithmark_name") if ngrams(words, 8) & a.ban8: return a.rej("test_overlap_8gram") if a.contaminated(text, a.ban13): return a.rej("eval_13gram") h = hashlib.sha1(" ".join(norm_words(p)).encode()).hexdigest() if h in a.seen: return a.rej("duplicate") a.seen.add(h) return {"problem": p, "solution": sol, "answers": [s["answer"], s2["answer"]], "grade": max(s["grade"], s2["grade"]), "topics": [s["topic"], s2["topic"]], "format": s["format"], "setting": s["setting"], "generator": MODEL} def flush(a, force=False): while len(a.buf) >= a.args.shard_size or (force and a.buf): chunk, a.buf = a.buf[:a.args.shard_size], a.buf[a.args.shard_size:] with open(a.out / f"items-{a.shard:05d}.jsonl", "w") as f: for it in chunk: f.write(json.dumps(it) + "\n") a.shard += 1 (a.out / "stats.json").write_text(json.dumps(a.stats, indent=2)) async def pace(a): while getattr(a, "paused_until", 0) > time.time(): a.stats["paused_until"] = time.strftime("%a %I:%M %p", time.localtime(a.paused_until)); a.flush() await asyncio.sleep(min(300, a.paused_until - time.time())) a.stats.pop("paused_until", None) async with a.lock: now = time.time(); wait = a.next_slot - now if wait > 0: await asyncio.sleep(wait) a.next_slot = max(now, a.next_slot) + 1 / a.args.rps async def _one(a, sess, sem): batch = [specs.sample_pair(a.rng) for _ in range(a.args.per_request)] body = {"model": MODEL, "temperature": 1.0, "top_p": 0.95, "max_tokens": 32000, **({"chat_template_kwargs": TEMPLATE_KW} if TEMPLATE_KW else {}), "messages": [{"role": "user", "content": build_prompt(batch, a.rng)}]} for attempt in range(10): await a.pace() try: async with sess.post(API, json=body, headers={"Authorization": f"Bearer {KEY}"}) as r: if r.status == 429 and "quota" in (await r.text()).lower(): ra = r.headers.get("Retry-After") a.paused_until = max(getattr(a, "paused_until", 0), time.time() + min(1800, float(ra) if ra and ra.replace(".", "").isdigit() else 1800)) a.stats["quota_pauses"] = a.stats.get("quota_pauses", 0) + 1 continue if r.status == 429 or r.status >= 500: ra = r.headers.get("Retry-After") a.next_slot = max(a.next_slot, time.time() + (float(ra) if ra and ra.replace(".", "").isdigit() else 1.0)) a.stats["retries"] = a.stats.get("retries", 0) + 1 await asyncio.sleep(min(30, 2 ** attempt)); continue d = await r.json() break except Exception: await asyncio.sleep(2 ** attempt) else: a.stats["http_errors"] += 1; return a.stats["requests"] += 1 a.stats["tokens_out"] += d.get("usage", {}).get("completion_tokens", 0) txt = ((d.get("choices") or [{}])[0].get("message") or {}).get("content") or "" m = re.search(r"\[.*\]", txt, re.S) try: items = json.loads(m.group(0)) if m else [] except Exception: items = [] if not items: return a.rej("bad_json") for it in items: try: s = batch[int(it.get("id"))] except Exception: a.rej("bad_id"); continue k = a.check(s, it) if k: a.buf.append(k); a.stats["kept"] += 1 a.flush() async def one(a, sess, sem): try: await a._one(sess, sem) except Exception as e: a.stats["worker_errors"] = a.stats.get("worker_errors", 0) + 1 a.stats["last_worker_error"] = repr(e)[:200] finally: sem.release() async def run(a): sem = asyncio.Semaphore(a.args.inflight); tasks = set(); t0 = time.time() a.lock = asyncio.Lock(); a.next_slot = time.time() async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=600)) as sess: while a.stats["kept"] < a.args.items: await sem.acquire() while a.next_slot - time.time() > 2: await asyncio.sleep(0.2) # do not queue far ahead of the pacer t = asyncio.create_task(a.one(sess, sem)); tasks.add(t); t.add_done_callback(tasks.discard) if a.stats["requests"] and a.stats["requests"] % 200 == 0: el = time.time() - t0 print(f"{time.strftime('%I:%M %p')} req {a.stats['requests']} kept {a.stats['kept']:,} " f"out-tok {a.stats['tokens_out']/1e6:.1f}M ({a.stats['tokens_out']/el:.0f}/s) rejects {a.stats['reject']}", flush=True) await asyncio.gather(*tasks) a.flush(force=True) def main(): ap = argparse.ArgumentParser(); ap.add_argument("out"); ap.add_argument("--items", type=int, default=1000) ap.add_argument("--per-request", type=int, default=20); ap.add_argument("--rps", type=float, default=2.8) ap.add_argument("--inflight", type=int, default=96); ap.add_argument("--seed", type=int, default=20261015) ap.add_argument("--shard-size", type=int, default=50000) args = ap.parse_args(); out = Path(args.out); out.mkdir(parents=True, exist_ok=True) asyncio.run(Gen(out, args).run()) if __name__ == "__main__": main()