summaryrefslogtreecommitdiff
path: root/libremath/generate.py
diff options
context:
space:
mode:
Diffstat (limited to 'libremath/generate.py')
-rw-r--r--libremath/generate.py247
1 files changed, 247 insertions, 0 deletions
diff --git a/libremath/generate.py b/libremath/generate.py
new file mode 100644
index 0000000..0042f05
--- /dev/null
+++ b/libremath/generate.py
@@ -0,0 +1,247 @@
+"""LibreMath-1.0 generator: K2-Horizon-375B writes problems around specs.py's facts.
+
+ python libremath/generate.py OUT_DIR --items 200000 [--per-request 20] [--rps 2.8] [--seed N]
+
+Every request carries `per-request` specs, each with its own randomly chosen format,
+setting and names. The reply is a JSON list; each item is kept only if
+ * its final answer equals the computed answer,
+ * the problem ends with a question mark (ArithMark items are unfinished sentences),
+ * it uses none of ArithMark's 53 character names,
+ * it shares no 8-word span with any ArithMark, SVAMP or ASDiv test item,
+ * it passes the corpus-wide 13-gram ban (HellaSwag, ARC, PIQA, ArithMark),
+ * it is not a duplicate of an earlier problem.
+Kept items go to OUT_DIR/items-XXXXX.jsonl; counters and rejects to OUT_DIR/stats.json.
+Resumable: shards already written are counted and the seed advances past them.
+"""
+import argparse, asyncio, hashlib, json, random, re, sys, time
+from pathlib import Path
+import aiohttp
+
+ROOT = Path(__file__).resolve().parent
+sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT.parent))
+import specs
+from arithcheck import bad_steps
+
+import os
+API = os.environ.get("LIBREMATH_API", "https://api.ifm.ai/v1/chat/completions")
+MODEL = os.environ.get("LIBREMATH_MODEL", "IFM/K2-Horizon-375B-A23B")
+TEMPLATE_KW = json.loads(os.environ.get("LIBREMATH_TEMPLATE_KW", '{"reasoning_effort": "low"}'))
+import os
+KEY = open(Path(os.environ.get("LIBREMATH_KEY_FILE", Path.home() / ".config/libremath/api_key"))).read().strip()
+
+FORMATS = {
+ "word_problem": "a word problem that ends with a question, then a step-by-step solution",
+ "textbook_example": "a textbook-style worked example: state the problem as a question, then explain each step as a teacher would",
+ "dialogue": "a short dialogue in which one person asks the question and another works it out aloud",
+ "find_the_mistake": "a question, a student's incorrect attempt, and a correction that explains the error and reaches the right answer",
+ "concept_then_example": "two or three sentences explaining the math idea, then an example question and its solution",
+ "estimate_then_solve": "a question, a quick estimate, then the exact solution and a check that it is reasonable",
+ "two_methods": "a question solved two different ways that reach the same answer",
+ "real_world_planning": "a planning question someone faces in daily life, solved step by step",
+}
+SETTINGS = ("bakery|soccer tournament|public library|aquarium|science fair|road trip|community garden|recycling drive|school band|"
+ "train station|camping trip|school store|wildlife park|robotics club|dairy farm|history museum|theater rehearsal|"
+ "bicycle shop|kitchen|craft fair|weather station|space camp|swimming meet|bookbinding workshop|food bank|ferry|"
+ "orchard|chess club|pottery studio|hiking trail|bus route|lemonade stand|carpentry project|marathon|zoo nursery|"
+ "art class|recording studio|coral reef survey|ski lodge|post office|pet shelter|harbor|tea shop|rugby practice|"
+ "greenhouse|bird count|dance recital|basketball league|volcano exhibit|street market|ice rink|bowling alley|"
+ "beekeeping|newspaper delivery|chemistry lab|planetarium|tailor shop|fishing trip|quilting circle|apple cider mill|"
+ "school election|cricket match|map making|kite festival|summer reading program|hospital supply room|airport|"
+ "film club|vegetable stand|furniture store|scout troop|tree planting|cooking contest|science museum").split("|")
+NAMES = ("Aarav Adaeze Aiko Akira Alejandro Amara Anaya Andrei Annika Arjun Astrid Ayaan Aziz Bea Benedikt Bianca Bilal "
+ "Carmen Chidi Chloe Darius Dmitri Elif Emeka Esme Farah Fatima Felix Freya Gabriel Gustavo Hana Hiroshi Ines Ingrid "
+ "Isaac Jamal Jasper Javier Joaquin Jonah Kai Kalani Keanu Kenji Kofi Laila Lars Leila Lorenzo Lucia Magnus Malik "
+ "Marisol Matteo Miguel Mila Minh Nadia Naomi Niamh Nikhil Oluwaseun Oscar Pablo Paloma Quentin Rafael Rania Rohan "
+ "Rosa Saanvi Santiago Selin Seo-yeon Siddharth Silas Siobhan Sunita Tariq Teodor Thandiwe Tomas Valentina Wren "
+ "Xiomara Yara Yusuf Zainab Zara Ziad").split()
+
+
+def norm_words(s): return re.sub(r"[^0-9a-z]+", " ", s.lower()).split()
+
+
+def ngrams(words, n): return {" ".join(words[i:i + n]) for i in range(len(words) - n + 1)}
+
+
+def load_bans():
+ am = [json.loads(l) for l in open(Path.home() / ".cache/huggingface/hub/datasets--AxiomicLabs--ArithMark-3.0/snapshots/6f6e59dd9b7e2c63455f7af7f838f9ecc3d0a746/arithmark-3.jsonl")]
+ am_names = {re.match(r"[A-Z][a-z]+", r["ctx"]).group(0) for r in am if re.match(r"[A-Z][a-z]+", r["ctx"])}
+ am_names -= {"At", "In", "The", "A", "An", "On", "For", "During", "After", "Each", "Every", "Last"}
+ texts = [r["ctx"] for r in am]
+ from datasets import load_dataset
+ texts += [r["Body"] + " " + r["Question"] for r in load_dataset("ChilleD/SVAMP", split="test")]
+ texts += [r["body"] + " " + r["question"] for r in load_dataset("EleutherAI/asdiv", split="validation")]
+ ban8 = set().union(*(ngrams(norm_words(t), 8) for t in texts))
+ import prepare
+ return am_names, ban8, prepare.eval_ngrams(lambda *a: None), prepare.contaminated
+
+
+def same_answer(got, want):
+ """Compare the model's final answer with the computed one, tolerating units, $ signs,
+ commas and words around the number ("56 meters", "$11.71", "3/4 of the pizza")."""
+ from fractions import Fraction
+ g, w = str(got).strip(), str(want)
+ if re.fullmatch(r"\d{1,2}:\d{2} [AP]M", w):
+ m = re.search(r"(\d{1,2}:\d{2})\s*([AaPp])\.?\s*[Mm]", g)
+ return bool(m) and f"{m.group(1)} {m.group(2).upper()}M" == w
+ g = g.replace(",", "")
+ if "/" in w:
+ m = re.findall(r"-?\d+\s*/\s*\d+", g)
+ try: return bool(m) and Fraction(m[-1].replace(" ", "")) == Fraction(w)
+ except Exception: return False
+ nums = re.findall(r"-?\d+(?:\.\d+)?", g)
+ try: return bool(nums) and abs(float(nums[-1]) - float(w)) < 1e-9
+ except Exception: return False
+
+
+def build_prompt(batch, rng):
+ """batch: list of (spec_a, spec_b). Each item is one scenario with two questions."""
+ lines = []
+ for i, (sa, sb) in enumerate(batch):
+ fmt = rng.choice(list(FORMATS)); setting = rng.choice(SETTINGS); names = rng.sample(NAMES, 2)
+ for x in (sa, sb): x["format"], x["setting"], x["names"] = fmt, setting, names
+ lines.append(f'{i}. format: {FORMATS[fmt]}. Setting: {setting}. People (use only if needed): {names[0]}, {names[1]}.\n'
+ f' Question A: grade {sa["grade"]}, topic "{sa["topic"]}", facts: {sa["facts"]}. Correct answer: {sa["answer"]}.\n'
+ f' Question B: grade {sb["grade"]}, topic "{sb["topic"]}", facts: {sb["facts"]}. Correct answer: {sb["answer"]}.')
+ return ("You are writing items for an open educational math dataset. For each spec below, write one original item: a short "
+ "scenario in the given setting followed by two questions, (a) and (b), that use the given facts. Vary wording, sentence "
+ "structure and length widely between items. Each question must be a direct question ending with a question mark, and the "
+ "item's text must end with question (b). Use realistic quantities and units. The solution must work through (a) and then "
+ "(b) step by step, explaining why each step is done, and finish each part with a one-sentence check that the answer is "
+ "reasonable. Answers must equal the given correct answers. Write plain text only: no markdown, bold, headings or bullet symbols.\n\n" + "\n".join(lines) +
+ '\n\nReply with only a JSON array, one object per spec in order: '
+ '[{"id": 0, "problem": "...", "solution": "...", "answer_a": "...", "answer_b": "..."}, ...]')
+
+
+class Gen:
+ def __init__(a, out, args):
+ a.out, a.args = out, args; a.stats = {"requests": 0, "http_errors": 0, "kept": 0, "tokens_out": 0,
+ "reject": {}}
+ a.am_names, a.ban8, a.ban13, a.contaminated = load_bans()
+ a.seen = set(); a.buf = []; a.shard = len(list(out.glob("items-*.jsonl")))
+ a.rng = random.Random(args.seed + 1_000_003 * a.shard)
+
+ def rej(a, why): a.stats["reject"][why] = a.stats["reject"].get(why, 0) + 1
+
+ def answer_of(a, it, keys, sol):
+ fa = next((it[k] for k in keys if it.get(k) not in (None, "")), None)
+ return fa
+
+ def check(a, pair, it):
+ s, s2 = pair
+ p, sol = (re.sub(r"\*\*|__|^#+\s*", "", str(it.get(k, "")), flags=re.M).strip() for k in ("problem", "solution"))
+ if not p or not sol: return a.rej("empty")
+ fa = a.answer_of(it, ("answer_a", "a", "final_answer_a"), sol)
+ fb = a.answer_of(it, ("answer_b", "b", "final_answer_b"), sol)
+ if fa is None or fb is None: return a.rej("missing_answer")
+ if not same_answer(fb, s2["answer"]): return a.rej("wrong_answer")
+ if not same_answer(fa, s["answer"]):
+ if a.stats["reject"].get("wrong_answer", 0) < 200:
+ with open(a.out / "rejects_wrong_answer.jsonl", "a") as f:
+ f.write(json.dumps({"spec": {k: s[k] for k in ("topic", "facts", "answer")}, "got": fa,
+ "problem": p[:300]}) + "\n")
+ return a.rej("wrong_answer")
+ if not p.rstrip().endswith("?"): return a.rej("not_a_question")
+ if s.get("format") != "find_the_mistake" and bad_steps(sol): return a.rej("bad_arithmetic_step")
+ text = p + "\n" + sol; words = norm_words(text)
+ if set(re.findall(r"[A-Z][a-z]+", text)) & a.am_names: return a.rej("arithmark_name")
+ if ngrams(words, 8) & a.ban8: return a.rej("test_overlap_8gram")
+ if a.contaminated(text, a.ban13): return a.rej("eval_13gram")
+ h = hashlib.sha1(" ".join(norm_words(p)).encode()).hexdigest()
+ if h in a.seen: return a.rej("duplicate")
+ a.seen.add(h)
+ return {"problem": p, "solution": sol, "answers": [s["answer"], s2["answer"]], "grade": max(s["grade"], s2["grade"]),
+ "topics": [s["topic"], s2["topic"]], "format": s["format"], "setting": s["setting"], "generator": MODEL}
+
+ def flush(a, force=False):
+ while len(a.buf) >= a.args.shard_size or (force and a.buf):
+ chunk, a.buf = a.buf[:a.args.shard_size], a.buf[a.args.shard_size:]
+ with open(a.out / f"items-{a.shard:05d}.jsonl", "w") as f:
+ for it in chunk: f.write(json.dumps(it) + "\n")
+ a.shard += 1
+ (a.out / "stats.json").write_text(json.dumps(a.stats, indent=2))
+
+ async def pace(a):
+ while getattr(a, "paused_until", 0) > time.time():
+ a.stats["paused_until"] = time.strftime("%a %I:%M %p", time.localtime(a.paused_until)); a.flush()
+ await asyncio.sleep(min(300, a.paused_until - time.time()))
+ a.stats.pop("paused_until", None)
+ async with a.lock:
+ now = time.time(); wait = a.next_slot - now
+ if wait > 0: await asyncio.sleep(wait)
+ a.next_slot = max(now, a.next_slot) + 1 / a.args.rps
+
+ async def _one(a, sess, sem):
+ batch = [specs.sample_pair(a.rng) for _ in range(a.args.per_request)]
+ body = {"model": MODEL, "temperature": 1.0, "top_p": 0.95, "max_tokens": 32000,
+ **({"chat_template_kwargs": TEMPLATE_KW} if TEMPLATE_KW else {}),
+ "messages": [{"role": "user", "content": build_prompt(batch, a.rng)}]}
+ for attempt in range(10):
+ await a.pace()
+ try:
+ async with sess.post(API, json=body, headers={"Authorization": f"Bearer {KEY}"}) as r:
+ if r.status == 429 and "quota" in (await r.text()).lower():
+ ra = r.headers.get("Retry-After")
+ a.paused_until = max(getattr(a, "paused_until", 0), time.time() + min(1800, float(ra) if ra and ra.replace(".", "").isdigit() else 1800))
+ a.stats["quota_pauses"] = a.stats.get("quota_pauses", 0) + 1
+ continue
+ if r.status == 429 or r.status >= 500:
+ ra = r.headers.get("Retry-After")
+ a.next_slot = max(a.next_slot, time.time() + (float(ra) if ra and ra.replace(".", "").isdigit() else 1.0))
+ a.stats["retries"] = a.stats.get("retries", 0) + 1
+ await asyncio.sleep(min(30, 2 ** attempt)); continue
+ d = await r.json()
+ break
+ except Exception:
+ await asyncio.sleep(2 ** attempt)
+ else:
+ a.stats["http_errors"] += 1; return
+ a.stats["requests"] += 1
+ a.stats["tokens_out"] += d.get("usage", {}).get("completion_tokens", 0)
+ txt = ((d.get("choices") or [{}])[0].get("message") or {}).get("content") or ""
+ m = re.search(r"\[.*\]", txt, re.S)
+ try: items = json.loads(m.group(0)) if m else []
+ except Exception: items = []
+ if not items: return a.rej("bad_json")
+ for it in items:
+ try: s = batch[int(it.get("id"))]
+ except Exception: a.rej("bad_id"); continue
+ k = a.check(s, it)
+ if k: a.buf.append(k); a.stats["kept"] += 1
+ a.flush()
+
+ async def one(a, sess, sem):
+ try:
+ await a._one(sess, sem)
+ except Exception as e:
+ a.stats["worker_errors"] = a.stats.get("worker_errors", 0) + 1
+ a.stats["last_worker_error"] = repr(e)[:200]
+ finally:
+ sem.release()
+
+ async def run(a):
+ sem = asyncio.Semaphore(a.args.inflight); tasks = set(); t0 = time.time()
+ a.lock = asyncio.Lock(); a.next_slot = time.time()
+ async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=600)) as sess:
+ while a.stats["kept"] < a.args.items:
+ await sem.acquire()
+ while a.next_slot - time.time() > 2: await asyncio.sleep(0.2) # do not queue far ahead of the pacer
+ t = asyncio.create_task(a.one(sess, sem)); tasks.add(t); t.add_done_callback(tasks.discard)
+ if a.stats["requests"] and a.stats["requests"] % 200 == 0:
+ el = time.time() - t0
+ print(f"{time.strftime('%I:%M %p')} req {a.stats['requests']} kept {a.stats['kept']:,} "
+ f"out-tok {a.stats['tokens_out']/1e6:.1f}M ({a.stats['tokens_out']/el:.0f}/s) rejects {a.stats['reject']}", flush=True)
+ await asyncio.gather(*tasks)
+ a.flush(force=True)
+
+
+def main():
+ ap = argparse.ArgumentParser(); ap.add_argument("out"); ap.add_argument("--items", type=int, default=1000)
+ ap.add_argument("--per-request", type=int, default=20); ap.add_argument("--rps", type=float, default=2.8)
+ ap.add_argument("--inflight", type=int, default=96); ap.add_argument("--seed", type=int, default=20261015)
+ ap.add_argument("--shard-size", type=int, default=50000)
+ args = ap.parse_args(); out = Path(args.out); out.mkdir(parents=True, exist_ok=True)
+ asyncio.run(Gen(out, args).run())
+
+
+if __name__ == "__main__":
+ main()