From 5d2e4252e695644c34be605b44a14bd80a98a0a6 Mon Sep 17 00:00:00 2001 From: jdj Date: Wed, 18 Nov 2026 14:47:33 -0800 Subject: Boris-2 training code The code, configuration and data filters used to train Boris-2 (144M, 200B tokens, single V100): model, training loop, data preparation, n-gram table, V100 kernels, final stages and the exact run configs. --- libremath/generate.py | 247 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 247 insertions(+) create mode 100644 libremath/generate.py (limited to 'libremath/generate.py') diff --git a/libremath/generate.py b/libremath/generate.py new file mode 100644 index 0000000..0042f05 --- /dev/null +++ b/libremath/generate.py @@ -0,0 +1,247 @@ +"""LibreMath-1.0 generator: K2-Horizon-375B writes problems around specs.py's facts. + + python libremath/generate.py OUT_DIR --items 200000 [--per-request 20] [--rps 2.8] [--seed N] + +Every request carries `per-request` specs, each with its own randomly chosen format, +setting and names. The reply is a JSON list; each item is kept only if + * its final answer equals the computed answer, + * the problem ends with a question mark (ArithMark items are unfinished sentences), + * it uses none of ArithMark's 53 character names, + * it shares no 8-word span with any ArithMark, SVAMP or ASDiv test item, + * it passes the corpus-wide 13-gram ban (HellaSwag, ARC, PIQA, ArithMark), + * it is not a duplicate of an earlier problem. +Kept items go to OUT_DIR/items-XXXXX.jsonl; counters and rejects to OUT_DIR/stats.json. +Resumable: shards already written are counted and the seed advances past them. +""" +import argparse, asyncio, hashlib, json, random, re, sys, time +from pathlib import Path +import aiohttp + +ROOT = Path(__file__).resolve().parent +sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT.parent)) +import specs +from arithcheck import bad_steps + +import os +API = os.environ.get("LIBREMATH_API", "https://api.ifm.ai/v1/chat/completions") +MODEL = os.environ.get("LIBREMATH_MODEL", "IFM/K2-Horizon-375B-A23B") +TEMPLATE_KW = json.loads(os.environ.get("LIBREMATH_TEMPLATE_KW", '{"reasoning_effort": "low"}')) +import os +KEY = open(Path(os.environ.get("LIBREMATH_KEY_FILE", Path.home() / ".config/libremath/api_key"))).read().strip() + +FORMATS = { + "word_problem": "a word problem that ends with a question, then a step-by-step solution", + "textbook_example": "a textbook-style worked example: state the problem as a question, then explain each step as a teacher would", + "dialogue": "a short dialogue in which one person asks the question and another works it out aloud", + "find_the_mistake": "a question, a student's incorrect attempt, and a correction that explains the error and reaches the right answer", + "concept_then_example": "two or three sentences explaining the math idea, then an example question and its solution", + "estimate_then_solve": "a question, a quick estimate, then the exact solution and a check that it is reasonable", + "two_methods": "a question solved two different ways that reach the same answer", + "real_world_planning": "a planning question someone faces in daily life, solved step by step", +} +SETTINGS = ("bakery|soccer tournament|public library|aquarium|science fair|road trip|community garden|recycling drive|school band|" + "train station|camping trip|school store|wildlife park|robotics club|dairy farm|history museum|theater rehearsal|" + "bicycle shop|kitchen|craft fair|weather station|space camp|swimming meet|bookbinding workshop|food bank|ferry|" + "orchard|chess club|pottery studio|hiking trail|bus route|lemonade stand|carpentry project|marathon|zoo nursery|" + "art class|recording studio|coral reef survey|ski lodge|post office|pet shelter|harbor|tea shop|rugby practice|" + "greenhouse|bird count|dance recital|basketball league|volcano exhibit|street market|ice rink|bowling alley|" + "beekeeping|newspaper delivery|chemistry lab|planetarium|tailor shop|fishing trip|quilting circle|apple cider mill|" + "school election|cricket match|map making|kite festival|summer reading program|hospital supply room|airport|" + "film club|vegetable stand|furniture store|scout troop|tree planting|cooking contest|science museum").split("|") +NAMES = ("Aarav Adaeze Aiko Akira Alejandro Amara Anaya Andrei Annika Arjun Astrid Ayaan Aziz Bea Benedikt Bianca Bilal " + "Carmen Chidi Chloe Darius Dmitri Elif Emeka Esme Farah Fatima Felix Freya Gabriel Gustavo Hana Hiroshi Ines Ingrid " + "Isaac Jamal Jasper Javier Joaquin Jonah Kai Kalani Keanu Kenji Kofi Laila Lars Leila Lorenzo Lucia Magnus Malik " + "Marisol Matteo Miguel Mila Minh Nadia Naomi Niamh Nikhil Oluwaseun Oscar Pablo Paloma Quentin Rafael Rania Rohan " + "Rosa Saanvi Santiago Selin Seo-yeon Siddharth Silas Siobhan Sunita Tariq Teodor Thandiwe Tomas Valentina Wren " + "Xiomara Yara Yusuf Zainab Zara Ziad").split() + + +def norm_words(s): return re.sub(r"[^0-9a-z]+", " ", s.lower()).split() + + +def ngrams(words, n): return {" ".join(words[i:i + n]) for i in range(len(words) - n + 1)} + + +def load_bans(): + am = [json.loads(l) for l in open(Path.home() / ".cache/huggingface/hub/datasets--AxiomicLabs--ArithMark-3.0/snapshots/6f6e59dd9b7e2c63455f7af7f838f9ecc3d0a746/arithmark-3.jsonl")] + am_names = {re.match(r"[A-Z][a-z]+", r["ctx"]).group(0) for r in am if re.match(r"[A-Z][a-z]+", r["ctx"])} + am_names -= {"At", "In", "The", "A", "An", "On", "For", "During", "After", "Each", "Every", "Last"} + texts = [r["ctx"] for r in am] + from datasets import load_dataset + texts += [r["Body"] + " " + r["Question"] for r in load_dataset("ChilleD/SVAMP", split="test")] + texts += [r["body"] + " " + r["question"] for r in load_dataset("EleutherAI/asdiv", split="validation")] + ban8 = set().union(*(ngrams(norm_words(t), 8) for t in texts)) + import prepare + return am_names, ban8, prepare.eval_ngrams(lambda *a: None), prepare.contaminated + + +def same_answer(got, want): + """Compare the model's final answer with the computed one, tolerating units, $ signs, + commas and words around the number ("56 meters", "$11.71", "3/4 of the pizza").""" + from fractions import Fraction + g, w = str(got).strip(), str(want) + if re.fullmatch(r"\d{1,2}:\d{2} [AP]M", w): + m = re.search(r"(\d{1,2}:\d{2})\s*([AaPp])\.?\s*[Mm]", g) + return bool(m) and f"{m.group(1)} {m.group(2).upper()}M" == w + g = g.replace(",", "") + if "/" in w: + m = re.findall(r"-?\d+\s*/\s*\d+", g) + try: return bool(m) and Fraction(m[-1].replace(" ", "")) == Fraction(w) + except Exception: return False + nums = re.findall(r"-?\d+(?:\.\d+)?", g) + try: return bool(nums) and abs(float(nums[-1]) - float(w)) < 1e-9 + except Exception: return False + + +def build_prompt(batch, rng): + """batch: list of (spec_a, spec_b). Each item is one scenario with two questions.""" + lines = [] + for i, (sa, sb) in enumerate(batch): + fmt = rng.choice(list(FORMATS)); setting = rng.choice(SETTINGS); names = rng.sample(NAMES, 2) + for x in (sa, sb): x["format"], x["setting"], x["names"] = fmt, setting, names + lines.append(f'{i}. format: {FORMATS[fmt]}. Setting: {setting}. People (use only if needed): {names[0]}, {names[1]}.\n' + f' Question A: grade {sa["grade"]}, topic "{sa["topic"]}", facts: {sa["facts"]}. Correct answer: {sa["answer"]}.\n' + f' Question B: grade {sb["grade"]}, topic "{sb["topic"]}", facts: {sb["facts"]}. Correct answer: {sb["answer"]}.') + return ("You are writing items for an open educational math dataset. For each spec below, write one original item: a short " + "scenario in the given setting followed by two questions, (a) and (b), that use the given facts. Vary wording, sentence " + "structure and length widely between items. Each question must be a direct question ending with a question mark, and the " + "item's text must end with question (b). Use realistic quantities and units. The solution must work through (a) and then " + "(b) step by step, explaining why each step is done, and finish each part with a one-sentence check that the answer is " + "reasonable. Answers must equal the given correct answers. Write plain text only: no markdown, bold, headings or bullet symbols.\n\n" + "\n".join(lines) + + '\n\nReply with only a JSON array, one object per spec in order: ' + '[{"id": 0, "problem": "...", "solution": "...", "answer_a": "...", "answer_b": "..."}, ...]') + + +class Gen: + def __init__(a, out, args): + a.out, a.args = out, args; a.stats = {"requests": 0, "http_errors": 0, "kept": 0, "tokens_out": 0, + "reject": {}} + a.am_names, a.ban8, a.ban13, a.contaminated = load_bans() + a.seen = set(); a.buf = []; a.shard = len(list(out.glob("items-*.jsonl"))) + a.rng = random.Random(args.seed + 1_000_003 * a.shard) + + def rej(a, why): a.stats["reject"][why] = a.stats["reject"].get(why, 0) + 1 + + def answer_of(a, it, keys, sol): + fa = next((it[k] for k in keys if it.get(k) not in (None, "")), None) + return fa + + def check(a, pair, it): + s, s2 = pair + p, sol = (re.sub(r"\*\*|__|^#+\s*", "", str(it.get(k, "")), flags=re.M).strip() for k in ("problem", "solution")) + if not p or not sol: return a.rej("empty") + fa = a.answer_of(it, ("answer_a", "a", "final_answer_a"), sol) + fb = a.answer_of(it, ("answer_b", "b", "final_answer_b"), sol) + if fa is None or fb is None: return a.rej("missing_answer") + if not same_answer(fb, s2["answer"]): return a.rej("wrong_answer") + if not same_answer(fa, s["answer"]): + if a.stats["reject"].get("wrong_answer", 0) < 200: + with open(a.out / "rejects_wrong_answer.jsonl", "a") as f: + f.write(json.dumps({"spec": {k: s[k] for k in ("topic", "facts", "answer")}, "got": fa, + "problem": p[:300]}) + "\n") + return a.rej("wrong_answer") + if not p.rstrip().endswith("?"): return a.rej("not_a_question") + if s.get("format") != "find_the_mistake" and bad_steps(sol): return a.rej("bad_arithmetic_step") + text = p + "\n" + sol; words = norm_words(text) + if set(re.findall(r"[A-Z][a-z]+", text)) & a.am_names: return a.rej("arithmark_name") + if ngrams(words, 8) & a.ban8: return a.rej("test_overlap_8gram") + if a.contaminated(text, a.ban13): return a.rej("eval_13gram") + h = hashlib.sha1(" ".join(norm_words(p)).encode()).hexdigest() + if h in a.seen: return a.rej("duplicate") + a.seen.add(h) + return {"problem": p, "solution": sol, "answers": [s["answer"], s2["answer"]], "grade": max(s["grade"], s2["grade"]), + "topics": [s["topic"], s2["topic"]], "format": s["format"], "setting": s["setting"], "generator": MODEL} + + def flush(a, force=False): + while len(a.buf) >= a.args.shard_size or (force and a.buf): + chunk, a.buf = a.buf[:a.args.shard_size], a.buf[a.args.shard_size:] + with open(a.out / f"items-{a.shard:05d}.jsonl", "w") as f: + for it in chunk: f.write(json.dumps(it) + "\n") + a.shard += 1 + (a.out / "stats.json").write_text(json.dumps(a.stats, indent=2)) + + async def pace(a): + while getattr(a, "paused_until", 0) > time.time(): + a.stats["paused_until"] = time.strftime("%a %I:%M %p", time.localtime(a.paused_until)); a.flush() + await asyncio.sleep(min(300, a.paused_until - time.time())) + a.stats.pop("paused_until", None) + async with a.lock: + now = time.time(); wait = a.next_slot - now + if wait > 0: await asyncio.sleep(wait) + a.next_slot = max(now, a.next_slot) + 1 / a.args.rps + + async def _one(a, sess, sem): + batch = [specs.sample_pair(a.rng) for _ in range(a.args.per_request)] + body = {"model": MODEL, "temperature": 1.0, "top_p": 0.95, "max_tokens": 32000, + **({"chat_template_kwargs": TEMPLATE_KW} if TEMPLATE_KW else {}), + "messages": [{"role": "user", "content": build_prompt(batch, a.rng)}]} + for attempt in range(10): + await a.pace() + try: + async with sess.post(API, json=body, headers={"Authorization": f"Bearer {KEY}"}) as r: + if r.status == 429 and "quota" in (await r.text()).lower(): + ra = r.headers.get("Retry-After") + a.paused_until = max(getattr(a, "paused_until", 0), time.time() + min(1800, float(ra) if ra and ra.replace(".", "").isdigit() else 1800)) + a.stats["quota_pauses"] = a.stats.get("quota_pauses", 0) + 1 + continue + if r.status == 429 or r.status >= 500: + ra = r.headers.get("Retry-After") + a.next_slot = max(a.next_slot, time.time() + (float(ra) if ra and ra.replace(".", "").isdigit() else 1.0)) + a.stats["retries"] = a.stats.get("retries", 0) + 1 + await asyncio.sleep(min(30, 2 ** attempt)); continue + d = await r.json() + break + except Exception: + await asyncio.sleep(2 ** attempt) + else: + a.stats["http_errors"] += 1; return + a.stats["requests"] += 1 + a.stats["tokens_out"] += d.get("usage", {}).get("completion_tokens", 0) + txt = ((d.get("choices") or [{}])[0].get("message") or {}).get("content") or "" + m = re.search(r"\[.*\]", txt, re.S) + try: items = json.loads(m.group(0)) if m else [] + except Exception: items = [] + if not items: return a.rej("bad_json") + for it in items: + try: s = batch[int(it.get("id"))] + except Exception: a.rej("bad_id"); continue + k = a.check(s, it) + if k: a.buf.append(k); a.stats["kept"] += 1 + a.flush() + + async def one(a, sess, sem): + try: + await a._one(sess, sem) + except Exception as e: + a.stats["worker_errors"] = a.stats.get("worker_errors", 0) + 1 + a.stats["last_worker_error"] = repr(e)[:200] + finally: + sem.release() + + async def run(a): + sem = asyncio.Semaphore(a.args.inflight); tasks = set(); t0 = time.time() + a.lock = asyncio.Lock(); a.next_slot = time.time() + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=600)) as sess: + while a.stats["kept"] < a.args.items: + await sem.acquire() + while a.next_slot - time.time() > 2: await asyncio.sleep(0.2) # do not queue far ahead of the pacer + t = asyncio.create_task(a.one(sess, sem)); tasks.add(t); t.add_done_callback(tasks.discard) + if a.stats["requests"] and a.stats["requests"] % 200 == 0: + el = time.time() - t0 + print(f"{time.strftime('%I:%M %p')} req {a.stats['requests']} kept {a.stats['kept']:,} " + f"out-tok {a.stats['tokens_out']/1e6:.1f}M ({a.stats['tokens_out']/el:.0f}/s) rejects {a.stats['reject']}", flush=True) + await asyncio.gather(*tasks) + a.flush(force=True) + + +def main(): + ap = argparse.ArgumentParser(); ap.add_argument("out"); ap.add_argument("--items", type=int, default=1000) + ap.add_argument("--per-request", type=int, default=20); ap.add_argument("--rps", type=float, default=2.8) + ap.add_argument("--inflight", type=int, default=96); ap.add_argument("--seed", type=int, default=20261015) + ap.add_argument("--shard-size", type=int, default=50000) + args = ap.parse_args(); out = Path(args.out); out.mkdir(parents=True, exist_ok=True) + asyncio.run(Gen(out, args).run()) + + +if __name__ == "__main__": + main() -- cgit v1.2.3