From 5d2e4252e695644c34be605b44a14bd80a98a0a6 Mon Sep 17 00:00:00 2001 From: jdj Date: Wed, 18 Nov 2026 14:47:33 -0800 Subject: Boris-2 training code The code, configuration and data filters used to train Boris-2 (144M, 200B tokens, single V100): model, training loop, data preparation, n-gram table, V100 kernels, final stages and the exact run configs. --- libremath/arithcheck.py | 27 +++++ libremath/compose.py | 117 +++++++++++++++++++ libremath/fragments.py | 134 ++++++++++++++++++++++ libremath/generate.py | 247 +++++++++++++++++++++++++++++++++++++++ libremath/review_fragments.py | 68 +++++++++++ libremath/specs.py | 204 +++++++++++++++++++++++++++++++++ libremath/tgen.py | 261 ++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 1058 insertions(+) create mode 100644 libremath/arithcheck.py create mode 100644 libremath/compose.py create mode 100644 libremath/fragments.py create mode 100644 libremath/generate.py create mode 100644 libremath/review_fragments.py create mode 100644 libremath/specs.py create mode 100644 libremath/tgen.py (limited to 'libremath') diff --git a/libremath/arithcheck.py b/libremath/arithcheck.py new file mode 100644 index 0000000..d8f78ac --- /dev/null +++ b/libremath/arithcheck.py @@ -0,0 +1,27 @@ +"""Recompute every simple binary step "a op b = c" written in a solution. +Returns the list of wrong steps (empty if all check out). Times (10:20), ratios (3:4) and +fractions written inside larger expressions are skipped rather than misread.""" +import re +NUM = r"\d{1,3}(?:,\d{3})+(?:\.\d+)?|\d+(?:\.\d+)?" +STEP = re.compile(rf"(? 1e-6 and abs(round(v, 2) - c) < 0.006: continue # rounded quotient + if c == int(a // b): continue # whole-number quotient (remainder stated separately) + if abs(v - c) > 1e-6 * max(1, abs(v)) and abs(round(v, 2) - c) > 1e-9: + bad.append(m.group(0)) + return bad diff --git a/libremath/compose.py b/libremath/compose.py new file mode 100644 index 0000000..46f8bb5 --- /dev/null +++ b/libremath/compose.py @@ -0,0 +1,117 @@ +"""LibreMath composer (2026-10-23): unlimited grade-school word problems from the K2-written +fragment bank (fragments.py), with every number and answer computed here. + + python libremath/compose.py BANK.json OUT_DIR --items 1000000 [--seed 1] +""" +import argparse, json, random, re, sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import generate as G +from tgen import SETTINGS_EXTRA + +PH = re.compile(r"\{(\w+)\}") + + +def fill(t, env): + t = re.sub(r"\b([Ee])(very|ach) \{(cont|obj)\}", lambda m: ("All the {" if m.group(1) == "E" else "all the {") + m.group(3) + "}" + if m.group(2) == "very" else m.group(1) + "ach of the {" + m.group(3) + "}", t) # plural nouns: "every boxes" + out = PH.sub(lambda m: str(env[m.group(1)]), t) + out = re.sub(r"\b([Tt]he) the\b", r"\1", out) + out = out.strip(); out = out[:1].upper() + out[1:] + return out if out[-1:] in ".!?\"'”" else out + "." + + +ANSWER_TAGS = {"money": r"cost|spen|paid|pay|price|dollar|bill|\$", "remove": r"\bleft\b|remain|still", "two_step": r"\bleft\b|remain|still", + "compare": r"more|differ|ahead|extra", "share": r"\beach\b|per\b|every", "combine": r"total|altogether|in all|together|sum", + "groups": r"total|altogether|in all|together"} +ALL_TAGS = "|".join(ANSWER_TAGS.values()) + r"|collect|gather|found|bought|made|earn|save" + + +def money(x): + return f"{x:.2f}" + + +class Composer: + def __init__(a, bank, seed): + a.b = bank; a.r = random.Random(seed) + a.places = G.SETTINGS + SETTINGS_EXTRA; a.names = G.NAMES + + def pick(a, cat): return a.r.choice(a.b[cat]) + + def answer_for(a, kind): + if not hasattr(a, "_ans"): + gen = [f for f in a.b["answer"] if not re.search(ALL_TAGS, f, re.I)] + a._ans = {k: (gen if k != "money" else [f for f in gen if "{obj}" not in f]) + [f for f in a.b["answer"] if re.search(v, f, re.I) and not re.search( + "|".join(v2 for k2, v2 in ANSWER_TAGS.items() if v2 != v), f, re.I)] for k, v in ANSWER_TAGS.items()} + return a.r.choice(a._ans[kind]) + + def numbers(a, kind): + r = a.r; big = r.random() < 0.35 + if kind in ("combine", "remove", "compare"): + hi = r.choice([20, 50, 100, 100, 500, 1000]) if not big else 5000 + x, y = r.randint(2, hi), r.randint(2, hi) + return (max(x, y), min(x, y)) if kind != "combine" else (x, y) + if kind == "groups": return r.randint(2, 12 if not big else 40), r.randint(2, 12 if not big else 99) + if kind == "share": + b = r.randint(2, 12); return b * r.randint(2, 15 if not big else 60), b + if kind == "money": return r.randint(2, 12), r.randint(25, 1999) / 100 + + def item(a): + r = a.r; A, B = r.sample(a.names, 2); o = r.choice(a.b["objects"]); place = r.choice(a.places) + kind = r.choice(["combine", "remove", "compare", "groups", "groups", "share", "money", "two_step"]) + base = "groups" if kind == "two_step" and r.random() < 0.6 else ("combine" if kind == "two_step" else kind) + x, y = a.numbers(base) + if x == y and base in ("remove", "compare"): x += 1 + ans = {"combine": lambda: x + y, "remove": lambda: x - y, "compare": lambda: x - y, "groups": lambda: x * y, + "share": lambda: x // y, "money": lambda: round(x * y, 2)}[base]() + env = {"A": A, "B": B, "obj": o["obj"], "cont": o["cont"], "place": "the " + place, "a": x, "b": y, + "ans": money(ans) if base == "money" else ans, "p": money(y) if base == "money" else y, "d": r.randint(2, 60)} + prob = [fill(a.pick(f"{base}_setup"), env)] + steps = [fill(a.pick(f"{base}_step"), env)] + final = ans + if kind == "two_step": + c = r.randint(1, max(1, ans - 1)); env2 = dict(env, x=ans, c=c, ans=ans - c) + prob.append(fill(a.pick("followup_remove"), env2)); q = fill(a.pick("followup_q"), env2) + steps.append(fill(a.pick("followup_step"), env2)); final = ans - c; env = env2 + else: + q = fill(a.pick(f"{base}_q"), env) + if r.random() < 0.25: # irrelevant fact, before or after the setup + prob.insert(r.choice([0, len(prob)]), fill(a.pick("distractor"), env)) + if r.random() < 0.7: prob.insert(0, fill(a.pick("opener"), env)) + problem = " ".join(prob + [q]) + solution = " ".join(steps + [fill(a.answer_for(kind), env)]) + if r.random() < 0.3: problem, solution = "Question: " + problem, "Answer: " + solution + return {"problem": problem, "solution": solution, "answers": [str(env["ans"])], "kind": kind, + "objects": o, "generator": "LibreMath fragment composer (fragments by IFM/K2-Horizon)"} + + +def distinct(texts, n=4): + tot = 0; s = set() + for t in texts: + w = re.sub(r"\d+", "#", t.lower()).split(); g = [tuple(w[i:i + n]) for i in range(len(w) - n + 1)]; tot += len(g); s.update(g) + return len(s) / max(tot, 1) + + +def main(): + ap = argparse.ArgumentParser(); ap.add_argument("bank"); ap.add_argument("out") + ap.add_argument("--items", type=int, default=100000); ap.add_argument("--seed", type=int, default=1) + ap.add_argument("--no-filter", action="store_true") + a = ap.parse_args(); out = Path(a.out); out.mkdir(parents=True, exist_ok=True) + bank = json.load(open(a.bank)); c = Composer(bank, a.seed) + am_names, ban8 = set(), set() + if not a.no_filter: + am_names, ban8, _, _ = G.load_bans() + seen = set(); kept = rej = 0 + with open(out / f"items-seed{a.seed}.jsonl", "w") as f: + while kept < a.items: + it = c.item(); text = it["problem"] + " " + it["solution"] + h = hash(it["problem"]) + if h in seen or (set(re.findall(r"[A-Z][a-z]+", text)) & am_names) or (ban8 and G.ngrams(G.norm_words(text), 8) & ban8): + rej += 1; continue + seen.add(h); f.write(json.dumps(it) + "\n"); kept += 1 + S = [json.loads(l) for l in open(out / f"items-seed{a.seed}.jsonl")][:20000] + print(f"kept {kept:,}, rejected {rej:,}; distinct 4-gram ratio (numbers masked) on 20k: {distinct([s['problem'] + ' ' + s['solution'] for s in S]):.3f}") + + +if __name__ == "__main__": + main() diff --git a/libremath/fragments.py b/libremath/fragments.py new file mode 100644 index 0000000..4be2380 --- /dev/null +++ b/libremath/fragments.py @@ -0,0 +1,134 @@ +"""LibreMath fragment bank (2026-10-23): K2 writes reusable sentence fragments ONCE; compose.py +assembles unlimited problems from them with exact, code-computed math. + +Each category has a fixed set of placeholders. A fragment is kept only if it uses exactly the +allowed placeholders, includes the required ones, contains no literal digits, and (for solution +steps) contains the exact equation string the composer will fill. + + python libremath/fragments.py OUT.json --calls-per-category 12 +""" +import argparse, asyncio, json, random, re, sys, time +from pathlib import Path +import aiohttp + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import generate as G + +# category: (description, required placeholders, allowed placeholders, kind) +CATS = { + "opener": ("an opening sentence that sets a scene at or around {place}; it may mention {A}", {"place"}, {"place", "A"}, "s"), + "distractor": ("a sentence giving one irrelevant fact that includes the number {d} (not needed to solve any question); it may mention {A} or {place}", {"d"}, {"d", "A", "place"}, "s"), + "combine_setup": ("one or two sentences saying {A} has {a} {obj} and then gets {b} more (buys, finds, receives, picks, is given...)", {"A", "a", "b", "obj"}, {"A", "a", "b", "obj", "place"}, "s"), + "combine_q": ("a question asking how many {obj} {A} has now in total", {"obj"}, {"A", "obj"}, "q"), + "combine_step": ("a solution sentence explaining why we add, containing exactly '{a} + {b} = {ans}'", {"a", "b", "ans"}, {"a", "b", "ans", "A", "obj"}, "s"), + "remove_setup": ("one or two sentences saying {A} starts with {a} {obj} and then {b} of them are used, given away, sold, eaten or lost", {"A", "a", "b", "obj"}, {"A", "a", "b", "obj", "place"}, "s"), + "remove_q": ("a question asking how many {obj} {A} has left", {"obj"}, {"A", "obj"}, "q"), + "remove_step": ("a solution sentence explaining why we subtract, containing exactly '{a} - {b} = {ans}'", {"a", "b", "ans"}, {"a", "b", "ans", "A", "obj"}, "s"), + "compare_setup": ("one or two sentences saying {A} has {a} {obj} and {B} has {b} {obj}", {"A", "B", "a", "b"}, {"A", "B", "a", "b", "obj", "place"}, "s"), + "compare_q": ("a question asking how many more {obj} {A} has than {B}", {"A", "B"}, {"A", "B", "obj"}, "q"), + "compare_step": ("a solution sentence explaining the difference, containing exactly '{a} - {b} = {ans}'", {"a", "b", "ans"}, {"a", "b", "ans", "A", "B", "obj"}, "s"), + "groups_setup": ("one or two sentences saying there are {a} {cont} with {b} {obj} in each (packed, arranged, stacked...); may mention {A}", {"a", "b", "cont", "obj"}, {"A", "a", "b", "cont", "obj", "place"}, "s"), + "groups_q": ("a question asking how many {obj} there are altogether", {"obj"}, {"A", "obj", "cont"}, "q"), + "groups_step": ("a solution sentence explaining why we multiply, containing exactly '{a} × {b} = {ans}'", {"a", "b", "ans"}, {"a", "b", "ans", "obj", "cont"}, "s"), + "share_setup": ("one or two sentences saying {A} shares or splits {a} {obj} equally into {b} {cont}", {"A", "a", "b", "obj", "cont"}, {"A", "a", "b", "obj", "cont", "place"}, "s"), + "share_q": ("a question asking how many {obj} go in each of the {cont}", {"obj", "cont"}, {"A", "obj", "cont"}, "q"), + "share_step": ("a solution sentence explaining why we divide, containing exactly '{a} ÷ {b} = {ans}'", {"a", "b", "ans"}, {"a", "b", "ans", "obj", "cont"}, "s"), + "followup_remove": ("a follow-up sentence (for after a total is known) saying {c} of those {obj} are then used, sold or given away", {"c", "obj"}, {"c", "obj", "A", "B"}, "s"), + "followup_q": ("a question asking how many {obj} are left after that", set(), {"obj", "A"}, "q"), + "followup_step": ("a solution sentence for the second step, containing exactly '{x} - {c} = {ans}'", {"x", "c", "ans"}, {"x", "c", "ans", "obj"}, "s"), + "money_setup": ("one or two sentences saying {A} buys {a} {obj} that cost {p} dollars each", {"A", "a", "obj", "p"}, {"A", "a", "obj", "p", "place"}, "s"), + "money_q": ("a question asking how much {A} spends in total", {"A"}, {"A", "obj"}, "q"), + "money_step": ("a solution sentence explaining the total cost, containing exactly '{a} × ${p} = ${ans}'", {"a", "p", "ans"}, {"a", "p", "ans", "obj", "A"}, "s"), + "answer": ("a short final sentence stating the answer {ans}; it may mention {obj} or {A}", {"ans"}, {"ans", "obj", "A"}, "s"), +} +TONES = ["plain and simple, for young children", "like a friendly teacher", "like a math textbook", "lively and story-like", + "short and direct", "detailed, with a little extra description", "conversational", "matter-of-fact"] +PH = re.compile(r"\{(\w+)\}") + + +def valid(frag, cat): + desc, req, allowed, kind = CATS[cat] + if not isinstance(frag, str) or not (12 <= len(frag) <= 260): return False + if re.search(r"\d", PH.sub("", frag)): return False + ph = set(PH.findall(frag)) + if not req <= ph or not ph <= allowed: return False + if kind == "q" and not frag.rstrip().endswith("?"): return False + if kind == "s" and frag.rstrip().endswith("?"): return False + eq = re.search(r"'(.*)'", desc) + if eq and eq.group(1) not in frag: return False + return True + + +def prompt(cat, n, tone): + if cat == "objects": + return (f"List {n} pairs of things for grade-school math word problems: a plural countable item and a plural container or " + f"group it could be packed or arranged in (for example marbles / jars, chairs / rows, stickers / sheets). Cover many " + f"different domains (food, sports, school, nature, tools, crafts, animals, travel, music, building). Reply with only a " + f'JSON array of objects like {{"obj": "marbles", "cont": "jars"}}.') + desc = CATS[cat][0] + return (f"Write {n} different English fragments for building grade-school math word problems. Each fragment is {desc}. " + f"Use the placeholders in curly braces exactly as written (they are filled in later by a program); never write a digit. " + + ("Do not mention any specific item, food, animal or object; the items being counted are filled in later. " + if cat in ("opener", "distractor") else + "Never name a specific item, food, animal, place or object: refer to the items only as {obj} and {cont} " + "(or words like 'them', 'each one', 'the items'), because the program fills them in. ") + + f"Style: {tone}. Vary vocabulary and sentence structure as much as possible; no two fragments should start the same way. " + f"Plain text, no markdown. Reply with only a JSON array of strings.") + + +async def main_async(a): + out = Path(a.out); bank = json.loads(out.read_text()) if out.exists() else {c: [] for c in list(CATS) + ["objects"]} + sem = asyncio.Semaphore(a.inflight); lock = asyncio.Lock(); state = {"next": time.time(), "tok": 0} + rng = random.Random(a.seed) + + async def call(sess, cat): + async with sem: + body = {"model": G.MODEL, "temperature": 1.0, "top_p": 0.95, "max_tokens": 16000, + **({"chat_template_kwargs": G.TEMPLATE_KW} if G.TEMPLATE_KW else {}), + "messages": [{"role": "user", "content": prompt(cat, a.per_call, rng.choice(TONES))}]} + for attempt in range(8): + async with lock: + w = state["next"] - time.time() + if w > 0: await asyncio.sleep(w) + state["next"] = max(time.time(), state["next"]) + 1 / a.rps + try: + async with sess.post(G.API, json=body, headers={"Authorization": f"Bearer {G.KEY}"}) as r: + if r.status == 429: + txt = await r.text() + await asyncio.sleep(1800 if "quota" in txt.lower() else 5 * (attempt + 1)); continue + if r.status >= 500: await asyncio.sleep(5 * (attempt + 1)); continue + d = await r.json(); break + except Exception: + await asyncio.sleep(5 * (attempt + 1)) + else: + return cat, [] + state["tok"] += d.get("usage", {}).get("completion_tokens", 0) + txt = ((d.get("choices") or [{}])[0].get("message") or {}).get("content") or "" + m = re.search(r"\[.*\]", txt, re.S) + try: arr = json.loads(m.group(0)) if m else [] + except Exception: arr = [] + if cat == "objects": + return cat, [{"obj": o["obj"].strip().lower(), "cont": o["cont"].strip().lower()} for o in arr + if isinstance(o, dict) and all(isinstance(o.get(k), str) and re.fullmatch(r"[a-zA-Z' -]{3,30}", o[k].strip()) for k in ("obj", "cont"))] + return cat, [f.strip() for f in arr if valid(f, cat)] + + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=600)) as sess: + tasks = [call(sess, c) for c in CATS for _ in range(a.calls_per_category)] + [call(sess, "objects") for _ in range(a.calls_per_category)] + for fut in asyncio.as_completed(tasks): + cat, frags = await fut + seen = {json.dumps(f, sort_keys=True) for f in bank[cat]} + bank[cat] += [f for f in frags if json.dumps(f, sort_keys=True) not in seen] + out.write_text(json.dumps(bank, indent=1)) + print(f" {cat}: +{len(frags)} -> {len(bank[cat])} (output tokens so far {state['tok']:,})", flush=True) + print({c: len(v) for c, v in bank.items()}) + + +def main(): + ap = argparse.ArgumentParser(); ap.add_argument("out") + ap.add_argument("--calls-per-category", type=int, default=12); ap.add_argument("--per-call", type=int, default=40) + ap.add_argument("--rps", type=float, default=1.0); ap.add_argument("--inflight", type=int, default=16); ap.add_argument("--seed", type=int, default=7) + asyncio.run(main_async(ap.parse_args())) + + +if __name__ == "__main__": + main() diff --git a/libremath/generate.py b/libremath/generate.py new file mode 100644 index 0000000..0042f05 --- /dev/null +++ b/libremath/generate.py @@ -0,0 +1,247 @@ +"""LibreMath-1.0 generator: K2-Horizon-375B writes problems around specs.py's facts. + + python libremath/generate.py OUT_DIR --items 200000 [--per-request 20] [--rps 2.8] [--seed N] + +Every request carries `per-request` specs, each with its own randomly chosen format, +setting and names. The reply is a JSON list; each item is kept only if + * its final answer equals the computed answer, + * the problem ends with a question mark (ArithMark items are unfinished sentences), + * it uses none of ArithMark's 53 character names, + * it shares no 8-word span with any ArithMark, SVAMP or ASDiv test item, + * it passes the corpus-wide 13-gram ban (HellaSwag, ARC, PIQA, ArithMark), + * it is not a duplicate of an earlier problem. +Kept items go to OUT_DIR/items-XXXXX.jsonl; counters and rejects to OUT_DIR/stats.json. +Resumable: shards already written are counted and the seed advances past them. +""" +import argparse, asyncio, hashlib, json, random, re, sys, time +from pathlib import Path +import aiohttp + +ROOT = Path(__file__).resolve().parent +sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT.parent)) +import specs +from arithcheck import bad_steps + +import os +API = os.environ.get("LIBREMATH_API", "https://api.ifm.ai/v1/chat/completions") +MODEL = os.environ.get("LIBREMATH_MODEL", "IFM/K2-Horizon-375B-A23B") +TEMPLATE_KW = json.loads(os.environ.get("LIBREMATH_TEMPLATE_KW", '{"reasoning_effort": "low"}')) +import os +KEY = open(Path(os.environ.get("LIBREMATH_KEY_FILE", Path.home() / ".config/libremath/api_key"))).read().strip() + +FORMATS = { + "word_problem": "a word problem that ends with a question, then a step-by-step solution", + "textbook_example": "a textbook-style worked example: state the problem as a question, then explain each step as a teacher would", + "dialogue": "a short dialogue in which one person asks the question and another works it out aloud", + "find_the_mistake": "a question, a student's incorrect attempt, and a correction that explains the error and reaches the right answer", + "concept_then_example": "two or three sentences explaining the math idea, then an example question and its solution", + "estimate_then_solve": "a question, a quick estimate, then the exact solution and a check that it is reasonable", + "two_methods": "a question solved two different ways that reach the same answer", + "real_world_planning": "a planning question someone faces in daily life, solved step by step", +} +SETTINGS = ("bakery|soccer tournament|public library|aquarium|science fair|road trip|community garden|recycling drive|school band|" + "train station|camping trip|school store|wildlife park|robotics club|dairy farm|history museum|theater rehearsal|" + "bicycle shop|kitchen|craft fair|weather station|space camp|swimming meet|bookbinding workshop|food bank|ferry|" + "orchard|chess club|pottery studio|hiking trail|bus route|lemonade stand|carpentry project|marathon|zoo nursery|" + "art class|recording studio|coral reef survey|ski lodge|post office|pet shelter|harbor|tea shop|rugby practice|" + "greenhouse|bird count|dance recital|basketball league|volcano exhibit|street market|ice rink|bowling alley|" + "beekeeping|newspaper delivery|chemistry lab|planetarium|tailor shop|fishing trip|quilting circle|apple cider mill|" + "school election|cricket match|map making|kite festival|summer reading program|hospital supply room|airport|" + "film club|vegetable stand|furniture store|scout troop|tree planting|cooking contest|science museum").split("|") +NAMES = ("Aarav Adaeze Aiko Akira Alejandro Amara Anaya Andrei Annika Arjun Astrid Ayaan Aziz Bea Benedikt Bianca Bilal " + "Carmen Chidi Chloe Darius Dmitri Elif Emeka Esme Farah Fatima Felix Freya Gabriel Gustavo Hana Hiroshi Ines Ingrid " + "Isaac Jamal Jasper Javier Joaquin Jonah Kai Kalani Keanu Kenji Kofi Laila Lars Leila Lorenzo Lucia Magnus Malik " + "Marisol Matteo Miguel Mila Minh Nadia Naomi Niamh Nikhil Oluwaseun Oscar Pablo Paloma Quentin Rafael Rania Rohan " + "Rosa Saanvi Santiago Selin Seo-yeon Siddharth Silas Siobhan Sunita Tariq Teodor Thandiwe Tomas Valentina Wren " + "Xiomara Yara Yusuf Zainab Zara Ziad").split() + + +def norm_words(s): return re.sub(r"[^0-9a-z]+", " ", s.lower()).split() + + +def ngrams(words, n): return {" ".join(words[i:i + n]) for i in range(len(words) - n + 1)} + + +def load_bans(): + am = [json.loads(l) for l in open(Path.home() / ".cache/huggingface/hub/datasets--AxiomicLabs--ArithMark-3.0/snapshots/6f6e59dd9b7e2c63455f7af7f838f9ecc3d0a746/arithmark-3.jsonl")] + am_names = {re.match(r"[A-Z][a-z]+", r["ctx"]).group(0) for r in am if re.match(r"[A-Z][a-z]+", r["ctx"])} + am_names -= {"At", "In", "The", "A", "An", "On", "For", "During", "After", "Each", "Every", "Last"} + texts = [r["ctx"] for r in am] + from datasets import load_dataset + texts += [r["Body"] + " " + r["Question"] for r in load_dataset("ChilleD/SVAMP", split="test")] + texts += [r["body"] + " " + r["question"] for r in load_dataset("EleutherAI/asdiv", split="validation")] + ban8 = set().union(*(ngrams(norm_words(t), 8) for t in texts)) + import prepare + return am_names, ban8, prepare.eval_ngrams(lambda *a: None), prepare.contaminated + + +def same_answer(got, want): + """Compare the model's final answer with the computed one, tolerating units, $ signs, + commas and words around the number ("56 meters", "$11.71", "3/4 of the pizza").""" + from fractions import Fraction + g, w = str(got).strip(), str(want) + if re.fullmatch(r"\d{1,2}:\d{2} [AP]M", w): + m = re.search(r"(\d{1,2}:\d{2})\s*([AaPp])\.?\s*[Mm]", g) + return bool(m) and f"{m.group(1)} {m.group(2).upper()}M" == w + g = g.replace(",", "") + if "/" in w: + m = re.findall(r"-?\d+\s*/\s*\d+", g) + try: return bool(m) and Fraction(m[-1].replace(" ", "")) == Fraction(w) + except Exception: return False + nums = re.findall(r"-?\d+(?:\.\d+)?", g) + try: return bool(nums) and abs(float(nums[-1]) - float(w)) < 1e-9 + except Exception: return False + + +def build_prompt(batch, rng): + """batch: list of (spec_a, spec_b). Each item is one scenario with two questions.""" + lines = [] + for i, (sa, sb) in enumerate(batch): + fmt = rng.choice(list(FORMATS)); setting = rng.choice(SETTINGS); names = rng.sample(NAMES, 2) + for x in (sa, sb): x["format"], x["setting"], x["names"] = fmt, setting, names + lines.append(f'{i}. format: {FORMATS[fmt]}. Setting: {setting}. People (use only if needed): {names[0]}, {names[1]}.\n' + f' Question A: grade {sa["grade"]}, topic "{sa["topic"]}", facts: {sa["facts"]}. Correct answer: {sa["answer"]}.\n' + f' Question B: grade {sb["grade"]}, topic "{sb["topic"]}", facts: {sb["facts"]}. Correct answer: {sb["answer"]}.') + return ("You are writing items for an open educational math dataset. For each spec below, write one original item: a short " + "scenario in the given setting followed by two questions, (a) and (b), that use the given facts. Vary wording, sentence " + "structure and length widely between items. Each question must be a direct question ending with a question mark, and the " + "item's text must end with question (b). Use realistic quantities and units. The solution must work through (a) and then " + "(b) step by step, explaining why each step is done, and finish each part with a one-sentence check that the answer is " + "reasonable. Answers must equal the given correct answers. Write plain text only: no markdown, bold, headings or bullet symbols.\n\n" + "\n".join(lines) + + '\n\nReply with only a JSON array, one object per spec in order: ' + '[{"id": 0, "problem": "...", "solution": "...", "answer_a": "...", "answer_b": "..."}, ...]') + + +class Gen: + def __init__(a, out, args): + a.out, a.args = out, args; a.stats = {"requests": 0, "http_errors": 0, "kept": 0, "tokens_out": 0, + "reject": {}} + a.am_names, a.ban8, a.ban13, a.contaminated = load_bans() + a.seen = set(); a.buf = []; a.shard = len(list(out.glob("items-*.jsonl"))) + a.rng = random.Random(args.seed + 1_000_003 * a.shard) + + def rej(a, why): a.stats["reject"][why] = a.stats["reject"].get(why, 0) + 1 + + def answer_of(a, it, keys, sol): + fa = next((it[k] for k in keys if it.get(k) not in (None, "")), None) + return fa + + def check(a, pair, it): + s, s2 = pair + p, sol = (re.sub(r"\*\*|__|^#+\s*", "", str(it.get(k, "")), flags=re.M).strip() for k in ("problem", "solution")) + if not p or not sol: return a.rej("empty") + fa = a.answer_of(it, ("answer_a", "a", "final_answer_a"), sol) + fb = a.answer_of(it, ("answer_b", "b", "final_answer_b"), sol) + if fa is None or fb is None: return a.rej("missing_answer") + if not same_answer(fb, s2["answer"]): return a.rej("wrong_answer") + if not same_answer(fa, s["answer"]): + if a.stats["reject"].get("wrong_answer", 0) < 200: + with open(a.out / "rejects_wrong_answer.jsonl", "a") as f: + f.write(json.dumps({"spec": {k: s[k] for k in ("topic", "facts", "answer")}, "got": fa, + "problem": p[:300]}) + "\n") + return a.rej("wrong_answer") + if not p.rstrip().endswith("?"): return a.rej("not_a_question") + if s.get("format") != "find_the_mistake" and bad_steps(sol): return a.rej("bad_arithmetic_step") + text = p + "\n" + sol; words = norm_words(text) + if set(re.findall(r"[A-Z][a-z]+", text)) & a.am_names: return a.rej("arithmark_name") + if ngrams(words, 8) & a.ban8: return a.rej("test_overlap_8gram") + if a.contaminated(text, a.ban13): return a.rej("eval_13gram") + h = hashlib.sha1(" ".join(norm_words(p)).encode()).hexdigest() + if h in a.seen: return a.rej("duplicate") + a.seen.add(h) + return {"problem": p, "solution": sol, "answers": [s["answer"], s2["answer"]], "grade": max(s["grade"], s2["grade"]), + "topics": [s["topic"], s2["topic"]], "format": s["format"], "setting": s["setting"], "generator": MODEL} + + def flush(a, force=False): + while len(a.buf) >= a.args.shard_size or (force and a.buf): + chunk, a.buf = a.buf[:a.args.shard_size], a.buf[a.args.shard_size:] + with open(a.out / f"items-{a.shard:05d}.jsonl", "w") as f: + for it in chunk: f.write(json.dumps(it) + "\n") + a.shard += 1 + (a.out / "stats.json").write_text(json.dumps(a.stats, indent=2)) + + async def pace(a): + while getattr(a, "paused_until", 0) > time.time(): + a.stats["paused_until"] = time.strftime("%a %I:%M %p", time.localtime(a.paused_until)); a.flush() + await asyncio.sleep(min(300, a.paused_until - time.time())) + a.stats.pop("paused_until", None) + async with a.lock: + now = time.time(); wait = a.next_slot - now + if wait > 0: await asyncio.sleep(wait) + a.next_slot = max(now, a.next_slot) + 1 / a.args.rps + + async def _one(a, sess, sem): + batch = [specs.sample_pair(a.rng) for _ in range(a.args.per_request)] + body = {"model": MODEL, "temperature": 1.0, "top_p": 0.95, "max_tokens": 32000, + **({"chat_template_kwargs": TEMPLATE_KW} if TEMPLATE_KW else {}), + "messages": [{"role": "user", "content": build_prompt(batch, a.rng)}]} + for attempt in range(10): + await a.pace() + try: + async with sess.post(API, json=body, headers={"Authorization": f"Bearer {KEY}"}) as r: + if r.status == 429 and "quota" in (await r.text()).lower(): + ra = r.headers.get("Retry-After") + a.paused_until = max(getattr(a, "paused_until", 0), time.time() + min(1800, float(ra) if ra and ra.replace(".", "").isdigit() else 1800)) + a.stats["quota_pauses"] = a.stats.get("quota_pauses", 0) + 1 + continue + if r.status == 429 or r.status >= 500: + ra = r.headers.get("Retry-After") + a.next_slot = max(a.next_slot, time.time() + (float(ra) if ra and ra.replace(".", "").isdigit() else 1.0)) + a.stats["retries"] = a.stats.get("retries", 0) + 1 + await asyncio.sleep(min(30, 2 ** attempt)); continue + d = await r.json() + break + except Exception: + await asyncio.sleep(2 ** attempt) + else: + a.stats["http_errors"] += 1; return + a.stats["requests"] += 1 + a.stats["tokens_out"] += d.get("usage", {}).get("completion_tokens", 0) + txt = ((d.get("choices") or [{}])[0].get("message") or {}).get("content") or "" + m = re.search(r"\[.*\]", txt, re.S) + try: items = json.loads(m.group(0)) if m else [] + except Exception: items = [] + if not items: return a.rej("bad_json") + for it in items: + try: s = batch[int(it.get("id"))] + except Exception: a.rej("bad_id"); continue + k = a.check(s, it) + if k: a.buf.append(k); a.stats["kept"] += 1 + a.flush() + + async def one(a, sess, sem): + try: + await a._one(sess, sem) + except Exception as e: + a.stats["worker_errors"] = a.stats.get("worker_errors", 0) + 1 + a.stats["last_worker_error"] = repr(e)[:200] + finally: + sem.release() + + async def run(a): + sem = asyncio.Semaphore(a.args.inflight); tasks = set(); t0 = time.time() + a.lock = asyncio.Lock(); a.next_slot = time.time() + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=600)) as sess: + while a.stats["kept"] < a.args.items: + await sem.acquire() + while a.next_slot - time.time() > 2: await asyncio.sleep(0.2) # do not queue far ahead of the pacer + t = asyncio.create_task(a.one(sess, sem)); tasks.add(t); t.add_done_callback(tasks.discard) + if a.stats["requests"] and a.stats["requests"] % 200 == 0: + el = time.time() - t0 + print(f"{time.strftime('%I:%M %p')} req {a.stats['requests']} kept {a.stats['kept']:,} " + f"out-tok {a.stats['tokens_out']/1e6:.1f}M ({a.stats['tokens_out']/el:.0f}/s) rejects {a.stats['reject']}", flush=True) + await asyncio.gather(*tasks) + a.flush(force=True) + + +def main(): + ap = argparse.ArgumentParser(); ap.add_argument("out"); ap.add_argument("--items", type=int, default=1000) + ap.add_argument("--per-request", type=int, default=20); ap.add_argument("--rps", type=float, default=2.8) + ap.add_argument("--inflight", type=int, default=96); ap.add_argument("--seed", type=int, default=20261015) + ap.add_argument("--shard-size", type=int, default=50000) + args = ap.parse_args(); out = Path(args.out); out.mkdir(parents=True, exist_ok=True) + asyncio.run(Gen(out, args).run()) + + +if __name__ == "__main__": + main() diff --git a/libremath/review_fragments.py b/libremath/review_fragments.py new file mode 100644 index 0000000..4352c3a --- /dev/null +++ b/libremath/review_fragments.py @@ -0,0 +1,68 @@ +"""One-time K2 review of the fragment bank (2026-10-23): keep only fragments that read naturally +for ANY items/containers the composer may plug in, and object/container pairs that make sense. + + python libremath/review_fragments.py BANK.json REVIEWED.json +""" +import asyncio, json, re, sys, time +from pathlib import Path +import aiohttp +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import generate as G +from fragments import CATS + +RULE = ("A program fills the curly-brace placeholders: {A}/{B} are people's names, {obj} is ANY plural countable item " + "(e.g. marbles, cookies, stickers), {cont} is ANY plural container or group (e.g. jars, boxes, rows), {place} is a " + "location, and letters like {a} {b} {c} {d} {x} {p} {ans} are numbers.") + + +def ask(cat, frags): + if cat == "objects": + body = "\n".join(f"{i}. {f['obj']} / {f['cont']}" for i, f in enumerate(frags)) + q = ("Each line is a plural countable item and a plural container or grouping for a grade-school math problem " + "(\"there are N with M in each\"). Mark a line bad if the item would not sensibly be packed, " + "arranged or grouped in that container, or if either word is not a plural noun.") + else: + body = "\n".join(f"{i}. {f}" for i, f in enumerate(frags)) + q = (f"{RULE} Each line is meant to be: {CATS[cat][0]}. Mark a line bad if it names or implies specific items, " + "food, animals, actions or imagery that would clash with arbitrary {obj}/{cont}, if it would be ungrammatical " + "with a plural {obj} or {cont}, if its math meaning is wrong for the description, or if it is awkward or confusing.") + return q + "\n\n" + body + "\n\nReply with only a JSON array of the line numbers that are bad." + + +async def main_async(src, dst): + bank = json.load(open(src)); out = {k: [] for k in bank}; sem = asyncio.Semaphore(16); lock = asyncio.Lock(); nxt = [time.time()] + jobs = [(cat, bank[cat][i:i + 40]) for cat in bank for i in range(0, len(bank[cat]), 40)] + + async def run(sess, cat, frags): + async with sem: + body = {"model": G.MODEL, "temperature": 0.2, "max_tokens": 8000, + **({"chat_template_kwargs": G.TEMPLATE_KW} if G.TEMPLATE_KW else {}), + "messages": [{"role": "user", "content": ask(cat, frags)}]} + for attempt in range(8): + async with lock: + w = nxt[0] - time.time() + if w > 0: await asyncio.sleep(w) + nxt[0] = max(time.time(), nxt[0]) + 1.0 + try: + async with sess.post(G.API, json=body, headers={"Authorization": f"Bearer {G.KEY}"}) as r: + if r.status in (429,) or r.status >= 500: await asyncio.sleep(10 * (attempt + 1)); continue + d = await r.json(); break + except Exception: + await asyncio.sleep(5 * (attempt + 1)) + else: + return cat, [], len(frags) + txt = ((d.get("choices") or [{}])[0].get("message") or {}).get("content") or "" + m = re.search(r"\[[\d,\s]*\]", txt) + if not m: return cat, [], len(frags) # unparseable review: drop the batch (be strict) + bad = set(json.loads(m.group(0))) + return cat, [f for i, f in enumerate(frags) if i not in bad], len(frags) + + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=600)) as sess: + for fut in asyncio.as_completed([run(sess, c, f) for c, f in jobs]): + cat, keep, n = await fut; out[cat] += keep + json.dump(out, open(dst, "w"), indent=1) + print({k: f"{len(out[k])}/{len(bank[k])}" for k in bank}) + + +if __name__ == "__main__": + asyncio.run(main_async(sys.argv[1], sys.argv[2])) diff --git a/libremath/specs.py b/libremath/specs.py new file mode 100644 index 0000000..e68dce9 --- /dev/null +++ b/libremath/specs.py @@ -0,0 +1,204 @@ +"""LibreMath problem specs: curriculum topic, grade, the facts a problem must use, and the +exact answer, all computed here. The language model only writes the problem and solution +around these facts; anything it gets wrong is discarded downstream. + +Covers grades 1-6: whole-number operations at several scales, place value, rounding, +comparison, money, elapsed time, measurement and conversion, perimeter/area/volume, +fractions, decimals, data (mean/range/tables), factors and multiples, order of operations, +patterns, ratios, percents and multi-step problems with irrelevant information. +""" +import math, random +from fractions import Fraction + +R = random.Random() + + +def _r(a, b): return R.randint(a, b) + + +def _t(h, m): # minutes since midnight -> "h:mm AM/PM" + h, m = divmod(h * 60 + m, 60) + h %= 24 + return f"{(h % 12) or 12}:{m:02d} {'AM' if h < 12 else 'PM'}" + + +def add_sub(): + scale = R.choice([20, 100, 1000, 10000]) + a, b = _r(scale // 10, scale), _r(1, scale) + if R.random() < 0.5: + return 2 if scale <= 100 else 3, "addition", f"{a} and {b} are combined", a + b + a, b = max(a, b), min(a, b) + return 2 if scale <= 100 else 3, "subtraction", f"start with {a}, remove or compare {b}", a - b + + +def mult(): + a, b = R.choice([(_r(2, 12), _r(2, 12)), (_r(11, 99), _r(2, 9)), (_r(12, 99), _r(11, 40))]) + return 3 if max(a, b) <= 12 else 4, "multiplication", f"{a} groups of {b}", a * b + + +def div(): + b = _r(2, 12); q = _r(2, 30 if R.random() < 0.7 else 120); rem = _r(0, b - 1) if R.random() < 0.3 else 0 + a = b * q + rem + if rem: + kind = R.choice(["quotient", "remainder", "round_up"]) + ans = {"quotient": q, "remainder": rem, "round_up": q + 1}[kind] + how = {"quotient": "how many full groups", "remainder": "how many are left over", + "round_up": "how many groups are needed so none are left out"}[kind] + return 4, "division with remainder", f"{a} items split into groups of {b}; ask {how}", ans + return 3, "division", f"{a} items shared equally into {b} groups", q + + +def place_value(): + n = _r(1000, 999999); d = R.choice(["thousands", "hundreds", "tens"]) + v = {"thousands": 1000, "hundreds": 100, "tens": 10}[d] + if R.random() < 0.5: + return 4, "rounding", f"round {n} to the nearest {d[:-1]}", int(math.floor(n / v + 0.5) * v) + return 4, "place value", f"the value of the {d} digit in {n}", (n // v % 10) * v + + +def money(): + items = _r(2, 4); prices = [Fraction(_r(25, 1999), 100) for _ in range(items)] + paid = Fraction(math.ceil(float(sum(prices))) + R.choice([0, 1, 5, 10, 20]), 1) + if R.random() < 0.5: + return 3, "money", "items costing " + ", ".join(f"${float(p):.2f}" for p in prices) + " are bought; find the total", f"{float(sum(prices)):.2f}" + return 4, "making change", "items costing " + ", ".join(f"${float(p):.2f}" for p in prices) + f" are paid for with ${float(paid):.2f}; find the change", f"{float(paid - sum(prices)):.2f}" + + +def elapsed_time(): + h, m = _r(6, 18), R.choice(range(0, 60, 5)); d = R.choice([15, 20, 25, 35, 40, 45, 50, 70, 85, 95, 110, 135]) + if R.random() < 0.5: + return 3, "elapsed time", f"starts at {_t(h, m)} and lasts {d} minutes; find the end time", _t(h, m + d) + return 3, "elapsed time", f"starts at {_t(h, m)} and ends at {_t(h, m + d)}; find how many minutes", d + + +def measurement(): + kind = R.choice(["m_cm", "kg_g", "l_ml", "ft_in", "hr_min", "yd_ft"]) + big, small, k = {"m_cm": ("meters", "centimeters", 100), "kg_g": ("kilograms", "grams", 1000), + "l_ml": ("liters", "milliliters", 1000), "ft_in": ("feet", "inches", 12), + "hr_min": ("hours", "minutes", 60), "yd_ft": ("yards", "feet", 3)}[kind] + a, b = _r(1, 12), _r(1, k - 1) + return 4, "unit conversion", f"{a} {big} and {b} {small}, expressed in {small}", a * k + b + + +def geometry(): + kind = R.choice(["perim_rect", "area_rect", "area_l", "volume", "perim_poly", "missing_side"]) + l, w, h = _r(2, 30), _r(2, 20), _r(2, 12) + if kind == "perim_rect": return 3, "perimeter", f"a rectangle {l} by {w}", 2 * (l + w) + if kind == "area_rect": return 3, "area", f"a rectangle {l} by {w}", l * w + if kind == "volume": return 5, "volume", f"a box {l} by {w} by {h}", l * w * h + if kind == "perim_poly": + s = [_r(3, 25) for _ in range(_r(3, 6))] + return 3, "perimeter", "a shape with sides " + ", ".join(map(str, s)), sum(s) + if kind == "missing_side": + return 4, "area", f"a rectangle with area {l * w} and one side {l}; find the other side", w + a, b = _r(2, 10), _r(2, 10) + return 4, "area", f"an L-shape made of a {l} by {w} rectangle and a {a} by {b} rectangle", l * w + a * b + + +def fractions(): + kind = R.choice(["of_whole", "add_like", "sub_like", "compare", "equiv"]) + d = R.choice([2, 3, 4, 5, 6, 8, 10, 12]) + if kind == "of_whole": + n = _r(1, d - 1); k = _r(2, 12) * d + return 4, "fraction of a number", f"{n}/{d} of {k}", n * k // d + a, b = _r(1, d - 1), _r(1, d - 1) + if kind == "add_like": + s = Fraction(a + b, d); return 4, "adding fractions", f"{a}/{d} plus {b}/{d}, simplified", str(s) + if kind == "sub_like": + a, b = max(a, b), min(a, b); s = Fraction(a - b, d) + return 4, "subtracting fractions", f"{a}/{d} minus {b}/{d}, simplified", str(s) + if kind == "equiv": + k = _r(2, 6); return 4, "equivalent fractions", f"{a}/{d} written with denominator {d * k}; find the numerator", a * k + d2 = R.choice([2, 3, 4, 5, 6, 8]); c = _r(1, d2 - 1) + if Fraction(a, d) == Fraction(c, d2): c = (c % (d2 - 1)) + 1 + big = f"{a}/{d}" if Fraction(a, d) > Fraction(c, d2) else f"{c}/{d2}" + return 4, "comparing fractions", f"which is larger, {a}/{d} or {c}/{d2}", big + + +def decimals(): + a, b = Fraction(_r(10, 9999), 100), Fraction(_r(10, 9999), 100) + if R.random() < 0.5: return 5, "decimal addition", f"{float(a):.2f} and {float(b):.2f} combined", f"{float(a + b):.2f}" + a, b = max(a, b), min(a, b); return 5, "decimal subtraction", f"{float(a):.2f} minus {float(b):.2f}", f"{float(a - b):.2f}" + + +def data(): + k = _r(4, 7); xs = [_r(1, 60) for _ in range(k)] + kind = R.choice(["mean", "range", "total", "diff"]) + if kind == "mean": + xs[-1] += (-sum(xs)) % k + return 5, "mean", "values " + ", ".join(map(str, xs)), sum(xs) // k + if kind == "range": return 5, "range", "values " + ", ".join(map(str, xs)), max(xs) - min(xs) + if kind == "total": return 3, "reading a table", "a table with values " + ", ".join(map(str, xs)) + "; find the total", sum(xs) + return 3, "reading a table", "a table with values " + ", ".join(map(str, xs)) + "; find the difference between the largest and smallest category", max(xs) - min(xs) + + +def factors(): + kind = R.choice(["lcm", "gcf", "count_factors", "multiple"]) + a, b = _r(2, 15), _r(2, 15) + if kind == "lcm": return 5, "least common multiple", f"two things repeat every {a} and every {b}; when do they first match", a * b // math.gcd(a, b) + if kind == "gcf": + a, b = a * _r(1, 6), b * _r(1, 6); return 5, "greatest common factor", f"{a} and {b} split into the largest equal groups", math.gcd(a, b) + if kind == "count_factors": + n = _r(6, 100); return 4, "factors", f"how many factors {n} has", sum(1 for i in range(1, n + 1) if n % i == 0) + n, k = _r(3, 12), _r(4, 15); return 3, "multiples", f"the {k}th multiple of {n}", n * k + + +def order_ops(): + a, b, c, d = _r(2, 20), _r(2, 12), _r(2, 12), _r(1, 30) + expr = R.choice([f"{a} + {b} × {c}", f"({a} + {b}) × {c}", f"{a} × {b} - {d}", f"{a * b} ÷ {b} + {c}", f"{a} + {b} × {c} - {d}"]) + val = eval(expr.replace("×", "*").replace("÷", "//")) + return 5, "order of operations", f"the expression {expr}", val + + +def patterns(): + s, d, n = _r(1, 80), R.choice([2, 3, 4, 5, 6, 7, 9, 11, 12, 15, 25]), _r(5, 10) + if R.random() < 0.3: d = -d; s += abs(d) * (n + 2) + seq = [s + d * i for i in range(4)] + return 3, "number patterns", "the sequence " + ", ".join(map(str, seq)) + f"; find term number {n}", s + d * (n - 1) + + +def ratio_percent(): + kind = R.choice(["ratio", "percent", "unit_rate", "discount"]) + if kind == "ratio": + a, b, k = _r(1, 9), _r(1, 9), _r(2, 12); return 6, "ratios", f"a {a}:{b} ratio, with {a * k} of the first kind; find the second", b * k + if kind == "percent": + p = R.choice([10, 20, 25, 30, 40, 50, 60, 75, 80]); n = R.choice([20, 40, 60, 80, 120, 200, 240, 400]) + return 6, "percent of a number", f"{p}% of {n}", p * n // 100 + if kind == "unit_rate": + r, t = _r(2, 15), _r(2, 12); return 6, "unit rate", f"{r * t} done in {t} equal parts; find the amount per part", r + p = R.choice([10, 20, 25, 50]); price = R.choice([20, 40, 60, 80, 100, 120]) + return 6, "discounts", f"a ${price} item with {p}% off; find the sale price", price - price * p // 100 + + +def multi_step(): + a, b, c, d = _r(3, 12), _r(4, 25), _r(5, 60), _r(2, 9) + form = R.choice([(f"{a} packs of {b}, then {c} more are added", a * b + c), + (f"{a} packs of {b}, then {c} are given away", a * b - c if a * b > c else a * b + c), + (f"{a * d} items split into {d} equal groups, then each group gets {b} more", a + b), + (f"saving ${b} a week for {a} weeks, then spending ${c}", a * b - c if a * b > c else a * b + c)]) + return R.choice([3, 4, 5]), "multi-step", form[0] + "; include one irrelevant number", form[1] + + +TOPICS = [(add_sub, 14), (mult, 9), (div, 8), (place_value, 4), (money, 7), (elapsed_time, 5), (measurement, 6), + (geometry, 8), (fractions, 8), (decimals, 4), (data, 5), (factors, 5), (order_ops, 3), (patterns, 4), + (ratio_percent, 5), (multi_step, 9)] + + +def sample(rng=None): + global R + if rng is not None: R = rng + fn = R.choices([t for t, _ in TOPICS], weights=[w for _, w in TOPICS])[0] + grade, topic, facts, ans = fn() + return {"grade": grade, "topic": topic, "facts": facts, "answer": str(ans)} + + +def sample_pair(rng=None): + """Two specs from the same topic family, so one scenario can hold both questions.""" + global R + if rng is not None: R = rng + fn = R.choices([t for t, _ in TOPICS], weights=[w for _, w in TOPICS])[0] + out = [] + for _ in range(2): + grade, topic, facts, ans = fn() + out.append({"grade": grade, "topic": topic, "facts": facts, "answer": str(ans)}) + return tuple(out) diff --git a/libremath/tgen.py b/libremath/tgen.py new file mode 100644 index 0000000..2bd7975 --- /dev/null +++ b/libremath/tgen.py @@ -0,0 +1,261 @@ +"""LibreMath template generator (prototype, 2026-10-15). + +K2 writes problem TEMPLATES rather than problems: a two-part scenario and worked solution +with {placeholders}, integer ranges for each variable, formulas for every derived value, +constraints the numbers must satisfy, and one worked example. Code then + 1. checks the formulas reproduce the example's answers (catches wrong math in the template), + 2. fills the template --fills times with fresh numbers and names, computing every value, + 3. runs each filled item through the same filters as generate.py. + + python libremath/tgen.py OUT_DIR --templates 200 [--per-request 30] [--fills 10] +""" +import argparse, ast, asyncio, hashlib, json, math, operator, random, re, sys, time +from pathlib import Path +import aiohttp + +ROOT = Path(__file__).resolve().parent +sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT.parent)) +import generate as G +from arithcheck import bad_steps + +TOPICS = ["addition with regrouping", "subtraction with borrowing", "multiplication", "division with and without remainders", + "money and making change", "elapsed time in minutes", "unit conversion", "perimeter", "area", "volume", + "fractions of a quantity", "comparing and ordering numbers", "place value and rounding", "mean and range of data", + "factors and multiples", "number patterns", "ratios and rates", "percents and discounts", "multi-step problems", + "problems with extra information that is not needed"] + +SETTINGS_EXTRA = ["lighthouse keeper's log", "food truck", "bike repair shop", "bird-watching trip", "school band rehearsal", "farmers' co-op", + "ice rink", "pottery class", "train station", "chess club", "dog shelter", "beekeeping farm", "tailor's shop", "rock-climbing gym", + "community radio station", "ferry crossing", "orchard harvest", "board-game café", "museum gift shop", "volunteer fire station", + "robotics team", "knitting circle", "camping weekend", "bowling league", "hardware store", "space camp", "tide pool walk", + "mail sorting room", "theater set building", "kite festival", "snow-day sledding hill", "zoo feeding schedule", "puppet show", + "bus route planning", "recipe test kitchen", "weather station", "carpentry workshop", "town election count", "ballet recital", + "fishing pier", "recycling plant tour", "swim meet", "solar panel installation", "art gallery", "vegetable stand", "map-making club", + "quilt fair", "laundromat", "apartment move", "family reunion", "neighborhood cleanup", "printing press", "coffee roastery", + "veterinary clinic", "skate park", "sailing lesson", "film club", "bakery delivery route", "science museum planetarium"] +OPENINGS = ["start with a person's name", "start with a time or day", "start with a short quotation from someone", "start with a fact about the setting", + "start with an instruction to the reader", "start with a small table or list of numbers", "start with a question", "start in the middle of an action"] +NQ = [1, 1, 1, 2, 2, 3] # questions per item: half single-question (like most word-problem sets), some 2- and 3-part + + +# --- safe arithmetic --------------------------------------------------------- # +OPS = {ast.Add: operator.add, ast.Sub: operator.sub, ast.Mult: operator.mul, ast.Div: operator.truediv, + ast.FloorDiv: operator.floordiv, ast.Mod: operator.mod} +CMP = {ast.Eq: operator.eq, ast.NotEq: operator.ne, ast.Lt: operator.lt, ast.LtE: operator.le, + ast.Gt: operator.gt, ast.GtE: operator.ge} +def _round(x, n=0): # half-up, as taught in school (Python's round() is banker's rounding: round(25, -1) == 20) + v = math.floor(x * 10 ** n + 0.5) / 10 ** n + return int(v) if n <= 0 else v + + +FUNCS = {"min": min, "max": max, "round": _round, "abs": abs} + + +def ev(expr, env): + def go(n): + if isinstance(n, ast.Expression): return go(n.body) + if isinstance(n, ast.Constant) and isinstance(n.value, (int, float)): return n.value + if isinstance(n, ast.Name): return env[n.id] + if isinstance(n, ast.BinOp) and type(n.op) in OPS: return OPS[type(n.op)](go(n.left), go(n.right)) + if isinstance(n, ast.UnaryOp) and isinstance(n.op, ast.USub): return -go(n.operand) + if isinstance(n, ast.Compare): + l = go(n.left) + for op, c in zip(n.ops, n.comparators): + r = go(c) + if not CMP[type(op)](l, r): return False + l = r + return True + if isinstance(n, ast.BoolOp): + vals = [go(v) for v in n.values] + return all(vals) if isinstance(n.op, ast.And) else any(vals) + if isinstance(n, ast.Call) and isinstance(n.func, ast.Name) and n.func.id in FUNCS: + return FUNCS[n.func.id](*[go(a) for a in n.args]) + raise ValueError(f"disallowed: {ast.dump(n)[:60]}") + expr = str(expr).replace("||", " or ").replace("&&", " and ") + return go(ast.parse(expr, mode="eval")) + + +def fmt(v): + if isinstance(v, float): + if abs(v - round(v)) < 1e-9: return str(int(round(v))) + return f"{v:.2f}" + return str(v) + + +PH = re.compile(r"\{(\w+)\}") + + +def render(text, env): # noqa: E302 + return PH.sub(lambda m: fmt(env[m.group(1)]) if m.group(1) in env else m.group(0), text) + + +def sample_env(t, rng, tries=200): + for _ in range(tries): + env = {} + for k, (lo, hi) in t["variables"].items(): env[k] = rng.randint(int(lo), int(hi)) + for k, f in t["derived"].items(): env[k] = ev(f, env) + if all(ev(c, env) for c in t.get("constraints", [])) and all( + (isinstance(env[k], int) or abs(env[k] * 100 - round(env[k] * 100)) < 1e-6) and env[k] >= 0 for k in t["derived"]): + return env + return None + + +def build_prompt(specs): + lines = [] + for i, s in enumerate(specs): + qs = "one question" if s["nq"] == 1 else f"{s['nq']} questions labelled " + ", ".join(f"({c})" for c in "abc"[:s["nq"]]) + lines.append(f'{i}. grade {s["grade"]}, topic: {s["topic"]}, format: {G.FORMATS[s["format"]]}, setting: {s["setting"]}; ' + f'{qs}; {s["opening"]}' + ("; end each part of the solution with a one-sentence check" if s["check"] else "")) + return ("You are writing reusable TEMPLATES for an open educational math dataset. For each spec, write one template: a short " + "scenario followed by the requested question(s), plus a worked step-by-step solution that explains each step. Every " + "number in the problem and solution must be a {placeholder}: input variables with an integer range, and derived values " + "defined by formulas over other placeholders using + - * / // % min max round. Never write a computed number literally. " + "Use {name1} and {name2} for people. You may also add word placeholders for objects or units, listing 4-6 interchangeable " + "options under \"words\" (e.g. {\"snack\": [\"muffins\", \"bagels\", \"scones\", \"rolls\"]}). The problem text must end " + "with the last question and a question mark. Plain text only, no markdown. Make every template different in wording, " + "sentence structure and length; do not reuse openings like \"At the\" or \"In the\".\n\n" + "\n".join(lines) + + '\n\nReply with only a JSON array, one object per spec in order:\n' + '[{"id": 0, "problem": "...{name1}...{a} {snack}...?", "solution": "...{a} x {b} = {total}...", ' + '"variables": {"a": [2, 12], "b": [3, 9]}, "words": {"snack": ["muffins", "bagels", "scones", "rolls"]}, ' + '"derived": {"total": "a*b"}, "constraints": ["total > 10"], "answers": ["total"], ' + '"example": {"values": {"a": 4, "b": 6}, "answers": [24]}}, ...]') + + +class TGen(G.Gen): + def __init__(a, out, args): + a.args = args; a.out = out + a.stats = {"requests": 0, "http_errors": 0, "templates_ok": 0, "templates_bad": 0, "kept": 0, "tokens_out": 0, "reject": {}} + a.am_names, a.ban8, a.ban13, a.contaminated = G.load_bans() + a.seen = set(); a.buf = []; a.shard = len(list(out.glob("items-*.jsonl"))) + a.rng = random.Random(args.seed + 1_000_003 * a.shard) + a.tpl_out = open(out / "templates.jsonl", "a") + + def check_template(a, t): + try: + for k in ("problem", "solution", "variables", "derived", "answers", "example"): t[k] + if not t["problem"].rstrip().endswith("?"): return a.rej("tpl_not_question") + names = set(PH.findall(t["problem"] + t["solution"])) - {"name1", "name2"} + if not names <= set(t["variables"]) | set(t["derived"]) | set(t.get("words") or {}): return a.rej("tpl_unknown_placeholder") + if re.search(r"(? 1 and not all(f"({c})" in t["problem"] for c in "abc"[:nq]): return a.rej("tpl_missing_parts") + if len(t["answers"]) != nq or not all("{" + k + "}" in t["solution"] for k in t["answers"]): return a.rej("tpl_answer_not_in_solution") + words = t.get("words") or {} + if not isinstance(words, dict) or any(not isinstance(v, list) or len(v) < 2 for v in words.values()): return a.rej("tpl_bad_words") + consts = {str(n.value) for f in list(t["derived"].values()) + list(t.get("constraints", [])) + for n in ast.walk(ast.parse(str(f).replace("||", " or ").replace("&&", " and "), mode="eval")) + if isinstance(n, ast.Constant) and isinstance(n.value, (int, float))} + ok = consts | {str(i) for i in range(13)} | {"24", "60", "100", "1000", "365", "52"} + sol_lits = re.findall(r"(? 1e-6 for g, w in zip(got, want)): return a.rej("tpl_example_mismatch") + return True + except Exception: + return a.rej("tpl_invalid") + + def fill(a, t, spec): + out = [] + for _ in range(a.args.fills): + env = sample_env(t, a.rng) + if env is None: a.rej("fill_unsatisfiable"); break + n1, n2 = a.rng.sample(G.NAMES, 2); env.update(name1=n1, name2=n2) + for k, opts in (t.get("words") or {}).items(): + if isinstance(opts, list) and opts: env[k] = a.rng.choice([str(o) for o in opts]) + if len(t["answers"]) > 1 and len({fmt(env[k]) for k in t["answers"]}) < len(t["answers"]): a.rej("fill_same_answers"); continue + if any(env[k] in [env.get(v) for v in t["variables"]] for k in t["answers"]): a.rej("fill_answer_is_input"); continue + p, sol = (re.sub(r"(?<=\d) \* (?=\d)", " × ", render(t[k], env)) for k in ("problem", "solution")) + if PH.search(p + sol): a.rej("fill_leftover_placeholder"); continue + text = p + "\n" + sol; words = G.norm_words(text) + if bad_steps(sol) and spec["format"] != "find_the_mistake": a.rej("fill_bad_step"); continue + if set(re.findall(r"[A-Z][a-z]+", text)) & a.am_names: a.rej("arithmark_name"); continue + if G.ngrams(words, 8) & a.ban8: a.rej("test_overlap_8gram"); continue + if a.contaminated(text, a.ban13): a.rej("eval_13gram"); continue + h = hashlib.sha1(" ".join(G.norm_words(p)).encode()).hexdigest() + if h in a.seen: a.rej("duplicate"); continue + a.seen.add(h) + out.append({"problem": p, "solution": sol, "answers": [fmt(env[k]) for k in t["answers"]], + "grade": spec["grade"], "topics": [spec["topic"]], "format": spec["format"], "setting": spec["setting"], + "generator": G.MODEL + " template", "template_id": t["_tid"]}) + return out + + async def _one(a, sess, sem): + specs = [{"grade": a.rng.randint(1, 6), "topic": a.rng.choice(TOPICS), "format": a.rng.choice(list(G.FORMATS)), + "setting": a.rng.choice(G.SETTINGS + SETTINGS_EXTRA), "nq": a.rng.choice(NQ), "opening": a.rng.choice(OPENINGS), + "check": a.rng.random() < 0.3} for _ in range(a.args.per_request)] + body = {"model": G.MODEL, "temperature": 1.0, "top_p": 0.95, "max_tokens": 32000, + **({"chat_template_kwargs": G.TEMPLATE_KW} if G.TEMPLATE_KW else {}), + "messages": [{"role": "user", "content": build_prompt(specs)}]} + d = None + for attempt in range(10): + await a.pace() + try: + async with sess.post(G.API, json=body, headers={"Authorization": f"Bearer {G.KEY}"}) as r: + if r.status == 429 and "quota" in (await r.text()).lower(): + ra = r.headers.get("Retry-After") + a.paused_until = time.time() + min(1800, float(ra) if ra and ra.replace(".", "").isdigit() else 1800); continue + if r.status == 429 or r.status >= 500: + a.next_slot = max(a.next_slot, time.time() + 1.0); await asyncio.sleep(min(30, 2 ** attempt)); continue + d = await r.json(); break + except Exception: + await asyncio.sleep(2 ** attempt) + if d is None: a.stats["http_errors"] += 1; return + a.stats["requests"] += 1; a.stats["tokens_out"] += d.get("usage", {}).get("completion_tokens", 0) + txt = ((d.get("choices") or [{}])[0].get("message") or {}).get("content") or "" + # Salvage every complete object even if the array is malformed or truncated. + dec, tpls, i = json.JSONDecoder(), [], 0 + while (i := txt.find('{"id"', i)) != -1: + try: o, j = dec.raw_decode(txt, i); tpls.append(o); i = j + except Exception: i += 5 + if not tpls: + if a.stats.get("raw_saved", 0) < 5: + a.stats["raw_saved"] = a.stats.get("raw_saved", 0) + 1 + (a.out / f"raw_bad_{a.stats['raw_saved']}.txt").write_text(txt[:20000]) + return a.rej("bad_json") + a.stats["objects_parsed"] = a.stats.get("objects_parsed", 0) + len(tpls) + for t in tpls: + try: spec = specs[int(t.get("id"))] + except Exception: a.rej("bad_id"); continue + t["_topic"] = spec["topic"]; t["_nq"] = spec["nq"] + if a.check_template(t) is not True: a.stats["templates_bad"] += 1; continue + t["_tid"] = hashlib.sha1(json.dumps(t, sort_keys=True).encode()).hexdigest()[:12] + try: items = a.fill(t, spec) + except Exception: a.rej("fill_error"); items = [] + if items: + a.stats["templates_ok"] += 1; a.tpl_out.write(json.dumps({**t, "spec": spec}) + "\n"); a.tpl_out.flush() + a.buf.extend(items); a.stats["kept"] += len(items) + a.flush() + + async def run(a): + sem = asyncio.Semaphore(a.args.inflight); tasks = set() + a.lock = asyncio.Lock(); a.next_slot = time.time() + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=900)) as sess: + while a.stats["templates_ok"] < a.args.templates and a.stats["requests"] + len(tasks) < a.args.max_requests: + await sem.acquire() + while a.next_slot - time.time() > 2: await asyncio.sleep(0.2) + t = asyncio.create_task(a.one(sess, sem)); tasks.add(t); t.add_done_callback(tasks.discard) + await asyncio.gather(*tasks) + a.flush(force=True) + + +def main(): + ap = argparse.ArgumentParser(); ap.add_argument("out"); ap.add_argument("--templates", type=int, default=200) + ap.add_argument("--per-request", type=int, default=30); ap.add_argument("--fills", type=int, default=10) + ap.add_argument("--max-requests", type=int, default=10**9) + ap.add_argument("--rps", type=float, default=2.8); ap.add_argument("--inflight", type=int, default=64) + ap.add_argument("--seed", type=int, default=20261018); ap.add_argument("--shard-size", type=int, default=50000) + args = ap.parse_args(); out = Path(args.out); out.mkdir(parents=True, exist_ok=True) + asyncio.run(TGen(out, args).run()) + + +if __name__ == "__main__": + main() -- cgit v1.2.3