diff options
Diffstat (limited to 'libremath/tgen.py')
| -rw-r--r-- | libremath/tgen.py | 261 |
1 files changed, 261 insertions, 0 deletions
diff --git a/libremath/tgen.py b/libremath/tgen.py new file mode 100644 index 0000000..2bd7975 --- /dev/null +++ b/libremath/tgen.py @@ -0,0 +1,261 @@ +"""LibreMath template generator (prototype, 2026-10-15). + +K2 writes problem TEMPLATES rather than problems: a two-part scenario and worked solution +with {placeholders}, integer ranges for each variable, formulas for every derived value, +constraints the numbers must satisfy, and one worked example. Code then + 1. checks the formulas reproduce the example's answers (catches wrong math in the template), + 2. fills the template --fills times with fresh numbers and names, computing every value, + 3. runs each filled item through the same filters as generate.py. + + python libremath/tgen.py OUT_DIR --templates 200 [--per-request 30] [--fills 10] +""" +import argparse, ast, asyncio, hashlib, json, math, operator, random, re, sys, time +from pathlib import Path +import aiohttp + +ROOT = Path(__file__).resolve().parent +sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT.parent)) +import generate as G +from arithcheck import bad_steps + +TOPICS = ["addition with regrouping", "subtraction with borrowing", "multiplication", "division with and without remainders", + "money and making change", "elapsed time in minutes", "unit conversion", "perimeter", "area", "volume", + "fractions of a quantity", "comparing and ordering numbers", "place value and rounding", "mean and range of data", + "factors and multiples", "number patterns", "ratios and rates", "percents and discounts", "multi-step problems", + "problems with extra information that is not needed"] + +SETTINGS_EXTRA = ["lighthouse keeper's log", "food truck", "bike repair shop", "bird-watching trip", "school band rehearsal", "farmers' co-op", + "ice rink", "pottery class", "train station", "chess club", "dog shelter", "beekeeping farm", "tailor's shop", "rock-climbing gym", + "community radio station", "ferry crossing", "orchard harvest", "board-game café", "museum gift shop", "volunteer fire station", + "robotics team", "knitting circle", "camping weekend", "bowling league", "hardware store", "space camp", "tide pool walk", + "mail sorting room", "theater set building", "kite festival", "snow-day sledding hill", "zoo feeding schedule", "puppet show", + "bus route planning", "recipe test kitchen", "weather station", "carpentry workshop", "town election count", "ballet recital", + "fishing pier", "recycling plant tour", "swim meet", "solar panel installation", "art gallery", "vegetable stand", "map-making club", + "quilt fair", "laundromat", "apartment move", "family reunion", "neighborhood cleanup", "printing press", "coffee roastery", + "veterinary clinic", "skate park", "sailing lesson", "film club", "bakery delivery route", "science museum planetarium"] +OPENINGS = ["start with a person's name", "start with a time or day", "start with a short quotation from someone", "start with a fact about the setting", + "start with an instruction to the reader", "start with a small table or list of numbers", "start with a question", "start in the middle of an action"] +NQ = [1, 1, 1, 2, 2, 3] # questions per item: half single-question (like most word-problem sets), some 2- and 3-part + + +# --- safe arithmetic --------------------------------------------------------- # +OPS = {ast.Add: operator.add, ast.Sub: operator.sub, ast.Mult: operator.mul, ast.Div: operator.truediv, + ast.FloorDiv: operator.floordiv, ast.Mod: operator.mod} +CMP = {ast.Eq: operator.eq, ast.NotEq: operator.ne, ast.Lt: operator.lt, ast.LtE: operator.le, + ast.Gt: operator.gt, ast.GtE: operator.ge} +def _round(x, n=0): # half-up, as taught in school (Python's round() is banker's rounding: round(25, -1) == 20) + v = math.floor(x * 10 ** n + 0.5) / 10 ** n + return int(v) if n <= 0 else v + + +FUNCS = {"min": min, "max": max, "round": _round, "abs": abs} + + +def ev(expr, env): + def go(n): + if isinstance(n, ast.Expression): return go(n.body) + if isinstance(n, ast.Constant) and isinstance(n.value, (int, float)): return n.value + if isinstance(n, ast.Name): return env[n.id] + if isinstance(n, ast.BinOp) and type(n.op) in OPS: return OPS[type(n.op)](go(n.left), go(n.right)) + if isinstance(n, ast.UnaryOp) and isinstance(n.op, ast.USub): return -go(n.operand) + if isinstance(n, ast.Compare): + l = go(n.left) + for op, c in zip(n.ops, n.comparators): + r = go(c) + if not CMP[type(op)](l, r): return False + l = r + return True + if isinstance(n, ast.BoolOp): + vals = [go(v) for v in n.values] + return all(vals) if isinstance(n.op, ast.And) else any(vals) + if isinstance(n, ast.Call) and isinstance(n.func, ast.Name) and n.func.id in FUNCS: + return FUNCS[n.func.id](*[go(a) for a in n.args]) + raise ValueError(f"disallowed: {ast.dump(n)[:60]}") + expr = str(expr).replace("||", " or ").replace("&&", " and ") + return go(ast.parse(expr, mode="eval")) + + +def fmt(v): + if isinstance(v, float): + if abs(v - round(v)) < 1e-9: return str(int(round(v))) + return f"{v:.2f}" + return str(v) + + +PH = re.compile(r"\{(\w+)\}") + + +def render(text, env): # noqa: E302 + return PH.sub(lambda m: fmt(env[m.group(1)]) if m.group(1) in env else m.group(0), text) + + +def sample_env(t, rng, tries=200): + for _ in range(tries): + env = {} + for k, (lo, hi) in t["variables"].items(): env[k] = rng.randint(int(lo), int(hi)) + for k, f in t["derived"].items(): env[k] = ev(f, env) + if all(ev(c, env) for c in t.get("constraints", [])) and all( + (isinstance(env[k], int) or abs(env[k] * 100 - round(env[k] * 100)) < 1e-6) and env[k] >= 0 for k in t["derived"]): + return env + return None + + +def build_prompt(specs): + lines = [] + for i, s in enumerate(specs): + qs = "one question" if s["nq"] == 1 else f"{s['nq']} questions labelled " + ", ".join(f"({c})" for c in "abc"[:s["nq"]]) + lines.append(f'{i}. grade {s["grade"]}, topic: {s["topic"]}, format: {G.FORMATS[s["format"]]}, setting: {s["setting"]}; ' + f'{qs}; {s["opening"]}' + ("; end each part of the solution with a one-sentence check" if s["check"] else "")) + return ("You are writing reusable TEMPLATES for an open educational math dataset. For each spec, write one template: a short " + "scenario followed by the requested question(s), plus a worked step-by-step solution that explains each step. Every " + "number in the problem and solution must be a {placeholder}: input variables with an integer range, and derived values " + "defined by formulas over other placeholders using + - * / // % min max round. Never write a computed number literally. " + "Use {name1} and {name2} for people. You may also add word placeholders for objects or units, listing 4-6 interchangeable " + "options under \"words\" (e.g. {\"snack\": [\"muffins\", \"bagels\", \"scones\", \"rolls\"]}). The problem text must end " + "with the last question and a question mark. Plain text only, no markdown. Make every template different in wording, " + "sentence structure and length; do not reuse openings like \"At the\" or \"In the\".\n\n" + "\n".join(lines) + + '\n\nReply with only a JSON array, one object per spec in order:\n' + '[{"id": 0, "problem": "...{name1}...{a} {snack}...?", "solution": "...{a} x {b} = {total}...", ' + '"variables": {"a": [2, 12], "b": [3, 9]}, "words": {"snack": ["muffins", "bagels", "scones", "rolls"]}, ' + '"derived": {"total": "a*b"}, "constraints": ["total > 10"], "answers": ["total"], ' + '"example": {"values": {"a": 4, "b": 6}, "answers": [24]}}, ...]') + + +class TGen(G.Gen): + def __init__(a, out, args): + a.args = args; a.out = out + a.stats = {"requests": 0, "http_errors": 0, "templates_ok": 0, "templates_bad": 0, "kept": 0, "tokens_out": 0, "reject": {}} + a.am_names, a.ban8, a.ban13, a.contaminated = G.load_bans() + a.seen = set(); a.buf = []; a.shard = len(list(out.glob("items-*.jsonl"))) + a.rng = random.Random(args.seed + 1_000_003 * a.shard) + a.tpl_out = open(out / "templates.jsonl", "a") + + def check_template(a, t): + try: + for k in ("problem", "solution", "variables", "derived", "answers", "example"): t[k] + if not t["problem"].rstrip().endswith("?"): return a.rej("tpl_not_question") + names = set(PH.findall(t["problem"] + t["solution"])) - {"name1", "name2"} + if not names <= set(t["variables"]) | set(t["derived"]) | set(t.get("words") or {}): return a.rej("tpl_unknown_placeholder") + if re.search(r"(?<![{\w])\d{2,}(?![\w}])", re.sub(r"\{\w+\}", "", t["problem"])): return a.rej("tpl_literal_number") + nq = t["_nq"] + if nq > 1 and not all(f"({c})" in t["problem"] for c in "abc"[:nq]): return a.rej("tpl_missing_parts") + if len(t["answers"]) != nq or not all("{" + k + "}" in t["solution"] for k in t["answers"]): return a.rej("tpl_answer_not_in_solution") + words = t.get("words") or {} + if not isinstance(words, dict) or any(not isinstance(v, list) or len(v) < 2 for v in words.values()): return a.rej("tpl_bad_words") + consts = {str(n.value) for f in list(t["derived"].values()) + list(t.get("constraints", [])) + for n in ast.walk(ast.parse(str(f).replace("||", " or ").replace("&&", " and "), mode="eval")) + if isinstance(n, ast.Constant) and isinstance(n.value, (int, float))} + ok = consts | {str(i) for i in range(13)} | {"24", "60", "100", "1000", "365", "52"} + sol_lits = re.findall(r"(?<![{\w.])\d+(?:\.\d+)?(?![\w}])", re.sub(r"\{\w+\}", "", t["solution"])) + if any(x not in ok for x in sol_lits): return a.rej("tpl_solution_literal") + txt = t["problem"] + " " + t["solution"] + if re.search(r"//|\*\*|\{\w+\}\s*%|\s%\s", txt): return a.rej("tpl_code_operator") + if "nearest {" in txt: return a.rej("tpl_round_to_variable") + if re.search(r"\{\w+\}\s*[-+*/x×]\s*\{\w+\}", t["problem"]) and "order of operations" not in t.get("_topic", ""): + return a.rej("tpl_expression_in_problem") + env = dict(t["example"]["values"]) + for k, f in t["derived"].items(): env[k] = ev(f, env) + if not all(ev(c, env) for c in t.get("constraints", [])): return a.rej("tpl_example_violates_constraints") + got = [env[k] for k in t["answers"]] + want = t["example"]["answers"] + if len(got) != len(want) or any(abs(float(g) - float(w)) > 1e-6 for g, w in zip(got, want)): return a.rej("tpl_example_mismatch") + return True + except Exception: + return a.rej("tpl_invalid") + + def fill(a, t, spec): + out = [] + for _ in range(a.args.fills): + env = sample_env(t, a.rng) + if env is None: a.rej("fill_unsatisfiable"); break + n1, n2 = a.rng.sample(G.NAMES, 2); env.update(name1=n1, name2=n2) + for k, opts in (t.get("words") or {}).items(): + if isinstance(opts, list) and opts: env[k] = a.rng.choice([str(o) for o in opts]) + if len(t["answers"]) > 1 and len({fmt(env[k]) for k in t["answers"]}) < len(t["answers"]): a.rej("fill_same_answers"); continue + if any(env[k] in [env.get(v) for v in t["variables"]] for k in t["answers"]): a.rej("fill_answer_is_input"); continue + p, sol = (re.sub(r"(?<=\d) \* (?=\d)", " × ", render(t[k], env)) for k in ("problem", "solution")) + if PH.search(p + sol): a.rej("fill_leftover_placeholder"); continue + text = p + "\n" + sol; words = G.norm_words(text) + if bad_steps(sol) and spec["format"] != "find_the_mistake": a.rej("fill_bad_step"); continue + if set(re.findall(r"[A-Z][a-z]+", text)) & a.am_names: a.rej("arithmark_name"); continue + if G.ngrams(words, 8) & a.ban8: a.rej("test_overlap_8gram"); continue + if a.contaminated(text, a.ban13): a.rej("eval_13gram"); continue + h = hashlib.sha1(" ".join(G.norm_words(p)).encode()).hexdigest() + if h in a.seen: a.rej("duplicate"); continue + a.seen.add(h) + out.append({"problem": p, "solution": sol, "answers": [fmt(env[k]) for k in t["answers"]], + "grade": spec["grade"], "topics": [spec["topic"]], "format": spec["format"], "setting": spec["setting"], + "generator": G.MODEL + " template", "template_id": t["_tid"]}) + return out + + async def _one(a, sess, sem): + specs = [{"grade": a.rng.randint(1, 6), "topic": a.rng.choice(TOPICS), "format": a.rng.choice(list(G.FORMATS)), + "setting": a.rng.choice(G.SETTINGS + SETTINGS_EXTRA), "nq": a.rng.choice(NQ), "opening": a.rng.choice(OPENINGS), + "check": a.rng.random() < 0.3} for _ in range(a.args.per_request)] + body = {"model": G.MODEL, "temperature": 1.0, "top_p": 0.95, "max_tokens": 32000, + **({"chat_template_kwargs": G.TEMPLATE_KW} if G.TEMPLATE_KW else {}), + "messages": [{"role": "user", "content": build_prompt(specs)}]} + d = None + for attempt in range(10): + await a.pace() + try: + async with sess.post(G.API, json=body, headers={"Authorization": f"Bearer {G.KEY}"}) as r: + if r.status == 429 and "quota" in (await r.text()).lower(): + ra = r.headers.get("Retry-After") + a.paused_until = time.time() + min(1800, float(ra) if ra and ra.replace(".", "").isdigit() else 1800); continue + if r.status == 429 or r.status >= 500: + a.next_slot = max(a.next_slot, time.time() + 1.0); await asyncio.sleep(min(30, 2 ** attempt)); continue + d = await r.json(); break + except Exception: + await asyncio.sleep(2 ** attempt) + if d is None: a.stats["http_errors"] += 1; return + a.stats["requests"] += 1; a.stats["tokens_out"] += d.get("usage", {}).get("completion_tokens", 0) + txt = ((d.get("choices") or [{}])[0].get("message") or {}).get("content") or "" + # Salvage every complete object even if the array is malformed or truncated. + dec, tpls, i = json.JSONDecoder(), [], 0 + while (i := txt.find('{"id"', i)) != -1: + try: o, j = dec.raw_decode(txt, i); tpls.append(o); i = j + except Exception: i += 5 + if not tpls: + if a.stats.get("raw_saved", 0) < 5: + a.stats["raw_saved"] = a.stats.get("raw_saved", 0) + 1 + (a.out / f"raw_bad_{a.stats['raw_saved']}.txt").write_text(txt[:20000]) + return a.rej("bad_json") + a.stats["objects_parsed"] = a.stats.get("objects_parsed", 0) + len(tpls) + for t in tpls: + try: spec = specs[int(t.get("id"))] + except Exception: a.rej("bad_id"); continue + t["_topic"] = spec["topic"]; t["_nq"] = spec["nq"] + if a.check_template(t) is not True: a.stats["templates_bad"] += 1; continue + t["_tid"] = hashlib.sha1(json.dumps(t, sort_keys=True).encode()).hexdigest()[:12] + try: items = a.fill(t, spec) + except Exception: a.rej("fill_error"); items = [] + if items: + a.stats["templates_ok"] += 1; a.tpl_out.write(json.dumps({**t, "spec": spec}) + "\n"); a.tpl_out.flush() + a.buf.extend(items); a.stats["kept"] += len(items) + a.flush() + + async def run(a): + sem = asyncio.Semaphore(a.args.inflight); tasks = set() + a.lock = asyncio.Lock(); a.next_slot = time.time() + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=900)) as sess: + while a.stats["templates_ok"] < a.args.templates and a.stats["requests"] + len(tasks) < a.args.max_requests: + await sem.acquire() + while a.next_slot - time.time() > 2: await asyncio.sleep(0.2) + t = asyncio.create_task(a.one(sess, sem)); tasks.add(t); t.add_done_callback(tasks.discard) + await asyncio.gather(*tasks) + a.flush(force=True) + + +def main(): + ap = argparse.ArgumentParser(); ap.add_argument("out"); ap.add_argument("--templates", type=int, default=200) + ap.add_argument("--per-request", type=int, default=30); ap.add_argument("--fills", type=int, default=10) + ap.add_argument("--max-requests", type=int, default=10**9) + ap.add_argument("--rps", type=float, default=2.8); ap.add_argument("--inflight", type=int, default=64) + ap.add_argument("--seed", type=int, default=20261018); ap.add_argument("--shard-size", type=int, default=50000) + args = ap.parse_args(); out = Path(args.out); out.mkdir(parents=True, exist_ok=True) + asyncio.run(TGen(out, args).run()) + + +if __name__ == "__main__": + main() |
