"""LibreMath fragment bank (2026-10-23): K2 writes reusable sentence fragments ONCE; compose.py assembles unlimited problems from them with exact, code-computed math. Each category has a fixed set of placeholders. A fragment is kept only if it uses exactly the allowed placeholders, includes the required ones, contains no literal digits, and (for solution steps) contains the exact equation string the composer will fill. python libremath/fragments.py OUT.json --calls-per-category 12 """ import argparse, asyncio, json, random, re, sys, time from pathlib import Path import aiohttp sys.path.insert(0, str(Path(__file__).resolve().parent)) import generate as G # category: (description, required placeholders, allowed placeholders, kind) CATS = { "opener": ("an opening sentence that sets a scene at or around {place}; it may mention {A}", {"place"}, {"place", "A"}, "s"), "distractor": ("a sentence giving one irrelevant fact that includes the number {d} (not needed to solve any question); it may mention {A} or {place}", {"d"}, {"d", "A", "place"}, "s"), "combine_setup": ("one or two sentences saying {A} has {a} {obj} and then gets {b} more (buys, finds, receives, picks, is given...)", {"A", "a", "b", "obj"}, {"A", "a", "b", "obj", "place"}, "s"), "combine_q": ("a question asking how many {obj} {A} has now in total", {"obj"}, {"A", "obj"}, "q"), "combine_step": ("a solution sentence explaining why we add, containing exactly '{a} + {b} = {ans}'", {"a", "b", "ans"}, {"a", "b", "ans", "A", "obj"}, "s"), "remove_setup": ("one or two sentences saying {A} starts with {a} {obj} and then {b} of them are used, given away, sold, eaten or lost", {"A", "a", "b", "obj"}, {"A", "a", "b", "obj", "place"}, "s"), "remove_q": ("a question asking how many {obj} {A} has left", {"obj"}, {"A", "obj"}, "q"), "remove_step": ("a solution sentence explaining why we subtract, containing exactly '{a} - {b} = {ans}'", {"a", "b", "ans"}, {"a", "b", "ans", "A", "obj"}, "s"), "compare_setup": ("one or two sentences saying {A} has {a} {obj} and {B} has {b} {obj}", {"A", "B", "a", "b"}, {"A", "B", "a", "b", "obj", "place"}, "s"), "compare_q": ("a question asking how many more {obj} {A} has than {B}", {"A", "B"}, {"A", "B", "obj"}, "q"), "compare_step": ("a solution sentence explaining the difference, containing exactly '{a} - {b} = {ans}'", {"a", "b", "ans"}, {"a", "b", "ans", "A", "B", "obj"}, "s"), "groups_setup": ("one or two sentences saying there are {a} {cont} with {b} {obj} in each (packed, arranged, stacked...); may mention {A}", {"a", "b", "cont", "obj"}, {"A", "a", "b", "cont", "obj", "place"}, "s"), "groups_q": ("a question asking how many {obj} there are altogether", {"obj"}, {"A", "obj", "cont"}, "q"), "groups_step": ("a solution sentence explaining why we multiply, containing exactly '{a} × {b} = {ans}'", {"a", "b", "ans"}, {"a", "b", "ans", "obj", "cont"}, "s"), "share_setup": ("one or two sentences saying {A} shares or splits {a} {obj} equally into {b} {cont}", {"A", "a", "b", "obj", "cont"}, {"A", "a", "b", "obj", "cont", "place"}, "s"), "share_q": ("a question asking how many {obj} go in each of the {cont}", {"obj", "cont"}, {"A", "obj", "cont"}, "q"), "share_step": ("a solution sentence explaining why we divide, containing exactly '{a} ÷ {b} = {ans}'", {"a", "b", "ans"}, {"a", "b", "ans", "obj", "cont"}, "s"), "followup_remove": ("a follow-up sentence (for after a total is known) saying {c} of those {obj} are then used, sold or given away", {"c", "obj"}, {"c", "obj", "A", "B"}, "s"), "followup_q": ("a question asking how many {obj} are left after that", set(), {"obj", "A"}, "q"), "followup_step": ("a solution sentence for the second step, containing exactly '{x} - {c} = {ans}'", {"x", "c", "ans"}, {"x", "c", "ans", "obj"}, "s"), "money_setup": ("one or two sentences saying {A} buys {a} {obj} that cost {p} dollars each", {"A", "a", "obj", "p"}, {"A", "a", "obj", "p", "place"}, "s"), "money_q": ("a question asking how much {A} spends in total", {"A"}, {"A", "obj"}, "q"), "money_step": ("a solution sentence explaining the total cost, containing exactly '{a} × ${p} = ${ans}'", {"a", "p", "ans"}, {"a", "p", "ans", "obj", "A"}, "s"), "answer": ("a short final sentence stating the answer {ans}; it may mention {obj} or {A}", {"ans"}, {"ans", "obj", "A"}, "s"), } TONES = ["plain and simple, for young children", "like a friendly teacher", "like a math textbook", "lively and story-like", "short and direct", "detailed, with a little extra description", "conversational", "matter-of-fact"] PH = re.compile(r"\{(\w+)\}") def valid(frag, cat): desc, req, allowed, kind = CATS[cat] if not isinstance(frag, str) or not (12 <= len(frag) <= 260): return False if re.search(r"\d", PH.sub("", frag)): return False ph = set(PH.findall(frag)) if not req <= ph or not ph <= allowed: return False if kind == "q" and not frag.rstrip().endswith("?"): return False if kind == "s" and frag.rstrip().endswith("?"): return False eq = re.search(r"'(.*)'", desc) if eq and eq.group(1) not in frag: return False return True def prompt(cat, n, tone): if cat == "objects": return (f"List {n} pairs of things for grade-school math word problems: a plural countable item and a plural container or " f"group it could be packed or arranged in (for example marbles / jars, chairs / rows, stickers / sheets). Cover many " f"different domains (food, sports, school, nature, tools, crafts, animals, travel, music, building). Reply with only a " f'JSON array of objects like {{"obj": "marbles", "cont": "jars"}}.') desc = CATS[cat][0] return (f"Write {n} different English fragments for building grade-school math word problems. Each fragment is {desc}. " f"Use the placeholders in curly braces exactly as written (they are filled in later by a program); never write a digit. " + ("Do not mention any specific item, food, animal or object; the items being counted are filled in later. " if cat in ("opener", "distractor") else "Never name a specific item, food, animal, place or object: refer to the items only as {obj} and {cont} " "(or words like 'them', 'each one', 'the items'), because the program fills them in. ") + f"Style: {tone}. Vary vocabulary and sentence structure as much as possible; no two fragments should start the same way. " f"Plain text, no markdown. Reply with only a JSON array of strings.") async def main_async(a): out = Path(a.out); bank = json.loads(out.read_text()) if out.exists() else {c: [] for c in list(CATS) + ["objects"]} sem = asyncio.Semaphore(a.inflight); lock = asyncio.Lock(); state = {"next": time.time(), "tok": 0} rng = random.Random(a.seed) async def call(sess, cat): async with sem: body = {"model": G.MODEL, "temperature": 1.0, "top_p": 0.95, "max_tokens": 16000, **({"chat_template_kwargs": G.TEMPLATE_KW} if G.TEMPLATE_KW else {}), "messages": [{"role": "user", "content": prompt(cat, a.per_call, rng.choice(TONES))}]} for attempt in range(8): async with lock: w = state["next"] - time.time() if w > 0: await asyncio.sleep(w) state["next"] = max(time.time(), state["next"]) + 1 / a.rps try: async with sess.post(G.API, json=body, headers={"Authorization": f"Bearer {G.KEY}"}) as r: if r.status == 429: txt = await r.text() await asyncio.sleep(1800 if "quota" in txt.lower() else 5 * (attempt + 1)); continue if r.status >= 500: await asyncio.sleep(5 * (attempt + 1)); continue d = await r.json(); break except Exception: await asyncio.sleep(5 * (attempt + 1)) else: return cat, [] state["tok"] += d.get("usage", {}).get("completion_tokens", 0) txt = ((d.get("choices") or [{}])[0].get("message") or {}).get("content") or "" m = re.search(r"\[.*\]", txt, re.S) try: arr = json.loads(m.group(0)) if m else [] except Exception: arr = [] if cat == "objects": return cat, [{"obj": o["obj"].strip().lower(), "cont": o["cont"].strip().lower()} for o in arr if isinstance(o, dict) and all(isinstance(o.get(k), str) and re.fullmatch(r"[a-zA-Z' -]{3,30}", o[k].strip()) for k in ("obj", "cont"))] return cat, [f.strip() for f in arr if valid(f, cat)] async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=600)) as sess: tasks = [call(sess, c) for c in CATS for _ in range(a.calls_per_category)] + [call(sess, "objects") for _ in range(a.calls_per_category)] for fut in asyncio.as_completed(tasks): cat, frags = await fut seen = {json.dumps(f, sort_keys=True) for f in bank[cat]} bank[cat] += [f for f in frags if json.dumps(f, sort_keys=True) not in seen] out.write_text(json.dumps(bank, indent=1)) print(f" {cat}: +{len(frags)} -> {len(bank[cat])} (output tokens so far {state['tok']:,})", flush=True) print({c: len(v) for c, v in bank.items()}) def main(): ap = argparse.ArgumentParser(); ap.add_argument("out") ap.add_argument("--calls-per-category", type=int, default=12); ap.add_argument("--per-call", type=int, default=40) ap.add_argument("--rps", type=float, default=1.0); ap.add_argument("--inflight", type=int, default=16); ap.add_argument("--seed", type=int, default=7) asyncio.run(main_async(ap.parse_args())) if __name__ == "__main__": main()