1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
|
"""LibreMath template generator (prototype, 2026-10-15).
K2 writes problem TEMPLATES rather than problems: a two-part scenario and worked solution
with {placeholders}, integer ranges for each variable, formulas for every derived value,
constraints the numbers must satisfy, and one worked example. Code then
1. checks the formulas reproduce the example's answers (catches wrong math in the template),
2. fills the template --fills times with fresh numbers and names, computing every value,
3. runs each filled item through the same filters as generate.py.
python libremath/tgen.py OUT_DIR --templates 200 [--per-request 30] [--fills 10]
"""
import argparse, ast, asyncio, hashlib, json, math, operator, random, re, sys, time
from pathlib import Path
import aiohttp
ROOT = Path(__file__).resolve().parent
sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT.parent))
import generate as G
from arithcheck import bad_steps
TOPICS = ["addition with regrouping", "subtraction with borrowing", "multiplication", "division with and without remainders",
"money and making change", "elapsed time in minutes", "unit conversion", "perimeter", "area", "volume",
"fractions of a quantity", "comparing and ordering numbers", "place value and rounding", "mean and range of data",
"factors and multiples", "number patterns", "ratios and rates", "percents and discounts", "multi-step problems",
"problems with extra information that is not needed"]
SETTINGS_EXTRA = ["lighthouse keeper's log", "food truck", "bike repair shop", "bird-watching trip", "school band rehearsal", "farmers' co-op",
"ice rink", "pottery class", "train station", "chess club", "dog shelter", "beekeeping farm", "tailor's shop", "rock-climbing gym",
"community radio station", "ferry crossing", "orchard harvest", "board-game café", "museum gift shop", "volunteer fire station",
"robotics team", "knitting circle", "camping weekend", "bowling league", "hardware store", "space camp", "tide pool walk",
"mail sorting room", "theater set building", "kite festival", "snow-day sledding hill", "zoo feeding schedule", "puppet show",
"bus route planning", "recipe test kitchen", "weather station", "carpentry workshop", "town election count", "ballet recital",
"fishing pier", "recycling plant tour", "swim meet", "solar panel installation", "art gallery", "vegetable stand", "map-making club",
"quilt fair", "laundromat", "apartment move", "family reunion", "neighborhood cleanup", "printing press", "coffee roastery",
"veterinary clinic", "skate park", "sailing lesson", "film club", "bakery delivery route", "science museum planetarium"]
OPENINGS = ["start with a person's name", "start with a time or day", "start with a short quotation from someone", "start with a fact about the setting",
"start with an instruction to the reader", "start with a small table or list of numbers", "start with a question", "start in the middle of an action"]
NQ = [1, 1, 1, 2, 2, 3] # questions per item: half single-question (like most word-problem sets), some 2- and 3-part
# --- safe arithmetic --------------------------------------------------------- #
OPS = {ast.Add: operator.add, ast.Sub: operator.sub, ast.Mult: operator.mul, ast.Div: operator.truediv,
ast.FloorDiv: operator.floordiv, ast.Mod: operator.mod}
CMP = {ast.Eq: operator.eq, ast.NotEq: operator.ne, ast.Lt: operator.lt, ast.LtE: operator.le,
ast.Gt: operator.gt, ast.GtE: operator.ge}
def _round(x, n=0): # half-up, as taught in school (Python's round() is banker's rounding: round(25, -1) == 20)
v = math.floor(x * 10 ** n + 0.5) / 10 ** n
return int(v) if n <= 0 else v
FUNCS = {"min": min, "max": max, "round": _round, "abs": abs}
def ev(expr, env):
def go(n):
if isinstance(n, ast.Expression): return go(n.body)
if isinstance(n, ast.Constant) and isinstance(n.value, (int, float)): return n.value
if isinstance(n, ast.Name): return env[n.id]
if isinstance(n, ast.BinOp) and type(n.op) in OPS: return OPS[type(n.op)](go(n.left), go(n.right))
if isinstance(n, ast.UnaryOp) and isinstance(n.op, ast.USub): return -go(n.operand)
if isinstance(n, ast.Compare):
l = go(n.left)
for op, c in zip(n.ops, n.comparators):
r = go(c)
if not CMP[type(op)](l, r): return False
l = r
return True
if isinstance(n, ast.BoolOp):
vals = [go(v) for v in n.values]
return all(vals) if isinstance(n.op, ast.And) else any(vals)
if isinstance(n, ast.Call) and isinstance(n.func, ast.Name) and n.func.id in FUNCS:
return FUNCS[n.func.id](*[go(a) for a in n.args])
raise ValueError(f"disallowed: {ast.dump(n)[:60]}")
expr = str(expr).replace("||", " or ").replace("&&", " and ")
return go(ast.parse(expr, mode="eval"))
def fmt(v):
if isinstance(v, float):
if abs(v - round(v)) < 1e-9: return str(int(round(v)))
return f"{v:.2f}"
return str(v)
PH = re.compile(r"\{(\w+)\}")
def render(text, env): # noqa: E302
return PH.sub(lambda m: fmt(env[m.group(1)]) if m.group(1) in env else m.group(0), text)
def sample_env(t, rng, tries=200):
for _ in range(tries):
env = {}
for k, (lo, hi) in t["variables"].items(): env[k] = rng.randint(int(lo), int(hi))
for k, f in t["derived"].items(): env[k] = ev(f, env)
if all(ev(c, env) for c in t.get("constraints", [])) and all(
(isinstance(env[k], int) or abs(env[k] * 100 - round(env[k] * 100)) < 1e-6) and env[k] >= 0 for k in t["derived"]):
return env
return None
def build_prompt(specs):
lines = []
for i, s in enumerate(specs):
qs = "one question" if s["nq"] == 1 else f"{s['nq']} questions labelled " + ", ".join(f"({c})" for c in "abc"[:s["nq"]])
lines.append(f'{i}. grade {s["grade"]}, topic: {s["topic"]}, format: {G.FORMATS[s["format"]]}, setting: {s["setting"]}; '
f'{qs}; {s["opening"]}' + ("; end each part of the solution with a one-sentence check" if s["check"] else ""))
return ("You are writing reusable TEMPLATES for an open educational math dataset. For each spec, write one template: a short "
"scenario followed by the requested question(s), plus a worked step-by-step solution that explains each step. Every "
"number in the problem and solution must be a {placeholder}: input variables with an integer range, and derived values "
"defined by formulas over other placeholders using + - * / // % min max round. Never write a computed number literally. "
"Use {name1} and {name2} for people. You may also add word placeholders for objects or units, listing 4-6 interchangeable "
"options under \"words\" (e.g. {\"snack\": [\"muffins\", \"bagels\", \"scones\", \"rolls\"]}). The problem text must end "
"with the last question and a question mark. Plain text only, no markdown. Make every template different in wording, "
"sentence structure and length; do not reuse openings like \"At the\" or \"In the\".\n\n" + "\n".join(lines) +
'\n\nReply with only a JSON array, one object per spec in order:\n'
'[{"id": 0, "problem": "...{name1}...{a} {snack}...?", "solution": "...{a} x {b} = {total}...", '
'"variables": {"a": [2, 12], "b": [3, 9]}, "words": {"snack": ["muffins", "bagels", "scones", "rolls"]}, '
'"derived": {"total": "a*b"}, "constraints": ["total > 10"], "answers": ["total"], '
'"example": {"values": {"a": 4, "b": 6}, "answers": [24]}}, ...]')
class TGen(G.Gen):
def __init__(a, out, args):
a.args = args; a.out = out
a.stats = {"requests": 0, "http_errors": 0, "templates_ok": 0, "templates_bad": 0, "kept": 0, "tokens_out": 0, "reject": {}}
a.am_names, a.ban8, a.ban13, a.contaminated = G.load_bans()
a.seen = set(); a.buf = []; a.shard = len(list(out.glob("items-*.jsonl")))
a.rng = random.Random(args.seed + 1_000_003 * a.shard)
a.tpl_out = open(out / "templates.jsonl", "a")
def check_template(a, t):
try:
for k in ("problem", "solution", "variables", "derived", "answers", "example"): t[k]
if not t["problem"].rstrip().endswith("?"): return a.rej("tpl_not_question")
names = set(PH.findall(t["problem"] + t["solution"])) - {"name1", "name2"}
if not names <= set(t["variables"]) | set(t["derived"]) | set(t.get("words") or {}): return a.rej("tpl_unknown_placeholder")
if re.search(r"(?<![{\w])\d{2,}(?![\w}])", re.sub(r"\{\w+\}", "", t["problem"])): return a.rej("tpl_literal_number")
nq = t["_nq"]
if nq > 1 and not all(f"({c})" in t["problem"] for c in "abc"[:nq]): return a.rej("tpl_missing_parts")
if len(t["answers"]) != nq or not all("{" + k + "}" in t["solution"] for k in t["answers"]): return a.rej("tpl_answer_not_in_solution")
words = t.get("words") or {}
if not isinstance(words, dict) or any(not isinstance(v, list) or len(v) < 2 for v in words.values()): return a.rej("tpl_bad_words")
consts = {str(n.value) for f in list(t["derived"].values()) + list(t.get("constraints", []))
for n in ast.walk(ast.parse(str(f).replace("||", " or ").replace("&&", " and "), mode="eval"))
if isinstance(n, ast.Constant) and isinstance(n.value, (int, float))}
ok = consts | {str(i) for i in range(13)} | {"24", "60", "100", "1000", "365", "52"}
sol_lits = re.findall(r"(?<![{\w.])\d+(?:\.\d+)?(?![\w}])", re.sub(r"\{\w+\}", "", t["solution"]))
if any(x not in ok for x in sol_lits): return a.rej("tpl_solution_literal")
txt = t["problem"] + " " + t["solution"]
if re.search(r"//|\*\*|\{\w+\}\s*%|\s%\s", txt): return a.rej("tpl_code_operator")
if "nearest {" in txt: return a.rej("tpl_round_to_variable")
if re.search(r"\{\w+\}\s*[-+*/x×]\s*\{\w+\}", t["problem"]) and "order of operations" not in t.get("_topic", ""):
return a.rej("tpl_expression_in_problem")
env = dict(t["example"]["values"])
for k, f in t["derived"].items(): env[k] = ev(f, env)
if not all(ev(c, env) for c in t.get("constraints", [])): return a.rej("tpl_example_violates_constraints")
got = [env[k] for k in t["answers"]]
want = t["example"]["answers"]
if len(got) != len(want) or any(abs(float(g) - float(w)) > 1e-6 for g, w in zip(got, want)): return a.rej("tpl_example_mismatch")
return True
except Exception:
return a.rej("tpl_invalid")
def fill(a, t, spec):
out = []
for _ in range(a.args.fills):
env = sample_env(t, a.rng)
if env is None: a.rej("fill_unsatisfiable"); break
n1, n2 = a.rng.sample(G.NAMES, 2); env.update(name1=n1, name2=n2)
for k, opts in (t.get("words") or {}).items():
if isinstance(opts, list) and opts: env[k] = a.rng.choice([str(o) for o in opts])
if len(t["answers"]) > 1 and len({fmt(env[k]) for k in t["answers"]}) < len(t["answers"]): a.rej("fill_same_answers"); continue
if any(env[k] in [env.get(v) for v in t["variables"]] for k in t["answers"]): a.rej("fill_answer_is_input"); continue
p, sol = (re.sub(r"(?<=\d) \* (?=\d)", " × ", render(t[k], env)) for k in ("problem", "solution"))
if PH.search(p + sol): a.rej("fill_leftover_placeholder"); continue
text = p + "\n" + sol; words = G.norm_words(text)
if bad_steps(sol) and spec["format"] != "find_the_mistake": a.rej("fill_bad_step"); continue
if set(re.findall(r"[A-Z][a-z]+", text)) & a.am_names: a.rej("arithmark_name"); continue
if G.ngrams(words, 8) & a.ban8: a.rej("test_overlap_8gram"); continue
if a.contaminated(text, a.ban13): a.rej("eval_13gram"); continue
h = hashlib.sha1(" ".join(G.norm_words(p)).encode()).hexdigest()
if h in a.seen: a.rej("duplicate"); continue
a.seen.add(h)
out.append({"problem": p, "solution": sol, "answers": [fmt(env[k]) for k in t["answers"]],
"grade": spec["grade"], "topics": [spec["topic"]], "format": spec["format"], "setting": spec["setting"],
"generator": G.MODEL + " template", "template_id": t["_tid"]})
return out
async def _one(a, sess, sem):
specs = [{"grade": a.rng.randint(1, 6), "topic": a.rng.choice(TOPICS), "format": a.rng.choice(list(G.FORMATS)),
"setting": a.rng.choice(G.SETTINGS + SETTINGS_EXTRA), "nq": a.rng.choice(NQ), "opening": a.rng.choice(OPENINGS),
"check": a.rng.random() < 0.3} for _ in range(a.args.per_request)]
body = {"model": G.MODEL, "temperature": 1.0, "top_p": 0.95, "max_tokens": 32000,
**({"chat_template_kwargs": G.TEMPLATE_KW} if G.TEMPLATE_KW else {}),
"messages": [{"role": "user", "content": build_prompt(specs)}]}
d = None
for attempt in range(10):
await a.pace()
try:
async with sess.post(G.API, json=body, headers={"Authorization": f"Bearer {G.KEY}"}) as r:
if r.status == 429 and "quota" in (await r.text()).lower():
ra = r.headers.get("Retry-After")
a.paused_until = time.time() + min(1800, float(ra) if ra and ra.replace(".", "").isdigit() else 1800); continue
if r.status == 429 or r.status >= 500:
a.next_slot = max(a.next_slot, time.time() + 1.0); await asyncio.sleep(min(30, 2 ** attempt)); continue
d = await r.json(); break
except Exception:
await asyncio.sleep(2 ** attempt)
if d is None: a.stats["http_errors"] += 1; return
a.stats["requests"] += 1; a.stats["tokens_out"] += d.get("usage", {}).get("completion_tokens", 0)
txt = ((d.get("choices") or [{}])[0].get("message") or {}).get("content") or ""
# Salvage every complete object even if the array is malformed or truncated.
dec, tpls, i = json.JSONDecoder(), [], 0
while (i := txt.find('{"id"', i)) != -1:
try: o, j = dec.raw_decode(txt, i); tpls.append(o); i = j
except Exception: i += 5
if not tpls:
if a.stats.get("raw_saved", 0) < 5:
a.stats["raw_saved"] = a.stats.get("raw_saved", 0) + 1
(a.out / f"raw_bad_{a.stats['raw_saved']}.txt").write_text(txt[:20000])
return a.rej("bad_json")
a.stats["objects_parsed"] = a.stats.get("objects_parsed", 0) + len(tpls)
for t in tpls:
try: spec = specs[int(t.get("id"))]
except Exception: a.rej("bad_id"); continue
t["_topic"] = spec["topic"]; t["_nq"] = spec["nq"]
if a.check_template(t) is not True: a.stats["templates_bad"] += 1; continue
t["_tid"] = hashlib.sha1(json.dumps(t, sort_keys=True).encode()).hexdigest()[:12]
try: items = a.fill(t, spec)
except Exception: a.rej("fill_error"); items = []
if items:
a.stats["templates_ok"] += 1; a.tpl_out.write(json.dumps({**t, "spec": spec}) + "\n"); a.tpl_out.flush()
a.buf.extend(items); a.stats["kept"] += len(items)
a.flush()
async def run(a):
sem = asyncio.Semaphore(a.args.inflight); tasks = set()
a.lock = asyncio.Lock(); a.next_slot = time.time()
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=900)) as sess:
while a.stats["templates_ok"] < a.args.templates and a.stats["requests"] + len(tasks) < a.args.max_requests:
await sem.acquire()
while a.next_slot - time.time() > 2: await asyncio.sleep(0.2)
t = asyncio.create_task(a.one(sess, sem)); tasks.add(t); t.add_done_callback(tasks.discard)
await asyncio.gather(*tasks)
a.flush(force=True)
def main():
ap = argparse.ArgumentParser(); ap.add_argument("out"); ap.add_argument("--templates", type=int, default=200)
ap.add_argument("--per-request", type=int, default=30); ap.add_argument("--fills", type=int, default=10)
ap.add_argument("--max-requests", type=int, default=10**9)
ap.add_argument("--rps", type=float, default=2.8); ap.add_argument("--inflight", type=int, default=64)
ap.add_argument("--seed", type=int, default=20261018); ap.add_argument("--shard-size", type=int, default=50000)
args = ap.parse_args(); out = Path(args.out); out.mkdir(parents=True, exist_ok=True)
asyncio.run(TGen(out, args).run())
if __name__ == "__main__":
main()
|