1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
|
"""LibreMath composer (2026-10-23): unlimited grade-school word problems from the K2-written
fragment bank (fragments.py), with every number and answer computed here.
python libremath/compose.py BANK.json OUT_DIR --items 1000000 [--seed 1]
"""
import argparse, json, random, re, sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import generate as G
from tgen import SETTINGS_EXTRA
PH = re.compile(r"\{(\w+)\}")
def fill(t, env):
t = re.sub(r"\b([Ee])(very|ach) \{(cont|obj)\}", lambda m: ("All the {" if m.group(1) == "E" else "all the {") + m.group(3) + "}"
if m.group(2) == "very" else m.group(1) + "ach of the {" + m.group(3) + "}", t) # plural nouns: "every boxes"
out = PH.sub(lambda m: str(env[m.group(1)]), t)
out = re.sub(r"\b([Tt]he) the\b", r"\1", out)
out = out.strip(); out = out[:1].upper() + out[1:]
return out if out[-1:] in ".!?\"'”" else out + "."
ANSWER_TAGS = {"money": r"cost|spen|paid|pay|price|dollar|bill|\$", "remove": r"\bleft\b|remain|still", "two_step": r"\bleft\b|remain|still",
"compare": r"more|differ|ahead|extra", "share": r"\beach\b|per\b|every", "combine": r"total|altogether|in all|together|sum",
"groups": r"total|altogether|in all|together"}
ALL_TAGS = "|".join(ANSWER_TAGS.values()) + r"|collect|gather|found|bought|made|earn|save"
def money(x):
return f"{x:.2f}"
class Composer:
def __init__(a, bank, seed):
a.b = bank; a.r = random.Random(seed)
a.places = G.SETTINGS + SETTINGS_EXTRA; a.names = G.NAMES
def pick(a, cat): return a.r.choice(a.b[cat])
def answer_for(a, kind):
if not hasattr(a, "_ans"):
gen = [f for f in a.b["answer"] if not re.search(ALL_TAGS, f, re.I)]
a._ans = {k: (gen if k != "money" else [f for f in gen if "{obj}" not in f]) + [f for f in a.b["answer"] if re.search(v, f, re.I) and not re.search(
"|".join(v2 for k2, v2 in ANSWER_TAGS.items() if v2 != v), f, re.I)] for k, v in ANSWER_TAGS.items()}
return a.r.choice(a._ans[kind])
def numbers(a, kind):
r = a.r; big = r.random() < 0.35
if kind in ("combine", "remove", "compare"):
hi = r.choice([20, 50, 100, 100, 500, 1000]) if not big else 5000
x, y = r.randint(2, hi), r.randint(2, hi)
return (max(x, y), min(x, y)) if kind != "combine" else (x, y)
if kind == "groups": return r.randint(2, 12 if not big else 40), r.randint(2, 12 if not big else 99)
if kind == "share":
b = r.randint(2, 12); return b * r.randint(2, 15 if not big else 60), b
if kind == "money": return r.randint(2, 12), r.randint(25, 1999) / 100
def item(a):
r = a.r; A, B = r.sample(a.names, 2); o = r.choice(a.b["objects"]); place = r.choice(a.places)
kind = r.choice(["combine", "remove", "compare", "groups", "groups", "share", "money", "two_step"])
base = "groups" if kind == "two_step" and r.random() < 0.6 else ("combine" if kind == "two_step" else kind)
x, y = a.numbers(base)
if x == y and base in ("remove", "compare"): x += 1
ans = {"combine": lambda: x + y, "remove": lambda: x - y, "compare": lambda: x - y, "groups": lambda: x * y,
"share": lambda: x // y, "money": lambda: round(x * y, 2)}[base]()
env = {"A": A, "B": B, "obj": o["obj"], "cont": o["cont"], "place": "the " + place, "a": x, "b": y,
"ans": money(ans) if base == "money" else ans, "p": money(y) if base == "money" else y, "d": r.randint(2, 60)}
prob = [fill(a.pick(f"{base}_setup"), env)]
steps = [fill(a.pick(f"{base}_step"), env)]
final = ans
if kind == "two_step":
c = r.randint(1, max(1, ans - 1)); env2 = dict(env, x=ans, c=c, ans=ans - c)
prob.append(fill(a.pick("followup_remove"), env2)); q = fill(a.pick("followup_q"), env2)
steps.append(fill(a.pick("followup_step"), env2)); final = ans - c; env = env2
else:
q = fill(a.pick(f"{base}_q"), env)
if r.random() < 0.25: # irrelevant fact, before or after the setup
prob.insert(r.choice([0, len(prob)]), fill(a.pick("distractor"), env))
if r.random() < 0.7: prob.insert(0, fill(a.pick("opener"), env))
problem = " ".join(prob + [q])
solution = " ".join(steps + [fill(a.answer_for(kind), env)])
if r.random() < 0.3: problem, solution = "Question: " + problem, "Answer: " + solution
return {"problem": problem, "solution": solution, "answers": [str(env["ans"])], "kind": kind,
"objects": o, "generator": "LibreMath fragment composer (fragments by IFM/K2-Horizon)"}
def distinct(texts, n=4):
tot = 0; s = set()
for t in texts:
w = re.sub(r"\d+", "#", t.lower()).split(); g = [tuple(w[i:i + n]) for i in range(len(w) - n + 1)]; tot += len(g); s.update(g)
return len(s) / max(tot, 1)
def main():
ap = argparse.ArgumentParser(); ap.add_argument("bank"); ap.add_argument("out")
ap.add_argument("--items", type=int, default=100000); ap.add_argument("--seed", type=int, default=1)
ap.add_argument("--no-filter", action="store_true")
a = ap.parse_args(); out = Path(a.out); out.mkdir(parents=True, exist_ok=True)
bank = json.load(open(a.bank)); c = Composer(bank, a.seed)
am_names, ban8 = set(), set()
if not a.no_filter:
am_names, ban8, _, _ = G.load_bans()
seen = set(); kept = rej = 0
with open(out / f"items-seed{a.seed}.jsonl", "w") as f:
while kept < a.items:
it = c.item(); text = it["problem"] + " " + it["solution"]
h = hash(it["problem"])
if h in seen or (set(re.findall(r"[A-Z][a-z]+", text)) & am_names) or (ban8 and G.ngrams(G.norm_words(text), 8) & ban8):
rej += 1; continue
seen.add(h); f.write(json.dumps(it) + "\n"); kept += 1
S = [json.loads(l) for l in open(out / f"items-seed{a.seed}.jsonl")][:20000]
print(f"kept {kept:,}, rejected {rej:,}; distinct 4-gram ratio (numbers masked) on 20k: {distinct([s['problem'] + ' ' + s['solution'] for s in S]):.3f}")
if __name__ == "__main__":
main()
|