summaryrefslogtreecommitdiff
path: root/libremath/generate.py
blob: 0042f056a2bad97abc14bf4d4fc63bf835f86a18 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
"""LibreMath-1.0 generator: K2-Horizon-375B writes problems around specs.py's facts.

    python libremath/generate.py OUT_DIR --items 200000 [--per-request 20] [--rps 2.8] [--seed N]

Every request carries `per-request` specs, each with its own randomly chosen format,
setting and names. The reply is a JSON list; each item is kept only if
  * its final answer equals the computed answer,
  * the problem ends with a question mark (ArithMark items are unfinished sentences),
  * it uses none of ArithMark's 53 character names,
  * it shares no 8-word span with any ArithMark, SVAMP or ASDiv test item,
  * it passes the corpus-wide 13-gram ban (HellaSwag, ARC, PIQA, ArithMark),
  * it is not a duplicate of an earlier problem.
Kept items go to OUT_DIR/items-XXXXX.jsonl; counters and rejects to OUT_DIR/stats.json.
Resumable: shards already written are counted and the seed advances past them.
"""
import argparse, asyncio, hashlib, json, random, re, sys, time
from pathlib import Path
import aiohttp

ROOT = Path(__file__).resolve().parent
sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT.parent))
import specs
from arithcheck import bad_steps

import os
API = os.environ.get("LIBREMATH_API", "https://api.ifm.ai/v1/chat/completions")
MODEL = os.environ.get("LIBREMATH_MODEL", "IFM/K2-Horizon-375B-A23B")
TEMPLATE_KW = json.loads(os.environ.get("LIBREMATH_TEMPLATE_KW", '{"reasoning_effort": "low"}'))
import os
KEY = open(Path(os.environ.get("LIBREMATH_KEY_FILE", Path.home() / ".config/libremath/api_key"))).read().strip()

FORMATS = {
    "word_problem": "a word problem that ends with a question, then a step-by-step solution",
    "textbook_example": "a textbook-style worked example: state the problem as a question, then explain each step as a teacher would",
    "dialogue": "a short dialogue in which one person asks the question and another works it out aloud",
    "find_the_mistake": "a question, a student's incorrect attempt, and a correction that explains the error and reaches the right answer",
    "concept_then_example": "two or three sentences explaining the math idea, then an example question and its solution",
    "estimate_then_solve": "a question, a quick estimate, then the exact solution and a check that it is reasonable",
    "two_methods": "a question solved two different ways that reach the same answer",
    "real_world_planning": "a planning question someone faces in daily life, solved step by step",
}
SETTINGS = ("bakery|soccer tournament|public library|aquarium|science fair|road trip|community garden|recycling drive|school band|"
            "train station|camping trip|school store|wildlife park|robotics club|dairy farm|history museum|theater rehearsal|"
            "bicycle shop|kitchen|craft fair|weather station|space camp|swimming meet|bookbinding workshop|food bank|ferry|"
            "orchard|chess club|pottery studio|hiking trail|bus route|lemonade stand|carpentry project|marathon|zoo nursery|"
            "art class|recording studio|coral reef survey|ski lodge|post office|pet shelter|harbor|tea shop|rugby practice|"
            "greenhouse|bird count|dance recital|basketball league|volcano exhibit|street market|ice rink|bowling alley|"
            "beekeeping|newspaper delivery|chemistry lab|planetarium|tailor shop|fishing trip|quilting circle|apple cider mill|"
            "school election|cricket match|map making|kite festival|summer reading program|hospital supply room|airport|"
            "film club|vegetable stand|furniture store|scout troop|tree planting|cooking contest|science museum").split("|")
NAMES = ("Aarav Adaeze Aiko Akira Alejandro Amara Anaya Andrei Annika Arjun Astrid Ayaan Aziz Bea Benedikt Bianca Bilal "
         "Carmen Chidi Chloe Darius Dmitri Elif Emeka Esme Farah Fatima Felix Freya Gabriel Gustavo Hana Hiroshi Ines Ingrid "
         "Isaac Jamal Jasper Javier Joaquin Jonah Kai Kalani Keanu Kenji Kofi Laila Lars Leila Lorenzo Lucia Magnus Malik "
         "Marisol Matteo Miguel Mila Minh Nadia Naomi Niamh Nikhil Oluwaseun Oscar Pablo Paloma Quentin Rafael Rania Rohan "
         "Rosa Saanvi Santiago Selin Seo-yeon Siddharth Silas Siobhan Sunita Tariq Teodor Thandiwe Tomas Valentina Wren "
         "Xiomara Yara Yusuf Zainab Zara Ziad").split()


def norm_words(s): return re.sub(r"[^0-9a-z]+", " ", s.lower()).split()


def ngrams(words, n): return {" ".join(words[i:i + n]) for i in range(len(words) - n + 1)}


def load_bans():
    am = [json.loads(l) for l in open(Path.home() / ".cache/huggingface/hub/datasets--AxiomicLabs--ArithMark-3.0/snapshots/6f6e59dd9b7e2c63455f7af7f838f9ecc3d0a746/arithmark-3.jsonl")]
    am_names = {re.match(r"[A-Z][a-z]+", r["ctx"]).group(0) for r in am if re.match(r"[A-Z][a-z]+", r["ctx"])}
    am_names -= {"At", "In", "The", "A", "An", "On", "For", "During", "After", "Each", "Every", "Last"}
    texts = [r["ctx"] for r in am]
    from datasets import load_dataset
    texts += [r["Body"] + " " + r["Question"] for r in load_dataset("ChilleD/SVAMP", split="test")]
    texts += [r["body"] + " " + r["question"] for r in load_dataset("EleutherAI/asdiv", split="validation")]
    ban8 = set().union(*(ngrams(norm_words(t), 8) for t in texts))
    import prepare
    return am_names, ban8, prepare.eval_ngrams(lambda *a: None), prepare.contaminated


def same_answer(got, want):
    """Compare the model's final answer with the computed one, tolerating units, $ signs,
    commas and words around the number ("56 meters", "$11.71", "3/4 of the pizza")."""
    from fractions import Fraction
    g, w = str(got).strip(), str(want)
    if re.fullmatch(r"\d{1,2}:\d{2} [AP]M", w):
        m = re.search(r"(\d{1,2}:\d{2})\s*([AaPp])\.?\s*[Mm]", g)
        return bool(m) and f"{m.group(1)} {m.group(2).upper()}M" == w
    g = g.replace(",", "")
    if "/" in w:
        m = re.findall(r"-?\d+\s*/\s*\d+", g)
        try: return bool(m) and Fraction(m[-1].replace(" ", "")) == Fraction(w)
        except Exception: return False
    nums = re.findall(r"-?\d+(?:\.\d+)?", g)
    try: return bool(nums) and abs(float(nums[-1]) - float(w)) < 1e-9
    except Exception: return False


def build_prompt(batch, rng):
    """batch: list of (spec_a, spec_b). Each item is one scenario with two questions."""
    lines = []
    for i, (sa, sb) in enumerate(batch):
        fmt = rng.choice(list(FORMATS)); setting = rng.choice(SETTINGS); names = rng.sample(NAMES, 2)
        for x in (sa, sb): x["format"], x["setting"], x["names"] = fmt, setting, names
        lines.append(f'{i}. format: {FORMATS[fmt]}. Setting: {setting}. People (use only if needed): {names[0]}, {names[1]}.\n'
                     f'   Question A: grade {sa["grade"]}, topic "{sa["topic"]}", facts: {sa["facts"]}. Correct answer: {sa["answer"]}.\n'
                     f'   Question B: grade {sb["grade"]}, topic "{sb["topic"]}", facts: {sb["facts"]}. Correct answer: {sb["answer"]}.')
    return ("You are writing items for an open educational math dataset. For each spec below, write one original item: a short "
            "scenario in the given setting followed by two questions, (a) and (b), that use the given facts. Vary wording, sentence "
            "structure and length widely between items. Each question must be a direct question ending with a question mark, and the "
            "item's text must end with question (b). Use realistic quantities and units. The solution must work through (a) and then "
            "(b) step by step, explaining why each step is done, and finish each part with a one-sentence check that the answer is "
            "reasonable. Answers must equal the given correct answers. Write plain text only: no markdown, bold, headings or bullet symbols.\n\n" + "\n".join(lines) +
            '\n\nReply with only a JSON array, one object per spec in order: '
            '[{"id": 0, "problem": "...", "solution": "...", "answer_a": "...", "answer_b": "..."}, ...]')


class Gen:
    def __init__(a, out, args):
        a.out, a.args = out, args; a.stats = {"requests": 0, "http_errors": 0, "kept": 0, "tokens_out": 0,
                                              "reject": {}}
        a.am_names, a.ban8, a.ban13, a.contaminated = load_bans()
        a.seen = set(); a.buf = []; a.shard = len(list(out.glob("items-*.jsonl")))
        a.rng = random.Random(args.seed + 1_000_003 * a.shard)

    def rej(a, why): a.stats["reject"][why] = a.stats["reject"].get(why, 0) + 1

    def answer_of(a, it, keys, sol):
        fa = next((it[k] for k in keys if it.get(k) not in (None, "")), None)
        return fa

    def check(a, pair, it):
        s, s2 = pair
        p, sol = (re.sub(r"\*\*|__|^#+\s*", "", str(it.get(k, "")), flags=re.M).strip() for k in ("problem", "solution"))
        if not p or not sol: return a.rej("empty")
        fa = a.answer_of(it, ("answer_a", "a", "final_answer_a"), sol)
        fb = a.answer_of(it, ("answer_b", "b", "final_answer_b"), sol)
        if fa is None or fb is None: return a.rej("missing_answer")
        if not same_answer(fb, s2["answer"]): return a.rej("wrong_answer")
        if not same_answer(fa, s["answer"]):
            if a.stats["reject"].get("wrong_answer", 0) < 200:
                with open(a.out / "rejects_wrong_answer.jsonl", "a") as f:
                    f.write(json.dumps({"spec": {k: s[k] for k in ("topic", "facts", "answer")}, "got": fa,
                                        "problem": p[:300]}) + "\n")
            return a.rej("wrong_answer")
        if not p.rstrip().endswith("?"): return a.rej("not_a_question")
        if s.get("format") != "find_the_mistake" and bad_steps(sol): return a.rej("bad_arithmetic_step")
        text = p + "\n" + sol; words = norm_words(text)
        if set(re.findall(r"[A-Z][a-z]+", text)) & a.am_names: return a.rej("arithmark_name")
        if ngrams(words, 8) & a.ban8: return a.rej("test_overlap_8gram")
        if a.contaminated(text, a.ban13): return a.rej("eval_13gram")
        h = hashlib.sha1(" ".join(norm_words(p)).encode()).hexdigest()
        if h in a.seen: return a.rej("duplicate")
        a.seen.add(h)
        return {"problem": p, "solution": sol, "answers": [s["answer"], s2["answer"]], "grade": max(s["grade"], s2["grade"]),
                "topics": [s["topic"], s2["topic"]], "format": s["format"], "setting": s["setting"], "generator": MODEL}

    def flush(a, force=False):
        while len(a.buf) >= a.args.shard_size or (force and a.buf):
            chunk, a.buf = a.buf[:a.args.shard_size], a.buf[a.args.shard_size:]
            with open(a.out / f"items-{a.shard:05d}.jsonl", "w") as f:
                for it in chunk: f.write(json.dumps(it) + "\n")
            a.shard += 1
        (a.out / "stats.json").write_text(json.dumps(a.stats, indent=2))

    async def pace(a):
        while getattr(a, "paused_until", 0) > time.time():
            a.stats["paused_until"] = time.strftime("%a %I:%M %p", time.localtime(a.paused_until)); a.flush()
            await asyncio.sleep(min(300, a.paused_until - time.time()))
        a.stats.pop("paused_until", None)
        async with a.lock:
            now = time.time(); wait = a.next_slot - now
            if wait > 0: await asyncio.sleep(wait)
            a.next_slot = max(now, a.next_slot) + 1 / a.args.rps

    async def _one(a, sess, sem):
        batch = [specs.sample_pair(a.rng) for _ in range(a.args.per_request)]
        body = {"model": MODEL, "temperature": 1.0, "top_p": 0.95, "max_tokens": 32000,
                **({"chat_template_kwargs": TEMPLATE_KW} if TEMPLATE_KW else {}),
                "messages": [{"role": "user", "content": build_prompt(batch, a.rng)}]}
        for attempt in range(10):
            await a.pace()
            try:
                async with sess.post(API, json=body, headers={"Authorization": f"Bearer {KEY}"}) as r:
                    if r.status == 429 and "quota" in (await r.text()).lower():
                        ra = r.headers.get("Retry-After")
                        a.paused_until = max(getattr(a, "paused_until", 0), time.time() + min(1800, float(ra) if ra and ra.replace(".", "").isdigit() else 1800))
                        a.stats["quota_pauses"] = a.stats.get("quota_pauses", 0) + 1
                        continue
                    if r.status == 429 or r.status >= 500:
                        ra = r.headers.get("Retry-After")
                        a.next_slot = max(a.next_slot, time.time() + (float(ra) if ra and ra.replace(".", "").isdigit() else 1.0))
                        a.stats["retries"] = a.stats.get("retries", 0) + 1
                        await asyncio.sleep(min(30, 2 ** attempt)); continue
                    d = await r.json()
                    break
            except Exception:
                await asyncio.sleep(2 ** attempt)
        else:
            a.stats["http_errors"] += 1; return
        a.stats["requests"] += 1
        a.stats["tokens_out"] += d.get("usage", {}).get("completion_tokens", 0)
        txt = ((d.get("choices") or [{}])[0].get("message") or {}).get("content") or ""
        m = re.search(r"\[.*\]", txt, re.S)
        try: items = json.loads(m.group(0)) if m else []
        except Exception: items = []
        if not items: return a.rej("bad_json")
        for it in items:
            try: s = batch[int(it.get("id"))]
            except Exception: a.rej("bad_id"); continue
            k = a.check(s, it)
            if k: a.buf.append(k); a.stats["kept"] += 1
        a.flush()

    async def one(a, sess, sem):
        try:
            await a._one(sess, sem)
        except Exception as e:
            a.stats["worker_errors"] = a.stats.get("worker_errors", 0) + 1
            a.stats["last_worker_error"] = repr(e)[:200]
        finally:
            sem.release()

    async def run(a):
        sem = asyncio.Semaphore(a.args.inflight); tasks = set(); t0 = time.time()
        a.lock = asyncio.Lock(); a.next_slot = time.time()
        async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=600)) as sess:
            while a.stats["kept"] < a.args.items:
                await sem.acquire()
                while a.next_slot - time.time() > 2: await asyncio.sleep(0.2)   # do not queue far ahead of the pacer
                t = asyncio.create_task(a.one(sess, sem)); tasks.add(t); t.add_done_callback(tasks.discard)
                if a.stats["requests"] and a.stats["requests"] % 200 == 0:
                    el = time.time() - t0
                    print(f"{time.strftime('%I:%M %p')} req {a.stats['requests']} kept {a.stats['kept']:,} "
                          f"out-tok {a.stats['tokens_out']/1e6:.1f}M ({a.stats['tokens_out']/el:.0f}/s) rejects {a.stats['reject']}", flush=True)
            await asyncio.gather(*tasks)
        a.flush(force=True)


def main():
    ap = argparse.ArgumentParser(); ap.add_argument("out"); ap.add_argument("--items", type=int, default=1000)
    ap.add_argument("--per-request", type=int, default=20); ap.add_argument("--rps", type=float, default=2.8)
    ap.add_argument("--inflight", type=int, default=96); ap.add_argument("--seed", type=int, default=20261015)
    ap.add_argument("--shard-size", type=int, default=50000)
    args = ap.parse_args(); out = Path(args.out); out.mkdir(parents=True, exist_ok=True)
    asyncio.run(Gen(out, args).run())


if __name__ == "__main__":
    main()