summaryrefslogtreecommitdiff
path: root/libremath/review_fragments.py
blob: 4352c3a68c68b7436413cfd4162aa3f282dd29af (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
"""One-time K2 review of the fragment bank (2026-10-23): keep only fragments that read naturally
for ANY items/containers the composer may plug in, and object/container pairs that make sense.

    python libremath/review_fragments.py BANK.json REVIEWED.json
"""
import asyncio, json, re, sys, time
from pathlib import Path
import aiohttp
sys.path.insert(0, str(Path(__file__).resolve().parent))
import generate as G
from fragments import CATS

RULE = ("A program fills the curly-brace placeholders: {A}/{B} are people's names, {obj} is ANY plural countable item "
        "(e.g. marbles, cookies, stickers), {cont} is ANY plural container or group (e.g. jars, boxes, rows), {place} is a "
        "location, and letters like {a} {b} {c} {d} {x} {p} {ans} are numbers.")


def ask(cat, frags):
    if cat == "objects":
        body = "\n".join(f"{i}. {f['obj']} / {f['cont']}" for i, f in enumerate(frags))
        q = ("Each line is a plural countable item and a plural container or grouping for a grade-school math problem "
             "(\"there are N <containers> with M <items> in each\"). Mark a line bad if the item would not sensibly be packed, "
             "arranged or grouped in that container, or if either word is not a plural noun.")
    else:
        body = "\n".join(f"{i}. {f}" for i, f in enumerate(frags))
        q = (f"{RULE} Each line is meant to be: {CATS[cat][0]}. Mark a line bad if it names or implies specific items, "
             "food, animals, actions or imagery that would clash with arbitrary {obj}/{cont}, if it would be ungrammatical "
             "with a plural {obj} or {cont}, if its math meaning is wrong for the description, or if it is awkward or confusing.")
    return q + "\n\n" + body + "\n\nReply with only a JSON array of the line numbers that are bad."


async def main_async(src, dst):
    bank = json.load(open(src)); out = {k: [] for k in bank}; sem = asyncio.Semaphore(16); lock = asyncio.Lock(); nxt = [time.time()]
    jobs = [(cat, bank[cat][i:i + 40]) for cat in bank for i in range(0, len(bank[cat]), 40)]

    async def run(sess, cat, frags):
        async with sem:
            body = {"model": G.MODEL, "temperature": 0.2, "max_tokens": 8000,
                    **({"chat_template_kwargs": G.TEMPLATE_KW} if G.TEMPLATE_KW else {}),
                    "messages": [{"role": "user", "content": ask(cat, frags)}]}
            for attempt in range(8):
                async with lock:
                    w = nxt[0] - time.time()
                    if w > 0: await asyncio.sleep(w)
                    nxt[0] = max(time.time(), nxt[0]) + 1.0
                try:
                    async with sess.post(G.API, json=body, headers={"Authorization": f"Bearer {G.KEY}"}) as r:
                        if r.status in (429,) or r.status >= 500: await asyncio.sleep(10 * (attempt + 1)); continue
                        d = await r.json(); break
                except Exception:
                    await asyncio.sleep(5 * (attempt + 1))
            else:
                return cat, [], len(frags)
            txt = ((d.get("choices") or [{}])[0].get("message") or {}).get("content") or ""
            m = re.search(r"\[[\d,\s]*\]", txt)
            if not m: return cat, [], len(frags)                         # unparseable review: drop the batch (be strict)
            bad = set(json.loads(m.group(0)))
            return cat, [f for i, f in enumerate(frags) if i not in bad], len(frags)

    async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=600)) as sess:
        for fut in asyncio.as_completed([run(sess, c, f) for c, f in jobs]):
            cat, keep, n = await fut; out[cat] += keep
    json.dump(out, open(dst, "w"), indent=1)
    print({k: f"{len(out[k])}/{len(bank[k])}" for k in bank})


if __name__ == "__main__":
    asyncio.run(main_async(sys.argv[1], sys.argv[2]))