From 5d2e4252e695644c34be605b44a14bd80a98a0a6 Mon Sep 17 00:00:00 2001 From: jdj Date: Wed, 18 Nov 2026 14:47:33 -0800 Subject: Boris-2 training code The code, configuration and data filters used to train Boris-2 (144M, 200B tokens, single V100): model, training loop, data preparation, n-gram table, V100 kernels, final stages and the exact run configs. --- libremath/fragments.py | 134 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 libremath/fragments.py (limited to 'libremath/fragments.py') diff --git a/libremath/fragments.py b/libremath/fragments.py new file mode 100644 index 0000000..4be2380 --- /dev/null +++ b/libremath/fragments.py @@ -0,0 +1,134 @@ +"""LibreMath fragment bank (2026-10-23): K2 writes reusable sentence fragments ONCE; compose.py +assembles unlimited problems from them with exact, code-computed math. + +Each category has a fixed set of placeholders. A fragment is kept only if it uses exactly the +allowed placeholders, includes the required ones, contains no literal digits, and (for solution +steps) contains the exact equation string the composer will fill. + + python libremath/fragments.py OUT.json --calls-per-category 12 +""" +import argparse, asyncio, json, random, re, sys, time +from pathlib import Path +import aiohttp + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import generate as G + +# category: (description, required placeholders, allowed placeholders, kind) +CATS = { + "opener": ("an opening sentence that sets a scene at or around {place}; it may mention {A}", {"place"}, {"place", "A"}, "s"), + "distractor": ("a sentence giving one irrelevant fact that includes the number {d} (not needed to solve any question); it may mention {A} or {place}", {"d"}, {"d", "A", "place"}, "s"), + "combine_setup": ("one or two sentences saying {A} has {a} {obj} and then gets {b} more (buys, finds, receives, picks, is given...)", {"A", "a", "b", "obj"}, {"A", "a", "b", "obj", "place"}, "s"), + "combine_q": ("a question asking how many {obj} {A} has now in total", {"obj"}, {"A", "obj"}, "q"), + "combine_step": ("a solution sentence explaining why we add, containing exactly '{a} + {b} = {ans}'", {"a", "b", "ans"}, {"a", "b", "ans", "A", "obj"}, "s"), + "remove_setup": ("one or two sentences saying {A} starts with {a} {obj} and then {b} of them are used, given away, sold, eaten or lost", {"A", "a", "b", "obj"}, {"A", "a", "b", "obj", "place"}, "s"), + "remove_q": ("a question asking how many {obj} {A} has left", {"obj"}, {"A", "obj"}, "q"), + "remove_step": ("a solution sentence explaining why we subtract, containing exactly '{a} - {b} = {ans}'", {"a", "b", "ans"}, {"a", "b", "ans", "A", "obj"}, "s"), + "compare_setup": ("one or two sentences saying {A} has {a} {obj} and {B} has {b} {obj}", {"A", "B", "a", "b"}, {"A", "B", "a", "b", "obj", "place"}, "s"), + "compare_q": ("a question asking how many more {obj} {A} has than {B}", {"A", "B"}, {"A", "B", "obj"}, "q"), + "compare_step": ("a solution sentence explaining the difference, containing exactly '{a} - {b} = {ans}'", {"a", "b", "ans"}, {"a", "b", "ans", "A", "B", "obj"}, "s"), + "groups_setup": ("one or two sentences saying there are {a} {cont} with {b} {obj} in each (packed, arranged, stacked...); may mention {A}", {"a", "b", "cont", "obj"}, {"A", "a", "b", "cont", "obj", "place"}, "s"), + "groups_q": ("a question asking how many {obj} there are altogether", {"obj"}, {"A", "obj", "cont"}, "q"), + "groups_step": ("a solution sentence explaining why we multiply, containing exactly '{a} × {b} = {ans}'", {"a", "b", "ans"}, {"a", "b", "ans", "obj", "cont"}, "s"), + "share_setup": ("one or two sentences saying {A} shares or splits {a} {obj} equally into {b} {cont}", {"A", "a", "b", "obj", "cont"}, {"A", "a", "b", "obj", "cont", "place"}, "s"), + "share_q": ("a question asking how many {obj} go in each of the {cont}", {"obj", "cont"}, {"A", "obj", "cont"}, "q"), + "share_step": ("a solution sentence explaining why we divide, containing exactly '{a} ÷ {b} = {ans}'", {"a", "b", "ans"}, {"a", "b", "ans", "obj", "cont"}, "s"), + "followup_remove": ("a follow-up sentence (for after a total is known) saying {c} of those {obj} are then used, sold or given away", {"c", "obj"}, {"c", "obj", "A", "B"}, "s"), + "followup_q": ("a question asking how many {obj} are left after that", set(), {"obj", "A"}, "q"), + "followup_step": ("a solution sentence for the second step, containing exactly '{x} - {c} = {ans}'", {"x", "c", "ans"}, {"x", "c", "ans", "obj"}, "s"), + "money_setup": ("one or two sentences saying {A} buys {a} {obj} that cost {p} dollars each", {"A", "a", "obj", "p"}, {"A", "a", "obj", "p", "place"}, "s"), + "money_q": ("a question asking how much {A} spends in total", {"A"}, {"A", "obj"}, "q"), + "money_step": ("a solution sentence explaining the total cost, containing exactly '{a} × ${p} = ${ans}'", {"a", "p", "ans"}, {"a", "p", "ans", "obj", "A"}, "s"), + "answer": ("a short final sentence stating the answer {ans}; it may mention {obj} or {A}", {"ans"}, {"ans", "obj", "A"}, "s"), +} +TONES = ["plain and simple, for young children", "like a friendly teacher", "like a math textbook", "lively and story-like", + "short and direct", "detailed, with a little extra description", "conversational", "matter-of-fact"] +PH = re.compile(r"\{(\w+)\}") + + +def valid(frag, cat): + desc, req, allowed, kind = CATS[cat] + if not isinstance(frag, str) or not (12 <= len(frag) <= 260): return False + if re.search(r"\d", PH.sub("", frag)): return False + ph = set(PH.findall(frag)) + if not req <= ph or not ph <= allowed: return False + if kind == "q" and not frag.rstrip().endswith("?"): return False + if kind == "s" and frag.rstrip().endswith("?"): return False + eq = re.search(r"'(.*)'", desc) + if eq and eq.group(1) not in frag: return False + return True + + +def prompt(cat, n, tone): + if cat == "objects": + return (f"List {n} pairs of things for grade-school math word problems: a plural countable item and a plural container or " + f"group it could be packed or arranged in (for example marbles / jars, chairs / rows, stickers / sheets). Cover many " + f"different domains (food, sports, school, nature, tools, crafts, animals, travel, music, building). Reply with only a " + f'JSON array of objects like {{"obj": "marbles", "cont": "jars"}}.') + desc = CATS[cat][0] + return (f"Write {n} different English fragments for building grade-school math word problems. Each fragment is {desc}. " + f"Use the placeholders in curly braces exactly as written (they are filled in later by a program); never write a digit. " + + ("Do not mention any specific item, food, animal or object; the items being counted are filled in later. " + if cat in ("opener", "distractor") else + "Never name a specific item, food, animal, place or object: refer to the items only as {obj} and {cont} " + "(or words like 'them', 'each one', 'the items'), because the program fills them in. ") + + f"Style: {tone}. Vary vocabulary and sentence structure as much as possible; no two fragments should start the same way. " + f"Plain text, no markdown. Reply with only a JSON array of strings.") + + +async def main_async(a): + out = Path(a.out); bank = json.loads(out.read_text()) if out.exists() else {c: [] for c in list(CATS) + ["objects"]} + sem = asyncio.Semaphore(a.inflight); lock = asyncio.Lock(); state = {"next": time.time(), "tok": 0} + rng = random.Random(a.seed) + + async def call(sess, cat): + async with sem: + body = {"model": G.MODEL, "temperature": 1.0, "top_p": 0.95, "max_tokens": 16000, + **({"chat_template_kwargs": G.TEMPLATE_KW} if G.TEMPLATE_KW else {}), + "messages": [{"role": "user", "content": prompt(cat, a.per_call, rng.choice(TONES))}]} + for attempt in range(8): + async with lock: + w = state["next"] - time.time() + if w > 0: await asyncio.sleep(w) + state["next"] = max(time.time(), state["next"]) + 1 / a.rps + try: + async with sess.post(G.API, json=body, headers={"Authorization": f"Bearer {G.KEY}"}) as r: + if r.status == 429: + txt = await r.text() + await asyncio.sleep(1800 if "quota" in txt.lower() else 5 * (attempt + 1)); continue + if r.status >= 500: await asyncio.sleep(5 * (attempt + 1)); continue + d = await r.json(); break + except Exception: + await asyncio.sleep(5 * (attempt + 1)) + else: + return cat, [] + state["tok"] += d.get("usage", {}).get("completion_tokens", 0) + txt = ((d.get("choices") or [{}])[0].get("message") or {}).get("content") or "" + m = re.search(r"\[.*\]", txt, re.S) + try: arr = json.loads(m.group(0)) if m else [] + except Exception: arr = [] + if cat == "objects": + return cat, [{"obj": o["obj"].strip().lower(), "cont": o["cont"].strip().lower()} for o in arr + if isinstance(o, dict) and all(isinstance(o.get(k), str) and re.fullmatch(r"[a-zA-Z' -]{3,30}", o[k].strip()) for k in ("obj", "cont"))] + return cat, [f.strip() for f in arr if valid(f, cat)] + + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=600)) as sess: + tasks = [call(sess, c) for c in CATS for _ in range(a.calls_per_category)] + [call(sess, "objects") for _ in range(a.calls_per_category)] + for fut in asyncio.as_completed(tasks): + cat, frags = await fut + seen = {json.dumps(f, sort_keys=True) for f in bank[cat]} + bank[cat] += [f for f in frags if json.dumps(f, sort_keys=True) not in seen] + out.write_text(json.dumps(bank, indent=1)) + print(f" {cat}: +{len(frags)} -> {len(bank[cat])} (output tokens so far {state['tok']:,})", flush=True) + print({c: len(v) for c, v in bank.items()}) + + +def main(): + ap = argparse.ArgumentParser(); ap.add_argument("out") + ap.add_argument("--calls-per-category", type=int, default=12); ap.add_argument("--per-call", type=int, default=40) + ap.add_argument("--rps", type=float, default=1.0); ap.add_argument("--inflight", type=int, default=16); ap.add_argument("--seed", type=int, default=7) + asyncio.run(main_async(ap.parse_args())) + + +if __name__ == "__main__": + main() -- cgit v1.2.3