summaryrefslogtreecommitdiff
path: root/libremath/compose.py
diff options
context:
space:
mode:
authorjdj <jdj@kunix.org>2026-11-18 14:47:33 -0800
committerjdj <jdj@kunix.org>2026-11-18 14:47:33 -0800
commit5d2e4252e695644c34be605b44a14bd80a98a0a6 (patch)
tree311a72eda4459af7547b9cd05c0956eafb235f2d /libremath/compose.py
downloadboris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.tar.gz
boris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.zip
Boris-2 training code
The code, configuration and data filters used to train Boris-2 (144M, 200B tokens, single V100): model, training loop, data preparation, n-gram table, V100 kernels, final stages and the exact run configs.
Diffstat (limited to 'libremath/compose.py')
-rw-r--r--libremath/compose.py117
1 files changed, 117 insertions, 0 deletions
diff --git a/libremath/compose.py b/libremath/compose.py
new file mode 100644
index 0000000..46f8bb5
--- /dev/null
+++ b/libremath/compose.py
@@ -0,0 +1,117 @@
+"""LibreMath composer (2026-10-23): unlimited grade-school word problems from the K2-written
+fragment bank (fragments.py), with every number and answer computed here.
+
+ python libremath/compose.py BANK.json OUT_DIR --items 1000000 [--seed 1]
+"""
+import argparse, json, random, re, sys
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+import generate as G
+from tgen import SETTINGS_EXTRA
+
+PH = re.compile(r"\{(\w+)\}")
+
+
+def fill(t, env):
+ t = re.sub(r"\b([Ee])(very|ach) \{(cont|obj)\}", lambda m: ("All the {" if m.group(1) == "E" else "all the {") + m.group(3) + "}"
+ if m.group(2) == "very" else m.group(1) + "ach of the {" + m.group(3) + "}", t) # plural nouns: "every boxes"
+ out = PH.sub(lambda m: str(env[m.group(1)]), t)
+ out = re.sub(r"\b([Tt]he) the\b", r"\1", out)
+ out = out.strip(); out = out[:1].upper() + out[1:]
+ return out if out[-1:] in ".!?\"'”" else out + "."
+
+
+ANSWER_TAGS = {"money": r"cost|spen|paid|pay|price|dollar|bill|\$", "remove": r"\bleft\b|remain|still", "two_step": r"\bleft\b|remain|still",
+ "compare": r"more|differ|ahead|extra", "share": r"\beach\b|per\b|every", "combine": r"total|altogether|in all|together|sum",
+ "groups": r"total|altogether|in all|together"}
+ALL_TAGS = "|".join(ANSWER_TAGS.values()) + r"|collect|gather|found|bought|made|earn|save"
+
+
+def money(x):
+ return f"{x:.2f}"
+
+
+class Composer:
+ def __init__(a, bank, seed):
+ a.b = bank; a.r = random.Random(seed)
+ a.places = G.SETTINGS + SETTINGS_EXTRA; a.names = G.NAMES
+
+ def pick(a, cat): return a.r.choice(a.b[cat])
+
+ def answer_for(a, kind):
+ if not hasattr(a, "_ans"):
+ gen = [f for f in a.b["answer"] if not re.search(ALL_TAGS, f, re.I)]
+ a._ans = {k: (gen if k != "money" else [f for f in gen if "{obj}" not in f]) + [f for f in a.b["answer"] if re.search(v, f, re.I) and not re.search(
+ "|".join(v2 for k2, v2 in ANSWER_TAGS.items() if v2 != v), f, re.I)] for k, v in ANSWER_TAGS.items()}
+ return a.r.choice(a._ans[kind])
+
+ def numbers(a, kind):
+ r = a.r; big = r.random() < 0.35
+ if kind in ("combine", "remove", "compare"):
+ hi = r.choice([20, 50, 100, 100, 500, 1000]) if not big else 5000
+ x, y = r.randint(2, hi), r.randint(2, hi)
+ return (max(x, y), min(x, y)) if kind != "combine" else (x, y)
+ if kind == "groups": return r.randint(2, 12 if not big else 40), r.randint(2, 12 if not big else 99)
+ if kind == "share":
+ b = r.randint(2, 12); return b * r.randint(2, 15 if not big else 60), b
+ if kind == "money": return r.randint(2, 12), r.randint(25, 1999) / 100
+
+ def item(a):
+ r = a.r; A, B = r.sample(a.names, 2); o = r.choice(a.b["objects"]); place = r.choice(a.places)
+ kind = r.choice(["combine", "remove", "compare", "groups", "groups", "share", "money", "two_step"])
+ base = "groups" if kind == "two_step" and r.random() < 0.6 else ("combine" if kind == "two_step" else kind)
+ x, y = a.numbers(base)
+ if x == y and base in ("remove", "compare"): x += 1
+ ans = {"combine": lambda: x + y, "remove": lambda: x - y, "compare": lambda: x - y, "groups": lambda: x * y,
+ "share": lambda: x // y, "money": lambda: round(x * y, 2)}[base]()
+ env = {"A": A, "B": B, "obj": o["obj"], "cont": o["cont"], "place": "the " + place, "a": x, "b": y,
+ "ans": money(ans) if base == "money" else ans, "p": money(y) if base == "money" else y, "d": r.randint(2, 60)}
+ prob = [fill(a.pick(f"{base}_setup"), env)]
+ steps = [fill(a.pick(f"{base}_step"), env)]
+ final = ans
+ if kind == "two_step":
+ c = r.randint(1, max(1, ans - 1)); env2 = dict(env, x=ans, c=c, ans=ans - c)
+ prob.append(fill(a.pick("followup_remove"), env2)); q = fill(a.pick("followup_q"), env2)
+ steps.append(fill(a.pick("followup_step"), env2)); final = ans - c; env = env2
+ else:
+ q = fill(a.pick(f"{base}_q"), env)
+ if r.random() < 0.25: # irrelevant fact, before or after the setup
+ prob.insert(r.choice([0, len(prob)]), fill(a.pick("distractor"), env))
+ if r.random() < 0.7: prob.insert(0, fill(a.pick("opener"), env))
+ problem = " ".join(prob + [q])
+ solution = " ".join(steps + [fill(a.answer_for(kind), env)])
+ if r.random() < 0.3: problem, solution = "Question: " + problem, "Answer: " + solution
+ return {"problem": problem, "solution": solution, "answers": [str(env["ans"])], "kind": kind,
+ "objects": o, "generator": "LibreMath fragment composer (fragments by IFM/K2-Horizon)"}
+
+
+def distinct(texts, n=4):
+ tot = 0; s = set()
+ for t in texts:
+ w = re.sub(r"\d+", "#", t.lower()).split(); g = [tuple(w[i:i + n]) for i in range(len(w) - n + 1)]; tot += len(g); s.update(g)
+ return len(s) / max(tot, 1)
+
+
+def main():
+ ap = argparse.ArgumentParser(); ap.add_argument("bank"); ap.add_argument("out")
+ ap.add_argument("--items", type=int, default=100000); ap.add_argument("--seed", type=int, default=1)
+ ap.add_argument("--no-filter", action="store_true")
+ a = ap.parse_args(); out = Path(a.out); out.mkdir(parents=True, exist_ok=True)
+ bank = json.load(open(a.bank)); c = Composer(bank, a.seed)
+ am_names, ban8 = set(), set()
+ if not a.no_filter:
+ am_names, ban8, _, _ = G.load_bans()
+ seen = set(); kept = rej = 0
+ with open(out / f"items-seed{a.seed}.jsonl", "w") as f:
+ while kept < a.items:
+ it = c.item(); text = it["problem"] + " " + it["solution"]
+ h = hash(it["problem"])
+ if h in seen or (set(re.findall(r"[A-Z][a-z]+", text)) & am_names) or (ban8 and G.ngrams(G.norm_words(text), 8) & ban8):
+ rej += 1; continue
+ seen.add(h); f.write(json.dumps(it) + "\n"); kept += 1
+ S = [json.loads(l) for l in open(out / f"items-seed{a.seed}.jsonl")][:20000]
+ print(f"kept {kept:,}, rejected {rej:,}; distinct 4-gram ratio (numbers masked) on 20k: {distinct([s['problem'] + ' ' + s['solution'] for s in S]):.3f}")
+
+
+if __name__ == "__main__":
+ main()