summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorjdj <jdj@kunix.org>2026-11-18 14:47:33 -0800
committerjdj <jdj@kunix.org>2026-11-18 14:47:33 -0800
commit5d2e4252e695644c34be605b44a14bd80a98a0a6 (patch)
tree311a72eda4459af7547b9cd05c0956eafb235f2d
downloadboris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.tar.gz
boris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.zip
Boris-2 training code
The code, configuration and data filters used to train Boris-2 (144M, 200B tokens, single V100): model, training loop, data preparation, n-gram table, V100 kernels, final stages and the exact run configs.
-rw-r--r--.gitignore6
-rw-r--r--LICENSE202
-rw-r--r--README.md50
-rw-r--r--checkpoint.py254
-rw-r--r--config.py629
-rw-r--r--configs/boris-2-0929/RECIPE.md9
-rw-r--r--configs/boris-2-0929/config.before_2xbatch.json220
-rw-r--r--configs/boris-2-0929/config.before_decaymix.json145
-rw-r--r--configs/boris-2-0929/config.before_no8192.json225
-rw-r--r--configs/boris-2-0929/config.before_p2tail.json197
-rw-r--r--configs/boris-2-0929/config.json220
-rw-r--r--configs/boris-2-0929/final_decay_draws.json10
-rw-r--r--data.py488
-rw-r--r--docs/design-notes-0920.md218
-rw-r--r--eval.py358
-rwxr-xr-xfinalize_0920.sh22
-rw-r--r--libremath/arithcheck.py27
-rw-r--r--libremath/compose.py117
-rw-r--r--libremath/fragments.py134
-rw-r--r--libremath/generate.py247
-rw-r--r--libremath/review_fragments.py68
-rw-r--r--libremath/specs.py204
-rw-r--r--libremath/tgen.py261
-rw-r--r--model.py833
-rw-r--r--ngram.py414
-rw-r--r--optim.py293
-rw-r--r--preflight.py238
-rw-r--r--prepare.py645
-rwxr-xr-xprepare_0920.sh102
-rw-r--r--retokenize.py103
-rw-r--r--schedule.py106
-rw-r--r--scripts/hellaswag_title_split.py67
-rw-r--r--scripts/pilot_train.py191
-rw-r--r--scripts/prepare_orca.py23
-rw-r--r--scripts/prepare_wordprob.py187
-rw-r--r--scripts/sample.py47
-rw-r--r--scripts/scan_decay_contam.py17
-rw-r--r--sft_chat.py165
-rw-r--r--supervise_run.py34
-rw-r--r--tokenizer.py118
-rw-r--r--train.py596
-rw-r--r--verify.py1164
-rw-r--r--volta_kernels/README.md165
-rw-r--r--volta_kernels/__init__.py124
-rw-r--r--volta_kernels/attn.cu761
-rw-r--r--volta_kernels/canon_rope.py171
-rw-r--r--volta_kernels/canon_rope_bw.py327
-rw-r--r--volta_kernels/canon_rope_cuda.py266
-rw-r--r--volta_kernels/head.py168
-rw-r--r--volta_kernels/integ_ops.py256
50 files changed, 11892 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..d880138
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,6 @@
+__pycache__/
+*.pyc
+*.pt
+data
+checkpoints
+logs/
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..d645695
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..9a6e834
--- /dev/null
+++ b/README.md
@@ -0,0 +1,50 @@
+# Boris-2: training code
+
+The complete code used to train [Boris-2](https://huggingface.co/opencerebral/Boris-2), a 144M-parameter language model trained from scratch on 200B tokens on a single NVIDIA V100 32GB, and its post-trained variants (Instruct, Think, Think-Instruct).
+
+Weights, tokenizer and model card: <https://huggingface.co/opencerebral/Boris-2>.
+
+## What is here
+
+| Path | Contents |
+|---|---|
+| `config.py` | Every setting of the run: model shape, n-gram table, optimizer, schedule, data mixture phases. `python config.py` prints the parameter budget and asserts it. |
+| `model.py` | The model: decoder with fused Canon layers, a 15M-parameter n-gram lookup table and a float32 residual stream. |
+| `train.py` | The training loop: Muon + AdamW, fp16 autocast with GradScaler, rolling checkpoints, resume. |
+| `data.py` | Variable-length packing; attention, Canon and the n-gram lookup are masked at document boundaries. |
+| `ngram.py` | Builds the bigram/trigram table vocabulary from the training data. |
+| `optim.py`, `schedule.py`, `checkpoint.py` | Optimizers, warmup-stable-decay schedule with phase-interpolated mixtures, checkpoint format. |
+| `prepare.py`, `prepare_0920.sh`, `retokenize.py`, `tokenizer.py` | Downloading, filtering and tokenizing every source into shards. |
+| `eval.py`, `verify.py`, `preflight.py` | In-run evaluation, numerical verification of the fused paths against references, pre-launch checks. |
+| `volta_kernels/` | Hand-written CUDA/Triton kernels for the V100 (sm_70): varlen attention with fused Canon and RoPE, fused LM head. Used only when `BORIS_VOLTA_KERNELS=1` on an sm_70 GPU; every other GPU uses the stock PyTorch path. |
+| `sft_chat.py` | Supervised fine-tuning for the chat and reasoning variants. |
+| `scripts/` | The final training stages and the data filters used for the decay sources (see below). |
+| `libremath/` | The generator behind the LibreMath v1 dataset. |
+| `configs/boris-2-0929/` | The exact configuration of the released run, including the configuration before each mixture change, and the final decay draw weights. |
+| `docs/design-notes-0920.md` | Design notes written during the run (historical; some plans described there changed later). |
+
+## Reproducing Boris-2
+
+1. **Data.** `prepare_0920.sh` downloads and tokenizes every source with `prepare.py`. The sources, their share of each phase and the filters applied are in `configs/boris-2-0929/config.json` and the model card. Decay-only sources use the scripts in `scripts/`: `prepare_wordprob.py` builds the Cosmopedia tutorial sources (`cosmo2_how`, `cosmo_how_v1_tf`), dropping any article whose title matches a HellaSwag WikiHow item, plus the word-problem, MetaMathQA and LibreMath sources; `prepare_orca.py` builds Orca-Math. `hellaswag_title_split.py` and `scan_decay_contam.py` are the checks run on the result. `data/` is expected to point at the prepared shards.
+2. **n-gram table.** `python ngram.py build` samples the training mixture and writes the table vocabulary.
+3. **Main run.** `BORIS_RUN_DIR=<run dir> BORIS_VOLTA_KERNELS=1 python train.py --config configs/boris-2-0929/config.json`. Resume with `--resume`.
+4. **Final stages** (from the 200B checkpoint), with `scripts/pilot_train.py`:
+ - 30M tokens at 0.1x peak learning rate on the "nudge" mixture (`--peak 0.1`),
+ - 10M tokens of LibreMath v1 at 0.02x peak learning rate (`--peak 0.02`).
+ The exact mixtures and seeds are in `configs/boris-2-0929/RECIPE.md`.
+
+Paths such as `/media/joseph/Media/...` are the defaults of the original machine. Set `BORIS_RUN_DIR`, point `data/` at your shards and edit the remaining paths in the configs for your own setup.
+
+## Data licenses
+
+The training data comes from public datasets with their own licenses. Most are permissive (Apache-2.0, MIT, ODC-By, CC-BY-4.0). The largest source, ClimbMix, is published by NVIDIA under CC-BY-NC-4.0 (non-commercial); Boris-2 was trained on the `karpathy/climbmix-400b-shuffle` mirror of it. Check the license of each source before reusing the data.
+
+## Copyright & License
+
+*Copyright 2026 OpenCerebral*
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use these files except in compliance with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
diff --git a/checkpoint.py b/checkpoint.py
new file mode 100644
index 0000000..5da5921
--- /dev/null
+++ b/checkpoint.py
@@ -0,0 +1,254 @@
+"""Checkpoint save, load, rotate, and the weights-only export for eval.
+
+Three kinds, and they are not interchangeable:
+
+ rolling Every 45 minutes, two kept, rotated. This is what a run actually
+ resumes from. The 50B milestones are 25+ days apart; a run that is
+ expected to be interrupted cannot use them as its resume point.
+ full Every 50B tokens, permanent. Weights, Muon momentum buffers, AdamW
+ moments, the loss scale, the packer position and every RNG stream.
+ The end-of-stable one is kept forever and never rotated: variants
+ branch from it as a tree, and re-warming from a decayed model costs
+ a loss spike and a slow recovery.
+ export Every 10B tokens, weights only, fp16, ~287MB. Written to a watched
+ directory with a manifest line; eval.py on the second GPU picks it
+ up. Training never waits for eval and eval never touches GPU 0.
+
+Writes are atomic (tmp file, fsync, rename), so a checkpoint interrupted mid-
+write cannot leave a torn `latest.pt` behind. The GPU->CPU copy is synchronous
+and takes about a second; the disk write happens on a background thread, so the
+training loop stalls for the copy and not for the SATA SSD.
+"""
+from __future__ import annotations
+
+import json
+import os
+import random
+import re
+import threading
+import time
+from pathlib import Path
+
+import numpy as np
+import torch
+
+from config import Config
+
+_writer_lock = threading.Lock()
+
+
+def _atomic_write(blob, path: Path):
+ path = Path(path)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ tmp = path.with_suffix(path.suffix + ".tmp")
+ with open(tmp, "wb") as f:
+ torch.save(blob, f)
+ f.flush()
+ os.fsync(f.fileno())
+ os.replace(tmp, path)
+
+
+def _cpu_state(sd: dict) -> dict:
+ return {k: (v.detach().to("cpu", copy=True) if torch.is_tensor(v) else v)
+ for k, v in sd.items()}
+
+
+def _deep_cpu(obj):
+ if torch.is_tensor(obj):
+ return obj.detach().to("cpu", copy=True)
+ if isinstance(obj, dict):
+ return {k: _deep_cpu(v) for k, v in obj.items()}
+ if isinstance(obj, (list, tuple)):
+ return type(obj)(_deep_cpu(v) for v in obj)
+ return obj
+
+
+def save_full(path: Path, *, model, muon, adamw, scaler, packer, step: int,
+ tokens_seen: int, cfg: Config, ngram_fingerprint: str,
+ best_val: float = float("inf"), extra: dict | None = None,
+ background: bool = True, rotate_pattern: str | None = None,
+ rotate_keep: int | None = None):
+ raw = getattr(model, "_orig_mod", model)
+ blob = {
+ "format": 1,
+ "model": _cpu_state(raw.state_dict()),
+ "muon": _deep_cpu(muon.state_dict()),
+ "adamw": _deep_cpu(adamw.state_dict()),
+ "scaler": scaler.state_dict() if scaler is not None else None,
+ "packer": packer.state_dict() if packer is not None else None,
+ "step": int(step),
+ "tokens_seen": int(tokens_seen),
+ "best_val": float(best_val),
+ "config": cfg.to_json(),
+ "fingerprint": cfg.fingerprint(),
+ "ngram_fingerprint": ngram_fingerprint,
+ "rng": {"python": random.getstate(),
+ "numpy": np.random.get_state(),
+ "torch": torch.get_rng_state(),
+ "cuda": torch.cuda.get_rng_state_all() if torch.cuda.is_available() else None},
+ "extra": extra or {},
+ "wall": time.time(),
+ }
+
+ def _write():
+ with _writer_lock:
+ _atomic_write(blob, path)
+ if rotate_pattern is not None and rotate_keep is not None:
+ rotate(path.parent, rotate_pattern, rotate_keep)
+
+ if background:
+ t = threading.Thread(target=_write, daemon=False)
+ t.start()
+ return t
+ _write()
+ return None
+
+
+def save_export(path: Path, *, model, cfg: Config, step: int, tokens_seen: int,
+ manifest: Path, ngram_fingerprint: str, background: bool = True):
+ """Weights only, fp16, no optimizer state -- the eval handoff."""
+ raw = getattr(model, "_orig_mod", model)
+ sd = {k: v.detach().to("cpu", torch.float16, copy=True)
+ for k, v in raw.state_dict().items()}
+ blob = {"format": 1, "model": sd, "config": cfg.to_json(),
+ "step": int(step), "tokens_seen": int(tokens_seen),
+ "fingerprint": cfg.fingerprint(),
+ "ngram_fingerprint": ngram_fingerprint, "wall": time.time()}
+
+ def _write():
+ with _writer_lock:
+ _atomic_write(blob, path)
+ manifest.parent.mkdir(parents=True, exist_ok=True)
+ line = json.dumps({"path": str(path), "step": int(step),
+ "tokens_seen": int(tokens_seen),
+ "wall": time.time()})
+ with open(manifest, "a") as f:
+ f.write(line + "\n")
+ f.flush()
+ os.fsync(f.fileno())
+
+ if background:
+ t = threading.Thread(target=_write, daemon=False)
+ t.start()
+ return t
+ _write()
+ return None
+
+
+def _migrate_adamw_groups(blob, model, adamw):
+ """Re-key an older AdamW state onto the current param_groups by identity.
+
+ state_dict indexes parameters by position across all groups, so ANY change in
+ grouping -- 1 group (before 2026-09-17), 2 groups [embedding+ngram | norms+canon]
+ (before 2026-10-07), now 3 groups [embedding+ngram | norms | canon] -- moves
+ indices. load_state_dict refuses a group-count mismatch, and if counts lined up
+ it would pair a parameter with another one's exp_avg/exp_avg_sq. Rebuild the
+ saved order from the layout that wrote it and permute by parameter identity;
+ every moved moment tensor must match its parameter's shape.
+ """
+ from optim import _role, ADAMW_ROLES
+ saved = blob["adamw"]
+ live = [p for g in adamw.param_groups for p in g["params"]]
+ named = [(p, _role(p, n)) for n, p in model.named_parameters()
+ if p.requires_grad and _role(p, n) in ADAMW_ROLES]
+ layouts = {1: [{"embedding", "ngram_table", "norm", "canon"}],
+ 2: [{"embedding", "ngram_table"}, {"norm", "canon"}],
+ 3: [{"embedding", "ngram_table"}, {"norm"}, {"canon"}]}
+ k = len(saved["param_groups"])
+ sizes = [len(g["params"]) for g in saved["param_groups"]]
+ old_order = [p for roles in layouts[k] for p, r in named if r in roles]
+ if [sum(r in roles for _, r in named) for roles in layouts[k]] != sizes:
+ raise AssertionError(f"adamw migration: saved group sizes {sizes} do not "
+ f"match the {k}-group layout -- refusing to guess")
+ if [id(p) for p in old_order] == [id(p) for p in live] and k == len(adamw.param_groups):
+ return saved
+ if len(old_order) != len(live):
+ raise AssertionError(
+ f"adamw migration: {len(old_order)} saved tensors vs "
+ f"{len(live)} live tensors -- refusing to guess")
+ pos = {id(p): i for i, p in enumerate(old_order)}
+ perm = [pos[id(p)] for p in live]
+ old_state = saved["state"]
+ new_state = {j: old_state[o] for j, o in enumerate(perm) if o in old_state}
+ for j, st in new_state.items():
+ for key in ("exp_avg", "exp_avg_sq"):
+ if key in st:
+ assert st[key].shape == live[j].shape, (
+ f"adamw migration: {key} {tuple(st[key].shape)} landed on a "
+ f"{tuple(live[j].shape)} parameter")
+ tmpl = {kk: v for kk, v in saved["param_groups"][0].items() if kk != "params"}
+ groups, j = [], 0
+ for g in adamw.param_groups:
+ n = len(g["params"])
+ groups.append({**tmpl, "weight_decay": g["weight_decay"],
+ "params": list(range(j, j + n))})
+ j += n
+ return {"state": new_state, "param_groups": groups}
+
+
+def load_full(path: Path, *, model=None, muon=None, adamw=None, scaler=None,
+ packer=None, map_location="cpu", expect_fingerprint: str | None = None,
+ expect_ngram: str | None = None, strict: bool = True) -> dict:
+ blob = torch.load(Path(path), map_location=map_location, weights_only=False)
+
+ if expect_fingerprint is not None and blob.get("fingerprint") != expect_fingerprint:
+ raise SystemExit(
+ f"{path} was written by a run whose config fingerprint is "
+ f"{blob.get('fingerprint')}, this run is {expect_fingerprint}. The "
+ f"model, optimizer or schedule has changed; resuming would silently "
+ f"train a different thing. Pass --force-config if that is intended.")
+ expected_ngrams = ((expect_ngram,) if isinstance(expect_ngram, str)
+ else tuple(expect_ngram or ()))
+ if expected_ngrams and blob.get("ngram_fingerprint") not in (None, *expected_ngrams):
+ raise SystemExit(
+ f"{path} was trained against ngram_vocab.bin "
+ f"{blob.get('ngram_fingerprint')}, this run loaded {expected_ngrams}. "
+ f"Every row index in the table would mean a different n-gram.")
+
+ if model is not None:
+ raw = getattr(model, "_orig_mod", model)
+ raw.load_state_dict(blob["model"], strict=strict)
+ if muon is not None and blob.get("muon"):
+ muon.load_state_dict(blob["muon"])
+ muon._plan = None # rebuilt against the live Parameters
+ if adamw is not None and blob.get("adamw"):
+ adamw.load_state_dict(_migrate_adamw_groups(blob, raw, adamw))
+ if scaler is not None and blob.get("scaler"):
+ scaler.load_state_dict(blob["scaler"])
+ if packer is not None and blob.get("packer"):
+ packer.load_state_dict(blob["packer"])
+ r = blob.get("rng") or {}
+ if r:
+ random.setstate(r["python"])
+ np.random.set_state(r["numpy"])
+ torch.set_rng_state(r["torch"].cpu() if torch.is_tensor(r["torch"]) else r["torch"])
+ if r.get("cuda") is not None and torch.cuda.is_available():
+ try:
+ torch.cuda.set_rng_state_all([s.cpu() for s in r["cuda"]])
+ except Exception:
+ pass
+ return blob
+
+
+def rotate(directory: Path, pattern: str, keep: int) -> None:
+ d = Path(directory)
+ if not d.exists():
+ return
+ rx = re.compile(pattern)
+ files = sorted((p for p in d.iterdir() if rx.match(p.name)),
+ key=lambda p: p.stat().st_mtime)
+ for p in files[:-keep] if keep > 0 else files:
+ try:
+ p.unlink()
+ except OSError:
+ pass
+
+
+def latest_rolling(directory: Path) -> Path | None:
+ d = Path(directory)
+ if not d.exists():
+ return None
+ files = [p for p in d.iterdir() if re.match(r"rolling_\d+\.pt$", p.name)]
+ if not files:
+ return None
+ return max(files, key=lambda p: p.stat().st_mtime)
diff --git a/config.py b/config.py
new file mode 100644
index 0000000..02ec2c4
--- /dev/null
+++ b/config.py
@@ -0,0 +1,629 @@
+"""Resolved configuration for Boris-2-D112M-n15M.
+
+A 143.5M-parameter decoder-only model trained from scratch on 200B tokens on a
+single V100 32GB (sm_70). Every number that is a judgement call is a field in
+this file, and the whole object is serialised into every checkpoint, so a
+resumed run cannot silently disagree with the run that wrote the checkpoint.
+
+Lineage. The trunk shape (d512 x 31L), Canon layers and the n-gram table come
+from Boris-1.7/1.8, which validated them at 60M dense on a 3060. Boris-2 is not
+1.8 scaled up: the FFN aspect ratio is deliberately different (4.6x rather than
+2.1x, chosen for Volta's GEMM shapes as much as for capacity), Canon is fused
+into one kernel over the concatenated Q|K|V block instead of three convolutions,
+attention is varlen over a jagged nested tensor rather than dense-masked, the
+context length changes partway through the run, and Newton-Schulz is batched.
+
+Hardware. GPU 0 is the V100 and trains. GPU 1 is a 3060 and only ever runs
+eval.py, in a separate process, from weights-only checkpoints. There is no DDP
+here and there is not meant to be.
+"""
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass, field, fields
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent
+DATA_DIR = ROOT / "data" # -> /media/joseph/Media/boris-2-data
+CKPT_DIR = ROOT / "checkpoints" # -> /media/joseph/Media/boris-2-ckpt
+LOG_DIR = ROOT / "logs"
+EVAL_DIR = ROOT / "eval_out"
+RUN_0920_DIR = Path("/media/joseph/Media/Boris-2-0920")
+# The ACTIVE run directory (eval exports, checkpoints, run name). BORIS_RUN_DIR
+# selects a new run without touching 0920, whose n-gram vocab is shared.
+RUN_DIR = Path(__import__("os").environ.get("BORIS_RUN_DIR", "/media/joseph/Media/Boris-2-0929"))
+LEGACY_TOKENIZER_SHA256 = "0eabe7bb2467338230acaf6fc99321668929391a1bd90b7d27138ded627e8b7c"
+
+
+# --------------------------------------------------------------------------- #
+# Model
+# --------------------------------------------------------------------------- #
+@dataclass
+class ModelConfig:
+ architecture: str = "boris-2-canon-ngram"
+
+ # Depth-thin, carried over from 1.8. MobileLLM (2402.14905) finds depth
+ # matters independently of parameter count below 1B, with the optimum near
+ # 30 layers; d_model 512 is the hidden-size threshold replications identify,
+ # so the shape stops there rather than going thinner.
+ # 2026-09-17 rebalance. Measured against GPT-X2.5-135M, which beats Boris-2
+ # by 9.28 Int Index with PIQA + HellaSwag accounting for 79% of the gap.
+ # Those are context-integration tasks and attention was the component this
+ # model was starved of: 17.0M attention params (15.3% of each block) against
+ # their 26.5M (22.9%). GQA 4:1 exists to shrink the KV cache at INFERENCE --
+ # a cost this run never pays, since it trains on one V100 and is not served.
+ # So: full MHA (2 KV heads -> 8), depth 26 -> 31, and the FFN ratio pulled
+ # from 4.63x to 3.25x to fund it. The resulting 31-layer shape measures
+ # about 35.5k tok/s in the live 0917 run.
+ # d_model stays 512 so the 32,768 BPE, the
+ # n-gram table and its (512, 64) projection are all unchanged.
+ # attention 17.0M -> 32.5M (+91%)
+ # total 143,504,896 -> 143,724,032 (+0.15%)
+ d_model: int = 512
+ n_layer: int = 31
+ n_head: int = 8
+ n_kv_head: int = 8 # full MHA; was GQA 4:1
+ vocab_size: int = 32_768 # our own BPE; power of two, uint16 storage
+ # SHA-256 of tokenizer.json. Token ids are part of the model definition:
+ # two 32,768-entry BPEs are not checkpoint-compatible merely because their
+ # tensor shapes match. Keep this in the config fingerprint so a resume
+ # cannot silently attach old embeddings to a new vocabulary.
+ tokenizer_sha256: str = "f366974b96357d11a7e9930826efd81d77438638c0e21ea1517f620da72e05f4"
+ norm_eps: float = 1e-5
+ tie_embeddings: bool = True
+
+ # SwiGLU inner width. 2368 is not a conventional aspect ratio -- it is 4.63x
+ # d_model where 2.7x is normal -- and it is chosen twice over:
+ #
+ # Set explicitly rather than derived from a multiplier, so the fixed
+ # parameter budget cannot drift as the attention allocation changes.
+ ffn_hidden: int = 1664 # 3.25x; was 2368 (4.63x)
+
+ # --- Canon ---------------------------------------------------------- #
+ # Trainable causal depthwise 1-D convolutions with a residual, applied to
+ # the projected Q, K and V before attention (2607.18413, which ablated 17
+ # placements in a Qwen3 block and landed here). Kernel 3, no norm, no
+ # activation, zero-initialised so the layer starts as exactly the identity.
+ #
+ # Depthwise convolution is per-channel, so one convolution over the
+ # concatenated Q|K|V channel block is mathematically identical to three
+ # separate ones. model.py exploits that: 31 kernels per forward, not 93.
+ use_canon: bool = True
+ canon_kernel: int = 3
+
+ # --- n-gram table ---------------------------------------------------- #
+ # Construction-time arm. False builds the trunk with no table and no
+ # projection; the training script is otherwise byte-identical, so the
+ # control is this same file with one flag.
+ #
+ # The transformer is identical either way. False just drops the table and
+ # its projection; nothing else moves.
+ #
+ # use_ngram=True 143,504,896
+ # use_ngram=False 128,472,064 (same trunk, minus the 15M table)
+ use_ngram: bool = True
+
+ # --- Context length ------------------------------------------------- #
+ # Two stages. The model trains at 2048 for the bulk of the run and extends
+ # to 8192 at 80% of the token budget -- the same relative point X2.5 used
+ # (60B of 75B). A stage is (token_fraction, block_size, rope_theta).
+ #
+ # Extension is cheap here in a way it is not for dense-masked attention:
+ # attention runs varlen per document, so its cost is sum(len_i^2) and does
+ # not grow just because the packing window did. What actually costs is the
+ # long documents themselves -- which is the point of the stage, and why
+ # curriculum.long_doc_floor_tokens rises with it.
+ #
+ # A config, not a fixed choice: it is cheap to move once the loss curve is
+ # visible, and moving it does not invalidate any checkpoint.
+ context_stages: tuple = ((0.00, 2048, 10_000.0),
+ (0.80, 8192, 500_000.0))
+
+ @property
+ def head_dim(self) -> int:
+ assert self.d_model % self.n_head == 0
+ return self.d_model // self.n_head
+
+ @property
+ def q_dim(self) -> int:
+ return self.n_head * self.head_dim
+
+ @property
+ def kv_dim(self) -> int:
+ return self.n_kv_head * self.head_dim
+
+ @property
+ def canon_channels(self) -> int:
+ """One fused Canon acts on Q|K|V together."""
+ return self.q_dim + 2 * self.kv_dim
+
+ @property
+ def max_block_size(self) -> int:
+ return max(int(b) for _, b, _ in self.context_stages)
+
+ # -- parameter accounting; the single source of truth for the asserts -- #
+ def attn_params(self) -> int:
+ d = self.d_model
+ return d * self.q_dim + 2 * d * self.kv_dim + self.q_dim * d
+
+ def ffn_params(self) -> int:
+ return 3 * self.d_model * self.ffn_hidden
+
+ def canon_params(self) -> int:
+ return self.n_layer * self.canon_kernel * self.canon_channels if self.use_canon else 0
+
+ def norm_params(self) -> int:
+ return self.n_layer * 2 * self.d_model + self.d_model
+
+ def matmul_params(self) -> int:
+ """Params that drive a matmul -- what Muon operates on, trunk only."""
+ return self.n_layer * (self.attn_params() + self.ffn_params())
+
+ def embed_params(self) -> int:
+ return self.vocab_size * self.d_model * (1 if self.tie_embeddings else 2)
+
+
+# --------------------------------------------------------------------------- #
+# N-gram table
+# --------------------------------------------------------------------------- #
+@dataclass
+class NGramConfig:
+ """A lookup, never a matmul.
+
+ Rows are an explicit top-K vocabulary: a specific (t-1, t) or (t-2, t-1, t)
+ owns a row, or it falls to the single shared OOV row. No hashing, so two
+ unrelated phrases can never silently share a vector.
+
+ Layout is chosen for the gather, not for the file. Rows are written in
+ descending corpus frequency, so the hot head of the table is contiguous:
+ rank 64 in fp32 is 256 bytes per row, and the top ~24k rows -- which cover
+ the large majority of lookups -- fit inside the V100's 6MB L2. A table
+ ordered by n-gram id instead would scatter every batch across 60MB.
+ """
+
+ rank: int = 64 # 256B/row fp32: 4 aligned cache lines
+ bigram_rows: int = 210_938 # held-out optimum: about 90% of rows
+ trigram_rows: int = 23_437 # preserve the strongest specific phrases
+ oov_rows: int = 1
+
+ # Added to the residual stream after this many blocks. Not at the embedding
+ # sum: a retrieved phrase vector is meant to enter a stream that already
+ # carries some context, rather than be re-encoded from scratch by the full
+ # depth of the network. 1.7/1.8 used 2 and this keeps the comparison.
+ insert_after: int = 2
+
+ vocab_path: Path = RUN_0920_DIR / "ngram_vocab.bin"
+
+ target_params: int = 15_000_000
+ tolerance: float = 0.01
+
+ table_init_std: float = 0.02
+ proj_init_std: float = 0.02
+ # The projection is a 2-D linear map -> Muon. The table never is.
+ muon_on_proj: bool = True
+
+ @property
+ def n_rows(self) -> int:
+ return self.oov_rows + self.bigram_rows + self.trigram_rows
+
+ @property
+ def oov_row(self) -> int:
+ return 0
+
+ @property
+ def bigram_base(self) -> int:
+ return self.oov_rows
+
+ @property
+ def trigram_base(self) -> int:
+ return self.oov_rows + self.bigram_rows
+
+ def table_params(self, d_model: int) -> int:
+ return self.n_rows * self.rank + self.rank * d_model
+
+
+# --------------------------------------------------------------------------- #
+# Data
+# --------------------------------------------------------------------------- #
+@dataclass
+class DataConfig:
+ root: Path = DATA_DIR
+ # 2026-09-17, Boris-2-0917. The seven hand-weighted sources are gone.
+ # ClimbMix (2504.13161) is a 400B-token corpus whose mixture was SEARCHED
+ # rather than guessed: 1,000 topic clusters, pruned by ad/educational/
+ # informational/quality classifiers at a loose threshold, then mixed by
+ # iterative bootstrapping against the PIQA, ARC-E and HellaSwag validation
+ # sets. Those three are 79% of the Int Index gap Boris-2-0907 showed against
+ # GPT-X2.5-135M. At equal token budget it beats Nemotron-CC, DCLM, SmolLM-
+ # corpus and FineWeb-Edu, and its 350M model beats SmolLM-360M on exactly
+ # the two benchmarks 0907 was weakest on (PIQA +1.03, HellaSwag +2.75).
+ # We take 200B of the 369B (under our BPE) -- sound only because the
+ # karpathy/climbmix-400b-shuffle mirror is pre-shuffled, so a prefix is a
+ # representative sample. Written as climbmix + climbmix_s1..s15 and
+ # stitched by MultiStore.
+ # finemath is the one source kept from 0907: ArithMark-3 was the single
+ # benchmark where 0907 was competitive (37.40 vs 38.40), and ClimbMix was
+ # not optimised for arithmetic.
+ sources: tuple = ("climbmix", "finemath")
+ val_path: Path = DATA_DIR / "val.bin"
+ val_source: str = "climbmix_val"
+ # Documents longer than this are split across rows rather than truncated,
+ # so nothing in the corpus is unreachable.
+ max_doc_tokens: int = 65_536
+ # Rows are packed, not EOS-concatenated: attention and Canon both isolate
+ # documents, so a row may hold many and none of them leak into another.
+ eos_id: int = 0
+ prefetch_depth: int = 4
+
+
+@dataclass
+class CurriculumConfig:
+ """Token-threshold mixture weights, interpolated between phases.
+
+ Never a static ratio: the decay phase is supposed to consolidate onto the
+ densest data, and that only happens if the mixture is a function of
+ progress. Weights are normalised at use, so they need not sum to 1 here.
+ """
+
+ enabled: bool = True
+ smooth: bool = True # interpolate between phase endpoints
+
+ # (name, progress_at_end, weights)
+ # Decay consolidates onto the densest data and drops general web text
+ # entirely: Ultra-FineWeb's L2 weight moves to its L3 refinement (Q&A pairs
+ # and multi-style rewrites of the same corpus), which is what openbmb built
+ # L3 for. Weights ramp smoothly, so L3 averages ~0.125 across the decay
+ # phase rather than 0.25.
+ # 2026-09-17: a first phase was added. The run previously fed 30% synthetic
+ # and structured text (cosmopedia + finephrase) from step 0. GPT-X2.5 runs
+ # 98% natural web (58% FineWeb-Edu + 40% DCLM) and introduces Cosmopedia
+ # only at 40B of 75B. Boris-2's two weakest benchmarks -- PIQA and HellaSwag,
+ # 79% of the Int Index gap -- are exactly what broad natural text teaches,
+ # and Boris-2 scores WORSE on held-out fw-edu (3.2017) than on its own val
+ # (3.0420), i.e. it is under-exposed. Phase 0 is held constant (mixture_at
+ # does not interpolate into the first phase), then ramps to the original
+ # stable weights by 0.87. Decay is unchanged.
+ # FLAT. ClimbMix's weights are the output of a search over the whole
+ # mixture; re-weighting it by phase would undo the thing we adopted it for.
+ # The WSD decay phase still decays the LEARNING RATE -- it just no longer
+ # swaps the data underneath it. One phase ending at 1.00 means mixture_at()
+ # returns these weights constantly (it does not interpolate into phase 0).
+ phases: tuple = (
+ ("flat", 1.00, {"climbmix": 0.95, "finemath": 0.05}),
+ )
+
+ # When the context stage lengthens, short documents stop teaching long
+ # context. From that point the packer prefers documents at least this long
+ # where a source has them, falling back when it does not.
+ long_doc_floor_tokens: int = 4096
+ long_doc_target_frac: float = 0.5 # of tokens drawn, once extended
+
+
+# --------------------------------------------------------------------------- #
+# Optimisation
+# --------------------------------------------------------------------------- #
+@dataclass
+class OptimConfig:
+ """Muon on 2-D matmul weights, AdamW on everything else.
+
+ The split is by role, assigned at construction, never by sniffing ndim.
+ Canon kernels are (C, 3) tensors -- two-dimensional, and emphatically not
+ matrices to orthogonalise. A shape rule would hand all 31 of them to Muon
+ and nothing would ever complain.
+ """
+
+ muon_lr: float = 0.02
+ muon_momentum: float = 0.95
+ muon_nesterov: bool = True
+ muon_ns_steps: int = 5
+ muon_weight_decay: float = 0.1
+ # Newton-Schulz over same-shaped parameters stacked into one bmm. At 31
+ # layers this collapses ~2000 small launches per step into ~60. Run in
+ # fp32: 1.8 measured that a bf16 iteration diverges 5.4e-02 relative from
+ # the fp32 path because five steps amplify rounding well past bf16 epsilon,
+ # and on sm_70 the fp16 alternative is worse. The matrices are small enough
+ # that fp32 costs little -- preflight.py reports the measured share.
+ muon_batched: bool = True
+ muon_ns_dtype: str = "float32"
+ # Scale the orthogonalised update by 0.2*sqrt(max(m,n)) so one LR behaves
+ # across aspect ratios as different as 512x512 and 2368x512. With a 4.63x
+ # FFN this matters more than it does at a conventional width.
+ muon_rms_scale: bool = True
+
+ adamw_lr: float = 3e-3 # embeddings, table, head, norms, canon
+ adamw_betas: tuple = (0.9, 0.95)
+ adamw_eps: float = 1e-8
+ # Embedding (16.8M) and the n-gram table (~15M) are 22% of the model and had
+ # NO restraint: both measured +21.1% in norm over 20.8B->30.1B while every
+ # Muon-decayed matmul stayed within 1%. See optim.build_optimizers.
+ adamw_weight_decay: float = 0.1 # embedding + ngram_table only
+ adamw_norm_weight_decay: float = 0.0 # norm gains: conventional, keep 0
+ # 2026-10-07: canon (31 x (768, 3), weight 0 = identity) grew ~+20% per 10B
+ # in 0929 (block 11: 16.3 -> 20.2 -> 24.9 over 20/30/40B, max |w| 13.5) while
+ # the norm1 gains in front of it shrank ~12% per 10B to compensate. Decay it
+ # toward identity like the other AdamW tensors. Own group: see optim.py.
+ adamw_canon_weight_decay: float = 0.1
+
+ # 2026-09-17: the fused LM head walks the vocab in chunks and its peak is
+ # one (chunk, 32768) fp32 logits tensor -- 268 MB at 2048. Boris-2-0917 OOMs
+ # at 31.28/31.73 GB with that, because preflight measures peak_allocated in
+ # a clean process and cannot see allocator fragmentation (1.0 GB reserved
+ # unallocated here) or the 280 MB another process holds on the same card.
+ # 1024 halves the head's footprint for a few percent of throughput.
+ lm_head_chunk: int = 1024
+ grad_clip: float = 4.0 # was 1.0; median gn fits 0.0654*D_B^0.646,
+ # crossing 1.0 at ~68B and reaching ~2.0 by 200B
+
+
+@dataclass
+class ScheduleConfig:
+ """WSD. Warmup 1%, stable to 87%, 1-sqrt decay over the final 13%."""
+
+ warmup_frac: float = 0.01 # 2B tokens
+ stable_frac: float = 0.87
+ decay_shape: str = "1-sqrt" # 1-sqrt | linear | cosine
+ final_lr_frac: float = 0.0
+
+
+# --------------------------------------------------------------------------- #
+# Run
+# --------------------------------------------------------------------------- #
+@dataclass
+class TrainConfig:
+ total_tokens: int = 200_000_000_000
+ tokens_per_step: int = 524_288 # 0.5M; step count 381,470
+
+ # Micro-batch per context stage, in ROWS. Sweep with `preflight --sweep`
+ # and set from measurement; early measurements expire, so re-sweep after
+ # any change to the model or the kernels.
+ # Measured on the V100, 2026-09-06, with `preflight.py --sweep`, including
+ # the worst case of one full-block document per row:
+ #
+ # ctx 2048 mb 8 27.9k tok/s 13.0 GB | mb 16 37.7k tok/s 23.9 GB
+ # ctx 8192 mb 2 26.7k tok/s 13.1 GB | mb 4 32.6k tok/s 24.0 GB
+ #
+ # Both stages hold 32,768 tokens per micro-batch, so activation memory is
+ # roughly unchanged when the context lengthens and the accumulation count
+ # stays at 16 -- the step remains exactly tokens_per_step either way. Peak
+ # is 23.8 GB of 31.7, and the worst case measures the same because the
+ # memory-efficient attention kernel is O(N), not O(N^2).
+ #
+ # Re-sweep after any change to the model or the kernels: the optimum moves
+ # as bottlenecks fall.
+ micro_batch: dict = field(default_factory=lambda: {2048: 16, 8192: 4})
+
+ device: str = "cuda:0"
+ eval_device: str = "cuda:1" # eval.py only; training never touches it
+ seed: int = 20260906
+
+ # fp16 everywhere. sm_70 tensor cores are fp16-in/fp32-accumulate and have
+ # no bf16 path at all; TF32 does not exist before sm_80. Both are asserted
+ # at startup rather than assumed.
+ amp_dtype: str = "float16"
+ init_scale: float = 2.0 ** 16
+ # GradScaler has no floor of its own, and that is fatal here. Below roughly
+ # 1e-30 `unscale_` computes grad/scale and overflows fp32 to inf, which sets
+ # found_inf, which halves the scale again: the collapse becomes
+ # self-reinforcing and has no exit. That is exactly how the run that reached
+ # step 3,810 died -- the scale bled 131,072 -> 0 over 170 steps while the
+ # weights stayed perfectly finite, and every step after that was skipped.
+ # Floored at 1.0 the same event costs a few skipped steps instead of the run.
+ min_loss_scale: float = 1.0
+ # Ceiling, for the other end. The scale grew to 131,072 at step ~2,000 and
+ # the first overflow appeared at 3,460; halving the ceiling buys a factor of
+ # two of headroom in the backward and costs nothing but the smallest
+ # gradients, which are already far below where they matter.
+ max_loss_scale: float = 2.0 ** 15
+ # Steps pinned at min_loss_scale before the run stops itself. At the floor
+ # nothing is training -- every step is being skipped -- so this is the real
+ # early warning, ~350 steps ahead of the loss going non-finite.
+ abort_after_floored: int = 100
+
+ compile_regions: bool = True # fixed-shape regions only, never the model
+ # FALSE, and this is not a tuning knob -- an fp16 residual stream cannot
+ # survive 31 layers of this model. The stream grows with depth, and at step
+ # 3,456 (1% of the budget) it measured:
+ # block 0 3,062 block 16 40,640
+ # block 9 16,144 block 24 39,264
+ # block 25 65,617 vs fp16's max of 65,504 <-- over by 113
+ # It saturates to inf, and _rmsnorm then computes rsqrt(inf) = 0 and
+ # inf * 0 = NaN. That NaN is what drove GradScaler to halve its scale
+ # forever (halving a loss scale cannot fix a FORWARD overflow), which is how
+ # two runs died. Recomputed in fp32 the same batch reaches 65,617 and stays
+ # finite. Measured cost: 39.77k -> 36.73k tok/s and 24.0 -> 25.1 GB, about
+ # +4.3 days on the 200B budget. Boris-1.7 never hit this because bf16 on the
+ # 3060 has fp32's exponent range; sm_70 has no bf16 at all.
+ fp16_residual: bool = False
+ # Recompute block activations in the backward instead of storing them, to
+ # buy a larger micro-batch. Costs one extra forward per checkpointed block
+ # (~+11% compute) and buys fatter GEMMs. Whether that trade is positive is
+ # a measurement, not an argument -- preflight.py --sweep with it on/off.
+ # 2026-09-17: ON for Boris-2-0917. Without it the run OOMs -- 30.17 GB
+ # allocated + 0.78 GB fragmentation + 0.28 GB held by another process on
+ # the same card = 31.28 of 31.73. preflight reads ~1 GB LOW because it
+ # measures peak_allocated in a clean process and cannot see either.
+ # every=4 returns 6.5 GB (29.2 -> 22.7 worst-case) for 5.3% throughput.
+ grad_checkpoint: bool = True
+ grad_checkpoint_every: int = 4 # 1 = every block, 4 = every 4th
+
+ # Resume granularity. The 50B checkpoints are archival milestones; they are
+ # not the thing you resume from. A run that will be interrupted needs a
+ # rolling checkpoint on a wall-clock timer.
+ rolling_ckpt_minutes: int = 45
+ rolling_ckpt_keep: int = 2
+ full_ckpt_every_tokens: int = 50_000_000_000
+ export_every_tokens: int = 10_000_000_000 # weights-only, for eval.py
+ log_every_steps: int = 10
+ # Consecutive steps of non-finite loss OR non-finite grad norm before the
+ # run stops itself. One or two are normal (GradScaler probing its scale);
+ # 200 means something is broken and every further step is wasted.
+ abort_after_bad: int = 200
+ val_every_tokens: int = 1_000_000_000
+
+
+@dataclass
+class EvalConfig:
+ device: str = "cuda:1"
+ out_path: Path = RUN_DIR / "eval" / "results.jsonl"
+ export_dir: Path = RUN_DIR / "checkpoints" / "exports"
+ manifest: Path = RUN_DIR / "checkpoints" / "exports" / "manifest.jsonl"
+ tasks: tuple = ("hellaswag", "arc_easy", "arc_challenge", "piqa", "arithmark3")
+ val_tokens: int = 10_000_000
+ batch_size: int = 16
+ poll_seconds: int = 120
+
+
+@dataclass
+class Config:
+ model: ModelConfig = field(default_factory=ModelConfig)
+ ngram: NGramConfig = field(default_factory=NGramConfig)
+ data: DataConfig = field(default_factory=DataConfig)
+ curriculum: CurriculumConfig = field(default_factory=CurriculumConfig)
+ optim: OptimConfig = field(default_factory=OptimConfig)
+ schedule: ScheduleConfig = field(default_factory=ScheduleConfig)
+ train: TrainConfig = field(default_factory=TrainConfig)
+ eval: EvalConfig = field(default_factory=EvalConfig)
+
+ run_name: str = RUN_DIR.name
+ checkpoint_dir: Path = RUN_DIR / "checkpoints"
+ # verify.py builds 4- and 6-layer models to keep the isolation probes fast;
+ # they are not the run, so the band assert does not apply to them.
+ enforce_budget: bool = True
+
+ # -- derived ------------------------------------------------------------ #
+ @property
+ def total_steps(self) -> int:
+ return self.train.total_tokens // self.train.tokens_per_step
+
+ def block_size_at(self, progress: float) -> tuple[int, float]:
+ """(block_size, rope_theta) for this point in the run."""
+ bs, theta = self.model.context_stages[0][1], self.model.context_stages[0][2]
+ for frac, b, th in self.model.context_stages:
+ if progress >= frac:
+ bs, theta = int(b), float(th)
+ return bs, theta
+
+ def micro_batch_at(self, progress: float) -> int:
+ bs, _ = self.block_size_at(progress)
+ return int(self.train.micro_batch[bs])
+
+ def accum_at(self, progress: float) -> int:
+ bs = self.block_size_at(progress)[0]
+ mb = self.micro_batch_at(progress)
+ tokens_per_micro = mb * bs
+ assert self.train.tokens_per_step % tokens_per_micro == 0, (
+ f"tokens_per_step {self.train.tokens_per_step} is not divisible by "
+ f"micro_batch {mb} x block_size {bs} = {tokens_per_micro}; the step "
+ f"size would change when the context stage does")
+ return self.train.tokens_per_step // tokens_per_micro
+
+ # -- parameter budget ---------------------------------------------------- #
+ def param_budget(self) -> dict:
+ m, n = self.model, self.ngram
+ on = m.use_ngram
+ d = {
+ "attention": m.n_layer * m.attn_params(),
+ "ffn": m.n_layer * m.ffn_params(),
+ "canon": m.canon_params(),
+ "norms": m.norm_params(),
+ "embedding": m.embed_params(),
+ "ngram_table": n.n_rows * n.rank if on else 0,
+ "ngram_proj": n.rank * m.d_model if on else 0,
+ }
+ d["trunk"] = d["attention"] + d["ffn"] + d["canon"] + d["norms"]
+ d["matmul"] = d["attention"] + d["ffn"] + d["ngram_proj"]
+ d["total"] = d["trunk"] + d["embedding"] + d["ngram_table"] + d["ngram_proj"]
+ return d
+
+ def assert_budget(self, ceiling: int = 150_000_000,
+ band: tuple[int, int] = (140_000_000, 145_000_000)) -> None:
+ b = self.param_budget()
+ if not self.enforce_budget:
+ return
+ assert b["total"] <= ceiling, (
+ f"total parameters {b['total']:,} exceeds the hard ceiling {ceiling:,}")
+ # The band describes the full model. With the table off the trunk is
+ # unchanged and the total is simply 15M lower, which is the point.
+ if not self.model.use_ngram:
+ return
+ assert band[0] <= b["total"] <= band[1], (
+ f"total parameters {b['total']:,} outside the target band "
+ f"{band[0]:,}-{band[1]:,}; change model.ffn_hidden")
+ tp = self.ngram.table_params(self.model.d_model)
+ tgt, tol = self.ngram.target_params, self.ngram.tolerance
+ assert abs(tp - tgt) <= tol * tgt, (
+ f"n-gram table {tp:,} is more than {tol:.0%} from its {tgt:,} target")
+
+ # -- serialisation ------------------------------------------------------- #
+ def to_dict(self) -> dict:
+ def conv(o):
+ if isinstance(o, Path):
+ return str(o)
+ if isinstance(o, dict):
+ return {str(k): conv(v) for k, v in o.items()}
+ if isinstance(o, (list, tuple)):
+ return [conv(x) for x in o]
+ return o
+ return {f.name: conv(asdict(getattr(self, f.name))
+ if hasattr(getattr(self, f.name), "__dataclass_fields__")
+ else getattr(self, f.name))
+ for f in fields(self)}
+
+ def to_json(self) -> str:
+ return json.dumps(self.to_dict(), indent=2, sort_keys=True)
+
+ def fingerprint(self) -> str:
+ """Hash of everything that must match to resume a checkpoint."""
+ import hashlib
+ keep = {k: self.to_dict()[k] for k in ("model", "ngram", "optim",
+ "schedule", "train")}
+ return hashlib.sha256(json.dumps(keep, sort_keys=True).encode()).hexdigest()[:16]
+
+ @staticmethod
+ def from_json(s: str) -> "Config":
+ raw = json.loads(s)
+ cfg = Config()
+ # Checkpoints predating the tokenizer guard used the original 09-06
+ # BPE. Give those serialized configs their real identity rather than
+ # inheriting the 0920 default because the field is absent.
+ if "model" in raw and "tokenizer_sha256" not in raw["model"]:
+ cfg.model.tokenizer_sha256 = LEGACY_TOKENIZER_SHA256
+ if (raw.get("model", {}).get("tokenizer_sha256", LEGACY_TOKENIZER_SHA256)
+ == LEGACY_TOKENIZER_SHA256):
+ cfg.data.val_source = "val"
+ for name, sub in raw.items():
+ if not hasattr(cfg, name):
+ continue
+ cur = getattr(cfg, name)
+ if hasattr(cur, "__dataclass_fields__") and isinstance(sub, dict):
+ for k, v in sub.items():
+ if not hasattr(cur, k):
+ continue
+ old = getattr(cur, k)
+ if isinstance(old, Path):
+ v = Path(v)
+ elif isinstance(old, tuple) and isinstance(v, list):
+ v = tuple(tuple(x) if isinstance(x, list) else x for x in v)
+ elif isinstance(old, dict) and isinstance(v, dict):
+ v = {int(kk) if str(kk).isdigit() else kk: vv for kk, vv in v.items()}
+ setattr(cur, k, v)
+ else:
+ if isinstance(cur, Path):
+ sub = Path(sub)
+ setattr(cfg, name, sub)
+ return cfg
+
+
+if __name__ == "__main__":
+ cfg = Config()
+ b = cfg.param_budget()
+ w = max(len(k) for k in b)
+ for k in ("attention", "ffn", "canon", "norms", "trunk", "embedding",
+ "ngram_table", "ngram_proj", "matmul", "total"):
+ print(f" {k:<{w}} {b[k]:>13,}")
+ cfg.assert_budget()
+ print(f"\n steps {cfg.total_steps:,}")
+ for frac, bs, th in cfg.model.context_stages:
+ print(f" ctx from {frac:>5.0%} block {bs:>5} theta {th:>9,.0f} "
+ f"micro_batch {cfg.train.micro_batch[bs]} accum {cfg.accum_at(frac)}")
+ print(f" fingerprint {cfg.fingerprint()}")
diff --git a/configs/boris-2-0929/RECIPE.md b/configs/boris-2-0929/RECIPE.md
new file mode 100644
index 0000000..087ec5f
--- /dev/null
+++ b/configs/boris-2-0929/RECIPE.md
@@ -0,0 +1,9 @@
+# Boris-2 release candidate (2026-11-11)
+Official-style Open SLM Index 34.19 (HellaSwag 50.26, ARC-E 59.81, ARC-C 32.00, PIQA 72.91, ArithMark-3 official 45.10).
+Lineage:
+1. Boris-2-0929 final.pt (200B tokens, WSD 1-sqrt decay, doubled batch from 170B).
+2. "Nudge": 30M tokens, mix climbmix .55 / cosmo2_how .30 / cosmo_how_v1_tf .05 / ultra_math_l3 .07 / metamath_gsm .02 /
+ orca_math .01 (per-doc draw weights), LR 0.1x peak, 1-sqrt to 0, seed 1234 (probe/pilot_train.py --peak 0.1).
+3. 10M tokens of LibreMath v1 only (decontaminated: 13-gram + 8-gram ArithMark/SVAMP/ASDiv bans), LR 0.02x peak,
+ 1-sqrt to 0, fresh optimizer state (probe/pilot_train.py --peak 0.02).
+Selected from ~10 short final-stage candidates by benchmark score -- disclose in the model card.
diff --git a/configs/boris-2-0929/config.before_2xbatch.json b/configs/boris-2-0929/config.before_2xbatch.json
new file mode 100644
index 0000000..f138731
--- /dev/null
+++ b/configs/boris-2-0929/config.before_2xbatch.json
@@ -0,0 +1,220 @@
+{
+ "curriculum": {
+ "enabled": true,
+ "long_doc_floor_tokens": 4096,
+ "long_doc_target_frac": 0.5,
+ "phases": [
+ [
+ "flat",
+ 0.75,
+ {
+ "climbmix": 0.932109,
+ "ultra_math_l3": 0.064783,
+ "orca_math": 0.003108,
+ "cosmo2_how": 0.0,
+ "cosmo_how_v1_tf": 0.0,
+ "metamath_gsm": 0.0,
+ "elem_wordprob": 0.0,
+ "dclm_r0920": 0.0
+ }
+ ],
+ [
+ "pre_decay_ramp",
+ 0.87,
+ {
+ "climbmix": 0.821333,
+ "ultra_math_l3": 0.081144,
+ "orca_math": 0.003244,
+ "cosmo2_how": 0.085976,
+ "cosmo_how_v1_tf": 0.008302,
+ "metamath_gsm": 0.0,
+ "elem_wordprob": 0.0,
+ "dclm_r0920": 0.0
+ }
+ ],
+ [
+ "decay_in",
+ 0.875,
+ {
+ "climbmix": 0.775534,
+ "ultra_math_l3": 0.07862,
+ "orca_math": 0.003143,
+ "cosmo2_how": 0.083301,
+ "cosmo_how_v1_tf": 0.008044,
+ "metamath_gsm": 0.025126,
+ "elem_wordprob": 0.020974,
+ "dclm_r0920": 0.005257
+ }
+ ],
+ [
+ "decay",
+ 0.9825,
+ {
+ "climbmix": 0.775534,
+ "ultra_math_l3": 0.07862,
+ "orca_math": 0.003143,
+ "cosmo2_how": 0.083301,
+ "cosmo_how_v1_tf": 0.008044,
+ "metamath_gsm": 0.025126,
+ "elem_wordprob": 0.020974,
+ "dclm_r0920": 0.005257
+ }
+ ],
+ [
+ "p2_tail_in",
+ 0.9875,
+ {
+ "climbmix": 0.580561,
+ "ultra_math_l3": 0.053042,
+ "metamath_gsm": 0.135615,
+ "elem_wordprob": 0.141507,
+ "dclm_r0920": 0.012769,
+ "orca_math": 0.002545,
+ "cosmo2_how": 0.067493,
+ "cosmo_how_v1_tf": 0.006468
+ }
+ ],
+ [
+ "p2_tail",
+ 1.0,
+ {
+ "climbmix": 0.580561,
+ "ultra_math_l3": 0.053042,
+ "metamath_gsm": 0.135615,
+ "elem_wordprob": 0.141507,
+ "dclm_r0920": 0.012769,
+ "orca_math": 0.002545,
+ "cosmo2_how": 0.067493,
+ "cosmo_how_v1_tf": 0.006468
+ }
+ ]
+ ],
+ "smooth": true
+ },
+ "checkpoint_dir": "/media/joseph/Media/Boris-2-0929/checkpoints",
+ "data": {
+ "eos_id": 0,
+ "max_doc_tokens": 65536,
+ "prefetch_depth": 4,
+ "root": "/home/joseph/boris-2-D112M-n15M/data",
+ "sources": [
+ "climbmix",
+ "ultra_math_l3",
+ "orca_math",
+ "cosmo2_how",
+ "cosmo_how_v1_tf",
+ "metamath_gsm",
+ "elem_wordprob",
+ "dclm_r0920"
+ ],
+ "val_path": "/home/joseph/boris-2-D112M-n15M/data/val.bin",
+ "val_source": "climbmix_val"
+ },
+ "enforce_budget": true,
+ "eval": {
+ "batch_size": 128,
+ "device": "cuda:1",
+ "export_dir": "/media/joseph/Media/Boris-2-0929/checkpoints/exports",
+ "manifest": "/media/joseph/Media/Boris-2-0929/checkpoints/exports/manifest.jsonl",
+ "out_path": "/media/joseph/Media/Boris-2-0929/eval/results.jsonl",
+ "poll_seconds": 120,
+ "tasks": [
+ "hellaswag",
+ "arc_easy",
+ "arc_challenge",
+ "piqa",
+ "arithmark3"
+ ],
+ "val_tokens": 10000000
+ },
+ "model": {
+ "architecture": "boris-2-canon-ngram",
+ "canon_kernel": 3,
+ "context_stages": [
+ [
+ 0.0,
+ 2048,
+ 10000.0
+ ]
+ ],
+ "d_model": 512,
+ "ffn_hidden": 1664,
+ "n_head": 8,
+ "n_kv_head": 8,
+ "n_layer": 31,
+ "norm_eps": 1e-05,
+ "tie_embeddings": true,
+ "tokenizer_sha256": "f366974b96357d11a7e9930826efd81d77438638c0e21ea1517f620da72e05f4",
+ "use_canon": true,
+ "use_ngram": true,
+ "vocab_size": 32768
+ },
+ "ngram": {
+ "bigram_rows": 210938,
+ "insert_after": 2,
+ "muon_on_proj": true,
+ "oov_rows": 1,
+ "proj_init_std": 0.02,
+ "rank": 64,
+ "table_init_std": 0.02,
+ "target_params": 15000000,
+ "tolerance": 0.01,
+ "trigram_rows": 23437,
+ "vocab_path": "/media/joseph/Media/Boris-2-0920/ngram_vocab.bin"
+ },
+ "optim": {
+ "adamw_betas": [
+ 0.9,
+ 0.95
+ ],
+ "adamw_eps": 1e-08,
+ "adamw_lr": 0.003,
+ "adamw_norm_weight_decay": 0.0,
+ "adamw_weight_decay": 0.1,
+ "grad_clip": 4.0,
+ "lm_head_chunk": 8192,
+ "muon_batched": true,
+ "muon_lr": 0.0025,
+ "muon_momentum": 0.95,
+ "muon_nesterov": true,
+ "muon_ns_dtype": "float32",
+ "muon_ns_steps": 5,
+ "muon_rms_scale": true,
+ "muon_weight_decay": 0.1,
+ "adamw_canon_weight_decay": 0.1
+ },
+ "run_name": "Boris-2-0929",
+ "schedule": {
+ "decay_shape": "1-sqrt",
+ "final_lr_frac": 0.0,
+ "stable_frac": 0.87,
+ "warmup_frac": 0.01
+ },
+ "train": {
+ "abort_after_bad": 200,
+ "abort_after_floored": 100,
+ "amp_dtype": "float16",
+ "compile_regions": true,
+ "device": "cuda:0",
+ "eval_device": "cuda:1",
+ "export_every_tokens": 10000000000,
+ "fp16_residual": false,
+ "full_ckpt_every_tokens": 50000000000,
+ "grad_checkpoint": false,
+ "grad_checkpoint_every": 8,
+ "init_scale": 65536.0,
+ "log_every_steps": 10,
+ "max_loss_scale": 32768.0,
+ "micro_batch": {
+ "2048": 16,
+ "8192": 4
+ },
+ "min_loss_scale": 1.0,
+ "rolling_ckpt_keep": 2,
+ "rolling_ckpt_minutes": 45,
+ "seed": 20260906,
+ "tokens_per_step": 524288,
+ "total_tokens": 200000000000,
+ "val_every_tokens": 1000000000
+ }
+} \ No newline at end of file
diff --git a/configs/boris-2-0929/config.before_decaymix.json b/configs/boris-2-0929/config.before_decaymix.json
new file mode 100644
index 0000000..e74cbc0
--- /dev/null
+++ b/configs/boris-2-0929/config.before_decaymix.json
@@ -0,0 +1,145 @@
+{
+ "curriculum": {
+ "enabled": true,
+ "long_doc_floor_tokens": 4096,
+ "long_doc_target_frac": 0.5,
+ "phases": [
+ [
+ "flat",
+ 1.0,
+ {
+ "climbmix": 0.932109,
+ "ultra_math_l3": 0.064783,
+ "orca_math": 0.003108
+ }
+ ]
+ ],
+ "smooth": true
+ },
+ "checkpoint_dir": "/media/joseph/Media/Boris-2-0929/checkpoints",
+ "data": {
+ "eos_id": 0,
+ "max_doc_tokens": 65536,
+ "prefetch_depth": 4,
+ "root": "/home/joseph/boris-2-D112M-n15M/data",
+ "sources": [
+ "climbmix",
+ "ultra_math_l3",
+ "orca_math"
+ ],
+ "val_path": "/home/joseph/boris-2-D112M-n15M/data/val.bin",
+ "val_source": "climbmix_val"
+ },
+ "enforce_budget": true,
+ "eval": {
+ "batch_size": 128,
+ "device": "cuda:1",
+ "export_dir": "/media/joseph/Media/Boris-2-0929/checkpoints/exports",
+ "manifest": "/media/joseph/Media/Boris-2-0929/checkpoints/exports/manifest.jsonl",
+ "out_path": "/media/joseph/Media/Boris-2-0929/eval/results.jsonl",
+ "poll_seconds": 120,
+ "tasks": [
+ "hellaswag",
+ "arc_easy",
+ "arc_challenge",
+ "piqa",
+ "arithmark3"
+ ],
+ "val_tokens": 10000000
+ },
+ "model": {
+ "architecture": "boris-2-canon-ngram",
+ "canon_kernel": 3,
+ "context_stages": [
+ [
+ 0.0,
+ 2048,
+ 10000.0
+ ],
+ [
+ 0.8,
+ 8192,
+ 500000.0
+ ]
+ ],
+ "d_model": 512,
+ "ffn_hidden": 1664,
+ "n_head": 8,
+ "n_kv_head": 8,
+ "n_layer": 31,
+ "norm_eps": 1e-05,
+ "tie_embeddings": true,
+ "tokenizer_sha256": "f366974b96357d11a7e9930826efd81d77438638c0e21ea1517f620da72e05f4",
+ "use_canon": true,
+ "use_ngram": true,
+ "vocab_size": 32768
+ },
+ "ngram": {
+ "bigram_rows": 210938,
+ "insert_after": 2,
+ "muon_on_proj": true,
+ "oov_rows": 1,
+ "proj_init_std": 0.02,
+ "rank": 64,
+ "table_init_std": 0.02,
+ "target_params": 15000000,
+ "tolerance": 0.01,
+ "trigram_rows": 23437,
+ "vocab_path": "/media/joseph/Media/Boris-2-0920/ngram_vocab.bin"
+ },
+ "optim": {
+ "adamw_betas": [
+ 0.9,
+ 0.95
+ ],
+ "adamw_eps": 1e-08,
+ "adamw_lr": 0.003,
+ "adamw_norm_weight_decay": 0.0,
+ "adamw_weight_decay": 0.1,
+ "grad_clip": 4.0,
+ "lm_head_chunk": 8192,
+ "muon_batched": true,
+ "muon_lr": 0.0025,
+ "muon_momentum": 0.95,
+ "muon_nesterov": true,
+ "muon_ns_dtype": "float32",
+ "muon_ns_steps": 5,
+ "muon_rms_scale": true,
+ "muon_weight_decay": 0.1,
+ "adamw_canon_weight_decay": 0.1
+ },
+ "run_name": "Boris-2-0929",
+ "schedule": {
+ "decay_shape": "1-sqrt",
+ "final_lr_frac": 0.0,
+ "stable_frac": 0.87,
+ "warmup_frac": 0.01
+ },
+ "train": {
+ "abort_after_bad": 200,
+ "abort_after_floored": 100,
+ "amp_dtype": "float16",
+ "compile_regions": true,
+ "device": "cuda:0",
+ "eval_device": "cuda:1",
+ "export_every_tokens": 10000000000,
+ "fp16_residual": false,
+ "full_ckpt_every_tokens": 50000000000,
+ "grad_checkpoint": false,
+ "grad_checkpoint_every": 8,
+ "init_scale": 65536.0,
+ "log_every_steps": 10,
+ "max_loss_scale": 32768.0,
+ "micro_batch": {
+ "2048": 16,
+ "8192": 4
+ },
+ "min_loss_scale": 1.0,
+ "rolling_ckpt_keep": 2,
+ "rolling_ckpt_minutes": 45,
+ "seed": 20260906,
+ "tokens_per_step": 524288,
+ "total_tokens": 200000000000,
+ "val_every_tokens": 1000000000
+ }
+} \ No newline at end of file
diff --git a/configs/boris-2-0929/config.before_no8192.json b/configs/boris-2-0929/config.before_no8192.json
new file mode 100644
index 0000000..37a77ea
--- /dev/null
+++ b/configs/boris-2-0929/config.before_no8192.json
@@ -0,0 +1,225 @@
+{
+ "curriculum": {
+ "enabled": true,
+ "long_doc_floor_tokens": 4096,
+ "long_doc_target_frac": 0.5,
+ "phases": [
+ [
+ "flat",
+ 0.75,
+ {
+ "climbmix": 0.932109,
+ "ultra_math_l3": 0.064783,
+ "orca_math": 0.003108,
+ "cosmo2_how": 0.0,
+ "cosmo_how_v1_tf": 0.0,
+ "metamath_gsm": 0.0,
+ "elem_wordprob": 0.0,
+ "dclm_r0920": 0.0
+ }
+ ],
+ [
+ "pre_decay_ramp",
+ 0.87,
+ {
+ "climbmix": 0.821333,
+ "ultra_math_l3": 0.081144,
+ "orca_math": 0.003244,
+ "cosmo2_how": 0.085976,
+ "cosmo_how_v1_tf": 0.008302,
+ "metamath_gsm": 0.0,
+ "elem_wordprob": 0.0,
+ "dclm_r0920": 0.0
+ }
+ ],
+ [
+ "decay_in",
+ 0.875,
+ {
+ "climbmix": 0.775534,
+ "ultra_math_l3": 0.07862,
+ "orca_math": 0.003143,
+ "cosmo2_how": 0.083301,
+ "cosmo_how_v1_tf": 0.008044,
+ "metamath_gsm": 0.025126,
+ "elem_wordprob": 0.020974,
+ "dclm_r0920": 0.005257
+ }
+ ],
+ [
+ "decay",
+ 0.9825,
+ {
+ "climbmix": 0.775534,
+ "ultra_math_l3": 0.07862,
+ "orca_math": 0.003143,
+ "cosmo2_how": 0.083301,
+ "cosmo_how_v1_tf": 0.008044,
+ "metamath_gsm": 0.025126,
+ "elem_wordprob": 0.020974,
+ "dclm_r0920": 0.005257
+ }
+ ],
+ [
+ "p2_tail_in",
+ 0.9875,
+ {
+ "climbmix": 0.580561,
+ "ultra_math_l3": 0.053042,
+ "metamath_gsm": 0.135615,
+ "elem_wordprob": 0.141507,
+ "dclm_r0920": 0.012769,
+ "orca_math": 0.002545,
+ "cosmo2_how": 0.067493,
+ "cosmo_how_v1_tf": 0.006468
+ }
+ ],
+ [
+ "p2_tail",
+ 1.0,
+ {
+ "climbmix": 0.580561,
+ "ultra_math_l3": 0.053042,
+ "metamath_gsm": 0.135615,
+ "elem_wordprob": 0.141507,
+ "dclm_r0920": 0.012769,
+ "orca_math": 0.002545,
+ "cosmo2_how": 0.067493,
+ "cosmo_how_v1_tf": 0.006468
+ }
+ ]
+ ],
+ "smooth": true
+ },
+ "checkpoint_dir": "/media/joseph/Media/Boris-2-0929/checkpoints",
+ "data": {
+ "eos_id": 0,
+ "max_doc_tokens": 65536,
+ "prefetch_depth": 4,
+ "root": "/home/joseph/boris-2-D112M-n15M/data",
+ "sources": [
+ "climbmix",
+ "ultra_math_l3",
+ "orca_math",
+ "cosmo2_how",
+ "cosmo_how_v1_tf",
+ "metamath_gsm",
+ "elem_wordprob",
+ "dclm_r0920"
+ ],
+ "val_path": "/home/joseph/boris-2-D112M-n15M/data/val.bin",
+ "val_source": "climbmix_val"
+ },
+ "enforce_budget": true,
+ "eval": {
+ "batch_size": 128,
+ "device": "cuda:1",
+ "export_dir": "/media/joseph/Media/Boris-2-0929/checkpoints/exports",
+ "manifest": "/media/joseph/Media/Boris-2-0929/checkpoints/exports/manifest.jsonl",
+ "out_path": "/media/joseph/Media/Boris-2-0929/eval/results.jsonl",
+ "poll_seconds": 120,
+ "tasks": [
+ "hellaswag",
+ "arc_easy",
+ "arc_challenge",
+ "piqa",
+ "arithmark3"
+ ],
+ "val_tokens": 10000000
+ },
+ "model": {
+ "architecture": "boris-2-canon-ngram",
+ "canon_kernel": 3,
+ "context_stages": [
+ [
+ 0.0,
+ 2048,
+ 10000.0
+ ],
+ [
+ 0.8,
+ 8192,
+ 500000.0
+ ]
+ ],
+ "d_model": 512,
+ "ffn_hidden": 1664,
+ "n_head": 8,
+ "n_kv_head": 8,
+ "n_layer": 31,
+ "norm_eps": 1e-05,
+ "tie_embeddings": true,
+ "tokenizer_sha256": "f366974b96357d11a7e9930826efd81d77438638c0e21ea1517f620da72e05f4",
+ "use_canon": true,
+ "use_ngram": true,
+ "vocab_size": 32768
+ },
+ "ngram": {
+ "bigram_rows": 210938,
+ "insert_after": 2,
+ "muon_on_proj": true,
+ "oov_rows": 1,
+ "proj_init_std": 0.02,
+ "rank": 64,
+ "table_init_std": 0.02,
+ "target_params": 15000000,
+ "tolerance": 0.01,
+ "trigram_rows": 23437,
+ "vocab_path": "/media/joseph/Media/Boris-2-0920/ngram_vocab.bin"
+ },
+ "optim": {
+ "adamw_betas": [
+ 0.9,
+ 0.95
+ ],
+ "adamw_eps": 1e-08,
+ "adamw_lr": 0.003,
+ "adamw_norm_weight_decay": 0.0,
+ "adamw_weight_decay": 0.1,
+ "grad_clip": 4.0,
+ "lm_head_chunk": 8192,
+ "muon_batched": true,
+ "muon_lr": 0.0025,
+ "muon_momentum": 0.95,
+ "muon_nesterov": true,
+ "muon_ns_dtype": "float32",
+ "muon_ns_steps": 5,
+ "muon_rms_scale": true,
+ "muon_weight_decay": 0.1,
+ "adamw_canon_weight_decay": 0.1
+ },
+ "run_name": "Boris-2-0929",
+ "schedule": {
+ "decay_shape": "1-sqrt",
+ "final_lr_frac": 0.0,
+ "stable_frac": 0.87,
+ "warmup_frac": 0.01
+ },
+ "train": {
+ "abort_after_bad": 200,
+ "abort_after_floored": 100,
+ "amp_dtype": "float16",
+ "compile_regions": true,
+ "device": "cuda:0",
+ "eval_device": "cuda:1",
+ "export_every_tokens": 10000000000,
+ "fp16_residual": false,
+ "full_ckpt_every_tokens": 50000000000,
+ "grad_checkpoint": false,
+ "grad_checkpoint_every": 8,
+ "init_scale": 65536.0,
+ "log_every_steps": 10,
+ "max_loss_scale": 32768.0,
+ "micro_batch": {
+ "2048": 16,
+ "8192": 4
+ },
+ "min_loss_scale": 1.0,
+ "rolling_ckpt_keep": 2,
+ "rolling_ckpt_minutes": 45,
+ "seed": 20260906,
+ "tokens_per_step": 524288,
+ "total_tokens": 200000000000,
+ "val_every_tokens": 1000000000
+ }
+} \ No newline at end of file
diff --git a/configs/boris-2-0929/config.before_p2tail.json b/configs/boris-2-0929/config.before_p2tail.json
new file mode 100644
index 0000000..6c34a6c
--- /dev/null
+++ b/configs/boris-2-0929/config.before_p2tail.json
@@ -0,0 +1,197 @@
+{
+ "curriculum": {
+ "enabled": true,
+ "long_doc_floor_tokens": 4096,
+ "long_doc_target_frac": 0.5,
+ "phases": [
+ [
+ "flat",
+ 0.75,
+ {
+ "climbmix": 0.932109,
+ "ultra_math_l3": 0.064783,
+ "orca_math": 0.003108,
+ "cosmo2_how": 0.0,
+ "cosmo_how_v1_tf": 0.0,
+ "metamath_gsm": 0.0,
+ "elem_wordprob": 0.0,
+ "dclm_r0920": 0.0
+ }
+ ],
+ [
+ "pre_decay_ramp",
+ 0.87,
+ {
+ "climbmix": 0.821333,
+ "ultra_math_l3": 0.081144,
+ "orca_math": 0.003244,
+ "cosmo2_how": 0.085976,
+ "cosmo_how_v1_tf": 0.008302,
+ "metamath_gsm": 0.0,
+ "elem_wordprob": 0.0,
+ "dclm_r0920": 0.0
+ }
+ ],
+ [
+ "decay_in",
+ 0.875,
+ {
+ "climbmix": 0.775534,
+ "ultra_math_l3": 0.07862,
+ "orca_math": 0.003143,
+ "cosmo2_how": 0.083301,
+ "cosmo_how_v1_tf": 0.008044,
+ "metamath_gsm": 0.025126,
+ "elem_wordprob": 0.020974,
+ "dclm_r0920": 0.005257
+ }
+ ],
+ [
+ "decay",
+ 1.0,
+ {
+ "climbmix": 0.775534,
+ "ultra_math_l3": 0.07862,
+ "orca_math": 0.003143,
+ "cosmo2_how": 0.083301,
+ "cosmo_how_v1_tf": 0.008044,
+ "metamath_gsm": 0.025126,
+ "elem_wordprob": 0.020974,
+ "dclm_r0920": 0.005257
+ }
+ ]
+ ],
+ "smooth": true
+ },
+ "checkpoint_dir": "/media/joseph/Media/Boris-2-0929/checkpoints",
+ "data": {
+ "eos_id": 0,
+ "max_doc_tokens": 65536,
+ "prefetch_depth": 4,
+ "root": "/home/joseph/boris-2-D112M-n15M/data",
+ "sources": [
+ "climbmix",
+ "ultra_math_l3",
+ "orca_math",
+ "cosmo2_how",
+ "cosmo_how_v1_tf",
+ "metamath_gsm",
+ "elem_wordprob",
+ "dclm_r0920"
+ ],
+ "val_path": "/home/joseph/boris-2-D112M-n15M/data/val.bin",
+ "val_source": "climbmix_val"
+ },
+ "enforce_budget": true,
+ "eval": {
+ "batch_size": 128,
+ "device": "cuda:1",
+ "export_dir": "/media/joseph/Media/Boris-2-0929/checkpoints/exports",
+ "manifest": "/media/joseph/Media/Boris-2-0929/checkpoints/exports/manifest.jsonl",
+ "out_path": "/media/joseph/Media/Boris-2-0929/eval/results.jsonl",
+ "poll_seconds": 120,
+ "tasks": [
+ "hellaswag",
+ "arc_easy",
+ "arc_challenge",
+ "piqa",
+ "arithmark3"
+ ],
+ "val_tokens": 10000000
+ },
+ "model": {
+ "architecture": "boris-2-canon-ngram",
+ "canon_kernel": 3,
+ "context_stages": [
+ [
+ 0.0,
+ 2048,
+ 10000.0
+ ],
+ [
+ 0.8,
+ 8192,
+ 500000.0
+ ]
+ ],
+ "d_model": 512,
+ "ffn_hidden": 1664,
+ "n_head": 8,
+ "n_kv_head": 8,
+ "n_layer": 31,
+ "norm_eps": 1e-05,
+ "tie_embeddings": true,
+ "tokenizer_sha256": "f366974b96357d11a7e9930826efd81d77438638c0e21ea1517f620da72e05f4",
+ "use_canon": true,
+ "use_ngram": true,
+ "vocab_size": 32768
+ },
+ "ngram": {
+ "bigram_rows": 210938,
+ "insert_after": 2,
+ "muon_on_proj": true,
+ "oov_rows": 1,
+ "proj_init_std": 0.02,
+ "rank": 64,
+ "table_init_std": 0.02,
+ "target_params": 15000000,
+ "tolerance": 0.01,
+ "trigram_rows": 23437,
+ "vocab_path": "/media/joseph/Media/Boris-2-0920/ngram_vocab.bin"
+ },
+ "optim": {
+ "adamw_betas": [
+ 0.9,
+ 0.95
+ ],
+ "adamw_eps": 1e-08,
+ "adamw_lr": 0.003,
+ "adamw_norm_weight_decay": 0.0,
+ "adamw_weight_decay": 0.1,
+ "grad_clip": 4.0,
+ "lm_head_chunk": 8192,
+ "muon_batched": true,
+ "muon_lr": 0.0025,
+ "muon_momentum": 0.95,
+ "muon_nesterov": true,
+ "muon_ns_dtype": "float32",
+ "muon_ns_steps": 5,
+ "muon_rms_scale": true,
+ "muon_weight_decay": 0.1,
+ "adamw_canon_weight_decay": 0.1
+ },
+ "run_name": "Boris-2-0929",
+ "schedule": {
+ "decay_shape": "1-sqrt",
+ "final_lr_frac": 0.0,
+ "stable_frac": 0.87,
+ "warmup_frac": 0.01
+ },
+ "train": {
+ "abort_after_bad": 200,
+ "abort_after_floored": 100,
+ "amp_dtype": "float16",
+ "compile_regions": true,
+ "device": "cuda:0",
+ "eval_device": "cuda:1",
+ "export_every_tokens": 10000000000,
+ "fp16_residual": false,
+ "full_ckpt_every_tokens": 50000000000,
+ "grad_checkpoint": false,
+ "grad_checkpoint_every": 8,
+ "init_scale": 65536.0,
+ "log_every_steps": 10,
+ "max_loss_scale": 32768.0,
+ "micro_batch": {
+ "2048": 16,
+ "8192": 4
+ },
+ "min_loss_scale": 1.0,
+ "rolling_ckpt_keep": 2,
+ "rolling_ckpt_minutes": 45,
+ "seed": 20260906,
+ "tokens_per_step": 524288,
+ "total_tokens": 200000000000,
+ "val_every_tokens": 1000000000
+ }
+} \ No newline at end of file
diff --git a/configs/boris-2-0929/config.json b/configs/boris-2-0929/config.json
new file mode 100644
index 0000000..ac90eac
--- /dev/null
+++ b/configs/boris-2-0929/config.json
@@ -0,0 +1,220 @@
+{
+ "curriculum": {
+ "enabled": true,
+ "long_doc_floor_tokens": 4096,
+ "long_doc_target_frac": 0.5,
+ "phases": [
+ [
+ "flat",
+ 0.75,
+ {
+ "climbmix": 0.932109,
+ "ultra_math_l3": 0.064783,
+ "orca_math": 0.003108,
+ "cosmo2_how": 0.0,
+ "cosmo_how_v1_tf": 0.0,
+ "metamath_gsm": 0.0,
+ "elem_wordprob": 0.0,
+ "dclm_r0920": 0.0
+ }
+ ],
+ [
+ "pre_decay_ramp",
+ 0.87,
+ {
+ "climbmix": 0.821333,
+ "ultra_math_l3": 0.081144,
+ "orca_math": 0.003244,
+ "cosmo2_how": 0.085976,
+ "cosmo_how_v1_tf": 0.008302,
+ "metamath_gsm": 0.0,
+ "elem_wordprob": 0.0,
+ "dclm_r0920": 0.0
+ }
+ ],
+ [
+ "decay_in",
+ 0.875,
+ {
+ "climbmix": 0.775534,
+ "ultra_math_l3": 0.07862,
+ "orca_math": 0.003143,
+ "cosmo2_how": 0.083301,
+ "cosmo_how_v1_tf": 0.008044,
+ "metamath_gsm": 0.025126,
+ "elem_wordprob": 0.020974,
+ "dclm_r0920": 0.005257
+ }
+ ],
+ [
+ "decay",
+ 0.9825,
+ {
+ "climbmix": 0.775534,
+ "ultra_math_l3": 0.07862,
+ "orca_math": 0.003143,
+ "cosmo2_how": 0.083301,
+ "cosmo_how_v1_tf": 0.008044,
+ "metamath_gsm": 0.025126,
+ "elem_wordprob": 0.020974,
+ "dclm_r0920": 0.005257
+ }
+ ],
+ [
+ "p2_tail_in",
+ 0.9875,
+ {
+ "climbmix": 0.580561,
+ "ultra_math_l3": 0.053042,
+ "metamath_gsm": 0.135615,
+ "elem_wordprob": 0.141507,
+ "dclm_r0920": 0.012769,
+ "orca_math": 0.002545,
+ "cosmo2_how": 0.067493,
+ "cosmo_how_v1_tf": 0.006468
+ }
+ ],
+ [
+ "p2_tail",
+ 1.0,
+ {
+ "climbmix": 0.580561,
+ "ultra_math_l3": 0.053042,
+ "metamath_gsm": 0.135615,
+ "elem_wordprob": 0.141507,
+ "dclm_r0920": 0.012769,
+ "orca_math": 0.002545,
+ "cosmo2_how": 0.067493,
+ "cosmo_how_v1_tf": 0.006468
+ }
+ ]
+ ],
+ "smooth": true
+ },
+ "checkpoint_dir": "/media/joseph/Media/Boris-2-0929/checkpoints",
+ "data": {
+ "eos_id": 0,
+ "max_doc_tokens": 65536,
+ "prefetch_depth": 4,
+ "root": "/home/joseph/boris-2-D112M-n15M/data",
+ "sources": [
+ "climbmix",
+ "ultra_math_l3",
+ "orca_math",
+ "cosmo2_how",
+ "cosmo_how_v1_tf",
+ "metamath_gsm",
+ "elem_wordprob",
+ "dclm_r0920"
+ ],
+ "val_path": "/home/joseph/boris-2-D112M-n15M/data/val.bin",
+ "val_source": "climbmix_val"
+ },
+ "enforce_budget": true,
+ "eval": {
+ "batch_size": 128,
+ "device": "cuda:1",
+ "export_dir": "/media/joseph/Media/Boris-2-0929/checkpoints/exports",
+ "manifest": "/media/joseph/Media/Boris-2-0929/checkpoints/exports/manifest.jsonl",
+ "out_path": "/media/joseph/Media/Boris-2-0929/eval/results.jsonl",
+ "poll_seconds": 120,
+ "tasks": [
+ "hellaswag",
+ "arc_easy",
+ "arc_challenge",
+ "piqa",
+ "arithmark3"
+ ],
+ "val_tokens": 10000000
+ },
+ "model": {
+ "architecture": "boris-2-canon-ngram",
+ "canon_kernel": 3,
+ "context_stages": [
+ [
+ 0.0,
+ 2048,
+ 10000.0
+ ]
+ ],
+ "d_model": 512,
+ "ffn_hidden": 1664,
+ "n_head": 8,
+ "n_kv_head": 8,
+ "n_layer": 31,
+ "norm_eps": 1e-05,
+ "tie_embeddings": true,
+ "tokenizer_sha256": "f366974b96357d11a7e9930826efd81d77438638c0e21ea1517f620da72e05f4",
+ "use_canon": true,
+ "use_ngram": true,
+ "vocab_size": 32768
+ },
+ "ngram": {
+ "bigram_rows": 210938,
+ "insert_after": 2,
+ "muon_on_proj": true,
+ "oov_rows": 1,
+ "proj_init_std": 0.02,
+ "rank": 64,
+ "table_init_std": 0.02,
+ "target_params": 15000000,
+ "tolerance": 0.01,
+ "trigram_rows": 23437,
+ "vocab_path": "/media/joseph/Media/Boris-2-0920/ngram_vocab.bin"
+ },
+ "optim": {
+ "adamw_betas": [
+ 0.9,
+ 0.95
+ ],
+ "adamw_eps": 1e-08,
+ "adamw_lr": 0.003,
+ "adamw_norm_weight_decay": 0.0,
+ "adamw_weight_decay": 0.1,
+ "grad_clip": 4.0,
+ "lm_head_chunk": 8192,
+ "muon_batched": true,
+ "muon_lr": 0.0025,
+ "muon_momentum": 0.95,
+ "muon_nesterov": true,
+ "muon_ns_dtype": "float32",
+ "muon_ns_steps": 5,
+ "muon_rms_scale": true,
+ "muon_weight_decay": 0.1,
+ "adamw_canon_weight_decay": 0.1
+ },
+ "run_name": "Boris-2-0929",
+ "schedule": {
+ "decay_shape": "1-sqrt",
+ "final_lr_frac": 0.0,
+ "stable_frac": 0.87,
+ "warmup_frac": 0.01
+ },
+ "train": {
+ "abort_after_bad": 200,
+ "abort_after_floored": 100,
+ "amp_dtype": "float16",
+ "compile_regions": true,
+ "device": "cuda:0",
+ "eval_device": "cuda:1",
+ "export_every_tokens": 10000000000,
+ "fp16_residual": false,
+ "full_ckpt_every_tokens": 50000000000,
+ "grad_checkpoint": false,
+ "grad_checkpoint_every": 8,
+ "init_scale": 65536.0,
+ "log_every_steps": 10,
+ "max_loss_scale": 32768.0,
+ "micro_batch": {
+ "2048": 16,
+ "8192": 4
+ },
+ "min_loss_scale": 1.0,
+ "rolling_ckpt_keep": 2,
+ "rolling_ckpt_minutes": 45,
+ "seed": 20260906,
+ "tokens_per_step": 1048576,
+ "total_tokens": 200000000000,
+ "val_every_tokens": 1000000000
+ }
+} \ No newline at end of file
diff --git a/configs/boris-2-0929/final_decay_draws.json b/configs/boris-2-0929/final_decay_draws.json
new file mode 100644
index 0000000..0150d5c
--- /dev/null
+++ b/configs/boris-2-0929/final_decay_draws.json
@@ -0,0 +1,10 @@
+{
+ "climbmix": 0.775534,
+ "ultra_math_l3": 0.07862,
+ "cosmo2_how": 0.083301,
+ "cosmo_how_v1_tf": 0.008044,
+ "metamath_gsm": 0.025126,
+ "elem_wordprob": 0.020974,
+ "dclm_r0920": 0.005257,
+ "orca_math": 0.003143
+} \ No newline at end of file
diff --git a/data.py b/data.py
new file mode 100644
index 0000000..0d71175
--- /dev/null
+++ b/data.py
@@ -0,0 +1,488 @@
+"""Varlen-packed data pipeline.
+
+Each source is a memory-mapped uint16 token file plus a uint64 index of document
+start offsets. Batches are packed, not EOS-concatenated: documents are laid end
+to end into rows of `block_size`, and the packer emits the document boundaries
+directly. It knows them for free while laying documents down, so nothing later
+has to re-derive them with a GPU scan and a host sync on every forward.
+
+Four tensors come out of the packer besides the tokens, and every one of them
+exists to stop a document seeing another document:
+
+ offsets document boundaries into the flattened (B*T) stream. Feeds the
+ jagged nested tensor, so attention is per-document.
+ boundary (B, T) bool, True where a document starts. Masks the Canon taps
+ and the n-gram history.
+ pos (B, T) int32, position within the document. RoPE restarts at each
+ document rather than running on for the whole row.
+ targets next token within the document; the last position of a document
+ predicts nothing and is IGNORE.
+
+Everything stochastic runs on a torch.Generator whose state is checkpointed,
+together with the partially consumed document carried between batches, so a
+resumed run replays the exact same token stream.
+"""
+from __future__ import annotations
+
+import queue
+import threading
+from pathlib import Path
+import json
+
+import numpy as np
+import torch
+
+from config import Config
+from schedule import long_doc_pressure, mixture_at
+
+IGNORE = -100
+
+
+_DIGITS = None
+
+
+def _digit_split(doc):
+ """BORIS_DIGIT_SPLIT=1 (pilot only): rewrite every number token as single digits,
+ e.g. "Ġ999" -> "Ġ" "9" "9" "9", exactly as data/tokenizer_digits/tokenizer.json encodes
+ text (verified on 4,000 ClimbMix documents: 3,997 identical). Off by default."""
+ global _DIGITS
+ import os
+ if os.environ.get("BORIS_DIGIT_SPLIT") != "1":
+ return doc
+ if _DIGITS is None:
+ z = np.load(Path(__file__).resolve().parent / "data/tokenizer_digits/expand.npz")
+ _DIGITS = (z["flat"], z["starts"], z["lens"])
+ flat, starts, lens = _DIGITS
+ L = lens[doc]
+ if (L == 1).all():
+ return doc
+ idx = np.repeat(starts[doc], L) + (np.arange(int(L.sum())) - np.repeat(np.cumsum(L) - L, L))
+ return flat[idx]
+
+
+def validate_tokenization(directory, cfg):
+ """Fail closed if a prepared 0920 shard uses different token IDs."""
+ from config import LEGACY_TOKENIZER_SHA256
+ if cfg.model.tokenizer_sha256 == LEGACY_TOKENIZER_SHA256:
+ return
+ path = Path(directory) / "retokenize_0920.json"
+ if not path.exists():
+ raise ValueError(f"missing tokenizer provenance: {path}")
+ got = json.loads(path.read_text()).get("new_tokenizer_sha256")
+ if got != cfg.model.tokenizer_sha256:
+ raise ValueError(f"tokenizer mismatch in {directory}: {got}")
+
+
+def open_validation(cfg):
+ directory = Path(cfg.data.root) / cfg.data.val_source
+ validate_tokenization(directory, cfg)
+ return SourceStore.open(cfg.data.root, cfg.data.val_source)
+
+
+class SourceStore:
+ """One tokenized source: tokens.bin (uint16) + docs.idx (uint64)."""
+
+ def __init__(self, name: str, bin_path: Path, idx_path: Path):
+ self.name = name
+ self.tokens = np.memmap(bin_path, dtype=np.uint16, mode="r")
+ # Keep the multi-GB document index file mapped instead of copying it
+ # into anonymous RAM. Random lookup only needs two adjacent entries.
+ self.offsets = np.memmap(idx_path, dtype=np.uint64, mode="r")
+ if self.offsets.size < 2:
+ raise ValueError(f"{idx_path}: need at least two offsets")
+ self.n_docs = self.offsets.size - 1
+ raw_lengths = np.diff(self.offsets)
+ if raw_lengths.size and raw_lengths.max() > np.iinfo(np.uint32).max:
+ raise ValueError(f"{idx_path}: a document exceeds uint32 token length")
+ self.lengths = raw_lengths.astype(np.uint32)
+ self._long_cache: dict[int, np.ndarray] = {}
+
+ @property
+ def n_tokens(self) -> int:
+ return int(self.offsets[-1])
+
+ def doc(self, i: int) -> np.ndarray:
+ a, b = int(self.offsets[i]), int(self.offsets[i + 1])
+ return self.tokens[a:b]
+
+ def long_docs(self, floor: int) -> np.ndarray:
+ """Indices of documents at least `floor` tokens long, computed once.
+
+ Built from the length array, which is already in memory -- no pass over
+ the token file, which for a 200GB source would be its own job.
+ """
+ if floor not in self._long_cache:
+ self._long_cache[floor] = np.flatnonzero(self.lengths >= floor)
+ return self._long_cache[floor]
+
+ @staticmethod
+ def open(root: Path, name: str) -> "SourceStore":
+ d = Path(root) / name
+ return SourceStore(name, d / "tokens.bin", d / "docs.idx")
+
+
+class MultiStore:
+ """Several shard directories presented as one logical source.
+
+ A source too large for one prepare process is split across workers by data
+ file range, each writing its own directory. Stitching them here means the
+ shards are never concatenated on disk -- copying 200GB to merge them would
+ cost as much as downloading them and need the free space twice over.
+
+ Document indices are global: doc i belongs to whichever shard's cumulative
+ range contains it.
+ """
+
+ def __init__(self, name: str, stores: list[SourceStore]):
+ self.name = name
+ self.stores = stores
+ counts = np.array([s.n_docs for s in stores], dtype=np.int64)
+ self._bounds = np.concatenate([[0], np.cumsum(counts)])
+ self.n_docs = int(self._bounds[-1])
+ self.lengths = np.concatenate([s.lengths for s in stores])
+ # MultiStore owns the combined length index; retaining every per-shard
+ # copy doubled resident memory (about 2.5 GB for ClimbMix).
+ for s in stores:
+ s.lengths = np.empty(0, dtype=np.uint32)
+ self._long_cache: dict[int, np.ndarray] = {}
+
+ @property
+ def n_tokens(self) -> int:
+ return sum(s.n_tokens for s in self.stores)
+
+ def doc(self, i: int) -> np.ndarray:
+ j = int(np.searchsorted(self._bounds, i, side="right")) - 1
+ return self.stores[j].doc(i - int(self._bounds[j]))
+
+ def long_docs(self, floor: int) -> np.ndarray:
+ if floor not in self._long_cache:
+ self._long_cache[floor] = np.flatnonzero(self.lengths >= floor)
+ return self._long_cache[floor]
+
+
+class VarlenPacker:
+ """Draws documents per the curriculum and packs them into fixed-width rows."""
+
+ def __init__(self, cfg: Config, stores: dict[str, SourceStore], seed: int,
+ split: str = "train"):
+ self.cfg = cfg
+ self.stores = stores
+ self.split = split
+ self.gen = torch.Generator().manual_seed(seed)
+ self._carry: np.ndarray | None = None
+ self._carry_name: str | None = None
+ self._carry_pos = 0 # position within the carried document
+ self.docs_seen = 0
+ self.source_tokens = {k: 0 for k in stores}
+
+ # -- sampling ------------------------------------------------------- #
+ def _pick_source(self, weights) -> str:
+ names = [n for n in weights if n in self.stores and weights[n] > 0]
+ if not names:
+ names = sorted(self.stores)
+ probs = torch.ones(len(names))
+ else:
+ probs = torch.tensor([weights[n] for n in names], dtype=torch.float)
+ i = int(torch.multinomial(probs, 1, generator=self.gen).item())
+ return names[i]
+
+ def _long_draw_prob(self, name: str, st, floor: int, token_frac: float) -> float:
+ """Per-draw probability of taking a long document so that `token_frac` of
+ this source's TOKENS come from documents >= floor.
+
+ long_doc_target_frac is a share of tokens (config.py; the stage-2 plan was
+ "20B of 40B tokens from docs >= 4096"). It used to be applied as a per-draw
+ probability, and since long documents average ~13x the mean length, 0.5 of
+ draws was ~94% of stage-2 tokens. With q the long-pool probability, a
+ uniform draw otherwise, L the mean length, Ll the long-pool mean and
+ Eu = E[len * 1(len >= floor)] under a uniform draw:
+ tokens_long / tokens = (q Ll + (1-q) Eu) / (q Ll + (1-q) L) = f
+ => q = (f L - Eu) / ((1-f) Ll - Eu + f L), clamped to [0, 1].
+ """
+ key = (name, floor, token_frac)
+ cache = self.__dict__.setdefault("_long_prob_cache", {})
+ if key not in cache:
+ lens = st.lengths.astype(np.float64)
+ L = lens.mean() if lens.size else 0.0
+ longm = lens >= floor
+ Ll = lens[longm].mean() if longm.any() else 0.0
+ Eu = (lens * longm).mean() if lens.size else 0.0
+ den = (1.0 - token_frac) * Ll - Eu + token_frac * L
+ q = (token_frac * L - Eu) / den if den > 0 else 0.0
+ cache[key] = float(min(max(q, 0.0), 1.0))
+ return cache[key]
+
+ def _next_doc(self, weights, floor: int, long_frac: float):
+ name = self._pick_source(weights)
+ st = self.stores[name]
+ q = self._long_draw_prob(name, st, floor, long_frac) if floor > 0 else 0.0
+ want_long = floor > 0 and float(torch.rand(1, generator=self.gen)) < q
+ pool = st.long_docs(floor) if want_long else None
+ if pool is not None and pool.size:
+ j = int(torch.randint(pool.size, (1,), generator=self.gen))
+ i = int(pool[j])
+ else:
+ i = int(torch.randint(st.n_docs, (1,), generator=self.gen))
+ self.docs_seen += 1
+ return name, _digit_split(np.asarray(st.doc(i), dtype=np.int64))
+
+ # -- packing -------------------------------------------------------- #
+ def batch(self, micro_batch: int, block: int, progress: float = 0.0):
+ weights = (mixture_at(progress, self.cfg) if self.split == "train"
+ else {n: 1.0 for n in self.stores})
+ floor, long_frac = (long_doc_pressure(progress, self.cfg)
+ if self.split == "train" else (0, 0.0))
+ B, T = micro_batch, block
+ xs = np.zeros((B, T), dtype=np.int64)
+ ys = np.full((B, T), IGNORE, dtype=np.int64)
+ pos = np.zeros((B, T), dtype=np.int64)
+ bnd = np.zeros((B, T), dtype=bool)
+ bounds: list[int] = []
+ used: dict[str, int] = {}
+
+ for b in range(B):
+ p = 0
+ while p < T:
+ if self._carry is not None and self._carry.size:
+ # Positions restart on a carried chunk. The chunk opens a
+ # new attention segment (its own bounds entry, its own bnd
+ # marker, Canon and the n-gram table both masked across it),
+ # so it cannot attend to the part of the document already
+ # emitted -- continuing the count described a context the
+ # chunk does not have, and RoPE is relative, so restarting
+ # is identical for every pair it can actually see. It also
+ # keeps pos inside the RoPE table: max_doc_tokens is 65,536
+ # against a table of max_block_size, and a carried document
+ # ran pos to 33,789 and indexed off the end of it.
+ name, d, start = self._carry_name, self._carry, 0
+ self._carry = None
+ else:
+ name, d = self._next_doc(weights, floor, long_frac)
+ start = 0
+ # Long documents are emitted over as many row segments as
+ # necessary. The old slice discarded every token after
+ # max_doc_tokens despite the configuration promising that
+ # documents were split, making 882.5M stored corpus tokens
+ # unreachable.
+ take = min(d.size, T - p)
+ bounds.append(b * T + p)
+ bnd[b, p] = True
+ xs[b, p:p + take] = d[:take]
+ pos[b, p:p + take] = np.arange(start, start + take)
+ # y = next token inside this document; the final slot of a
+ # document predicts nothing and stays IGNORE.
+ if take > 1:
+ ys[b, p:p + take - 1] = d[1:take]
+ if take < d.size:
+ # The document continues in the next row: carry the tail,
+ # and the last kept token does have a next token.
+ ys[b, p + take - 1] = d[take]
+ self._carry = d[take:]
+ self._carry_name = name
+ self._carry_pos = 0
+ used[name] = used.get(name, 0) + take
+ p += take
+
+ for k, v in used.items():
+ self.source_tokens[k] = self.source_tokens.get(k, 0) + v
+ bounds.append(B * T)
+ bounds = np.asarray(bounds, dtype=np.int64)
+ # sum(len^2) is the exact attention cost of this batch. Computed here,
+ # where the lengths are already known, so the MFU line reports what
+ # attention actually did rather than the T^2 a dense mask would have.
+ lens = np.diff(bounds)
+ stats = {"used": used, "sumsq": int((lens * lens).sum()),
+ "tokens": B * T, "docs": int(lens.size),
+ # Carried so the nested-tensor SDPA path never has to derive
+ # them from the offsets with a host sync.
+ "min_seqlen": int(lens.min()), "max_seqlen": int(lens.max())}
+ return (xs, ys, pos, bnd, bounds, stats)
+
+ # -- resumability ---------------------------------------------------- #
+ def state_dict(self) -> dict:
+ return {"gen": self.gen.get_state(),
+ "carry": None if self._carry is None else self._carry.copy(),
+ "carry_name": self._carry_name,
+ "carry_pos": self._carry_pos,
+ "docs_seen": self.docs_seen,
+ "source_tokens": dict(self.source_tokens)}
+
+ def load_state_dict(self, st: dict) -> None:
+ if st is None:
+ return
+ # .cpu(): checkpoints are loaded with map_location=<cuda device>, which
+ # moves this RNG state onto the GPU too, and a CPU Generator's set_state
+ # demands a CPU ByteTensor ("RNG state must be a torch.ByteTensor").
+ # Resume had never actually been run, so this went unnoticed until the
+ # first real one.
+ g = st["gen"]
+ self.gen.set_state(g.cpu().to(torch.uint8) if torch.is_tensor(g)
+ else torch.as_tensor(g, dtype=torch.uint8))
+ c = st.get("carry")
+ name = st.get("carry_name")
+ # A checkpoint carries the half-consumed document it was mid-way
+ # through, tagged with its source name. Resuming that checkpoint onto a
+ # DIFFERENT corpus -- an anneal probe, or 0907 weights on ClimbMix --
+ # leaves that name pointing at a source that no longer exists, and
+ # every later _pick_source lookup would KeyError. Drop the carry: one
+ # truncated document at the resume point costs nothing.
+ if name is not None and name not in self.stores:
+ c, name = None, None
+ self._carry = None if c is None else np.asarray(c)
+ self._carry_name = name
+ self._carry_pos = 0 if name is None else int(st.get("carry_pos", 0))
+ self.docs_seen = int(st.get("docs_seen", 0))
+ self.source_tokens.update(st.get("source_tokens", {}))
+
+
+# --------------------------------------------------------------------------- #
+# Prefetch
+# --------------------------------------------------------------------------- #
+class Prefetcher:
+ """One producer thread, pinned staging buffers, non-blocking H2D.
+
+ A single producer, not a pool: the packer's RNG stream is the run's
+ reproducibility guarantee, and several workers drawing from it would make
+ the token order depend on thread scheduling. Packing is numpy slicing over a
+ memmap and costs far less than a step, so one thread stays well ahead.
+ """
+
+ def __init__(self, packer: VarlenPacker, cfg: Config, device: str):
+ self.packer = packer
+ self.cfg = cfg
+ self.device = device
+ self.q: queue.Queue = queue.Queue(maxsize=cfg.data.prefetch_depth)
+ self._shape = None # (micro_batch, block, progress)
+ self._stop = threading.Event()
+ self._t = threading.Thread(target=self._run, daemon=True)
+ self._lock = threading.Lock()
+ # The producer advances packer ahead of training. Checkpoints must store
+ # the state after the last CONSUMED batch, not its speculative state.
+ self._committed_state = packer.state_dict()
+ self._generation = 0
+ self._error = None
+
+ def configure(self, micro_batch: int, block: int, progress: float):
+ with self._lock:
+ if (self._shape is not None and
+ self._shape[:2] != (micro_batch, block)):
+ # Rewind speculative producer work before a shape transition.
+ # Otherwise every queued 2048-token batch discarded at the 8192
+ # stage silently disappears from the reproducible token stream.
+ try:
+ while True:
+ self.q.get_nowait()
+ except queue.Empty:
+ pass
+ self.packer.load_state_dict(self._committed_state)
+ self._generation += 1
+ self._shape = (micro_batch, block, progress)
+
+ def start(self):
+ self._t.start()
+ return self
+
+ def stop(self):
+ self._stop.set()
+ try:
+ while True:
+ self.q.get_nowait()
+ except queue.Empty:
+ pass
+ self._t.join(timeout=5)
+
+ def state_dict(self):
+ with self._lock:
+ return self._committed_state
+
+ def _run(self):
+ try:
+ self._produce()
+ except BaseException as exc:
+ self._error = exc
+ self._stop.set()
+
+ def _produce(self):
+ while not self._stop.is_set():
+ with self._lock:
+ shape = self._shape
+ generation = self._generation
+ if shape is not None:
+ mb, block, progress = shape
+ out = self.packer.batch(mb, block, progress)
+ state = self.packer.state_dict()
+ if shape is None:
+ self._stop.wait(0.01)
+ continue
+ xs, ys, pos, bnd, bounds, stats = out
+ item = (torch.from_numpy(xs).pin_memory(),
+ torch.from_numpy(ys).pin_memory(),
+ torch.from_numpy(pos).pin_memory(),
+ torch.from_numpy(bnd).pin_memory(),
+ torch.from_numpy(bounds).pin_memory(),
+ stats, (mb, block), state, generation)
+ while not self._stop.is_set():
+ try:
+ self.q.put(item, timeout=0.25)
+ break
+ except queue.Full:
+ continue
+
+ def next(self, micro_batch: int, block: int):
+ """Blocks only if the producer has fallen behind; drops stale shapes
+ after a context stage change."""
+ while True:
+ if self._error is not None:
+ raise RuntimeError("data prefetch failed") from self._error
+ try:
+ xs, ys, pos, bnd, bounds, stats, shape, state, generation = self.q.get(timeout=1)
+ except queue.Empty:
+ if self._stop.is_set():
+ raise RuntimeError("data prefetch stopped") from self._error
+ continue
+ if shape == (micro_batch, block) and generation == self._generation:
+ with self._lock:
+ self._committed_state = state
+ break
+ d = self.device
+ return (xs.to(d, non_blocking=True), ys.to(d, non_blocking=True),
+ pos.to(d, non_blocking=True), bnd.to(d, non_blocking=True),
+ bounds.to(d, non_blocking=True), stats)
+
+
+def open_sources(cfg: Config, log=print) -> dict[str, SourceStore]:
+ stores = {}
+ missing = []
+ for name in cfg.data.sources:
+ root = Path(cfg.data.root)
+ # A shard that has not written anything yet has an empty tokens.bin,
+ # which cannot be mmapped. Skip it: it contributes nothing until it
+ # does, and it will be picked up next time the run starts.
+ dirs = [d for d in sorted(root.glob(f"{name}"))
+ + sorted(root.glob(f"{name}_s*"))
+ if (d / "tokens.bin").exists() and (d / "docs.idx").exists()
+ and (d / "tokens.bin").stat().st_size > 0
+ and (d / "docs.idx").stat().st_size > 8]
+ if not dirs:
+ missing.append(name)
+ continue
+ for d in dirs:
+ validate_tokenization(d, cfg)
+ parts = [SourceStore(name, d / "tokens.bin", d / "docs.idx") for d in dirs]
+ stores[name] = parts[0] if len(parts) == 1 else MultiStore(name, parts)
+ if len(parts) > 1:
+ log(f" {name}: {len(parts)} shards stitched "
+ f"({', '.join(d.name for d in dirs)})")
+ if not stores:
+ raise SystemExit(
+ f"no prepared sources under {cfg.data.root}. Run prepare.py first.")
+ if missing:
+ log(f" WARNING: sources not prepared and excluded from the mixture: "
+ f"{', '.join(missing)}")
+ tot = sum(s.n_tokens for s in stores.values())
+ log(f" {len(stores)} sources, {tot/1e9:.1f}B tokens, "
+ f"{sum(s.n_docs for s in stores.values())/1e6:.1f}M documents")
+ for n, s in sorted(stores.items()):
+ log(f" {n:<16}{s.n_tokens/1e9:>8.2f}B {s.n_docs/1e6:>7.2f}M docs "
+ f"mean len {s.n_tokens/max(s.n_docs,1):>6.0f}")
+ return stores
diff --git a/docs/design-notes-0920.md b/docs/design-notes-0920.md
new file mode 100644
index 0000000..df4ddf6
--- /dev/null
+++ b/docs/design-notes-0920.md
@@ -0,0 +1,218 @@
+# Boris-2-0920
+
+A 143.7M-parameter decoder-only language model trained from scratch on 200B
+tokens on a single V100 32GB. Deep-narrow trunk (d512 x 31L) with fused Canon
+layers, a 15M-parameter n-gram lookup table, Muon on the matmul weights, a WSD
+schedule, and a context length that extends from 2048 to 8192 at 80% of the run.
+
+ GPU 0 Tesla V100 32GB (sm_70) trains. fp16 only -- no bf16 path exists.
+ GPU 1 RTX 3060 12GB (sm_86) runs eval.py, in its own process, never
+ blocking or pausing the training loop.
+
+## Parameters
+
+| | |
+|---|---|
+| attention (31 x 1,048,576) | 32,505,856 |
+| SwiGLU FFN, hidden 1664 | 79,233,024 |
+| Canon depthwise kernels | 142,848 |
+| RMSNorms | 32,256 |
+| tied embedding / LM head | 16,777,216 |
+| n-gram table (234,376 x 64) + projection | 15,032,832 |
+| **total** | **143,724,032** |
+
+Muon holds 111,771,648 parameters; AdamW holds the remaining trainable tensors.
+`python config.py` prints this and asserts it; `train.py` prints the
+per-parameter assignment and refuses to start if it disagrees with the budget.
+
+The 0917 rebalance moved capacity from the FFN into full multi-head attention
+while preserving the same overall size class. The FFN width is 1664 (3.25x).
+
+## Layout
+
+ config.py every judgement call, serialised into every checkpoint
+ tokenizer.py our own 32,768-token byte-level BPE
+ prepare.py stream -> decontaminate -> tokenize -> uint16 shards
+ ngram.py the n-gram table: file format, builder, module
+ model.py the model; fused Canon, varlen attention, fused LM head
+ optim.py Muon (shape-batched Newton-Schulz), AdamW, the split
+ schedule.py WSD, the mixture curriculum, the context stages
+ data.py varlen packer + prefetcher
+ checkpoint.py rolling / full / export, atomic
+ train.py the loop
+ eval.py standalone, on GPU 1, from a weights-only export
+ verify.py 31 correctness checks
+ preflight.py throughput, micro-batch sweep, ablations
+
+`data/` and `checkpoints/` are symlinks onto `/media/joseph/Media` -- `/home` has
+20GB free and 200B tokens is 400GB.
+
+## Order of operations
+
+ python prepare.py plan # what the curriculum will draw
+ python prepare.py sample --mb 2000
+ python tokenizer.py train --sample data/tokenizer_sample.txt
+ python prepare.py source --name ultra_fineweb --tokens 106e9
+ python prepare.py source --name cosmopedia --tokens 33e9
+ python prepare.py source --name finephrase --tokens 34e9
+ python prepare.py source --name fineweb_edu --tokens 19e9
+ python prepare.py source --name dclm --tokens 10e9
+ python prepare.py val --tokens 2e7
+ python ngram.py build --shards data --out data/ngram_vocab.bin
+
+ python verify.py # 31 checks, all must pass
+ python preflight.py --sweep 4,8,16 # set micro_batch from measurement
+
+ python train.py # or --resume
+ python eval.py --watch # on the other GPU, separately
+
+`prepare.py` is resumable and writes incrementally, so training can start on the
+first sources while the rest are still arriving.
+
+## Measured throughput (V100, 2026-09-06)
+
+ ctx 2048, micro_batch 16 40.1k tok/s 33.7 TFLOPS 27.0% MFU 23.9 GB
+ ctx 8192, micro_batch 4 32.6k tok/s 28.5 TFLOPS 22.8% MFU 24.0 GB
+
+Worst case (one full-block document per row) measures 31.2k / 23.8 GB at ctx
+2048 and 15.8k / 23.8 GB at ctx 8192 -- memory is unchanged, because the
+memory-efficient attention kernel is O(N), so the chosen micro-batches are safe.
+Throughput falls because attention costs sum(len_i^2).
+
+Two-stage projection for 200B: roughly 51 days at ctx 2048 for the first 160B,
+plus roughly 21 days at ctx 8192 for the last 40B -- call it **~72 days**. The
+single-stage "200B in" column that preflight prints is optimistic by exactly
+this amount.
+
+What each optimisation is worth, measured by removing it:
+
+| removed | throughput | vs. all |
+|---|---|---|
+| nothing (all optimisations, mb 16) | 37.6k tok/s | — |
+| compiled regions (at mb 8, since mb 16 OOMs without them) | 21.4k vs 29.1k | **-26%** and +34% memory |
+| fp16 residual stream | 35.0k | -6.8% |
+| batched Newton-Schulz | 35.6k | -5.1% |
+| fp16 reduced-precision GEMM reduction | 34.9k | -6.9% |
+| fused Canon (i.e. what Canon costs) | 38.6k | +2.6% |
+
+The starting point before this round of work was 31.3k tok/s at 21.0% MFU. The
+compiled regions, the block-fused projection weights and the compiled LM-head
+chain took it to 37.6k at 25.3%; removing the host syncs took it to 40.1k at
+27.0%.
+
+**Host syncs were worth 6.7%.** The nested-tensor SDPA path derives min and max
+sequence length from the offsets with a `.item()` on every call -- 131 host syncs
+per forward, about 2,100 per optimizer step, each one draining CPU run-ahead. The
+packer knows both while it is laying documents down, so they are passed to
+`nested_tensor_from_jagged` explicitly and forward+backward now performs exactly
+one sync. Accumulating the step loss as a GPU tensor rather than calling
+`float()` per micro-step removes the last one from the inner loop.
+`probe/syncs.py` counts them; run it after touching the attention path.
+
+Canon costing only 2.6% is the point of fusing it: the same layers as three
+separate convolutions per block measured 4x that.
+
+## What is not the textbook spelling, and why
+
+Every one of these is a measured win on this card, not a preference. The
+measurements are in the docstrings of the modules they belong to.
+
+**Only fixed-shape regions are compiled, and they are wide.** Each block is two
+compiled regions -- norm+QKV+Canon+RoPE, then o_proj+residual+norm+SwiGLU+
+residual -- with only attention between them. Written as separate eager modules
+the RMSNorms upcast to fp32 and materialise a (B,T,512) fp32 tensor 53 times;
+inside the regions Inductor keeps the reduction in fp32 without ever writing it.
+This is worth 36% and a quarter of the activation memory. Whole-model compile is
+still not used: document counts vary, `cu_seqlens` changes length every step, and
+Inductor recompiles mid-run.
+
+**Q/K/V is one Parameter, and so is gate/up.** They carry a `_boris_blocks` tag
+and Muon orthogonalises each row range independently -- identical to treating
+them as separate Parameters, since the quintic iteration acts on one matrix at a
+time, and `verify.py` asserts the two spellings give bit-identical updates. This
+removes a `torch.cat` per layer per micro-step from the forward and its backward
+slice.
+
+**Canon is one kernel over the fused Q|K|V block.** Depthwise convolution is
+per-channel, so one 1536-channel convolution equals three separate ones on
+Q/K/V channel ranges -- 31 kernels per forward, not 93. It is spelled as three
+shifted multiply-adds on the native (B,T,C) layout inside one compiled region
+rather than `F.conv1d(groups=C)`, which needs a transpose whose following
+reshape materialises a copy. The older grouped-convolution spelling was
+substantially slower on the V100.
+
+**Attention is varlen over a jagged nested tensor.** One SDPA launch for the
+whole batch and cost `sum(len_i^2)` instead of `T^2`. At ctx 2048 with realistic
+document lengths, fwd+bwd: nested 11.1 ms, dense block-diagonal mask 27.0 ms,
+per-document loop 45.8 ms -- and nested is bit-identical to the loop.
+
+**The LM head never materialises (N, V) logits.** dx and dW are computed during
+the forward walk over vocab chunks, so there is no recomputation and the total
+GEMM work equals the unfused head. 4.2x less memory, and *more* accurate than
+the unfused fp16 path: dx 8.4e-04 vs 1.3e-02, dW 2.0e-04 vs 3.9e-03, both
+against fp32.
+
+**Newton-Schulz is batched by shape.** 218 logical matrices in a few distinct shapes
+become four bmm chains -- about 60 kernel launches per optimizer step instead of
+2,300. In fp32: five steps amplify rounding well past bf16 epsilon, and sm_70
+has no bf16 tensor cores to gain from.
+
+**The residual stream is fp32.** The live V100 run measured activations above
+fp16's finite range. Compiled regions still cast matmul inputs under autocast,
+so tensor-core GEMMs remain fp16 while residual additions stay finite.
+
+**The packer emits document boundaries directly.** It knows them while laying
+documents down; deriving them later costs a GPU scan and a host sync per forward.
+
+## Document isolation
+
+Rows hold several packed documents and three separate mechanisms keep them
+apart: varlen attention, the Canon boundary mask, and the n-gram history mask.
+Kernel-3 Canon stacked 31 deep reaches 62 tokens back, so without the mask every
+block pulls the previous document's tail across the boundary.
+
+`verify.py` de-zeroes the Canon kernels before probing for a leak. They are
+zero-initialised, so at init the convolution contributes exactly nothing and an
+isolation test passes no matter how badly the model leaks -- which is how the
+75M run shipped that bug behind a green test for weeks.
+
+## Interruption
+
+The run is expected to be interrupted. A rolling checkpoint every 45 minutes
+(two kept, atomic) carries the weights, both optimizer states, the loss scale,
+the packer's RNG and its carried document tail, so a resume replays the identical
+token stream. The 50B full checkpoints are archival milestones, not the resume
+point. `stable.pt`, written at the end of the stable phase, is permanent and
+never rotated: variants branch from it as a tree, and re-warming from a fully
+decayed model costs a loss spike and a slow recovery.
+
+A resume refuses to load a checkpoint whose config fingerprint or n-gram vocab
+fingerprint disagrees with the running configuration. `--force-config` overrides.
+
+## Eval
+
+`eval.py --watch` polls the export manifest and appends one row per checkpoint to
+`eval_out/results.jsonl`, keyed by `tokens_seen`, so the run plots directly
+against training tokens. All five benchmarks are multiple-choice log-likelihood,
+ArithMark-3 included; both raw and length-normalised accuracy are recorded, plus
+held-out loss and perplexity on a fixed decontaminated split.
+
+## September 22 repair and active run
+
+Boris-2-0920 was restarted from initialization on September 22 at 18:19 PDT,
+with the same 143,724,032 parameters and 200B-token target. The previous attempt
+is preserved under `/media/joseph/Media/Boris-2-0920/failed_20260922`.
+
+The confirmed fused-head overflow was fixed by widening the dW GEMM output
+before undoing its protective scaling. Validation now uses the retokenized
+`climbmix_val` source. Regression checks call the production head under autocast;
+64 verification checks passed, followed by 20 finite V100 updates from the failed
+checkpoint with zero skips. Details and source snapshots are under
+`/media/joseph/Media/Boris-2-0920/repair_20260922/`.
+
+Current status is `/media/joseph/Media/Boris-2-0920/process_status.json` and the
+live console log is `/media/joseph/Media/Boris-2-0920/train.log`.
+Resume with `bash logs/launch_chain.sh --resume`. The launcher uses
+`supervise_run.py`; duplicate supervisors are refused by an exclusive lock.
+Missing resume checkpoints and accidental fresh launches into a nonempty
+checkpoint directory are errors. Do not run the old `finalize_0920.sh` to resume.
diff --git a/eval.py b/eval.py
new file mode 100644
index 0000000..88f692a
--- /dev/null
+++ b/eval.py
@@ -0,0 +1,358 @@
+"""Standalone evaluation, on the second GPU, from a weights-only export.
+
+ python eval.py --watch # poll the export manifest forever
+ python eval.py --ckpt checkpoints/exports/export_0010B.pt
+ python eval.py --ckpt ... --tasks hellaswag,piqa
+
+Runs in its own process on cuda:1 and never touches the training GPU. Training
+writes an export every 10B tokens and appends a manifest line; this picks it up,
+scores it, and appends one row per checkpoint to eval_out/results.jsonl keyed by
+tokens_seen -- so the whole run plots directly against training tokens.
+
+All five benchmarks are multiple-choice log-likelihood, ArithMark-3 included
+(its records are HellaSwag-shaped: a context and four endings). Both the raw
+sum and the byte-length-normalised score are reported: acc_norm is the number
+the Open SLM Leaderboard shows for HellaSwag, ARC and PIQA, and raw accuracy is
+the more meaningful one for ArithMark, where all endings are nearly the same
+length and normalising only adds noise.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import os
+import re
+import time
+from pathlib import Path
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+
+from config import Config
+from model import Boris2
+from ngram import NGramVocab
+from tokenizer import BorisTokenizer
+
+HF_CACHE = Path.home() / ".cache" / "huggingface"
+
+
+# --------------------------------------------------------------------------- #
+# Task loading
+# --------------------------------------------------------------------------- #
+def _hf(name, *args, **kw):
+ from datasets import load_dataset
+ prev = os.environ.get("HF_HUB_OFFLINE")
+ os.environ.setdefault("HF_HUB_OFFLINE", "1") # never hit the network mid-run
+ try:
+ return load_dataset(name, *args, **kw)
+ finally:
+ if prev is None:
+ os.environ.pop("HF_HUB_OFFLINE", None)
+
+
+def load_hellaswag():
+ d = _hf("Rowan/hellaswag", split="validation")
+ def clean(s):
+ s = s.strip().replace(" [title]", ". ")
+ s = re.sub(r"\[.*?\]", "", s).replace(" ", " ")
+ return s
+ out = []
+ for r in d:
+ if str(r["label"]).strip() == "":
+ continue
+ # Match lm-evaluation-harness/Open SLM Leaderboard preprocessing.
+ ctx = r["ctx_a"] + " " + r["ctx_b"].capitalize()
+ out.append({"ctx": clean(r["activity_label"] + ": " + ctx),
+ "choices": [" " + clean(x) for x in r["endings"]],
+ "gold": int(r["label"])})
+ return out
+
+
+def load_arc(which):
+ d = _hf("allenai/ai2_arc", which, split="test")
+ out = []
+ for r in d:
+ ch = r["choices"]
+ labels = list(ch["label"])
+ if r["answerKey"] not in labels:
+ continue
+ out.append({"ctx": "Question: " + r["question"] + "\nAnswer:",
+ "choices": [" " + t for t in ch["text"]],
+ "gold": labels.index(r["answerKey"])})
+ return out
+
+
+def load_piqa():
+ # baber/piqa, not ybisk/piqa: the latter ships a loading script, and
+ # datasets 5.0 refuses those outright. Same 1,838 validation items.
+ d = _hf("baber/piqa", split="validation")
+ return [{"ctx": "Question: " + r["goal"] + "\nAnswer:",
+ "choices": [" " + r["sol1"], " " + r["sol2"]],
+ "gold": int(r["label"])} for r in d]
+
+
+def load_arithmark3():
+ root = HF_CACHE / "hub" / "datasets--AxiomicLabs--ArithMark-3.0" / "snapshots"
+ files = sorted(root.rglob("*.jsonl"))
+ if not files:
+ raise FileNotFoundError("ArithMark-3.0 is not in the HF cache")
+ out = []
+ with open(files[0]) as f:
+ for line in f:
+ r = json.loads(line)
+ out.append({"ctx": r["ctx"], "choices": list(r["endings"]),
+ "gold": int(r["label"])})
+ return out
+
+
+TASKS = {
+ "hellaswag": load_hellaswag,
+ "arc_easy": lambda: load_arc("ARC-Easy"),
+ "arc_challenge": lambda: load_arc("ARC-Challenge"),
+ "piqa": load_piqa,
+ "arithmark3": load_arithmark3,
+}
+
+
+# --------------------------------------------------------------------------- #
+# Scoring
+# --------------------------------------------------------------------------- #
+@torch.no_grad()
+def score_batch(model, tok, items, dev, batch_size=16, max_len=1024):
+ """Summed log-probability of each continuation given its context.
+
+ Sequences are right-padded, which is safe under causal attention: a padded
+ position can only affect positions after it, and none of those are scored.
+ Only the positions that predict continuation tokens are projected through
+ the LM head, so the (B, T, V) logit tensor is never built.
+ """
+ flat = []
+ for i, it in enumerate(items):
+ c = tok.encode(it["ctx"])
+ for j, ch in enumerate(it["choices"]):
+ full, offsets = tok.encode_with_offsets(it["ctx"] + ch)
+ # Include a token that straddles the text boundary. Its probability
+ # is the first one that depends on continuation characters.
+ cut = next((q for q, (_, end) in enumerate(offsets)
+ if end > len(it["ctx"])), len(full))
+ left = max(0, len(full) - max_len)
+ seq = full[left:]
+ cont_start = max(0, cut - left)
+ n_cont = max(0, len(seq) - max(cont_start, 1))
+ # The leading target delimiter is scored but lm-eval excludes it
+ # from the character count used by acc_norm.
+ norm_chars = len(ch[1:]) if ch.startswith(" ") else len(ch)
+ flat.append((i, j, seq, n_cont, norm_chars))
+
+ order = sorted(range(len(flat)), key=lambda i: -len(flat[i][2]))
+ scores: dict[tuple, tuple] = {}
+ W = model.head_weight()
+
+ for a in range(0, len(order), batch_size):
+ chunk = [flat[i] for i in order[a:a + batch_size]]
+ T = max(len(s) for _, _, s, _, _ in chunk)
+ B = len(chunk)
+ x = torch.zeros(B, T, dtype=torch.int64)
+ pos = torch.zeros(B, T, dtype=torch.int64)
+ bnd = torch.zeros(B, T, dtype=torch.bool)
+ bnd[:, 0] = True
+ for b, (_, _, s, _, _) in enumerate(chunk):
+ x[b, :len(s)] = torch.tensor(s)
+ pos[b] = torch.arange(T)
+ off = torch.arange(B + 1, dtype=torch.int64) * T
+ x, pos, bnd, off = x.to(dev), pos.to(dev), bnd.to(dev), off.to(dev)
+
+ with torch.autocast("cuda", dtype=torch.float16):
+ h = model(x, pos, off, bnd, None, model._eval_theta)
+
+ for b, (i, j, s, n_cont, nch) in enumerate(chunk):
+ end = len(s) - 1
+ start = end - n_cont
+ sel = h[b, start:end].float() # positions predicting cont
+ logits = F.linear(sel, W.float())
+ lp = torch.log_softmax(logits, -1)
+ tgt = torch.tensor(s[start + 1:end + 1], device=dev)
+ tot = float(lp.gather(1, tgt[:, None]).sum())
+ scores[(i, j)] = (tot, nch)
+
+ n_hit = n_hit_norm = 0
+ for i, it in enumerate(items):
+ raw = [scores[(i, j)][0] for j in range(len(it["choices"]))]
+ # lm-evaluation-harness acc_norm divides by continuation characters.
+ norm = [scores[(i, j)][0] / max(scores[(i, j)][1], 1)
+ for j in range(len(it["choices"]))]
+ n_hit += int(int(np.argmax(raw)) == it["gold"])
+ n_hit_norm += int(int(np.argmax(norm)) == it["gold"])
+ n = len(items)
+ return {"acc": n_hit / n, "acc_norm": n_hit_norm / n, "n": n}
+
+
+@torch.no_grad()
+def val_loss(model, cfg, dev, max_tokens=10_000_000, block=2048, batch=8):
+ """Held-out loss on a fixed split the packer never draws from."""
+ from data import open_validation, VarlenPacker
+ vdir = Path(cfg.data.root) / cfg.data.val_source
+ if not (vdir / "tokens.bin").exists():
+ return None
+ st = open_validation(cfg)
+ c2 = Config.from_json(cfg.to_json())
+ c2.enforce_budget = False
+ c2.curriculum.enabled = False
+ p = VarlenPacker(c2, {"val": st}, 999, split="val")
+ tot, n, seen = 0.0, 0, 0
+ while seen < max_tokens:
+ xs, ys, pos, bnd, bounds, stats = p.batch(batch, block, 0.0)
+ t = lambda z: torch.from_numpy(z).to(dev)
+ with torch.autocast("cuda", dtype=torch.float16):
+ loss = model(t(xs), t(pos), t(bounds), t(bnd), t(ys), model._eval_theta)
+ valid = int((ys != -100).sum())
+ tot += float(loss) * valid; n += valid; seen += stats["tokens"]
+ return tot / max(n, 1)
+
+
+# --------------------------------------------------------------------------- #
+def load_export(path: Path, dev: str):
+ # The allocator statistics and empty_cache() act on the CURRENT device, not
+ # on the device the tensors live on. eval runs on cuda:1 while cuda:0 is
+ # busy training, so this is not optional.
+ torch.cuda.set_device(dev)
+ blob = torch.load(path, map_location="cpu", weights_only=False)
+ cfg = Config.from_json(blob["config"])
+ # The training head chunk (8192 on the V100) makes the 3060's stock head
+ # materialize 1 GB fp32 logit chunks; the chunk only sets memory, not the loss.
+ cfg.optim.lm_head_chunk = min(cfg.optim.lm_head_chunk, 2048)
+ model = Boris2(cfg).to(dev)
+ sd = {k: v.to(torch.float32) for k, v in blob["model"].items()}
+ model.load_state_dict(sd)
+ if model.ngram is not None:
+ vocab = NGramVocab.load(cfg.ngram.vocab_path)
+ got = vocab.fingerprint()
+ want = blob.get("ngram_fingerprint")
+ if want is not None and want not in (got, vocab.legacy_fingerprint()):
+ raise RuntimeError(f"checkpoint expects n-gram vocab {want}, loaded {got}")
+ model.ngram.load_vocab(vocab, dev)
+ model.eval()
+ progress = blob["tokens_seen"] / cfg.train.total_tokens
+ model._eval_theta = cfg.block_size_at(progress)[1]
+ return model, cfg, blob
+
+
+def evaluate(path: Path, dev: str, tasks, out: Path, val_tokens: int, batch: int,
+ no_ngram: bool = False):
+ t0 = time.time()
+ model, cfg, blob = load_export(path, dev)
+ if no_ngram:
+ model.ngram_enabled = False
+ tok = BorisTokenizer.load(Path(cfg.data.root) / "tokenizer")
+ tok_fp = BorisTokenizer.fingerprint(Path(cfg.data.root) / "tokenizer")
+ if tok_fp != cfg.model.tokenizer_sha256:
+ raise RuntimeError(
+ f"tokenizer fingerprint {tok_fp} does not match checkpoint config "
+ f"{cfg.model.tokenizer_sha256}")
+ row = {"checkpoint": path.name, "step": blob["step"],
+ "tokens_seen": blob["tokens_seen"],
+ "tokens_b": blob["tokens_seen"] / 1e9, "wall": time.time(),
+ "ngram": (model.ngram is not None) and not no_ngram}
+
+ for name in tasks:
+ if name not in TASKS:
+ print(f" unknown task {name}"); continue
+ try:
+ items = TASKS[name]()
+ except Exception as e:
+ print(f" {name}: unavailable ({type(e).__name__}: {str(e)[:90]})")
+ continue
+ r = score_batch(model, tok, items, dev, batch_size=batch)
+ # The Open SLM Leaderboard uses length-normalized accuracy for all
+ # five tasks, including ArithMark-3.
+ row[name] = r["acc_norm"]
+ row[name + "_acc"] = r["acc"]
+ row[name + "_acc_norm"] = r["acc_norm"]
+ row[name + "_n"] = r["n"]
+ print(f" {name:<15} acc {r['acc']:.4f} acc_norm {r['acc_norm']:.4f} "
+ f"n={r['n']}")
+
+ # Compile only now: val batches are fixed-shape (8 x 2048), so the regions
+ # compile once; the benchmark batches above vary in length and would
+ # recompile per shape. Measured on the 3060: val 101 s -> 60 s per 2M
+ # tokens, loss equal to 4 decimals.
+ model.maybe_compile()
+ vl = val_loss(model, cfg, dev, val_tokens)
+ if vl is not None:
+ row["val_loss"] = vl
+ row["val_ppl"] = math.exp(min(vl, 20))
+ print(f" {'val':<15} loss {vl:.4f} ppl {math.exp(min(vl,20)):.3f}")
+
+ keys = [k for k in tasks if k in row]
+ if keys:
+ row["mean"] = sum(row[k] for k in keys) / len(keys)
+ print(f" {'MEAN':<15} {row['mean']:.4f}")
+ row["eval_seconds"] = time.time() - t0
+
+ out.parent.mkdir(parents=True, exist_ok=True)
+ with open(out, "a") as f:
+ f.write(json.dumps(row) + "\n")
+ del model
+ torch.cuda.empty_cache()
+ return row
+
+
+def watch(cfg: Config, dev, tasks, out, val_tokens, batch, poll):
+ done = set()
+ if out.exists():
+ for line in out.read_text().splitlines():
+ try:
+ done.add(json.loads(line)["checkpoint"])
+ except Exception:
+ pass
+ print(f" watching {cfg.eval.manifest} ({len(done)} already scored)")
+ while True:
+ pending = []
+ if cfg.eval.manifest.exists():
+ for line in cfg.eval.manifest.read_text().splitlines():
+ try:
+ m = json.loads(line)
+ except Exception:
+ continue
+ p = Path(m["path"])
+ if p.name not in done and p.exists():
+ pending.append(p)
+ for p in sorted(set(pending)):
+ print(f"\n === {p.name} ===")
+ try:
+ evaluate(p, dev, tasks, out, val_tokens, batch)
+ done.add(p.name)
+ except Exception as e:
+ print(f" failed on {p.name}: {type(e).__name__}: {e}")
+ time.sleep(poll)
+
+
+def main():
+ cfg = Config()
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--ckpt", type=Path, default=None)
+ ap.add_argument("--watch", action="store_true")
+ ap.add_argument("--device", default=cfg.eval.device)
+ ap.add_argument("--tasks", default=",".join(cfg.eval.tasks))
+ ap.add_argument("--out", type=Path, default=cfg.eval.out_path)
+ ap.add_argument("--val-tokens", type=int, default=cfg.eval.val_tokens)
+ ap.add_argument("--batch", type=int, default=cfg.eval.batch_size)
+ ap.add_argument("--no-ngram", action="store_true",
+ help="score with the table switched off -- reports how much "
+ "the trained model DEPENDS on it, not what it gained")
+ a = ap.parse_args()
+ tasks = [t.strip() for t in a.tasks.split(",") if t.strip()]
+
+ if a.watch:
+ watch(cfg, a.device, tasks, a.out, a.val_tokens, a.batch, cfg.eval.poll_seconds)
+ elif a.ckpt:
+ evaluate(a.ckpt, a.device, tasks, a.out, a.val_tokens, a.batch,
+ no_ngram=a.no_ngram)
+ else:
+ raise SystemExit("pass --ckpt PATH or --watch")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/finalize_0920.sh b/finalize_0920.sh
new file mode 100755
index 0000000..64ba388
--- /dev/null
+++ b/finalize_0920.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -euo pipefail
+repo=/home/joseph/boris-2-D112M-n15M
+run=/media/joseph/Media/Boris-2-0920
+exec >>"$run/finalize_0920.log" 2>&1
+
+echo "[$(date -Is)] building 90/10 n-gram vocabulary"
+nice -n 5 ionice -c2 -n5 "$repo/.venv/bin/python" "$repo/ngram.py" build \
+ --config "$run/config.json" --out "$run/ngram_vocab.bin"
+
+echo "[$(date -Is)] verification"
+"$repo/.venv/bin/python" "$repo/verify.py" --quick --device cuda:1
+
+echo "[$(date -Is)] V100 micro-batch confirmation"
+"$repo/.venv/bin/python" "$repo/preflight.py" --device cuda:0 \
+ --block 2048 --sweep 8,16 --iters 4
+
+setsid "$repo/.venv/bin/python" "$repo/eval.py" --watch --device cuda:1 \
+ >"$run/eval_watch.log" 2>&1 < /dev/null &
+
+echo "[$(date -Is)] launching Boris-2-0920"
+exec "$repo/.venv/bin/python" "$repo/train.py" --config "$run/config.json"
diff --git a/libremath/arithcheck.py b/libremath/arithcheck.py
new file mode 100644
index 0000000..d8f78ac
--- /dev/null
+++ b/libremath/arithcheck.py
@@ -0,0 +1,27 @@
+"""Recompute every simple binary step "a op b = c" written in a solution.
+Returns the list of wrong steps (empty if all check out). Times (10:20), ratios (3:4) and
+fractions written inside larger expressions are skipped rather than misread."""
+import re
+NUM = r"\d{1,3}(?:,\d{3})+(?:\.\d+)?|\d+(?:\.\d+)?"
+STEP = re.compile(rf"(?<![\d.:/,])(?<!\d\s)({NUM})\s*([+\-−–×x*÷/])\s*({NUM})\s*=\s*\$?({NUM})(?![\d:/])(?!,\d)(?!\s*[+\-−–×x*÷/(^])")
+
+
+def bad_steps(text):
+ bad = []
+ for m in STEP.finditer(text):
+ a, op, b, c = m.groups()
+ # skip if this is the tail of a longer expression, e.g. "2 + 3 × 4 = 14"
+ pre = text[max(0, m.start() - 3):m.start()]
+ if re.search(r"[+\-−–×x*÷/(]\s*$", pre): continue
+ a, b, c = (float(v.replace(",", "")) for v in (a, b, c))
+ if op in "×x*": v = a * b
+ elif op == "+": v = a + b
+ elif op in "-−–": v = a - b
+ else:
+ if b == 0: continue
+ v = a / b
+ if abs(v - c) > 1e-6 and abs(round(v, 2) - c) < 0.006: continue # rounded quotient
+ if c == int(a // b): continue # whole-number quotient (remainder stated separately)
+ if abs(v - c) > 1e-6 * max(1, abs(v)) and abs(round(v, 2) - c) > 1e-9:
+ bad.append(m.group(0))
+ return bad
diff --git a/libremath/compose.py b/libremath/compose.py
new file mode 100644
index 0000000..46f8bb5
--- /dev/null
+++ b/libremath/compose.py
@@ -0,0 +1,117 @@
+"""LibreMath composer (2026-10-23): unlimited grade-school word problems from the K2-written
+fragment bank (fragments.py), with every number and answer computed here.
+
+ python libremath/compose.py BANK.json OUT_DIR --items 1000000 [--seed 1]
+"""
+import argparse, json, random, re, sys
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+import generate as G
+from tgen import SETTINGS_EXTRA
+
+PH = re.compile(r"\{(\w+)\}")
+
+
+def fill(t, env):
+ t = re.sub(r"\b([Ee])(very|ach) \{(cont|obj)\}", lambda m: ("All the {" if m.group(1) == "E" else "all the {") + m.group(3) + "}"
+ if m.group(2) == "very" else m.group(1) + "ach of the {" + m.group(3) + "}", t) # plural nouns: "every boxes"
+ out = PH.sub(lambda m: str(env[m.group(1)]), t)
+ out = re.sub(r"\b([Tt]he) the\b", r"\1", out)
+ out = out.strip(); out = out[:1].upper() + out[1:]
+ return out if out[-1:] in ".!?\"'”" else out + "."
+
+
+ANSWER_TAGS = {"money": r"cost|spen|paid|pay|price|dollar|bill|\$", "remove": r"\bleft\b|remain|still", "two_step": r"\bleft\b|remain|still",
+ "compare": r"more|differ|ahead|extra", "share": r"\beach\b|per\b|every", "combine": r"total|altogether|in all|together|sum",
+ "groups": r"total|altogether|in all|together"}
+ALL_TAGS = "|".join(ANSWER_TAGS.values()) + r"|collect|gather|found|bought|made|earn|save"
+
+
+def money(x):
+ return f"{x:.2f}"
+
+
+class Composer:
+ def __init__(a, bank, seed):
+ a.b = bank; a.r = random.Random(seed)
+ a.places = G.SETTINGS + SETTINGS_EXTRA; a.names = G.NAMES
+
+ def pick(a, cat): return a.r.choice(a.b[cat])
+
+ def answer_for(a, kind):
+ if not hasattr(a, "_ans"):
+ gen = [f for f in a.b["answer"] if not re.search(ALL_TAGS, f, re.I)]
+ a._ans = {k: (gen if k != "money" else [f for f in gen if "{obj}" not in f]) + [f for f in a.b["answer"] if re.search(v, f, re.I) and not re.search(
+ "|".join(v2 for k2, v2 in ANSWER_TAGS.items() if v2 != v), f, re.I)] for k, v in ANSWER_TAGS.items()}
+ return a.r.choice(a._ans[kind])
+
+ def numbers(a, kind):
+ r = a.r; big = r.random() < 0.35
+ if kind in ("combine", "remove", "compare"):
+ hi = r.choice([20, 50, 100, 100, 500, 1000]) if not big else 5000
+ x, y = r.randint(2, hi), r.randint(2, hi)
+ return (max(x, y), min(x, y)) if kind != "combine" else (x, y)
+ if kind == "groups": return r.randint(2, 12 if not big else 40), r.randint(2, 12 if not big else 99)
+ if kind == "share":
+ b = r.randint(2, 12); return b * r.randint(2, 15 if not big else 60), b
+ if kind == "money": return r.randint(2, 12), r.randint(25, 1999) / 100
+
+ def item(a):
+ r = a.r; A, B = r.sample(a.names, 2); o = r.choice(a.b["objects"]); place = r.choice(a.places)
+ kind = r.choice(["combine", "remove", "compare", "groups", "groups", "share", "money", "two_step"])
+ base = "groups" if kind == "two_step" and r.random() < 0.6 else ("combine" if kind == "two_step" else kind)
+ x, y = a.numbers(base)
+ if x == y and base in ("remove", "compare"): x += 1
+ ans = {"combine": lambda: x + y, "remove": lambda: x - y, "compare": lambda: x - y, "groups": lambda: x * y,
+ "share": lambda: x // y, "money": lambda: round(x * y, 2)}[base]()
+ env = {"A": A, "B": B, "obj": o["obj"], "cont": o["cont"], "place": "the " + place, "a": x, "b": y,
+ "ans": money(ans) if base == "money" else ans, "p": money(y) if base == "money" else y, "d": r.randint(2, 60)}
+ prob = [fill(a.pick(f"{base}_setup"), env)]
+ steps = [fill(a.pick(f"{base}_step"), env)]
+ final = ans
+ if kind == "two_step":
+ c = r.randint(1, max(1, ans - 1)); env2 = dict(env, x=ans, c=c, ans=ans - c)
+ prob.append(fill(a.pick("followup_remove"), env2)); q = fill(a.pick("followup_q"), env2)
+ steps.append(fill(a.pick("followup_step"), env2)); final = ans - c; env = env2
+ else:
+ q = fill(a.pick(f"{base}_q"), env)
+ if r.random() < 0.25: # irrelevant fact, before or after the setup
+ prob.insert(r.choice([0, len(prob)]), fill(a.pick("distractor"), env))
+ if r.random() < 0.7: prob.insert(0, fill(a.pick("opener"), env))
+ problem = " ".join(prob + [q])
+ solution = " ".join(steps + [fill(a.answer_for(kind), env)])
+ if r.random() < 0.3: problem, solution = "Question: " + problem, "Answer: " + solution
+ return {"problem": problem, "solution": solution, "answers": [str(env["ans"])], "kind": kind,
+ "objects": o, "generator": "LibreMath fragment composer (fragments by IFM/K2-Horizon)"}
+
+
+def distinct(texts, n=4):
+ tot = 0; s = set()
+ for t in texts:
+ w = re.sub(r"\d+", "#", t.lower()).split(); g = [tuple(w[i:i + n]) for i in range(len(w) - n + 1)]; tot += len(g); s.update(g)
+ return len(s) / max(tot, 1)
+
+
+def main():
+ ap = argparse.ArgumentParser(); ap.add_argument("bank"); ap.add_argument("out")
+ ap.add_argument("--items", type=int, default=100000); ap.add_argument("--seed", type=int, default=1)
+ ap.add_argument("--no-filter", action="store_true")
+ a = ap.parse_args(); out = Path(a.out); out.mkdir(parents=True, exist_ok=True)
+ bank = json.load(open(a.bank)); c = Composer(bank, a.seed)
+ am_names, ban8 = set(), set()
+ if not a.no_filter:
+ am_names, ban8, _, _ = G.load_bans()
+ seen = set(); kept = rej = 0
+ with open(out / f"items-seed{a.seed}.jsonl", "w") as f:
+ while kept < a.items:
+ it = c.item(); text = it["problem"] + " " + it["solution"]
+ h = hash(it["problem"])
+ if h in seen or (set(re.findall(r"[A-Z][a-z]+", text)) & am_names) or (ban8 and G.ngrams(G.norm_words(text), 8) & ban8):
+ rej += 1; continue
+ seen.add(h); f.write(json.dumps(it) + "\n"); kept += 1
+ S = [json.loads(l) for l in open(out / f"items-seed{a.seed}.jsonl")][:20000]
+ print(f"kept {kept:,}, rejected {rej:,}; distinct 4-gram ratio (numbers masked) on 20k: {distinct([s['problem'] + ' ' + s['solution'] for s in S]):.3f}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/libremath/fragments.py b/libremath/fragments.py
new file mode 100644
index 0000000..4be2380
--- /dev/null
+++ b/libremath/fragments.py
@@ -0,0 +1,134 @@
+"""LibreMath fragment bank (2026-10-23): K2 writes reusable sentence fragments ONCE; compose.py
+assembles unlimited problems from them with exact, code-computed math.
+
+Each category has a fixed set of placeholders. A fragment is kept only if it uses exactly the
+allowed placeholders, includes the required ones, contains no literal digits, and (for solution
+steps) contains the exact equation string the composer will fill.
+
+ python libremath/fragments.py OUT.json --calls-per-category 12
+"""
+import argparse, asyncio, json, random, re, sys, time
+from pathlib import Path
+import aiohttp
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+import generate as G
+
+# category: (description, required placeholders, allowed placeholders, kind)
+CATS = {
+ "opener": ("an opening sentence that sets a scene at or around {place}; it may mention {A}", {"place"}, {"place", "A"}, "s"),
+ "distractor": ("a sentence giving one irrelevant fact that includes the number {d} (not needed to solve any question); it may mention {A} or {place}", {"d"}, {"d", "A", "place"}, "s"),
+ "combine_setup": ("one or two sentences saying {A} has {a} {obj} and then gets {b} more (buys, finds, receives, picks, is given...)", {"A", "a", "b", "obj"}, {"A", "a", "b", "obj", "place"}, "s"),
+ "combine_q": ("a question asking how many {obj} {A} has now in total", {"obj"}, {"A", "obj"}, "q"),
+ "combine_step": ("a solution sentence explaining why we add, containing exactly '{a} + {b} = {ans}'", {"a", "b", "ans"}, {"a", "b", "ans", "A", "obj"}, "s"),
+ "remove_setup": ("one or two sentences saying {A} starts with {a} {obj} and then {b} of them are used, given away, sold, eaten or lost", {"A", "a", "b", "obj"}, {"A", "a", "b", "obj", "place"}, "s"),
+ "remove_q": ("a question asking how many {obj} {A} has left", {"obj"}, {"A", "obj"}, "q"),
+ "remove_step": ("a solution sentence explaining why we subtract, containing exactly '{a} - {b} = {ans}'", {"a", "b", "ans"}, {"a", "b", "ans", "A", "obj"}, "s"),
+ "compare_setup": ("one or two sentences saying {A} has {a} {obj} and {B} has {b} {obj}", {"A", "B", "a", "b"}, {"A", "B", "a", "b", "obj", "place"}, "s"),
+ "compare_q": ("a question asking how many more {obj} {A} has than {B}", {"A", "B"}, {"A", "B", "obj"}, "q"),
+ "compare_step": ("a solution sentence explaining the difference, containing exactly '{a} - {b} = {ans}'", {"a", "b", "ans"}, {"a", "b", "ans", "A", "B", "obj"}, "s"),
+ "groups_setup": ("one or two sentences saying there are {a} {cont} with {b} {obj} in each (packed, arranged, stacked...); may mention {A}", {"a", "b", "cont", "obj"}, {"A", "a", "b", "cont", "obj", "place"}, "s"),
+ "groups_q": ("a question asking how many {obj} there are altogether", {"obj"}, {"A", "obj", "cont"}, "q"),
+ "groups_step": ("a solution sentence explaining why we multiply, containing exactly '{a} × {b} = {ans}'", {"a", "b", "ans"}, {"a", "b", "ans", "obj", "cont"}, "s"),
+ "share_setup": ("one or two sentences saying {A} shares or splits {a} {obj} equally into {b} {cont}", {"A", "a", "b", "obj", "cont"}, {"A", "a", "b", "obj", "cont", "place"}, "s"),
+ "share_q": ("a question asking how many {obj} go in each of the {cont}", {"obj", "cont"}, {"A", "obj", "cont"}, "q"),
+ "share_step": ("a solution sentence explaining why we divide, containing exactly '{a} ÷ {b} = {ans}'", {"a", "b", "ans"}, {"a", "b", "ans", "obj", "cont"}, "s"),
+ "followup_remove": ("a follow-up sentence (for after a total is known) saying {c} of those {obj} are then used, sold or given away", {"c", "obj"}, {"c", "obj", "A", "B"}, "s"),
+ "followup_q": ("a question asking how many {obj} are left after that", set(), {"obj", "A"}, "q"),
+ "followup_step": ("a solution sentence for the second step, containing exactly '{x} - {c} = {ans}'", {"x", "c", "ans"}, {"x", "c", "ans", "obj"}, "s"),
+ "money_setup": ("one or two sentences saying {A} buys {a} {obj} that cost {p} dollars each", {"A", "a", "obj", "p"}, {"A", "a", "obj", "p", "place"}, "s"),
+ "money_q": ("a question asking how much {A} spends in total", {"A"}, {"A", "obj"}, "q"),
+ "money_step": ("a solution sentence explaining the total cost, containing exactly '{a} × ${p} = ${ans}'", {"a", "p", "ans"}, {"a", "p", "ans", "obj", "A"}, "s"),
+ "answer": ("a short final sentence stating the answer {ans}; it may mention {obj} or {A}", {"ans"}, {"ans", "obj", "A"}, "s"),
+}
+TONES = ["plain and simple, for young children", "like a friendly teacher", "like a math textbook", "lively and story-like",
+ "short and direct", "detailed, with a little extra description", "conversational", "matter-of-fact"]
+PH = re.compile(r"\{(\w+)\}")
+
+
+def valid(frag, cat):
+ desc, req, allowed, kind = CATS[cat]
+ if not isinstance(frag, str) or not (12 <= len(frag) <= 260): return False
+ if re.search(r"\d", PH.sub("", frag)): return False
+ ph = set(PH.findall(frag))
+ if not req <= ph or not ph <= allowed: return False
+ if kind == "q" and not frag.rstrip().endswith("?"): return False
+ if kind == "s" and frag.rstrip().endswith("?"): return False
+ eq = re.search(r"'(.*)'", desc)
+ if eq and eq.group(1) not in frag: return False
+ return True
+
+
+def prompt(cat, n, tone):
+ if cat == "objects":
+ return (f"List {n} pairs of things for grade-school math word problems: a plural countable item and a plural container or "
+ f"group it could be packed or arranged in (for example marbles / jars, chairs / rows, stickers / sheets). Cover many "
+ f"different domains (food, sports, school, nature, tools, crafts, animals, travel, music, building). Reply with only a "
+ f'JSON array of objects like {{"obj": "marbles", "cont": "jars"}}.')
+ desc = CATS[cat][0]
+ return (f"Write {n} different English fragments for building grade-school math word problems. Each fragment is {desc}. "
+ f"Use the placeholders in curly braces exactly as written (they are filled in later by a program); never write a digit. "
+ + ("Do not mention any specific item, food, animal or object; the items being counted are filled in later. "
+ if cat in ("opener", "distractor") else
+ "Never name a specific item, food, animal, place or object: refer to the items only as {obj} and {cont} "
+ "(or words like 'them', 'each one', 'the items'), because the program fills them in. ")
+ + f"Style: {tone}. Vary vocabulary and sentence structure as much as possible; no two fragments should start the same way. "
+ f"Plain text, no markdown. Reply with only a JSON array of strings.")
+
+
+async def main_async(a):
+ out = Path(a.out); bank = json.loads(out.read_text()) if out.exists() else {c: [] for c in list(CATS) + ["objects"]}
+ sem = asyncio.Semaphore(a.inflight); lock = asyncio.Lock(); state = {"next": time.time(), "tok": 0}
+ rng = random.Random(a.seed)
+
+ async def call(sess, cat):
+ async with sem:
+ body = {"model": G.MODEL, "temperature": 1.0, "top_p": 0.95, "max_tokens": 16000,
+ **({"chat_template_kwargs": G.TEMPLATE_KW} if G.TEMPLATE_KW else {}),
+ "messages": [{"role": "user", "content": prompt(cat, a.per_call, rng.choice(TONES))}]}
+ for attempt in range(8):
+ async with lock:
+ w = state["next"] - time.time()
+ if w > 0: await asyncio.sleep(w)
+ state["next"] = max(time.time(), state["next"]) + 1 / a.rps
+ try:
+ async with sess.post(G.API, json=body, headers={"Authorization": f"Bearer {G.KEY}"}) as r:
+ if r.status == 429:
+ txt = await r.text()
+ await asyncio.sleep(1800 if "quota" in txt.lower() else 5 * (attempt + 1)); continue
+ if r.status >= 500: await asyncio.sleep(5 * (attempt + 1)); continue
+ d = await r.json(); break
+ except Exception:
+ await asyncio.sleep(5 * (attempt + 1))
+ else:
+ return cat, []
+ state["tok"] += d.get("usage", {}).get("completion_tokens", 0)
+ txt = ((d.get("choices") or [{}])[0].get("message") or {}).get("content") or ""
+ m = re.search(r"\[.*\]", txt, re.S)
+ try: arr = json.loads(m.group(0)) if m else []
+ except Exception: arr = []
+ if cat == "objects":
+ return cat, [{"obj": o["obj"].strip().lower(), "cont": o["cont"].strip().lower()} for o in arr
+ if isinstance(o, dict) and all(isinstance(o.get(k), str) and re.fullmatch(r"[a-zA-Z' -]{3,30}", o[k].strip()) for k in ("obj", "cont"))]
+ return cat, [f.strip() for f in arr if valid(f, cat)]
+
+ async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=600)) as sess:
+ tasks = [call(sess, c) for c in CATS for _ in range(a.calls_per_category)] + [call(sess, "objects") for _ in range(a.calls_per_category)]
+ for fut in asyncio.as_completed(tasks):
+ cat, frags = await fut
+ seen = {json.dumps(f, sort_keys=True) for f in bank[cat]}
+ bank[cat] += [f for f in frags if json.dumps(f, sort_keys=True) not in seen]
+ out.write_text(json.dumps(bank, indent=1))
+ print(f" {cat}: +{len(frags)} -> {len(bank[cat])} (output tokens so far {state['tok']:,})", flush=True)
+ print({c: len(v) for c, v in bank.items()})
+
+
+def main():
+ ap = argparse.ArgumentParser(); ap.add_argument("out")
+ ap.add_argument("--calls-per-category", type=int, default=12); ap.add_argument("--per-call", type=int, default=40)
+ ap.add_argument("--rps", type=float, default=1.0); ap.add_argument("--inflight", type=int, default=16); ap.add_argument("--seed", type=int, default=7)
+ asyncio.run(main_async(ap.parse_args()))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/libremath/generate.py b/libremath/generate.py
new file mode 100644
index 0000000..0042f05
--- /dev/null
+++ b/libremath/generate.py
@@ -0,0 +1,247 @@
+"""LibreMath-1.0 generator: K2-Horizon-375B writes problems around specs.py's facts.
+
+ python libremath/generate.py OUT_DIR --items 200000 [--per-request 20] [--rps 2.8] [--seed N]
+
+Every request carries `per-request` specs, each with its own randomly chosen format,
+setting and names. The reply is a JSON list; each item is kept only if
+ * its final answer equals the computed answer,
+ * the problem ends with a question mark (ArithMark items are unfinished sentences),
+ * it uses none of ArithMark's 53 character names,
+ * it shares no 8-word span with any ArithMark, SVAMP or ASDiv test item,
+ * it passes the corpus-wide 13-gram ban (HellaSwag, ARC, PIQA, ArithMark),
+ * it is not a duplicate of an earlier problem.
+Kept items go to OUT_DIR/items-XXXXX.jsonl; counters and rejects to OUT_DIR/stats.json.
+Resumable: shards already written are counted and the seed advances past them.
+"""
+import argparse, asyncio, hashlib, json, random, re, sys, time
+from pathlib import Path
+import aiohttp
+
+ROOT = Path(__file__).resolve().parent
+sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT.parent))
+import specs
+from arithcheck import bad_steps
+
+import os
+API = os.environ.get("LIBREMATH_API", "https://api.ifm.ai/v1/chat/completions")
+MODEL = os.environ.get("LIBREMATH_MODEL", "IFM/K2-Horizon-375B-A23B")
+TEMPLATE_KW = json.loads(os.environ.get("LIBREMATH_TEMPLATE_KW", '{"reasoning_effort": "low"}'))
+import os
+KEY = open(Path(os.environ.get("LIBREMATH_KEY_FILE", Path.home() / ".config/libremath/api_key"))).read().strip()
+
+FORMATS = {
+ "word_problem": "a word problem that ends with a question, then a step-by-step solution",
+ "textbook_example": "a textbook-style worked example: state the problem as a question, then explain each step as a teacher would",
+ "dialogue": "a short dialogue in which one person asks the question and another works it out aloud",
+ "find_the_mistake": "a question, a student's incorrect attempt, and a correction that explains the error and reaches the right answer",
+ "concept_then_example": "two or three sentences explaining the math idea, then an example question and its solution",
+ "estimate_then_solve": "a question, a quick estimate, then the exact solution and a check that it is reasonable",
+ "two_methods": "a question solved two different ways that reach the same answer",
+ "real_world_planning": "a planning question someone faces in daily life, solved step by step",
+}
+SETTINGS = ("bakery|soccer tournament|public library|aquarium|science fair|road trip|community garden|recycling drive|school band|"
+ "train station|camping trip|school store|wildlife park|robotics club|dairy farm|history museum|theater rehearsal|"
+ "bicycle shop|kitchen|craft fair|weather station|space camp|swimming meet|bookbinding workshop|food bank|ferry|"
+ "orchard|chess club|pottery studio|hiking trail|bus route|lemonade stand|carpentry project|marathon|zoo nursery|"
+ "art class|recording studio|coral reef survey|ski lodge|post office|pet shelter|harbor|tea shop|rugby practice|"
+ "greenhouse|bird count|dance recital|basketball league|volcano exhibit|street market|ice rink|bowling alley|"
+ "beekeeping|newspaper delivery|chemistry lab|planetarium|tailor shop|fishing trip|quilting circle|apple cider mill|"
+ "school election|cricket match|map making|kite festival|summer reading program|hospital supply room|airport|"
+ "film club|vegetable stand|furniture store|scout troop|tree planting|cooking contest|science museum").split("|")
+NAMES = ("Aarav Adaeze Aiko Akira Alejandro Amara Anaya Andrei Annika Arjun Astrid Ayaan Aziz Bea Benedikt Bianca Bilal "
+ "Carmen Chidi Chloe Darius Dmitri Elif Emeka Esme Farah Fatima Felix Freya Gabriel Gustavo Hana Hiroshi Ines Ingrid "
+ "Isaac Jamal Jasper Javier Joaquin Jonah Kai Kalani Keanu Kenji Kofi Laila Lars Leila Lorenzo Lucia Magnus Malik "
+ "Marisol Matteo Miguel Mila Minh Nadia Naomi Niamh Nikhil Oluwaseun Oscar Pablo Paloma Quentin Rafael Rania Rohan "
+ "Rosa Saanvi Santiago Selin Seo-yeon Siddharth Silas Siobhan Sunita Tariq Teodor Thandiwe Tomas Valentina Wren "
+ "Xiomara Yara Yusuf Zainab Zara Ziad").split()
+
+
+def norm_words(s): return re.sub(r"[^0-9a-z]+", " ", s.lower()).split()
+
+
+def ngrams(words, n): return {" ".join(words[i:i + n]) for i in range(len(words) - n + 1)}
+
+
+def load_bans():
+ am = [json.loads(l) for l in open(Path.home() / ".cache/huggingface/hub/datasets--AxiomicLabs--ArithMark-3.0/snapshots/6f6e59dd9b7e2c63455f7af7f838f9ecc3d0a746/arithmark-3.jsonl")]
+ am_names = {re.match(r"[A-Z][a-z]+", r["ctx"]).group(0) for r in am if re.match(r"[A-Z][a-z]+", r["ctx"])}
+ am_names -= {"At", "In", "The", "A", "An", "On", "For", "During", "After", "Each", "Every", "Last"}
+ texts = [r["ctx"] for r in am]
+ from datasets import load_dataset
+ texts += [r["Body"] + " " + r["Question"] for r in load_dataset("ChilleD/SVAMP", split="test")]
+ texts += [r["body"] + " " + r["question"] for r in load_dataset("EleutherAI/asdiv", split="validation")]
+ ban8 = set().union(*(ngrams(norm_words(t), 8) for t in texts))
+ import prepare
+ return am_names, ban8, prepare.eval_ngrams(lambda *a: None), prepare.contaminated
+
+
+def same_answer(got, want):
+ """Compare the model's final answer with the computed one, tolerating units, $ signs,
+ commas and words around the number ("56 meters", "$11.71", "3/4 of the pizza")."""
+ from fractions import Fraction
+ g, w = str(got).strip(), str(want)
+ if re.fullmatch(r"\d{1,2}:\d{2} [AP]M", w):
+ m = re.search(r"(\d{1,2}:\d{2})\s*([AaPp])\.?\s*[Mm]", g)
+ return bool(m) and f"{m.group(1)} {m.group(2).upper()}M" == w
+ g = g.replace(",", "")
+ if "/" in w:
+ m = re.findall(r"-?\d+\s*/\s*\d+", g)
+ try: return bool(m) and Fraction(m[-1].replace(" ", "")) == Fraction(w)
+ except Exception: return False
+ nums = re.findall(r"-?\d+(?:\.\d+)?", g)
+ try: return bool(nums) and abs(float(nums[-1]) - float(w)) < 1e-9
+ except Exception: return False
+
+
+def build_prompt(batch, rng):
+ """batch: list of (spec_a, spec_b). Each item is one scenario with two questions."""
+ lines = []
+ for i, (sa, sb) in enumerate(batch):
+ fmt = rng.choice(list(FORMATS)); setting = rng.choice(SETTINGS); names = rng.sample(NAMES, 2)
+ for x in (sa, sb): x["format"], x["setting"], x["names"] = fmt, setting, names
+ lines.append(f'{i}. format: {FORMATS[fmt]}. Setting: {setting}. People (use only if needed): {names[0]}, {names[1]}.\n'
+ f' Question A: grade {sa["grade"]}, topic "{sa["topic"]}", facts: {sa["facts"]}. Correct answer: {sa["answer"]}.\n'
+ f' Question B: grade {sb["grade"]}, topic "{sb["topic"]}", facts: {sb["facts"]}. Correct answer: {sb["answer"]}.')
+ return ("You are writing items for an open educational math dataset. For each spec below, write one original item: a short "
+ "scenario in the given setting followed by two questions, (a) and (b), that use the given facts. Vary wording, sentence "
+ "structure and length widely between items. Each question must be a direct question ending with a question mark, and the "
+ "item's text must end with question (b). Use realistic quantities and units. The solution must work through (a) and then "
+ "(b) step by step, explaining why each step is done, and finish each part with a one-sentence check that the answer is "
+ "reasonable. Answers must equal the given correct answers. Write plain text only: no markdown, bold, headings or bullet symbols.\n\n" + "\n".join(lines) +
+ '\n\nReply with only a JSON array, one object per spec in order: '
+ '[{"id": 0, "problem": "...", "solution": "...", "answer_a": "...", "answer_b": "..."}, ...]')
+
+
+class Gen:
+ def __init__(a, out, args):
+ a.out, a.args = out, args; a.stats = {"requests": 0, "http_errors": 0, "kept": 0, "tokens_out": 0,
+ "reject": {}}
+ a.am_names, a.ban8, a.ban13, a.contaminated = load_bans()
+ a.seen = set(); a.buf = []; a.shard = len(list(out.glob("items-*.jsonl")))
+ a.rng = random.Random(args.seed + 1_000_003 * a.shard)
+
+ def rej(a, why): a.stats["reject"][why] = a.stats["reject"].get(why, 0) + 1
+
+ def answer_of(a, it, keys, sol):
+ fa = next((it[k] for k in keys if it.get(k) not in (None, "")), None)
+ return fa
+
+ def check(a, pair, it):
+ s, s2 = pair
+ p, sol = (re.sub(r"\*\*|__|^#+\s*", "", str(it.get(k, "")), flags=re.M).strip() for k in ("problem", "solution"))
+ if not p or not sol: return a.rej("empty")
+ fa = a.answer_of(it, ("answer_a", "a", "final_answer_a"), sol)
+ fb = a.answer_of(it, ("answer_b", "b", "final_answer_b"), sol)
+ if fa is None or fb is None: return a.rej("missing_answer")
+ if not same_answer(fb, s2["answer"]): return a.rej("wrong_answer")
+ if not same_answer(fa, s["answer"]):
+ if a.stats["reject"].get("wrong_answer", 0) < 200:
+ with open(a.out / "rejects_wrong_answer.jsonl", "a") as f:
+ f.write(json.dumps({"spec": {k: s[k] for k in ("topic", "facts", "answer")}, "got": fa,
+ "problem": p[:300]}) + "\n")
+ return a.rej("wrong_answer")
+ if not p.rstrip().endswith("?"): return a.rej("not_a_question")
+ if s.get("format") != "find_the_mistake" and bad_steps(sol): return a.rej("bad_arithmetic_step")
+ text = p + "\n" + sol; words = norm_words(text)
+ if set(re.findall(r"[A-Z][a-z]+", text)) & a.am_names: return a.rej("arithmark_name")
+ if ngrams(words, 8) & a.ban8: return a.rej("test_overlap_8gram")
+ if a.contaminated(text, a.ban13): return a.rej("eval_13gram")
+ h = hashlib.sha1(" ".join(norm_words(p)).encode()).hexdigest()
+ if h in a.seen: return a.rej("duplicate")
+ a.seen.add(h)
+ return {"problem": p, "solution": sol, "answers": [s["answer"], s2["answer"]], "grade": max(s["grade"], s2["grade"]),
+ "topics": [s["topic"], s2["topic"]], "format": s["format"], "setting": s["setting"], "generator": MODEL}
+
+ def flush(a, force=False):
+ while len(a.buf) >= a.args.shard_size or (force and a.buf):
+ chunk, a.buf = a.buf[:a.args.shard_size], a.buf[a.args.shard_size:]
+ with open(a.out / f"items-{a.shard:05d}.jsonl", "w") as f:
+ for it in chunk: f.write(json.dumps(it) + "\n")
+ a.shard += 1
+ (a.out / "stats.json").write_text(json.dumps(a.stats, indent=2))
+
+ async def pace(a):
+ while getattr(a, "paused_until", 0) > time.time():
+ a.stats["paused_until"] = time.strftime("%a %I:%M %p", time.localtime(a.paused_until)); a.flush()
+ await asyncio.sleep(min(300, a.paused_until - time.time()))
+ a.stats.pop("paused_until", None)
+ async with a.lock:
+ now = time.time(); wait = a.next_slot - now
+ if wait > 0: await asyncio.sleep(wait)
+ a.next_slot = max(now, a.next_slot) + 1 / a.args.rps
+
+ async def _one(a, sess, sem):
+ batch = [specs.sample_pair(a.rng) for _ in range(a.args.per_request)]
+ body = {"model": MODEL, "temperature": 1.0, "top_p": 0.95, "max_tokens": 32000,
+ **({"chat_template_kwargs": TEMPLATE_KW} if TEMPLATE_KW else {}),
+ "messages": [{"role": "user", "content": build_prompt(batch, a.rng)}]}
+ for attempt in range(10):
+ await a.pace()
+ try:
+ async with sess.post(API, json=body, headers={"Authorization": f"Bearer {KEY}"}) as r:
+ if r.status == 429 and "quota" in (await r.text()).lower():
+ ra = r.headers.get("Retry-After")
+ a.paused_until = max(getattr(a, "paused_until", 0), time.time() + min(1800, float(ra) if ra and ra.replace(".", "").isdigit() else 1800))
+ a.stats["quota_pauses"] = a.stats.get("quota_pauses", 0) + 1
+ continue
+ if r.status == 429 or r.status >= 500:
+ ra = r.headers.get("Retry-After")
+ a.next_slot = max(a.next_slot, time.time() + (float(ra) if ra and ra.replace(".", "").isdigit() else 1.0))
+ a.stats["retries"] = a.stats.get("retries", 0) + 1
+ await asyncio.sleep(min(30, 2 ** attempt)); continue
+ d = await r.json()
+ break
+ except Exception:
+ await asyncio.sleep(2 ** attempt)
+ else:
+ a.stats["http_errors"] += 1; return
+ a.stats["requests"] += 1
+ a.stats["tokens_out"] += d.get("usage", {}).get("completion_tokens", 0)
+ txt = ((d.get("choices") or [{}])[0].get("message") or {}).get("content") or ""
+ m = re.search(r"\[.*\]", txt, re.S)
+ try: items = json.loads(m.group(0)) if m else []
+ except Exception: items = []
+ if not items: return a.rej("bad_json")
+ for it in items:
+ try: s = batch[int(it.get("id"))]
+ except Exception: a.rej("bad_id"); continue
+ k = a.check(s, it)
+ if k: a.buf.append(k); a.stats["kept"] += 1
+ a.flush()
+
+ async def one(a, sess, sem):
+ try:
+ await a._one(sess, sem)
+ except Exception as e:
+ a.stats["worker_errors"] = a.stats.get("worker_errors", 0) + 1
+ a.stats["last_worker_error"] = repr(e)[:200]
+ finally:
+ sem.release()
+
+ async def run(a):
+ sem = asyncio.Semaphore(a.args.inflight); tasks = set(); t0 = time.time()
+ a.lock = asyncio.Lock(); a.next_slot = time.time()
+ async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=600)) as sess:
+ while a.stats["kept"] < a.args.items:
+ await sem.acquire()
+ while a.next_slot - time.time() > 2: await asyncio.sleep(0.2) # do not queue far ahead of the pacer
+ t = asyncio.create_task(a.one(sess, sem)); tasks.add(t); t.add_done_callback(tasks.discard)
+ if a.stats["requests"] and a.stats["requests"] % 200 == 0:
+ el = time.time() - t0
+ print(f"{time.strftime('%I:%M %p')} req {a.stats['requests']} kept {a.stats['kept']:,} "
+ f"out-tok {a.stats['tokens_out']/1e6:.1f}M ({a.stats['tokens_out']/el:.0f}/s) rejects {a.stats['reject']}", flush=True)
+ await asyncio.gather(*tasks)
+ a.flush(force=True)
+
+
+def main():
+ ap = argparse.ArgumentParser(); ap.add_argument("out"); ap.add_argument("--items", type=int, default=1000)
+ ap.add_argument("--per-request", type=int, default=20); ap.add_argument("--rps", type=float, default=2.8)
+ ap.add_argument("--inflight", type=int, default=96); ap.add_argument("--seed", type=int, default=20261015)
+ ap.add_argument("--shard-size", type=int, default=50000)
+ args = ap.parse_args(); out = Path(args.out); out.mkdir(parents=True, exist_ok=True)
+ asyncio.run(Gen(out, args).run())
+
+
+if __name__ == "__main__":
+ main()
diff --git a/libremath/review_fragments.py b/libremath/review_fragments.py
new file mode 100644
index 0000000..4352c3a
--- /dev/null
+++ b/libremath/review_fragments.py
@@ -0,0 +1,68 @@
+"""One-time K2 review of the fragment bank (2026-10-23): keep only fragments that read naturally
+for ANY items/containers the composer may plug in, and object/container pairs that make sense.
+
+ python libremath/review_fragments.py BANK.json REVIEWED.json
+"""
+import asyncio, json, re, sys, time
+from pathlib import Path
+import aiohttp
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+import generate as G
+from fragments import CATS
+
+RULE = ("A program fills the curly-brace placeholders: {A}/{B} are people's names, {obj} is ANY plural countable item "
+ "(e.g. marbles, cookies, stickers), {cont} is ANY plural container or group (e.g. jars, boxes, rows), {place} is a "
+ "location, and letters like {a} {b} {c} {d} {x} {p} {ans} are numbers.")
+
+
+def ask(cat, frags):
+ if cat == "objects":
+ body = "\n".join(f"{i}. {f['obj']} / {f['cont']}" for i, f in enumerate(frags))
+ q = ("Each line is a plural countable item and a plural container or grouping for a grade-school math problem "
+ "(\"there are N <containers> with M <items> in each\"). Mark a line bad if the item would not sensibly be packed, "
+ "arranged or grouped in that container, or if either word is not a plural noun.")
+ else:
+ body = "\n".join(f"{i}. {f}" for i, f in enumerate(frags))
+ q = (f"{RULE} Each line is meant to be: {CATS[cat][0]}. Mark a line bad if it names or implies specific items, "
+ "food, animals, actions or imagery that would clash with arbitrary {obj}/{cont}, if it would be ungrammatical "
+ "with a plural {obj} or {cont}, if its math meaning is wrong for the description, or if it is awkward or confusing.")
+ return q + "\n\n" + body + "\n\nReply with only a JSON array of the line numbers that are bad."
+
+
+async def main_async(src, dst):
+ bank = json.load(open(src)); out = {k: [] for k in bank}; sem = asyncio.Semaphore(16); lock = asyncio.Lock(); nxt = [time.time()]
+ jobs = [(cat, bank[cat][i:i + 40]) for cat in bank for i in range(0, len(bank[cat]), 40)]
+
+ async def run(sess, cat, frags):
+ async with sem:
+ body = {"model": G.MODEL, "temperature": 0.2, "max_tokens": 8000,
+ **({"chat_template_kwargs": G.TEMPLATE_KW} if G.TEMPLATE_KW else {}),
+ "messages": [{"role": "user", "content": ask(cat, frags)}]}
+ for attempt in range(8):
+ async with lock:
+ w = nxt[0] - time.time()
+ if w > 0: await asyncio.sleep(w)
+ nxt[0] = max(time.time(), nxt[0]) + 1.0
+ try:
+ async with sess.post(G.API, json=body, headers={"Authorization": f"Bearer {G.KEY}"}) as r:
+ if r.status in (429,) or r.status >= 500: await asyncio.sleep(10 * (attempt + 1)); continue
+ d = await r.json(); break
+ except Exception:
+ await asyncio.sleep(5 * (attempt + 1))
+ else:
+ return cat, [], len(frags)
+ txt = ((d.get("choices") or [{}])[0].get("message") or {}).get("content") or ""
+ m = re.search(r"\[[\d,\s]*\]", txt)
+ if not m: return cat, [], len(frags) # unparseable review: drop the batch (be strict)
+ bad = set(json.loads(m.group(0)))
+ return cat, [f for i, f in enumerate(frags) if i not in bad], len(frags)
+
+ async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=600)) as sess:
+ for fut in asyncio.as_completed([run(sess, c, f) for c, f in jobs]):
+ cat, keep, n = await fut; out[cat] += keep
+ json.dump(out, open(dst, "w"), indent=1)
+ print({k: f"{len(out[k])}/{len(bank[k])}" for k in bank})
+
+
+if __name__ == "__main__":
+ asyncio.run(main_async(sys.argv[1], sys.argv[2]))
diff --git a/libremath/specs.py b/libremath/specs.py
new file mode 100644
index 0000000..e68dce9
--- /dev/null
+++ b/libremath/specs.py
@@ -0,0 +1,204 @@
+"""LibreMath problem specs: curriculum topic, grade, the facts a problem must use, and the
+exact answer, all computed here. The language model only writes the problem and solution
+around these facts; anything it gets wrong is discarded downstream.
+
+Covers grades 1-6: whole-number operations at several scales, place value, rounding,
+comparison, money, elapsed time, measurement and conversion, perimeter/area/volume,
+fractions, decimals, data (mean/range/tables), factors and multiples, order of operations,
+patterns, ratios, percents and multi-step problems with irrelevant information.
+"""
+import math, random
+from fractions import Fraction
+
+R = random.Random()
+
+
+def _r(a, b): return R.randint(a, b)
+
+
+def _t(h, m): # minutes since midnight -> "h:mm AM/PM"
+ h, m = divmod(h * 60 + m, 60)
+ h %= 24
+ return f"{(h % 12) or 12}:{m:02d} {'AM' if h < 12 else 'PM'}"
+
+
+def add_sub():
+ scale = R.choice([20, 100, 1000, 10000])
+ a, b = _r(scale // 10, scale), _r(1, scale)
+ if R.random() < 0.5:
+ return 2 if scale <= 100 else 3, "addition", f"{a} and {b} are combined", a + b
+ a, b = max(a, b), min(a, b)
+ return 2 if scale <= 100 else 3, "subtraction", f"start with {a}, remove or compare {b}", a - b
+
+
+def mult():
+ a, b = R.choice([(_r(2, 12), _r(2, 12)), (_r(11, 99), _r(2, 9)), (_r(12, 99), _r(11, 40))])
+ return 3 if max(a, b) <= 12 else 4, "multiplication", f"{a} groups of {b}", a * b
+
+
+def div():
+ b = _r(2, 12); q = _r(2, 30 if R.random() < 0.7 else 120); rem = _r(0, b - 1) if R.random() < 0.3 else 0
+ a = b * q + rem
+ if rem:
+ kind = R.choice(["quotient", "remainder", "round_up"])
+ ans = {"quotient": q, "remainder": rem, "round_up": q + 1}[kind]
+ how = {"quotient": "how many full groups", "remainder": "how many are left over",
+ "round_up": "how many groups are needed so none are left out"}[kind]
+ return 4, "division with remainder", f"{a} items split into groups of {b}; ask {how}", ans
+ return 3, "division", f"{a} items shared equally into {b} groups", q
+
+
+def place_value():
+ n = _r(1000, 999999); d = R.choice(["thousands", "hundreds", "tens"])
+ v = {"thousands": 1000, "hundreds": 100, "tens": 10}[d]
+ if R.random() < 0.5:
+ return 4, "rounding", f"round {n} to the nearest {d[:-1]}", int(math.floor(n / v + 0.5) * v)
+ return 4, "place value", f"the value of the {d} digit in {n}", (n // v % 10) * v
+
+
+def money():
+ items = _r(2, 4); prices = [Fraction(_r(25, 1999), 100) for _ in range(items)]
+ paid = Fraction(math.ceil(float(sum(prices))) + R.choice([0, 1, 5, 10, 20]), 1)
+ if R.random() < 0.5:
+ return 3, "money", "items costing " + ", ".join(f"${float(p):.2f}" for p in prices) + " are bought; find the total", f"{float(sum(prices)):.2f}"
+ return 4, "making change", "items costing " + ", ".join(f"${float(p):.2f}" for p in prices) + f" are paid for with ${float(paid):.2f}; find the change", f"{float(paid - sum(prices)):.2f}"
+
+
+def elapsed_time():
+ h, m = _r(6, 18), R.choice(range(0, 60, 5)); d = R.choice([15, 20, 25, 35, 40, 45, 50, 70, 85, 95, 110, 135])
+ if R.random() < 0.5:
+ return 3, "elapsed time", f"starts at {_t(h, m)} and lasts {d} minutes; find the end time", _t(h, m + d)
+ return 3, "elapsed time", f"starts at {_t(h, m)} and ends at {_t(h, m + d)}; find how many minutes", d
+
+
+def measurement():
+ kind = R.choice(["m_cm", "kg_g", "l_ml", "ft_in", "hr_min", "yd_ft"])
+ big, small, k = {"m_cm": ("meters", "centimeters", 100), "kg_g": ("kilograms", "grams", 1000),
+ "l_ml": ("liters", "milliliters", 1000), "ft_in": ("feet", "inches", 12),
+ "hr_min": ("hours", "minutes", 60), "yd_ft": ("yards", "feet", 3)}[kind]
+ a, b = _r(1, 12), _r(1, k - 1)
+ return 4, "unit conversion", f"{a} {big} and {b} {small}, expressed in {small}", a * k + b
+
+
+def geometry():
+ kind = R.choice(["perim_rect", "area_rect", "area_l", "volume", "perim_poly", "missing_side"])
+ l, w, h = _r(2, 30), _r(2, 20), _r(2, 12)
+ if kind == "perim_rect": return 3, "perimeter", f"a rectangle {l} by {w}", 2 * (l + w)
+ if kind == "area_rect": return 3, "area", f"a rectangle {l} by {w}", l * w
+ if kind == "volume": return 5, "volume", f"a box {l} by {w} by {h}", l * w * h
+ if kind == "perim_poly":
+ s = [_r(3, 25) for _ in range(_r(3, 6))]
+ return 3, "perimeter", "a shape with sides " + ", ".join(map(str, s)), sum(s)
+ if kind == "missing_side":
+ return 4, "area", f"a rectangle with area {l * w} and one side {l}; find the other side", w
+ a, b = _r(2, 10), _r(2, 10)
+ return 4, "area", f"an L-shape made of a {l} by {w} rectangle and a {a} by {b} rectangle", l * w + a * b
+
+
+def fractions():
+ kind = R.choice(["of_whole", "add_like", "sub_like", "compare", "equiv"])
+ d = R.choice([2, 3, 4, 5, 6, 8, 10, 12])
+ if kind == "of_whole":
+ n = _r(1, d - 1); k = _r(2, 12) * d
+ return 4, "fraction of a number", f"{n}/{d} of {k}", n * k // d
+ a, b = _r(1, d - 1), _r(1, d - 1)
+ if kind == "add_like":
+ s = Fraction(a + b, d); return 4, "adding fractions", f"{a}/{d} plus {b}/{d}, simplified", str(s)
+ if kind == "sub_like":
+ a, b = max(a, b), min(a, b); s = Fraction(a - b, d)
+ return 4, "subtracting fractions", f"{a}/{d} minus {b}/{d}, simplified", str(s)
+ if kind == "equiv":
+ k = _r(2, 6); return 4, "equivalent fractions", f"{a}/{d} written with denominator {d * k}; find the numerator", a * k
+ d2 = R.choice([2, 3, 4, 5, 6, 8]); c = _r(1, d2 - 1)
+ if Fraction(a, d) == Fraction(c, d2): c = (c % (d2 - 1)) + 1
+ big = f"{a}/{d}" if Fraction(a, d) > Fraction(c, d2) else f"{c}/{d2}"
+ return 4, "comparing fractions", f"which is larger, {a}/{d} or {c}/{d2}", big
+
+
+def decimals():
+ a, b = Fraction(_r(10, 9999), 100), Fraction(_r(10, 9999), 100)
+ if R.random() < 0.5: return 5, "decimal addition", f"{float(a):.2f} and {float(b):.2f} combined", f"{float(a + b):.2f}"
+ a, b = max(a, b), min(a, b); return 5, "decimal subtraction", f"{float(a):.2f} minus {float(b):.2f}", f"{float(a - b):.2f}"
+
+
+def data():
+ k = _r(4, 7); xs = [_r(1, 60) for _ in range(k)]
+ kind = R.choice(["mean", "range", "total", "diff"])
+ if kind == "mean":
+ xs[-1] += (-sum(xs)) % k
+ return 5, "mean", "values " + ", ".join(map(str, xs)), sum(xs) // k
+ if kind == "range": return 5, "range", "values " + ", ".join(map(str, xs)), max(xs) - min(xs)
+ if kind == "total": return 3, "reading a table", "a table with values " + ", ".join(map(str, xs)) + "; find the total", sum(xs)
+ return 3, "reading a table", "a table with values " + ", ".join(map(str, xs)) + "; find the difference between the largest and smallest category", max(xs) - min(xs)
+
+
+def factors():
+ kind = R.choice(["lcm", "gcf", "count_factors", "multiple"])
+ a, b = _r(2, 15), _r(2, 15)
+ if kind == "lcm": return 5, "least common multiple", f"two things repeat every {a} and every {b}; when do they first match", a * b // math.gcd(a, b)
+ if kind == "gcf":
+ a, b = a * _r(1, 6), b * _r(1, 6); return 5, "greatest common factor", f"{a} and {b} split into the largest equal groups", math.gcd(a, b)
+ if kind == "count_factors":
+ n = _r(6, 100); return 4, "factors", f"how many factors {n} has", sum(1 for i in range(1, n + 1) if n % i == 0)
+ n, k = _r(3, 12), _r(4, 15); return 3, "multiples", f"the {k}th multiple of {n}", n * k
+
+
+def order_ops():
+ a, b, c, d = _r(2, 20), _r(2, 12), _r(2, 12), _r(1, 30)
+ expr = R.choice([f"{a} + {b} × {c}", f"({a} + {b}) × {c}", f"{a} × {b} - {d}", f"{a * b} ÷ {b} + {c}", f"{a} + {b} × {c} - {d}"])
+ val = eval(expr.replace("×", "*").replace("÷", "//"))
+ return 5, "order of operations", f"the expression {expr}", val
+
+
+def patterns():
+ s, d, n = _r(1, 80), R.choice([2, 3, 4, 5, 6, 7, 9, 11, 12, 15, 25]), _r(5, 10)
+ if R.random() < 0.3: d = -d; s += abs(d) * (n + 2)
+ seq = [s + d * i for i in range(4)]
+ return 3, "number patterns", "the sequence " + ", ".join(map(str, seq)) + f"; find term number {n}", s + d * (n - 1)
+
+
+def ratio_percent():
+ kind = R.choice(["ratio", "percent", "unit_rate", "discount"])
+ if kind == "ratio":
+ a, b, k = _r(1, 9), _r(1, 9), _r(2, 12); return 6, "ratios", f"a {a}:{b} ratio, with {a * k} of the first kind; find the second", b * k
+ if kind == "percent":
+ p = R.choice([10, 20, 25, 30, 40, 50, 60, 75, 80]); n = R.choice([20, 40, 60, 80, 120, 200, 240, 400])
+ return 6, "percent of a number", f"{p}% of {n}", p * n // 100
+ if kind == "unit_rate":
+ r, t = _r(2, 15), _r(2, 12); return 6, "unit rate", f"{r * t} done in {t} equal parts; find the amount per part", r
+ p = R.choice([10, 20, 25, 50]); price = R.choice([20, 40, 60, 80, 100, 120])
+ return 6, "discounts", f"a ${price} item with {p}% off; find the sale price", price - price * p // 100
+
+
+def multi_step():
+ a, b, c, d = _r(3, 12), _r(4, 25), _r(5, 60), _r(2, 9)
+ form = R.choice([(f"{a} packs of {b}, then {c} more are added", a * b + c),
+ (f"{a} packs of {b}, then {c} are given away", a * b - c if a * b > c else a * b + c),
+ (f"{a * d} items split into {d} equal groups, then each group gets {b} more", a + b),
+ (f"saving ${b} a week for {a} weeks, then spending ${c}", a * b - c if a * b > c else a * b + c)])
+ return R.choice([3, 4, 5]), "multi-step", form[0] + "; include one irrelevant number", form[1]
+
+
+TOPICS = [(add_sub, 14), (mult, 9), (div, 8), (place_value, 4), (money, 7), (elapsed_time, 5), (measurement, 6),
+ (geometry, 8), (fractions, 8), (decimals, 4), (data, 5), (factors, 5), (order_ops, 3), (patterns, 4),
+ (ratio_percent, 5), (multi_step, 9)]
+
+
+def sample(rng=None):
+ global R
+ if rng is not None: R = rng
+ fn = R.choices([t for t, _ in TOPICS], weights=[w for _, w in TOPICS])[0]
+ grade, topic, facts, ans = fn()
+ return {"grade": grade, "topic": topic, "facts": facts, "answer": str(ans)}
+
+
+def sample_pair(rng=None):
+ """Two specs from the same topic family, so one scenario can hold both questions."""
+ global R
+ if rng is not None: R = rng
+ fn = R.choices([t for t, _ in TOPICS], weights=[w for _, w in TOPICS])[0]
+ out = []
+ for _ in range(2):
+ grade, topic, facts, ans = fn()
+ out.append({"grade": grade, "topic": topic, "facts": facts, "answer": str(ans)})
+ return tuple(out)
diff --git a/libremath/tgen.py b/libremath/tgen.py
new file mode 100644
index 0000000..2bd7975
--- /dev/null
+++ b/libremath/tgen.py
@@ -0,0 +1,261 @@
+"""LibreMath template generator (prototype, 2026-10-15).
+
+K2 writes problem TEMPLATES rather than problems: a two-part scenario and worked solution
+with {placeholders}, integer ranges for each variable, formulas for every derived value,
+constraints the numbers must satisfy, and one worked example. Code then
+ 1. checks the formulas reproduce the example's answers (catches wrong math in the template),
+ 2. fills the template --fills times with fresh numbers and names, computing every value,
+ 3. runs each filled item through the same filters as generate.py.
+
+ python libremath/tgen.py OUT_DIR --templates 200 [--per-request 30] [--fills 10]
+"""
+import argparse, ast, asyncio, hashlib, json, math, operator, random, re, sys, time
+from pathlib import Path
+import aiohttp
+
+ROOT = Path(__file__).resolve().parent
+sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT.parent))
+import generate as G
+from arithcheck import bad_steps
+
+TOPICS = ["addition with regrouping", "subtraction with borrowing", "multiplication", "division with and without remainders",
+ "money and making change", "elapsed time in minutes", "unit conversion", "perimeter", "area", "volume",
+ "fractions of a quantity", "comparing and ordering numbers", "place value and rounding", "mean and range of data",
+ "factors and multiples", "number patterns", "ratios and rates", "percents and discounts", "multi-step problems",
+ "problems with extra information that is not needed"]
+
+SETTINGS_EXTRA = ["lighthouse keeper's log", "food truck", "bike repair shop", "bird-watching trip", "school band rehearsal", "farmers' co-op",
+ "ice rink", "pottery class", "train station", "chess club", "dog shelter", "beekeeping farm", "tailor's shop", "rock-climbing gym",
+ "community radio station", "ferry crossing", "orchard harvest", "board-game café", "museum gift shop", "volunteer fire station",
+ "robotics team", "knitting circle", "camping weekend", "bowling league", "hardware store", "space camp", "tide pool walk",
+ "mail sorting room", "theater set building", "kite festival", "snow-day sledding hill", "zoo feeding schedule", "puppet show",
+ "bus route planning", "recipe test kitchen", "weather station", "carpentry workshop", "town election count", "ballet recital",
+ "fishing pier", "recycling plant tour", "swim meet", "solar panel installation", "art gallery", "vegetable stand", "map-making club",
+ "quilt fair", "laundromat", "apartment move", "family reunion", "neighborhood cleanup", "printing press", "coffee roastery",
+ "veterinary clinic", "skate park", "sailing lesson", "film club", "bakery delivery route", "science museum planetarium"]
+OPENINGS = ["start with a person's name", "start with a time or day", "start with a short quotation from someone", "start with a fact about the setting",
+ "start with an instruction to the reader", "start with a small table or list of numbers", "start with a question", "start in the middle of an action"]
+NQ = [1, 1, 1, 2, 2, 3] # questions per item: half single-question (like most word-problem sets), some 2- and 3-part
+
+
+# --- safe arithmetic --------------------------------------------------------- #
+OPS = {ast.Add: operator.add, ast.Sub: operator.sub, ast.Mult: operator.mul, ast.Div: operator.truediv,
+ ast.FloorDiv: operator.floordiv, ast.Mod: operator.mod}
+CMP = {ast.Eq: operator.eq, ast.NotEq: operator.ne, ast.Lt: operator.lt, ast.LtE: operator.le,
+ ast.Gt: operator.gt, ast.GtE: operator.ge}
+def _round(x, n=0): # half-up, as taught in school (Python's round() is banker's rounding: round(25, -1) == 20)
+ v = math.floor(x * 10 ** n + 0.5) / 10 ** n
+ return int(v) if n <= 0 else v
+
+
+FUNCS = {"min": min, "max": max, "round": _round, "abs": abs}
+
+
+def ev(expr, env):
+ def go(n):
+ if isinstance(n, ast.Expression): return go(n.body)
+ if isinstance(n, ast.Constant) and isinstance(n.value, (int, float)): return n.value
+ if isinstance(n, ast.Name): return env[n.id]
+ if isinstance(n, ast.BinOp) and type(n.op) in OPS: return OPS[type(n.op)](go(n.left), go(n.right))
+ if isinstance(n, ast.UnaryOp) and isinstance(n.op, ast.USub): return -go(n.operand)
+ if isinstance(n, ast.Compare):
+ l = go(n.left)
+ for op, c in zip(n.ops, n.comparators):
+ r = go(c)
+ if not CMP[type(op)](l, r): return False
+ l = r
+ return True
+ if isinstance(n, ast.BoolOp):
+ vals = [go(v) for v in n.values]
+ return all(vals) if isinstance(n.op, ast.And) else any(vals)
+ if isinstance(n, ast.Call) and isinstance(n.func, ast.Name) and n.func.id in FUNCS:
+ return FUNCS[n.func.id](*[go(a) for a in n.args])
+ raise ValueError(f"disallowed: {ast.dump(n)[:60]}")
+ expr = str(expr).replace("||", " or ").replace("&&", " and ")
+ return go(ast.parse(expr, mode="eval"))
+
+
+def fmt(v):
+ if isinstance(v, float):
+ if abs(v - round(v)) < 1e-9: return str(int(round(v)))
+ return f"{v:.2f}"
+ return str(v)
+
+
+PH = re.compile(r"\{(\w+)\}")
+
+
+def render(text, env): # noqa: E302
+ return PH.sub(lambda m: fmt(env[m.group(1)]) if m.group(1) in env else m.group(0), text)
+
+
+def sample_env(t, rng, tries=200):
+ for _ in range(tries):
+ env = {}
+ for k, (lo, hi) in t["variables"].items(): env[k] = rng.randint(int(lo), int(hi))
+ for k, f in t["derived"].items(): env[k] = ev(f, env)
+ if all(ev(c, env) for c in t.get("constraints", [])) and all(
+ (isinstance(env[k], int) or abs(env[k] * 100 - round(env[k] * 100)) < 1e-6) and env[k] >= 0 for k in t["derived"]):
+ return env
+ return None
+
+
+def build_prompt(specs):
+ lines = []
+ for i, s in enumerate(specs):
+ qs = "one question" if s["nq"] == 1 else f"{s['nq']} questions labelled " + ", ".join(f"({c})" for c in "abc"[:s["nq"]])
+ lines.append(f'{i}. grade {s["grade"]}, topic: {s["topic"]}, format: {G.FORMATS[s["format"]]}, setting: {s["setting"]}; '
+ f'{qs}; {s["opening"]}' + ("; end each part of the solution with a one-sentence check" if s["check"] else ""))
+ return ("You are writing reusable TEMPLATES for an open educational math dataset. For each spec, write one template: a short "
+ "scenario followed by the requested question(s), plus a worked step-by-step solution that explains each step. Every "
+ "number in the problem and solution must be a {placeholder}: input variables with an integer range, and derived values "
+ "defined by formulas over other placeholders using + - * / // % min max round. Never write a computed number literally. "
+ "Use {name1} and {name2} for people. You may also add word placeholders for objects or units, listing 4-6 interchangeable "
+ "options under \"words\" (e.g. {\"snack\": [\"muffins\", \"bagels\", \"scones\", \"rolls\"]}). The problem text must end "
+ "with the last question and a question mark. Plain text only, no markdown. Make every template different in wording, "
+ "sentence structure and length; do not reuse openings like \"At the\" or \"In the\".\n\n" + "\n".join(lines) +
+ '\n\nReply with only a JSON array, one object per spec in order:\n'
+ '[{"id": 0, "problem": "...{name1}...{a} {snack}...?", "solution": "...{a} x {b} = {total}...", '
+ '"variables": {"a": [2, 12], "b": [3, 9]}, "words": {"snack": ["muffins", "bagels", "scones", "rolls"]}, '
+ '"derived": {"total": "a*b"}, "constraints": ["total > 10"], "answers": ["total"], '
+ '"example": {"values": {"a": 4, "b": 6}, "answers": [24]}}, ...]')
+
+
+class TGen(G.Gen):
+ def __init__(a, out, args):
+ a.args = args; a.out = out
+ a.stats = {"requests": 0, "http_errors": 0, "templates_ok": 0, "templates_bad": 0, "kept": 0, "tokens_out": 0, "reject": {}}
+ a.am_names, a.ban8, a.ban13, a.contaminated = G.load_bans()
+ a.seen = set(); a.buf = []; a.shard = len(list(out.glob("items-*.jsonl")))
+ a.rng = random.Random(args.seed + 1_000_003 * a.shard)
+ a.tpl_out = open(out / "templates.jsonl", "a")
+
+ def check_template(a, t):
+ try:
+ for k in ("problem", "solution", "variables", "derived", "answers", "example"): t[k]
+ if not t["problem"].rstrip().endswith("?"): return a.rej("tpl_not_question")
+ names = set(PH.findall(t["problem"] + t["solution"])) - {"name1", "name2"}
+ if not names <= set(t["variables"]) | set(t["derived"]) | set(t.get("words") or {}): return a.rej("tpl_unknown_placeholder")
+ if re.search(r"(?<![{\w])\d{2,}(?![\w}])", re.sub(r"\{\w+\}", "", t["problem"])): return a.rej("tpl_literal_number")
+ nq = t["_nq"]
+ if nq > 1 and not all(f"({c})" in t["problem"] for c in "abc"[:nq]): return a.rej("tpl_missing_parts")
+ if len(t["answers"]) != nq or not all("{" + k + "}" in t["solution"] for k in t["answers"]): return a.rej("tpl_answer_not_in_solution")
+ words = t.get("words") or {}
+ if not isinstance(words, dict) or any(not isinstance(v, list) or len(v) < 2 for v in words.values()): return a.rej("tpl_bad_words")
+ consts = {str(n.value) for f in list(t["derived"].values()) + list(t.get("constraints", []))
+ for n in ast.walk(ast.parse(str(f).replace("||", " or ").replace("&&", " and "), mode="eval"))
+ if isinstance(n, ast.Constant) and isinstance(n.value, (int, float))}
+ ok = consts | {str(i) for i in range(13)} | {"24", "60", "100", "1000", "365", "52"}
+ sol_lits = re.findall(r"(?<![{\w.])\d+(?:\.\d+)?(?![\w}])", re.sub(r"\{\w+\}", "", t["solution"]))
+ if any(x not in ok for x in sol_lits): return a.rej("tpl_solution_literal")
+ txt = t["problem"] + " " + t["solution"]
+ if re.search(r"//|\*\*|\{\w+\}\s*%|\s%\s", txt): return a.rej("tpl_code_operator")
+ if "nearest {" in txt: return a.rej("tpl_round_to_variable")
+ if re.search(r"\{\w+\}\s*[-+*/x×]\s*\{\w+\}", t["problem"]) and "order of operations" not in t.get("_topic", ""):
+ return a.rej("tpl_expression_in_problem")
+ env = dict(t["example"]["values"])
+ for k, f in t["derived"].items(): env[k] = ev(f, env)
+ if not all(ev(c, env) for c in t.get("constraints", [])): return a.rej("tpl_example_violates_constraints")
+ got = [env[k] for k in t["answers"]]
+ want = t["example"]["answers"]
+ if len(got) != len(want) or any(abs(float(g) - float(w)) > 1e-6 for g, w in zip(got, want)): return a.rej("tpl_example_mismatch")
+ return True
+ except Exception:
+ return a.rej("tpl_invalid")
+
+ def fill(a, t, spec):
+ out = []
+ for _ in range(a.args.fills):
+ env = sample_env(t, a.rng)
+ if env is None: a.rej("fill_unsatisfiable"); break
+ n1, n2 = a.rng.sample(G.NAMES, 2); env.update(name1=n1, name2=n2)
+ for k, opts in (t.get("words") or {}).items():
+ if isinstance(opts, list) and opts: env[k] = a.rng.choice([str(o) for o in opts])
+ if len(t["answers"]) > 1 and len({fmt(env[k]) for k in t["answers"]}) < len(t["answers"]): a.rej("fill_same_answers"); continue
+ if any(env[k] in [env.get(v) for v in t["variables"]] for k in t["answers"]): a.rej("fill_answer_is_input"); continue
+ p, sol = (re.sub(r"(?<=\d) \* (?=\d)", " × ", render(t[k], env)) for k in ("problem", "solution"))
+ if PH.search(p + sol): a.rej("fill_leftover_placeholder"); continue
+ text = p + "\n" + sol; words = G.norm_words(text)
+ if bad_steps(sol) and spec["format"] != "find_the_mistake": a.rej("fill_bad_step"); continue
+ if set(re.findall(r"[A-Z][a-z]+", text)) & a.am_names: a.rej("arithmark_name"); continue
+ if G.ngrams(words, 8) & a.ban8: a.rej("test_overlap_8gram"); continue
+ if a.contaminated(text, a.ban13): a.rej("eval_13gram"); continue
+ h = hashlib.sha1(" ".join(G.norm_words(p)).encode()).hexdigest()
+ if h in a.seen: a.rej("duplicate"); continue
+ a.seen.add(h)
+ out.append({"problem": p, "solution": sol, "answers": [fmt(env[k]) for k in t["answers"]],
+ "grade": spec["grade"], "topics": [spec["topic"]], "format": spec["format"], "setting": spec["setting"],
+ "generator": G.MODEL + " template", "template_id": t["_tid"]})
+ return out
+
+ async def _one(a, sess, sem):
+ specs = [{"grade": a.rng.randint(1, 6), "topic": a.rng.choice(TOPICS), "format": a.rng.choice(list(G.FORMATS)),
+ "setting": a.rng.choice(G.SETTINGS + SETTINGS_EXTRA), "nq": a.rng.choice(NQ), "opening": a.rng.choice(OPENINGS),
+ "check": a.rng.random() < 0.3} for _ in range(a.args.per_request)]
+ body = {"model": G.MODEL, "temperature": 1.0, "top_p": 0.95, "max_tokens": 32000,
+ **({"chat_template_kwargs": G.TEMPLATE_KW} if G.TEMPLATE_KW else {}),
+ "messages": [{"role": "user", "content": build_prompt(specs)}]}
+ d = None
+ for attempt in range(10):
+ await a.pace()
+ try:
+ async with sess.post(G.API, json=body, headers={"Authorization": f"Bearer {G.KEY}"}) as r:
+ if r.status == 429 and "quota" in (await r.text()).lower():
+ ra = r.headers.get("Retry-After")
+ a.paused_until = time.time() + min(1800, float(ra) if ra and ra.replace(".", "").isdigit() else 1800); continue
+ if r.status == 429 or r.status >= 500:
+ a.next_slot = max(a.next_slot, time.time() + 1.0); await asyncio.sleep(min(30, 2 ** attempt)); continue
+ d = await r.json(); break
+ except Exception:
+ await asyncio.sleep(2 ** attempt)
+ if d is None: a.stats["http_errors"] += 1; return
+ a.stats["requests"] += 1; a.stats["tokens_out"] += d.get("usage", {}).get("completion_tokens", 0)
+ txt = ((d.get("choices") or [{}])[0].get("message") or {}).get("content") or ""
+ # Salvage every complete object even if the array is malformed or truncated.
+ dec, tpls, i = json.JSONDecoder(), [], 0
+ while (i := txt.find('{"id"', i)) != -1:
+ try: o, j = dec.raw_decode(txt, i); tpls.append(o); i = j
+ except Exception: i += 5
+ if not tpls:
+ if a.stats.get("raw_saved", 0) < 5:
+ a.stats["raw_saved"] = a.stats.get("raw_saved", 0) + 1
+ (a.out / f"raw_bad_{a.stats['raw_saved']}.txt").write_text(txt[:20000])
+ return a.rej("bad_json")
+ a.stats["objects_parsed"] = a.stats.get("objects_parsed", 0) + len(tpls)
+ for t in tpls:
+ try: spec = specs[int(t.get("id"))]
+ except Exception: a.rej("bad_id"); continue
+ t["_topic"] = spec["topic"]; t["_nq"] = spec["nq"]
+ if a.check_template(t) is not True: a.stats["templates_bad"] += 1; continue
+ t["_tid"] = hashlib.sha1(json.dumps(t, sort_keys=True).encode()).hexdigest()[:12]
+ try: items = a.fill(t, spec)
+ except Exception: a.rej("fill_error"); items = []
+ if items:
+ a.stats["templates_ok"] += 1; a.tpl_out.write(json.dumps({**t, "spec": spec}) + "\n"); a.tpl_out.flush()
+ a.buf.extend(items); a.stats["kept"] += len(items)
+ a.flush()
+
+ async def run(a):
+ sem = asyncio.Semaphore(a.args.inflight); tasks = set()
+ a.lock = asyncio.Lock(); a.next_slot = time.time()
+ async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=900)) as sess:
+ while a.stats["templates_ok"] < a.args.templates and a.stats["requests"] + len(tasks) < a.args.max_requests:
+ await sem.acquire()
+ while a.next_slot - time.time() > 2: await asyncio.sleep(0.2)
+ t = asyncio.create_task(a.one(sess, sem)); tasks.add(t); t.add_done_callback(tasks.discard)
+ await asyncio.gather(*tasks)
+ a.flush(force=True)
+
+
+def main():
+ ap = argparse.ArgumentParser(); ap.add_argument("out"); ap.add_argument("--templates", type=int, default=200)
+ ap.add_argument("--per-request", type=int, default=30); ap.add_argument("--fills", type=int, default=10)
+ ap.add_argument("--max-requests", type=int, default=10**9)
+ ap.add_argument("--rps", type=float, default=2.8); ap.add_argument("--inflight", type=int, default=64)
+ ap.add_argument("--seed", type=int, default=20261018); ap.add_argument("--shard-size", type=int, default=50000)
+ args = ap.parse_args(); out = Path(args.out); out.mkdir(parents=True, exist_ok=True)
+ asyncio.run(TGen(out, args).run())
+
+
+if __name__ == "__main__":
+ main()
diff --git a/model.py b/model.py
new file mode 100644
index 0000000..69d0db5
--- /dev/null
+++ b/model.py
@@ -0,0 +1,833 @@
+"""Boris-2: a d512 x 31L decoder-only transformer with fused Canon layers.
+
+Shape of a block:
+
+ h = h + o_proj( attn( canon( qkv_proj( norm(h) ) ) ) )
+ h = h + down( silu(gate) * up )( norm(h) )
+
+Four things here are not the textbook spelling, and each is a measured win on
+sm_70 rather than a preference:
+
+1. Q, K and V are one GEMM. The weights stay three separate Parameters and are
+ concatenated at call time, so Muon still orthogonalises each independently --
+ fusing the Parameters themselves would change what the optimizer operates on
+ and would not be lossless. The concatenated weight is ~1.2MB against a
+ 16,384-row activation.
+
+2. Canon is one kernel over the concatenated Q|K|V block. Depthwise convolution
+ is per-channel, so a single 768-channel convolution equals three separate
+ ones on the Q/K/V channel ranges: 31 kernels per forward instead of 93. It is
+ spelled as three shifted multiply-adds on the native (B,T,C) layout rather
+ than F.conv1d(groups=C), which needs a (B,T,C)->(B,C,T) transpose whose
+ following reshape materialises a copy. Measured on this V100, fwd+bwd over
+ the real shape: 4.30 ms for conv1d, 1.10 ms for the compiled shift form --
+ substantially faster than the transposing grouped-convolution spelling.
+
+3. Canon is document-aware. Kernel 3 stacked 31 deep reaches 62 tokens back, and
+ rows hold several packed documents, so without masking every block pulls the
+ previous document's tail across the boundary. The mask costs 0.10 ms/layer.
+ This is the bug the 75M run found the hard way; verify.py probes for it with
+ the output projections de-zeroed, because at init a zero-init projection
+ makes the test pass no matter how badly the model leaks.
+
+4. Attention is varlen over a jagged nested tensor: one SDPA launch for the
+ whole batch, cost sum(len_i^2) rather than T^2. Measured against the two
+ alternatives at ctx 2048 with realistic document lengths, fwd+bwd:
+ nested 11.09 ms, dense block-diagonal mask 26.97 ms, per-document loop
+ 45.84 ms. Nested is bit-identical to the loop (max diff 0.0).
+
+The residual stream is fp32 on the V100 because measured massive activations can
+exceed fp16's finite range. Matmul inputs are cast by the compiled regions under
+autocast, preserving tensor-core execution without risking a forward overflow.
+"""
+from __future__ import annotations
+
+import math
+import os
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from config import Config, ModelConfig
+from ngram import NGramTable
+from volta_kernels.integ_ops import AddNorm, LinAcc, TiedEmbedding
+
+IGNORE = -100
+
+# Hand-written sm_70 kernels (volta_kernels/): varlen FlashAttention with mma.sync
+# and a one-pass LM-head softmax. Opt-in, because they are adopted by a running
+# 200B-token job at a resume: BORIS_VOLTA_KERNELS=1. Same operand dtypes and fp32
+# accumulation as the paths they replace; verify.py checks both against fp64.
+# They only engage on an sm_70 device -- elsewhere (eval on the 3060) the stock
+# paths run regardless.
+VOLTA_KERNELS = os.environ.get("BORIS_VOLTA_KERNELS", "0") == "1"
+
+
+def _volta(t: torch.Tensor) -> bool:
+ return VOLTA_KERNELS and t.is_cuda and torch.cuda.get_device_capability(t.device) == (7, 0)
+
+
+# --------------------------------------------------------------------------- #
+# Compiled fixed-shape regions
+# --------------------------------------------------------------------------- #
+# Whole-model compile is not used: the 75M run measured it 28% slower in
+# realistic conditions because document counts vary and Inductor recompiles
+# mid-run. These regions never see a ragged shape -- only (B, T, C) with B and T
+# fixed for the whole context stage -- so each compiles once per stage.
+
+def _rmsnorm(x, w, eps: float):
+ f = x.float()
+ f = f * torch.rsqrt(f.pow(2).mean(-1, keepdim=True) + eps)
+ return (f * w.float()).to(x.dtype)
+
+
+def _pre_attn(h, w_norm, eps: float, w_qkv, canon_w, m1, m2, cos, sin,
+ n_head: int, n_kv_head: int, head_dim: int, use_canon: bool,
+ out16: bool = False):
+ """Norm, one GEMM, one depthwise convolution, one rotation -- one region.
+
+ The RMSNorm is inside the region on purpose. Written as its own eager module
+ it upcasts to fp32, and that materialises a (B, T, 512) fp32 tensor for
+ every one of the 53 norms in the model -- 25% of forward+backward was
+ elementwise traffic before these regions were widened, and this was most of
+ it. Inductor keeps the reduction in fp32 without ever writing the fp32
+ tensor out.
+
+ canon_w is (C, 3) with column 0 the weight on token t, column 1 on t-1 and
+ column 2 on t-2. m1 masks the t-1 tap and m2 the t-2 tap at document
+ boundaries; both are (B, T, 1) and are computed once per forward, not per
+ layer.
+ """
+ f = h.float()
+ x = (f * torch.rsqrt(f.pow(2).mean(-1, keepdim=True) + eps)
+ * w_norm.float()).to(h.dtype)
+ qkv = F.linear(x, w_qkv) # (B, T, C)
+ if use_canon:
+ w = canon_w.to(qkv.dtype)
+ s1 = F.pad(qkv, (0, 0, 1, 0))[:, :-1] * m1
+ s2 = F.pad(qkv, (0, 0, 2, 0))[:, :-2] * m2
+ qkv = qkv + qkv * w[:, 0] + s1 * w[:, 1] + s2 * w[:, 2]
+
+ B, T, _ = qkv.shape
+ qd, kd = n_head * head_dim, n_kv_head * head_dim
+ q = qkv[..., :qd].view(B, T, n_head, head_dim)
+ k = qkv[..., qd:qd + kd].view(B, T, n_kv_head, head_dim)
+ v = qkv[..., qd + kd:].view(B, T, n_kv_head, head_dim)
+
+ def rope(t):
+ f = t.float()
+ a, b = f.chunk(2, dim=-1)
+ return torch.cat([a * cos - b * sin, a * sin + b * cos], -1).to(t.dtype)
+
+ if out16:
+ # The Volta attention takes fp16. Casting here rounds the same fp32
+ # values autocast would round, but inside the region's epilogue, so the
+ # fp32 q/k/v are never written out and read back by separate casts.
+ h16 = torch.float16
+ return rope(q).to(h16), rope(k).to(h16), v.to(h16)
+ return rope(q), rope(k), v
+
+
+def _pre_attn_qkv(h, w_norm, eps: float, w_qkv):
+ """Norm and the fused QKV GEMM only -- the Volta path's region.
+
+ Canon and RoPE then run as one fused Triton pass (volta_kernels/canon_rope.py)
+ instead of Inductor's pad-copy / combine / rotate kernels.
+ """
+ f = h.float()
+ x = (f * torch.rsqrt(f.pow(2).mean(-1, keepdim=True) + eps)
+ * w_norm.float()).to(h.dtype)
+ return F.linear(x, w_qkv)
+
+
+def _post_then_pre(h, o, w_o, w_norm2, eps2, w_gu, w_down, w_norm1n, eps1n, w_qkvn,
+ extra):
+ """Block i's FFN half and block i+1's norm + QKV GEMM as ONE region.
+
+ As separate regions the residual h crossed a graph boundary between two
+ consumers, so its two backward gradient streams were summed by an eager
+ fp32 add (~192 MB of traffic per layer) and h was written and read back.
+ Here Inductor fuses that sum into the region's backward. `extra` is the
+ n-gram residual when block i+1 is where it is inserted (it depends only on
+ the token ids), else None -- each value gets its own compiled graph.
+ """
+ h = _post_attn(h, o, w_o, w_norm2, eps2, w_gu, w_down)
+ if extra is not None:
+ h = h + extra
+ return h, _pre_attn_qkv(h, w_norm1n, eps1n, w_qkvn)
+
+
+def _post_then_final_norm(h, o, w_o, w_norm2, eps2, w_gu, w_down, w_normf, epsf):
+ """The last block's FFN half and the final RMSNorm as one region."""
+ h = _post_attn(h, o, w_o, w_norm2, eps2, w_gu, w_down)
+ return _rmsnorm(h, w_normf, epsf)
+
+
+# Power-of-two headroom for the FFN down projection's fp16 output. 1/8 puts the
+# measured step-4,520 peak (~65.5k) at ~8.2k, leaving 8x before it overflows
+# again, and costs only inputs below 4.9e-4 -- 1.6e-8 of the peak, which
+# contribute nothing to a sum dominated by terms 6e7 times larger.
+DOWN_SHRINK = 0.125
+
+
+def _post_attn(h, o, w_o, w_norm, eps: float, w_gu, w_down):
+ """Output projection, residual, norm, SwiGLU, residual -- one region.
+
+ Everything in a block except the attention itself. The two residual adds and
+ the second norm were separate eager kernels over the widest tensors in the
+ model; inside the region Inductor folds them into the GEMM epilogues, and
+ the fp32 norm intermediate is never written out.
+ """
+ h = h + F.linear(o, w_o)
+ f = h.float()
+ x = (f * torch.rsqrt(f.pow(2).mean(-1, keepdim=True) + eps)
+ * w_norm.float()).to(h.dtype)
+ gu = F.linear(x, w_gu)
+ g, u = gu.chunk(2, dim=-1)
+ # The down projection is the one GEMM here whose fp16 OUTPUT can overflow.
+ # Its inputs are finite and so is the true result -- it is the fp16 range
+ # that runs out. Channel 228 of the residual is a "massive activation",
+ # 173x the RMS (78,115 against an RMS of 451 at block 16), and the FFN has
+ # to write into it, so at step 4,520 this output crossed 65,504 and became
+ # inf; norm_f then turned that into NaN via rsqrt(inf)=0, inf*0.
+ # Pre-scaling w_down is exact (a power of two), keeps the GEMM on tensor
+ # cores, costs one pass over a 1.2M-element weight rather than the 77M-
+ # element activation, and is divided back out in fp32 where there is room.
+ # Every partial sum is scaled too, so this is safe even with cuBLAS's
+ # reduced-precision fp16 reductions enabled.
+ return h + F.linear(F.silu(g) * u,
+ w_down * DOWN_SHRINK).float() * (1.0 / DOWN_SHRINK)
+
+
+# --------------------------------------------------------------------------- #
+# Merged-path regions with the swarm additions (volta_kernels/integ_ops.py)
+# --------------------------------------------------------------------------- #
+# ft = (wacc, norm) -- which additions a compiled region uses:
+# wacc: weight-gradient GEMMs accumulate into the fp32 .grad (accs given)
+# norm: residual-add + RMSNorm backward and norm-weight grad in one Triton pass
+# Weights arrive either as the fp32 Parameters (cast here, as autocast would) or
+# as the cached, pre-scaled fp16 copies (Boris2._w16); both give the same fp16
+# operand, so the forward is bitwise the live one either way.
+FT_ALL = (True, True)
+
+
+def _w16(w, scale: float = 1.0):
+ if w.dtype == torch.float16:
+ return w # cached copy, already scaled
+ return (w * scale).to(torch.float16) if scale != 1.0 else w.to(torch.float16)
+
+
+def _lin_x(x, w, acc, scale: float = 1.0):
+ if acc is None:
+ if w.dtype == torch.float16:
+ return F.linear(x, w)
+ return F.linear(x, w * scale if scale != 1.0 else w)
+ return LinAcc.apply(x.to(torch.float16), _w16(w, scale), acc, scale)
+
+
+def _norm_x(h, w, eps: float):
+ f = h.float()
+ return (f * torch.rsqrt(f.pow(2).mean(-1, keepdim=True) + eps)
+ * w.float()).to(h.dtype)
+
+
+def _ffn_x(h, o, w_o, w_norm, eps: float, w_gu, w_down, a_o, a_gu, a_down, ft):
+ """Block FFN half up to (but excluding) the down projection's residual add:
+ returns h1 = h + o_proj(o) and the fp16 down output (pre-scaled by
+ DOWN_SHRINK), so the next norm can fold the add into its backward."""
+ wacc, norm = ft
+ a = _lin_x(o, w_o, a_o)
+ if norm:
+ h1, x = AddNorm.apply(h, a, w_norm, eps, 1.0, False)
+ else:
+ h1 = h + a
+ x = _norm_x(h1, w_norm, eps)
+ gu = _lin_x(x, w_gu, a_gu)
+ g, u = gu.chunk(2, dim=-1)
+ return h1, _lin_x(F.silu(g) * u, w_down, a_down, DOWN_SHRINK)
+
+
+def _post_then_pre_x(h, o, w_o, w_norm2, eps2, w_gu, w_down, w_norm1n, eps1n, w_qkvn,
+ extra, accs, ft):
+ a_o, a_gu, a_down, a_qkvn = accs if accs is not None else (None,) * 4
+ h1, dn = _ffn_x(h, o, w_o, w_norm2, eps2, w_gu, w_down, a_o, a_gu, a_down, ft)
+ if ft[1] and extra is None:
+ h2, x = AddNorm.apply(h1, dn, w_norm1n, eps1n, 1.0 / DOWN_SHRINK, True)
+ else:
+ h2 = h1 + dn.float() * (1.0 / DOWN_SHRINK)
+ if extra is not None:
+ h2 = h2 + extra
+ x = _norm_x(h2, w_norm1n, eps1n)
+ return h2, _lin_x(x, w_qkvn, a_qkvn)
+
+
+def _post_then_final_norm_x(h, o, w_o, w_norm2, eps2, w_gu, w_down, w_normf, epsf,
+ accs, ft):
+ a_o, a_gu, a_down = accs if accs is not None else (None,) * 3
+ h1, dn = _ffn_x(h, o, w_o, w_norm2, eps2, w_gu, w_down, a_o, a_gu, a_down, ft)
+ return _rmsnorm(h1 + dn.float() * (1.0 / DOWN_SHRINK), w_normf, epsf)
+
+
+def _pre_attn_qkv_x(h, w_norm, eps: float, w_qkv, acc):
+ return _lin_x(_norm_x(h, w_norm, eps), w_qkv, acc)
+
+
+# --------------------------------------------------------------------------- #
+# Fused linear cross-entropy
+# --------------------------------------------------------------------------- #
+def _head_chunk(logits, safe, vc, scale: float):
+ """logsumexp, loss, and the scaled softmax, as one fused pass.
+
+ Eager, this chain is five separate passes over a (chunk, 32768) fp32 tensor
+ -- at micro_batch 16 that is 4.3 GB read and written five times per step.
+ Inductor folds it to two.
+ """
+ lse = torch.logsumexp(logits, dim=-1)
+ picked = logits.gather(1, safe[:, None]).squeeze(1)
+ loss = torch.where(vc, lse - picked, torch.zeros_like(lse)).sum()
+ p = (logits - lse[:, None]).exp() * (vc[:, None].to(logits.dtype) * scale)
+ return loss, p
+
+
+_head_chunk_c = None
+
+
+class _FusedLMHead(torch.autograd.Function):
+ """Cross-entropy over the LM head without ever holding (N, V) logits.
+
+ The naive head materialises the logits three times -- fp16 from the
+ projection, fp32 inside cross_entropy, and again for the backward -- which
+ at N=16,384 and V=32,768 is 2.1 GB a copy and sets the micro-batch.
+
+ The head's gradient depends on nothing downstream except a scalar, so this
+ computes dx and dW during the forward walk over vocab-sized chunks and
+ keeps only those. No recomputation in the backward: total GEMM work is the
+ same as the unfused head, and peak memory is one chunk of logits.
+
+ Gradients are held at sum scale (not divided by the token count) and lifted
+ by a further constant: a softmax over 32,768 classes has typical entries
+ near 3e-5, which is SUBNORMAL in fp16 (the smallest normal is 6.1e-5), so
+ the natural scaling would compute the whole dx GEMM on values with a few
+ bits of mantissa. GRAD_SCALE lifts them clear of that. The factor, the 1/N
+ and the loss scale are all divided out together in the backward.
+
+ The constant is bounded from ABOVE by the dW GEMM, which was missed until
+ real data hit it. g = S*(p - onehot), so the true token carries -S, and
+ dW sums that over every token in the chunk sharing a vocab row. Measured on
+ a real batch: the most frequent target appears 89 times in a 2048-token
+ chunk and |x| reaches 5.4, so at S=4096 the largest |dW| entry is 504,389
+ -- 7.7x past fp16's 65,504. It saturated to Inf, Inf-Inf gave NaN in the
+ tied embedding gradient, and the run died on step 2 with loss nan.
+
+ Measured max |dW| against S, same batch:
+
+ S=4096 504,389 OVERFLOW S=256 31,524 (2.1x margin)
+ S=1024 126,097 OVERFLOW S=64 7,881 (8.3x margin)
+
+ 64 is chosen over a value that splits the difference because the two risks
+ are not symmetric. Both grow over a long run -- |x| rises as the residual
+ stream grows, and a confident softmax makes small probabilities smaller --
+ but an overflow is a NaN that ends the run, while an underflow silently
+ drops terms that were already negligible. At S=64 the smallest nonzero
+ |p*S| is 1.48e-4, still 2.4x above fp16's min normal.
+ """
+
+ GRAD_SCALE = 64.0
+ # PERF.md's last unapplied win: accumulate the head's dW in fp16 rather than
+ # fp32. Their +10.1% came from an fp32 accumulate forcing a non-tensor-core
+ # sgemm; ours is already a tensor-core fp16 GEMM with only the accumulator
+ # buffer in fp32, so the mechanism differs. Measured either way.
+ FP16_DW = False
+
+ # Shrinks g for the dW GEMM only (a power of two, so it is exact and is
+ # divided back out losslessly in fp32). dx keeps the full GRAD_SCALE.
+ DW_SHRINK = 0.25
+
+ @staticmethod
+ def forward(ctx, x, W, targets, chunk: int):
+ N, d = x.shape
+ V = W.shape[0]
+ # Cast once, explicitly, rather than relying on autocast to do it: this
+ # runs inside a custom Function and is called from verify.py outside any
+ # autocast region.
+ Wc = W.to(x.dtype)
+ valid = targets != IGNORE
+ # Keep this scalar on-device. int(valid.sum()) was one forced CPU/GPU
+ # synchronization per micro-batch: sixteen pipeline stalls per step.
+ n_valid = valid.sum().clamp_min(1)
+ loss_sum = x.new_zeros((), dtype=torch.float32)
+ dx = torch.empty_like(x)
+ dW = torch.zeros_like(W, dtype=(torch.float16 if _FusedLMHead.FP16_DW
+ else torch.float32))
+
+ for a in range(0, N, chunk):
+ b = min(a + chunk, N)
+ xc, tc = x[a:b], targets[a:b]
+ logits = F.linear(xc, Wc).float() # (c, V)
+ safe = tc.clamp_min(0)
+ vc = (tc != IGNORE)
+ S = _FusedLMHead.GRAD_SCALE
+ loss, p = _head_chunk_c(logits, safe, vc, S)
+ loss_sum += loss
+ del logits
+ # The one-hot subtraction touches c elements, not c*V, so it stays
+ # outside the fused pass.
+ p.scatter_add_(1, safe[:, None], (-S * vc).to(p.dtype)[:, None])
+ g = p.to(x.dtype)
+ del p
+ dx[a:b] = F.linear(g, Wc.t())
+ # dW is the one GEMM here whose fp16 OUTPUT can overflow, and no
+ # loss scale can save it: g and xc are built inside this Function,
+ # so GradScaler halving its scale changes nothing while found_inf
+ # skips AdamW forever. That froze the embeddings, head, norms, Canon
+ # and n-gram table at step ~230 while Muon kept training the trunk
+ # against them; loss bottomed at 6.48 and climbed to nan by 350.
+ # Bound it by construction instead of by choosing a constant:
+ # rescale xc by a power of two so |xn| <= 1 (exact, no rounding),
+ # and shrink g for this GEMM only. |sum| <= S*DW_SHRINK*chunk = 32k
+ # then holds for ANY data, including a chunk of one repeated token,
+ # and both factors are divided back out in fp32.
+ r2 = torch.exp2(torch.ceil(torch.log2(
+ xc.abs().amax().float().clamp_min(1e-30))))
+ xn = xc / r2.to(xc.dtype)
+ gd = g * _FusedLMHead.DW_SHRINK
+ back = r2 / _FusedLMHead.DW_SHRINK
+ dw = gd.t() @ xn
+ # Restore magnitude AFTER widening. Multiplying in fp16 undoes
+ # the overflow protection before the fp32 accumulator sees it.
+ dW += (dw.float() * back).to(dW.dtype)
+
+ ctx.w_dtype = W.dtype
+ ctx.save_for_backward(dx, dW)
+ ctx.n_valid = n_valid
+ return loss_sum / n_valid
+
+ @staticmethod
+ def backward(ctx, grad_out):
+ dx, dW = ctx.saved_tensors
+ s = grad_out / (ctx.n_valid * _FusedLMHead.GRAD_SCALE)
+ # In fp32, then cast. s is ~2.8e-7 without a loss scale, which is
+ # subnormal in fp16 and would carry about two bits of mantissa -- it
+ # cost 7e-02 relative on dx before this was float. Under GradScaler s
+ # lands near 1e-2 and the difference is invisible, which is exactly why
+ # it would never have shown up in the loss curve.
+ return (dx.float() * s).to(dx.dtype), (dW.float() * s).to(ctx.w_dtype), None, None
+
+
+def fused_lm_loss(x, W, targets, chunk: int = 2048, stash: bool = False):
+ if _volta(x):
+ from volta_kernels.head import fused_lm_loss as volta_lm_loss
+ return volta_lm_loss(x, W, targets, chunk, stash)
+ assert not stash
+ global _head_chunk_c
+ if _head_chunk_c is None:
+ _head_chunk_c = torch.compile(_head_chunk, dynamic=False)
+ return _FusedLMHead.apply(x, W, targets, chunk)
+
+
+# --------------------------------------------------------------------------- #
+# Modules
+# --------------------------------------------------------------------------- #
+class RMSNorm(nn.Module):
+ def __init__(self, dim: int, eps: float):
+ super().__init__()
+ self.weight = nn.Parameter(torch.ones(dim))
+ self.eps = eps
+ self.weight._boris_role = "norm"
+
+ def forward(self, x):
+ return _rmsnorm(x, self.weight, self.eps)
+
+
+class Attention(nn.Module):
+ """Causal GQA (8 query heads over 2 KV heads) with fused Canon, varlen."""
+
+ def __init__(self, cfg: ModelConfig):
+ super().__init__()
+ self.cfg = cfg
+ d, hd = cfg.d_model, cfg.head_dim
+ self.n_head, self.n_kv_head, self.head_dim = cfg.n_head, cfg.n_kv_head, hd
+ self.rep = cfg.n_head // cfg.n_kv_head
+
+ # Q, K and V are ONE Parameter, and Muon orthogonalises its three row
+ # blocks independently -- which is exactly what it would do to three
+ # separate Parameters, because the quintic iteration acts on each matrix
+ # alone. Storing them fused removes a torch.cat per layer per
+ # micro-step (2.4% of forward+backward) and its backward slice, without
+ # changing what the optimizer operates on. verify.py asserts the two
+ # spellings produce identical updates.
+ self.qkv_proj = nn.Parameter(torch.empty(cfg.q_dim + 2 * cfg.kv_dim, d))
+ self.qkv_proj._boris_role = "matmul"
+ self.qkv_proj._boris_blocks = ((0, cfg.q_dim),
+ (cfg.q_dim, cfg.q_dim + cfg.kv_dim),
+ (cfg.q_dim + cfg.kv_dim, cfg.q_dim + 2 * cfg.kv_dim))
+ self.o_proj = nn.Parameter(torch.empty(d, cfg.q_dim))
+ self.o_proj._boris_role = "matmul"
+
+ if cfg.use_canon:
+ # (C, 3), zero-init: the layer is exactly the identity at step 0, so
+ # a Canon run and a no-Canon run start from the same function. The
+ # gradient w.r.t. the kernel is dL/dy * x and is non-zero, so it
+ # trains from the first step regardless.
+ self.canon = nn.Parameter(torch.zeros(cfg.canon_channels, cfg.canon_kernel))
+ self.canon._boris_role = "canon"
+ else:
+ self.register_parameter("canon", None)
+
+ def forward(self, h, norm_w, eps, ctx):
+ volta = self.head_dim == 64 and _volta(h)
+ if volta and self.canon is not None and self.rep == 1:
+ from volta_kernels.canon_rope_cuda import canon_rope
+ qkv = ctx.pre_attn_qkv(h, norm_w, eps, self.qkv_proj)
+ q, k, v = canon_rope(qkv, self.canon, ctx.m1, ctx.m2, ctx.cos, ctx.sin)
+ else:
+ q, k, v = ctx.pre_attn(h, norm_w, eps, self.qkv_proj, self.canon,
+ ctx.m1, ctx.m2, ctx.cos, ctx.sin,
+ self.n_head, self.n_kv_head, self.head_dim,
+ self.canon is not None, volta)
+ B, T = q.shape[0], q.shape[1]
+ q = q.reshape(B * T, self.n_head, self.head_dim)
+ # rep == 1 under full MHA: repeat_interleave(1) is a no-op that still
+ # allocates and copies a (B*T, n_head, head_dim) tensor -- 33 MB per
+ # call per layer at micro_batch 16, twice (k and v), on every block in
+ # forward AND backward. Skip it.
+ k = k.reshape(B * T, self.n_kv_head, self.head_dim)
+ v = v.reshape(B * T, self.n_kv_head, self.head_dim)
+ if self.rep > 1:
+ k = k.repeat_interleave(self.rep, 1)
+ v = v.repeat_interleave(self.rep, 1)
+
+ off = ctx.offsets
+ if volta:
+ from volta_kernels import varlen_causal_attention
+ # q/k/v arrive fp16 from the region (out16): the cast autocast
+ # would apply inside SDPA, since the Canon masks keep them fp32.
+ o = varlen_causal_attention(q, k, v, off)
+ return o.reshape(B, T, self.n_head * self.head_dim)
+ nj = lambda t: torch.nested.nested_tensor_from_jagged(
+ t, off, min_seqlen=ctx.min_seqlen, max_seqlen=ctx.max_seqlen)
+ nq = nj(q).transpose(1, 2)
+ nk = nj(k).transpose(1, 2)
+ nv = nj(v).transpose(1, 2)
+ o = F.scaled_dot_product_attention(nq, nk, nv, is_causal=True)
+ return o.transpose(1, 2).values().reshape(B, T, self.n_head * self.head_dim)
+
+
+class FFN(nn.Module):
+ """SwiGLU, inner width 2368. gate and up are one GEMM at call time."""
+
+ def __init__(self, cfg: ModelConfig):
+ super().__init__()
+ h, d = cfg.ffn_hidden, cfg.d_model
+ # gate and up fused for the same reason q/k/v are; Muon still sees two
+ # independent (2368, 512) matrices.
+ self.gate_up_proj = nn.Parameter(torch.empty(2 * h, d))
+ self.gate_up_proj._boris_role = "matmul"
+ self.gate_up_proj._boris_blocks = ((0, h), (h, 2 * h))
+ self.down_proj = nn.Parameter(torch.empty(d, h))
+ self.down_proj._boris_role = "matmul"
+
+
+class Block(nn.Module):
+ def __init__(self, cfg: ModelConfig):
+ super().__init__()
+ self.norm1 = RMSNorm(cfg.d_model, cfg.norm_eps)
+ self.attn = Attention(cfg)
+ self.norm2 = RMSNorm(cfg.d_model, cfg.norm_eps)
+ self.ffn = FFN(cfg)
+
+ def forward(self, h, ctx):
+ o = self.attn(h, self.norm1.weight, self.norm1.eps, ctx)
+ return ctx.post_attn(h, o, self.attn.o_proj, self.norm2.weight,
+ self.norm2.eps, self.ffn.gate_up_proj,
+ self.ffn.down_proj)
+
+
+class RunContext:
+ """Everything shared across all blocks for one forward.
+
+ RoPE tables are gathered once here rather than once per attention layer,
+ and the Canon boundary masks once rather than once per Canon. Holding the
+ compiled callables here keeps a single compiled graph per stage shared by
+ every block.
+ """
+
+ def __init__(self, cos, sin, m1, m2, offsets, pre_attn, post_attn,
+ min_seqlen=None, max_seqlen=None, pre_attn_qkv=None):
+ self.pre_attn_qkv = pre_attn_qkv
+ self.cos, self.sin = cos, sin
+ self.m1, self.m2 = m1, m2
+ self.offsets = offsets
+ self.pre_attn, self.post_attn = pre_attn, post_attn
+ # Without these, every SDPA call derives them from the offsets with a
+ # .max().item() -- a host sync, 131 of them per forward, each stalling
+ # CPU run-ahead. The packer knows both while it is laying documents
+ # down, so they cost nothing to carry.
+ self.min_seqlen, self.max_seqlen = min_seqlen, max_seqlen
+
+
+class Boris2(nn.Module):
+ def __init__(self, cfg: Config):
+ super().__init__()
+ self.cfg = cfg
+ m = cfg.model
+ self.embed = nn.Embedding(m.vocab_size, m.d_model)
+ self.embed.weight._boris_role = "embedding"
+ self.blocks = nn.ModuleList([Block(m) for _ in range(m.n_layer)])
+ self.norm_f = RMSNorm(m.d_model, m.norm_eps)
+ self.ngram = NGramTable(cfg.ngram, m.d_model) if m.use_ngram else None
+ # Runtime toggle, for scoring one trained checkpoint both ways. Note
+ # what it measures: after training, the trunk has co-adapted to the
+ # table, so switching it off reports how much the model DEPENDS on it,
+ # not how much it GAINED from it. The gain needs the use_ngram=False
+ # training arm.
+ self.ngram_enabled = True
+ if not m.tie_embeddings:
+ self.lm_head = nn.Parameter(torch.empty(m.vocab_size, m.d_model))
+ self.lm_head._boris_role = "embedding"
+
+ self._rope_cache: dict[tuple, tuple] = {}
+ self._pre_attn = _pre_attn
+ self._pre_attn_qkv = _pre_attn_qkv
+ self._post_attn = _post_attn
+ self._post_then_pre = _post_then_pre
+ self._post_then_final_norm = _post_then_final_norm
+ self._pre_attn_qkv_x = _pre_attn_qkv_x
+ self._post_then_pre_x = _post_then_pre_x
+ self._post_then_final_norm_x = _post_then_final_norm_x
+ # Swarm additions on the merged path (see volta_kernels/integ_ops.py):
+ # (wgrad_acc, fused norm backward) and the tied embedding's fused grad
+ # accumulate. w16_cache (fp16 weight copies cast once per optimizer
+ # step) is OFF: ~0.15% at mb 4, within noise, and it is only correct
+ # while every step starts with zero_grad(set_to_none=True).
+ self.ft = FT_ALL
+ self.tied_acc = True
+ self.w16_cache = False
+ self._w16c: dict = {}
+ self._compiled = False
+ self.apply_init()
+
+ # -- init ------------------------------------------------------------- #
+ def apply_init(self):
+ m = self.cfg.model
+ nn.init.normal_(self.embed.weight, std=0.02)
+ if not m.tie_embeddings:
+ nn.init.normal_(self.lm_head, std=0.02)
+ # Residual-path projections are scaled down so the residual variance
+ # does not grow with depth -- at 31 layers this matters more than it
+ # does at 10.
+ res_scale = 1.0 / math.sqrt(2 * m.n_layer)
+ for blk in self.blocks:
+ for p in (blk.attn.qkv_proj, blk.ffn.gate_up_proj):
+ nn.init.normal_(p, std=0.02)
+ for p in (blk.attn.o_proj, blk.ffn.down_proj):
+ nn.init.normal_(p, std=0.02 * res_scale)
+ if blk.attn.canon is not None:
+ nn.init.zeros_(blk.attn.canon)
+
+ def maybe_compile(self):
+ if self._compiled or not self.cfg.train.compile_regions:
+ return
+ self._pre_attn = torch.compile(_pre_attn, dynamic=False)
+ self._pre_attn_qkv = torch.compile(_pre_attn_qkv, dynamic=False)
+ self._post_then_pre = torch.compile(_post_then_pre, dynamic=False)
+ self._post_then_final_norm = torch.compile(_post_then_final_norm, dynamic=False)
+ self._post_attn = torch.compile(_post_attn, dynamic=False)
+ self._pre_attn_qkv_x = torch.compile(_pre_attn_qkv_x, dynamic=False)
+ self._post_then_pre_x = torch.compile(_post_then_pre_x, dynamic=False)
+ self._post_then_final_norm_x = torch.compile(_post_then_final_norm_x, dynamic=False)
+ self._compiled = True
+
+ def head_weight(self):
+ return self.embed.weight if self.cfg.model.tie_embeddings else self.lm_head
+
+ # -- rope ------------------------------------------------------------- #
+ def rope_tables(self, pos, theta: float, device, dtype=torch.float32):
+ """cos/sin for (B, T) per-document positions, gathered once per forward.
+
+ Positions restart at every document, so a document packed at row offset
+ 1500 is rotated as though it began at 0. RoPE is relative, so this is
+ not strictly required inside a document -- but it keeps the absolute
+ range a document sees equal to its own length, which is what the
+ context stage is defined in terms of.
+ """
+ hd = self.cfg.model.head_dim
+ key = (theta, hd, device)
+ if key not in self._rope_cache:
+ inv = 1.0 / (theta ** (torch.arange(0, hd, 2, device=device).float() / hd))
+ t = torch.arange(self.cfg.model.max_block_size, device=device).float()
+ f = torch.outer(t, inv)
+ self._rope_cache[key] = (f.cos().to(dtype), f.sin().to(dtype))
+ cos, sin = self._rope_cache[key]
+ return cos[pos].unsqueeze(2), sin[pos].unsqueeze(2) # (B, T, 1, hd/2)
+
+ # -- forward ---------------------------------------------------------- #
+ def forward(self, idx, pos, offsets, boundary, targets=None,
+ theta: float = 10_000.0, min_seqlen=None, max_seqlen=None):
+ """idx/pos/boundary are (B, T); offsets indexes the flattened B*T stream."""
+ cos, sin = self.rope_tables(pos, theta, idx.device)
+ # m1: is t-1 in this document? m2: is t-2? Once per forward, not per layer.
+ b1 = boundary
+ b2 = boundary | F.pad(boundary, (1, 0))[:, :-1]
+ dt = torch.float16 if self.cfg.train.fp16_residual else torch.float32
+ m1 = (~b1).to(dt)[:, :, None]
+ m2 = (~b2).to(dt)[:, :, None]
+ ctx = RunContext(cos, sin, m1, m2, offsets, self._pre_attn, self._post_attn,
+ min_seqlen, max_seqlen, self._pre_attn_qkv)
+
+ gc = self.cfg.train.grad_checkpoint
+ every = max(1, self.cfg.train.grad_checkpoint_every)
+ a0 = self.blocks[0].attn
+ merged = (_volta(idx) and a0.canon is not None and a0.rep == 1 and a0.head_dim == 64
+ and not (gc and self.training) and not self.cfg.train.fp16_residual
+ and not getattr(self, "_no_merge", False))
+ # Tied-embedding fused accumulate: only when the head's backward is
+ # certain to run first and stash its dW (training forward with targets).
+ stash = (merged and self.tied_acc and targets is not None
+ and torch.is_grad_enabled() and self.cfg.model.tie_embeddings
+ and self.embed.weight.requires_grad)
+ if stash:
+ h = TiedEmbedding.apply(idx, self.embed.weight)
+ else:
+ h = self.embed(idx)
+ if self.cfg.train.fp16_residual:
+ h = h.to(torch.float16)
+
+ if merged:
+ h = self._forward_merged(h, idx, boundary, ctx)
+ if targets is None:
+ return h
+ return fused_lm_loss(h.reshape(-1, h.shape[-1]), self.head_weight(),
+ targets.reshape(-1), self.cfg.optim.lm_head_chunk,
+ stash)
+ for i, blk in enumerate(self.blocks):
+ if (self.ngram is not None and self.ngram_enabled
+ and i == self.cfg.ngram.insert_after):
+ h = h + self.ngram(idx, boundary, h.dtype)
+ if gc and self.training and (i % every == 0):
+ h = torch.utils.checkpoint.checkpoint(
+ blk, h, ctx, use_reentrant=False)
+ else:
+ h = blk(h, ctx)
+
+ h = self.norm_f(h)
+ if targets is None:
+ return h
+ loss = fused_lm_loss(h.reshape(-1, h.shape[-1]), self.head_weight(),
+ targets.reshape(-1), self.cfg.optim.lm_head_chunk)
+ return loss
+
+ @staticmethod
+ def _acc(p):
+ """p.grad as the fp32 accumulator the weight-gradient GEMMs add into;
+ created (zeroed) on the first micro-step after zero_grad(set_to_none)."""
+ if not p.requires_grad:
+ return None
+ if p.grad is None:
+ p.grad = torch.zeros_like(p, memory_format=torch.contiguous_format)
+ return p.grad
+
+ def _wq(self, p, acc, scale: float = 1.0):
+ """The weight as the region should receive it: the fp32 Parameter
+ (cast inside the region), or -- with w16_cache -- an fp16 copy cast once
+ per optimizer step. The copy is keyed on the identity of p.grad: every
+ step starts with zero_grad(set_to_none=True), so a new accumulator means
+ the weights may have moved. (p._version can NOT be used: Muon updates
+ p.data in place, which does not bump the version counter.)"""
+ if not (self.w16_cache and acc is not None):
+ return p
+ import weakref
+ e = self._w16c.get(id(p))
+ if e is not None and e[1]() is acc:
+ return e[0]
+ with torch.no_grad():
+ w = (p * scale if scale != 1.0 else p).to(torch.float16)
+ self._w16c[id(p)] = (w, weakref.ref(acc))
+ return w
+
+ def _forward_merged(self, h, idx, boundary, ctx):
+ """Blocks with each residual boundary inside one compiled region
+ (Volta path, no gradient checkpointing). Same operations, same order,
+ as the per-block path; returns norm_f(h)."""
+ if torch.is_grad_enabled() and any(self.ft):
+ return self._forward_merged_x(h, idx, boundary, ctx)
+ return self._forward_merged_stock(h, idx, boundary, ctx)
+
+ def _forward_merged_x(self, h, idx, boundary, ctx):
+ """_forward_merged with the swarm additions (self.ft, self.w16_cache)."""
+ from volta_kernels import canon_rope_attention
+ ft = self.ft
+ wacc = ft[0]
+ A = self._acc if wacc else (lambda p: None)
+ W = self._wq
+ blocks = self.blocks
+ L = len(blocks)
+ ins = self.cfg.ngram.insert_after if (self.ngram is not None
+ and self.ngram_enabled) else -1
+ ng = self.ngram(idx, boundary, h.dtype) if ins >= 0 else None
+ if ins == 0:
+ h = h + ng
+ B, T = h.shape[0], h.shape[1]
+ b0 = blocks[0]
+ a0 = A(b0.attn.qkv_proj)
+ qkv = self._pre_attn_qkv_x(h, b0.norm1.weight, b0.norm1.eps,
+ W(b0.attn.qkv_proj, a0), a0)
+ for i, blk in enumerate(blocks):
+ a = blk.attn
+ o = canon_rope_attention(qkv, a.canon, ctx.m1, ctx.m2, ctx.cos, ctx.sin,
+ ctx.offsets).reshape(B, T, a.n_head * a.head_dim)
+ f = blk.ffn
+ ao, agu, adn = A(a.o_proj), A(f.gate_up_proj), A(f.down_proj)
+ wo, wgu, wdn = W(a.o_proj, ao), W(f.gate_up_proj, agu), W(f.down_proj, adn, DOWN_SHRINK)
+ if i + 1 < L:
+ nb = blocks[i + 1]
+ aq = A(nb.attn.qkv_proj)
+ h, qkv = self._post_then_pre_x(
+ h, o, wo, blk.norm2.weight, blk.norm2.eps, wgu, wdn,
+ nb.norm1.weight, nb.norm1.eps, W(nb.attn.qkv_proj, aq),
+ ng if i + 1 == ins else None,
+ (ao, agu, adn, aq) if wacc else None, ft)
+ else:
+ h = self._post_then_final_norm_x(
+ h, o, wo, blk.norm2.weight, blk.norm2.eps, wgu, wdn,
+ self.norm_f.weight, self.norm_f.eps,
+ (ao, agu, adn) if wacc else None, ft)
+ return h
+
+ def _forward_merged_stock(self, h, idx, boundary, ctx):
+ from volta_kernels import canon_rope_attention
+ blocks = self.blocks
+ L = len(blocks)
+ ins = self.cfg.ngram.insert_after if (self.ngram is not None
+ and self.ngram_enabled) else -1
+ ng = self.ngram(idx, boundary, h.dtype) if ins >= 0 else None
+ if ins == 0:
+ h = h + ng
+ B, T = h.shape[0], h.shape[1]
+ b0 = blocks[0]
+ qkv = self._pre_attn_qkv(h, b0.norm1.weight, b0.norm1.eps, b0.attn.qkv_proj)
+ for i, blk in enumerate(blocks):
+ a = blk.attn
+ # Canon+RoPE+attention as one autograd Function: dQ reaches the
+ # Canon backward in fp32 (see volta_kernels._CanonRopeAttn).
+ o = canon_rope_attention(qkv, a.canon, ctx.m1, ctx.m2, ctx.cos, ctx.sin,
+ ctx.offsets).reshape(B, T, a.n_head * a.head_dim)
+ if i + 1 < L:
+ nb = blocks[i + 1]
+ h, qkv = self._post_then_pre(
+ h, o, a.o_proj, blk.norm2.weight, blk.norm2.eps,
+ blk.ffn.gate_up_proj, blk.ffn.down_proj,
+ nb.norm1.weight, nb.norm1.eps, nb.attn.qkv_proj,
+ ng if i + 1 == ins else None)
+ else:
+ h = self._post_then_final_norm(
+ h, o, a.o_proj, blk.norm2.weight, blk.norm2.eps,
+ blk.ffn.gate_up_proj, blk.ffn.down_proj,
+ self.norm_f.weight, self.norm_f.eps)
+ return h
+
+ @torch.no_grad()
+ def logits_for_eval(self, idx, pos, offsets, boundary, theta: float = 10_000.0):
+ h = self.forward(idx, pos, offsets, boundary, None, theta)
+ return F.linear(h, self.head_weight())
diff --git a/ngram.py b/ngram.py
new file mode 100644
index 0000000..f847d6b
--- /dev/null
+++ b/ngram.py
@@ -0,0 +1,414 @@
+"""The n-gram embedding table: file format, builder, and the module.
+
+The idea. A dense transformer has to reconstruct "New York City" or "in
+accordance with" out of its weights every time it sees one. This table is keyed
+on the literal last two or three token ids, so the model can retrieve a learned
+vector for that specific phrase instead. It holds 15M parameters and costs one
+gather per position -- no matmul, so compute per token barely moves.
+
+Explicit top-K, never hashing. A row belongs to one specific (t-1, t) or
+(t-2, t-1, t); anything outside the top-K falls to a single shared OOV row.
+Hashed buckets would let two unrelated phrases share a vector, which silently
+changes what is being measured.
+
+Layout is chosen for the gather. Row indices are assigned in descending corpus
+frequency, so the rows a batch actually touches are clustered at the head of the
+table: rank 64 in fp32 is 256 bytes, and the top ~24k rows fit in the V100's 6MB
+L2. Assigning row indices by n-gram id instead would scatter every batch across
+the full 60MB table, and the gather is bandwidth-bound.
+
+The file. `ngram_vocab.bin` maps n-gram to row index as two sorted key arrays
+with parallel row arrays:
+
+ header (64 bytes)
+ bigram_keys int64[n_bigram] sorted ascending, key = a*V + b
+ bigram_rows int32[n_bigram]
+ trigram_keys int64[n_trigram] sorted ascending, key = (a*V + b)*V + c
+ trigram_rows int32[n_trigram]
+
+Lookup is torch.searchsorted plus an equality check -- exact, no probe loop, and
+a miss is a miss rather than a collision. Trigram keys peak at V^3 = 3.5e13 for
+V = 32,768, comfortably inside int64.
+
+ python ngram.py build --shards data/shards --out data/ngram_vocab.bin
+"""
+from __future__ import annotations
+
+import argparse
+import struct
+import sys
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+import numpy as np
+import torch
+import torch.nn as nn
+
+MAGIC = b"BNGRAM2\0"
+HEADER_FMT = "<8sIIQQ" # magic, version, vocab, n_bi, n_tri
+HEADER_SIZE = 64
+VERSION = 2
+
+
+@dataclass
+class NGramVocab:
+ vocab_size: int
+ bigram_keys: np.ndarray
+ bigram_rows: np.ndarray
+ trigram_keys: np.ndarray
+ trigram_rows: np.ndarray
+
+ def save(self, path: Path) -> None:
+ path = Path(path)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ head = struct.pack(HEADER_FMT, MAGIC, VERSION, self.vocab_size,
+ self.bigram_keys.size, self.trigram_keys.size)
+ with open(path, "wb") as f:
+ f.write(head + b"\0" * (HEADER_SIZE - len(head)))
+ for a in (self.bigram_keys.astype(np.int64),
+ self.bigram_rows.astype(np.int32),
+ self.trigram_keys.astype(np.int64),
+ self.trigram_rows.astype(np.int32)):
+ f.write(a.tobytes())
+
+ @staticmethod
+ def load(path: Path) -> "NGramVocab":
+ path = Path(path)
+ with open(path, "rb") as f:
+ head = f.read(HEADER_SIZE)
+ magic, ver, vocab, nb, nt = struct.unpack(HEADER_FMT, head[:struct.calcsize(HEADER_FMT)])
+ if magic != MAGIC:
+ raise ValueError(f"{path}: not an ngram vocab file")
+ if ver != VERSION:
+ raise ValueError(f"{path}: version {ver}, expected {VERSION}")
+ parts = (f.read(nb * 8), f.read(nb * 4),
+ f.read(nt * 8), f.read(nt * 4))
+ if tuple(map(len, parts)) != (nb * 8, nb * 4, nt * 8, nt * 4):
+ raise ValueError(f"{path}: truncated n-gram vocabulary")
+ if f.read(1):
+ raise ValueError(f"{path}: trailing data after n-gram vocabulary")
+ bk = np.frombuffer(parts[0], dtype=np.int64)
+ br = np.frombuffer(parts[1], dtype=np.int32)
+ tk = np.frombuffer(parts[2], dtype=np.int64)
+ tr = np.frombuffer(parts[3], dtype=np.int32)
+ if ((bk.size > 1 and np.any(bk[1:] <= bk[:-1])) or
+ (tk.size > 1 and np.any(tk[1:] <= tk[:-1]))):
+ raise ValueError(f"{path}: keys are not strictly increasing")
+ return NGramVocab(vocab, bk, br, tk, tr)
+
+ def fingerprint(self) -> str:
+ import hashlib
+ h = hashlib.sha256()
+ for a in (self.bigram_keys, self.bigram_rows, self.trigram_keys, self.trigram_rows):
+ # The mapping file is only a few MB. Hashing just the first MB let a
+ # changed tail pass the resume guard while assigning learned rows to
+ # different phrases.
+ h.update(np.ascontiguousarray(a).tobytes())
+ h.update(str(a.size).encode())
+ return h.hexdigest()[:16]
+
+ def legacy_fingerprint(self) -> str:
+ """Fingerprint written by checkpoints before the full-file hash fix."""
+ import hashlib
+ h = hashlib.sha256()
+ for a in (self.bigram_keys, self.bigram_rows,
+ self.trigram_keys, self.trigram_rows):
+ h.update(np.ascontiguousarray(a).tobytes()[:1 << 20])
+ h.update(str(a.size).encode())
+ return h.hexdigest()[:16]
+
+
+# --------------------------------------------------------------------------- #
+# Module
+# --------------------------------------------------------------------------- #
+class NGramTable(nn.Module):
+ """Lookup on (t-1, t) and (t-2, t-1, t), projected into the residual stream.
+
+ Trigram wins where both hit: it is the more specific key. Positions whose
+ history would cross a packed document boundary fall to the OOV row, for the
+ same reason Canon is masked there -- the previous document is not context.
+ """
+
+ def __init__(self, cfg, d_model: int):
+ super().__init__()
+ self.cfg = cfg
+ self.d_model = d_model
+ self.table = nn.Parameter(torch.empty(cfg.n_rows, cfg.rank))
+ self.proj = nn.Parameter(torch.empty(d_model, cfg.rank))
+ nn.init.normal_(self.table, std=cfg.table_init_std)
+ nn.init.normal_(self.proj, std=cfg.proj_init_std)
+ self.table._boris_role = "ngram_table" # a gather, never Muon
+ self.proj._boris_role = "matmul" if cfg.muon_on_proj else "ngram_table"
+
+ # Empty until load_vocab(); registered so they move with .to(device)
+ # and are excluded from the checkpoint (they live in ngram_vocab.bin,
+ # whose fingerprint is asserted at resume).
+ self.register_buffer("bigram_keys", torch.zeros(0, dtype=torch.int64), persistent=False)
+ self.register_buffer("bigram_rows", torch.zeros(0, dtype=torch.int64), persistent=False)
+ self.register_buffer("trigram_keys", torch.zeros(0, dtype=torch.int64), persistent=False)
+ self.register_buffer("trigram_rows", torch.zeros(0, dtype=torch.int64), persistent=False)
+ self.vocab_size = 0
+ self.loaded = False
+
+ def load_vocab(self, vocab: NGramVocab, device=None) -> None:
+ dev = device or self.table.device
+ self.bigram_keys = torch.from_numpy(vocab.bigram_keys.copy()).to(dev)
+ self.bigram_rows = torch.from_numpy(vocab.bigram_rows.astype(np.int64)).to(dev)
+ self.trigram_keys = torch.from_numpy(vocab.trigram_keys.copy()).to(dev)
+ self.trigram_rows = torch.from_numpy(vocab.trigram_rows.astype(np.int64)).to(dev)
+ self.vocab_size = int(vocab.vocab_size)
+ n = self.cfg.n_rows
+ hi = max(int(self.bigram_rows.max()) if self.bigram_rows.numel() else -1,
+ int(self.trigram_rows.max()) if self.trigram_rows.numel() else -1)
+ assert hi < n, (
+ f"ngram_vocab.bin holds row index {hi}, the configured table has "
+ f"{n} rows")
+ self.loaded = True
+
+ @staticmethod
+ def _lookup(keys, rows, query, oov: int):
+ """Exact sorted lookup: searchsorted + equality, one fused kernel."""
+ if keys.numel() == 0:
+ return torch.full_like(query, oov)
+ i = torch.searchsorted(keys, query.clamp_min(0))
+ i = i.clamp_max(keys.numel() - 1)
+ hit = keys[i] == query
+ return torch.where(hit, rows[i], torch.full_like(query, oov))
+
+ def forward(self, idx, boundary, dtype):
+ if not self.loaded:
+ raise RuntimeError("NGramTable.load_vocab() was never called")
+ B, T = idx.shape
+ V = self.vocab_size
+ oov = self.cfg.oov_row
+ i0 = idx.to(torch.int64)
+ i1 = torch.roll(i0, 1, dims=1)
+ i2 = torch.roll(i0, 2, dims=1)
+
+ # Same masks Canon uses: t-1 present, and t-2 present.
+ ok1 = ~boundary
+ ok2 = ~(boundary | torch.roll(boundary, 1, dims=1))
+ ok1[:, 0] = False
+ ok2[:, :2] = False
+
+ bi_key = torch.where(ok1, i1 * V + i0, torch.full_like(i0, -1))
+ tri_key = torch.where(ok2, (i2 * V + i1) * V + i0, torch.full_like(i0, -1))
+
+ row = self._lookup(self.bigram_keys, self.bigram_rows, bi_key, oov)
+ tri = self._lookup(self.trigram_keys, self.trigram_rows, tri_key, oov)
+ row = torch.where(tri != oov, tri, row) # trigram is more specific
+
+ vec = torch.nn.functional.embedding(row, self.table) # (B, T, rank)
+ return torch.nn.functional.linear(vec.to(dtype), self.proj.to(dtype))
+
+ @torch.no_grad()
+ def hit_rate(self, idx, boundary) -> dict:
+ """Fraction of positions served by a real row -- logged at startup and
+ periodically; a collapse here means the table has stopped earning its
+ parameters."""
+ B, T = idx.shape
+ V, oov = self.vocab_size, self.cfg.oov_row
+ i0 = idx.to(torch.int64)
+ i1, i2 = torch.roll(i0, 1, 1), torch.roll(i0, 2, 1)
+ ok1 = ~boundary.clone(); ok1[:, 0] = False
+ ok2 = ~(boundary | torch.roll(boundary, 1, 1)); ok2[:, :2] = False
+ bi = self._lookup(self.bigram_keys, self.bigram_rows,
+ torch.where(ok1, i1 * V + i0, torch.full_like(i0, -1)), oov)
+ tri = self._lookup(self.trigram_keys, self.trigram_rows,
+ torch.where(ok2, (i2 * V + i1) * V + i0, torch.full_like(i0, -1)), oov)
+ n = B * T
+ return {"bigram": float((bi != oov).sum()) / n,
+ "trigram": float((tri != oov).sum()) / n,
+ "any": float(((bi != oov) | (tri != oov)).sum()) / n}
+
+
+# --------------------------------------------------------------------------- #
+# Builder
+# --------------------------------------------------------------------------- #
+def _topk_counts(stream, n_keep: int, chunk: int, cap: int, order: int, vocab: int,
+ log=print):
+ """Approximate top-n_keep keys by count over a token stream.
+
+ Exact counting is not affordable -- a few billion tokens produce hundreds of
+ millions of distinct bigrams. This is lossy counting: accumulate exact counts
+ per chunk, merge, and when the accumulator exceeds `cap` drop the tail below
+ a rising threshold. Frequent n-grams, which are the only ones that get a row,
+ survive every prune; the approximation only ever affects the boundary of the
+ top-K, where the difference between rank 140,624 and 140,626 is noise.
+ """
+ keys_acc = np.zeros(0, dtype=np.int64)
+ cnt_acc = np.zeros(0, dtype=np.int64)
+ floor = 1
+ seen = 0
+ t0 = time.time()
+ for toks in stream:
+ toks = toks.astype(np.int64, copy=False)
+ if toks.size <= order:
+ continue
+ if order == 2:
+ ctx = np.lib.stride_tricks.sliding_window_view(toks, 2)
+ ctx = ctx[np.all(ctx != 0, axis=1)]
+ k = ctx[:, 0] * vocab + ctx[:, 1]
+ else:
+ ctx = np.lib.stride_tricks.sliding_window_view(toks, 3)
+ ctx = ctx[np.all(ctx != 0, axis=1)]
+ k = (ctx[:, 0] * vocab + ctx[:, 1]) * vocab + ctx[:, 2]
+ seen += k.size
+ for a in range(0, k.size, chunk):
+ u, c = np.unique(k[a:a + chunk], return_counts=True)
+ keys_acc = np.concatenate([keys_acc, u])
+ cnt_acc = np.concatenate([cnt_acc, c.astype(np.int64)])
+ if keys_acc.size > cap:
+ o = np.argsort(keys_acc, kind="stable")
+ keys_acc, cnt_acc = keys_acc[o], cnt_acc[o]
+ uk, start = np.unique(keys_acc, return_index=True)
+ uc = np.add.reduceat(cnt_acc, start)
+ while uk.size > cap // 2:
+ floor += 1
+ m = uc >= floor
+ uk, uc = uk[m], uc[m]
+ keys_acc, cnt_acc = uk, uc
+ log(f" merged: {seen/1e9:.2f}B {order}-grams, "
+ f"{uk.size/1e6:.1f}M kept, floor {floor}, "
+ f"{time.time()-t0:.0f}s")
+ o = np.argsort(keys_acc, kind="stable")
+ keys_acc, cnt_acc = keys_acc[o], cnt_acc[o]
+ uk, start = np.unique(keys_acc, return_index=True)
+ uc = np.add.reduceat(cnt_acc, start)
+ if uk.size > n_keep:
+ top = np.argpartition(uc, -n_keep)[-n_keep:]
+ uk, uc = uk[top], uc[top]
+ log(f" {order}-gram: {seen/1e9:.2f}B seen, {uk.size:,} rows kept, "
+ f"min count {int(uc.min()) if uc.size else 0}")
+ return uk, uc
+
+
+def _source_dirs(root: Path, name: str) -> list[Path]:
+ """A source's directories: data/<name>/ plus any data/<name>_sN/ shards."""
+ root = Path(root)
+ return [d for d in sorted(root.glob(name)) + sorted(root.glob(f"{name}_s*"))
+ if (d / "tokens.bin").exists() and (d / "tokens.bin").stat().st_size > 0]
+
+
+def _sample_stream(root: Path, cfg, limit_tokens: int,
+ chunk_tokens: int = 25_000_000, seed: int = 0, log=print):
+ """Token chunks sampled to match the training distribution.
+
+ Two things this must not do. It must not walk the sources in directory
+ order and stop when the budget runs out -- that would build the table's
+ vocabulary out of whichever source sorts first, and with Ultra-FineWeb split
+ across four shard directories the bias is severe. And it must not read only
+ the head of each file: a shard is written in the order documents arrived
+ from the Hub, so its first 3B tokens are one slice of Common Crawl rather
+ than a sample of it.
+
+ So: budget is split across sources by their stable-phase mixture weight,
+ split again evenly across each source's shards, and drawn from evenly spaced
+ offsets within each shard. Chunks are emitted round-robin so the lossy
+ counter's prune floor sees a representative mix throughout rather than one
+ source at a time.
+ """
+ from schedule import mixture_at
+ doc_weights = mixture_at(
+ (cfg.schedule.warmup_frac + cfg.schedule.stable_frac) / 2, cfg)
+ # VarlenPacker chooses a source once per document. Convert those draw
+ # probabilities to the token shares the table actually sees; FineMath docs
+ # are 2.4x longer, so a nominal 5% of documents is about 11% of tokens.
+ means = {}
+ for name in doc_weights:
+ dirs = _source_dirs(root, name)
+ nt = nd = 0
+ for d in dirs:
+ idx = np.memmap(d / "docs.idx", dtype=np.uint64, mode="r")
+ nt += int(idx[-1]); nd += idx.size - 1
+ means[name] = nt / max(nd, 1)
+ z = sum(doc_weights[n] * means[n] for n in doc_weights) or 1.0
+ weights = {n: doc_weights[n] * means[n] / z for n in doc_weights}
+
+ readers = []
+ for name, share in sorted(weights.items()):
+ if share <= 0:
+ continue
+ dirs = _source_dirs(root, name)
+ if not dirs:
+ log(f" {name}: not prepared, excluded from the n-gram sample")
+ continue
+ per_dir = int(limit_tokens * share) // len(dirs)
+ for d in dirs:
+ a = np.memmap(d / "tokens.bin", dtype=np.uint16, mode="r")
+ n_chunks = max(1, per_dir // chunk_tokens)
+ if a.size <= chunk_tokens:
+ offs = [0]
+ else:
+ offs = np.linspace(0, a.size - chunk_tokens, n_chunks).astype(np.int64)
+ readers.append([d.name, a, list(offs), per_dir])
+ log(f" {name:<16} {d.name:<20} {per_dir/1e9:5.2f}B from "
+ f"{len(offs)} spans of {a.size/1e9:.2f}B")
+
+ used = {r[0]: 0 for r in readers}
+ alive = True
+ while alive and sum(used.values()) < limit_tokens:
+ alive = False
+ for name, a, offs, budget in readers:
+ if not offs or used[name] >= budget:
+ continue
+ o = offs.pop(0)
+ b = np.asarray(a[o:o + chunk_tokens])
+ used[name] += b.size
+ alive = True
+ yield b
+
+
+def build(root: Path, out: Path, cfg, full_cfg, vocab_size: int,
+ sample_tokens: int = 3_000_000_000, cap: int = 120_000_000):
+ print(f" sampling {sample_tokens/1e9:.1f}B tokens, weighted by the "
+ f"stable-phase mixture")
+ if not _source_dirs(root, "*"):
+ raise SystemExit(f"no prepared sources under {root}")
+
+ bk, bc = _topk_counts(_sample_stream(root, full_cfg, sample_tokens),
+ cfg.bigram_rows, 20_000_000, cap, 2, vocab_size)
+ tk, tc = _topk_counts(_sample_stream(root, full_cfg, sample_tokens,
+ log=lambda *a: None),
+ cfg.trigram_rows, 20_000_000, cap, 3, vocab_size)
+
+ # Row indices in descending frequency: hot rows contiguous at the head of
+ # each block. Keys are then sorted ascending for searchsorted; the row array
+ # rides along and is deliberately not sorted.
+ def assign(keys, counts, base):
+ rank = np.argsort(-counts, kind="stable")
+ rows = np.empty(keys.size, dtype=np.int64)
+ rows[rank] = base + np.arange(keys.size)
+ o = np.argsort(keys, kind="stable")
+ return keys[o], rows[o]
+
+ bkey, brow = assign(bk, bc, cfg.bigram_base)
+ tkey, trow = assign(tk, tc, cfg.trigram_base)
+ v = NGramVocab(vocab_size, bkey, brow, tkey, trow)
+ v.save(out)
+ print(f" wrote {out} ({bkey.size:,} bigram + {tkey.size:,} trigram rows, "
+ f"fingerprint {v.fingerprint()})")
+
+
+if __name__ == "__main__":
+ sys.path.insert(0, str(Path(__file__).resolve().parent))
+ from config import Config
+
+ ap = argparse.ArgumentParser()
+ ap.add_argument("cmd", choices=["build", "inspect"])
+ ap.add_argument("--shards", type=Path, default=None,
+ help="prepared-data root (defaults to config.data.root)")
+ ap.add_argument("--out", type=Path, default=None)
+ ap.add_argument("--sample-tokens", type=int, default=3_000_000_000)
+ ap.add_argument("--config", type=Path, default=None)
+ a = ap.parse_args()
+ cfg = Config.from_json(a.config.read_text()) if a.config else Config()
+ shards = a.shards or Path(cfg.data.root)
+ out = a.out or cfg.ngram.vocab_path
+ if a.cmd == "build":
+ build(shards, out, cfg.ngram, cfg, cfg.model.vocab_size, a.sample_tokens)
+ else:
+ v = NGramVocab.load(out)
+ print(f" vocab {v.vocab_size} bigrams {v.bigram_keys.size:,} "
+ f"trigrams {v.trigram_keys.size:,} fingerprint {v.fingerprint()}")
diff --git a/optim.py b/optim.py
new file mode 100644
index 0000000..7d87abb
--- /dev/null
+++ b/optim.py
@@ -0,0 +1,293 @@
+"""Muon on the 2-D matmul weights, AdamW on everything else.
+
+The split is by role, tagged on each Parameter at construction, and never by
+sniffing ndim. Canon kernels are (768, 3) tensors: two-dimensional, and
+emphatically not matrices to orthogonalise. A shape rule would hand all 26 of
+them to Muon and nothing would ever complain -- the run would simply be a little
+worse for several weeks. `build_optimizers` prints the full assignment and
+asserts it against config.param_budget() before the first step.
+
+Newton-Schulz is batched across same-shaped matrices: the quintic iteration
+becomes a handful of bmm chains instead of 183 independent ones, ~60 kernel
+launches per optimizer step in place of ~2,300. It runs in fp32 -- 1.8 measured
+a bf16 iteration diverging 5.4e-02 relative from the fp32 path, because five
+steps amplify rounding well past bf16 epsilon, and sm_70 has no bf16 tensor
+cores to gain from anyway.
+
+Some Parameters hold several logical matrices. Q, K and V live in one (768, 512)
+tensor and gate/up in one (4736, 512), because storing them fused removes a
+torch.cat per layer per micro-step from the forward. Those Parameters carry a
+`_boris_blocks` tag giving the row range of each logical matrix, and Muon
+orthogonalises each range independently -- which is precisely what it would do
+to separate Parameters, since the iteration acts on one matrix at a time.
+Momentum is elementwise and so is unaffected by the split.
+"""
+from __future__ import annotations
+
+import math
+
+import torch
+from torch.optim import AdamW
+
+MUON_ROLES = {"matmul"}
+ADAMW_ROLES = {"embedding", "ngram_table", "canon", "norm"}
+# Roles whose SCALE is a free parameter that nothing else restrains.
+DECAY_ROLES = {"embedding", "ngram_table"}
+
+# Quintic coefficients from Jordan et al.; tuned so the iteration converges on
+# the singular values of a Frobenius-normalised matrix in five steps.
+_NS_ABC = (3.4445, -4.7750, 2.0315)
+
+
+@torch.no_grad()
+def newton_schulz(X: torch.Tensor, steps: int = 5, eps: float = 1e-7) -> torch.Tensor:
+ """Orthogonalise a batch of matrices. X is (G, m, n) with m <= n.
+
+ Frobenius-normalised per matrix, so every matrix in the batch converges on
+ its own scale rather than the batch's.
+ """
+ a, b, c = _NS_ABC
+ X = X / (X.norm(dim=(1, 2), keepdim=True) + eps)
+ for _ in range(steps):
+ # Same polynomial, with the scalar terms folded into the GEMM epilogues
+ # (swarm muon_ns "fused32"): B = b*A + c*A@A and X = a*X + B@X are one
+ # baddbmm each instead of a bmm plus two elementwise passes. Error vs
+ # fp64 equal or lower (4.185e-6 -> 4.143e-6 median), ~7% faster.
+ A = torch.bmm(X, X.transpose(1, 2))
+ B = torch.baddbmm(A, A, A, beta=b, alpha=c)
+ X = torch.baddbmm(X, B, X, beta=a)
+ return X
+
+
+class Muon(torch.optim.Optimizer):
+ """Momentum, then an orthogonalised update, on 2-D matmul weights only."""
+
+ def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True, ns_steps=5,
+ weight_decay=0.1, batched=True, ns_dtype=torch.float32,
+ rms_scale=True):
+ super().__init__(list(params), dict(lr=lr, momentum=momentum,
+ nesterov=nesterov, ns_steps=ns_steps,
+ weight_decay=weight_decay))
+ self.batched = batched
+ self.ns_dtype = ns_dtype
+ self.rms_scale = rms_scale
+ self._plan: list | None = None
+
+ @staticmethod
+ def _units(group):
+ """(parameter, row range or None, matrix shape) for every logical matrix."""
+ out = []
+ for p in group["params"]:
+ if p.grad is None:
+ continue
+ blocks = getattr(p, "_boris_blocks", None)
+ if blocks is None:
+ out.append((p, None, tuple(p.shape)))
+ else:
+ for a, b in blocks:
+ out.append((p, (int(a), int(b)), (int(b) - int(a), p.shape[1])))
+ return out
+
+ def _build_plan(self, group):
+ """Bucket logical matrices by canonical shape once; shapes never change.
+
+ Orthogonalisation commutes with transpose, so a (2368, 512) matrix and a
+ (512, 2368) one share a bucket -- the wide orientation is canonical and
+ the narrow ones are transposed in and back out.
+ """
+ buckets: dict[tuple, list] = {}
+ for p, rng, shape in self._units(group):
+ m, n = shape
+ buckets.setdefault((min(m, n), max(m, n)), []).append((p, rng, shape, m > n))
+ return list(buckets.items())
+
+ @torch.no_grad()
+ def step(self, closure=None):
+ loss = closure() if closure is not None else None
+ for group in self.param_groups:
+ lr, mom = group["lr"], group["momentum"]
+ nesterov, wd = group["nesterov"], group["weight_decay"]
+ steps = group["ns_steps"]
+
+ # momentum, in place, on every parameter
+ bufs = []
+ for p in group["params"]:
+ if p.grad is None:
+ continue
+ st = self.state[p]
+ if "momentum_buffer" not in st:
+ st["momentum_buffer"] = torch.zeros_like(p)
+ buf = st["momentum_buffer"]
+ buf.mul_(mom).add_(p.grad)
+ g = p.grad.add(buf, alpha=mom) if nesterov else buf
+ bufs.append((p, g))
+
+ gmap = {id(p): g for p, g in bufs}
+
+ if not self.batched:
+ for p, rng, shape in self._units(group):
+ g = gmap[id(p)]
+ g = g if rng is None else g[rng[0]:rng[1]]
+ tr = shape[0] > shape[1]
+ x = (g.t() if tr else g).to(self.ns_dtype)[None]
+ o = newton_schulz(x, steps)[0]
+ self._apply(p, rng, shape, o.t() if tr else o, lr, wd)
+ continue
+
+ if self._plan is None:
+ self._plan = self._build_plan(group)
+ for _key, members in self._plan:
+ stack = torch.stack([
+ ((gmap[id(p)] if rng is None else gmap[id(p)][rng[0]:rng[1]]).t()
+ if tr else
+ (gmap[id(p)] if rng is None else gmap[id(p)][rng[0]:rng[1]])
+ ).to(self.ns_dtype)
+ for p, rng, shape, tr in members])
+ out = newton_schulz(stack, steps)
+ for i, (p, rng, shape, tr) in enumerate(members):
+ o = out[i]
+ self._apply(p, rng, shape, o.t() if tr else o, lr, wd)
+ return loss
+
+ def _apply(self, p, rng, shape, o, lr, wd):
+ # 0.2*sqrt(max(m,n)) makes the update RMS the same for a 512x512 matrix
+ # and a 512x2368 one, so a single LR is meaningful across a 4.63x FFN.
+ # Computed from the LOGICAL matrix shape, not the storage tensor.
+ s = 0.2 * math.sqrt(max(shape)) if self.rms_scale else 1.0
+ t = p.data if rng is None else p.data[rng[0]:rng[1]]
+ if wd:
+ t.mul_(1 - lr * wd)
+ t.add_(o.to(t.dtype), alpha=-lr * s)
+
+
+# --------------------------------------------------------------------------- #
+# The split
+# --------------------------------------------------------------------------- #
+def _role(p, name: str) -> str:
+ r = getattr(p, "_boris_role", None)
+ if r is None:
+ raise AssertionError(
+ f"parameter '{name}' has no _boris_role tag. Every parameter must be "
+ f"assigned to an optimizer explicitly at construction -- an untagged "
+ f"parameter is a misconfiguration, not a default.")
+ return r
+
+
+def build_optimizers(model, cfg, log=print):
+ muon_params, adamw_params = [], []
+ adamw_roles = []
+ rows = []
+ for name, p in model.named_parameters():
+ if not p.requires_grad:
+ continue
+ r = _role(p, name)
+ if r in MUON_ROLES:
+ muon_params.append(p); which = "muon"
+ elif r in ADAMW_ROLES:
+ adamw_params.append(p); adamw_roles.append((p, r)); which = "adamw"
+ else:
+ raise AssertionError(f"parameter '{name}' has unknown role '{r}'")
+ rows.append((name, tuple(p.shape), p.numel(), r, which))
+
+ o = cfg.optim
+ muon = Muon(muon_params, lr=o.muon_lr, momentum=o.muon_momentum,
+ nesterov=o.muon_nesterov, ns_steps=o.muon_ns_steps,
+ weight_decay=o.muon_weight_decay, batched=o.muon_batched,
+ ns_dtype=getattr(torch, o.muon_ns_dtype), rms_scale=o.muon_rms_scale)
+ # 2026-09-17: AdamW used ONE weight_decay for every role, and it was 0.0.
+ # Muon decays its matmuls at 0.1 and they are flat to within 1% over 9.3B
+ # tokens; the undecayed AdamW tensors are not. Measured 20.8B -> 30.1B:
+ # embed.weight 1879 -> 2275 (+21.1%) wd 0.0
+ # ngram.table 2747 -> 3326 (+21.1%) wd 0.0
+ # qkv_proj 167 -> 168 ( +0.4%) wd 0.1
+ # down_proj 327 -> 324 ( -1.0%) wd 0.1
+ # Nothing restrains the two largest tensors, so they inflate, norm_f shrinks
+ # to hold logit scale roughly constant (8.05 -> 7.12), and 66% of the total
+ # gradient power ends up in that one 512-vector fighting the embedding.
+ # AdamW's step is ~lr regardless of scale, so a growing ||w|| means the
+ # RELATIVE update lr/||w|| decays -- embedding directions calcify. Row norms
+ # grow ~+1.07 per 5B (linear), which extrapolates to ~4x by 200B.
+ # Decay the two big tensors; leave norms and canon alone, as is conventional
+ # -- norm gains are a scale, not a representation, and canon is near-flat
+ # already (+0.8% to +5.7%).
+ #
+ # 2026-10-07: canon is no longer near-flat. At the corrected Muon lr (0929) it
+ # grew ~+20% per 10B with nothing restraining it, and the norm1 gains shrank
+ # to compensate. It gets its own group (adamw_canon_weight_decay); the group
+ # ORDER is [decay, norms, canon] and checkpoint._migrate_adamw_groups remaps
+ # older 1- and 2-group states onto it by parameter identity.
+ decay = [p for p, r in adamw_roles if r in DECAY_ROLES]
+ norms = [p for p, r in adamw_roles if r not in DECAY_ROLES and r != "canon"]
+ canon = [p for p, r in adamw_roles if r == "canon"]
+ adamw = AdamW([{"params": decay, "weight_decay": o.adamw_weight_decay},
+ {"params": norms, "weight_decay": o.adamw_norm_weight_decay},
+ {"params": canon, "weight_decay": o.adamw_canon_weight_decay}],
+ lr=o.adamw_lr, betas=tuple(o.adamw_betas),
+ eps=o.adamw_eps, fused=torch.cuda.is_available())
+ n = lambda ps: f"{len(ps)} tensors ({sum(p.numel() for p in ps):,} params)"
+ log(f"\n adamw decay groups: {n(decay)} at wd {o.adamw_weight_decay}, "
+ f"{n(norms)} at wd {o.adamw_norm_weight_decay}, "
+ f"canon {n(canon)} at wd {o.adamw_canon_weight_decay}")
+
+ _report_and_assert(rows, muon_params, adamw_params, model, cfg, log)
+ return muon, adamw
+
+
+def _report_and_assert(rows, muon_params, adamw_params, model, cfg, log):
+ by_role: dict[str, list] = {}
+ for name, shape, n, role, which in rows:
+ by_role.setdefault(role, []).append((name, shape, n, which))
+
+ log("\n parameter groups")
+ log(" " + "-" * 74)
+ log(f" {'role':<13}{'optimizer':<10}{'tensors':>9}{'parameters':>15} example shape")
+ total = 0
+ for role in ("matmul", "embedding", "ngram_table", "canon", "norm"):
+ if role not in by_role:
+ continue
+ e = by_role[role]
+ n = sum(x[2] for x in e)
+ total += n
+ log(f" {role:<13}{e[0][3]:<10}{len(e):>9}{n:>15,} {e[0][1]}")
+ log(" " + "-" * 74)
+ n_muon = sum(p.numel() for p in muon_params)
+ n_adamw = sum(p.numel() for p in adamw_params)
+ n_mat = sum(len(getattr(p, "_boris_blocks", (None,))) for p in muon_params)
+ log(f" {'MUON':<23}{len(muon_params):>9}{n_muon:>15,} "
+ f"{n_mat} logical matrices")
+ log(f" {'ADAMW':<23}{len(adamw_params):>9}{n_adamw:>15,}")
+ log(f" {'TOTAL':<23}{len(rows):>9}{total:>15,}")
+
+ ids_m = {id(p) for p in muon_params}
+ ids_a = {id(p) for p in adamw_params}
+ assert not (ids_m & ids_a), "a parameter landed in both optimizers"
+ all_ids = {id(p) for p in model.parameters() if p.requires_grad}
+ assert ids_m | ids_a == all_ids, (
+ f"{len(all_ids - (ids_m | ids_a))} trainable parameters are in no optimizer")
+
+ for p in muon_params:
+ assert p.ndim == 2 and min(p.shape) > 3, (
+ f"a non-matrix of shape {tuple(p.shape)} reached Muon")
+ blocks = getattr(p, "_boris_blocks", None)
+ if blocks is not None:
+ assert blocks[0][0] == 0 and blocks[-1][1] == p.shape[0], (
+ f"_boris_blocks {blocks} do not tile the {tuple(p.shape)} tensor")
+ for (a1, b1), (a2, _) in zip(blocks, blocks[1:]):
+ assert b1 == a2, f"_boris_blocks {blocks} overlap or leave a gap"
+ b = cfg.param_budget()
+ assert n_muon == b["matmul"], (
+ f"Muon holds {n_muon:,} parameters, the budget says {b['matmul']:,}")
+ assert total == b["total"], (
+ f"model has {total:,} parameters, the budget says {b['total']:,}")
+ cfg.assert_budget()
+ log(f" budget assertions pass; fingerprint {cfg.fingerprint()}\n")
+
+
+def set_lr(muon, adamw, cfg, scale: float) -> tuple[float, float]:
+ ml = cfg.optim.muon_lr * scale
+ al = cfg.optim.adamw_lr * scale
+ for g in muon.param_groups:
+ g["lr"] = ml
+ for g in adamw.param_groups:
+ g["lr"] = al
+ return ml, al
diff --git a/preflight.py b/preflight.py
new file mode 100644
index 0000000..4481b8b
--- /dev/null
+++ b/preflight.py
@@ -0,0 +1,238 @@
+"""Measure throughput before committing weeks to a setting.
+
+ python preflight.py # measure the configured setting
+ python preflight.py --sweep 4,8,16 # micro-batch sweep
+ python preflight.py --ablate # what each optimisation is worth
+ python preflight.py --block 8192 --sweep 1,2,4
+
+Every number in the plan is an estimate until this has run on the actual card
+with the actual shapes. Re-run it after any change to the model or the kernels:
+the optimum micro-batch moves as bottlenecks fall, so an early measurement
+expires.
+"""
+from __future__ import annotations
+
+import argparse
+import math
+import os
+
+os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
+
+import time
+from pathlib import Path
+
+import numpy as np
+import torch
+
+from config import Config
+from model import Boris2
+from ngram import NGramVocab
+from optim import build_optimizers, set_lr
+from train import step_flops
+
+
+def synth_vocab(cfg, seed=0):
+ V = cfg.model.vocab_size
+ rng = np.random.default_rng(seed)
+ bk = np.unique(rng.integers(0, V * V, size=cfg.ngram.bigram_rows).astype(np.int64))
+ tk = np.unique(rng.integers(0, V * V * V, size=cfg.ngram.trigram_rows).astype(np.int64))
+ return NGramVocab(V, bk, np.arange(bk.size) + cfg.ngram.bigram_base,
+ tk, np.arange(tk.size) + cfg.ngram.trigram_base)
+
+
+def make_batch(cfg, dev, B, T, seed=0, worst_case=False):
+ """Realistic packed rows: lognormal document lengths, mean ~560 tokens.
+
+ worst_case makes every row a single document of the full block size. That is
+ the largest attention any batch can produce -- cost is sum(len_i^2), so one
+ 2048-token document costs what fourteen 560-token ones do -- and it is the
+ case the chosen micro_batch has to survive, not the average one.
+ """
+ g = torch.Generator().manual_seed(seed)
+ bounds, pos, bnd = [], np.zeros((B, T), np.int64), np.zeros((B, T), bool)
+ for b in range(B):
+ p = 0
+ while p < T:
+ L = T if worst_case else int(min(T - p, max(32, torch.empty(1).log_normal_(6.4, 0.9, generator=g).item())))
+ bounds.append(b * T + p)
+ bnd[b, p] = True
+ pos[b, p:p + L] = np.arange(L)
+ p += L
+ bounds.append(B * T)
+ lens = np.diff(np.asarray(bounds))
+ V = cfg.model.vocab_size
+ t = lambda a, d=torch.int64: torch.from_numpy(np.ascontiguousarray(a)).to(dev, d)
+ return (torch.randint(0, V, (B, T), device=dev),
+ torch.randint(0, V, (B, T), device=dev),
+ t(pos), t(bnd, torch.bool), t(np.asarray(bounds)),
+ int((lens * lens).sum()), int(lens.min()), int(lens.max()))
+
+
+def build(cfg, dev, compile_regions):
+ cfg.train.compile_regions = compile_regions
+ m = Boris2(cfg).to(dev)
+ if m.ngram is not None:
+ m.ngram.load_vocab(synth_vocab(cfg), dev)
+ if compile_regions:
+ m.maybe_compile()
+ return m
+
+
+def measure(cfg, dev, B, T, iters=6, warmup=3, compile_regions=True, log=print,
+ worst_case=False):
+ torch.cuda.set_device(dev)
+ torch.cuda.empty_cache()
+ torch.cuda.reset_peak_memory_stats(dev)
+ m = build(cfg, dev, compile_regions)
+ muon, adamw = build_optimizers(m, cfg, log=lambda *a: None)
+ scaler = torch.amp.GradScaler("cuda", init_scale=2. ** 16)
+ batches = [make_batch(cfg, dev, B, T, seed=i, worst_case=worst_case)
+ for i in range(3)]
+
+ def one(i):
+ x, y, pos, bnd, off, sumsq, lo, hi = batches[i % len(batches)]
+ with torch.autocast("cuda", dtype=torch.float16):
+ loss = m(x, pos, off, bnd, y, 10_000.0, lo, hi)
+ scaler.scale(loss).backward()
+ scaler.unscale_(muon); scaler.unscale_(adamw)
+ # Match train.py. clip_grad_norm_ plus two scaler.step calls benchmarked
+ # a different synchronization and overflow path than the real run, so
+ # its micro-batch ranking could be wrong by the margin we care about.
+ grads = [p.grad for p in m.parameters() if p.grad is not None]
+ norms = torch._foreach_norm(grads)
+ gnorm = torch.linalg.vector_norm(torch.stack(norms))
+ coef = torch.where(torch.isfinite(gnorm),
+ (cfg.optim.grad_clip / (gnorm + 1e-6)).clamp(max=1.0),
+ torch.ones_like(gnorm))
+ torch._foreach_mul_(grads, coef)
+ if math.isfinite(float(loss.detach())) and math.isfinite(float(gnorm)):
+ muon.step(); adamw.step()
+ scaler.update()
+ muon.zero_grad(set_to_none=True); adamw.zero_grad(set_to_none=True)
+ return sumsq
+
+ for i in range(warmup):
+ one(i)
+ torch.cuda.synchronize()
+ t0 = time.perf_counter()
+ ss = 0
+ for i in range(iters):
+ ss += one(i)
+ torch.cuda.synchronize()
+ dt = time.perf_counter() - t0
+
+ tok = B * T * iters
+ tps = tok / dt
+ fl = step_flops(cfg, tok, ss) / dt / 1e12
+ peak = torch.cuda.max_memory_allocated(dev) / 2 ** 30
+ del m, muon, adamw
+ torch.cuda.empty_cache()
+ return tps, fl, peak
+
+
+def report_row(label, tps, fl, peak, total_tokens, peak_tflops):
+ days = total_tokens / tps / 86400
+ print(f" {label:<22}{tps/1e3:>9.2f}k tok/s{fl:>9.1f} TF"
+ f"{100*fl/peak_tflops:>7.1f}%{peak:>8.1f} GB{days:>9.1f} d")
+
+
+def main():
+ cfg = Config()
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--device", default=cfg.train.device)
+ ap.add_argument("--block", type=int, default=None)
+ ap.add_argument("--sweep", default=None)
+ ap.add_argument("--ablate", action="store_true")
+ ap.add_argument("--worst-case", action="store_true",
+ help="one full-block document per row: the largest attention "
+ "any batch can produce")
+ ap.add_argument("--iters", type=int, default=6)
+ a = ap.parse_args()
+
+ dev = a.device
+ # Set the current device before anything allocates. torch.cuda.empty_cache()
+ # and the allocator statistics act on the CURRENT device, not on the device
+ # the tensors are on -- leaving it at cuda:0 while measuring cuda:1 mixes
+ # two allocators and surfaces as an illegal memory access, not as an error
+ # that names the cause.
+ torch.cuda.set_device(dev)
+ torch.backends.cuda.matmul.allow_tf32 = False
+ torch.backends.cuda.matmul.allow_fp16_reduced_precision_reduction = True
+ torch.backends.cudnn.benchmark = True
+ p = torch.cuda.get_device_properties(dev)
+ # fp16 tensor-core peak: V100 SXM2 125, PCIe 112, and GeForce halves the
+ # fp32-accumulate rate. Only used to turn TFLOPS into a percentage.
+ peak_tflops = 125.0 if p.major == 7 else (2 * p.multi_processor_count * 128 * 2
+ * 1.7e9 / 1e12)
+ T = a.block or cfg.model.context_stages[0][1]
+
+ print(f"\n preflight: {cfg.run_name} on {p.name} (sm_{p.major}{p.minor}, "
+ f"{p.total_memory/2**30:.1f} GB)")
+ print(f" block {T}, step {cfg.train.tokens_per_step:,} tokens, "
+ f"budget {cfg.train.total_tokens/1e9:.0f}B")
+ print(f" assumed fp16 peak {peak_tflops:.0f} TFLOPS\n")
+ print(f" {'setting':<22}{'throughput':>17}{'':>9}{'MFU':>7}{'peak':>8}{'200B in':>9}")
+ print(" " + "-" * 72)
+
+ if a.sweep:
+ for mb in [int(x) for x in a.sweep.split(",")]:
+ if cfg.train.tokens_per_step % (mb * T):
+ print(f" micro_batch {mb:<10}skipped: {mb}x{T} does not divide "
+ f"the {cfg.train.tokens_per_step:,}-token step")
+ continue
+ try:
+ tps, fl, peak = measure(cfg, dev, mb, T, iters=a.iters,
+ worst_case=a.worst_case)
+ report_row(f"micro_batch {mb}" + (" worst-case" if a.worst_case else ""),
+ tps, fl, peak, cfg.train.total_tokens, peak_tflops)
+ except torch.OutOfMemoryError:
+ print(f" micro_batch {mb:<10}OOM")
+ torch.cuda.empty_cache()
+ elif a.ablate:
+ mb = cfg.train.micro_batch[T]
+ variants = [("all optimisations", dict()),
+ ("no compiled regions", dict(compile=False)),
+ ("no fused Canon", dict(canon=False)),
+ ("fp32 residual", dict(fp16res=False)),
+ ("unbatched Newton-Schulz", dict(batched=False))]
+ for label, v in variants:
+ c = Config.from_json(cfg.to_json())
+ if v.get("canon") is False:
+ c.model.use_canon = False
+ if v.get("fp16res") is False:
+ c.train.fp16_residual = False
+ if v.get("batched") is False:
+ c.optim.muon_batched = False
+ try:
+ tps, fl, peak = measure(c, dev, mb, T, iters=a.iters,
+ compile_regions=v.get("compile", True))
+ report_row(label, tps, fl, peak, cfg.train.total_tokens, peak_tflops)
+ except torch.OutOfMemoryError:
+ print(f" {label:<22}OOM"); torch.cuda.empty_cache()
+ else:
+ mb = cfg.train.micro_batch[T]
+ tps, fl, peak = measure(cfg, dev, mb, T, iters=a.iters,
+ worst_case=a.worst_case)
+ report_row(f"micro_batch {mb}" + (" worst-case" if a.worst_case else ""),
+ tps, fl, peak, cfg.train.total_tokens, peak_tflops)
+
+ print(" " + "-" * 72)
+ print(" The '200B in' column extrapolates ONE stage across the whole budget,")
+ print(" which no real run does. The honest projection is two stages:")
+ frac = cfg.model.context_stages[-1][0]
+ t1 = cfg.train.total_tokens * frac
+ t2 = cfg.train.total_tokens - t1
+ print(f" 0-{frac:.0%} {t1/1e9:.0f}B at ctx {cfg.model.context_stages[0][1]} "
+ f"(micro_batch {cfg.train.micro_batch[cfg.model.context_stages[0][1]]})")
+ print(f" {frac:.0%}-100% {t2/1e9:.0f}B at ctx {cfg.model.context_stages[-1][1]} "
+ f"(micro_batch {cfg.train.micro_batch[cfg.model.context_stages[-1][1]]})")
+ print(" and the second stage is much slower per token, because attention")
+ print(" costs sum(len_i^2) and the curriculum deliberately feeds it long")
+ print(" documents once the window opens. Run --block 8192 to measure it.\n")
+ print(" Peak VRAM is measured on lognormal document lengths. Re-run with")
+ print(" --worst-case (one full-block document per row) before committing a")
+ print(" micro_batch: that is the largest attention any batch can produce.\n")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/prepare.py b/prepare.py
new file mode 100644
index 0000000..d4f0188
--- /dev/null
+++ b/prepare.py
@@ -0,0 +1,645 @@
+"""Corpus preparation: stream, decontaminate, tokenize, shard.
+
+ python prepare.py plan # token targets per source
+ python prepare.py sample --mb 2000 # text sample for the tokenizer
+ python prepare.py source --name ultra_fineweb --tokens 120e9
+ python prepare.py val --tokens 20e6
+ python prepare.py status
+
+Sources are streamed from the Hub and written straight out as uint16 tokens.
+Nothing keeps the raw text: 200B tokens is ~400GB stored, and the text it came
+from is more than twice that -- there is 664GB free on the Media SSD and 20GB on
+/home, so the token files live on Media and the text never lands at all.
+
+Each source becomes data/<name>/tokens.bin (uint16) plus docs.idx (uint64
+document start offsets). Both are appended to incrementally and a progress file
+records how far the stream got, so preparation is resumable and training can
+start on the first sources while the rest are still arriving.
+
+Decontamination runs inline: any document sharing a 13-gram with an evaluation
+item is dropped before it is ever written. Filtering afterwards would mean
+rewriting a 200GB file.
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import re
+import os
+import sys
+import time
+from pathlib import Path
+
+import numpy as np
+
+from config import DATA_DIR, Config
+from tokenizer import BorisTokenizer
+
+os.environ.setdefault("TOKENIZERS_PARALLELISM", "true")
+
+# name -> (repo, configs, split, text field). Resolved against the Hub on
+# 2026-09-06 with probe/sources_probe.py, not from memory -- Ultra-FineWeb
+# splits by language rather than "train", and finephrase has five configs, so
+# both of those failed when they were guessed. Run `prepare.py check` after any
+# edit here; it opens every source and pulls one record.
+#
+# `configs` may be a tuple: the source is consumed in that order until its token
+# target is met. Cosmopedia needs this -- no single config holds enough.
+SOURCES = {
+ "ultra_fineweb": ("openbmb/Ultra-FineWeb", ("default",), "en", "content"),
+ "cosmopedia": ("HuggingFaceTB/cosmopedia",
+ ("web_samples_v2", "web_samples_v1", "auto_math_text",
+ "stories", "openstax", "stanford", "wikihow", "khanacademy"),
+ "train", "text"),
+ "finephrase": ("HuggingFaceFW/finephrase", ("all",), "train", "text"),
+ "fineweb_edu": ("HuggingFaceFW/fineweb-edu", ("sample-100BT",), "train", "text"),
+ "dclm": ("mlfoundations/dclm-baseline-1.0-parquet", ("default",),
+ "train", "text"),
+ # L3 of openbmb's L0-L4 tiering: Ultra-FineWeb (the L2 selected layer) put
+ # through Q&A generation and multi-style rewriting with MiniCPM4/Qwen3.
+ # Their own decay-phase data for MiniCPM5-1B, which is exactly what we use
+ # it for -- it carries the decay weight that would otherwise go to L2
+ # general web text.
+ # Math. The eval suite includes ArithMark-3 (arithmetic reasoning) and the
+ # rest of this corpus has no dedicated mathematical content -- an omission,
+ # not a choice. finemath-4plus is the high-quality tier: ~39GB of text
+ # filtered from CommonCrawl by a classifier trained on Llama-3.1-70B
+ # annotations, roughly 8B tokens under our BPE.
+ # 4plus first (the high-quality tier, ~8B tokens under our BPE), then
+ # 3plus as the fallback -- the curriculum draws 10.7B, more than 4plus holds.
+ "finemath": ("HuggingFaceTB/finemath",
+ ("finemath-4plus", "finemath-3plus"), "train", "text"),
+ # 2026-10-10: elementary math facts. Preview-1 sat at chance on ArithMark
+ # division/patterns/multiplication where GPT-X3 (10% UltraData-Math L3 for
+ # its whole run) scores 64-92%. L3 is rewritten math in four styles; one
+ # worker per config so the four stay balanced.
+ "ultra_math_l3": ("openbmb/UltraData-Math",
+ ("UltraData-Math-L3-QA-Synthetic",
+ "UltraData-Math-L3-Textbook-Exercise-Synthetic",
+ "UltraData-Math-L3-Multi-Style-Synthetic",
+ "UltraData-Math-L3-Conversation-Synthetic"),
+ "train", "content"),
+ "ultra_fineweb_l3": ("openbmb/Ultra-FineWeb-L3",
+ ("Ultra-FineWeb-L3-en-QA-Synthetic",
+ "Ultra-FineWeb-L3-en-Multi-Style-Synthetic"),
+ "train", "content"),
+}
+
+NGRAM_N = 13
+BAN_CACHE = DATA_DIR / "decontam_ngrams.npy"
+
+
+# --------------------------------------------------------------------------- #
+_NON_ALNUM = re.compile(r"[^0-9a-z]+")
+
+
+def _norm(s: str) -> list[str]:
+ """Lowercase, split on anything non-alphanumeric.
+
+ A per-character Python generator here ran once over every document in the
+ corpus -- hundreds of billions of interpreted operations. The regex does the
+ same work in C.
+ """
+ return _NON_ALNUM.sub(" ", s.lower()).split()
+
+
+def _h(s: str) -> int:
+ """Stable 64-bit hash.
+
+ Not builtin hash(): PYTHONHASHSEED randomises string hashing per process, so
+ a set built in one process is meaningless in another -- and the ban set is
+ built once and shared by five. 64 bits keeps false positives negligible
+ against ~900k banned n-grams; a 32-bit hash would falsely drop ~0.8% of all
+ documents.
+ """
+ return int.from_bytes(hashlib.blake2b(s.encode("utf-8", "ignore"),
+ digest_size=8).digest(), "big",
+ signed=True)
+
+
+def eval_ngrams(log=print, use_cache: bool = True):
+ """Sorted array of banned 13-gram hashes, over every evaluation item.
+
+ Built once and cached: five prepare processes run in parallel and each was
+ rebuilding this from scratch.
+ """
+ import numpy as _np
+ if use_cache and BAN_CACHE.exists():
+ a = _np.load(BAN_CACHE)
+ log(f" decontamination: {a.size:,} banned n-grams (cached)")
+ return a
+
+ import eval as E
+ out = set()
+ missing = []
+ for name, loader in E.TASKS.items():
+ try:
+ items = loader()
+ except Exception as e:
+ missing.append(f"{name} ({type(e).__name__}: {str(e)[:60]})")
+ continue
+ n0 = len(out)
+ for it in items:
+ for text in [it["ctx"]] + list(it["choices"]):
+ w = _norm(text)
+ for i in range(max(0, len(w) - NGRAM_N + 1)):
+ out.add(_h(" ".join(w[i:i + NGRAM_N])))
+ log(f" {name}: {len(items)} items, +{len(out)-n0:,} n-grams")
+ if missing:
+ # Never a warning. A benchmark that silently skips decontamination
+ # produces a score that looks like a result and is not one.
+ raise SystemExit(
+ " decontamination could not load: " + "; ".join(missing) +
+ "\n Refusing to write training data that has not been checked "
+ "against every evaluation set. Fix the loader in eval.py TASKS, or "
+ "pass --no-decontam if you accept scoring on contaminated data.")
+ a = _np.sort(_np.fromiter(out, dtype=_np.int64, count=len(out)))
+ BAN_CACHE.parent.mkdir(parents=True, exist_ok=True)
+ _np.save(BAN_CACHE, a)
+ log(f" decontamination: {a.size:,} banned n-grams -> {BAN_CACHE.name}")
+ return a
+
+
+def contaminated(text: str, ban) -> bool:
+ """True if the document shares a 13-gram window with an evaluation item.
+
+ The document side steps by NGRAM_N rather than by 1. The evaluation side
+ holds every overlapping window, so any shared span of 2*NGRAM_N-1 = 25 words
+ or more still contains one aligned document window and is always caught --
+ at a thirteenth of the work. Scanning every overlapping window here was
+ ~160 billion hashes over the full corpus, which does not finish.
+ """
+ w = _norm(text)
+ if len(w) < NGRAM_N or ban.size == 0:
+ return False
+ keys = np.fromiter(
+ (_h(" ".join(w[i:i + NGRAM_N]))
+ for i in range(0, len(w) - NGRAM_N + 1, NGRAM_N)),
+ dtype=np.int64)
+ if keys.size == 0:
+ return False
+ i = np.searchsorted(ban, keys)
+ i[i >= ban.size] = ban.size - 1
+ return bool((ban[i] == keys).any())
+
+
+# --------------------------------------------------------------------------- #
+class ShardWriter:
+ """Appends uint16 tokens and uint64 document offsets, crash-safely.
+
+ The offsets file is the authority: on resume the token file is truncated
+ back to the last complete document, so a kill mid-write cannot leave a torn
+ final document that the packer would read as real text.
+ """
+
+ def __init__(self, d: Path, eos_id: int):
+ self.d = Path(d); self.d.mkdir(parents=True, exist_ok=True)
+ self.bin = self.d / "tokens.bin"
+ self.idx = self.d / "docs.idx"
+ self.eos = eos_id
+ if not self.idx.exists():
+ np.array([0], dtype=np.uint64).tofile(self.idx)
+ off = np.fromfile(self.idx, dtype=np.uint64)
+ self.n_tokens = int(off[-1])
+ self.n_docs = off.size - 1
+ if self.bin.exists() and self.bin.stat().st_size != self.n_tokens * 2:
+ with open(self.bin, "r+b") as f:
+ f.truncate(self.n_tokens * 2)
+ self.fb = open(self.bin, "ab")
+ self.fi = open(self.idx, "ab")
+
+ def write(self, docs: list[list[int]]):
+ offs = []
+ buf = []
+ n = self.n_tokens
+ for ids in docs:
+ if not ids:
+ continue
+ ids = ids + [self.eos]
+ buf.append(np.asarray(ids, dtype=np.uint16))
+ n += len(ids)
+ offs.append(n)
+ if not offs:
+ return 0
+ np.concatenate(buf).tofile(self.fb)
+ np.asarray(offs, dtype=np.uint64).tofile(self.fi)
+ added = n - self.n_tokens
+ self.n_tokens = n
+ self.n_docs += len(offs)
+ return added
+
+ def flush(self):
+ self.fb.flush(); os.fsync(self.fb.fileno())
+ self.fi.flush(); os.fsync(self.fi.fileno())
+
+ def close(self):
+ self.flush(); self.fb.close(); self.fi.close()
+
+
+def progress_path(name: str) -> Path:
+ return DATA_DIR / name / "progress.json"
+
+
+def load_progress(name: str) -> dict:
+ p = progress_path(name)
+ return json.loads(p.read_text()) if p.exists() else {"seen": 0, "kept": 0,
+ "dropped": 0, "tokens": 0}
+
+
+def save_progress(name: str, d: dict):
+ p = progress_path(name)
+ p.parent.mkdir(parents=True, exist_ok=True)
+ p.write_text(json.dumps(d, indent=2))
+
+
+# --------------------------------------------------------------------------- #
+def resolve_files(repo: str, conf: str, split: str) -> list[str]:
+ """The concrete data files behind one config/split, in order.
+
+ Streaming a whole config and skipping consumed records on resume re-fetches
+ and re-decodes everything already done -- the cost of resuming grows with
+ how far you got, which is the wrong shape for a job that runs for a day.
+ Iterating file by file makes resume O(1): finished files are never touched
+ again.
+ """
+ from datasets import load_dataset_builder
+ b = load_dataset_builder(repo, conf)
+ df = b.config.data_files
+ files = df[split] if split in df else list(df.values())[0]
+ return [str(f) for f in files]
+
+
+def _stream_file(path: str, field: str):
+ """One data file, one column."""
+ from datasets import load_dataset
+ fmt = "json" if ".json" in path else "parquet"
+ ds = load_dataset(fmt, data_files=[path], split="train", streaming=True)
+ try:
+ ds = ds.select_columns([field]) # decode one column, not twelve
+ except Exception:
+ pass
+ return ds
+
+
+def prepare_source(name: str, target_tokens: int, cfg: Config, decontam=True,
+ batch=1000, log=print, file_start: int = 0,
+ file_end: int | None = None, out: str | None = None,
+ configs: tuple | None = None):
+ """Tokenize one source into data/<out or name>/.
+
+ file_start/file_end bound which data files this worker owns. A source whose
+ files are large enough is otherwise one serial process and sets the finish
+ time for the whole job: Ultra-FineWeb needs 106B tokens and was projected at
+ 48h alone while the link and CPU both had headroom. Disjoint file ranges let
+ several workers share it, and because resume is already file-indexed the
+ bookkeeping is just a bound on the loop.
+
+ data.py stitches data/<name>/ and data/<name>_s*/ back into one logical
+ source, so the shards are never copied or merged.
+ """
+ if name not in SOURCES:
+ raise SystemExit(f"unknown source '{name}'. Known: {', '.join(SOURCES)}")
+ repo, confs, split, field = SOURCES[name]
+ if configs:
+ confs = tuple(configs)
+ tok = BorisTokenizer.load(DATA_DIR / "tokenizer")
+ ban = eval_ngrams(log) if decontam else np.zeros(0, dtype=np.int64)
+ out = out or name
+ w = ShardWriter(DATA_DIR / out, tok.eos_id)
+ prog = load_progress(out)
+ # tokens.bin is appended immediately; progress.json is only checkpointed at
+ # file boundaries and every 200M tokens. A worker killed mid-file therefore
+ # has more on disk than its progress claims, and resuming at the recorded
+ # file would write those documents twice. Trust the writer, and skip the
+ # partially-consumed file -- losing the rest of one file is free, duplicate
+ # documents in the corpus are not.
+ if prog.get("tokens", 0) != w.n_tokens:
+ log(f" progress lagged the writer ({prog.get('tokens',0)/1e9:.3f}B "
+ f"recorded vs {w.n_tokens/1e9:.3f}B on disk); resuming past the "
+ f"partial file")
+ prog["tokens"] = w.n_tokens
+ if w.n_tokens:
+ prog["file_idx"] = prog.get("file_idx", file_start) + 1
+ log(f" {name}: {repo} {list(confs)}")
+ log(f" already {prog['tokens']/1e9:.2f}B tokens in {w.n_docs:,} documents")
+ if w.n_tokens >= target_tokens:
+ log(" target already met"); return
+
+ t0 = time.time()
+ texts, n_since = [], 0
+ try:
+ while prog.get("cfg_idx", 0) < len(confs) and prog["tokens"] < target_tokens:
+ conf = confs[prog.get("cfg_idx", 0)]
+ files = resolve_files(repo, conf, split)
+ hi = len(files) if file_end is None else min(file_end, len(files))
+ if prog.get("file_idx", 0) < file_start:
+ prog["file_idx"] = file_start
+ log(f" config {conf}: {len(files)} files, this worker owns "
+ f"[{file_start}, {hi}), resuming at file {prog.get('file_idx', 0)}")
+ while (prog.get("file_idx", 0) < hi
+ and prog["tokens"] < target_tokens):
+ fi = prog.get("file_idx", 0)
+ # A transient network error must not be fatal. Five of eight workers
+ # died overnight on a single httpx "client has been closed" -- one
+ # WiFi blip -- and sat idle for hours with the job half done. Retry
+ # the file; on persistent failure skip it, because every source here
+ # has far more files than the token budget needs.
+ attempt, rec_iter = 0, None
+ while attempt < 5:
+ try:
+ rec_iter = _stream_file(files[fi], field)
+ break
+ except Exception as e:
+ attempt += 1
+ log(f" file {fi}: open failed ({type(e).__name__}), "
+ f"retry {attempt}/5 in {2**attempt}s")
+ time.sleep(2 ** attempt)
+ if rec_iter is None:
+ log(f" file {fi}: unreadable after 5 attempts, skipping")
+ prog["file_idx"] = fi + 1
+ save_progress(out, prog)
+ continue
+ try:
+ for rec in rec_iter:
+ t = rec.get(field) or ""
+ if not t:
+ continue
+ if ban.size and contaminated(t, ban):
+ prog["dropped"] += 1
+ continue
+ texts.append(t)
+ if len(texts) >= batch:
+ added = w.write(tok.encode_batch(texts))
+ prog["kept"] += len(texts); prog["tokens"] += added
+ texts = []; n_since += added
+ if n_since > 200_000_000:
+ w.flush(); save_progress(out, prog); n_since = 0
+ r = prog["tokens"] / max(time.time() - t0, 1e-9)
+ log(f" {prog['tokens']/1e9:7.2f}B / {target_tokens/1e9:.0f}B "
+ f"file {fi+1}/{len(files)} {w.n_docs/1e6:6.2f}M docs "
+ f"{r/1e6:5.2f}M tok/s dropped {prog['dropped']:,} "
+ f"eta {(target_tokens-prog['tokens'])/max(r,1)/3600:5.1f}h")
+ if prog["tokens"] >= target_tokens:
+ break
+ except Exception as e:
+ # Mid-file failure: the documents already written are safe and
+ # the reconciliation on the next start skips the partial file.
+ log(f" file {fi}: stream failed mid-way ({type(e).__name__}: "
+ f"{str(e)[:80]}); moving on")
+ # This file is fully consumed; resume will never fetch it again.
+ prog["file_idx"] = fi + 1
+ if texts:
+ added = w.write(tok.encode_batch(texts))
+ prog["kept"] += len(texts); prog["tokens"] += added
+ texts = []; n_since += added
+ w.flush(); save_progress(out, prog)
+ if prog["tokens"] < target_tokens:
+ log(f" config {conf} exhausted at {prog['tokens']/1e9:.2f}B")
+ prog["cfg_idx"] = prog.get("cfg_idx", 0) + 1
+ prog["file_idx"] = file_start
+ save_progress(out, prog)
+ finally:
+ if texts:
+ prog["tokens"] += w.write(tok.encode_batch(texts))
+ prog["kept"] += len(texts)
+ w.close(); save_progress(out, prog)
+ log(f" done: {prog['tokens']/1e9:.2f}B tokens, {w.n_docs:,} documents, "
+ f"{prog['dropped']:,} dropped as contaminated")
+
+
+def prepare_sample(mb: int, cfg: Config, log=print):
+ """A stratified text sample for training the tokenizer."""
+ from datasets import load_dataset
+ out = DATA_DIR / "tokenizer_sample.txt"
+ out.parent.mkdir(parents=True, exist_ok=True)
+ per = mb * 1024 * 1024 // len(SOURCES)
+ with open(out, "w") as f:
+ for name, (repo, confs, split, field) in SOURCES.items():
+ n = 0
+ # Not skipped on failure. The tokenizer is the one artifact that
+ # cannot be changed later without invalidating every token file, the
+ # n-gram vocab and every checkpoint, so training it on four of five
+ # sources because one quietly failed is not an acceptable outcome.
+ ds = load_dataset(repo, confs[0], split=split, streaming=True)
+ for rec in ds:
+ t = rec.get(field) or ""
+ if not t:
+ continue
+ f.write(t.replace("\x00", "") + "\n")
+ n += len(t)
+ if n >= per:
+ break
+ log(f" {name}: {n/1e6:.0f} MB")
+ log(f" wrote {out} ({out.stat().st_size/1e6:.0f} MB)")
+
+
+def _val_start(name: str, confs: tuple, log=print):
+ """The (config, file index) where this source's untouched files begin.
+
+ The old implementation streamed a config from record zero and called next()
+ five million times to get past what training consumed. That re-downloads and
+ re-decodes tens of gigabytes to produce three megabytes of tokens, and with
+ every shard's "seen" counter at 0 it wasn't even reliably disjoint. The
+ download already recorded, per shard, which config and file it stopped at,
+ so the disjoint region is known exactly and reaching it costs nothing.
+ """
+ shards = [name] + sorted(
+ p.parent.name for p in DATA_DIR.glob(f"{name}_s*/progress.json"))
+ progs = [load_progress(s) for s in shards
+ if progress_path(s).exists()]
+ if not progs:
+ return confs[0], 0
+ max_cfg = max(p.get("cfg_idx", 0) for p in progs)
+ if max_cfg + 1 < len(confs):
+ return confs[max_cfg + 1], 0 # a whole config training never opened
+ hi = max(p.get("file_idx", 0) for p in progs
+ if p.get("cfg_idx", 0) == max_cfg)
+ return confs[max_cfg], hi + 2 # +2: skip the partially-read file
+
+
+def prepare_val(tokens: int, cfg: Config, log=print):
+ """A fixed held-out split, drawn from every source, that the training packer
+ never opens -- data.open_sources only ever loads cfg.data.sources.
+
+ Resumable per source: data/val/progress.json records which sources are
+ finished, so an interrupted run does not redo them.
+ """
+ tok = BorisTokenizer.load(DATA_DIR / "tokenizer")
+ w = ShardWriter(DATA_DIR / "val", tok.eos_id)
+ prog = load_progress("val")
+ done = set(prog.get("done", []))
+ per = tokens // len(SOURCES)
+ for name, (repo, confs, split, field) in SOURCES.items():
+ if name in done:
+ log(f" {name}: already in val split"); continue
+ try:
+ conf, fi = _val_start(name, confs, log)
+ files = resolve_files(repo, conf, split)
+ except Exception as e:
+ log(f" {name}: skipped ({type(e).__name__}: {e})"); continue
+ if fi >= len(files):
+ log(f" {name}: no untouched files left in {conf}; skipped")
+ continue
+ got, texts = 0, []
+ t0 = time.time()
+ while got < per and fi < len(files):
+ try:
+ recs = _stream_file(files[fi], field)
+ except Exception as e:
+ log(f" {name} file {fi}: {type(e).__name__}, skipping")
+ fi += 1; continue
+ for rec in recs:
+ t = rec.get(field) or ""
+ if not t:
+ continue
+ texts.append(t)
+ if len(texts) >= 500:
+ got += w.write(tok.encode_batch(texts)); texts = []
+ if got >= per:
+ break
+ fi += 1
+ if texts:
+ got += w.write(tok.encode_batch(texts))
+ log(f" {name}: {got/1e6:.1f}M val tokens from {conf} "
+ f"(files from {fi}) in {time.time()-t0:.0f}s")
+ done.add(name)
+ save_progress("val", {"done": sorted(done), "tokens": w.n_tokens})
+ w.close()
+ log(f" val split: {w.n_tokens/1e6:.1f}M tokens, {w.n_docs:,} documents")
+
+
+def check(cfg: Config, log=print):
+ """Open every source and pull one record. Run this before downloading.
+
+ Every field in SOURCES is a guess until something has actually read from the
+ repo -- two of the five were wrong the first time, and one of them only
+ surfaced after the sample step had already run.
+ """
+ from datasets import load_dataset
+ bad = 0
+ for name, (repo, confs, split, field) in SOURCES.items():
+ for conf in confs:
+ try:
+ ds = load_dataset(repo, conf, split=split, streaming=True)
+ rec = next(iter(ds))
+ t = rec.get(field)
+ if not isinstance(t, str) or not t:
+ raise ValueError(
+ f"field '{field}' is {type(t).__name__}, not text; "
+ f"available: {list(rec.keys())}")
+ log(f" ok {name:<15}{repo} [{conf}] / {split} / {field} "
+ f"first record {len(t):,} chars")
+ except Exception as e:
+ bad += 1
+ log(f" FAIL {name:<15}{repo} [{conf}] / {split} / {field}")
+ log(f" {type(e).__name__}: {str(e)[:160]}")
+ if bad:
+ raise SystemExit(f"\n {bad} source/config pairs unreachable -- fix SOURCES "
+ f"before downloading 400GB against a broken registry")
+ log("\n all sources reachable")
+
+
+def plan(cfg: Config, log=print):
+ """Token target per source: what the curriculum will actually draw."""
+ from schedule import mixture_at
+ total = cfg.train.total_tokens
+ steps = 200
+ draw = {n: 0.0 for n in cfg.data.sources}
+ means = {}
+ for n in cfg.data.sources:
+ dirs = [d for d in sorted(DATA_DIR.glob(n)) +
+ sorted(DATA_DIR.glob(f"{n}_s*")) if (d / "docs.idx").exists()]
+ nt = nd = 0
+ for d in dirs:
+ o = np.memmap(d / "docs.idx", dtype=np.uint64, mode="r")
+ nt += int(o[-1]); nd += o.size - 1
+ means[n] = nt / max(nd, 1)
+ for i in range(steps):
+ w = mixture_at((i + 0.5) / steps, cfg)
+ # The packer samples sources per document, so turn document draw
+ # probabilities into expected token shares before reporting capacity.
+ z = sum(w[n] * means[n] for n in w) or 1.0
+ for k, v in w.items():
+ draw[k] += (v * means[k] / z) * total / steps
+ log(f" {cfg.train.total_tokens/1e9:.0f}B token budget, "
+ f"{cfg.total_steps:,} steps\n")
+ log(f" {'source':<16}{'draws':>10}{'unique needed':>16}{'on disk':>12}")
+ log(" " + "-" * 56)
+ for n in cfg.data.sources:
+ have = 0
+ p = DATA_DIR / n / "docs.idx"
+ if p.exists():
+ have = int(np.fromfile(p, dtype=np.uint64)[-1])
+ log(f" {n:<16}{draw[n]/1e9:>9.1f}B{draw[n]/1e9:>15.1f}B"
+ f"{have/1e9:>11.2f}B")
+ log(" " + "-" * 56)
+ log(f" {'TOTAL':<16}{sum(draw.values())/1e9:>9.1f}B"
+ f"{sum(draw.values())/1e9:>15.1f}B")
+ log(f"\n stored as uint16: {sum(draw.values())*2/1e12:.2f} TB at one epoch.")
+ log(f" Repeating a smaller unique corpus is the intended fallback -- set a")
+ log(f" lower --tokens per source and the packer will resample it.")
+
+
+def status(cfg: Config, log=print):
+ log(f" {'source':<16}{'tokens':>12}{'docs':>12}{'dropped':>10}{'GB':>8}")
+ log(" " + "-" * 58)
+ for n in list(cfg.data.sources) + ["val"]:
+ dirs = [d for d in sorted(DATA_DIR.glob(n)) + sorted(DATA_DIR.glob(f"{n}_s*"))
+ if (d / "docs.idx").exists()]
+ if not dirs:
+ log(f" {n:<16}{'-':>12}"); continue
+ tot = docs = dropped = 0
+ for d in dirs:
+ off = np.fromfile(d / "docs.idx", dtype=np.uint64)
+ tot += int(off[-1]); docs += off.size - 1
+ dropped += load_progress(d.name).get("dropped", 0)
+ tag = f"{n} ({len(dirs)}sh)" if len(dirs) > 1 else n
+ log(f" {tag:<16}{tot/1e9:>11.2f}B{docs/1e6:>11.2f}M"
+ f"{dropped:>10,}{tot*2/2**30:>8.1f}")
+
+
+def main():
+ cfg = Config()
+ ap = argparse.ArgumentParser()
+ sub = ap.add_subparsers(dest="cmd", required=True)
+ sub.add_parser("plan"); sub.add_parser("status"); sub.add_parser("check")
+ s = sub.add_parser("sample"); s.add_argument("--mb", type=int, default=2000)
+ s = sub.add_parser("source")
+ s.add_argument("--name", required=True)
+ s.add_argument("--tokens", type=float, required=True)
+ s.add_argument("--no-decontam", action="store_true")
+ s.add_argument("--configs", default=None,
+ help="comma-separated config override, so two workers can "
+ "split a multi-config source between them")
+ s.add_argument("--file-start", type=int, default=0)
+ s.add_argument("--file-end", type=int, default=None)
+ s.add_argument("--out", default=None,
+ help="output directory name; use <name>_sN for a shard")
+ s.add_argument("--batch", type=int, default=250,
+ help="documents per encode_batch call. Each HF Encoding "
+ "carries ids, offsets, attention and word-id arrays, so "
+ "this sets the transient memory peak per worker; 1000 "
+ "across eight workers drove the box 16GB into swap.")
+ s = sub.add_parser("val"); s.add_argument("--tokens", type=float, default=2e7)
+ a = ap.parse_args()
+
+ if a.cmd == "check":
+ check(cfg)
+ elif a.cmd == "plan":
+ plan(cfg)
+ elif a.cmd == "status":
+ status(cfg)
+ elif a.cmd == "sample":
+ prepare_sample(a.mb, cfg)
+ elif a.cmd == "source":
+ prepare_source(a.name, int(a.tokens), cfg, decontam=not a.no_decontam,
+ file_start=a.file_start, file_end=a.file_end, out=a.out,
+ batch=a.batch,
+ configs=tuple(a.configs.split(",")) if a.configs else None)
+ elif a.cmd == "val":
+ prepare_val(int(a.tokens), cfg)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/prepare_0920.sh b/prepare_0920.sh
new file mode 100755
index 0000000..768c7fa
--- /dev/null
+++ b/prepare_0920.sh
@@ -0,0 +1,102 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+repo=/home/joseph/boris-2-D112M-n15M
+data=/media/joseph/Media/boris-2-data
+run=/media/joseph/Media/Boris-2-0920
+old_tok="$data/tokenizer"
+new_tok="$run/tokenizer"
+log="$run/prepare_0920.log"
+
+exec >>"$log" 2>&1
+echo "[$(date -Is)] Boris-2-0920 preparation started"
+
+sources=(
+ climbmix climbmix_s1 climbmix_s2 climbmix_s3
+ climbmix_s4 climbmix_s5 climbmix_s6 climbmix_s7
+ climbmix_s8 climbmix_s9 climbmix_s10 climbmix_s11
+ climbmix_s12 climbmix_s13 climbmix_s14 climbmix_s15
+ finemath finemath_s1
+)
+
+convert_one() {
+ local name=$1 out="$run/data/$1"
+ if [[ -f "$out/retokenize.json" ]]; then
+ echo "[$(date -Is)] $name already converted"
+ return
+ fi
+ if [[ -e "$out/tokens.bin" || -e "$out/docs.idx" ]]; then
+ echo "[$(date -Is)] incomplete output for $name; refusing to overwrite" >&2
+ return 2
+ fi
+ echo "[$(date -Is)] converting $name"
+ RAYON_NUM_THREADS=4 nice -n 10 ionice -c2 -n7 \
+ "$repo/.venv/bin/python" "$repo/retokenize.py" \
+ "$data/$name" "$old_tok" "$new_tok" --out "$out" --batch-docs 2048
+}
+
+promote_one() {
+ local name=$1 src="$data/$1" out="$run/data/$1"
+ [[ -f "$out/retokenize.json" ]] || { echo "missing manifest for $name" >&2; return 2; }
+ [[ -f "$out/tokens.bin" && -f "$out/docs.idx" ]] || { echo "missing output for $name" >&2; return 2; }
+ echo "[$(date -Is)] promoting $name"
+ mv "$src/tokens.bin" "$src/tokens.bin.0917"
+ mv "$src/docs.idx" "$src/docs.idx.0917"
+ mv "$out/tokens.bin" "$src/tokens.bin"
+ mv "$out/docs.idx" "$src/docs.idx"
+ if ! "$repo/.venv/bin/python" - "$src" <<'PY'
+import sys
+from pathlib import Path
+import numpy as np
+p=Path(sys.argv[1]); off=np.memmap(p/'docs.idx',dtype=np.uint64,mode='r')
+assert off.size >= 2 and int(off[0]) == 0
+assert int(off[-1])*2 == (p/'tokens.bin').stat().st_size
+assert np.all(off[1:] > off[:-1])
+PY
+ then
+ mv "$src/tokens.bin" "$out/tokens.bin.bad"
+ mv "$src/docs.idx" "$out/docs.idx.bad"
+ mv "$src/tokens.bin.0917" "$src/tokens.bin"
+ mv "$src/docs.idx.0917" "$src/docs.idx"
+ echo "validation failed and $name was rolled back" >&2
+ return 2
+ fi
+ cp "$out/retokenize.json" "$src/retokenize_0920.json"
+ rm "$src/tokens.bin.0917" "$src/docs.idx.0917"
+ echo "[$(date -Is)] promoted and removed superseded token files for $name"
+}
+
+for ((i=0; i<${#sources[@]}; i+=4)); do
+ pids=(); names=()
+ for name in "${sources[@]:i:4}"; do
+ convert_one "$name" & pids+=("$!"); names+=("$name")
+ done
+ for pid in "${pids[@]}"; do wait "$pid"; done
+ for name in "${names[@]}"; do promote_one "$name"; done
+done
+
+# The validation split was started separately before this orchestrator.
+while pgrep -f "retokenize.py $data/climbmix_val " >/dev/null; do sleep 20; done
+promote_one climbmix_val
+
+if [[ ! -d "$data/tokenizer-0917" ]]; then
+ mv "$data/tokenizer" "$data/tokenizer-0917"
+fi
+cp -a "$new_tok" "$data/tokenizer"
+
+echo "[$(date -Is)] building 90/10 n-gram vocabulary"
+nice -n 10 ionice -c2 -n7 "$repo/.venv/bin/python" "$repo/ngram.py" build \
+ --config "$run/config.json" --out "$run/ngram_vocab.bin"
+
+echo "[$(date -Is)] final verification"
+"$repo/.venv/bin/python" "$repo/verify.py" --quick --device cuda:1
+
+echo "[$(date -Is)] V100 micro-batch confirmation"
+"$repo/.venv/bin/python" "$repo/preflight.py" --device cuda:0 \
+ --block 2048 --sweep 8,16 --iters 4
+
+setsid "$repo/.venv/bin/python" "$repo/eval.py" --watch --device cuda:1 \
+ >"$run/eval_watch.log" 2>&1 < /dev/null &
+
+echo "[$(date -Is)] launching Boris-2-0920"
+exec "$repo/.venv/bin/python" "$repo/train.py" --config "$run/config.json"
diff --git a/retokenize.py b/retokenize.py
new file mode 100644
index 0000000..6e9b88d
--- /dev/null
+++ b/retokenize.py
@@ -0,0 +1,103 @@
+"""Transactionally convert prepared uint16 shards to a different Boris BPE.
+
+The source format preserves document boundaries, so decoding each old document
+and encoding it with the new byte-level BPE is lossless. Conversion writes a
+sibling directory and never mutates the source shard. A separate, explicit
+swap step is required after training has stopped and validation has passed.
+
+Example:
+ python retokenize.py data/climbmix data/tokenizer data/tokenizer-0920 \
+ --out data/climbmix.retokenized-0920
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import time
+from pathlib import Path
+
+import numpy as np
+
+from tokenizer import BorisTokenizer
+
+
+def convert(source: Path, old_dir: Path, new_dir: Path, out: Path,
+ batch_docs: int = 2048) -> None:
+ source, out = Path(source), Path(out)
+ if source.resolve() == out.resolve():
+ raise SystemExit("output must differ from source; in-place conversion is unsafe")
+ out.mkdir(parents=True, exist_ok=True)
+ out_tokens, out_idx = out / "tokens.bin", out / "docs.idx"
+ if out_tokens.exists() or out_idx.exists():
+ raise SystemExit(f"{out}: output files already exist; refusing to append or overwrite")
+
+ old, new = BorisTokenizer.load(old_dir), BorisTokenizer.load(new_dir)
+ tokens = np.memmap(source / "tokens.bin", dtype=np.uint16, mode="r")
+ offsets = np.memmap(source / "docs.idx", dtype=np.uint64, mode="r")
+ if offsets.size < 2 or int(offsets[0]) != 0 or int(offsets[-1]) != tokens.size:
+ raise SystemExit(f"{source}: inconsistent token/index files")
+
+ t0 = time.time()
+ written = 0
+ with open(out_tokens, "wb") as ft, open(out_idx, "wb") as fi:
+ np.asarray([0], dtype=np.uint64).tofile(fi)
+ for start in range(0, offsets.size - 1, batch_docs):
+ stop = min(start + batch_docs, offsets.size - 1)
+ texts = []
+ for i in range(start, stop):
+ a, b = int(offsets[i]), int(offsets[i + 1])
+ ids = tokens[a:b]
+ if ids.size and int(ids[-1]) == old.eos_id:
+ ids = ids[:-1]
+ texts.append(old.decode(ids))
+ encoded = new.encode_batch(texts)
+ chunks, ends = [], []
+ for text, ids in zip(texts, encoded):
+ # Both tokenizers are byte-level; this makes corruption a hard
+ # failure before the source shard is ever considered for swap.
+ if new.decode(ids) != text:
+ raise RuntimeError(f"round-trip failed at document {start + len(ends)}")
+ chunk = np.asarray(ids + [new.eos_id], dtype=np.uint16)
+ chunks.append(chunk)
+ written += chunk.size
+ ends.append(written)
+ if chunks:
+ np.concatenate(chunks).tofile(ft)
+ np.asarray(ends, dtype=np.uint64).tofile(fi)
+ if start == 0 or (start // batch_docs) % 100 == 0:
+ rate = written / max(time.time() - t0, 1e-9)
+ print(f" {stop:,}/{offsets.size-1:,} docs {written/1e9:.3f}B tokens "
+ f"{rate/1e6:.2f}M tok/s", flush=True)
+ ft.flush(); os.fsync(ft.fileno())
+ fi.flush(); os.fsync(fi.fileno())
+
+ check_idx = np.memmap(out_idx, dtype=np.uint64, mode="r")
+ expected_bytes = int(check_idx[-1]) * np.dtype(np.uint16).itemsize
+ if check_idx.size != offsets.size or out_tokens.stat().st_size != expected_bytes:
+ raise RuntimeError("converted shard failed final size/index validation")
+ manifest = {
+ "source": str(source.resolve()),
+ "documents": int(offsets.size - 1),
+ "old_tokens": int(tokens.size),
+ "new_tokens": int(check_idx[-1]),
+ "old_tokenizer_sha256": BorisTokenizer.fingerprint(old_dir),
+ "new_tokenizer_sha256": BorisTokenizer.fingerprint(new_dir),
+ }
+ (out / "retokenize.json").write_text(json.dumps(manifest, indent=2) + "\n")
+ print(json.dumps(manifest, indent=2))
+
+
+def main() -> None:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("source", type=Path)
+ ap.add_argument("old_tokenizer", type=Path)
+ ap.add_argument("new_tokenizer", type=Path)
+ ap.add_argument("--out", type=Path, required=True)
+ ap.add_argument("--batch-docs", type=int, default=2048)
+ a = ap.parse_args()
+ convert(a.source, a.old_tokenizer, a.new_tokenizer, a.out, a.batch_docs)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/schedule.py b/schedule.py
new file mode 100644
index 0000000..904cfe4
--- /dev/null
+++ b/schedule.py
@@ -0,0 +1,106 @@
+"""WSD learning rate, the mixture curriculum, and the context-length stages.
+
+Everything here is a function of `progress` in [0, 1] -- tokens seen over the
+token budget -- so a resumed run recovers the whole schedule from `tokens_seen`
+alone and cannot drift out of phase with the run that wrote the checkpoint.
+"""
+from __future__ import annotations
+
+import math
+
+from config import Config
+
+
+def phase_at(progress: float, cfg: Config) -> str:
+ s = cfg.schedule
+ if progress < s.warmup_frac:
+ return "warmup"
+ if progress < s.stable_frac:
+ return "stable"
+ return "decay"
+
+
+def lr_scale(progress: float, cfg: Config) -> float:
+ """Multiplier on each optimizer's base LR, in [0, 1].
+
+ 1-sqrt decay rather than cosine: it holds the high LR longer and then drops
+ fast, which is what makes the decay phase a consolidation onto the dense
+ data rather than a long slow taper through it.
+ """
+ s = cfg.schedule
+ p = min(max(progress, 0.0), 1.0)
+ if p < s.warmup_frac:
+ return p / max(s.warmup_frac, 1e-12)
+ if p < s.stable_frac:
+ return 1.0
+ frac = (p - s.stable_frac) / max(1.0 - s.stable_frac, 1e-12)
+ if s.decay_shape == "1-sqrt":
+ v = 1.0 - math.sqrt(frac)
+ elif s.decay_shape == "linear":
+ v = 1.0 - frac
+ else:
+ v = 0.5 * (1.0 + math.cos(math.pi * frac))
+ return s.final_lr_frac + (1 - s.final_lr_frac) * max(v, 0.0)
+
+
+def mixture_at(progress: float, cfg: Config) -> dict:
+ """Source weights at this point in the run, normalised to sum to 1."""
+ c = cfg.curriculum
+ names = list(cfg.data.sources)
+ if not c.enabled:
+ return {n: 1.0 / len(names) for n in names}
+
+ phases = [(n, float(e), {k: float(w.get(k, 0.0)) for k in names})
+ for n, e, w in c.phases]
+ i = 0
+ while i < len(phases) - 1 and progress >= phases[i][1]:
+ i += 1
+ _, end, target = phases[i]
+ start = 0.0 if i == 0 else phases[i - 1][1]
+
+ if not c.smooth or i == 0:
+ w = target
+ else:
+ prev = phases[i - 1][2]
+ f = min(max((progress - start) / max(end - start, 1e-12), 0.0), 1.0)
+ w = {k: prev[k] + f * (target[k] - prev[k]) for k in names}
+ tot = sum(w.values()) or 1.0
+ return {k: v / tot for k, v in w.items()}
+
+
+def context_at(progress: float, cfg: Config) -> tuple[int, float, int, int]:
+ """(block_size, rope_theta, micro_batch, accum_steps)."""
+ bs, theta = cfg.block_size_at(progress)
+ return bs, theta, cfg.micro_batch_at(progress), cfg.accum_at(progress)
+
+
+def long_doc_pressure(progress: float, cfg: Config) -> tuple[int, float]:
+ """(minimum length preferred, target share of TOKENS from such documents).
+
+ The packer converts the token share into a per-draw probability per source
+ (VarlenPacker._long_draw_prob); it is not itself a draw probability.
+
+ Zero until the context stage lengthens: at 2048 there is nothing to gain
+ from hunting for 4k-token documents, and biasing the mixture toward long
+ documents is a change to the data distribution, not just to the packing.
+ """
+ bs, _ = cfg.block_size_at(progress)
+ base = cfg.model.context_stages[0][1]
+ if bs <= base:
+ return 0, 0.0
+ return cfg.curriculum.long_doc_floor_tokens, cfg.curriculum.long_doc_target_frac
+
+
+def phase_boundaries(cfg: Config) -> list[tuple[str, int, int]]:
+ total = cfg.total_steps
+ s = cfg.schedule
+ return [("warmup", 0, int(total * s.warmup_frac)),
+ ("stable", int(total * s.warmup_frac), int(total * s.stable_frac)),
+ ("decay", int(total * s.stable_frac), total)]
+
+
+def stable_end_tokens(cfg: Config) -> int:
+ """The end-of-stable checkpoint is permanent: variants branch from it as a
+ tree, and re-warming from a fully decayed model costs a loss spike and a
+ slow recovery."""
+ return int(cfg.train.total_tokens * cfg.schedule.stable_frac)
diff --git a/scripts/hellaswag_title_split.py b/scripts/hellaswag_title_split.py
new file mode 100644
index 0000000..da83229
--- /dev/null
+++ b/scripts/hellaswag_title_split.py
@@ -0,0 +1,67 @@
+"""HellaSwag gain split by WikiHow-title overlap with the decay's how-to data
+(the check cagliostro-v3.5's card ran). Items: HellaSwag validation, in eval.py's order.
+Groups: WikiHow items whose article title is also a Cosmopedia-wikihow tutorial title,
+WikiHow items whose title is not, and ActivityNet items (no WikiHow link at all).
+Per-item acc_norm before/after the decay; a real skill gain shows up about equally in
+the first two groups, memorised topics would show up only in the first.
+
+ python probe/hellaswag_title_split.py BEFORE.pt AFTER.pt OUT.json
+"""
+import json, re, sys, inspect
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+import numpy as np, torch, pyarrow.parquet as pq
+from huggingface_hub import hf_hub_download
+from datasets import load_dataset
+import eval as E
+from tokenizer import BorisTokenizer
+
+def norm(t):
+ t = t.lower().strip()
+ t = re.sub(r"^how to\s+", "", t)
+ return re.sub(r"[^a-z0-9 ]+", "", t).strip()
+
+titles = set()
+for k in range(2):
+ p = hf_hub_download("HuggingFaceTB/cosmopedia", f"data/wikihow/train-0000{k}-of-00002.parquet", repo_type="dataset")
+ for pr in pq.read_table(p, columns=["prompt"]).column("prompt").to_pylist():
+ m = re.search(r'tutorial on "(.+?)"', pr)
+ if m: titles.add(norm(m.group(1)))
+
+raw = [r for r in load_dataset("Rowan/hellaswag", split="validation") if str(r["label"]).strip() != ""]
+group = []
+for r in raw:
+ if r["source_id"].startswith("activitynet"):
+ group.append("activitynet"); continue
+ m = re.search(r"\[header\]\s*(.+?)\s*\[title\]", r["ctx_a"])
+ group.append("wikihow_title_in_data" if m and norm(m.group(1)) in titles else
+ "wikihow_title_not_in_data" if m else "wikihow_no_title")
+items = E.TASKS["hellaswag"]()
+assert len(items) == len(group)
+
+src = inspect.getsource(E.score_batch).replace(" n_hit = n_hit_norm = 0", " return scores\n n_hit = 0")
+ns = dict(vars(E)); exec(src, ns); score = ns["score_batch"]
+dev = "cuda:1"; torch.cuda.set_device(dev)
+right = {}
+for p in sys.argv[1:3]:
+ model, cfg, blob = E.load_export(Path(p), dev)
+ tok = BorisTokenizer.load(Path(cfg.data.root) / "tokenizer")
+ with torch.no_grad():
+ s = score(model, tok, items, dev, batch_size=256)
+ right[p] = np.array([int(np.argmax([s[(i, j)][0] / max(s[(i, j)][1], 1) for j in range(len(it["choices"]))]) == it["gold"])
+ for i, it in enumerate(items)])
+ del model; torch.cuda.empty_cache()
+a, b = right[sys.argv[1]], right[sys.argv[2]]
+g = np.array(group); out = {"cosmopedia_wikihow_titles": len(titles)}
+wk = np.isin(g, ["wikihow_title_in_data", "wikihow_title_not_in_data"])
+out["title_overlap_share_of_titled_wikihow_items"] = float((g == "wikihow_title_in_data").sum() / max(wk.sum(), 1))
+print(f"Cosmopedia wikihow titles: {len(titles):,}; titled WikiHow items whose title is in the data: "
+ f"{100*out['title_overlap_share_of_titled_wikihow_items']:.1f}%")
+for name in ["wikihow_title_in_data", "wikihow_title_not_in_data", "wikihow_no_title", "activitynet", "all"]:
+ m = np.ones(len(g), bool) if name == "all" else (g == name)
+ if not m.any(): continue
+ d = (b[m] - a[m]).astype(float); se = d.std(ddof=1) / np.sqrt(m.sum()) if m.sum() > 1 else 0.0
+ out[name] = {"n": int(m.sum()), "before": float(a[m].mean() * 100), "after": float(b[m].mean() * 100),
+ "gain": float(d.mean() * 100), "gain_95ci": float(1.96 * se * 100)}
+ print(f" {name:28s} n={m.sum():5d} {a[m].mean()*100:6.2f} -> {b[m].mean()*100:6.2f} gain {d.mean()*100:+5.2f} ± {1.96*se*100:.2f}")
+json.dump(out, open(sys.argv[3], "w"), indent=2)
diff --git a/scripts/pilot_train.py b/scripts/pilot_train.py
new file mode 100644
index 0000000..5d7e85f
--- /dev/null
+++ b/scripts/pilot_train.py
@@ -0,0 +1,191 @@
+"""Pilot branches from a real Boris checkpoint (2026-10-10).
+
+Generalises branch_decay.py: either GPU, the Volta kernels on the V100, the main
+run's batch, a constant or 1-sqrt-decaying LR (optionally to a non-zero floor),
+and an optional output-superposition phase (TST, Peng et al. 2026, section 5.2):
+inputs stay ordinary tokens, but for the first `--bag-tokens` tokens each position
+is trained to predict the bag of its next s tokens -- uniform 1/s multi-hot cross
+entropy on the same LM head -- then training reverts to next-token prediction.
+Bag targets never cross a document boundary (the packer's IGNORE marks a document's
+last position; a target is valid only if every step to it is valid).
+
+Every arm with the same --seed sees the same token stream.
+
+ python probe/pilot_train.py NAME --tokens 3e8 --mix climbmix:0.95,finemath:0.05 \
+ [--dev cuda:0 --mb 16 --tps 524288] [--lr const|decay --final-frac 0.0] \
+ [--bag 4 --bag-tokens 1.5e8] --ckpt START.pt --out DIR
+"""
+from __future__ import annotations
+
+import argparse, json, math, sys, time
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+import torch
+import torch.nn.functional as F
+from torch.utils.checkpoint import checkpoint
+
+from config import Config
+from data import Prefetcher, VarlenPacker, open_sources
+from model import Boris2
+from ngram import NGramVocab
+from optim import build_optimizers, set_lr
+import checkpoint as ckpt_mod
+
+IGNORE = -100
+
+
+def bag_targets(y, s):
+ """(B,T) next-token targets -> (B,T,s) targets for t+1..t+s and their validity."""
+ B, T = y.shape
+ tg = torch.full((B, T, s), IGNORE, dtype=y.dtype, device=y.device)
+ ok = torch.zeros(B, T, s, dtype=torch.bool, device=y.device)
+ run = torch.ones(B, T, dtype=torch.bool, device=y.device)
+ for j in range(s):
+ yj = torch.full_like(y, IGNORE)
+ yj[:, :T - j] = y[:, j:]
+ run = run & (yj != IGNORE)
+ tg[:, :, j] = torch.where(run, yj, torch.full_like(yj, IGNORE))
+ ok[:, :, j] = run
+ return tg, ok
+
+
+def _bag_chunk(h, W, tg, ok):
+ lg = F.linear(h.to(torch.float16), W.to(torch.float16)).float()
+ lp = lg.gather(1, tg.clamp_min(0)) - torch.logsumexp(lg, -1, keepdim=True)
+ n = ok.sum(1)
+ per = (lp * ok).sum(1) / n.clamp_min(1)
+ return -(per * (n > 0)).sum(), (n > 0).sum()
+
+
+def bag_loss(h, W, y, s, chunk=2048):
+ tg, ok = bag_targets(y, s)
+ h, tg, ok = h.reshape(-1, h.shape[-1]), tg.reshape(-1, s), ok.reshape(-1, s)
+ tot, cnt = 0.0, 0
+ for a in range(0, h.shape[0], chunk):
+ l, c = checkpoint(_bag_chunk, h[a:a + chunk], W, tg[a:a + chunk], ok[a:a + chunk],
+ use_reentrant=False)
+ tot = tot + l; cnt = cnt + c
+ return tot / max(int(cnt), 1)
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("name"); ap.add_argument("--tokens", type=float, required=True)
+ ap.add_argument("--mix", required=True, help="per-document draw weights src:w,...")
+ ap.add_argument("--dev", default="cuda:1"); ap.add_argument("--mb", type=int, default=2)
+ ap.add_argument("--tps", type=int, default=131072)
+ ap.add_argument("--lr", choices=["const", "decay", "wsd"], default="decay")
+ ap.add_argument("--decay-tokens", type=float, default=0)
+ ap.add_argument("--final-frac", type=float, default=0.0)
+ ap.add_argument("--bag", type=int, default=0); ap.add_argument("--bag-tokens", type=float, default=0)
+ ap.add_argument("--ckpt", required=True); ap.add_argument("--out", required=True)
+ ap.add_argument("--seed", type=int, default=1234)
+ ap.add_argument("--peak", type=float, default=1.0, help="multiplier on the config's peak LRs (0.1 = a tenth)")
+ ap.add_argument("--compile", action="store_true")
+ ap.add_argument("--max-steps", type=int, default=None)
+ ap.add_argument("--block", type=int, default=2048, help="sequence length (Boris-2-Nano: 1024)")
+ ap.add_argument("--mix-before", default=None, help="mix used before --switch-tokens; --mix after it")
+ ap.add_argument("--switch-tokens", type=float, default=0)
+ a = ap.parse_args()
+ dev = a.dev; torch.cuda.set_device(dev)
+ parse = lambda m: {k: float(v) for k, v in (kv.split(":") for kv in m.split(","))}
+ mix = parse(a.mix); mix0 = parse(a.mix_before) if a.mix_before else None
+
+ blob = torch.load(a.ckpt, map_location="cpu", weights_only=False)
+ cfg = Config.from_json(blob["config"])
+ t0_tokens, ngfp = blob["tokens_seen"], blob.get("ngram_fingerprint")
+ allsrc = dict.fromkeys(list(mix) + list(mix0 or {}))
+ cfg.data.sources = tuple(allsrc)
+ full = lambda m: {k: m.get(k, 0.0) for k in allsrc}
+ cfg.curriculum.phases = [["pilot", 1.0, full(mix0 or mix)]]
+ cfg.curriculum.smooth = False
+ cfg.optim.lm_head_chunk = min(cfg.optim.lm_head_chunk, 8192 if "cuda:0" in dev else 2048)
+ cfg.train.device = dev
+ cfg.train.tokens_per_step = a.tps
+ cfg.train.micro_batch = {a.block: a.mb, 8192: max(1, a.mb // 4)}
+ torch.manual_seed(a.seed)
+
+ stores = open_sources(cfg, lambda *x: None)
+ packer = VarlenPacker(cfg, stores, a.seed, split="train")
+ model = Boris2(cfg).to(dev)
+ model.load_state_dict({k: v.float() for k, v in blob["model"].items()})
+ if model.ngram is not None:
+ model.ngram.load_vocab(NGramVocab.load(cfg.ngram.vocab_path), dev)
+ if a.compile:
+ model.maybe_compile()
+ muon, adamw = build_optimizers(model, cfg, lambda *x: None)
+ if "muon" in blob: # model-only exports start with fresh optimizer state
+ muon.load_state_dict(blob["muon"])
+ adamw.load_state_dict(ckpt_mod._migrate_adamw_groups(blob, model, adamw))
+ scaler = torch.amp.GradScaler("cuda", init_scale=2.0 ** 15)
+ theta = float(cfg.block_size_at(t0_tokens / cfg.train.total_tokens)[1])
+ del blob
+
+ accum = a.tps // (a.mb * a.block)
+ steps = int(a.tokens) // a.tps
+ if a.max_steps: steps = min(steps, a.max_steps)
+ bag_steps = int(a.bag_tokens) // a.tps if a.bag else 0
+ pre = Prefetcher(packer, cfg, dev); pre.configure(a.mb, a.block, 0.0); pre.start()
+ from train import gpu_temp, device_index
+ di = device_index(dev)
+ W = model.head_weight()
+ out = Path(a.out); out.mkdir(parents=True, exist_ok=True)
+ log = open(out / f"{a.name}.log", "a")
+ t0 = time.time(); tokens = 0
+ switch_step = int(a.switch_tokens) // a.tps if mix0 else -1
+ for step in range(steps):
+ if step == switch_step: # change the data mix in place (same packer state otherwise)
+ pre.stop()
+ cfg.curriculum.phases = [["pilot", 1.0, full(mix)]]
+ packer = VarlenPacker(cfg, stores, a.seed + 1, split="train")
+ pre = Prefetcher(packer, cfg, dev); pre.configure(a.mb, a.block, 0.0); pre.start()
+ print(f"{a.name} step {step}: switched data mix", flush=True); log.write(f"switched mix at step {step}\n")
+ if step % 5 == 0 and gpu_temp(di) >= 79:
+ while gpu_temp(di) > 76:
+ time.sleep(2)
+ frac = step / steps
+ if a.lr == "wsd":
+ dsteps = max(1, int(a.decay_tokens) // a.tps); d0 = steps - dsteps
+ scale = 1.0 if step < d0 else a.final_frac + (1 - a.final_frac) * (1.0 - math.sqrt((step - d0) / dsteps))
+ else:
+ scale = 1.0 if a.lr == "const" else a.final_frac + (1 - a.final_frac) * (1.0 - math.sqrt(frac))
+ set_lr(muon, adamw, cfg, scale * a.peak)
+ in_bag = step < bag_steps
+ acc = torch.zeros((), device=dev)
+ for _ in range(accum):
+ x, y, pos, bnd, bounds, st = pre.next(a.mb, a.block)
+ with torch.autocast("cuda", dtype=torch.float16):
+ if in_bag:
+ h = model(x, pos, bounds, bnd, None, theta, st["min_seqlen"], st["max_seqlen"])
+ loss = bag_loss(h, W, y, a.bag)
+ else:
+ loss = model(x, pos, bounds, bnd, y, theta, st["min_seqlen"], st["max_seqlen"])
+ scaler.scale(loss / accum).backward()
+ acc += loss.detach() / accum; tokens += st["tokens"]
+ scaler.unscale_(muon); scaler.unscale_(adamw)
+ gl = [p.grad for p in model.parameters() if p.grad is not None]
+ gn = float(torch.linalg.vector_norm(torch.stack(torch._foreach_norm(gl))))
+ if math.isfinite(gn):
+ if gn >= cfg.optim.grad_clip:
+ torch._foreach_mul_(gl, cfg.optim.grad_clip / (gn + 1e-6))
+ muon.step(); adamw.step()
+ scaler.update()
+ if scaler.get_scale() > 2.0 ** 15:
+ scaler.update(2.0 ** 15)
+ muon.zero_grad(set_to_none=True); adamw.zero_grad(set_to_none=True)
+ if step % 10 == 0 or step == steps - 1:
+ msg = (f"{a.name} step {step}/{steps} {'BAG' if in_bag else 'ntp'} loss {float(acc):.4f} "
+ f"gn {gn:.3f} lr_scale {scale:.3f} {tokens/(time.time()-t0)/1e3:.1f}k tok/s "
+ f"temp {gpu_temp(di)}")
+ print(msg, flush=True); log.write(msg + "\n"); log.flush()
+ pre.stop()
+ exp = out / f"pilot_{a.name}.pt"
+ torch.save({"model": model.state_dict(), "config": cfg.to_json(), "step": steps,
+ "tokens_seen": t0_tokens + tokens, "ngram_fingerprint": ngfp}, exp)
+ print(json.dumps({"name": a.name, "tokens": tokens, "minutes": (time.time() - t0) / 60,
+ "bag": a.bag, "bag_steps": bag_steps, "lr": a.lr, "export": str(exp)}), flush=True)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/prepare_orca.py b/scripts/prepare_orca.py
new file mode 100644
index 0000000..37f27b4
--- /dev/null
+++ b/scripts/prepare_orca.py
@@ -0,0 +1,23 @@
+"""microsoft/orca-math-word-problems-200k -> data/orca_math (question + answer as one
+document), 13-gram decontaminated against the eval suite like every other source."""
+import sys, json
+sys.path.insert(0, "/home/joseph/boris-2-D112M-n15M")
+from datasets import load_dataset
+import prepare
+from tokenizer import BorisTokenizer
+tok = BorisTokenizer.load(prepare.DATA_DIR / "tokenizer"); ban = prepare.eval_ngrams()
+w = prepare.ShardWriter(prepare.DATA_DIR / "orca_math", tok.eos_id)
+assert w.n_tokens == 0, "data/orca_math already has tokens"
+ds = load_dataset("microsoft/orca-math-word-problems-200k", split="train")
+kept = dropped = toks = 0; batch = []
+for r in ds:
+ t = r["question"].strip() + "\n\n" + r["answer"].strip()
+ if prepare.contaminated(t, ban): dropped += 1; continue
+ batch.append(t)
+ if len(batch) == 1000:
+ toks += w.write(tok.encode_batch(batch)); kept += len(batch); batch = []
+if batch: toks += w.write(tok.encode_batch(batch)); kept += len(batch)
+w.close()
+json.dump({"source": "microsoft/orca-math-word-problems-200k", "documents": kept, "tokens": toks,
+ "dropped_contaminated": dropped}, open(prepare.DATA_DIR / "orca_math" / "prepare_meta.json", "w"), indent=2)
+print("orca_math:", kept, "docs", toks, "tokens", dropped, "dropped as contaminated")
diff --git a/scripts/prepare_wordprob.py b/scripts/prepare_wordprob.py
new file mode 100644
index 0000000..cfa0cd6
--- /dev/null
+++ b/scripts/prepare_wordprob.py
@@ -0,0 +1,187 @@
+"""Public word-problem datasets -> Boris token shards (2026-10-10).
+
+ python probe/prepare_wordprob.py tinygsm --tokens 1.5e9
+ python probe/prepare_wordprob.py elem
+
+tinygsm: TinyGSM/TinyGSM (MIT) -- question, then its Python solution.
+elem: MU-NLPC Calc-ape210k and Calc-mawps, training splits only (MIT) -- question, then
+ "Answer: <equation> = <result>".
+Every document goes through the same 13-gram filter as the rest of the corpus (the ban
+set covers HellaSwag, ARC, PIQA and every ArithMark-3 item). Writes data/<name>/ with a
+provenance file.
+"""
+import argparse, hashlib, json, sys
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+from datasets import load_dataset
+import prepare
+from tokenizer import BorisTokenizer
+
+
+def tinygsm():
+ for r in load_dataset("TinyGSM/TinyGSM", split="train", streaming=True):
+ yield r["question"].strip() + "\n\n" + r["code"].strip()
+
+
+def elem():
+ # Training splits only: ASDiv-A and SVAMP exist only as test sets and are left out.
+ for name in ("MU-NLPC/Calc-ape210k", "MU-NLPC/Calc-mawps"):
+ for split in ("train",):
+ for r in load_dataset(name, split=split):
+ q = r["question"].strip()
+ eq = (r.get("equation") or "").replace("x =", "").strip()
+ res = str(r.get("result") or "").strip()
+ yield f"{q}\nAnswer: {eq} = {res}" if eq and res and eq != res else f"{q}\nAnswer: {res}"
+
+
+def _raw(repo, fn):
+ from huggingface_hub import hf_hub_download
+ return json.load(open(hf_hub_download(repo, fn, repo_type="dataset")))
+
+
+def metamath_gsm():
+ # GSM8K-derived rows only (rephrasings, self-verification, FOBAR); the MATH rows are competition level.
+ for r in _raw("meta-math/MetaMathQA", "MetaMathQA-395K.json"):
+ if r["type"].startswith("GSM"):
+ yield r["query"].strip() + "\n" + r["response"].strip()
+
+
+def gsm8k_aug():
+ import re
+ # Training file only; the repo also ships gsm8k_test.json, which is never read.
+ d = _raw("zen-E/GSM8k-Aug", "gsm8k_aug_train.json") # columnar: {question: [...], cot: [...], answer: [...]}
+ for r in (dict(zip(d, vals)) for vals in zip(*d.values())):
+ steps = re.findall(r"<<([^=<>]+)=([^<>]+)>>", r.get("cot", ""))
+ work = "\n".join(f"{a} = {b}" for a, b in steps)
+ yield r["question"].strip() + ("\n" + work if work else "") + f"\nAnswer: {str(r['answer']).strip()}"
+
+
+def grade_lessons():
+ for r in _raw("ajibawa-2023/Maths-Grade-School", "Maths-Grade-School-0-Final.json"):
+ t = (r.get("output") or "").strip()
+ if t:
+ yield t
+
+
+def omi2_gsm():
+ # Full OpenMathInstruct-2 (CC-BY-4.0), grade-school rows only: gsm8k + augmented_gsm8k (~2.6M problems).
+ from huggingface_hub import hf_hub_download
+ import pyarrow.parquet as pq
+ for k in range(32):
+ t = pq.read_table(hf_hub_download("nvidia/OpenMathInstruct-2", f"data/train-{k:05d}-of-00032.parquet", repo_type="dataset"),
+ columns=["problem", "generated_solution", "problem_source"])
+ for pr, so, src in zip(*(t.column(c).to_pylist() for c in ("problem", "generated_solution", "problem_source"))):
+ if "gsm" in src:
+ yield pr.strip() + "\n" + so.strip()
+
+
+def libremath():
+ # LibreMath v1 (K2-Horizon, Apache-2.0): single-part and two-part items generated so far.
+ import glob
+ for d in ("v1", "v1_pairs", "v1_pairs_key2"):
+ for f in sorted(glob.glob(f"/media/joseph/Media/libremath/{d}/items-*.jsonl")):
+ for l in open(f):
+ r = json.loads(l)
+ yield r["problem"].strip() + "\n" + r["solution"].strip()
+
+
+def libremath_tpl():
+ # LibreMath v2: problems filled from K2-Horizon-written templates (both keys), every value computed in code.
+ import glob
+ for d in ("v2_templates_key1", "v2_templates_key2"):
+ for f in sorted(glob.glob(f"/media/joseph/Media/libremath/{d}/items-*.jsonl")):
+ for l in open(f):
+ r = json.loads(l)
+ yield r["problem"].strip() + "\n" + r["solution"].strip()
+
+
+def _hs_banned_titles():
+ import re
+ norm = lambda t: re.sub(r"[^a-z0-9 ]", "", t.lower().replace("how to", "")).split()
+ banned = set()
+ for sp in ("validation", "test"):
+ for r in load_dataset("Rowan/hellaswag", split=sp):
+ m = re.match(r"\[header\] (.*?) \[", r["ctx_a"] + " [")
+ if r["source_id"].startswith("wikihow") and m: banned.add(" ".join(norm(m.group(1))))
+ return banned, norm
+
+
+def cosmo1_how_tf():
+ # Cosmopedia v1 "wikihow" (the set used in Preview-1/2), now also dropping any article whose title
+ # matches a HellaSwag WikiHow item, as for cosmo2_how.
+ import re
+ from huggingface_hub import hf_hub_download
+ import pyarrow.parquet as pq
+ banned, norm = _hs_banned_titles(); dropped = 0
+ for k in range(2):
+ t = pq.read_table(hf_hub_download("HuggingFaceTB/cosmopedia", f"data/wikihow/train-0000{k}-of-00002.parquet", repo_type="dataset"),
+ columns=["prompt", "text"])
+ for pr, tx in zip(t.column("prompt").to_pylist(), t.column("text").to_pylist()):
+ m = re.search(r'tutorial on "(.+?)"', pr)
+ if m and " ".join(norm(m.group(1))) in banned:
+ dropped += 1; continue
+ yield tx.strip()
+ print(f" {dropped:,} dropped for a HellaSwag title", flush=True)
+
+
+def cosmo2_how():
+ # Cosmopedia v2 (SmolLM-Corpus, ODC-BY; Mixtral-8x7B generations), "wikihow" format only.
+ # HellaSwag draws on WikiHow, so every document goes through the 13-gram eval filter below.
+ # Stricter than the 13-gram filter: some articles were prompted with real WikiHow titles, so any article
+ # whose title matches the title of a HellaSwag WikiHow item (validation or test) is dropped outright.
+ import re
+ norm = lambda t: re.sub(r"[^a-z0-9 ]", "", t.lower().replace("how to", "")).split()
+ banned = set()
+ for sp in ("validation", "test"):
+ for r in load_dataset("Rowan/hellaswag", split=sp):
+ m = re.match(r"\[header\] (.*?) \[", r["ctx_a"] + " [")
+ if r["source_id"].startswith("wikihow") and m: banned.add(" ".join(norm(m.group(1))))
+ print(f" {len(banned):,} HellaSwag WikiHow titles banned", flush=True)
+ dropped = 0
+ import os
+ start = int(os.environ.get("COSMO2_START_SHARD", "0")) # resume after a network failure: shards are read in order
+ files = [f"hf://datasets/HuggingFaceTB/smollm-corpus/cosmopedia-v2/train-{k:05d}-of-00104.parquet" for k in range(start, 104)]
+ for r in load_dataset("parquet", data_files=files, split="train", streaming=True):
+ if r["format"] != "wikihow": continue
+ m = re.search(r'"([^"]+)" in the style of WikiHow', r["prompt"]) or re.match(r"\s*Title:\s*(.*)", r["text"])
+ if m and " ".join(norm(m.group(1))) in banned:
+ dropped += 1
+ if dropped % 1000 == 0: print(f" {dropped:,} dropped for a HellaSwag title", flush=True)
+ continue
+ yield r["text"].strip()
+
+
+GEN = {"tinygsm": (tinygsm, "tinygsm"), "elem": (elem, "elem_wordprob"), "metamath_gsm": (metamath_gsm, "metamath_gsm"),
+ "gsm8k_aug": (gsm8k_aug, "gsm8k_aug"), "grade_lessons": (grade_lessons, "grade_lessons"),
+ "omi2_gsm": (omi2_gsm, "omi2_gsm"), "libremath": (libremath, "libremath_v1"), "cosmo2_how": (cosmo2_how, "cosmo2_how"), "libremath_tpl": (libremath_tpl, "libremath_tpl"), "cosmo1_how_tf": (cosmo1_how_tf, "cosmo_how_v1_tf")}
+
+
+def main():
+ ap = argparse.ArgumentParser(); ap.add_argument("name", choices=list(GEN))
+ ap.add_argument("--tokens", type=float, default=1e12)
+ a = ap.parse_args()
+ fn, outname = GEN[a.name]
+ out = prepare.DATA_DIR / outname
+ tok = BorisTokenizer.load(prepare.DATA_DIR / "tokenizer"); ban = prepare.eval_ngrams()
+ import os
+ w = prepare.ShardWriter(out, tok.eos_id); assert w.n_tokens == 0 or os.environ.get("COSMO2_START_SHARD"), f"{out} not empty"
+ kept = dropped = toks = 0; batch = []
+ gen = fn()
+ for t in gen:
+ if prepare.contaminated(t, ban):
+ dropped += 1; continue
+ batch.append(t)
+ if len(batch) == 1000:
+ toks += w.write(tok.encode_batch(batch)); kept += len(batch); batch = []
+ if kept % 200000 == 0: print(f" {kept:,} docs {toks/1e6:.0f}M tokens {dropped} dropped", flush=True)
+ if toks >= a.tokens: break
+ if batch: toks += w.write(tok.encode_batch(batch)); kept += len(batch)
+ w.close()
+ sha = hashlib.sha256(open(prepare.DATA_DIR / "tokenizer" / "tokenizer.json", "rb").read()).hexdigest()
+ json.dump({"source": a.name, "documents": kept, "new_tokens": toks, "dropped_contaminated": dropped,
+ "new_tokenizer_sha256": sha}, open(out / "retokenize_0920.json", "w"), indent=2)
+ print(f"{out.name}: {kept:,} docs, {toks:,} tokens, {dropped} dropped as contaminated")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/sample.py b/scripts/sample.py
new file mode 100644
index 0000000..b19a600
--- /dev/null
+++ b/scripts/sample.py
@@ -0,0 +1,47 @@
+"""Sample text from the newest Boris-2-0929 rolling checkpoint on the 3060 (no KV cache;
+recomputes the full prefix each token, fine for short samples)."""
+import glob, sys
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+import torch, torch.nn.functional as F
+import eval as E
+from tokenizer import BorisTokenizer
+
+dev = "cuda:1"
+ck = sorted(glob.glob("/media/joseph/Media/Boris-2-0929/checkpoints/rolling_*.pt"))[-1]
+b = torch.load(ck, map_location="cpu", weights_only=False)
+tmp = "/home/joseph/boris-2-D112M-n15M/probe/_sample_export.pt"
+torch.save({k: b[k] for k in ("model", "config", "step", "tokens_seen", "ngram_fingerprint")}, tmp)
+del b
+model, cfg, blob = E.load_export(Path(tmp), dev)
+Path(tmp).unlink()
+tok = BorisTokenizer.load(Path(cfg.data.root) / "tokenizer")
+W = model.head_weight().float()
+print(f"checkpoint step {blob['step']} tokens {blob['tokens_seen']/1e9:.2f}B\n")
+
+prompts = sys.argv[1:] or [
+ "The capital of France is",
+ "Once upon a time, in a small village by the sea,",
+ "Photosynthesis is the process by which",
+ "To make a cup of tea, first",
+ "Question: If I have 12 apples and give away 5, how many do I have left?\nAnswer:",
+ "The most important invention of the 20th century was",
+]
+torch.manual_seed(0)
+for p in prompts:
+ ids = tok.encode(p)
+ for _ in range(60):
+ T = len(ids)
+ x = torch.tensor([ids], device=dev)
+ pos = torch.arange(T, device=dev)[None]
+ bnd = torch.zeros(1, T, dtype=torch.bool, device=dev); bnd[0, 0] = True
+ off = torch.tensor([0, T], device=dev)
+ with torch.no_grad(), torch.autocast("cuda", dtype=torch.float16):
+ h = model(x, pos, off, bnd, None, model._eval_theta)
+ logits = F.linear(h[0, -1].float(), W) / 0.7
+ top = torch.topk(logits, 40)
+ nxt = int(top.indices[torch.multinomial(F.softmax(top.values, -1), 1)])
+ if nxt == tok.eos_id:
+ break
+ ids.append(nxt)
+ print("=" * 70); print(tok.decode(ids)); print()
diff --git a/scripts/scan_decay_contam.py b/scripts/scan_decay_contam.py
new file mode 100644
index 0000000..2577a05
--- /dev/null
+++ b/scripts/scan_decay_contam.py
@@ -0,0 +1,17 @@
+"""13-gram contamination scan of the decay sources (same ban set as prepare.py)."""
+import sys, json
+from pathlib import Path
+sys.path.insert(0, "/home/joseph/boris-2-D112M-n15M")
+import numpy as np
+import prepare
+from tokenizer import BorisTokenizer
+ROOT = Path("/home/joseph/boris-2-D112M-n15M/data")
+tok = BorisTokenizer.load(ROOT / "tokenizer"); ban = prepare.eval_ngrams()
+for name in sys.argv[1:]:
+ ix = np.memmap(ROOT / name / "docs.idx", dtype=np.uint64, mode="r")
+ tt = np.memmap(ROOT / name / "tokens.bin", dtype=np.uint16, mode="r")
+ bad = []
+ for i in range(len(ix) - 1):
+ t = tok.tk.decode(tt[int(ix[i]):int(ix[i + 1])].tolist(), skip_special_tokens=True)
+ if prepare.contaminated(t, ban): bad.append(i)
+ print(json.dumps({"source": name, "docs": len(ix) - 1, "contaminated": len(bad), "ids": bad}), flush=True)
diff --git a/sft_chat.py b/sft_chat.py
new file mode 100644
index 0000000..e258e28
--- /dev/null
+++ b/sft_chat.py
@@ -0,0 +1,165 @@
+"""Chat SFT for Boris-2 on smol-smoltalk (2026-11-11) -> Boris-2-Instruct.
+
+ChatML-style turns written as plain text (no new tokens, so the 32,768 vocabulary, the n-gram table and the
+export path are unchanged):
+ <|im_start|>user\n{content}<|im_end|>\n<|im_start|>assistant\n{content}<|im_end|>\n ... <|endoftext|>
+Loss only on assistant contents and their closing "<|im_end|>". Conversations are packed as separate documents
+into 2048-token rows (attention, Canon and the n-gram lookup reset at each one). Holdout = smol-smoltalk test.
+
+ BORIS_VOLTA_KERNELS=1 python sft_chat.py START.pt OUT.pt [--lr 1e-4 --epochs 1]
+"""
+import argparse, math, os, random, sys, time
+from array import array
+from pathlib import Path
+import numpy as np, torch
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+import eval as E
+from tokenizer import BorisTokenizer
+
+IGNORE, T = -100, 2048
+
+
+def encode_conv(msgs, tok, eos):
+ """-> (ids, sup): sup[t] = 1 if token t is a training target (assistant content + its <|im_end|>\n)."""
+ ids, sup = [], []
+ for m in msgs:
+ head = tok.encode(f"<|im_start|>{m['role']}\n")
+ body = tok.encode(m["content"].strip() + "<|im_end|>") + tok.encode("\n")
+ ids += head + body; sup += [0] * len(head) + [int(m["role"] == "assistant")] * len(body)
+ ids.append(eos); sup.append(0)
+ return ids[:T], sup[:T]
+
+
+def tokenize(split, tok, eos, cache, limit=None):
+ """Flat uint16 tokens + uint8 target mask + int64 starts, cached as .npy (Python int lists took ~25 GB)."""
+ if (cache / "ids.npy").exists():
+ return tuple(np.load(cache / f"{k}.npy") for k in ("ids", "sup", "start"))
+ ids, sup, start = array("H"), array("B"), array("q", [0])
+ for i, row in enumerate(split):
+ if limit and i >= limit: break
+ a, b = encode_conv(row["messages"], tok, eos); ids.extend(a); sup.extend(b); start.append(len(ids))
+ out = (np.frombuffer(ids, np.uint16).copy(), np.frombuffer(sup, np.uint8).copy(), np.frombuffer(start, np.int64).copy())
+ cache.mkdir(parents=True, exist_ok=True)
+ for k, v in zip(("ids", "sup", "start"), out): np.save(cache / f"{k}.npy", v)
+ return out
+
+
+def rows(lens, shuffle, seed):
+ order = list(range(len(lens)))
+ if shuffle: random.Random(seed).shuffle(order)
+ out, cur, n = [], [], 0
+ for e in order:
+ L = int(lens[e])
+ if n + L > T: out.append(cur); cur, n = [], 0
+ cur.append(e); n += L
+ if cur: out.append(cur)
+ return out
+
+
+def batch(rl, data, eos, dev):
+ ids, sup, start = data
+ B = len(rl); idx = torch.full((B, T), eos, dtype=torch.long); tgt = torch.full((B, T), IGNORE, dtype=torch.long)
+ pos = torch.zeros((B, T), dtype=torch.long); bnd = torch.zeros((B, T), dtype=torch.bool); offs, lens = [0], []
+ for b, row in enumerate(rl):
+ t = 0
+ for e in row:
+ s0, s1 = int(start[e]), int(start[e + 1]); L = s1 - s0
+ x = torch.from_numpy(ids[s0:s1].astype(np.int64)); m = torch.from_numpy(sup[s0:s1].astype(bool))
+ lab = torch.full((L,), IGNORE, dtype=torch.long); lab[:-1] = torch.where(m[1:], x[1:], IGNORE)
+ idx[b, t:t + L] = x; tgt[b, t:t + L] = lab # position t predicts token t+1
+ bnd[b, t] = True; pos[b, t:t + L] = torch.arange(L); lens.append(L); t += L; offs.append(b * T + t)
+ if t < T:
+ bnd[b, t] = True; pos[b, t:] = torch.arange(T - t); lens.append(T - t); offs.append(b * T + T)
+ o = torch.tensor(offs, dtype=torch.int64)
+ return idx.to(dev), pos.to(dev), o.to(dev), bnd.to(dev), tgt.to(dev), min(lens), max(lens)
+
+
+def openthink(path, fmt):
+ """OpenThink rows -> {"train", "test"} lists of chats. plain: the trace as-is ("... Final answer: X").
+ think: "<think>\\n{reasoning}\\n</think>\\n\\nFinal answer: X" (empty think block for answer-only rows).
+ Holdout = 800 rows chosen with a fixed seed (same split for every format and seed)."""
+ import json
+ recs = [json.loads(l) for l in open(path)]
+ def reply(t):
+ t = t.strip()
+ if fmt == "plain": return t
+ head, _, ans = t.rpartition("Final answer:")
+ return f"<think>\n{head.strip()}\n</think>\n\nFinal answer:{ans}"
+ chats = [{"messages": [{"role": "user", "content": r["question"]}, {"role": "assistant", "content": reply(r["trace"])}]} for r in recs]
+ order = list(range(len(chats))); random.Random(1234).shuffle(order); hold = set(order[:800])
+ return {"train": [c for i, c in enumerate(chats) if i not in hold], "test": [c for i, c in enumerate(chats) if i in hold]}
+
+
+def main():
+ ap = argparse.ArgumentParser(); ap.add_argument("start"); ap.add_argument("out")
+ ap.add_argument("--lr", type=float, default=1e-4); ap.add_argument("--epochs", type=int, default=1)
+ ap.add_argument("--mb", type=int, default=16); ap.add_argument("--accum", type=int, default=4)
+ ap.add_argument("--dev", default="cuda:0"); ap.add_argument("--max-steps", type=int, default=0)
+ ap.add_argument("--data", default=None, help="OpenThink jsonl (question/trace) instead of smol-smoltalk")
+ ap.add_argument("--fmt", default="plain", choices=("plain", "think")); ap.add_argument("--seed", type=int, default=0)
+ ap.add_argument("--cache", default="/media/joseph/Media/instruct/tok")
+ ap.add_argument("--mix", type=int, default=0, help="with --data: smol-smoltalk + N copies of the OpenThink train split; "
+ "smol-smoltalk replies get an empty think block when --fmt think")
+ a = ap.parse_args(); dev = a.dev; torch.manual_seed(a.seed)
+ os.environ.setdefault("HF_DATASETS_CACHE", "/media/joseph/Media/hf-cache/datasets")
+ from datasets import load_dataset
+ ds = openthink(a.data, a.fmt) if a.data else load_dataset("HuggingFaceTB/smol-smoltalk")
+ if a.data and a.mix:
+ import itertools
+ st = load_dataset("HuggingFaceTB/smol-smoltalk")
+ def wrap(c):
+ if a.fmt != "think": return c
+ return {"messages": [dict(m, content=f"<think>\n\n</think>\n\n{m['content'].strip()}") if m["role"] == "assistant" else m
+ for m in c["messages"]]}
+ ds = {"train": itertools.chain((wrap(c) for c in st["train"]), *[ds["train"]] * a.mix),
+ "test": [wrap(c) for c in itertools.islice(st["test"], 800)] + ds["test"]}
+ model, cfg, blob = E.load_export(Path(a.start), dev)
+ tok = BorisTokenizer.load(Path(cfg.data.root) / "tokenizer"); eos = cfg.data.eos_id
+ t0 = time.time()
+ C = Path(a.cache)
+ tr = tokenize(ds["train"], tok, eos, C / "train"); va = tokenize(ds["test"], tok, eos, C / "test", None if a.mix else 800)
+ print(f"tokenized {len(tr[2]) - 1} train conversations ({tr[0].size/1e6:.0f}M tokens, {int(tr[1].sum())/1e6:.0f}M supervised) in {time.time()-t0:.0f}s", flush=True)
+ theta = model._eval_theta
+ opt = torch.optim.AdamW(model.parameters(), lr=a.lr, betas=(0.9, 0.95), weight_decay=0.0, fused=True)
+ scaler = torch.amp.GradScaler("cuda", init_scale=2.0 ** 15)
+ R0 = rows(np.diff(tr[2]), False, 0); steps = math.ceil(len(R0) / (a.mb * a.accum)) * a.epochs
+ if a.max_steps: steps = min(steps, a.max_steps)
+ warm = max(1, steps // 30); VR = rows(np.diff(va[2]), False, 0)
+
+ def val():
+ model.eval(); tot = n = 0.0
+ with torch.no_grad():
+ for i in range(0, len(VR), a.mb):
+ x, p, o, bd, y, mn, mx = batch(VR[i:i + a.mb], va, eos, dev)
+ with torch.autocast("cuda", dtype=torch.float16):
+ l = model(x, p, o, bd, y, theta, mn, mx)
+ k = int((y != IGNORE).sum()); tot += float(l) * k; n += k
+ model.train(); return tot / max(n, 1)
+
+ print(f"{steps} steps of {a.mb * a.accum} rows; holdout loss before {val():.4f}", flush=True)
+ model.train(); step = 0; t0 = time.time(); seen = 0
+ for ep in range(a.epochs):
+ R = rows(np.diff(tr[2]), True, 100 + ep + 1000 * a.seed)
+ for i in range(0, len(R), a.mb * a.accum):
+ if step >= steps: break
+ lr = a.lr * min(1, (step + 1) / warm) * 0.5 * (1 + math.cos(math.pi * step / steps))
+ for g in opt.param_groups: g["lr"] = lr
+ chunk = R[i:i + a.mb * a.accum]
+ for j in range(0, len(chunk), a.mb):
+ x, p, o, bd, y, mn, mx = batch(chunk[j:j + a.mb], tr, eos, dev)
+ with torch.autocast("cuda", dtype=torch.float16):
+ loss = model(x, p, o, bd, y, theta, mn, mx)
+ scaler.scale(loss * len(chunk[j:j + a.mb]) / len(chunk)).backward(); seen += x.numel()
+ scaler.unscale_(opt); torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
+ scaler.step(opt); scaler.update(); opt.zero_grad(set_to_none=True); step += 1
+ if step % 50 == 0 or step == steps:
+ print(f"step {step}/{steps} loss {float(loss):.4f} lr {lr:.2e} {seen/(time.time()-t0)/1e3:.1f}k tok/s", flush=True)
+ if step % 500 == 0: print(f" holdout loss {val():.4f}", flush=True)
+ print(f"final holdout loss {val():.4f}", flush=True)
+ keep = set(blob["model"]); sd = model.state_dict()
+ blob["model"] = {k: sd[k].detach().float().cpu() for k in keep}
+ torch.save(blob, a.out); print("saved", a.out, flush=True)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/supervise_run.py b/supervise_run.py
new file mode 100644
index 0000000..731ab59
--- /dev/null
+++ b/supervise_run.py
@@ -0,0 +1,34 @@
+"""Launch one training process and retain its exit status for the local dashboard."""
+import datetime
+import fcntl
+import json
+import os
+from pathlib import Path
+import subprocess
+import shutil
+import sys
+
+ROOT = Path(__file__).resolve().parent
+RUN = Path(os.environ.get('BORIS_RUN_DIR', '/media/joseph/Media/Boris-2-0929'))
+lock = open(RUN / 'training.lock', 'w')
+fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+status = {'started_at': datetime.datetime.now(datetime.timezone.utc).isoformat()}
+def publish(**fields):
+ status.update(fields)
+ temp = RUN / 'process_status.json.tmp'
+ temp.write_text(json.dumps(status, indent=2))
+ temp.replace(RUN / 'process_status.json')
+
+with open(RUN / 'train.log', 'a', buffering=1) as out:
+ child = subprocess.Popen([sys.executable, '-u', str(ROOT / 'train.py'),
+ '--config', str(RUN / 'config.json'), *sys.argv[1:]],
+ cwd=ROOT, stdout=out, stderr=subprocess.STDOUT)
+ (ROOT / 'logs/train.pid').write_text(str(child.pid))
+ publish(state='running', pid=child.pid)
+ code = child.wait()
+ publish(state='completed' if code == 0 else 'failed', exit_code=code,
+ stopped_at=datetime.datetime.now(datetime.timezone.utc).isoformat())
+ if shutil.which('notify-send'):
+ subprocess.run(['notify-send', '--urgency=critical', f'{RUN.name} stopped',
+ f'Training exited with code {code}. See {RUN}/train.log'], check=False)
+sys.exit(code)
diff --git a/tokenizer.py b/tokenizer.py
new file mode 100644
index 0000000..74ea732
--- /dev/null
+++ b/tokenizer.py
@@ -0,0 +1,118 @@
+"""Our own byte-level BPE, trained on this corpus.
+
+32,768 tokens. A power of two needs no padding for the tensor cores, stores in
+uint16, and costs 16.8M tied embedding parameters against 25.8M for a 50k
+vocabulary -- at a 143.5M budget that difference is most of a layer's worth of
+FFN. Byte-level, so every input round-trips and there is no unknown token.
+
+ python tokenizer.py train --sample data/tokenizer_sample.txt
+ python tokenizer.py inspect
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+from pathlib import Path
+
+EOS = "<|endoftext|>"
+
+
+class BorisTokenizer:
+ def __init__(self, tk, eos_id: int):
+ self.tk = tk
+ self.eos_id = eos_id
+
+ @property
+ def vocab_size(self) -> int:
+ return self.tk.get_vocab_size()
+
+ def encode(self, text: str) -> list[int]:
+ return self.tk.encode(text, add_special_tokens=False).ids
+
+ def encode_with_offsets(self, text: str) -> tuple[list[int], list[tuple[int, int]]]:
+ """Token ids and offsets into the original string.
+
+ Evaluation needs the offsets because BPE is not prefix-stable: the last
+ context token can merge with the first continuation characters. Splitting
+ ``encode(context + choice)`` at ``len(encode(context))`` can therefore
+ score the wrong tokens.
+ """
+ e = self.tk.encode(text, add_special_tokens=False)
+ return e.ids, e.offsets
+
+ def encode_batch(self, texts: list[str]) -> list[list[int]]:
+ return [e.ids for e in self.tk.encode_batch(texts, add_special_tokens=False)]
+
+ def decode(self, ids) -> str:
+ return self.tk.decode(list(ids), skip_special_tokens=False)
+
+ def save(self, d: Path):
+ d = Path(d); d.mkdir(parents=True, exist_ok=True)
+ self.tk.save(str(d / "tokenizer.json"))
+ (d / "meta.json").write_text(json.dumps(
+ {"eos_id": self.eos_id, "vocab_size": self.vocab_size}, indent=2))
+
+ @staticmethod
+ def fingerprint(d: Path) -> str:
+ """Full tokenizer identity used by checkpoint/config compatibility."""
+ h = hashlib.sha256()
+ with open(Path(d) / "tokenizer.json", "rb") as f:
+ for chunk in iter(lambda: f.read(1 << 20), b""):
+ h.update(chunk)
+ return h.hexdigest()
+
+ @staticmethod
+ def load(d: Path) -> "BorisTokenizer":
+ from tokenizers import Tokenizer
+ d = Path(d)
+ import os
+ if os.environ.get("BORIS_DIGIT_SPLIT") == "1" and (d.parent / "tokenizer_digits").exists():
+ tk = Tokenizer.from_file(str(d.parent / "tokenizer_digits" / "tokenizer.json"))
+ else:
+ tk = Tokenizer.from_file(str(d / "tokenizer.json"))
+ meta = json.loads((d / "meta.json").read_text())
+ return BorisTokenizer(tk, meta["eos_id"])
+
+ @staticmethod
+ def train(files: list[Path], vocab_size: int, out: Path) -> "BorisTokenizer":
+ from tokenizers import Tokenizer, decoders, models, pre_tokenizers, trainers
+ tk = Tokenizer(models.BPE(unk_token=None))
+ # ByteLevel with no initial-space prefix: documents are packed rather
+ # than sentence-fed, so a leading space would be wrong at every
+ # document start.
+ tk.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
+ tk.decoder = decoders.ByteLevel()
+ trainer = trainers.BpeTrainer(
+ vocab_size=vocab_size, special_tokens=[EOS], show_progress=True,
+ initial_alphabet=pre_tokenizers.ByteLevel.alphabet())
+ tk.train([str(f) for f in files], trainer)
+ t = BorisTokenizer(tk, tk.token_to_id(EOS))
+ assert t.eos_id == 0, f"{EOS} must be id 0, got {t.eos_id}"
+ assert t.vocab_size == vocab_size, (
+ f"trained {t.vocab_size} tokens, wanted {vocab_size}: the sample is "
+ f"probably too small to reach the target vocabulary")
+ t.save(out)
+ return t
+
+
+if __name__ == "__main__":
+ from config import DATA_DIR, Config
+ ap = argparse.ArgumentParser()
+ ap.add_argument("cmd", choices=["train", "inspect"])
+ ap.add_argument("--sample", type=Path, nargs="+",
+ default=[DATA_DIR / "tokenizer_sample.txt"])
+ ap.add_argument("--out", type=Path, default=DATA_DIR / "tokenizer")
+ a = ap.parse_args()
+ cfg = Config()
+ if a.cmd == "train":
+ t = BorisTokenizer.train(a.sample, cfg.model.vocab_size, a.out)
+ print(f" trained {t.vocab_size} tokens -> {a.out}")
+ else:
+ t = BorisTokenizer.load(a.out)
+ s = "The quick brown fox jumps over the lazy dog. 3 + 19 = 22."
+ ids = t.encode(s)
+ print(f" vocab {t.vocab_size} eos {t.eos_id}")
+ print(f" {len(ids)} tokens for {len(s)} chars ({len(s)/len(ids):.2f} ch/tok)")
+ assert t.decode(ids) == s, "byte-level BPE must round-trip exactly"
+ print(" round-trip ok")
diff --git a/train.py b/train.py
new file mode 100644
index 0000000..113b873
--- /dev/null
+++ b/train.py
@@ -0,0 +1,596 @@
+"""Boris-2 pretraining loop: 200B tokens, fp16, single V100.
+
+Run:
+ python train.py # fresh
+ python train.py --resume # from the newest rolling checkpoint
+ python train.py --resume path.pt # from a specific one
+
+The loop assumes it will be interrupted. Everything needed to continue -- the
+weights, both optimizer states, the loss scale, the packer's RNG and its carried
+document tail -- goes into a rolling checkpoint every 45 minutes, and a resume
+replays the identical token stream from there.
+"""
+from __future__ import annotations
+
+import os
+
+os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
+
+import argparse
+import json
+import math
+import subprocess
+import sys
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+
+import torch
+
+from config import LOG_DIR, Config
+import checkpoint as ckpt
+from data import Prefetcher, VarlenPacker, open_sources, open_validation
+from model import Boris2
+from ngram import NGramVocab
+from optim import build_optimizers, set_lr
+from schedule import (context_at, lr_scale, mixture_at, phase_at,
+ stable_end_tokens)
+from tokenizer import BorisTokenizer
+
+
+# --------------------------------------------------------------------------- #
+def assert_environment(cfg: Config, log):
+ """Catch a silently-wrong precision setup in seconds rather than on day 30."""
+ if not torch.cuda.is_available():
+ raise SystemExit("no CUDA device")
+ dev = torch.device(cfg.train.device)
+ p = torch.cuda.get_device_properties(dev)
+ log(f" device {p.name} sm_{p.major}{p.minor} "
+ f"{p.total_memory/2**30:.1f} GB {p.multi_processor_count} SMs")
+ if p.major != 7:
+ log(f" WARNING: this run is tuned for sm_70; found sm_{p.major}{p.minor}")
+
+ assert cfg.train.amp_dtype == "float16", (
+ "sm_70 tensor cores are fp16-in/fp32-accumulate and have no bf16 path; "
+ "bf16 would run in software and TF32 does not exist before sm_80")
+ torch.backends.cuda.matmul.allow_tf32 = False
+ torch.backends.cudnn.allow_tf32 = False
+ torch.backends.cudnn.benchmark = True
+ # Measured +7.4% end to end on this card. It lets cuBLAS reduce fp16 GEMM
+ # split-k partials in fp16; checked against fp32 references in verify.py,
+ # where the LM head gradients are unaffected at 9e-04 / 2e-04. It is the
+ # default, and it is set explicitly so a future torch cannot flip it
+ # silently and cost a week.
+ torch.backends.cuda.matmul.allow_fp16_reduced_precision_reduction = True
+
+ # Prove autocast is actually engaged and producing fp16, rather than the
+ # loop quietly running fp32 for weeks at a quarter of the throughput.
+ a = torch.randn(256, 256, device=dev)
+ with torch.autocast("cuda", dtype=torch.float16):
+ out = a @ a
+ assert out.dtype == torch.float16, (
+ f"autocast produced {out.dtype}: fp16 is not engaged")
+ log(f" precision fp16 autocast + GradScaler, TF32 off, bf16 unused")
+
+
+_NVML = None
+
+
+def _nvml_temp(index: int) -> float:
+ """Temperature through libnvidia-ml directly: ~18 us, versus ~14 ms to fork
+ nvidia-smi (whose indices are NVML's, so the numbering is the same)."""
+ global _NVML
+ import ctypes
+ if _NVML is None:
+ lib = ctypes.CDLL("libnvidia-ml.so.1")
+ if lib.nvmlInit_v2() != 0:
+ raise RuntimeError("nvmlInit failed")
+ _NVML = (lib, {})
+ lib, handles = _NVML
+ if index not in handles:
+ h = ctypes.c_void_p()
+ if lib.nvmlDeviceGetHandleByIndex_v2(index, ctypes.byref(h)) != 0:
+ raise RuntimeError("nvml handle failed")
+ handles[index] = h
+ t = ctypes.c_uint()
+ if lib.nvmlDeviceGetTemperature(handles[index], 0, ctypes.byref(t)) != 0:
+ raise RuntimeError("nvml temperature read failed")
+ return float(t.value)
+
+
+def gpu_temp(index: int = 0) -> float:
+ try:
+ return _nvml_temp(index)
+ except Exception:
+ pass
+ try:
+ r = subprocess.run(["nvidia-smi", f"--id={index}",
+ "--query-gpu=temperature.gpu", "--format=csv,noheader"],
+ capture_output=True, text=True, timeout=4)
+ return float(r.stdout.strip().splitlines()[0])
+ except Exception:
+ return float("nan")
+
+
+def device_index(device: str | torch.device) -> int:
+ """Physical CUDA index used by diagnostics; never assume cuda:0."""
+ d = torch.device(device)
+ return torch.cuda.current_device() if d.index is None else d.index
+
+
+class Logger:
+ def __init__(self, path: Path):
+ path.parent.mkdir(parents=True, exist_ok=True)
+ self.f = open(path, "a")
+
+ def line(self, s: str):
+ print(s, flush=True)
+ self.f.write(json.dumps({"msg": s, "timestamp": datetime.now(timezone.utc).isoformat()}) + "\n")
+ self.f.flush()
+
+ def row(self, d: dict):
+ d = {**d, "timestamp": datetime.now(timezone.utc).isoformat()}
+ self.f.write(json.dumps(d) + "\n")
+ self.f.flush()
+
+
+def step_flops(cfg: Config, tokens: int, sumsq: int) -> float:
+ """Exact fwd+bwd FLOPs for a step, attention included at what it really did.
+
+ Attention is varlen, so its cost is sum(len_i^2) over the documents actually
+ packed -- not block_size^2. Reporting the dense figure would overstate the
+ model's utilisation by roughly the block sparsity, which at ctx 2048 is 57%.
+ """
+ m = cfg.model
+ b = cfg.param_budget()
+ matmul = 2 * b["matmul"] * tokens
+ head = 2 * m.vocab_size * m.d_model * tokens
+ attn = m.n_layer * 2 * (m.n_head * m.head_dim) * sumsq
+ return 3.0 * (matmul + head + attn)
+
+
+# --------------------------------------------------------------------------- #
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--resume", nargs="?", const="auto", default=None)
+ ap.add_argument("--config", type=Path, default=None)
+ ap.add_argument("--max-steps", type=int, default=None)
+ ap.add_argument("--force-config", action="store_true")
+ ap.add_argument("--no-compile", action="store_true")
+ a = ap.parse_args()
+
+ cfg = Config.from_json(a.config.read_text()) if a.config else Config()
+ ckpt_dir = Path(cfg.checkpoint_dir)
+ if not a.resume and ckpt_dir.exists() and any(ckpt_dir.glob("*.pt")):
+ raise SystemExit("checkpoint directory is nonempty: use --resume or a new directory")
+ if a.no_compile:
+ cfg.train.compile_regions = False
+ log = Logger(LOG_DIR / f"{cfg.run_name}.jsonl")
+ log.line(f"\n {cfg.run_name}")
+ log.line(" " + "=" * 74)
+ assert_environment(cfg, log.line)
+
+ torch.manual_seed(cfg.train.seed)
+ dev = cfg.train.device
+ torch.cuda.set_device(dev)
+
+ tok_dir = Path(cfg.data.root) / "tokenizer"
+ tok_fp = BorisTokenizer.fingerprint(tok_dir)
+ if tok_fp != cfg.model.tokenizer_sha256:
+ raise SystemExit(
+ f"tokenizer fingerprint {tok_fp} at {tok_dir} does not match "
+ f"config {cfg.model.tokenizer_sha256}; token ids and embedding rows "
+ "would disagree")
+
+ # -- data ---------------------------------------------------------- #
+ stores = open_sources(cfg, log.line)
+ packer = VarlenPacker(cfg, stores, cfg.train.seed, split="train")
+ # The held-out split is just another prepared source directory, and it is
+ # deliberately not in cfg.data.sources, so the training packer can never
+ # draw from it.
+ val_packer = None
+ if (Path(cfg.data.root) / cfg.data.val_source / "tokens.bin").exists():
+ vcfg = Config.from_json(cfg.to_json())
+ vcfg.curriculum.enabled = False
+ val_packer = VarlenPacker(vcfg, {"val": open_validation(cfg)},
+ 12345, split="val")
+ log.line(f" val split "
+ f"{val_packer.stores['val'].n_tokens/1e6:.1f}M tokens held out")
+
+ # -- model --------------------------------------------------------- #
+ model = Boris2(cfg).to(dev)
+ if model.ngram is not None:
+ vocab = NGramVocab.load(cfg.ngram.vocab_path)
+ assert vocab.vocab_size == cfg.model.vocab_size, (
+ f"ngram_vocab.bin was built for vocab {vocab.vocab_size}, model is "
+ f"{cfg.model.vocab_size}")
+ model.ngram.load_vocab(vocab, dev)
+ ng_fp = vocab.fingerprint()
+ ng_compatible = (ng_fp, vocab.legacy_fingerprint())
+ log.line(f" ngram vocab {vocab.bigram_keys.size:,} bigram + "
+ f"{vocab.trigram_keys.size:,} trigram rows, fingerprint {ng_fp}")
+ else:
+ ng_fp = "disabled"
+ ng_compatible = (ng_fp,)
+ log.line(" ngram table OFF (use_ngram=False): dense control arm")
+ if not a.no_compile:
+ model.maybe_compile()
+
+ muon, adamw = build_optimizers(model, cfg, log.line)
+ scaler = torch.amp.GradScaler("cuda", init_scale=cfg.train.init_scale)
+
+ # -- resume -------------------------------------------------------- #
+ step, tokens_seen, best_val = 0, 0, float("inf")
+ bad_grad = bad_loss = floored = 0 # consecutive non-finite log intervals
+ if a.resume:
+ path = (ckpt.latest_rolling(ckpt_dir) if a.resume == "auto"
+ else Path(a.resume))
+ if path is None:
+ raise SystemExit("--resume requested but no rolling checkpoint exists")
+ else:
+ blob = ckpt.load_full(
+ path, model=model, muon=muon, adamw=adamw, scaler=scaler,
+ packer=packer, map_location=dev,
+ expect_fingerprint=None if a.force_config else cfg.fingerprint(),
+ expect_ngram=None if a.force_config else ng_compatible)
+ step, tokens_seen = blob["step"], blob["tokens_seen"]
+ best_val = blob.get("best_val", float("inf"))
+ log.line(f" resumed {path.name} at step {step:,}, "
+ f"{tokens_seen/1e9:.2f}B tokens")
+
+ # -- run ------------------------------------------------------------ #
+ # The run ends when the token budget is spent, not at a fixed step count: tokens_per_step may change at a
+ # resume (2026-11-05: doubled for the final decay). Step numbers keep counting up, because rolling
+ # checkpoints are named and pruned by step. For an unchanged batch this equals cfg.total_steps.
+ total_steps = a.max_steps or (step + max(0, cfg.train.total_tokens - tokens_seen) // cfg.train.tokens_per_step)
+ progress = tokens_seen / cfg.train.total_tokens
+ block, theta, mb, accum = context_at(progress, cfg)
+ pre = Prefetcher(packer, cfg, dev)
+ pre.configure(mb, block, progress)
+ pre.start()
+
+ log.line(f" budget {cfg.train.total_tokens/1e9:.0f}B tokens over "
+ f"{cfg.total_steps:,} steps of {cfg.train.tokens_per_step:,}")
+ log.line(f" context block {block}, theta {theta:,.0f}, "
+ f"micro_batch {mb} x {accum} accum")
+ log.line(" " + "=" * 74 + "\n")
+
+ t_run = time.time()
+ t_ckpt = time.time()
+ next_export = ((tokens_seen // cfg.train.export_every_tokens) + 1) * cfg.train.export_every_tokens
+ next_full = ((tokens_seen // cfg.train.full_ckpt_every_tokens) + 1) * cfg.train.full_ckpt_every_tokens
+ next_val = ((tokens_seen // cfg.train.val_every_tokens) + 1) * cfg.train.val_every_tokens
+ stable_end = stable_end_tokens(cfg)
+ stable_saved = tokens_seen >= stable_end
+ attempt = step
+ win_t, win_tok, win_loss, win_flops = time.time(), 0, 0.0, 0.0
+ win_steps = 0
+ retry_batches = []
+ consecutive_skips = 0
+
+ temp_poll = 0
+ bad_temp_reads = 0
+ temperature = gpu_temp(device_index(dev))
+ while step < total_steps:
+ # Leave headroom below the user's 80 C cap without changing power limits.
+ # gpu_temp() reads NVML directly (~18 us), so every step is free again;
+ # it falls back to forking nvidia-smi (~14 ms), where every 5th step
+ # keeps the idle GPU time small.
+ temp_poll += 1
+ if _NVML is not None or temp_poll % 5 == 1:
+ temperature = gpu_temp(device_index(dev))
+ # One failed nvidia-smi read used to end the run; tolerate a few in a
+ # row (still refusing to train unmonitored for long).
+ bad_temp_reads = 0 if math.isfinite(temperature) else bad_temp_reads + 1
+ if bad_temp_reads >= 3:
+ raise RuntimeError("cannot read GPU temperature 3 times running; "
+ "refusing unmonitored training")
+ if math.isfinite(temperature) and temperature >= 78:
+ # The sensor flickers 77 <-> 78 at steady load, so a single reading
+ # of 78 paused the run for 5 s of idle GPU. Confirm it a second
+ # later; the thresholds (pause at 78, resume at 75, cap 80) are
+ # unchanged and a real climb cannot gain 2 C in a second.
+ time.sleep(1)
+ temperature = gpu_temp(device_index(dev))
+ if math.isfinite(temperature) and temperature >= 78:
+ log.line(f"thermal pause at {temperature:.0f} C; waiting for <=75 C")
+ fails = 0
+ while not (math.isfinite(temperature) and temperature <= 75):
+ time.sleep(1)
+ temperature = gpu_temp(device_index(dev))
+ fails = 0 if math.isfinite(temperature) else fails + 1
+ if fails >= 3:
+ raise RuntimeError("GPU temperature unavailable during cooldown")
+ progress = tokens_seen / cfg.train.total_tokens
+ nb, ntheta, nmb, naccum = context_at(progress, cfg)
+ if (nb, nmb) != (block, mb):
+ log.line(f"\n context stage block {block} -> {nb}, "
+ f"theta {theta:,.0f} -> {ntheta:,.0f}, "
+ f"micro_batch {mb} -> {nmb} (one recompile expected)\n")
+ block, theta, mb, accum = nb, ntheta, nmb, naccum
+ pre.configure(mb, block, progress)
+ torch.cuda.empty_cache()
+ else:
+ theta = ntheta
+ pre.configure(mb, block, progress)
+
+ scale = lr_scale(progress, cfg)
+ mlr, alr = set_lr(muon, adamw, cfg, scale)
+
+ # Accumulated on the GPU. Reading it per micro-step with float() forces
+ # a host sync 16 times a step and destroys CPU run-ahead; it is read
+ # once, at log time, well after the fact.
+ loss_acc = torch.zeros((), device=dev)
+ tok_acc, sumsq_acc = 0, 0
+ for micro in range(accum):
+ if micro == len(retry_batches):
+ retry_batches.append(pre.next(mb, block))
+ x, y, pos, bnd, bounds, stats = retry_batches[micro]
+ with torch.autocast("cuda", dtype=torch.float16):
+ loss = model(x, pos, bounds, bnd, y, theta,
+ stats["min_seqlen"], stats["max_seqlen"])
+ scaler.scale(loss / accum).backward()
+ loss_acc += loss.detach() / accum
+ tok_acc += stats["tokens"]
+ sumsq_acc += stats["sumsq"]
+
+ grad_list = [p.grad for p in model.parameters() if p.grad is not None]
+ scaler.unscale_(muon)
+ scaler.unscale_(adamw)
+ # Not clip_grad_norm_: it scales every gradient by a coefficient derived
+ # from the norm over ALL parameters, so a single Inf anywhere -- routine
+ # on the first steps, where init_scale is 2^16 and GradScaler is meant
+ # to overflow, back off and skip -- makes the coefficient non-finite and
+ # writes NaN into gradients that were fine. found_inf is recorded per
+ # optimizer at unscale_ time, so Muon would skip while AdamW stepped on
+ # the freshly-poisoned grads, and 0 * NaN = NaN defeats even the lr-0
+ # warmup step. That is exactly how this run died: step 1 loss 10.4739,
+ # step 2 loss nan. Leave the gradients untouched when the norm is not
+ # finite and let GradScaler do its job.
+ norms = torch._foreach_norm(grad_list)
+ gnorm = torch.linalg.vector_norm(torch.stack(norms))
+ coef = torch.where(torch.isfinite(gnorm),
+ (cfg.optim.grad_clip / (gnorm + 1e-6)).clamp(max=1.0),
+ torch.ones_like(gnorm))
+ # A tensor coefficient, and torch.where rather than a Python branch:
+ # reading gnorm here would be a host sync on every step.
+ pre_scale = scaler.get_scale()
+ # GradScaler makes its skip decision per optimizer. That allowed a head
+ # overflow to skip AdamW while Muon still changed the trunk: a partial
+ # update that desynchronised the model. Both optimizers see unscaled
+ # gradients here, so make one global finite decision and step both or
+ # neither. The scalar reads replace the loss read already performed
+ # below; they do not add another synchronization point.
+ loss_value = float(loss_acc)
+ gnorm_value = float(gnorm)
+ # coef is exactly 1.0 unless the norm reaches the clip (it never has in
+ # this run), and x * 1.0 == x, so skipping the multiply is bitwise the
+ # same update. It is applied, as before, whenever clipping engages.
+ if math.isfinite(gnorm_value) and gnorm_value + 1e-6 >= cfg.optim.grad_clip:
+ torch._foreach_mul_(grad_list, coef)
+ did_step = math.isfinite(loss_value) and math.isfinite(gnorm_value)
+ if did_step:
+ muon.step()
+ adamw.step()
+ scaler.update()
+ # A non-finite LOSS is a forward overflow, and halving the loss scale
+ # cannot touch it -- the forward does not depend on the scale at all.
+ # GradScaler halves anyway, every time, and that ratchet is what walked
+ # two runs down to the floor. Restore the scale on those steps: the step
+ # is still skipped (found_inf), so the weights are safe, and a rare
+ # forward NaN costs exactly one step instead of compounding. Genuine
+ # gradient overflow, where the loss is finite, still halves normally.
+ # One host sync per step, not the sixteen a per-micro-batch check costs.
+ if not math.isfinite(loss_value):
+ scaler.update(pre_scale)
+ # Clamp the loss scale at both ends. GradScaler will otherwise run the
+ # scale down without limit, and below ~1e-30 unscale_ (grad/scale)
+ # overflows fp32 to inf, sets found_inf, and halves it again -- a
+ # self-reinforcing collapse with no exit, which is what ended the run at
+ # step 3,810 with the weights still perfectly finite. The ceiling is the
+ # other half: the scale had grown to 131,072 before the first overflow.
+ _s = scaler.get_scale()
+ if _s < cfg.train.min_loss_scale:
+ scaler.update(cfg.train.min_loss_scale)
+ floored += 1
+ elif _s > cfg.train.max_loss_scale:
+ scaler.update(cfg.train.max_loss_scale)
+ floored = 0
+ else:
+ floored = 0
+ muon.zero_grad(set_to_none=True)
+ adamw.zero_grad(set_to_none=True)
+
+ if did_step:
+ step += 1
+ tokens_seen += tok_acc
+ retry_batches.clear()
+ consecutive_skips = 0
+ else:
+ consecutive_skips += 1
+ # A non-finite LOSS is a forward overflow: deterministic, so retrying
+ # the identical batch only repeats it until the 20-skip abort. Move
+ # on to fresh data. Gradient overflow (finite loss) still retries the
+ # same batch, where the halved loss scale changes the outcome.
+ if not math.isfinite(loss_value):
+ retry_batches.clear()
+ log.row({"event": "skipped_update", "step": step,
+ "attempt": attempt + 1, "loss": loss_value,
+ "grad_norm": gnorm_value, "scale": scaler.get_scale(),
+ "consecutive_skips": consecutive_skips})
+ if consecutive_skips >= 20:
+ log.line(f"ABORT: 20 consecutive failed updates at step {step}; same batch retried")
+ ckpt.save_full(ckpt_dir / f"abort_{step:09d}.pt", model=model,
+ muon=muon, adamw=adamw, scaler=scaler,
+ packer=pre, step=step, tokens_seen=tokens_seen,
+ cfg=cfg, ngram_fingerprint=ng_fp,
+ best_val=best_val, background=False)
+ raise SystemExit(3)
+ attempt += 1
+ win_tok += tok_acc
+ win_loss += loss_acc
+ win_steps += 1
+ win_flops += step_flops(cfg, tok_acc, sumsq_acc)
+
+ # -- logging ---------------------------------------------------- #
+ if attempt % cfg.train.log_every_steps == 0:
+ dt = time.time() - win_t
+ tps = win_tok / max(dt, 1e-9)
+ tflops = win_flops / max(dt, 1e-9) / 1e12
+ eta = (cfg.train.total_tokens - tokens_seen) / max(tps, 1e-9)
+ temp = gpu_temp(device_index(dev))
+ mem = torch.cuda.max_memory_allocated() / 2**30
+ log.row({"step": step, "attempt": attempt, "tokens": tokens_seen,
+ "loss": float(win_loss) / max(win_steps, 1),
+ "lr_muon": mlr, "lr_adamw": alr, "grad_norm": gnorm_value,
+ "scale": scaler.get_scale(), "tok_s": tps,
+ "tflops": tflops, "block": block, "temp": temp,
+ "vram_gb": mem, "phase": phase_at(progress, cfg),
+ "wall": time.time() - t_run})
+ print(f" step {step:>7,}/{total_steps:,} {progress:>6.2%} "
+ f"loss {float(win_loss)/max(win_steps, 1):6.4f} "
+ f"lr {mlr:.2e} gn {gnorm_value:5.2f} "
+ f"s{math.log2(max(scaler.get_scale(), 1e-45)):+03.0f} "
+ f"{tps/1e3:6.1f}k tok/s {tflops:5.1f} TF "
+ f"{temp:.0f}C {mem:4.1f}G "
+ f"eta {eta/86400:5.1f}d", flush=True)
+
+ # -- health ------------------------------------------------- #
+ # A non-finite grad norm is normal in ones and twos: GradScaler
+ # probes for its loss scale by overflowing on purpose. PERSISTENT
+ # non-finite is a different animal -- it means found_inf fires every
+ # step and an optimizer is being skipped every step. That is what
+ # happened at step 230 of the first launch: the head's dW GEMM
+ # overflowed fp16 independently of the loss scale, so AdamW never
+ # stepped again, the embeddings and head froze while Muon kept
+ # training the trunk, and the loss climbed from 6.48 to nan over
+ # 180 steps and then burned 3,300 more. Neither the temperature nor
+ # the throughput showed anything wrong. Stop instead of cooking.
+ bad_grad = 0 if math.isfinite(gnorm_value) else bad_grad + 1
+ # Divide by the steps ACTUALLY in this window, not the nominal
+ # window size. A resume lands mid-window (step 57,340 with
+ # log_every_steps=10), so the first line after every resume
+ # under-reported the loss by up to 10x -- 0.337 for a model
+ # whose real loss was 3.37.
+ win_mean = float(win_loss) / max(win_steps, 1)
+ bad_loss = 0 if math.isfinite(win_mean) else bad_loss + 1
+ if bad_loss * cfg.train.log_every_steps >= cfg.train.abort_after_bad:
+ print(f"\n ABORT: loss has been non-finite for "
+ f"{bad_loss * cfg.train.log_every_steps} steps.\n",
+ flush=True)
+ ckpt.save_full(ckpt_dir / f"abort_{step:09d}.pt", model=model,
+ muon=muon, adamw=adamw, scaler=scaler,
+ packer=pre, step=step, tokens_seen=tokens_seen,
+ cfg=cfg, ngram_fingerprint=ng_fp,
+ best_val=best_val, background=False)
+ raise SystemExit(3)
+ if floored >= cfg.train.abort_after_floored:
+ print(f"\n ABORT: the loss scale has been pinned at its floor "
+ f"({cfg.train.min_loss_scale}) for {floored} steps -- "
+ f"every step is being skipped and nothing is training.\n",
+ flush=True)
+ ckpt.save_full(ckpt_dir / f"abort_{step:09d}.pt", model=model,
+ muon=muon, adamw=adamw, scaler=scaler,
+ packer=pre, step=step, tokens_seen=tokens_seen,
+ cfg=cfg, ngram_fingerprint=ng_fp,
+ best_val=best_val, background=False)
+ raise SystemExit(3)
+ if bad_grad * cfg.train.log_every_steps >= cfg.train.abort_after_bad:
+ print(f"\n ABORT: the gradient norm has been non-finite for "
+ f"{bad_grad * cfg.train.log_every_steps} steps -- an "
+ f"optimizer is being skipped every step.\n", flush=True)
+ ckpt.save_full(ckpt_dir / f"abort_{step:09d}.pt", model=model,
+ muon=muon, adamw=adamw, scaler=scaler,
+ packer=pre, step=step, tokens_seen=tokens_seen,
+ cfg=cfg, ngram_fingerprint=ng_fp,
+ best_val=best_val, background=False)
+ raise SystemExit(3)
+
+ win_t, win_tok, win_loss, win_flops = time.time(), 0, 0.0, 0.0
+ win_steps = 0
+
+ # -- validation ------------------------------------------------- #
+ if val_packer is not None and tokens_seen >= next_val:
+ next_val += cfg.train.val_every_tokens
+ vl = quick_val(model, val_packer, cfg, dev, block, theta, progress)
+ best_val = min(best_val, vl)
+ log.row({"step": step, "tokens": tokens_seen, "val_loss": vl,
+ "val_ppl": math.exp(min(vl, 20))})
+ print(f" val {vl:.4f} ppl {math.exp(min(vl,20)):.2f}", flush=True)
+
+ # -- checkpoints ------------------------------------------------- #
+ if (time.time() - t_ckpt) / 60 >= cfg.train.rolling_ckpt_minutes:
+ t_ckpt = time.time()
+ p = ckpt_dir / f"rolling_{step:09d}.pt"
+ ckpt.save_full(p, model=model, muon=muon, adamw=adamw, scaler=scaler,
+ packer=pre, step=step, tokens_seen=tokens_seen,
+ cfg=cfg, ngram_fingerprint=ng_fp, best_val=best_val,
+ rotate_pattern=r"rolling_\d+\.pt$",
+ rotate_keep=cfg.train.rolling_ckpt_keep)
+ log.line(f" rolling checkpoint {p.name}")
+
+ if tokens_seen >= next_export:
+ next_export += cfg.train.export_every_tokens
+ d = cfg.eval.export_dir
+ p = d / f"export_{tokens_seen//10**9:04d}B.pt"
+ ckpt.save_export(p, model=model, cfg=cfg, step=step,
+ tokens_seen=tokens_seen, manifest=cfg.eval.manifest,
+ ngram_fingerprint=ng_fp)
+ log.line(f" export {p.name} -> eval queue")
+
+ if tokens_seen >= next_full:
+ next_full += cfg.train.full_ckpt_every_tokens
+ p = ckpt_dir / f"full_{tokens_seen//10**9:04d}B.pt"
+ ckpt.save_full(p, model=model, muon=muon, adamw=adamw, scaler=scaler,
+ packer=pre, step=step, tokens_seen=tokens_seen,
+ cfg=cfg, ngram_fingerprint=ng_fp, best_val=best_val)
+ log.line(f" full checkpoint {p.name}")
+
+ if not stable_saved and tokens_seen >= stable_end:
+ stable_saved = True
+ p = ckpt_dir / "stable.pt"
+ ckpt.save_full(p, model=model, muon=muon, adamw=adamw, scaler=scaler,
+ packer=pre, step=step, tokens_seen=tokens_seen,
+ cfg=cfg, ngram_fingerprint=ng_fp, best_val=best_val,
+ background=False)
+ log.line(f" END OF STABLE PHASE -> stable.pt (permanent, never rotated)")
+
+ pre.stop()
+ p = ckpt_dir / "final.pt"
+ ckpt.save_full(p, model=model, muon=muon, adamw=adamw, scaler=scaler,
+ packer=pre, step=step, tokens_seen=tokens_seen, cfg=cfg,
+ ngram_fingerprint=ng_fp, best_val=best_val, background=False)
+ log.line(f"\n done: {tokens_seen/1e9:.2f}B tokens in "
+ f"{(time.time()-t_run)/86400:.2f} days -> {p}")
+
+
+@torch.no_grad()
+def quick_val(model, val_packer, cfg, dev, block, theta, progress,
+ batches: int = 16):
+ """A short in-loop validation. The full suite runs on the second GPU."""
+ model.eval()
+ tot, n = 0.0, 0
+ # Score the SAME held-out batches every time. The packer's RNG used to run
+ # on from call to call (and restart at its seed on every resume), so each
+ # validation drew a fresh sample and consecutive values wandered by about
+ # +/-0.05 -- larger than the effects being tracked. Snapshot the state on
+ # the first call and rewind to it on every call.
+ if not hasattr(val_packer, "_boris_val_state0"):
+ val_packer._boris_val_state0 = val_packer.state_dict()
+ val_packer.load_state_dict(val_packer._boris_val_state0)
+ # Stage-appropriate micro-batch: using the 2048-token value at block 8192
+ # would quadruple the validation activation footprint and OOM late in run.
+ mb = cfg.micro_batch_at(progress)
+ for _ in range(batches):
+ xs, ys, pos, bnd, bounds, _ = val_packer.batch(mb, block, 0.0)
+ t = lambda z: torch.from_numpy(z).to(dev)
+ with torch.autocast("cuda", dtype=torch.float16):
+ loss = model(t(xs), t(pos), t(bounds), t(bnd), t(ys), theta)
+ valid = int((ys != -100).sum())
+ tot += float(loss) * valid; n += valid
+ model.train()
+ return tot / max(n, 1)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/verify.py b/verify.py
new file mode 100644
index 0000000..9418e07
--- /dev/null
+++ b/verify.py
@@ -0,0 +1,1164 @@
+"""Correctness checks. Run before starting a run, and after touching a kernel.
+
+ python verify.py [--device cuda:0] [--quick]
+
+The checks that matter most are the isolation ones. Boris-2 packs several
+documents into every row, and three separate mechanisms have to keep them apart:
+varlen attention, the Canon boundary mask, and the n-gram history mask. A leak
+in any of them is invisible in the loss curve -- the model simply learns from
+context it will never have at inference.
+
+Note how check_document_isolation de-zeroes the Canon kernels first. They are
+zero-initialised, so at init the convolution contributes exactly nothing and the
+test passes no matter how badly it leaks. The 75M run shipped that bug for weeks
+behind a test that looked green.
+"""
+from __future__ import annotations
+
+import argparse
+import math
+import os
+import sys
+import atexit
+import shutil
+import tempfile
+from pathlib import Path
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from config import Config
+from model import Boris2, IGNORE, fused_lm_loss, _pre_attn
+from ngram import NGramVocab
+from optim import Muon, build_optimizers, newton_schulz
+from schedule import lr_scale, mixture_at, context_at
+
+PASS, FAIL = [], []
+
+
+_TMPDIRS = []
+
+
+def _tmpdir():
+ """mkdtemp that actually cleans up.
+
+ verify.py used bare tempfile.mkdtemp() and never removed anything. /tmp here
+ is a 16 GB tmpfs -- RAM -- and each run leaves ~500 MB of packer fixtures
+ behind, so 89 runs had filled it completely. That is not just a disk
+ problem: it was eating the machine's memory and getting background jobs
+ OOM-killed.
+ """
+ d = tempfile.mkdtemp()
+ _TMPDIRS.append(d)
+ return d
+
+
+@atexit.register
+def _cleanup_tmpdirs():
+ for d in _TMPDIRS:
+ shutil.rmtree(d, ignore_errors=True)
+
+
+def check(name, cond, detail=""):
+ (PASS if cond else FAIL).append(name)
+ mark = "ok " if cond else "FAIL"
+ print(f" [{mark}] {name}" + (f" {detail}" if detail else ""))
+ return cond
+
+
+# --------------------------------------------------------------------------- #
+def synth_vocab(cfg, seed=0):
+ V = cfg.model.vocab_size
+ rng = np.random.default_rng(seed)
+ bk = np.unique(rng.integers(0, V * V, size=cfg.ngram.bigram_rows).astype(np.int64))
+ tk = np.unique(rng.integers(0, V * V * V, size=cfg.ngram.trigram_rows).astype(np.int64))
+ return NGramVocab(V, bk, np.arange(bk.size) + cfg.ngram.bigram_base,
+ tk, np.arange(tk.size) + cfg.ngram.trigram_base)
+
+
+def synth_batch(cfg, dev, B, T, lens_per_row, seed=0):
+ g = torch.Generator().manual_seed(seed)
+ V = cfg.model.vocab_size
+ x = torch.randint(0, V, (B, T), generator=g).to(dev)
+ pos = torch.zeros(B, T, dtype=torch.int64)
+ bnd = torch.zeros(B, T, dtype=torch.bool)
+ bounds = []
+ for b, lens in enumerate(lens_per_row):
+ p = 0
+ for L in lens:
+ bounds.append(b * T + p)
+ bnd[b, p] = True
+ pos[b, p:p + L] = torch.arange(L)
+ p += L
+ assert p == T, "row must be exactly full"
+ bounds.append(B * T)
+ return (x, pos.to(dev), torch.tensor(bounds, dtype=torch.int64).to(dev), bnd.to(dev))
+
+
+def build(cfg, dev, compile_regions=False):
+ cfg.train.compile_regions = compile_regions
+ m = Boris2(cfg).to(dev)
+ if m.ngram is not None:
+ m.ngram.load_vocab(synth_vocab(cfg), dev)
+ if compile_regions:
+ m.maybe_compile()
+ return m
+
+
+# --------------------------------------------------------------------------- #
+def check_fused_head(cfg, dev):
+ """The fused head must equal an ordinary cross-entropy, loss and gradient."""
+ torch.manual_seed(0)
+ N, d, V = 1024, cfg.model.d_model, cfg.model.vocab_size
+ x = torch.randn(N, d, device=dev, dtype=torch.float16, requires_grad=True)
+ W = (torch.randn(V, d, device=dev) * 0.02).requires_grad_(True)
+ t = torch.randint(0, V, (N,), device=dev)
+ t[::7] = IGNORE
+
+ loss = fused_lm_loss(x, W, t, chunk=256)
+ loss.backward()
+ gx, gW = x.grad.clone(), W.grad.clone()
+
+ # Reference in fp32. Comparing against the fp16 unfused head would compare
+ # two approximations: its grad_logits reach the GEMM at ~3e-8, far below
+ # what fp16 can represent, so it is the less accurate of the two.
+ x2 = x.detach().float().clone().requires_grad_(True)
+ W2 = W.detach().float().clone().requires_grad_(True)
+ ref = F.cross_entropy(F.linear(x2, W2), t, ignore_index=IGNORE)
+ ref.backward()
+
+ dl = abs(float(loss) - float(ref))
+ rx = (gx.float() - x2.grad).norm() / (x2.grad.norm() + 1e-9)
+ rw = (gW - W2.grad).norm() / (W2.grad.norm() + 1e-9)
+ check("fused LM head: loss matches fp32 cross_entropy", dl < 5e-3, f"|d| = {dl:.2e}")
+ check("fused LM head: dx matches fp32", float(rx) < 5e-3, f"rel-L2 {float(rx):.2e}")
+ check("fused LM head: dW matches fp32", float(rw) < 5e-3, f"rel-L2 {float(rw):.2e}")
+ # The point of the fusion: never hold (N, V) logits. Measured against the
+ # unfused head on the same inputs, at a realistic token count.
+ def peak_of(fn, N2=8192):
+ xx = torch.randn(N2, d, device=dev, dtype=torch.float16, requires_grad=True)
+ WW = (torch.randn(V, d, device=dev) * 0.02).requires_grad_(True)
+ tt = torch.randint(0, V, (N2,), device=dev)
+ torch.cuda.synchronize(); torch.cuda.empty_cache()
+ base = torch.cuda.memory_allocated(dev)
+ torch.cuda.reset_peak_memory_stats(dev)
+ fn(xx, WW, tt).backward()
+ torch.cuda.synchronize()
+ out = torch.cuda.max_memory_allocated(dev) - base
+ del xx, WW, tt
+ torch.cuda.empty_cache()
+ return out
+
+ fused_peak = peak_of(lambda a, b, c: fused_lm_loss(a, b, c))
+ plain_peak = peak_of(lambda a, b, c: F.cross_entropy(
+ F.linear(a, b.to(a.dtype)).float(), c, ignore_index=IGNORE))
+ check("fused LM head: uses far less memory than the unfused head",
+ fused_peak < plain_peak / 3,
+ f"{fused_peak/2**20:.0f} MB vs {plain_peak/2**20:.0f} MB "
+ f"({plain_peak/max(fused_peak,1):.1f}x)")
+
+
+def check_repeated_target_head(cfg, dev):
+ """Exercise the production Function under autocast, including its rescale."""
+ torch.manual_seed(91)
+ for dtype in (torch.float32, torch.float16):
+ x = torch.full((1024, 32), 16.0, device=dev, dtype=dtype, requires_grad=True)
+ w = (torch.randn(256, 32, device=dev) * .01).requires_grad_()
+ target = torch.zeros(1024, device=dev, dtype=torch.long)
+ with torch.autocast("cuda", dtype=torch.float16):
+ loss = fused_lm_loss(x, w, target, chunk=1024)
+ loss.backward()
+ xr = x.detach().float().requires_grad_()
+ wr = w.detach().clone().requires_grad_()
+ ref = F.cross_entropy(F.linear(xr, wr), target)
+ ref.backward()
+ rel = (w.grad - wr.grad).norm() / wr.grad.norm()
+ check(f"production head repeated-target dW {dtype}",
+ bool(torch.isfinite(w.grad).all()) and float(rel) < .005,
+ f"relative error {float(rel):.3g}")
+
+
+def check_canon_causality(cfg, dev):
+ """A change at position t must never move an output at t' < t."""
+ torch.manual_seed(0)
+ C, K = cfg.model.canon_channels, cfg.model.canon_kernel
+ B, T = 1, 64
+ x = torch.randn(B, T, cfg.model.d_model, device=dev, dtype=torch.float16)
+ w_qkv = torch.randn(C, cfg.model.d_model, device=dev, dtype=torch.float16) * 0.02
+ canon = torch.randn(C, K, device=dev) * 0.5
+ m1 = torch.ones(B, T, 1, device=dev, dtype=torch.float16)
+ m2 = torch.ones(B, T, 1, device=dev, dtype=torch.float16)
+ hd = cfg.model.head_dim
+ cos = torch.ones(B, T, 1, hd // 2, device=dev)
+ sin = torch.zeros(B, T, 1, hd // 2, device=dev)
+ nw = torch.ones(cfg.model.d_model, device=dev)
+ args = (nw, 1e-5, w_qkv, canon, m1, m2, cos, sin, cfg.model.n_head,
+ cfg.model.n_kv_head, hd, True)
+ q0 = _pre_attn(x, *args)[0]
+ x2 = x.clone(); x2[0, 32] += 10.0
+ q1 = _pre_attn(x2, *args)[0]
+ check("Canon is strictly causal", torch.equal(q0[0, :32], q1[0, :32]))
+ moved = int((q0[0, 32:35] != q1[0, 32:35]).any(-1).any(-1).sum())
+ check("Canon reaches exactly kernel-1 tokens forward", moved == 3,
+ f"{moved} of the next 3 positions moved")
+
+
+def check_document_isolation(cfg, dev):
+ """End-to-end: a change in one document must not move another.
+
+ Canon kernels are randomised first. Zero-initialised, they contribute
+ nothing and this test would pass on a model that leaks badly.
+ """
+ torch.manual_seed(0)
+ small = Config.from_json(cfg.to_json())
+ small.enforce_budget = False
+ small.model.n_layer = 6
+ m = build(small, dev)
+ for blk in m.blocks: # de-zero: the whole point
+ torch.nn.init.normal_(blk.attn.canon, std=0.5)
+
+ B, T = 1, 512
+ lens = [[200, 312]]
+ x, pos, bounds, bnd = synth_batch(small, dev, B, T, lens, seed=3)
+ with torch.autocast("cuda", dtype=torch.float16):
+ h0 = m(x, pos, bounds, bnd, None, 10_000.0)
+ x2 = x.clone(); x2[0, :200] = torch.randint(0, small.model.vocab_size,
+ (200,), device=dev)
+ h1 = m(x2, pos, bounds, bnd, None, 10_000.0)
+ drift = float((h0[0, 200:] - h1[0, 200:]).abs().max())
+ check("document isolation: rewriting doc A never moves doc B", drift == 0.0,
+ f"max logit drift {drift:.3e}")
+
+ same = float((h0[0, :200] - h1[0, :200]).abs().max())
+ check("the probe actually perturbs doc A", same > 0.0, f"max change {same:.3e}")
+
+
+def check_attention_matches_loop(cfg, dev):
+ """Nested varlen attention must equal a per-document causal SDPA."""
+ torch.manual_seed(0)
+ H, KV, D = cfg.model.n_head, cfg.model.n_kv_head, cfg.model.head_dim
+ lens = [300, 212, 512]
+ N = sum(lens)
+ off = torch.tensor([0] + list(np.cumsum(lens)), device=dev)
+ q = torch.randn(N, H, D, device=dev, dtype=torch.float16)
+ k = torch.randn(N, H, D, device=dev, dtype=torch.float16)
+ v = torch.randn(N, H, D, device=dev, dtype=torch.float16)
+ nj = lambda t: torch.nested.nested_tensor_from_jagged(t, off).transpose(1, 2)
+ o = F.scaled_dot_product_attention(nj(q), nj(k), nj(v), is_causal=True)
+ o = o.transpose(1, 2).values()
+ ref = torch.empty_like(q)
+ p = 0
+ for L in lens:
+ s = F.scaled_dot_product_attention(
+ q[p:p+L].transpose(0, 1)[None], k[p:p+L].transpose(0, 1)[None],
+ v[p:p+L].transpose(0, 1)[None], is_causal=True)
+ ref[p:p+L] = s[0].transpose(0, 1)
+ p += L
+ d = float((o.float() - ref.float()).abs().max())
+ check("varlen attention equals a per-document loop", d == 0.0, f"max diff {d:.1e}")
+
+
+def check_volta_kernels(cfg, dev):
+ """volta_kernels/: the sm_70 attention and LM head, held to the stock paths.
+
+ Runs whenever the device is sm_70, whether or not BORIS_VOLTA_KERNELS is set,
+ so the kernels are proven before anyone opts in. The bar is the stock path's
+ own error against an fp64 reference, measured here on the same inputs --
+ "same precision" as a number, not an adjective.
+ """
+ if torch.cuda.get_device_capability(dev) != (7, 0):
+ print(" [skip] volta kernels: device is not sm_70")
+ return
+ import model as M
+ import volta_kernels as VK
+ from volta_kernels.head import fused_lm_loss as volta_lm_loss
+ torch.manual_seed(0)
+ H, D = cfg.model.n_head, cfg.model.head_dim
+ # Edge lengths around the 64-row tile, a length-1 document, and one long one.
+ lens = [1, 63, 64, 65, 127, 128, 300, 1300]
+ N = sum(lens)
+ off = torch.tensor([0] + list(np.cumsum(lens)), device=dev)
+ # v as a strided view of a fused qkv, exactly as the model hands it over.
+ qkv = (torch.randn(N, 3 * H * D, device=dev) * 1.5).half()
+ q = qkv[:, :H * D].reshape(N, H, D).contiguous().requires_grad_()
+ k = qkv[:, H * D:2 * H * D].reshape(N, H, D).contiguous().requires_grad_()
+ vb = qkv.clone().requires_grad_()
+ v = vb[:, 2 * H * D:].view(N, H, D)
+ go = torch.randn(N, H, D, device=dev).half()
+
+ def run(fn):
+ for t in (q, k, vb):
+ t.grad = None
+ o = fn()
+ o.backward(go)
+ return [o.detach().double(), q.grad.double(), k.grad.double(),
+ vb.grad[:, 2 * H * D:].reshape(N, H, D).double()]
+
+ def stock():
+ nj = lambda t: torch.nested.nested_tensor_from_jagged(t, off).transpose(1, 2)
+ return F.scaled_dot_product_attention(nj(q), nj(k), nj(v), is_causal=True
+ ).transpose(1, 2).values()
+
+ qq, kk, vv = (t.detach().double().requires_grad_() for t in (q, k, v))
+ outs, p = [], 0
+ for L in lens:
+ sl = slice(p, p + L)
+ outs.append(F.scaled_dot_product_attention(
+ qq[sl].transpose(0, 1), kk[sl].transpose(0, 1), vv[sl].transpose(0, 1),
+ is_causal=True).transpose(0, 1))
+ p += L
+ ref_o = torch.cat(outs)
+ ref_o.backward(go.double())
+ ref = [ref_o.detach(), qq.grad, kk.grad, vv.grad]
+ S = run(stock)
+ V = run(lambda: VK.varlen_causal_attention(q, k, v, off))
+ rel = lambda a, b: float((a - b).norm() / b.norm())
+ for name, i in (("output", 0), ("dq", 1), ("dk", 2), ("dv", 3)):
+ es, ev = rel(S[i], ref[i]), rel(V[i], ref[i])
+ check(f"volta attention: {name} as accurate as memory-efficient SDPA",
+ ev <= 1.1 * es and math.isfinite(ev), f"rel-L2 vs fp64 {ev:.2e} (stock {es:.2e})")
+
+ # Isolation and causality, exactly: perturb one document / one late token.
+ def fwd_only(qx, kx, vx):
+ return VK.varlen_causal_attention(qx, kx, vx, off.clone()).float()
+ q0, k0, v0 = (t.detach().clone() for t in (q, k, v))
+ o0 = fwd_only(q0, k0, v0)
+ a, b = int(off[6]), int(off[7]) # the 300-token document
+ q1, k1, v1 = q0.clone(), k0.clone(), v0.clone()
+ for t in (q1, k1, v1):
+ t[a:b] = torch.randn_like(t[a:b])
+ o1 = fwd_only(q1, k1, v1)
+ outside = torch.cat([o1[:a] - o0[:a], o1[b:] - o0[b:]]).abs().max().item()
+ check("volta attention: rewriting one document never moves another",
+ outside == 0.0 and (o1[a:b] - o0[a:b]).abs().max().item() > 0,
+ f"max drift outside {outside:.1e}")
+ t_ = a + 150
+ k2, v2 = k0.clone(), v0.clone()
+ k2[t_] = torch.randn_like(k2[t_]); v2[t_] = torch.randn_like(v2[t_])
+ o2 = fwd_only(q0, k2, v2)
+ before = (o2[a:t_] - o0[a:t_]).abs().max().item()
+ at = (o2[t_] - o0[t_]).abs().max().item()
+ check("volta attention: causal (a key never reaches an earlier query)",
+ before == 0.0 and at > 0, f"drift before {before:.1e}, at {at:.1e}")
+
+ # The per-offsets tile table must not be served stale after an in-place edit.
+ o3 = off.clone()
+ _, t_before, _ = VK.tile_table(o3) # doc 0 has 1 token: 1 tile
+ o3[1] = 65 # in place: now 65 tokens, 2 tiles
+ cu, tcum, perm = VK.tile_table(o3)
+ fresh = (o3[1:] - o3[:-1] + 63) // 64 # what the table must now say
+ check("volta attention: tile table follows in-place edits of the offsets",
+ int(cu[1]) == 65 and int(tcum[-1]) == int(fresh.clamp_min(0).sum())
+ and int(fresh[int(perm[0])]) == int(fresh.max()))
+ lens_ = (cu[1:] - cu[:-1] + 63) // 64
+ check("volta attention: tile table launches the longest documents first",
+ bool((lens_[perm.long()][:-1] >= lens_[perm.long()][1:]).all())
+ and sorted(perm.tolist()) == list(range(len(lens))))
+
+ # LM head: the one-pass Triton softmax against the production head.
+ torch.manual_seed(1)
+ Vv, d = cfg.model.vocab_size, cfg.model.d_model
+ Nn = 2048
+ W = (torch.randn(Vv, d, device=dev) * 0.05)
+ x = torch.randn(Nn, d, device=dev) * 3.0
+ y = torch.randint(0, Vv, (Nn,), device=dev)
+ y[::5] = IGNORE
+
+ def head(fn, xx, yy):
+ xx = xx.clone().requires_grad_(); ww = W.clone().requires_grad_()
+ with torch.autocast("cuda", dtype=torch.float16):
+ loss = fn(xx, ww, yy, 1024)
+ loss.backward()
+ return loss.detach().double(), xx.grad.double(), ww.grad.double()
+
+ was = M.VOLTA_KERNELS
+ M.VOLTA_KERNELS = False
+ try:
+ P = head(M.fused_lm_loss, x, y)
+ finally:
+ M.VOLTA_KERNELS = was
+ Q = head(volta_lm_loss, x, y)
+ xr = x.double().requires_grad_(); wr = W.double().requires_grad_()
+ lr_ = F.cross_entropy(xr @ wr.T, y, ignore_index=IGNORE)
+ lr_.backward()
+ check("volta head: loss equals the production head",
+ abs(float(Q[0] - P[0])) <= 1e-6 * abs(float(P[0])), f"{float(Q[0]):.7f} vs {float(P[0]):.7f}")
+ for name, i, r in (("dx", 1, xr.grad), ("dW", 2, wr.grad)):
+ ep, eq = rel(P[i], r), rel(Q[i], r)
+ check(f"volta head: {name} as accurate as the production head",
+ eq <= 1.02 * ep, f"rel-L2 vs fp64 {eq:.3e} (production {ep:.3e})")
+ # The dW overflow guard survives: one token repeated through a whole chunk.
+ xs = x[:1].repeat(1024, 1) * 5.0
+ ys = y.clone()[:1024]; ys[:] = 7
+ _, dxs, dws = head(volta_lm_loss, xs, ys)
+ check("volta head: a chunk of one repeated token stays finite",
+ bool(torch.isfinite(dws).all() and torch.isfinite(dxs).all()))
+
+ # Stage two (80% of the run) packs documents up to 8192 tokens. fp64 would
+ # need a 4 GB score matrix, so the reference is fp32 memory-efficient SDPA.
+ torch.manual_seed(3)
+ lens8 = [8192, 3000, 5000]
+ N8 = sum(lens8)
+ off8 = torch.tensor([0] + list(np.cumsum(lens8)), device=dev)
+ q8, k8, v8, g8 = (torch.randn(N8, H, D, device=dev).half() for _ in range(4))
+
+ def attn_grads(fn, dtype):
+ # clone: for fp16, .to() returns the SAME tensor, and the two runs would
+ # accumulate into each other's .grad
+ ts = [t.detach().to(dtype).clone().requires_grad_() for t in (q8, k8, v8)]
+ o = fn(*ts)
+ o.backward(g8.to(dtype))
+ return [o.detach().double()] + [t.grad.double() for t in ts]
+
+ def nested(a, b, c):
+ nj = lambda t: torch.nested.nested_tensor_from_jagged(t, off8).transpose(1, 2)
+ return F.scaled_dot_product_attention(nj(a), nj(b), nj(c), is_causal=True
+ ).transpose(1, 2).values()
+ ref8 = attn_grads(nested, torch.float32)
+ S8 = attn_grads(nested, torch.float16)
+ V8 = attn_grads(lambda a, b, c: VK.varlen_causal_attention(a, b, c, off8), torch.float16)
+ worst = max(rel(V8[i], ref8[i]) / rel(S8[i], ref8[i]) for i in range(4))
+ check("volta attention: an 8192-token document as accurate as stock SDPA",
+ worst <= 1.1, f"worst error ratio vs stock {worst:.3f} (o, dq, dk, dv vs fp32)")
+
+ # A head call whose row count is not a multiple of the 1024-row chunk.
+ xp, yp = x[:1024 + 300], y[:1024 + 300]
+ M.VOLTA_KERNELS = False
+ try:
+ Pp = head(M.fused_lm_loss, xp, yp)
+ finally:
+ M.VOLTA_KERNELS = was
+ Qp = head(volta_lm_loss, xp, yp)
+ check("volta head: a partial final chunk matches the production head",
+ abs(float(Qp[0] - Pp[0])) <= 1e-6 * abs(float(Pp[0]))
+ and rel(Qp[1], Pp[1]) < 1e-3 and rel(Qp[2], Pp[2]) < 1e-3,
+ f"dx {rel(Qp[1], Pp[1]):.1e}, dW {rel(Qp[2], Pp[2]):.1e} apart")
+
+ # Fused Canon + RoPE: precision against fp64 next to the eager region, and
+ # document isolation with non-zero Canon weights (zero-init would hide a leak).
+ from volta_kernels.canon_rope_cuda import canon_rope # what the model uses
+ torch.manual_seed(4)
+ Bc, Tc = 2, 256
+ bndc = torch.zeros(Bc, Tc, dtype=torch.bool, device=dev)
+ bndc[:, 0] = True; bndc[0, 100] = True; bndc[1, 7] = True; bndc[1, 8] = True
+ b2c = bndc | F.pad(bndc, (1, 0))[:, :-1]
+ m1c = (~bndc).float()[:, :, None]; m2c = (~b2c).float()[:, :, None]
+ posc = torch.zeros(Bc, Tc, dtype=torch.long, device=dev)
+ for b in range(Bc):
+ p0 = 0
+ for t_ in range(Tc):
+ if bndc[b, t_]: p0 = t_
+ posc[b, t_] = t_ - p0
+ invc = 1.0 / (10000.0 ** (torch.arange(0, 64, 2, device=dev).float() / 64))
+ fc = torch.outer(torch.arange(Tc, device=dev).float(), invc)
+ cosc, sinc = fc.cos()[posc].unsqueeze(2), fc.sin()[posc].unsqueeze(2)
+ qkvc = (torch.randn(Bc, Tc, 1536, device=dev) * 2).half()
+ cwc = torch.randn(1536, 3, device=dev) * 0.3
+ gsc = [torch.randn(Bc, Tc, 8, 64, device=dev).half() for _ in range(3)]
+
+ def region(a, w_, dt):
+ ww = w_.to(torch.float16).to(dt)
+ s1 = F.pad(a, (0, 0, 1, 0))[:, :-1] * m1c.to(dt)
+ s2 = F.pad(a, (0, 0, 2, 0))[:, :-2] * m2c.to(dt)
+ y_ = a + a * ww[:, 0] + s1 * ww[:, 1] + s2 * ww[:, 2]
+ c_, s_ = cosc.to(dt), sinc.to(dt)
+ def rope(t_):
+ a_, b_ = t_.chunk(2, dim=-1)
+ return torch.cat([a_ * c_ - b_ * s_, a_ * s_ + b_ * c_], -1)
+ v_ = lambda i: y_[..., 512 * i:512 * (i + 1)].reshape(Bc, Tc, 8, 64)
+ return rope(v_(0)), rope(v_(1)), v_(2)
+
+ def cr_run(fn, dt):
+ a = qkvc.detach().to(dt).clone().requires_grad_()
+ w_ = cwc.detach().clone().requires_grad_()
+ outs = fn(a, w_)
+ torch.autograd.backward(outs, [g.to(outs[0].dtype) for g in gsc])
+ return [o.detach().double() for o in outs] + [a.grad.double()]
+ ref_c = cr_run(lambda a, w_: region(a, w_, torch.float64), torch.float64)
+ eag_c = cr_run(lambda a, w_: tuple(o.to(torch.float16) for o in
+ region(a.float(), w_, torch.float32)), torch.float16)
+ fus_c = cr_run(lambda a, w_: canon_rope(a, w_, m1c, m2c, cosc, sinc), torch.float16)
+ worst = max(rel(fus_c[i], ref_c[i]) / rel(eag_c[i], ref_c[i]) for i in range(4))
+ check("volta canon+rope: q, k, v, dqkv as accurate as the eager region",
+ worst <= 1.1, f"worst error ratio {worst:.3f}")
+ q0_, k0_, v0_ = canon_rope(qkvc, cwc, m1c, m2c, cosc, sinc)
+ pert = qkvc.clone(); pert[0, :100] = torch.randn_like(pert[0, :100]).half()
+ q1_, k1_, v1_ = canon_rope(pert, cwc, m1c, m2c, cosc, sinc)
+ leak = max(float((a_[0, 100:].float() - b_[0, 100:].float()).abs().max())
+ for a_, b_ in ((q0_, q1_), (k0_, k1_), (v0_, v1_)))
+ moved = float((v1_[0, :100].float() - v0_[0, :100].float()).abs().max())
+ check("volta canon+rope: a document boundary stops the Canon taps exactly",
+ leak == 0.0 and moved > 0, f"drift past boundary {leak:.1e}")
+
+ # Whole model, both paths, against an fp32 (no autocast) reference, at the
+ # run's loss scale. Canon de-zeroed so every mechanism is live.
+ small = Config.from_json(cfg.to_json())
+ small.enforce_budget = False
+ small.model.n_layer = 4
+ torch.manual_seed(2)
+ m = build(small, dev)
+ for blk in m.blocks:
+ torch.nn.init.normal_(blk.attn.canon, std=0.1)
+ lens2 = [[700, 64, 1, 1283], [2048]]
+ xb, pb, ob, bb = synth_batch(small, dev, 2, 2048, lens2, seed=5)
+ yb = torch.roll(xb, -1, 1)
+
+ def grads(volta, amp):
+ M.VOLTA_KERNELS = volta
+ try:
+ for prm in m.parameters():
+ prm.grad = None
+ with torch.autocast("cuda", dtype=torch.float16, enabled=amp):
+ loss = m(xb, pb, ob.clone(), bb, yb, 10_000.0)
+ (loss * 32768.0).backward()
+ finally:
+ M.VOLTA_KERNELS = was
+ return {n: prm.grad.double() / 32768.0 for n, prm in m.named_parameters()
+ if prm.grad is not None}
+ gr, gs, gv = grads(False, False), grads(False, True), grads(True, True)
+ # The production path with checkpointing OFF merges each block's FFN half
+ # with the next block's norm+QKV into one region (model._forward_merged).
+ small.train.grad_checkpoint = False
+ gmerged = grads(True, True)
+ m._no_merge = True
+ gunmerged = grads(True, True)
+ m._no_merge = False
+ small.train.grad_checkpoint = True
+ em = sorted(rel(gmerged[n], gr[n]) for n in gr)
+ eu = sorted(rel(gunmerged[n], gr[n]) for n in gr)
+ check("volta merged regions: gradients as close to fp32 as per-block regions",
+ em[len(em) // 2] <= 1.05 * eu[len(eu) // 2] and max(em) <= 1.05 * max(eu),
+ f"median {em[len(em) // 2]:.3e} (per-block {eu[len(eu) // 2]:.3e})")
+ es = sorted(rel(gs[n], gr[n]) for n in gr)
+ ev = sorted(rel(gv[n], gr[n]) for n in gr)
+ med_s, med_v = es[len(es) // 2], ev[len(ev) // 2]
+ check("volta model: gradients as close to fp32 as the stock fp16 path",
+ med_v <= 1.1 * med_s and max(ev) <= 1.25 * max(es),
+ f"median rel-L2 {med_v:.3e} (stock {med_s:.3e}), worst {max(ev):.3e} (stock {max(es):.3e})")
+
+
+def check_ngram_isolation(cfg, dev):
+ """The n-gram key must not span a document boundary."""
+ m = build(cfg, dev)
+ B, T = 1, 32
+ x = torch.arange(T, device=dev)[None] % cfg.model.vocab_size
+ bnd = torch.zeros(B, T, dtype=torch.bool, device=dev)
+ bnd[0, 0] = True; bnd[0, 16] = True
+ out = m.ngram(x, bnd, torch.float32)
+ x2 = x.clone(); x2[0, 14:16] = (x2[0, 14:16] + 7) % cfg.model.vocab_size
+ out2 = m.ngram(x2, bnd, torch.float32)
+ d = float((out[0, 16:] - out2[0, 16:]).abs().max())
+ check("n-gram history stops at a document boundary", d == 0.0, f"max diff {d:.1e}")
+
+
+def check_muon(cfg, dev):
+ """Batched Newton-Schulz must agree with the unbatched one, and orthogonalise."""
+ torch.manual_seed(0)
+ G = torch.randn(4, 512, 2368, device=dev)
+ batched = newton_schulz(G, 5)
+ single = torch.stack([newton_schulz(G[i:i+1], 5)[0] for i in range(4)])
+ r = float((batched - single).norm() / single.norm())
+ check("batched Newton-Schulz equals the unbatched one", r < 1e-5,
+ f"rel-L2 {r:.2e}")
+ s = torch.linalg.svdvals(batched[0].float())
+ check("Newton-Schulz orthogonalises (singular values near 1)",
+ 0.7 < float(s.mean()) < 1.3, f"mean sv {float(s.mean()):.3f}, "
+ f"min {float(s.min()):.3f}, max {float(s.max()):.3f}")
+
+ # Fusing q/k/v into one Parameter must not change the update. This is the
+ # whole justification for storing them fused, so it is asserted, not assumed.
+ torch.manual_seed(1)
+ fused = torch.nn.Parameter(torch.randn(768, 512, device=dev))
+ fused._boris_role = "matmul"
+ fused._boris_blocks = ((0, 512), (512, 640), (640, 768))
+ grad = torch.randn(768, 512, device=dev)
+ fused.grad = grad.clone()
+ parts = []
+ for a, b in fused._boris_blocks:
+ q = torch.nn.Parameter(fused.data[a:b].clone())
+ q._boris_role = "matmul"
+ q.grad = grad[a:b].clone()
+ parts.append(q)
+ kw = dict(lr=0.02, momentum=0.95, nesterov=True, ns_steps=5,
+ weight_decay=0.1, batched=True)
+ Muon([fused], **kw).step()
+ Muon(parts, **kw).step()
+ sep = torch.cat([q.data for q in parts], 0)
+ d = float((fused.data - sep).abs().max())
+ check("a block-fused Parameter updates exactly like separate ones", d == 0.0,
+ f"max diff {d:.1e}")
+
+ # A (768, 3) Canon kernel must never reach Muon.
+ m = build(cfg, dev)
+ muon, adamw = build_optimizers(m, cfg, log=lambda *a: None)
+ canon_ids = {id(b.attn.canon) for b in m.blocks}
+ in_muon = any(id(p) in canon_ids for g in muon.param_groups for p in g["params"])
+ check("Canon kernels are in AdamW, not Muon", not in_muon)
+ tbl = id(m.ngram.table)
+ in_muon_tbl = any(id(p) == tbl for g in muon.param_groups for p in g["params"])
+ check("the n-gram table is in AdamW, not Muon", not in_muon_tbl)
+
+
+def check_ngram_ablation(cfg, dev):
+ """Both arms must build, and the runtime toggle must actually do something."""
+ off = Config.from_json(cfg.to_json())
+ off.model.use_ngram = False
+ b_on, b_off = cfg.param_budget()["total"], off.param_budget()["total"]
+ check("use_ngram leaves the transformer untouched",
+ cfg.param_budget()["trunk"] == off.param_budget()["trunk"]
+ and b_on - b_off == cfg.ngram.table_params(cfg.model.d_model),
+ f"{b_on:,} with table, {b_off:,} without, same {b_off - cfg.model.embed_params():,} trunk")
+ off.model.n_layer = 4
+ off.enforce_budget = False
+ m = build(off, dev)
+ check("use_ngram=False builds no table and no projection",
+ m.ngram is None and not any("ngram" in n for n, _ in m.named_parameters()))
+ build_optimizers(m, off, log=lambda *a: None)
+ del m
+ torch.cuda.empty_cache()
+
+ on = Config.from_json(cfg.to_json())
+ on.model.n_layer = 4
+ on.enforce_budget = False
+ m = build(on, dev)
+ x, pos, bounds, bnd = synth_batch(on, dev, 1, 256, [[128, 128]], seed=5)
+ with torch.autocast("cuda", dtype=torch.float16):
+ a = m(x, pos, bounds, bnd, None, 10_000.0)
+ m.ngram_enabled = False
+ b = m(x, pos, bounds, bnd, None, 10_000.0)
+ m.ngram_enabled = True
+ c = m(x, pos, bounds, bnd, None, 10_000.0)
+ check("the runtime toggle changes the output", not torch.equal(a, b),
+ f"max diff {float((a.float()-b.float()).abs().max()):.3e}")
+ check("the runtime toggle is reversible", torch.equal(a, c))
+ del m
+ torch.cuda.empty_cache()
+
+
+def check_schedule(cfg):
+ s = cfg.schedule
+ w, st = s.warmup_frac, s.stable_frac
+ ok = (abs(lr_scale(0.0, cfg)) < 1e-9 and abs(lr_scale(w, cfg) - 1) < 1e-9
+ and abs(lr_scale((w + st) / 2, cfg) - 1) < 1e-9
+ and abs(lr_scale(1.0, cfg) - s.final_lr_frac) < 1e-9)
+ check("WSD: 0 at start, 1 across the stable phase, floor at the end", ok)
+ mono = all(lr_scale(p, cfg) >= lr_scale(p + 0.005, cfg) - 1e-12
+ for p in np.arange(st, 1.0, 0.005))
+ check("WSD: the decay never goes back up", mono)
+
+ for p in (0.0, 0.5, 0.86, 0.94, 1.0):
+ w_ = mixture_at(p, cfg)
+ assert abs(sum(w_.values()) - 1) < 1e-9
+ end = mixture_at(1.0, cfg)
+ # 2026-09-17, Boris-2-0917. The four checks here used to assert the old
+ # hand-weighted decay mixture (raw web to zero, curated > 0.7). That corpus
+ # is gone. ClimbMix's weights are a SEARCH RESULT over the whole mixture, so
+ # the property worth protecting is the opposite one: that we do not
+ # re-weight it by phase and quietly undo the optimisation we adopted it for.
+ flat = all(abs(mixture_at(p, cfg)["climbmix"] - end["climbmix"]) < 1e-9
+ for p in (0.0, 0.25, 0.5, 0.87, 0.95))
+ check("ClimbMix weights are flat across the run, not re-weighted by phase",
+ flat, f"climbmix held at {end['climbmix']:.2f} throughout")
+ check("ClimbMix is the overwhelming majority of the mixture",
+ end["climbmix"] >= 0.90, f"{end['climbmix']:.2f} of the mixture")
+ # finemath is the only source carried over from 0907. ArithMark-3 was the
+ # one benchmark 0907 was competitive on (37.40 vs GPT-X2.5's 38.40) and
+ # ClimbMix was optimised against PIQA/ARC-E/HellaSwag, not arithmetic.
+ check("finemath is retained, so ArithMark-3 is not abandoned",
+ 0.02 <= end.get("finemath", 0.0) <= 0.10,
+ f"{end.get('finemath', 0.0):.2f} of the mixture")
+ check("the mixture names only sources that exist on disk",
+ set(cfg.data.sources) == set(end),
+ f"{sorted(cfg.data.sources)}")
+
+ b0 = context_at(0.0, cfg)
+ b1 = context_at(0.99, cfg)
+ check("context extends later in the run", b1[0] > b0[0],
+ f"{b0[0]} -> {b1[0]} at theta {b0[1]:,.0f} -> {b1[1]:,.0f}")
+ check("the step size is identical across context stages",
+ b0[2] * b0[0] * b0[3] == b1[2] * b1[0] * b1[3] == cfg.train.tokens_per_step,
+ f"{b0[2]}x{b0[0]}x{b0[3]} = {b0[2]*b0[0]*b0[3]:,}")
+
+
+def check_packer(cfg, dev):
+ """Targets are next-token within a document, never across one."""
+ from data import VarlenPacker, SourceStore
+ d = Path(_tmpdir()) / "src"
+ d.mkdir(parents=True)
+ rng = np.random.default_rng(0)
+ lens = rng.integers(20, 400, size=400)
+ toks = rng.integers(0, cfg.model.vocab_size, size=int(lens.sum())).astype(np.uint16)
+ toks.tofile(d / "tokens.bin")
+ np.concatenate([[0], np.cumsum(lens)]).astype(np.uint64).tofile(d / "docs.idx")
+ st = SourceStore.open(d.parent, "src")
+ cfg2 = Config.from_json(cfg.to_json())
+ cfg2.data.sources = ("src",)
+ cfg2.curriculum.enabled = False
+ p = VarlenPacker(cfg2, {"src": st}, 7)
+ xs, ys, pos, bnd, bounds, stats = p.batch(2, 256, 0.0)
+
+ ok_bounds = (bounds[0] == 0 and bounds[-1] == 2 * 256
+ and np.all(np.diff(bounds) > 0))
+ check("packer: offsets are increasing and cover the batch exactly", ok_bounds)
+ flat_b = np.zeros(2 * 256, dtype=bool)
+ flat_b[bounds[:-1]] = True
+ check("packer: boundary mask agrees with the offsets",
+ np.array_equal(flat_b, bnd.reshape(-1)))
+ fx, fy = xs.reshape(-1), ys.reshape(-1)
+ bad = 0
+ for i in range(len(bounds) - 1):
+ a, b_ = bounds[i], bounds[i + 1]
+ if b_ - a > 1 and not np.array_equal(fy[a:b_ - 1], fx[a + 1:b_]):
+ bad += 1
+ if fy[b_ - 1] != IGNORE and i + 1 < len(bounds) - 1:
+ # only legal when the document was carried into the next row
+ pass
+ check("packer: targets are next-token inside each document", bad == 0,
+ f"{bad} documents mismatched")
+ check("packer: positions restart at every document",
+ all(pos.reshape(-1)[bounds[i]] == 0
+ for i in range(len(bounds) - 1)))
+
+ # The RoPE table is exactly max_block_size long and is indexed by pos, so a
+ # position past its end is an out-of-bounds gather -- a device-side assert
+ # thousands of steps in, reported asynchronously at whatever kernel runs
+ # next. Carried chunks of a long document used to keep counting, and with
+ # max_doc_tokens at 65,536 they ran off the end. Sample where long-document
+ # pressure is highest, at both context stages.
+ lim = cfg.model.max_block_size
+ worst, worst_at = 0, None
+ for prog, blk in ((0.0, 256), (0.5, 256), (0.85, 512), (0.95, 512)):
+ for _ in range(12):
+ _x, _y, po, _b, bo, _st = p.batch(4, blk, prog)
+ m = int(po.max())
+ if m > worst:
+ worst, worst_at = m, (prog, blk)
+ if not all(po.reshape(-1)[bo[i]] == 0 for i in range(len(bo) - 1)):
+ worst, worst_at = lim + 1, (prog, blk) # force a failure
+ check("packer: positions stay inside the RoPE table", worst < lim,
+ f"max pos {worst:,} of {lim:,}"
+ + (f" at progress {worst_at[0]}, block {worst_at[1]}" if worst_at else ""))
+
+ # Resumability: the same state must replay the same batch.
+ s = p.state_dict()
+ a1 = p.batch(2, 256, 0.0)[0]
+ p.load_state_dict(s)
+ a2 = p.batch(2, 256, 0.0)[0]
+ check("packer: a restored state replays the identical batch",
+ np.array_equal(a1, a2))
+
+ from data import Prefetcher, validate_tokenization, open_validation
+ pref = Prefetcher(p, cfg2, dev)
+ pref.configure(2, 256, 0.0)
+ pref.start()
+ try:
+ pref.next(2, 256)
+ committed = pref.state_dict()
+ reference = VarlenPacker(cfg2, {"src": st}, 7)
+ reference.load_state_dict(committed)
+ expected = reference.batch(2, 512)[0]
+ pref.configure(2, 512, 0.0)
+ actual = pref.next(2, 512)[0].cpu().numpy()
+ check("prefetch: shape transition replays committed state",
+ np.array_equal(actual, expected))
+ finally:
+ pref.stop()
+
+ def broken_batch(*args):
+ raise ValueError("deliberate producer failure")
+ p.batch = broken_batch
+ pref = Prefetcher(p, cfg2, dev)
+ pref.configure(2, 256, 0.0)
+ pref.start()
+ propagated = False
+ try:
+ pref.next(2, 256)
+ except RuntimeError as exc:
+ propagated = isinstance(exc.__cause__, ValueError)
+ finally:
+ pref.stop()
+ check("prefetch: producer failure reaches training instead of hanging", propagated)
+ rejected = False
+ try:
+ validate_tokenization(d, cfg)
+ except ValueError:
+ rejected = True
+ check("data: rejects shards without matching tokenizer provenance", rejected)
+ check("validation: current tokenizer provenance verified", open_validation(cfg).n_tokens > 0)
+
+
+def check_multistore(cfg):
+ """Sharded sources must stitch back into one correctly indexed source.
+
+ A source too large for one prepare process is split across workers by data
+ file range; data.py presents the shard directories as one logical source
+ rather than concatenating 200GB on disk. If the global-to-shard index
+ mapping is wrong, training silently reads the wrong documents.
+ """
+ import tempfile
+ from data import MultiStore, SourceStore
+ root = Path(_tmpdir())
+ rng = np.random.default_rng(0)
+ truth, parts = [], []
+ for sh in range(3):
+ d = root / f"src_s{sh}"; d.mkdir()
+ lens = rng.integers(5, 50, size=7)
+ docs = [rng.integers(1, 1000, size=L).astype(np.uint16) for L in lens]
+ truth += docs
+ np.concatenate(docs).tofile(d / "tokens.bin")
+ np.concatenate([[0], np.cumsum(lens)]).astype(np.uint64).tofile(d / "docs.idx")
+ parts.append(SourceStore("src", d / "tokens.bin", d / "docs.idx"))
+ m = MultiStore("src", parts)
+ check("sharded source: document count and token count add up",
+ m.n_docs == len(truth) and m.n_tokens == sum(len(t) for t in truth))
+ check("sharded source: every global index maps to the right document",
+ all(np.array_equal(m.doc(i), truth[i]) for i in range(m.n_docs)))
+ exp = np.flatnonzero(np.array([len(t) for t in truth]) >= 30)
+ check("sharded source: long-document selection spans shards",
+ np.array_equal(m.long_docs(30), exp))
+
+
+def check_checkpoint(cfg, dev):
+ import checkpoint as C
+ small = Config.from_json(cfg.to_json())
+ small.enforce_budget = False
+ small.model.n_layer = 4
+ m = build(small, dev)
+ muon, adamw = build_optimizers(m, small, log=lambda *a: None)
+ for p in m.parameters():
+ p.grad = torch.randn_like(p)
+ muon.step(); adamw.step()
+ scaler = torch.amp.GradScaler("cuda")
+ tmp = Path(_tmpdir()) / "c.pt"
+ C.save_full(tmp, model=m, muon=muon, adamw=adamw, scaler=scaler, packer=None,
+ step=5, tokens_seen=123, cfg=small, ngram_fingerprint="deadbeef",
+ background=False)
+ before = {k: v.clone() for k, v in m.state_dict().items()}
+ for p in m.parameters():
+ p.data.add_(torch.randn_like(p))
+ blob = C.load_full(tmp, model=m, muon=muon, adamw=adamw, scaler=scaler,
+ map_location=dev, expect_fingerprint=small.fingerprint(),
+ expect_ngram="deadbeef")
+ same = all(torch.equal(before[k].to(dev), v) for k, v in m.state_dict().items())
+ check("checkpoint: weights round-trip exactly", same)
+ check("checkpoint: step and token count survive",
+ blob["step"] == 5 and blob["tokens_seen"] == 123)
+ mom = next(iter(muon.state.values())).get("momentum_buffer")
+ check("checkpoint: Muon momentum buffers are restored", mom is not None)
+
+ # The packer round-trip, on the real device. This check used to pass
+ # packer=None, so the resume path was never actually exercised: the first
+ # real --resume died on it. map_location=<cuda> moves the packer's saved RNG
+ # state onto the GPU, and a CPU Generator's set_state rejects a CUDA tensor.
+ import data as D
+ srcs = D.open_sources(cfg, log=lambda *a, **k: None)
+ pk = D.VarlenPacker(cfg, srcs, seed=0)
+ for _ in range(3):
+ pk.batch(2, 256, 0.0)
+ tmp2 = Path(_tmpdir()) / "p.pt"
+ C.save_full(tmp2, model=m, muon=muon, adamw=adamw, scaler=scaler, packer=pk,
+ step=7, tokens_seen=99, cfg=small, ngram_fingerprint="deadbeef",
+ background=False)
+ want = pk.batch(2, 256, 0.0)[0]
+ pk2 = D.VarlenPacker(cfg, srcs, seed=0)
+ ok_resume, err = True, ""
+ try:
+ C.load_full(tmp2, model=m, muon=muon, adamw=adamw, scaler=scaler,
+ packer=pk2, map_location=dev,
+ expect_fingerprint=small.fingerprint(),
+ expect_ngram="deadbeef")
+ except Exception as e:
+ ok_resume, err = False, f"{type(e).__name__}: {e}"
+ check("checkpoint: the packer resumes from a GPU map_location",
+ ok_resume, err)
+ if ok_resume:
+ got = pk2.batch(2, 256, 0.0)[0]
+ check("checkpoint: a resumed packer replays the identical next batch",
+ np.array_equal(want, got),
+ "same tokens" if np.array_equal(want, got) else "DIVERGED")
+ try:
+ C.load_full(tmp, expect_fingerprint="0000000000000000")
+ check("checkpoint: a config change is refused, not silently accepted", False)
+ except SystemExit:
+ check("checkpoint: a config change is refused, not silently accepted", True)
+
+
+def check_tensor_cores(cfg, dev):
+ """Prove fp16 autocast reaches a tensor-core kernel rather than sgemm."""
+ from torch.profiler import ProfilerActivity, profile
+ a = torch.randn(4096, 512, device=dev)
+ b = torch.randn(512, 2368, device=dev)
+ with profile(activities=[ProfilerActivity.CUDA]) as pr:
+ with torch.autocast("cuda", dtype=torch.float16):
+ for _ in range(3):
+ (a @ b).sum()
+ torch.cuda.synchronize()
+ names = " ".join(e.key for e in pr.key_averages()).lower()
+ tc = any(s in names for s in ("h884", "884gemm", "hgemm", "s16816", "16816",
+ "tensor", "cutlass"))
+ check("fp16 GEMMs dispatch to a tensor-core kernel", tc,
+ [e.key for e in pr.key_averages() if "gemm" in e.key.lower()][:1])
+
+
+def check_down_proj_headroom(cfg, dev):
+ """The FFN down projection's fp16 output range -- failure #2, step 4,520.
+
+ Channel 228 of the residual is a massive activation, 173x the RMS (78,115
+ against 451 at block 16). The FFN writes into it, so this GEMM's fp16 OUTPUT
+ crossed 65,504 and became inf, which norm_f turned into NaN. Inputs and
+ weights were all finite; only the range ran out. DOWN_SHRINK pre-scales
+ w_down by a power of two, which is exact and keeps the GEMM on tensor cores.
+ """
+ import model as M
+ peak = 65_504.0 / M.DOWN_SHRINK # what the output may now reach
+ check("down proj: DOWN_SHRINK is a power of two (so the scaling is exact)",
+ M.DOWN_SHRINK > 0 and
+ abs(math.log2(M.DOWN_SHRINK) - round(math.log2(M.DOWN_SHRINK))) < 1e-12,
+ f"{M.DOWN_SHRINK} = 2^{round(math.log2(M.DOWN_SHRINK))}")
+ check("down proj: headroom covers the measured step-4,520 peak",
+ peak > 4 * 65_504.0,
+ f"overflows at {peak:,.0f} now, was 65,504 ({1/M.DOWN_SHRINK:.0f}x)")
+
+ # the scaling must not change the result: same GEMM, both ways, in fp32
+ torch.manual_seed(0)
+ x = (torch.randn(4096, 2368, device=dev, dtype=torch.float16) * 40)
+ w = (torch.randn(512, 2368, device=dev, dtype=torch.float16) * 0.5)
+ plain = F.linear(x.float(), w.float())
+ scaled = F.linear(x, w * M.DOWN_SHRINK).float() * (1.0 / M.DOWN_SHRINK)
+ rel = float((scaled - plain).norm() / plain.norm())
+ check("down proj: pre-scaling the weight matches the unscaled GEMM",
+ rel < 2e-3, f"rel-L2 {rel:.2e}")
+
+ # And it must rescue the real failure shape: an output that lands just past
+ # 65,504, the way step 4,520's did -- not an absurd one. Headroom is 8x, not
+ # infinite, so the test is calibrated to the measured overshoot.
+ torch.manual_seed(1)
+ big = torch.randn(2048, 2368, device=dev, dtype=torch.float16)
+ wb = torch.randn(512, 2368, device=dev, dtype=torch.float16)
+ ref = F.linear(big.float(), wb.float()).abs().amax()
+ big = (big * (1.6 * 65_504.0 / float(ref)) ** 0.5) # target ~1.6x over
+ wb = (wb * (1.6 * 65_504.0 / float(ref)) ** 0.5)
+ true_peak = float(F.linear(big.float(), wb.float()).abs().amax())
+ unscaled_bad = not torch.isfinite(F.linear(big, wb)).all()
+ scaled = F.linear(big, wb * M.DOWN_SHRINK).float() * (1.0 / M.DOWN_SHRINK)
+ scaled_ok = bool(torch.isfinite(scaled).all())
+ check("down proj: an output just past 65,504 overflows unscaled, survives scaled",
+ bool(unscaled_bad) and scaled_ok,
+ f"true peak {true_peak:,.0f}: unscaled overflows={bool(unscaled_bad)}, "
+ f"scaled finite={scaled_ok}")
+
+
+def check_residual_range(cfg, dev):
+ """The overflow that killed two runs, reproduced from the measured value.
+
+ The residual stream grows with depth. At step 3,456 -- 1% into the budget --
+ it reached 65,617 at block 25 against fp16's maximum of 65,504. It saturates
+ to inf, and _rmsnorm's rsqrt(inf) = 0 turns that into inf * 0 = NaN. No loss
+ scale can help: the overflow is in the forward pass, so GradScaler halves its
+ scale forever against a cause halving cannot touch.
+ """
+ from model import _rmsnorm
+ peak = 65_617.3 # measured, block 25, step 3,456
+ w = torch.ones(512, device=dev)
+ x = torch.full((4, 512), peak, device=dev)
+
+ h16 = x.to(torch.float16)
+ check("residual: the measured peak really does overflow fp16",
+ not torch.isfinite(h16).all(),
+ f"{peak:,.0f} vs fp16 max 65,504")
+ out16 = _rmsnorm(h16, w, cfg.model.norm_eps)
+ check("residual: an overflowed fp16 residual makes RMSNorm produce NaN",
+ bool(torch.isnan(out16).any()),
+ "rsqrt(inf)=0 then inf*0=nan -- the exact failure")
+
+ out32 = _rmsnorm(x, w, cfg.model.norm_eps)
+ check("residual: the same value in fp32 stays finite",
+ bool(torch.isfinite(out32).all()),
+ f"max |out| {float(out32.abs().amax()):.3f}")
+ check("residual: fp16_residual is off",
+ cfg.train.fp16_residual is False,
+ "fp32 residual stream (costs ~7.6% throughput)")
+
+
+def check_loss_scale_floor(cfg, dev):
+ """The scale collapse that ended the step-3,810 run, reproduced and fenced.
+
+ GradScaler has no floor. Once the scale falls below ~1e-30, unscale_ computes
+ grad/scale, overflows fp32 to inf, sets found_inf, and halves the scale
+ again -- self-reinforcing, no exit, every step skipped from then on. The run
+ died that way with its weights still perfectly finite. This drives a scaler
+ into that spiral and asserts the clamp in train.py stops it.
+ """
+ w = torch.zeros(4, device=dev, requires_grad=True)
+ sc = torch.amp.GradScaler("cuda", init_scale=cfg.train.init_scale)
+ opt = torch.optim.SGD([w], lr=0.0)
+
+ # Unclamped: 80 consecutive overflows.
+ bare = torch.amp.GradScaler("cuda", init_scale=cfg.train.init_scale)
+ for _ in range(80):
+ bare.scale(torch.zeros((), device=dev)) # lazy-inits _scale
+ w.grad = torch.full_like(w, float("inf"))
+ bare.unscale_(opt); bare.step(opt); bare.update()
+ collapsed = bare.get_scale()
+
+ floored_steps = 0
+ for _ in range(80):
+ sc.scale(torch.zeros((), device=dev))
+ w.grad = torch.full_like(w, float("inf"))
+ sc.unscale_(opt); sc.step(opt); sc.update()
+ if sc.get_scale() < cfg.train.min_loss_scale:
+ sc.update(cfg.train.min_loss_scale); floored_steps += 1
+ held = sc.get_scale()
+
+ check("loss scale: unclamped GradScaler does collapse (the bug)",
+ collapsed < 1e-12,
+ f"80 overflows drive it to {collapsed:.3g}")
+ check("loss scale: the clamp holds it at the floor",
+ held == cfg.train.min_loss_scale and math.isfinite(held),
+ f"held at {held:g}, floored on {floored_steps} steps")
+ # And the floor must be high enough that unscale_ cannot overflow fp32.
+ safe = cfg.train.min_loss_scale > 1e-30
+ check("loss scale: the floor is above the fp32 unscale_ overflow point",
+ safe, f"floor {cfg.train.min_loss_scale:g} vs ~1e-30")
+ check("loss scale: the ceiling is at or below 2^15",
+ cfg.train.max_loss_scale <= 2.0 ** 15,
+ f"max {cfg.train.max_loss_scale:,.0f}")
+
+
+def check_head_range(cfg, dev):
+ """The fused head's dW GEMM must not overflow fp16 on realistic text.
+
+ g = S*(p - onehot): the true token carries -S, and dW sums it over every
+ token in the chunk with the same target. Synthetic benchmark data is
+ uniform over the vocabulary and never stacks a target often enough to
+ matter; real text does. At GRAD_SCALE 4096 this saturated to Inf on step 1,
+ Inf-Inf put NaN in the tied embedding gradient, and the run died on step 2.
+ Assert on a batch of the actual corpus, and assert margin, not just
+ finiteness -- |x| grows over a long run and this must not sit on the edge.
+ """
+ import torch.nn.functional as F
+ import data as D, ngram as N, model as M
+ from pathlib import Path
+ srcs = D.open_sources(cfg, log=lambda *a, **k: None)
+ pk = D.VarlenPacker(cfg, srcs, seed=0)
+ m = M.Boris2(cfg).to(dev)
+ if m.ngram is not None:
+ m.ngram.load_vocab(N.NGramVocab.load(cfg.ngram.vocab_path), dev)
+ m.train()
+ x, y, pos, bnd, bounds, st = pk.batch(8, 2048, 0.0)
+ t = lambda a, dt: torch.as_tensor(a, dtype=dt, device=dev)
+ with torch.no_grad(), torch.autocast("cuda", dtype=torch.float16):
+ h = m(t(x, torch.long), t(pos, torch.long), t(bounds, torch.long),
+ t(bnd, torch.bool), None, 10_000.0, st["min_seqlen"], st["max_seqlen"])
+ xf = h.reshape(-1, h.shape[-1])
+ tg = t(y, torch.long).reshape(-1)
+ # Test the actual custom Function, not a separately reimplemented GEMM.
+ # The old test bypassed its broken fp16 rescale and therefore passed while
+ # production overflowed. Compare full gradients with fp32 cross-entropy.
+ c = cfg.optim.lm_head_chunk
+ for magnitude, repeated in ((1.0, False), (64.0, True)):
+ xx = (xf[:c].detach() * magnitude).requires_grad_()
+ ww = m.head_weight().detach().clone().requires_grad_()
+ tt = tg[:c].clone()
+ if repeated:
+ tt.fill_(int(tg[tg != IGNORE][0]))
+ with torch.autocast("cuda", dtype=torch.float16):
+ loss = M.fused_lm_loss(xx, ww, tt, chunk=c)
+ loss.backward()
+ xr = xx.detach().float().requires_grad_()
+ wr = ww.detach().clone().requires_grad_()
+ ref = F.cross_entropy(F.linear(xr, wr), tt, ignore_index=IGNORE)
+ ref.backward()
+ rel = (ww.grad - wr.grad).norm() / wr.grad.norm().clamp_min(1e-12)
+ check(f"production head real text: magnitude={magnitude:g}, repeated={repeated}",
+ bool(torch.isfinite(ww.grad).all()) and float(rel) < .01,
+ f"dW relative error {float(rel):.3g}")
+ del xx, ww, xr, wr, loss, ref
+
+ del m
+ torch.cuda.empty_cache()
+
+
+def check_compile(cfg, dev):
+ """The compiled regions must produce the same numbers as eager."""
+ small = Config.from_json(cfg.to_json())
+ small.enforce_budget = False
+ small.model.n_layer = 4
+ torch.manual_seed(0)
+ me = build(small, dev, compile_regions=False)
+ sd = me.state_dict()
+ torch.manual_seed(0)
+ mc = build(small, dev, compile_regions=True)
+ mc.load_state_dict(sd)
+ for m_ in (me, mc):
+ for blk in m_.blocks:
+ torch.nn.init.normal_(blk.attn.canon, std=0.3)
+ mc.load_state_dict(me.state_dict())
+ x, pos, bounds, bnd = synth_batch(small, dev, 1, 512, [[200, 312]], seed=1)
+ with torch.autocast("cuda", dtype=torch.float16):
+ a = me(x, pos, bounds, bnd, None, 10_000.0)
+ b = mc(x, pos, bounds, bnd, None, 10_000.0)
+ r = float((a.float() - b.float()).norm() / (a.float().norm() + 1e-9))
+ check("compiled regions match eager", r < 5e-3, f"rel-L2 {r:.2e}")
+
+
+# --------------------------------------------------------------------------- #
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--device", default="cuda:0")
+ ap.add_argument("--quick", action="store_true")
+ a = ap.parse_args()
+ dev = a.device
+ cfg = Config()
+ torch.cuda.set_device(dev)
+ torch.backends.cuda.matmul.allow_tf32 = False
+
+ print(f"\n verify: {cfg.run_name} on {dev}")
+ print(" " + "=" * 74)
+ print("\n budget and optimizer split")
+ m = build(cfg, dev)
+ build_optimizers(m, cfg)
+ del m
+ torch.cuda.empty_cache()
+
+ print(" numerics")
+ check_fused_head(cfg, dev)
+ check_repeated_target_head(cfg, dev)
+ check_head_range(cfg, dev)
+ check_loss_scale_floor(cfg, dev)
+ check_residual_range(cfg, dev)
+ check_down_proj_headroom(cfg, dev)
+ check_muon(cfg, dev)
+ torch.cuda.empty_cache()
+
+ print("\n isolation")
+ check_canon_causality(cfg, dev)
+ check_attention_matches_loop(cfg, dev)
+ check_volta_kernels(cfg, dev)
+ check_ngram_isolation(cfg, dev)
+ check_ngram_ablation(cfg, dev)
+ check_document_isolation(cfg, dev)
+ torch.cuda.empty_cache()
+
+ print("\n schedule and data")
+ check_schedule(cfg)
+ check_packer(cfg, dev)
+ check_multistore(cfg)
+
+ print("\n plumbing")
+ check_tensor_cores(cfg, dev)
+ check_checkpoint(cfg, dev)
+ if not a.quick:
+ check_compile(cfg, dev)
+
+ print("\n " + "=" * 74)
+ print(f" {len(PASS)} passed, {len(FAIL)} failed")
+ for f in FAIL:
+ print(f" FAILED: {f}")
+ sys.exit(1 if FAIL else 0)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/volta_kernels/README.md b/volta_kernels/README.md
new file mode 100644
index 0000000..dd80018
--- /dev/null
+++ b/volta_kernels/README.md
@@ -0,0 +1,165 @@
+# volta_kernels — hand-written sm_70 kernels for Boris-2
+
+Opt-in: `BORIS_VOLTA_KERNELS=1`. Engages only on an sm_70 device (the V100);
+eval on the 3060 always runs the stock paths. Unset, nothing changes.
+
+ BORIS_VOLTA_KERNELS=1 bash logs/launch_chain.sh --resume
+
+First use JIT-builds `attn.cu` with nvcc into `build/` (~1 min, then cached).
+
+## What is here
+
+| file | replaces | measured (V100, beside the live run) |
+|---|---|---|
+| `attn.cu` fwd | CUTLASS mem-efficient SDPA fwd (nested/jagged) | ~2x |
+| `attn.cu` bwd | mem-efficient SDPA bwd (was ~4x its own fwd) | ~2x |
+| `head.py` | `_FusedLMHead` elementwise middle + per-chunk casts | 1.8x, -25% peak mem |
+| `model._pre_attn(out16)` | separate fp32->fp16 casts of q/k/v | fused into the region |
+
+End to end, 8-layer slice of the live config, full micro-step: **1.41x**
+contended (stable over 3 repeats), 2.05x in one uncontended window. The
+31-layer mb-16 number is only observable on the run itself.
+
+## Why CUDA and not Triton
+
+On sm_70, Triton's `tl.dot` compiles to plain FMA -- no tensor cores (checked:
+2,048 `fma.rn` in the PTX, no `mma`). The attention uses raw
+`mma.sync.aligned.m8n8k4` (HMMA.884, what CUTLASS sm70 uses). Fragment layouts,
+verified empirically on this card:
+
+ A.row: row = j + 4hi, k = 0..3 A.col: k = j, row = 4hi + 0..3
+ B.col: n = j + 4hi, k = 0..3 B.row: k = j, n = 4hi + 0..3
+ D f32: row = (lane&1) + (i&2) + 4hi, col = (i&4) + (lane&2) + (i&1)
+ j = lane%4, hi = lane>=16; quad-pair qp = (lane>>2)&3 owns one 8x8 tile
+
+The head's softmax is Triton (elementwise/reduction, no dot), which is fine.
+
+## Precision
+
+Same operand dtypes and fp32 accumulation as what they replace. `verify.py`
+`check_volta_kernels` (always runs on sm_70) holds every output to the stock
+path's own error against fp64 on the same inputs:
+
+ attention out 2.22e-4 (= stock) dq 3.66e-4 (stock 4.23e-4)
+ dk 3.51e-4 (stock 4.05e-4) dv 2.88e-4 (= stock)
+ head dx, dW identical error to production; loss equal to 9 digits;
+ ~0.02% of dx elements differ by 1 ulp (lse reduction order)
+ model median grad rel-L2 vs fp32 1.29e-3 (stock 1.30-1.32e-3), at the
+ run's 2^15 loss scale. Without a loss scale the two fp16 paths
+ differ ~6% on early layers -- subnormal noise, not a bug.
+
+Document isolation and causality are exact (zero drift). dQ uses fp32 atomics,
+so attention backward is not bitwise deterministic run to run (fp32 order only).
+
+## Gotchas found on the way
+
+- q/k/v leave `_pre_attn` fp32, not fp16: the Canon masks m1/m2 are fp32 and
+ promote. Stock SDPA casts them via autocast; the Volta path does the same cast
+ inside the compiled region (`out16`).
+- In production the head's `g` is fp32 (x is fp32), and autocast re-casts W to
+ fp16 twice per chunk. The rewrite casts once per call.
+- `dW.addcmul_(fp16, fp32)` takes torch's slow unvectorized path (0.9 ms/chunk);
+ replaced by a Triton FMA. Exact, because `back` is a power of two.
+- `__launch_bounds__(128, 3)` makes both kernels slower (spills): 248/216 regs,
+ 2 CTAs/SM is the best measured point.
+- Remaining bwd split (ablation): dQ phase ~26%, tile loads ~6%, atomics ~4%.
+
+## Live result and later experiments (2026-09-27)
+
+Adopted at step 28,239: **35.5k -> 45.2k tok/s, 30.5 -> 38.8 TF, MFU 24.4% ->
+31.0%**, ETA 60 -> 47.5 days. `logs/launch_chain.sh` exports
+`BORIS_VOLTA_KERNELS=1` by default.
+
+Measured afterwards, so nobody re-tries them blind:
+
+- **Megakernel / launch overhead.** Whole micro-step captured as one CUDA graph
+ vs eager, identical kernels: 3.3% faster at micro_batch 2 (8-layer slice),
+ i.e. ~0.5% at production mb 16. That is the ceiling for anything whose win is
+ removing launches; the GEMMs (~55% of the step) are cuBLAS.
+- **Skipping fully-masked MMA blocks on diagonal tiles** (bitwise-identical
+ results): fwd 4.5% and bwd 18.6% SLOWER. Warp-dependent loop bounds break the
+ unrolled MMA schedule, and the CTA waits at the barrier for its busiest warp
+ anyway. Dropped.
+- **Adopted, no speed effect:** gd written over the logits buffer (-64 MB);
+ shape/device/metadata guards in the extension.
+- verify.py now also covers an 8192-token document (stage two; fp32 reference,
+ error ratio vs stock 1.000) and a head call with a partial final chunk.
+- Test gotcha: `t.to(torch.float16)` on an fp16 tensor returns the same tensor,
+ so two gradient runs "on copies" accumulate into one .grad. Clone.
+- **Megakernel, stage 1** (`experiments/mega_ffn2.cu`): the whole FFN half of a
+ block forward (RMSNorm -> gate/up -> SiLU*up -> down -> residual) in one
+ kernel, intermediate never leaving shared memory. Precision identical to stock
+ (FFN-delta error vs fp64 6.454e-4 both). Speed at mb 16: 23.7 ms first build,
+ 16.1 ms with register-prefetched weights, vs **6.5 ms** Inductor+cuBLAS -- 2.5x
+ SLOWER. Structural: holding a full 512-wide x tile costs 64 KB of shared
+ memory, so a CTA can own only 64 rows, each weight byte is reused by only 64
+ rows (cuBLAS reuses across 128x128 tiles), and one CTA per SM leaves nothing
+ to hide the barriers. The fusion's best case is avoiding the gate/up write,
+ and training's backward needs gate/up saved anyway. Not pursued further.
+
+## Round 2 (2026-09-27 evening) -- reviewed with GPT-6 Sol and Muse Spark
+
+Adopted (dormant until the next relaunch; verify 14/14):
+- **Head dW via fp32-output GEMM** (GPT's idea): `torch.addmm(dW, g^T, x16,
+ beta=0|1, out_dtype=float32)` selects `volta_s884gemm_fp16` + fp32 split-k
+ reduce. dW is never rounded to fp16, so the x-rescale / DW_SHRINK overflow
+ guard and the second fp16 operand (gd, 64 MB write per chunk) are gone.
+ **Head 12.6% faster; dW error vs fp64 5.24e-4 -> 5.01e-4 (fixture).**
+- **Forward K/V register prefetch**: next tile's global loads issued before the
+ current tile's MMAs (fwd has register headroom: 216). **fwd 4.5% faster,
+ bitwise-identical output.**
+
+Measured and rejected:
+- Megakernel FFN at 128 rows (column split, gate/up recomputed): 24.3 ms vs
+ 6.5 ms stock (and 16.0 ms at 64 rows). 1.67x FLOPs plus spills.
+- Blocked two-pass head softmax with online max/sum + evict_first (Muse): no
+ gain over the one-pass kernel.
+- Rejected on inspection: aliasing P into Ks in fwd (race -- other warps still
+ read Ks for QK^T), folding scale*log2e into fp16 Q (changes rounding), Q/dO
+ register prefetch in bwd (248 regs, no headroom).
+- **Fused Canon + RoPE** (`canon_rope.py`, Triton, fwd + bwd): replaces
+ Inductor's zero-padded fp32 copies of qkv, the 3-tap combine, RoPE and their
+ backward passes. 1.96x on that region; same error as the region for q/k/v and
+ dqkv, weight gradient closer to fp64. Isolation exact at document boundaries.
+ Whole 8-layer slice: 1.41x -> **1.538x** vs stock (contended), i.e. ~+9% over
+ the kernels live since step 28,239. verify.py: 80/80.
+- **Canon + RoPE in CUDA** (`canon_rope_cuda.py`, from a second Claude Code
+ reviewer): sliding window down the sequence, 8 channels per thread, 16-byte
+ I/O. Bitwise-identical q/k/v/dqkv to the Triton version (checked
+ independently), 3.55x the compiled region at mb 16, **+4.1% whole micro-step
+ at full size**. The model now uses this; `canon_rope.py` (Triton) is kept as
+ the reference. verify.py 80/80.
+- **Longest-document-first CTA order** (GPT-6 Sol's research lead): the tile
+ table cumsums over documents sorted by tile count and passes the permutation
+ to both kernels. Bitwise-identical; idle V100, mb 16: fwd 1.010 -> 0.945 ms
+ (-6.5%), bwd 2.712 -> 2.559 ms (-5.6%); ctx 8192 about -6% both. verify 81/81.
+- **GEMM algorithm sweep** (TunableOp, idle V100): cuBLAS "Default" won every
+ block shape. The GEMMs already run at 54-59% of the 125 TF peak (QKV 73.5 TF,
+ gate/up 71, down 74, o_proj 67). There is little left to take from them.
+- Attention, by contrast, runs at ~24-27% of peak (fwd ~30 TF, bwd ~27 TF
+ counted). That gap is the largest remaining one.
+- **Masks only where needed** (after an ablation showed softmax math is ~19%
+ of the forward): a CTA-uniform branch skips the per-element integer mask on
+ every tile except the diagonal and a document's last tile. Bitwise-identical
+ (dQ differs only by its usual atomic-order noise, measured equal to
+ old-vs-old). Idle V100: fwd -4.9%, bwd -2.1% at both 2048 and 8192.
+ Tried and dropped: skipping the output rescale when no max moved (no gain),
+ folding the scale into ex2's FMA (+1.2% more but not bitwise).
+- Volta attention literature (2026-09): SparkAttention's backward gains need
+ fp16 accumulation (excluded); its register-shuffle C->A layout trick targets
+ the P shared-memory round trip, which the ablation shows costs nothing here.
+ No public fp32-accumulation varlen training kernel for sm_70 beats this one.
+- **Head dx written fp32 from the GEMM** (`out_dtype=float32`): cuBLAS can no
+ longer pick split-k kernels that reduce partials in fp16. dx error vs fp64
+ 5.31e-4 -> 3.43e-4 and 2.9% faster; also makes a larger lm_head_chunk safe
+ (chunk 4096: head -11.6%, dx 3.37e-4; with fp16 dx the same chunk raised dx
+ error 41%). verify 81/81, model median grad error 1.208e-3 (stock 1.293e-3).
+- **Hardware correction:** this PG500-216 has **72 SMs** (not 80) and HBM at
+ 1107 MHz. fp16 tensor peak = 72 x 8 x 128 x 1.38 GHz = **101.7 TF**, not 125;
+ every MFU quoted against 125 understates by 1.23x. Measured device copy
+ 956 GB/s. So the GEMMs run at ~70-73% of real peak, and the ~680 GB/s
+ elementwise kernels have ~30% headroom.
+- Tried and reverted: one-thread-per-row bwd_pre_kernel (3.6x SLOWER -- the
+ warp-per-row version was already coalesced at ~680 GB/s). The SwiGLU backward
+ pairing lead was dropped on reading Inductor's kernel: the "redundant" reads
+ hit cache.
diff --git a/volta_kernels/__init__.py b/volta_kernels/__init__.py
new file mode 100644
index 0000000..06b9547
--- /dev/null
+++ b/volta_kernels/__init__.py
@@ -0,0 +1,124 @@
+"""Hand-written sm_70 kernels for Boris-2. See README.md in this directory.
+
+Built on first use with torch.utils.cpp_extension (nvcc, -arch=sm_70) into
+volta_kernels/build/, which is kept between runs so a resume does not recompile.
+"""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+import torch
+
+_HERE = Path(__file__).resolve().parent
+_ext = None
+
+
+def _load():
+ global _ext
+ if _ext is None:
+ from torch.utils.cpp_extension import load
+ build = _HERE / "build_v4" # new dir: never relink the .so a running trainer has mapped
+ build.mkdir(exist_ok=True)
+ os.environ.setdefault("TORCH_CUDA_ARCH_LIST", "7.0")
+ _ext = load(name="boris_volta_attn_v4", sources=[str(_HERE / "attn.cu")],
+ extra_cuda_cflags=["-O3", "-arch=sm_70", "-lineinfo"],
+ build_directory=str(build), verbose=False)
+ return _ext
+
+
+def tile_table(offsets: torch.Tensor):
+ """(cu, tcum, perm) int32 for a document-offset tensor, cached on the tensor.
+
+ tcum is the cumulative 64-tile count over documents sorted LONGEST FIRST and
+ perm maps that order back to document indices, so the heaviest CTAs launch
+ first and short documents fill the tail of the grid. Launch order only: the
+ results are bitwise-identical to packer order; measured fwd -6.5%, bwd -5.6%.
+
+ One forward calls attention 31 times (plus recomputation) with the same
+ offsets, so the table is built once. Pure GPU ops -- no host sync.
+ """
+ tab = getattr(offsets, "_volta_tab", None)
+ # Keyed on the version counter too: an in-place edit of the offsets must
+ # never be served a stale table.
+ if tab is None or tab[3] != offsets._version:
+ cu = offsets.to(torch.int32)
+ tiles = (cu[1:] - cu[:-1] + 63) // 64
+ order = torch.argsort(tiles, descending=True, stable=True)
+ tcum = torch.zeros_like(cu)
+ tcum[1:] = torch.cumsum(tiles[order], 0)
+ tab = (cu, tcum, order.to(torch.int32), offsets._version)
+ offsets._volta_tab = tab
+ return tab[0], tab[1], tab[2]
+
+
+class _VarlenAttn(torch.autograd.Function):
+ @staticmethod
+ def forward(ctx, q, k, v, cu, tcum, perm, scale):
+ ext = _load()
+ o, lse = ext.fwd(q, k, v, cu, tcum, perm, scale)
+ ctx.save_for_backward(q, k, v, o, lse, cu, tcum, perm)
+ ctx.scale = scale
+ return o
+
+ @staticmethod
+ def backward(ctx, do):
+ q, k, v, o, lse, cu, tcum, perm = ctx.saved_tensors
+ dq, dk, dv = _load().bwd(do, q, k, v, o, lse, cu, tcum, perm, ctx.scale)
+ return dq, dk, dv, None, None, None, None
+
+
+def varlen_causal_attention(q, k, v, offsets, scale: float | None = None):
+ """Causal attention within each document of a packed (N, H, 64) fp16 stream.
+
+ Equivalent to SDPA(is_causal=True) over a jagged nested tensor built from
+ `offsets` (the packer's document boundaries into the flattened stream).
+ """
+ if scale is None:
+ scale = q.shape[-1] ** -0.5
+ cu, tcum, perm = tile_table(offsets)
+ return _VarlenAttn.apply(q, k, v, cu, tcum, perm, scale)
+
+
+class _CanonRopeAttn(torch.autograd.Function):
+ """Canon+RoPE and varlen attention as ONE autograd Function.
+
+ Kept separate, the attention backward rounded dQ to fp16 in its own pass
+ (bwd_post_kernel) and the Canon backward read it back: ~100 MB of traffic
+ per layer and an fp16 rounding of every dQ. Merged, the Canon backward
+ reads the raw fp32 dQ accumulator and applies the (power-of-two) softmax
+ scale itself. Measured: dqkv error vs fp64 1.2038e-3 -> 1.1955e-3, region
+ -1.0% at micro_batch 16.
+ """
+
+ @staticmethod
+ def forward(ctx, qkv, w, m1, m2, cos, sin, offsets, scale):
+ from .canon_rope_bw import ext as canon_ext
+ B, T, _ = qkv.shape
+ N = B * T
+ m1f, m2f = m1.reshape(N).contiguous(), m2.reshape(N).contiguous()
+ cf, sf = cos.reshape(N, 32).contiguous(), sin.reshape(N, 32).contiguous()
+ # canon_rope_bw (swarm canon_bw): bitwise-identical q/k/v, fwd -33% / bwd -17%
+ q, k, v = canon_ext.fwd(qkv, w.contiguous(), m1f, m2f, cf, sf, 4, 64, 4, 1)
+ cu, tcum, perm = tile_table(offsets)
+ qv, kv, vv = q.view(N, 8, 64), k.view(N, 8, 64), v.view(N, 8, 64)
+ o, lse = _load().fwd(qv, kv, vv, cu, tcum, perm, scale)
+ ctx.save_for_backward(qkv, w, m1f, m2f, cf, sf, qv, kv, vv, o, lse, cu, tcum, perm)
+ ctx.scale = scale
+ return o
+
+ @staticmethod
+ def backward(ctx, do):
+ from .canon_rope_bw import ext as canon_ext
+ qkv, w, m1f, m2f, cf, sf, q, k, v, o, lse, cu, tcum, perm = ctx.saved_tensors
+ B, T, _ = qkv.shape
+ dqacc, dk, dv = _load().bwd_raw(do, q, k, v, o, lse, cu, tcum, perm, ctx.scale)
+ dqkv, part = canon_ext.bwd(qkv, w.contiguous(), m1f, m2f, cf, sf,
+ dqacc.view(B, T, 8, 64), dk.view(B, T, 8, 64),
+ dv.view(B, T, 8, 64), ctx.scale, True, 4, 2, 1, False)
+ return dqkv, part.sum(0).to(w.dtype), None, None, None, None, None, None
+
+
+def canon_rope_attention(qkv, canon_w, m1, m2, cos, sin, offsets):
+ """qkv (B, T, 1536) fp16 -> Canon + RoPE -> causal varlen attention -> (N, 8, 64)."""
+ return _CanonRopeAttn.apply(qkv, canon_w, m1, m2, cos, sin, offsets, 0.125)
diff --git a/volta_kernels/attn.cu b/volta_kernels/attn.cu
new file mode 100644
index 0000000..1725723
--- /dev/null
+++ b/volta_kernels/attn.cu
@@ -0,0 +1,761 @@
+// swarm/integ_attn: merged attention kernel = live attn.cu +
+// attn_softmax padded smem (row stride 84 halves), fwd software pipelining (double-buffered K,
+// next QK^T overlapping softmax), unrolled full-tile dQ atomic loop;
+// bwd ILP kept as a knob, default off (see BWD_ILP) (polynomial exp NOT included)
+// attn_long 1-D grid, heads interleaved in the global longest-first launch order
+// attn_bwd mode 6: next query tile's Q, dO, lse, D prefetched into registers (BWD_PF)
+// o, lse, dk, dv bitwise identical to the live kernel; dq differs only by fp32 atomic order.
+// Varlen causal FlashAttention for sm_70 (V100), head_dim 64, fp16 in/out.
+//
+// Why this exists: PyTorch's only SDPA backend that trains on sm_70 is the
+// CUTLASS memory-efficient kernel, whose backward measured ~4x its own forward
+// on Boris-2's shapes (FlashAttention-2 does ~2.5x). Triton cannot help here:
+// on sm_70 tl.dot lowers to plain FMA (no tensor cores), so this is CUDA with
+// raw mma.sync.m8n8k4 -- the same HMMA.884 instruction CUTLASS's sm70 path uses.
+//
+// Precision is the memory-efficient kernel's precision class, deliberately:
+// * every GEMM takes fp16 operands and accumulates in fp32
+// * softmax statistics (max, sum, logsumexp) are fp32
+// * P and dS are rounded to fp16 only as GEMM operands, as in any fp16 kernel
+// * dQ is accumulated in an fp32 buffer across key tiles and rounded once
+//
+// Fragment layouts of m8n8k4 (verified empirically on this card, see
+// volta_kernels/README.md). j = lane%4, hi = lane>=16, quad-pair qp = (lane>>2)&3:
+// A.row : row = j + 4hi, k = 0..3 A.col : k = j, row = 4hi + 0..3
+// B.col : n = j + 4hi, k = 0..3 B.row : k = j, n = 4hi + 0..3
+// D f32 : row = (lane&1) + (i&2) + 4hi, col = (i&4) + (lane&2) + (i&1)
+// Each quad-pair computes its own 8x8 tile; the four are arranged 2x2 into a
+// 16x16 warp tile: m offset 8*(qp&1), n offset 8*(qp>>1).
+#include <torch/extension.h>
+#include <ATen/cuda/CUDAContext.h>
+#include <cuda_fp16.h>
+#include <stdint.h>
+
+namespace {
+
+constexpr int HD = 64; // head dim
+constexpr int TL = 64; // tile edge (queries or keys)
+constexpr int NT = 128; // 4 warps, 16 tile rows each
+
+// 64x64 fp16 tiles live unpadded in shared memory with the 8-byte segments of
+// each row XOR-swizzled. The mapping makes every fragment access pattern used
+// below (A.row, B.col: 8 rows x one segment; B.row, A.col: 4 rows x two
+// segments) hit distinct banks within each half-warp.
+__device__ __forceinline__ int swz(int r) { return ((r & 3) << 2) | ((r >> 2) & 3); }
+__device__ __forceinline__ int so(int r, int c) {
+ return r * 64 + ((((c >> 2) ^ swz(r))) << 2) + (c & 3);
+}
+
+// Compile-time knobs (defaults = the adopted combination).
+// BWD_ILP (attn_softmax) and BWD_PF (attn_bwd mode 6) combine without spills (255 regs, 2 CTAs/SM),
+// but with the prefetch on, ILP adds nothing measurable (bwd vs live: PF only 0.944/0.945 at
+// 16x2048, 0.900 at 4x8192; PF+ILP 0.946/0.947 and 0.897). PF alone wins at stage 1 and leaves
+// 17 registers of headroom (238), so ILP is off by default; set -DBWD_ILP=1 to re-test.
+#ifndef BWD_ILP
+#define BWD_ILP 0 // bwd: S^T MMAs first, then exp(S^T) and the dP^T MMAs in one basic block
+#endif
+#ifndef BWD_PF
+#define BWD_PF 1 // bwd: next query tile's Q/dO/lse/D prefetched into registers
+#endif
+
+// Padded layout: row stride 84 halves = 42 words. Every fragment pattern used
+// here (8 rows {0..3,8..11} x 8B; 4 rows x 2 segments 16B apart; P's 32-bit
+// stores; commit's 2 rows x 8 segments) hits distinct banks per half-warp, and
+// all per-ks / per-nb offsets become compile-time immediates.
+constexpr int LDP = 84;
+template <bool P> __device__ __forceinline__ int lo(int r, int c) {
+ if constexpr (P) return r * LDP + c; else return so(r, c);
+}
+
+__device__ __forceinline__ uint2 lds2(const half* t, int off) {
+ return *reinterpret_cast<const uint2*>(t + off);
+}
+
+#define DEF_MMA(LA, LB) \
+ __device__ __forceinline__ void mma_##LA##_##LB(float* d, uint2 a, uint2 b) { \
+ asm("mma.sync.aligned.m8n8k4." #LA "." #LB ".f32.f16.f16.f32 " \
+ "{%0,%1,%2,%3,%4,%5,%6,%7}, {%8,%9}, {%10,%11}, {%0,%1,%2,%3,%4,%5,%6,%7};" \
+ : "+f"(d[0]), "+f"(d[1]), "+f"(d[2]), "+f"(d[3]), \
+ "+f"(d[4]), "+f"(d[5]), "+f"(d[6]), "+f"(d[7]) \
+ : "r"(a.x), "r"(a.y), "r"(b.x), "r"(b.y)); \
+ }
+DEF_MMA(row, col)
+DEF_MMA(row, row)
+DEF_MMA(col, row)
+
+__device__ __forceinline__ float ex2(float x) {
+ float y;
+ asm("ex2.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x));
+ return y;
+}
+
+__device__ __forceinline__ unsigned pack_h2(float a, float b) {
+ __half2 h = __floats2half2_rn(a, b);
+ return *reinterpret_cast<unsigned*>(&h);
+}
+
+// Copy rows [row0, row0+64) of one head (64 contiguous halves at token stride
+// `ts`) into a swizzled tile. Rows at or past `nrows` are zero-filled.
+template <bool PD = false>
+__device__ __forceinline__ void load_tile(half* t, const half* g, int ts, int row0,
+ int nrows, int tid) {
+#pragma unroll
+ for (int it = 0; it < (TL * 8) / NT; ++it) {
+ int idx = tid + it * NT;
+ int r = idx >> 3, c = (idx & 7) * 8;
+ uint4 v = make_uint4(0, 0, 0, 0);
+ if (row0 + r < nrows)
+ v = *reinterpret_cast<const uint4*>(g + (long)(row0 + r) * ts + c);
+ *reinterpret_cast<uint2*>(t + lo<PD>(r, c)) = make_uint2(v.x, v.y);
+ *reinterpret_cast<uint2*>(t + lo<PD>(r, c + 4)) = make_uint2(v.z, v.w);
+ }
+}
+
+// Register-staged copy of one 64x64 tile: global loads issued early, the
+// swizzled shared-memory stores later, so the load latency overlaps compute.
+struct TileRegs { uint4 v[(TL * 8) / NT]; };
+__device__ __forceinline__ void fetch_tile(TileRegs& R, const half* g, int ts, int row0, int nrows, int tid) {
+#pragma unroll
+ for (int it = 0; it < (TL * 8) / NT; ++it) {
+ int idx = tid + it * NT, r = idx >> 3, c = (idx & 7) * 8;
+ R.v[it] = (row0 + r < nrows) ? *reinterpret_cast<const uint4*>(g + (long)(row0 + r) * ts + c)
+ : make_uint4(0, 0, 0, 0);
+ }
+}
+template <bool PD = false>
+__device__ __forceinline__ void commit_tile(half* t, const TileRegs& R, int tid) {
+#pragma unroll
+ for (int it = 0; it < (TL * 8) / NT; ++it) {
+ int idx = tid + it * NT, r = idx >> 3, c = (idx & 7) * 8;
+ *reinterpret_cast<uint2*>(t + lo<PD>(r, c)) = make_uint2(R.v[it].x, R.v[it].y);
+ *reinterpret_cast<uint2*>(t + lo<PD>(r, c + 4)) = make_uint2(R.v[it].z, R.v[it].w);
+ }
+}
+
+// Map a flat block index onto (document, tile). tcum[d] is the number of tiles
+// in documents before d; tcum[ndocs] the total.
+__device__ __forceinline__ bool find_tile(const int* tcum, const int* perm, int ndocs, int b,
+ int& doc, int& local, int& ntiles) {
+ if (b >= tcum[ndocs]) return false;
+ int lo = 0, hi = ndocs - 1;
+ while (lo < hi) {
+ int mid = (lo + hi + 1) >> 1;
+ if (tcum[mid] <= b) lo = mid; else hi = mid - 1;
+ }
+ doc = perm[lo]; // tcum is over documents sorted longest-first
+ local = b - tcum[lo];
+ ntiles = tcum[lo + 1] - tcum[lo];
+ return true;
+}
+
+// --------------------------------------------------------------------------
+// Forward. One CTA = one 64-query tile of one document, one head.
+// --------------------------------------------------------------------------
+__global__ void __launch_bounds__(NT) fwd_kernel(
+ const half* __restrict__ Q, const half* __restrict__ K, const half* __restrict__ V,
+ half* __restrict__ O, float* __restrict__ LSE, const int* __restrict__ cu,
+ const int* __restrict__ tcum, const int* __restrict__ perm, int ndocs, int N, int H,
+ int sq, int sk, int sv, int s_o, float scale_log2) {
+ // Padded tiles (row stride LDP); K double-buffered for the software pipeline.
+ __shared__ __align__(16) half Ks2[2][TL * LDP];
+ __shared__ __align__(16) half Vs[TL * LDP];
+ __shared__ __align__(16) half Ps[4][16 * LDP];
+
+ // 1-D grid, heads interleaved: the longest-first order holds across heads.
+ const int h = blockIdx.x % H;
+ int doc, local, ntiles;
+ if (!find_tile(tcum, perm, ndocs, blockIdx.x / H, doc, local, ntiles)) return;
+ const int t = ntiles - 1 - local; // longest-running tiles launch first
+ const int s0 = cu[doc], L = cu[doc + 1] - s0;
+
+ const int tid = threadIdx.x, w = tid >> 5, lane = tid & 31;
+ const int j = lane & 3, hi = lane >> 4, qp = (lane >> 2) & 3;
+ const int mo = 8 * (qp & 1), no = 8 * (qp >> 1);
+
+ uint2 qf[16];
+ {
+ int qrow = t * TL + 16 * w + mo + j + 4 * hi;
+ const half* g = Q + (long)(s0 + qrow) * sq + h * HD;
+#pragma unroll
+ for (int ks = 0; ks < 16; ++ks)
+ qf[ks] = qrow < L ? *reinterpret_cast<const uint2*>(g + 4 * ks) : make_uint2(0, 0);
+ }
+ const int qr0 = t * TL + 16 * w + mo + (lane & 1) + 4 * hi; // i&2 == 0
+ float m[2] = {-INFINITY, -INFINITY}, l[2] = {0.f, 0.f};
+ float o[4][8];
+#pragma unroll
+ for (int a = 0; a < 4; ++a)
+#pragma unroll
+ for (int i = 0; i < 8; ++i) o[a][i] = 0.f;
+
+ half* P = Ps[w];
+ const half* Kg = K + (long)s0 * sk + h * HD;
+ const half* Vg = V + (long)s0 * sv + h * HD;
+
+ float s[4][8];
+ TileRegs kr, vr;
+ // Prologue: K_0 committed, S_0 computed; the loop body computes S_{kt+1}
+ // while doing softmax(S_kt). Ks2[kt&1] holds K_kt.
+ fetch_tile(kr, Kg, sk, 0, L, tid);
+ fetch_tile(vr, Vg, sv, 0, L, tid);
+ commit_tile<true>(Ks2[0], kr, tid);
+ if (t > 0) fetch_tile(kr, Kg, sk, TL, L, tid);
+ __syncthreads();
+#pragma unroll
+ for (int a = 0; a < 4; ++a)
+#pragma unroll
+ for (int i = 0; i < 8; ++i) s[a][i] = 0.f;
+#pragma unroll
+ for (int ks = 0; ks < 16; ++ks)
+#pragma unroll
+ for (int nb = 0; nb < 4; ++nb)
+ mma_row_col(s[nb], qf[ks], lds2(Ks2[0], lo<true>(nb * 16 + no + j + 4 * hi, 4 * ks)));
+
+ for (int kt = 0; kt <= t; ++kt) {
+ __syncthreads(); // everyone done with Vs (PV of kt-1) and Ks2[(kt+1)&1] (S_kt's MMAs)
+ commit_tile<true>(Vs, vr, tid);
+ if (kt < t) commit_tile<true>(Ks2[(kt + 1) & 1], kr, tid);
+ if (kt < t) {
+ fetch_tile(vr, Vg, sv, (kt + 1) * TL, L, tid);
+ if (kt + 1 < t) fetch_tile(kr, Kg, sk, (kt + 2) * TL, L, tid);
+ }
+ __syncthreads();
+
+ const bool diag = (kt == t);
+ const bool masked = diag || kt * TL + TL > L;
+ float mx[2] = {m[0], m[1]};
+ // Mask by select only; the max, the next tile's MMAs and the exp share one
+ // basic block so ptxas can interleave HMMA with the softmax FP work.
+ if (masked) {
+#pragma unroll
+ for (int nb = 0; nb < 4; ++nb)
+#pragma unroll
+ for (int i = 0; i < 8; ++i) {
+ int key = kt * TL + nb * 16 + no + (i & 4) + (lane & 2) + (i & 1);
+ int qr = qr0 + (i & 2);
+ bool ok = key < L && (!diag || key <= qr);
+ s[nb][i] = ok ? s[nb][i] : -INFINITY;
+ }
+ }
+ float sn[4][8];
+ {
+ const half* Kn = Ks2[(kt + 1) & 1]; // stale on the last tile; result unused
+#pragma unroll
+ for (int a = 0; a < 4; ++a)
+#pragma unroll
+ for (int i = 0; i < 8; ++i) sn[a][i] = 0.f;
+#pragma unroll
+ for (int ks = 0; ks < 16; ++ks)
+#pragma unroll
+ for (int nb = 0; nb < 4; ++nb)
+ mma_row_col(sn[nb], qf[ks], lds2(Kn, lo<true>(nb * 16 + no + j + 4 * hi, 4 * ks)));
+ }
+#pragma unroll
+ for (int nb = 0; nb < 4; ++nb)
+#pragma unroll
+ for (int i = 0; i < 8; ++i) mx[(i >> 1) & 1] = fmaxf(mx[(i >> 1) & 1], s[nb][i]);
+#pragma unroll
+ for (int r = 0; r < 2; ++r) {
+ mx[r] = fmaxf(mx[r], __shfl_xor_sync(0xffffffff, mx[r], 2));
+ mx[r] = fmaxf(mx[r], __shfl_xor_sync(0xffffffff, mx[r], 8));
+ }
+ float alpha[2];
+#pragma unroll
+ for (int r = 0; r < 2; ++r) {
+ alpha[r] = ex2((m[r] - mx[r]) * scale_log2); // exp2(-inf) = 0 on the first tile
+ m[r] = mx[r];
+ l[r] *= alpha[r];
+ }
+#pragma unroll
+ for (int nb = 0; nb < 4; ++nb)
+#pragma unroll
+ for (int i = 0; i < 8; ++i) {
+ int r = (i >> 1) & 1;
+ float p = ex2(fmaf(s[nb][i], scale_log2, -m[r] * scale_log2));
+ s[nb][i] = p;
+ l[r] += p;
+ o[nb][i] *= alpha[r];
+ }
+
+ __syncwarp();
+#pragma unroll
+ for (int nb = 0; nb < 4; ++nb)
+#pragma unroll
+ for (int i = 0; i < 8; i += 2) {
+ int row = mo + (lane & 1) + (i & 2) + 4 * hi;
+ int col = nb * 16 + no + (i & 4) + (lane & 2);
+ *reinterpret_cast<unsigned*>(P + lo<true>(row, col)) = pack_h2(s[nb][i], s[nb][i + 1]);
+ }
+ __syncwarp();
+#pragma unroll
+ for (int ks = 0; ks < 16; ++ks) {
+ uint2 a = lds2(P, lo<true>(mo + j + 4 * hi, 4 * ks));
+#pragma unroll
+ for (int nb = 0; nb < 4; ++nb)
+ mma_row_row(o[nb], a, lds2(Vs, lo<true>(4 * ks + j, nb * 16 + no + 4 * hi)));
+ }
+#pragma unroll
+ for (int a = 0; a < 4; ++a)
+#pragma unroll
+ for (int i = 0; i < 8; ++i) s[a][i] = sn[a][i];
+ }
+
+#pragma unroll
+ for (int r = 0; r < 2; ++r) {
+ l[r] += __shfl_xor_sync(0xffffffff, l[r], 2);
+ l[r] += __shfl_xor_sync(0xffffffff, l[r], 8);
+ }
+#pragma unroll
+ for (int r = 0; r < 2; ++r) {
+ int qr = qr0 + 2 * r;
+ if (qr >= L) continue;
+ float inv = 1.f / l[r];
+ half* og = O + (long)(s0 + qr) * s_o + h * HD;
+#pragma unroll
+ for (int nb = 0; nb < 4; ++nb)
+#pragma unroll
+ for (int i = 2 * r; i < 8; i += 4) {
+ int col = nb * 16 + no + (i & 4) + (lane & 2);
+ *reinterpret_cast<unsigned*>(og + col) = pack_h2(o[nb][i] * inv, o[nb][i + 1] * inv);
+ }
+ if ((lane & 2) == 0 && no == 0) LSE[(long)h * N + s0 + qr] = m[r] * scale_log2 + __log2f(l[r]);
+ }
+}
+
+// --------------------------------------------------------------------------
+// Backward. One CTA = one 64-key tile of one document, one head; it walks the
+// query tiles at and after it. dK/dV stay in registers; dQ is accumulated in
+// an fp32 buffer with atomics (same as FlashAttention-2).
+// --------------------------------------------------------------------------
+__global__ void __launch_bounds__(NT) bwd_kernel(
+ const half* __restrict__ Q, const half* __restrict__ K, const half* __restrict__ V,
+ const half* __restrict__ dO, const float* __restrict__ LSE,
+ const float* __restrict__ Dv, float* __restrict__ dQacc, half* __restrict__ dK,
+ half* __restrict__ dV, const int* __restrict__ cu, const int* __restrict__ tcum,
+ const int* __restrict__ perm,
+ int ndocs, int N, int H, int sq, int sk, int sv, int sdo, int sdk, int sdv,
+ float scale_log2, float scale) {
+ __shared__ __align__(16) half Ks[TL * HD];
+ __shared__ __align__(16) half Vs[TL * HD];
+ // Q and dO for the current query tile (padded, row stride LDP). Once dK/dV
+ // are done with them they are dead for the iteration, and the same bytes
+ // stage dQ in fp32.
+ __shared__ __align__(16) half QdOs[2 * TL * LDP];
+ half* Qs = QdOs;
+ half* dOs = QdOs + TL * LDP;
+ float* dQs = reinterpret_cast<float*>(QdOs);
+ __shared__ __align__(16) half St[TL * TL]; // P^T, then dS^T (key rows, query cols)
+ __shared__ float lse_s[TL], d_s[TL];
+
+ // 1-D grid, heads interleaved: the longest-first order holds across heads.
+ const int h = blockIdx.x % H;
+ int doc, kt, ntiles;
+ if (!find_tile(tcum, perm, ndocs, blockIdx.x / H, doc, kt, ntiles)) return;
+ const int s0 = cu[doc], L = cu[doc + 1] - s0;
+
+ const int tid = threadIdx.x, w = tid >> 5, lane = tid & 31;
+ const int j = lane & 3, hi = lane >> 4, qp = (lane >> 2) & 3;
+ const int mo = 8 * (qp & 1), no = 8 * (qp >> 1);
+
+ load_tile(Ks, K + (long)s0 * sk + h * HD, sk, kt * TL, L, tid);
+ load_tile(Vs, V + (long)s0 * sv + h * HD, sv, kt * TL, L, tid);
+
+ float dk[4][8], dv[4][8];
+#pragma unroll
+ for (int a = 0; a < 4; ++a)
+#pragma unroll
+ for (int i = 0; i < 8; ++i) dk[a][i] = dv[a][i] = 0.f;
+
+ const int kr0 = kt * TL + 16 * w + mo + (lane & 1) + 4 * hi; // this lane's key rows (i&2==0)
+ half* Sw = St + 16 * w * 64; // this warp's 16 key rows
+ const half* Qg = Q + (long)s0 * sq + h * HD;
+ const half* dOg = dO + (long)s0 * sdo + h * HD;
+#if BWD_PF
+ // The next query tile's Q/dO (and lse/D) are fetched into registers while
+ // the current tile computes, like the forward's K/V prefetch.
+ TileRegs qreg, dreg;
+ float lse_n = 0.f, d_n = 0.f;
+ fetch_tile(qreg, Qg, sq, kt * TL, L, tid);
+ fetch_tile(dreg, dOg, sdo, kt * TL, L, tid);
+ if (tid < TL) {
+ int qr = kt * TL + tid;
+ if (qr < L) { lse_n = LSE[(long)h * N + s0 + qr]; d_n = Dv[(long)h * N + s0 + qr]; }
+ }
+#endif
+
+ for (int qt = kt; qt < ntiles; ++qt) {
+ __syncthreads();
+#if BWD_PF
+ commit_tile<true>(Qs, qreg, tid);
+ commit_tile<true>(dOs, dreg, tid);
+ if (tid < TL) { lse_s[tid] = lse_n; d_s[tid] = d_n; }
+ if (qt + 1 < ntiles) {
+ fetch_tile(qreg, Qg, sq, (qt + 1) * TL, L, tid);
+ fetch_tile(dreg, dOg, sdo, (qt + 1) * TL, L, tid);
+ if (tid < TL) {
+ int qr = (qt + 1) * TL + tid;
+ bool ok = qr < L;
+ lse_n = ok ? LSE[(long)h * N + s0 + qr] : 0.f;
+ d_n = ok ? Dv[(long)h * N + s0 + qr] : 0.f;
+ }
+ }
+#else
+ load_tile<true>(Qs, Qg, sq, qt * TL, L, tid);
+ load_tile<true>(dOs, dOg, sdo, qt * TL, L, tid);
+ if (tid < TL) {
+ int qr = qt * TL + tid;
+ bool ok = qr < L;
+ lse_s[tid] = ok ? LSE[(long)h * N + s0 + qr] : 0.f;
+ d_s[tid] = ok ? Dv[(long)h * N + s0 + qr] : 0.f;
+ }
+#endif
+ __syncthreads();
+
+ // S^T = K_w Q^T and dP^T = V_w dO^T, both (16 keys x 64 queries).
+ float st[4][8], dp[4][8];
+#pragma unroll
+ for (int a = 0; a < 4; ++a)
+#pragma unroll
+ for (int i = 0; i < 8; ++i) st[a][i] = dp[a][i] = 0.f;
+#if BWD_ILP
+ // S^T first; the dP^T MMAs are issued in the same basic block as exp(S^T)
+ // so ptxas interleaves HMMA with the MUFU/FFMA work.
+#pragma unroll
+ for (int ks = 0; ks < 16; ++ks) {
+ uint2 ak = lds2(Ks, so(16 * w + mo + j + 4 * hi, 4 * ks));
+#pragma unroll
+ for (int nb = 0; nb < 4; ++nb)
+ mma_row_col(st[nb], ak, lds2(Qs, lo<true>(nb * 16 + no + j + 4 * hi, 4 * ks)));
+ }
+#define DP_MMAS \
+ _Pragma("unroll") for (int ks = 0; ks < 16; ++ks) { \
+ uint2 av = lds2(Vs, so(16 * w + mo + j + 4 * hi, 4 * ks)); \
+ _Pragma("unroll") for (int nb = 0; nb < 4; ++nb) \
+ mma_row_col(dp[nb], av, lds2(dOs, lo<true>(nb * 16 + no + j + 4 * hi, 4 * ks))); \
+ }
+#define DS_FROM_P \
+ _Pragma("unroll") for (int nb = 0; nb < 4; ++nb) \
+ _Pragma("unroll") for (int i = 0; i < 8; ++i) { \
+ int c = nb * 16 + no + (i & 4) + (lane & 2) + (i & 1); \
+ dp[nb][i] = st[nb][i] * (dp[nb][i] - d_s[c]); \
+ }
+#else
+#pragma unroll
+ for (int ks = 0; ks < 16; ++ks) {
+ uint2 ak = lds2(Ks, so(16 * w + mo + j + 4 * hi, 4 * ks));
+ uint2 av = lds2(Vs, so(16 * w + mo + j + 4 * hi, 4 * ks));
+#pragma unroll
+ for (int nb = 0; nb < 4; ++nb) {
+ int n = nb * 16 + no + j + 4 * hi;
+ mma_row_col(st[nb], ak, lds2(Qs, lo<true>(n, 4 * ks)));
+ mma_row_col(dp[nb], av, lds2(dOs, lo<true>(n, 4 * ks)));
+ }
+ }
+#endif
+
+ const bool diag = (qt == kt);
+ // Masks are needed only on the diagonal tile and a document's last query
+ // tile (a partial key tile is always diagonal); uniform across the CTA.
+ if (diag || qt * TL + TL > L) {
+#pragma unroll
+ for (int nb = 0; nb < 4; ++nb)
+#pragma unroll
+ for (int i = 0; i < 8; ++i) {
+ int c = nb * 16 + no + (i & 4) + (lane & 2) + (i & 1);
+ int qr = qt * TL + c;
+ int kr = kr0 + (i & 2);
+ bool ok = qr < L && kr < L && (!diag || kr <= qr);
+ float p = ok ? ex2(fmaf(st[nb][i], scale_log2, -lse_s[c])) : 0.f;
+ st[nb][i] = p;
+#if !BWD_ILP
+ dp[nb][i] = p * (dp[nb][i] - d_s[c]);
+#endif
+ }
+#if BWD_ILP
+ DP_MMAS
+ DS_FROM_P
+#endif
+ } else {
+#pragma unroll
+ for (int nb = 0; nb < 4; ++nb)
+#pragma unroll
+ for (int i = 0; i < 8; ++i) {
+ int c = nb * 16 + no + (i & 4) + (lane & 2) + (i & 1);
+ float p = ex2(fmaf(st[nb][i], scale_log2, -lse_s[c]));
+ st[nb][i] = p;
+#if !BWD_ILP
+ dp[nb][i] = p * (dp[nb][i] - d_s[c]);
+#endif
+ }
+#if BWD_ILP
+ DP_MMAS
+ DS_FROM_P
+#endif
+ }
+#undef DP_MMAS
+#undef DS_FROM_P
+
+ // dV_w += P^T_w dO
+ __syncwarp();
+#pragma unroll
+ for (int nb = 0; nb < 4; ++nb)
+#pragma unroll
+ for (int i = 0; i < 8; i += 2) {
+ int row = mo + (lane & 1) + (i & 2) + 4 * hi;
+ int col = nb * 16 + no + (i & 4) + (lane & 2);
+ *reinterpret_cast<unsigned*>(Sw + so(row, col)) = pack_h2(st[nb][i], st[nb][i + 1]);
+ }
+ __syncwarp();
+#pragma unroll
+ for (int ks = 0; ks < 16; ++ks) {
+ uint2 a = lds2(Sw, so(mo + j + 4 * hi, 4 * ks));
+#pragma unroll
+ for (int nb = 0; nb < 4; ++nb)
+ mma_row_row(dv[nb], a, lds2(dOs, lo<true>(4 * ks + j, nb * 16 + no + 4 * hi)));
+ }
+
+ // dK_w += dS^T_w Q
+ __syncwarp();
+#pragma unroll
+ for (int nb = 0; nb < 4; ++nb)
+#pragma unroll
+ for (int i = 0; i < 8; i += 2) {
+ int row = mo + (lane & 1) + (i & 2) + 4 * hi;
+ int col = nb * 16 + no + (i & 4) + (lane & 2);
+ *reinterpret_cast<unsigned*>(Sw + so(row, col)) = pack_h2(dp[nb][i], dp[nb][i + 1]);
+ }
+ __syncwarp();
+#pragma unroll
+ for (int ks = 0; ks < 16; ++ks) {
+ uint2 a = lds2(Sw, so(mo + j + 4 * hi, 4 * ks));
+#pragma unroll
+ for (int nb = 0; nb < 4; ++nb)
+ mma_row_row(dk[nb], a, lds2(Qs, lo<true>(4 * ks + j, nb * 16 + no + 4 * hi)));
+ }
+
+ // dQ (64 queries x 64 dims) += dS K; warp w takes queries 16w..16w+15.
+ // Its A operand is dS = (dS^T)^T, read straight from St as A.col.
+ __syncthreads();
+ float dq[4][8];
+#pragma unroll
+ for (int a = 0; a < 4; ++a)
+#pragma unroll
+ for (int i = 0; i < 8; ++i) dq[a][i] = 0.f;
+#pragma unroll
+ for (int ks = 0; ks < 16; ++ks) {
+ uint2 a = lds2(St, so(4 * ks + j, 16 * w + mo + 4 * hi));
+#pragma unroll
+ for (int nb = 0; nb < 4; ++nb)
+ mma_col_row(dq[nb], a, lds2(Ks, so(4 * ks + j, nb * 16 + no + 4 * hi)));
+ }
+ // Stage the warp's 16x64 dQ slice in shared memory and issue the atomics
+ // row-contiguous: each warp-wide RED then covers 128 consecutive bytes
+ // (4 sectors) instead of scattering over 8 rows 2KB apart. Row stride is
+ // 64 floats; float2 slots are XOR-swizzled by row so both the fragment-
+ // layout writes and the row reads are bank-conflict free.
+ float* dQw = dQs + 16 * w * 64;
+#pragma unroll
+ for (int nb = 0; nb < 4; ++nb)
+#pragma unroll
+ for (int i = 0; i < 8; i += 2) {
+ int row = mo + (lane & 1) + (i & 2) + 4 * hi;
+ int c2 = (nb * 16 + no + (i & 4) + (lane & 2)) >> 1;
+ int f = ((row & 1) << 1) | (row & 8);
+ *reinterpret_cast<float2*>(dQw + row * 64 + 2 * (c2 ^ f)) =
+ make_float2(dq[nb][i], dq[nb][i + 1]);
+ }
+ __syncwarp();
+ {
+ const int qbase = qt * TL + 16 * w;
+ const int nrow = min(16, L - qbase);
+ if (nrow == 16) {
+ // full 16-row slice: unrolled, smem offsets are 4 lane bases + immediates,
+ // global offsets one 64-bit pointer + row * (H*HD) increments.
+ float* gp = dQacc + ((long)(s0 + qbase) * H + h) * HD + lane;
+ const long rs = (long)H * HD;
+ const int x0 = lane >> 1, b0 = lane & 1;
+#pragma unroll
+ for (int row = 0; row < 16; ++row) {
+ const int f = ((row & 1) << 1) | (row & 8);
+ const float* sr = dQw + row * 64 + 2 * (x0 ^ f) + b0;
+ atomicAdd(gp, sr[0]);
+ atomicAdd(gp + 32, sr[32]);
+ gp += rs;
+ }
+ } else
+ for (int it = 0; it < 2 * nrow; ++it) {
+ int row = it >> 1, col = (it & 1) * 32 + lane;
+ int f = ((row & 1) << 1) | (row & 8);
+ float v = dQw[row * 64 + 2 * ((col >> 1) ^ f) + (col & 1)];
+ atomicAdd(dQacc + ((long)(s0 + qbase + row) * H + h) * HD + col, v);
+ }
+ }
+ }
+
+#pragma unroll
+ for (int r = 0; r < 2; ++r) {
+ int kr = kr0 + 2 * r;
+ if (kr >= L) continue;
+ half* gk = dK + (long)(s0 + kr) * sdk + h * HD;
+ half* gv = dV + (long)(s0 + kr) * sdv + h * HD;
+#pragma unroll
+ for (int nb = 0; nb < 4; ++nb)
+#pragma unroll
+ for (int i = 2 * r; i < 8; i += 4) {
+ int col = nb * 16 + no + (i & 4) + (lane & 2);
+ *reinterpret_cast<unsigned*>(gk + col) = pack_h2(dk[nb][i] * scale, dk[nb][i + 1] * scale);
+ *reinterpret_cast<unsigned*>(gv + col) = pack_h2(dv[nb][i], dv[nb][i + 1]);
+ }
+ }
+}
+
+// D[h][n] = sum_d dO[n,h,d] * O[n,h,d] in fp32. One warp per (n, h) row pair.
+__global__ void bwd_pre_kernel(const half* __restrict__ dO, const half* __restrict__ O,
+ float* __restrict__ Dv, int N, int H, int sdo, int s_o) {
+ int gw = (blockIdx.x * blockDim.x + threadIdx.x) >> 5, lane = threadIdx.x & 31;
+ if (gw >= N * H) return;
+ int n = gw / H, h = gw % H;
+ __half2 a = reinterpret_cast<const __half2*>(dO + (long)n * sdo + h * HD)[lane];
+ __half2 b = reinterpret_cast<const __half2*>(O + (long)n * s_o + h * HD)[lane];
+ float2 fa = __half22float2(a), fb = __half22float2(b);
+ float acc = fa.x * fb.x + fa.y * fb.y;
+#pragma unroll
+ for (int o = 16; o; o >>= 1) acc += __shfl_xor_sync(0xffffffff, acc, o);
+ if (lane == 0) Dv[(long)h * N + n] = acc;
+}
+
+// dq = fp16(dQacc * scale), four elements per thread.
+__global__ void bwd_post_kernel(const float4* __restrict__ acc, uint2* __restrict__ dq,
+ long n4, float scale) {
+ long i = blockIdx.x * (long)blockDim.x + threadIdx.x;
+ if (i >= n4) return;
+ float4 v = acc[i];
+ dq[i] = make_uint2(pack_h2(v.x * scale, v.y * scale), pack_h2(v.z * scale, v.w * scale));
+}
+
+void check_qkv(const torch::Tensor& t, const char* name) {
+ TORCH_CHECK(t.is_cuda() && t.scalar_type() == at::kHalf, name, " must be cuda fp16");
+ TORCH_CHECK(t.dim() == 3 && t.size(2) == HD && t.stride(2) == 1 && t.stride(1) == HD,
+ name, " must be (N, H, 64) with contiguous heads");
+ TORCH_CHECK(t.stride(0) % 8 == 0, name, " token stride must be a multiple of 8");
+}
+
+void check_meta(const torch::Tensor& q, const torch::Tensor& k, const torch::Tensor& v,
+ const torch::Tensor& cu, const torch::Tensor& tcum) {
+ TORCH_CHECK(k.sizes() == q.sizes() && v.sizes() == q.sizes(),
+ "q, k, v must have the same (N, H, 64) shape");
+ TORCH_CHECK(k.device() == q.device() && v.device() == q.device() &&
+ cu.device() == q.device() && tcum.device() == q.device(),
+ "q, k, v, cu, tcum must be on one device");
+ TORCH_CHECK(cu.scalar_type() == at::kInt && tcum.scalar_type() == at::kInt &&
+ cu.dim() == 1 && tcum.dim() == 1 && cu.is_contiguous() && tcum.is_contiguous(),
+ "cu and tcum must be contiguous 1-D int32");
+ TORCH_CHECK(cu.numel() >= 2 && tcum.numel() == cu.numel(),
+ "cu and tcum must both have ndocs+1 entries");
+}
+
+} // namespace
+
+// q, k, v: (N, H, 64) fp16. cu: (ndocs+1,) int32 offsets. tcum: (ndocs+1,) int32
+// cumulative 64-tile counts. Returns (o, lse) with lse (H, N) fp32 in log2 units.
+std::vector<torch::Tensor> fwd(torch::Tensor q, torch::Tensor k, torch::Tensor v,
+ torch::Tensor cu, torch::Tensor tcum, torch::Tensor perm, double scale) {
+ check_qkv(q, "q"); check_qkv(k, "k"); check_qkv(v, "v");
+ check_meta(q, k, v, cu, tcum);
+ const int N = q.size(0), H = q.size(1), ndocs = cu.numel() - 1;
+ auto o = torch::empty({N, H, HD}, q.options());
+ auto lse = torch::empty({H, N}, q.options().dtype(torch::kFloat));
+ // 1-D grid, heads interleaved (b = blockIdx.x / H); >= H * sum of ceil(len/64)
+ dim3 grid(((N + TL - 1) / TL + ndocs) * H);
+ fwd_kernel<<<grid, NT, 0, at::cuda::getCurrentCUDAStream()>>>(
+ (const half*)q.data_ptr(), (const half*)k.data_ptr(), (const half*)v.data_ptr(),
+ (half*)o.data_ptr(), lse.data_ptr<float>(), cu.data_ptr<int>(), tcum.data_ptr<int>(), perm.data_ptr<int>(),
+ ndocs, N, H, q.stride(0), k.stride(0), v.stride(0), o.stride(0),
+ (float)(scale * 1.4426950408889634));
+ C10_CUDA_KERNEL_LAUNCH_CHECK();
+ return {o, lse};
+}
+
+// Returns (dq, dk, dv), each (N, H, 64) fp16.
+std::vector<torch::Tensor> bwd_raw(torch::Tensor dout, torch::Tensor q, torch::Tensor k,
+ torch::Tensor v, torch::Tensor o, torch::Tensor lse,
+ torch::Tensor cu, torch::Tensor tcum, torch::Tensor perm, double scale) {
+ check_qkv(q, "q"); check_qkv(k, "k"); check_qkv(v, "v");
+ check_meta(q, k, v, cu, tcum);
+ dout = dout.contiguous();
+ check_qkv(dout, "dout");
+ TORCH_CHECK(dout.sizes() == q.sizes() && o.sizes() == q.sizes() && o.stride(2) == 1 &&
+ o.stride(1) == HD && dout.device() == q.device() && o.device() == q.device(),
+ "dout and o must match q");
+ TORCH_CHECK(lse.scalar_type() == at::kFloat && lse.is_contiguous() && lse.dim() == 2 &&
+ lse.size(0) == q.size(1) && lse.size(1) == q.size(0), "lse must be (H, N) fp32");
+ const int N = q.size(0), H = q.size(1), ndocs = cu.numel() - 1;
+ // D_i = rowsum(dO * O), fp32, laid out (H, N) like lse.
+ auto stream = at::cuda::getCurrentCUDAStream();
+ auto Dv = torch::empty({H, N}, q.options().dtype(torch::kFloat));
+ bwd_pre_kernel<<<(N * H * 32 + 255) / 256, 256, 0, stream>>>(
+ (const half*)dout.data_ptr(), (const half*)o.data_ptr(), Dv.data_ptr<float>(), N, H,
+ dout.stride(0), o.stride(0));
+ auto dqacc = torch::zeros({N, H, HD}, q.options().dtype(torch::kFloat));
+ auto dk = torch::empty({N, H, HD}, q.options());
+ auto dv = torch::empty({N, H, HD}, q.options());
+ // 1-D grid, heads interleaved (b = blockIdx.x / H); >= H * sum of ceil(len/64)
+ dim3 grid(((N + TL - 1) / TL + ndocs) * H);
+ bwd_kernel<<<grid, NT, 0, stream>>>(
+ (const half*)q.data_ptr(), (const half*)k.data_ptr(), (const half*)v.data_ptr(),
+ (const half*)dout.data_ptr(), lse.data_ptr<float>(), Dv.data_ptr<float>(),
+ dqacc.data_ptr<float>(), (half*)dk.data_ptr(), (half*)dv.data_ptr(),
+ cu.data_ptr<int>(), tcum.data_ptr<int>(), perm.data_ptr<int>(), ndocs, N, H, q.stride(0), k.stride(0),
+ v.stride(0), dout.stride(0), dk.stride(0), dv.stride(0),
+ (float)(scale * 1.4426950408889634), (float)scale);
+ C10_CUDA_KERNEL_LAUNCH_CHECK();
+ // raw fp32 dQ accumulator, NOT yet multiplied by the softmax scale
+ return {dqacc, dk, dv};
+}
+
+std::vector<torch::Tensor> bwd(torch::Tensor dout, torch::Tensor q, torch::Tensor k,
+ torch::Tensor v, torch::Tensor o, torch::Tensor lse,
+ torch::Tensor cu, torch::Tensor tcum, torch::Tensor perm, double scale) {
+ check_qkv(q, "q"); check_qkv(k, "k"); check_qkv(v, "v");
+ check_meta(q, k, v, cu, tcum);
+ dout = dout.contiguous();
+ check_qkv(dout, "dout");
+ TORCH_CHECK(dout.sizes() == q.sizes() && o.sizes() == q.sizes() && o.stride(2) == 1 &&
+ o.stride(1) == HD && dout.device() == q.device() && o.device() == q.device(),
+ "dout and o must match q");
+ TORCH_CHECK(lse.scalar_type() == at::kFloat && lse.is_contiguous() && lse.dim() == 2 &&
+ lse.size(0) == q.size(1) && lse.size(1) == q.size(0), "lse must be (H, N) fp32");
+ const int N = q.size(0), H = q.size(1), ndocs = cu.numel() - 1;
+ // D_i = rowsum(dO * O), fp32, laid out (H, N) like lse.
+ auto stream = at::cuda::getCurrentCUDAStream();
+ auto Dv = torch::empty({H, N}, q.options().dtype(torch::kFloat));
+ bwd_pre_kernel<<<(N * H * 32 + 255) / 256, 256, 0, stream>>>(
+ (const half*)dout.data_ptr(), (const half*)o.data_ptr(), Dv.data_ptr<float>(), N, H,
+ dout.stride(0), o.stride(0));
+ auto dqacc = torch::zeros({N, H, HD}, q.options().dtype(torch::kFloat));
+ auto dk = torch::empty({N, H, HD}, q.options());
+ auto dv = torch::empty({N, H, HD}, q.options());
+ // 1-D grid, heads interleaved (b = blockIdx.x / H); >= H * sum of ceil(len/64)
+ dim3 grid(((N + TL - 1) / TL + ndocs) * H);
+ bwd_kernel<<<grid, NT, 0, stream>>>(
+ (const half*)q.data_ptr(), (const half*)k.data_ptr(), (const half*)v.data_ptr(),
+ (const half*)dout.data_ptr(), lse.data_ptr<float>(), Dv.data_ptr<float>(),
+ dqacc.data_ptr<float>(), (half*)dk.data_ptr(), (half*)dv.data_ptr(),
+ cu.data_ptr<int>(), tcum.data_ptr<int>(), perm.data_ptr<int>(), ndocs, N, H, q.stride(0), k.stride(0),
+ v.stride(0), dout.stride(0), dk.stride(0), dv.stride(0),
+ (float)(scale * 1.4426950408889634), (float)scale);
+ C10_CUDA_KERNEL_LAUNCH_CHECK();
+ auto dq = torch::empty({N, H, HD}, q.options());
+ long n4 = (long)N * H * HD / 4;
+ bwd_post_kernel<<<(n4 + 255) / 256, 256, 0, stream>>>(
+ reinterpret_cast<const float4*>(dqacc.data_ptr<float>()),
+ reinterpret_cast<uint2*>(dq.data_ptr()), n4, (float)scale);
+ C10_CUDA_KERNEL_LAUNCH_CHECK();
+ return {dq, dk, dv};
+}
+
+PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
+ m.def("fwd", &fwd);
+ m.def("bwd", &bwd);
+ m.def("bwd_raw", &bwd_raw);
+}
diff --git a/volta_kernels/canon_rope.py b/volta_kernels/canon_rope.py
new file mode 100644
index 0000000..08b6d8c
--- /dev/null
+++ b/volta_kernels/canon_rope.py
@@ -0,0 +1,171 @@
+"""Fused Canon + RoPE for the Volta path: one Triton pass forward, one backward.
+
+What it replaces, inside the compiled _pre_attn region (Inductor's choices):
+ forward constant_pad_nd x2 materialise zero-padded fp32 copies of qkv
+ add_mul the 3-tap combine, reading those copies
+ to_copy_cat RoPE on q and k, cast to fp16
+ backward slice_backward / pad backward / a 1536x3 weight-grad reduction,
+ RoPE backward, as separate passes over (N, 1536)
+
+Semantics are the region's, operation for operation:
+ w = fp16(canon_w) (the region casts it to qkv's dtype)
+ y[n] = x[n] + x[n]*w0 + (x[n-1]*m1[n])*w1 + (x[n-2]*m2[n])*w2 fp32
+ q, k = rope(y) in fp32, then fp16; v = fp16(y)
+m1/m2 are the per-token document masks (0 where t-1 / t-2 is in another
+document) and the shifts never cross a packed row. The backward is the exact
+transpose; the weight gradient is reduced in fp32 and rounded to fp16 before
+widening, as the gradient of the region's fp16 cast of canon_w is.
+"""
+from __future__ import annotations
+
+import torch
+import triton
+import triton.language as tl
+
+C = 1536 # q | k | v channels
+HALF = 32 # RoPE pairs (d, d+32) within a 64-wide head
+
+
+@triton.jit
+def _canon_half(qkv, w, rows, ch, mm1, mm2, ok1, ok2):
+ x0 = tl.load(qkv + rows).to(tl.float32)
+ x1 = tl.load(qkv + rows - 1536, mask=ok1, other=0.0).to(tl.float32)
+ x2 = tl.load(qkv + rows - 3072, mask=ok2, other=0.0).to(tl.float32)
+ w0 = tl.load(w + ch * 3).to(tl.float16).to(tl.float32)[None, :]
+ w1 = tl.load(w + ch * 3 + 1).to(tl.float16).to(tl.float32)[None, :]
+ w2 = tl.load(w + ch * 3 + 2).to(tl.float16).to(tl.float32)[None, :]
+ return x0 + x0 * w0 + (x1 * mm1) * w1 + (x2 * mm2) * w2
+
+
+@triton.jit
+def _fwd(qkv, w, m1, m2, cos, sin, out, T, GROUP: tl.constexpr,
+ TB: tl.constexpr, HALF: tl.constexpr):
+ """TB tokens x one 64-wide head of one group (0 q, 1 k, 2 v)."""
+ pid_n = tl.program_id(0)
+ h = tl.program_id(1)
+ n = pid_n * TB + tl.arange(0, TB)
+ d = tl.arange(0, HALF)
+ t = n % T
+ ca = GROUP * 512 + h * 64 + d
+ cb = ca + HALF
+ ok1 = (t >= 1)[:, None]
+ ok2 = (t >= 2)[:, None]
+ mm1 = tl.load(m1 + n)[:, None]
+ mm2 = tl.load(m2 + n)[:, None]
+ ya = _canon_half(qkv, w, n[:, None] * 1536 + ca[None, :], ca, mm1, mm2, ok1, ok2)
+ yb = _canon_half(qkv, w, n[:, None] * 1536 + cb[None, :], cb, mm1, mm2, ok1, ok2)
+ orow = n[:, None] * 512 + h * 64 + d[None, :]
+ if GROUP < 2:
+ cs = tl.load(cos + n[:, None] * HALF + d[None, :])
+ sn = tl.load(sin + n[:, None] * HALF + d[None, :])
+ oa = ya * cs - yb * sn
+ ob = ya * sn + yb * cs
+ else:
+ oa = ya
+ ob = yb
+ tl.store(out + orow, oa.to(tl.float16))
+ tl.store(out + orow + HALF, ob.to(tl.float16))
+
+
+@triton.jit
+def _dy(src, cos, sin, n, t, S: tl.constexpr, d, ocol, T, ROPE: tl.constexpr,
+ HALF: tl.constexpr):
+ """Gradient w.r.t. the Canon output at token n+S, un-rotated; 0 past the row."""
+ nn = n + S
+ ok = ((t + S) < T)[:, None]
+ ga = tl.load(src + nn[:, None] * 512 + ocol, mask=ok, other=0.0).to(tl.float32)
+ gb = tl.load(src + nn[:, None] * 512 + ocol + HALF, mask=ok, other=0.0).to(tl.float32)
+ if ROPE:
+ cs = tl.load(cos + nn[:, None] * HALF + d[None, :], mask=ok, other=0.0)
+ sn = tl.load(sin + nn[:, None] * HALF + d[None, :], mask=ok, other=0.0)
+ return ga * cs + gb * sn, gb * cs - ga * sn
+ return ga, gb
+
+
+@triton.jit
+def _bwd_half(qkv, w, dqkv, dw_part, pid_n, n, ch, dy0, dy1, dy2,
+ mm1, mm2, mm1n, mm2n, p1, p2):
+ w0 = tl.load(w + ch * 3).to(tl.float16).to(tl.float32)[None, :]
+ w1 = tl.load(w + ch * 3 + 1).to(tl.float16).to(tl.float32)[None, :]
+ w2 = tl.load(w + ch * 3 + 2).to(tl.float16).to(tl.float32)[None, :]
+ rows = n[:, None] * 1536 + ch[None, :]
+ # dx[n] = dy[n](1 + w0) + dy[n+1] m1[n+1] w1 + dy[n+2] m2[n+2] w2
+ dx = dy0 + dy0 * w0 + (dy1 * mm1n) * w1 + (dy2 * mm2n) * w2
+ tl.store(dqkv + rows, dx.to(tl.float16))
+ x0 = tl.load(qkv + rows).to(tl.float32)
+ x1 = tl.load(qkv + rows - 1536, mask=p1, other=0.0).to(tl.float32)
+ x2 = tl.load(qkv + rows - 3072, mask=p2, other=0.0).to(tl.float32)
+ base = dw_part + pid_n * (1536 * 3) + ch * 3
+ tl.store(base, tl.sum(dy0 * x0, 0))
+ tl.store(base + 1, tl.sum(dy0 * (x1 * mm1), 0))
+ tl.store(base + 2, tl.sum(dy0 * (x2 * mm2), 0))
+
+
+@triton.jit
+def _bwd(qkv, w, m1, m2, cos, sin, src, dqkv, dw_part, T, GROUP: tl.constexpr,
+ TB: tl.constexpr, HALF: tl.constexpr):
+ pid_n = tl.program_id(0)
+ h = tl.program_id(1)
+ n = pid_n * TB + tl.arange(0, TB)
+ d = tl.arange(0, HALF)
+ t = n % T
+ ca = GROUP * 512 + h * 64 + d
+ cb = ca + HALF
+ ocol = h * 64 + d[None, :]
+ ROPE: tl.constexpr = GROUP < 2
+ dya0, dyb0 = _dy(src, cos, sin, n, t, 0, d, ocol, T, ROPE, HALF)
+ dya1, dyb1 = _dy(src, cos, sin, n, t, 1, d, ocol, T, ROPE, HALF)
+ dya2, dyb2 = _dy(src, cos, sin, n, t, 2, d, ocol, T, ROPE, HALF)
+ mm1n = tl.load(m1 + n + 1, mask=(t + 1) < T, other=0.0)[:, None]
+ mm2n = tl.load(m2 + n + 2, mask=(t + 2) < T, other=0.0)[:, None]
+ mm1 = tl.load(m1 + n)[:, None]
+ mm2 = tl.load(m2 + n)[:, None]
+ p1 = (t >= 1)[:, None]
+ p2 = (t >= 2)[:, None]
+ _bwd_half(qkv, w, dqkv, dw_part, pid_n, n, ca, dya0, dya1, dya2, mm1, mm2, mm1n, mm2n, p1, p2)
+ _bwd_half(qkv, w, dqkv, dw_part, pid_n, n, cb, dyb0, dyb1, dyb2, mm1, mm2, mm1n, mm2n, p1, p2)
+
+
+TB = 32
+
+
+class CanonRope(torch.autograd.Function):
+ @staticmethod
+ def forward(ctx, qkv, canon_w, m1, m2, cos, sin):
+ B, T, Cc = qkv.shape
+ assert Cc == C and qkv.dtype == torch.float16 and qkv.is_contiguous()
+ N = B * T
+ assert N % TB == 0
+ q = torch.empty((B, T, 8, 64), device=qkv.device, dtype=torch.float16)
+ k = torch.empty_like(q)
+ v = torch.empty_like(q)
+ m1f = m1.reshape(N).contiguous()
+ m2f = m2.reshape(N).contiguous()
+ cosf = cos.reshape(N, HALF).contiguous()
+ sinf = sin.reshape(N, HALF).contiguous()
+ for g_, out in enumerate((q, k, v)):
+ _fwd[(N // TB, 8)](qkv, canon_w, m1f, m2f, cosf, sinf, out, T, GROUP=g_,
+ TB=TB, HALF=HALF, num_warps=4)
+ ctx.save_for_backward(qkv, canon_w, m1f, m2f, cosf, sinf)
+ ctx.T = T
+ return q, k, v
+
+ @staticmethod
+ def backward(ctx, gq, gk, gv):
+ qkv, canon_w, m1f, m2f, cosf, sinf = ctx.saved_tensors
+ B, T, _ = qkv.shape
+ N = B * T
+ gq, gk, gv = (g.to(torch.float16).contiguous() for g in (gq, gk, gv))
+ dqkv = torch.empty_like(qkv)
+ part = torch.empty((N // TB, C, 3), device=qkv.device, dtype=torch.float32)
+ for g_, src in enumerate((gq, gk, gv)):
+ _bwd[(N // TB, 8)](qkv, canon_w, m1f, m2f, cosf, sinf, src, dqkv, part, T,
+ GROUP=g_, TB=TB, HALF=HALF, num_warps=4)
+ # fp32 reduction, then the fp16 rounding the region's cast of canon_w implies
+ dw = part.sum(0).to(torch.float16).to(canon_w.dtype)
+ return dqkv, dw, None, None, None, None
+
+
+def canon_rope(qkv, canon_w, m1, m2, cos, sin):
+ """qkv (B,T,1536) fp16 -> q, k, v (B,T,8,64) fp16, Canon + RoPE applied."""
+ return CanonRope.apply(qkv, canon_w, m1, m2, cos, sin)
diff --git a/volta_kernels/canon_rope_bw.py b/volta_kernels/canon_rope_bw.py
new file mode 100644
index 0000000..172de86
--- /dev/null
+++ b/volta_kernels/canon_rope_bw.py
@@ -0,0 +1,327 @@
+"""Canon + RoPE in CUDA, bandwidth rewrite (swarm agent canon_bw, 2026-09-27).
+
+Same math, same rounding as volta_kernels/canon_rope_cuda.py (q/k/v and dqkv
+bitwise identical; Canon weight-gradient partials bitwise identical, so the
+weight gradient is too when reduced the same way).
+
+Why the old kernels sit at ~65% of DRAM bandwidth (V100, 72 SMs):
+ fwd 88 regs x 192 thr -> 3 CTAs/SM (18 warps), grid 512 = 2.37 waves
+ (last wave 37% full), one 16-B load in flight per thread.
+ bwd 162 regs x 192 thr -> 2 CTAs/SM (12 warps), grid 256 = 1.78 waves.
+What changes here:
+ * CPT channels per thread (default 4 instead of 8) and one q/k/v group per
+ CTA (g = blockIdx.x % 3, so the q, k and v CTAs of one token chunk run
+ side by side and share cos/sin and halo rows through L2): ~half the
+ registers, 128-thread CTAs, 3x more CTAs -> high occupancy, small tail.
+ * U-deep register prefetch of the streamed rows (qkv / gradients / masks).
+ * bwd keeps the 128-token chunk and the backward walk order, so the Canon
+ weight-gradient partials are the same fp32 sums in the same order.
+"""
+import os, torch
+from torch.utils.cpp_extension import load_inline
+
+SRC = r"""
+#include <torch/extension.h>
+#include <ATen/cuda/CUDAContext.h>
+#include <cuda_fp16.h>
+
+template <int CPT> struct HV; // CPT fp16 values in one vector load
+template <> struct HV<8> { typedef uint4 T; };
+template <> struct HV<4> { typedef uint2 T; };
+template <> struct HV<2> { typedef unsigned int T; };
+template <int CPT> struct FV; // CPT fp32 values
+template <> struct FV<8> { typedef struct { float4 a, b; } T; };
+template <> struct FV<4> { typedef struct { float4 a; } T; };
+template <> struct FV<2> { typedef struct { float2 a; } T; };
+
+template <int CPT> __device__ __forceinline__ typename HV<CPT>::T ldh(const half* p) {
+ return *reinterpret_cast<const typename HV<CPT>::T*>(p); }
+template <int CPT> __device__ __forceinline__ void cvh(const typename HV<CPT>::T& u, float* x) {
+ const __half2* h = reinterpret_cast<const __half2*>(&u);
+#pragma unroll
+ for (int i = 0; i < CPT / 2; ++i) { float2 f = __half22float2(h[i]); x[2*i] = f.x; x[2*i+1] = f.y; }
+}
+template <int CPT> __device__ __forceinline__ void sth(half* p, const float* o) {
+ typename HV<CPT>::T u; __half2* h = reinterpret_cast<__half2*>(&u);
+#pragma unroll
+ for (int i = 0; i < CPT / 2; ++i) h[i] = __floats2half2_rn(o[2*i], o[2*i+1]);
+ *reinterpret_cast<typename HV<CPT>::T*>(p) = u;
+}
+template <int CPT> __device__ __forceinline__ void ldcs(const float* p, float* x) {
+ if constexpr (CPT == 2) { float2 f = *reinterpret_cast<const float2*>(p); x[0] = f.x; x[1] = f.y; }
+ else {
+ const float4* q = reinterpret_cast<const float4*>(p);
+#pragma unroll
+ for (int j = 0; j < CPT / 4; ++j) { float4 f = q[j]; x[4*j] = f.x; x[4*j+1] = f.y; x[4*j+2] = f.z; x[4*j+3] = f.w; }
+ }
+}
+
+// ---------------------------------------------------------------- forward
+// grid: (N/FCH) * 3 CTAs of TPB = 512/CPT threads; CTA b handles group g = b % 3
+// (q, k, v) of token chunk b / 3.
+template <int CPT, int FCH, int U, int MINB>
+__global__ void __launch_bounds__(512 / CPT, MINB) crbw_fwd(
+ const half* __restrict__ qkv, const float* __restrict__ w, const float* __restrict__ m1,
+ const float* __restrict__ m2, const float* __restrict__ cs, const float* __restrict__ sn,
+ half* __restrict__ q, half* __restrict__ k, half* __restrict__ v, int T) {
+ constexpr int PX = 32 / CPT; // lane distance of the RoPE partner (d +/- 32)
+ const int g = blockIdx.x % 3;
+ const int col = CPT * threadIdx.x, ch = g * 512 + col, d0 = col & 63;
+ const long n0 = (long)(blockIdx.x / 3) * FCH;
+ const int t0 = n0 % T;
+ float w0[CPT], w1[CPT], w2[CPT];
+#pragma unroll
+ for (int i = 0; i < CPT; ++i) { // the region casts canon_w to fp16
+ w0[i] = __half2float(__float2half(w[(ch + i) * 3 + 0]));
+ w1[i] = __half2float(__float2half(w[(ch + i) * 3 + 1]));
+ w2[i] = __half2float(__float2half(w[(ch + i) * 3 + 2]));
+ }
+ const half* src = qkv + ch;
+ float x1[CPT], x2[CPT], x0[CPT];
+#pragma unroll
+ for (int i = 0; i < CPT; ++i) x1[i] = x2[i] = 0.f;
+ if (t0 >= 1) cvh<CPT>(ldh<CPT>(src + (n0 - 1) * 1536), x1);
+ if (t0 >= 2) cvh<CPT>(ldh<CPT>(src + (n0 - 2) * 1536), x2);
+ half* out = (g == 0 ? q : (g == 1 ? k : v)) + col;
+ const int d = d0 & 31;
+ typename HV<CPT>::T raw[U]; float A1[U], A2[U];
+#pragma unroll
+ for (int u = 0; u < U; ++u) { raw[u] = ldh<CPT>(src + (n0 + u) * 1536); A1[u] = m1[n0 + u]; A2[u] = m2[n0 + u]; }
+#pragma unroll 1
+ for (int s0 = 0; s0 < FCH; s0 += U) {
+ typename HV<CPT>::T cur[U]; float C1[U], C2[U];
+#pragma unroll
+ for (int u = 0; u < U; ++u) { cur[u] = raw[u]; C1[u] = A1[u]; C2[u] = A2[u]; }
+ if (s0 + U < FCH) {
+#pragma unroll
+ for (int u = 0; u < U; ++u) {
+ const long n = n0 + s0 + U + u;
+ raw[u] = ldh<CPT>(src + n * 1536); A1[u] = m1[n]; A2[u] = m2[n];
+ }
+ }
+#pragma unroll
+ for (int u = 0; u < U; ++u) {
+ const long n = n0 + s0 + u;
+ cvh<CPT>(cur[u], x0);
+ const float a1 = C1[u], a2 = C2[u];
+ float y[CPT];
+#pragma unroll
+ for (int i = 0; i < CPT; ++i)
+ y[i] = x0[i] + x0[i] * w0[i] + (x1[i] * a1) * w1[i] + (x2[i] * a2) * w2[i];
+ float o[CPT];
+ if (g < 2) {
+ float c[CPT], sv[CPT];
+ ldcs<CPT>(cs + n * 32 + d, c); ldcs<CPT>(sn + n * 32 + d, sv);
+#pragma unroll
+ for (int i = 0; i < CPT; ++i) {
+ float p = __shfl_xor_sync(0xffffffff, y[i], PX);
+ o[i] = d0 < 32 ? fmaf(y[i], c[i], -(p * sv[i])) : fmaf(y[i], c[i], p * sv[i]);
+ }
+ } else {
+#pragma unroll
+ for (int i = 0; i < CPT; ++i) o[i] = y[i];
+ }
+ sth<CPT>(out + n * 512, o);
+#pragma unroll
+ for (int i = 0; i < CPT; ++i) { x2[i] = x1[i]; x1[i] = x0[i]; }
+ }
+ }
+}
+
+// ---------------------------------------------------------------- backward
+// Q32: the q-gradient is the attention backward's raw fp32 dQ accumulator, times qscale.
+// CH must stay 128 for the weight-gradient partials to be the old kernel's sums.
+template <int CPT, int CH, int U, bool Q32, int MINB>
+__global__ void __launch_bounds__(512 / CPT, MINB) crbw_bwd(
+ const half* __restrict__ qkv, const float* __restrict__ w, const float* __restrict__ m1,
+ const float* __restrict__ m2, const float* __restrict__ cs, const float* __restrict__ sn,
+ const void* __restrict__ gq_, const half* __restrict__ gk, const half* __restrict__ gv,
+ half* __restrict__ dqkv, float* __restrict__ part, int T, float qscale) {
+ constexpr int PX = 32 / CPT;
+ const int g = blockIdx.x % 3;
+ const int col = CPT * threadIdx.x, ch = g * 512 + col, d0 = col & 63, d = d0 & 31;
+ const long chunk = blockIdx.x / 3, n0 = chunk * CH;
+ const int t0 = n0 % T;
+ const bool f32 = Q32 && g == 0;
+ const half* G = (g == 0 ? reinterpret_cast<const half*>(gq_) : (g == 1 ? gk : gv)) + col;
+ const float* G32 = reinterpret_cast<const float*>(gq_) + col;
+ float w0[CPT], w1[CPT], w2[CPT];
+#pragma unroll
+ for (int i = 0; i < CPT; ++i) {
+ w0[i] = __half2float(__float2half(w[(ch + i) * 3 + 0]));
+ w1[i] = __half2float(__float2half(w[(ch + i) * 3 + 1]));
+ w2[i] = __half2float(__float2half(w[(ch + i) * 3 + 2]));
+ }
+ // raw gradient row load (fp32 path: CPT floats; fp16 path: CPT halves in .a's bytes)
+ typedef typename FV<CPT>::T F;
+ auto ldg_ = [&](long n, F& r) {
+ if (f32) r = *reinterpret_cast<const F*>(G32 + n * 512);
+ else *reinterpret_cast<typename HV<CPT>::T*>(&r) = ldh<CPT>(G + n * 512);
+ };
+ // dy(n) from a raw gradient row: un-rotated gradient w.r.t. the Canon output.
+ auto dy = [&](long n, const F& r, float* out) {
+ float gg[CPT];
+ if (f32) {
+ const float* f = reinterpret_cast<const float*>(&r);
+#pragma unroll
+ for (int i = 0; i < CPT; ++i) gg[i] = f[i] * qscale;
+ } else {
+ cvh<CPT>(*reinterpret_cast<const typename HV<CPT>::T*>(&r), gg);
+ }
+ if (g < 2) {
+ float c[CPT], sv[CPT];
+ ldcs<CPT>(cs + n * 32 + d, c); ldcs<CPT>(sn + n * 32 + d, sv);
+#pragma unroll
+ for (int i = 0; i < CPT; ++i) {
+ float p = __shfl_xor_sync(0xffffffff, gg[i], PX);
+ float ga = d0 < 32 ? gg[i] : p, gb = d0 < 32 ? p : gg[i];
+ out[i] = d0 < 32 ? fmaf(ga, c[i], gb * sv[i]) : fmaf(gb, c[i], -(ga * sv[i]));
+ }
+ } else {
+#pragma unroll
+ for (int i = 0; i < CPT; ++i) out[i] = gg[i];
+ }
+ };
+ const half* X = qkv + ch;
+ const long nl = n0 + CH - 1;
+ const int tl = t0 + CH - 1;
+ float d0v[CPT], d1v[CPT], d2v[CPT], x0[CPT], xm1[CPT], xm2[CPT];
+ float mm1n = 0.f, mm2n = 0.f, m2p1 = 0.f;
+#pragma unroll
+ for (int i = 0; i < CPT; ++i) d1v[i] = d2v[i] = 0.f;
+ { F r; float tmp[CPT];
+ long a = (tl + 1 < T) ? nl + 1 : nl; ldg_(a, r); dy(a, r, tmp);
+ if (tl + 1 < T) { mm1n = m1[nl + 1]; m2p1 = m2[nl + 1];
+#pragma unroll
+ for (int i = 0; i < CPT; ++i) d1v[i] = tmp[i]; }
+ long b = (tl + 2 < T) ? nl + 2 : nl; ldg_(b, r); dy(b, r, tmp);
+ if (tl + 2 < T) { mm2n = m2[nl + 2];
+#pragma unroll
+ for (int i = 0; i < CPT; ++i) d2v[i] = tmp[i]; } }
+ cvh<CPT>(ldh<CPT>(X + nl * 1536), x0);
+ cvh<CPT>(ldh<CPT>(X + (nl - 1) * 1536), xm1);
+ cvh<CPT>(ldh<CPT>(X + (nl - 2) * 1536), xm2);
+ float a0[CPT], a1[CPT], a2[CPT];
+#pragma unroll
+ for (int i = 0; i < CPT; ++i) a0[i] = a1[i] = a2[i] = 0.f;
+ // prefetch ring: for step s (token n = n0+s) we need grad row n, masks n, x row n-3.
+ F rg[U]; typename HV<CPT>::T rx[U]; float R1[U], R2[U];
+ auto fetch = [&](int s, int u) {
+ const long n = n0 + s; const int t = t0 + s;
+ ldg_(n, rg[u]); R1[u] = m1[n]; R2[u] = m2[n];
+ if (t - 3 >= 0) rx[u] = ldh<CPT>(X + (n - 3) * 1536);
+ };
+#pragma unroll
+ for (int u = 0; u < U; ++u) fetch(CH - 1 - u, u);
+#pragma unroll 1
+ for (int sb = CH - 1; sb >= 0; sb -= U) {
+ F cg[U]; typename HV<CPT>::T cx[U]; float C1[U], C2[U];
+#pragma unroll
+ for (int u = 0; u < U; ++u) { cg[u] = rg[u]; cx[u] = rx[u]; C1[u] = R1[u]; C2[u] = R2[u]; }
+ if (sb - U >= 0) {
+#pragma unroll
+ for (int u = 0; u < U; ++u) fetch(sb - U - u, u);
+ }
+#pragma unroll
+ for (int u = 0; u < U; ++u) {
+ const int s = sb - u;
+ const long n = n0 + s; const int t = t0 + s;
+ dy(n, cg[u], d0v);
+ const float m1n = C1[u], m2n = C2[u];
+ float dx[CPT];
+#pragma unroll
+ for (int i = 0; i < CPT; ++i) {
+ dx[i] = d0v[i] + d0v[i] * w0[i] + (d1v[i] * mm1n) * w1[i] + (d2v[i] * mm2n) * w2[i];
+ a0[i] += d0v[i] * x0[i];
+ a1[i] += d0v[i] * (xm1[i] * m1n);
+ a2[i] += d0v[i] * (xm2[i] * m2n);
+ }
+ sth<CPT>(dqkv + n * 1536 + ch, dx);
+ mm2n = m2p1; m2p1 = m2n; mm1n = m1n;
+#pragma unroll
+ for (int i = 0; i < CPT; ++i) { d2v[i] = d1v[i]; d1v[i] = d0v[i]; x0[i] = xm1[i]; xm1[i] = xm2[i]; }
+ if (s > 0) {
+ if (t - 3 >= 0) cvh<CPT>(cx[u], xm2);
+ else {
+#pragma unroll
+ for (int i = 0; i < CPT; ++i) xm2[i] = 0.f; }
+ if (t - 1 < 1) {
+#pragma unroll
+ for (int i = 0; i < CPT; ++i) xm1[i] = 0.f; }
+ }
+ }
+ }
+ float* pp = part + chunk * 1536 * 3 + ch * 3;
+#pragma unroll
+ for (int i = 0; i < CPT; ++i) { pp[3*i] = a0[i]; pp[3*i+1] = a1[i]; pp[3*i+2] = a2[i]; }
+}
+
+// Deterministic Canon weight-gradient reduction: sum over chunks in fp64 (more
+// precise than the fp32 torch sum), one thread per (channel, tap).
+__global__ void crbw_wsum(const float* __restrict__ part, float* __restrict__ dw, int nchunk) {
+ const int j = blockIdx.x * blockDim.x + threadIdx.x;
+ if (j >= 1536 * 3) return;
+ double s = 0.0;
+ for (int c = 0; c < nchunk; ++c) s += (double)part[(long)c * 4608 + j];
+ dw[j] = (float)s;
+}
+
+#define FWD_CFG(CPT, FCH, U, MB) \
+ if (cpt == CPT && fch == FCH && u == U && minb == MB) { \
+ TORCH_CHECK(T % FCH == 0); \
+ crbw_fwd<CPT, FCH, U, MB><<<(N / FCH) * 3, 512 / CPT, 0, st>>>(P(qkv), w.data_ptr<float>(), m1.data_ptr<float>(), \
+ m2.data_ptr<float>(), cs.data_ptr<float>(), sn.data_ptr<float>(), P(q), P(k), P(v), T); \
+ C10_CUDA_KERNEL_LAUNCH_CHECK(); return {q, k, v}; }
+#define P(t) reinterpret_cast<half*>(t.data_ptr())
+
+std::vector<torch::Tensor> fwd(torch::Tensor qkv, torch::Tensor w, torch::Tensor m1, torch::Tensor m2,
+ torch::Tensor cs, torch::Tensor sn, int64_t cpt, int64_t fch, int64_t u, int64_t minb) {
+ const int B = qkv.size(0), T = qkv.size(1); const long N = (long)B * T;
+ auto q = torch::empty({B, T, 8, 64}, qkv.options()), k = torch::empty_like(q), v = torch::empty_like(q);
+ auto st = at::cuda::getCurrentCUDAStream();
+ FWD_CFG(4, 64, 4, 1) FWD_CFG(4, 64, 4, 6) FWD_CFG(4, 64, 4, 8) FWD_CFG(4, 64, 2, 8) FWD_CFG(2, 64, 4, 4)
+ FWD_CFG(2, 64, 2, 4) FWD_CFG(4, 32, 4, 6) FWD_CFG(4, 64, 1, 8) FWD_CFG(2, 64, 4, 1)
+ TORCH_CHECK(false, "unsupported fwd config");
+}
+
+#define BWD_CFG(CPT, U, MB) \
+ if (cpt == CPT && u == U && minb == MB) { \
+ if (q32) crbw_bwd<CPT, 128, U, true, MB><<<(N / 128) * 3, 512 / CPT, 0, st>>>(P(qkv), w.data_ptr<float>(), \
+ m1.data_ptr<float>(), m2.data_ptr<float>(), cs.data_ptr<float>(), sn.data_ptr<float>(), gq.data_ptr(), \
+ P(gk), P(gv), P(dqkv), part.data_ptr<float>(), T, (float)qscale); \
+ else crbw_bwd<CPT, 128, U, false, MB><<<(N / 128) * 3, 512 / CPT, 0, st>>>(P(qkv), w.data_ptr<float>(), \
+ m1.data_ptr<float>(), m2.data_ptr<float>(), cs.data_ptr<float>(), sn.data_ptr<float>(), gq.data_ptr(), \
+ P(gk), P(gv), P(dqkv), part.data_ptr<float>(), T, (float)qscale); \
+ C10_CUDA_KERNEL_LAUNCH_CHECK(); ok = true; }
+
+// q32: gq is the (B, T, 8, 64) fp32 raw dQ accumulator; qscale folds in the softmax scale.
+// wsum: 0 -> return part (N/128, 1536, 3) like the old op; 1 -> fused fp64 reduction, returns dw (1536, 3).
+std::vector<torch::Tensor> bwd(torch::Tensor qkv, torch::Tensor w, torch::Tensor m1, torch::Tensor m2,
+ torch::Tensor cs, torch::Tensor sn, torch::Tensor gq, torch::Tensor gk,
+ torch::Tensor gv, double qscale, bool q32, int64_t cpt, int64_t u, int64_t minb, bool wsum) {
+ const int B = qkv.size(0), T = qkv.size(1); const long N = (long)B * T;
+ TORCH_CHECK(T % 128 == 0, "T must be a multiple of 128");
+ TORCH_CHECK(gq.is_contiguous() && gk.is_contiguous() && gv.is_contiguous());
+ TORCH_CHECK(!q32 || gq.scalar_type() == at::kFloat, "gq32 must be fp32");
+ auto dqkv = torch::empty_like(qkv);
+ auto part = torch::empty({N / 128, 1536, 3}, qkv.options().dtype(torch::kFloat));
+ auto st = at::cuda::getCurrentCUDAStream();
+ bool ok = false;
+ BWD_CFG(4, 2, 1) BWD_CFG(4, 2, 5) BWD_CFG(4, 2, 6) BWD_CFG(4, 1, 6) BWD_CFG(4, 1, 1) BWD_CFG(2, 2, 3)
+ BWD_CFG(2, 1, 3) BWD_CFG(2, 2, 1) BWD_CFG(4, 4, 5)
+ TORCH_CHECK(ok, "unsupported bwd config");
+ if (!wsum) return {dqkv, part};
+ auto dw = torch::empty({1536, 3}, part.options());
+ crbw_wsum<<<(4608 + 127) / 128, 128, 0, st>>>(part.data_ptr<float>(), dw.data_ptr<float>(), (int)(N / 128));
+ C10_CUDA_KERNEL_LAUNCH_CHECK();
+ return {dqkv, dw};
+}
+"""
+_here = os.path.dirname(os.path.abspath(__file__))
+_build = os.path.join(_here, "build_canon_bw"); os.makedirs(_build, exist_ok=True)
+os.environ.setdefault("TORCH_CUDA_ARCH_LIST", "7.0")
+ext = load_inline("boris_volta_canon_bw", cpp_sources=
+ "std::vector<torch::Tensor> fwd(torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, int64_t, int64_t, int64_t, int64_t);"
+ "std::vector<torch::Tensor> bwd(torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, double, bool, int64_t, int64_t, int64_t, bool);",
+ cuda_sources=SRC, functions=["fwd", "bwd"], extra_cuda_cflags=["-O3", "-arch=sm_70"],
+ build_directory=_build)
diff --git a/volta_kernels/canon_rope_cuda.py b/volta_kernels/canon_rope_cuda.py
new file mode 100644
index 0000000..e632212
--- /dev/null
+++ b/volta_kernels/canon_rope_cuda.py
@@ -0,0 +1,266 @@
+"""Canon + RoPE in CUDA: a sliding window down the sequence (drop-in for canon_rope.py).
+
+Written by a second Claude Code reviewer on 2026-09-27; adopted after review.
+Each thread owns 8 channels and walks 64 (fwd) / 128 (bwd, backwards) tokens,
+keeping the two previous tokens in registers instead of re-reading neighbours
+as the Triton version does, with 16-byte loads and stores throughout.
+Measured at full size (31 layers, mb 16, idle V100): fwd 0.61 -> 0.32 ms,
+bwd 1.20 -> 0.57 ms, whole micro-step +4.1%. q, k, v, dqkv and the loss are
+bitwise-identical to the Triton kernel (its RoPE rounding order is copied:
+fmaf(y, c, -(p*s))); the Canon weight gradient differs only in fp32 summation
+order (same error vs fp64). Requires T % 128 == 0 (2048 and 8192 both are).
+"""
+import os, torch
+from torch.utils.cpp_extension import load_inline
+SRC = r"""
+#include <torch/extension.h>
+#include <ATen/cuda/CUDAContext.h>
+#include <cuda_fp16.h>
+constexpr int FCH = 64; // tokens per block (sliding window down the sequence)
+__global__ void __launch_bounds__(192) canon_rope_fwd(
+ const half* __restrict__ qkv, const float* __restrict__ w, const float* __restrict__ m1,
+ const float* __restrict__ m2, const float* __restrict__ cs, const float* __restrict__ sn,
+ half* __restrict__ q, half* __restrict__ k, half* __restrict__ v, int T) {
+ const int c8 = threadIdx.x, ch = 8 * c8, g = ch >> 9, col = ch & 511, d0 = col & 63;
+ const long n0 = (long)blockIdx.x * FCH;
+ const int t0 = n0 % T;
+ float w0[8], w1[8], w2[8];
+#pragma unroll
+ for (int i = 0; i < 8; ++i) { // the region casts canon_w to fp16
+ w0[i] = __half2float(__float2half(w[(ch + i) * 3 + 0]));
+ w1[i] = __half2float(__float2half(w[(ch + i) * 3 + 1]));
+ w2[i] = __half2float(__float2half(w[(ch + i) * 3 + 2]));
+ }
+ auto ld = [&](long n, float* x) {
+ uint4 u = *reinterpret_cast<const uint4*>(qkv + n * 1536 + ch);
+ const __half2* h = reinterpret_cast<const __half2*>(&u);
+#pragma unroll
+ for (int i = 0; i < 4; ++i) { float2 f = __half22float2(h[i]); x[2*i] = f.x; x[2*i+1] = f.y; }
+ };
+ float x1[8], x2[8], x0[8];
+#pragma unroll
+ for (int i = 0; i < 8; ++i) x1[i] = x2[i] = 0.f;
+ if (t0 >= 1) ld(n0 - 1, x1);
+ if (t0 >= 2) ld(n0 - 2, x2);
+ half* out = g == 0 ? q : (g == 1 ? k : v);
+#pragma unroll 8
+ for (int s = 0; s < FCH; ++s) {
+ const long n = n0 + s;
+ ld(n, x0);
+ const float a1 = m1[n], a2 = m2[n];
+ float y[8];
+#pragma unroll
+ for (int i = 0; i < 8; ++i)
+ y[i] = x0[i] + x0[i] * w0[i] + (x1[i] * a1) * w1[i] + (x2[i] * a2) * w2[i];
+ float o[8];
+ if (g < 2) {
+ const int d = d0 & 31;
+ const float4* cp = reinterpret_cast<const float4*>(cs + n * 32 + d);
+ const float4* sp = reinterpret_cast<const float4*>(sn + n * 32 + d);
+ float4 c0 = cp[0], c1 = cp[1], s0 = sp[0], s1 = sp[1];
+ float c[8] = {c0.x, c0.y, c0.z, c0.w, c1.x, c1.y, c1.z, c1.w};
+ float sv[8] = {s0.x, s0.y, s0.z, s0.w, s1.x, s1.y, s1.z, s1.w};
+#pragma unroll
+ for (int i = 0; i < 8; ++i) {
+ float p = __shfl_xor_sync(0xffffffff, y[i], 4); // partner channel d +/- 32
+ o[i] = d0 < 32 ? fmaf(y[i], c[i], -(p * sv[i])) : fmaf(y[i], c[i], p * sv[i]);
+ }
+ } else {
+#pragma unroll
+ for (int i = 0; i < 8; ++i) o[i] = y[i];
+ }
+ uint4 u;
+ __half2* h = reinterpret_cast<__half2*>(&u);
+#pragma unroll
+ for (int i = 0; i < 4; ++i) h[i] = __floats2half2_rn(o[2*i], o[2*i+1]);
+ *reinterpret_cast<uint4*>(out + n * 512 + col) = u;
+#pragma unroll
+ for (int i = 0; i < 8; ++i) { x2[i] = x1[i]; x1[i] = x0[i]; }
+ }
+}
+std::vector<torch::Tensor> fwd(torch::Tensor qkv, torch::Tensor w, torch::Tensor m1, torch::Tensor m2,
+ torch::Tensor cs, torch::Tensor sn) {
+ const int B = qkv.size(0), T = qkv.size(1); const long N = (long)B * T;
+ TORCH_CHECK(T % FCH == 0);
+ auto q = torch::empty({B, T, 8, 64}, qkv.options()), k = torch::empty_like(q), v = torch::empty_like(q);
+ canon_rope_fwd<<<N / FCH, 192, 0, at::cuda::getCurrentCUDAStream()>>>(
+ (const half*)qkv.data_ptr(), w.data_ptr<float>(), m1.data_ptr<float>(), m2.data_ptr<float>(),
+ cs.data_ptr<float>(), sn.data_ptr<float>(), (half*)q.data_ptr(), (half*)k.data_ptr(),
+ (half*)v.data_ptr(), T);
+ return {q, k, v};
+}
+
+constexpr int CH = 128;
+__device__ __forceinline__ void ld8(const half* p, float* x) {
+ uint4 u = *reinterpret_cast<const uint4*>(p);
+ const __half2* h = reinterpret_cast<const __half2*>(&u);
+#pragma unroll
+ for (int i = 0; i < 4; ++i) { float2 f = __half22float2(h[i]); x[2*i] = f.x; x[2*i+1] = f.y; }
+}
+// Q32: the q-gradient arrives as the attention backward's raw fp32 dQ
+// accumulator (times qscale) instead of an fp16 tensor, so it is never rounded.
+template <bool Q32>
+__global__ void __launch_bounds__(192) canon_rope_bwd(
+ const half* __restrict__ qkv, const float* __restrict__ w, const float* __restrict__ m1,
+ const float* __restrict__ m2, const float* __restrict__ cs, const float* __restrict__ sn,
+ const void* __restrict__ gq_, const half* __restrict__ gk, const half* __restrict__ gv,
+ half* __restrict__ dqkv, float* __restrict__ part, int T, float qscale) {
+ const half* gq = reinterpret_cast<const half*>(gq_);
+ const float* gq32 = reinterpret_cast<const float*>(gq_);
+ const int c8 = threadIdx.x, ch = 8 * c8, g = ch >> 9, col = ch & 511, d0 = col & 63;
+ const long n0 = (long)blockIdx.x * CH;
+ const int t0 = n0 % T;
+ const half* G = g == 0 ? gq : (g == 1 ? gk : gv);
+ float w0[8], w1[8], w2[8];
+#pragma unroll
+ for (int i = 0; i < 8; ++i) {
+ w0[i] = __half2float(__float2half(w[(ch + i) * 3 + 0]));
+ w1[i] = __half2float(__float2half(w[(ch + i) * 3 + 1]));
+ w2[i] = __half2float(__float2half(w[(ch + i) * 3 + 2]));
+ }
+ // dy(n): gradient w.r.t. the Canon output at token n, un-rotated.
+ auto dy = [&](long n, float* out) {
+ float gg[8];
+ if (Q32 && g == 0) {
+ const float4* p4 = reinterpret_cast<const float4*>(gq32 + n * 512 + col);
+ float4 a = p4[0], b = p4[1];
+ gg[0] = a.x * qscale; gg[1] = a.y * qscale; gg[2] = a.z * qscale; gg[3] = a.w * qscale;
+ gg[4] = b.x * qscale; gg[5] = b.y * qscale; gg[6] = b.z * qscale; gg[7] = b.w * qscale;
+ } else {
+ ld8(G + n * 512 + col, gg);
+ }
+ if (g < 2) {
+ const int d = d0 & 31;
+ const float4* cp = reinterpret_cast<const float4*>(cs + n * 32 + d);
+ const float4* sp = reinterpret_cast<const float4*>(sn + n * 32 + d);
+ float4 c0 = cp[0], c1 = cp[1], s0 = sp[0], s1 = sp[1];
+ float cs_[8] = {c0.x, c0.y, c0.z, c0.w, c1.x, c1.y, c1.z, c1.w};
+ float sn_[8] = {s0.x, s0.y, s0.z, s0.w, s1.x, s1.y, s1.z, s1.w};
+#pragma unroll
+ for (int i = 0; i < 8; ++i) {
+ float p = __shfl_xor_sync(0xffffffff, gg[i], 4);
+ float ga = d0 < 32 ? gg[i] : p, gb = d0 < 32 ? p : gg[i];
+ out[i] = d0 < 32 ? fmaf(ga, cs_[i], gb * sn_[i]) : fmaf(gb, cs_[i], -(ga * sn_[i]));
+ }
+ } else {
+#pragma unroll
+ for (int i = 0; i < 8; ++i) out[i] = gg[i];
+ }
+ };
+ const long nl = n0 + CH - 1; // walk from the chunk's last token backwards
+ const int tl = t0 + CH - 1;
+ float d0v[8], d1v[8], d2v[8], x0[8], xm1[8], xm2[8];
+ float mm1n = 0.f, mm2n = 0.f, m2p1 = 0.f; // m2p1 = m2[n+1]
+#pragma unroll
+ for (int i = 0; i < 8; ++i) d1v[i] = d2v[i] = 0.f;
+ // dy at n+1, n+2 must be computed by every lane (shuffles), masked afterwards.
+ { float tmp[8];
+ long a = (tl + 1 < T) ? nl + 1 : nl; dy(a, tmp);
+ if (tl + 1 < T) { mm1n = m1[nl + 1]; m2p1 = m2[nl + 1];
+#pragma unroll
+ for (int i = 0; i < 8; ++i) d1v[i] = tmp[i]; }
+ long b = (tl + 2 < T) ? nl + 2 : nl; dy(b, tmp);
+ if (tl + 2 < T) { mm2n = m2[nl + 2];
+#pragma unroll
+ for (int i = 0; i < 8; ++i) d2v[i] = tmp[i]; } }
+ ld8(qkv + nl * 1536 + ch, x0);
+ ld8(qkv + (nl - 1) * 1536 + ch, xm1); // tl >= 31 so both exist
+ ld8(qkv + (nl - 2) * 1536 + ch, xm2);
+ float a0[8], a1[8], a2[8];
+#pragma unroll
+ for (int i = 0; i < 8; ++i) a0[i] = a1[i] = a2[i] = 0.f;
+ #pragma unroll 8
+ for (int s = CH - 1; s >= 0; --s) {
+ const long n = n0 + s; const int t = t0 + s;
+ dy(n, d0v);
+ const float m1n = m1[n], m2n = m2[n];
+ float dx[8];
+#pragma unroll
+ for (int i = 0; i < 8; ++i) {
+ dx[i] = d0v[i] + d0v[i] * w0[i] + (d1v[i] * mm1n) * w1[i] + (d2v[i] * mm2n) * w2[i];
+ a0[i] += d0v[i] * x0[i];
+ a1[i] += d0v[i] * (xm1[i] * m1n);
+ a2[i] += d0v[i] * (xm2[i] * m2n);
+ }
+ uint4 u; __half2* h = reinterpret_cast<__half2*>(&u);
+#pragma unroll
+ for (int i = 0; i < 4; ++i) h[i] = __floats2half2_rn(dx[2*i], dx[2*i+1]);
+ *reinterpret_cast<uint4*>(dqkv + n * 1536 + ch) = u;
+ mm2n = m2p1; m2p1 = m2n; mm1n = m1n;
+#pragma unroll
+ for (int i = 0; i < 8; ++i) { d2v[i] = d1v[i]; d1v[i] = d0v[i]; x0[i] = xm1[i]; xm1[i] = xm2[i]; }
+ if (s > 0) {
+ if (t - 3 >= 0) ld8(qkv + (n - 3) * 1536 + ch, xm2);
+ else {
+#pragma unroll
+ for (int i = 0; i < 8; ++i) xm2[i] = 0.f; }
+ if (t - 1 < 1) { // next token (t-1) is t==0: its t-1 tap is outside the row
+#pragma unroll
+ for (int i = 0; i < 8; ++i) xm1[i] = 0.f; }
+ }
+ }
+ float* pp = part + (long)blockIdx.x * 1536 * 3 + ch * 3;
+#pragma unroll
+ for (int i = 0; i < 8; ++i) { pp[3*i] = a0[i]; pp[3*i+1] = a1[i]; pp[3*i+2] = a2[i]; }
+}
+std::vector<torch::Tensor> bwd(torch::Tensor qkv, torch::Tensor w, torch::Tensor m1, torch::Tensor m2,
+ torch::Tensor cs, torch::Tensor sn, torch::Tensor gq, torch::Tensor gk,
+ torch::Tensor gv) {
+ const int B = qkv.size(0), T = qkv.size(1); const long N = (long)B * T;
+ auto dqkv = torch::empty_like(qkv);
+ auto part = torch::empty({N / CH, 1536, 3}, qkv.options().dtype(torch::kFloat));
+ canon_rope_bwd<false><<<N / CH, 192, 0, at::cuda::getCurrentCUDAStream()>>>(
+ (const half*)qkv.data_ptr(), w.data_ptr<float>(), m1.data_ptr<float>(), m2.data_ptr<float>(),
+ cs.data_ptr<float>(), sn.data_ptr<float>(), gq.data_ptr(), (const half*)gk.data_ptr(),
+ (const half*)gv.data_ptr(), (half*)dqkv.data_ptr(), part.data_ptr<float>(), T, 1.f);
+ C10_CUDA_KERNEL_LAUNCH_CHECK();
+ return {dqkv, part};
+}
+// gq32: (B, T, 8, 64) fp32 raw dQ accumulator; qscale folds in the softmax scale.
+std::vector<torch::Tensor> bwd_q32(torch::Tensor qkv, torch::Tensor w, torch::Tensor m1, torch::Tensor m2,
+ torch::Tensor cs, torch::Tensor sn, torch::Tensor gq32, torch::Tensor gk,
+ torch::Tensor gv, double qscale) {
+ TORCH_CHECK(gq32.scalar_type() == at::kFloat && gq32.is_contiguous(), "gq32 must be contiguous fp32");
+ const int B = qkv.size(0), T = qkv.size(1); const long N = (long)B * T;
+ TORCH_CHECK(T % CH == 0, "T must be a multiple of 128");
+ auto dqkv = torch::empty_like(qkv);
+ auto part = torch::empty({N / CH, 1536, 3}, qkv.options().dtype(torch::kFloat));
+ canon_rope_bwd<true><<<N / CH, 192, 0, at::cuda::getCurrentCUDAStream()>>>(
+ (const half*)qkv.data_ptr(), w.data_ptr<float>(), m1.data_ptr<float>(), m2.data_ptr<float>(),
+ cs.data_ptr<float>(), sn.data_ptr<float>(), gq32.data_ptr(), (const half*)gk.data_ptr(),
+ (const half*)gv.data_ptr(), (half*)dqkv.data_ptr(), part.data_ptr<float>(), T, (float)qscale);
+ C10_CUDA_KERNEL_LAUNCH_CHECK();
+ return {dqkv, part};
+}
+"""
+_here = os.path.dirname(os.path.abspath(__file__)); os.makedirs(os.path.join(_here, "build_canon"), exist_ok=True)
+os.environ.setdefault("TORCH_CUDA_ARCH_LIST", "7.0")
+ext = load_inline("boris_volta_canon", cpp_sources=
+ "std::vector<torch::Tensor> fwd(torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor);"
+ "std::vector<torch::Tensor> bwd(torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor);"
+ "std::vector<torch::Tensor> bwd_q32(torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, double);",
+ cuda_sources=SRC, functions=["fwd", "bwd", "bwd_q32"], extra_cuda_cflags=["-O3", "-arch=sm_70"],
+ build_directory=os.path.join(_here, "build_canon"))
+
+class CanonRopeFast(torch.autograd.Function):
+ @staticmethod
+ def forward(ctx, qkv, canon_w, m1, m2, cos, sin):
+ B, T, C = qkv.shape; N = B * T
+ assert C == 1536 and qkv.dtype == torch.float16 and qkv.is_contiguous() and T % 128 == 0
+ m1f, m2f = m1.reshape(N).contiguous(), m2.reshape(N).contiguous()
+ cf, sf = cos.reshape(N, 32).contiguous(), sin.reshape(N, 32).contiguous()
+ q, k, v = ext.fwd(qkv, canon_w.contiguous(), m1f, m2f, cf, sf)
+ ctx.save_for_backward(qkv, canon_w, m1f, m2f, cf, sf)
+ return q, k, v
+ @staticmethod
+ def backward(ctx, gq, gk, gv):
+ qkv, w, m1f, m2f, cf, sf = ctx.saved_tensors
+ gq, gk, gv = (g.to(torch.float16).contiguous() for g in (gq, gk, gv))
+ dqkv, part = ext.bwd(qkv, w.contiguous(), m1f, m2f, cf, sf, gq, gk, gv)
+ # fp32 straight from the fp32 reduction. Rounding through fp16 (as the
+ # eager region's cast implied) left ~3% of entries subnormal on the live
+ # checkpoint; keeping fp32 is strictly more precise.
+ return dqkv, part.sum(0).to(w.dtype), None, None, None, None
+
+def canon_rope(qkv, canon_w, m1, m2, cos, sin):
+ return CanonRopeFast.apply(qkv, canon_w, m1, m2, cos, sin)
diff --git a/volta_kernels/head.py b/volta_kernels/head.py
new file mode 100644
index 0000000..35c3974
--- /dev/null
+++ b/volta_kernels/head.py
@@ -0,0 +1,168 @@
+"""Fused LM head (cross-entropy without (N, V) logits), with the elementwise
+middle rewritten as one Triton pass.
+
+Same algorithm, same constants and the same operand precision as
+model._FusedLMHead -- read that docstring for why GRAD_SCALE is 64, why the dW
+GEMM is shrunk, and why the backward multiplier stays fp32. What changes is only
+how many times each chunk's logits cross memory.
+
+Production spelling, per 1024-row chunk at micro_batch 16 (x fp32 because the
+residual stream is fp32):
+
+ F.linear(xc, Wc) autocast casts W (32768x512 fp32) to fp16 -- again
+ .float() fp16 logits -> fp32 copy 128 MB
+ _head_chunk_c 3 passes over fp32 logits, writes fp32 p 128 MB
+ scatter_add_ one-hot, in place
+ p.to(x.dtype) a no-op: x is fp32, so g is fp32
+ F.linear(g, Wc.t()) autocast casts g to fp16 and W to fp16 again
+ dx[a:b] = ... fp16 -> fp32 copy into dx
+ g * DW_SHRINK another fp32 pass 128 MB
+ gd.t() @ xn autocast casts gd^T (strided) to fp16
+
+Here W and x are cast once per call, not twice per chunk, and one kernel reads
+the fp16 logits once and writes the two fp16 operands the GEMMs were being fed
+anyway: fp16(p'), where p' is the scaled softmax minus the one-hot. dW is
+accumulated with an fp32-output GEMM, so it is never rounded to fp16 at all --
+the production path's overflow guard (x rescale, DW_SHRINK) has nothing left to
+guard. lse differs from Inductor's only in fp32 reduction order.
+"""
+from __future__ import annotations
+
+import torch
+import torch.nn.functional as F
+import triton
+import triton.language as tl
+
+IGNORE = -100
+
+
+@triton.jit
+def _softmax_grad_kernel(logits, tgt, g_out, row_loss, V: tl.constexpr, S: tl.constexpr):
+ """One program per row: the whole 32k-wide row stays in registers.
+
+ g_out = fp16(S * softmax - S * onehot), the operand both head GEMMs take.
+ (A blocked two-pass version with online max/sum measured no faster.)
+ """
+ row = tl.program_id(0).to(tl.int64)
+ cols = tl.arange(0, V)
+ x = tl.load(logits + row * V + cols).to(tl.float32)
+ t = tl.load(tgt + row)
+ valid = t != -100
+ safe = tl.where(valid, t, 0)
+ # Inductor's logsumexp decomposition, including its guard for an
+ # all-infinite row.
+ m = tl.max(x, 0)
+ m = tl.where(tl.abs(m) == float("inf"), 0.0, m)
+ lse = tl.log(tl.sum(tl.exp(x - m), 0)) + m
+ vs = valid.to(tl.float32) * S
+ p = tl.exp(x - lse) * vs
+ hit = cols == safe
+ p = tl.where(hit, p + (-S * valid.to(tl.float32)), p)
+ tl.store(g_out + row * V + cols, p.to(tl.float16))
+ picked = tl.sum(tl.where(hit, x, 0.0), 0)
+ tl.store(row_loss + row, tl.where(valid, lse - picked, 0.0))
+
+
+class FusedLMHead(torch.autograd.Function):
+ GRAD_SCALE = 64.0
+
+ @staticmethod
+ def forward(ctx, x, W, targets, chunk: int):
+ N, d = x.shape
+ V = W.shape[0]
+ W16 = W.to(torch.float16)
+ x16 = x.to(torch.float16)
+ valid = targets != IGNORE
+ n_valid = valid.sum().clamp_min(1)
+ # dx is written fp32 straight from the GEMM's fp32 accumulator. With an
+ # fp16 output cuBLAS may pick split-k kernels that sum partials in fp16
+ # (allowed by reduced-precision reduction); fp32 output rules that out.
+ # Measured on the real fixture: dx error vs fp64 5.31e-4 -> 3.43e-4,
+ # and 2.9% faster.
+ dx16 = torch.empty((N, d), device=x.device, dtype=torch.float32)
+ # zeros, not empty: do not rely on beta=0 never reading the buffer in the
+ # out_dtype addmm path (0.07 ms per micro-step)
+ dW = torch.zeros_like(W, dtype=torch.float32)
+ row_loss = torch.empty(N, device=x.device, dtype=torch.float32)
+ c = min(chunk, N)
+ logits = torch.empty((c, V), device=x.device, dtype=torch.float16)
+ g = torch.empty_like(logits)
+ S = FusedLMHead.GRAD_SCALE
+
+ for a in range(0, N, chunk):
+ b = min(a + chunk, N)
+ n = b - a
+ lg = logits[:n]
+ torch.mm(x16[a:b], W16.t(), out=lg)
+ _softmax_grad_kernel[(n,)](lg, targets[a:b], g[:n], row_loss[a:b],
+ V=V, S=S, num_warps=16)
+ torch.mm(g[:n], W16, out_dtype=torch.float32, out=dx16[a:b])
+ # dW: fp16 operands, fp32 accumulation AND fp32 output (cuBLAS
+ # s884gemm + fp32 split-k reduce), accumulated across chunks with
+ # beta=1. The production head rounds every chunk's dW to fp16 and so
+ # needs the |x| <= 1 rescale and DW_SHRINK to keep that output from
+ # overflowing; with an fp32 output there is nothing to overflow.
+ # Strictly more precise (dW rel-L2 vs fp64 5.24e-4 -> 5.01e-4 on
+ # the real fixture), 12.6% faster, one fewer 64 MB write per chunk.
+ torch.addmm(dW, g[:n].t(), x16[a:b], beta=0 if a == 0 else 1,
+ out_dtype=torch.float32, out=dW)
+
+ ctx.w_dtype = W.dtype
+ ctx.x_dtype = x.dtype
+ ctx.W_ref = W
+ ctx.save_for_backward(dx16, dW)
+ ctx.n_valid = n_valid
+ return row_loss.sum() / n_valid
+
+ @staticmethod
+ def backward(ctx, grad_out):
+ dx16, dW = ctx.saved_tensors
+ s = grad_out / (ctx.n_valid * FusedLMHead.GRAD_SCALE)
+ return ((dx16.float() * s).to(ctx.x_dtype), (dW * s).to(ctx.w_dtype),
+ None, None)
+
+
+class FusedLMHeadStash(FusedLMHead):
+ """Same forward. The backward hands (dW, s) to the tied embedding's backward
+ (integ_ops.TiedEmbedding), which folds dW*s, the embedding segment sums and
+ the .grad accumulate into one pass, instead of returning dW*s to autograd."""
+
+ @staticmethod
+ def backward(ctx, grad_out):
+ from .integ_ops import stash_head_grad
+ dx16, dW = ctx.saved_tensors
+ s = grad_out / (ctx.n_valid * FusedLMHead.GRAD_SCALE)
+ stash_head_grad(ctx.W_ref, dW, s)
+ return (dx16.float() * s).to(ctx.x_dtype), None, None, None
+
+
+def _loss_only(x, W, targets, chunk):
+ N = x.shape[0]
+ V = W.shape[0]
+ W16 = W.to(torch.float16)
+ x16 = x.to(torch.float16)
+ n_valid = (targets != IGNORE).sum().clamp_min(1)
+ row_loss = torch.empty(N, device=x.device, dtype=torch.float32)
+ c = min(chunk, N)
+ logits = torch.empty((c, V), device=x.device, dtype=torch.float16)
+ g = torch.empty_like(logits)
+ for a in range(0, N, chunk):
+ b = min(a + chunk, N)
+ n = b - a
+ torch.mm(x16[a:b], W16.t(), out=logits[:n])
+ _softmax_grad_kernel[(n,)](logits[:n], targets[a:b], g[:n], row_loss[a:b],
+ V=V, S=FusedLMHead.GRAD_SCALE, num_warps=16)
+ return row_loss.sum() / n_valid
+
+
+def fused_lm_loss(x, W, targets, chunk: int = 1024, stash: bool = False):
+ if not torch.is_grad_enabled():
+ # Validation (quick_val runs under no_grad): no backward will come, so
+ # skip the dx and dW GEMMs entirely -- 2.25x faster head. Same logits
+ # GEMM and softmax kernel, so the loss is bitwise the training path's.
+ # (Checked here, not inside the Function: its forward always runs with
+ # grad mode off, and the tied weight keeps requires_grad under no_grad.)
+ return _loss_only(x, W, targets, chunk)
+ if stash:
+ return FusedLMHeadStash.apply(x, W, targets, chunk)
+ return FusedLMHead.apply(x, W, targets, chunk)
diff --git a/volta_kernels/integ_ops.py b/volta_kernels/integ_ops.py
new file mode 100644
index 0000000..8f05f01
--- /dev/null
+++ b/volta_kernels/integ_ops.py
@@ -0,0 +1,256 @@
+"""Model-side additions adopted from the kernel swarm, integrated on the merged
+(BORIS_VOLTA_KERNELS=1) path. Everything here is used by model.py's
+`_forward_merged`; nothing changes the forward values (the loss is bitwise the
+live one).
+
+1. wgrad_acc (block_autograd b, = resid_fp32 V1 = precision_plus dw):
+ the weight-gradient GEMM of every trunk matmul accumulates straight into
+ the fp32 p.grad (cuBLAS beta=1, fp32 output). Removes the fp16 rounding of
+ dW (and of its split-k partials), the fp16->fp32 cast and AccumulateGrad's
+ add_. dW error vs fp64 ~100x lower.
+2. norm_bwd (elem_audit): residual-add + RMSNorm backward and the norm-weight
+ gradient in ONE Triton pass (16-row blocks, dw partials in registers)
+ instead of Inductor's row kernel + a split column reduction that re-reads
+ the same 100-134 MB. Row arithmetic replicates Inductor's live kernel, so
+ dh/dh16 (and hence every other gradient) are bitwise the live ones.
+ (3. the CUTLASS SwiGLU epilogue was integrated, measured bitwise but not
+ faster on the whole micro-step, and dropped -- see ../epi_dropped.py.)
+4. TiedEmbedding (ngram_embed A): the head's dW*s, the embedding's segment
+ sums and the .grad accumulate in one pass (bitwise).
+"""
+from __future__ import annotations
+
+import torch
+import torch.nn.functional as F
+import triton
+import triton.language as tl
+from torch.library import triton_op, wrap_triton
+
+F16, F32 = torch.float16, torch.float32
+
+
+# --------------------------------------------------------------------------- #
+# 1. weight gradients accumulated by the GEMM into fp32 .grad
+# --------------------------------------------------------------------------- #
+@torch.library.custom_op("integ::wgrad_acc", mutates_args=("acc",))
+def wgrad_acc(dy: torch.Tensor, x: torch.Tensor, acc: torch.Tensor, alpha: float) -> None:
+ dy2 = dy.reshape(-1, dy.shape[-1])
+ x2 = x.reshape(-1, x.shape[-1])
+ torch.addmm(acc, dy2.t(), x2, beta=1, alpha=alpha, out_dtype=F32, out=acc)
+
+
+@wgrad_acc.register_fake
+def _(dy, x, acc, alpha):
+ return None
+
+
+class LinAcc(torch.autograd.Function):
+ """y = x16 @ w16^T. The weight gradient alpha * dy^T x16 is ADDED into `acc`
+ (the fp32 p.grad) by the GEMM itself and not returned to autograd. w16 is
+ the (pre-scaled) fp16 weight; alpha undoes nothing -- it is the scale the
+ forward applied to the fp32 weight (DOWN_SHRINK), i.e. d(w*s)/dw."""
+
+ @staticmethod
+ def forward(ctx, x16, w16, acc, alpha: float):
+ ctx.save_for_backward(x16, w16, acc)
+ ctx.alpha = alpha
+ return F.linear(x16, w16)
+
+ @staticmethod
+ def backward(ctx, dy):
+ x16, w16, acc = ctx.saved_tensors
+ torch.ops.integ.wgrad_acc(dy, x16, acc, ctx.alpha)
+ return dy @ w16, None, None, None
+
+
+# --------------------------------------------------------------------------- #
+# 2. fused residual-add + RMSNorm backward with the norm-weight gradient
+# --------------------------------------------------------------------------- #
+@triton.jit
+def _norm_bwd_bw_kernel(h, a, rstd, w, gres, gx, dh, dh16, dwp, ROWS: tl.constexpr,
+ D: tl.constexpr, HAS_A: tl.constexpr, OSCALE: tl.constexpr):
+ """Row-for-row the arithmetic of Inductor's live norm-backward row kernel
+ (triton_per_fused__to_copy_add_div_mul_pow_sum_view_*: one 512-wide row as a
+ 1-D block, 4 warps, num_stages 1, the same op DAG and so the same FMA
+ contractions), so dh and dh16 are BITWISE the live ones and every other
+ gradient stays bitwise too. The only change: the norm-weight gradient
+ sum_rows(gx * (h1*rstd)) -- the same per-element products as Inductor's
+ separate reduction kernel -- is accumulated here in fp32 over ROWS rows
+ (partials then summed in fp64), instead of by a second pass that re-reads
+ gx, h and a.
+ d = g_res + (gx*w)*rstd + ((sum(gx*w*h1)*-0.5)*rstd^3/D) * (2*h1)"""
+ pid = tl.program_id(0).to(tl.int64)
+ cols = tl.arange(0, D)
+ wv = tl.load(w + cols, eviction_policy="evict_last")
+ acc = tl.zeros([D], tl.float32)
+ for i in range(ROWS):
+ r = pid * ROWS + i
+ off = r * D + cols
+ g = tl.load(gx + off).to(tl.float32)
+ hv = tl.load(h + off)
+ if HAS_A:
+ hv = hv + tl.load(a + off).to(tl.float32)
+ gr = tl.load(gres + off)
+ rs = tl.load(rstd + r)
+ gw = g * wv
+ sm = tl.sum(gw * hv, 0)
+ t = gr + gw * rs
+ k = ((sm * -0.5) * (rs * rs * rs)) * (1.0 / D)
+ d = t + k * (hv * 2.0)
+ tl.store(dh + off, d)
+ if OSCALE == 1.0:
+ tl.store(dh16 + off, d.to(tl.float16))
+ else:
+ tl.store(dh16 + off, (d * OSCALE).to(tl.float16))
+ acc += g * (hv * rs)
+ tl.store(dwp + pid * D + cols, acc)
+
+
+@triton.jit
+def _colsum64_kernel(p, out, NR, D: tl.constexpr, BC: tl.constexpr, BR: tl.constexpr):
+ """Stage 1 of an fp64 column sum: out[k, c] = sum of rows k*BR..k*BR+BR-1 of p
+ (fp32 in, fp64 accumulate and out). Grid (D // BC, cdiv(NR, BR))."""
+ c = tl.program_id(0) * BC + tl.arange(0, BC)
+ k = tl.program_id(1)
+ r = k * BR + tl.arange(0, BR)
+ x = tl.load(p + r[:, None] * D + c[None, :], mask=r[:, None] < NR, other=0.0)
+ tl.store(out + k * D + c, tl.sum(x.to(tl.float64), 0))
+
+
+def _colsum64(p):
+ """fp32 (NR, D) -> fp32 (D,) column sums with an fp64 accumulator (deterministic)."""
+ NR, D = p.shape
+ BR = 64
+ nk = triton.cdiv(NR, BR)
+ part = torch.empty((nk, D), device=p.device, dtype=torch.float64)
+ wrap_triton(_colsum64_kernel)[(D // 64, nk)](p, part, NR, D=D, BC=64, BR=BR, num_warps=4)
+ return part.sum(0).to(F32)
+
+
+_NB_ROWS = 16 # rows per program: fp32 dw partial per 16 rows, partials summed in fp64
+
+
+def _norm_bwd_launch(h, a, rstd, w, gres, gx, oscale: float, has_a: bool):
+ N, D = h.shape
+ dh = torch.empty_like(h)
+ dh16 = torch.empty((N, D), device=h.device, dtype=F16)
+ dwp = torch.empty((N // _NB_ROWS, D), device=h.device, dtype=F32)
+ wrap_triton(_norm_bwd_bw_kernel)[(N // _NB_ROWS,)](
+ h, a, rstd, w, gres, gx, dh, dh16, dwp, ROWS=_NB_ROWS, D=D,
+ HAS_A=has_a, OSCALE=oscale, num_warps=4, num_stages=1)
+ return dh, dh16, _colsum64(dwp)
+
+
+@triton_op("integ::norm_bwd_a", mutates_args={})
+def norm_bwd_a(h: torch.Tensor, a: torch.Tensor, rstd: torch.Tensor, w: torch.Tensor,
+ gres: torch.Tensor, gx: torch.Tensor, oscale: float
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ return _norm_bwd_launch(h, a, rstd, w, gres, gx, oscale, True)
+
+
+@triton_op("integ::norm_bwd_h", mutates_args={})
+def norm_bwd_h(h: torch.Tensor, rstd: torch.Tensor, w: torch.Tensor,
+ gres: torch.Tensor, gx: torch.Tensor, oscale: float
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ return _norm_bwd_launch(h, h, rstd, w, gres, gx, oscale, False)
+
+
+class AddNorm(torch.autograd.Function):
+ """(h1, x16) = (h + s*a.float(), fp16(rmsnorm(h1) * w)).
+
+ Forward: the same torch ops as the live region, so Inductor emits the same
+ fused kernel (bitwise). Backward: one Triton pass (_norm_bwd_bw_kernel) that
+ reproduces the live row kernel bit for bit -- dh1 (fp32, to the residual)
+ and da16 = fp16(s*dh1) (to the o_proj / down GEMM backward) -- and also
+ accumulates the norm-weight gradient sum_rows(gx*xhat), which the live graph
+ computes in a second kernel re-reading gx, h and a.
+
+ save_out=False saves (h, a) -- the tensors the live graph already keeps for
+ this norm (h is the previous region's output, a the o_proj output).
+ save_out=True saves h1 instead -- for the next block's norm1, where h1 is
+ this region's output and is kept anyway as the next region's input.
+ """
+
+ @staticmethod
+ def forward(ctx, h, a, w, eps: float, s: float, save_out: bool):
+ h1 = h + a.float() if s == 1.0 else h + a.float() * s
+ rstd = torch.rsqrt(h1.pow(2).mean(-1, keepdim=True) + eps)
+ x16 = (h1 * rstd * w.float()).to(F16)
+ if save_out:
+ ctx.save_for_backward(h1, rstd, w)
+ else:
+ ctx.save_for_backward(h, a, rstd, w)
+ ctx.s, ctx.save_out = s, save_out
+ return h1, x16
+
+ @staticmethod
+ def backward(ctx, gh1, gx):
+ D = gx.shape[-1]
+ if ctx.save_out:
+ h1, rstd, w = ctx.saved_tensors
+ shp = h1.shape
+ dh, da, dw = torch.ops.integ.norm_bwd_h(
+ h1.reshape(-1, D), rstd.reshape(-1), w,
+ gh1.reshape(-1, D).contiguous(), gx.reshape(-1, D).contiguous(), ctx.s)
+ else:
+ h, a, rstd, w = ctx.saved_tensors
+ shp = h.shape
+ dh, da, dw = torch.ops.integ.norm_bwd_a(
+ h.reshape(-1, D), a.reshape(-1, D), rstd.reshape(-1), w,
+ gh1.reshape(-1, D).contiguous(), gx.reshape(-1, D).contiguous(), ctx.s)
+ return dh.view(shp), da.view(shp), dw, None, None, None
+
+
+# --------------------------------------------------------------------------- #
+# 4. tied embedding: acc + (head dW * s + embedding segment sums) in one pass
+# --------------------------------------------------------------------------- #
+@triton.jit
+def _tied_accum_dense(acc_ptr, dw_ptr, s_ptr, seg_ptr, out_ptr, n,
+ HAS_ACC: tl.constexpr, BLOCK: tl.constexpr):
+ offs = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)
+ m = offs < n
+ s = tl.load(s_ptr)
+ v = tl.load(dw_ptr + offs, mask=m) * s + tl.load(seg_ptr + offs, mask=m)
+ if HAS_ACC:
+ v = tl.load(acc_ptr + offs, mask=m) + v
+ tl.store(out_ptr + offs, v, mask=m)
+
+
+_STASH: dict = {}
+
+
+def stash_head_grad(W, dW, s):
+ """Called by the head's backward in place of returning dW*s for W."""
+ _STASH[id(W)] = (dW, s)
+
+
+class TiedEmbedding(torch.autograd.Function):
+ """F.embedding(idx, W) for the tied weight. Its backward adds
+ (head dW * s) + segment sums into W.grad in one pass -- the exact rounding
+ sequence of the stock path (dW*s rounded, + seg, + acc; FMA contraction
+ off), in place of: the dW*s pass, autograd's tied-weight sum and
+ AccumulateGrad's add_. Requires the head to stash its dW (stash=True)."""
+
+ @staticmethod
+ def forward(ctx, idx, W):
+ ctx.save_for_backward(idx)
+ ctx.W = W
+ return F.embedding(idx, W)
+
+ @staticmethod
+ def backward(ctx, g):
+ (idx,) = ctx.saved_tensors
+ W = ctx.W
+ dW, s = _STASH.pop(id(W))
+ seg = torch.ops.aten.embedding_dense_backward(
+ g.reshape(-1, g.shape[-1]), idx.reshape(-1), W.shape[0], -1, False)
+ has = W.grad is not None
+ out = W.grad if has else seg # first micro-step: in place into seg
+ n = W.numel()
+ BLOCK = 2048
+ _tied_accum_dense[(triton.cdiv(n, BLOCK),)](
+ out, dW, s.reshape(1).float(), seg, out, n, HAS_ACC=has, BLOCK=BLOCK,
+ num_warps=4, enable_fp_fusion=False)
+ if not has:
+ W.grad = out
+ return None, None