summaryrefslogtreecommitdiff
path: root/config.py
blob: 02ec2c4c6db2911a4f0ce9e3100b6e0286bac6bf (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
"""Resolved configuration for Boris-2-D112M-n15M.

A 143.5M-parameter decoder-only model trained from scratch on 200B tokens on a
single V100 32GB (sm_70). Every number that is a judgement call is a field in
this file, and the whole object is serialised into every checkpoint, so a
resumed run cannot silently disagree with the run that wrote the checkpoint.

Lineage. The trunk shape (d512 x 31L), Canon layers and the n-gram table come
from Boris-1.7/1.8, which validated them at 60M dense on a 3060. Boris-2 is not
1.8 scaled up: the FFN aspect ratio is deliberately different (4.6x rather than
2.1x, chosen for Volta's GEMM shapes as much as for capacity), Canon is fused
into one kernel over the concatenated Q|K|V block instead of three convolutions,
attention is varlen over a jagged nested tensor rather than dense-masked, the
context length changes partway through the run, and Newton-Schulz is batched.

Hardware. GPU 0 is the V100 and trains. GPU 1 is a 3060 and only ever runs
eval.py, in a separate process, from weights-only checkpoints. There is no DDP
here and there is not meant to be.
"""
from __future__ import annotations

import json
from dataclasses import asdict, dataclass, field, fields
from pathlib import Path

ROOT = Path(__file__).resolve().parent
DATA_DIR = ROOT / "data"                 # -> /media/joseph/Media/boris-2-data
CKPT_DIR = ROOT / "checkpoints"          # -> /media/joseph/Media/boris-2-ckpt
LOG_DIR = ROOT / "logs"
EVAL_DIR = ROOT / "eval_out"
RUN_0920_DIR = Path("/media/joseph/Media/Boris-2-0920")
# The ACTIVE run directory (eval exports, checkpoints, run name). BORIS_RUN_DIR
# selects a new run without touching 0920, whose n-gram vocab is shared.
RUN_DIR = Path(__import__("os").environ.get("BORIS_RUN_DIR", "/media/joseph/Media/Boris-2-0929"))
LEGACY_TOKENIZER_SHA256 = "0eabe7bb2467338230acaf6fc99321668929391a1bd90b7d27138ded627e8b7c"


# --------------------------------------------------------------------------- #
# Model
# --------------------------------------------------------------------------- #
@dataclass
class ModelConfig:
    architecture: str = "boris-2-canon-ngram"

    # Depth-thin, carried over from 1.8. MobileLLM (2402.14905) finds depth
    # matters independently of parameter count below 1B, with the optimum near
    # 30 layers; d_model 512 is the hidden-size threshold replications identify,
    # so the shape stops there rather than going thinner.
    # 2026-09-17 rebalance. Measured against GPT-X2.5-135M, which beats Boris-2
    # by 9.28 Int Index with PIQA + HellaSwag accounting for 79% of the gap.
    # Those are context-integration tasks and attention was the component this
    # model was starved of: 17.0M attention params (15.3% of each block) against
    # their 26.5M (22.9%). GQA 4:1 exists to shrink the KV cache at INFERENCE --
    # a cost this run never pays, since it trains on one V100 and is not served.
    # So: full MHA (2 KV heads -> 8), depth 26 -> 31, and the FFN ratio pulled
    # from 4.63x to 3.25x to fund it. The resulting 31-layer shape measures
    # about 35.5k tok/s in the live 0917 run.
    # d_model stays 512 so the 32,768 BPE, the
    # n-gram table and its (512, 64) projection are all unchanged.
    #   attention   17.0M -> 32.5M   (+91%)
    #   total      143,504,896 -> 143,724,032   (+0.15%)
    d_model: int = 512
    n_layer: int = 31
    n_head: int = 8
    n_kv_head: int = 8                   # full MHA; was GQA 4:1
    vocab_size: int = 32_768             # our own BPE; power of two, uint16 storage
    # SHA-256 of tokenizer.json. Token ids are part of the model definition:
    # two 32,768-entry BPEs are not checkpoint-compatible merely because their
    # tensor shapes match. Keep this in the config fingerprint so a resume
    # cannot silently attach old embeddings to a new vocabulary.
    tokenizer_sha256: str = "f366974b96357d11a7e9930826efd81d77438638c0e21ea1517f620da72e05f4"
    norm_eps: float = 1e-5
    tie_embeddings: bool = True

    # SwiGLU inner width. 2368 is not a conventional aspect ratio -- it is 4.63x
    # d_model where 2.7x is normal -- and it is chosen twice over:
    #
    # Set explicitly rather than derived from a multiplier, so the fixed
    # parameter budget cannot drift as the attention allocation changes.
    ffn_hidden: int = 1664               # 3.25x; was 2368 (4.63x)

    # --- Canon ---------------------------------------------------------- #
    # Trainable causal depthwise 1-D convolutions with a residual, applied to
    # the projected Q, K and V before attention (2607.18413, which ablated 17
    # placements in a Qwen3 block and landed here). Kernel 3, no norm, no
    # activation, zero-initialised so the layer starts as exactly the identity.
    #
    # Depthwise convolution is per-channel, so one convolution over the
    # concatenated Q|K|V channel block is mathematically identical to three
    # separate ones. model.py exploits that: 31 kernels per forward, not 93.
    use_canon: bool = True
    canon_kernel: int = 3

    # --- n-gram table ---------------------------------------------------- #
    # Construction-time arm. False builds the trunk with no table and no
    # projection; the training script is otherwise byte-identical, so the
    # control is this same file with one flag.
    #
    # The transformer is identical either way. False just drops the table and
    # its projection; nothing else moves.
    #
    #     use_ngram=True    143,504,896
    #     use_ngram=False   128,472,064   (same trunk, minus the 15M table)
    use_ngram: bool = True

    # --- Context length ------------------------------------------------- #
    # Two stages. The model trains at 2048 for the bulk of the run and extends
    # to 8192 at 80% of the token budget -- the same relative point X2.5 used
    # (60B of 75B). A stage is (token_fraction, block_size, rope_theta).
    #
    # Extension is cheap here in a way it is not for dense-masked attention:
    # attention runs varlen per document, so its cost is sum(len_i^2) and does
    # not grow just because the packing window did. What actually costs is the
    # long documents themselves -- which is the point of the stage, and why
    # curriculum.long_doc_floor_tokens rises with it.
    #
    # A config, not a fixed choice: it is cheap to move once the loss curve is
    # visible, and moving it does not invalidate any checkpoint.
    context_stages: tuple = ((0.00, 2048, 10_000.0),
                             (0.80, 8192, 500_000.0))

    @property
    def head_dim(self) -> int:
        assert self.d_model % self.n_head == 0
        return self.d_model // self.n_head

    @property
    def q_dim(self) -> int:
        return self.n_head * self.head_dim

    @property
    def kv_dim(self) -> int:
        return self.n_kv_head * self.head_dim

    @property
    def canon_channels(self) -> int:
        """One fused Canon acts on Q|K|V together."""
        return self.q_dim + 2 * self.kv_dim

    @property
    def max_block_size(self) -> int:
        return max(int(b) for _, b, _ in self.context_stages)

    # -- parameter accounting; the single source of truth for the asserts -- #
    def attn_params(self) -> int:
        d = self.d_model
        return d * self.q_dim + 2 * d * self.kv_dim + self.q_dim * d

    def ffn_params(self) -> int:
        return 3 * self.d_model * self.ffn_hidden

    def canon_params(self) -> int:
        return self.n_layer * self.canon_kernel * self.canon_channels if self.use_canon else 0

    def norm_params(self) -> int:
        return self.n_layer * 2 * self.d_model + self.d_model

    def matmul_params(self) -> int:
        """Params that drive a matmul -- what Muon operates on, trunk only."""
        return self.n_layer * (self.attn_params() + self.ffn_params())

    def embed_params(self) -> int:
        return self.vocab_size * self.d_model * (1 if self.tie_embeddings else 2)


# --------------------------------------------------------------------------- #
# N-gram table
# --------------------------------------------------------------------------- #
@dataclass
class NGramConfig:
    """A lookup, never a matmul.

    Rows are an explicit top-K vocabulary: a specific (t-1, t) or (t-2, t-1, t)
    owns a row, or it falls to the single shared OOV row. No hashing, so two
    unrelated phrases can never silently share a vector.

    Layout is chosen for the gather, not for the file. Rows are written in
    descending corpus frequency, so the hot head of the table is contiguous:
    rank 64 in fp32 is 256 bytes per row, and the top ~24k rows -- which cover
    the large majority of lookups -- fit inside the V100's 6MB L2. A table
    ordered by n-gram id instead would scatter every batch across 60MB.
    """

    rank: int = 64                       # 256B/row fp32: 4 aligned cache lines
    bigram_rows: int = 210_938           # held-out optimum: about 90% of rows
    trigram_rows: int = 23_437           # preserve the strongest specific phrases
    oov_rows: int = 1

    # Added to the residual stream after this many blocks. Not at the embedding
    # sum: a retrieved phrase vector is meant to enter a stream that already
    # carries some context, rather than be re-encoded from scratch by the full
    # depth of the network. 1.7/1.8 used 2 and this keeps the comparison.
    insert_after: int = 2

    vocab_path: Path = RUN_0920_DIR / "ngram_vocab.bin"

    target_params: int = 15_000_000
    tolerance: float = 0.01

    table_init_std: float = 0.02
    proj_init_std: float = 0.02
    # The projection is a 2-D linear map -> Muon. The table never is.
    muon_on_proj: bool = True

    @property
    def n_rows(self) -> int:
        return self.oov_rows + self.bigram_rows + self.trigram_rows

    @property
    def oov_row(self) -> int:
        return 0

    @property
    def bigram_base(self) -> int:
        return self.oov_rows

    @property
    def trigram_base(self) -> int:
        return self.oov_rows + self.bigram_rows

    def table_params(self, d_model: int) -> int:
        return self.n_rows * self.rank + self.rank * d_model


# --------------------------------------------------------------------------- #
# Data
# --------------------------------------------------------------------------- #
@dataclass
class DataConfig:
    root: Path = DATA_DIR
    # 2026-09-17, Boris-2-0917. The seven hand-weighted sources are gone.
    # ClimbMix (2504.13161) is a 400B-token corpus whose mixture was SEARCHED
    # rather than guessed: 1,000 topic clusters, pruned by ad/educational/
    # informational/quality classifiers at a loose threshold, then mixed by
    # iterative bootstrapping against the PIQA, ARC-E and HellaSwag validation
    # sets. Those three are 79% of the Int Index gap Boris-2-0907 showed against
    # GPT-X2.5-135M. At equal token budget it beats Nemotron-CC, DCLM, SmolLM-
    # corpus and FineWeb-Edu, and its 350M model beats SmolLM-360M on exactly
    # the two benchmarks 0907 was weakest on (PIQA +1.03, HellaSwag +2.75).
    # We take 200B of the 369B (under our BPE) -- sound only because the
    # karpathy/climbmix-400b-shuffle mirror is pre-shuffled, so a prefix is a
    # representative sample. Written as climbmix + climbmix_s1..s15 and
    # stitched by MultiStore.
    # finemath is the one source kept from 0907: ArithMark-3 was the single
    # benchmark where 0907 was competitive (37.40 vs 38.40), and ClimbMix was
    # not optimised for arithmetic.
    sources: tuple = ("climbmix", "finemath")
    val_path: Path = DATA_DIR / "val.bin"
    val_source: str = "climbmix_val"
    # Documents longer than this are split across rows rather than truncated,
    # so nothing in the corpus is unreachable.
    max_doc_tokens: int = 65_536
    # Rows are packed, not EOS-concatenated: attention and Canon both isolate
    # documents, so a row may hold many and none of them leak into another.
    eos_id: int = 0
    prefetch_depth: int = 4


@dataclass
class CurriculumConfig:
    """Token-threshold mixture weights, interpolated between phases.

    Never a static ratio: the decay phase is supposed to consolidate onto the
    densest data, and that only happens if the mixture is a function of
    progress. Weights are normalised at use, so they need not sum to 1 here.
    """

    enabled: bool = True
    smooth: bool = True                  # interpolate between phase endpoints

    # (name, progress_at_end, weights)
    # Decay consolidates onto the densest data and drops general web text
    # entirely: Ultra-FineWeb's L2 weight moves to its L3 refinement (Q&A pairs
    # and multi-style rewrites of the same corpus), which is what openbmb built
    # L3 for. Weights ramp smoothly, so L3 averages ~0.125 across the decay
    # phase rather than 0.25.
    # 2026-09-17: a first phase was added. The run previously fed 30% synthetic
    # and structured text (cosmopedia + finephrase) from step 0. GPT-X2.5 runs
    # 98% natural web (58% FineWeb-Edu + 40% DCLM) and introduces Cosmopedia
    # only at 40B of 75B. Boris-2's two weakest benchmarks -- PIQA and HellaSwag,
    # 79% of the Int Index gap -- are exactly what broad natural text teaches,
    # and Boris-2 scores WORSE on held-out fw-edu (3.2017) than on its own val
    # (3.0420), i.e. it is under-exposed. Phase 0 is held constant (mixture_at
    # does not interpolate into the first phase), then ramps to the original
    # stable weights by 0.87. Decay is unchanged.
    # FLAT. ClimbMix's weights are the output of a search over the whole
    # mixture; re-weighting it by phase would undo the thing we adopted it for.
    # The WSD decay phase still decays the LEARNING RATE -- it just no longer
    # swaps the data underneath it. One phase ending at 1.00 means mixture_at()
    # returns these weights constantly (it does not interpolate into phase 0).
    phases: tuple = (
        ("flat", 1.00, {"climbmix": 0.95, "finemath": 0.05}),
    )

    # When the context stage lengthens, short documents stop teaching long
    # context. From that point the packer prefers documents at least this long
    # where a source has them, falling back when it does not.
    long_doc_floor_tokens: int = 4096
    long_doc_target_frac: float = 0.5    # of tokens drawn, once extended


# --------------------------------------------------------------------------- #
# Optimisation
# --------------------------------------------------------------------------- #
@dataclass
class OptimConfig:
    """Muon on 2-D matmul weights, AdamW on everything else.

    The split is by role, assigned at construction, never by sniffing ndim.
    Canon kernels are (C, 3) tensors -- two-dimensional, and emphatically not
    matrices to orthogonalise. A shape rule would hand all 31 of them to Muon
    and nothing would ever complain.
    """

    muon_lr: float = 0.02
    muon_momentum: float = 0.95
    muon_nesterov: bool = True
    muon_ns_steps: int = 5
    muon_weight_decay: float = 0.1
    # Newton-Schulz over same-shaped parameters stacked into one bmm. At 31
    # layers this collapses ~2000 small launches per step into ~60. Run in
    # fp32: 1.8 measured that a bf16 iteration diverges 5.4e-02 relative from
    # the fp32 path because five steps amplify rounding well past bf16 epsilon,
    # and on sm_70 the fp16 alternative is worse. The matrices are small enough
    # that fp32 costs little -- preflight.py reports the measured share.
    muon_batched: bool = True
    muon_ns_dtype: str = "float32"
    # Scale the orthogonalised update by 0.2*sqrt(max(m,n)) so one LR behaves
    # across aspect ratios as different as 512x512 and 2368x512. With a 4.63x
    # FFN this matters more than it does at a conventional width.
    muon_rms_scale: bool = True

    adamw_lr: float = 3e-3               # embeddings, table, head, norms, canon
    adamw_betas: tuple = (0.9, 0.95)
    adamw_eps: float = 1e-8
    # Embedding (16.8M) and the n-gram table (~15M) are 22% of the model and had
    # NO restraint: both measured +21.1% in norm over 20.8B->30.1B while every
    # Muon-decayed matmul stayed within 1%. See optim.build_optimizers.
    adamw_weight_decay: float = 0.1      # embedding + ngram_table only
    adamw_norm_weight_decay: float = 0.0 # norm gains: conventional, keep 0
    # 2026-10-07: canon (31 x (768, 3), weight 0 = identity) grew ~+20% per 10B
    # in 0929 (block 11: 16.3 -> 20.2 -> 24.9 over 20/30/40B, max |w| 13.5) while
    # the norm1 gains in front of it shrank ~12% per 10B to compensate. Decay it
    # toward identity like the other AdamW tensors. Own group: see optim.py.
    adamw_canon_weight_decay: float = 0.1

    # 2026-09-17: the fused LM head walks the vocab in chunks and its peak is
    # one (chunk, 32768) fp32 logits tensor -- 268 MB at 2048. Boris-2-0917 OOMs
    # at 31.28/31.73 GB with that, because preflight measures peak_allocated in
    # a clean process and cannot see allocator fragmentation (1.0 GB reserved
    # unallocated here) or the 280 MB another process holds on the same card.
    # 1024 halves the head's footprint for a few percent of throughput.
    lm_head_chunk: int = 1024
    grad_clip: float = 4.0   # was 1.0; median gn fits 0.0654*D_B^0.646,
                             # crossing 1.0 at ~68B and reaching ~2.0 by 200B


@dataclass
class ScheduleConfig:
    """WSD. Warmup 1%, stable to 87%, 1-sqrt decay over the final 13%."""

    warmup_frac: float = 0.01            # 2B tokens
    stable_frac: float = 0.87
    decay_shape: str = "1-sqrt"          # 1-sqrt | linear | cosine
    final_lr_frac: float = 0.0


# --------------------------------------------------------------------------- #
# Run
# --------------------------------------------------------------------------- #
@dataclass
class TrainConfig:
    total_tokens: int = 200_000_000_000
    tokens_per_step: int = 524_288       # 0.5M; step count 381,470

    # Micro-batch per context stage, in ROWS. Sweep with `preflight --sweep`
    # and set from measurement; early measurements expire, so re-sweep after
    # any change to the model or the kernels.
    # Measured on the V100, 2026-09-06, with `preflight.py --sweep`, including
    # the worst case of one full-block document per row:
    #
    #   ctx 2048   mb 8  27.9k tok/s 13.0 GB | mb 16  37.7k tok/s 23.9 GB
    #   ctx 8192   mb 2  26.7k tok/s 13.1 GB | mb 4   32.6k tok/s 24.0 GB
    #
    # Both stages hold 32,768 tokens per micro-batch, so activation memory is
    # roughly unchanged when the context lengthens and the accumulation count
    # stays at 16 -- the step remains exactly tokens_per_step either way. Peak
    # is 23.8 GB of 31.7, and the worst case measures the same because the
    # memory-efficient attention kernel is O(N), not O(N^2).
    #
    # Re-sweep after any change to the model or the kernels: the optimum moves
    # as bottlenecks fall.
    micro_batch: dict = field(default_factory=lambda: {2048: 16, 8192: 4})

    device: str = "cuda:0"
    eval_device: str = "cuda:1"          # eval.py only; training never touches it
    seed: int = 20260906

    # fp16 everywhere. sm_70 tensor cores are fp16-in/fp32-accumulate and have
    # no bf16 path at all; TF32 does not exist before sm_80. Both are asserted
    # at startup rather than assumed.
    amp_dtype: str = "float16"
    init_scale: float = 2.0 ** 16
    # GradScaler has no floor of its own, and that is fatal here. Below roughly
    # 1e-30 `unscale_` computes grad/scale and overflows fp32 to inf, which sets
    # found_inf, which halves the scale again: the collapse becomes
    # self-reinforcing and has no exit. That is exactly how the run that reached
    # step 3,810 died -- the scale bled 131,072 -> 0 over 170 steps while the
    # weights stayed perfectly finite, and every step after that was skipped.
    # Floored at 1.0 the same event costs a few skipped steps instead of the run.
    min_loss_scale: float = 1.0
    # Ceiling, for the other end. The scale grew to 131,072 at step ~2,000 and
    # the first overflow appeared at 3,460; halving the ceiling buys a factor of
    # two of headroom in the backward and costs nothing but the smallest
    # gradients, which are already far below where they matter.
    max_loss_scale: float = 2.0 ** 15
    # Steps pinned at min_loss_scale before the run stops itself. At the floor
    # nothing is training -- every step is being skipped -- so this is the real
    # early warning, ~350 steps ahead of the loss going non-finite.
    abort_after_floored: int = 100

    compile_regions: bool = True         # fixed-shape regions only, never the model
    # FALSE, and this is not a tuning knob -- an fp16 residual stream cannot
    # survive 31 layers of this model. The stream grows with depth, and at step
    # 3,456 (1% of the budget) it measured:
    #     block  0   3,062      block 16  40,640
    #     block  9  16,144      block 24  39,264
    #     block 25  65,617   vs fp16's max of 65,504   <-- over by 113
    # It saturates to inf, and _rmsnorm then computes rsqrt(inf) = 0 and
    # inf * 0 = NaN. That NaN is what drove GradScaler to halve its scale
    # forever (halving a loss scale cannot fix a FORWARD overflow), which is how
    # two runs died. Recomputed in fp32 the same batch reaches 65,617 and stays
    # finite. Measured cost: 39.77k -> 36.73k tok/s and 24.0 -> 25.1 GB, about
    # +4.3 days on the 200B budget. Boris-1.7 never hit this because bf16 on the
    # 3060 has fp32's exponent range; sm_70 has no bf16 at all.
    fp16_residual: bool = False
    # Recompute block activations in the backward instead of storing them, to
    # buy a larger micro-batch. Costs one extra forward per checkpointed block
    # (~+11% compute) and buys fatter GEMMs. Whether that trade is positive is
    # a measurement, not an argument -- preflight.py --sweep with it on/off.
    # 2026-09-17: ON for Boris-2-0917. Without it the run OOMs -- 30.17 GB
    # allocated + 0.78 GB fragmentation + 0.28 GB held by another process on
    # the same card = 31.28 of 31.73. preflight reads ~1 GB LOW because it
    # measures peak_allocated in a clean process and cannot see either.
    # every=4 returns 6.5 GB (29.2 -> 22.7 worst-case) for 5.3% throughput.
    grad_checkpoint: bool = True
    grad_checkpoint_every: int = 4       # 1 = every block, 4 = every 4th

    # Resume granularity. The 50B checkpoints are archival milestones; they are
    # not the thing you resume from. A run that will be interrupted needs a
    # rolling checkpoint on a wall-clock timer.
    rolling_ckpt_minutes: int = 45
    rolling_ckpt_keep: int = 2
    full_ckpt_every_tokens: int = 50_000_000_000
    export_every_tokens: int = 10_000_000_000     # weights-only, for eval.py
    log_every_steps: int = 10
    # Consecutive steps of non-finite loss OR non-finite grad norm before the
    # run stops itself. One or two are normal (GradScaler probing its scale);
    # 200 means something is broken and every further step is wasted.
    abort_after_bad: int = 200
    val_every_tokens: int = 1_000_000_000


@dataclass
class EvalConfig:
    device: str = "cuda:1"
    out_path: Path = RUN_DIR / "eval" / "results.jsonl"
    export_dir: Path = RUN_DIR / "checkpoints" / "exports"
    manifest: Path = RUN_DIR / "checkpoints" / "exports" / "manifest.jsonl"
    tasks: tuple = ("hellaswag", "arc_easy", "arc_challenge", "piqa", "arithmark3")
    val_tokens: int = 10_000_000
    batch_size: int = 16
    poll_seconds: int = 120


@dataclass
class Config:
    model: ModelConfig = field(default_factory=ModelConfig)
    ngram: NGramConfig = field(default_factory=NGramConfig)
    data: DataConfig = field(default_factory=DataConfig)
    curriculum: CurriculumConfig = field(default_factory=CurriculumConfig)
    optim: OptimConfig = field(default_factory=OptimConfig)
    schedule: ScheduleConfig = field(default_factory=ScheduleConfig)
    train: TrainConfig = field(default_factory=TrainConfig)
    eval: EvalConfig = field(default_factory=EvalConfig)

    run_name: str = RUN_DIR.name
    checkpoint_dir: Path = RUN_DIR / "checkpoints"
    # verify.py builds 4- and 6-layer models to keep the isolation probes fast;
    # they are not the run, so the band assert does not apply to them.
    enforce_budget: bool = True

    # -- derived ------------------------------------------------------------ #
    @property
    def total_steps(self) -> int:
        return self.train.total_tokens // self.train.tokens_per_step

    def block_size_at(self, progress: float) -> tuple[int, float]:
        """(block_size, rope_theta) for this point in the run."""
        bs, theta = self.model.context_stages[0][1], self.model.context_stages[0][2]
        for frac, b, th in self.model.context_stages:
            if progress >= frac:
                bs, theta = int(b), float(th)
        return bs, theta

    def micro_batch_at(self, progress: float) -> int:
        bs, _ = self.block_size_at(progress)
        return int(self.train.micro_batch[bs])

    def accum_at(self, progress: float) -> int:
        bs = self.block_size_at(progress)[0]
        mb = self.micro_batch_at(progress)
        tokens_per_micro = mb * bs
        assert self.train.tokens_per_step % tokens_per_micro == 0, (
            f"tokens_per_step {self.train.tokens_per_step} is not divisible by "
            f"micro_batch {mb} x block_size {bs} = {tokens_per_micro}; the step "
            f"size would change when the context stage does")
        return self.train.tokens_per_step // tokens_per_micro

    # -- parameter budget ---------------------------------------------------- #
    def param_budget(self) -> dict:
        m, n = self.model, self.ngram
        on = m.use_ngram
        d = {
            "attention": m.n_layer * m.attn_params(),
            "ffn": m.n_layer * m.ffn_params(),
            "canon": m.canon_params(),
            "norms": m.norm_params(),
            "embedding": m.embed_params(),
            "ngram_table": n.n_rows * n.rank if on else 0,
            "ngram_proj": n.rank * m.d_model if on else 0,
        }
        d["trunk"] = d["attention"] + d["ffn"] + d["canon"] + d["norms"]
        d["matmul"] = d["attention"] + d["ffn"] + d["ngram_proj"]
        d["total"] = d["trunk"] + d["embedding"] + d["ngram_table"] + d["ngram_proj"]
        return d

    def assert_budget(self, ceiling: int = 150_000_000,
                      band: tuple[int, int] = (140_000_000, 145_000_000)) -> None:
        b = self.param_budget()
        if not self.enforce_budget:
            return
        assert b["total"] <= ceiling, (
            f"total parameters {b['total']:,} exceeds the hard ceiling {ceiling:,}")
        # The band describes the full model. With the table off the trunk is
        # unchanged and the total is simply 15M lower, which is the point.
        if not self.model.use_ngram:
            return
        assert band[0] <= b["total"] <= band[1], (
            f"total parameters {b['total']:,} outside the target band "
            f"{band[0]:,}-{band[1]:,}; change model.ffn_hidden")
        tp = self.ngram.table_params(self.model.d_model)
        tgt, tol = self.ngram.target_params, self.ngram.tolerance
        assert abs(tp - tgt) <= tol * tgt, (
            f"n-gram table {tp:,} is more than {tol:.0%} from its {tgt:,} target")

    # -- serialisation ------------------------------------------------------- #
    def to_dict(self) -> dict:
        def conv(o):
            if isinstance(o, Path):
                return str(o)
            if isinstance(o, dict):
                return {str(k): conv(v) for k, v in o.items()}
            if isinstance(o, (list, tuple)):
                return [conv(x) for x in o]
            return o
        return {f.name: conv(asdict(getattr(self, f.name))
                             if hasattr(getattr(self, f.name), "__dataclass_fields__")
                             else getattr(self, f.name))
                for f in fields(self)}

    def to_json(self) -> str:
        return json.dumps(self.to_dict(), indent=2, sort_keys=True)

    def fingerprint(self) -> str:
        """Hash of everything that must match to resume a checkpoint."""
        import hashlib
        keep = {k: self.to_dict()[k] for k in ("model", "ngram", "optim",
                                               "schedule", "train")}
        return hashlib.sha256(json.dumps(keep, sort_keys=True).encode()).hexdigest()[:16]

    @staticmethod
    def from_json(s: str) -> "Config":
        raw = json.loads(s)
        cfg = Config()
        # Checkpoints predating the tokenizer guard used the original 09-06
        # BPE. Give those serialized configs their real identity rather than
        # inheriting the 0920 default because the field is absent.
        if "model" in raw and "tokenizer_sha256" not in raw["model"]:
            cfg.model.tokenizer_sha256 = LEGACY_TOKENIZER_SHA256
        if (raw.get("model", {}).get("tokenizer_sha256", LEGACY_TOKENIZER_SHA256)
                == LEGACY_TOKENIZER_SHA256):
            cfg.data.val_source = "val"
        for name, sub in raw.items():
            if not hasattr(cfg, name):
                continue
            cur = getattr(cfg, name)
            if hasattr(cur, "__dataclass_fields__") and isinstance(sub, dict):
                for k, v in sub.items():
                    if not hasattr(cur, k):
                        continue
                    old = getattr(cur, k)
                    if isinstance(old, Path):
                        v = Path(v)
                    elif isinstance(old, tuple) and isinstance(v, list):
                        v = tuple(tuple(x) if isinstance(x, list) else x for x in v)
                    elif isinstance(old, dict) and isinstance(v, dict):
                        v = {int(kk) if str(kk).isdigit() else kk: vv for kk, vv in v.items()}
                    setattr(cur, k, v)
            else:
                if isinstance(cur, Path):
                    sub = Path(sub)
                setattr(cfg, name, sub)
        return cfg


if __name__ == "__main__":
    cfg = Config()
    b = cfg.param_budget()
    w = max(len(k) for k in b)
    for k in ("attention", "ffn", "canon", "norms", "trunk", "embedding",
              "ngram_table", "ngram_proj", "matmul", "total"):
        print(f"  {k:<{w}}  {b[k]:>13,}")
    cfg.assert_budget()
    print(f"\n  steps            {cfg.total_steps:,}")
    for frac, bs, th in cfg.model.context_stages:
        print(f"  ctx from {frac:>5.0%}    block {bs:>5}  theta {th:>9,.0f}  "
              f"micro_batch {cfg.train.micro_batch[bs]}  accum {cfg.accum_at(frac)}")
    print(f"  fingerprint      {cfg.fingerprint()}")