summaryrefslogtreecommitdiff
path: root/config.py
diff options
context:
space:
mode:
Diffstat (limited to 'config.py')
-rw-r--r--config.py629
1 files changed, 629 insertions, 0 deletions
diff --git a/config.py b/config.py
new file mode 100644
index 0000000..02ec2c4
--- /dev/null
+++ b/config.py
@@ -0,0 +1,629 @@
+"""Resolved configuration for Boris-2-D112M-n15M.
+
+A 143.5M-parameter decoder-only model trained from scratch on 200B tokens on a
+single V100 32GB (sm_70). Every number that is a judgement call is a field in
+this file, and the whole object is serialised into every checkpoint, so a
+resumed run cannot silently disagree with the run that wrote the checkpoint.
+
+Lineage. The trunk shape (d512 x 31L), Canon layers and the n-gram table come
+from Boris-1.7/1.8, which validated them at 60M dense on a 3060. Boris-2 is not
+1.8 scaled up: the FFN aspect ratio is deliberately different (4.6x rather than
+2.1x, chosen for Volta's GEMM shapes as much as for capacity), Canon is fused
+into one kernel over the concatenated Q|K|V block instead of three convolutions,
+attention is varlen over a jagged nested tensor rather than dense-masked, the
+context length changes partway through the run, and Newton-Schulz is batched.
+
+Hardware. GPU 0 is the V100 and trains. GPU 1 is a 3060 and only ever runs
+eval.py, in a separate process, from weights-only checkpoints. There is no DDP
+here and there is not meant to be.
+"""
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass, field, fields
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent
+DATA_DIR = ROOT / "data" # -> /media/joseph/Media/boris-2-data
+CKPT_DIR = ROOT / "checkpoints" # -> /media/joseph/Media/boris-2-ckpt
+LOG_DIR = ROOT / "logs"
+EVAL_DIR = ROOT / "eval_out"
+RUN_0920_DIR = Path("/media/joseph/Media/Boris-2-0920")
+# The ACTIVE run directory (eval exports, checkpoints, run name). BORIS_RUN_DIR
+# selects a new run without touching 0920, whose n-gram vocab is shared.
+RUN_DIR = Path(__import__("os").environ.get("BORIS_RUN_DIR", "/media/joseph/Media/Boris-2-0929"))
+LEGACY_TOKENIZER_SHA256 = "0eabe7bb2467338230acaf6fc99321668929391a1bd90b7d27138ded627e8b7c"
+
+
+# --------------------------------------------------------------------------- #
+# Model
+# --------------------------------------------------------------------------- #
+@dataclass
+class ModelConfig:
+ architecture: str = "boris-2-canon-ngram"
+
+ # Depth-thin, carried over from 1.8. MobileLLM (2402.14905) finds depth
+ # matters independently of parameter count below 1B, with the optimum near
+ # 30 layers; d_model 512 is the hidden-size threshold replications identify,
+ # so the shape stops there rather than going thinner.
+ # 2026-09-17 rebalance. Measured against GPT-X2.5-135M, which beats Boris-2
+ # by 9.28 Int Index with PIQA + HellaSwag accounting for 79% of the gap.
+ # Those are context-integration tasks and attention was the component this
+ # model was starved of: 17.0M attention params (15.3% of each block) against
+ # their 26.5M (22.9%). GQA 4:1 exists to shrink the KV cache at INFERENCE --
+ # a cost this run never pays, since it trains on one V100 and is not served.
+ # So: full MHA (2 KV heads -> 8), depth 26 -> 31, and the FFN ratio pulled
+ # from 4.63x to 3.25x to fund it. The resulting 31-layer shape measures
+ # about 35.5k tok/s in the live 0917 run.
+ # d_model stays 512 so the 32,768 BPE, the
+ # n-gram table and its (512, 64) projection are all unchanged.
+ # attention 17.0M -> 32.5M (+91%)
+ # total 143,504,896 -> 143,724,032 (+0.15%)
+ d_model: int = 512
+ n_layer: int = 31
+ n_head: int = 8
+ n_kv_head: int = 8 # full MHA; was GQA 4:1
+ vocab_size: int = 32_768 # our own BPE; power of two, uint16 storage
+ # SHA-256 of tokenizer.json. Token ids are part of the model definition:
+ # two 32,768-entry BPEs are not checkpoint-compatible merely because their
+ # tensor shapes match. Keep this in the config fingerprint so a resume
+ # cannot silently attach old embeddings to a new vocabulary.
+ tokenizer_sha256: str = "f366974b96357d11a7e9930826efd81d77438638c0e21ea1517f620da72e05f4"
+ norm_eps: float = 1e-5
+ tie_embeddings: bool = True
+
+ # SwiGLU inner width. 2368 is not a conventional aspect ratio -- it is 4.63x
+ # d_model where 2.7x is normal -- and it is chosen twice over:
+ #
+ # Set explicitly rather than derived from a multiplier, so the fixed
+ # parameter budget cannot drift as the attention allocation changes.
+ ffn_hidden: int = 1664 # 3.25x; was 2368 (4.63x)
+
+ # --- Canon ---------------------------------------------------------- #
+ # Trainable causal depthwise 1-D convolutions with a residual, applied to
+ # the projected Q, K and V before attention (2607.18413, which ablated 17
+ # placements in a Qwen3 block and landed here). Kernel 3, no norm, no
+ # activation, zero-initialised so the layer starts as exactly the identity.
+ #
+ # Depthwise convolution is per-channel, so one convolution over the
+ # concatenated Q|K|V channel block is mathematically identical to three
+ # separate ones. model.py exploits that: 31 kernels per forward, not 93.
+ use_canon: bool = True
+ canon_kernel: int = 3
+
+ # --- n-gram table ---------------------------------------------------- #
+ # Construction-time arm. False builds the trunk with no table and no
+ # projection; the training script is otherwise byte-identical, so the
+ # control is this same file with one flag.
+ #
+ # The transformer is identical either way. False just drops the table and
+ # its projection; nothing else moves.
+ #
+ # use_ngram=True 143,504,896
+ # use_ngram=False 128,472,064 (same trunk, minus the 15M table)
+ use_ngram: bool = True
+
+ # --- Context length ------------------------------------------------- #
+ # Two stages. The model trains at 2048 for the bulk of the run and extends
+ # to 8192 at 80% of the token budget -- the same relative point X2.5 used
+ # (60B of 75B). A stage is (token_fraction, block_size, rope_theta).
+ #
+ # Extension is cheap here in a way it is not for dense-masked attention:
+ # attention runs varlen per document, so its cost is sum(len_i^2) and does
+ # not grow just because the packing window did. What actually costs is the
+ # long documents themselves -- which is the point of the stage, and why
+ # curriculum.long_doc_floor_tokens rises with it.
+ #
+ # A config, not a fixed choice: it is cheap to move once the loss curve is
+ # visible, and moving it does not invalidate any checkpoint.
+ context_stages: tuple = ((0.00, 2048, 10_000.0),
+ (0.80, 8192, 500_000.0))
+
+ @property
+ def head_dim(self) -> int:
+ assert self.d_model % self.n_head == 0
+ return self.d_model // self.n_head
+
+ @property
+ def q_dim(self) -> int:
+ return self.n_head * self.head_dim
+
+ @property
+ def kv_dim(self) -> int:
+ return self.n_kv_head * self.head_dim
+
+ @property
+ def canon_channels(self) -> int:
+ """One fused Canon acts on Q|K|V together."""
+ return self.q_dim + 2 * self.kv_dim
+
+ @property
+ def max_block_size(self) -> int:
+ return max(int(b) for _, b, _ in self.context_stages)
+
+ # -- parameter accounting; the single source of truth for the asserts -- #
+ def attn_params(self) -> int:
+ d = self.d_model
+ return d * self.q_dim + 2 * d * self.kv_dim + self.q_dim * d
+
+ def ffn_params(self) -> int:
+ return 3 * self.d_model * self.ffn_hidden
+
+ def canon_params(self) -> int:
+ return self.n_layer * self.canon_kernel * self.canon_channels if self.use_canon else 0
+
+ def norm_params(self) -> int:
+ return self.n_layer * 2 * self.d_model + self.d_model
+
+ def matmul_params(self) -> int:
+ """Params that drive a matmul -- what Muon operates on, trunk only."""
+ return self.n_layer * (self.attn_params() + self.ffn_params())
+
+ def embed_params(self) -> int:
+ return self.vocab_size * self.d_model * (1 if self.tie_embeddings else 2)
+
+
+# --------------------------------------------------------------------------- #
+# N-gram table
+# --------------------------------------------------------------------------- #
+@dataclass
+class NGramConfig:
+ """A lookup, never a matmul.
+
+ Rows are an explicit top-K vocabulary: a specific (t-1, t) or (t-2, t-1, t)
+ owns a row, or it falls to the single shared OOV row. No hashing, so two
+ unrelated phrases can never silently share a vector.
+
+ Layout is chosen for the gather, not for the file. Rows are written in
+ descending corpus frequency, so the hot head of the table is contiguous:
+ rank 64 in fp32 is 256 bytes per row, and the top ~24k rows -- which cover
+ the large majority of lookups -- fit inside the V100's 6MB L2. A table
+ ordered by n-gram id instead would scatter every batch across 60MB.
+ """
+
+ rank: int = 64 # 256B/row fp32: 4 aligned cache lines
+ bigram_rows: int = 210_938 # held-out optimum: about 90% of rows
+ trigram_rows: int = 23_437 # preserve the strongest specific phrases
+ oov_rows: int = 1
+
+ # Added to the residual stream after this many blocks. Not at the embedding
+ # sum: a retrieved phrase vector is meant to enter a stream that already
+ # carries some context, rather than be re-encoded from scratch by the full
+ # depth of the network. 1.7/1.8 used 2 and this keeps the comparison.
+ insert_after: int = 2
+
+ vocab_path: Path = RUN_0920_DIR / "ngram_vocab.bin"
+
+ target_params: int = 15_000_000
+ tolerance: float = 0.01
+
+ table_init_std: float = 0.02
+ proj_init_std: float = 0.02
+ # The projection is a 2-D linear map -> Muon. The table never is.
+ muon_on_proj: bool = True
+
+ @property
+ def n_rows(self) -> int:
+ return self.oov_rows + self.bigram_rows + self.trigram_rows
+
+ @property
+ def oov_row(self) -> int:
+ return 0
+
+ @property
+ def bigram_base(self) -> int:
+ return self.oov_rows
+
+ @property
+ def trigram_base(self) -> int:
+ return self.oov_rows + self.bigram_rows
+
+ def table_params(self, d_model: int) -> int:
+ return self.n_rows * self.rank + self.rank * d_model
+
+
+# --------------------------------------------------------------------------- #
+# Data
+# --------------------------------------------------------------------------- #
+@dataclass
+class DataConfig:
+ root: Path = DATA_DIR
+ # 2026-09-17, Boris-2-0917. The seven hand-weighted sources are gone.
+ # ClimbMix (2504.13161) is a 400B-token corpus whose mixture was SEARCHED
+ # rather than guessed: 1,000 topic clusters, pruned by ad/educational/
+ # informational/quality classifiers at a loose threshold, then mixed by
+ # iterative bootstrapping against the PIQA, ARC-E and HellaSwag validation
+ # sets. Those three are 79% of the Int Index gap Boris-2-0907 showed against
+ # GPT-X2.5-135M. At equal token budget it beats Nemotron-CC, DCLM, SmolLM-
+ # corpus and FineWeb-Edu, and its 350M model beats SmolLM-360M on exactly
+ # the two benchmarks 0907 was weakest on (PIQA +1.03, HellaSwag +2.75).
+ # We take 200B of the 369B (under our BPE) -- sound only because the
+ # karpathy/climbmix-400b-shuffle mirror is pre-shuffled, so a prefix is a
+ # representative sample. Written as climbmix + climbmix_s1..s15 and
+ # stitched by MultiStore.
+ # finemath is the one source kept from 0907: ArithMark-3 was the single
+ # benchmark where 0907 was competitive (37.40 vs 38.40), and ClimbMix was
+ # not optimised for arithmetic.
+ sources: tuple = ("climbmix", "finemath")
+ val_path: Path = DATA_DIR / "val.bin"
+ val_source: str = "climbmix_val"
+ # Documents longer than this are split across rows rather than truncated,
+ # so nothing in the corpus is unreachable.
+ max_doc_tokens: int = 65_536
+ # Rows are packed, not EOS-concatenated: attention and Canon both isolate
+ # documents, so a row may hold many and none of them leak into another.
+ eos_id: int = 0
+ prefetch_depth: int = 4
+
+
+@dataclass
+class CurriculumConfig:
+ """Token-threshold mixture weights, interpolated between phases.
+
+ Never a static ratio: the decay phase is supposed to consolidate onto the
+ densest data, and that only happens if the mixture is a function of
+ progress. Weights are normalised at use, so they need not sum to 1 here.
+ """
+
+ enabled: bool = True
+ smooth: bool = True # interpolate between phase endpoints
+
+ # (name, progress_at_end, weights)
+ # Decay consolidates onto the densest data and drops general web text
+ # entirely: Ultra-FineWeb's L2 weight moves to its L3 refinement (Q&A pairs
+ # and multi-style rewrites of the same corpus), which is what openbmb built
+ # L3 for. Weights ramp smoothly, so L3 averages ~0.125 across the decay
+ # phase rather than 0.25.
+ # 2026-09-17: a first phase was added. The run previously fed 30% synthetic
+ # and structured text (cosmopedia + finephrase) from step 0. GPT-X2.5 runs
+ # 98% natural web (58% FineWeb-Edu + 40% DCLM) and introduces Cosmopedia
+ # only at 40B of 75B. Boris-2's two weakest benchmarks -- PIQA and HellaSwag,
+ # 79% of the Int Index gap -- are exactly what broad natural text teaches,
+ # and Boris-2 scores WORSE on held-out fw-edu (3.2017) than on its own val
+ # (3.0420), i.e. it is under-exposed. Phase 0 is held constant (mixture_at
+ # does not interpolate into the first phase), then ramps to the original
+ # stable weights by 0.87. Decay is unchanged.
+ # FLAT. ClimbMix's weights are the output of a search over the whole
+ # mixture; re-weighting it by phase would undo the thing we adopted it for.
+ # The WSD decay phase still decays the LEARNING RATE -- it just no longer
+ # swaps the data underneath it. One phase ending at 1.00 means mixture_at()
+ # returns these weights constantly (it does not interpolate into phase 0).
+ phases: tuple = (
+ ("flat", 1.00, {"climbmix": 0.95, "finemath": 0.05}),
+ )
+
+ # When the context stage lengthens, short documents stop teaching long
+ # context. From that point the packer prefers documents at least this long
+ # where a source has them, falling back when it does not.
+ long_doc_floor_tokens: int = 4096
+ long_doc_target_frac: float = 0.5 # of tokens drawn, once extended
+
+
+# --------------------------------------------------------------------------- #
+# Optimisation
+# --------------------------------------------------------------------------- #
+@dataclass
+class OptimConfig:
+ """Muon on 2-D matmul weights, AdamW on everything else.
+
+ The split is by role, assigned at construction, never by sniffing ndim.
+ Canon kernels are (C, 3) tensors -- two-dimensional, and emphatically not
+ matrices to orthogonalise. A shape rule would hand all 31 of them to Muon
+ and nothing would ever complain.
+ """
+
+ muon_lr: float = 0.02
+ muon_momentum: float = 0.95
+ muon_nesterov: bool = True
+ muon_ns_steps: int = 5
+ muon_weight_decay: float = 0.1
+ # Newton-Schulz over same-shaped parameters stacked into one bmm. At 31
+ # layers this collapses ~2000 small launches per step into ~60. Run in
+ # fp32: 1.8 measured that a bf16 iteration diverges 5.4e-02 relative from
+ # the fp32 path because five steps amplify rounding well past bf16 epsilon,
+ # and on sm_70 the fp16 alternative is worse. The matrices are small enough
+ # that fp32 costs little -- preflight.py reports the measured share.
+ muon_batched: bool = True
+ muon_ns_dtype: str = "float32"
+ # Scale the orthogonalised update by 0.2*sqrt(max(m,n)) so one LR behaves
+ # across aspect ratios as different as 512x512 and 2368x512. With a 4.63x
+ # FFN this matters more than it does at a conventional width.
+ muon_rms_scale: bool = True
+
+ adamw_lr: float = 3e-3 # embeddings, table, head, norms, canon
+ adamw_betas: tuple = (0.9, 0.95)
+ adamw_eps: float = 1e-8
+ # Embedding (16.8M) and the n-gram table (~15M) are 22% of the model and had
+ # NO restraint: both measured +21.1% in norm over 20.8B->30.1B while every
+ # Muon-decayed matmul stayed within 1%. See optim.build_optimizers.
+ adamw_weight_decay: float = 0.1 # embedding + ngram_table only
+ adamw_norm_weight_decay: float = 0.0 # norm gains: conventional, keep 0
+ # 2026-10-07: canon (31 x (768, 3), weight 0 = identity) grew ~+20% per 10B
+ # in 0929 (block 11: 16.3 -> 20.2 -> 24.9 over 20/30/40B, max |w| 13.5) while
+ # the norm1 gains in front of it shrank ~12% per 10B to compensate. Decay it
+ # toward identity like the other AdamW tensors. Own group: see optim.py.
+ adamw_canon_weight_decay: float = 0.1
+
+ # 2026-09-17: the fused LM head walks the vocab in chunks and its peak is
+ # one (chunk, 32768) fp32 logits tensor -- 268 MB at 2048. Boris-2-0917 OOMs
+ # at 31.28/31.73 GB with that, because preflight measures peak_allocated in
+ # a clean process and cannot see allocator fragmentation (1.0 GB reserved
+ # unallocated here) or the 280 MB another process holds on the same card.
+ # 1024 halves the head's footprint for a few percent of throughput.
+ lm_head_chunk: int = 1024
+ grad_clip: float = 4.0 # was 1.0; median gn fits 0.0654*D_B^0.646,
+ # crossing 1.0 at ~68B and reaching ~2.0 by 200B
+
+
+@dataclass
+class ScheduleConfig:
+ """WSD. Warmup 1%, stable to 87%, 1-sqrt decay over the final 13%."""
+
+ warmup_frac: float = 0.01 # 2B tokens
+ stable_frac: float = 0.87
+ decay_shape: str = "1-sqrt" # 1-sqrt | linear | cosine
+ final_lr_frac: float = 0.0
+
+
+# --------------------------------------------------------------------------- #
+# Run
+# --------------------------------------------------------------------------- #
+@dataclass
+class TrainConfig:
+ total_tokens: int = 200_000_000_000
+ tokens_per_step: int = 524_288 # 0.5M; step count 381,470
+
+ # Micro-batch per context stage, in ROWS. Sweep with `preflight --sweep`
+ # and set from measurement; early measurements expire, so re-sweep after
+ # any change to the model or the kernels.
+ # Measured on the V100, 2026-09-06, with `preflight.py --sweep`, including
+ # the worst case of one full-block document per row:
+ #
+ # ctx 2048 mb 8 27.9k tok/s 13.0 GB | mb 16 37.7k tok/s 23.9 GB
+ # ctx 8192 mb 2 26.7k tok/s 13.1 GB | mb 4 32.6k tok/s 24.0 GB
+ #
+ # Both stages hold 32,768 tokens per micro-batch, so activation memory is
+ # roughly unchanged when the context lengthens and the accumulation count
+ # stays at 16 -- the step remains exactly tokens_per_step either way. Peak
+ # is 23.8 GB of 31.7, and the worst case measures the same because the
+ # memory-efficient attention kernel is O(N), not O(N^2).
+ #
+ # Re-sweep after any change to the model or the kernels: the optimum moves
+ # as bottlenecks fall.
+ micro_batch: dict = field(default_factory=lambda: {2048: 16, 8192: 4})
+
+ device: str = "cuda:0"
+ eval_device: str = "cuda:1" # eval.py only; training never touches it
+ seed: int = 20260906
+
+ # fp16 everywhere. sm_70 tensor cores are fp16-in/fp32-accumulate and have
+ # no bf16 path at all; TF32 does not exist before sm_80. Both are asserted
+ # at startup rather than assumed.
+ amp_dtype: str = "float16"
+ init_scale: float = 2.0 ** 16
+ # GradScaler has no floor of its own, and that is fatal here. Below roughly
+ # 1e-30 `unscale_` computes grad/scale and overflows fp32 to inf, which sets
+ # found_inf, which halves the scale again: the collapse becomes
+ # self-reinforcing and has no exit. That is exactly how the run that reached
+ # step 3,810 died -- the scale bled 131,072 -> 0 over 170 steps while the
+ # weights stayed perfectly finite, and every step after that was skipped.
+ # Floored at 1.0 the same event costs a few skipped steps instead of the run.
+ min_loss_scale: float = 1.0
+ # Ceiling, for the other end. The scale grew to 131,072 at step ~2,000 and
+ # the first overflow appeared at 3,460; halving the ceiling buys a factor of
+ # two of headroom in the backward and costs nothing but the smallest
+ # gradients, which are already far below where they matter.
+ max_loss_scale: float = 2.0 ** 15
+ # Steps pinned at min_loss_scale before the run stops itself. At the floor
+ # nothing is training -- every step is being skipped -- so this is the real
+ # early warning, ~350 steps ahead of the loss going non-finite.
+ abort_after_floored: int = 100
+
+ compile_regions: bool = True # fixed-shape regions only, never the model
+ # FALSE, and this is not a tuning knob -- an fp16 residual stream cannot
+ # survive 31 layers of this model. The stream grows with depth, and at step
+ # 3,456 (1% of the budget) it measured:
+ # block 0 3,062 block 16 40,640
+ # block 9 16,144 block 24 39,264
+ # block 25 65,617 vs fp16's max of 65,504 <-- over by 113
+ # It saturates to inf, and _rmsnorm then computes rsqrt(inf) = 0 and
+ # inf * 0 = NaN. That NaN is what drove GradScaler to halve its scale
+ # forever (halving a loss scale cannot fix a FORWARD overflow), which is how
+ # two runs died. Recomputed in fp32 the same batch reaches 65,617 and stays
+ # finite. Measured cost: 39.77k -> 36.73k tok/s and 24.0 -> 25.1 GB, about
+ # +4.3 days on the 200B budget. Boris-1.7 never hit this because bf16 on the
+ # 3060 has fp32's exponent range; sm_70 has no bf16 at all.
+ fp16_residual: bool = False
+ # Recompute block activations in the backward instead of storing them, to
+ # buy a larger micro-batch. Costs one extra forward per checkpointed block
+ # (~+11% compute) and buys fatter GEMMs. Whether that trade is positive is
+ # a measurement, not an argument -- preflight.py --sweep with it on/off.
+ # 2026-09-17: ON for Boris-2-0917. Without it the run OOMs -- 30.17 GB
+ # allocated + 0.78 GB fragmentation + 0.28 GB held by another process on
+ # the same card = 31.28 of 31.73. preflight reads ~1 GB LOW because it
+ # measures peak_allocated in a clean process and cannot see either.
+ # every=4 returns 6.5 GB (29.2 -> 22.7 worst-case) for 5.3% throughput.
+ grad_checkpoint: bool = True
+ grad_checkpoint_every: int = 4 # 1 = every block, 4 = every 4th
+
+ # Resume granularity. The 50B checkpoints are archival milestones; they are
+ # not the thing you resume from. A run that will be interrupted needs a
+ # rolling checkpoint on a wall-clock timer.
+ rolling_ckpt_minutes: int = 45
+ rolling_ckpt_keep: int = 2
+ full_ckpt_every_tokens: int = 50_000_000_000
+ export_every_tokens: int = 10_000_000_000 # weights-only, for eval.py
+ log_every_steps: int = 10
+ # Consecutive steps of non-finite loss OR non-finite grad norm before the
+ # run stops itself. One or two are normal (GradScaler probing its scale);
+ # 200 means something is broken and every further step is wasted.
+ abort_after_bad: int = 200
+ val_every_tokens: int = 1_000_000_000
+
+
+@dataclass
+class EvalConfig:
+ device: str = "cuda:1"
+ out_path: Path = RUN_DIR / "eval" / "results.jsonl"
+ export_dir: Path = RUN_DIR / "checkpoints" / "exports"
+ manifest: Path = RUN_DIR / "checkpoints" / "exports" / "manifest.jsonl"
+ tasks: tuple = ("hellaswag", "arc_easy", "arc_challenge", "piqa", "arithmark3")
+ val_tokens: int = 10_000_000
+ batch_size: int = 16
+ poll_seconds: int = 120
+
+
+@dataclass
+class Config:
+ model: ModelConfig = field(default_factory=ModelConfig)
+ ngram: NGramConfig = field(default_factory=NGramConfig)
+ data: DataConfig = field(default_factory=DataConfig)
+ curriculum: CurriculumConfig = field(default_factory=CurriculumConfig)
+ optim: OptimConfig = field(default_factory=OptimConfig)
+ schedule: ScheduleConfig = field(default_factory=ScheduleConfig)
+ train: TrainConfig = field(default_factory=TrainConfig)
+ eval: EvalConfig = field(default_factory=EvalConfig)
+
+ run_name: str = RUN_DIR.name
+ checkpoint_dir: Path = RUN_DIR / "checkpoints"
+ # verify.py builds 4- and 6-layer models to keep the isolation probes fast;
+ # they are not the run, so the band assert does not apply to them.
+ enforce_budget: bool = True
+
+ # -- derived ------------------------------------------------------------ #
+ @property
+ def total_steps(self) -> int:
+ return self.train.total_tokens // self.train.tokens_per_step
+
+ def block_size_at(self, progress: float) -> tuple[int, float]:
+ """(block_size, rope_theta) for this point in the run."""
+ bs, theta = self.model.context_stages[0][1], self.model.context_stages[0][2]
+ for frac, b, th in self.model.context_stages:
+ if progress >= frac:
+ bs, theta = int(b), float(th)
+ return bs, theta
+
+ def micro_batch_at(self, progress: float) -> int:
+ bs, _ = self.block_size_at(progress)
+ return int(self.train.micro_batch[bs])
+
+ def accum_at(self, progress: float) -> int:
+ bs = self.block_size_at(progress)[0]
+ mb = self.micro_batch_at(progress)
+ tokens_per_micro = mb * bs
+ assert self.train.tokens_per_step % tokens_per_micro == 0, (
+ f"tokens_per_step {self.train.tokens_per_step} is not divisible by "
+ f"micro_batch {mb} x block_size {bs} = {tokens_per_micro}; the step "
+ f"size would change when the context stage does")
+ return self.train.tokens_per_step // tokens_per_micro
+
+ # -- parameter budget ---------------------------------------------------- #
+ def param_budget(self) -> dict:
+ m, n = self.model, self.ngram
+ on = m.use_ngram
+ d = {
+ "attention": m.n_layer * m.attn_params(),
+ "ffn": m.n_layer * m.ffn_params(),
+ "canon": m.canon_params(),
+ "norms": m.norm_params(),
+ "embedding": m.embed_params(),
+ "ngram_table": n.n_rows * n.rank if on else 0,
+ "ngram_proj": n.rank * m.d_model if on else 0,
+ }
+ d["trunk"] = d["attention"] + d["ffn"] + d["canon"] + d["norms"]
+ d["matmul"] = d["attention"] + d["ffn"] + d["ngram_proj"]
+ d["total"] = d["trunk"] + d["embedding"] + d["ngram_table"] + d["ngram_proj"]
+ return d
+
+ def assert_budget(self, ceiling: int = 150_000_000,
+ band: tuple[int, int] = (140_000_000, 145_000_000)) -> None:
+ b = self.param_budget()
+ if not self.enforce_budget:
+ return
+ assert b["total"] <= ceiling, (
+ f"total parameters {b['total']:,} exceeds the hard ceiling {ceiling:,}")
+ # The band describes the full model. With the table off the trunk is
+ # unchanged and the total is simply 15M lower, which is the point.
+ if not self.model.use_ngram:
+ return
+ assert band[0] <= b["total"] <= band[1], (
+ f"total parameters {b['total']:,} outside the target band "
+ f"{band[0]:,}-{band[1]:,}; change model.ffn_hidden")
+ tp = self.ngram.table_params(self.model.d_model)
+ tgt, tol = self.ngram.target_params, self.ngram.tolerance
+ assert abs(tp - tgt) <= tol * tgt, (
+ f"n-gram table {tp:,} is more than {tol:.0%} from its {tgt:,} target")
+
+ # -- serialisation ------------------------------------------------------- #
+ def to_dict(self) -> dict:
+ def conv(o):
+ if isinstance(o, Path):
+ return str(o)
+ if isinstance(o, dict):
+ return {str(k): conv(v) for k, v in o.items()}
+ if isinstance(o, (list, tuple)):
+ return [conv(x) for x in o]
+ return o
+ return {f.name: conv(asdict(getattr(self, f.name))
+ if hasattr(getattr(self, f.name), "__dataclass_fields__")
+ else getattr(self, f.name))
+ for f in fields(self)}
+
+ def to_json(self) -> str:
+ return json.dumps(self.to_dict(), indent=2, sort_keys=True)
+
+ def fingerprint(self) -> str:
+ """Hash of everything that must match to resume a checkpoint."""
+ import hashlib
+ keep = {k: self.to_dict()[k] for k in ("model", "ngram", "optim",
+ "schedule", "train")}
+ return hashlib.sha256(json.dumps(keep, sort_keys=True).encode()).hexdigest()[:16]
+
+ @staticmethod
+ def from_json(s: str) -> "Config":
+ raw = json.loads(s)
+ cfg = Config()
+ # Checkpoints predating the tokenizer guard used the original 09-06
+ # BPE. Give those serialized configs their real identity rather than
+ # inheriting the 0920 default because the field is absent.
+ if "model" in raw and "tokenizer_sha256" not in raw["model"]:
+ cfg.model.tokenizer_sha256 = LEGACY_TOKENIZER_SHA256
+ if (raw.get("model", {}).get("tokenizer_sha256", LEGACY_TOKENIZER_SHA256)
+ == LEGACY_TOKENIZER_SHA256):
+ cfg.data.val_source = "val"
+ for name, sub in raw.items():
+ if not hasattr(cfg, name):
+ continue
+ cur = getattr(cfg, name)
+ if hasattr(cur, "__dataclass_fields__") and isinstance(sub, dict):
+ for k, v in sub.items():
+ if not hasattr(cur, k):
+ continue
+ old = getattr(cur, k)
+ if isinstance(old, Path):
+ v = Path(v)
+ elif isinstance(old, tuple) and isinstance(v, list):
+ v = tuple(tuple(x) if isinstance(x, list) else x for x in v)
+ elif isinstance(old, dict) and isinstance(v, dict):
+ v = {int(kk) if str(kk).isdigit() else kk: vv for kk, vv in v.items()}
+ setattr(cur, k, v)
+ else:
+ if isinstance(cur, Path):
+ sub = Path(sub)
+ setattr(cfg, name, sub)
+ return cfg
+
+
+if __name__ == "__main__":
+ cfg = Config()
+ b = cfg.param_budget()
+ w = max(len(k) for k in b)
+ for k in ("attention", "ffn", "canon", "norms", "trunk", "embedding",
+ "ngram_table", "ngram_proj", "matmul", "total"):
+ print(f" {k:<{w}} {b[k]:>13,}")
+ cfg.assert_budget()
+ print(f"\n steps {cfg.total_steps:,}")
+ for frac, bs, th in cfg.model.context_stages:
+ print(f" ctx from {frac:>5.0%} block {bs:>5} theta {th:>9,.0f} "
+ f"micro_batch {cfg.train.micro_batch[bs]} accum {cfg.accum_at(frac)}")
+ print(f" fingerprint {cfg.fingerprint()}")