diff options
Diffstat (limited to 'config.py')
| -rw-r--r-- | config.py | 629 |
1 files changed, 629 insertions, 0 deletions
diff --git a/config.py b/config.py new file mode 100644 index 0000000..02ec2c4 --- /dev/null +++ b/config.py @@ -0,0 +1,629 @@ +"""Resolved configuration for Boris-2-D112M-n15M. + +A 143.5M-parameter decoder-only model trained from scratch on 200B tokens on a +single V100 32GB (sm_70). Every number that is a judgement call is a field in +this file, and the whole object is serialised into every checkpoint, so a +resumed run cannot silently disagree with the run that wrote the checkpoint. + +Lineage. The trunk shape (d512 x 31L), Canon layers and the n-gram table come +from Boris-1.7/1.8, which validated them at 60M dense on a 3060. Boris-2 is not +1.8 scaled up: the FFN aspect ratio is deliberately different (4.6x rather than +2.1x, chosen for Volta's GEMM shapes as much as for capacity), Canon is fused +into one kernel over the concatenated Q|K|V block instead of three convolutions, +attention is varlen over a jagged nested tensor rather than dense-masked, the +context length changes partway through the run, and Newton-Schulz is batched. + +Hardware. GPU 0 is the V100 and trains. GPU 1 is a 3060 and only ever runs +eval.py, in a separate process, from weights-only checkpoints. There is no DDP +here and there is not meant to be. +""" +from __future__ import annotations + +import json +from dataclasses import asdict, dataclass, field, fields +from pathlib import Path + +ROOT = Path(__file__).resolve().parent +DATA_DIR = ROOT / "data" # -> /media/joseph/Media/boris-2-data +CKPT_DIR = ROOT / "checkpoints" # -> /media/joseph/Media/boris-2-ckpt +LOG_DIR = ROOT / "logs" +EVAL_DIR = ROOT / "eval_out" +RUN_0920_DIR = Path("/media/joseph/Media/Boris-2-0920") +# The ACTIVE run directory (eval exports, checkpoints, run name). BORIS_RUN_DIR +# selects a new run without touching 0920, whose n-gram vocab is shared. +RUN_DIR = Path(__import__("os").environ.get("BORIS_RUN_DIR", "/media/joseph/Media/Boris-2-0929")) +LEGACY_TOKENIZER_SHA256 = "0eabe7bb2467338230acaf6fc99321668929391a1bd90b7d27138ded627e8b7c" + + +# --------------------------------------------------------------------------- # +# Model +# --------------------------------------------------------------------------- # +@dataclass +class ModelConfig: + architecture: str = "boris-2-canon-ngram" + + # Depth-thin, carried over from 1.8. MobileLLM (2402.14905) finds depth + # matters independently of parameter count below 1B, with the optimum near + # 30 layers; d_model 512 is the hidden-size threshold replications identify, + # so the shape stops there rather than going thinner. + # 2026-09-17 rebalance. Measured against GPT-X2.5-135M, which beats Boris-2 + # by 9.28 Int Index with PIQA + HellaSwag accounting for 79% of the gap. + # Those are context-integration tasks and attention was the component this + # model was starved of: 17.0M attention params (15.3% of each block) against + # their 26.5M (22.9%). GQA 4:1 exists to shrink the KV cache at INFERENCE -- + # a cost this run never pays, since it trains on one V100 and is not served. + # So: full MHA (2 KV heads -> 8), depth 26 -> 31, and the FFN ratio pulled + # from 4.63x to 3.25x to fund it. The resulting 31-layer shape measures + # about 35.5k tok/s in the live 0917 run. + # d_model stays 512 so the 32,768 BPE, the + # n-gram table and its (512, 64) projection are all unchanged. + # attention 17.0M -> 32.5M (+91%) + # total 143,504,896 -> 143,724,032 (+0.15%) + d_model: int = 512 + n_layer: int = 31 + n_head: int = 8 + n_kv_head: int = 8 # full MHA; was GQA 4:1 + vocab_size: int = 32_768 # our own BPE; power of two, uint16 storage + # SHA-256 of tokenizer.json. Token ids are part of the model definition: + # two 32,768-entry BPEs are not checkpoint-compatible merely because their + # tensor shapes match. Keep this in the config fingerprint so a resume + # cannot silently attach old embeddings to a new vocabulary. + tokenizer_sha256: str = "f366974b96357d11a7e9930826efd81d77438638c0e21ea1517f620da72e05f4" + norm_eps: float = 1e-5 + tie_embeddings: bool = True + + # SwiGLU inner width. 2368 is not a conventional aspect ratio -- it is 4.63x + # d_model where 2.7x is normal -- and it is chosen twice over: + # + # Set explicitly rather than derived from a multiplier, so the fixed + # parameter budget cannot drift as the attention allocation changes. + ffn_hidden: int = 1664 # 3.25x; was 2368 (4.63x) + + # --- Canon ---------------------------------------------------------- # + # Trainable causal depthwise 1-D convolutions with a residual, applied to + # the projected Q, K and V before attention (2607.18413, which ablated 17 + # placements in a Qwen3 block and landed here). Kernel 3, no norm, no + # activation, zero-initialised so the layer starts as exactly the identity. + # + # Depthwise convolution is per-channel, so one convolution over the + # concatenated Q|K|V channel block is mathematically identical to three + # separate ones. model.py exploits that: 31 kernels per forward, not 93. + use_canon: bool = True + canon_kernel: int = 3 + + # --- n-gram table ---------------------------------------------------- # + # Construction-time arm. False builds the trunk with no table and no + # projection; the training script is otherwise byte-identical, so the + # control is this same file with one flag. + # + # The transformer is identical either way. False just drops the table and + # its projection; nothing else moves. + # + # use_ngram=True 143,504,896 + # use_ngram=False 128,472,064 (same trunk, minus the 15M table) + use_ngram: bool = True + + # --- Context length ------------------------------------------------- # + # Two stages. The model trains at 2048 for the bulk of the run and extends + # to 8192 at 80% of the token budget -- the same relative point X2.5 used + # (60B of 75B). A stage is (token_fraction, block_size, rope_theta). + # + # Extension is cheap here in a way it is not for dense-masked attention: + # attention runs varlen per document, so its cost is sum(len_i^2) and does + # not grow just because the packing window did. What actually costs is the + # long documents themselves -- which is the point of the stage, and why + # curriculum.long_doc_floor_tokens rises with it. + # + # A config, not a fixed choice: it is cheap to move once the loss curve is + # visible, and moving it does not invalidate any checkpoint. + context_stages: tuple = ((0.00, 2048, 10_000.0), + (0.80, 8192, 500_000.0)) + + @property + def head_dim(self) -> int: + assert self.d_model % self.n_head == 0 + return self.d_model // self.n_head + + @property + def q_dim(self) -> int: + return self.n_head * self.head_dim + + @property + def kv_dim(self) -> int: + return self.n_kv_head * self.head_dim + + @property + def canon_channels(self) -> int: + """One fused Canon acts on Q|K|V together.""" + return self.q_dim + 2 * self.kv_dim + + @property + def max_block_size(self) -> int: + return max(int(b) for _, b, _ in self.context_stages) + + # -- parameter accounting; the single source of truth for the asserts -- # + def attn_params(self) -> int: + d = self.d_model + return d * self.q_dim + 2 * d * self.kv_dim + self.q_dim * d + + def ffn_params(self) -> int: + return 3 * self.d_model * self.ffn_hidden + + def canon_params(self) -> int: + return self.n_layer * self.canon_kernel * self.canon_channels if self.use_canon else 0 + + def norm_params(self) -> int: + return self.n_layer * 2 * self.d_model + self.d_model + + def matmul_params(self) -> int: + """Params that drive a matmul -- what Muon operates on, trunk only.""" + return self.n_layer * (self.attn_params() + self.ffn_params()) + + def embed_params(self) -> int: + return self.vocab_size * self.d_model * (1 if self.tie_embeddings else 2) + + +# --------------------------------------------------------------------------- # +# N-gram table +# --------------------------------------------------------------------------- # +@dataclass +class NGramConfig: + """A lookup, never a matmul. + + Rows are an explicit top-K vocabulary: a specific (t-1, t) or (t-2, t-1, t) + owns a row, or it falls to the single shared OOV row. No hashing, so two + unrelated phrases can never silently share a vector. + + Layout is chosen for the gather, not for the file. Rows are written in + descending corpus frequency, so the hot head of the table is contiguous: + rank 64 in fp32 is 256 bytes per row, and the top ~24k rows -- which cover + the large majority of lookups -- fit inside the V100's 6MB L2. A table + ordered by n-gram id instead would scatter every batch across 60MB. + """ + + rank: int = 64 # 256B/row fp32: 4 aligned cache lines + bigram_rows: int = 210_938 # held-out optimum: about 90% of rows + trigram_rows: int = 23_437 # preserve the strongest specific phrases + oov_rows: int = 1 + + # Added to the residual stream after this many blocks. Not at the embedding + # sum: a retrieved phrase vector is meant to enter a stream that already + # carries some context, rather than be re-encoded from scratch by the full + # depth of the network. 1.7/1.8 used 2 and this keeps the comparison. + insert_after: int = 2 + + vocab_path: Path = RUN_0920_DIR / "ngram_vocab.bin" + + target_params: int = 15_000_000 + tolerance: float = 0.01 + + table_init_std: float = 0.02 + proj_init_std: float = 0.02 + # The projection is a 2-D linear map -> Muon. The table never is. + muon_on_proj: bool = True + + @property + def n_rows(self) -> int: + return self.oov_rows + self.bigram_rows + self.trigram_rows + + @property + def oov_row(self) -> int: + return 0 + + @property + def bigram_base(self) -> int: + return self.oov_rows + + @property + def trigram_base(self) -> int: + return self.oov_rows + self.bigram_rows + + def table_params(self, d_model: int) -> int: + return self.n_rows * self.rank + self.rank * d_model + + +# --------------------------------------------------------------------------- # +# Data +# --------------------------------------------------------------------------- # +@dataclass +class DataConfig: + root: Path = DATA_DIR + # 2026-09-17, Boris-2-0917. The seven hand-weighted sources are gone. + # ClimbMix (2504.13161) is a 400B-token corpus whose mixture was SEARCHED + # rather than guessed: 1,000 topic clusters, pruned by ad/educational/ + # informational/quality classifiers at a loose threshold, then mixed by + # iterative bootstrapping against the PIQA, ARC-E and HellaSwag validation + # sets. Those three are 79% of the Int Index gap Boris-2-0907 showed against + # GPT-X2.5-135M. At equal token budget it beats Nemotron-CC, DCLM, SmolLM- + # corpus and FineWeb-Edu, and its 350M model beats SmolLM-360M on exactly + # the two benchmarks 0907 was weakest on (PIQA +1.03, HellaSwag +2.75). + # We take 200B of the 369B (under our BPE) -- sound only because the + # karpathy/climbmix-400b-shuffle mirror is pre-shuffled, so a prefix is a + # representative sample. Written as climbmix + climbmix_s1..s15 and + # stitched by MultiStore. + # finemath is the one source kept from 0907: ArithMark-3 was the single + # benchmark where 0907 was competitive (37.40 vs 38.40), and ClimbMix was + # not optimised for arithmetic. + sources: tuple = ("climbmix", "finemath") + val_path: Path = DATA_DIR / "val.bin" + val_source: str = "climbmix_val" + # Documents longer than this are split across rows rather than truncated, + # so nothing in the corpus is unreachable. + max_doc_tokens: int = 65_536 + # Rows are packed, not EOS-concatenated: attention and Canon both isolate + # documents, so a row may hold many and none of them leak into another. + eos_id: int = 0 + prefetch_depth: int = 4 + + +@dataclass +class CurriculumConfig: + """Token-threshold mixture weights, interpolated between phases. + + Never a static ratio: the decay phase is supposed to consolidate onto the + densest data, and that only happens if the mixture is a function of + progress. Weights are normalised at use, so they need not sum to 1 here. + """ + + enabled: bool = True + smooth: bool = True # interpolate between phase endpoints + + # (name, progress_at_end, weights) + # Decay consolidates onto the densest data and drops general web text + # entirely: Ultra-FineWeb's L2 weight moves to its L3 refinement (Q&A pairs + # and multi-style rewrites of the same corpus), which is what openbmb built + # L3 for. Weights ramp smoothly, so L3 averages ~0.125 across the decay + # phase rather than 0.25. + # 2026-09-17: a first phase was added. The run previously fed 30% synthetic + # and structured text (cosmopedia + finephrase) from step 0. GPT-X2.5 runs + # 98% natural web (58% FineWeb-Edu + 40% DCLM) and introduces Cosmopedia + # only at 40B of 75B. Boris-2's two weakest benchmarks -- PIQA and HellaSwag, + # 79% of the Int Index gap -- are exactly what broad natural text teaches, + # and Boris-2 scores WORSE on held-out fw-edu (3.2017) than on its own val + # (3.0420), i.e. it is under-exposed. Phase 0 is held constant (mixture_at + # does not interpolate into the first phase), then ramps to the original + # stable weights by 0.87. Decay is unchanged. + # FLAT. ClimbMix's weights are the output of a search over the whole + # mixture; re-weighting it by phase would undo the thing we adopted it for. + # The WSD decay phase still decays the LEARNING RATE -- it just no longer + # swaps the data underneath it. One phase ending at 1.00 means mixture_at() + # returns these weights constantly (it does not interpolate into phase 0). + phases: tuple = ( + ("flat", 1.00, {"climbmix": 0.95, "finemath": 0.05}), + ) + + # When the context stage lengthens, short documents stop teaching long + # context. From that point the packer prefers documents at least this long + # where a source has them, falling back when it does not. + long_doc_floor_tokens: int = 4096 + long_doc_target_frac: float = 0.5 # of tokens drawn, once extended + + +# --------------------------------------------------------------------------- # +# Optimisation +# --------------------------------------------------------------------------- # +@dataclass +class OptimConfig: + """Muon on 2-D matmul weights, AdamW on everything else. + + The split is by role, assigned at construction, never by sniffing ndim. + Canon kernels are (C, 3) tensors -- two-dimensional, and emphatically not + matrices to orthogonalise. A shape rule would hand all 31 of them to Muon + and nothing would ever complain. + """ + + muon_lr: float = 0.02 + muon_momentum: float = 0.95 + muon_nesterov: bool = True + muon_ns_steps: int = 5 + muon_weight_decay: float = 0.1 + # Newton-Schulz over same-shaped parameters stacked into one bmm. At 31 + # layers this collapses ~2000 small launches per step into ~60. Run in + # fp32: 1.8 measured that a bf16 iteration diverges 5.4e-02 relative from + # the fp32 path because five steps amplify rounding well past bf16 epsilon, + # and on sm_70 the fp16 alternative is worse. The matrices are small enough + # that fp32 costs little -- preflight.py reports the measured share. + muon_batched: bool = True + muon_ns_dtype: str = "float32" + # Scale the orthogonalised update by 0.2*sqrt(max(m,n)) so one LR behaves + # across aspect ratios as different as 512x512 and 2368x512. With a 4.63x + # FFN this matters more than it does at a conventional width. + muon_rms_scale: bool = True + + adamw_lr: float = 3e-3 # embeddings, table, head, norms, canon + adamw_betas: tuple = (0.9, 0.95) + adamw_eps: float = 1e-8 + # Embedding (16.8M) and the n-gram table (~15M) are 22% of the model and had + # NO restraint: both measured +21.1% in norm over 20.8B->30.1B while every + # Muon-decayed matmul stayed within 1%. See optim.build_optimizers. + adamw_weight_decay: float = 0.1 # embedding + ngram_table only + adamw_norm_weight_decay: float = 0.0 # norm gains: conventional, keep 0 + # 2026-10-07: canon (31 x (768, 3), weight 0 = identity) grew ~+20% per 10B + # in 0929 (block 11: 16.3 -> 20.2 -> 24.9 over 20/30/40B, max |w| 13.5) while + # the norm1 gains in front of it shrank ~12% per 10B to compensate. Decay it + # toward identity like the other AdamW tensors. Own group: see optim.py. + adamw_canon_weight_decay: float = 0.1 + + # 2026-09-17: the fused LM head walks the vocab in chunks and its peak is + # one (chunk, 32768) fp32 logits tensor -- 268 MB at 2048. Boris-2-0917 OOMs + # at 31.28/31.73 GB with that, because preflight measures peak_allocated in + # a clean process and cannot see allocator fragmentation (1.0 GB reserved + # unallocated here) or the 280 MB another process holds on the same card. + # 1024 halves the head's footprint for a few percent of throughput. + lm_head_chunk: int = 1024 + grad_clip: float = 4.0 # was 1.0; median gn fits 0.0654*D_B^0.646, + # crossing 1.0 at ~68B and reaching ~2.0 by 200B + + +@dataclass +class ScheduleConfig: + """WSD. Warmup 1%, stable to 87%, 1-sqrt decay over the final 13%.""" + + warmup_frac: float = 0.01 # 2B tokens + stable_frac: float = 0.87 + decay_shape: str = "1-sqrt" # 1-sqrt | linear | cosine + final_lr_frac: float = 0.0 + + +# --------------------------------------------------------------------------- # +# Run +# --------------------------------------------------------------------------- # +@dataclass +class TrainConfig: + total_tokens: int = 200_000_000_000 + tokens_per_step: int = 524_288 # 0.5M; step count 381,470 + + # Micro-batch per context stage, in ROWS. Sweep with `preflight --sweep` + # and set from measurement; early measurements expire, so re-sweep after + # any change to the model or the kernels. + # Measured on the V100, 2026-09-06, with `preflight.py --sweep`, including + # the worst case of one full-block document per row: + # + # ctx 2048 mb 8 27.9k tok/s 13.0 GB | mb 16 37.7k tok/s 23.9 GB + # ctx 8192 mb 2 26.7k tok/s 13.1 GB | mb 4 32.6k tok/s 24.0 GB + # + # Both stages hold 32,768 tokens per micro-batch, so activation memory is + # roughly unchanged when the context lengthens and the accumulation count + # stays at 16 -- the step remains exactly tokens_per_step either way. Peak + # is 23.8 GB of 31.7, and the worst case measures the same because the + # memory-efficient attention kernel is O(N), not O(N^2). + # + # Re-sweep after any change to the model or the kernels: the optimum moves + # as bottlenecks fall. + micro_batch: dict = field(default_factory=lambda: {2048: 16, 8192: 4}) + + device: str = "cuda:0" + eval_device: str = "cuda:1" # eval.py only; training never touches it + seed: int = 20260906 + + # fp16 everywhere. sm_70 tensor cores are fp16-in/fp32-accumulate and have + # no bf16 path at all; TF32 does not exist before sm_80. Both are asserted + # at startup rather than assumed. + amp_dtype: str = "float16" + init_scale: float = 2.0 ** 16 + # GradScaler has no floor of its own, and that is fatal here. Below roughly + # 1e-30 `unscale_` computes grad/scale and overflows fp32 to inf, which sets + # found_inf, which halves the scale again: the collapse becomes + # self-reinforcing and has no exit. That is exactly how the run that reached + # step 3,810 died -- the scale bled 131,072 -> 0 over 170 steps while the + # weights stayed perfectly finite, and every step after that was skipped. + # Floored at 1.0 the same event costs a few skipped steps instead of the run. + min_loss_scale: float = 1.0 + # Ceiling, for the other end. The scale grew to 131,072 at step ~2,000 and + # the first overflow appeared at 3,460; halving the ceiling buys a factor of + # two of headroom in the backward and costs nothing but the smallest + # gradients, which are already far below where they matter. + max_loss_scale: float = 2.0 ** 15 + # Steps pinned at min_loss_scale before the run stops itself. At the floor + # nothing is training -- every step is being skipped -- so this is the real + # early warning, ~350 steps ahead of the loss going non-finite. + abort_after_floored: int = 100 + + compile_regions: bool = True # fixed-shape regions only, never the model + # FALSE, and this is not a tuning knob -- an fp16 residual stream cannot + # survive 31 layers of this model. The stream grows with depth, and at step + # 3,456 (1% of the budget) it measured: + # block 0 3,062 block 16 40,640 + # block 9 16,144 block 24 39,264 + # block 25 65,617 vs fp16's max of 65,504 <-- over by 113 + # It saturates to inf, and _rmsnorm then computes rsqrt(inf) = 0 and + # inf * 0 = NaN. That NaN is what drove GradScaler to halve its scale + # forever (halving a loss scale cannot fix a FORWARD overflow), which is how + # two runs died. Recomputed in fp32 the same batch reaches 65,617 and stays + # finite. Measured cost: 39.77k -> 36.73k tok/s and 24.0 -> 25.1 GB, about + # +4.3 days on the 200B budget. Boris-1.7 never hit this because bf16 on the + # 3060 has fp32's exponent range; sm_70 has no bf16 at all. + fp16_residual: bool = False + # Recompute block activations in the backward instead of storing them, to + # buy a larger micro-batch. Costs one extra forward per checkpointed block + # (~+11% compute) and buys fatter GEMMs. Whether that trade is positive is + # a measurement, not an argument -- preflight.py --sweep with it on/off. + # 2026-09-17: ON for Boris-2-0917. Without it the run OOMs -- 30.17 GB + # allocated + 0.78 GB fragmentation + 0.28 GB held by another process on + # the same card = 31.28 of 31.73. preflight reads ~1 GB LOW because it + # measures peak_allocated in a clean process and cannot see either. + # every=4 returns 6.5 GB (29.2 -> 22.7 worst-case) for 5.3% throughput. + grad_checkpoint: bool = True + grad_checkpoint_every: int = 4 # 1 = every block, 4 = every 4th + + # Resume granularity. The 50B checkpoints are archival milestones; they are + # not the thing you resume from. A run that will be interrupted needs a + # rolling checkpoint on a wall-clock timer. + rolling_ckpt_minutes: int = 45 + rolling_ckpt_keep: int = 2 + full_ckpt_every_tokens: int = 50_000_000_000 + export_every_tokens: int = 10_000_000_000 # weights-only, for eval.py + log_every_steps: int = 10 + # Consecutive steps of non-finite loss OR non-finite grad norm before the + # run stops itself. One or two are normal (GradScaler probing its scale); + # 200 means something is broken and every further step is wasted. + abort_after_bad: int = 200 + val_every_tokens: int = 1_000_000_000 + + +@dataclass +class EvalConfig: + device: str = "cuda:1" + out_path: Path = RUN_DIR / "eval" / "results.jsonl" + export_dir: Path = RUN_DIR / "checkpoints" / "exports" + manifest: Path = RUN_DIR / "checkpoints" / "exports" / "manifest.jsonl" + tasks: tuple = ("hellaswag", "arc_easy", "arc_challenge", "piqa", "arithmark3") + val_tokens: int = 10_000_000 + batch_size: int = 16 + poll_seconds: int = 120 + + +@dataclass +class Config: + model: ModelConfig = field(default_factory=ModelConfig) + ngram: NGramConfig = field(default_factory=NGramConfig) + data: DataConfig = field(default_factory=DataConfig) + curriculum: CurriculumConfig = field(default_factory=CurriculumConfig) + optim: OptimConfig = field(default_factory=OptimConfig) + schedule: ScheduleConfig = field(default_factory=ScheduleConfig) + train: TrainConfig = field(default_factory=TrainConfig) + eval: EvalConfig = field(default_factory=EvalConfig) + + run_name: str = RUN_DIR.name + checkpoint_dir: Path = RUN_DIR / "checkpoints" + # verify.py builds 4- and 6-layer models to keep the isolation probes fast; + # they are not the run, so the band assert does not apply to them. + enforce_budget: bool = True + + # -- derived ------------------------------------------------------------ # + @property + def total_steps(self) -> int: + return self.train.total_tokens // self.train.tokens_per_step + + def block_size_at(self, progress: float) -> tuple[int, float]: + """(block_size, rope_theta) for this point in the run.""" + bs, theta = self.model.context_stages[0][1], self.model.context_stages[0][2] + for frac, b, th in self.model.context_stages: + if progress >= frac: + bs, theta = int(b), float(th) + return bs, theta + + def micro_batch_at(self, progress: float) -> int: + bs, _ = self.block_size_at(progress) + return int(self.train.micro_batch[bs]) + + def accum_at(self, progress: float) -> int: + bs = self.block_size_at(progress)[0] + mb = self.micro_batch_at(progress) + tokens_per_micro = mb * bs + assert self.train.tokens_per_step % tokens_per_micro == 0, ( + f"tokens_per_step {self.train.tokens_per_step} is not divisible by " + f"micro_batch {mb} x block_size {bs} = {tokens_per_micro}; the step " + f"size would change when the context stage does") + return self.train.tokens_per_step // tokens_per_micro + + # -- parameter budget ---------------------------------------------------- # + def param_budget(self) -> dict: + m, n = self.model, self.ngram + on = m.use_ngram + d = { + "attention": m.n_layer * m.attn_params(), + "ffn": m.n_layer * m.ffn_params(), + "canon": m.canon_params(), + "norms": m.norm_params(), + "embedding": m.embed_params(), + "ngram_table": n.n_rows * n.rank if on else 0, + "ngram_proj": n.rank * m.d_model if on else 0, + } + d["trunk"] = d["attention"] + d["ffn"] + d["canon"] + d["norms"] + d["matmul"] = d["attention"] + d["ffn"] + d["ngram_proj"] + d["total"] = d["trunk"] + d["embedding"] + d["ngram_table"] + d["ngram_proj"] + return d + + def assert_budget(self, ceiling: int = 150_000_000, + band: tuple[int, int] = (140_000_000, 145_000_000)) -> None: + b = self.param_budget() + if not self.enforce_budget: + return + assert b["total"] <= ceiling, ( + f"total parameters {b['total']:,} exceeds the hard ceiling {ceiling:,}") + # The band describes the full model. With the table off the trunk is + # unchanged and the total is simply 15M lower, which is the point. + if not self.model.use_ngram: + return + assert band[0] <= b["total"] <= band[1], ( + f"total parameters {b['total']:,} outside the target band " + f"{band[0]:,}-{band[1]:,}; change model.ffn_hidden") + tp = self.ngram.table_params(self.model.d_model) + tgt, tol = self.ngram.target_params, self.ngram.tolerance + assert abs(tp - tgt) <= tol * tgt, ( + f"n-gram table {tp:,} is more than {tol:.0%} from its {tgt:,} target") + + # -- serialisation ------------------------------------------------------- # + def to_dict(self) -> dict: + def conv(o): + if isinstance(o, Path): + return str(o) + if isinstance(o, dict): + return {str(k): conv(v) for k, v in o.items()} + if isinstance(o, (list, tuple)): + return [conv(x) for x in o] + return o + return {f.name: conv(asdict(getattr(self, f.name)) + if hasattr(getattr(self, f.name), "__dataclass_fields__") + else getattr(self, f.name)) + for f in fields(self)} + + def to_json(self) -> str: + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + + def fingerprint(self) -> str: + """Hash of everything that must match to resume a checkpoint.""" + import hashlib + keep = {k: self.to_dict()[k] for k in ("model", "ngram", "optim", + "schedule", "train")} + return hashlib.sha256(json.dumps(keep, sort_keys=True).encode()).hexdigest()[:16] + + @staticmethod + def from_json(s: str) -> "Config": + raw = json.loads(s) + cfg = Config() + # Checkpoints predating the tokenizer guard used the original 09-06 + # BPE. Give those serialized configs their real identity rather than + # inheriting the 0920 default because the field is absent. + if "model" in raw and "tokenizer_sha256" not in raw["model"]: + cfg.model.tokenizer_sha256 = LEGACY_TOKENIZER_SHA256 + if (raw.get("model", {}).get("tokenizer_sha256", LEGACY_TOKENIZER_SHA256) + == LEGACY_TOKENIZER_SHA256): + cfg.data.val_source = "val" + for name, sub in raw.items(): + if not hasattr(cfg, name): + continue + cur = getattr(cfg, name) + if hasattr(cur, "__dataclass_fields__") and isinstance(sub, dict): + for k, v in sub.items(): + if not hasattr(cur, k): + continue + old = getattr(cur, k) + if isinstance(old, Path): + v = Path(v) + elif isinstance(old, tuple) and isinstance(v, list): + v = tuple(tuple(x) if isinstance(x, list) else x for x in v) + elif isinstance(old, dict) and isinstance(v, dict): + v = {int(kk) if str(kk).isdigit() else kk: vv for kk, vv in v.items()} + setattr(cur, k, v) + else: + if isinstance(cur, Path): + sub = Path(sub) + setattr(cfg, name, sub) + return cfg + + +if __name__ == "__main__": + cfg = Config() + b = cfg.param_budget() + w = max(len(k) for k in b) + for k in ("attention", "ffn", "canon", "norms", "trunk", "embedding", + "ngram_table", "ngram_proj", "matmul", "total"): + print(f" {k:<{w}} {b[k]:>13,}") + cfg.assert_budget() + print(f"\n steps {cfg.total_steps:,}") + for frac, bs, th in cfg.model.context_stages: + print(f" ctx from {frac:>5.0%} block {bs:>5} theta {th:>9,.0f} " + f"micro_batch {cfg.train.micro_batch[bs]} accum {cfg.accum_at(frac)}") + print(f" fingerprint {cfg.fingerprint()}") |
