"""Resolved configuration for Boris-2-D112M-n15M. A 143.5M-parameter decoder-only model trained from scratch on 200B tokens on a single V100 32GB (sm_70). Every number that is a judgement call is a field in this file, and the whole object is serialised into every checkpoint, so a resumed run cannot silently disagree with the run that wrote the checkpoint. Lineage. The trunk shape (d512 x 31L), Canon layers and the n-gram table come from Boris-1.7/1.8, which validated them at 60M dense on a 3060. Boris-2 is not 1.8 scaled up: the FFN aspect ratio is deliberately different (4.6x rather than 2.1x, chosen for Volta's GEMM shapes as much as for capacity), Canon is fused into one kernel over the concatenated Q|K|V block instead of three convolutions, attention is varlen over a jagged nested tensor rather than dense-masked, the context length changes partway through the run, and Newton-Schulz is batched. Hardware. GPU 0 is the V100 and trains. GPU 1 is a 3060 and only ever runs eval.py, in a separate process, from weights-only checkpoints. There is no DDP here and there is not meant to be. """ from __future__ import annotations import json from dataclasses import asdict, dataclass, field, fields from pathlib import Path ROOT = Path(__file__).resolve().parent DATA_DIR = ROOT / "data" # -> /media/joseph/Media/boris-2-data CKPT_DIR = ROOT / "checkpoints" # -> /media/joseph/Media/boris-2-ckpt LOG_DIR = ROOT / "logs" EVAL_DIR = ROOT / "eval_out" RUN_0920_DIR = Path("/media/joseph/Media/Boris-2-0920") # The ACTIVE run directory (eval exports, checkpoints, run name). BORIS_RUN_DIR # selects a new run without touching 0920, whose n-gram vocab is shared. RUN_DIR = Path(__import__("os").environ.get("BORIS_RUN_DIR", "/media/joseph/Media/Boris-2-0929")) LEGACY_TOKENIZER_SHA256 = "0eabe7bb2467338230acaf6fc99321668929391a1bd90b7d27138ded627e8b7c" # --------------------------------------------------------------------------- # # Model # --------------------------------------------------------------------------- # @dataclass class ModelConfig: architecture: str = "boris-2-canon-ngram" # Depth-thin, carried over from 1.8. MobileLLM (2402.14905) finds depth # matters independently of parameter count below 1B, with the optimum near # 30 layers; d_model 512 is the hidden-size threshold replications identify, # so the shape stops there rather than going thinner. # 2026-09-17 rebalance. Measured against GPT-X2.5-135M, which beats Boris-2 # by 9.28 Int Index with PIQA + HellaSwag accounting for 79% of the gap. # Those are context-integration tasks and attention was the component this # model was starved of: 17.0M attention params (15.3% of each block) against # their 26.5M (22.9%). GQA 4:1 exists to shrink the KV cache at INFERENCE -- # a cost this run never pays, since it trains on one V100 and is not served. # So: full MHA (2 KV heads -> 8), depth 26 -> 31, and the FFN ratio pulled # from 4.63x to 3.25x to fund it. The resulting 31-layer shape measures # about 35.5k tok/s in the live 0917 run. # d_model stays 512 so the 32,768 BPE, the # n-gram table and its (512, 64) projection are all unchanged. # attention 17.0M -> 32.5M (+91%) # total 143,504,896 -> 143,724,032 (+0.15%) d_model: int = 512 n_layer: int = 31 n_head: int = 8 n_kv_head: int = 8 # full MHA; was GQA 4:1 vocab_size: int = 32_768 # our own BPE; power of two, uint16 storage # SHA-256 of tokenizer.json. Token ids are part of the model definition: # two 32,768-entry BPEs are not checkpoint-compatible merely because their # tensor shapes match. Keep this in the config fingerprint so a resume # cannot silently attach old embeddings to a new vocabulary. tokenizer_sha256: str = "f366974b96357d11a7e9930826efd81d77438638c0e21ea1517f620da72e05f4" norm_eps: float = 1e-5 tie_embeddings: bool = True # SwiGLU inner width. 2368 is not a conventional aspect ratio -- it is 4.63x # d_model where 2.7x is normal -- and it is chosen twice over: # # Set explicitly rather than derived from a multiplier, so the fixed # parameter budget cannot drift as the attention allocation changes. ffn_hidden: int = 1664 # 3.25x; was 2368 (4.63x) # --- Canon ---------------------------------------------------------- # # Trainable causal depthwise 1-D convolutions with a residual, applied to # the projected Q, K and V before attention (2607.18413, which ablated 17 # placements in a Qwen3 block and landed here). Kernel 3, no norm, no # activation, zero-initialised so the layer starts as exactly the identity. # # Depthwise convolution is per-channel, so one convolution over the # concatenated Q|K|V channel block is mathematically identical to three # separate ones. model.py exploits that: 31 kernels per forward, not 93. use_canon: bool = True canon_kernel: int = 3 # --- n-gram table ---------------------------------------------------- # # Construction-time arm. False builds the trunk with no table and no # projection; the training script is otherwise byte-identical, so the # control is this same file with one flag. # # The transformer is identical either way. False just drops the table and # its projection; nothing else moves. # # use_ngram=True 143,504,896 # use_ngram=False 128,472,064 (same trunk, minus the 15M table) use_ngram: bool = True # --- Context length ------------------------------------------------- # # Two stages. The model trains at 2048 for the bulk of the run and extends # to 8192 at 80% of the token budget -- the same relative point X2.5 used # (60B of 75B). A stage is (token_fraction, block_size, rope_theta). # # Extension is cheap here in a way it is not for dense-masked attention: # attention runs varlen per document, so its cost is sum(len_i^2) and does # not grow just because the packing window did. What actually costs is the # long documents themselves -- which is the point of the stage, and why # curriculum.long_doc_floor_tokens rises with it. # # A config, not a fixed choice: it is cheap to move once the loss curve is # visible, and moving it does not invalidate any checkpoint. context_stages: tuple = ((0.00, 2048, 10_000.0), (0.80, 8192, 500_000.0)) @property def head_dim(self) -> int: assert self.d_model % self.n_head == 0 return self.d_model // self.n_head @property def q_dim(self) -> int: return self.n_head * self.head_dim @property def kv_dim(self) -> int: return self.n_kv_head * self.head_dim @property def canon_channels(self) -> int: """One fused Canon acts on Q|K|V together.""" return self.q_dim + 2 * self.kv_dim @property def max_block_size(self) -> int: return max(int(b) for _, b, _ in self.context_stages) # -- parameter accounting; the single source of truth for the asserts -- # def attn_params(self) -> int: d = self.d_model return d * self.q_dim + 2 * d * self.kv_dim + self.q_dim * d def ffn_params(self) -> int: return 3 * self.d_model * self.ffn_hidden def canon_params(self) -> int: return self.n_layer * self.canon_kernel * self.canon_channels if self.use_canon else 0 def norm_params(self) -> int: return self.n_layer * 2 * self.d_model + self.d_model def matmul_params(self) -> int: """Params that drive a matmul -- what Muon operates on, trunk only.""" return self.n_layer * (self.attn_params() + self.ffn_params()) def embed_params(self) -> int: return self.vocab_size * self.d_model * (1 if self.tie_embeddings else 2) # --------------------------------------------------------------------------- # # N-gram table # --------------------------------------------------------------------------- # @dataclass class NGramConfig: """A lookup, never a matmul. Rows are an explicit top-K vocabulary: a specific (t-1, t) or (t-2, t-1, t) owns a row, or it falls to the single shared OOV row. No hashing, so two unrelated phrases can never silently share a vector. Layout is chosen for the gather, not for the file. Rows are written in descending corpus frequency, so the hot head of the table is contiguous: rank 64 in fp32 is 256 bytes per row, and the top ~24k rows -- which cover the large majority of lookups -- fit inside the V100's 6MB L2. A table ordered by n-gram id instead would scatter every batch across 60MB. """ rank: int = 64 # 256B/row fp32: 4 aligned cache lines bigram_rows: int = 210_938 # held-out optimum: about 90% of rows trigram_rows: int = 23_437 # preserve the strongest specific phrases oov_rows: int = 1 # Added to the residual stream after this many blocks. Not at the embedding # sum: a retrieved phrase vector is meant to enter a stream that already # carries some context, rather than be re-encoded from scratch by the full # depth of the network. 1.7/1.8 used 2 and this keeps the comparison. insert_after: int = 2 vocab_path: Path = RUN_0920_DIR / "ngram_vocab.bin" target_params: int = 15_000_000 tolerance: float = 0.01 table_init_std: float = 0.02 proj_init_std: float = 0.02 # The projection is a 2-D linear map -> Muon. The table never is. muon_on_proj: bool = True @property def n_rows(self) -> int: return self.oov_rows + self.bigram_rows + self.trigram_rows @property def oov_row(self) -> int: return 0 @property def bigram_base(self) -> int: return self.oov_rows @property def trigram_base(self) -> int: return self.oov_rows + self.bigram_rows def table_params(self, d_model: int) -> int: return self.n_rows * self.rank + self.rank * d_model # --------------------------------------------------------------------------- # # Data # --------------------------------------------------------------------------- # @dataclass class DataConfig: root: Path = DATA_DIR # 2026-09-17, Boris-2-0917. The seven hand-weighted sources are gone. # ClimbMix (2504.13161) is a 400B-token corpus whose mixture was SEARCHED # rather than guessed: 1,000 topic clusters, pruned by ad/educational/ # informational/quality classifiers at a loose threshold, then mixed by # iterative bootstrapping against the PIQA, ARC-E and HellaSwag validation # sets. Those three are 79% of the Int Index gap Boris-2-0907 showed against # GPT-X2.5-135M. At equal token budget it beats Nemotron-CC, DCLM, SmolLM- # corpus and FineWeb-Edu, and its 350M model beats SmolLM-360M on exactly # the two benchmarks 0907 was weakest on (PIQA +1.03, HellaSwag +2.75). # We take 200B of the 369B (under our BPE) -- sound only because the # karpathy/climbmix-400b-shuffle mirror is pre-shuffled, so a prefix is a # representative sample. Written as climbmix + climbmix_s1..s15 and # stitched by MultiStore. # finemath is the one source kept from 0907: ArithMark-3 was the single # benchmark where 0907 was competitive (37.40 vs 38.40), and ClimbMix was # not optimised for arithmetic. sources: tuple = ("climbmix", "finemath") val_path: Path = DATA_DIR / "val.bin" val_source: str = "climbmix_val" # Documents longer than this are split across rows rather than truncated, # so nothing in the corpus is unreachable. max_doc_tokens: int = 65_536 # Rows are packed, not EOS-concatenated: attention and Canon both isolate # documents, so a row may hold many and none of them leak into another. eos_id: int = 0 prefetch_depth: int = 4 @dataclass class CurriculumConfig: """Token-threshold mixture weights, interpolated between phases. Never a static ratio: the decay phase is supposed to consolidate onto the densest data, and that only happens if the mixture is a function of progress. Weights are normalised at use, so they need not sum to 1 here. """ enabled: bool = True smooth: bool = True # interpolate between phase endpoints # (name, progress_at_end, weights) # Decay consolidates onto the densest data and drops general web text # entirely: Ultra-FineWeb's L2 weight moves to its L3 refinement (Q&A pairs # and multi-style rewrites of the same corpus), which is what openbmb built # L3 for. Weights ramp smoothly, so L3 averages ~0.125 across the decay # phase rather than 0.25. # 2026-09-17: a first phase was added. The run previously fed 30% synthetic # and structured text (cosmopedia + finephrase) from step 0. GPT-X2.5 runs # 98% natural web (58% FineWeb-Edu + 40% DCLM) and introduces Cosmopedia # only at 40B of 75B. Boris-2's two weakest benchmarks -- PIQA and HellaSwag, # 79% of the Int Index gap -- are exactly what broad natural text teaches, # and Boris-2 scores WORSE on held-out fw-edu (3.2017) than on its own val # (3.0420), i.e. it is under-exposed. Phase 0 is held constant (mixture_at # does not interpolate into the first phase), then ramps to the original # stable weights by 0.87. Decay is unchanged. # FLAT. ClimbMix's weights are the output of a search over the whole # mixture; re-weighting it by phase would undo the thing we adopted it for. # The WSD decay phase still decays the LEARNING RATE -- it just no longer # swaps the data underneath it. One phase ending at 1.00 means mixture_at() # returns these weights constantly (it does not interpolate into phase 0). phases: tuple = ( ("flat", 1.00, {"climbmix": 0.95, "finemath": 0.05}), ) # When the context stage lengthens, short documents stop teaching long # context. From that point the packer prefers documents at least this long # where a source has them, falling back when it does not. long_doc_floor_tokens: int = 4096 long_doc_target_frac: float = 0.5 # of tokens drawn, once extended # --------------------------------------------------------------------------- # # Optimisation # --------------------------------------------------------------------------- # @dataclass class OptimConfig: """Muon on 2-D matmul weights, AdamW on everything else. The split is by role, assigned at construction, never by sniffing ndim. Canon kernels are (C, 3) tensors -- two-dimensional, and emphatically not matrices to orthogonalise. A shape rule would hand all 31 of them to Muon and nothing would ever complain. """ muon_lr: float = 0.02 muon_momentum: float = 0.95 muon_nesterov: bool = True muon_ns_steps: int = 5 muon_weight_decay: float = 0.1 # Newton-Schulz over same-shaped parameters stacked into one bmm. At 31 # layers this collapses ~2000 small launches per step into ~60. Run in # fp32: 1.8 measured that a bf16 iteration diverges 5.4e-02 relative from # the fp32 path because five steps amplify rounding well past bf16 epsilon, # and on sm_70 the fp16 alternative is worse. The matrices are small enough # that fp32 costs little -- preflight.py reports the measured share. muon_batched: bool = True muon_ns_dtype: str = "float32" # Scale the orthogonalised update by 0.2*sqrt(max(m,n)) so one LR behaves # across aspect ratios as different as 512x512 and 2368x512. With a 4.63x # FFN this matters more than it does at a conventional width. muon_rms_scale: bool = True adamw_lr: float = 3e-3 # embeddings, table, head, norms, canon adamw_betas: tuple = (0.9, 0.95) adamw_eps: float = 1e-8 # Embedding (16.8M) and the n-gram table (~15M) are 22% of the model and had # NO restraint: both measured +21.1% in norm over 20.8B->30.1B while every # Muon-decayed matmul stayed within 1%. See optim.build_optimizers. adamw_weight_decay: float = 0.1 # embedding + ngram_table only adamw_norm_weight_decay: float = 0.0 # norm gains: conventional, keep 0 # 2026-10-07: canon (31 x (768, 3), weight 0 = identity) grew ~+20% per 10B # in 0929 (block 11: 16.3 -> 20.2 -> 24.9 over 20/30/40B, max |w| 13.5) while # the norm1 gains in front of it shrank ~12% per 10B to compensate. Decay it # toward identity like the other AdamW tensors. Own group: see optim.py. adamw_canon_weight_decay: float = 0.1 # 2026-09-17: the fused LM head walks the vocab in chunks and its peak is # one (chunk, 32768) fp32 logits tensor -- 268 MB at 2048. Boris-2-0917 OOMs # at 31.28/31.73 GB with that, because preflight measures peak_allocated in # a clean process and cannot see allocator fragmentation (1.0 GB reserved # unallocated here) or the 280 MB another process holds on the same card. # 1024 halves the head's footprint for a few percent of throughput. lm_head_chunk: int = 1024 grad_clip: float = 4.0 # was 1.0; median gn fits 0.0654*D_B^0.646, # crossing 1.0 at ~68B and reaching ~2.0 by 200B @dataclass class ScheduleConfig: """WSD. Warmup 1%, stable to 87%, 1-sqrt decay over the final 13%.""" warmup_frac: float = 0.01 # 2B tokens stable_frac: float = 0.87 decay_shape: str = "1-sqrt" # 1-sqrt | linear | cosine final_lr_frac: float = 0.0 # --------------------------------------------------------------------------- # # Run # --------------------------------------------------------------------------- # @dataclass class TrainConfig: total_tokens: int = 200_000_000_000 tokens_per_step: int = 524_288 # 0.5M; step count 381,470 # Micro-batch per context stage, in ROWS. Sweep with `preflight --sweep` # and set from measurement; early measurements expire, so re-sweep after # any change to the model or the kernels. # Measured on the V100, 2026-09-06, with `preflight.py --sweep`, including # the worst case of one full-block document per row: # # ctx 2048 mb 8 27.9k tok/s 13.0 GB | mb 16 37.7k tok/s 23.9 GB # ctx 8192 mb 2 26.7k tok/s 13.1 GB | mb 4 32.6k tok/s 24.0 GB # # Both stages hold 32,768 tokens per micro-batch, so activation memory is # roughly unchanged when the context lengthens and the accumulation count # stays at 16 -- the step remains exactly tokens_per_step either way. Peak # is 23.8 GB of 31.7, and the worst case measures the same because the # memory-efficient attention kernel is O(N), not O(N^2). # # Re-sweep after any change to the model or the kernels: the optimum moves # as bottlenecks fall. micro_batch: dict = field(default_factory=lambda: {2048: 16, 8192: 4}) device: str = "cuda:0" eval_device: str = "cuda:1" # eval.py only; training never touches it seed: int = 20260906 # fp16 everywhere. sm_70 tensor cores are fp16-in/fp32-accumulate and have # no bf16 path at all; TF32 does not exist before sm_80. Both are asserted # at startup rather than assumed. amp_dtype: str = "float16" init_scale: float = 2.0 ** 16 # GradScaler has no floor of its own, and that is fatal here. Below roughly # 1e-30 `unscale_` computes grad/scale and overflows fp32 to inf, which sets # found_inf, which halves the scale again: the collapse becomes # self-reinforcing and has no exit. That is exactly how the run that reached # step 3,810 died -- the scale bled 131,072 -> 0 over 170 steps while the # weights stayed perfectly finite, and every step after that was skipped. # Floored at 1.0 the same event costs a few skipped steps instead of the run. min_loss_scale: float = 1.0 # Ceiling, for the other end. The scale grew to 131,072 at step ~2,000 and # the first overflow appeared at 3,460; halving the ceiling buys a factor of # two of headroom in the backward and costs nothing but the smallest # gradients, which are already far below where they matter. max_loss_scale: float = 2.0 ** 15 # Steps pinned at min_loss_scale before the run stops itself. At the floor # nothing is training -- every step is being skipped -- so this is the real # early warning, ~350 steps ahead of the loss going non-finite. abort_after_floored: int = 100 compile_regions: bool = True # fixed-shape regions only, never the model # FALSE, and this is not a tuning knob -- an fp16 residual stream cannot # survive 31 layers of this model. The stream grows with depth, and at step # 3,456 (1% of the budget) it measured: # block 0 3,062 block 16 40,640 # block 9 16,144 block 24 39,264 # block 25 65,617 vs fp16's max of 65,504 <-- over by 113 # It saturates to inf, and _rmsnorm then computes rsqrt(inf) = 0 and # inf * 0 = NaN. That NaN is what drove GradScaler to halve its scale # forever (halving a loss scale cannot fix a FORWARD overflow), which is how # two runs died. Recomputed in fp32 the same batch reaches 65,617 and stays # finite. Measured cost: 39.77k -> 36.73k tok/s and 24.0 -> 25.1 GB, about # +4.3 days on the 200B budget. Boris-1.7 never hit this because bf16 on the # 3060 has fp32's exponent range; sm_70 has no bf16 at all. fp16_residual: bool = False # Recompute block activations in the backward instead of storing them, to # buy a larger micro-batch. Costs one extra forward per checkpointed block # (~+11% compute) and buys fatter GEMMs. Whether that trade is positive is # a measurement, not an argument -- preflight.py --sweep with it on/off. # 2026-09-17: ON for Boris-2-0917. Without it the run OOMs -- 30.17 GB # allocated + 0.78 GB fragmentation + 0.28 GB held by another process on # the same card = 31.28 of 31.73. preflight reads ~1 GB LOW because it # measures peak_allocated in a clean process and cannot see either. # every=4 returns 6.5 GB (29.2 -> 22.7 worst-case) for 5.3% throughput. grad_checkpoint: bool = True grad_checkpoint_every: int = 4 # 1 = every block, 4 = every 4th # Resume granularity. The 50B checkpoints are archival milestones; they are # not the thing you resume from. A run that will be interrupted needs a # rolling checkpoint on a wall-clock timer. rolling_ckpt_minutes: int = 45 rolling_ckpt_keep: int = 2 full_ckpt_every_tokens: int = 50_000_000_000 export_every_tokens: int = 10_000_000_000 # weights-only, for eval.py log_every_steps: int = 10 # Consecutive steps of non-finite loss OR non-finite grad norm before the # run stops itself. One or two are normal (GradScaler probing its scale); # 200 means something is broken and every further step is wasted. abort_after_bad: int = 200 val_every_tokens: int = 1_000_000_000 @dataclass class EvalConfig: device: str = "cuda:1" out_path: Path = RUN_DIR / "eval" / "results.jsonl" export_dir: Path = RUN_DIR / "checkpoints" / "exports" manifest: Path = RUN_DIR / "checkpoints" / "exports" / "manifest.jsonl" tasks: tuple = ("hellaswag", "arc_easy", "arc_challenge", "piqa", "arithmark3") val_tokens: int = 10_000_000 batch_size: int = 16 poll_seconds: int = 120 @dataclass class Config: model: ModelConfig = field(default_factory=ModelConfig) ngram: NGramConfig = field(default_factory=NGramConfig) data: DataConfig = field(default_factory=DataConfig) curriculum: CurriculumConfig = field(default_factory=CurriculumConfig) optim: OptimConfig = field(default_factory=OptimConfig) schedule: ScheduleConfig = field(default_factory=ScheduleConfig) train: TrainConfig = field(default_factory=TrainConfig) eval: EvalConfig = field(default_factory=EvalConfig) run_name: str = RUN_DIR.name checkpoint_dir: Path = RUN_DIR / "checkpoints" # verify.py builds 4- and 6-layer models to keep the isolation probes fast; # they are not the run, so the band assert does not apply to them. enforce_budget: bool = True # -- derived ------------------------------------------------------------ # @property def total_steps(self) -> int: return self.train.total_tokens // self.train.tokens_per_step def block_size_at(self, progress: float) -> tuple[int, float]: """(block_size, rope_theta) for this point in the run.""" bs, theta = self.model.context_stages[0][1], self.model.context_stages[0][2] for frac, b, th in self.model.context_stages: if progress >= frac: bs, theta = int(b), float(th) return bs, theta def micro_batch_at(self, progress: float) -> int: bs, _ = self.block_size_at(progress) return int(self.train.micro_batch[bs]) def accum_at(self, progress: float) -> int: bs = self.block_size_at(progress)[0] mb = self.micro_batch_at(progress) tokens_per_micro = mb * bs assert self.train.tokens_per_step % tokens_per_micro == 0, ( f"tokens_per_step {self.train.tokens_per_step} is not divisible by " f"micro_batch {mb} x block_size {bs} = {tokens_per_micro}; the step " f"size would change when the context stage does") return self.train.tokens_per_step // tokens_per_micro # -- parameter budget ---------------------------------------------------- # def param_budget(self) -> dict: m, n = self.model, self.ngram on = m.use_ngram d = { "attention": m.n_layer * m.attn_params(), "ffn": m.n_layer * m.ffn_params(), "canon": m.canon_params(), "norms": m.norm_params(), "embedding": m.embed_params(), "ngram_table": n.n_rows * n.rank if on else 0, "ngram_proj": n.rank * m.d_model if on else 0, } d["trunk"] = d["attention"] + d["ffn"] + d["canon"] + d["norms"] d["matmul"] = d["attention"] + d["ffn"] + d["ngram_proj"] d["total"] = d["trunk"] + d["embedding"] + d["ngram_table"] + d["ngram_proj"] return d def assert_budget(self, ceiling: int = 150_000_000, band: tuple[int, int] = (140_000_000, 145_000_000)) -> None: b = self.param_budget() if not self.enforce_budget: return assert b["total"] <= ceiling, ( f"total parameters {b['total']:,} exceeds the hard ceiling {ceiling:,}") # The band describes the full model. With the table off the trunk is # unchanged and the total is simply 15M lower, which is the point. if not self.model.use_ngram: return assert band[0] <= b["total"] <= band[1], ( f"total parameters {b['total']:,} outside the target band " f"{band[0]:,}-{band[1]:,}; change model.ffn_hidden") tp = self.ngram.table_params(self.model.d_model) tgt, tol = self.ngram.target_params, self.ngram.tolerance assert abs(tp - tgt) <= tol * tgt, ( f"n-gram table {tp:,} is more than {tol:.0%} from its {tgt:,} target") # -- serialisation ------------------------------------------------------- # def to_dict(self) -> dict: def conv(o): if isinstance(o, Path): return str(o) if isinstance(o, dict): return {str(k): conv(v) for k, v in o.items()} if isinstance(o, (list, tuple)): return [conv(x) for x in o] return o return {f.name: conv(asdict(getattr(self, f.name)) if hasattr(getattr(self, f.name), "__dataclass_fields__") else getattr(self, f.name)) for f in fields(self)} def to_json(self) -> str: return json.dumps(self.to_dict(), indent=2, sort_keys=True) def fingerprint(self) -> str: """Hash of everything that must match to resume a checkpoint.""" import hashlib keep = {k: self.to_dict()[k] for k in ("model", "ngram", "optim", "schedule", "train")} return hashlib.sha256(json.dumps(keep, sort_keys=True).encode()).hexdigest()[:16] @staticmethod def from_json(s: str) -> "Config": raw = json.loads(s) cfg = Config() # Checkpoints predating the tokenizer guard used the original 09-06 # BPE. Give those serialized configs their real identity rather than # inheriting the 0920 default because the field is absent. if "model" in raw and "tokenizer_sha256" not in raw["model"]: cfg.model.tokenizer_sha256 = LEGACY_TOKENIZER_SHA256 if (raw.get("model", {}).get("tokenizer_sha256", LEGACY_TOKENIZER_SHA256) == LEGACY_TOKENIZER_SHA256): cfg.data.val_source = "val" for name, sub in raw.items(): if not hasattr(cfg, name): continue cur = getattr(cfg, name) if hasattr(cur, "__dataclass_fields__") and isinstance(sub, dict): for k, v in sub.items(): if not hasattr(cur, k): continue old = getattr(cur, k) if isinstance(old, Path): v = Path(v) elif isinstance(old, tuple) and isinstance(v, list): v = tuple(tuple(x) if isinstance(x, list) else x for x in v) elif isinstance(old, dict) and isinstance(v, dict): v = {int(kk) if str(kk).isdigit() else kk: vv for kk, vv in v.items()} setattr(cur, k, v) else: if isinstance(cur, Path): sub = Path(sub) setattr(cfg, name, sub) return cfg if __name__ == "__main__": cfg = Config() b = cfg.param_budget() w = max(len(k) for k in b) for k in ("attention", "ffn", "canon", "norms", "trunk", "embedding", "ngram_table", "ngram_proj", "matmul", "total"): print(f" {k:<{w}} {b[k]:>13,}") cfg.assert_budget() print(f"\n steps {cfg.total_steps:,}") for frac, bs, th in cfg.model.context_stages: print(f" ctx from {frac:>5.0%} block {bs:>5} theta {th:>9,.0f} " f"micro_batch {cfg.train.micro_batch[bs]} accum {cfg.accum_at(frac)}") print(f" fingerprint {cfg.fingerprint()}")