"""WSD learning rate, the mixture curriculum, and the context-length stages. Everything here is a function of `progress` in [0, 1] -- tokens seen over the token budget -- so a resumed run recovers the whole schedule from `tokens_seen` alone and cannot drift out of phase with the run that wrote the checkpoint. """ from __future__ import annotations import math from config import Config def phase_at(progress: float, cfg: Config) -> str: s = cfg.schedule if progress < s.warmup_frac: return "warmup" if progress < s.stable_frac: return "stable" return "decay" def lr_scale(progress: float, cfg: Config) -> float: """Multiplier on each optimizer's base LR, in [0, 1]. 1-sqrt decay rather than cosine: it holds the high LR longer and then drops fast, which is what makes the decay phase a consolidation onto the dense data rather than a long slow taper through it. """ s = cfg.schedule p = min(max(progress, 0.0), 1.0) if p < s.warmup_frac: return p / max(s.warmup_frac, 1e-12) if p < s.stable_frac: return 1.0 frac = (p - s.stable_frac) / max(1.0 - s.stable_frac, 1e-12) if s.decay_shape == "1-sqrt": v = 1.0 - math.sqrt(frac) elif s.decay_shape == "linear": v = 1.0 - frac else: v = 0.5 * (1.0 + math.cos(math.pi * frac)) return s.final_lr_frac + (1 - s.final_lr_frac) * max(v, 0.0) def mixture_at(progress: float, cfg: Config) -> dict: """Source weights at this point in the run, normalised to sum to 1.""" c = cfg.curriculum names = list(cfg.data.sources) if not c.enabled: return {n: 1.0 / len(names) for n in names} phases = [(n, float(e), {k: float(w.get(k, 0.0)) for k in names}) for n, e, w in c.phases] i = 0 while i < len(phases) - 1 and progress >= phases[i][1]: i += 1 _, end, target = phases[i] start = 0.0 if i == 0 else phases[i - 1][1] if not c.smooth or i == 0: w = target else: prev = phases[i - 1][2] f = min(max((progress - start) / max(end - start, 1e-12), 0.0), 1.0) w = {k: prev[k] + f * (target[k] - prev[k]) for k in names} tot = sum(w.values()) or 1.0 return {k: v / tot for k, v in w.items()} def context_at(progress: float, cfg: Config) -> tuple[int, float, int, int]: """(block_size, rope_theta, micro_batch, accum_steps).""" bs, theta = cfg.block_size_at(progress) return bs, theta, cfg.micro_batch_at(progress), cfg.accum_at(progress) def long_doc_pressure(progress: float, cfg: Config) -> tuple[int, float]: """(minimum length preferred, target share of TOKENS from such documents). The packer converts the token share into a per-draw probability per source (VarlenPacker._long_draw_prob); it is not itself a draw probability. Zero until the context stage lengthens: at 2048 there is nothing to gain from hunting for 4k-token documents, and biasing the mixture toward long documents is a change to the data distribution, not just to the packing. """ bs, _ = cfg.block_size_at(progress) base = cfg.model.context_stages[0][1] if bs <= base: return 0, 0.0 return cfg.curriculum.long_doc_floor_tokens, cfg.curriculum.long_doc_target_frac def phase_boundaries(cfg: Config) -> list[tuple[str, int, int]]: total = cfg.total_steps s = cfg.schedule return [("warmup", 0, int(total * s.warmup_frac)), ("stable", int(total * s.warmup_frac), int(total * s.stable_frac)), ("decay", int(total * s.stable_frac), total)] def stable_end_tokens(cfg: Config) -> int: """The end-of-stable checkpoint is permanent: variants branch from it as a tree, and re-warming from a fully decayed model costs a loss spike and a slow recovery.""" return int(cfg.train.total_tokens * cfg.schedule.stable_frac)