diff options
Diffstat (limited to 'schedule.py')
| -rw-r--r-- | schedule.py | 106 |
1 files changed, 106 insertions, 0 deletions
diff --git a/schedule.py b/schedule.py new file mode 100644 index 0000000..904cfe4 --- /dev/null +++ b/schedule.py @@ -0,0 +1,106 @@ +"""WSD learning rate, the mixture curriculum, and the context-length stages. + +Everything here is a function of `progress` in [0, 1] -- tokens seen over the +token budget -- so a resumed run recovers the whole schedule from `tokens_seen` +alone and cannot drift out of phase with the run that wrote the checkpoint. +""" +from __future__ import annotations + +import math + +from config import Config + + +def phase_at(progress: float, cfg: Config) -> str: + s = cfg.schedule + if progress < s.warmup_frac: + return "warmup" + if progress < s.stable_frac: + return "stable" + return "decay" + + +def lr_scale(progress: float, cfg: Config) -> float: + """Multiplier on each optimizer's base LR, in [0, 1]. + + 1-sqrt decay rather than cosine: it holds the high LR longer and then drops + fast, which is what makes the decay phase a consolidation onto the dense + data rather than a long slow taper through it. + """ + s = cfg.schedule + p = min(max(progress, 0.0), 1.0) + if p < s.warmup_frac: + return p / max(s.warmup_frac, 1e-12) + if p < s.stable_frac: + return 1.0 + frac = (p - s.stable_frac) / max(1.0 - s.stable_frac, 1e-12) + if s.decay_shape == "1-sqrt": + v = 1.0 - math.sqrt(frac) + elif s.decay_shape == "linear": + v = 1.0 - frac + else: + v = 0.5 * (1.0 + math.cos(math.pi * frac)) + return s.final_lr_frac + (1 - s.final_lr_frac) * max(v, 0.0) + + +def mixture_at(progress: float, cfg: Config) -> dict: + """Source weights at this point in the run, normalised to sum to 1.""" + c = cfg.curriculum + names = list(cfg.data.sources) + if not c.enabled: + return {n: 1.0 / len(names) for n in names} + + phases = [(n, float(e), {k: float(w.get(k, 0.0)) for k in names}) + for n, e, w in c.phases] + i = 0 + while i < len(phases) - 1 and progress >= phases[i][1]: + i += 1 + _, end, target = phases[i] + start = 0.0 if i == 0 else phases[i - 1][1] + + if not c.smooth or i == 0: + w = target + else: + prev = phases[i - 1][2] + f = min(max((progress - start) / max(end - start, 1e-12), 0.0), 1.0) + w = {k: prev[k] + f * (target[k] - prev[k]) for k in names} + tot = sum(w.values()) or 1.0 + return {k: v / tot for k, v in w.items()} + + +def context_at(progress: float, cfg: Config) -> tuple[int, float, int, int]: + """(block_size, rope_theta, micro_batch, accum_steps).""" + bs, theta = cfg.block_size_at(progress) + return bs, theta, cfg.micro_batch_at(progress), cfg.accum_at(progress) + + +def long_doc_pressure(progress: float, cfg: Config) -> tuple[int, float]: + """(minimum length preferred, target share of TOKENS from such documents). + + The packer converts the token share into a per-draw probability per source + (VarlenPacker._long_draw_prob); it is not itself a draw probability. + + Zero until the context stage lengthens: at 2048 there is nothing to gain + from hunting for 4k-token documents, and biasing the mixture toward long + documents is a change to the data distribution, not just to the packing. + """ + bs, _ = cfg.block_size_at(progress) + base = cfg.model.context_stages[0][1] + if bs <= base: + return 0, 0.0 + return cfg.curriculum.long_doc_floor_tokens, cfg.curriculum.long_doc_target_frac + + +def phase_boundaries(cfg: Config) -> list[tuple[str, int, int]]: + total = cfg.total_steps + s = cfg.schedule + return [("warmup", 0, int(total * s.warmup_frac)), + ("stable", int(total * s.warmup_frac), int(total * s.stable_frac)), + ("decay", int(total * s.stable_frac), total)] + + +def stable_end_tokens(cfg: Config) -> int: + """The end-of-stable checkpoint is permanent: variants branch from it as a + tree, and re-warming from a fully decayed model costs a loss spike and a + slow recovery.""" + return int(cfg.train.total_tokens * cfg.schedule.stable_frac) |
