summaryrefslogtreecommitdiff
path: root/schedule.py
diff options
context:
space:
mode:
Diffstat (limited to 'schedule.py')
-rw-r--r--schedule.py106
1 files changed, 106 insertions, 0 deletions
diff --git a/schedule.py b/schedule.py
new file mode 100644
index 0000000..904cfe4
--- /dev/null
+++ b/schedule.py
@@ -0,0 +1,106 @@
+"""WSD learning rate, the mixture curriculum, and the context-length stages.
+
+Everything here is a function of `progress` in [0, 1] -- tokens seen over the
+token budget -- so a resumed run recovers the whole schedule from `tokens_seen`
+alone and cannot drift out of phase with the run that wrote the checkpoint.
+"""
+from __future__ import annotations
+
+import math
+
+from config import Config
+
+
+def phase_at(progress: float, cfg: Config) -> str:
+ s = cfg.schedule
+ if progress < s.warmup_frac:
+ return "warmup"
+ if progress < s.stable_frac:
+ return "stable"
+ return "decay"
+
+
+def lr_scale(progress: float, cfg: Config) -> float:
+ """Multiplier on each optimizer's base LR, in [0, 1].
+
+ 1-sqrt decay rather than cosine: it holds the high LR longer and then drops
+ fast, which is what makes the decay phase a consolidation onto the dense
+ data rather than a long slow taper through it.
+ """
+ s = cfg.schedule
+ p = min(max(progress, 0.0), 1.0)
+ if p < s.warmup_frac:
+ return p / max(s.warmup_frac, 1e-12)
+ if p < s.stable_frac:
+ return 1.0
+ frac = (p - s.stable_frac) / max(1.0 - s.stable_frac, 1e-12)
+ if s.decay_shape == "1-sqrt":
+ v = 1.0 - math.sqrt(frac)
+ elif s.decay_shape == "linear":
+ v = 1.0 - frac
+ else:
+ v = 0.5 * (1.0 + math.cos(math.pi * frac))
+ return s.final_lr_frac + (1 - s.final_lr_frac) * max(v, 0.0)
+
+
+def mixture_at(progress: float, cfg: Config) -> dict:
+ """Source weights at this point in the run, normalised to sum to 1."""
+ c = cfg.curriculum
+ names = list(cfg.data.sources)
+ if not c.enabled:
+ return {n: 1.0 / len(names) for n in names}
+
+ phases = [(n, float(e), {k: float(w.get(k, 0.0)) for k in names})
+ for n, e, w in c.phases]
+ i = 0
+ while i < len(phases) - 1 and progress >= phases[i][1]:
+ i += 1
+ _, end, target = phases[i]
+ start = 0.0 if i == 0 else phases[i - 1][1]
+
+ if not c.smooth or i == 0:
+ w = target
+ else:
+ prev = phases[i - 1][2]
+ f = min(max((progress - start) / max(end - start, 1e-12), 0.0), 1.0)
+ w = {k: prev[k] + f * (target[k] - prev[k]) for k in names}
+ tot = sum(w.values()) or 1.0
+ return {k: v / tot for k, v in w.items()}
+
+
+def context_at(progress: float, cfg: Config) -> tuple[int, float, int, int]:
+ """(block_size, rope_theta, micro_batch, accum_steps)."""
+ bs, theta = cfg.block_size_at(progress)
+ return bs, theta, cfg.micro_batch_at(progress), cfg.accum_at(progress)
+
+
+def long_doc_pressure(progress: float, cfg: Config) -> tuple[int, float]:
+ """(minimum length preferred, target share of TOKENS from such documents).
+
+ The packer converts the token share into a per-draw probability per source
+ (VarlenPacker._long_draw_prob); it is not itself a draw probability.
+
+ Zero until the context stage lengthens: at 2048 there is nothing to gain
+ from hunting for 4k-token documents, and biasing the mixture toward long
+ documents is a change to the data distribution, not just to the packing.
+ """
+ bs, _ = cfg.block_size_at(progress)
+ base = cfg.model.context_stages[0][1]
+ if bs <= base:
+ return 0, 0.0
+ return cfg.curriculum.long_doc_floor_tokens, cfg.curriculum.long_doc_target_frac
+
+
+def phase_boundaries(cfg: Config) -> list[tuple[str, int, int]]:
+ total = cfg.total_steps
+ s = cfg.schedule
+ return [("warmup", 0, int(total * s.warmup_frac)),
+ ("stable", int(total * s.warmup_frac), int(total * s.stable_frac)),
+ ("decay", int(total * s.stable_frac), total)]
+
+
+def stable_end_tokens(cfg: Config) -> int:
+ """The end-of-stable checkpoint is permanent: variants branch from it as a
+ tree, and re-warming from a fully decayed model costs a loss spike and a
+ slow recovery."""
+ return int(cfg.train.total_tokens * cfg.schedule.stable_frac)