1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
|
"""WSD learning rate, the mixture curriculum, and the context-length stages.
Everything here is a function of `progress` in [0, 1] -- tokens seen over the
token budget -- so a resumed run recovers the whole schedule from `tokens_seen`
alone and cannot drift out of phase with the run that wrote the checkpoint.
"""
from __future__ import annotations
import math
from config import Config
def phase_at(progress: float, cfg: Config) -> str:
s = cfg.schedule
if progress < s.warmup_frac:
return "warmup"
if progress < s.stable_frac:
return "stable"
return "decay"
def lr_scale(progress: float, cfg: Config) -> float:
"""Multiplier on each optimizer's base LR, in [0, 1].
1-sqrt decay rather than cosine: it holds the high LR longer and then drops
fast, which is what makes the decay phase a consolidation onto the dense
data rather than a long slow taper through it.
"""
s = cfg.schedule
p = min(max(progress, 0.0), 1.0)
if p < s.warmup_frac:
return p / max(s.warmup_frac, 1e-12)
if p < s.stable_frac:
return 1.0
frac = (p - s.stable_frac) / max(1.0 - s.stable_frac, 1e-12)
if s.decay_shape == "1-sqrt":
v = 1.0 - math.sqrt(frac)
elif s.decay_shape == "linear":
v = 1.0 - frac
else:
v = 0.5 * (1.0 + math.cos(math.pi * frac))
return s.final_lr_frac + (1 - s.final_lr_frac) * max(v, 0.0)
def mixture_at(progress: float, cfg: Config) -> dict:
"""Source weights at this point in the run, normalised to sum to 1."""
c = cfg.curriculum
names = list(cfg.data.sources)
if not c.enabled:
return {n: 1.0 / len(names) for n in names}
phases = [(n, float(e), {k: float(w.get(k, 0.0)) for k in names})
for n, e, w in c.phases]
i = 0
while i < len(phases) - 1 and progress >= phases[i][1]:
i += 1
_, end, target = phases[i]
start = 0.0 if i == 0 else phases[i - 1][1]
if not c.smooth or i == 0:
w = target
else:
prev = phases[i - 1][2]
f = min(max((progress - start) / max(end - start, 1e-12), 0.0), 1.0)
w = {k: prev[k] + f * (target[k] - prev[k]) for k in names}
tot = sum(w.values()) or 1.0
return {k: v / tot for k, v in w.items()}
def context_at(progress: float, cfg: Config) -> tuple[int, float, int, int]:
"""(block_size, rope_theta, micro_batch, accum_steps)."""
bs, theta = cfg.block_size_at(progress)
return bs, theta, cfg.micro_batch_at(progress), cfg.accum_at(progress)
def long_doc_pressure(progress: float, cfg: Config) -> tuple[int, float]:
"""(minimum length preferred, target share of TOKENS from such documents).
The packer converts the token share into a per-draw probability per source
(VarlenPacker._long_draw_prob); it is not itself a draw probability.
Zero until the context stage lengthens: at 2048 there is nothing to gain
from hunting for 4k-token documents, and biasing the mixture toward long
documents is a change to the data distribution, not just to the packing.
"""
bs, _ = cfg.block_size_at(progress)
base = cfg.model.context_stages[0][1]
if bs <= base:
return 0, 0.0
return cfg.curriculum.long_doc_floor_tokens, cfg.curriculum.long_doc_target_frac
def phase_boundaries(cfg: Config) -> list[tuple[str, int, int]]:
total = cfg.total_steps
s = cfg.schedule
return [("warmup", 0, int(total * s.warmup_frac)),
("stable", int(total * s.warmup_frac), int(total * s.stable_frac)),
("decay", int(total * s.stable_frac), total)]
def stable_end_tokens(cfg: Config) -> int:
"""The end-of-stable checkpoint is permanent: variants branch from it as a
tree, and re-warming from a fully decayed model costs a loss spike and a
slow recovery."""
return int(cfg.train.total_tokens * cfg.schedule.stable_frac)
|