summaryrefslogtreecommitdiff
path: root/schedule.py
blob: 904cfe4234fffb59a93b7d0d2ce353a2cc082e48 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
"""WSD learning rate, the mixture curriculum, and the context-length stages.

Everything here is a function of `progress` in [0, 1] -- tokens seen over the
token budget -- so a resumed run recovers the whole schedule from `tokens_seen`
alone and cannot drift out of phase with the run that wrote the checkpoint.
"""
from __future__ import annotations

import math

from config import Config


def phase_at(progress: float, cfg: Config) -> str:
    s = cfg.schedule
    if progress < s.warmup_frac:
        return "warmup"
    if progress < s.stable_frac:
        return "stable"
    return "decay"


def lr_scale(progress: float, cfg: Config) -> float:
    """Multiplier on each optimizer's base LR, in [0, 1].

    1-sqrt decay rather than cosine: it holds the high LR longer and then drops
    fast, which is what makes the decay phase a consolidation onto the dense
    data rather than a long slow taper through it.
    """
    s = cfg.schedule
    p = min(max(progress, 0.0), 1.0)
    if p < s.warmup_frac:
        return p / max(s.warmup_frac, 1e-12)
    if p < s.stable_frac:
        return 1.0
    frac = (p - s.stable_frac) / max(1.0 - s.stable_frac, 1e-12)
    if s.decay_shape == "1-sqrt":
        v = 1.0 - math.sqrt(frac)
    elif s.decay_shape == "linear":
        v = 1.0 - frac
    else:
        v = 0.5 * (1.0 + math.cos(math.pi * frac))
    return s.final_lr_frac + (1 - s.final_lr_frac) * max(v, 0.0)


def mixture_at(progress: float, cfg: Config) -> dict:
    """Source weights at this point in the run, normalised to sum to 1."""
    c = cfg.curriculum
    names = list(cfg.data.sources)
    if not c.enabled:
        return {n: 1.0 / len(names) for n in names}

    phases = [(n, float(e), {k: float(w.get(k, 0.0)) for k in names})
              for n, e, w in c.phases]
    i = 0
    while i < len(phases) - 1 and progress >= phases[i][1]:
        i += 1
    _, end, target = phases[i]
    start = 0.0 if i == 0 else phases[i - 1][1]

    if not c.smooth or i == 0:
        w = target
    else:
        prev = phases[i - 1][2]
        f = min(max((progress - start) / max(end - start, 1e-12), 0.0), 1.0)
        w = {k: prev[k] + f * (target[k] - prev[k]) for k in names}
    tot = sum(w.values()) or 1.0
    return {k: v / tot for k, v in w.items()}


def context_at(progress: float, cfg: Config) -> tuple[int, float, int, int]:
    """(block_size, rope_theta, micro_batch, accum_steps)."""
    bs, theta = cfg.block_size_at(progress)
    return bs, theta, cfg.micro_batch_at(progress), cfg.accum_at(progress)


def long_doc_pressure(progress: float, cfg: Config) -> tuple[int, float]:
    """(minimum length preferred, target share of TOKENS from such documents).

    The packer converts the token share into a per-draw probability per source
    (VarlenPacker._long_draw_prob); it is not itself a draw probability.

    Zero until the context stage lengthens: at 2048 there is nothing to gain
    from hunting for 4k-token documents, and biasing the mixture toward long
    documents is a change to the data distribution, not just to the packing.
    """
    bs, _ = cfg.block_size_at(progress)
    base = cfg.model.context_stages[0][1]
    if bs <= base:
        return 0, 0.0
    return cfg.curriculum.long_doc_floor_tokens, cfg.curriculum.long_doc_target_frac


def phase_boundaries(cfg: Config) -> list[tuple[str, int, int]]:
    total = cfg.total_steps
    s = cfg.schedule
    return [("warmup", 0, int(total * s.warmup_frac)),
            ("stable", int(total * s.warmup_frac), int(total * s.stable_frac)),
            ("decay", int(total * s.stable_frac), total)]


def stable_end_tokens(cfg: Config) -> int:
    """The end-of-stable checkpoint is permanent: variants branch from it as a
    tree, and re-warming from a fully decayed model costs a loss spike and a
    slow recovery."""
    return int(cfg.train.total_tokens * cfg.schedule.stable_frac)