1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
|
{
"curriculum": {
"enabled": true,
"long_doc_floor_tokens": 4096,
"long_doc_target_frac": 0.5,
"phases": [
[
"flat",
0.75,
{
"climbmix": 0.932109,
"ultra_math_l3": 0.064783,
"orca_math": 0.003108,
"cosmo2_how": 0.0,
"cosmo_how_v1_tf": 0.0,
"metamath_gsm": 0.0,
"elem_wordprob": 0.0,
"dclm_r0920": 0.0
}
],
[
"pre_decay_ramp",
0.87,
{
"climbmix": 0.821333,
"ultra_math_l3": 0.081144,
"orca_math": 0.003244,
"cosmo2_how": 0.085976,
"cosmo_how_v1_tf": 0.008302,
"metamath_gsm": 0.0,
"elem_wordprob": 0.0,
"dclm_r0920": 0.0
}
],
[
"decay_in",
0.875,
{
"climbmix": 0.775534,
"ultra_math_l3": 0.07862,
"orca_math": 0.003143,
"cosmo2_how": 0.083301,
"cosmo_how_v1_tf": 0.008044,
"metamath_gsm": 0.025126,
"elem_wordprob": 0.020974,
"dclm_r0920": 0.005257
}
],
[
"decay",
0.9825,
{
"climbmix": 0.775534,
"ultra_math_l3": 0.07862,
"orca_math": 0.003143,
"cosmo2_how": 0.083301,
"cosmo_how_v1_tf": 0.008044,
"metamath_gsm": 0.025126,
"elem_wordprob": 0.020974,
"dclm_r0920": 0.005257
}
],
[
"p2_tail_in",
0.9875,
{
"climbmix": 0.580561,
"ultra_math_l3": 0.053042,
"metamath_gsm": 0.135615,
"elem_wordprob": 0.141507,
"dclm_r0920": 0.012769,
"orca_math": 0.002545,
"cosmo2_how": 0.067493,
"cosmo_how_v1_tf": 0.006468
}
],
[
"p2_tail",
1.0,
{
"climbmix": 0.580561,
"ultra_math_l3": 0.053042,
"metamath_gsm": 0.135615,
"elem_wordprob": 0.141507,
"dclm_r0920": 0.012769,
"orca_math": 0.002545,
"cosmo2_how": 0.067493,
"cosmo_how_v1_tf": 0.006468
}
]
],
"smooth": true
},
"checkpoint_dir": "/media/joseph/Media/Boris-2-0929/checkpoints",
"data": {
"eos_id": 0,
"max_doc_tokens": 65536,
"prefetch_depth": 4,
"root": "/home/joseph/boris-2-D112M-n15M/data",
"sources": [
"climbmix",
"ultra_math_l3",
"orca_math",
"cosmo2_how",
"cosmo_how_v1_tf",
"metamath_gsm",
"elem_wordprob",
"dclm_r0920"
],
"val_path": "/home/joseph/boris-2-D112M-n15M/data/val.bin",
"val_source": "climbmix_val"
},
"enforce_budget": true,
"eval": {
"batch_size": 128,
"device": "cuda:1",
"export_dir": "/media/joseph/Media/Boris-2-0929/checkpoints/exports",
"manifest": "/media/joseph/Media/Boris-2-0929/checkpoints/exports/manifest.jsonl",
"out_path": "/media/joseph/Media/Boris-2-0929/eval/results.jsonl",
"poll_seconds": 120,
"tasks": [
"hellaswag",
"arc_easy",
"arc_challenge",
"piqa",
"arithmark3"
],
"val_tokens": 10000000
},
"model": {
"architecture": "boris-2-canon-ngram",
"canon_kernel": 3,
"context_stages": [
[
0.0,
2048,
10000.0
]
],
"d_model": 512,
"ffn_hidden": 1664,
"n_head": 8,
"n_kv_head": 8,
"n_layer": 31,
"norm_eps": 1e-05,
"tie_embeddings": true,
"tokenizer_sha256": "f366974b96357d11a7e9930826efd81d77438638c0e21ea1517f620da72e05f4",
"use_canon": true,
"use_ngram": true,
"vocab_size": 32768
},
"ngram": {
"bigram_rows": 210938,
"insert_after": 2,
"muon_on_proj": true,
"oov_rows": 1,
"proj_init_std": 0.02,
"rank": 64,
"table_init_std": 0.02,
"target_params": 15000000,
"tolerance": 0.01,
"trigram_rows": 23437,
"vocab_path": "/media/joseph/Media/Boris-2-0920/ngram_vocab.bin"
},
"optim": {
"adamw_betas": [
0.9,
0.95
],
"adamw_eps": 1e-08,
"adamw_lr": 0.003,
"adamw_norm_weight_decay": 0.0,
"adamw_weight_decay": 0.1,
"grad_clip": 4.0,
"lm_head_chunk": 8192,
"muon_batched": true,
"muon_lr": 0.0025,
"muon_momentum": 0.95,
"muon_nesterov": true,
"muon_ns_dtype": "float32",
"muon_ns_steps": 5,
"muon_rms_scale": true,
"muon_weight_decay": 0.1,
"adamw_canon_weight_decay": 0.1
},
"run_name": "Boris-2-0929",
"schedule": {
"decay_shape": "1-sqrt",
"final_lr_frac": 0.0,
"stable_frac": 0.87,
"warmup_frac": 0.01
},
"train": {
"abort_after_bad": 200,
"abort_after_floored": 100,
"amp_dtype": "float16",
"compile_regions": true,
"device": "cuda:0",
"eval_device": "cuda:1",
"export_every_tokens": 10000000000,
"fp16_residual": false,
"full_ckpt_every_tokens": 50000000000,
"grad_checkpoint": false,
"grad_checkpoint_every": 8,
"init_scale": 65536.0,
"log_every_steps": 10,
"max_loss_scale": 32768.0,
"micro_batch": {
"2048": 16,
"8192": 4
},
"min_loss_scale": 1.0,
"rolling_ckpt_keep": 2,
"rolling_ckpt_minutes": 45,
"seed": 20260906,
"tokens_per_step": 1048576,
"total_tokens": 200000000000,
"val_every_tokens": 1000000000
}
}
|