1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
|
"""Boris-2 pretraining loop: 200B tokens, fp16, single V100.
Run:
python train.py # fresh
python train.py --resume # from the newest rolling checkpoint
python train.py --resume path.pt # from a specific one
The loop assumes it will be interrupted. Everything needed to continue -- the
weights, both optimizer states, the loss scale, the packer's RNG and its carried
document tail -- goes into a rolling checkpoint every 45 minutes, and a resume
replays the identical token stream from there.
"""
from __future__ import annotations
import os
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
import argparse
import json
import math
import subprocess
import sys
import time
from datetime import datetime, timezone
from pathlib import Path
import torch
from config import LOG_DIR, Config
import checkpoint as ckpt
from data import Prefetcher, VarlenPacker, open_sources, open_validation
from model import Boris2
from ngram import NGramVocab
from optim import build_optimizers, set_lr
from schedule import (context_at, lr_scale, mixture_at, phase_at,
stable_end_tokens)
from tokenizer import BorisTokenizer
# --------------------------------------------------------------------------- #
def assert_environment(cfg: Config, log):
"""Catch a silently-wrong precision setup in seconds rather than on day 30."""
if not torch.cuda.is_available():
raise SystemExit("no CUDA device")
dev = torch.device(cfg.train.device)
p = torch.cuda.get_device_properties(dev)
log(f" device {p.name} sm_{p.major}{p.minor} "
f"{p.total_memory/2**30:.1f} GB {p.multi_processor_count} SMs")
if p.major != 7:
log(f" WARNING: this run is tuned for sm_70; found sm_{p.major}{p.minor}")
assert cfg.train.amp_dtype == "float16", (
"sm_70 tensor cores are fp16-in/fp32-accumulate and have no bf16 path; "
"bf16 would run in software and TF32 does not exist before sm_80")
torch.backends.cuda.matmul.allow_tf32 = False
torch.backends.cudnn.allow_tf32 = False
torch.backends.cudnn.benchmark = True
# Measured +7.4% end to end on this card. It lets cuBLAS reduce fp16 GEMM
# split-k partials in fp16; checked against fp32 references in verify.py,
# where the LM head gradients are unaffected at 9e-04 / 2e-04. It is the
# default, and it is set explicitly so a future torch cannot flip it
# silently and cost a week.
torch.backends.cuda.matmul.allow_fp16_reduced_precision_reduction = True
# Prove autocast is actually engaged and producing fp16, rather than the
# loop quietly running fp32 for weeks at a quarter of the throughput.
a = torch.randn(256, 256, device=dev)
with torch.autocast("cuda", dtype=torch.float16):
out = a @ a
assert out.dtype == torch.float16, (
f"autocast produced {out.dtype}: fp16 is not engaged")
log(f" precision fp16 autocast + GradScaler, TF32 off, bf16 unused")
_NVML = None
def _nvml_temp(index: int) -> float:
"""Temperature through libnvidia-ml directly: ~18 us, versus ~14 ms to fork
nvidia-smi (whose indices are NVML's, so the numbering is the same)."""
global _NVML
import ctypes
if _NVML is None:
lib = ctypes.CDLL("libnvidia-ml.so.1")
if lib.nvmlInit_v2() != 0:
raise RuntimeError("nvmlInit failed")
_NVML = (lib, {})
lib, handles = _NVML
if index not in handles:
h = ctypes.c_void_p()
if lib.nvmlDeviceGetHandleByIndex_v2(index, ctypes.byref(h)) != 0:
raise RuntimeError("nvml handle failed")
handles[index] = h
t = ctypes.c_uint()
if lib.nvmlDeviceGetTemperature(handles[index], 0, ctypes.byref(t)) != 0:
raise RuntimeError("nvml temperature read failed")
return float(t.value)
def gpu_temp(index: int = 0) -> float:
try:
return _nvml_temp(index)
except Exception:
pass
try:
r = subprocess.run(["nvidia-smi", f"--id={index}",
"--query-gpu=temperature.gpu", "--format=csv,noheader"],
capture_output=True, text=True, timeout=4)
return float(r.stdout.strip().splitlines()[0])
except Exception:
return float("nan")
def device_index(device: str | torch.device) -> int:
"""Physical CUDA index used by diagnostics; never assume cuda:0."""
d = torch.device(device)
return torch.cuda.current_device() if d.index is None else d.index
class Logger:
def __init__(self, path: Path):
path.parent.mkdir(parents=True, exist_ok=True)
self.f = open(path, "a")
def line(self, s: str):
print(s, flush=True)
self.f.write(json.dumps({"msg": s, "timestamp": datetime.now(timezone.utc).isoformat()}) + "\n")
self.f.flush()
def row(self, d: dict):
d = {**d, "timestamp": datetime.now(timezone.utc).isoformat()}
self.f.write(json.dumps(d) + "\n")
self.f.flush()
def step_flops(cfg: Config, tokens: int, sumsq: int) -> float:
"""Exact fwd+bwd FLOPs for a step, attention included at what it really did.
Attention is varlen, so its cost is sum(len_i^2) over the documents actually
packed -- not block_size^2. Reporting the dense figure would overstate the
model's utilisation by roughly the block sparsity, which at ctx 2048 is 57%.
"""
m = cfg.model
b = cfg.param_budget()
matmul = 2 * b["matmul"] * tokens
head = 2 * m.vocab_size * m.d_model * tokens
attn = m.n_layer * 2 * (m.n_head * m.head_dim) * sumsq
return 3.0 * (matmul + head + attn)
# --------------------------------------------------------------------------- #
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--resume", nargs="?", const="auto", default=None)
ap.add_argument("--config", type=Path, default=None)
ap.add_argument("--max-steps", type=int, default=None)
ap.add_argument("--force-config", action="store_true")
ap.add_argument("--no-compile", action="store_true")
a = ap.parse_args()
cfg = Config.from_json(a.config.read_text()) if a.config else Config()
ckpt_dir = Path(cfg.checkpoint_dir)
if not a.resume and ckpt_dir.exists() and any(ckpt_dir.glob("*.pt")):
raise SystemExit("checkpoint directory is nonempty: use --resume or a new directory")
if a.no_compile:
cfg.train.compile_regions = False
log = Logger(LOG_DIR / f"{cfg.run_name}.jsonl")
log.line(f"\n {cfg.run_name}")
log.line(" " + "=" * 74)
assert_environment(cfg, log.line)
torch.manual_seed(cfg.train.seed)
dev = cfg.train.device
torch.cuda.set_device(dev)
tok_dir = Path(cfg.data.root) / "tokenizer"
tok_fp = BorisTokenizer.fingerprint(tok_dir)
if tok_fp != cfg.model.tokenizer_sha256:
raise SystemExit(
f"tokenizer fingerprint {tok_fp} at {tok_dir} does not match "
f"config {cfg.model.tokenizer_sha256}; token ids and embedding rows "
"would disagree")
# -- data ---------------------------------------------------------- #
stores = open_sources(cfg, log.line)
packer = VarlenPacker(cfg, stores, cfg.train.seed, split="train")
# The held-out split is just another prepared source directory, and it is
# deliberately not in cfg.data.sources, so the training packer can never
# draw from it.
val_packer = None
if (Path(cfg.data.root) / cfg.data.val_source / "tokens.bin").exists():
vcfg = Config.from_json(cfg.to_json())
vcfg.curriculum.enabled = False
val_packer = VarlenPacker(vcfg, {"val": open_validation(cfg)},
12345, split="val")
log.line(f" val split "
f"{val_packer.stores['val'].n_tokens/1e6:.1f}M tokens held out")
# -- model --------------------------------------------------------- #
model = Boris2(cfg).to(dev)
if model.ngram is not None:
vocab = NGramVocab.load(cfg.ngram.vocab_path)
assert vocab.vocab_size == cfg.model.vocab_size, (
f"ngram_vocab.bin was built for vocab {vocab.vocab_size}, model is "
f"{cfg.model.vocab_size}")
model.ngram.load_vocab(vocab, dev)
ng_fp = vocab.fingerprint()
ng_compatible = (ng_fp, vocab.legacy_fingerprint())
log.line(f" ngram vocab {vocab.bigram_keys.size:,} bigram + "
f"{vocab.trigram_keys.size:,} trigram rows, fingerprint {ng_fp}")
else:
ng_fp = "disabled"
ng_compatible = (ng_fp,)
log.line(" ngram table OFF (use_ngram=False): dense control arm")
if not a.no_compile:
model.maybe_compile()
muon, adamw = build_optimizers(model, cfg, log.line)
scaler = torch.amp.GradScaler("cuda", init_scale=cfg.train.init_scale)
# -- resume -------------------------------------------------------- #
step, tokens_seen, best_val = 0, 0, float("inf")
bad_grad = bad_loss = floored = 0 # consecutive non-finite log intervals
if a.resume:
path = (ckpt.latest_rolling(ckpt_dir) if a.resume == "auto"
else Path(a.resume))
if path is None:
raise SystemExit("--resume requested but no rolling checkpoint exists")
else:
blob = ckpt.load_full(
path, model=model, muon=muon, adamw=adamw, scaler=scaler,
packer=packer, map_location=dev,
expect_fingerprint=None if a.force_config else cfg.fingerprint(),
expect_ngram=None if a.force_config else ng_compatible)
step, tokens_seen = blob["step"], blob["tokens_seen"]
best_val = blob.get("best_val", float("inf"))
log.line(f" resumed {path.name} at step {step:,}, "
f"{tokens_seen/1e9:.2f}B tokens")
# -- run ------------------------------------------------------------ #
# The run ends when the token budget is spent, not at a fixed step count: tokens_per_step may change at a
# resume (2026-11-05: doubled for the final decay). Step numbers keep counting up, because rolling
# checkpoints are named and pruned by step. For an unchanged batch this equals cfg.total_steps.
total_steps = a.max_steps or (step + max(0, cfg.train.total_tokens - tokens_seen) // cfg.train.tokens_per_step)
progress = tokens_seen / cfg.train.total_tokens
block, theta, mb, accum = context_at(progress, cfg)
pre = Prefetcher(packer, cfg, dev)
pre.configure(mb, block, progress)
pre.start()
log.line(f" budget {cfg.train.total_tokens/1e9:.0f}B tokens over "
f"{cfg.total_steps:,} steps of {cfg.train.tokens_per_step:,}")
log.line(f" context block {block}, theta {theta:,.0f}, "
f"micro_batch {mb} x {accum} accum")
log.line(" " + "=" * 74 + "\n")
t_run = time.time()
t_ckpt = time.time()
next_export = ((tokens_seen // cfg.train.export_every_tokens) + 1) * cfg.train.export_every_tokens
next_full = ((tokens_seen // cfg.train.full_ckpt_every_tokens) + 1) * cfg.train.full_ckpt_every_tokens
next_val = ((tokens_seen // cfg.train.val_every_tokens) + 1) * cfg.train.val_every_tokens
stable_end = stable_end_tokens(cfg)
stable_saved = tokens_seen >= stable_end
attempt = step
win_t, win_tok, win_loss, win_flops = time.time(), 0, 0.0, 0.0
win_steps = 0
retry_batches = []
consecutive_skips = 0
temp_poll = 0
bad_temp_reads = 0
temperature = gpu_temp(device_index(dev))
while step < total_steps:
# Leave headroom below the user's 80 C cap without changing power limits.
# gpu_temp() reads NVML directly (~18 us), so every step is free again;
# it falls back to forking nvidia-smi (~14 ms), where every 5th step
# keeps the idle GPU time small.
temp_poll += 1
if _NVML is not None or temp_poll % 5 == 1:
temperature = gpu_temp(device_index(dev))
# One failed nvidia-smi read used to end the run; tolerate a few in a
# row (still refusing to train unmonitored for long).
bad_temp_reads = 0 if math.isfinite(temperature) else bad_temp_reads + 1
if bad_temp_reads >= 3:
raise RuntimeError("cannot read GPU temperature 3 times running; "
"refusing unmonitored training")
if math.isfinite(temperature) and temperature >= 78:
# The sensor flickers 77 <-> 78 at steady load, so a single reading
# of 78 paused the run for 5 s of idle GPU. Confirm it a second
# later; the thresholds (pause at 78, resume at 75, cap 80) are
# unchanged and a real climb cannot gain 2 C in a second.
time.sleep(1)
temperature = gpu_temp(device_index(dev))
if math.isfinite(temperature) and temperature >= 78:
log.line(f"thermal pause at {temperature:.0f} C; waiting for <=75 C")
fails = 0
while not (math.isfinite(temperature) and temperature <= 75):
time.sleep(1)
temperature = gpu_temp(device_index(dev))
fails = 0 if math.isfinite(temperature) else fails + 1
if fails >= 3:
raise RuntimeError("GPU temperature unavailable during cooldown")
progress = tokens_seen / cfg.train.total_tokens
nb, ntheta, nmb, naccum = context_at(progress, cfg)
if (nb, nmb) != (block, mb):
log.line(f"\n context stage block {block} -> {nb}, "
f"theta {theta:,.0f} -> {ntheta:,.0f}, "
f"micro_batch {mb} -> {nmb} (one recompile expected)\n")
block, theta, mb, accum = nb, ntheta, nmb, naccum
pre.configure(mb, block, progress)
torch.cuda.empty_cache()
else:
theta = ntheta
pre.configure(mb, block, progress)
scale = lr_scale(progress, cfg)
mlr, alr = set_lr(muon, adamw, cfg, scale)
# Accumulated on the GPU. Reading it per micro-step with float() forces
# a host sync 16 times a step and destroys CPU run-ahead; it is read
# once, at log time, well after the fact.
loss_acc = torch.zeros((), device=dev)
tok_acc, sumsq_acc = 0, 0
for micro in range(accum):
if micro == len(retry_batches):
retry_batches.append(pre.next(mb, block))
x, y, pos, bnd, bounds, stats = retry_batches[micro]
with torch.autocast("cuda", dtype=torch.float16):
loss = model(x, pos, bounds, bnd, y, theta,
stats["min_seqlen"], stats["max_seqlen"])
scaler.scale(loss / accum).backward()
loss_acc += loss.detach() / accum
tok_acc += stats["tokens"]
sumsq_acc += stats["sumsq"]
grad_list = [p.grad for p in model.parameters() if p.grad is not None]
scaler.unscale_(muon)
scaler.unscale_(adamw)
# Not clip_grad_norm_: it scales every gradient by a coefficient derived
# from the norm over ALL parameters, so a single Inf anywhere -- routine
# on the first steps, where init_scale is 2^16 and GradScaler is meant
# to overflow, back off and skip -- makes the coefficient non-finite and
# writes NaN into gradients that were fine. found_inf is recorded per
# optimizer at unscale_ time, so Muon would skip while AdamW stepped on
# the freshly-poisoned grads, and 0 * NaN = NaN defeats even the lr-0
# warmup step. That is exactly how this run died: step 1 loss 10.4739,
# step 2 loss nan. Leave the gradients untouched when the norm is not
# finite and let GradScaler do its job.
norms = torch._foreach_norm(grad_list)
gnorm = torch.linalg.vector_norm(torch.stack(norms))
coef = torch.where(torch.isfinite(gnorm),
(cfg.optim.grad_clip / (gnorm + 1e-6)).clamp(max=1.0),
torch.ones_like(gnorm))
# A tensor coefficient, and torch.where rather than a Python branch:
# reading gnorm here would be a host sync on every step.
pre_scale = scaler.get_scale()
# GradScaler makes its skip decision per optimizer. That allowed a head
# overflow to skip AdamW while Muon still changed the trunk: a partial
# update that desynchronised the model. Both optimizers see unscaled
# gradients here, so make one global finite decision and step both or
# neither. The scalar reads replace the loss read already performed
# below; they do not add another synchronization point.
loss_value = float(loss_acc)
gnorm_value = float(gnorm)
# coef is exactly 1.0 unless the norm reaches the clip (it never has in
# this run), and x * 1.0 == x, so skipping the multiply is bitwise the
# same update. It is applied, as before, whenever clipping engages.
if math.isfinite(gnorm_value) and gnorm_value + 1e-6 >= cfg.optim.grad_clip:
torch._foreach_mul_(grad_list, coef)
did_step = math.isfinite(loss_value) and math.isfinite(gnorm_value)
if did_step:
muon.step()
adamw.step()
scaler.update()
# A non-finite LOSS is a forward overflow, and halving the loss scale
# cannot touch it -- the forward does not depend on the scale at all.
# GradScaler halves anyway, every time, and that ratchet is what walked
# two runs down to the floor. Restore the scale on those steps: the step
# is still skipped (found_inf), so the weights are safe, and a rare
# forward NaN costs exactly one step instead of compounding. Genuine
# gradient overflow, where the loss is finite, still halves normally.
# One host sync per step, not the sixteen a per-micro-batch check costs.
if not math.isfinite(loss_value):
scaler.update(pre_scale)
# Clamp the loss scale at both ends. GradScaler will otherwise run the
# scale down without limit, and below ~1e-30 unscale_ (grad/scale)
# overflows fp32 to inf, sets found_inf, and halves it again -- a
# self-reinforcing collapse with no exit, which is what ended the run at
# step 3,810 with the weights still perfectly finite. The ceiling is the
# other half: the scale had grown to 131,072 before the first overflow.
_s = scaler.get_scale()
if _s < cfg.train.min_loss_scale:
scaler.update(cfg.train.min_loss_scale)
floored += 1
elif _s > cfg.train.max_loss_scale:
scaler.update(cfg.train.max_loss_scale)
floored = 0
else:
floored = 0
muon.zero_grad(set_to_none=True)
adamw.zero_grad(set_to_none=True)
if did_step:
step += 1
tokens_seen += tok_acc
retry_batches.clear()
consecutive_skips = 0
else:
consecutive_skips += 1
# A non-finite LOSS is a forward overflow: deterministic, so retrying
# the identical batch only repeats it until the 20-skip abort. Move
# on to fresh data. Gradient overflow (finite loss) still retries the
# same batch, where the halved loss scale changes the outcome.
if not math.isfinite(loss_value):
retry_batches.clear()
log.row({"event": "skipped_update", "step": step,
"attempt": attempt + 1, "loss": loss_value,
"grad_norm": gnorm_value, "scale": scaler.get_scale(),
"consecutive_skips": consecutive_skips})
if consecutive_skips >= 20:
log.line(f"ABORT: 20 consecutive failed updates at step {step}; same batch retried")
ckpt.save_full(ckpt_dir / f"abort_{step:09d}.pt", model=model,
muon=muon, adamw=adamw, scaler=scaler,
packer=pre, step=step, tokens_seen=tokens_seen,
cfg=cfg, ngram_fingerprint=ng_fp,
best_val=best_val, background=False)
raise SystemExit(3)
attempt += 1
win_tok += tok_acc
win_loss += loss_acc
win_steps += 1
win_flops += step_flops(cfg, tok_acc, sumsq_acc)
# -- logging ---------------------------------------------------- #
if attempt % cfg.train.log_every_steps == 0:
dt = time.time() - win_t
tps = win_tok / max(dt, 1e-9)
tflops = win_flops / max(dt, 1e-9) / 1e12
eta = (cfg.train.total_tokens - tokens_seen) / max(tps, 1e-9)
temp = gpu_temp(device_index(dev))
mem = torch.cuda.max_memory_allocated() / 2**30
log.row({"step": step, "attempt": attempt, "tokens": tokens_seen,
"loss": float(win_loss) / max(win_steps, 1),
"lr_muon": mlr, "lr_adamw": alr, "grad_norm": gnorm_value,
"scale": scaler.get_scale(), "tok_s": tps,
"tflops": tflops, "block": block, "temp": temp,
"vram_gb": mem, "phase": phase_at(progress, cfg),
"wall": time.time() - t_run})
print(f" step {step:>7,}/{total_steps:,} {progress:>6.2%} "
f"loss {float(win_loss)/max(win_steps, 1):6.4f} "
f"lr {mlr:.2e} gn {gnorm_value:5.2f} "
f"s{math.log2(max(scaler.get_scale(), 1e-45)):+03.0f} "
f"{tps/1e3:6.1f}k tok/s {tflops:5.1f} TF "
f"{temp:.0f}C {mem:4.1f}G "
f"eta {eta/86400:5.1f}d", flush=True)
# -- health ------------------------------------------------- #
# A non-finite grad norm is normal in ones and twos: GradScaler
# probes for its loss scale by overflowing on purpose. PERSISTENT
# non-finite is a different animal -- it means found_inf fires every
# step and an optimizer is being skipped every step. That is what
# happened at step 230 of the first launch: the head's dW GEMM
# overflowed fp16 independently of the loss scale, so AdamW never
# stepped again, the embeddings and head froze while Muon kept
# training the trunk, and the loss climbed from 6.48 to nan over
# 180 steps and then burned 3,300 more. Neither the temperature nor
# the throughput showed anything wrong. Stop instead of cooking.
bad_grad = 0 if math.isfinite(gnorm_value) else bad_grad + 1
# Divide by the steps ACTUALLY in this window, not the nominal
# window size. A resume lands mid-window (step 57,340 with
# log_every_steps=10), so the first line after every resume
# under-reported the loss by up to 10x -- 0.337 for a model
# whose real loss was 3.37.
win_mean = float(win_loss) / max(win_steps, 1)
bad_loss = 0 if math.isfinite(win_mean) else bad_loss + 1
if bad_loss * cfg.train.log_every_steps >= cfg.train.abort_after_bad:
print(f"\n ABORT: loss has been non-finite for "
f"{bad_loss * cfg.train.log_every_steps} steps.\n",
flush=True)
ckpt.save_full(ckpt_dir / f"abort_{step:09d}.pt", model=model,
muon=muon, adamw=adamw, scaler=scaler,
packer=pre, step=step, tokens_seen=tokens_seen,
cfg=cfg, ngram_fingerprint=ng_fp,
best_val=best_val, background=False)
raise SystemExit(3)
if floored >= cfg.train.abort_after_floored:
print(f"\n ABORT: the loss scale has been pinned at its floor "
f"({cfg.train.min_loss_scale}) for {floored} steps -- "
f"every step is being skipped and nothing is training.\n",
flush=True)
ckpt.save_full(ckpt_dir / f"abort_{step:09d}.pt", model=model,
muon=muon, adamw=adamw, scaler=scaler,
packer=pre, step=step, tokens_seen=tokens_seen,
cfg=cfg, ngram_fingerprint=ng_fp,
best_val=best_val, background=False)
raise SystemExit(3)
if bad_grad * cfg.train.log_every_steps >= cfg.train.abort_after_bad:
print(f"\n ABORT: the gradient norm has been non-finite for "
f"{bad_grad * cfg.train.log_every_steps} steps -- an "
f"optimizer is being skipped every step.\n", flush=True)
ckpt.save_full(ckpt_dir / f"abort_{step:09d}.pt", model=model,
muon=muon, adamw=adamw, scaler=scaler,
packer=pre, step=step, tokens_seen=tokens_seen,
cfg=cfg, ngram_fingerprint=ng_fp,
best_val=best_val, background=False)
raise SystemExit(3)
win_t, win_tok, win_loss, win_flops = time.time(), 0, 0.0, 0.0
win_steps = 0
# -- validation ------------------------------------------------- #
if val_packer is not None and tokens_seen >= next_val:
next_val += cfg.train.val_every_tokens
vl = quick_val(model, val_packer, cfg, dev, block, theta, progress)
best_val = min(best_val, vl)
log.row({"step": step, "tokens": tokens_seen, "val_loss": vl,
"val_ppl": math.exp(min(vl, 20))})
print(f" val {vl:.4f} ppl {math.exp(min(vl,20)):.2f}", flush=True)
# -- checkpoints ------------------------------------------------- #
if (time.time() - t_ckpt) / 60 >= cfg.train.rolling_ckpt_minutes:
t_ckpt = time.time()
p = ckpt_dir / f"rolling_{step:09d}.pt"
ckpt.save_full(p, model=model, muon=muon, adamw=adamw, scaler=scaler,
packer=pre, step=step, tokens_seen=tokens_seen,
cfg=cfg, ngram_fingerprint=ng_fp, best_val=best_val,
rotate_pattern=r"rolling_\d+\.pt$",
rotate_keep=cfg.train.rolling_ckpt_keep)
log.line(f" rolling checkpoint {p.name}")
if tokens_seen >= next_export:
next_export += cfg.train.export_every_tokens
d = cfg.eval.export_dir
p = d / f"export_{tokens_seen//10**9:04d}B.pt"
ckpt.save_export(p, model=model, cfg=cfg, step=step,
tokens_seen=tokens_seen, manifest=cfg.eval.manifest,
ngram_fingerprint=ng_fp)
log.line(f" export {p.name} -> eval queue")
if tokens_seen >= next_full:
next_full += cfg.train.full_ckpt_every_tokens
p = ckpt_dir / f"full_{tokens_seen//10**9:04d}B.pt"
ckpt.save_full(p, model=model, muon=muon, adamw=adamw, scaler=scaler,
packer=pre, step=step, tokens_seen=tokens_seen,
cfg=cfg, ngram_fingerprint=ng_fp, best_val=best_val)
log.line(f" full checkpoint {p.name}")
if not stable_saved and tokens_seen >= stable_end:
stable_saved = True
p = ckpt_dir / "stable.pt"
ckpt.save_full(p, model=model, muon=muon, adamw=adamw, scaler=scaler,
packer=pre, step=step, tokens_seen=tokens_seen,
cfg=cfg, ngram_fingerprint=ng_fp, best_val=best_val,
background=False)
log.line(f" END OF STABLE PHASE -> stable.pt (permanent, never rotated)")
pre.stop()
p = ckpt_dir / "final.pt"
ckpt.save_full(p, model=model, muon=muon, adamw=adamw, scaler=scaler,
packer=pre, step=step, tokens_seen=tokens_seen, cfg=cfg,
ngram_fingerprint=ng_fp, best_val=best_val, background=False)
log.line(f"\n done: {tokens_seen/1e9:.2f}B tokens in "
f"{(time.time()-t_run)/86400:.2f} days -> {p}")
@torch.no_grad()
def quick_val(model, val_packer, cfg, dev, block, theta, progress,
batches: int = 16):
"""A short in-loop validation. The full suite runs on the second GPU."""
model.eval()
tot, n = 0.0, 0
# Score the SAME held-out batches every time. The packer's RNG used to run
# on from call to call (and restart at its seed on every resume), so each
# validation drew a fresh sample and consecutive values wandered by about
# +/-0.05 -- larger than the effects being tracked. Snapshot the state on
# the first call and rewind to it on every call.
if not hasattr(val_packer, "_boris_val_state0"):
val_packer._boris_val_state0 = val_packer.state_dict()
val_packer.load_state_dict(val_packer._boris_val_state0)
# Stage-appropriate micro-batch: using the 2048-token value at block 8192
# would quadruple the validation activation footprint and OOM late in run.
mb = cfg.micro_batch_at(progress)
for _ in range(batches):
xs, ys, pos, bnd, bounds, _ = val_packer.batch(mb, block, 0.0)
t = lambda z: torch.from_numpy(z).to(dev)
with torch.autocast("cuda", dtype=torch.float16):
loss = model(t(xs), t(pos), t(bounds), t(bnd), t(ys), theta)
valid = int((ys != -100).sum())
tot += float(loss) * valid; n += valid
model.train()
return tot / max(n, 1)
if __name__ == "__main__":
main()
|