From 5d2e4252e695644c34be605b44a14bd80a98a0a6 Mon Sep 17 00:00:00 2001 From: jdj Date: Wed, 18 Nov 2026 14:47:33 -0800 Subject: Boris-2 training code The code, configuration and data filters used to train Boris-2 (144M, 200B tokens, single V100): model, training loop, data preparation, n-gram table, V100 kernels, final stages and the exact run configs. --- configs/boris-2-0929/config.json | 220 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 configs/boris-2-0929/config.json (limited to 'configs/boris-2-0929/config.json') diff --git a/configs/boris-2-0929/config.json b/configs/boris-2-0929/config.json new file mode 100644 index 0000000..ac90eac --- /dev/null +++ b/configs/boris-2-0929/config.json @@ -0,0 +1,220 @@ +{ + "curriculum": { + "enabled": true, + "long_doc_floor_tokens": 4096, + "long_doc_target_frac": 0.5, + "phases": [ + [ + "flat", + 0.75, + { + "climbmix": 0.932109, + "ultra_math_l3": 0.064783, + "orca_math": 0.003108, + "cosmo2_how": 0.0, + "cosmo_how_v1_tf": 0.0, + "metamath_gsm": 0.0, + "elem_wordprob": 0.0, + "dclm_r0920": 0.0 + } + ], + [ + "pre_decay_ramp", + 0.87, + { + "climbmix": 0.821333, + "ultra_math_l3": 0.081144, + "orca_math": 0.003244, + "cosmo2_how": 0.085976, + "cosmo_how_v1_tf": 0.008302, + "metamath_gsm": 0.0, + "elem_wordprob": 0.0, + "dclm_r0920": 0.0 + } + ], + [ + "decay_in", + 0.875, + { + "climbmix": 0.775534, + "ultra_math_l3": 0.07862, + "orca_math": 0.003143, + "cosmo2_how": 0.083301, + "cosmo_how_v1_tf": 0.008044, + "metamath_gsm": 0.025126, + "elem_wordprob": 0.020974, + "dclm_r0920": 0.005257 + } + ], + [ + "decay", + 0.9825, + { + "climbmix": 0.775534, + "ultra_math_l3": 0.07862, + "orca_math": 0.003143, + "cosmo2_how": 0.083301, + "cosmo_how_v1_tf": 0.008044, + "metamath_gsm": 0.025126, + "elem_wordprob": 0.020974, + "dclm_r0920": 0.005257 + } + ], + [ + "p2_tail_in", + 0.9875, + { + "climbmix": 0.580561, + "ultra_math_l3": 0.053042, + "metamath_gsm": 0.135615, + "elem_wordprob": 0.141507, + "dclm_r0920": 0.012769, + "orca_math": 0.002545, + "cosmo2_how": 0.067493, + "cosmo_how_v1_tf": 0.006468 + } + ], + [ + "p2_tail", + 1.0, + { + "climbmix": 0.580561, + "ultra_math_l3": 0.053042, + "metamath_gsm": 0.135615, + "elem_wordprob": 0.141507, + "dclm_r0920": 0.012769, + "orca_math": 0.002545, + "cosmo2_how": 0.067493, + "cosmo_how_v1_tf": 0.006468 + } + ] + ], + "smooth": true + }, + "checkpoint_dir": "/media/joseph/Media/Boris-2-0929/checkpoints", + "data": { + "eos_id": 0, + "max_doc_tokens": 65536, + "prefetch_depth": 4, + "root": "/home/joseph/boris-2-D112M-n15M/data", + "sources": [ + "climbmix", + "ultra_math_l3", + "orca_math", + "cosmo2_how", + "cosmo_how_v1_tf", + "metamath_gsm", + "elem_wordprob", + "dclm_r0920" + ], + "val_path": "/home/joseph/boris-2-D112M-n15M/data/val.bin", + "val_source": "climbmix_val" + }, + "enforce_budget": true, + "eval": { + "batch_size": 128, + "device": "cuda:1", + "export_dir": "/media/joseph/Media/Boris-2-0929/checkpoints/exports", + "manifest": "/media/joseph/Media/Boris-2-0929/checkpoints/exports/manifest.jsonl", + "out_path": "/media/joseph/Media/Boris-2-0929/eval/results.jsonl", + "poll_seconds": 120, + "tasks": [ + "hellaswag", + "arc_easy", + "arc_challenge", + "piqa", + "arithmark3" + ], + "val_tokens": 10000000 + }, + "model": { + "architecture": "boris-2-canon-ngram", + "canon_kernel": 3, + "context_stages": [ + [ + 0.0, + 2048, + 10000.0 + ] + ], + "d_model": 512, + "ffn_hidden": 1664, + "n_head": 8, + "n_kv_head": 8, + "n_layer": 31, + "norm_eps": 1e-05, + "tie_embeddings": true, + "tokenizer_sha256": "f366974b96357d11a7e9930826efd81d77438638c0e21ea1517f620da72e05f4", + "use_canon": true, + "use_ngram": true, + "vocab_size": 32768 + }, + "ngram": { + "bigram_rows": 210938, + "insert_after": 2, + "muon_on_proj": true, + "oov_rows": 1, + "proj_init_std": 0.02, + "rank": 64, + "table_init_std": 0.02, + "target_params": 15000000, + "tolerance": 0.01, + "trigram_rows": 23437, + "vocab_path": "/media/joseph/Media/Boris-2-0920/ngram_vocab.bin" + }, + "optim": { + "adamw_betas": [ + 0.9, + 0.95 + ], + "adamw_eps": 1e-08, + "adamw_lr": 0.003, + "adamw_norm_weight_decay": 0.0, + "adamw_weight_decay": 0.1, + "grad_clip": 4.0, + "lm_head_chunk": 8192, + "muon_batched": true, + "muon_lr": 0.0025, + "muon_momentum": 0.95, + "muon_nesterov": true, + "muon_ns_dtype": "float32", + "muon_ns_steps": 5, + "muon_rms_scale": true, + "muon_weight_decay": 0.1, + "adamw_canon_weight_decay": 0.1 + }, + "run_name": "Boris-2-0929", + "schedule": { + "decay_shape": "1-sqrt", + "final_lr_frac": 0.0, + "stable_frac": 0.87, + "warmup_frac": 0.01 + }, + "train": { + "abort_after_bad": 200, + "abort_after_floored": 100, + "amp_dtype": "float16", + "compile_regions": true, + "device": "cuda:0", + "eval_device": "cuda:1", + "export_every_tokens": 10000000000, + "fp16_residual": false, + "full_ckpt_every_tokens": 50000000000, + "grad_checkpoint": false, + "grad_checkpoint_every": 8, + "init_scale": 65536.0, + "log_every_steps": 10, + "max_loss_scale": 32768.0, + "micro_batch": { + "2048": 16, + "8192": 4 + }, + "min_loss_scale": 1.0, + "rolling_ckpt_keep": 2, + "rolling_ckpt_minutes": 45, + "seed": 20260906, + "tokens_per_step": 1048576, + "total_tokens": 200000000000, + "val_every_tokens": 1000000000 + } +} \ No newline at end of file -- cgit v1.2.3