summaryrefslogtreecommitdiff
path: root/configs/boris-2-0929/config.before_no8192.json
diff options
context:
space:
mode:
authorjdj <jdj@kunix.org>2026-11-18 14:47:33 -0800
committerjdj <jdj@kunix.org>2026-11-18 14:47:33 -0800
commit5d2e4252e695644c34be605b44a14bd80a98a0a6 (patch)
tree311a72eda4459af7547b9cd05c0956eafb235f2d /configs/boris-2-0929/config.before_no8192.json
downloadboris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.tar.gz
boris-2-5d2e4252e695644c34be605b44a14bd80a98a0a6.zip
Boris-2 training code
The code, configuration and data filters used to train Boris-2 (144M, 200B tokens, single V100): model, training loop, data preparation, n-gram table, V100 kernels, final stages and the exact run configs.
Diffstat (limited to 'configs/boris-2-0929/config.before_no8192.json')
-rw-r--r--configs/boris-2-0929/config.before_no8192.json225
1 files changed, 225 insertions, 0 deletions
diff --git a/configs/boris-2-0929/config.before_no8192.json b/configs/boris-2-0929/config.before_no8192.json
new file mode 100644
index 0000000..37a77ea
--- /dev/null
+++ b/configs/boris-2-0929/config.before_no8192.json
@@ -0,0 +1,225 @@
+{
+ "curriculum": {
+ "enabled": true,
+ "long_doc_floor_tokens": 4096,
+ "long_doc_target_frac": 0.5,
+ "phases": [
+ [
+ "flat",
+ 0.75,
+ {
+ "climbmix": 0.932109,
+ "ultra_math_l3": 0.064783,
+ "orca_math": 0.003108,
+ "cosmo2_how": 0.0,
+ "cosmo_how_v1_tf": 0.0,
+ "metamath_gsm": 0.0,
+ "elem_wordprob": 0.0,
+ "dclm_r0920": 0.0
+ }
+ ],
+ [
+ "pre_decay_ramp",
+ 0.87,
+ {
+ "climbmix": 0.821333,
+ "ultra_math_l3": 0.081144,
+ "orca_math": 0.003244,
+ "cosmo2_how": 0.085976,
+ "cosmo_how_v1_tf": 0.008302,
+ "metamath_gsm": 0.0,
+ "elem_wordprob": 0.0,
+ "dclm_r0920": 0.0
+ }
+ ],
+ [
+ "decay_in",
+ 0.875,
+ {
+ "climbmix": 0.775534,
+ "ultra_math_l3": 0.07862,
+ "orca_math": 0.003143,
+ "cosmo2_how": 0.083301,
+ "cosmo_how_v1_tf": 0.008044,
+ "metamath_gsm": 0.025126,
+ "elem_wordprob": 0.020974,
+ "dclm_r0920": 0.005257
+ }
+ ],
+ [
+ "decay",
+ 0.9825,
+ {
+ "climbmix": 0.775534,
+ "ultra_math_l3": 0.07862,
+ "orca_math": 0.003143,
+ "cosmo2_how": 0.083301,
+ "cosmo_how_v1_tf": 0.008044,
+ "metamath_gsm": 0.025126,
+ "elem_wordprob": 0.020974,
+ "dclm_r0920": 0.005257
+ }
+ ],
+ [
+ "p2_tail_in",
+ 0.9875,
+ {
+ "climbmix": 0.580561,
+ "ultra_math_l3": 0.053042,
+ "metamath_gsm": 0.135615,
+ "elem_wordprob": 0.141507,
+ "dclm_r0920": 0.012769,
+ "orca_math": 0.002545,
+ "cosmo2_how": 0.067493,
+ "cosmo_how_v1_tf": 0.006468
+ }
+ ],
+ [
+ "p2_tail",
+ 1.0,
+ {
+ "climbmix": 0.580561,
+ "ultra_math_l3": 0.053042,
+ "metamath_gsm": 0.135615,
+ "elem_wordprob": 0.141507,
+ "dclm_r0920": 0.012769,
+ "orca_math": 0.002545,
+ "cosmo2_how": 0.067493,
+ "cosmo_how_v1_tf": 0.006468
+ }
+ ]
+ ],
+ "smooth": true
+ },
+ "checkpoint_dir": "/media/joseph/Media/Boris-2-0929/checkpoints",
+ "data": {
+ "eos_id": 0,
+ "max_doc_tokens": 65536,
+ "prefetch_depth": 4,
+ "root": "/home/joseph/boris-2-D112M-n15M/data",
+ "sources": [
+ "climbmix",
+ "ultra_math_l3",
+ "orca_math",
+ "cosmo2_how",
+ "cosmo_how_v1_tf",
+ "metamath_gsm",
+ "elem_wordprob",
+ "dclm_r0920"
+ ],
+ "val_path": "/home/joseph/boris-2-D112M-n15M/data/val.bin",
+ "val_source": "climbmix_val"
+ },
+ "enforce_budget": true,
+ "eval": {
+ "batch_size": 128,
+ "device": "cuda:1",
+ "export_dir": "/media/joseph/Media/Boris-2-0929/checkpoints/exports",
+ "manifest": "/media/joseph/Media/Boris-2-0929/checkpoints/exports/manifest.jsonl",
+ "out_path": "/media/joseph/Media/Boris-2-0929/eval/results.jsonl",
+ "poll_seconds": 120,
+ "tasks": [
+ "hellaswag",
+ "arc_easy",
+ "arc_challenge",
+ "piqa",
+ "arithmark3"
+ ],
+ "val_tokens": 10000000
+ },
+ "model": {
+ "architecture": "boris-2-canon-ngram",
+ "canon_kernel": 3,
+ "context_stages": [
+ [
+ 0.0,
+ 2048,
+ 10000.0
+ ],
+ [
+ 0.8,
+ 8192,
+ 500000.0
+ ]
+ ],
+ "d_model": 512,
+ "ffn_hidden": 1664,
+ "n_head": 8,
+ "n_kv_head": 8,
+ "n_layer": 31,
+ "norm_eps": 1e-05,
+ "tie_embeddings": true,
+ "tokenizer_sha256": "f366974b96357d11a7e9930826efd81d77438638c0e21ea1517f620da72e05f4",
+ "use_canon": true,
+ "use_ngram": true,
+ "vocab_size": 32768
+ },
+ "ngram": {
+ "bigram_rows": 210938,
+ "insert_after": 2,
+ "muon_on_proj": true,
+ "oov_rows": 1,
+ "proj_init_std": 0.02,
+ "rank": 64,
+ "table_init_std": 0.02,
+ "target_params": 15000000,
+ "tolerance": 0.01,
+ "trigram_rows": 23437,
+ "vocab_path": "/media/joseph/Media/Boris-2-0920/ngram_vocab.bin"
+ },
+ "optim": {
+ "adamw_betas": [
+ 0.9,
+ 0.95
+ ],
+ "adamw_eps": 1e-08,
+ "adamw_lr": 0.003,
+ "adamw_norm_weight_decay": 0.0,
+ "adamw_weight_decay": 0.1,
+ "grad_clip": 4.0,
+ "lm_head_chunk": 8192,
+ "muon_batched": true,
+ "muon_lr": 0.0025,
+ "muon_momentum": 0.95,
+ "muon_nesterov": true,
+ "muon_ns_dtype": "float32",
+ "muon_ns_steps": 5,
+ "muon_rms_scale": true,
+ "muon_weight_decay": 0.1,
+ "adamw_canon_weight_decay": 0.1
+ },
+ "run_name": "Boris-2-0929",
+ "schedule": {
+ "decay_shape": "1-sqrt",
+ "final_lr_frac": 0.0,
+ "stable_frac": 0.87,
+ "warmup_frac": 0.01
+ },
+ "train": {
+ "abort_after_bad": 200,
+ "abort_after_floored": 100,
+ "amp_dtype": "float16",
+ "compile_regions": true,
+ "device": "cuda:0",
+ "eval_device": "cuda:1",
+ "export_every_tokens": 10000000000,
+ "fp16_residual": false,
+ "full_ckpt_every_tokens": 50000000000,
+ "grad_checkpoint": false,
+ "grad_checkpoint_every": 8,
+ "init_scale": 65536.0,
+ "log_every_steps": 10,
+ "max_loss_scale": 32768.0,
+ "micro_batch": {
+ "2048": 16,
+ "8192": 4
+ },
+ "min_loss_scale": 1.0,
+ "rolling_ckpt_keep": 2,
+ "rolling_ckpt_minutes": 45,
+ "seed": 20260906,
+ "tokens_per_step": 524288,
+ "total_tokens": 200000000000,
+ "val_every_tokens": 1000000000
+ }
+} \ No newline at end of file