{ "curriculum": { "enabled": true, "long_doc_floor_tokens": 4096, "long_doc_target_frac": 0.5, "phases": [ [ "flat", 0.75, { "climbmix": 0.932109, "ultra_math_l3": 0.064783, "orca_math": 0.003108, "cosmo2_how": 0.0, "cosmo_how_v1_tf": 0.0, "metamath_gsm": 0.0, "elem_wordprob": 0.0, "dclm_r0920": 0.0 } ], [ "pre_decay_ramp", 0.87, { "climbmix": 0.821333, "ultra_math_l3": 0.081144, "orca_math": 0.003244, "cosmo2_how": 0.085976, "cosmo_how_v1_tf": 0.008302, "metamath_gsm": 0.0, "elem_wordprob": 0.0, "dclm_r0920": 0.0 } ], [ "decay_in", 0.875, { "climbmix": 0.775534, "ultra_math_l3": 0.07862, "orca_math": 0.003143, "cosmo2_how": 0.083301, "cosmo_how_v1_tf": 0.008044, "metamath_gsm": 0.025126, "elem_wordprob": 0.020974, "dclm_r0920": 0.005257 } ], [ "decay", 0.9825, { "climbmix": 0.775534, "ultra_math_l3": 0.07862, "orca_math": 0.003143, "cosmo2_how": 0.083301, "cosmo_how_v1_tf": 0.008044, "metamath_gsm": 0.025126, "elem_wordprob": 0.020974, "dclm_r0920": 0.005257 } ], [ "p2_tail_in", 0.9875, { "climbmix": 0.580561, "ultra_math_l3": 0.053042, "metamath_gsm": 0.135615, "elem_wordprob": 0.141507, "dclm_r0920": 0.012769, "orca_math": 0.002545, "cosmo2_how": 0.067493, "cosmo_how_v1_tf": 0.006468 } ], [ "p2_tail", 1.0, { "climbmix": 0.580561, "ultra_math_l3": 0.053042, "metamath_gsm": 0.135615, "elem_wordprob": 0.141507, "dclm_r0920": 0.012769, "orca_math": 0.002545, "cosmo2_how": 0.067493, "cosmo_how_v1_tf": 0.006468 } ] ], "smooth": true }, "checkpoint_dir": "/media/joseph/Media/Boris-2-0929/checkpoints", "data": { "eos_id": 0, "max_doc_tokens": 65536, "prefetch_depth": 4, "root": "/home/joseph/boris-2-D112M-n15M/data", "sources": [ "climbmix", "ultra_math_l3", "orca_math", "cosmo2_how", "cosmo_how_v1_tf", "metamath_gsm", "elem_wordprob", "dclm_r0920" ], "val_path": "/home/joseph/boris-2-D112M-n15M/data/val.bin", "val_source": "climbmix_val" }, "enforce_budget": true, "eval": { "batch_size": 128, "device": "cuda:1", "export_dir": "/media/joseph/Media/Boris-2-0929/checkpoints/exports", "manifest": "/media/joseph/Media/Boris-2-0929/checkpoints/exports/manifest.jsonl", "out_path": "/media/joseph/Media/Boris-2-0929/eval/results.jsonl", "poll_seconds": 120, "tasks": [ "hellaswag", "arc_easy", "arc_challenge", "piqa", "arithmark3" ], "val_tokens": 10000000 }, "model": { "architecture": "boris-2-canon-ngram", "canon_kernel": 3, "context_stages": [ [ 0.0, 2048, 10000.0 ] ], "d_model": 512, "ffn_hidden": 1664, "n_head": 8, "n_kv_head": 8, "n_layer": 31, "norm_eps": 1e-05, "tie_embeddings": true, "tokenizer_sha256": "f366974b96357d11a7e9930826efd81d77438638c0e21ea1517f620da72e05f4", "use_canon": true, "use_ngram": true, "vocab_size": 32768 }, "ngram": { "bigram_rows": 210938, "insert_after": 2, "muon_on_proj": true, "oov_rows": 1, "proj_init_std": 0.02, "rank": 64, "table_init_std": 0.02, "target_params": 15000000, "tolerance": 0.01, "trigram_rows": 23437, "vocab_path": "/media/joseph/Media/Boris-2-0920/ngram_vocab.bin" }, "optim": { "adamw_betas": [ 0.9, 0.95 ], "adamw_eps": 1e-08, "adamw_lr": 0.003, "adamw_norm_weight_decay": 0.0, "adamw_weight_decay": 0.1, "grad_clip": 4.0, "lm_head_chunk": 8192, "muon_batched": true, "muon_lr": 0.0025, "muon_momentum": 0.95, "muon_nesterov": true, "muon_ns_dtype": "float32", "muon_ns_steps": 5, "muon_rms_scale": true, "muon_weight_decay": 0.1, "adamw_canon_weight_decay": 0.1 }, "run_name": "Boris-2-0929", "schedule": { "decay_shape": "1-sqrt", "final_lr_frac": 0.0, "stable_frac": 0.87, "warmup_frac": 0.01 }, "train": { "abort_after_bad": 200, "abort_after_floored": 100, "amp_dtype": "float16", "compile_regions": true, "device": "cuda:0", "eval_device": "cuda:1", "export_every_tokens": 10000000000, "fp16_residual": false, "full_ckpt_every_tokens": 50000000000, "grad_checkpoint": false, "grad_checkpoint_every": 8, "init_scale": 65536.0, "log_every_steps": 10, "max_loss_scale": 32768.0, "micro_batch": { "2048": 16, "8192": 4 }, "min_loss_scale": 1.0, "rolling_ckpt_keep": 2, "rolling_ckpt_minutes": 45, "seed": 20260906, "tokens_per_step": 524288, "total_tokens": 200000000000, "val_every_tokens": 1000000000 } }