summaryrefslogtreecommitdiff
path: root/configs/boris-2-0929
diff options
context:
space:
mode:
Diffstat (limited to 'configs/boris-2-0929')
-rw-r--r--configs/boris-2-0929/RECIPE.md9
-rw-r--r--configs/boris-2-0929/config.before_2xbatch.json220
-rw-r--r--configs/boris-2-0929/config.before_decaymix.json145
-rw-r--r--configs/boris-2-0929/config.before_no8192.json225
-rw-r--r--configs/boris-2-0929/config.before_p2tail.json197
-rw-r--r--configs/boris-2-0929/config.json220
-rw-r--r--configs/boris-2-0929/final_decay_draws.json10
7 files changed, 1026 insertions, 0 deletions
diff --git a/configs/boris-2-0929/RECIPE.md b/configs/boris-2-0929/RECIPE.md
new file mode 100644
index 0000000..087ec5f
--- /dev/null
+++ b/configs/boris-2-0929/RECIPE.md
@@ -0,0 +1,9 @@
+# Boris-2 release candidate (2026-11-11)
+Official-style Open SLM Index 34.19 (HellaSwag 50.26, ARC-E 59.81, ARC-C 32.00, PIQA 72.91, ArithMark-3 official 45.10).
+Lineage:
+1. Boris-2-0929 final.pt (200B tokens, WSD 1-sqrt decay, doubled batch from 170B).
+2. "Nudge": 30M tokens, mix climbmix .55 / cosmo2_how .30 / cosmo_how_v1_tf .05 / ultra_math_l3 .07 / metamath_gsm .02 /
+ orca_math .01 (per-doc draw weights), LR 0.1x peak, 1-sqrt to 0, seed 1234 (probe/pilot_train.py --peak 0.1).
+3. 10M tokens of LibreMath v1 only (decontaminated: 13-gram + 8-gram ArithMark/SVAMP/ASDiv bans), LR 0.02x peak,
+ 1-sqrt to 0, fresh optimizer state (probe/pilot_train.py --peak 0.02).
+Selected from ~10 short final-stage candidates by benchmark score -- disclose in the model card.
diff --git a/configs/boris-2-0929/config.before_2xbatch.json b/configs/boris-2-0929/config.before_2xbatch.json
new file mode 100644
index 0000000..f138731
--- /dev/null
+++ b/configs/boris-2-0929/config.before_2xbatch.json
@@ -0,0 +1,220 @@
+{
+ "curriculum": {
+ "enabled": true,
+ "long_doc_floor_tokens": 4096,
+ "long_doc_target_frac": 0.5,
+ "phases": [
+ [
+ "flat",
+ 0.75,
+ {
+ "climbmix": 0.932109,
+ "ultra_math_l3": 0.064783,
+ "orca_math": 0.003108,
+ "cosmo2_how": 0.0,
+ "cosmo_how_v1_tf": 0.0,
+ "metamath_gsm": 0.0,
+ "elem_wordprob": 0.0,
+ "dclm_r0920": 0.0
+ }
+ ],
+ [
+ "pre_decay_ramp",
+ 0.87,
+ {
+ "climbmix": 0.821333,
+ "ultra_math_l3": 0.081144,
+ "orca_math": 0.003244,
+ "cosmo2_how": 0.085976,
+ "cosmo_how_v1_tf": 0.008302,
+ "metamath_gsm": 0.0,
+ "elem_wordprob": 0.0,
+ "dclm_r0920": 0.0
+ }
+ ],
+ [
+ "decay_in",
+ 0.875,
+ {
+ "climbmix": 0.775534,
+ "ultra_math_l3": 0.07862,
+ "orca_math": 0.003143,
+ "cosmo2_how": 0.083301,
+ "cosmo_how_v1_tf": 0.008044,
+ "metamath_gsm": 0.025126,
+ "elem_wordprob": 0.020974,
+ "dclm_r0920": 0.005257
+ }
+ ],
+ [
+ "decay",
+ 0.9825,
+ {
+ "climbmix": 0.775534,
+ "ultra_math_l3": 0.07862,
+ "orca_math": 0.003143,
+ "cosmo2_how": 0.083301,
+ "cosmo_how_v1_tf": 0.008044,
+ "metamath_gsm": 0.025126,
+ "elem_wordprob": 0.020974,
+ "dclm_r0920": 0.005257
+ }
+ ],
+ [
+ "p2_tail_in",
+ 0.9875,
+ {
+ "climbmix": 0.580561,
+ "ultra_math_l3": 0.053042,
+ "metamath_gsm": 0.135615,
+ "elem_wordprob": 0.141507,
+ "dclm_r0920": 0.012769,
+ "orca_math": 0.002545,
+ "cosmo2_how": 0.067493,
+ "cosmo_how_v1_tf": 0.006468
+ }
+ ],
+ [
+ "p2_tail",
+ 1.0,
+ {
+ "climbmix": 0.580561,
+ "ultra_math_l3": 0.053042,
+ "metamath_gsm": 0.135615,
+ "elem_wordprob": 0.141507,
+ "dclm_r0920": 0.012769,
+ "orca_math": 0.002545,
+ "cosmo2_how": 0.067493,
+ "cosmo_how_v1_tf": 0.006468
+ }
+ ]
+ ],
+ "smooth": true
+ },
+ "checkpoint_dir": "/media/joseph/Media/Boris-2-0929/checkpoints",
+ "data": {
+ "eos_id": 0,
+ "max_doc_tokens": 65536,
+ "prefetch_depth": 4,
+ "root": "/home/joseph/boris-2-D112M-n15M/data",
+ "sources": [
+ "climbmix",
+ "ultra_math_l3",
+ "orca_math",
+ "cosmo2_how",
+ "cosmo_how_v1_tf",
+ "metamath_gsm",
+ "elem_wordprob",
+ "dclm_r0920"
+ ],
+ "val_path": "/home/joseph/boris-2-D112M-n15M/data/val.bin",
+ "val_source": "climbmix_val"
+ },
+ "enforce_budget": true,
+ "eval": {
+ "batch_size": 128,
+ "device": "cuda:1",
+ "export_dir": "/media/joseph/Media/Boris-2-0929/checkpoints/exports",
+ "manifest": "/media/joseph/Media/Boris-2-0929/checkpoints/exports/manifest.jsonl",
+ "out_path": "/media/joseph/Media/Boris-2-0929/eval/results.jsonl",
+ "poll_seconds": 120,
+ "tasks": [
+ "hellaswag",
+ "arc_easy",
+ "arc_challenge",
+ "piqa",
+ "arithmark3"
+ ],
+ "val_tokens": 10000000
+ },
+ "model": {
+ "architecture": "boris-2-canon-ngram",
+ "canon_kernel": 3,
+ "context_stages": [
+ [
+ 0.0,
+ 2048,
+ 10000.0
+ ]
+ ],
+ "d_model": 512,
+ "ffn_hidden": 1664,
+ "n_head": 8,
+ "n_kv_head": 8,
+ "n_layer": 31,
+ "norm_eps": 1e-05,
+ "tie_embeddings": true,
+ "tokenizer_sha256": "f366974b96357d11a7e9930826efd81d77438638c0e21ea1517f620da72e05f4",
+ "use_canon": true,
+ "use_ngram": true,
+ "vocab_size": 32768
+ },
+ "ngram": {
+ "bigram_rows": 210938,
+ "insert_after": 2,
+ "muon_on_proj": true,
+ "oov_rows": 1,
+ "proj_init_std": 0.02,
+ "rank": 64,
+ "table_init_std": 0.02,
+ "target_params": 15000000,
+ "tolerance": 0.01,
+ "trigram_rows": 23437,
+ "vocab_path": "/media/joseph/Media/Boris-2-0920/ngram_vocab.bin"
+ },
+ "optim": {
+ "adamw_betas": [
+ 0.9,
+ 0.95
+ ],
+ "adamw_eps": 1e-08,
+ "adamw_lr": 0.003,
+ "adamw_norm_weight_decay": 0.0,
+ "adamw_weight_decay": 0.1,
+ "grad_clip": 4.0,
+ "lm_head_chunk": 8192,
+ "muon_batched": true,
+ "muon_lr": 0.0025,
+ "muon_momentum": 0.95,
+ "muon_nesterov": true,
+ "muon_ns_dtype": "float32",
+ "muon_ns_steps": 5,
+ "muon_rms_scale": true,
+ "muon_weight_decay": 0.1,
+ "adamw_canon_weight_decay": 0.1
+ },
+ "run_name": "Boris-2-0929",
+ "schedule": {
+ "decay_shape": "1-sqrt",
+ "final_lr_frac": 0.0,
+ "stable_frac": 0.87,
+ "warmup_frac": 0.01
+ },
+ "train": {
+ "abort_after_bad": 200,
+ "abort_after_floored": 100,
+ "amp_dtype": "float16",
+ "compile_regions": true,
+ "device": "cuda:0",
+ "eval_device": "cuda:1",
+ "export_every_tokens": 10000000000,
+ "fp16_residual": false,
+ "full_ckpt_every_tokens": 50000000000,
+ "grad_checkpoint": false,
+ "grad_checkpoint_every": 8,
+ "init_scale": 65536.0,
+ "log_every_steps": 10,
+ "max_loss_scale": 32768.0,
+ "micro_batch": {
+ "2048": 16,
+ "8192": 4
+ },
+ "min_loss_scale": 1.0,
+ "rolling_ckpt_keep": 2,
+ "rolling_ckpt_minutes": 45,
+ "seed": 20260906,
+ "tokens_per_step": 524288,
+ "total_tokens": 200000000000,
+ "val_every_tokens": 1000000000
+ }
+} \ No newline at end of file
diff --git a/configs/boris-2-0929/config.before_decaymix.json b/configs/boris-2-0929/config.before_decaymix.json
new file mode 100644
index 0000000..e74cbc0
--- /dev/null
+++ b/configs/boris-2-0929/config.before_decaymix.json
@@ -0,0 +1,145 @@
+{
+ "curriculum": {
+ "enabled": true,
+ "long_doc_floor_tokens": 4096,
+ "long_doc_target_frac": 0.5,
+ "phases": [
+ [
+ "flat",
+ 1.0,
+ {
+ "climbmix": 0.932109,
+ "ultra_math_l3": 0.064783,
+ "orca_math": 0.003108
+ }
+ ]
+ ],
+ "smooth": true
+ },
+ "checkpoint_dir": "/media/joseph/Media/Boris-2-0929/checkpoints",
+ "data": {
+ "eos_id": 0,
+ "max_doc_tokens": 65536,
+ "prefetch_depth": 4,
+ "root": "/home/joseph/boris-2-D112M-n15M/data",
+ "sources": [
+ "climbmix",
+ "ultra_math_l3",
+ "orca_math"
+ ],
+ "val_path": "/home/joseph/boris-2-D112M-n15M/data/val.bin",
+ "val_source": "climbmix_val"
+ },
+ "enforce_budget": true,
+ "eval": {
+ "batch_size": 128,
+ "device": "cuda:1",
+ "export_dir": "/media/joseph/Media/Boris-2-0929/checkpoints/exports",
+ "manifest": "/media/joseph/Media/Boris-2-0929/checkpoints/exports/manifest.jsonl",
+ "out_path": "/media/joseph/Media/Boris-2-0929/eval/results.jsonl",
+ "poll_seconds": 120,
+ "tasks": [
+ "hellaswag",
+ "arc_easy",
+ "arc_challenge",
+ "piqa",
+ "arithmark3"
+ ],
+ "val_tokens": 10000000
+ },
+ "model": {
+ "architecture": "boris-2-canon-ngram",
+ "canon_kernel": 3,
+ "context_stages": [
+ [
+ 0.0,
+ 2048,
+ 10000.0
+ ],
+ [
+ 0.8,
+ 8192,
+ 500000.0
+ ]
+ ],
+ "d_model": 512,
+ "ffn_hidden": 1664,
+ "n_head": 8,
+ "n_kv_head": 8,
+ "n_layer": 31,
+ "norm_eps": 1e-05,
+ "tie_embeddings": true,
+ "tokenizer_sha256": "f366974b96357d11a7e9930826efd81d77438638c0e21ea1517f620da72e05f4",
+ "use_canon": true,
+ "use_ngram": true,
+ "vocab_size": 32768
+ },
+ "ngram": {
+ "bigram_rows": 210938,
+ "insert_after": 2,
+ "muon_on_proj": true,
+ "oov_rows": 1,
+ "proj_init_std": 0.02,
+ "rank": 64,
+ "table_init_std": 0.02,
+ "target_params": 15000000,
+ "tolerance": 0.01,
+ "trigram_rows": 23437,
+ "vocab_path": "/media/joseph/Media/Boris-2-0920/ngram_vocab.bin"
+ },
+ "optim": {
+ "adamw_betas": [
+ 0.9,
+ 0.95
+ ],
+ "adamw_eps": 1e-08,
+ "adamw_lr": 0.003,
+ "adamw_norm_weight_decay": 0.0,
+ "adamw_weight_decay": 0.1,
+ "grad_clip": 4.0,
+ "lm_head_chunk": 8192,
+ "muon_batched": true,
+ "muon_lr": 0.0025,
+ "muon_momentum": 0.95,
+ "muon_nesterov": true,
+ "muon_ns_dtype": "float32",
+ "muon_ns_steps": 5,
+ "muon_rms_scale": true,
+ "muon_weight_decay": 0.1,
+ "adamw_canon_weight_decay": 0.1
+ },
+ "run_name": "Boris-2-0929",
+ "schedule": {
+ "decay_shape": "1-sqrt",
+ "final_lr_frac": 0.0,
+ "stable_frac": 0.87,
+ "warmup_frac": 0.01
+ },
+ "train": {
+ "abort_after_bad": 200,
+ "abort_after_floored": 100,
+ "amp_dtype": "float16",
+ "compile_regions": true,
+ "device": "cuda:0",
+ "eval_device": "cuda:1",
+ "export_every_tokens": 10000000000,
+ "fp16_residual": false,
+ "full_ckpt_every_tokens": 50000000000,
+ "grad_checkpoint": false,
+ "grad_checkpoint_every": 8,
+ "init_scale": 65536.0,
+ "log_every_steps": 10,
+ "max_loss_scale": 32768.0,
+ "micro_batch": {
+ "2048": 16,
+ "8192": 4
+ },
+ "min_loss_scale": 1.0,
+ "rolling_ckpt_keep": 2,
+ "rolling_ckpt_minutes": 45,
+ "seed": 20260906,
+ "tokens_per_step": 524288,
+ "total_tokens": 200000000000,
+ "val_every_tokens": 1000000000
+ }
+} \ No newline at end of file
diff --git a/configs/boris-2-0929/config.before_no8192.json b/configs/boris-2-0929/config.before_no8192.json
new file mode 100644
index 0000000..37a77ea
--- /dev/null
+++ b/configs/boris-2-0929/config.before_no8192.json
@@ -0,0 +1,225 @@
+{
+ "curriculum": {
+ "enabled": true,
+ "long_doc_floor_tokens": 4096,
+ "long_doc_target_frac": 0.5,
+ "phases": [
+ [
+ "flat",
+ 0.75,
+ {
+ "climbmix": 0.932109,
+ "ultra_math_l3": 0.064783,
+ "orca_math": 0.003108,
+ "cosmo2_how": 0.0,
+ "cosmo_how_v1_tf": 0.0,
+ "metamath_gsm": 0.0,
+ "elem_wordprob": 0.0,
+ "dclm_r0920": 0.0
+ }
+ ],
+ [
+ "pre_decay_ramp",
+ 0.87,
+ {
+ "climbmix": 0.821333,
+ "ultra_math_l3": 0.081144,
+ "orca_math": 0.003244,
+ "cosmo2_how": 0.085976,
+ "cosmo_how_v1_tf": 0.008302,
+ "metamath_gsm": 0.0,
+ "elem_wordprob": 0.0,
+ "dclm_r0920": 0.0
+ }
+ ],
+ [
+ "decay_in",
+ 0.875,
+ {
+ "climbmix": 0.775534,
+ "ultra_math_l3": 0.07862,
+ "orca_math": 0.003143,
+ "cosmo2_how": 0.083301,
+ "cosmo_how_v1_tf": 0.008044,
+ "metamath_gsm": 0.025126,
+ "elem_wordprob": 0.020974,
+ "dclm_r0920": 0.005257
+ }
+ ],
+ [
+ "decay",
+ 0.9825,
+ {
+ "climbmix": 0.775534,
+ "ultra_math_l3": 0.07862,
+ "orca_math": 0.003143,
+ "cosmo2_how": 0.083301,
+ "cosmo_how_v1_tf": 0.008044,
+ "metamath_gsm": 0.025126,
+ "elem_wordprob": 0.020974,
+ "dclm_r0920": 0.005257
+ }
+ ],
+ [
+ "p2_tail_in",
+ 0.9875,
+ {
+ "climbmix": 0.580561,
+ "ultra_math_l3": 0.053042,
+ "metamath_gsm": 0.135615,
+ "elem_wordprob": 0.141507,
+ "dclm_r0920": 0.012769,
+ "orca_math": 0.002545,
+ "cosmo2_how": 0.067493,
+ "cosmo_how_v1_tf": 0.006468
+ }
+ ],
+ [
+ "p2_tail",
+ 1.0,
+ {
+ "climbmix": 0.580561,
+ "ultra_math_l3": 0.053042,
+ "metamath_gsm": 0.135615,
+ "elem_wordprob": 0.141507,
+ "dclm_r0920": 0.012769,
+ "orca_math": 0.002545,
+ "cosmo2_how": 0.067493,
+ "cosmo_how_v1_tf": 0.006468
+ }
+ ]
+ ],
+ "smooth": true
+ },
+ "checkpoint_dir": "/media/joseph/Media/Boris-2-0929/checkpoints",
+ "data": {
+ "eos_id": 0,
+ "max_doc_tokens": 65536,
+ "prefetch_depth": 4,
+ "root": "/home/joseph/boris-2-D112M-n15M/data",
+ "sources": [
+ "climbmix",
+ "ultra_math_l3",
+ "orca_math",
+ "cosmo2_how",
+ "cosmo_how_v1_tf",
+ "metamath_gsm",
+ "elem_wordprob",
+ "dclm_r0920"
+ ],
+ "val_path": "/home/joseph/boris-2-D112M-n15M/data/val.bin",
+ "val_source": "climbmix_val"
+ },
+ "enforce_budget": true,
+ "eval": {
+ "batch_size": 128,
+ "device": "cuda:1",
+ "export_dir": "/media/joseph/Media/Boris-2-0929/checkpoints/exports",
+ "manifest": "/media/joseph/Media/Boris-2-0929/checkpoints/exports/manifest.jsonl",
+ "out_path": "/media/joseph/Media/Boris-2-0929/eval/results.jsonl",
+ "poll_seconds": 120,
+ "tasks": [
+ "hellaswag",
+ "arc_easy",
+ "arc_challenge",
+ "piqa",
+ "arithmark3"
+ ],
+ "val_tokens": 10000000
+ },
+ "model": {
+ "architecture": "boris-2-canon-ngram",
+ "canon_kernel": 3,
+ "context_stages": [
+ [
+ 0.0,
+ 2048,
+ 10000.0
+ ],
+ [
+ 0.8,
+ 8192,
+ 500000.0
+ ]
+ ],
+ "d_model": 512,
+ "ffn_hidden": 1664,
+ "n_head": 8,
+ "n_kv_head": 8,
+ "n_layer": 31,
+ "norm_eps": 1e-05,
+ "tie_embeddings": true,
+ "tokenizer_sha256": "f366974b96357d11a7e9930826efd81d77438638c0e21ea1517f620da72e05f4",
+ "use_canon": true,
+ "use_ngram": true,
+ "vocab_size": 32768
+ },
+ "ngram": {
+ "bigram_rows": 210938,
+ "insert_after": 2,
+ "muon_on_proj": true,
+ "oov_rows": 1,
+ "proj_init_std": 0.02,
+ "rank": 64,
+ "table_init_std": 0.02,
+ "target_params": 15000000,
+ "tolerance": 0.01,
+ "trigram_rows": 23437,
+ "vocab_path": "/media/joseph/Media/Boris-2-0920/ngram_vocab.bin"
+ },
+ "optim": {
+ "adamw_betas": [
+ 0.9,
+ 0.95
+ ],
+ "adamw_eps": 1e-08,
+ "adamw_lr": 0.003,
+ "adamw_norm_weight_decay": 0.0,
+ "adamw_weight_decay": 0.1,
+ "grad_clip": 4.0,
+ "lm_head_chunk": 8192,
+ "muon_batched": true,
+ "muon_lr": 0.0025,
+ "muon_momentum": 0.95,
+ "muon_nesterov": true,
+ "muon_ns_dtype": "float32",
+ "muon_ns_steps": 5,
+ "muon_rms_scale": true,
+ "muon_weight_decay": 0.1,
+ "adamw_canon_weight_decay": 0.1
+ },
+ "run_name": "Boris-2-0929",
+ "schedule": {
+ "decay_shape": "1-sqrt",
+ "final_lr_frac": 0.0,
+ "stable_frac": 0.87,
+ "warmup_frac": 0.01
+ },
+ "train": {
+ "abort_after_bad": 200,
+ "abort_after_floored": 100,
+ "amp_dtype": "float16",
+ "compile_regions": true,
+ "device": "cuda:0",
+ "eval_device": "cuda:1",
+ "export_every_tokens": 10000000000,
+ "fp16_residual": false,
+ "full_ckpt_every_tokens": 50000000000,
+ "grad_checkpoint": false,
+ "grad_checkpoint_every": 8,
+ "init_scale": 65536.0,
+ "log_every_steps": 10,
+ "max_loss_scale": 32768.0,
+ "micro_batch": {
+ "2048": 16,
+ "8192": 4
+ },
+ "min_loss_scale": 1.0,
+ "rolling_ckpt_keep": 2,
+ "rolling_ckpt_minutes": 45,
+ "seed": 20260906,
+ "tokens_per_step": 524288,
+ "total_tokens": 200000000000,
+ "val_every_tokens": 1000000000
+ }
+} \ No newline at end of file
diff --git a/configs/boris-2-0929/config.before_p2tail.json b/configs/boris-2-0929/config.before_p2tail.json
new file mode 100644
index 0000000..6c34a6c
--- /dev/null
+++ b/configs/boris-2-0929/config.before_p2tail.json
@@ -0,0 +1,197 @@
+{
+ "curriculum": {
+ "enabled": true,
+ "long_doc_floor_tokens": 4096,
+ "long_doc_target_frac": 0.5,
+ "phases": [
+ [
+ "flat",
+ 0.75,
+ {
+ "climbmix": 0.932109,
+ "ultra_math_l3": 0.064783,
+ "orca_math": 0.003108,
+ "cosmo2_how": 0.0,
+ "cosmo_how_v1_tf": 0.0,
+ "metamath_gsm": 0.0,
+ "elem_wordprob": 0.0,
+ "dclm_r0920": 0.0
+ }
+ ],
+ [
+ "pre_decay_ramp",
+ 0.87,
+ {
+ "climbmix": 0.821333,
+ "ultra_math_l3": 0.081144,
+ "orca_math": 0.003244,
+ "cosmo2_how": 0.085976,
+ "cosmo_how_v1_tf": 0.008302,
+ "metamath_gsm": 0.0,
+ "elem_wordprob": 0.0,
+ "dclm_r0920": 0.0
+ }
+ ],
+ [
+ "decay_in",
+ 0.875,
+ {
+ "climbmix": 0.775534,
+ "ultra_math_l3": 0.07862,
+ "orca_math": 0.003143,
+ "cosmo2_how": 0.083301,
+ "cosmo_how_v1_tf": 0.008044,
+ "metamath_gsm": 0.025126,
+ "elem_wordprob": 0.020974,
+ "dclm_r0920": 0.005257
+ }
+ ],
+ [
+ "decay",
+ 1.0,
+ {
+ "climbmix": 0.775534,
+ "ultra_math_l3": 0.07862,
+ "orca_math": 0.003143,
+ "cosmo2_how": 0.083301,
+ "cosmo_how_v1_tf": 0.008044,
+ "metamath_gsm": 0.025126,
+ "elem_wordprob": 0.020974,
+ "dclm_r0920": 0.005257
+ }
+ ]
+ ],
+ "smooth": true
+ },
+ "checkpoint_dir": "/media/joseph/Media/Boris-2-0929/checkpoints",
+ "data": {
+ "eos_id": 0,
+ "max_doc_tokens": 65536,
+ "prefetch_depth": 4,
+ "root": "/home/joseph/boris-2-D112M-n15M/data",
+ "sources": [
+ "climbmix",
+ "ultra_math_l3",
+ "orca_math",
+ "cosmo2_how",
+ "cosmo_how_v1_tf",
+ "metamath_gsm",
+ "elem_wordprob",
+ "dclm_r0920"
+ ],
+ "val_path": "/home/joseph/boris-2-D112M-n15M/data/val.bin",
+ "val_source": "climbmix_val"
+ },
+ "enforce_budget": true,
+ "eval": {
+ "batch_size": 128,
+ "device": "cuda:1",
+ "export_dir": "/media/joseph/Media/Boris-2-0929/checkpoints/exports",
+ "manifest": "/media/joseph/Media/Boris-2-0929/checkpoints/exports/manifest.jsonl",
+ "out_path": "/media/joseph/Media/Boris-2-0929/eval/results.jsonl",
+ "poll_seconds": 120,
+ "tasks": [
+ "hellaswag",
+ "arc_easy",
+ "arc_challenge",
+ "piqa",
+ "arithmark3"
+ ],
+ "val_tokens": 10000000
+ },
+ "model": {
+ "architecture": "boris-2-canon-ngram",
+ "canon_kernel": 3,
+ "context_stages": [
+ [
+ 0.0,
+ 2048,
+ 10000.0
+ ],
+ [
+ 0.8,
+ 8192,
+ 500000.0
+ ]
+ ],
+ "d_model": 512,
+ "ffn_hidden": 1664,
+ "n_head": 8,
+ "n_kv_head": 8,
+ "n_layer": 31,
+ "norm_eps": 1e-05,
+ "tie_embeddings": true,
+ "tokenizer_sha256": "f366974b96357d11a7e9930826efd81d77438638c0e21ea1517f620da72e05f4",
+ "use_canon": true,
+ "use_ngram": true,
+ "vocab_size": 32768
+ },
+ "ngram": {
+ "bigram_rows": 210938,
+ "insert_after": 2,
+ "muon_on_proj": true,
+ "oov_rows": 1,
+ "proj_init_std": 0.02,
+ "rank": 64,
+ "table_init_std": 0.02,
+ "target_params": 15000000,
+ "tolerance": 0.01,
+ "trigram_rows": 23437,
+ "vocab_path": "/media/joseph/Media/Boris-2-0920/ngram_vocab.bin"
+ },
+ "optim": {
+ "adamw_betas": [
+ 0.9,
+ 0.95
+ ],
+ "adamw_eps": 1e-08,
+ "adamw_lr": 0.003,
+ "adamw_norm_weight_decay": 0.0,
+ "adamw_weight_decay": 0.1,
+ "grad_clip": 4.0,
+ "lm_head_chunk": 8192,
+ "muon_batched": true,
+ "muon_lr": 0.0025,
+ "muon_momentum": 0.95,
+ "muon_nesterov": true,
+ "muon_ns_dtype": "float32",
+ "muon_ns_steps": 5,
+ "muon_rms_scale": true,
+ "muon_weight_decay": 0.1,
+ "adamw_canon_weight_decay": 0.1
+ },
+ "run_name": "Boris-2-0929",
+ "schedule": {
+ "decay_shape": "1-sqrt",
+ "final_lr_frac": 0.0,
+ "stable_frac": 0.87,
+ "warmup_frac": 0.01
+ },
+ "train": {
+ "abort_after_bad": 200,
+ "abort_after_floored": 100,
+ "amp_dtype": "float16",
+ "compile_regions": true,
+ "device": "cuda:0",
+ "eval_device": "cuda:1",
+ "export_every_tokens": 10000000000,
+ "fp16_residual": false,
+ "full_ckpt_every_tokens": 50000000000,
+ "grad_checkpoint": false,
+ "grad_checkpoint_every": 8,
+ "init_scale": 65536.0,
+ "log_every_steps": 10,
+ "max_loss_scale": 32768.0,
+ "micro_batch": {
+ "2048": 16,
+ "8192": 4
+ },
+ "min_loss_scale": 1.0,
+ "rolling_ckpt_keep": 2,
+ "rolling_ckpt_minutes": 45,
+ "seed": 20260906,
+ "tokens_per_step": 524288,
+ "total_tokens": 200000000000,
+ "val_every_tokens": 1000000000
+ }
+} \ No newline at end of file
diff --git a/configs/boris-2-0929/config.json b/configs/boris-2-0929/config.json
new file mode 100644
index 0000000..ac90eac
--- /dev/null
+++ b/configs/boris-2-0929/config.json
@@ -0,0 +1,220 @@
+{
+ "curriculum": {
+ "enabled": true,
+ "long_doc_floor_tokens": 4096,
+ "long_doc_target_frac": 0.5,
+ "phases": [
+ [
+ "flat",
+ 0.75,
+ {
+ "climbmix": 0.932109,
+ "ultra_math_l3": 0.064783,
+ "orca_math": 0.003108,
+ "cosmo2_how": 0.0,
+ "cosmo_how_v1_tf": 0.0,
+ "metamath_gsm": 0.0,
+ "elem_wordprob": 0.0,
+ "dclm_r0920": 0.0
+ }
+ ],
+ [
+ "pre_decay_ramp",
+ 0.87,
+ {
+ "climbmix": 0.821333,
+ "ultra_math_l3": 0.081144,
+ "orca_math": 0.003244,
+ "cosmo2_how": 0.085976,
+ "cosmo_how_v1_tf": 0.008302,
+ "metamath_gsm": 0.0,
+ "elem_wordprob": 0.0,
+ "dclm_r0920": 0.0
+ }
+ ],
+ [
+ "decay_in",
+ 0.875,
+ {
+ "climbmix": 0.775534,
+ "ultra_math_l3": 0.07862,
+ "orca_math": 0.003143,
+ "cosmo2_how": 0.083301,
+ "cosmo_how_v1_tf": 0.008044,
+ "metamath_gsm": 0.025126,
+ "elem_wordprob": 0.020974,
+ "dclm_r0920": 0.005257
+ }
+ ],
+ [
+ "decay",
+ 0.9825,
+ {
+ "climbmix": 0.775534,
+ "ultra_math_l3": 0.07862,
+ "orca_math": 0.003143,
+ "cosmo2_how": 0.083301,
+ "cosmo_how_v1_tf": 0.008044,
+ "metamath_gsm": 0.025126,
+ "elem_wordprob": 0.020974,
+ "dclm_r0920": 0.005257
+ }
+ ],
+ [
+ "p2_tail_in",
+ 0.9875,
+ {
+ "climbmix": 0.580561,
+ "ultra_math_l3": 0.053042,
+ "metamath_gsm": 0.135615,
+ "elem_wordprob": 0.141507,
+ "dclm_r0920": 0.012769,
+ "orca_math": 0.002545,
+ "cosmo2_how": 0.067493,
+ "cosmo_how_v1_tf": 0.006468
+ }
+ ],
+ [
+ "p2_tail",
+ 1.0,
+ {
+ "climbmix": 0.580561,
+ "ultra_math_l3": 0.053042,
+ "metamath_gsm": 0.135615,
+ "elem_wordprob": 0.141507,
+ "dclm_r0920": 0.012769,
+ "orca_math": 0.002545,
+ "cosmo2_how": 0.067493,
+ "cosmo_how_v1_tf": 0.006468
+ }
+ ]
+ ],
+ "smooth": true
+ },
+ "checkpoint_dir": "/media/joseph/Media/Boris-2-0929/checkpoints",
+ "data": {
+ "eos_id": 0,
+ "max_doc_tokens": 65536,
+ "prefetch_depth": 4,
+ "root": "/home/joseph/boris-2-D112M-n15M/data",
+ "sources": [
+ "climbmix",
+ "ultra_math_l3",
+ "orca_math",
+ "cosmo2_how",
+ "cosmo_how_v1_tf",
+ "metamath_gsm",
+ "elem_wordprob",
+ "dclm_r0920"
+ ],
+ "val_path": "/home/joseph/boris-2-D112M-n15M/data/val.bin",
+ "val_source": "climbmix_val"
+ },
+ "enforce_budget": true,
+ "eval": {
+ "batch_size": 128,
+ "device": "cuda:1",
+ "export_dir": "/media/joseph/Media/Boris-2-0929/checkpoints/exports",
+ "manifest": "/media/joseph/Media/Boris-2-0929/checkpoints/exports/manifest.jsonl",
+ "out_path": "/media/joseph/Media/Boris-2-0929/eval/results.jsonl",
+ "poll_seconds": 120,
+ "tasks": [
+ "hellaswag",
+ "arc_easy",
+ "arc_challenge",
+ "piqa",
+ "arithmark3"
+ ],
+ "val_tokens": 10000000
+ },
+ "model": {
+ "architecture": "boris-2-canon-ngram",
+ "canon_kernel": 3,
+ "context_stages": [
+ [
+ 0.0,
+ 2048,
+ 10000.0
+ ]
+ ],
+ "d_model": 512,
+ "ffn_hidden": 1664,
+ "n_head": 8,
+ "n_kv_head": 8,
+ "n_layer": 31,
+ "norm_eps": 1e-05,
+ "tie_embeddings": true,
+ "tokenizer_sha256": "f366974b96357d11a7e9930826efd81d77438638c0e21ea1517f620da72e05f4",
+ "use_canon": true,
+ "use_ngram": true,
+ "vocab_size": 32768
+ },
+ "ngram": {
+ "bigram_rows": 210938,
+ "insert_after": 2,
+ "muon_on_proj": true,
+ "oov_rows": 1,
+ "proj_init_std": 0.02,
+ "rank": 64,
+ "table_init_std": 0.02,
+ "target_params": 15000000,
+ "tolerance": 0.01,
+ "trigram_rows": 23437,
+ "vocab_path": "/media/joseph/Media/Boris-2-0920/ngram_vocab.bin"
+ },
+ "optim": {
+ "adamw_betas": [
+ 0.9,
+ 0.95
+ ],
+ "adamw_eps": 1e-08,
+ "adamw_lr": 0.003,
+ "adamw_norm_weight_decay": 0.0,
+ "adamw_weight_decay": 0.1,
+ "grad_clip": 4.0,
+ "lm_head_chunk": 8192,
+ "muon_batched": true,
+ "muon_lr": 0.0025,
+ "muon_momentum": 0.95,
+ "muon_nesterov": true,
+ "muon_ns_dtype": "float32",
+ "muon_ns_steps": 5,
+ "muon_rms_scale": true,
+ "muon_weight_decay": 0.1,
+ "adamw_canon_weight_decay": 0.1
+ },
+ "run_name": "Boris-2-0929",
+ "schedule": {
+ "decay_shape": "1-sqrt",
+ "final_lr_frac": 0.0,
+ "stable_frac": 0.87,
+ "warmup_frac": 0.01
+ },
+ "train": {
+ "abort_after_bad": 200,
+ "abort_after_floored": 100,
+ "amp_dtype": "float16",
+ "compile_regions": true,
+ "device": "cuda:0",
+ "eval_device": "cuda:1",
+ "export_every_tokens": 10000000000,
+ "fp16_residual": false,
+ "full_ckpt_every_tokens": 50000000000,
+ "grad_checkpoint": false,
+ "grad_checkpoint_every": 8,
+ "init_scale": 65536.0,
+ "log_every_steps": 10,
+ "max_loss_scale": 32768.0,
+ "micro_batch": {
+ "2048": 16,
+ "8192": 4
+ },
+ "min_loss_scale": 1.0,
+ "rolling_ckpt_keep": 2,
+ "rolling_ckpt_minutes": 45,
+ "seed": 20260906,
+ "tokens_per_step": 1048576,
+ "total_tokens": 200000000000,
+ "val_every_tokens": 1000000000
+ }
+} \ No newline at end of file
diff --git a/configs/boris-2-0929/final_decay_draws.json b/configs/boris-2-0929/final_decay_draws.json
new file mode 100644
index 0000000..0150d5c
--- /dev/null
+++ b/configs/boris-2-0929/final_decay_draws.json
@@ -0,0 +1,10 @@
+{
+ "climbmix": 0.775534,
+ "ultra_math_l3": 0.07862,
+ "cosmo2_how": 0.083301,
+ "cosmo_how_v1_tf": 0.008044,
+ "metamath_gsm": 0.025126,
+ "elem_wordprob": 0.020974,
+ "dclm_r0920": 0.005257,
+ "orca_math": 0.003143
+} \ No newline at end of file