diff --git a/crates/ml-alpha/src/rl/isv_slots.rs b/crates/ml-alpha/src/rl/isv_slots.rs index 51c1b1a53..e46a8c1ca 100644 --- a/crates/ml-alpha/src/rl/isv_slots.rs +++ b/crates/ml-alpha/src/rl/isv_slots.rs @@ -1886,6 +1886,24 @@ pub const RL_POLICY_GATE_ENTROPY_MEAN_INDEX: usize = 781; /// single action. pub const RL_POLICY_PER_HEAD_ENTROPY_MEAN_BASE_INDEX: usize = 782; +/// Multiplier applied to the gate Adam learning rates (W_gate, b_gate) on +/// top of the shared `RL_LR_PI_INDEX` (slot 411) value. Bootstrap 5.0 per +/// Phase 2A-D fix B1 (2026-06-03): 3-seed mid-smoke at HEAD 5f10bcde3 +/// (flag-on) showed gate_entropy_mean pinned at MAX log(K) ≈ 1.0983 for +/// 2000 steps with argmax_mass = 1.000 — the gate gradient +/// `grad_gate_probs[k] = Σ_a pi_probs_k[k,a]·grad_pi_probs[a]` is +/// structurally weak near uniform g_k ≈ 1/K because all heads contribute +/// similarly, so W_gate moves too slowly to escape the uniform basin. +/// Scaling only the gate Adams (NOT the head Adams) accelerates routing +/// learning without destabilising head training. Range [0.5, 50.0] for +/// the ISV controller — outside that we expect either degenerate noise +/// (too high) or no effect (too low). Slot defaults to 0.0 sentinel when +/// the multi-head flag is off; the bootstrap write at slot 790 happens +/// inside the same `if self.use_multi_head_policy { ... }` block as +/// slots 761-764, so flag-off bit-equality is preserved (0² contributes +/// 0 to the `isv_state` checksum). +pub const RL_POLICY_GATE_LR_MULTIPLIER_INDEX: usize = 790; + /// Last RL-allocated slot index (exclusive). /// Pre-risk-stack: 662. Post-Fix-B: 685. Post-v9 (warmup boundary): 696. /// Post-regime-observer (F1.1): 716. Post-warmed-flag addendum: 717. @@ -1902,4 +1920,5 @@ pub const RL_POLICY_PER_HEAD_ENTROPY_MEAN_BASE_INDEX: usize = 782; /// Post-rollout-buffer+GAE Phase 1B-A (3 PPO control slots): 760. /// Post-multi-head-policy Phase 2A-A (4 policy-gating slots): 764. /// Post-multi-head-policy Phase 2A-D (25 aggregate diag slots): 789. -pub const RL_SLOTS_END: usize = 790; +/// Post-Phase 2A-D fix B1 (gate LR multiplier slot 790): 790. +pub const RL_SLOTS_END: usize = 791; diff --git a/crates/ml-alpha/src/trainer/integrated.rs b/crates/ml-alpha/src/trainer/integrated.rs index 901d5889d..415130405 100644 --- a/crates/ml-alpha/src/trainer/integrated.rs +++ b/crates/ml-alpha/src/trainer/integrated.rs @@ -4167,11 +4167,16 @@ impl IntegratedTrainer { // * head_entropy_floor = log(N_ACTIONS)·0.4 ≈ 0.959 // * aux_prior_beta = 0.05 (5% of policy-loss magnitude) if self.use_multi_head_policy { - let mhp_isv_constants: [(usize, f32); 4] = [ + // Phase 2A-D fix B1 (2026-06-03): slot 790 gate-LR multiplier + // bootstrap 5.0 — addresses gate-not-learning observed in + // 3-seed mid-smoke at HEAD 5f10bcde3 (gate_entropy stayed + // pinned at MAX, argmax_mass = 1.0). See slot doc-comment. + let mhp_isv_constants: [(usize, f32); 5] = [ (crate::rl::isv_slots::RL_POLICY_NUM_HEADS_INDEX, 3.0_f32), (crate::rl::isv_slots::RL_POLICY_GATING_ENTROPY_FLOOR_INDEX, 0.549_f32), (crate::rl::isv_slots::RL_POLICY_HEAD_ENTROPY_FLOOR_INDEX, 0.959_f32), (crate::rl::isv_slots::RL_POLICY_AUX_PRIOR_BETA_INDEX, 0.05_f32), + (crate::rl::isv_slots::RL_POLICY_GATE_LR_MULTIPLIER_INDEX, 5.0_f32), ]; for (slot, value) in mhp_isv_constants.iter() { let slot_i32 = *slot as i32; @@ -5522,11 +5527,21 @@ impl IntegratedTrainer { // Phase 2A-C: mirror lr_pi onto the multi-head Adam state when // the gate is on. ISV[RL_LR_PI_INDEX] remains the single source // of truth — no separate slot for multi-head LR. + // + // Phase 2A-D fix B1 (2026-06-03): gate Adams use lr_pi × ISV[790] + // (RL_POLICY_GATE_LR_MULTIPLIER_INDEX, bootstrap 5.0) so the + // routing gradient — structurally weak near uniform g_k ≈ 1/K — + // can escape the uniform basin without amplifying head training. + // Heads keep raw lr_pi. if self.use_multi_head_policy { + let gate_lr_mult = self.read_isv_host( + crate::rl::isv_slots::RL_POLICY_GATE_LR_MULTIPLIER_INDEX, + ); + let lr_gate = lr_pi * gate_lr_mult; if let Some(a) = self.mhp_w_heads_adam.as_mut() { a.lr = lr_pi; } if let Some(a) = self.mhp_b_heads_adam.as_mut() { a.lr = lr_pi; } - if let Some(a) = self.mhp_w_gate_adam.as_mut() { a.lr = lr_pi; } - if let Some(a) = self.mhp_b_gate_adam.as_mut() { a.lr = lr_pi; } + if let Some(a) = self.mhp_w_gate_adam.as_mut() { a.lr = lr_gate; } + if let Some(a) = self.mhp_b_gate_adam.as_mut() { a.lr = lr_gate; } } self.value_w_adam.lr = lr_v; self.value_b_adam.lr = lr_v;