diff --git a/crates/ml/src/trainers/dqn/smoke_tests/mod.rs b/crates/ml/src/trainers/dqn/smoke_tests/mod.rs index 12a29580d..ae02f7b8f 100644 --- a/crates/ml/src/trainers/dqn/smoke_tests/mod.rs +++ b/crates/ml/src/trainers/dqn/smoke_tests/mod.rs @@ -17,7 +17,7 @@ mod hyperopt; #[cfg(test)] mod regression; #[cfg(test)] -pub mod reward_v8; +mod reward_v8; #[cfg(test)] mod generalization; #[cfg(test)] diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index 67e306eb3..6bba5e18a 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -1821,7 +1821,28 @@ impl DQNTrainer { .unwrap_or(1.0); let grad_norm = avg_grad.abs(); - let ens_disagreement = 0.1f32; // placeholder — D6/N6 task will replace with real ensemble pairwise Q-gap + + // Ensemble disagreement: dispersion across the 4 per-branch Q-gaps as a + // proxy for true per-head ensemble disagreement (per-head Q readback isn't + // wired in this codebase). When all branches collapse to similar Q-gap, the + // range shrinks → low disagreement → D6/N6 oracle score rises. When + // branches preserve diverse Q-gaps (healthy), the range is wider. Maps + // directly into the smoothstep(0.01, 0.1) thresholds in from_raw and the + // D6 oracle's own smoothstep with the same window. + let ens_disagreement = if let Some(ref fused) = self.fused_ctx { + let gaps = fused.per_branch_q_gap_ema(); + let mut gmin = f32::INFINITY; + let mut gmax = f32::NEG_INFINITY; + for &g in &gaps { + if g.is_finite() { + if g < gmin { gmin = g; } + if g > gmax { gmax = g; } + } + } + if gmin.is_finite() && gmax.is_finite() { (gmax - gmin).max(0.0) } else { 0.1 } + } else { + 0.1 + }; // Update EMAs. self.health_ema.update(q_gap_signal, q_var_signal, grad_norm); @@ -2082,8 +2103,8 @@ impl DQNTrainer { self.learning_health.components.spectral_gap_norm, self.last_cql_alpha_eff.unwrap_or(0.0), self.last_iqn_budget_eff.unwrap_or(0.40), - self.last_cql_budget_eff.unwrap_or(0.10), - self.last_c51_budget_eff.unwrap_or(0.45), + self.last_cql_budget_eff.unwrap_or(0.00), + self.last_c51_budget_eff.unwrap_or(0.55), self.last_tau_eff.unwrap_or(0.005), self.last_sarsa_tau_factor.unwrap_or(1.0), self.last_gamma_eff.unwrap_or(0.99),