diff --git a/crates/ml/src/dqn/dqn.rs b/crates/ml/src/dqn/dqn.rs index 85bd71379..682ea550e 100644 --- a/crates/ml/src/dqn/dqn.rs +++ b/crates/ml/src/dqn/dqn.rs @@ -1453,9 +1453,12 @@ impl DQN { // During warmup period: always use random exploration (epsilon=1.0) let in_warmup = self.total_steps <= self.config.warmup_steps as u64; - // C2 FIX: Noisy nets are sole exploration mechanism — no epsilon floor. + // When noisy nets are active, use noisy_epsilon_floor as a minimum random + // exploration rate instead of the full epsilon schedule. This prevents + // triple-stacking (noisy + epsilon + count) while ensuring some random + // actions feed diverse experiences into the replay buffer. let effective_epsilon = if self.config.use_noisy_nets { - 0.0_f32 // Noisy nets only — epsilon removed to prevent triple-stacking + self.config.noisy_epsilon_floor // Minimum exploration floor for noisy nets } else { self.epsilon // Standard epsilon-greedy exploration }; @@ -1500,9 +1503,20 @@ impl DQN { .map_err(|e| MLError::ModelError(format!("Expected Q computation failed: {}", e)))? }; - // C2 FIX: Count bonus removed from Q-value computation. - // Noisy nets are the sole exploration mechanism. Count bonus is - // kept for diversity metrics only (record_action + get_count_bonuses). + // Add UCB count bonus to Q-values to encourage visiting under-explored actions. + // Noisy nets + epsilon floor handle random exploration; count bonus handles + // directed exploration toward actions the agent has been ignoring. + let action_scores = if self.config.use_count_bonus { + let bonuses = self.count_bonus.bonuses(); + let bonus_tensor = Tensor::from_vec( + bonuses.iter().map(|b| *b as f32).collect::>(), + (1, self.config.num_actions), + &self.device, + ).map_err(|e| MLError::ModelError(format!("Count bonus tensor: {}", e)))?; + action_scores.add(&bonus_tensor)? + } else { + action_scores + }; // Argmax over actions let best_action_idx = action_scores @@ -1516,7 +1530,18 @@ impl DQN { // Standard Q-network action selection let q_values = self.forward(&state_tensor)?; - // C2 FIX: Count bonus removed from Q-values (metrics only). + // Add UCB count bonus to Q-values for directed exploration + let q_values = if self.config.use_count_bonus { + let bonuses = self.count_bonus.bonuses(); + let bonus_tensor = Tensor::from_vec( + bonuses.iter().map(|b| *b as f32).collect::>(), + (1, self.config.num_actions), + &self.device, + ).map_err(|e| MLError::ModelError(format!("Count bonus tensor: {}", e)))?; + q_values.add(&bonus_tensor)? + } else { + q_values + }; let best_action_idx = q_values .argmax(1)? @@ -1583,9 +1608,9 @@ impl DQN { // During warmup period: always use random exploration (epsilon=1.0) let in_warmup = self.total_steps <= self.config.warmup_steps as u64; - // C2 FIX: Noisy nets are sole exploration mechanism — no epsilon floor. + // Noisy nets + epsilon floor (same logic as select_action) let effective_epsilon = if self.config.use_noisy_nets { - 0.0_f32 // C2: Noisy nets only — no epsilon floor + self.config.noisy_epsilon_floor } else { self.epsilon }; @@ -1624,7 +1649,18 @@ impl DQN { .map_err(|e| MLError::ModelError(format!("Expected Q computation failed: {}", e)))? }; - // C2 FIX: Count bonus removed from Q-values (metrics only). + // Add UCB count bonus for directed exploration + let action_scores = if self.config.use_count_bonus { + let bonuses = self.count_bonus.bonuses(); + let bonus_tensor = Tensor::from_vec( + bonuses.iter().map(|b| *b as f32).collect::>(), + (1, self.config.num_actions), + &self.device, + ).map_err(|e| MLError::ModelError(format!("Count bonus tensor: {}", e)))?; + action_scores.add(&bonus_tensor)? + } else { + action_scores + }; let conf = Self::softmax_confidence(&action_scores)?; @@ -1639,7 +1675,18 @@ impl DQN { // Standard Q-network path let q_values = self.forward(&state_tensor)?; - // C2 FIX: Count bonus removed from Q-values (metrics only). + // Add UCB count bonus for directed exploration + let q_values = if self.config.use_count_bonus { + let bonuses = self.count_bonus.bonuses(); + let bonus_tensor = Tensor::from_vec( + bonuses.iter().map(|b| *b as f32).collect::>(), + (1, self.config.num_actions), + &self.device, + ).map_err(|e| MLError::ModelError(format!("Count bonus tensor: {}", e)))?; + q_values.add(&bonus_tensor)? + } else { + q_values + }; let conf = Self::softmax_confidence(&q_values)?; diff --git a/crates/ml/src/hyperopt/adapters/dqn.rs b/crates/ml/src/hyperopt/adapters/dqn.rs index b3665731b..6fe9ca82a 100644 --- a/crates/ml/src/hyperopt/adapters/dqn.rs +++ b/crates/ml/src/hyperopt/adapters/dqn.rs @@ -462,7 +462,7 @@ impl Default for DQNParams { num_quantiles: 32, qr_kappa: 1.0, hidden_dim_base: 256, // Conservative default (backward compatible) - noisy_epsilon_floor: 0.0, // C2: Noisy nets provide exploration; no epsilon floor during training + noisy_epsilon_floor: 0.05, // Minimum random exploration when noisy nets active count_bonus_coefficient: 0.1, // UCB exploration bonus cql_alpha: 0.1, // CQL regularization: mild penalty on OOD Q-values eval_softmax_temp: 0.1, // Default: nearly greedy (hyperopt will tune) @@ -514,8 +514,8 @@ impl ParameterSpace for DQNParams { // GPU-dynamic network sizing (1D) (256.0, 4096.0), // 22: hidden_dim_base (linear, step=256) - // Exploration anti-collapse (1D) - (0.0, 0.05), // 23: noisy_epsilon_floor (linear, 0.0=noisy-nets-only) + // Exploration anti-collapse (1D) — minimum random action probability when noisy nets active + (0.03, 0.15), // 23: noisy_epsilon_floor (linear, ensures replay buffer diversity) // CQL regularization (1D) (0.0, 0.5), // 24: cql_alpha (linear) @@ -579,7 +579,7 @@ impl ParameterSpace for DQNParams { let hidden_dim_base = ((x[22].round() / 256.0).round() * 256.0).clamp(256.0, 4096.0) as usize; // Exploration anti-collapse - let noisy_epsilon_floor = x[23].clamp(0.0, 0.05); + let noisy_epsilon_floor = x[23].clamp(0.03, 0.15); // CQL regularization let cql_alpha = x[24].clamp(0.0, 0.5); @@ -3612,7 +3612,7 @@ mod tests { // Curiosity + exploration assert_eq!(bounds[20], (0.01, 0.5)); // curiosity_weight assert_eq!(bounds[22], (256.0, 4096.0)); // hidden_dim_base - assert_eq!(bounds[23], (0.0, 0.05)); // noisy_epsilon_floor + assert_eq!(bounds[23], (0.03, 0.15)); // noisy_epsilon_floor // CQL regularization assert_eq!(bounds[24], (0.0, 0.5)); // cql_alpha @@ -3697,7 +3697,7 @@ mod tests { 0.01, // curiosity_weight min 0.0001_f64.ln(), // tau min 256.0, // hidden_dim_base min - 0.0, // noisy_epsilon_floor min + 0.03, // noisy_epsilon_floor min 0.0, // cql_alpha min 0.5_f64.ln(), // eval_softmax_temp min 0.0, // 26: lr_decay_type (constant) @@ -3723,7 +3723,7 @@ mod tests { 0.5, // curiosity_weight max 0.01_f64.ln(), // tau max 4096.0, // hidden_dim_base max - 0.05, // noisy_epsilon_floor max + 0.15, // noisy_epsilon_floor max 0.5, // cql_alpha max 2.0_f64.ln(), // eval_softmax_temp max 2.0, // 26: lr_decay_type (cosine) diff --git a/crates/ml/src/trainers/dqn/trainer.rs b/crates/ml/src/trainers/dqn/trainer.rs index 48bf555a3..c14b832ad 100644 --- a/crates/ml/src/trainers/dqn/trainer.rs +++ b/crates/ml/src/trainers/dqn/trainer.rs @@ -1473,15 +1473,14 @@ impl DQNTrainer { info!(" ❌ Noisy Networks"); } - // BUG #40 FIX: When noisy nets are enabled, set stored epsilon to 0. - // Without this fix, epsilon starts at 1.0 and never decays (update_epsilon - // is skipped when use_noisy_nets=true), causing 100% random action selection. - // NOTE: select_action() also applies noisy_epsilon_floor (default 5%) as a - // safety floor for 45-action spaces — that floor is separate from this field. + // When noisy nets are enabled, epsilon-greedy is replaced by noisy_epsilon_floor. + // Set stored epsilon to the floor value so get_epsilon() reports it accurately. + // select_action() reads noisy_epsilon_floor from DQNConfig directly. if self.hyperparams.use_noisy_nets { + let floor = self.hyperparams.noisy_epsilon_floor.unwrap_or(0.05); let mut agent = self.agent.write().await; - agent.set_epsilon(0.0); - info!(" 🔧 BUG #40: Stored epsilon set to 0.0 (noisy nets + noisy_epsilon_floor provide exploration)"); + agent.set_epsilon(floor); + info!(" Noisy nets active: epsilon set to noisy_epsilon_floor={:.4}", floor); } // Training loop @@ -4640,24 +4639,24 @@ mod tests { assert!(!names.contains(&"count_bonus_coefficient"), "count_bonus_coefficient must not be in search space"); } - /// Verify noisy_epsilon_floor defaults to 0.0 (noisy nets only). + /// Verify noisy_epsilon_floor defaults to 0.05 (minimum exploration). #[test] - fn test_c2_noisy_epsilon_floor_defaults_to_zero() { + fn test_noisy_epsilon_floor_default() { let params = crate::hyperopt::adapters::dqn::DQNParams::default(); assert!( - (params.noisy_epsilon_floor - 0.0).abs() < f64::EPSILON, - "noisy_epsilon_floor must default to 0.0 (noisy nets only)" + (params.noisy_epsilon_floor - 0.05).abs() < f64::EPSILON, + "noisy_epsilon_floor must default to 0.05" ); } - /// Verify noisy_epsilon_floor search bounds are [0.0, 0.05]. + /// Verify noisy_epsilon_floor search bounds are [0.03, 0.15]. #[test] - fn test_c2_noisy_epsilon_floor_bounds() { + fn test_noisy_epsilon_floor_bounds() { let bounds = crate::hyperopt::adapters::dqn::DQNParams::continuous_bounds(); // Index 23 = noisy_epsilon_floor let (lo, hi) = bounds[23]; - assert!((lo - 0.0).abs() < f64::EPSILON, "noisy_epsilon_floor lower bound must be 0.0"); - assert!((hi - 0.05).abs() < f64::EPSILON, "noisy_epsilon_floor upper bound must be 0.05"); + assert!((lo - 0.03).abs() < f64::EPSILON, "noisy_epsilon_floor lower bound must be 0.03"); + assert!((hi - 0.15).abs() < f64::EPSILON, "noisy_epsilon_floor upper bound must be 0.15"); } /// Verify count_bonus_coefficient is fixed at 0.1 (not tunable).