diff --git a/crates/ml/src/dqn/dqn.rs b/crates/ml/src/dqn/dqn.rs index 400eedae5..bd8f2a56a 100644 --- a/crates/ml/src/dqn/dqn.rs +++ b/crates/ml/src/dqn/dqn.rs @@ -235,8 +235,8 @@ impl Default for DQNConfig { dueling_hidden_dim: 128, use_distributional: true, // BUG #36 FIXED: C51 re-enabled num_atoms: 51, - v_min: -2.0, // Bug #5: Corrected from -10.0 - v_max: 2.0, // Bug #5: Corrected from 10.0 + v_min: -10.0, // Widened: [-2,2] compressed Q-values, [-10,10] gives C51 room to separate 45 actions + v_max: 10.0, // Widened: [-2,2] compressed Q-values, [-10,10] gives C51 room to separate 45 actions use_noisy_nets: false, noisy_sigma_init: 0.5, enable_q_value_clipping: true, @@ -246,17 +246,17 @@ impl Default for DQNConfig { gradient_collapse_patience: 100, // Entropy regularization: prevent 45-action collapse - entropy_coefficient: 0.01, + entropy_coefficient: 0.05, // 5x stronger anti-collapse for 45-action space noisy_epsilon_floor: 0.05, use_count_bonus: true, count_bonus_coefficient: 0.1, // CQL: Enabled by default for offline training use_cql: true, - cql_alpha: 1.0, + cql_alpha: 0.1, // Reduced from 1.0: full strength added ~ln(45)=3.8 loss penalty, crushing Q-value differentiation - // IQN: Enabled by default (complementary to C51) - use_iqn: true, + // IQN: Disabled: IQN trains base q_network but inference uses dist_dueling network (zero gradients) + use_iqn: false, iqn_num_quantiles: 64, iqn_kappa: 1.0, iqn_embedding_dim: 64, @@ -3150,6 +3150,13 @@ impl DQN { self.count_bonus.reset(); } + /// Get count bonuses for all actions (UCB exploration). + /// Returns a Vec of length num_actions with exploration bonuses. + pub fn get_count_bonuses(&self) -> Vec { + let bonuses = self.count_bonus.bonuses(); + bonuses.iter().map(|b| *b as f32).collect() + } + /// Update target network by copying weights from main network fn update_target_network(&mut self) -> Result<(), MLError> { // Wave 11.6: Update target network (hybrid > dueling > standard)