From 71bc2e7d65cfeeea6a8f128d372962d5c369442b Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Wed, 4 Mar 2026 21:01:34 +0100 Subject: [PATCH] =?UTF-8?q?fix(ml):=20correct=20DQNConfig=20defaults=20?= =?UTF-8?q?=E2=80=94=20widen=20v=5Fmin/v=5Fmax,=20reduce=20cql=5Falpha,=20?= =?UTF-8?q?disable=20IQN?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - v_min/v_max: -2/+2 → -10/+10 (C51 needs room to separate 45 actions) - entropy_coefficient: 0.01 → 0.05 (5x stronger anti-collapse) - cql_alpha: 1.0 → 0.1 (was adding ~3.8 loss penalty per step) - use_iqn: true → false (IQN trains q_network but inference uses dist_dueling) - Added get_count_bonuses() method for batch action selection Co-Authored-By: Claude Opus 4.6 --- crates/ml/src/dqn/dqn.rs | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/crates/ml/src/dqn/dqn.rs b/crates/ml/src/dqn/dqn.rs index 400eedae5..bd8f2a56a 100644 --- a/crates/ml/src/dqn/dqn.rs +++ b/crates/ml/src/dqn/dqn.rs @@ -235,8 +235,8 @@ impl Default for DQNConfig { dueling_hidden_dim: 128, use_distributional: true, // BUG #36 FIXED: C51 re-enabled num_atoms: 51, - v_min: -2.0, // Bug #5: Corrected from -10.0 - v_max: 2.0, // Bug #5: Corrected from 10.0 + v_min: -10.0, // Widened: [-2,2] compressed Q-values, [-10,10] gives C51 room to separate 45 actions + v_max: 10.0, // Widened: [-2,2] compressed Q-values, [-10,10] gives C51 room to separate 45 actions use_noisy_nets: false, noisy_sigma_init: 0.5, enable_q_value_clipping: true, @@ -246,17 +246,17 @@ impl Default for DQNConfig { gradient_collapse_patience: 100, // Entropy regularization: prevent 45-action collapse - entropy_coefficient: 0.01, + entropy_coefficient: 0.05, // 5x stronger anti-collapse for 45-action space noisy_epsilon_floor: 0.05, use_count_bonus: true, count_bonus_coefficient: 0.1, // CQL: Enabled by default for offline training use_cql: true, - cql_alpha: 1.0, + cql_alpha: 0.1, // Reduced from 1.0: full strength added ~ln(45)=3.8 loss penalty, crushing Q-value differentiation - // IQN: Enabled by default (complementary to C51) - use_iqn: true, + // IQN: Disabled: IQN trains base q_network but inference uses dist_dueling network (zero gradients) + use_iqn: false, iqn_num_quantiles: 64, iqn_kappa: 1.0, iqn_embedding_dim: 64, @@ -3150,6 +3150,13 @@ impl DQN { self.count_bonus.reset(); } + /// Get count bonuses for all actions (UCB exploration). + /// Returns a Vec of length num_actions with exploration bonuses. + pub fn get_count_bonuses(&self) -> Vec { + let bonuses = self.count_bonus.bonuses(); + bonuses.iter().map(|b| *b as f32).collect() + } + /// Update target network by copying weights from main network fn update_target_network(&mut self) -> Result<(), MLError> { // Wave 11.6: Update target network (hybrid > dueling > standard)