diff --git a/ml/src/trainers/dqn.rs b/ml/src/trainers/dqn.rs index f38a5fbb2..6f60ddc28 100644 --- a/ml/src/trainers/dqn.rs +++ b/ml/src/trainers/dqn.rs @@ -1958,51 +1958,56 @@ impl DQNTrainer { self.feature_stats = Some(stats); info!("✅ WAVE 3 FIX #2: Feature normalization enabled for epoch {}+", stats_collection_epochs + 1); - // BUG #38 FIX: Clear replay buffer and reset target network - info!("🔄 BUG #38 FIX: Clearing replay buffer (removing pre-normalized experiences)"); - self.clear_replay_buffer().await?; - - info!("🔄 BUG #38 FIX: Resetting target network (updating to normalized feature space)"); - self.reset_target_network().await?; - - // Adaptive C51 bounds + // Adaptive C51 bounds (MOVED BEFORE buffer clear to use pre-normalized experiences) if self.hyperparams.use_distributional { info!("🎯 C51 Adaptive Bounds: Collecting Q-value statistics from Phase 1..."); - + + // Add buffer state logging + let buffer_size = self.agent.read().await.get_replay_buffer_size()?; + info!(" Buffer state: {} experiences (min required: {})", + buffer_size, self.hyperparams.min_replay_size); + match self.collect_qvalue_statistics().await { Ok(q_stats) => { let old_v_min = self.hyperparams.v_min; let old_v_max = self.hyperparams.v_max; let (new_v_min, new_v_max) = Self::calculate_adaptive_bounds(&q_stats, 0.3); - + let old_range = old_v_max - old_v_min; let new_range = new_v_max - new_v_min; let q_range = q_stats.max - q_stats.min; let old_coverage = if q_range > 0.0 { (old_range / q_range) * 100.0 } else { 0.0 }; let new_coverage = if q_range > 0.0 { (new_range / q_range) * 100.0 } else { 0.0 }; - - info!(" Phase 1 Q-range: [{:.2}, {:.2}] (mean: {:.2}, samples: {})", + + info!(" Phase 1 Q-range: [{:.2}, {:.2}] (mean: {:.2}, samples: {})", q_stats.min, q_stats.max, q_stats.mean, q_stats.sample_count); info!(" Old bounds: ({:.2}, {:.2}) → coverage: {:.2}%", old_v_min, old_v_max, old_coverage); info!(" New bounds: ({:.2}, {:.2}) → coverage: {:.2}%", new_v_min, new_v_max, new_coverage); - + self.reinit_categorical_distribution(new_v_min, new_v_max).await?; - + // Update hyperparams for logging self.hyperparams.v_min = new_v_min; self.hyperparams.v_max = new_v_max; - + info!("✅ C51 distribution reinitialized successfully"); }, Err(e) => { warn!("⚠️ Failed to collect Q-stats: {}", e); - warn!(" Continuing with fixed bounds ({:.2}, {:.2})", + warn!(" Continuing with fixed bounds ({:.2}, {:.2})", self.hyperparams.v_min, self.hyperparams.v_max); } } } + + // BUG #38 FIX: Clear replay buffer and reset target network (AFTER C51 stats collection) + info!("🔄 BUG #38 FIX: Clearing replay buffer (removing pre-normalized experiences)"); + self.clear_replay_buffer().await?; + + info!("🔄 BUG #38 FIX: Resetting target network (updating to normalized feature space)"); + self.reset_target_network().await?; } else if epoch < 10 && epoch % 2 == 0 { // Log progress during stats collection phase info!("📊 WAVE 3 FIX #2: Collecting feature statistics (epoch {}/10)", epoch + 1); diff --git a/ml/trained_models/dqn_epoch_17.safetensors b/ml/trained_models/dqn_epoch_17.safetensors new file mode 100644 index 000000000..714dcd408 Binary files /dev/null and b/ml/trained_models/dqn_epoch_17.safetensors differ diff --git a/ml/trained_models/dqn_epoch_8.safetensors b/ml/trained_models/dqn_epoch_8.safetensors index fe9d7b28b..714dcd408 100644 Binary files a/ml/trained_models/dqn_epoch_8.safetensors and b/ml/trained_models/dqn_epoch_8.safetensors differ diff --git a/ml/trained_models/dqn_final_epoch15.safetensors b/ml/trained_models/dqn_final_epoch15.safetensors new file mode 100644 index 000000000..714dcd408 Binary files /dev/null and b/ml/trained_models/dqn_final_epoch15.safetensors differ