From 5ace5dd24f565146af93b72eb0b0cde934042cc6 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Fri, 6 Mar 2026 15:29:02 +0100 Subject: [PATCH] =?UTF-8?q?fix(ml):=20DQN=20hyperopt=20overhaul=20?= =?UTF-8?q?=E2=80=94=20C2=20triple=20exploration=20+=20eval=205-action=20c?= =?UTF-8?q?ompat?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Critical fixes: - hyperopt backtest: ExposureLevel::from_index() replaces FactoredAction::from_index() which mapped all DQN indices 0-4 to Short100 (every trial ran all-short) - evaluate_baseline: same fix + DQN num_actions default 5, PPO hardcoded 45 - simulate_chunk_trades: is_dqn dispatch for DQN vs PPO action decoding C2 triple exploration stacking: - Removed count bonus UCB from Q-value computation in both batch paths (noisy nets are sole exploration mechanism) - Narrowed noisy_epsilon_floor from [0.02, 0.10] to [0.0, 0.05], default 0.0 - Removed count_bonus_coefficient from search space (30D → 29D) - Count bonus module kept for diversity metrics tracking only Smoke tests: 6 new tests verifying 5-action space, 29D search space, epsilon floor defaults, count bonus coefficient fixed at 0.1 2735 tests passed, 0 clippy warnings. Co-Authored-By: Claude Opus 4.6 --- crates/ml/examples/evaluate_baseline.rs | 35 +++++--- crates/ml/src/hyperopt/adapters/dqn.rs | 105 ++++++++++++---------- crates/ml/src/trainers/dqn/trainer.rs | 113 +++++++++++++++++------- 3 files changed, 162 insertions(+), 91 deletions(-) diff --git a/crates/ml/examples/evaluate_baseline.rs b/crates/ml/examples/evaluate_baseline.rs index b47f164bc..8d0c60889 100644 --- a/crates/ml/examples/evaluate_baseline.rs +++ b/crates/ml/examples/evaluate_baseline.rs @@ -42,7 +42,7 @@ use tracing::{error, info, warn}; use common::metrics::{server as metrics_server, training_metrics as tm}; use ml::common::action::{ExposureLevel, FactoredAction, OrderType as ActionOrderType, Urgency}; -use ml::dqn::{DQNConfig, DQN}; +use ml::dqn::{DQNConfig, OrderRouter, DQN}; #[allow(unreachable_pub)] mod baseline_common; @@ -92,8 +92,8 @@ struct Args { #[arg(long, default_value_t = 54)] feature_dim: usize, - /// Number of actions for the DQN/PPO action space (45 = 5 exposure x 3 order x 3 urgency) - #[arg(long, default_value_t = 45)] + /// Number of actions (5 exposure levels for DQN, pass --num-actions 45 for PPO) + #[arg(long, default_value_t = 5)] num_actions: usize, /// Walk-forward: initial training window in months @@ -459,15 +459,26 @@ fn simulate_chunk_trades( returns: &mut Vec, action_counts: &mut [usize; 3], model_name: &str, + is_dqn: bool, ) { for (i, &action_idx) in action_indices.iter().enumerate() { let bar_idx = chunk_start + i; - let action = match FactoredAction::from_index(action_idx) { - Ok(fa) => fa, - Err(e) => { - warn!(" [{}] from_index({}) error at bar {}: {}", model_name, action_idx, bar_idx, e); - FactoredAction::new(ExposureLevel::Flat, ActionOrderType::Market, Urgency::Normal) + let action = if is_dqn { + match ExposureLevel::from_index(action_idx) { + Ok(exposure) => OrderRouter::route_default(exposure), + Err(e) => { + warn!(" [{}] exposure_from_index({}) error at bar {}: {}", model_name, action_idx, bar_idx, e); + FactoredAction::new(ExposureLevel::Flat, ActionOrderType::Market, Urgency::Normal) + } + } + } else { + match FactoredAction::from_index(action_idx) { + Ok(fa) => fa, + Err(e) => { + warn!(" [{}] from_index({}) error at bar {}: {}", model_name, action_idx, bar_idx, e); + FactoredAction::new(ExposureLevel::Flat, ActionOrderType::Market, Urgency::Normal) + } } }; @@ -673,7 +684,7 @@ fn evaluate_dqn_fold( // Simulate trade and update portfolio state (B3: state synced per bar) simulate_chunk_trades( &action_indices, bar_idx, test_bars, args, - &mut portfolio, &mut returns, &mut action_counts, "DQN", + &mut portfolio, &mut returns, &mut action_counts, "DQN", true, ); } @@ -716,11 +727,11 @@ fn evaluate_ppo_fold( ); } - // Create PPO config matching training + // Create PPO config matching training — PPO always uses 45 factored actions #[allow(clippy::integer_division)] let config = PPOConfig { state_dim: args.feature_dim, - num_actions: args.num_actions, + num_actions: 45, policy_hidden_dims: { let base = hp_usize(hp, "hidden_dim_base").unwrap_or(256); let align = |x: usize| x.div_ceil(8) * 8; // tensor-core alignment (must match PpoTrainer) @@ -823,7 +834,7 @@ fn evaluate_ppo_fold( // 3. Sequential trade simulation on CPU — updates portfolio state simulate_chunk_trades( &action_indices, chunk_start, test_bars, args, - &mut portfolio, &mut returns, &mut action_counts, "PPO", + &mut portfolio, &mut returns, &mut action_counts, "PPO", false, ); } diff --git a/crates/ml/src/hyperopt/adapters/dqn.rs b/crates/ml/src/hyperopt/adapters/dqn.rs index c67b69ecd..a892779a5 100644 --- a/crates/ml/src/hyperopt/adapters/dqn.rs +++ b/crates/ml/src/hyperopt/adapters/dqn.rs @@ -3,7 +3,7 @@ //! This module provides a production-ready adapter for optimizing DQN //! hyperparameters using the generic optimization framework. It implements: //! -//! - 30D continuous parameter space with log-scale handling +//! - 29D continuous parameter space with log-scale handling //! - Training wrapper that integrates with existing DQN pipeline //! - Backtest-based objective using Sharpe ratio (production metric) //! - Prioritized Experience Replay (PER) tuning for Rainbow DQN performance (+25-40% convergence speed) @@ -153,7 +153,7 @@ const TRIAL_VRAM_MB: f64 = 7000.0; /// Corrected: was 0.02 (20KB) — 100x too high const MB_PER_SAMPLE: f64 = 0.0005; -/// DQN hyperparameter space (31D continuous - WAVE 11 with Rainbow booleans hardcoded to TRUE) +/// DQN hyperparameter space (29D continuous - C2c: count_bonus_coefficient removed from search) /// /// Defines the hyperparameters to optimize for DQN training: /// **Base Parameters (11D from Wave 1-2)**: @@ -462,7 +462,7 @@ impl Default for DQNParams { num_quantiles: 32, qr_kappa: 1.0, hidden_dim_base: 256, // Conservative default (backward compatible) - noisy_epsilon_floor: 0.05, // Minimum random exploration with noisy nets + noisy_epsilon_floor: 0.0, // C2: Noisy nets provide exploration; no epsilon floor during training count_bonus_coefficient: 0.1, // UCB exploration bonus cql_alpha: 0.1, // CQL regularization: mild penalty on OOD Q-values eval_softmax_temp: 0.1, // Default: nearly greedy (hyperopt will tune) @@ -473,7 +473,7 @@ impl Default for DQNParams { impl ParameterSpace for DQNParams { fn continuous_bounds() -> Vec<(f64, f64)> { - // 30D search space (C3: hold_penalty_weight removed — hold reward is neutral 0.0) + // 29D search space (C3: hold_penalty_weight removed, C2c: count_bonus_coefficient removed) // 16 exotic parameters fixed to validated defaults in from_continuous() // This makes PSO (20 particles) and TPE dramatically more effective vec![ @@ -515,7 +515,7 @@ impl ParameterSpace for DQNParams { (256.0, 4096.0), // 22: hidden_dim_base (linear, step=256) // Exploration anti-collapse (1D) - (0.02, 0.10), // 23: noisy_epsilon_floor (linear) + (0.0, 0.05), // 23: noisy_epsilon_floor (linear, 0.0=noisy-nets-only) // CQL regularization (1D) (0.0, 0.5), // 24: cql_alpha (linear) @@ -527,21 +527,20 @@ impl ParameterSpace for DQNParams { (0.0, 2.0), // 26: lr_decay_type (discrete: 0=constant, 1=linear, 2=cosine) (2.0, 6.0), // 27: min_epochs_before_stopping (discrete) (1.1, 2.0), // 28: minimum_profit_factor (linear) - - // Count-based exploration bonus (1D — kept for diversity metrics) - (0.05, 1.0), // 29: count_bonus_coefficient (linear) + // C2c: count_bonus_coefficient removed — count bonus no longer modifies Q-values ] } fn from_continuous(x: &[f64]) -> Result { - if x.len() != 30 { + if x.len() != 29 { return Err(MLError::ConfigError { - reason: format!("Expected 30 continuous parameters, got {}", x.len()), + reason: format!("Expected 29 continuous parameters, got {}", x.len()), }); } - // === 30 TUNED parameters (from search space) === + // === 29 TUNED parameters (from search space) === // C3: hold_penalty_weight removed — hold reward is neutral 0.0 + // C2c: count_bonus_coefficient removed — count bonus no longer modifies Q-values let learning_rate = x[0].exp(); let mut batch_size = x[1].round().max(64.0) as usize; let buffer_size = x[3].exp().round().max(50_000.0) as usize; @@ -580,7 +579,7 @@ impl ParameterSpace for DQNParams { let hidden_dim_base = ((x[22].round() / 256.0).round() * 256.0).clamp(256.0, 4096.0) as usize; // Exploration anti-collapse - let noisy_epsilon_floor = x[23].clamp(0.02, 0.10); + let noisy_epsilon_floor = x[23].clamp(0.0, 0.05); // CQL regularization let cql_alpha = x[24].clamp(0.0, 0.5); @@ -593,9 +592,6 @@ impl ParameterSpace for DQNParams { let min_epochs_before_stopping = x[27].round().clamp(2.0, 6.0) as usize; let minimum_profit_factor = x[28].clamp(1.1, 2.0); - // Count-based exploration bonus - let count_bonus_coefficient = x[29].clamp(0.05, 1.0); - // === 15 FIXED parameters (validated defaults, removed from search) === let kelly_min_trades: usize = 20; let ensemble_size = 5.0; @@ -678,7 +674,7 @@ impl ParameterSpace for DQNParams { qr_kappa, hidden_dim_base, noisy_epsilon_floor, - count_bonus_coefficient, + count_bonus_coefficient: 0.1, // C2c: fixed default (not in search space) cql_alpha, eval_softmax_temp, min_epochs_before_stopping, @@ -688,7 +684,7 @@ impl ParameterSpace for DQNParams { } fn to_continuous(&self) -> Vec { - // 30D search space — C3: hold_penalty_weight removed + // 29D search space — C3: hold_penalty_weight removed, C2c: count_bonus_coefficient removed vec![ self.learning_rate.ln(), // 0 self.batch_size as f64, // 1 @@ -719,12 +715,11 @@ impl ParameterSpace for DQNParams { self.lr_decay_type, // 26 self.min_epochs_before_stopping as f64, // 27 self.minimum_profit_factor, // 28 - self.count_bonus_coefficient, // 29 ] } fn param_names() -> Vec<&'static str> { - // 30 tuned parameters — C3: hold_penalty_weight removed + // 29 tuned parameters — C3: hold_penalty_weight removed, C2c: count_bonus_coefficient removed vec![ "learning_rate", // 0 "batch_size", // 1 @@ -755,7 +750,6 @@ impl ParameterSpace for DQNParams { "lr_decay_type", // 26 "min_epochs_before_stopping", // 27 "minimum_profit_factor", // 28 - "count_bonus_coefficient", // 29 ] } @@ -767,7 +761,7 @@ impl ParameterSpace for DQNParams { batch_bound.1 = max_batch; } } - // Cap hidden_dim_base by VRAM (index 22 in 30D space) + // Cap hidden_dim_base by VRAM (index 22 in 29D space) let max_base = budget.max_hidden_dim_base(4, 256, 54, 45); if let Some(dim_bound) = bounds.get_mut(22) { dim_bound.1 = max_base as f64; @@ -2533,8 +2527,8 @@ impl HyperparameterOptimizable for DQNTrainer { use_cql: true, cql_alpha: params.cql_alpha, - // BUG #7: minimum_profit_factor — now tunable via hyperopt (idx 30, [1.1, 2.0]) - // Previously hardcoded to 1.5, meaning dimension 30 of the search space was wasted. + // BUG #7: minimum_profit_factor — now tunable via hyperopt (idx 28, [1.1, 2.0]) + // Previously hardcoded to 1.5, meaning this dimension of the search space was wasted. minimum_profit_factor: params.minimum_profit_factor, // Phase 3: GPU experience collection (defaults, not in search space) @@ -2937,7 +2931,8 @@ impl HyperparameterOptimizable for DQNTrainer { let close = win_prices[bar_idx] as f32; unique_actions.insert(action_idx); - let factored = crate::dqn::FactoredAction::from_index(action_idx)?; + let exposure = crate::dqn::action_space::ExposureLevel::from_index(action_idx)?; + let factored = crate::dqn::OrderRouter::route_default(exposure); let bar = OHLCVBarF32 { timestamp: (abs_start + i) as i64, @@ -2982,7 +2977,7 @@ impl HyperparameterOptimizable for DQNTrainer { MaxDD {:.2}%, Return {:.2}%, unique_actions={}/{}, conv_fail={}", win_idx + 1, WINDOW_COUNT, ohlcv_bars.len(), wm.total_trades, wm.sharpe_ratio, wm.win_rate, wm.max_drawdown_pct, - wm.total_return_pct, unique_actions.len(), 45, conversion_failures, + wm.total_return_pct, unique_actions.len(), 5, conversion_failures, ); all_unique_actions.extend(&unique_actions); @@ -3473,8 +3468,8 @@ mod tests { #[test] fn test_dqn_params_roundtrip() { - // Roundtrip test for the 30D search space - // Only the 30 tuned parameters roundtrip; the 17 fixed params get defaults from from_continuous + // Roundtrip test for the 29D search space + // Only the 29 tuned parameters roundtrip; the 17 fixed params get defaults from from_continuous let params = DQNParams { learning_rate: 3.37e-05, batch_size: 92, @@ -3535,10 +3530,10 @@ mod tests { }; let continuous = params.to_continuous(); - assert_eq!(continuous.len(), 30, "to_continuous must return 30D vector"); + assert_eq!(continuous.len(), 29, "to_continuous must return 29D vector"); let recovered = DQNParams::from_continuous(&continuous).unwrap(); - // Tuned parameters must roundtrip exactly (30D — C3: hold_penalty_weight removed) + // Tuned parameters must roundtrip exactly (29D — C3: hold_penalty_weight removed, C2c: count_bonus_coefficient removed) assert!((recovered.learning_rate - params.learning_rate).abs() < 1e-6); assert_eq!(recovered.batch_size, params.batch_size); assert!((recovered.gamma - params.gamma).abs() < 1e-6); @@ -3572,13 +3567,13 @@ mod tests { assert!((recovered.activation_type - 1.0).abs() < 1e-6); // LeakyReLU assert_eq!(recovered.num_quantiles, 64); assert!((recovered.qr_kappa - 1.0).abs() < 1e-6); - assert!((recovered.count_bonus_coefficient - params.count_bonus_coefficient).abs() < 1e-6); + assert!((recovered.count_bonus_coefficient - 0.1).abs() < 1e-6); // C2c: fixed default } #[test] fn test_dqn_params_bounds() { let bounds = DQNParams::continuous_bounds(); - assert_eq!(bounds.len(), 30); // 30D search space (C3: hold_penalty_weight removed) + assert_eq!(bounds.len(), 29); // 29D search space (C3: hold_penalty_weight removed, C2c: count_bonus_coefficient removed) // Check log-scale bounds are reasonable assert!(bounds[0].0 < bounds[0].1); // learning_rate @@ -3617,7 +3612,7 @@ mod tests { // Curiosity + exploration assert_eq!(bounds[20], (0.01, 0.5)); // curiosity_weight assert_eq!(bounds[22], (256.0, 4096.0)); // hidden_dim_base - assert_eq!(bounds[23], (0.02, 0.10)); // noisy_epsilon_floor + assert_eq!(bounds[23], (0.0, 0.05)); // noisy_epsilon_floor // CQL regularization assert_eq!(bounds[24], (0.0, 0.5)); // cql_alpha @@ -3626,7 +3621,7 @@ mod tests { #[test] fn test_param_names() { let names = DQNParams::param_names(); - assert_eq!(names.len(), 30); // 30D search space (C3: hold_penalty_weight removed) + assert_eq!(names.len(), 29); // 29D search space (C3: hold_penalty_weight removed, C2c: count_bonus_coefficient removed) assert_eq!(names[0], "learning_rate"); assert_eq!(names[1], "batch_size"); assert_eq!(names[2], "gamma"); @@ -3656,13 +3651,12 @@ mod tests { assert_eq!(names[26], "lr_decay_type"); assert_eq!(names[27], "min_epochs_before_stopping"); assert_eq!(names[28], "minimum_profit_factor"); - assert_eq!(names[29], "count_bonus_coefficient"); } #[test] fn test_per_params_always_enabled() { // Test that PER is always enabled with tunable alpha/beta parameters - // 30D search space (C3: hold_penalty_weight removed) + // 29D search space (C3: hold_penalty_weight removed, C2c: count_bonus_coefficient removed) let continuous = vec![ 3e-5_f64.ln(), 92.0, 0.9588, 97_273_f64.ln(), 4.0, 24.77_f64.ln(), 0.15, 1.0, 0.6, 0.4, // 8-9: per_alpha, per_beta_start @@ -3680,7 +3674,6 @@ mod tests { 2.0, // 26: lr_decay_type (cosine) 4.0, // 27: min_epochs_before_stopping 1.5, // 28: minimum_profit_factor - 0.2, // 29: count_bonus_coefficient ]; let params = DQNParams::from_continuous(&continuous).unwrap(); @@ -3692,7 +3685,7 @@ mod tests { assert!(params.use_noisy_nets); assert!((params.warmup_ratio - 0.0).abs() < 1e-6); // Always fixed to 0.0 - // Test PER parameter bounds (min values) — 30D + // Test PER parameter bounds (min values) — 29D let continuous_min = vec![ 2e-5_f64.ln(), 64.0, 0.95, 50_000_f64.ln(), 1.0, 10.0_f64.ln(), 0.05, 0.5, 0.4, 0.2, // per_alpha min, per_beta_start min @@ -3704,13 +3697,12 @@ mod tests { 0.01, // curiosity_weight min 0.0001_f64.ln(), // tau min 256.0, // hidden_dim_base min - 0.02, // noisy_epsilon_floor min + 0.0, // noisy_epsilon_floor min 0.0, // cql_alpha min 0.5_f64.ln(), // eval_softmax_temp min 0.0, // 26: lr_decay_type (constant) 2.0, // 27: min_epochs_before_stopping min 1.1, // 28: minimum_profit_factor min - 0.05, // 29: count_bonus_coefficient min ]; let params_min = DQNParams::from_continuous(&continuous_min).unwrap(); assert!((params_min.per_alpha - 0.4).abs() < 1e-6); @@ -3719,7 +3711,7 @@ mod tests { assert!(params_min.use_distributional); assert!(params_min.use_noisy_nets); - // Test PER parameter bounds (max values) — 30D + // Test PER parameter bounds (max values) — 29D let continuous_max = vec![ 8e-5_f64.ln(), 4096.0, 0.99, 100_000_f64.ln(), 4.0, 40.0_f64.ln(), 0.5, 2.0, 0.8, 0.6, // per_alpha max, per_beta_start max @@ -3731,13 +3723,12 @@ mod tests { 0.5, // curiosity_weight max 0.01_f64.ln(), // tau max 4096.0, // hidden_dim_base max - 0.10, // noisy_epsilon_floor max + 0.05, // noisy_epsilon_floor max 0.5, // cql_alpha max 2.0_f64.ln(), // eval_softmax_temp max 2.0, // 26: lr_decay_type (cosine) 6.0, // 27: min_epochs_before_stopping max 2.0, // 28: minimum_profit_factor max - 1.0, // 29: count_bonus_coefficient max ]; let params_max = DQNParams::from_continuous(&continuous_max).unwrap(); assert!((params_max.per_alpha - 0.8).abs() < 1e-6); @@ -4037,7 +4028,7 @@ mod tests { fn test_qr_dqn_roundtrip_continuous() { let params = DQNParams::default(); let continuous = params.to_continuous(); - assert_eq!(continuous.len(), 30, "Should have 30 continuous dimensions"); + assert_eq!(continuous.len(), 29, "Should have 29 continuous dimensions"); let roundtrip = DQNParams::from_continuous(&continuous).unwrap(); // num_quantiles and qr_kappa are now fixed defaults (not in search space) assert_eq!(roundtrip.num_quantiles, 64); // Fixed default @@ -4062,7 +4053,7 @@ mod tests { fn test_batch_size_respects_wide_bounds() { // Simulate PSO choosing batch_size=2048 (within VRAM-aware bounds) let bounds = DQNParams::continuous_bounds(); - let mut params = vec![0.0_f64; 30]; // 30D (C3: hold_penalty_weight removed) + let mut params = vec![0.0_f64; 29]; // 29D (C3: hold_penalty_weight removed, C2c: count_bonus_coefficient removed) params[1] = 2048.0; // batch_size (index 1) // Fill other required params with valid defaults params[0] = (1e-4_f64).ln(); // learning_rate @@ -4074,8 +4065,8 @@ mod tests { params[7] = 0.5; // transaction_cost_multiplier params[8] = 0.6; // per_alpha params[9] = 0.4; // per_beta_start - // Remaining params (indices 10-29) -- use midpoint of bounds - for i in 10..30 { + // Remaining params (indices 10-28) -- use midpoint of bounds + for i in 10..29 { params[i] = (bounds[i].0 + bounds[i].1) / 2.0; } let result = DQNParams::from_continuous(¶ms).unwrap(); @@ -4199,9 +4190,9 @@ mod tests { ); // from_continuous should clamp to 0.5 even if value is below - let mut low_temp_vec = vec![0.0_f64; 30]; // 30D (C3: hold_penalty_weight removed) + let mut low_temp_vec = vec![0.0_f64; 29]; // 29D (C3: hold_penalty_weight removed, C2c: count_bonus_coefficient removed) // Set all to midpoint of bounds - for i in 0..30 { + for i in 0..29 { low_temp_vec[i] = (bounds[i].0 + bounds[i].1) / 2.0; } low_temp_vec[25] = 0.01_f64.ln(); // Way below floor (eval_softmax_temp at index 25) @@ -4261,4 +4252,22 @@ mod tests { objective ); } + + #[test] + fn test_dqn_exposure_indices_produce_distinct_actions() { + use crate::dqn::action_space::ExposureLevel; + use crate::dqn::order_router::OrderRouter; + + // DQN outputs 0-4. Each must map to a distinct exposure level. + let actions: Vec<_> = (0..5) + .map(|idx| { + let exposure = ExposureLevel::from_index(idx).expect("valid index 0-4"); + OrderRouter::route_default(exposure) + }) + .collect(); + + // All 5 exposures must be distinct + let exposures: std::collections::HashSet<_> = actions.iter().map(|a| a.exposure).collect(); + assert_eq!(exposures.len(), 5, "DQN 5-action space must produce 5 distinct exposure levels"); + } } diff --git a/crates/ml/src/trainers/dqn/trainer.rs b/crates/ml/src/trainers/dqn/trainer.rs index 1bcde3141..004f78928 100644 --- a/crates/ml/src/trainers/dqn/trainer.rs +++ b/crates/ml/src/trainers/dqn/trainer.rs @@ -3289,22 +3289,9 @@ impl DQNTrainer { .forward(&batch_tensor) .map_err(|e| anyhow::anyhow!("Batched forward pass failed: {}", e))?; - // Apply count bonus (UCB exploration) to Q-values before argmax. - // Previously only in DQN::select_action() (single-sample path), - // making the batch training path have no UCB exploration. - let use_cb = agent.use_count_bonus(); - let n_actions = agent.num_actions(); - let batch_q_values = if use_cb { - let bonuses = agent.get_count_bonuses(); - let bonus_tensor = Tensor::from_vec(bonuses, (1, n_actions), batch_q_values.device()) - .map_err(|e| anyhow::anyhow!("Failed to create bonus tensor: {}", e))? - .to_dtype(batch_q_values.dtype()) - .map_err(|e| anyhow::anyhow!("Failed to cast bonus tensor: {}", e))?; - batch_q_values.broadcast_add(&bonus_tensor) - .map_err(|e| anyhow::anyhow!("Failed to add count bonus: {}", e))? - } else { - batch_q_values - }; + // C2 FIX: Count bonus removed from Q-value computation. + // Noisy nets are the sole exploration mechanism during training. + // Count bonus kept for diversity metrics only (record_action tracking). drop(agent); // Release lock early @@ -3364,21 +3351,9 @@ impl DQNTrainer { .forward(batch_tensor) .map_err(|e| anyhow::anyhow!("GPU batched forward pass failed: {}", e))?; - // Apply count bonus (UCB exploration) to Q-values before argmax. - // Mirrors the same fix in select_actions_batch() above. - let use_cb = agent.use_count_bonus(); - let n_actions = agent.num_actions(); - let batch_q_values = if use_cb { - let bonuses = agent.get_count_bonuses(); - let bonus_tensor = Tensor::from_vec(bonuses, (1, n_actions), batch_q_values.device()) - .map_err(|e| anyhow::anyhow!("Failed to create bonus tensor: {}", e))? - .to_dtype(batch_q_values.dtype()) - .map_err(|e| anyhow::anyhow!("Failed to cast bonus tensor: {}", e))?; - batch_q_values.broadcast_add(&bonus_tensor) - .map_err(|e| anyhow::anyhow!("Failed to add count bonus: {}", e))? - } else { - batch_q_values - }; + // C2 FIX: Count bonus removed from Q-value computation. + // Noisy nets are the sole exploration mechanism during training. + // Count bonus kept for diversity metrics only (record_action tracking). drop(agent); @@ -4151,6 +4126,7 @@ fn gpu_batch_to_experiences( #[cfg(test)] mod tests { use super::*; + use crate::hyperopt::ParameterSpace; // Helper function to create test hyperparameters // Uses conservative defaults suitable for testing @@ -4624,4 +4600,79 @@ mod tests { let batch = budget.max_batch_size(50.0, 0.0005, 64.0, 8192.0); assert!((batch.unwrap_or(0.0) - 8192.0).abs() < 1.0, "H100 should hit 8192 ceiling, got {:?}", batch); } + + // ── C2 Overhaul Smoke Tests ───────────────────────────────────────── + + /// Verify DQN action space is 5 exposure levels (not 45 factored actions). + #[test] + fn test_c2_dqn_default_num_actions_is_5() { + let config = crate::dqn::DQNConfig::default(); + assert_eq!(config.num_actions, 5, "DQN default must be 5 exposure-level actions"); + } + + /// Verify 5 exposure indices produce 5 distinct exposure levels. + #[test] + fn test_c2_five_actions_produce_distinct_exposures() { + use crate::dqn::action_space::ExposureLevel; + use crate::dqn::order_router::OrderRouter; + + let actions: Vec<_> = (0..5) + .filter_map(|idx| ExposureLevel::from_index(idx).ok()) + .map(|e| OrderRouter::route_default(e)) + .collect(); + + assert_eq!(actions.len(), 5); + + let unique: std::collections::HashSet<_> = actions.iter().map(|a| a.exposure).collect(); + assert_eq!(unique.len(), 5, "All 5 exposure levels must be distinct"); + } + + /// Verify hyperopt search space is 29D (count_bonus_coefficient removed). + #[test] + fn test_c2_search_space_is_29d() { + let bounds = crate::hyperopt::adapters::dqn::DQNParams::continuous_bounds(); + assert_eq!(bounds.len(), 29, "Search space must be 29D after C2c removal"); + + let names = crate::hyperopt::adapters::dqn::DQNParams::param_names(); + assert_eq!(names.len(), 29); + assert!(!names.contains(&"count_bonus_coefficient"), "count_bonus_coefficient must not be in search space"); + } + + /// Verify noisy_epsilon_floor defaults to 0.0 (noisy nets only). + #[test] + fn test_c2_noisy_epsilon_floor_defaults_to_zero() { + let params = crate::hyperopt::adapters::dqn::DQNParams::default(); + assert!( + (params.noisy_epsilon_floor - 0.0).abs() < f64::EPSILON, + "noisy_epsilon_floor must default to 0.0 (noisy nets only)" + ); + } + + /// Verify noisy_epsilon_floor search bounds are [0.0, 0.05]. + #[test] + fn test_c2_noisy_epsilon_floor_bounds() { + let bounds = crate::hyperopt::adapters::dqn::DQNParams::continuous_bounds(); + // Index 23 = noisy_epsilon_floor + let (lo, hi) = bounds[23]; + assert!((lo - 0.0).abs() < f64::EPSILON, "noisy_epsilon_floor lower bound must be 0.0"); + assert!((hi - 0.05).abs() < f64::EPSILON, "noisy_epsilon_floor upper bound must be 0.05"); + } + + /// Verify count_bonus_coefficient is fixed at 0.1 (not tunable). + #[test] + fn test_c2_count_bonus_coefficient_fixed() { + let params = crate::hyperopt::adapters::dqn::DQNParams::default(); + assert!( + (params.count_bonus_coefficient - 0.1).abs() < f64::EPSILON, + "count_bonus_coefficient must be fixed at 0.1" + ); + + // Roundtrip through from_continuous should preserve fixed value + let continuous = params.to_continuous(); + let recovered = crate::hyperopt::adapters::dqn::DQNParams::from_continuous(&continuous).unwrap(); + assert!( + (recovered.count_bonus_coefficient - 0.1).abs() < f64::EPSILON, + "count_bonus_coefficient must remain 0.1 after roundtrip" + ); + } }