diff --git a/crates/ml/src/hyperopt/adapters/dqn.rs b/crates/ml/src/hyperopt/adapters/dqn.rs index a9e336ee8..b69998809 100644 --- a/crates/ml/src/hyperopt/adapters/dqn.rs +++ b/crates/ml/src/hyperopt/adapters/dqn.rs @@ -310,10 +310,9 @@ pub struct DQNParams { pub beta_entropy: f64, // Note: variance_cap is NOT in search space, it's fixed in DQNHyperparameters - // WAVE 26 P1.5: Learning rate warmup ratio (0.0-0.2 = 0-20% of training) - /// Learning rate warmup ratio (0.0-0.2) - /// Fraction of training steps to use for linear LR warmup from 0 to target LR - /// 0.0 = no warmup, 0.1 = 10% warmup, 0.2 = 20% warmup + // NOTE: warmup_ratio removed from search space — warmup_steps forced to 0 because + // the batch training path (select_actions_batch) never increments DQN::total_steps. + // Kept in struct for serde backward compat with saved JSON results. pub warmup_ratio: f64, // WAVE 26 P1.8: Curiosity-Driven Exploration @@ -474,11 +473,11 @@ impl Default for DQNParams { impl ParameterSpace for DQNParams { fn continuous_bounds() -> Vec<(f64, f64)> { - // 31D search space (expanded from 27D) + // 30D search space (reduced from 31D: warmup_ratio removed — no effect in batch training) // 16 exotic parameters fixed to validated defaults in from_continuous() // This makes PSO (20 particles) and TPE dramatically more effective // - // Kept: the 31 params that genuinely affect trading performance + // Kept: the 30 params that genuinely affect trading performance // Fixed: ensemble/architecture params that rarely deviate from defaults vec![ // Base parameters (11D) @@ -529,35 +528,32 @@ impl ParameterSpace for DQNParams { // (raised from 0.1: temps below 0.3 produce degenerate trials with 1-3/45 actions) (0.5_f64.ln(), 2.0_f64.ln()), // 26: eval_softmax_temp (log scale) - // === NEW: Training dynamics parameters (4D) === - // These were hardcoded but critically affect convergence and OOS generalization. - - // LR warmup ratio (fraction of training spent ramping LR from 0 to target) - // Prevents early Q-value collapse when starting from random weights - (0.0, 0.15), // 27: warmup_ratio (linear, 0.0=no warmup, 0.15=15% warmup) + // === Training dynamics parameters (3D, indices 27-29) === + // warmup_ratio REMOVED: batch training path never increments total_steps, + // so warmup_steps > 0 causes train_step() to return (0.0, 0.0) forever. // Learning rate schedule (0=constant, 1=linear decay, 2=cosine annealing) // Cosine annealing prevents Q-value inflation in late epochs - (0.0, 2.0), // 28: lr_decay_type (discrete: round to nearest int) + (0.0, 2.0), // 27: lr_decay_type (discrete: round to nearest int) // Minimum epochs before early stopping fires (prevents premature termination) // For 8-epoch hyperopt trials, range [2,6] lets optimizer find the sweet spot - (2.0, 6.0), // 29: min_epochs_before_stopping (discrete: round to int) + (2.0, 6.0), // 28: min_epochs_before_stopping (discrete: round to int) // Minimum profit factor for trade execution (BUG #7: margin above breakeven) // 1.1 = 10% above costs, 2.0 = 100% above costs. Filters marginal trades. - (1.1, 2.0), // 30: minimum_profit_factor (linear) + (1.1, 2.0), // 29: minimum_profit_factor (linear) ] } fn from_continuous(x: &[f64]) -> Result { - if x.len() != 31 { + if x.len() != 30 { return Err(MLError::ConfigError { - reason: format!("Expected 31 continuous parameters, got {}", x.len()), + reason: format!("Expected 30 continuous parameters, got {}", x.len()), }); } - // === 31 TUNED parameters (from search space) === + // === 30 TUNED parameters (from search space) === let learning_rate = x[0].exp(); let mut batch_size = x[1].round().max(64.0) as usize; // Only enforce floor. PSO + HardwareBudget control the upper bound. let buffer_size = x[3].exp().round().max(50_000.0) as usize; @@ -605,11 +601,11 @@ impl ParameterSpace for DQNParams { // Eval softmax temperature (floor 0.5 — below this, action diversity collapses) let eval_softmax_temp = x[26].exp().clamp(0.5, 2.0); - // === NEW: Training dynamics parameters (4D, indices 27-30) === - let warmup_ratio = x[27].clamp(0.0, 0.15); - let lr_decay_type = x[28].round().clamp(0.0, 2.0); // 0=constant, 1=linear, 2=cosine - let min_epochs_before_stopping = x[29].round().clamp(2.0, 6.0) as usize; - let minimum_profit_factor = x[30].clamp(1.1, 2.0); + // === Training dynamics parameters (3D, indices 27-29) === + // warmup_ratio removed: batch training never increments total_steps → always stuck in warmup + let lr_decay_type = x[27].round().clamp(0.0, 2.0); // 0=constant, 1=linear, 2=cosine + let min_epochs_before_stopping = x[28].round().clamp(2.0, 6.0) as usize; + let minimum_profit_factor = x[29].clamp(1.1, 2.0); // === 16 FIXED parameters (validated defaults, removed from search) === let kelly_min_trades: usize = 20; @@ -683,7 +679,7 @@ impl ParameterSpace for DQNParams { beta_variance, beta_disagreement, beta_entropy, - warmup_ratio, + warmup_ratio: 0.0, // Fixed: batch training never increments total_steps curiosity_weight, tau, td_error_clamp_max, @@ -713,8 +709,9 @@ impl ParameterSpace for DQNParams { } fn to_continuous(&self) -> Vec { - // 31D search space — only the tuned parameters + // 30D search space — only the tuned parameters // Fixed parameters are NOT emitted (they get their defaults in from_continuous) + // warmup_ratio excluded: forced to 0.0 (batch training never increments total_steps) vec![ self.learning_rate.ln(), // 0 self.batch_size as f64, // 1 @@ -743,16 +740,16 @@ impl ParameterSpace for DQNParams { self.noisy_epsilon_floor, // 24 self.cql_alpha, // 25 self.eval_softmax_temp.ln(), // 26 - // Training dynamics (4D) - self.warmup_ratio, // 27 - self.lr_decay_type, // 28 - self.min_epochs_before_stopping as f64, // 29 - self.minimum_profit_factor, // 30 + // Training dynamics (3D) + self.lr_decay_type, // 27 + self.min_epochs_before_stopping as f64, // 28 + self.minimum_profit_factor, // 29 ] } fn param_names() -> Vec<&'static str> { - // 31 tuned parameters (matches continuous_bounds / from_continuous / to_continuous) + // 30 tuned parameters (matches continuous_bounds / from_continuous / to_continuous) + // warmup_ratio removed: forced to 0.0 (batch training never increments total_steps) vec![ "learning_rate", // 0 "batch_size", // 1 @@ -781,10 +778,9 @@ impl ParameterSpace for DQNParams { "noisy_epsilon_floor", // 24 "cql_alpha", // 25 "eval_softmax_temp", // 26 - "warmup_ratio", // 27 - "lr_decay_type", // 28 - "min_epochs_before_stopping", // 29 - "minimum_profit_factor", // 30 + "lr_decay_type", // 27 + "min_epochs_before_stopping", // 28 + "minimum_profit_factor", // 29 ] } @@ -796,7 +792,7 @@ impl ParameterSpace for DQNParams { batch_bound.1 = max_batch; } } - // Cap hidden_dim_base by VRAM (index 23 in 31D space) + // Cap hidden_dim_base by VRAM (index 23 in 30D space) let max_base = budget.max_hidden_dim_base(4, 256, 54, 45); if let Some(dim_bound) = bounds.get_mut(23) { dim_bound.1 = max_base as f64; @@ -2270,7 +2266,6 @@ impl HyperparameterOptimizable for DQNTrainer { info!(" N-steps: {}", params.n_steps); info!(" Num atoms: {}", params.num_atoms); info!(" Eval softmax temp: {:.3}", params.eval_softmax_temp); - info!(" Warmup ratio: {:.3}", params.warmup_ratio); info!(" LR decay type: {} (0=const, 1=linear, 2=cosine)", params.lr_decay_type.round() as i32); info!(" Min epochs before stopping: {}", params.min_epochs_before_stopping); info!(" Minimum profit factor: {:.2}", params.minimum_profit_factor); @@ -2406,9 +2401,11 @@ impl HyperparameterOptimizable for DQNTrainer { // Note: tau is set later at line 2086 (WAVE 26 P1.12) target_update_mode: self.target_update_mode.clone(), target_update_frequency: self.target_update_frequency, - // Warmup steps computed from warmup_ratio × epochs × ~2000 steps/epoch - // For 8-epoch hyperopt trials: ratio 0.1 → ~1600 steps warmup - warmup_steps: (params.warmup_ratio * (self.epochs as f64) * 2000.0) as usize, + // CRITICAL FIX: warmup_steps MUST be 0 for hyperopt. + // The batch training path (select_actions_batch → forward()) does NOT + // increment DQN::total_steps. Only select_action() does. So if warmup_steps > 0, + // train_step() returns (0.0, 0.0) forever — zero loss, zero gradients, no learning. + warmup_steps: 0, // P2-A Enhancement #[allow(clippy::cast_possible_truncation)] @@ -3432,8 +3429,8 @@ mod tests { #[test] fn test_dqn_params_roundtrip() { - // Roundtrip test for the 31D search space - // Only the 31 tuned parameters roundtrip; the 16 fixed params get defaults from from_continuous + // Roundtrip test for the 30D search space + // Only the 30 tuned parameters roundtrip; the 17 fixed params get defaults from from_continuous let params = DQNParams { learning_rate: 3.37e-05, batch_size: 92, @@ -3459,8 +3456,8 @@ mod tests { noisy_sigma_init: 0.5, // Now tuned (moved from fixed): minimum_profit_factor: 1.5, - warmup_ratio: 0.05, // 5% warmup - lr_decay_type: 2.0, // Cosine annealing + warmup_ratio: 0.0, // Fixed to 0.0 (not in search space) + lr_decay_type: 2.0, // Cosine annealing min_epochs_before_stopping: 4, // Still fixed — these will be overwritten by from_continuous defaults weight_decay: 1e-4, @@ -3494,10 +3491,10 @@ mod tests { }; let continuous = params.to_continuous(); - assert_eq!(continuous.len(), 31, "to_continuous must return 31D vector"); + assert_eq!(continuous.len(), 30, "to_continuous must return 30D vector"); let recovered = DQNParams::from_continuous(&continuous).unwrap(); - // Tuned parameters must roundtrip exactly (31D) + // Tuned parameters must roundtrip exactly (30D) assert!((recovered.learning_rate - params.learning_rate).abs() < 1e-6); assert_eq!(recovered.batch_size, params.batch_size); assert!((recovered.gamma - params.gamma).abs() < 1e-6); @@ -3516,13 +3513,13 @@ mod tests { assert!((recovered.noisy_epsilon_floor - params.noisy_epsilon_floor).abs() < 1e-6); assert!((recovered.cql_alpha - params.cql_alpha).abs() < 1e-6); assert!((recovered.eval_softmax_temp - params.eval_softmax_temp).abs() < 1e-3); - // New tuned params: - assert!((recovered.warmup_ratio - params.warmup_ratio).abs() < 1e-6); + // warmup_ratio is fixed to 0.0, not in search space + assert!((recovered.warmup_ratio - 0.0).abs() < 1e-6); assert!((recovered.lr_decay_type - params.lr_decay_type).abs() < 1e-6); assert_eq!(recovered.min_epochs_before_stopping, params.min_epochs_before_stopping); assert!((recovered.minimum_profit_factor - params.minimum_profit_factor).abs() < 1e-6); - // Fixed parameters must get their validated defaults (16 remaining) + // Fixed parameters must get their validated defaults (17 remaining, including warmup_ratio) assert_eq!(recovered.kelly_min_trades, 20); assert!((recovered.ensemble_size - 5.0).abs() < 1e-6); assert!((recovered.td_error_clamp_max - 10.0).abs() < 1e-6); @@ -3537,7 +3534,7 @@ mod tests { #[test] fn test_dqn_params_bounds() { let bounds = DQNParams::continuous_bounds(); - assert_eq!(bounds.len(), 31); // 31D search space + assert_eq!(bounds.len(), 30); // 30D search space // Check log-scale bounds are reasonable assert!(bounds[0].0 < bounds[0].1); // learning_rate @@ -3586,7 +3583,7 @@ mod tests { #[test] fn test_param_names() { let names = DQNParams::param_names(); - assert_eq!(names.len(), 31); // 31D search space + assert_eq!(names.len(), 30); // 30D search space assert_eq!(names[0], "learning_rate"); assert_eq!(names[1], "batch_size"); assert_eq!(names[2], "gamma"); @@ -3613,12 +3610,16 @@ mod tests { assert_eq!(names[23], "hidden_dim_base"); assert_eq!(names[24], "noisy_epsilon_floor"); assert_eq!(names[25], "cql_alpha"); + assert_eq!(names[26], "eval_softmax_temp"); + assert_eq!(names[27], "lr_decay_type"); + assert_eq!(names[28], "min_epochs_before_stopping"); + assert_eq!(names[29], "minimum_profit_factor"); } #[test] fn test_per_params_always_enabled() { // Test that PER is always enabled with tunable alpha/beta parameters - // 31D search space + // 30D search space (warmup_ratio removed) let continuous = vec![ 3e-5_f64.ln(), 92.0, 0.9588, 97_273_f64.ln(), 1.404, 4.0, 24.77_f64.ln(), 0.05, 1.0, 0.6, 0.4, // 9-10: per_alpha, per_beta_start @@ -3633,10 +3634,9 @@ mod tests { 0.05, // 24: noisy_epsilon_floor 0.1, // 25: cql_alpha 0.8_f64.ln(), // 26: eval_softmax_temp (>= 0.5 floor) - 0.05, // 27: warmup_ratio - 2.0, // 28: lr_decay_type (cosine) - 4.0, // 29: min_epochs_before_stopping - 1.5, // 30: minimum_profit_factor + 2.0, // 27: lr_decay_type (cosine) + 4.0, // 28: min_epochs_before_stopping + 1.5, // 29: minimum_profit_factor ]; let params = DQNParams::from_continuous(&continuous).unwrap(); @@ -3646,8 +3646,9 @@ mod tests { assert!(params.use_dueling); assert!(params.use_distributional); assert!(params.use_noisy_nets); + assert!((params.warmup_ratio - 0.0).abs() < 1e-6); // Always fixed to 0.0 - // Test PER parameter bounds (min values) — 31D + // Test PER parameter bounds (min values) — 30D let continuous_min = vec![ 2e-5_f64.ln(), 64.0, 0.95, 50_000_f64.ln(), 1.0, 1.0, 10.0_f64.ln(), 0.01, 0.5, 0.4, 0.2, // per_alpha min, per_beta_start min @@ -3662,10 +3663,9 @@ mod tests { 0.02, // noisy_epsilon_floor min 0.0, // cql_alpha min 0.5_f64.ln(), // eval_softmax_temp min - 0.0, // 27: warmup_ratio min - 0.0, // 28: lr_decay_type (constant) - 2.0, // 29: min_epochs_before_stopping min - 1.1, // 30: minimum_profit_factor min + 0.0, // 27: lr_decay_type (constant) + 2.0, // 28: min_epochs_before_stopping min + 1.1, // 29: minimum_profit_factor min ]; let params_min = DQNParams::from_continuous(&continuous_min).unwrap(); assert!((params_min.per_alpha - 0.4).abs() < 1e-6); @@ -3674,7 +3674,7 @@ mod tests { assert!(params_min.use_distributional); assert!(params_min.use_noisy_nets); - // Test PER parameter bounds (max values) — 31D + // Test PER parameter bounds (max values) — 30D let continuous_max = vec![ 8e-5_f64.ln(), 4096.0, 0.99, 100_000_f64.ln(), 2.0, 4.0, 40.0_f64.ln(), 0.2, 2.0, 0.8, 0.6, // per_alpha max, per_beta_start max @@ -3689,10 +3689,9 @@ mod tests { 0.10, // noisy_epsilon_floor max 0.5, // cql_alpha max 2.0_f64.ln(), // eval_softmax_temp max - 0.15, // 27: warmup_ratio max - 2.0, // 28: lr_decay_type (cosine) - 6.0, // 29: min_epochs_before_stopping max - 2.0, // 30: minimum_profit_factor max + 2.0, // 27: lr_decay_type (cosine) + 6.0, // 28: min_epochs_before_stopping max + 2.0, // 29: minimum_profit_factor max ]; let params_max = DQNParams::from_continuous(&continuous_max).unwrap(); assert!((params_max.per_alpha - 0.8).abs() < 1e-6); @@ -3992,7 +3991,7 @@ mod tests { fn test_qr_dqn_roundtrip_continuous() { let params = DQNParams::default(); let continuous = params.to_continuous(); - assert_eq!(continuous.len(), 31, "Should have 31 continuous dimensions"); + assert_eq!(continuous.len(), 30, "Should have 30 continuous dimensions"); let roundtrip = DQNParams::from_continuous(&continuous).unwrap(); // num_quantiles and qr_kappa are now fixed defaults (not in search space) assert_eq!(roundtrip.num_quantiles, 64); // Fixed default @@ -4017,7 +4016,7 @@ mod tests { fn test_batch_size_respects_wide_bounds() { // Simulate PSO choosing batch_size=2048 (within VRAM-aware bounds) let bounds = DQNParams::continuous_bounds(); - let mut params = vec![0.0_f64; 31]; + let mut params = vec![0.0_f64; 30]; params[1] = 2048.0; // batch_size (index 1) // Fill other required params with valid defaults params[0] = (1e-4_f64).ln(); // learning_rate @@ -4030,8 +4029,8 @@ mod tests { params[8] = 0.5; // transaction_cost_multiplier params[9] = 0.6; // per_alpha params[10] = 0.4; // per_beta_start - // Remaining params (indices 11-30) -- use midpoint of bounds - for i in 11..31 { + // Remaining params (indices 11-29) -- use midpoint of bounds + for i in 11..30 { params[i] = (bounds[i].0 + bounds[i].1) / 2.0; } let result = DQNParams::from_continuous(¶ms).unwrap(); @@ -4155,9 +4154,9 @@ mod tests { ); // from_continuous should clamp to 0.5 even if value is below - let mut low_temp_vec = vec![0.0_f64; 31]; + let mut low_temp_vec = vec![0.0_f64; 30]; // Set all to midpoint of bounds - for i in 0..31 { + for i in 0..30 { low_temp_vec[i] = (bounds[i].0 + bounds[i].1) / 2.0; } low_temp_vec[26] = 0.01_f64.ln(); // Way below floor