diff --git a/crates/ml/src/hyperopt/adapters/dqn.rs b/crates/ml/src/hyperopt/adapters/dqn.rs index 5f305824f..d3ae74c70 100644 --- a/crates/ml/src/hyperopt/adapters/dqn.rs +++ b/crates/ml/src/hyperopt/adapters/dqn.rs @@ -160,14 +160,17 @@ const TRIAL_VRAM_MB: f64 = 7000.0; /// Corrected: was 0.02 (20KB) — 100x too high const MB_PER_SAMPLE: f64 = 0.0005; -/// DQN hyperparameter space (26D continuous -- exploration stacking removed) +/// DQN hyperparameter space (28D continuous) /// -/// C2 FIX: Reduced from 28D to 26D by removing exploration stacking: -/// - curiosity_weight: FIXED to 0.0 (C2: conflicts with NoisyNet, not in search space) -/// - noisy_epsilon_floor: FIXED to 0.0 (C2: NoisyNet handles exploration, not in search space) -/// - entropy_coefficient: NARROWED from (0.05, 0.5) to (0.005, 0.05) -/// - count_bonus_coefficient: FIXED to 0.0 (was 0.1, conflicts with NoisyNet) -/// NoisyNet (sigma in network weights) is the sole primary exploration mechanism. +/// Fixed params (not in search space): +/// - curiosity_weight: 0.0 (conflicts with NoisyNet) +/// - noisy_epsilon_floor: 0.0 (NoisyNet handles exploration) +/// +/// Tuned params restored in C1-C4: +/// - entropy_coefficient: (0.05, 0.5) — 10x wider for Q-values ~7.0 +/// - weight_decay: log(1e-4, 1e-2) — 10x stronger to prevent overfitting +/// - count_bonus_coefficient: (0.0, 0.3) — UCB exploration bonus, complementary to NoisyNet +/// - sharpe_weight: (0.0, 0.5) — tunable Sharpe ratio blending in composite reward /// /// ## Parameter Scaling /// @@ -453,7 +456,7 @@ impl Default for DQNParams { impl ParameterSpace for DQNParams { fn continuous_bounds() -> Vec<(f64, f64)> { - // 27D search space (C3: re-enabled count_bonus_coefficient for exploration) + // 28D search space (C4: added sharpe_weight) // Fixed: curiosity_weight=0.0, noisy_epsilon_floor=0.0 // NoisyNet (noisy_sigma_init) + count bonus (UCB) provide exploration. vec![ @@ -503,13 +506,16 @@ impl ParameterSpace for DQNParams { // Exploration (1D) — C3 FIX: re-enabled count bonus (UCB, complementary to NoisyNet) (0.0, 0.3), // 26: count_bonus_coefficient (UCB exploration bonus β) + + // Risk-adjusted returns (1D) — C4 FIX: wire from search space instead of hardcoded + (0.0, 0.5), // 27: sharpe_weight (Sharpe ratio blending in composite reward) ] } fn from_continuous(x: &[f64]) -> Result { - if x.len() != 27 { + if x.len() != 28 { return Err(MLError::ConfigError { - reason: format!("Expected 27 continuous parameters, got {}", x.len()), + reason: format!("Expected 28 continuous parameters, got {}", x.len()), }); } @@ -570,7 +576,7 @@ impl ParameterSpace for DQNParams { let beta_entropy = 0.2; let td_error_clamp_max = 10.0; let batch_diversity_cooldown = 50.0; - let sharpe_weight = 0.3; // 30% Sharpe blending (matches DqnParams default) + let sharpe_weight = x[27].clamp(0.0, 0.5); // C4 FIX: tunable Sharpe blending let eval_softmax_temp = 1.0; // Fixed: backtest uses greedy argmax (Fix A) let gae_lambda = 0.95; let noisy_sigma_initial = 0.5; @@ -655,7 +661,7 @@ impl ParameterSpace for DQNParams { } fn to_continuous(&self) -> Vec { - // 27D search space (C3: re-enabled count_bonus_coefficient) + // 28D search space (C4: added sharpe_weight) vec![ self.learning_rate.ln(), // 0 self.batch_size as f64, // 1 @@ -677,18 +683,19 @@ impl ParameterSpace for DQNParams { self.kelly_fractional, // 17 self.kelly_max_fraction, // 18 self.volatility_window as f64, // 19 - self.tau.ln(), // 20 (C2: was 21) - self.hidden_dim_base as f64, // 21 (C2: was 22) - self.cql_alpha, // 22 (C2: was 24) - self.lr_decay_type, // 23 (C2: was 25) - self.dsr_eta.ln(), // 24: dsr_eta (log scale) + self.tau.ln(), // 20 + self.hidden_dim_base as f64, // 21 + self.cql_alpha, // 22 + self.lr_decay_type, // 23 + self.dsr_eta.ln(), // 24 self.minimum_profit_factor, // 25 - self.count_bonus_coefficient, // 26: UCB exploration bonus + self.count_bonus_coefficient, // 26 + self.sharpe_weight, // 27: C4 FIX ] } fn param_names() -> Vec<&'static str> { - // 27 tuned parameters (C3: re-enabled count_bonus_coefficient) + // 28 tuned parameters (C4: added sharpe_weight) vec![ "learning_rate", // 0 "batch_size", // 1 @@ -714,9 +721,10 @@ impl ParameterSpace for DQNParams { "hidden_dim_base", // 21 "cql_alpha", // 22 "lr_decay_type", // 23 - "dsr_eta", // 24 + "dsr_eta", // 24 "minimum_profit_factor", // 25 "count_bonus_coefficient", // 26 + "sharpe_weight", // 27: C4 FIX ] } @@ -3541,7 +3549,7 @@ mod tests { }; let continuous = params.to_continuous(); - assert_eq!(continuous.len(), 27, "to_continuous must return 27D vector"); + assert_eq!(continuous.len(), 28, "to_continuous must return 28D vector"); let recovered = DQNParams::from_continuous(&continuous).unwrap(); // Tuned parameters must roundtrip exactly (26D) @@ -3584,7 +3592,7 @@ mod tests { #[test] fn test_dqn_params_bounds() { let bounds = DQNParams::continuous_bounds(); - assert_eq!(bounds.len(), 27); // C3: 27D (re-added count_bonus_coefficient) + assert_eq!(bounds.len(), 28); // C4: 28D (added sharpe_weight) // Check log-scale bounds are reasonable assert!(bounds[0].0 < bounds[0].1); // learning_rate @@ -3628,7 +3636,7 @@ mod tests { #[test] fn test_param_names() { let names = DQNParams::param_names(); - assert_eq!(names.len(), 27); // C3: 27D (re-added count_bonus_coefficient) + assert_eq!(names.len(), 28); // C4: 28D (added sharpe_weight) assert_eq!(names[0], "learning_rate"); assert_eq!(names[1], "batch_size"); assert_eq!(names[2], "gamma"); @@ -3649,19 +3657,20 @@ mod tests { assert_eq!(names[17], "kelly_fractional"); assert_eq!(names[18], "kelly_max_fraction"); assert_eq!(names[19], "volatility_window"); - assert_eq!(names[20], "tau"); // C2: was 21 - assert_eq!(names[21], "hidden_dim_base"); // C2: was 22 - assert_eq!(names[22], "cql_alpha"); // C2: was 24 - assert_eq!(names[23], "lr_decay_type"); // C2: was 25 - assert_eq!(names[24], "dsr_eta"); // replaces min_epochs_before_stopping - assert_eq!(names[25], "minimum_profit_factor"); // C2: was 27 - assert_eq!(names[26], "count_bonus_coefficient"); // C3: re-added + assert_eq!(names[20], "tau"); + assert_eq!(names[21], "hidden_dim_base"); + assert_eq!(names[22], "cql_alpha"); + assert_eq!(names[23], "lr_decay_type"); + assert_eq!(names[24], "dsr_eta"); + assert_eq!(names[25], "minimum_profit_factor"); + assert_eq!(names[26], "count_bonus_coefficient"); + assert_eq!(names[27], "sharpe_weight"); // C4 } #[test] fn test_per_params_always_enabled() { // Test that PER is always enabled with tunable alpha/beta parameters - // 27D search space (C3: re-added count_bonus_coefficient) + // 28D search space (C4: added sharpe_weight) let continuous = vec![ 3e-5_f64.ln(), 92.0, 0.92, 97_273_f64.ln(), 4.0, 24.77_f64.ln(), 0.03, 1.0, 0.6, 0.4, // 8-9: per_alpha, per_beta_start @@ -3670,13 +3679,14 @@ mod tests { 1e-4_f64.ln(), // 16: weight_decay 0.5, 0.25, // 17-18: kelly_fractional, kelly_max_fraction 20.0, // 19: volatility_window - 0.007_f64.ln(), // 20: tau (was 21) - 512.0, // 21: hidden_dim_base (was 22) - 0.1, // 22: cql_alpha (was 24) - 2.0, // 23: lr_decay_type (was 25) + 0.007_f64.ln(), // 20: tau + 512.0, // 21: hidden_dim_base + 0.1, // 22: cql_alpha + 2.0, // 23: lr_decay_type 0.02_f64.ln(), // 24: dsr_eta (log scale) - 1.5, // 25: minimum_profit_factor (was 27) - 0.1, // 26: count_bonus_coefficient (C3) + 1.5, // 25: minimum_profit_factor + 0.1, // 26: count_bonus_coefficient + 0.3, // 27: sharpe_weight (C4) ]; let params = DQNParams::from_continuous(&continuous).unwrap(); @@ -3690,13 +3700,13 @@ mod tests { assert!((params.curiosity_weight - 0.0).abs() < 1e-6); // C2: fixed assert!((params.noisy_epsilon_floor - 0.0).abs() < 1e-6); // C2: fixed - // Test PER parameter bounds (min values) -- 27D + // Test PER parameter bounds (min values) -- 28D let continuous_min = vec![ 2e-5_f64.ln(), 64.0, 0.88, 50_000_f64.ln(), 1.0, 10.0_f64.ln(), 0.05, 0.5, 0.4, 0.2, // per_alpha min, per_beta_start min -30.0, 5.0, 0.1_f64.ln(), // v_min, v_max, noisy_sigma_init 128.0, 3.0, 51.0, // dueling_hidden_dim, n_steps (min=3), num_atoms - 1e-4_f64.ln(), // weight_decay min (C2: 10x stronger) + 1e-4_f64.ln(), // weight_decay min 0.25, 0.1, // kelly_fractional, kelly_max_fraction 10.0, // volatility_window 0.005_f64.ln(), // 20: tau min @@ -3705,7 +3715,8 @@ mod tests { 0.0, // 23: lr_decay_type (constant) 0.001_f64.ln(), // 24: dsr_eta min (log scale) 1.1, // 25: minimum_profit_factor min - 0.0, // 26: count_bonus_coefficient min (C3) + 0.0, // 26: count_bonus_coefficient min + 0.0, // 27: sharpe_weight min (C4) ]; let params_min = DQNParams::from_continuous(&continuous_min).unwrap(); assert!((params_min.per_alpha - 0.4).abs() < 1e-6); @@ -3714,22 +3725,23 @@ mod tests { assert!(params_min.use_distributional); assert!(params_min.use_noisy_nets); - // Test PER parameter bounds (max values) -- 27D + // Test PER parameter bounds (max values) -- 28D let continuous_max = vec![ 8e-5_f64.ln(), 512.0, 0.99, 100_000_f64.ln(), 4.0, 40.0_f64.ln(), 0.5, 2.0, 0.8, 0.6, // per_alpha max, per_beta_start max -5.0, 30.0, 1.0_f64.ln(), // v_min, v_max, noisy_sigma_init 512.0, 5.0, 201.0, // dueling_hidden_dim, n_steps, num_atoms - 1e-2_f64.ln(), // weight_decay max (C2: 10x stronger) + 1e-2_f64.ln(), // weight_decay max 0.75, 0.5, // kelly_fractional, kelly_max_fraction 30.0, // volatility_window 0.01_f64.ln(), // 20: tau max 1024.0, // 21: hidden_dim_base max - 1.0, // 22: cql_alpha max (full offline-RL range) + 1.0, // 22: cql_alpha max 2.0, // 23: lr_decay_type (cosine) 0.05_f64.ln(), // 24: dsr_eta max (log scale) 2.0, // 25: minimum_profit_factor max - 0.3, // 26: count_bonus_coefficient max (C3) + 0.3, // 26: count_bonus_coefficient max + 0.5, // 27: sharpe_weight max (C4) ]; let params_max = DQNParams::from_continuous(&continuous_max).unwrap(); assert!((params_max.per_alpha - 0.8).abs() < 1e-6); @@ -4029,7 +4041,7 @@ mod tests { fn test_qr_dqn_roundtrip_continuous() { let params = DQNParams::default(); let continuous = params.to_continuous(); - assert_eq!(continuous.len(), 27, "Should have 27 continuous dimensions"); + assert_eq!(continuous.len(), 28, "Should have 28 continuous dimensions"); let roundtrip = DQNParams::from_continuous(&continuous).unwrap(); // Default num_atoms=51 <= 100, so QR-DQN disabled, num_quantiles=64 (C51 default) assert!(!roundtrip.use_qr_dqn, "num_atoms=51 should use C51, not QR-DQN"); @@ -4040,7 +4052,7 @@ mod tests { #[test] fn test_qr_dqn_activation_threshold() { let bounds = DQNParams::continuous_bounds(); - let mut params = vec![0.0_f64; 27]; // C3: 27D + let mut params = vec![0.0_f64; 28]; // C4: 28D // Fill with valid defaults params[0] = (1e-4_f64).ln(); // learning_rate params[1] = 128.0; // batch_size @@ -4053,7 +4065,7 @@ mod tests { params[8] = 0.6; // per_alpha params[9] = 0.4; // per_beta_start // Fill remaining with midpoint of bounds - for i in 10..27 { + for i in 10..28 { params[i] = (bounds[i].0 + bounds[i].1) / 2.0; } @@ -4104,7 +4116,7 @@ mod tests { fn test_batch_size_respects_wide_bounds() { // Simulate PSO choosing batch_size=2048 (within VRAM-aware bounds) let bounds = DQNParams::continuous_bounds(); - let mut params = vec![0.0_f64; 27]; // C3: 27D + let mut params = vec![0.0_f64; 28]; // C4: 28D params[1] = 2048.0; // batch_size (index 1) // Fill other required params with valid defaults params[0] = (1e-4_f64).ln(); // learning_rate @@ -4112,12 +4124,12 @@ mod tests { params[3] = (100_000.0_f64).ln(); // buffer_size params[4] = 3.0; // max_position_absolute params[5] = (25.0_f64).ln(); // huber_delta - params[6] = 0.02; // entropy_coefficient (C2: within 0.005..0.05 range) + params[6] = 0.02; // entropy_coefficient params[7] = 0.5; // transaction_cost_multiplier params[8] = 0.6; // per_alpha params[9] = 0.4; // per_beta_start - // Remaining params (indices 10-26) -- use midpoint of bounds - for i in 10..27 { + // Remaining params (indices 10-27) -- use midpoint of bounds + for i in 10..28 { params[i] = (bounds[i].0 + bounds[i].1) / 2.0; } let result = DQNParams::from_continuous(¶ms).unwrap(); @@ -4227,8 +4239,8 @@ mod tests { // eval_softmax_temp removed from search space (backtest uses greedy argmax). // Verify from_continuous always sets it to fixed 1.0 regardless of input. let bounds = DQNParams::continuous_bounds(); - let mut vec = vec![0.0_f64; 27]; // C3: 27D - for i in 0..27 { + let mut vec = vec![0.0_f64; 28]; // C4: 28D + for i in 0..28 { vec[i] = (bounds[i].0 + bounds[i].1) / 2.0; } let params = DQNParams::from_continuous(&vec).unwrap(); diff --git a/crates/ml/src/trainers/dqn/trainer.rs b/crates/ml/src/trainers/dqn/trainer.rs index 27c44f17e..076f45e3d 100644 --- a/crates/ml/src/trainers/dqn/trainer.rs +++ b/crates/ml/src/trainers/dqn/trainer.rs @@ -5041,15 +5041,16 @@ mod tests { assert_eq!(unique.len(), 5, "All 5 exposure levels must be distinct"); } - /// Verify hyperopt search space is 27D (C3: re-added count_bonus_coefficient). + /// Verify hyperopt search space is 28D (C4: added sharpe_weight). #[test] fn test_c3_search_space_is_27d() { let bounds = crate::hyperopt::adapters::dqn::DQNParams::continuous_bounds(); - assert_eq!(bounds.len(), 27, "Search space must be 27D (C3: count_bonus_coefficient re-added)"); + assert_eq!(bounds.len(), 28, "Search space must be 28D (C4: sharpe_weight added)"); let names = crate::hyperopt::adapters::dqn::DQNParams::param_names(); - assert_eq!(names.len(), 27); + assert_eq!(names.len(), 28); assert!(names.contains(&"count_bonus_coefficient"), "count_bonus_coefficient must be in search space (C3)"); + assert!(names.contains(&"sharpe_weight"), "sharpe_weight must be in search space (C4)"); assert!(!names.contains(&"curiosity_weight"), "curiosity_weight must not be in search space"); assert!(!names.contains(&"noisy_epsilon_floor"), "noisy_epsilon_floor must not be in search space"); }