From f0e6845fb3ce90cde70f4db6cfaa1fc0a634a31c Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Wed, 4 Mar 2026 21:02:59 +0100 Subject: [PATCH] feat(ml): add cql_alpha to 26D hyperopt search space, widen v_min/v_max MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Expand search space from 25D to 26D with cql_alpha (0.0-0.5) - v_min bounds: (-3,-1) → (-15,-3), v_max: (1,3) → (3,15) - Default use_qr_dqn: true → false (IQN disabled) - Wire cql_alpha into DQNHyperparameters construction - Update all 18 adapter tests for new dimensions and ranges Co-Authored-By: Claude Opus 4.6 --- crates/ml/src/dqn/regime_conditional.rs | 12 +++ crates/ml/src/hyperopt/adapters/dqn.rs | 103 +++++++++++++++--------- 2 files changed, 77 insertions(+), 38 deletions(-) diff --git a/crates/ml/src/dqn/regime_conditional.rs b/crates/ml/src/dqn/regime_conditional.rs index 7e66cb9cc..50c879edf 100644 --- a/crates/ml/src/dqn/regime_conditional.rs +++ b/crates/ml/src/dqn/regime_conditional.rs @@ -477,6 +477,18 @@ impl RegimeConditionalDQN { self.volatile_head.reset_count_bonus(); } + /// Get count bonuses for all actions (UCB exploration). + /// Uses trending head as representative (same convention as get_epsilon). + pub fn get_count_bonuses(&self) -> Vec { + self.trending_head.get_count_bonuses() + } + + /// Get shared DQN config (all heads share the same config). + /// Returns trending head's config as representative. + pub fn config(&self) -> &super::DQNConfig { + &self.trending_head.config + } + /// Update epsilon for specific regime head pub fn update_epsilon(&mut self, regime: RegimeType) { match regime { diff --git a/crates/ml/src/hyperopt/adapters/dqn.rs b/crates/ml/src/hyperopt/adapters/dqn.rs index b1b46c744..67c0d6339 100644 --- a/crates/ml/src/hyperopt/adapters/dqn.rs +++ b/crates/ml/src/hyperopt/adapters/dqn.rs @@ -3,7 +3,7 @@ //! This module provides a production-ready adapter for optimizing DQN //! hyperparameters using the generic optimization framework. It implements: //! -//! - 11D continuous parameter space with log-scale handling (9D previous + 2D PER) +//! - 26D continuous parameter space with log-scale handling (25D previous + 1D CQL) //! - Training wrapper that integrates with existing DQN pipeline //! - Backtest-based objective using Sharpe ratio (production metric) //! - Prioritized Experience Replay (PER) tuning for Rainbow DQN performance (+25-40% convergence speed) @@ -377,6 +377,10 @@ pub struct DQNParams { /// Count-based exploration bonus coefficient (0.01-0.5, default 0.1) /// UCB-style bonus: β * sqrt(log(N) / (1 + n_a)) pub count_bonus_coefficient: f64, + + /// Conservative Q-Learning (CQL) regularization alpha (0.0-0.5) + /// Penalizes overestimation of OOD actions: 0.0=disabled, 0.1=mild, 0.5=moderate + pub cql_alpha: f64, } impl Default for DQNParams { @@ -409,8 +413,8 @@ impl Default for DQNParams { // ============================================================================= use_distributional: true, // ENABLED: BUG #36 fixed, full Rainbow DQN C51 num_atoms: 51, // Wave 2.3: Rainbow DQN standard - v_min: -2.0, // Bug #5: Corrected from -1000.0 - v_max: 2.0, // Bug #5: Corrected from 1000.0 + v_min: -10.0, // Widened for C51 action separation (45 actions need wider support) + v_max: 10.0, // Widened for C51 action separation (45 actions need wider support) use_noisy_nets: true, // Wave 8: Default ENABLED for full Rainbow DQN noisy_sigma_init: 0.5, // Wave 2.4: Rainbow DQN standard minimum_profit_factor: 1.5, // BUG #7: Default 50% margin above breakeven @@ -442,24 +446,25 @@ impl Default for DQNParams { use_residual: false, // Default: disabled norm_type: 0.0, // Default: LayerNorm activation_type: 0.0, // Default: ReLU - // QR-DQN: Enabled by default (replaces disabled C51) - use_qr_dqn: true, + // QR-DQN: Disabled — IQN conflicts with C51 (train-test distribution mismatch) + use_qr_dqn: false, num_quantiles: 32, qr_kappa: 1.0, hidden_dim_base: 256, // Conservative default (backward compatible) noisy_epsilon_floor: 0.05, // Minimum random exploration with noisy nets count_bonus_coefficient: 0.1, // UCB exploration bonus + cql_alpha: 0.1, // CQL regularization: mild penalty on OOD Q-values } } } impl ParameterSpace for DQNParams { fn continuous_bounds() -> Vec<(f64, f64)> { - // 25D reduced search space (down from 45D) + // 26D search space (25D + cql_alpha) // 20 exotic parameters fixed to validated defaults in from_continuous() // This makes PSO (20 particles) and TPE dramatically more effective // - // Kept: the 25 params that genuinely affect trading performance + // Kept: the 26 params that genuinely affect trading performance // Fixed: ensemble/architecture/scheduling params that rarely deviate from defaults vec![ // Base parameters (11D) @@ -476,8 +481,8 @@ impl ParameterSpace for DQNParams { (0.2, 0.6), // 10: per_beta_start (linear) // Rainbow DQN extensions (6D) - (-3.0, -1.0), // 11: v_min (linear) - Bug #5 fix: center -2.0 - (1.0, 3.0), // 12: v_max (linear) - Bug #5 fix: center +2.0 + (-15.0, -3.0), // 11: v_min (linear) - widened for C51 action separation + (3.0, 15.0), // 12: v_max (linear) - widened for C51 action separation (0.1_f64.ln(), 1.0_f64.ln()), // 13: noisy_sigma_init (log scale) (128.0, 512.0), // 14: dueling_hidden_dim (linear, step=128) (1.0, 5.0), // 15: n_steps (linear, int) @@ -502,17 +507,20 @@ impl ParameterSpace for DQNParams { // Exploration anti-collapse (1D) (0.02, 0.10), // 24: noisy_epsilon_floor (linear, minimum random exploration) + + // CQL regularization (1D) + (0.0, 0.5), // 25: cql_alpha (linear, 0.0=disabled, 0.1=mild, 0.5=moderate) ] } fn from_continuous(x: &[f64]) -> Result { - if x.len() != 25 { + if x.len() != 26 { return Err(MLError::ConfigError { - reason: format!("Expected 25 continuous parameters (reduced from 45D), got {}", x.len()), + reason: format!("Expected 26 continuous parameters (25D + cql_alpha), got {}", x.len()), }); } - // === 25 TUNED parameters (from search space) === + // === 26 TUNED parameters (from search space) === let learning_rate = x[0].exp(); let mut batch_size = x[1].round().max(64.0) as usize; // Only enforce floor. PSO + HardwareBudget control the upper bound. let buffer_size = x[3].exp().round().max(50_000.0) as usize; @@ -525,8 +533,8 @@ impl ParameterSpace for DQNParams { let per_beta_start = x[10].clamp(0.2, 0.6); // Rainbow DQN extensions - let v_min = x[11].clamp(-3.0, -1.0); - let v_max = x[12].clamp(1.0, 3.0); + let v_min = x[11].clamp(-15.0, -3.0); + let v_max = x[12].clamp(3.0, 15.0); let noisy_sigma_init = x[13].exp().clamp(0.1, 1.0); let dueling_hidden_dim = (x[14].round() / 128.0).round() * 128.0; let dueling_hidden_dim = dueling_hidden_dim.clamp(128.0, 512.0) as usize; @@ -554,6 +562,9 @@ impl ParameterSpace for DQNParams { // Exploration anti-collapse let noisy_epsilon_floor = x[24].clamp(0.02, 0.10); + // CQL regularization + let cql_alpha = x[25].clamp(0.0, 0.5); + // === 20 FIXED parameters (validated defaults, removed from search) === let minimum_profit_factor = 1.5; let kelly_min_trades: usize = 20; @@ -644,19 +655,20 @@ impl ParameterSpace for DQNParams { use_residual: false, norm_type, activation_type, - use_qr_dqn: true, + use_qr_dqn: false, // IQN disabled: conflicts with C51 (train-test mismatch) num_quantiles, qr_kappa, hidden_dim_base, noisy_epsilon_floor, count_bonus_coefficient, + cql_alpha, }; Ok(params) } fn to_continuous(&self) -> Vec { - // 25D reduced space — only the tuned parameters + // 26D search space — only the tuned parameters // Fixed parameters are NOT emitted (they get their defaults in from_continuous) vec![ self.learning_rate.ln(), // 0 @@ -684,11 +696,12 @@ impl ParameterSpace for DQNParams { self.tau.ln(), // 22 self.hidden_dim_base as f64, // 23 self.noisy_epsilon_floor, // 24 + self.cql_alpha, // 25 ] } fn param_names() -> Vec<&'static str> { - // 25 tuned parameters (matches continuous_bounds / from_continuous / to_continuous) + // 26 tuned parameters (matches continuous_bounds / from_continuous / to_continuous) vec![ "learning_rate", // 0 "batch_size", // 1 @@ -715,6 +728,7 @@ impl ParameterSpace for DQNParams { "tau", // 22 "hidden_dim_base", // 23 "noisy_epsilon_floor", // 24 + "cql_alpha", // 25 ] } @@ -726,7 +740,7 @@ impl ParameterSpace for DQNParams { batch_bound.1 = max_batch; } } - // Cap hidden_dim_base by VRAM (index 23 in 25D space) + // Cap hidden_dim_base by VRAM (index 23 in 26D space) let max_base = budget.max_hidden_dim_base(4, 256, 54, 45); if let Some(dim_bound) = bounds.get_mut(23) { dim_bound.1 = max_base as f64; @@ -2411,6 +2425,10 @@ impl HyperparameterOptimizable for DQNTrainer { num_quantiles: params.num_quantiles, qr_kappa: params.qr_kappa, + // Conservative Q-Learning (CQL) — wired from hyperopt search space + use_cql: true, + cql_alpha: params.cql_alpha, + // Phase 3: GPU experience collection (defaults, not in search space) gpu_n_episodes: 128, gpu_timesteps_per_episode: 500, @@ -3246,8 +3264,8 @@ mod tests { #[test] fn test_dqn_params_roundtrip() { - // Roundtrip test for the 25D reduced search space - // Only the 25 tuned parameters roundtrip; the 20 fixed params get defaults from from_continuous + // Roundtrip test for the 26D search space + // Only the 26 tuned parameters roundtrip; the 20 fixed params get defaults from from_continuous let params = DQNParams { learning_rate: 3.37e-05, batch_size: 92, @@ -3267,8 +3285,8 @@ mod tests { tau: 0.001, use_distributional: true, num_atoms: 51, - v_min: -2.0, - v_max: 2.0, + v_min: -5.0, + v_max: 5.0, use_noisy_nets: true, noisy_sigma_init: 0.5, // Fixed params — these will be overwritten by from_continuous defaults @@ -3301,10 +3319,11 @@ mod tests { hidden_dim_base: 512, noisy_epsilon_floor: 0.05, count_bonus_coefficient: 0.1, + cql_alpha: 0.15, }; let continuous = params.to_continuous(); - assert_eq!(continuous.len(), 25, "to_continuous must return 25D vector"); + assert_eq!(continuous.len(), 26, "to_continuous must return 26D vector"); let recovered = DQNParams::from_continuous(&continuous).unwrap(); // Tuned parameters must roundtrip exactly @@ -3324,6 +3343,7 @@ mod tests { assert!((recovered.tau - params.tau).abs() < 1e-6); assert_eq!(recovered.hidden_dim_base, params.hidden_dim_base); assert!((recovered.noisy_epsilon_floor - params.noisy_epsilon_floor).abs() < 1e-6); + assert!((recovered.cql_alpha - params.cql_alpha).abs() < 1e-6); // Fixed parameters must get their validated defaults assert!((recovered.minimum_profit_factor - 1.5).abs() < 1e-6); @@ -3343,7 +3363,7 @@ mod tests { #[test] fn test_dqn_params_bounds() { let bounds = DQNParams::continuous_bounds(); - assert_eq!(bounds.len(), 25); // 25D reduced search space (down from 45) + assert_eq!(bounds.len(), 26); // 26D search space (25D + cql_alpha) // Check log-scale bounds are reasonable assert!(bounds[0].0 < bounds[0].1); // learning_rate @@ -3369,8 +3389,8 @@ mod tests { assert_eq!(bounds[8], (0.5, 2.0)); // transaction_cost_multiplier assert_eq!(bounds[9], (0.4, 0.8)); // per_alpha assert_eq!(bounds[10], (0.2, 0.6)); // per_beta_start - assert_eq!(bounds[11], (-3.0, -1.0)); // v_min - assert_eq!(bounds[12], (1.0, 3.0)); // v_max + assert_eq!(bounds[11], (-15.0, -3.0)); // v_min (widened for C51 action separation) + assert_eq!(bounds[12], (3.0, 15.0)); // v_max (widened for C51 action separation) assert_eq!(bounds[14], (128.0, 512.0)); // dueling_hidden_dim assert_eq!(bounds[15], (1.0, 5.0)); // n_steps assert_eq!(bounds[16], (51.0, 201.0)); // num_atoms @@ -3384,12 +3404,15 @@ mod tests { assert_eq!(bounds[21], (0.01, 0.5)); // curiosity_weight assert_eq!(bounds[23], (256.0, 4096.0)); // hidden_dim_base assert_eq!(bounds[24], (0.02, 0.10)); // noisy_epsilon_floor + + // CQL regularization + assert_eq!(bounds[25], (0.0, 0.5)); // cql_alpha } #[test] fn test_param_names() { let names = DQNParams::param_names(); - assert_eq!(names.len(), 25); // 25D reduced search space + assert_eq!(names.len(), 26); // 26D search space assert_eq!(names[0], "learning_rate"); assert_eq!(names[1], "batch_size"); assert_eq!(names[2], "gamma"); @@ -3415,16 +3438,17 @@ mod tests { assert_eq!(names[22], "tau"); assert_eq!(names[23], "hidden_dim_base"); assert_eq!(names[24], "noisy_epsilon_floor"); + assert_eq!(names[25], "cql_alpha"); } #[test] fn test_per_params_always_enabled() { // Test that PER is always enabled with tunable alpha/beta parameters - // 25D reduced search space + // 26D search space let continuous = vec![ 3e-5_f64.ln(), 92.0, 0.9588, 97_273_f64.ln(), 1.404, 4.0, 24.77_f64.ln(), 0.01, 1.0, 0.6, 0.4, // 9-10: per_alpha, per_beta_start - -2.0, 2.0, 0.5_f64.ln(), // 11-13: v_min, v_max, noisy_sigma_init + -5.0, 5.0, 0.5_f64.ln(), // 11-13: v_min, v_max, noisy_sigma_init 256.0, 3.0, 101.0, // 14-16: dueling_hidden_dim, n_steps, num_atoms 1e-4_f64.ln(), // 17: weight_decay 0.5, 0.25, // 18-19: kelly_fractional, kelly_max_fraction @@ -3433,6 +3457,7 @@ mod tests { 0.005_f64.ln(), // 22: tau 512.0, // 23: hidden_dim_base 0.05, // 24: noisy_epsilon_floor + 0.1, // 25: cql_alpha ]; let params = DQNParams::from_continuous(&continuous).unwrap(); @@ -3443,11 +3468,11 @@ mod tests { assert!(params.use_distributional); assert!(params.use_noisy_nets); - // Test PER parameter bounds (min values) — 25D + // Test PER parameter bounds (min values) — 26D let continuous_min = vec![ 2e-5_f64.ln(), 64.0, 0.95, 50_000_f64.ln(), 1.0, 1.0, 10.0_f64.ln(), 0.0, 0.5, 0.4, 0.2, // per_alpha min, per_beta_start min - -3.0, 1.0, 0.1_f64.ln(), // v_min, v_max, noisy_sigma_init + -15.0, 3.0, 0.1_f64.ln(), // v_min, v_max, noisy_sigma_init 128.0, 1.0, 51.0, // dueling_hidden_dim, n_steps, num_atoms 1e-5_f64.ln(), // weight_decay min 0.25, 0.1, // kelly_fractional, kelly_max_fraction @@ -3456,6 +3481,7 @@ mod tests { 0.0001_f64.ln(), // tau min 256.0, // hidden_dim_base min 0.02, // noisy_epsilon_floor min + 0.0, // cql_alpha min ]; let params_min = DQNParams::from_continuous(&continuous_min).unwrap(); assert!((params_min.per_alpha - 0.4).abs() < 1e-6); @@ -3464,11 +3490,11 @@ mod tests { assert!(params_min.use_distributional); assert!(params_min.use_noisy_nets); - // Test PER parameter bounds (max values) — 25D + // Test PER parameter bounds (max values) — 26D let continuous_max = vec![ 8e-5_f64.ln(), 4096.0, 0.99, 100_000_f64.ln(), 2.0, 4.0, 40.0_f64.ln(), 0.1, 2.0, 0.8, 0.6, // per_alpha max, per_beta_start max - -1.0, 3.0, 1.0_f64.ln(), // v_min, v_max, noisy_sigma_init + -3.0, 15.0, 1.0_f64.ln(), // v_min, v_max, noisy_sigma_init 512.0, 5.0, 201.0, // dueling_hidden_dim, n_steps, num_atoms 1e-3_f64.ln(), // weight_decay max 1.0, 0.5, // kelly_fractional, kelly_max_fraction @@ -3477,6 +3503,7 @@ mod tests { 0.01_f64.ln(), // tau max 4096.0, // hidden_dim_base max 0.10, // noisy_epsilon_floor max + 0.5, // cql_alpha max ]; let params_max = DQNParams::from_continuous(&continuous_max).unwrap(); assert!((params_max.per_alpha - 0.8).abs() < 1e-6); @@ -3767,7 +3794,7 @@ mod tests { #[test] fn test_qr_dqn_params_in_default() { let params = DQNParams::default(); - assert!(params.use_qr_dqn, "QR-DQN should be enabled by default"); + assert!(!params.use_qr_dqn, "QR-DQN (IQN) should be disabled by default (conflicts with C51)"); assert_eq!(params.num_quantiles, 32, "Default num_quantiles should be 32"); assert!((params.qr_kappa - 1.0).abs() < f64::EPSILON, "Default kappa should be 1.0"); } @@ -3776,7 +3803,7 @@ mod tests { fn test_qr_dqn_roundtrip_continuous() { let params = DQNParams::default(); let continuous = params.to_continuous(); - assert_eq!(continuous.len(), 25, "Should have 25 continuous dimensions (reduced from 45D)"); + assert_eq!(continuous.len(), 26, "Should have 26 continuous dimensions"); let roundtrip = DQNParams::from_continuous(&continuous).unwrap(); // num_quantiles and qr_kappa are now fixed defaults (not in search space) assert_eq!(roundtrip.num_quantiles, 64); // Fixed default @@ -3800,7 +3827,7 @@ mod tests { #[test] fn test_batch_size_respects_wide_bounds() { // Simulate PSO choosing batch_size=2048 (within VRAM-aware bounds) - let mut params = vec![0.0_f64; 25]; + let mut params = vec![0.0_f64; 26]; params[1] = 2048.0; // batch_size (index 1) // Fill other required params with valid defaults params[0] = (1e-4_f64).ln(); // learning_rate @@ -3813,9 +3840,9 @@ mod tests { params[8] = 0.5; // transaction_cost_multiplier params[9] = 0.6; // per_alpha params[10] = 0.4; // per_beta_start - // Remaining params (indices 11-24) -- use midpoint of bounds + // Remaining params (indices 11-25) -- use midpoint of bounds let bounds = DQNParams::continuous_bounds(); - for i in 11..25 { + for i in 11..26 { params[i] = (bounds[i].0 + bounds[i].1) / 2.0; } let result = DQNParams::from_continuous(¶ms).unwrap();