diff --git a/ml/src/hyperopt/adapters/dqn.rs b/ml/src/hyperopt/adapters/dqn.rs index cfd999cff..5f0a707d5 100644 --- a/ml/src/hyperopt/adapters/dqn.rs +++ b/ml/src/hyperopt/adapters/dqn.rs @@ -320,6 +320,14 @@ pub struct DQNParams { /// Activation function type (0=ReLU, 1=LeakyReLU, 2=GELU, 3=Mish) /// Stored as f64 for hyperopt, cast to enum in train_with_params pub activation_type: f64, + + // QR-DQN (Quantile Regression DQN) — replaces disabled C51 + /// Enable QR-DQN for distributional RL (replaces C51 disabled by BUG #36) + pub use_qr_dqn: bool, + /// Number of quantiles for QR-DQN distribution (32-200) + pub num_quantiles: usize, + /// Kappa for quantile Huber loss (0.5-2.0, controls L1<->L2 transition) + pub qr_kappa: f64, } impl Default for DQNParams { @@ -391,13 +399,17 @@ impl Default for DQNParams { use_residual: false, // Default: disabled norm_type: 0.0, // Default: LayerNorm activation_type: 0.0, // Default: ReLU + // QR-DQN: Enabled by default (replaces disabled C51) + use_qr_dqn: true, + num_quantiles: 32, + qr_kappa: 1.0, } } } impl ParameterSpace for DQNParams { fn continuous_bounds() -> Vec<(f64, f64)> { - // CRITICAL GAP FIX: Expanded to 40D continuous space (added weight_decay) + // 42D continuous space (40D base + 2D QR-DQN: num_quantiles, qr_kappa) // Base parameters (11D from Wave 1-2): LR, batch, gamma, buffer, hold_penalty, max_pos, huber, entropy, tx_cost, per_alpha, per_beta // Rainbow extensions (6D): v_min, v_max, noisy_sigma_init, dueling_hidden_dim, n_steps, num_atoms // NOTE: v_min/v_max/num_atoms still tunable but UNUSED (C51 disabled due to BUG #36) @@ -477,14 +489,17 @@ impl ParameterSpace for DQNParams { // They default to false and can be enabled via CLI or config (0.0, 2.0), // 38: norm_type (0=LayerNorm, 1=RMSNorm, 2=None) (0.0, 3.0), // 39: activation_type (0=ReLU, 1=LeakyReLU, 2=GELU, 3=Mish) + // QR-DQN parameters (40D -> 42D) + (32.0, 200.0), // 40: num_quantiles (linear, integer) + (0.5_f64.ln(), 2.0_f64.ln()), // 41: qr_kappa (log scale) // WAVE 11: Rainbow DQN boolean parameters REMOVED from search space (always TRUE) ] } fn from_continuous(x: &[f64]) -> Result { - if x.len() != 40 { + if x.len() != 42 { return Err(MLError::ConfigError { - reason: format!("Expected 40 continuous parameters (CRITICAL GAP FIX: added weight_decay), got {}", x.len()), + reason: format!("Expected 42 continuous parameters (added QR-DQN), got {}", x.len()), }); } @@ -555,6 +570,10 @@ impl ParameterSpace for DQNParams { let norm_type = x[38].round().clamp(0.0, 2.0); // 0=LayerNorm, 1=RMSNorm, 2=None let activation_type = x[39].round().clamp(0.0, 3.0); // 0=ReLU, 1=LeakyReLU, 2=GELU, 3=Mish + // QR-DQN parameters + let num_quantiles = x[40].round().clamp(32.0, 200.0) as usize; + let qr_kappa = x[41].exp().clamp(0.5, 2.0); + // WAVE 11: Rainbow DQN boolean parameters are ALWAYS TRUE (removed from search space) // User requirement: "I want them enabled!" - no point in tuning boolean flags @@ -632,6 +651,10 @@ impl ParameterSpace for DQNParams { use_residual: false, norm_type, activation_type, + // QR-DQN (replaces disabled C51) + use_qr_dqn: true, // Always enabled + num_quantiles, + qr_kappa, }; // Note: HFT constraint validation moved to evaluate_objective (train_with_params) @@ -690,6 +713,9 @@ impl ParameterSpace for DQNParams { // WAVE 26 P1: Network architecture self.norm_type, self.activation_type, + // QR-DQN parameters (42D) + self.num_quantiles as f64, + self.qr_kappa.ln(), // WAVE 11: Rainbow DQN boolean parameters REMOVED (always TRUE, not tunable) ] } @@ -744,6 +770,9 @@ impl ParameterSpace for DQNParams { // WAVE 26 P1: Network architecture "norm_type", "activation_type", + // QR-DQN parameters + "num_quantiles", + "qr_kappa", // WAVE 11: Rainbow DQN boolean parameters REMOVED (always TRUE, not tunable) ] } @@ -3037,6 +3066,10 @@ mod tests { use_residual: false, norm_type: 0.0, activation_type: 0.0, + // QR-DQN + use_qr_dqn: true, + num_quantiles: 32, + qr_kappa: 1.0, }; let continuous = params.to_continuous(); @@ -3057,7 +3090,7 @@ mod tests { #[test] fn test_dqn_params_bounds() { let bounds = DQNParams::continuous_bounds(); - assert_eq!(bounds.len(), 40); // WAVE 26 P1.5: 40 continuous parameters (includes weight_decay + all WAVE 26 params) + assert_eq!(bounds.len(), 42); // 42 continuous parameters (added QR-DQN: num_quantiles, qr_kappa) // Check log-scale bounds are reasonable assert!(bounds[0].0 < bounds[0].1); // learning_rate @@ -3108,7 +3141,7 @@ mod tests { #[test] fn test_param_names() { let names = DQNParams::param_names(); - assert_eq!(names.len(), 40); // CRITICAL GAP FIX: 40 tunable hyperparameters (added weight_decay + all WAVE 26 params) + assert_eq!(names.len(), 42); // 42 tunable hyperparameters (added QR-DQN: num_quantiles, qr_kappa) assert_eq!(names[0], "learning_rate"); assert_eq!(names[1], "batch_size"); assert_eq!(names[2], "gamma"); @@ -3164,6 +3197,8 @@ mod tests { 1.0, 0.1, 0.95, 0.6, 0.3, // lr_decay_type, sharpe_weight, gae_lambda, noisy_sigma_initial, noisy_sigma_final // WAVE 26 P1: Network Architecture 0.0, 1.0, // norm_type (LayerNorm), activation_type (LeakyReLU) + // QR-DQN parameters + 64.0, 1.0_f64.ln(), // num_quantiles=64, qr_kappa=1.0 (ln(1.0)=0.0) ]; let params = DQNParams::from_continuous(&continuous).unwrap(); @@ -3198,6 +3233,8 @@ mod tests { 0.0, 0.0, 0.9, 0.4, 0.2, // lr_decay_type min, sharpe_weight min, gae_lambda min, noisy_sigma_initial min, noisy_sigma_final min // WAVE 26 P1: Network Architecture min 0.0, 0.0, // norm_type min (LayerNorm), activation_type min (ReLU) + // QR-DQN parameters min + 32.0, 0.5_f64.ln(), // num_quantiles min=32, qr_kappa min=0.5 ]; let params_min = DQNParams::from_continuous(&continuous_min).unwrap(); assert!((params_min.per_alpha - 0.4).abs() < 1e-6); @@ -3228,6 +3265,8 @@ mod tests { 2.0, 0.5, 0.99, 0.8, 0.5, // lr_decay_type max, sharpe_weight max, gae_lambda max, noisy_sigma_initial max, noisy_sigma_final max // WAVE 26 P1: Network Architecture max 2.0, 3.0, // norm_type max (None), activation_type max (Mish) + // QR-DQN parameters max + 200.0, 2.0_f64.ln(), // num_quantiles max=200, qr_kappa max=2.0 ]; let params_max = DQNParams::from_continuous(&continuous_max).unwrap(); assert!((params_max.per_alpha - 0.8).abs() < 1e-6); @@ -3587,4 +3626,22 @@ mod tests { objective_positive ); } + + #[test] + fn test_qr_dqn_params_in_default() { + let params = DQNParams::default(); + assert!(params.use_qr_dqn, "QR-DQN should be enabled by default"); + assert_eq!(params.num_quantiles, 32, "Default num_quantiles should be 32"); + assert!((params.qr_kappa - 1.0).abs() < f64::EPSILON, "Default kappa should be 1.0"); + } + + #[test] + fn test_qr_dqn_roundtrip_continuous() { + let params = DQNParams::default(); + let continuous = params.to_continuous(); + assert_eq!(continuous.len(), 42, "Should have 42 continuous dimensions"); + let roundtrip = DQNParams::from_continuous(&continuous).unwrap(); + assert_eq!(roundtrip.num_quantiles, params.num_quantiles); + assert!((roundtrip.qr_kappa - params.qr_kappa).abs() < 0.01); + } }