diff --git a/ml/src/hyperopt/adapters/dqn.rs b/ml/src/hyperopt/adapters/dqn.rs index 5f0a707d5..386d08b85 100644 --- a/ml/src/hyperopt/adapters/dqn.rs +++ b/ml/src/hyperopt/adapters/dqn.rs @@ -2239,6 +2239,11 @@ impl HyperparameterOptimizable for DQNTrainer { noisy_sigma_initial: 0.6, // Default: 60% initial noise noisy_sigma_final: 0.4, // Default: 40% final noise noisy_sigma_anneal_steps: 10000, // Default: 10K steps for annealing + + // QR-DQN (replaces disabled C51) + use_qr_dqn: params.use_qr_dqn, + num_quantiles: params.num_quantiles, + qr_kappa: params.qr_kappa, }; let data_path_str = self diff --git a/ml/src/trainers/dqn/config.rs b/ml/src/trainers/dqn/config.rs index 79b45bd64..ba5d8f5a0 100644 --- a/ml/src/trainers/dqn/config.rs +++ b/ml/src/trainers/dqn/config.rs @@ -579,6 +579,14 @@ pub struct DQNHyperparameters { /// Prevents overfitting by penalizing large weights via L2 regularization. /// Recommended: 1e-4 for standard training, 1e-3 for aggressive regularization. pub weight_decay: f64, + + // QR-DQN (Quantile Regression DQN) — replaces disabled C51 + /// Enable QR-DQN (IQN) for distributional RL + pub use_qr_dqn: bool, + /// Number of quantiles for QR-DQN (32-200) + pub num_quantiles: usize, + /// Kappa for quantile Huber loss (0.5-2.0) + pub qr_kappa: f64, } impl Default for DQNHyperparameters { @@ -748,6 +756,11 @@ impl DQNHyperparameters { // WAVE 30: L2 Weight Decay weight_decay: 1e-4, // Default: 0.0001 (standard regularization strength) + + // QR-DQN (replaces disabled C51) + use_qr_dqn: true, // Default: enabled (IQN distributional RL) + num_quantiles: 32, // Default: 32 quantiles + qr_kappa: 1.0, // Default: 1.0 (standard quantile Huber loss) } } } diff --git a/ml/src/trainers/dqn/trainer.rs b/ml/src/trainers/dqn/trainer.rs index c6b5c72f2..cb22f99f5 100644 --- a/ml/src/trainers/dqn/trainer.rs +++ b/ml/src/trainers/dqn/trainer.rs @@ -350,10 +350,10 @@ impl DQNTrainer { use_cql: true, cql_alpha: 1.0, - use_iqn: true, - iqn_num_quantiles: 64, - iqn_kappa: 1.0, - iqn_embedding_dim: 64, + use_iqn: hyperparams.use_qr_dqn, // Controlled by hyperopt + iqn_num_quantiles: hyperparams.num_quantiles, // Controlled by hyperopt + iqn_kappa: hyperparams.qr_kappa as f32, // Controlled by hyperopt (f64→f32) + iqn_embedding_dim: 64, // Fixed (not in search space) use_cvar_action_selection: false, cvar_alpha: 0.05, };