feat(dqn): wire QR-DQN QuantileNetwork through hyperopt to training loop
Add use_qr_dqn, num_quantiles, qr_kappa fields to DQNHyperparameters and pass them from DQNParams (hyperopt) through to DQNConfig (agent), replacing hardcoded IQN values in the trainer. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -2239,6 +2239,11 @@ impl HyperparameterOptimizable for DQNTrainer {
|
||||
noisy_sigma_initial: 0.6, // Default: 60% initial noise
|
||||
noisy_sigma_final: 0.4, // Default: 40% final noise
|
||||
noisy_sigma_anneal_steps: 10000, // Default: 10K steps for annealing
|
||||
|
||||
// QR-DQN (replaces disabled C51)
|
||||
use_qr_dqn: params.use_qr_dqn,
|
||||
num_quantiles: params.num_quantiles,
|
||||
qr_kappa: params.qr_kappa,
|
||||
};
|
||||
|
||||
let data_path_str = self
|
||||
|
||||
@@ -579,6 +579,14 @@ pub struct DQNHyperparameters {
|
||||
/// Prevents overfitting by penalizing large weights via L2 regularization.
|
||||
/// Recommended: 1e-4 for standard training, 1e-3 for aggressive regularization.
|
||||
pub weight_decay: f64,
|
||||
|
||||
// QR-DQN (Quantile Regression DQN) — replaces disabled C51
|
||||
/// Enable QR-DQN (IQN) for distributional RL
|
||||
pub use_qr_dqn: bool,
|
||||
/// Number of quantiles for QR-DQN (32-200)
|
||||
pub num_quantiles: usize,
|
||||
/// Kappa for quantile Huber loss (0.5-2.0)
|
||||
pub qr_kappa: f64,
|
||||
}
|
||||
|
||||
impl Default for DQNHyperparameters {
|
||||
@@ -748,6 +756,11 @@ impl DQNHyperparameters {
|
||||
|
||||
// WAVE 30: L2 Weight Decay
|
||||
weight_decay: 1e-4, // Default: 0.0001 (standard regularization strength)
|
||||
|
||||
// QR-DQN (replaces disabled C51)
|
||||
use_qr_dqn: true, // Default: enabled (IQN distributional RL)
|
||||
num_quantiles: 32, // Default: 32 quantiles
|
||||
qr_kappa: 1.0, // Default: 1.0 (standard quantile Huber loss)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -350,10 +350,10 @@ impl DQNTrainer {
|
||||
|
||||
use_cql: true,
|
||||
cql_alpha: 1.0,
|
||||
use_iqn: true,
|
||||
iqn_num_quantiles: 64,
|
||||
iqn_kappa: 1.0,
|
||||
iqn_embedding_dim: 64,
|
||||
use_iqn: hyperparams.use_qr_dqn, // Controlled by hyperopt
|
||||
iqn_num_quantiles: hyperparams.num_quantiles, // Controlled by hyperopt
|
||||
iqn_kappa: hyperparams.qr_kappa as f32, // Controlled by hyperopt (f64→f32)
|
||||
iqn_embedding_dim: 64, // Fixed (not in search space)
|
||||
use_cvar_action_selection: false,
|
||||
cvar_alpha: 0.05,
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user