feat(dqn): wire QR-DQN QuantileNetwork through hyperopt to training loop

Add use_qr_dqn, num_quantiles, qr_kappa fields to DQNHyperparameters
and pass them from DQNParams (hyperopt) through to DQNConfig (agent),
replacing hardcoded IQN values in the trainer.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-02-21 12:33:21 +01:00
parent e29b6ac307
commit 11d486021f
3 changed files with 22 additions and 4 deletions

View File

@@ -2239,6 +2239,11 @@ impl HyperparameterOptimizable for DQNTrainer {
noisy_sigma_initial: 0.6, // Default: 60% initial noise
noisy_sigma_final: 0.4, // Default: 40% final noise
noisy_sigma_anneal_steps: 10000, // Default: 10K steps for annealing
// QR-DQN (replaces disabled C51)
use_qr_dqn: params.use_qr_dqn,
num_quantiles: params.num_quantiles,
qr_kappa: params.qr_kappa,
};
let data_path_str = self

View File

@@ -579,6 +579,14 @@ pub struct DQNHyperparameters {
/// Prevents overfitting by penalizing large weights via L2 regularization.
/// Recommended: 1e-4 for standard training, 1e-3 for aggressive regularization.
pub weight_decay: f64,
// QR-DQN (Quantile Regression DQN) — replaces disabled C51
/// Enable QR-DQN (IQN) for distributional RL
pub use_qr_dqn: bool,
/// Number of quantiles for QR-DQN (32-200)
pub num_quantiles: usize,
/// Kappa for quantile Huber loss (0.5-2.0)
pub qr_kappa: f64,
}
impl Default for DQNHyperparameters {
@@ -748,6 +756,11 @@ impl DQNHyperparameters {
// WAVE 30: L2 Weight Decay
weight_decay: 1e-4, // Default: 0.0001 (standard regularization strength)
// QR-DQN (replaces disabled C51)
use_qr_dqn: true, // Default: enabled (IQN distributional RL)
num_quantiles: 32, // Default: 32 quantiles
qr_kappa: 1.0, // Default: 1.0 (standard quantile Huber loss)
}
}
}

View File

@@ -350,10 +350,10 @@ impl DQNTrainer {
use_cql: true,
cql_alpha: 1.0,
use_iqn: true,
iqn_num_quantiles: 64,
iqn_kappa: 1.0,
iqn_embedding_dim: 64,
use_iqn: hyperparams.use_qr_dqn, // Controlled by hyperopt
iqn_num_quantiles: hyperparams.num_quantiles, // Controlled by hyperopt
iqn_kappa: hyperparams.qr_kappa as f32, // Controlled by hyperopt (f64→f32)
iqn_embedding_dim: 64, // Fixed (not in search space)
use_cvar_action_selection: false,
cvar_alpha: 0.05,
};