feat(hyperopt): add QR-DQN parameters to DQN hyperopt search space (40D->42D)
Add num_quantiles and qr_kappa to the continuous search space for Quantile Regression DQN, replacing the disabled C51 distributional RL (BUG #36). QR-DQN is enabled by default with 32 quantiles and kappa=1.0. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
@@ -320,6 +320,14 @@ pub struct DQNParams {
|
||||
/// Activation function type (0=ReLU, 1=LeakyReLU, 2=GELU, 3=Mish)
|
||||
/// Stored as f64 for hyperopt, cast to enum in train_with_params
|
||||
pub activation_type: f64,
|
||||
|
||||
// QR-DQN (Quantile Regression DQN) — replaces disabled C51
|
||||
/// Enable QR-DQN for distributional RL (replaces C51 disabled by BUG #36)
|
||||
pub use_qr_dqn: bool,
|
||||
/// Number of quantiles for QR-DQN distribution (32-200)
|
||||
pub num_quantiles: usize,
|
||||
/// Kappa for quantile Huber loss (0.5-2.0, controls L1<->L2 transition)
|
||||
pub qr_kappa: f64,
|
||||
}
|
||||
|
||||
impl Default for DQNParams {
|
||||
@@ -391,13 +399,17 @@ impl Default for DQNParams {
|
||||
use_residual: false, // Default: disabled
|
||||
norm_type: 0.0, // Default: LayerNorm
|
||||
activation_type: 0.0, // Default: ReLU
|
||||
// QR-DQN: Enabled by default (replaces disabled C51)
|
||||
use_qr_dqn: true,
|
||||
num_quantiles: 32,
|
||||
qr_kappa: 1.0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ParameterSpace for DQNParams {
|
||||
fn continuous_bounds() -> Vec<(f64, f64)> {
|
||||
// CRITICAL GAP FIX: Expanded to 40D continuous space (added weight_decay)
|
||||
// 42D continuous space (40D base + 2D QR-DQN: num_quantiles, qr_kappa)
|
||||
// Base parameters (11D from Wave 1-2): LR, batch, gamma, buffer, hold_penalty, max_pos, huber, entropy, tx_cost, per_alpha, per_beta
|
||||
// Rainbow extensions (6D): v_min, v_max, noisy_sigma_init, dueling_hidden_dim, n_steps, num_atoms
|
||||
// NOTE: v_min/v_max/num_atoms still tunable but UNUSED (C51 disabled due to BUG #36)
|
||||
@@ -477,14 +489,17 @@ impl ParameterSpace for DQNParams {
|
||||
// They default to false and can be enabled via CLI or config
|
||||
(0.0, 2.0), // 38: norm_type (0=LayerNorm, 1=RMSNorm, 2=None)
|
||||
(0.0, 3.0), // 39: activation_type (0=ReLU, 1=LeakyReLU, 2=GELU, 3=Mish)
|
||||
// QR-DQN parameters (40D -> 42D)
|
||||
(32.0, 200.0), // 40: num_quantiles (linear, integer)
|
||||
(0.5_f64.ln(), 2.0_f64.ln()), // 41: qr_kappa (log scale)
|
||||
// WAVE 11: Rainbow DQN boolean parameters REMOVED from search space (always TRUE)
|
||||
]
|
||||
}
|
||||
|
||||
fn from_continuous(x: &[f64]) -> Result<Self, MLError> {
|
||||
if x.len() != 40 {
|
||||
if x.len() != 42 {
|
||||
return Err(MLError::ConfigError {
|
||||
reason: format!("Expected 40 continuous parameters (CRITICAL GAP FIX: added weight_decay), got {}", x.len()),
|
||||
reason: format!("Expected 42 continuous parameters (added QR-DQN), got {}", x.len()),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -555,6 +570,10 @@ impl ParameterSpace for DQNParams {
|
||||
let norm_type = x[38].round().clamp(0.0, 2.0); // 0=LayerNorm, 1=RMSNorm, 2=None
|
||||
let activation_type = x[39].round().clamp(0.0, 3.0); // 0=ReLU, 1=LeakyReLU, 2=GELU, 3=Mish
|
||||
|
||||
// QR-DQN parameters
|
||||
let num_quantiles = x[40].round().clamp(32.0, 200.0) as usize;
|
||||
let qr_kappa = x[41].exp().clamp(0.5, 2.0);
|
||||
|
||||
// WAVE 11: Rainbow DQN boolean parameters are ALWAYS TRUE (removed from search space)
|
||||
// User requirement: "I want them enabled!" - no point in tuning boolean flags
|
||||
|
||||
@@ -632,6 +651,10 @@ impl ParameterSpace for DQNParams {
|
||||
use_residual: false,
|
||||
norm_type,
|
||||
activation_type,
|
||||
// QR-DQN (replaces disabled C51)
|
||||
use_qr_dqn: true, // Always enabled
|
||||
num_quantiles,
|
||||
qr_kappa,
|
||||
};
|
||||
|
||||
// Note: HFT constraint validation moved to evaluate_objective (train_with_params)
|
||||
@@ -690,6 +713,9 @@ impl ParameterSpace for DQNParams {
|
||||
// WAVE 26 P1: Network architecture
|
||||
self.norm_type,
|
||||
self.activation_type,
|
||||
// QR-DQN parameters (42D)
|
||||
self.num_quantiles as f64,
|
||||
self.qr_kappa.ln(),
|
||||
// WAVE 11: Rainbow DQN boolean parameters REMOVED (always TRUE, not tunable)
|
||||
]
|
||||
}
|
||||
@@ -744,6 +770,9 @@ impl ParameterSpace for DQNParams {
|
||||
// WAVE 26 P1: Network architecture
|
||||
"norm_type",
|
||||
"activation_type",
|
||||
// QR-DQN parameters
|
||||
"num_quantiles",
|
||||
"qr_kappa",
|
||||
// WAVE 11: Rainbow DQN boolean parameters REMOVED (always TRUE, not tunable)
|
||||
]
|
||||
}
|
||||
@@ -3037,6 +3066,10 @@ mod tests {
|
||||
use_residual: false,
|
||||
norm_type: 0.0,
|
||||
activation_type: 0.0,
|
||||
// QR-DQN
|
||||
use_qr_dqn: true,
|
||||
num_quantiles: 32,
|
||||
qr_kappa: 1.0,
|
||||
};
|
||||
|
||||
let continuous = params.to_continuous();
|
||||
@@ -3057,7 +3090,7 @@ mod tests {
|
||||
#[test]
|
||||
fn test_dqn_params_bounds() {
|
||||
let bounds = DQNParams::continuous_bounds();
|
||||
assert_eq!(bounds.len(), 40); // WAVE 26 P1.5: 40 continuous parameters (includes weight_decay + all WAVE 26 params)
|
||||
assert_eq!(bounds.len(), 42); // 42 continuous parameters (added QR-DQN: num_quantiles, qr_kappa)
|
||||
|
||||
// Check log-scale bounds are reasonable
|
||||
assert!(bounds[0].0 < bounds[0].1); // learning_rate
|
||||
@@ -3108,7 +3141,7 @@ mod tests {
|
||||
#[test]
|
||||
fn test_param_names() {
|
||||
let names = DQNParams::param_names();
|
||||
assert_eq!(names.len(), 40); // CRITICAL GAP FIX: 40 tunable hyperparameters (added weight_decay + all WAVE 26 params)
|
||||
assert_eq!(names.len(), 42); // 42 tunable hyperparameters (added QR-DQN: num_quantiles, qr_kappa)
|
||||
assert_eq!(names[0], "learning_rate");
|
||||
assert_eq!(names[1], "batch_size");
|
||||
assert_eq!(names[2], "gamma");
|
||||
@@ -3164,6 +3197,8 @@ mod tests {
|
||||
1.0, 0.1, 0.95, 0.6, 0.3, // lr_decay_type, sharpe_weight, gae_lambda, noisy_sigma_initial, noisy_sigma_final
|
||||
// WAVE 26 P1: Network Architecture
|
||||
0.0, 1.0, // norm_type (LayerNorm), activation_type (LeakyReLU)
|
||||
// QR-DQN parameters
|
||||
64.0, 1.0_f64.ln(), // num_quantiles=64, qr_kappa=1.0 (ln(1.0)=0.0)
|
||||
];
|
||||
|
||||
let params = DQNParams::from_continuous(&continuous).unwrap();
|
||||
@@ -3198,6 +3233,8 @@ mod tests {
|
||||
0.0, 0.0, 0.9, 0.4, 0.2, // lr_decay_type min, sharpe_weight min, gae_lambda min, noisy_sigma_initial min, noisy_sigma_final min
|
||||
// WAVE 26 P1: Network Architecture min
|
||||
0.0, 0.0, // norm_type min (LayerNorm), activation_type min (ReLU)
|
||||
// QR-DQN parameters min
|
||||
32.0, 0.5_f64.ln(), // num_quantiles min=32, qr_kappa min=0.5
|
||||
];
|
||||
let params_min = DQNParams::from_continuous(&continuous_min).unwrap();
|
||||
assert!((params_min.per_alpha - 0.4).abs() < 1e-6);
|
||||
@@ -3228,6 +3265,8 @@ mod tests {
|
||||
2.0, 0.5, 0.99, 0.8, 0.5, // lr_decay_type max, sharpe_weight max, gae_lambda max, noisy_sigma_initial max, noisy_sigma_final max
|
||||
// WAVE 26 P1: Network Architecture max
|
||||
2.0, 3.0, // norm_type max (None), activation_type max (Mish)
|
||||
// QR-DQN parameters max
|
||||
200.0, 2.0_f64.ln(), // num_quantiles max=200, qr_kappa max=2.0
|
||||
];
|
||||
let params_max = DQNParams::from_continuous(&continuous_max).unwrap();
|
||||
assert!((params_max.per_alpha - 0.8).abs() < 1e-6);
|
||||
@@ -3587,4 +3626,22 @@ mod tests {
|
||||
objective_positive
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_qr_dqn_params_in_default() {
|
||||
let params = DQNParams::default();
|
||||
assert!(params.use_qr_dqn, "QR-DQN should be enabled by default");
|
||||
assert_eq!(params.num_quantiles, 32, "Default num_quantiles should be 32");
|
||||
assert!((params.qr_kappa - 1.0).abs() < f64::EPSILON, "Default kappa should be 1.0");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_qr_dqn_roundtrip_continuous() {
|
||||
let params = DQNParams::default();
|
||||
let continuous = params.to_continuous();
|
||||
assert_eq!(continuous.len(), 42, "Should have 42 continuous dimensions");
|
||||
let roundtrip = DQNParams::from_continuous(&continuous).unwrap();
|
||||
assert_eq!(roundtrip.num_quantiles, params.num_quantiles);
|
||||
assert!((roundtrip.qr_kappa - params.qr_kappa).abs() < 0.01);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user