feat(ml): add cql_alpha to 26D hyperopt search space, widen v_min/v_max
- Expand search space from 25D to 26D with cql_alpha (0.0-0.5) - v_min bounds: (-3,-1) → (-15,-3), v_max: (1,3) → (3,15) - Default use_qr_dqn: true → false (IQN disabled) - Wire cql_alpha into DQNHyperparameters construction - Update all 18 adapter tests for new dimensions and ranges Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
@@ -477,6 +477,18 @@ impl RegimeConditionalDQN {
|
||||
self.volatile_head.reset_count_bonus();
|
||||
}
|
||||
|
||||
/// Get count bonuses for all actions (UCB exploration).
|
||||
/// Uses trending head as representative (same convention as get_epsilon).
|
||||
pub fn get_count_bonuses(&self) -> Vec<f32> {
|
||||
self.trending_head.get_count_bonuses()
|
||||
}
|
||||
|
||||
/// Get shared DQN config (all heads share the same config).
|
||||
/// Returns trending head's config as representative.
|
||||
pub fn config(&self) -> &super::DQNConfig {
|
||||
&self.trending_head.config
|
||||
}
|
||||
|
||||
/// Update epsilon for specific regime head
|
||||
pub fn update_epsilon(&mut self, regime: RegimeType) {
|
||||
match regime {
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
//! This module provides a production-ready adapter for optimizing DQN
|
||||
//! hyperparameters using the generic optimization framework. It implements:
|
||||
//!
|
||||
//! - 11D continuous parameter space with log-scale handling (9D previous + 2D PER)
|
||||
//! - 26D continuous parameter space with log-scale handling (25D previous + 1D CQL)
|
||||
//! - Training wrapper that integrates with existing DQN pipeline
|
||||
//! - Backtest-based objective using Sharpe ratio (production metric)
|
||||
//! - Prioritized Experience Replay (PER) tuning for Rainbow DQN performance (+25-40% convergence speed)
|
||||
@@ -377,6 +377,10 @@ pub struct DQNParams {
|
||||
/// Count-based exploration bonus coefficient (0.01-0.5, default 0.1)
|
||||
/// UCB-style bonus: β * sqrt(log(N) / (1 + n_a))
|
||||
pub count_bonus_coefficient: f64,
|
||||
|
||||
/// Conservative Q-Learning (CQL) regularization alpha (0.0-0.5)
|
||||
/// Penalizes overestimation of OOD actions: 0.0=disabled, 0.1=mild, 0.5=moderate
|
||||
pub cql_alpha: f64,
|
||||
}
|
||||
|
||||
impl Default for DQNParams {
|
||||
@@ -409,8 +413,8 @@ impl Default for DQNParams {
|
||||
// =============================================================================
|
||||
use_distributional: true, // ENABLED: BUG #36 fixed, full Rainbow DQN C51
|
||||
num_atoms: 51, // Wave 2.3: Rainbow DQN standard
|
||||
v_min: -2.0, // Bug #5: Corrected from -1000.0
|
||||
v_max: 2.0, // Bug #5: Corrected from 1000.0
|
||||
v_min: -10.0, // Widened for C51 action separation (45 actions need wider support)
|
||||
v_max: 10.0, // Widened for C51 action separation (45 actions need wider support)
|
||||
use_noisy_nets: true, // Wave 8: Default ENABLED for full Rainbow DQN
|
||||
noisy_sigma_init: 0.5, // Wave 2.4: Rainbow DQN standard
|
||||
minimum_profit_factor: 1.5, // BUG #7: Default 50% margin above breakeven
|
||||
@@ -442,24 +446,25 @@ impl Default for DQNParams {
|
||||
use_residual: false, // Default: disabled
|
||||
norm_type: 0.0, // Default: LayerNorm
|
||||
activation_type: 0.0, // Default: ReLU
|
||||
// QR-DQN: Enabled by default (replaces disabled C51)
|
||||
use_qr_dqn: true,
|
||||
// QR-DQN: Disabled — IQN conflicts with C51 (train-test distribution mismatch)
|
||||
use_qr_dqn: false,
|
||||
num_quantiles: 32,
|
||||
qr_kappa: 1.0,
|
||||
hidden_dim_base: 256, // Conservative default (backward compatible)
|
||||
noisy_epsilon_floor: 0.05, // Minimum random exploration with noisy nets
|
||||
count_bonus_coefficient: 0.1, // UCB exploration bonus
|
||||
cql_alpha: 0.1, // CQL regularization: mild penalty on OOD Q-values
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ParameterSpace for DQNParams {
|
||||
fn continuous_bounds() -> Vec<(f64, f64)> {
|
||||
// 25D reduced search space (down from 45D)
|
||||
// 26D search space (25D + cql_alpha)
|
||||
// 20 exotic parameters fixed to validated defaults in from_continuous()
|
||||
// This makes PSO (20 particles) and TPE dramatically more effective
|
||||
//
|
||||
// Kept: the 25 params that genuinely affect trading performance
|
||||
// Kept: the 26 params that genuinely affect trading performance
|
||||
// Fixed: ensemble/architecture/scheduling params that rarely deviate from defaults
|
||||
vec![
|
||||
// Base parameters (11D)
|
||||
@@ -476,8 +481,8 @@ impl ParameterSpace for DQNParams {
|
||||
(0.2, 0.6), // 10: per_beta_start (linear)
|
||||
|
||||
// Rainbow DQN extensions (6D)
|
||||
(-3.0, -1.0), // 11: v_min (linear) - Bug #5 fix: center -2.0
|
||||
(1.0, 3.0), // 12: v_max (linear) - Bug #5 fix: center +2.0
|
||||
(-15.0, -3.0), // 11: v_min (linear) - widened for C51 action separation
|
||||
(3.0, 15.0), // 12: v_max (linear) - widened for C51 action separation
|
||||
(0.1_f64.ln(), 1.0_f64.ln()), // 13: noisy_sigma_init (log scale)
|
||||
(128.0, 512.0), // 14: dueling_hidden_dim (linear, step=128)
|
||||
(1.0, 5.0), // 15: n_steps (linear, int)
|
||||
@@ -502,17 +507,20 @@ impl ParameterSpace for DQNParams {
|
||||
|
||||
// Exploration anti-collapse (1D)
|
||||
(0.02, 0.10), // 24: noisy_epsilon_floor (linear, minimum random exploration)
|
||||
|
||||
// CQL regularization (1D)
|
||||
(0.0, 0.5), // 25: cql_alpha (linear, 0.0=disabled, 0.1=mild, 0.5=moderate)
|
||||
]
|
||||
}
|
||||
|
||||
fn from_continuous(x: &[f64]) -> Result<Self, MLError> {
|
||||
if x.len() != 25 {
|
||||
if x.len() != 26 {
|
||||
return Err(MLError::ConfigError {
|
||||
reason: format!("Expected 25 continuous parameters (reduced from 45D), got {}", x.len()),
|
||||
reason: format!("Expected 26 continuous parameters (25D + cql_alpha), got {}", x.len()),
|
||||
});
|
||||
}
|
||||
|
||||
// === 25 TUNED parameters (from search space) ===
|
||||
// === 26 TUNED parameters (from search space) ===
|
||||
let learning_rate = x[0].exp();
|
||||
let mut batch_size = x[1].round().max(64.0) as usize; // Only enforce floor. PSO + HardwareBudget control the upper bound.
|
||||
let buffer_size = x[3].exp().round().max(50_000.0) as usize;
|
||||
@@ -525,8 +533,8 @@ impl ParameterSpace for DQNParams {
|
||||
let per_beta_start = x[10].clamp(0.2, 0.6);
|
||||
|
||||
// Rainbow DQN extensions
|
||||
let v_min = x[11].clamp(-3.0, -1.0);
|
||||
let v_max = x[12].clamp(1.0, 3.0);
|
||||
let v_min = x[11].clamp(-15.0, -3.0);
|
||||
let v_max = x[12].clamp(3.0, 15.0);
|
||||
let noisy_sigma_init = x[13].exp().clamp(0.1, 1.0);
|
||||
let dueling_hidden_dim = (x[14].round() / 128.0).round() * 128.0;
|
||||
let dueling_hidden_dim = dueling_hidden_dim.clamp(128.0, 512.0) as usize;
|
||||
@@ -554,6 +562,9 @@ impl ParameterSpace for DQNParams {
|
||||
// Exploration anti-collapse
|
||||
let noisy_epsilon_floor = x[24].clamp(0.02, 0.10);
|
||||
|
||||
// CQL regularization
|
||||
let cql_alpha = x[25].clamp(0.0, 0.5);
|
||||
|
||||
// === 20 FIXED parameters (validated defaults, removed from search) ===
|
||||
let minimum_profit_factor = 1.5;
|
||||
let kelly_min_trades: usize = 20;
|
||||
@@ -644,19 +655,20 @@ impl ParameterSpace for DQNParams {
|
||||
use_residual: false,
|
||||
norm_type,
|
||||
activation_type,
|
||||
use_qr_dqn: true,
|
||||
use_qr_dqn: false, // IQN disabled: conflicts with C51 (train-test mismatch)
|
||||
num_quantiles,
|
||||
qr_kappa,
|
||||
hidden_dim_base,
|
||||
noisy_epsilon_floor,
|
||||
count_bonus_coefficient,
|
||||
cql_alpha,
|
||||
};
|
||||
|
||||
Ok(params)
|
||||
}
|
||||
|
||||
fn to_continuous(&self) -> Vec<f64> {
|
||||
// 25D reduced space — only the tuned parameters
|
||||
// 26D search space — only the tuned parameters
|
||||
// Fixed parameters are NOT emitted (they get their defaults in from_continuous)
|
||||
vec![
|
||||
self.learning_rate.ln(), // 0
|
||||
@@ -684,11 +696,12 @@ impl ParameterSpace for DQNParams {
|
||||
self.tau.ln(), // 22
|
||||
self.hidden_dim_base as f64, // 23
|
||||
self.noisy_epsilon_floor, // 24
|
||||
self.cql_alpha, // 25
|
||||
]
|
||||
}
|
||||
|
||||
fn param_names() -> Vec<&'static str> {
|
||||
// 25 tuned parameters (matches continuous_bounds / from_continuous / to_continuous)
|
||||
// 26 tuned parameters (matches continuous_bounds / from_continuous / to_continuous)
|
||||
vec![
|
||||
"learning_rate", // 0
|
||||
"batch_size", // 1
|
||||
@@ -715,6 +728,7 @@ impl ParameterSpace for DQNParams {
|
||||
"tau", // 22
|
||||
"hidden_dim_base", // 23
|
||||
"noisy_epsilon_floor", // 24
|
||||
"cql_alpha", // 25
|
||||
]
|
||||
}
|
||||
|
||||
@@ -726,7 +740,7 @@ impl ParameterSpace for DQNParams {
|
||||
batch_bound.1 = max_batch;
|
||||
}
|
||||
}
|
||||
// Cap hidden_dim_base by VRAM (index 23 in 25D space)
|
||||
// Cap hidden_dim_base by VRAM (index 23 in 26D space)
|
||||
let max_base = budget.max_hidden_dim_base(4, 256, 54, 45);
|
||||
if let Some(dim_bound) = bounds.get_mut(23) {
|
||||
dim_bound.1 = max_base as f64;
|
||||
@@ -2411,6 +2425,10 @@ impl HyperparameterOptimizable for DQNTrainer {
|
||||
num_quantiles: params.num_quantiles,
|
||||
qr_kappa: params.qr_kappa,
|
||||
|
||||
// Conservative Q-Learning (CQL) — wired from hyperopt search space
|
||||
use_cql: true,
|
||||
cql_alpha: params.cql_alpha,
|
||||
|
||||
// Phase 3: GPU experience collection (defaults, not in search space)
|
||||
gpu_n_episodes: 128,
|
||||
gpu_timesteps_per_episode: 500,
|
||||
@@ -3246,8 +3264,8 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_dqn_params_roundtrip() {
|
||||
// Roundtrip test for the 25D reduced search space
|
||||
// Only the 25 tuned parameters roundtrip; the 20 fixed params get defaults from from_continuous
|
||||
// Roundtrip test for the 26D search space
|
||||
// Only the 26 tuned parameters roundtrip; the 20 fixed params get defaults from from_continuous
|
||||
let params = DQNParams {
|
||||
learning_rate: 3.37e-05,
|
||||
batch_size: 92,
|
||||
@@ -3267,8 +3285,8 @@ mod tests {
|
||||
tau: 0.001,
|
||||
use_distributional: true,
|
||||
num_atoms: 51,
|
||||
v_min: -2.0,
|
||||
v_max: 2.0,
|
||||
v_min: -5.0,
|
||||
v_max: 5.0,
|
||||
use_noisy_nets: true,
|
||||
noisy_sigma_init: 0.5,
|
||||
// Fixed params — these will be overwritten by from_continuous defaults
|
||||
@@ -3301,10 +3319,11 @@ mod tests {
|
||||
hidden_dim_base: 512,
|
||||
noisy_epsilon_floor: 0.05,
|
||||
count_bonus_coefficient: 0.1,
|
||||
cql_alpha: 0.15,
|
||||
};
|
||||
|
||||
let continuous = params.to_continuous();
|
||||
assert_eq!(continuous.len(), 25, "to_continuous must return 25D vector");
|
||||
assert_eq!(continuous.len(), 26, "to_continuous must return 26D vector");
|
||||
let recovered = DQNParams::from_continuous(&continuous).unwrap();
|
||||
|
||||
// Tuned parameters must roundtrip exactly
|
||||
@@ -3324,6 +3343,7 @@ mod tests {
|
||||
assert!((recovered.tau - params.tau).abs() < 1e-6);
|
||||
assert_eq!(recovered.hidden_dim_base, params.hidden_dim_base);
|
||||
assert!((recovered.noisy_epsilon_floor - params.noisy_epsilon_floor).abs() < 1e-6);
|
||||
assert!((recovered.cql_alpha - params.cql_alpha).abs() < 1e-6);
|
||||
|
||||
// Fixed parameters must get their validated defaults
|
||||
assert!((recovered.minimum_profit_factor - 1.5).abs() < 1e-6);
|
||||
@@ -3343,7 +3363,7 @@ mod tests {
|
||||
#[test]
|
||||
fn test_dqn_params_bounds() {
|
||||
let bounds = DQNParams::continuous_bounds();
|
||||
assert_eq!(bounds.len(), 25); // 25D reduced search space (down from 45)
|
||||
assert_eq!(bounds.len(), 26); // 26D search space (25D + cql_alpha)
|
||||
|
||||
// Check log-scale bounds are reasonable
|
||||
assert!(bounds[0].0 < bounds[0].1); // learning_rate
|
||||
@@ -3369,8 +3389,8 @@ mod tests {
|
||||
assert_eq!(bounds[8], (0.5, 2.0)); // transaction_cost_multiplier
|
||||
assert_eq!(bounds[9], (0.4, 0.8)); // per_alpha
|
||||
assert_eq!(bounds[10], (0.2, 0.6)); // per_beta_start
|
||||
assert_eq!(bounds[11], (-3.0, -1.0)); // v_min
|
||||
assert_eq!(bounds[12], (1.0, 3.0)); // v_max
|
||||
assert_eq!(bounds[11], (-15.0, -3.0)); // v_min (widened for C51 action separation)
|
||||
assert_eq!(bounds[12], (3.0, 15.0)); // v_max (widened for C51 action separation)
|
||||
assert_eq!(bounds[14], (128.0, 512.0)); // dueling_hidden_dim
|
||||
assert_eq!(bounds[15], (1.0, 5.0)); // n_steps
|
||||
assert_eq!(bounds[16], (51.0, 201.0)); // num_atoms
|
||||
@@ -3384,12 +3404,15 @@ mod tests {
|
||||
assert_eq!(bounds[21], (0.01, 0.5)); // curiosity_weight
|
||||
assert_eq!(bounds[23], (256.0, 4096.0)); // hidden_dim_base
|
||||
assert_eq!(bounds[24], (0.02, 0.10)); // noisy_epsilon_floor
|
||||
|
||||
// CQL regularization
|
||||
assert_eq!(bounds[25], (0.0, 0.5)); // cql_alpha
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_param_names() {
|
||||
let names = DQNParams::param_names();
|
||||
assert_eq!(names.len(), 25); // 25D reduced search space
|
||||
assert_eq!(names.len(), 26); // 26D search space
|
||||
assert_eq!(names[0], "learning_rate");
|
||||
assert_eq!(names[1], "batch_size");
|
||||
assert_eq!(names[2], "gamma");
|
||||
@@ -3415,16 +3438,17 @@ mod tests {
|
||||
assert_eq!(names[22], "tau");
|
||||
assert_eq!(names[23], "hidden_dim_base");
|
||||
assert_eq!(names[24], "noisy_epsilon_floor");
|
||||
assert_eq!(names[25], "cql_alpha");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_per_params_always_enabled() {
|
||||
// Test that PER is always enabled with tunable alpha/beta parameters
|
||||
// 25D reduced search space
|
||||
// 26D search space
|
||||
let continuous = vec![
|
||||
3e-5_f64.ln(), 92.0, 0.9588, 97_273_f64.ln(), 1.404, 4.0, 24.77_f64.ln(), 0.01, 1.0,
|
||||
0.6, 0.4, // 9-10: per_alpha, per_beta_start
|
||||
-2.0, 2.0, 0.5_f64.ln(), // 11-13: v_min, v_max, noisy_sigma_init
|
||||
-5.0, 5.0, 0.5_f64.ln(), // 11-13: v_min, v_max, noisy_sigma_init
|
||||
256.0, 3.0, 101.0, // 14-16: dueling_hidden_dim, n_steps, num_atoms
|
||||
1e-4_f64.ln(), // 17: weight_decay
|
||||
0.5, 0.25, // 18-19: kelly_fractional, kelly_max_fraction
|
||||
@@ -3433,6 +3457,7 @@ mod tests {
|
||||
0.005_f64.ln(), // 22: tau
|
||||
512.0, // 23: hidden_dim_base
|
||||
0.05, // 24: noisy_epsilon_floor
|
||||
0.1, // 25: cql_alpha
|
||||
];
|
||||
|
||||
let params = DQNParams::from_continuous(&continuous).unwrap();
|
||||
@@ -3443,11 +3468,11 @@ mod tests {
|
||||
assert!(params.use_distributional);
|
||||
assert!(params.use_noisy_nets);
|
||||
|
||||
// Test PER parameter bounds (min values) — 25D
|
||||
// Test PER parameter bounds (min values) — 26D
|
||||
let continuous_min = vec![
|
||||
2e-5_f64.ln(), 64.0, 0.95, 50_000_f64.ln(), 1.0, 1.0, 10.0_f64.ln(), 0.0, 0.5,
|
||||
0.4, 0.2, // per_alpha min, per_beta_start min
|
||||
-3.0, 1.0, 0.1_f64.ln(), // v_min, v_max, noisy_sigma_init
|
||||
-15.0, 3.0, 0.1_f64.ln(), // v_min, v_max, noisy_sigma_init
|
||||
128.0, 1.0, 51.0, // dueling_hidden_dim, n_steps, num_atoms
|
||||
1e-5_f64.ln(), // weight_decay min
|
||||
0.25, 0.1, // kelly_fractional, kelly_max_fraction
|
||||
@@ -3456,6 +3481,7 @@ mod tests {
|
||||
0.0001_f64.ln(), // tau min
|
||||
256.0, // hidden_dim_base min
|
||||
0.02, // noisy_epsilon_floor min
|
||||
0.0, // cql_alpha min
|
||||
];
|
||||
let params_min = DQNParams::from_continuous(&continuous_min).unwrap();
|
||||
assert!((params_min.per_alpha - 0.4).abs() < 1e-6);
|
||||
@@ -3464,11 +3490,11 @@ mod tests {
|
||||
assert!(params_min.use_distributional);
|
||||
assert!(params_min.use_noisy_nets);
|
||||
|
||||
// Test PER parameter bounds (max values) — 25D
|
||||
// Test PER parameter bounds (max values) — 26D
|
||||
let continuous_max = vec![
|
||||
8e-5_f64.ln(), 4096.0, 0.99, 100_000_f64.ln(), 2.0, 4.0, 40.0_f64.ln(), 0.1, 2.0,
|
||||
0.8, 0.6, // per_alpha max, per_beta_start max
|
||||
-1.0, 3.0, 1.0_f64.ln(), // v_min, v_max, noisy_sigma_init
|
||||
-3.0, 15.0, 1.0_f64.ln(), // v_min, v_max, noisy_sigma_init
|
||||
512.0, 5.0, 201.0, // dueling_hidden_dim, n_steps, num_atoms
|
||||
1e-3_f64.ln(), // weight_decay max
|
||||
1.0, 0.5, // kelly_fractional, kelly_max_fraction
|
||||
@@ -3477,6 +3503,7 @@ mod tests {
|
||||
0.01_f64.ln(), // tau max
|
||||
4096.0, // hidden_dim_base max
|
||||
0.10, // noisy_epsilon_floor max
|
||||
0.5, // cql_alpha max
|
||||
];
|
||||
let params_max = DQNParams::from_continuous(&continuous_max).unwrap();
|
||||
assert!((params_max.per_alpha - 0.8).abs() < 1e-6);
|
||||
@@ -3767,7 +3794,7 @@ mod tests {
|
||||
#[test]
|
||||
fn test_qr_dqn_params_in_default() {
|
||||
let params = DQNParams::default();
|
||||
assert!(params.use_qr_dqn, "QR-DQN should be enabled by default");
|
||||
assert!(!params.use_qr_dqn, "QR-DQN (IQN) should be disabled by default (conflicts with C51)");
|
||||
assert_eq!(params.num_quantiles, 32, "Default num_quantiles should be 32");
|
||||
assert!((params.qr_kappa - 1.0).abs() < f64::EPSILON, "Default kappa should be 1.0");
|
||||
}
|
||||
@@ -3776,7 +3803,7 @@ mod tests {
|
||||
fn test_qr_dqn_roundtrip_continuous() {
|
||||
let params = DQNParams::default();
|
||||
let continuous = params.to_continuous();
|
||||
assert_eq!(continuous.len(), 25, "Should have 25 continuous dimensions (reduced from 45D)");
|
||||
assert_eq!(continuous.len(), 26, "Should have 26 continuous dimensions");
|
||||
let roundtrip = DQNParams::from_continuous(&continuous).unwrap();
|
||||
// num_quantiles and qr_kappa are now fixed defaults (not in search space)
|
||||
assert_eq!(roundtrip.num_quantiles, 64); // Fixed default
|
||||
@@ -3800,7 +3827,7 @@ mod tests {
|
||||
#[test]
|
||||
fn test_batch_size_respects_wide_bounds() {
|
||||
// Simulate PSO choosing batch_size=2048 (within VRAM-aware bounds)
|
||||
let mut params = vec![0.0_f64; 25];
|
||||
let mut params = vec![0.0_f64; 26];
|
||||
params[1] = 2048.0; // batch_size (index 1)
|
||||
// Fill other required params with valid defaults
|
||||
params[0] = (1e-4_f64).ln(); // learning_rate
|
||||
@@ -3813,9 +3840,9 @@ mod tests {
|
||||
params[8] = 0.5; // transaction_cost_multiplier
|
||||
params[9] = 0.6; // per_alpha
|
||||
params[10] = 0.4; // per_beta_start
|
||||
// Remaining params (indices 11-24) -- use midpoint of bounds
|
||||
// Remaining params (indices 11-25) -- use midpoint of bounds
|
||||
let bounds = DQNParams::continuous_bounds();
|
||||
for i in 11..25 {
|
||||
for i in 11..26 {
|
||||
params[i] = (bounds[i].0 + bounds[i].1) / 2.0;
|
||||
}
|
||||
let result = DQNParams::from_continuous(¶ms).unwrap();
|
||||
|
||||
Reference in New Issue
Block a user