gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: hyperopt 31D→38D — 7 composite reward weight dimensions

Browse Source 
Expand the DQN hyperopt parameter space from 31D to 38D by adding 7 GPU
composite reward weights (w_dsr, w_pnl, w_dd, w_idle, dd_threshold,
loss_aversion, time_decay_rate) at indices 31-37.

- Remove hold_penalty_weight from DQNParams (replaced by w_idle)
- Add #[serde(default)] for backward compat with old JSON results
- Phase Fast fixes reward weights to defaults (not searched)
- Wire reward weights from DQNParams → DQNHyperparameters in train_with_params
- Fix all tests: 134 hyperopt + 7 ensemble + 5 JSON export tests pass
- Fix stale 40D ensemble tests → 38D layout (were already broken pre-change)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-22 21:13:31 +01:00
parent a2fde27d09
commit ad5423e016
4 changed files with 274 additions and 271 deletions
Download Patch File Download Diff File Expand all files Collapse all files

248
crates/ml/src/hyperopt/adapters/dqn.rs
View File

@@ -3,7 +3,7 @@
//! This module provides a production-ready adapter for optimizing DQN
//! hyperparameters using the generic optimization framework. It implements:
//!
//! - 31D continuous parameter space with log-scale handling
//! - 38D continuous parameter space with log-scale handling
//! - Training wrapper that integrates with existing DQN pipeline
//! - Backtest-based objective using Sharpe ratio (production metric)
//! - Prioritized Experience Replay (PER) tuning for Rainbow DQN performance (+25-40% convergence speed)
@@ -144,7 +144,7 @@ const TRIAL_VRAM_MB: f64 = 7000.0;
/// Corrected: was 0.02 (20KB) — 100x too high
const MB_PER_SAMPLE: f64 = 0.0005;
/// DQN hyperparameter space (31D continuous)
/// DQN hyperparameter space (38D continuous)
///
/// Fixed params (not in search space):
/// - curiosity_weight: 0.0 (conflicts with NoisyNet)
@@ -156,13 +156,23 @@ const MB_PER_SAMPLE: f64 = 0.0005;
/// - count_bonus_coefficient: (0.0, 0.3) — UCB exploration bonus, complementary to NoisyNet
/// - sharpe_weight: (0.0, 0.5) — tunable Sharpe ratio blending in composite reward
///
/// Composite reward weights (indices 31-37):
/// - w_dsr: DSR (Sharpe) weight
/// - w_pnl: Normalized PnL weight
/// - w_dd: Drawdown penalty weight
/// - w_idle: Idle penalty weight (replaces hold_penalty_weight)
/// - dd_threshold: Drawdown tolerance before penalty
/// - loss_aversion: Asymmetric loss scaling (prospect theory)
/// - time_decay_rate: Position staleness rent per step
///
/// ## Parameter Scaling
///
/// - **Log-scale**: Learning rate, buffer_size, huber_delta, noisy_sigma_init, tau, weight_decay
/// - **Linear scale**: Batch size, gamma, max_position, entropy, tx_cost, per_alpha, per_beta, v_min, v_max, dueling_hidden_dim, n_steps, num_atoms
/// - **Linear scale**: Batch size, gamma, max_position, entropy, tx_cost, per_alpha, per_beta, v_min, v_max, dueling_hidden_dim, n_steps, num_atoms, reward weights
///
/// This scaling ensures efficient exploration by egobox's Bayesian optimization.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(default)]
pub struct DQNParams {
/// Learning rate for Adam optimizer (log-scale)
pub learning_rate: f64,
@@ -172,8 +182,6 @@ pub struct DQNParams {
pub gamma: f64,
/// Replay buffer capacity (log-scale)
pub buffer_size: usize,
/// HOLD penalty weight (linear scale: 1.0 - 2.0 for HFT active trading) - OPTIMIZED from [0.5, 5.0]
pub hold_penalty_weight: f64,
// movement_threshold removed - now fixed at 0.02 (2%) to align with production
/// Maximum absolute position size for action masking (4.0-8.0 contracts) - OPTIMIZED from [1.0, 10.0]
@@ -375,6 +383,22 @@ pub struct DQNParams {
/// Only in search space for large GPUs (≥40GB): effective_batch = batch_size × accum_steps.
/// H100 with batch=1024, accum=4 → effective batch 4096 (more stable gradients).
pub gradient_accumulation_steps: usize,
// === Composite Reward Weights (indices 31-37 in search space) ===
/// DSR (Sharpe) weight in composite reward (0.1-2.0)
pub w_dsr: f64,
/// Normalized PnL weight in composite reward (0.0-1.0)
pub w_pnl: f64,
/// Drawdown penalty weight in composite reward (0.0-5.0)
pub w_dd: f64,
/// Idle penalty weight — replaces hold_penalty_weight (0.0-0.1)
pub w_idle: f64,
/// Drawdown tolerance before penalty kicks in (0.01-0.10)
pub dd_threshold: f64,
/// Asymmetric loss scaling factor — prospect theory (1.0-3.0)
pub loss_aversion: f64,
/// Position staleness rent per step (0.0001-0.005)
pub time_decay_rate: f64,
}
impl Default for DQNParams {
@@ -384,7 +408,6 @@ impl Default for DQNParams {
batch_size: 128,
gamma: 0.95,
buffer_size: 100_000,
hold_penalty_weight: 0.0, // C3: Neutral hold reward — not in search space
max_position_absolute: 2.0, // BLOCKER #2: Default matches production (±2.0)
huber_delta: 10.0, // Conservative default (hyperopt can scale to 15-40)
entropy_coefficient: 0.02, // C2: SAC-style regularization only (narrowed from 0.1)
@@ -455,13 +478,21 @@ impl Default for DQNParams {
branch_hidden_dim: 128, // Default: 128 hidden units per branching head
use_branching: true, // Default: branching enabled (3 heads)
gradient_accumulation_steps: 1, // Default: no accumulation
// Composite reward weights (defaults match dqn-production.toml)
w_dsr: 1.0,
w_pnl: 0.3,
w_dd: 1.0,
w_idle: 0.01,
dd_threshold: 0.02,
loss_aversion: 1.5,
time_decay_rate: 0.0005,
}
}
}
impl ParameterSpace for DQNParams {
fn continuous_bounds() -> Vec<(f64, f64)> {
// 31D search space (C7: added iqn_lambda)
// 38D search space (C8: added 7 composite reward weights at indices 31-37)
// All bounds loaded from config/training/dqn-hyperopt.toml [search_space].
// To change search ranges, edit the TOML — no code changes needed.
// Log-scale transforms are applied here; TOML stores linear values.
@@ -499,6 +530,15 @@ impl ParameterSpace for DQNParams {
let ga = b("gradient_accumulation_steps", (1.0, 1.0));
let iq = b("iqn_lambda", (0.0, 2.0));
// Composite reward weights (indices 31-37)
let mut rw_dsr = b("w_dsr", (0.1, 2.0));
let mut rw_pnl = b("w_pnl", (0.0, 1.0));
let mut rw_dd = b("w_dd", (0.0, 5.0));
let mut rw_idle = b("w_idle", (0.0, 0.1));
let mut rw_ddt = b("dd_threshold", (0.01, 0.10));
let mut rw_la = b("loss_aversion", (1.0, 3.0));
let mut rw_tdr = b("time_decay_rate", (0.0001, 0.005));
// Two-phase hyperopt: fix architecture OR dynamics params to single-point bounds.
// Single-point bounds (v, v) make the optimizer trivially converge on those
// dimensions, focusing search power on the free dimensions.
@@ -508,7 +548,7 @@ impl ParameterSpace for DQNParams {
match phase {
HyperoptPhase::Fast => {
// Phase 1: fix architecture to small network from [phase_fast] TOML.
// Search only learning dynamics (~15D free, ~5D fixed).
// Also fix reward weights to defaults (search learning dynamics only).
// All values MUST exist in [phase_fast] — no silent defaults.
let pf = hp.phase_fast.as_ref()
.expect("[phase_fast] section missing from dqn-hyperopt.toml — required for --phase fast");
@@ -529,14 +569,38 @@ impl ParameterSpace for DQNParams {
dh = (fix_dh, fix_dh);
vr = (fix_vr, fix_vr);
// Fix reward weights to defaults in Phase Fast (not searched)
let fix_w_dsr = pf.w_dsr
.expect("phase_fast.w_dsr missing from dqn-hyperopt.toml");
let fix_w_pnl = pf.w_pnl
.expect("phase_fast.w_pnl missing from dqn-hyperopt.toml");
let fix_w_dd = pf.w_dd
.expect("phase_fast.w_dd missing from dqn-hyperopt.toml");
let fix_w_idle = pf.w_idle
.expect("phase_fast.w_idle missing from dqn-hyperopt.toml");
let fix_ddt = pf.dd_threshold
.expect("phase_fast.dd_threshold missing from dqn-hyperopt.toml");
let fix_la = pf.loss_aversion
.expect("phase_fast.loss_aversion missing from dqn-hyperopt.toml");
let fix_tdr = pf.time_decay_rate
.expect("phase_fast.time_decay_rate missing from dqn-hyperopt.toml");
rw_dsr = (fix_w_dsr, fix_w_dsr);
rw_pnl = (fix_w_pnl, fix_w_pnl);
rw_dd = (fix_w_dd, fix_w_dd);
rw_idle = (fix_w_idle, fix_w_idle);
rw_ddt = (fix_ddt, fix_ddt);
rw_la = (fix_la, fix_la);
rw_tdr = (fix_tdr, fix_tdr);
tracing::info!(
"Phase FAST: fixed architecture (hidden_dim={}, num_atoms={}, branch_hidden={}, dueling_hidden={}, v_range={})",
"Phase FAST: fixed architecture (hidden_dim={}, num_atoms={}, branch_hidden={}, dueling_hidden={}, v_range={}) + reward weights",
fix_hdim, fix_na, fix_bh, fix_dh, fix_vr
);
}
HyperoptPhase::Full(ref best_vec) => {
// Phase 2: fix learning dynamics from Phase 1 best params.
// Search only architecture dims (~5D free, ~26D fixed).
// Search only architecture dims (~5D free, ~33D fixed).
// The best_vec is the raw continuous vector from Phase 1.
// Fix all non-architecture dims to Phase 1's best values.
// Architecture dims (indices 10, 13, 15, 21, 28) stay free.
@@ -577,6 +641,14 @@ impl ParameterSpace for DQNParams {
bh, // 28: branch_hidden_dim
ga, // 29: gradient_accumulation_steps
iq, // 30: iqn_lambda
// Composite reward weights (indices 31-37)
rw_dsr, // 31: w_dsr
rw_pnl, // 32: w_pnl
rw_dd, // 33: w_dd
rw_idle, // 34: w_idle
rw_ddt, // 35: dd_threshold
rw_la, // 36: loss_aversion
rw_tdr, // 37: time_decay_rate
];
// Phase FULL: fix all non-architecture dims to Phase 1 best values.
@@ -595,8 +667,8 @@ impl ParameterSpace for DQNParams {
}
fn from_continuous(x: &[f64]) -> Result<Self, MLError> {
if x.len() != 31 {
return Err(MLError::ConfigError(format!("Expected 31 continuous parameters, got {}", x.len())));
if x.len() != 38 {
return Err(MLError::ConfigError(format!("Expected 38 continuous parameters, got {}", x.len())));
}
// === 26 TUNED parameters (from search space) ===
@@ -691,6 +763,15 @@ impl ParameterSpace for DQNParams {
// C7: IQN dual-head lambda (idx 30)
let iqn_lambda = x[30].clamp(0.0, 2.0);
// C8: Composite reward weights (indices 31-37)
let w_dsr = x[31].clamp(0.1, 2.0);
let w_pnl = x[32].clamp(0.0, 1.0);
let w_dd = x[33].clamp(0.0, 5.0);
let w_idle = x[34].clamp(0.0, 0.1);
let dd_threshold = x[35].clamp(0.01, 0.10);
let loss_aversion = x[36].clamp(1.0, 3.0);
let time_decay_rate = x[37].clamp(0.0001, 0.005);
// WAVE 6 FIX #2: Batch size floor for high learning rates
if learning_rate > 2e-4 && batch_size < 120 {
tracing::info!(
@@ -706,7 +787,6 @@ impl ParameterSpace for DQNParams {
batch_size,
gamma: x[2].clamp(0.88, 0.99),
buffer_size,
hold_penalty_weight: 0.0, // C3: Neutral hold reward — not in search space
max_position_absolute,
huber_delta,
entropy_coefficient,
@@ -762,13 +842,21 @@ impl ParameterSpace for DQNParams {
branch_hidden_dim,
use_branching,
gradient_accumulation_steps,
// C8: Composite reward weights (indices 31-37)
w_dsr,
w_pnl,
w_dd,
w_idle,
dd_threshold,
loss_aversion,
time_decay_rate,
};
Ok(params)
}
fn to_continuous(&self) -> Vec<f64> {
// 31D search space (C7: added iqn_lambda)
// 38D search space (C8: added 7 composite reward weights at indices 31-37)
vec![
self.learning_rate.ln(), // 0
self.batch_size as f64, // 1
@@ -801,11 +889,19 @@ impl ParameterSpace for DQNParams {
self.branch_hidden_dim as f64, // 28: C5 branch_hidden_dim
self.gradient_accumulation_steps as f64, // 29: C6 gradient accumulation
self.iqn_lambda, // 30: C7 IQN dual-head loss weight
// C8: Composite reward weights (indices 31-37)
self.w_dsr, // 31
self.w_pnl, // 32
self.w_dd, // 33
self.w_idle, // 34
self.dd_threshold, // 35
self.loss_aversion, // 36
self.time_decay_rate, // 37
]
}
fn param_names() -> Vec<&'static str> {
// 31 tuned parameters (C7: added iqn_lambda)
// 38 tuned parameters (C8: added 7 composite reward weights)
vec![
"learning_rate", // 0
"batch_size", // 1
@@ -838,6 +934,14 @@ impl ParameterSpace for DQNParams {
"branch_hidden_dim", // 28: C5 branching head capacity
"gradient_accumulation_steps", // 29: C6 effective batch multiplier
"iqn_lambda", // 30: C7 IQN dual-head loss weight
// C8: Composite reward weights
"w_dsr", // 31
"w_pnl", // 32
"w_dd", // 33
"w_idle", // 34
"dd_threshold", // 35
"loss_aversion", // 36
"time_decay_rate", // 37
]
}
@@ -2425,7 +2529,7 @@ impl HyperparameterOptimizable for DQNTrainer {
// Fix 1: Clamp buffer size to max (4GB GPU constraint)
let clamped_buffer_size = params.buffer_size.min(self.buffer_size_max);
info!("Training DQN with parameters (Wave 6.4: 31D search space):");
info!("Training DQN with parameters (C8: 38D search space):");
info!(" Learning rate: {:.6}", params.learning_rate);
info!(" Batch size: {}", params.batch_size);
info!(" Gamma: {:.3}", params.gamma);
@@ -2433,7 +2537,6 @@ impl HyperparameterOptimizable for DQNTrainer {
" Buffer size: {} (requested: {})",
clamped_buffer_size, params.buffer_size
);
info!(" Hold penalty weight: {:.3}", params.hold_penalty_weight);
info!(" Max position: {:.1}", params.max_position_absolute);
info!(" Huber delta: {:.3}", params.huber_delta);
info!(" Entropy coeff: {:.4}", params.entropy_coefficient);
@@ -2626,7 +2729,7 @@ impl HyperparameterOptimizable for DQNTrainer {
huber_delta: params.huber_delta, // WAVE 17: Use hyperopt value instead of fixed 1.0
use_double_dqn: true, // Production feature: --use-double-dqn
gradient_clip_norm: Some(gradient_clip_norm), // WAVE 64: Dynamic clipping (5.0 for high LR, 10.0 for low LR)
hold_penalty_weight: params.hold_penalty_weight, // Use optimized value from search
hold_penalty_weight: params.w_idle, // Deprecated: mapped from w_idle for backward compat
movement_threshold: 0.02, // Aligned with production (fixed at 2%, train_dqn.rs:296)
// WAVE 16 (Agent 38): Use stored configuration values (allow CLI overrides)
enable_preprocessing: self.enable_preprocessing,
@@ -2718,7 +2821,7 @@ impl HyperparameterOptimizable for DQNTrainer {
gradient_collapse_multiplier: 100.0, // Adaptive threshold (LR × 100)
gradient_collapse_patience: 5, // 5 consecutive epochs before early stop
// LR scheduling — tunable via hyperopt search space (idx 23 in 31D)
// LR scheduling — tunable via hyperopt search space (idx 23 in 38D)
// 0=constant, 1=linear decay, 2=cosine annealing
// BUG FIX: total_steps = self.epochs (scheduler steps once per epoch,
// NOT per training step). Previous calc used epochs × steps_per_epoch
@@ -2853,6 +2956,15 @@ impl HyperparameterOptimizable for DQNTrainer {
wf_val_fraction: 0.125,
wf_test_fraction: 0.125,
wf_step_fraction: 0.125,
// C8: Composite reward weights — wired from hyperopt search space (indices 31-37)
w_dsr: params.w_dsr,
w_pnl: params.w_pnl,
w_dd: params.w_dd,
w_idle: params.w_idle,
dd_threshold: params.dd_threshold,
loss_aversion: params.loss_aversion,
time_decay_rate: params.time_decay_rate,
..Default::default()
};
@@ -3603,14 +3715,13 @@ mod tests {
#[test]
fn test_dqn_params_roundtrip() {
// Roundtrip test for the 26D search space (C2: removed curiosity_weight + noisy_epsilon_floor)
// Only the 26 tuned parameters roundtrip; fixed params get defaults from from_continuous
// Roundtrip test for the 38D search space (C8: 7 composite reward weights added)
// Only the tuned parameters roundtrip; fixed params get defaults from from_continuous
let params = DQNParams {
learning_rate: 3.37e-05,
batch_size: 92,
gamma: 0.92,
buffer_size: 97_273,
hold_penalty_weight: 1.404,
max_position_absolute: 2.5,
huber_delta: 24.77,
entropy_coefficient: 0.03, // C2: narrowed range [0.005, 0.05]
@@ -3664,18 +3775,25 @@ mod tests {
branch_hidden_dim: 128,
use_branching: true,
gradient_accumulation_steps: 4,
// C8: Composite reward weights
w_dsr: 1.5,
w_pnl: 0.5,
w_dd: 2.0,
w_idle: 0.05,
dd_threshold: 0.04,
loss_aversion: 2.0,
time_decay_rate: 0.001,
};
let continuous = params.to_continuous();
assert_eq!(continuous.len(), 31, "to_continuous must return 31D vector");
assert_eq!(continuous.len(), 38, "to_continuous must return 38D vector");
let recovered = DQNParams::from_continuous(&continuous).unwrap();
// Tuned parameters must roundtrip exactly (26D)
// Tuned parameters must roundtrip exactly
assert!((recovered.learning_rate - params.learning_rate).abs() < 1e-6);
assert_eq!(recovered.batch_size, params.batch_size);
assert!((recovered.gamma - params.gamma).abs() < 1e-6);
assert_eq!(recovered.buffer_size, params.buffer_size);
assert!((recovered.hold_penalty_weight - 0.0).abs() < 1e-6); // C3: always 0.0
assert!((recovered.max_position_absolute - params.max_position_absolute).abs() < 1e-6);
assert!((recovered.per_alpha - params.per_alpha).abs() < 1e-6);
assert!((recovered.per_beta_start - params.per_beta_start).abs() < 1e-6);
@@ -3710,12 +3828,21 @@ mod tests {
assert_eq!(recovered.use_branching, params.use_branching);
// C6: gradient_accumulation_steps roundtrips via index 29
assert_eq!(recovered.gradient_accumulation_steps, params.gradient_accumulation_steps);
// C8: Composite reward weights roundtrip (indices 31-37)
assert!((recovered.w_dsr - params.w_dsr).abs() < 1e-6);
assert!((recovered.w_pnl - params.w_pnl).abs() < 1e-6);
assert!((recovered.w_dd - params.w_dd).abs() < 1e-6);
assert!((recovered.w_idle - params.w_idle).abs() < 1e-6);
assert!((recovered.dd_threshold - params.dd_threshold).abs() < 1e-6);
assert!((recovered.loss_aversion - params.loss_aversion).abs() < 1e-6);
assert!((recovered.time_decay_rate - params.time_decay_rate).abs() < 1e-6);
}
#[test]
fn test_dqn_params_bounds() {
let bounds = DQNParams::continuous_bounds();
assert_eq!(bounds.len(), 31); // C7: 31D (added iqn_lambda)
assert_eq!(bounds.len(), 38); // C8: 38D (added 7 reward weights)
// Check log-scale bounds are reasonable
assert!(bounds[0].0 < bounds[0].1); // learning_rate
@@ -3760,12 +3887,21 @@ mod tests {
assert_eq!(bounds[28], (64.0, 64.0)); // branch_hidden_dim (FIXED in Fast phase)
// C6: gradient_accumulation_steps (fixed at 1 for default GPUs)
assert_eq!(bounds[29], (1.0, 1.0)); // gradient_accumulation_steps
// C8: Composite reward weights (FIXED in Fast phase to defaults)
assert_eq!(bounds[31], (1.0, 1.0)); // w_dsr (FIXED in Fast phase)
assert_eq!(bounds[32], (0.3, 0.3)); // w_pnl (FIXED in Fast phase)
assert_eq!(bounds[33], (1.0, 1.0)); // w_dd (FIXED in Fast phase)
assert_eq!(bounds[34], (0.01, 0.01)); // w_idle (FIXED in Fast phase)
assert_eq!(bounds[35], (0.02, 0.02)); // dd_threshold (FIXED in Fast phase)
assert_eq!(bounds[36], (1.5, 1.5)); // loss_aversion (FIXED in Fast phase)
assert_eq!(bounds[37], (0.0005, 0.0005)); // time_decay_rate (FIXED in Fast phase)
}
#[test]
fn test_param_names() {
let names = DQNParams::param_names();
assert_eq!(names.len(), 31); // C7: 31D (added iqn_lambda)
assert_eq!(names.len(), 38); // C8: 38D (added 7 reward weights)
assert_eq!(names[0], "learning_rate");
assert_eq!(names[1], "batch_size");
assert_eq!(names[2], "gamma");
@@ -3801,7 +3937,7 @@ mod tests {
#[test]
fn test_per_params_always_enabled() {
// Test that PER is always enabled with tunable alpha/beta parameters
// 31D search space (C7: added iqn_lambda)
// 38D search space (C8: added 7 composite reward weights)
let continuous = vec![
3e-5_f64.ln(), 92.0, 0.92, 97_273_f64.ln(), 4.0, 24.77_f64.ln(), 0.03, 1.0,
0.6, 0.4, // 8-9: per_alpha, per_beta_start
@@ -3821,6 +3957,14 @@ mod tests {
128.0, // 28: branch_hidden_dim (C5)
1.0, // 29: gradient_accumulation_steps (C6)
0.25, // 30: iqn_lambda (C7)
// C8: Composite reward weights (indices 31-37)
1.0, // 31: w_dsr
0.3, // 32: w_pnl
1.0, // 33: w_dd
0.01, // 34: w_idle
0.02, // 35: dd_threshold
1.5, // 36: loss_aversion
0.0005, // 37: time_decay_rate
];
let params = DQNParams::from_continuous(&continuous).unwrap();
@@ -3837,7 +3981,7 @@ mod tests {
assert!((params.v_min - (-25.0)).abs() < 1e-6, "v_min should be -v_range");
assert!((params.v_max - 25.0).abs() < 1e-6, "v_max should be +v_range");
// Test PER parameter bounds (min values) -- 31D
// Test PER parameter bounds (min values) -- 38D
let continuous_min = vec![
2e-5_f64.ln(), 64.0, 0.88, 50_000_f64.ln(), 1.0, 10.0_f64.ln(), 0.05, 0.5,
0.4, 0.2, // per_alpha min, per_beta_start min
@@ -3857,6 +4001,14 @@ mod tests {
64.0, // 28: branch_hidden_dim min (C5)
1.0, // 29: gradient_accumulation_steps min (C6)
0.0, // 30: iqn_lambda min (C7)
// C8: Composite reward weights min
0.1, // 31: w_dsr min
0.0, // 32: w_pnl min
0.0, // 33: w_dd min
0.0, // 34: w_idle min
0.01, // 35: dd_threshold min
1.0, // 36: loss_aversion min
0.0001, // 37: time_decay_rate min
];
let params_min = DQNParams::from_continuous(&continuous_min).unwrap();
assert!((params_min.per_alpha - 0.4).abs() < 1e-6);
@@ -3867,7 +4019,7 @@ mod tests {
assert!((params_min.v_min - (-10.0)).abs() < 1e-6, "v_min should be -v_range at min");
assert!((params_min.v_max - 10.0).abs() < 1e-6, "v_max should be +v_range at min");
// Test PER parameter bounds (max values) -- 31D
// Test PER parameter bounds (max values) -- 38D
let continuous_max = vec![
8e-5_f64.ln(), 512.0, 0.99, 100_000_f64.ln(), 4.0, 40.0_f64.ln(), 0.5, 2.0,
0.8, 0.6, // per_alpha max, per_beta_start max
@@ -3887,6 +4039,14 @@ mod tests {
256.0, // 28: branch_hidden_dim max (C5)
1.0, // 29: gradient_accumulation_steps max (C6)
2.0, // 30: iqn_lambda max (C7)
// C8: Composite reward weights max
2.0, // 31: w_dsr max
1.0, // 32: w_pnl max
5.0, // 33: w_dd max
0.1, // 34: w_idle max
0.10, // 35: dd_threshold max
3.0, // 36: loss_aversion max
0.005, // 37: time_decay_rate max
];
let params_max = DQNParams::from_continuous(&continuous_max).unwrap();
assert!((params_max.per_alpha - 0.8).abs() < 1e-6);
@@ -3904,20 +4064,8 @@ mod tests {
let params = DQNParams::default();
assert!(params.validate_for_hft_trendfollowing().is_ok());
// Params with various hold_penalty/buffer combos should also pass
// Params with various buffer sizes should also pass
// (strategy-specific constraints were removed in Wave 16V)
let params_low_penalty = DQNParams {
hold_penalty_weight: 0.3,
..DQNParams::default()
};
assert!(params_low_penalty.validate_for_hft_trendfollowing().is_ok());
let params_high_penalty = DQNParams {
hold_penalty_weight: 4.5,
..DQNParams::default()
};
assert!(params_high_penalty.validate_for_hft_trendfollowing().is_ok());
let params_small_buffer = DQNParams {
buffer_size: 20_000,
..DQNParams::default()
@@ -4188,7 +4336,7 @@ mod tests {
fn test_qr_dqn_roundtrip_continuous() {
let params = DQNParams::default();
let continuous = params.to_continuous();
assert_eq!(continuous.len(), 31, "Should have 31 continuous dimensions");
assert_eq!(continuous.len(), 38, "Should have 38 continuous dimensions");
let roundtrip = DQNParams::from_continuous(&continuous).unwrap();
// Default num_atoms=51 <= 100, so QR-DQN disabled, num_quantiles=64 (C51 default)
assert!(!roundtrip.use_qr_dqn, "num_atoms=51 should use C51, not QR-DQN");
@@ -4199,7 +4347,7 @@ mod tests {
#[test]
fn test_qr_dqn_activation_threshold() {
let bounds = DQNParams::continuous_bounds();
let mut params = vec![0.0_f64; 31]; // C7: 31D
let mut params = vec![0.0_f64; 38]; // C8: 38D
// Fill with valid defaults
params[0] = (1e-4_f64).ln(); // learning_rate
params[1] = 128.0; // batch_size
@@ -4212,7 +4360,7 @@ mod tests {
params[8] = 0.6; // per_alpha
params[9] = 0.4; // per_beta_start
// Fill remaining with midpoint of bounds
for i in 10..31 {
for i in 10..38 {
params[i] = (bounds[i].0 + bounds[i].1) / 2.0;
}
@@ -4263,7 +4411,7 @@ mod tests {
fn test_batch_size_respects_wide_bounds() {
// Simulate PSO choosing batch_size=2048 (within VRAM-aware bounds)
let bounds = DQNParams::continuous_bounds();
let mut params = vec![0.0_f64; 31]; // C7: 31D
let mut params = vec![0.0_f64; 38]; // C8: 38D
params[1] = 2048.0; // batch_size (index 1)
// Fill other required params with valid defaults
params[0] = (1e-4_f64).ln(); // learning_rate
@@ -4276,7 +4424,7 @@ mod tests {
params[8] = 0.6; // per_alpha
params[9] = 0.4; // per_beta_start
// Remaining params (indices 10-29) -- use midpoint of bounds
for i in 10..31 {
for i in 10..38 {
params[i] = (bounds[i].0 + bounds[i].1) / 2.0;
}
let result = DQNParams::from_continuous(&params).unwrap();
@@ -4386,8 +4534,8 @@ mod tests {
// eval_softmax_temp removed from search space (backtest uses greedy argmax).
// Verify from_continuous always sets it to fixed 1.0 regardless of input.
let bounds = DQNParams::continuous_bounds();
let mut vec = vec![0.0_f64; 31]; // C7: 31D
for i in 0..31 {
let mut vec = vec![0.0_f64; 38]; // C8: 38D
for i in 0..38 {
vec[i] = (bounds[i].0 + bounds[i].1) / 2.0;
}
let params = DQNParams::from_continuous(&vec).unwrap();

6
crates/ml/src/training_profile.rs
View File

@@ -257,9 +257,9 @@ pub struct PsoSection {
pub social: Option<f64>,
}
/// Phase 1 ("fast") fixed architecture values.
/// When phase=fast, these architecture params are fixed to single-point bounds
/// so PSO searches only learning dynamics (~15D instead of 31D).
/// Phase 1 ("fast") fixed architecture and reward values.
/// When phase=fast, architecture params and reward weights are fixed to single-point
/// bounds so PSO searches only learning dynamics (~17D instead of 38D).
#[derive(Debug, Clone, Deserialize, Default)]
pub struct PhaseFastSection {
pub hidden_dim_base: Option<f64>,

5
crates/ml/tests/dqn_hyperopt_json_export_test.rs
View File

@@ -286,7 +286,6 @@ mod hyperopt_json_export_tests {
assert_eq!(saved_params.batch_size, roundtrip_params.batch_size);
assert_eq!(saved_params.gamma, roundtrip_params.gamma);
assert_eq!(saved_params.buffer_size, roundtrip_params.buffer_size);
assert_eq!(saved_params.hold_penalty_weight, roundtrip_params.hold_penalty_weight);
assert_eq!(saved_params.max_position_absolute, roundtrip_params.max_position_absolute);
assert_eq!(saved_params.huber_delta, roundtrip_params.huber_delta);
assert_eq!(saved_params.entropy_coefficient, roundtrip_params.entropy_coefficient);
@@ -361,12 +360,14 @@ mod hyperopt_json_export_tests {
let required_fields = vec![
"learning_rate", "batch_size", "gamma", "buffer_size",
"hold_penalty_weight", "max_position_absolute", "huber_delta",
"max_position_absolute", "huber_delta",
"entropy_coefficient", "transaction_cost_multiplier",
"use_per", "per_alpha", "per_beta_start",
"use_dueling", "dueling_hidden_dim", "n_steps", "tau",
"use_distributional", "num_atoms", "v_min", "v_max",
"use_noisy_nets", "noisy_sigma_init", "minimum_profit_factor",
"w_dsr", "w_pnl", "w_dd", "w_idle",
"dd_threshold", "loss_aversion", "time_decay_rate",
];
for field in required_fields {

286
crates/ml/tests/ensemble_hyperopt_tests.rs
View File

@@ -72,13 +72,11 @@
clippy::unnecessary_to_owned,
clippy::single_component_path_imports,
)]
//! TDD Tests for WAVE 26 P1.4: 6D Ensemble Uncertainty Hyperopt Integration
//! TDD Tests for Ensemble Uncertainty Fields in Hyperopt
//!
//! Validates that ensemble uncertainty parameters are properly integrated into hyperopt search space:
//! 1. DQNHyperparameters has 6 new fields
//! 2. DQNParams has 5 new fields (ensemble_size as f64)
//! 3. Search space includes 40 total parameters
//! 4. train_with_params correctly converts parameters
//! Validates that ensemble uncertainty parameters are properly set as fixed defaults
//! in the current 38D search space (ensemble params were removed from search space,
//! they are now fixed in from_continuous).
use ml::hyperopt::adapters::dqn::DQNParams;
use ml::hyperopt::ParameterSpace;
@@ -128,242 +126,98 @@ fn test_dqn_params_has_ensemble_fields() {
#[test]
fn test_ensemble_search_space_bounds() {
// WAVE 26 P1.4: Verify search space includes 5 ensemble continuous bounds
// Ensemble params (ensemble_size, beta_variance, etc.) are no longer in the
// continuous search space — they are fixed in from_continuous(). Verify that
// the 38D search space is properly sized and ensemble defaults are set.
let bounds = DQNParams::continuous_bounds();
assert_eq!(bounds.len(), 38, "Search space should have 38 continuous parameters total (C8: +7 reward weights)");
// Search space now has 40 total parameters
assert_eq!(bounds.len(), 40, "Search space should have 40 continuous parameters total");
// Extract the ensemble bounds (indices 23-27)
let ensemble_size_bounds = bounds[23];
let beta_variance_bounds = bounds[24];
let beta_disagreement_bounds = bounds[25];
let beta_entropy_bounds = bounds[26];
let variance_cap_bounds = bounds[27];
// Verify ensemble_size bounds: [3.0, 10.0]
assert_eq!(ensemble_size_bounds.0, 3.0, "Ensemble size min should be 3.0");
assert_eq!(ensemble_size_bounds.1, 10.0, "Ensemble size max should be 10.0");
// Verify beta_variance bounds: [0.1, 1.0]
assert_eq!(beta_variance_bounds.0, 0.1, "Beta variance min should be 0.1");
assert_eq!(beta_variance_bounds.1, 1.0, "Beta variance max should be 1.0");
// Verify beta_disagreement bounds: [0.1, 1.0]
assert_eq!(beta_disagreement_bounds.0, 0.1, "Beta disagreement min should be 0.1");
assert_eq!(beta_disagreement_bounds.1, 1.0, "Beta disagreement max should be 1.0");
// Verify beta_entropy bounds: [0.05, 0.5]
assert_eq!(beta_entropy_bounds.0, 0.05, "Beta entropy min should be 0.05");
assert_eq!(beta_entropy_bounds.1, 0.5, "Beta entropy max should be 0.5");
// Verify variance_cap bounds: [0.1, 2.0]
assert_eq!(variance_cap_bounds.0, 0.1, "Variance cap min should be 0.1");
assert_eq!(variance_cap_bounds.1, 2.0, "Variance cap max should be 2.0");
// Ensemble params are NOT in bounds — they're fixed in from_continuous.
// Verify a DQNParams default still has correct ensemble defaults.
let params = DQNParams::default();
assert!(!params.use_ensemble_uncertainty, "Default: ensemble disabled");
assert_eq!(params.ensemble_size, 5.0, "Default ensemble size should be 5.0");
assert!(params.beta_variance > 0.0, "Beta variance should be positive");
assert!(params.beta_disagreement > 0.0, "Beta disagreement should be positive");
assert!(params.beta_entropy > 0.0, "Beta entropy should be positive");
}
#[test]
fn test_from_continuous_validates_ensemble_params() {
// WAVE 26 P1.4: Test parameter conversion from continuous space
let mut x = vec![0.0; 40];
// Set base parameters to valid values (using defaults)
x[0] = (5e-5_f64).ln(); // learning_rate
x[1] = 128.0; // batch_size
x[2] = 0.99; // gamma
x[3] = (75_000_f64).ln(); // buffer_size
x[4] = 1.5; // hold_penalty_weight
x[5] = 6.0; // max_position_absolute
x[6] = (20.0_f64).ln(); // huber_delta
x[7] = 0.05; // entropy_coefficient
x[8] = 1.0; // transaction_cost_multiplier
x[9] = 0.6; // per_alpha
x[10] = 0.4; // per_beta_start
x[11] = -2.0; // v_min
x[12] = 2.0; // v_max
x[13] = (0.5_f64).ln(); // noisy_sigma_init
x[14] = 256.0; // dueling_hidden_dim
x[15] = 3.0; // n_steps
x[16] = 51.0; // num_atoms
x[17] = 1.5; // minimum_profit_factor
x[18] = (1e-4_f64).ln(); // weight_decay
x[19] = 0.5; // kelly_fractional
x[20] = 0.25; // kelly_max_fraction
x[21] = 20.0; // kelly_min_trades
x[22] = 20.0; // volatility_window
// Set ensemble parameters (indices 23-27)
x[23] = 5.0; // ensemble_size
x[24] = 0.5; // beta_variance
x[25] = 0.5; // beta_disagreement
x[26] = 0.2; // beta_entropy
x[27] = 1.0; // variance_cap
// Set additional parameters (indices 28-39)
x[28] = 0.1; // warmup_ratio
x[29] = 0.1; // curiosity_weight
x[30] = (0.005_f64).ln(); // tau
x[31] = 10.0; // td_error_clamp_max
x[32] = 50.0; // batch_diversity_cooldown
x[33] = 1.0; // lr_decay_type
x[34] = 0.1; // sharpe_weight
x[35] = 0.95; // gae_lambda
x[36] = 0.6; // noisy_sigma_initial
x[37] = 0.3; // noisy_sigma_final
x[38] = 0.0; // norm_type
x[39] = 1.0; // activation_type
// Ensemble params are now fixed in from_continuous() (not in 38D search space).
// Verify that from_continuous produces correct fixed ensemble defaults.
let bounds = DQNParams::continuous_bounds();
let x: Vec<f64> = bounds.iter().map(|(lo, hi)| (lo + hi) / 2.0).collect();
assert_eq!(x.len(), 38, "Should produce 38D midpoint vector");
let params = DQNParams::from_continuous(&x).expect("Should convert valid parameters");
// Verify ensemble parameters are correctly extracted
assert_eq!(params.ensemble_size, 5.0, "Ensemble size should be 5.0");
assert_eq!(params.beta_variance, 0.5, "Beta variance should be 0.5");
assert_eq!(params.beta_disagreement, 0.5, "Beta disagreement should be 0.5");
assert_eq!(params.beta_entropy, 0.2, "Beta entropy should be 0.2");
// Note: variance_cap is not in DQNParams, it's only in DQNHyperparameters
// This will be tested in the train_with_params integration test
// Ensemble parameters are fixed to conservative defaults
assert_eq!(params.ensemble_size, 5.0, "Ensemble size should be fixed to 5.0");
assert_eq!(params.beta_variance, 0.5, "Beta variance should be fixed to 0.5");
assert_eq!(params.beta_disagreement, 0.5, "Beta disagreement should be fixed to 0.5");
assert_eq!(params.beta_entropy, 0.2, "Beta entropy should be fixed to 0.2");
assert!(!params.use_ensemble_uncertainty, "Ensemble uncertainty should be disabled");
}
#[test]
fn test_from_continuous_clamps_ensemble_params() {
// WAVE 26 P1.4: Test that ensemble parameters are clamped to valid ranges
let mut x = vec![0.0; 40];
// Ensemble parameters are now fixed (not in search space), so clamping
// is not relevant. Instead verify that from_continuous rejects wrong-length input.
let x = vec![0.0; 40]; // Wrong length (should be 38)
let result = DQNParams::from_continuous(&x);
assert!(result.is_err(), "Should reject wrong-length (40) input");
// Set base parameters to valid values (minimal setup)
x[0] = (5e-5_f64).ln();
x[1] = 128.0;
x[2] = 0.99;
x[3] = (75_000_f64).ln();
x[4] = 1.5;
x[5] = 6.0;
x[6] = (20.0_f64).ln();
x[7] = 0.05;
x[8] = 1.0;
x[9] = 0.6;
x[10] = 0.4;
x[11] = -2.0;
x[12] = 2.0;
x[13] = (0.5_f64).ln();
x[14] = 256.0;
x[15] = 3.0;
x[16] = 51.0;
x[17] = 1.5;
x[18] = (1e-4_f64).ln();
x[19] = 0.5;
x[20] = 0.25;
x[21] = 20.0;
x[22] = 20.0;
let x2 = vec![0.0; 31]; // Also wrong length
let result2 = DQNParams::from_continuous(&x2);
assert!(result2.is_err(), "Should reject wrong-length (31) input");
// Set ensemble parameters to out-of-bounds values (indices 23-27)
x[23] = 15.0; // ensemble_size (should clamp to 10.0)
x[24] = 2.0; // beta_variance (should clamp to 1.0)
x[25] = -0.5; // beta_disagreement (should clamp to 0.1)
x[26] = 1.0; // beta_entropy (should clamp to 0.5)
x[27] = 5.0; // variance_cap (valid, should stay 5.0)
// Set additional parameters (indices 28-39)
x[28] = 0.1; // warmup_ratio
x[29] = 0.1; // curiosity_weight
x[30] = (0.005_f64).ln(); // tau
x[31] = 10.0; // td_error_clamp_max
x[32] = 50.0; // batch_diversity_cooldown
x[33] = 1.0; // lr_decay_type
x[34] = 0.1; // sharpe_weight
x[35] = 0.95; // gae_lambda
x[36] = 0.6; // noisy_sigma_initial
x[37] = 0.3; // noisy_sigma_final
x[38] = 0.0; // norm_type
x[39] = 1.0; // activation_type
let params = DQNParams::from_continuous(&x).expect("Should convert and clamp parameters");
// Verify clamping of ensemble parameters
assert_eq!(params.ensemble_size, 10.0, "Ensemble size should clamp to max 10.0");
assert_eq!(params.beta_variance, 1.0, "Beta variance should clamp to max 1.0");
assert_eq!(params.beta_disagreement, 0.1, "Beta disagreement should clamp to min 0.1");
assert_eq!(params.beta_entropy, 0.5, "Beta entropy should clamp to max 0.5");
// Correct length (38) with valid midpoints should succeed
let bounds = DQNParams::continuous_bounds();
let x38: Vec<f64> = bounds.iter().map(|(lo, hi)| (lo + hi) / 2.0).collect();
let params = DQNParams::from_continuous(&x38).expect("Should accept 38D input");
// Ensemble params are fixed regardless of input
assert_eq!(params.ensemble_size, 5.0, "Ensemble size fixed to 5.0");
}
#[test]
fn test_to_continuous_includes_ensemble_params() {
// WAVE 26 P1.4: Test that to_continuous() includes ensemble parameters
fn test_to_continuous_includes_reward_weights() {
// C8: Verify that to_continuous() includes 7 composite reward weights (indices 31-37)
let mut params = DQNParams::default();
// Set ensemble parameters to specific values
params.ensemble_size = 7.0;
params.beta_variance = 0.6;
params.beta_disagreement = 0.4;
params.beta_entropy = 0.3;
// Set reward weights to specific values
params.w_dsr = 1.5;
params.w_pnl = 0.5;
params.w_dd = 2.0;
params.w_idle = 0.05;
params.dd_threshold = 0.04;
params.loss_aversion = 2.0;
params.time_decay_rate = 0.001;
let continuous = params.to_continuous();
assert_eq!(continuous.len(), 38, "Continuous representation should have 38 values");
// Should have 40 values
assert_eq!(continuous.len(), 40, "Continuous representation should have 40 values");
// Verify reward weight positions (indices 31-37)
assert_eq!(continuous[31], 1.5, "w_dsr should be at position 31");
assert_eq!(continuous[32], 0.5, "w_pnl should be at position 32");
assert_eq!(continuous[33], 2.0, "w_dd should be at position 33");
assert_eq!(continuous[34], 0.05, "w_idle should be at position 34");
assert_eq!(continuous[35], 0.04, "dd_threshold should be at position 35");
assert_eq!(continuous[36], 2.0, "loss_aversion should be at position 36");
assert_eq!(continuous[37], 0.001, "time_decay_rate should be at position 37");
// Verify ensemble parameters are at positions 23-26
assert_eq!(continuous[23], 7.0, "Ensemble size should be at position 23");
assert_eq!(continuous[24], 0.6, "Beta variance should be at position 24");
assert_eq!(continuous[25], 0.4, "Beta disagreement should be at position 25");
assert_eq!(continuous[26], 0.3, "Beta entropy should be at position 26");
// Position 27 (variance_cap) is not in DQNParams, won't appear in to_continuous()
// Ensemble params are NOT in the continuous vector (they're fixed)
// Verify the vector doesn't include them
assert_eq!(continuous.len(), 38);
}
#[test]
fn test_ensemble_size_rounds_to_integer() {
// WAVE 26 P1.4: Ensemble size should be rounded to nearest integer
let mut x = vec![0.0; 40];
// Minimal valid setup
x[0] = (5e-5_f64).ln();
x[1] = 128.0;
x[2] = 0.99;
x[3] = (75_000_f64).ln();
x[4] = 1.5;
x[5] = 6.0;
x[6] = (20.0_f64).ln();
x[7] = 0.05;
x[8] = 1.0;
x[9] = 0.6;
x[10] = 0.4;
x[11] = -2.0;
x[12] = 2.0;
x[13] = (0.5_f64).ln();
x[14] = 256.0;
x[15] = 3.0;
x[16] = 51.0;
x[17] = 1.5;
x[18] = (1e-4_f64).ln();
x[19] = 0.5;
x[20] = 0.25;
x[21] = 20.0;
x[22] = 20.0;
// Test rounding behavior (ensemble parameters at indices 23-27)
x[23] = 5.3; // Should round to 5.0
x[24] = 0.5;
x[25] = 0.5;
x[26] = 0.2;
x[27] = 1.0;
// Set additional parameters (indices 28-39)
x[28] = 0.1; // warmup_ratio
x[29] = 0.1; // curiosity_weight
x[30] = (0.005_f64).ln(); // tau
x[31] = 10.0; // td_error_clamp_max
x[32] = 50.0; // batch_diversity_cooldown
x[33] = 1.0; // lr_decay_type
x[34] = 0.1; // sharpe_weight
x[35] = 0.95; // gae_lambda
x[36] = 0.6; // noisy_sigma_initial
x[37] = 0.3; // noisy_sigma_final
x[38] = 0.0; // norm_type
x[39] = 1.0; // activation_type
fn test_ensemble_size_fixed_in_from_continuous() {
// Ensemble size is now fixed at 5.0 in from_continuous() — not in search space.
// Verify that from_continuous always produces ensemble_size=5.0 regardless of input.
let bounds = DQNParams::continuous_bounds();
let x: Vec<f64> = bounds.iter().map(|(lo, hi)| (lo + hi) / 2.0).collect();
let params = DQNParams::from_continuous(&x).expect("Should convert");
assert_eq!(params.ensemble_size, 5.0, "5.3 should round to 5.0");
x[23] = 7.8; // Should round to 8.0
let params = DQNParams::from_continuous(&x).expect("Should convert");
assert_eq!(params.ensemble_size, 8.0, "7.8 should round to 8.0");
assert_eq!(params.ensemble_size, 5.0, "Ensemble size should always be fixed to 5.0");
assert!(!params.use_ensemble_uncertainty, "Ensemble uncertainty should be disabled");
}