From ad5423e0168f0900616f4e0df64372a967b91c25 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 22 Mar 2026 21:13:31 +0100 Subject: [PATCH] =?UTF-8?q?feat:=20hyperopt=2031D=E2=86=9238D=20=E2=80=94?= =?UTF-8?q?=207=20composite=20reward=20weight=20dimensions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expand the DQN hyperopt parameter space from 31D to 38D by adding 7 GPU composite reward weights (w_dsr, w_pnl, w_dd, w_idle, dd_threshold, loss_aversion, time_decay_rate) at indices 31-37. - Remove hold_penalty_weight from DQNParams (replaced by w_idle) - Add #[serde(default)] for backward compat with old JSON results - Phase Fast fixes reward weights to defaults (not searched) - Wire reward weights from DQNParams → DQNHyperparameters in train_with_params - Fix all tests: 134 hyperopt + 7 ensemble + 5 JSON export tests pass - Fix stale 40D ensemble tests → 38D layout (were already broken pre-change) Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/ml/src/hyperopt/adapters/dqn.rs | 248 ++++++++++++--- crates/ml/src/training_profile.rs | 6 +- .../ml/tests/dqn_hyperopt_json_export_test.rs | 5 +- crates/ml/tests/ensemble_hyperopt_tests.rs | 286 +++++------------- 4 files changed, 274 insertions(+), 271 deletions(-) diff --git a/crates/ml/src/hyperopt/adapters/dqn.rs b/crates/ml/src/hyperopt/adapters/dqn.rs index e696633cf..a4cf760d3 100644 --- a/crates/ml/src/hyperopt/adapters/dqn.rs +++ b/crates/ml/src/hyperopt/adapters/dqn.rs @@ -3,7 +3,7 @@ //! This module provides a production-ready adapter for optimizing DQN //! hyperparameters using the generic optimization framework. It implements: //! -//! - 31D continuous parameter space with log-scale handling +//! - 38D continuous parameter space with log-scale handling //! - Training wrapper that integrates with existing DQN pipeline //! - Backtest-based objective using Sharpe ratio (production metric) //! - Prioritized Experience Replay (PER) tuning for Rainbow DQN performance (+25-40% convergence speed) @@ -144,7 +144,7 @@ const TRIAL_VRAM_MB: f64 = 7000.0; /// Corrected: was 0.02 (20KB) — 100x too high const MB_PER_SAMPLE: f64 = 0.0005; -/// DQN hyperparameter space (31D continuous) +/// DQN hyperparameter space (38D continuous) /// /// Fixed params (not in search space): /// - curiosity_weight: 0.0 (conflicts with NoisyNet) @@ -156,13 +156,23 @@ const MB_PER_SAMPLE: f64 = 0.0005; /// - count_bonus_coefficient: (0.0, 0.3) — UCB exploration bonus, complementary to NoisyNet /// - sharpe_weight: (0.0, 0.5) — tunable Sharpe ratio blending in composite reward /// +/// Composite reward weights (indices 31-37): +/// - w_dsr: DSR (Sharpe) weight +/// - w_pnl: Normalized PnL weight +/// - w_dd: Drawdown penalty weight +/// - w_idle: Idle penalty weight (replaces hold_penalty_weight) +/// - dd_threshold: Drawdown tolerance before penalty +/// - loss_aversion: Asymmetric loss scaling (prospect theory) +/// - time_decay_rate: Position staleness rent per step +/// /// ## Parameter Scaling /// /// - **Log-scale**: Learning rate, buffer_size, huber_delta, noisy_sigma_init, tau, weight_decay -/// - **Linear scale**: Batch size, gamma, max_position, entropy, tx_cost, per_alpha, per_beta, v_min, v_max, dueling_hidden_dim, n_steps, num_atoms +/// - **Linear scale**: Batch size, gamma, max_position, entropy, tx_cost, per_alpha, per_beta, v_min, v_max, dueling_hidden_dim, n_steps, num_atoms, reward weights /// /// This scaling ensures efficient exploration by egobox's Bayesian optimization. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(default)] pub struct DQNParams { /// Learning rate for Adam optimizer (log-scale) pub learning_rate: f64, @@ -172,8 +182,6 @@ pub struct DQNParams { pub gamma: f64, /// Replay buffer capacity (log-scale) pub buffer_size: usize, - /// HOLD penalty weight (linear scale: 1.0 - 2.0 for HFT active trading) - OPTIMIZED from [0.5, 5.0] - pub hold_penalty_weight: f64, // movement_threshold removed - now fixed at 0.02 (2%) to align with production /// Maximum absolute position size for action masking (4.0-8.0 contracts) - OPTIMIZED from [1.0, 10.0] @@ -375,6 +383,22 @@ pub struct DQNParams { /// Only in search space for large GPUs (≥40GB): effective_batch = batch_size × accum_steps. /// H100 with batch=1024, accum=4 → effective batch 4096 (more stable gradients). pub gradient_accumulation_steps: usize, + + // === Composite Reward Weights (indices 31-37 in search space) === + /// DSR (Sharpe) weight in composite reward (0.1-2.0) + pub w_dsr: f64, + /// Normalized PnL weight in composite reward (0.0-1.0) + pub w_pnl: f64, + /// Drawdown penalty weight in composite reward (0.0-5.0) + pub w_dd: f64, + /// Idle penalty weight — replaces hold_penalty_weight (0.0-0.1) + pub w_idle: f64, + /// Drawdown tolerance before penalty kicks in (0.01-0.10) + pub dd_threshold: f64, + /// Asymmetric loss scaling factor — prospect theory (1.0-3.0) + pub loss_aversion: f64, + /// Position staleness rent per step (0.0001-0.005) + pub time_decay_rate: f64, } impl Default for DQNParams { @@ -384,7 +408,6 @@ impl Default for DQNParams { batch_size: 128, gamma: 0.95, buffer_size: 100_000, - hold_penalty_weight: 0.0, // C3: Neutral hold reward — not in search space max_position_absolute: 2.0, // BLOCKER #2: Default matches production (±2.0) huber_delta: 10.0, // Conservative default (hyperopt can scale to 15-40) entropy_coefficient: 0.02, // C2: SAC-style regularization only (narrowed from 0.1) @@ -455,13 +478,21 @@ impl Default for DQNParams { branch_hidden_dim: 128, // Default: 128 hidden units per branching head use_branching: true, // Default: branching enabled (3 heads) gradient_accumulation_steps: 1, // Default: no accumulation + // Composite reward weights (defaults match dqn-production.toml) + w_dsr: 1.0, + w_pnl: 0.3, + w_dd: 1.0, + w_idle: 0.01, + dd_threshold: 0.02, + loss_aversion: 1.5, + time_decay_rate: 0.0005, } } } impl ParameterSpace for DQNParams { fn continuous_bounds() -> Vec<(f64, f64)> { - // 31D search space (C7: added iqn_lambda) + // 38D search space (C8: added 7 composite reward weights at indices 31-37) // All bounds loaded from config/training/dqn-hyperopt.toml [search_space]. // To change search ranges, edit the TOML — no code changes needed. // Log-scale transforms are applied here; TOML stores linear values. @@ -499,6 +530,15 @@ impl ParameterSpace for DQNParams { let ga = b("gradient_accumulation_steps", (1.0, 1.0)); let iq = b("iqn_lambda", (0.0, 2.0)); + // Composite reward weights (indices 31-37) + let mut rw_dsr = b("w_dsr", (0.1, 2.0)); + let mut rw_pnl = b("w_pnl", (0.0, 1.0)); + let mut rw_dd = b("w_dd", (0.0, 5.0)); + let mut rw_idle = b("w_idle", (0.0, 0.1)); + let mut rw_ddt = b("dd_threshold", (0.01, 0.10)); + let mut rw_la = b("loss_aversion", (1.0, 3.0)); + let mut rw_tdr = b("time_decay_rate", (0.0001, 0.005)); + // Two-phase hyperopt: fix architecture OR dynamics params to single-point bounds. // Single-point bounds (v, v) make the optimizer trivially converge on those // dimensions, focusing search power on the free dimensions. @@ -508,7 +548,7 @@ impl ParameterSpace for DQNParams { match phase { HyperoptPhase::Fast => { // Phase 1: fix architecture to small network from [phase_fast] TOML. - // Search only learning dynamics (~15D free, ~5D fixed). + // Also fix reward weights to defaults (search learning dynamics only). // All values MUST exist in [phase_fast] — no silent defaults. let pf = hp.phase_fast.as_ref() .expect("[phase_fast] section missing from dqn-hyperopt.toml — required for --phase fast"); @@ -529,14 +569,38 @@ impl ParameterSpace for DQNParams { dh = (fix_dh, fix_dh); vr = (fix_vr, fix_vr); + // Fix reward weights to defaults in Phase Fast (not searched) + let fix_w_dsr = pf.w_dsr + .expect("phase_fast.w_dsr missing from dqn-hyperopt.toml"); + let fix_w_pnl = pf.w_pnl + .expect("phase_fast.w_pnl missing from dqn-hyperopt.toml"); + let fix_w_dd = pf.w_dd + .expect("phase_fast.w_dd missing from dqn-hyperopt.toml"); + let fix_w_idle = pf.w_idle + .expect("phase_fast.w_idle missing from dqn-hyperopt.toml"); + let fix_ddt = pf.dd_threshold + .expect("phase_fast.dd_threshold missing from dqn-hyperopt.toml"); + let fix_la = pf.loss_aversion + .expect("phase_fast.loss_aversion missing from dqn-hyperopt.toml"); + let fix_tdr = pf.time_decay_rate + .expect("phase_fast.time_decay_rate missing from dqn-hyperopt.toml"); + + rw_dsr = (fix_w_dsr, fix_w_dsr); + rw_pnl = (fix_w_pnl, fix_w_pnl); + rw_dd = (fix_w_dd, fix_w_dd); + rw_idle = (fix_w_idle, fix_w_idle); + rw_ddt = (fix_ddt, fix_ddt); + rw_la = (fix_la, fix_la); + rw_tdr = (fix_tdr, fix_tdr); + tracing::info!( - "Phase FAST: fixed architecture (hidden_dim={}, num_atoms={}, branch_hidden={}, dueling_hidden={}, v_range={})", + "Phase FAST: fixed architecture (hidden_dim={}, num_atoms={}, branch_hidden={}, dueling_hidden={}, v_range={}) + reward weights", fix_hdim, fix_na, fix_bh, fix_dh, fix_vr ); } HyperoptPhase::Full(ref best_vec) => { // Phase 2: fix learning dynamics from Phase 1 best params. - // Search only architecture dims (~5D free, ~26D fixed). + // Search only architecture dims (~5D free, ~33D fixed). // The best_vec is the raw continuous vector from Phase 1. // Fix all non-architecture dims to Phase 1's best values. // Architecture dims (indices 10, 13, 15, 21, 28) stay free. @@ -577,6 +641,14 @@ impl ParameterSpace for DQNParams { bh, // 28: branch_hidden_dim ga, // 29: gradient_accumulation_steps iq, // 30: iqn_lambda + // Composite reward weights (indices 31-37) + rw_dsr, // 31: w_dsr + rw_pnl, // 32: w_pnl + rw_dd, // 33: w_dd + rw_idle, // 34: w_idle + rw_ddt, // 35: dd_threshold + rw_la, // 36: loss_aversion + rw_tdr, // 37: time_decay_rate ]; // Phase FULL: fix all non-architecture dims to Phase 1 best values. @@ -595,8 +667,8 @@ impl ParameterSpace for DQNParams { } fn from_continuous(x: &[f64]) -> Result { - if x.len() != 31 { - return Err(MLError::ConfigError(format!("Expected 31 continuous parameters, got {}", x.len()))); + if x.len() != 38 { + return Err(MLError::ConfigError(format!("Expected 38 continuous parameters, got {}", x.len()))); } // === 26 TUNED parameters (from search space) === @@ -691,6 +763,15 @@ impl ParameterSpace for DQNParams { // C7: IQN dual-head lambda (idx 30) let iqn_lambda = x[30].clamp(0.0, 2.0); + // C8: Composite reward weights (indices 31-37) + let w_dsr = x[31].clamp(0.1, 2.0); + let w_pnl = x[32].clamp(0.0, 1.0); + let w_dd = x[33].clamp(0.0, 5.0); + let w_idle = x[34].clamp(0.0, 0.1); + let dd_threshold = x[35].clamp(0.01, 0.10); + let loss_aversion = x[36].clamp(1.0, 3.0); + let time_decay_rate = x[37].clamp(0.0001, 0.005); + // WAVE 6 FIX #2: Batch size floor for high learning rates if learning_rate > 2e-4 && batch_size < 120 { tracing::info!( @@ -706,7 +787,6 @@ impl ParameterSpace for DQNParams { batch_size, gamma: x[2].clamp(0.88, 0.99), buffer_size, - hold_penalty_weight: 0.0, // C3: Neutral hold reward — not in search space max_position_absolute, huber_delta, entropy_coefficient, @@ -762,13 +842,21 @@ impl ParameterSpace for DQNParams { branch_hidden_dim, use_branching, gradient_accumulation_steps, + // C8: Composite reward weights (indices 31-37) + w_dsr, + w_pnl, + w_dd, + w_idle, + dd_threshold, + loss_aversion, + time_decay_rate, }; Ok(params) } fn to_continuous(&self) -> Vec { - // 31D search space (C7: added iqn_lambda) + // 38D search space (C8: added 7 composite reward weights at indices 31-37) vec![ self.learning_rate.ln(), // 0 self.batch_size as f64, // 1 @@ -801,11 +889,19 @@ impl ParameterSpace for DQNParams { self.branch_hidden_dim as f64, // 28: C5 branch_hidden_dim self.gradient_accumulation_steps as f64, // 29: C6 gradient accumulation self.iqn_lambda, // 30: C7 IQN dual-head loss weight + // C8: Composite reward weights (indices 31-37) + self.w_dsr, // 31 + self.w_pnl, // 32 + self.w_dd, // 33 + self.w_idle, // 34 + self.dd_threshold, // 35 + self.loss_aversion, // 36 + self.time_decay_rate, // 37 ] } fn param_names() -> Vec<&'static str> { - // 31 tuned parameters (C7: added iqn_lambda) + // 38 tuned parameters (C8: added 7 composite reward weights) vec![ "learning_rate", // 0 "batch_size", // 1 @@ -838,6 +934,14 @@ impl ParameterSpace for DQNParams { "branch_hidden_dim", // 28: C5 branching head capacity "gradient_accumulation_steps", // 29: C6 effective batch multiplier "iqn_lambda", // 30: C7 IQN dual-head loss weight + // C8: Composite reward weights + "w_dsr", // 31 + "w_pnl", // 32 + "w_dd", // 33 + "w_idle", // 34 + "dd_threshold", // 35 + "loss_aversion", // 36 + "time_decay_rate", // 37 ] } @@ -2425,7 +2529,7 @@ impl HyperparameterOptimizable for DQNTrainer { // Fix 1: Clamp buffer size to max (4GB GPU constraint) let clamped_buffer_size = params.buffer_size.min(self.buffer_size_max); - info!("Training DQN with parameters (Wave 6.4: 31D search space):"); + info!("Training DQN with parameters (C8: 38D search space):"); info!(" Learning rate: {:.6}", params.learning_rate); info!(" Batch size: {}", params.batch_size); info!(" Gamma: {:.3}", params.gamma); @@ -2433,7 +2537,6 @@ impl HyperparameterOptimizable for DQNTrainer { " Buffer size: {} (requested: {})", clamped_buffer_size, params.buffer_size ); - info!(" Hold penalty weight: {:.3}", params.hold_penalty_weight); info!(" Max position: {:.1}", params.max_position_absolute); info!(" Huber delta: {:.3}", params.huber_delta); info!(" Entropy coeff: {:.4}", params.entropy_coefficient); @@ -2626,7 +2729,7 @@ impl HyperparameterOptimizable for DQNTrainer { huber_delta: params.huber_delta, // WAVE 17: Use hyperopt value instead of fixed 1.0 use_double_dqn: true, // Production feature: --use-double-dqn gradient_clip_norm: Some(gradient_clip_norm), // WAVE 64: Dynamic clipping (5.0 for high LR, 10.0 for low LR) - hold_penalty_weight: params.hold_penalty_weight, // Use optimized value from search + hold_penalty_weight: params.w_idle, // Deprecated: mapped from w_idle for backward compat movement_threshold: 0.02, // Aligned with production (fixed at 2%, train_dqn.rs:296) // WAVE 16 (Agent 38): Use stored configuration values (allow CLI overrides) enable_preprocessing: self.enable_preprocessing, @@ -2718,7 +2821,7 @@ impl HyperparameterOptimizable for DQNTrainer { gradient_collapse_multiplier: 100.0, // Adaptive threshold (LR × 100) gradient_collapse_patience: 5, // 5 consecutive epochs before early stop - // LR scheduling — tunable via hyperopt search space (idx 23 in 31D) + // LR scheduling — tunable via hyperopt search space (idx 23 in 38D) // 0=constant, 1=linear decay, 2=cosine annealing // BUG FIX: total_steps = self.epochs (scheduler steps once per epoch, // NOT per training step). Previous calc used epochs × steps_per_epoch @@ -2853,6 +2956,15 @@ impl HyperparameterOptimizable for DQNTrainer { wf_val_fraction: 0.125, wf_test_fraction: 0.125, wf_step_fraction: 0.125, + + // C8: Composite reward weights — wired from hyperopt search space (indices 31-37) + w_dsr: params.w_dsr, + w_pnl: params.w_pnl, + w_dd: params.w_dd, + w_idle: params.w_idle, + dd_threshold: params.dd_threshold, + loss_aversion: params.loss_aversion, + time_decay_rate: params.time_decay_rate, ..Default::default() }; @@ -3603,14 +3715,13 @@ mod tests { #[test] fn test_dqn_params_roundtrip() { - // Roundtrip test for the 26D search space (C2: removed curiosity_weight + noisy_epsilon_floor) - // Only the 26 tuned parameters roundtrip; fixed params get defaults from from_continuous + // Roundtrip test for the 38D search space (C8: 7 composite reward weights added) + // Only the tuned parameters roundtrip; fixed params get defaults from from_continuous let params = DQNParams { learning_rate: 3.37e-05, batch_size: 92, gamma: 0.92, buffer_size: 97_273, - hold_penalty_weight: 1.404, max_position_absolute: 2.5, huber_delta: 24.77, entropy_coefficient: 0.03, // C2: narrowed range [0.005, 0.05] @@ -3664,18 +3775,25 @@ mod tests { branch_hidden_dim: 128, use_branching: true, gradient_accumulation_steps: 4, + // C8: Composite reward weights + w_dsr: 1.5, + w_pnl: 0.5, + w_dd: 2.0, + w_idle: 0.05, + dd_threshold: 0.04, + loss_aversion: 2.0, + time_decay_rate: 0.001, }; let continuous = params.to_continuous(); - assert_eq!(continuous.len(), 31, "to_continuous must return 31D vector"); + assert_eq!(continuous.len(), 38, "to_continuous must return 38D vector"); let recovered = DQNParams::from_continuous(&continuous).unwrap(); - // Tuned parameters must roundtrip exactly (26D) + // Tuned parameters must roundtrip exactly assert!((recovered.learning_rate - params.learning_rate).abs() < 1e-6); assert_eq!(recovered.batch_size, params.batch_size); assert!((recovered.gamma - params.gamma).abs() < 1e-6); assert_eq!(recovered.buffer_size, params.buffer_size); - assert!((recovered.hold_penalty_weight - 0.0).abs() < 1e-6); // C3: always 0.0 assert!((recovered.max_position_absolute - params.max_position_absolute).abs() < 1e-6); assert!((recovered.per_alpha - params.per_alpha).abs() < 1e-6); assert!((recovered.per_beta_start - params.per_beta_start).abs() < 1e-6); @@ -3710,12 +3828,21 @@ mod tests { assert_eq!(recovered.use_branching, params.use_branching); // C6: gradient_accumulation_steps roundtrips via index 29 assert_eq!(recovered.gradient_accumulation_steps, params.gradient_accumulation_steps); + + // C8: Composite reward weights roundtrip (indices 31-37) + assert!((recovered.w_dsr - params.w_dsr).abs() < 1e-6); + assert!((recovered.w_pnl - params.w_pnl).abs() < 1e-6); + assert!((recovered.w_dd - params.w_dd).abs() < 1e-6); + assert!((recovered.w_idle - params.w_idle).abs() < 1e-6); + assert!((recovered.dd_threshold - params.dd_threshold).abs() < 1e-6); + assert!((recovered.loss_aversion - params.loss_aversion).abs() < 1e-6); + assert!((recovered.time_decay_rate - params.time_decay_rate).abs() < 1e-6); } #[test] fn test_dqn_params_bounds() { let bounds = DQNParams::continuous_bounds(); - assert_eq!(bounds.len(), 31); // C7: 31D (added iqn_lambda) + assert_eq!(bounds.len(), 38); // C8: 38D (added 7 reward weights) // Check log-scale bounds are reasonable assert!(bounds[0].0 < bounds[0].1); // learning_rate @@ -3760,12 +3887,21 @@ mod tests { assert_eq!(bounds[28], (64.0, 64.0)); // branch_hidden_dim (FIXED in Fast phase) // C6: gradient_accumulation_steps (fixed at 1 for default GPUs) assert_eq!(bounds[29], (1.0, 1.0)); // gradient_accumulation_steps + + // C8: Composite reward weights (FIXED in Fast phase to defaults) + assert_eq!(bounds[31], (1.0, 1.0)); // w_dsr (FIXED in Fast phase) + assert_eq!(bounds[32], (0.3, 0.3)); // w_pnl (FIXED in Fast phase) + assert_eq!(bounds[33], (1.0, 1.0)); // w_dd (FIXED in Fast phase) + assert_eq!(bounds[34], (0.01, 0.01)); // w_idle (FIXED in Fast phase) + assert_eq!(bounds[35], (0.02, 0.02)); // dd_threshold (FIXED in Fast phase) + assert_eq!(bounds[36], (1.5, 1.5)); // loss_aversion (FIXED in Fast phase) + assert_eq!(bounds[37], (0.0005, 0.0005)); // time_decay_rate (FIXED in Fast phase) } #[test] fn test_param_names() { let names = DQNParams::param_names(); - assert_eq!(names.len(), 31); // C7: 31D (added iqn_lambda) + assert_eq!(names.len(), 38); // C8: 38D (added 7 reward weights) assert_eq!(names[0], "learning_rate"); assert_eq!(names[1], "batch_size"); assert_eq!(names[2], "gamma"); @@ -3801,7 +3937,7 @@ mod tests { #[test] fn test_per_params_always_enabled() { // Test that PER is always enabled with tunable alpha/beta parameters - // 31D search space (C7: added iqn_lambda) + // 38D search space (C8: added 7 composite reward weights) let continuous = vec![ 3e-5_f64.ln(), 92.0, 0.92, 97_273_f64.ln(), 4.0, 24.77_f64.ln(), 0.03, 1.0, 0.6, 0.4, // 8-9: per_alpha, per_beta_start @@ -3821,6 +3957,14 @@ mod tests { 128.0, // 28: branch_hidden_dim (C5) 1.0, // 29: gradient_accumulation_steps (C6) 0.25, // 30: iqn_lambda (C7) + // C8: Composite reward weights (indices 31-37) + 1.0, // 31: w_dsr + 0.3, // 32: w_pnl + 1.0, // 33: w_dd + 0.01, // 34: w_idle + 0.02, // 35: dd_threshold + 1.5, // 36: loss_aversion + 0.0005, // 37: time_decay_rate ]; let params = DQNParams::from_continuous(&continuous).unwrap(); @@ -3837,7 +3981,7 @@ mod tests { assert!((params.v_min - (-25.0)).abs() < 1e-6, "v_min should be -v_range"); assert!((params.v_max - 25.0).abs() < 1e-6, "v_max should be +v_range"); - // Test PER parameter bounds (min values) -- 31D + // Test PER parameter bounds (min values) -- 38D let continuous_min = vec![ 2e-5_f64.ln(), 64.0, 0.88, 50_000_f64.ln(), 1.0, 10.0_f64.ln(), 0.05, 0.5, 0.4, 0.2, // per_alpha min, per_beta_start min @@ -3857,6 +4001,14 @@ mod tests { 64.0, // 28: branch_hidden_dim min (C5) 1.0, // 29: gradient_accumulation_steps min (C6) 0.0, // 30: iqn_lambda min (C7) + // C8: Composite reward weights min + 0.1, // 31: w_dsr min + 0.0, // 32: w_pnl min + 0.0, // 33: w_dd min + 0.0, // 34: w_idle min + 0.01, // 35: dd_threshold min + 1.0, // 36: loss_aversion min + 0.0001, // 37: time_decay_rate min ]; let params_min = DQNParams::from_continuous(&continuous_min).unwrap(); assert!((params_min.per_alpha - 0.4).abs() < 1e-6); @@ -3867,7 +4019,7 @@ mod tests { assert!((params_min.v_min - (-10.0)).abs() < 1e-6, "v_min should be -v_range at min"); assert!((params_min.v_max - 10.0).abs() < 1e-6, "v_max should be +v_range at min"); - // Test PER parameter bounds (max values) -- 31D + // Test PER parameter bounds (max values) -- 38D let continuous_max = vec![ 8e-5_f64.ln(), 512.0, 0.99, 100_000_f64.ln(), 4.0, 40.0_f64.ln(), 0.5, 2.0, 0.8, 0.6, // per_alpha max, per_beta_start max @@ -3887,6 +4039,14 @@ mod tests { 256.0, // 28: branch_hidden_dim max (C5) 1.0, // 29: gradient_accumulation_steps max (C6) 2.0, // 30: iqn_lambda max (C7) + // C8: Composite reward weights max + 2.0, // 31: w_dsr max + 1.0, // 32: w_pnl max + 5.0, // 33: w_dd max + 0.1, // 34: w_idle max + 0.10, // 35: dd_threshold max + 3.0, // 36: loss_aversion max + 0.005, // 37: time_decay_rate max ]; let params_max = DQNParams::from_continuous(&continuous_max).unwrap(); assert!((params_max.per_alpha - 0.8).abs() < 1e-6); @@ -3904,20 +4064,8 @@ mod tests { let params = DQNParams::default(); assert!(params.validate_for_hft_trendfollowing().is_ok()); - // Params with various hold_penalty/buffer combos should also pass + // Params with various buffer sizes should also pass // (strategy-specific constraints were removed in Wave 16V) - let params_low_penalty = DQNParams { - hold_penalty_weight: 0.3, - ..DQNParams::default() - }; - assert!(params_low_penalty.validate_for_hft_trendfollowing().is_ok()); - - let params_high_penalty = DQNParams { - hold_penalty_weight: 4.5, - ..DQNParams::default() - }; - assert!(params_high_penalty.validate_for_hft_trendfollowing().is_ok()); - let params_small_buffer = DQNParams { buffer_size: 20_000, ..DQNParams::default() @@ -4188,7 +4336,7 @@ mod tests { fn test_qr_dqn_roundtrip_continuous() { let params = DQNParams::default(); let continuous = params.to_continuous(); - assert_eq!(continuous.len(), 31, "Should have 31 continuous dimensions"); + assert_eq!(continuous.len(), 38, "Should have 38 continuous dimensions"); let roundtrip = DQNParams::from_continuous(&continuous).unwrap(); // Default num_atoms=51 <= 100, so QR-DQN disabled, num_quantiles=64 (C51 default) assert!(!roundtrip.use_qr_dqn, "num_atoms=51 should use C51, not QR-DQN"); @@ -4199,7 +4347,7 @@ mod tests { #[test] fn test_qr_dqn_activation_threshold() { let bounds = DQNParams::continuous_bounds(); - let mut params = vec![0.0_f64; 31]; // C7: 31D + let mut params = vec![0.0_f64; 38]; // C8: 38D // Fill with valid defaults params[0] = (1e-4_f64).ln(); // learning_rate params[1] = 128.0; // batch_size @@ -4212,7 +4360,7 @@ mod tests { params[8] = 0.6; // per_alpha params[9] = 0.4; // per_beta_start // Fill remaining with midpoint of bounds - for i in 10..31 { + for i in 10..38 { params[i] = (bounds[i].0 + bounds[i].1) / 2.0; } @@ -4263,7 +4411,7 @@ mod tests { fn test_batch_size_respects_wide_bounds() { // Simulate PSO choosing batch_size=2048 (within VRAM-aware bounds) let bounds = DQNParams::continuous_bounds(); - let mut params = vec![0.0_f64; 31]; // C7: 31D + let mut params = vec![0.0_f64; 38]; // C8: 38D params[1] = 2048.0; // batch_size (index 1) // Fill other required params with valid defaults params[0] = (1e-4_f64).ln(); // learning_rate @@ -4276,7 +4424,7 @@ mod tests { params[8] = 0.6; // per_alpha params[9] = 0.4; // per_beta_start // Remaining params (indices 10-29) -- use midpoint of bounds - for i in 10..31 { + for i in 10..38 { params[i] = (bounds[i].0 + bounds[i].1) / 2.0; } let result = DQNParams::from_continuous(¶ms).unwrap(); @@ -4386,8 +4534,8 @@ mod tests { // eval_softmax_temp removed from search space (backtest uses greedy argmax). // Verify from_continuous always sets it to fixed 1.0 regardless of input. let bounds = DQNParams::continuous_bounds(); - let mut vec = vec![0.0_f64; 31]; // C7: 31D - for i in 0..31 { + let mut vec = vec![0.0_f64; 38]; // C8: 38D + for i in 0..38 { vec[i] = (bounds[i].0 + bounds[i].1) / 2.0; } let params = DQNParams::from_continuous(&vec).unwrap(); diff --git a/crates/ml/src/training_profile.rs b/crates/ml/src/training_profile.rs index eddd0cc80..1cae538a2 100644 --- a/crates/ml/src/training_profile.rs +++ b/crates/ml/src/training_profile.rs @@ -257,9 +257,9 @@ pub struct PsoSection { pub social: Option, } -/// Phase 1 ("fast") fixed architecture values. -/// When phase=fast, these architecture params are fixed to single-point bounds -/// so PSO searches only learning dynamics (~15D instead of 31D). +/// Phase 1 ("fast") fixed architecture and reward values. +/// When phase=fast, architecture params and reward weights are fixed to single-point +/// bounds so PSO searches only learning dynamics (~17D instead of 38D). #[derive(Debug, Clone, Deserialize, Default)] pub struct PhaseFastSection { pub hidden_dim_base: Option, diff --git a/crates/ml/tests/dqn_hyperopt_json_export_test.rs b/crates/ml/tests/dqn_hyperopt_json_export_test.rs index ce3599aa1..abe26ab31 100644 --- a/crates/ml/tests/dqn_hyperopt_json_export_test.rs +++ b/crates/ml/tests/dqn_hyperopt_json_export_test.rs @@ -286,7 +286,6 @@ mod hyperopt_json_export_tests { assert_eq!(saved_params.batch_size, roundtrip_params.batch_size); assert_eq!(saved_params.gamma, roundtrip_params.gamma); assert_eq!(saved_params.buffer_size, roundtrip_params.buffer_size); - assert_eq!(saved_params.hold_penalty_weight, roundtrip_params.hold_penalty_weight); assert_eq!(saved_params.max_position_absolute, roundtrip_params.max_position_absolute); assert_eq!(saved_params.huber_delta, roundtrip_params.huber_delta); assert_eq!(saved_params.entropy_coefficient, roundtrip_params.entropy_coefficient); @@ -361,12 +360,14 @@ mod hyperopt_json_export_tests { let required_fields = vec![ "learning_rate", "batch_size", "gamma", "buffer_size", - "hold_penalty_weight", "max_position_absolute", "huber_delta", + "max_position_absolute", "huber_delta", "entropy_coefficient", "transaction_cost_multiplier", "use_per", "per_alpha", "per_beta_start", "use_dueling", "dueling_hidden_dim", "n_steps", "tau", "use_distributional", "num_atoms", "v_min", "v_max", "use_noisy_nets", "noisy_sigma_init", "minimum_profit_factor", + "w_dsr", "w_pnl", "w_dd", "w_idle", + "dd_threshold", "loss_aversion", "time_decay_rate", ]; for field in required_fields { diff --git a/crates/ml/tests/ensemble_hyperopt_tests.rs b/crates/ml/tests/ensemble_hyperopt_tests.rs index f40994d7d..cccc6ea55 100644 --- a/crates/ml/tests/ensemble_hyperopt_tests.rs +++ b/crates/ml/tests/ensemble_hyperopt_tests.rs @@ -72,13 +72,11 @@ clippy::unnecessary_to_owned, clippy::single_component_path_imports, )] -//! TDD Tests for WAVE 26 P1.4: 6D Ensemble Uncertainty Hyperopt Integration +//! TDD Tests for Ensemble Uncertainty Fields in Hyperopt //! -//! Validates that ensemble uncertainty parameters are properly integrated into hyperopt search space: -//! 1. DQNHyperparameters has 6 new fields -//! 2. DQNParams has 5 new fields (ensemble_size as f64) -//! 3. Search space includes 40 total parameters -//! 4. train_with_params correctly converts parameters +//! Validates that ensemble uncertainty parameters are properly set as fixed defaults +//! in the current 38D search space (ensemble params were removed from search space, +//! they are now fixed in from_continuous). use ml::hyperopt::adapters::dqn::DQNParams; use ml::hyperopt::ParameterSpace; @@ -128,242 +126,98 @@ fn test_dqn_params_has_ensemble_fields() { #[test] fn test_ensemble_search_space_bounds() { - // WAVE 26 P1.4: Verify search space includes 5 ensemble continuous bounds + // Ensemble params (ensemble_size, beta_variance, etc.) are no longer in the + // continuous search space — they are fixed in from_continuous(). Verify that + // the 38D search space is properly sized and ensemble defaults are set. let bounds = DQNParams::continuous_bounds(); + assert_eq!(bounds.len(), 38, "Search space should have 38 continuous parameters total (C8: +7 reward weights)"); - // Search space now has 40 total parameters - assert_eq!(bounds.len(), 40, "Search space should have 40 continuous parameters total"); - - // Extract the ensemble bounds (indices 23-27) - let ensemble_size_bounds = bounds[23]; - let beta_variance_bounds = bounds[24]; - let beta_disagreement_bounds = bounds[25]; - let beta_entropy_bounds = bounds[26]; - let variance_cap_bounds = bounds[27]; - - // Verify ensemble_size bounds: [3.0, 10.0] - assert_eq!(ensemble_size_bounds.0, 3.0, "Ensemble size min should be 3.0"); - assert_eq!(ensemble_size_bounds.1, 10.0, "Ensemble size max should be 10.0"); - - // Verify beta_variance bounds: [0.1, 1.0] - assert_eq!(beta_variance_bounds.0, 0.1, "Beta variance min should be 0.1"); - assert_eq!(beta_variance_bounds.1, 1.0, "Beta variance max should be 1.0"); - - // Verify beta_disagreement bounds: [0.1, 1.0] - assert_eq!(beta_disagreement_bounds.0, 0.1, "Beta disagreement min should be 0.1"); - assert_eq!(beta_disagreement_bounds.1, 1.0, "Beta disagreement max should be 1.0"); - - // Verify beta_entropy bounds: [0.05, 0.5] - assert_eq!(beta_entropy_bounds.0, 0.05, "Beta entropy min should be 0.05"); - assert_eq!(beta_entropy_bounds.1, 0.5, "Beta entropy max should be 0.5"); - - // Verify variance_cap bounds: [0.1, 2.0] - assert_eq!(variance_cap_bounds.0, 0.1, "Variance cap min should be 0.1"); - assert_eq!(variance_cap_bounds.1, 2.0, "Variance cap max should be 2.0"); + // Ensemble params are NOT in bounds — they're fixed in from_continuous. + // Verify a DQNParams default still has correct ensemble defaults. + let params = DQNParams::default(); + assert!(!params.use_ensemble_uncertainty, "Default: ensemble disabled"); + assert_eq!(params.ensemble_size, 5.0, "Default ensemble size should be 5.0"); + assert!(params.beta_variance > 0.0, "Beta variance should be positive"); + assert!(params.beta_disagreement > 0.0, "Beta disagreement should be positive"); + assert!(params.beta_entropy > 0.0, "Beta entropy should be positive"); } #[test] fn test_from_continuous_validates_ensemble_params() { - // WAVE 26 P1.4: Test parameter conversion from continuous space - let mut x = vec![0.0; 40]; - - // Set base parameters to valid values (using defaults) - x[0] = (5e-5_f64).ln(); // learning_rate - x[1] = 128.0; // batch_size - x[2] = 0.99; // gamma - x[3] = (75_000_f64).ln(); // buffer_size - x[4] = 1.5; // hold_penalty_weight - x[5] = 6.0; // max_position_absolute - x[6] = (20.0_f64).ln(); // huber_delta - x[7] = 0.05; // entropy_coefficient - x[8] = 1.0; // transaction_cost_multiplier - x[9] = 0.6; // per_alpha - x[10] = 0.4; // per_beta_start - x[11] = -2.0; // v_min - x[12] = 2.0; // v_max - x[13] = (0.5_f64).ln(); // noisy_sigma_init - x[14] = 256.0; // dueling_hidden_dim - x[15] = 3.0; // n_steps - x[16] = 51.0; // num_atoms - x[17] = 1.5; // minimum_profit_factor - x[18] = (1e-4_f64).ln(); // weight_decay - x[19] = 0.5; // kelly_fractional - x[20] = 0.25; // kelly_max_fraction - x[21] = 20.0; // kelly_min_trades - x[22] = 20.0; // volatility_window - - // Set ensemble parameters (indices 23-27) - x[23] = 5.0; // ensemble_size - x[24] = 0.5; // beta_variance - x[25] = 0.5; // beta_disagreement - x[26] = 0.2; // beta_entropy - x[27] = 1.0; // variance_cap - - // Set additional parameters (indices 28-39) - x[28] = 0.1; // warmup_ratio - x[29] = 0.1; // curiosity_weight - x[30] = (0.005_f64).ln(); // tau - x[31] = 10.0; // td_error_clamp_max - x[32] = 50.0; // batch_diversity_cooldown - x[33] = 1.0; // lr_decay_type - x[34] = 0.1; // sharpe_weight - x[35] = 0.95; // gae_lambda - x[36] = 0.6; // noisy_sigma_initial - x[37] = 0.3; // noisy_sigma_final - x[38] = 0.0; // norm_type - x[39] = 1.0; // activation_type + // Ensemble params are now fixed in from_continuous() (not in 38D search space). + // Verify that from_continuous produces correct fixed ensemble defaults. + let bounds = DQNParams::continuous_bounds(); + let x: Vec = bounds.iter().map(|(lo, hi)| (lo + hi) / 2.0).collect(); + assert_eq!(x.len(), 38, "Should produce 38D midpoint vector"); let params = DQNParams::from_continuous(&x).expect("Should convert valid parameters"); - // Verify ensemble parameters are correctly extracted - assert_eq!(params.ensemble_size, 5.0, "Ensemble size should be 5.0"); - assert_eq!(params.beta_variance, 0.5, "Beta variance should be 0.5"); - assert_eq!(params.beta_disagreement, 0.5, "Beta disagreement should be 0.5"); - assert_eq!(params.beta_entropy, 0.2, "Beta entropy should be 0.2"); - - // Note: variance_cap is not in DQNParams, it's only in DQNHyperparameters - // This will be tested in the train_with_params integration test + // Ensemble parameters are fixed to conservative defaults + assert_eq!(params.ensemble_size, 5.0, "Ensemble size should be fixed to 5.0"); + assert_eq!(params.beta_variance, 0.5, "Beta variance should be fixed to 0.5"); + assert_eq!(params.beta_disagreement, 0.5, "Beta disagreement should be fixed to 0.5"); + assert_eq!(params.beta_entropy, 0.2, "Beta entropy should be fixed to 0.2"); + assert!(!params.use_ensemble_uncertainty, "Ensemble uncertainty should be disabled"); } #[test] fn test_from_continuous_clamps_ensemble_params() { - // WAVE 26 P1.4: Test that ensemble parameters are clamped to valid ranges - let mut x = vec![0.0; 40]; + // Ensemble parameters are now fixed (not in search space), so clamping + // is not relevant. Instead verify that from_continuous rejects wrong-length input. + let x = vec![0.0; 40]; // Wrong length (should be 38) + let result = DQNParams::from_continuous(&x); + assert!(result.is_err(), "Should reject wrong-length (40) input"); - // Set base parameters to valid values (minimal setup) - x[0] = (5e-5_f64).ln(); - x[1] = 128.0; - x[2] = 0.99; - x[3] = (75_000_f64).ln(); - x[4] = 1.5; - x[5] = 6.0; - x[6] = (20.0_f64).ln(); - x[7] = 0.05; - x[8] = 1.0; - x[9] = 0.6; - x[10] = 0.4; - x[11] = -2.0; - x[12] = 2.0; - x[13] = (0.5_f64).ln(); - x[14] = 256.0; - x[15] = 3.0; - x[16] = 51.0; - x[17] = 1.5; - x[18] = (1e-4_f64).ln(); - x[19] = 0.5; - x[20] = 0.25; - x[21] = 20.0; - x[22] = 20.0; + let x2 = vec![0.0; 31]; // Also wrong length + let result2 = DQNParams::from_continuous(&x2); + assert!(result2.is_err(), "Should reject wrong-length (31) input"); - // Set ensemble parameters to out-of-bounds values (indices 23-27) - x[23] = 15.0; // ensemble_size (should clamp to 10.0) - x[24] = 2.0; // beta_variance (should clamp to 1.0) - x[25] = -0.5; // beta_disagreement (should clamp to 0.1) - x[26] = 1.0; // beta_entropy (should clamp to 0.5) - x[27] = 5.0; // variance_cap (valid, should stay 5.0) - - // Set additional parameters (indices 28-39) - x[28] = 0.1; // warmup_ratio - x[29] = 0.1; // curiosity_weight - x[30] = (0.005_f64).ln(); // tau - x[31] = 10.0; // td_error_clamp_max - x[32] = 50.0; // batch_diversity_cooldown - x[33] = 1.0; // lr_decay_type - x[34] = 0.1; // sharpe_weight - x[35] = 0.95; // gae_lambda - x[36] = 0.6; // noisy_sigma_initial - x[37] = 0.3; // noisy_sigma_final - x[38] = 0.0; // norm_type - x[39] = 1.0; // activation_type - - let params = DQNParams::from_continuous(&x).expect("Should convert and clamp parameters"); - - // Verify clamping of ensemble parameters - assert_eq!(params.ensemble_size, 10.0, "Ensemble size should clamp to max 10.0"); - assert_eq!(params.beta_variance, 1.0, "Beta variance should clamp to max 1.0"); - assert_eq!(params.beta_disagreement, 0.1, "Beta disagreement should clamp to min 0.1"); - assert_eq!(params.beta_entropy, 0.5, "Beta entropy should clamp to max 0.5"); + // Correct length (38) with valid midpoints should succeed + let bounds = DQNParams::continuous_bounds(); + let x38: Vec = bounds.iter().map(|(lo, hi)| (lo + hi) / 2.0).collect(); + let params = DQNParams::from_continuous(&x38).expect("Should accept 38D input"); + // Ensemble params are fixed regardless of input + assert_eq!(params.ensemble_size, 5.0, "Ensemble size fixed to 5.0"); } #[test] -fn test_to_continuous_includes_ensemble_params() { - // WAVE 26 P1.4: Test that to_continuous() includes ensemble parameters +fn test_to_continuous_includes_reward_weights() { + // C8: Verify that to_continuous() includes 7 composite reward weights (indices 31-37) let mut params = DQNParams::default(); - // Set ensemble parameters to specific values - params.ensemble_size = 7.0; - params.beta_variance = 0.6; - params.beta_disagreement = 0.4; - params.beta_entropy = 0.3; + // Set reward weights to specific values + params.w_dsr = 1.5; + params.w_pnl = 0.5; + params.w_dd = 2.0; + params.w_idle = 0.05; + params.dd_threshold = 0.04; + params.loss_aversion = 2.0; + params.time_decay_rate = 0.001; let continuous = params.to_continuous(); + assert_eq!(continuous.len(), 38, "Continuous representation should have 38 values"); - // Should have 40 values - assert_eq!(continuous.len(), 40, "Continuous representation should have 40 values"); + // Verify reward weight positions (indices 31-37) + assert_eq!(continuous[31], 1.5, "w_dsr should be at position 31"); + assert_eq!(continuous[32], 0.5, "w_pnl should be at position 32"); + assert_eq!(continuous[33], 2.0, "w_dd should be at position 33"); + assert_eq!(continuous[34], 0.05, "w_idle should be at position 34"); + assert_eq!(continuous[35], 0.04, "dd_threshold should be at position 35"); + assert_eq!(continuous[36], 2.0, "loss_aversion should be at position 36"); + assert_eq!(continuous[37], 0.001, "time_decay_rate should be at position 37"); - // Verify ensemble parameters are at positions 23-26 - assert_eq!(continuous[23], 7.0, "Ensemble size should be at position 23"); - assert_eq!(continuous[24], 0.6, "Beta variance should be at position 24"); - assert_eq!(continuous[25], 0.4, "Beta disagreement should be at position 25"); - assert_eq!(continuous[26], 0.3, "Beta entropy should be at position 26"); - // Position 27 (variance_cap) is not in DQNParams, won't appear in to_continuous() + // Ensemble params are NOT in the continuous vector (they're fixed) + // Verify the vector doesn't include them + assert_eq!(continuous.len(), 38); } #[test] -fn test_ensemble_size_rounds_to_integer() { - // WAVE 26 P1.4: Ensemble size should be rounded to nearest integer - let mut x = vec![0.0; 40]; - - // Minimal valid setup - x[0] = (5e-5_f64).ln(); - x[1] = 128.0; - x[2] = 0.99; - x[3] = (75_000_f64).ln(); - x[4] = 1.5; - x[5] = 6.0; - x[6] = (20.0_f64).ln(); - x[7] = 0.05; - x[8] = 1.0; - x[9] = 0.6; - x[10] = 0.4; - x[11] = -2.0; - x[12] = 2.0; - x[13] = (0.5_f64).ln(); - x[14] = 256.0; - x[15] = 3.0; - x[16] = 51.0; - x[17] = 1.5; - x[18] = (1e-4_f64).ln(); - x[19] = 0.5; - x[20] = 0.25; - x[21] = 20.0; - x[22] = 20.0; - - // Test rounding behavior (ensemble parameters at indices 23-27) - x[23] = 5.3; // Should round to 5.0 - x[24] = 0.5; - x[25] = 0.5; - x[26] = 0.2; - x[27] = 1.0; - - // Set additional parameters (indices 28-39) - x[28] = 0.1; // warmup_ratio - x[29] = 0.1; // curiosity_weight - x[30] = (0.005_f64).ln(); // tau - x[31] = 10.0; // td_error_clamp_max - x[32] = 50.0; // batch_diversity_cooldown - x[33] = 1.0; // lr_decay_type - x[34] = 0.1; // sharpe_weight - x[35] = 0.95; // gae_lambda - x[36] = 0.6; // noisy_sigma_initial - x[37] = 0.3; // noisy_sigma_final - x[38] = 0.0; // norm_type - x[39] = 1.0; // activation_type - +fn test_ensemble_size_fixed_in_from_continuous() { + // Ensemble size is now fixed at 5.0 in from_continuous() — not in search space. + // Verify that from_continuous always produces ensemble_size=5.0 regardless of input. + let bounds = DQNParams::continuous_bounds(); + let x: Vec = bounds.iter().map(|(lo, hi)| (lo + hi) / 2.0).collect(); let params = DQNParams::from_continuous(&x).expect("Should convert"); - assert_eq!(params.ensemble_size, 5.0, "5.3 should round to 5.0"); - - x[23] = 7.8; // Should round to 8.0 - let params = DQNParams::from_continuous(&x).expect("Should convert"); - assert_eq!(params.ensemble_size, 8.0, "7.8 should round to 8.0"); + assert_eq!(params.ensemble_size, 5.0, "Ensemble size should always be fixed to 5.0"); + assert!(!params.use_ensemble_uncertainty, "Ensemble uncertainty should be disabled"); }