diff --git a/crates/ml/src/dqn/dqn.rs b/crates/ml/src/dqn/dqn.rs index 617f29bc6..0104ffe5a 100644 --- a/crates/ml/src/dqn/dqn.rs +++ b/crates/ml/src/dqn/dqn.rs @@ -246,7 +246,7 @@ impl Default for DQNConfig { gradient_collapse_patience: 100, // Entropy regularization: prevent 45-action collapse - entropy_coefficient: 0.05, // 5x stronger anti-collapse for 45-action space + entropy_coefficient: 0.1, // Strong anti-collapse for 45-action factored space noisy_epsilon_floor: 0.05, use_count_bonus: true, count_bonus_coefficient: 0.1, @@ -3336,6 +3336,18 @@ impl DQN { self.epsilon } + /// Get the effective epsilon for exploration, respecting noisy_epsilon_floor. + /// In the single-sample path (select_action), this is handled inline. + /// This accessor exists for the batch training path (select_actions_batch) + /// which previously used get_epsilon() and got 0.0 with noisy nets — no exploration. + pub fn get_effective_epsilon(&self) -> f32 { + if self.config.use_noisy_nets { + self.config.noisy_epsilon_floor + } else { + self.epsilon + } + } + /// Set epsilon value (used for deterministic evaluation) pub fn set_epsilon(&mut self, epsilon: f64) { self.epsilon = epsilon.clamp(0.0, 1.0) as f32; diff --git a/crates/ml/src/hyperopt/adapters/dqn.rs b/crates/ml/src/hyperopt/adapters/dqn.rs index b69998809..7d3f906dc 100644 --- a/crates/ml/src/hyperopt/adapters/dqn.rs +++ b/crates/ml/src/hyperopt/adapters/dqn.rs @@ -404,7 +404,7 @@ impl Default for DQNParams { hold_penalty_weight: 0.01, // WAVE 10 Bug Fix: Align with CLI/RewardConfig defaults (was 2.0, causing 200x mismatch) max_position_absolute: 2.0, // BLOCKER #2: Default matches production (±2.0) huber_delta: 10.0, // Conservative default (hyperopt can scale to 15-40) - entropy_coefficient: 0.01, + entropy_coefficient: 0.1, // Stronger default for 45-action space transaction_cost_multiplier: 1.0, use_per: true, // P0: Default enabled for Rainbow DQN performance per_alpha: 0.6, // Rainbow DQN standard @@ -488,7 +488,7 @@ impl ParameterSpace for DQNParams { (1.0, 2.0), // 4: hold_penalty_weight (linear) (1.0, 4.0), // 5: max_position_absolute (linear) (10.0_f64.ln(), 40.0_f64.ln()), // 6: huber_delta (log scale: 10.0-40.0) - (0.01, 0.2), // 7: entropy_coefficient (WIDENED from [0.0,0.1] for 45-action factored space) + (0.05, 0.5), // 7: entropy_coefficient (WIDENED: 45-action space needs strong anti-collapse) (0.5, 2.0), // 8: transaction_cost_multiplier (linear) (0.4, 0.8), // 9: per_alpha (linear) (0.2, 0.6), // 10: per_beta_start (linear) @@ -543,13 +543,18 @@ impl ParameterSpace for DQNParams { // Minimum profit factor for trade execution (BUG #7: margin above breakeven) // 1.1 = 10% above costs, 2.0 = 100% above costs. Filters marginal trades. (1.1, 2.0), // 29: minimum_profit_factor (linear) + + // Count-based exploration bonus coefficient (1D) + // Controls UCB exploration bonus: β × sqrt(ln(N) / (1 + n_a)) + // Previously hardcoded at 0.1 — now searchable so TPE can tune exploration strength. + (0.05, 1.0), // 30: count_bonus_coefficient (linear) ] } fn from_continuous(x: &[f64]) -> Result { - if x.len() != 30 { + if x.len() != 31 { return Err(MLError::ConfigError { - reason: format!("Expected 30 continuous parameters, got {}", x.len()), + reason: format!("Expected 31 continuous parameters, got {}", x.len()), }); } @@ -607,7 +612,10 @@ impl ParameterSpace for DQNParams { let min_epochs_before_stopping = x[28].round().clamp(2.0, 6.0) as usize; let minimum_profit_factor = x[29].clamp(1.1, 2.0); - // === 16 FIXED parameters (validated defaults, removed from search) === + // Count-based exploration bonus (promoted from fixed to searchable) + let count_bonus_coefficient = x[30].clamp(0.05, 1.0); + + // === 15 FIXED parameters (validated defaults, removed from search) === let kelly_min_trades: usize = 20; let ensemble_size = 5.0; let beta_variance = 0.5; @@ -623,7 +631,6 @@ impl ParameterSpace for DQNParams { let activation_type = 1.0; // LeakyReLU let num_quantiles: usize = 64; let qr_kappa = 1.0; - let count_bonus_coefficient = 0.1; // variance_cap is not in DQNParams (fixed in DQNHyperparameters) // WAVE 6 FIX #2: Batch size floor for high learning rates @@ -709,7 +716,7 @@ impl ParameterSpace for DQNParams { } fn to_continuous(&self) -> Vec { - // 30D search space — only the tuned parameters + // 31D search space — only the tuned parameters // Fixed parameters are NOT emitted (they get their defaults in from_continuous) // warmup_ratio excluded: forced to 0.0 (batch training never increments total_steps) vec![ @@ -744,11 +751,13 @@ impl ParameterSpace for DQNParams { self.lr_decay_type, // 27 self.min_epochs_before_stopping as f64, // 28 self.minimum_profit_factor, // 29 + // Exploration (1D) + self.count_bonus_coefficient, // 30 ] } fn param_names() -> Vec<&'static str> { - // 30 tuned parameters (matches continuous_bounds / from_continuous / to_continuous) + // 31 tuned parameters (matches continuous_bounds / from_continuous / to_continuous) // warmup_ratio removed: forced to 0.0 (batch training never increments total_steps) vec![ "learning_rate", // 0 @@ -781,6 +790,7 @@ impl ParameterSpace for DQNParams { "lr_decay_type", // 27 "min_epochs_before_stopping", // 28 "minimum_profit_factor", // 29 + "count_bonus_coefficient", // 30 ] } @@ -792,7 +802,7 @@ impl ParameterSpace for DQNParams { batch_bound.1 = max_batch; } } - // Cap hidden_dim_base by VRAM (index 23 in 30D space) + // Cap hidden_dim_base by VRAM (index 23 in 31D space) let max_base = budget.max_hidden_dim_base(4, 256, 54, 45); if let Some(dim_bound) = bounds.get_mut(23) { dim_bound.1 = max_base as f64; @@ -3322,10 +3332,11 @@ impl HyperparameterOptimizable for DQNTrainer { // TRADE INSUFFICIENCY PENALTY: gives PSO gradient across degenerate plateau let trade_penalty = calculate_trade_insufficiency_penalty(backtest.total_trades); - // ACTION DIVERSITY SHORT-CIRCUIT: <10/45 unique actions = degenerate trial - // These produce phantom metrics (Sharpe 2317, 100% WR) by sitting in one - // exposure bucket. Composite score would be -369k, fooling TPE completely. - // Treat same as insufficient trades: return a graduated penalty. + // ACTION DIVERSITY PENALTY: smooth scaling across full 45-action range. + // - <10/45 unique actions: degenerate trial, short-circuit with heavy penalty + // (phantom metrics like Sharpe 2317, 100% WR from single exposure bucket) + // - 10-20/45: moderate penalty blended with composite score + // - >20/45: no penalty (healthy diversity) if backtest.unique_actions < 10 { let diversity_ratio = backtest.unique_actions as f64 / 10.0; // 8.0 for 1 action → 1.0 for 9 actions (same scale as trade penalty) @@ -3356,15 +3367,25 @@ impl HyperparameterOptimizable for DQNTrainer { // Component 2: HFT activity score (25% weight) let hft_activity = calculate_hft_activity_score_wave10(buy_pct, sell_pct, hold_pct); - // Combine: base objective + trade insufficiency penalty + // Graduated diversity penalty for 10-20 unique actions: + // 10 actions → 3.0 penalty, 20 actions → 0.0 penalty (linear ramp) + // >20 actions: no penalty (healthy diversity) + let mid_diversity_penalty = if backtest.unique_actions < 20 { + let ratio = (backtest.unique_actions as f64 - 10.0) / 10.0; // 0.0 at 10, 1.0 at 20 + 3.0 * (1.0 - ratio) + } else { + 0.0 + }; + + // Combine: base objective + trade insufficiency + diversity penalty let base_objective = -0.60 * composite_score + cvar_penalty + -0.25 * hft_activity + 0.15 * stability_penalty_raw; - let objective = base_objective + trade_penalty; + let objective = base_objective + trade_penalty + mid_diversity_penalty; info!( - "OBJECTIVE: {:.4} = base {:.4} + trade_penalty {:.4} | trades={} unique={}/45 composite={:.4}", - objective, base_objective, trade_penalty, backtest.total_trades, backtest.unique_actions, composite_score + "OBJECTIVE: {:.4} = base {:.4} + trade_penalty {:.4} + diversity_penalty {:.4} | trades={} unique={}/45 composite={:.4}", + objective, base_objective, trade_penalty, mid_diversity_penalty, backtest.total_trades, backtest.unique_actions, composite_score ); debug!( @@ -3439,7 +3460,7 @@ mod tests { hold_penalty_weight: 1.404, max_position_absolute: 2.5, huber_delta: 24.77, - entropy_coefficient: 0.05, // Widened range: [0.01, 0.2] + entropy_coefficient: 0.15, // Widened range: [0.05, 0.5] transaction_cost_multiplier: 1.0, use_per: true, per_alpha: 0.6, @@ -3491,10 +3512,10 @@ mod tests { }; let continuous = params.to_continuous(); - assert_eq!(continuous.len(), 30, "to_continuous must return 30D vector"); + assert_eq!(continuous.len(), 31, "to_continuous must return 31D vector"); let recovered = DQNParams::from_continuous(&continuous).unwrap(); - // Tuned parameters must roundtrip exactly (30D) + // Tuned parameters must roundtrip exactly (31D) assert!((recovered.learning_rate - params.learning_rate).abs() < 1e-6); assert_eq!(recovered.batch_size, params.batch_size); assert!((recovered.gamma - params.gamma).abs() < 1e-6); @@ -3528,13 +3549,13 @@ mod tests { assert!((recovered.activation_type - 1.0).abs() < 1e-6); // LeakyReLU assert_eq!(recovered.num_quantiles, 64); assert!((recovered.qr_kappa - 1.0).abs() < 1e-6); - assert!((recovered.count_bonus_coefficient - 0.1).abs() < 1e-6); + assert!((recovered.count_bonus_coefficient - params.count_bonus_coefficient).abs() < 1e-6); } #[test] fn test_dqn_params_bounds() { let bounds = DQNParams::continuous_bounds(); - assert_eq!(bounds.len(), 30); // 30D search space + assert_eq!(bounds.len(), 31); // 31D search space // Check log-scale bounds are reasonable assert!(bounds[0].0 < bounds[0].1); // learning_rate @@ -3556,7 +3577,7 @@ mod tests { assert_eq!(bounds[2], (0.95, 0.99)); // gamma assert_eq!(bounds[4], (1.0, 2.0)); // hold_penalty_weight assert_eq!(bounds[5], (1.0, 4.0)); // max_position_absolute - assert_eq!(bounds[7], (0.01, 0.2)); // entropy_coefficient (widened for 45-action space) + assert_eq!(bounds[7], (0.05, 0.5)); // entropy_coefficient (strong anti-collapse for 45-action space) assert_eq!(bounds[8], (0.5, 2.0)); // transaction_cost_multiplier assert_eq!(bounds[9], (0.4, 0.8)); // per_alpha assert_eq!(bounds[10], (0.2, 0.6)); // per_beta_start @@ -3583,7 +3604,7 @@ mod tests { #[test] fn test_param_names() { let names = DQNParams::param_names(); - assert_eq!(names.len(), 30); // 30D search space + assert_eq!(names.len(), 31); // 31D search space assert_eq!(names[0], "learning_rate"); assert_eq!(names[1], "batch_size"); assert_eq!(names[2], "gamma"); @@ -3614,14 +3635,15 @@ mod tests { assert_eq!(names[27], "lr_decay_type"); assert_eq!(names[28], "min_epochs_before_stopping"); assert_eq!(names[29], "minimum_profit_factor"); + assert_eq!(names[30], "count_bonus_coefficient"); } #[test] fn test_per_params_always_enabled() { // Test that PER is always enabled with tunable alpha/beta parameters - // 30D search space (warmup_ratio removed) + // 31D search space let continuous = vec![ - 3e-5_f64.ln(), 92.0, 0.9588, 97_273_f64.ln(), 1.404, 4.0, 24.77_f64.ln(), 0.05, 1.0, + 3e-5_f64.ln(), 92.0, 0.9588, 97_273_f64.ln(), 1.404, 4.0, 24.77_f64.ln(), 0.15, 1.0, 0.6, 0.4, // 9-10: per_alpha, per_beta_start -5.0, 5.0, 0.5_f64.ln(), // 11-13: v_min, v_max, noisy_sigma_init 256.0, 3.0, 101.0, // 14-16: dueling_hidden_dim, n_steps, num_atoms @@ -3637,6 +3659,7 @@ mod tests { 2.0, // 27: lr_decay_type (cosine) 4.0, // 28: min_epochs_before_stopping 1.5, // 29: minimum_profit_factor + 0.2, // 30: count_bonus_coefficient ]; let params = DQNParams::from_continuous(&continuous).unwrap(); @@ -3648,9 +3671,9 @@ mod tests { assert!(params.use_noisy_nets); assert!((params.warmup_ratio - 0.0).abs() < 1e-6); // Always fixed to 0.0 - // Test PER parameter bounds (min values) — 30D + // Test PER parameter bounds (min values) — 31D let continuous_min = vec![ - 2e-5_f64.ln(), 64.0, 0.95, 50_000_f64.ln(), 1.0, 1.0, 10.0_f64.ln(), 0.01, 0.5, + 2e-5_f64.ln(), 64.0, 0.95, 50_000_f64.ln(), 1.0, 1.0, 10.0_f64.ln(), 0.05, 0.5, 0.4, 0.2, // per_alpha min, per_beta_start min -15.0, 3.0, 0.1_f64.ln(), // v_min, v_max, noisy_sigma_init 128.0, 1.0, 51.0, // dueling_hidden_dim, n_steps, num_atoms @@ -3666,6 +3689,7 @@ mod tests { 0.0, // 27: lr_decay_type (constant) 2.0, // 28: min_epochs_before_stopping min 1.1, // 29: minimum_profit_factor min + 0.05, // 30: count_bonus_coefficient min ]; let params_min = DQNParams::from_continuous(&continuous_min).unwrap(); assert!((params_min.per_alpha - 0.4).abs() < 1e-6); @@ -3674,9 +3698,9 @@ mod tests { assert!(params_min.use_distributional); assert!(params_min.use_noisy_nets); - // Test PER parameter bounds (max values) — 30D + // Test PER parameter bounds (max values) — 31D let continuous_max = vec![ - 8e-5_f64.ln(), 4096.0, 0.99, 100_000_f64.ln(), 2.0, 4.0, 40.0_f64.ln(), 0.2, 2.0, + 8e-5_f64.ln(), 4096.0, 0.99, 100_000_f64.ln(), 2.0, 4.0, 40.0_f64.ln(), 0.5, 2.0, 0.8, 0.6, // per_alpha max, per_beta_start max -3.0, 15.0, 1.0_f64.ln(), // v_min, v_max, noisy_sigma_init 512.0, 5.0, 201.0, // dueling_hidden_dim, n_steps, num_atoms @@ -3692,6 +3716,7 @@ mod tests { 2.0, // 27: lr_decay_type (cosine) 6.0, // 28: min_epochs_before_stopping max 2.0, // 29: minimum_profit_factor max + 1.0, // 30: count_bonus_coefficient max ]; let params_max = DQNParams::from_continuous(&continuous_max).unwrap(); assert!((params_max.per_alpha - 0.8).abs() < 1e-6); @@ -3991,7 +4016,7 @@ mod tests { fn test_qr_dqn_roundtrip_continuous() { let params = DQNParams::default(); let continuous = params.to_continuous(); - assert_eq!(continuous.len(), 30, "Should have 30 continuous dimensions"); + assert_eq!(continuous.len(), 31, "Should have 31 continuous dimensions"); let roundtrip = DQNParams::from_continuous(&continuous).unwrap(); // num_quantiles and qr_kappa are now fixed defaults (not in search space) assert_eq!(roundtrip.num_quantiles, 64); // Fixed default @@ -4016,7 +4041,7 @@ mod tests { fn test_batch_size_respects_wide_bounds() { // Simulate PSO choosing batch_size=2048 (within VRAM-aware bounds) let bounds = DQNParams::continuous_bounds(); - let mut params = vec![0.0_f64; 30]; + let mut params = vec![0.0_f64; 31]; params[1] = 2048.0; // batch_size (index 1) // Fill other required params with valid defaults params[0] = (1e-4_f64).ln(); // learning_rate @@ -4025,12 +4050,12 @@ mod tests { params[4] = 1.5; // hold_penalty_weight params[5] = 3.0; // max_position_absolute params[6] = (25.0_f64).ln(); // huber_delta - params[7] = 0.05; // entropy_coefficient + params[7] = 0.15; // entropy_coefficient (within 0.05..0.5 range) params[8] = 0.5; // transaction_cost_multiplier params[9] = 0.6; // per_alpha params[10] = 0.4; // per_beta_start - // Remaining params (indices 11-29) -- use midpoint of bounds - for i in 11..30 { + // Remaining params (indices 11-30) -- use midpoint of bounds + for i in 11..31 { params[i] = (bounds[i].0 + bounds[i].1) / 2.0; } let result = DQNParams::from_continuous(¶ms).unwrap(); @@ -4154,9 +4179,9 @@ mod tests { ); // from_continuous should clamp to 0.5 even if value is below - let mut low_temp_vec = vec![0.0_f64; 30]; + let mut low_temp_vec = vec![0.0_f64; 31]; // Set all to midpoint of bounds - for i in 0..30 { + for i in 0..31 { low_temp_vec[i] = (bounds[i].0 + bounds[i].1) / 2.0; } low_temp_vec[26] = 0.01_f64.ln(); // Way below floor diff --git a/crates/ml/src/hyperopt/optimizer.rs b/crates/ml/src/hyperopt/optimizer.rs index 7cc227dd5..e47c8367b 100644 --- a/crates/ml/src/hyperopt/optimizer.rs +++ b/crates/ml/src/hyperopt/optimizer.rs @@ -1335,7 +1335,7 @@ impl ArgminOptimizerBuilder { } /// Set model name for per-trial Prometheus progress metrics - pub fn model_name(mut self, name: impl Into) -> Self { + pub fn model_name>(mut self, name: S) -> Self { self.model_name = Some(name.into()); self } diff --git a/crates/ml/src/trainers/dqn/config.rs b/crates/ml/src/trainers/dqn/config.rs index dbf9eeea1..19b6ac7af 100644 --- a/crates/ml/src/trainers/dqn/config.rs +++ b/crates/ml/src/trainers/dqn/config.rs @@ -156,6 +156,16 @@ impl DQNAgentType { } } + /// Get the effective epsilon for exploration, respecting noisy_epsilon_floor. + /// The batch training path must use this instead of get_epsilon() to avoid + /// zero exploration when noisy nets are enabled. + pub fn get_effective_epsilon(&self) -> f32 { + match self { + Self::Standard(agent) => agent.get_effective_epsilon(), + Self::RegimeConditional(agent) => agent.primary_head().get_effective_epsilon(), + } + } + /// Set epsilon value for exploration (all heads for regime-conditional) pub fn set_epsilon(&mut self, epsilon: f64) { match self { diff --git a/crates/ml/src/trainers/dqn/trainer.rs b/crates/ml/src/trainers/dqn/trainer.rs index b68066cba..07d8da38f 100644 --- a/crates/ml/src/trainers/dqn/trainer.rs +++ b/crates/ml/src/trainers/dqn/trainer.rs @@ -3250,8 +3250,10 @@ impl DQNTrainer { .to_dtype(training_dtype(&self.device)) .map_err(|e| anyhow::anyhow!("Failed to cast batched state tensor to training dtype: {}", e))?; - // WAVE 16S: Get volatility-adjusted epsilon for exploration - let base_epsilon = agent.get_epsilon() as f64; + // FIX: Use get_effective_epsilon() which respects noisy_epsilon_floor. + // Previously used get_epsilon() which returns the decayed epsilon (0.0 with noisy nets), + // making the batch path have ZERO random exploration — root cause of action collapse. + let base_epsilon = agent.get_effective_epsilon() as f64; let adjusted_epsilon = self.calculate_volatility_adjusted_epsilon(base_epsilon); let epsilon = adjusted_epsilon as f32; @@ -3323,8 +3325,10 @@ impl DQNTrainer { let agent = self.agent.read().await; - // WAVE 16S: Volatility-adjusted epsilon - let base_epsilon = agent.get_epsilon() as f64; + // FIX: Use get_effective_epsilon() which respects noisy_epsilon_floor. + // Same fix as select_actions_batch() — previously used get_epsilon() which + // returned 0.0 with noisy nets, causing zero exploration in GPU batch path. + let base_epsilon = agent.get_effective_epsilon() as f64; let adjusted_epsilon = self.calculate_volatility_adjusted_epsilon(base_epsilon); let epsilon = adjusted_epsilon as f32; debug!("Epsilon (GPU path): base={:.4}, volatility-adjusted={:.4}", base_epsilon, adjusted_epsilon);