From b0a51468857fb98d46d991985e5e64e2b47308ea Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Fri, 28 Nov 2025 09:29:38 +0100 Subject: [PATCH] feat(ml): Add weight_decay L2 regularization to DQN (P0.1 fix) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements critical P0.1 gap from DQN 2025 upgrade roadmap to address overfitting in hyperopt/trainer by enabling proper weight decay in AdamW. Changes: - Add weight_decay field to DQNHyperparameters (default: 1e-4) - Wire weight_decay to AdamW via Decay::DecoupledWeightDecay at 3 optimizer initialization points in agent.rs - Expand hyperopt search space to 40D with weight_decay [1e-5, 1e-3] log scale - Update from_continuous(), to_continuous(), param_names() for roundtrip This enables the modern best practice of decoupled weight decay (AdamW paper) which prevents co-adaptation of network weights and improves generalization. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- ml/src/dqn/agent.rs | 14 ++- ml/src/hyperopt/adapters/dqn.rs | 164 ++++++++++++++++++++------------ ml/src/trainers/dqn/config.rs | 9 ++ 3 files changed, 125 insertions(+), 62 deletions(-) diff --git a/ml/src/dqn/agent.rs b/ml/src/dqn/agent.rs index 6b86c2645..f4b53355c 100644 --- a/ml/src/dqn/agent.rs +++ b/ml/src/dqn/agent.rs @@ -180,6 +180,9 @@ pub struct DQNConfig { pub minimum_profit_factor: f32, /// BUG #4 FIX: Soft target update coefficient (Polyak averaging rate, default 0.005) pub tau: f64, + /// WAVE 30: L2 weight decay for AdamW optimizer (default: 1e-4, range: [1e-5, 1e-3]) + /// Prevents overfitting by penalizing large weights via L2 regularization. + pub weight_decay: f64, } impl Default for DQNConfig { @@ -198,6 +201,7 @@ impl Default for DQNConfig { epsilon_decay: 0.995, minimum_profit_factor: 1.5, // BUG #7 FIX: 50% margin above breakeven tau: 0.005, // BUG #4 FIX: Polyak averaging coefficient (0.5% new weights per update) + weight_decay: 1e-4, // WAVE 30: Standard L2 regularization strength } } } @@ -337,12 +341,13 @@ impl DQNAgent { // Initialize optimizer if not already done if self.optimizer.is_none() { + use candle_optimisers::Decay; let adam_params = ParamsAdam { lr: self.config.learning_rate, beta_1: 0.9, beta_2: 0.999, eps: 1e-8, - weight_decay: None, + weight_decay: Some(Decay::DecoupledWeightDecay(self.config.weight_decay)), amsgrad: false, }; self.optimizer = Some( @@ -739,12 +744,13 @@ impl DQNAgent { self.q_network.set_epsilon(checkpoint.epsilon); // Re-initialize optimizer with loaded parameters + use candle_optimisers::Decay; let adam_params = ParamsAdam { lr: self.config.learning_rate, beta_1: 0.9, beta_2: 0.999, eps: 1e-8, - weight_decay: None, + weight_decay: Some(Decay::DecoupledWeightDecay(self.config.weight_decay)), amsgrad: false, }; self.optimizer = Some( @@ -781,12 +787,13 @@ impl DQNAgent { let new_lr = current_lr * decay_factor; // Recreate optimizer with new learning rate + use candle_optimisers::Decay; let adam_params = ParamsAdam { lr: new_lr, beta_1: 0.9, beta_2: 0.999, eps: 1e-8, - weight_decay: None, + weight_decay: Some(Decay::DecoupledWeightDecay(self.config.weight_decay)), amsgrad: false, }; @@ -1316,6 +1323,7 @@ mod tests { epsilon_decay: 0.99, minimum_profit_factor: 1.5, tau: 0.005, + weight_decay: 1e-4, }; let agent = DQNAgent::new(config.clone())?; diff --git a/ml/src/hyperopt/adapters/dqn.rs b/ml/src/hyperopt/adapters/dqn.rs index 80f497f16..71453691b 100644 --- a/ml/src/hyperopt/adapters/dqn.rs +++ b/ml/src/hyperopt/adapters/dqn.rs @@ -248,7 +248,11 @@ pub struct DQNParams { /// BUG #7: Profit must exceed cost by this factor (1.5 = 50% margin above breakeven) /// Protects against marginal trades vulnerable to slippage pub minimum_profit_factor: f64, - + + /// Weight decay (L2 regularization) for Adam optimizer (log-scale: 1e-5 to 1e-3) + /// Prevents overfitting by penalizing large weights + pub weight_decay: f64, + // WAVE 19: Tunable Kelly risk parameters pub kelly_fractional: f64, pub kelly_max_fraction: f64, @@ -359,6 +363,7 @@ impl Default for DQNParams { use_noisy_nets: true, // Wave 8: Default ENABLED for full Rainbow DQN noisy_sigma_init: 0.5, // Wave 2.4: Rainbow DQN standard minimum_profit_factor: 1.5, // BUG #7: Default 50% margin above breakeven + weight_decay: 1e-4, // Default: 0.0001 (standard L2 regularization strength) kelly_fractional: 0.5, // WAVE 19: Default half-Kelly kelly_max_fraction: 0.25, // WAVE 19: Default 25% max position kelly_min_trades: 20, // WAVE 19: Default 20 trades minimum @@ -392,11 +397,12 @@ impl Default for DQNParams { impl ParameterSpace for DQNParams { fn continuous_bounds() -> Vec<(f64, f64)> { - // BUG #7: Expanded to 18D continuous space (added minimum_profit_factor) + // CRITICAL GAP FIX: Expanded to 40D continuous space (added weight_decay) // Base parameters (11D from Wave 1-2): LR, batch, gamma, buffer, hold_penalty, max_pos, huber, entropy, tx_cost, per_alpha, per_beta // Rainbow extensions (6D): v_min, v_max, noisy_sigma_init, dueling_hidden_dim, n_steps, num_atoms // NOTE: v_min/v_max/num_atoms still tunable but UNUSED (C51 disabled due to BUG #36) // Bug #7 addition (1D): minimum_profit_factor + // Weight decay addition (1D): weight_decay (L2 regularization) // Rainbow booleans: use_dueling=TRUE, use_distributional=FALSE (BUG #36), use_noisy_nets=TRUE // // WAVE 26 P1.5 FIX: Learning rate range EXPANDED to include production default 1e-4 @@ -430,52 +436,55 @@ impl ParameterSpace for DQNParams { // BUG #7: Minimum profit threshold (1D) (1.1, 2.0), // 17: minimum_profit_factor (linear) - Profit margin requirement - // WAVE 19: Kelly risk parameters (18D → 22D) - (0.25, 1.0), // 18: kelly_fractional - (0.1, 0.5), // 19: kelly_max_fraction - (10.0, 50.0), // 20: kelly_min_trades - (10.0, 30.0), // 21: volatility_window + // Weight decay (L2 regularization) (1D) + (1e-5_f64.ln(), 1e-3_f64.ln()), // 18: weight_decay (log scale) - L2 regularization strength - // WAVE 26 P1.4: Ensemble Uncertainty (22D → 27D) - (3.0, 10.0), // 22: ensemble_size (will be rounded to int) - (0.1, 1.0), // 23: beta_variance - (0.1, 1.0), // 24: beta_disagreement - (0.05, 0.5), // 25: beta_entropy - (0.1, 2.0), // 26: variance_cap (fixed in DQNHyperparameters, not tuned per-trial) + // WAVE 19: Kelly risk parameters (19D → 23D) + (0.25, 1.0), // 19: kelly_fractional + (0.1, 0.5), // 20: kelly_max_fraction + (10.0, 50.0), // 21: kelly_min_trades + (10.0, 30.0), // 22: volatility_window - // WAVE 26 P1.5: Learning rate warmup ratio (27D → 28D) - (0.0, 0.2), // 27: warmup_ratio (0-20% warmup) + // WAVE 26 P1.4: Ensemble Uncertainty (23D → 28D) + (3.0, 10.0), // 23: ensemble_size (will be rounded to int) + (0.1, 1.0), // 24: beta_variance + (0.1, 1.0), // 25: beta_disagreement + (0.05, 0.5), // 26: beta_entropy + (0.1, 2.0), // 27: variance_cap (fixed in DQNHyperparameters, not tuned per-trial) - // WAVE 26 P1.8: Curiosity-driven exploration (28D → 29D) - (0.0, 0.5), // 28: curiosity_weight (intrinsic reward scaling) + // WAVE 26 P1.5: Learning rate warmup ratio (28D → 29D) + (0.0, 0.2), // 28: warmup_ratio (0-20% warmup) - // WAVE 26 P1.12: Polyak soft update coefficient (29D → 30D) - (0.0001_f64.ln(), 0.01_f64.ln()), // 29: tau (log scale, 0.0001-0.01, Rainbow default: 0.001) + // WAVE 26 P1.8: Curiosity-driven exploration (29D → 30D) + (0.0, 0.5), // 29: curiosity_weight (intrinsic reward scaling) - // WAVE 26 P0: TD Error and Batch Diversity (30D → 32D) - (1.0, 100.0), // 30: td_error_clamp_max (linear, prevents extreme TD errors) - (10.0, 100.0), // 31: batch_diversity_cooldown (linear, diversity sampling frequency) + // WAVE 26 P1.12: Polyak soft update coefficient (30D → 31D) + (0.0001_f64.ln(), 0.01_f64.ln()), // 30: tau (log scale, 0.0001-0.01, Rainbow default: 0.001) - // WAVE 26 P1: Advanced Training Parameters (32D → 37D) - (0.0, 2.0), // 32: lr_decay_type (0=constant, 1=linear, 2=cosine) - (0.0, 0.5), // 33: sharpe_weight (risk-adjusted return weight) - (0.9, 0.99), // 34: gae_lambda (GAE bias-variance tradeoff) - (0.4, 0.8), // 35: noisy_sigma_initial (initial exploration noise) - (0.2, 0.5), // 36: noisy_sigma_final (final exploration noise) + // WAVE 26 P0: TD Error and Batch Diversity (31D → 33D) + (1.0, 100.0), // 31: td_error_clamp_max (linear, prevents extreme TD errors) + (10.0, 100.0), // 32: batch_diversity_cooldown (linear, diversity sampling frequency) - // WAVE 26 P1: Network Architecture (37D → 39D) + // WAVE 26 P1: Advanced Training Parameters (33D → 38D) + (0.0, 2.0), // 33: lr_decay_type (0=constant, 1=linear, 2=cosine) + (0.0, 0.5), // 34: sharpe_weight (risk-adjusted return weight) + (0.9, 0.99), // 35: gae_lambda (GAE bias-variance tradeoff) + (0.4, 0.8), // 36: noisy_sigma_initial (initial exploration noise) + (0.2, 0.5), // 37: noisy_sigma_final (final exploration noise) + + // WAVE 26 P1: Network Architecture (38D → 40D) // Note: Booleans (use_spectral_norm, use_attention, use_residual) NOT in search space // They default to false and can be enabled via CLI or config - (0.0, 2.0), // 37: norm_type (0=LayerNorm, 1=RMSNorm, 2=None) - (0.0, 3.0), // 38: activation_type (0=ReLU, 1=LeakyReLU, 2=GELU, 3=Mish) + (0.0, 2.0), // 38: norm_type (0=LayerNorm, 1=RMSNorm, 2=None) + (0.0, 3.0), // 39: activation_type (0=ReLU, 1=LeakyReLU, 2=GELU, 3=Mish) // WAVE 11: Rainbow DQN boolean parameters REMOVED from search space (always TRUE) ] } fn from_continuous(x: &[f64]) -> Result { - if x.len() != 39 { + if x.len() != 40 { return Err(MLError::ConfigError { - reason: format!("Expected 39 continuous parameters (WAVE 26: full integration), got {}", x.len()), + reason: format!("Expected 40 continuous parameters (CRITICAL GAP FIX: added weight_decay), got {}", x.len()), }); } @@ -506,42 +515,45 @@ impl ParameterSpace for DQNParams { // BUG #7: Minimum profit factor (18th parameter) let minimum_profit_factor = x[17].clamp(1.1, 2.0); - // WAVE 19: Extract Kelly parameters - let kelly_fractional = x[18].clamp(0.25, 1.0); - let kelly_max_fraction = x[19].clamp(0.1, 0.5); - let kelly_min_trades = x[20].round().clamp(10.0, 50.0) as usize; - let volatility_window = x[21].round().clamp(10.0, 30.0) as usize; + // CRITICAL GAP FIX: Weight decay (L2 regularization) (19th parameter) + let weight_decay = x[18].exp().clamp(1e-5, 1e-3); - // WAVE 26 P1.4: Extract ensemble uncertainty parameters - let ensemble_size = x[22].round().clamp(3.0, 10.0); - let beta_variance = x[23].clamp(0.1, 1.0); - let beta_disagreement = x[24].clamp(0.1, 1.0); - let beta_entropy = x[25].clamp(0.05, 0.5); - // Note: x[26] is variance_cap, but it's NOT in DQNParams (fixed in DQNHyperparameters) + // WAVE 19: Extract Kelly parameters (shifted by +1 due to weight_decay) + let kelly_fractional = x[19].clamp(0.25, 1.0); + let kelly_max_fraction = x[20].clamp(0.1, 0.5); + let kelly_min_trades = x[21].round().clamp(10.0, 50.0) as usize; + let volatility_window = x[22].round().clamp(10.0, 30.0) as usize; + + // WAVE 26 P1.4: Extract ensemble uncertainty parameters (shifted by +1 due to weight_decay) + let ensemble_size = x[23].round().clamp(3.0, 10.0); + let beta_variance = x[24].clamp(0.1, 1.0); + let beta_disagreement = x[25].clamp(0.1, 1.0); + let beta_entropy = x[26].clamp(0.05, 0.5); + // Note: x[27] is variance_cap, but it's NOT in DQNParams (fixed in DQNHyperparameters) // WAVE 26 P1.5: Extract warmup ratio - let warmup_ratio = x[27].clamp(0.0, 0.2); + let warmup_ratio = x[28].clamp(0.0, 0.2); // WAVE 26 P1.8: Extract curiosity weight - let curiosity_weight = x[28].clamp(0.0, 0.5); + let curiosity_weight = x[29].clamp(0.0, 0.5); // WAVE 26 P1.12: Extract tau (Polyak soft update coefficient) - let tau = x[29].exp().clamp(0.0001, 0.01); // Log scale: 0.0001-0.01, default: 0.001 + let tau = x[30].exp().clamp(0.0001, 0.01); // Log scale: 0.0001-0.01, default: 0.001 // WAVE 26 P0: Extract TD error and batch diversity parameters - let td_error_clamp_max = x[30].clamp(1.0, 100.0); - let batch_diversity_cooldown = x[31].clamp(10.0, 100.0); + let td_error_clamp_max = x[31].clamp(1.0, 100.0); + let batch_diversity_cooldown = x[32].clamp(10.0, 100.0); // WAVE 26 P1: Extract advanced training parameters - let lr_decay_type = x[32].round().clamp(0.0, 2.0); // 0=constant, 1=linear, 2=cosine - let sharpe_weight = x[33].clamp(0.0, 0.5); - let gae_lambda = x[34].clamp(0.9, 0.99); - let noisy_sigma_initial = x[35].clamp(0.4, 0.8); - let noisy_sigma_final = x[36].clamp(0.2, 0.5); + let lr_decay_type = x[33].round().clamp(0.0, 2.0); // 0=constant, 1=linear, 2=cosine + let sharpe_weight = x[34].clamp(0.0, 0.5); + let gae_lambda = x[35].clamp(0.9, 0.99); + let noisy_sigma_initial = x[36].clamp(0.4, 0.8); + let noisy_sigma_final = x[37].clamp(0.2, 0.5); // WAVE 26 P1: Extract network architecture parameters - let norm_type = x[37].round().clamp(0.0, 2.0); // 0=LayerNorm, 1=RMSNorm, 2=None - let activation_type = x[38].round().clamp(0.0, 3.0); // 0=ReLU, 1=LeakyReLU, 2=GELU, 3=Mish + let norm_type = x[38].round().clamp(0.0, 2.0); // 0=LayerNorm, 1=RMSNorm, 2=None + let activation_type = x[39].round().clamp(0.0, 3.0); // 0=ReLU, 1=LeakyReLU, 2=GELU, 3=Mish // WAVE 11: Rainbow DQN boolean parameters are ALWAYS TRUE (removed from search space) // User requirement: "I want them enabled!" - no point in tuning boolean flags @@ -590,6 +602,7 @@ impl ParameterSpace for DQNParams { use_noisy_nets: true, // WAVE 11: Always enabled for full Rainbow DQN noisy_sigma_init, minimum_profit_factor, // BUG #7: Configurable profit margin (1.1-2.0) + weight_decay, // CRITICAL GAP FIX: Now tunable in hyperopt search space (1e-5 to 1e-3) // WAVE 19: Kelly risk parameters kelly_fractional, kelly_max_fraction, @@ -649,7 +662,9 @@ impl ParameterSpace for DQNParams { self.num_atoms as f64, // Distributional atoms count // BUG #7: Minimum profit factor (18D) self.minimum_profit_factor, - // WAVE 19: Kelly risk parameters (22D) + // CRITICAL GAP FIX: Weight decay (19D) + self.weight_decay.ln(), + // WAVE 19: Kelly risk parameters (23D) self.kelly_fractional, self.kelly_max_fraction, self.kelly_min_trades as f64, @@ -662,6 +677,19 @@ impl ParameterSpace for DQNParams { 1.0, // variance_cap placeholder (not in DQNParams, fixed in DQNHyperparameters) self.warmup_ratio, self.curiosity_weight, + self.tau.ln(), // WAVE 26 P1.12: Polyak soft update coefficient (log scale) + // WAVE 26 P0: TD error and batch diversity + self.td_error_clamp_max, + self.batch_diversity_cooldown, + // WAVE 26 P1: Advanced training parameters + self.lr_decay_type, + self.sharpe_weight, + self.gae_lambda, + self.noisy_sigma_initial, + self.noisy_sigma_final, + // WAVE 26 P1: Network architecture + self.norm_type, + self.activation_type, // WAVE 11: Rainbow DQN boolean parameters REMOVED (always TRUE, not tunable) ] } @@ -688,7 +716,9 @@ impl ParameterSpace for DQNParams { "num_atoms", // Distributional atoms count // BUG #7: Minimum profit factor (18D) "minimum_profit_factor", - // WAVE 19: Kelly risk parameters (22D) + // CRITICAL GAP FIX: Weight decay (19D) + "weight_decay", + // WAVE 19: Kelly risk parameters (23D) "kelly_fractional", "kelly_max_fraction", "kelly_min_trades", @@ -701,6 +731,19 @@ impl ParameterSpace for DQNParams { "variance_cap", "warmup_ratio", "curiosity_weight", + "tau", // WAVE 26 P1.12: Polyak soft update coefficient + // WAVE 26 P0: TD error and batch diversity + "td_error_clamp_max", + "batch_diversity_cooldown", + // WAVE 26 P1: Advanced training parameters + "lr_decay_type", + "sharpe_weight", + "gae_lambda", + "noisy_sigma_initial", + "noisy_sigma_final", + // WAVE 26 P1: Network architecture + "norm_type", + "activation_type", // WAVE 11: Rainbow DQN boolean parameters REMOVED (always TRUE, not tunable) ] } @@ -2074,6 +2117,9 @@ impl HyperparameterOptimizable for DQNTrainer { lr_decay_steps: 1000, // Steps for LR decay lr_decay_rate: 0.99, // LR decay rate + // CRITICAL GAP FIX: Weight decay (L2 regularization) - now tunable via hyperopt + weight_decay: params.weight_decay, + // WAVE 26 P1.4: Ensemble Uncertainty Exploration (tunable via hyperopt) use_ensemble_uncertainty: params.use_ensemble_uncertainty, ensemble_size: params.ensemble_size.round() as usize, // Cast f64 to usize @@ -3008,7 +3054,7 @@ mod tests { #[test] fn test_param_names() { let names = DQNParams::param_names(); - assert_eq!(names.len(), 28); // WAVE 26 P1.5: 28 tunable hyperparameters (22 + 5 ensemble + 1 warmup_ratio) + assert_eq!(names.len(), 40); // CRITICAL GAP FIX: 40 tunable hyperparameters (added weight_decay + all WAVE 26 params) assert_eq!(names[0], "learning_rate"); assert_eq!(names[1], "batch_size"); assert_eq!(names[2], "gamma"); diff --git a/ml/src/trainers/dqn/config.rs b/ml/src/trainers/dqn/config.rs index 60c03c26f..5d5772d0b 100644 --- a/ml/src/trainers/dqn/config.rs +++ b/ml/src/trainers/dqn/config.rs @@ -532,6 +532,12 @@ pub struct DQNHyperparameters { pub noisy_sigma_final: f64, /// Steps for sigma annealing (default: 10000) pub noisy_sigma_anneal_steps: usize, + + // WAVE 30: L2 Weight Decay for Overfitting Prevention + /// L2 weight decay for AdamW optimizer (default: 1e-4, range: [1e-5, 1e-3]) + /// Prevents overfitting by penalizing large weights via L2 regularization. + /// Recommended: 1e-4 for standard training, 1e-3 for aggressive regularization. + pub weight_decay: f64, } impl Default for DQNHyperparameters { @@ -698,6 +704,9 @@ impl DQNHyperparameters { noisy_sigma_initial: 0.6, // Default: 60% initial noise noisy_sigma_final: 0.4, // Default: 40% final noise noisy_sigma_anneal_steps: 10000, // Default: 10K steps for annealing + + // WAVE 30: L2 Weight Decay + weight_decay: 1e-4, // Default: 0.0001 (standard regularization strength) } } }