diff --git a/crates/ml/src/trainers/dqn/config.rs b/crates/ml/src/trainers/dqn/config.rs index 03821ad80..4582ac57a 100644 --- a/crates/ml/src/trainers/dqn/config.rs +++ b/crates/ml/src/trainers/dqn/config.rs @@ -1342,6 +1342,11 @@ impl DQNHyperparameters { beta_penalty_strength: 0.3, // 30% reward reduction at full correlation // Gems & Pearls: generalization techniques + // Anti-LR multipliers retained at original 3.0/0.3 — the noise + // amplification problem (grad_norm spikes to 1.16M when Sharpe + // oscillates epoch-to-epoch, train-br8cb Fold 0) is now fixed by + // feeding the controller a temporally smoothed Sharpe instead of + // the raw last-epoch value. See training_loop.rs anti-LR block. anti_lr_good_mult: 3.0, // 3x LR when Sharpe is good (destabilize overfit) anti_lr_bad_mult: 0.3, // 0.3x LR when Sharpe is bad (stabilize) anti_lr_sharpe_threshold: 0.3, // Sharpe threshold for switching diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index 63d2195f0..e8417bee0 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -2476,20 +2476,39 @@ impl DQNTrainer { self.lr_scheduler.step(); let mut current_lr = self.lr_scheduler.get_lr(); - // #24 Anti-intuitive LR: use PREVIOUS epoch's Sharpe to adjust LR. - // When model was doing well, INCREASE LR to kick out of overfit minima. - // When struggling, decrease to stabilize. Opposite of standard practice. - // #24 Anti-intuitive LR: always active (one production path) - // Guard: skip anti-LR for first 5 epochs — early Sharpe is unreliable - // (random policy, tiny replay buffer, Sharpe from few trades). + // #24 Anti-intuitive LR: use a TEMPORALLY SMOOTHED Sharpe to adjust LR. + // When the model has been doing well, INCREASE LR to kick out of overfit + // minima. When struggling, decrease to stabilize. Opposite of standard + // practice. Always active. + // + // Previously this used `prev_sharpe = sharpe_history.last()`, feeding a + // single noisy epoch Sharpe into the controller. RL Sharpe oscillates + // widely epoch-to-epoch (observed: −20 / +30 swings), so the controller + // kept flipping multipliers and amplified its own input noise — + // gradient norms spiked to 1.16M on Fold 0 (train-br8cb). The raw + // multipliers (3.0 / 0.3) work fine when the controller sees a stable + // signal; the problem was the signal, not the magnitudes. + // + // Fix: feed a rolling-mean Sharpe over the same window as the warmup + // (reuses the one knob already in the code — no new hyperparameter). + // The window naturally filters per-epoch oscillation at the frequency + // the anti-LR logic wants to react on (multi-epoch trends), while + // letting genuine sustained improvement/degradation still trigger + // adjustments. let anti_lr_warmup = 5; if epoch >= anti_lr_warmup && !self.sharpe_history.is_empty() { - let prev_sharpe = self.sharpe_history.last().copied().unwrap_or(0.0); + let window = anti_lr_warmup.min(self.sharpe_history.len()); + let smoothed_sharpe: f64 = self.sharpe_history + .iter() + .rev() + .take(window) + .sum::() + / window as f64; let thresh = self.hyperparams.anti_lr_sharpe_threshold; let base_lr = self.lr_scheduler.get_initial_lr(); - let anti_mult = if prev_sharpe > thresh { + let anti_mult = if smoothed_sharpe > thresh { self.hyperparams.anti_lr_good_mult - } else if prev_sharpe < -thresh { + } else if smoothed_sharpe < -thresh { self.hyperparams.anti_lr_bad_mult } else { 1.0 @@ -2497,7 +2516,9 @@ impl DQNTrainer { current_lr = (current_lr * anti_mult).clamp(base_lr * 0.1, base_lr * 5.0); if (anti_mult - 1.0).abs() > 0.01 { info!( - epoch = epoch + 1, prev_sharpe = %format!("{:.3}", prev_sharpe), + epoch = epoch + 1, + smoothed_sharpe = %format!("{:.3}", smoothed_sharpe), + window = window, anti_mult = %format!("{:.1}x", anti_mult), lr = %format!("{:.2e}", current_lr), "Anti-intuitive LR adjustment"