diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index e8417bee0..e6d61d3c8 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -2476,39 +2476,59 @@ impl DQNTrainer { self.lr_scheduler.step(); let mut current_lr = self.lr_scheduler.get_lr(); - // #24 Anti-intuitive LR: use a TEMPORALLY SMOOTHED Sharpe to adjust LR. - // When the model has been doing well, INCREASE LR to kick out of overfit - // minima. When struggling, decrease to stabilize. Opposite of standard - // practice. Always active. + // #24 Anti-intuitive LR: ASYMMETRIC temporal filter. // - // Previously this used `prev_sharpe = sharpe_history.last()`, feeding a - // single noisy epoch Sharpe into the controller. RL Sharpe oscillates - // widely epoch-to-epoch (observed: −20 / +30 swings), so the controller - // kept flipping multipliers and amplified its own input noise — - // gradient norms spiked to 1.16M on Fold 0 (train-br8cb). The raw - // multipliers (3.0 / 0.3) work fine when the controller sees a stable - // signal; the problem was the signal, not the magnitudes. + // When training has been doing well, INCREASE LR to escape overfit + // minima; when struggling, DECREASE LR to stabilize. Original impl fed + // on raw last-epoch Sharpe — noisy, caused 1M+ grad spikes as the + // controller flipped every epoch (train-br8cb Fold 0). // - // Fix: feed a rolling-mean Sharpe over the same window as the warmup - // (reuses the one knob already in the code — no new hyperparameter). - // The window naturally filters per-epoch oscillation at the frequency - // the anti-LR logic wants to react on (multi-epoch trends), while - // letting genuine sustained improvement/degradation still trigger - // adjustments. + // A symmetric 5-window mean killed the noise but also killed exploration + // (train-v82b2): RL Sharpe is plentiful-bad and rare-good in early + // epochs, so the mean stays negative even when individual epochs find + // real improvements. The controller locked into 0.3× LR and never let + // the model kick out of its initial plateau. + // + // Correct structure is asymmetric: + // Good signal → fast response (short window, take max) + // — a single genuinely good epoch is meaningful and + // rare; don't wait for it to be sustained before + // letting the model exploit it. + // Bad signal → slow response (long window, take mean) + // — bad epochs are noisy and common; only dampen LR + // when we're confident the model is stuck. + // + // Both windows derive from `anti_lr_warmup` (one knob for the whole + // controller): full window for bad, half window for good. No new + // hyperparameter — the asymmetry is structural, not tuned. let anti_lr_warmup = 5; if epoch >= anti_lr_warmup && !self.sharpe_history.is_empty() { - let window = anti_lr_warmup.min(self.sharpe_history.len()); - let smoothed_sharpe: f64 = self.sharpe_history + let history_len = self.sharpe_history.len(); + let long_window = anti_lr_warmup.min(history_len); + let short_window = (anti_lr_warmup / 2).max(1).min(history_len); + + // Fast detector: best of recent few — responsive to new gains. + let recent_max: f64 = self.sharpe_history .iter() .rev() - .take(window) - .sum::() - / window as f64; + .take(short_window) + .copied() + .fold(f64::NEG_INFINITY, f64::max); + + // Slow detector: mean over longer window — filters noise. + let sustained_mean: f64 = self.sharpe_history + .iter() + .rev() + .take(long_window) + .sum::() / long_window as f64; + let thresh = self.hyperparams.anti_lr_sharpe_threshold; let base_lr = self.lr_scheduler.get_initial_lr(); - let anti_mult = if smoothed_sharpe > thresh { + // "Good" wins ties — we prefer exploration over dampening when + // both triggers fire (rare but possible in a noisy window). + let anti_mult = if recent_max > thresh { self.hyperparams.anti_lr_good_mult - } else if smoothed_sharpe < -thresh { + } else if sustained_mean < -thresh { self.hyperparams.anti_lr_bad_mult } else { 1.0 @@ -2517,8 +2537,8 @@ impl DQNTrainer { if (anti_mult - 1.0).abs() > 0.01 { info!( epoch = epoch + 1, - smoothed_sharpe = %format!("{:.3}", smoothed_sharpe), - window = window, + recent_max = %format!("{:.3}", recent_max), + sustained_mean = %format!("{:.3}", sustained_mean), anti_mult = %format!("{:.1}x", anti_mult), lr = %format!("{:.2e}", current_lr), "Anti-intuitive LR adjustment"