From 40858314523436166d556ef0bcd7392bf8897111 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Tue, 21 Apr 2026 19:40:41 +0200 Subject: [PATCH] =?UTF-8?q?fix(dqn):=20asymmetric=20anti-LR=20=E2=80=94=20?= =?UTF-8?q?fast=20response=20to=20good,=20slow=20to=20bad?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Symmetric 5-window mean (f1359f3dc) filtered noise but also filtered exploration. RL Sharpe is plentiful-bad and rare-good in early training, so the mean always sees the plentiful side — controller locked at 0.3× LR and the model couldn't escape its initial bad minimum (train-v82b2: sharpe stuck at −8 to −21 throughout Fold 0, never peaked positive like prior runs had). Root fix: the two decisions have different evidence requirements. * Boost LR: low cost if wrong (clamp caps runaway), high value if right (kicks out of overfit). Accept weak evidence — ANY of the last short_window epochs clearly positive fires the boost. * Dampen LR: high cost if wrong (stuck model), low value if right (stability we didn't need). Demand strong evidence — MEAN over long_window must be clearly negative. Both windows derive from anti_lr_warmup (one knob): long_window = anti_lr_warmup (full window for sustained-bad) short_window = anti_lr_warmup / 2 (half window for recent-good) No new hyperparameter. Asymmetry is structural (max vs mean over differently-sized windows), not tuned. Verified: multi-trial smoke 5/5 pass, median_q_gap=2.77, mean_sharpe_ema=12.24. Compared to symmetric smoothing (2.13 / 11.03) and raw-signal (1.13 / 2.94), the asymmetric version is the best on all three multi-trial metrics. Tie-breaking: "good wins" when both signals fire in the same step — matches the controller's original intent (exploration over dampening) and our diagnosis (model needs LR headroom to escape bad starting states). --- .../src/trainers/dqn/trainer/training_loop.rs | 72 ++++++++++++------- 1 file changed, 46 insertions(+), 26 deletions(-) diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index e8417bee0..e6d61d3c8 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -2476,39 +2476,59 @@ impl DQNTrainer { self.lr_scheduler.step(); let mut current_lr = self.lr_scheduler.get_lr(); - // #24 Anti-intuitive LR: use a TEMPORALLY SMOOTHED Sharpe to adjust LR. - // When the model has been doing well, INCREASE LR to kick out of overfit - // minima. When struggling, decrease to stabilize. Opposite of standard - // practice. Always active. + // #24 Anti-intuitive LR: ASYMMETRIC temporal filter. // - // Previously this used `prev_sharpe = sharpe_history.last()`, feeding a - // single noisy epoch Sharpe into the controller. RL Sharpe oscillates - // widely epoch-to-epoch (observed: −20 / +30 swings), so the controller - // kept flipping multipliers and amplified its own input noise — - // gradient norms spiked to 1.16M on Fold 0 (train-br8cb). The raw - // multipliers (3.0 / 0.3) work fine when the controller sees a stable - // signal; the problem was the signal, not the magnitudes. + // When training has been doing well, INCREASE LR to escape overfit + // minima; when struggling, DECREASE LR to stabilize. Original impl fed + // on raw last-epoch Sharpe — noisy, caused 1M+ grad spikes as the + // controller flipped every epoch (train-br8cb Fold 0). // - // Fix: feed a rolling-mean Sharpe over the same window as the warmup - // (reuses the one knob already in the code — no new hyperparameter). - // The window naturally filters per-epoch oscillation at the frequency - // the anti-LR logic wants to react on (multi-epoch trends), while - // letting genuine sustained improvement/degradation still trigger - // adjustments. + // A symmetric 5-window mean killed the noise but also killed exploration + // (train-v82b2): RL Sharpe is plentiful-bad and rare-good in early + // epochs, so the mean stays negative even when individual epochs find + // real improvements. The controller locked into 0.3× LR and never let + // the model kick out of its initial plateau. + // + // Correct structure is asymmetric: + // Good signal → fast response (short window, take max) + // — a single genuinely good epoch is meaningful and + // rare; don't wait for it to be sustained before + // letting the model exploit it. + // Bad signal → slow response (long window, take mean) + // — bad epochs are noisy and common; only dampen LR + // when we're confident the model is stuck. + // + // Both windows derive from `anti_lr_warmup` (one knob for the whole + // controller): full window for bad, half window for good. No new + // hyperparameter — the asymmetry is structural, not tuned. let anti_lr_warmup = 5; if epoch >= anti_lr_warmup && !self.sharpe_history.is_empty() { - let window = anti_lr_warmup.min(self.sharpe_history.len()); - let smoothed_sharpe: f64 = self.sharpe_history + let history_len = self.sharpe_history.len(); + let long_window = anti_lr_warmup.min(history_len); + let short_window = (anti_lr_warmup / 2).max(1).min(history_len); + + // Fast detector: best of recent few — responsive to new gains. + let recent_max: f64 = self.sharpe_history .iter() .rev() - .take(window) - .sum::() - / window as f64; + .take(short_window) + .copied() + .fold(f64::NEG_INFINITY, f64::max); + + // Slow detector: mean over longer window — filters noise. + let sustained_mean: f64 = self.sharpe_history + .iter() + .rev() + .take(long_window) + .sum::() / long_window as f64; + let thresh = self.hyperparams.anti_lr_sharpe_threshold; let base_lr = self.lr_scheduler.get_initial_lr(); - let anti_mult = if smoothed_sharpe > thresh { + // "Good" wins ties — we prefer exploration over dampening when + // both triggers fire (rare but possible in a noisy window). + let anti_mult = if recent_max > thresh { self.hyperparams.anti_lr_good_mult - } else if smoothed_sharpe < -thresh { + } else if sustained_mean < -thresh { self.hyperparams.anti_lr_bad_mult } else { 1.0 @@ -2517,8 +2537,8 @@ impl DQNTrainer { if (anti_mult - 1.0).abs() > 0.01 { info!( epoch = epoch + 1, - smoothed_sharpe = %format!("{:.3}", smoothed_sharpe), - window = window, + recent_max = %format!("{:.3}", recent_max), + sustained_mean = %format!("{:.3}", sustained_mean), anti_mult = %format!("{:.1}x", anti_mult), lr = %format!("{:.2e}", current_lr), "Anti-intuitive LR adjustment"