From f1359f3dcc12ab09ea69d6a73299f7f651b6ce52 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Tue, 21 Apr 2026 19:12:27 +0200 Subject: [PATCH] fix(dqn): temporal smoothing for anti-intuitive LR controller MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The anti-LR adjuster (config.rs#24) reacts to Sharpe swings by multiplying the learning rate — good Sharpe → ×3 to escape overfit minima, bad Sharpe → ×0.3 to stabilize. Previously it fed on raw `sharpe_history.last()`, which is a single noisy epoch value. In 30-epoch L40S smoke (train-br8cb Fold 0) per-epoch Sharpe oscillated between −20 and +30, so the controller flipped multipliers every epoch and amplified its own input noise — gradient norm spiked to 1.16M at Epoch 18 from a baseline of ~3000. Fix: feed the controller a rolling mean of the last `anti_lr_warmup` epochs (the same knob that already gates the controller on — no new hyperparameter). This filters per-epoch oscillation at the frequency the anti-LR logic wants to react on (multi-epoch trends), while still letting genuine sustained improvement or degradation trigger adjustments. Keeps the original 3.0 / 0.3 multipliers and [0.1, 5.0] clamp — the problem was the signal, not the magnitudes. Why temporal instead of tightening magnitudes: * Shrinking 3.0/0.3 → 1.5/0.7 reduces the symptom but keeps the structure — still amplifies noise, just less. * Smoothing removes the noise before the controller sees it, so the controller stays as aggressive as designed. * No new magic numbers — reuses anti_lr_warmup (=5) for both "don't-tune-yet" and "this-is-a-stable-horizon". Verified: multi-trial smoke 5/5 finite, 4/5 q_pass, median_q_gap=1.13. Slight reduction vs the fold-reset baseline (2.13) is expected — the smoother trades responsiveness for stability. Real validation is the L40S 20-epoch behaviour test queued after this commit. --- crates/ml/src/trainers/dqn/config.rs | 5 +++ .../src/trainers/dqn/trainer/training_loop.rs | 41 ++++++++++++++----- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/crates/ml/src/trainers/dqn/config.rs b/crates/ml/src/trainers/dqn/config.rs index 03821ad80..4582ac57a 100644 --- a/crates/ml/src/trainers/dqn/config.rs +++ b/crates/ml/src/trainers/dqn/config.rs @@ -1342,6 +1342,11 @@ impl DQNHyperparameters { beta_penalty_strength: 0.3, // 30% reward reduction at full correlation // Gems & Pearls: generalization techniques + // Anti-LR multipliers retained at original 3.0/0.3 — the noise + // amplification problem (grad_norm spikes to 1.16M when Sharpe + // oscillates epoch-to-epoch, train-br8cb Fold 0) is now fixed by + // feeding the controller a temporally smoothed Sharpe instead of + // the raw last-epoch value. See training_loop.rs anti-LR block. anti_lr_good_mult: 3.0, // 3x LR when Sharpe is good (destabilize overfit) anti_lr_bad_mult: 0.3, // 0.3x LR when Sharpe is bad (stabilize) anti_lr_sharpe_threshold: 0.3, // Sharpe threshold for switching diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index 63d2195f0..e8417bee0 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -2476,20 +2476,39 @@ impl DQNTrainer { self.lr_scheduler.step(); let mut current_lr = self.lr_scheduler.get_lr(); - // #24 Anti-intuitive LR: use PREVIOUS epoch's Sharpe to adjust LR. - // When model was doing well, INCREASE LR to kick out of overfit minima. - // When struggling, decrease to stabilize. Opposite of standard practice. - // #24 Anti-intuitive LR: always active (one production path) - // Guard: skip anti-LR for first 5 epochs — early Sharpe is unreliable - // (random policy, tiny replay buffer, Sharpe from few trades). + // #24 Anti-intuitive LR: use a TEMPORALLY SMOOTHED Sharpe to adjust LR. + // When the model has been doing well, INCREASE LR to kick out of overfit + // minima. When struggling, decrease to stabilize. Opposite of standard + // practice. Always active. + // + // Previously this used `prev_sharpe = sharpe_history.last()`, feeding a + // single noisy epoch Sharpe into the controller. RL Sharpe oscillates + // widely epoch-to-epoch (observed: −20 / +30 swings), so the controller + // kept flipping multipliers and amplified its own input noise — + // gradient norms spiked to 1.16M on Fold 0 (train-br8cb). The raw + // multipliers (3.0 / 0.3) work fine when the controller sees a stable + // signal; the problem was the signal, not the magnitudes. + // + // Fix: feed a rolling-mean Sharpe over the same window as the warmup + // (reuses the one knob already in the code — no new hyperparameter). + // The window naturally filters per-epoch oscillation at the frequency + // the anti-LR logic wants to react on (multi-epoch trends), while + // letting genuine sustained improvement/degradation still trigger + // adjustments. let anti_lr_warmup = 5; if epoch >= anti_lr_warmup && !self.sharpe_history.is_empty() { - let prev_sharpe = self.sharpe_history.last().copied().unwrap_or(0.0); + let window = anti_lr_warmup.min(self.sharpe_history.len()); + let smoothed_sharpe: f64 = self.sharpe_history + .iter() + .rev() + .take(window) + .sum::() + / window as f64; let thresh = self.hyperparams.anti_lr_sharpe_threshold; let base_lr = self.lr_scheduler.get_initial_lr(); - let anti_mult = if prev_sharpe > thresh { + let anti_mult = if smoothed_sharpe > thresh { self.hyperparams.anti_lr_good_mult - } else if prev_sharpe < -thresh { + } else if smoothed_sharpe < -thresh { self.hyperparams.anti_lr_bad_mult } else { 1.0 @@ -2497,7 +2516,9 @@ impl DQNTrainer { current_lr = (current_lr * anti_mult).clamp(base_lr * 0.1, base_lr * 5.0); if (anti_mult - 1.0).abs() > 0.01 { info!( - epoch = epoch + 1, prev_sharpe = %format!("{:.3}", prev_sharpe), + epoch = epoch + 1, + smoothed_sharpe = %format!("{:.3}", smoothed_sharpe), + window = window, anti_mult = %format!("{:.1}x", anti_mult), lr = %format!("{:.2e}", current_lr), "Anti-intuitive LR adjustment"