fix: eval v_range EMA uses adaptive alpha — fixed α=0.05 lagged on Q-shifts

When Q-mean shifted from +0.01 to -0.01, fixed α=0.05 took ~20 updates
to catch up → misaligned backtest atoms → val_Sharpe dipped to -20.
Adaptive alpha: α = |error| / (|error| + 0.01), clamped [0.01, 0.5].
Tracks fast on Q-shifts (α→0.5), smooths when stable (α→0.01).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-04-14 14:01:49 +02:00
parent 8c3b7f5c5e
commit 9c2f5d75b5

View File

@@ -1012,15 +1012,21 @@ impl GpuDqnTrainer {
/// Update eval v_range from observed Q-value statistics.
/// Called at epoch boundary after compute_q_stats.
pub fn update_eval_v_range(&mut self, q_mean: f32, q_std: f32) {
// EMA-smooth Q-stats to prevent eval v_range from jumping between steps.
// Without smoothing, per-sample IQL support produces noisy Q-stats that
// cause the backtest atom placement to fluctuate → inconsistent val_Sharpe.
// Adaptive-rate EMA: tracks fast when Q-values shift, smooths when stable.
// Alpha = prediction_error / (prediction_error + baseline). When the EMA
// is far from the current value (shift detected), alpha → 0.5 (fast).
// When close (stable), alpha → 0.01 (smooth). Self-calibrating.
if self.eval_q_mean_ema == 0.0 && self.eval_q_std_ema == 0.0 {
self.eval_q_mean_ema = q_mean;
self.eval_q_std_ema = q_std;
} else {
self.eval_q_mean_ema = 0.95 * self.eval_q_mean_ema + 0.05 * q_mean;
self.eval_q_std_ema = 0.95 * self.eval_q_std_ema + 0.05 * q_std;
let mean_err = (q_mean - self.eval_q_mean_ema).abs();
let std_err = (q_std - self.eval_q_std_ema).abs();
let baseline = 0.01_f32;
let alpha_mean = (mean_err / (mean_err + baseline)).clamp(0.01, 0.5);
let alpha_std = (std_err / (std_err + baseline)).clamp(0.01, 0.5);
self.eval_q_mean_ema = (1.0 - alpha_mean) * self.eval_q_mean_ema + alpha_mean * q_mean;
self.eval_q_std_ema = (1.0 - alpha_std) * self.eval_q_std_ema + alpha_std * q_std;
}
let half = (3.0 * self.eval_q_std_ema + 1.0).max(1.0);