fix: eval v_range EMA uses adaptive alpha — fixed α=0.05 lagged on Q-shifts
When Q-mean shifted from +0.01 to -0.01, fixed α=0.05 took ~20 updates to catch up → misaligned backtest atoms → val_Sharpe dipped to -20. Adaptive alpha: α = |error| / (|error| + 0.01), clamped [0.01, 0.5]. Tracks fast on Q-shifts (α→0.5), smooths when stable (α→0.01). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1012,15 +1012,21 @@ impl GpuDqnTrainer {
|
||||
/// Update eval v_range from observed Q-value statistics.
|
||||
/// Called at epoch boundary after compute_q_stats.
|
||||
pub fn update_eval_v_range(&mut self, q_mean: f32, q_std: f32) {
|
||||
// EMA-smooth Q-stats to prevent eval v_range from jumping between steps.
|
||||
// Without smoothing, per-sample IQL support produces noisy Q-stats that
|
||||
// cause the backtest atom placement to fluctuate → inconsistent val_Sharpe.
|
||||
// Adaptive-rate EMA: tracks fast when Q-values shift, smooths when stable.
|
||||
// Alpha = prediction_error / (prediction_error + baseline). When the EMA
|
||||
// is far from the current value (shift detected), alpha → 0.5 (fast).
|
||||
// When close (stable), alpha → 0.01 (smooth). Self-calibrating.
|
||||
if self.eval_q_mean_ema == 0.0 && self.eval_q_std_ema == 0.0 {
|
||||
self.eval_q_mean_ema = q_mean;
|
||||
self.eval_q_std_ema = q_std;
|
||||
} else {
|
||||
self.eval_q_mean_ema = 0.95 * self.eval_q_mean_ema + 0.05 * q_mean;
|
||||
self.eval_q_std_ema = 0.95 * self.eval_q_std_ema + 0.05 * q_std;
|
||||
let mean_err = (q_mean - self.eval_q_mean_ema).abs();
|
||||
let std_err = (q_std - self.eval_q_std_ema).abs();
|
||||
let baseline = 0.01_f32;
|
||||
let alpha_mean = (mean_err / (mean_err + baseline)).clamp(0.01, 0.5);
|
||||
let alpha_std = (std_err / (std_err + baseline)).clamp(0.01, 0.5);
|
||||
self.eval_q_mean_ema = (1.0 - alpha_mean) * self.eval_q_mean_ema + alpha_mean * q_mean;
|
||||
self.eval_q_std_ema = (1.0 - alpha_std) * self.eval_q_std_ema + alpha_std * q_std;
|
||||
}
|
||||
|
||||
let half = (3.0 * self.eval_q_std_ema + 1.0).max(1.0);
|
||||
|
||||
Reference in New Issue
Block a user