fix: validation backtest uses EMA-smoothed eval v_range — was fixed [-240, 240]

Fixed v_range from hyperparams (reward_scale/gamma=±240) was 4800x too
wide for Q-values near 0. All Q-values mapped to the same 2-3 center
atoms, making backtest action selection essentially random → val_Sharpe
jumped between -11 and +0.14 arbitrarily. Now uses the same
EMA-smoothed eval_v_range that the experience collector uses.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-04-14 13:39:44 +02:00
parent 5d877234b3
commit 8c3b7f5c5e
3 changed files with 12 additions and 4 deletions

View File

@@ -633,8 +633,8 @@ pub struct GpuDqnTrainer {
eval_v_range_buf: CudaSlice<f32>,
eval_v_range_ptr: u64,
/// EMA-smoothed Q-stats for stable eval v_range updates.
eval_q_mean_ema: f32,
eval_q_std_ema: f32,
pub(crate) eval_q_mean_ema: f32,
pub(crate) eval_q_std_ema: f32,
/// Adaptive gradient clip norm — pinned device-mapped host memory.
/// GPU reads via device pointer (graph-safe), host writes directly (no HtoD copy).

View File

@@ -2037,6 +2037,11 @@ impl FusedTrainingCtx {
pub(crate) fn per_sample_support_ptr(&self) -> u64 { self.gpu_iql.per_sample_support_ptr() }
/// Fixed wide v_range pointer for eval compute_expected_q (not C51 loss).
pub(crate) fn eval_v_range_ptr(&self) -> u64 { self.trainer.eval_v_range_ptr() }
/// Return EMA-smoothed eval v_range as [v_min, v_max].
pub(crate) fn eval_v_range(&self) -> [f32; 2] {
let half = (3.0 * self.trainer.eval_q_std_ema + 1.0_f32).max(1.0);
[self.trainer.eval_q_mean_ema - half, self.trainer.eval_q_mean_ema + half]
}
/// Update eval v_range from observed Q-value statistics.
pub(crate) fn update_eval_v_range(&mut self, q_mean: f32, q_std: f32) {
self.trainer.update_eval_v_range(q_mean, q_std);

View File

@@ -513,8 +513,11 @@ impl DQNTrainer {
let f = &*fused_ptr;
(f.online_dueling_ref() as *const _, f.online_branching_ref() as *const _)
};
// Eval v_range from hyperparams — must match the training config's v_min/v_max.
let vr = [self.hyperparams.computed_v_min() as f32, self.hyperparams.computed_v_max() as f32];
// Eval v_range from EMA-smoothed Q-stats — adaptive to actual Q-value scale.
// Fixed [-240, 240] was 4800x too wide for Q-values near 0, making backtest
// action selection essentially random (all Q-values map to same center atoms).
let fused_ref = unsafe { &*fused_ptr };
let vr = fused_ref.eval_v_range();
let agent = self.agent.read().await;
let network_dims = agent.network_dims();