fix: validation backtest uses EMA-smoothed eval v_range — was fixed [-240, 240]
Fixed v_range from hyperparams (reward_scale/gamma=±240) was 4800x too wide for Q-values near 0. All Q-values mapped to the same 2-3 center atoms, making backtest action selection essentially random → val_Sharpe jumped between -11 and +0.14 arbitrarily. Now uses the same EMA-smoothed eval_v_range that the experience collector uses. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -633,8 +633,8 @@ pub struct GpuDqnTrainer {
|
||||
eval_v_range_buf: CudaSlice<f32>,
|
||||
eval_v_range_ptr: u64,
|
||||
/// EMA-smoothed Q-stats for stable eval v_range updates.
|
||||
eval_q_mean_ema: f32,
|
||||
eval_q_std_ema: f32,
|
||||
pub(crate) eval_q_mean_ema: f32,
|
||||
pub(crate) eval_q_std_ema: f32,
|
||||
|
||||
/// Adaptive gradient clip norm — pinned device-mapped host memory.
|
||||
/// GPU reads via device pointer (graph-safe), host writes directly (no HtoD copy).
|
||||
|
||||
@@ -2037,6 +2037,11 @@ impl FusedTrainingCtx {
|
||||
pub(crate) fn per_sample_support_ptr(&self) -> u64 { self.gpu_iql.per_sample_support_ptr() }
|
||||
/// Fixed wide v_range pointer for eval compute_expected_q (not C51 loss).
|
||||
pub(crate) fn eval_v_range_ptr(&self) -> u64 { self.trainer.eval_v_range_ptr() }
|
||||
/// Return EMA-smoothed eval v_range as [v_min, v_max].
|
||||
pub(crate) fn eval_v_range(&self) -> [f32; 2] {
|
||||
let half = (3.0 * self.trainer.eval_q_std_ema + 1.0_f32).max(1.0);
|
||||
[self.trainer.eval_q_mean_ema - half, self.trainer.eval_q_mean_ema + half]
|
||||
}
|
||||
/// Update eval v_range from observed Q-value statistics.
|
||||
pub(crate) fn update_eval_v_range(&mut self, q_mean: f32, q_std: f32) {
|
||||
self.trainer.update_eval_v_range(q_mean, q_std);
|
||||
|
||||
@@ -513,8 +513,11 @@ impl DQNTrainer {
|
||||
let f = &*fused_ptr;
|
||||
(f.online_dueling_ref() as *const _, f.online_branching_ref() as *const _)
|
||||
};
|
||||
// Eval v_range from hyperparams — must match the training config's v_min/v_max.
|
||||
let vr = [self.hyperparams.computed_v_min() as f32, self.hyperparams.computed_v_max() as f32];
|
||||
// Eval v_range from EMA-smoothed Q-stats — adaptive to actual Q-value scale.
|
||||
// Fixed [-240, 240] was 4800x too wide for Q-values near 0, making backtest
|
||||
// action selection essentially random (all Q-values map to same center atoms).
|
||||
let fused_ref = unsafe { &*fused_ptr };
|
||||
let vr = fused_ref.eval_v_range();
|
||||
|
||||
let agent = self.agent.read().await;
|
||||
let network_dims = agent.network_dims();
|
||||
|
||||
Reference in New Issue
Block a user