From dab4478b47a9013550768fab5babf9ab6ee74b98 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 12 Apr 2026 06:56:37 +0200 Subject: [PATCH] =?UTF-8?q?fix:=20production=20v=5Fmin/v=5Fmax=20=C2=B115?= =?UTF-8?q?=E2=86=92=C2=B150=20for=20gamma=3D0.99=20+=20PopArt=20GPU=20buf?= =?UTF-8?q?fer=20reset?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v_min/v_max ±15 with gamma=0.99 only covers 15% of theoretical Q range (Q_max = 1/(1-0.99) = 100 for unit-variance PopArt rewards). C51 top atom saturates on sustained winners, degrading distributional learning. ±50 covers 95th percentile. PopArt GPU buffers (popart_mean, popart_var, popart_count) were never zeroed between walk-forward folds — fold 2's reward normalization was contaminated by fold 1's statistics. Now reset alongside Adam state in reset_adam_state(). Co-Authored-By: Claude Opus 4.6 (1M context) --- config/training/dqn-production.toml | 4 ++++ config/training/dqn-smoketest.toml | 4 ++++ crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs | 11 ++++++++++- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/config/training/dqn-production.toml b/config/training/dqn-production.toml index 2720654a1..fe765d8e0 100644 --- a/config/training/dqn-production.toml +++ b/config/training/dqn-production.toml @@ -36,6 +36,10 @@ trades_data_dir = "/data/futures-baseline-trades" [distributional] num_atoms = 51 +# gamma=0.99 → Q_max = R/(1-gamma) = 100 for unit-variance PopArt rewards. +# ±50 covers the 95th percentile of Q distribution (±15 default only covers 15%). +v_min = -50.0 +v_max = 50.0 [exploration] epsilon_start = 0.3 diff --git a/config/training/dqn-smoketest.toml b/config/training/dqn-smoketest.toml index 1821ea955..a8c383819 100644 --- a/config/training/dqn-smoketest.toml +++ b/config/training/dqn-smoketest.toml @@ -30,6 +30,10 @@ imbalance_bar_ewma_alpha = 0.1 [distributional] num_atoms = 51 +# gamma=0.95 → Q_max = R/(1-gamma) = 20 for PopArt-normalized rewards. +# ±15 covers 75% of theoretical Q range — adequate for smoke test. +v_min = -15.0 +v_max = 15.0 [exploration] epsilon_start = 0.3 diff --git a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs index 2ea122120..09e591701 100644 --- a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs +++ b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs @@ -942,7 +942,16 @@ impl GpuDqnTrainer { .map_err(|e| MLError::ModelError(format!("reset iqn t: {e}")))?; self.iqn_trunk_adam_step = 0; - tracing::info!("Adam optimizer state reset for new fold"); + // Reset PopArt running statistics — prevents fold 1's reward distribution + // from contaminating fold 2's normalization. + self.stream.memset_zeros(&mut self.popart_mean) + .map_err(|e| MLError::ModelError(format!("reset popart_mean: {e}")))?; + self.stream.memset_zeros(&mut self.popart_var) + .map_err(|e| MLError::ModelError(format!("reset popart_var: {e}")))?; + self.stream.memset_zeros(&mut self.popart_count) + .map_err(|e| MLError::ModelError(format!("reset popart_count: {e}")))?; + + tracing::info!("Adam optimizer + PopArt state reset for new fold"); Ok(()) } }