diff --git a/config/training/dqn-production.toml b/config/training/dqn-production.toml index 2720654a1..fe765d8e0 100644 --- a/config/training/dqn-production.toml +++ b/config/training/dqn-production.toml @@ -36,6 +36,10 @@ trades_data_dir = "/data/futures-baseline-trades" [distributional] num_atoms = 51 +# gamma=0.99 → Q_max = R/(1-gamma) = 100 for unit-variance PopArt rewards. +# ±50 covers the 95th percentile of Q distribution (±15 default only covers 15%). +v_min = -50.0 +v_max = 50.0 [exploration] epsilon_start = 0.3 diff --git a/config/training/dqn-smoketest.toml b/config/training/dqn-smoketest.toml index 1821ea955..a8c383819 100644 --- a/config/training/dqn-smoketest.toml +++ b/config/training/dqn-smoketest.toml @@ -30,6 +30,10 @@ imbalance_bar_ewma_alpha = 0.1 [distributional] num_atoms = 51 +# gamma=0.95 → Q_max = R/(1-gamma) = 20 for PopArt-normalized rewards. +# ±15 covers 75% of theoretical Q range — adequate for smoke test. +v_min = -15.0 +v_max = 15.0 [exploration] epsilon_start = 0.3 diff --git a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs index 2ea122120..09e591701 100644 --- a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs +++ b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs @@ -942,7 +942,16 @@ impl GpuDqnTrainer { .map_err(|e| MLError::ModelError(format!("reset iqn t: {e}")))?; self.iqn_trunk_adam_step = 0; - tracing::info!("Adam optimizer state reset for new fold"); + // Reset PopArt running statistics — prevents fold 1's reward distribution + // from contaminating fold 2's normalization. + self.stream.memset_zeros(&mut self.popart_mean) + .map_err(|e| MLError::ModelError(format!("reset popart_mean: {e}")))?; + self.stream.memset_zeros(&mut self.popart_var) + .map_err(|e| MLError::ModelError(format!("reset popart_var: {e}")))?; + self.stream.memset_zeros(&mut self.popart_count) + .map_err(|e| MLError::ModelError(format!("reset popart_count: {e}")))?; + + tracing::info!("Adam optimizer + PopArt state reset for new fold"); Ok(()) } }