diff --git a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs index 0f1cd6e18..1da348cbb 100644 --- a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs +++ b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs @@ -986,7 +986,7 @@ impl GpuDqnTrainer { .map_err(|e| MLError::ModelError(format!("reset popart_count: {e}")))?; // Reset adaptive C51 z-support — fold 2's Q-scale may differ from fold 1. - unsafe { *self.v_range_pinned = -1e-4; *self.v_range_pinned.add(1) = 1e-4; } + unsafe { *self.v_range_pinned = -0.005; *self.v_range_pinned.add(1) = 0.005; } // Keep adaptive gradient clip EMA across folds — gradient scale is a property // of the model architecture and loss function, not the data window. @@ -2480,7 +2480,7 @@ impl GpuDqnTrainer { .map_err(|e| MLError::ModelError(format!("pinned v_range alloc: {e}")))? as *mut f32 }; - unsafe { *v_range_pinned = -1e-4_f32; *v_range_pinned.add(1) = 1e-4_f32; } + unsafe { *v_range_pinned = -0.005_f32; *v_range_pinned.add(1) = 0.005_f32; } let v_range_dev_ptr = unsafe { let mut dev_ptr: u64 = 0; cudarc::driver::sys::cuMemHostGetDevicePointer_v2( @@ -6465,8 +6465,24 @@ impl GpuDqnTrainer { /// - Mean-centered: no wasted atoms on impossible Q-values pub fn adapt_v_range(&mut self, q_mean: f32, q_variance: f32) -> bool { const SIGMA_COVERAGE: f32 = 3.0; - // Absolute floor prevents collapse to a point at init (Q-mean=0, Q-var=0). - const ABS_FLOOR: f32 = 1e-4; + // Reward-scale floor: v_range must cover at least the single-step + // Bellman shift magnitude. With reward std ~R and gamma, a single + // Bellman projection shifts atoms by ~R. If v_range < R, the projection + // clips to boundary atoms → near-zero gradient → Q-values can't grow + // (stable fixed point trap). + // + // The experience kernel's rewards are PopArt-normalized to ~unit variance + // BUT the Q-values we observe are in the raw logit scale (pre-PopArt). + // With smoketest rewards std ≈ 0.007 and gamma=0.95: + // theoretical_q_range = R_std / (1-gamma) = 0.007/0.05 = 0.14 + // single_step_shift = R_std ≈ 0.007 + // + // ABS_FLOOR = single-step reward magnitude ensures the Bellman projection + // can shift atoms by at least 1 reward unit. This breaks the fixed point. + // The 0.007 comes from typical per-bar PnL with tx costs. + // At production scale (H100), PopArt variance stabilizes and this floor + // rarely binds because Q-values grow much faster. + const REWARD_SCALE_FLOOR: f32 = 0.01; // conservative: slightly above typical R_std let q_std = (q_variance.max(0.0) + 1e-10).sqrt(); let half_width = SIGMA_COVERAGE * q_std; @@ -6476,10 +6492,10 @@ impl GpuDqnTrainer { let target_min = q_mean - total_half; let target_max = q_mean + total_half; - // Adaptive minimum range: 20% of |Q-mean| + absolute floor. - // Fixed MIN_RANGE=0.02 was 200× too wide when Q-values are 0.0001, - // wasting atoms and keeping entropy at 100%. Adaptive scales with Q-magnitude. - let adaptive_min = (q_mean.abs() * 0.2 + ABS_FLOOR).max(ABS_FLOOR); + // The floor ensures v_range always covers at least the reward scale. + // This prevents the stable fixed point where tight v_range → tiny + // Bellman shift → tiny gradient → Q-values can't escape near-zero. + let adaptive_min = REWARD_SCALE_FLOOR; let (final_min, final_max) = if (target_max - target_min) < adaptive_min { (q_mean - adaptive_min * 0.5, q_mean + adaptive_min * 0.5) } else { diff --git a/crates/ml/src/trainers/dqn/fused_training.rs b/crates/ml/src/trainers/dqn/fused_training.rs index e04ce987d..286d7fc4c 100644 --- a/crates/ml/src/trainers/dqn/fused_training.rs +++ b/crates/ml/src/trainers/dqn/fused_training.rs @@ -758,6 +758,17 @@ impl FusedTrainingCtx { self.graph_aux = None; self.graph_spectral = None; self.graph_mega = None; + // Shrink-and-perturb at fold boundary — reduce overfit to previous fold. + // alpha=0.8: keep 80% of learned representations, add 20% random noise. + // Without this, the greedy policy from fold 1 produces val_Sharpe=-33 + // on fold 2 data and takes many epochs to recover. + let sp_alpha = 0.8_f32; + let sp_sigma = 0.01_f32; + if let Err(e) = self.trainer.shrink_and_perturb(sp_alpha, sp_sigma) { + tracing::warn!("Fold-boundary shrink-and-perturb failed (non-fatal): {e}"); + } else { + tracing::info!(alpha = sp_alpha, sigma = sp_sigma, "Fold-boundary shrink-and-perturb applied"); + } // Reset Adam optimizer state — each fold starts with fresh momentum. // Stale momentum from a previous fold causes weight explosion → Q-value -1e30. self.trainer.reset_adam_state()?;