diff --git a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs index d7727f39e..1f3a1205f 100644 --- a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs +++ b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs @@ -7617,18 +7617,16 @@ impl GpuDqnTrainer { unsafe { (*self.q_readback_pinned.add(6)).clamp(0.0, 1.0) } } - /// Coarse spectral-gap approximation on the last Q readback. + /// Coarse Q-collapse detector derived from the single-sample Q readback at + /// `q_readback_pinned[7..7+total_actions]`. /// - /// Reads per-action Q values from q_readback_pinned[7..7+total_actions] (single - /// sample from the previous reduce_current_q_stats call) and returns - /// 1 + 1/range as a cheap proxy for sigma_1/sigma_2. + /// Returns a LARGE value (~100) when all Q values are (nearly) equal — i.e. + /// rank-1 / Q-collapse — and a value close to 1.0 when the Q spread is wide. + /// Intentionally coarse: a full SVD across a batch of Q samples would require + /// a separate larger buffer and is out of scope for the host-side health path. /// - /// NOTE: This is intentionally a coarse approximation rather than a full SVD. - /// A proper sigma_1/sigma_2 would require a batch-sized Q matrix and cuSOLVER, - /// which adds complexity and a sync. The proxy rises when Q is dominated by a - /// single action (rank-1-like), which is the collapse signal we care about. - /// The spectral_gap thresholds in NormalizedComponents::from_raw (2..10) are - /// calibrated for this proxy, not for true singular values. + /// The downstream `NormalizedComponents::from_raw` thresholds are calibrated + /// for this behaviour (`spectral_gap_norm` drops toward 0 as this value rises). pub fn compute_q_spectral_gap(&self) -> f32 { if self.q_readback_pinned.is_null() { return 1.0; } // Layout: q_readback_pinned[0..7] = stats, [7..7+total_actions] = per-action Q values. diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index 7996577a5..71fa8c51c 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -1785,8 +1785,10 @@ impl DQNTrainer { // ── LearningHealth computation (Layer 1) ────────────────────────────── // Source signal adaptations vs. plan spec (A3 known deviations): - // - q_gap: mean of per_branch_q_gap_ema (4 branches) via fused_ctx — synchronous - // DtoH at epoch boundary (not on the hot path; acceptable). + // - q_gap: mean of per_branch_q_gap_ema (4 branches) via fused_ctx — + // NOTE: per_branch_q_gap_ema() performs a synchronous stream flush + DtoH + // (cuStreamSynchronize + cuMemcpyDtoH_v2). Acceptable at epoch boundary + // because it runs once per epoch; DO NOT move into a per-step loop. // - q_var: epoch_q_max - epoch_q_min proxy instead of QValueStatsResult.q_variance // (q_variance is not separately tracked as a scalar field; range is a sufficient proxy). // - grad_consistency: scalar proxy (successive grad_norm delta) instead of full Adam @@ -1804,8 +1806,14 @@ impl DQNTrainer { 0.0 }; - // Q-var proxy from epoch min/max (accumulated by update_epoch_q_stats each step). - let q_var_signal = (self.epoch_q_max - self.epoch_q_min).max(0.0); + // q_var proxy: range (max - min) of per-step Q values this epoch. If no + // steps ran (epoch_q_min still INFINITY), fall back to 0.0 rather than + // propagating NaN/-inf into health normalization. + let q_var_signal = if self.epoch_q_min.is_finite() && self.epoch_q_max.is_finite() { + (self.epoch_q_max - self.epoch_q_min).max(0.0) + } else { + 0.0 + }; // Atom utilization from q_readback_pinned[6] (one-step lag). let atom_util = self.fused_ctx.as_ref()