gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(A3): clarify docs and guard q_var against infinite sentinels

Browse Source
This commit is contained in:
jgrusewski
2026-04-20 19:43:24 +02:00
parent 6622d90906
commit 3e9a21d52b
2 changed files with 20 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs
View File

@@ -7617,18 +7617,16 @@ impl GpuDqnTrainer {
unsafe { (*self.q_readback_pinned.add(6)).clamp(0.0, 1.0) }
}
/// Coarse spectral-gap approximation on the last Q readback.
/// Coarse Q-collapse detector derived from the single-sample Q readback at
/// `q_readback_pinned[7..7+total_actions]`.
///
/// Reads per-action Q values from q_readback_pinned[7..7+total_actions] (single
/// sample from the previous reduce_current_q_stats call) and returns
/// 1 + 1/range as a cheap proxy for sigma_1/sigma_2.
/// Returns a LARGE value (~100) when all Q values are (nearly) equal — i.e.
/// rank-1 / Q-collapse — and a value close to 1.0 when the Q spread is wide.
/// Intentionally coarse: a full SVD across a batch of Q samples would require
/// a separate larger buffer and is out of scope for the host-side health path.
///
/// NOTE: This is intentionally a coarse approximation rather than a full SVD.
/// A proper sigma_1/sigma_2 would require a batch-sized Q matrix and cuSOLVER,
/// which adds complexity and a sync. The proxy rises when Q is dominated by a
/// single action (rank-1-like), which is the collapse signal we care about.
/// The spectral_gap thresholds in NormalizedComponents::from_raw (2..10) are
/// calibrated for this proxy, not for true singular values.
/// The downstream `NormalizedComponents::from_raw` thresholds are calibrated
/// for this behaviour (`spectral_gap_norm` drops toward 0 as this value rises).
pub fn compute_q_spectral_gap(&self) -> f32 {
if self.q_readback_pinned.is_null() { return 1.0; }
// Layout: q_readback_pinned[0..7] = stats, [7..7+total_actions] = per-action Q values.

16
crates/ml/src/trainers/dqn/trainer/training_loop.rs
View File

@@ -1785,8 +1785,10 @@ impl DQNTrainer {
// ── LearningHealth computation (Layer 1) ──────────────────────────────
// Source signal adaptations vs. plan spec (A3 known deviations):
// - q_gap: mean of per_branch_q_gap_ema (4 branches) via fused_ctx — synchronous
// DtoH at epoch boundary (not on the hot path; acceptable).
// - q_gap: mean of per_branch_q_gap_ema (4 branches) via fused_ctx —
// NOTE: per_branch_q_gap_ema() performs a synchronous stream flush + DtoH
// (cuStreamSynchronize + cuMemcpyDtoH_v2). Acceptable at epoch boundary
// because it runs once per epoch; DO NOT move into a per-step loop.
// - q_var: epoch_q_max - epoch_q_min proxy instead of QValueStatsResult.q_variance
// (q_variance is not separately tracked as a scalar field; range is a sufficient proxy).
// - grad_consistency: scalar proxy (successive grad_norm delta) instead of full Adam
@@ -1804,8 +1806,14 @@ impl DQNTrainer {
0.0
};
// Q-var proxy from epoch min/max (accumulated by update_epoch_q_stats each step).
let q_var_signal = (self.epoch_q_max - self.epoch_q_min).max(0.0);
// q_var proxy: range (max - min) of per-step Q values this epoch. If no
// steps ran (epoch_q_min still INFINITY), fall back to 0.0 rather than
// propagating NaN/-inf into health normalization.
let q_var_signal = if self.epoch_q_min.is_finite() && self.epoch_q_max.is_finite() {
(self.epoch_q_max - self.epoch_q_min).max(0.0)
} else {
0.0
};
// Atom utilization from q_readback_pinned[6] (one-step lag).
let atom_util = self.fused_ctx.as_ref()