diag(dqn): ISV v-range flow instrumentation (target=isv_vrange_diag)

Adds targeted tracing::info! at three call sites to diagnose why Q-value
range hits exactly +/-50 (config hard safety clamp) at every epoch after
the ISV v-range unification commits 9deda5f65 and 11df03785.

Instrumentation (all under target="isv_vrange_diag"):

1. update_eval_v_range (called at epoch-final Q-stats): per-branch
   (q_mean, q_std, q_gap, center, half, initialized_before) — captures
   whether cold init or warm EMA path produced the ISV write.

2. recompute_atom_positions (called at epoch N start): per-branch host
   read of ISV (center, half) — what the adaptive_atom kernel will
   consume on this epoch's forward pass.

3. warm_start_atom_positions (called mid-epoch, after experience
   collection): per-branch clamp bounds (b_min, b_max, isv_ptr_null),
   plus post-write per-branch (amin, amax) of the atoms just uploaded.

NOT PUSHED — local diagnostic commit for root-cause investigation.
This commit is contained in:
jgrusewski
2026-04-23 22:11:22 +02:00
parent 11df037855
commit 423ac460b3

View File

@@ -2153,6 +2153,24 @@ impl GpuDqnTrainer {
let out_ptr = atom_positions_buf_ptr + out_offset;
let branch_i32 = branch as i32;
// ISV v-range diag: log host-side ISV read used by the kernel.
// The kernel reads ISV via isv_dev_ptr (device mirror); the
// pinned mirror tracks the same writes from update_eval_v_range.
let (diag_center, diag_half) = if !self.isv_signals_pinned.is_null() {
unsafe {
let c = *self.isv_signals_pinned.add(V_CENTER_DIR_INDEX + 2 * branch);
let h = *self.isv_signals_pinned.add(V_CENTER_DIR_INDEX + 2 * branch + 1);
(c, h)
}
} else {
(f32::NAN, f32::NAN)
};
tracing::info!(
target: "isv_vrange_diag",
branch, diag_center, diag_half,
"recompute_atom_positions ISV read"
);
unsafe {
self.stream.launch_builder(&self.adaptive_atom_kernel)
.arg(&spacing_ptr)
@@ -2211,6 +2229,12 @@ impl GpuDqnTrainer {
if (b_max - b_min).abs() < 1e-6 { (v_min_f, v_max_f) }
else { (b_min, b_max) }
};
tracing::info!(
target: "isv_vrange_diag",
branch, b_min, b_max,
isv_ptr_null = isv_ptr.is_null(),
"warm_start clamp bounds"
);
let slot = &mut host_data[branch * na..(branch + 1) * na];
for (atom_idx, q) in quantiles.iter().enumerate() {
slot[atom_idx] = q.clamp(b_min, b_max);
@@ -2220,6 +2244,21 @@ impl GpuDqnTrainer {
self.stream.memcpy_htod(&host_data, &mut self.atom_positions_buf)
.map_err(|e| MLError::ModelError(format!("warm_start_atom_positions HtoD: {e}")))?;
// ISV v-range diag: log what we just wrote per branch (min/max of
// atom_positions) — host_data is the source of truth (no need for a
// DtoH roundtrip since memcpy_htod doesn't mutate source on failure
// and succeeded above).
for branch in 0..4_usize {
let slot = &host_data[branch * na..(branch + 1) * na];
let amin = slot.iter().cloned().fold(f32::INFINITY, f32::min);
let amax = slot.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
tracing::info!(
target: "isv_vrange_diag",
branch, amin, amax,
"warm_start atom_positions written"
);
}
Ok(())
}
@@ -3058,6 +3097,10 @@ impl GpuDqnTrainer {
let q_mean = stats.q_mean;
let q_std = stats.q_variance.max(0.0).sqrt();
let q_gap = per_branch_q_gaps[branch_idx].max(0.0);
// ISV v-range diag: capture the branch's initialization state
// BEFORE the EMA update so the log reflects which cold/warm path
// produced the (center, half) being written.
let initialized_before = self.eval_ema_initialized[branch_idx];
// Adaptive-rate EMA with baseline proportional to Q-std.
if !self.eval_ema_initialized[branch_idx] {
@@ -3090,6 +3133,13 @@ impl GpuDqnTrainer {
v_max_f - half,
);
tracing::info!(
target: "isv_vrange_diag",
branch_idx, q_mean, q_std, q_gap, center, half,
initialized = initialized_before,
"update_eval_v_range branch"
);
unsafe {
*self.isv_signals_pinned.add(V_CENTER_DIR_INDEX + 2 * branch_idx) = center;
*self.isv_signals_pinned.add(V_CENTER_DIR_INDEX + 2 * branch_idx + 1) = half;