diag(policy-quality): Task 0.4 — per-branch grad norm via pinned readback

Wires grad_ratio_mag_dir HEALTH_DIAG field with the magnitude head's
gradient L2 norm divided by the direction head's. H4 detection signal
(magnitude-head gradient starvation).

Implementation:
  * GpuDqnTrainer.grad_readback_pinned_ptr — pinned host buffer sized
    TOTAL_PARAMS f32, allocated once at construction. Plain pinned
    (not device-mapped) — written via memcpy_dtoh.
  * per_branch_grad_norms() -> [f32; 4]: sync stream, dtoh full grad
    buffer to pinned host, compute L2 norm per branch slice using
    compute_param_sizes + padded_byte_offset (branch tensors at indices
    8-11 / 12-15 / 16-19 / 20-23 for direction/magnitude/order/urgency).
  * grad_ratio_mag_dir() -> f32: convenience accessor for HEALTH_DIAG.
  * fused_training::FusedTrainingCtx::grad_ratio_mag_dir(&mut) — exposes
    via the wrapper used by training_loop.

Performance: pinned dtoh of ~2.7 MB (667K params × 4 bytes) takes ~0.5ms
on PCIe 4. Stream-sync cost is per-epoch (not per-step), acceptable for
diagnostic readback. Pageable-memory dtoh would be ~3-5x slower.

Drop free of the pinned buffer added alongside other pinned slots.

Per plan Task 0.4. Complete — no stubs.
This commit is contained in:
jgrusewski
2026-04-21 21:51:39 +02:00
parent 0310b1d1eb
commit bb42c99636
3 changed files with 111 additions and 3 deletions

View File

@@ -958,6 +958,13 @@ pub struct GpuDqnTrainer {
// ── Backward / Adam buffers (f32 master weights — GemmEx reads directly) ─
grad_buf: CudaSlice<f32>, // [TOTAL_PARAMS] f32 gradient accumulator
/// Task 0.4 — pinned host buffer for per-branch grad-norm readback.
/// Allocated once at construction sized to TOTAL_PARAMS f32. memcpy_dtoh
/// into pinned memory runs at PCIe bandwidth (~0.5 ms for 4 MB) instead
/// of pageable-host speed. Used only at epoch boundary, not in the
/// training graph capture region.
grad_readback_pinned_ptr: usize, // host-visible f32 buffer
grad_readback_pinned_capacity: usize, // element count (TOTAL_PARAMS)
params_buf: CudaSlice<f32>, // [TOTAL_PARAMS] f32 master online parameters (Adam operates here)
target_params_buf: CudaSlice<f32>, // [TOTAL_PARAMS] f32 master target parameters (EMA operates here)
m_buf: CudaSlice<f32>, // [TOTAL_PARAMS] Adam first moment (f32 for precision)
@@ -2027,6 +2034,12 @@ impl Drop for GpuDqnTrainer {
if !self.eval_v_range_pinned.is_null() {
let _ = unsafe { cudarc::driver::result::free_host(self.eval_v_range_pinned.cast()) };
}
// Task 0.4 — pinned grad-readback buffer.
if self.grad_readback_pinned_ptr != 0 {
let _ = unsafe {
cudarc::driver::result::free_host(self.grad_readback_pinned_ptr as *mut std::ffi::c_void)
};
}
if !self.per_branch_q_gaps_pinned.is_null() {
let _ = unsafe { cudarc::driver::result::free_host(self.per_branch_q_gaps_pinned.cast()) };
}
@@ -2084,6 +2097,76 @@ impl GpuDqnTrainer {
/// Device pointer to grad_norm (pinned device-mapped).
pub fn grad_norm_dev_ptr(&self) -> u64 { self.grad_norm_dev_ptr }
/// Task 0.4 — per-action-branch L2 grad norm.
///
/// Returns `[direction, magnitude, order, urgency]` L2 norms over each
/// branch's 4 weight tensors (fc weights, fc bias, out weights, out bias).
/// Branch 0 starts at param tensor index 8, each branch spans 4 indices.
///
/// Implementation: single `memcpy_dtoh` of the full grad buffer into a
/// **pinned** host slot allocated once at construction. Pinned host memory
/// gives PCIe-bandwidth-limited transfer (~0.5 ms for ~4 MB) instead of
/// pageable-memory copy. Per-branch norm reductions then run on host —
/// 4 small slice-sum-of-squares loops, sub-microsecond each.
///
/// Called once per epoch from HEALTH_DIAG — outside the training graph
/// capture region, so the readback cost is acceptable.
pub fn per_branch_grad_norms(&self) -> Result<[f32; 4], MLError> {
if self.grad_readback_pinned_ptr == 0 {
return Ok([0.0_f32; 4]);
}
let grad_len = self.grad_buf.len();
// Pinned buffer is sized TOTAL_PARAMS at construction; refuse if mismatch
// (defensive — should never happen, but better an error than UB).
if grad_len > self.grad_readback_pinned_capacity {
return Err(MLError::ModelError(format!(
"per_branch_grad_norms: grad_buf len {} exceeds pinned readback capacity {}",
grad_len, self.grad_readback_pinned_capacity
)));
}
// Stream-sync to ensure backward + Adam grad accumulation has completed
// before reading. Acceptable cost — runs once per epoch.
self.stream.synchronize().map_err(|e| {
MLError::ModelError(format!("per_branch_grad_norms sync: {e}"))
})?;
// Reinterpret pinned host slot as &mut [f32].
let host_slice: &mut [f32] = unsafe {
std::slice::from_raw_parts_mut(self.grad_readback_pinned_ptr as *mut f32, grad_len)
};
self.stream
.memcpy_dtoh(&self.grad_buf.slice(..grad_len), host_slice)
.map_err(|e| MLError::ModelError(format!("per_branch_grad_norms dtoh: {e}")))?;
let param_sizes = compute_param_sizes(&self.config);
let mut norms = [0.0_f32; 4];
for branch in 0..4_usize {
let first_tensor = 8 + branch * 4; // 8, 12, 16, 20
let last_tensor = first_tensor + 4; // exclusive
let mut sum_sq = 0.0_f64;
for tensor_idx in first_tensor..last_tensor {
let start_byte = padded_byte_offset(&param_sizes, tensor_idx);
let start_idx = (start_byte as usize) / std::mem::size_of::<f32>();
let len = param_sizes[tensor_idx];
let end_idx = (start_idx + len).min(host_slice.len());
for i in start_idx..end_idx {
let g = host_slice[i] as f64;
sum_sq += g * g;
}
}
norms[branch] = sum_sq.sqrt() as f32;
}
Ok(norms)
}
/// Task 0.4 — direction grad norm divided by magnitude grad norm.
/// Returns 0.0 if either norm is zero. H4 detection signal.
pub fn grad_ratio_mag_dir(&self) -> Result<f32, MLError> {
let n = self.per_branch_grad_norms()?;
let dir = n[0];
let mag = n[1];
Ok(if dir > 1e-9 { mag / dir } else { 0.0 })
}
/// Update per-branch liquid tau modulation — GPU kernel (RK4/Euler adaptive ODE).
///
/// Reads per_branch_q_gaps (pinned device-mapped) and qlstm_context from device.
@@ -5489,6 +5572,17 @@ impl GpuDqnTrainer {
buf
};
// Task 0.4 — pinned host buffer for grad readback.
// Plain pinned (NOT device-mapped) — written via memcpy_dtoh, not by GPU directly.
// Sized to TOTAL_PARAMS f32. Allocated once, reused every epoch.
let grad_readback_pinned_ptr: usize = unsafe {
let bytes = total_params * std::mem::size_of::<f32>();
cudarc::driver::result::malloc_host(bytes, 0)
.map_err(|e| MLError::ModelError(format!("pinned grad_readback alloc ({} bytes): {e}", bytes)))?
as usize
};
let grad_readback_pinned_capacity = total_params;
// eval_v_range — pinned device-mapped (zero-copy write from CPU).
let eval_v_range_pinned: *mut f32 = unsafe {
let flags = cudarc::driver::sys::CU_MEMHOSTALLOC_DEVICEMAP;
@@ -7340,6 +7434,8 @@ impl GpuDqnTrainer {
eval_td_snapshot,
eval_loss_snapshot,
grad_buf,
grad_readback_pinned_ptr,
grad_readback_pinned_capacity,
params_buf,
target_params_buf,
m_buf,

View File

@@ -884,6 +884,13 @@ impl FusedTrainingCtx {
self.trainer.predictive_loss_value()
}
/// Task 0.4 — per-action-branch grad-norm ratio (magnitude / direction)
/// for H4 detection. Reads grad_buf to a pinned host slot and computes
/// L2 norm per branch slice. Returns 0.0 on failure (non-fatal — diag only).
pub(crate) fn grad_ratio_mag_dir(&mut self) -> f32 {
self.trainer.grad_ratio_mag_dir().unwrap_or(0.0)
}
/// Flush the last in-flight async readback from the training step.
/// Called once at epoch end to retrieve the final step's scalars.
pub(crate) fn flush_readback(&mut self) -> Result<crate::cuda_pipeline::gpu_dqn_trainer::FusedTrainScalars> {

View File

@@ -2177,6 +2177,10 @@ impl DQNTrainer {
} else {
0.0
};
// Task 0.4 — per-branch grad-norm ratio (magnitude/direction). H4 signal.
let grad_ratio_mag_dir = self.fused_ctx.as_mut()
.map(|f| f.grad_ratio_mag_dir())
.unwrap_or(0.0);
// Track 1 action distribution: per-magnitude usage across the epoch.
// action_counts[9] layout is dir*3 + mag — Quarter = mag 0 (indices 0,3,6),
@@ -2299,9 +2303,10 @@ impl DQNTrainer {
g12_loss,
// Track 1 — magnitude (10 f32): q_full, q_half, q_quarter, var_scale,
// kelly_f, avg_win_ratio, grad_ratio_mag_dir, dist_q, dist_h, dist_f.
// Q-values, var_scale, Kelly, grad-ratio still stubbed (need GPU readback
// infrastructure from Task 0.3 follow-up); action dist wired now.
0.0_f32, 0.0_f32, 0.0_f32, 0.0_f32, 0.0_f32, 0.0_f32, 0.0_f32,
// Q-values, var_scale, Kelly remain to be wired by future Task 0.3 work.
// grad_ratio_mag_dir now real (Task 0.4); action_dist wired.
0.0_f32, 0.0_f32, 0.0_f32, 0.0_f32, 0.0_f32, 0.0_f32,
grad_ratio_mag_dir,
dist_q, dist_h, dist_f,
// Track 1 — trail (6 f32)
0.0_f32, 0.0_f32, 0.0_f32, 0.0_f32, 0.0_f32, 0.0_f32,