diag(policy-quality): Task 2.0 confirmation — expose absolute grad_dir / grad_mag norms

Task 0.4's grad_ratio_mag_dir returns 0.0 whenever dir_norm < 1e-9, so
the epoch-end reading of 0.0000 doesn't disambiguate "magnitude
starved" vs "direction starved". Task 2.0's per-component data showed
CQL and C51 each sending 100-400x more gradient to magnitude than
direction — implying direction is the starved one, not magnitude.

This adds a HEALTH_DIAG field exposing the raw absolute norms:

  grad_abs [dir=<sci-notation> mag=<sci-notation>]

Along the way uncovered + fixed two latent bugs that had been silently
zeroing the ratio signal since Task 0.4 landed:

1. `per_branch_grad_norms` read `grad_buf.len()` = total_params +
   cutlass_tile_pad (~4096 elements of GEMM tile padding) but the
   pinned readback slot was sized at construction to total_params
   exactly. The size check `grad_len > grad_readback_pinned_capacity`
   was always true, so the accessor returned Err on every call — and
   FusedTrainingCtx's proxy coerced Err to 0.0 via `.unwrap_or(0.0)`.
   Root cause for the "always 0.0000" grad_ratio_mag_dir. Fix: read
   only the first `total_params` prefix of grad_buf (the tail is pure
   GEMM padding, never holds gradient values).

2. Readback timing: process_epoch_boundary calls
   estimate_avg_q_value_with_early_stopping early, which replays
   `eval_forward_exec` — the SAME captured graph as forward_child
   whose first op is `cuMemsetD32Async(grad_buf, 0, total_params)`.
   Any grad_buf readback AFTER the avg_q call sees all zeros. Fix:
   snapshot grad_dir_abs / grad_mag_abs / grad_ratio_mag_dir at the
   TOP of process_epoch_boundary, before avg_q runs, and consume the
   cached values in the HEALTH_DIAG block.

Last 5 epochs of fold 3 on the magnitude_distribution baseline smoke
(FOXHUNT_TEST_DATA=test_data/futures-baseline):

  HEALTH_DIAG[15] ratio=42.85   grad_abs [dir=6.804900e0  mag=3.989081e2]
  HEALTH_DIAG[16] ratio=232.66  grad_abs [dir=6.500046e0  mag=3.603353e0]
  HEALTH_DIAG[17] ratio=318.34  grad_abs [dir=2.720317e-2 mag=1.713861e1]
  HEALTH_DIAG[18] ratio=367.59  grad_abs [dir=5.180866e-2 mag=8.076681e0]
  HEALTH_DIAG[19] ratio=298.55  grad_abs [dir=1.989553e-2 mag=6.498848e0]

Across all 60 epoch-boundary readings (3 folds × 20 epochs) dir ∈
[~4e-3, ~6e0] and mag ∈ [~5e-2, ~5e2], with ratio mag/dir consistently
50-400× (matching Task 2.0's per-component ratios). Neither branch is
near float precision — direction is PROPORTIONALLY starved, not
numerically zero.

Scenario confirmed: direction is starved relative to magnitude, NOT
the reverse. Phase 2's Task 2.1 (architectural fix on magnitude
branch) should pivot toward increasing direction's gradient flow
instead.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs

@@ -2196,13 +2196,20 @@ impl GpuDqnTrainer {
         if self.grad_readback_pinned_ptr == 0 {
             return Ok([0.0_f32; 4]);
         }
-        let grad_len = self.grad_buf.len();
-        // Pinned buffer is sized TOTAL_PARAMS at construction; refuse if mismatch
-        // (defensive — should never happen, but better an error than UB).
-        if grad_len > self.grad_readback_pinned_capacity {
+        // Read only the `total_params` prefix of grad_buf — the trailing
+        // `cutlass_tile_pad` slots are GEMM padding, never hold param gradients,
+        // and were never sized into `grad_readback_pinned_capacity`. The
+        // original check compared `grad_buf.len()` (= total_params + tile_pad)
+        // against the pinned capacity (= total_params) and returned Err — which
+        // the fused proxy silently coerced to 0.0, so both
+        // `grad_ratio_mag_dir` and `grad_norms_dir_mag_abs` reported zeros for
+        // every epoch instead of real per-branch norms. Readback is bounded to
+        // `total_params` which matches the pinned capacity exactly.
+        let read_len = self.total_params;
+        if read_len > self.grad_readback_pinned_capacity {
             return Err(MLError::ModelError(format!(
-                "per_branch_grad_norms: grad_buf len {} exceeds pinned readback capacity {}",
-                grad_len, self.grad_readback_pinned_capacity
+                "per_branch_grad_norms: total_params {} exceeds pinned readback capacity {}",
+                read_len, self.grad_readback_pinned_capacity
             )));
         }
         // Stream-sync to ensure backward + Adam grad accumulation has completed
@@ -2212,10 +2219,10 @@ impl GpuDqnTrainer {
         })?;
         // Reinterpret pinned host slot as &mut [f32].
         let host_slice: &mut [f32] = unsafe {
-            std::slice::from_raw_parts_mut(self.grad_readback_pinned_ptr as *mut f32, grad_len)
+            std::slice::from_raw_parts_mut(self.grad_readback_pinned_ptr as *mut f32, read_len)
         };
         self.stream
-            .memcpy_dtoh(&self.grad_buf.slice(..grad_len), host_slice)
+            .memcpy_dtoh(&self.grad_buf.slice(..read_len), host_slice)
             .map_err(|e| MLError::ModelError(format!("per_branch_grad_norms dtoh: {e}")))?;
 
         let param_sizes = compute_param_sizes(&self.config);
@@ -2248,6 +2255,18 @@ impl GpuDqnTrainer {
         Ok(if dir > 1e-9 { mag / dir } else { 0.0 })
     }
 
+    /// Absolute per-branch grad L2 norms, not ratios. Returns `(dir, mag)`.
+    ///
+    /// Task 2.0 confirmation accessor — `grad_ratio_mag_dir` collapses to 0.0
+    /// whenever `dir < 1e-9`, which masks whether direction is starved
+    /// (tiny but non-zero dir) vs magnitude is zero. Exposing the raw norms
+    /// disambiguates the two. Zero here means genuine near-float-precision
+    /// — don't coerce via threshold.
+    pub fn grad_norms_dir_mag_abs(&self) -> Result<(f32, f32), MLError> {
+        let n = self.per_branch_grad_norms()?;
+        Ok((n[0], n[1]))  // dir, mag
+    }
+
     /// Task 2.0 — in-graph DtoD snapshot of `grad_buf`'s branch 0+1 slice into
     /// the caller-supplied scratch buffer. Uses the `copy_f32` kernel (graph-
     /// captured DtoD) because `cuMemcpyDtoDAsync` is NOT captured by CUDA

crates/ml/src/trainers/dqn/fused_training.rs

@@ -891,6 +891,21 @@ impl FusedTrainingCtx {
         self.trainer.grad_ratio_mag_dir().unwrap_or(0.0)
     }
 
+    /// Task 2.0 confirmation — absolute per-branch grad L2 norms, returning
+    /// `(dir, mag)`. Complements `grad_ratio_mag_dir` — the ratio collapses
+    /// to 0.0 when `dir < 1e-9`, masking whether dir is starved (tiny) or
+    /// mag is zero. Raw norms disambiguate. Zeros on accessor failure
+    /// (non-fatal — diag only), matching the Task 0.5 / Task 2.0 pattern.
+    pub(crate) fn grad_norms_dir_mag_abs(&mut self) -> (f32, f32) {
+        match self.trainer.grad_norms_dir_mag_abs() {
+            Ok(v) => v,
+            Err(e) => {
+                tracing::warn!("grad_norms_dir_mag_abs failed: {e}");
+                (0.0, 0.0)
+            }
+        }
+    }
+
     /// Task 2.0 — refresh the cached per-component grad-norm arrays from the
     /// pinned result slot populated by the in-graph reduction kernel.
     /// Epoch-boundary cold path; stream-syncs once. Returns `Err` on

crates/ml/src/trainers/dqn/trainer/training_loop.rs

@@ -1776,6 +1776,24 @@ impl DQNTrainer {
             (0.0_f32, 0.0_f32)
         };
 
+        // Snapshot absolute per-branch grad L2 norms + their ratio BEFORE
+        // `estimate_avg_q_value_with_early_stopping` runs — that path replays
+        // `eval_forward_exec`, the same captured graph as `forward_child`, whose
+        // first op is `cuMemsetD32Async(grad_buf, 0, total_params)` (see
+        // `submit_forward_ops_main`). Without this early read, the HEALTH_DIAG
+        // accessor downstream would see an all-zero grad_buf — not "direction
+        // starved" or "magnitude zero", but a timing-induced wipe. Cache the
+        // real post-last-step values here and consume them in the HEALTH_DIAG
+        // block further down.
+        let (grad_dir_abs_cached, grad_mag_abs_cached) = self.fused_ctx
+            .as_mut()
+            .map(|f| f.grad_norms_dir_mag_abs())
+            .unwrap_or((0.0, 0.0));
+        let grad_ratio_mag_dir_cached = self.fused_ctx
+            .as_mut()
+            .map(|f| f.grad_ratio_mag_dir())
+            .unwrap_or(0.0);
+
         let avg_q = {
             let agent_clone = Arc::clone(&self.agent);
             let mut agent = agent_clone.write().await;
@@ -2224,9 +2242,18 @@ impl DQNTrainer {
                 0.0
             };
             // Task 0.4 — per-branch grad-norm ratio (magnitude/direction). H4 signal.
-            let grad_ratio_mag_dir = self.fused_ctx.as_mut()
-                .map(|f| f.grad_ratio_mag_dir())
-                .unwrap_or(0.0);
+            // Read at the TOP of `process_epoch_boundary` (before avg_q's eval
+            // forward replay wipes grad_buf via the captured memset) and cached.
+            let grad_ratio_mag_dir = grad_ratio_mag_dir_cached;
+            // Task 2.0 confirmation — absolute per-branch norms (dir, mag).
+            // `grad_ratio_mag_dir` coerces to 0.0 whenever dir < 1e-9, so the
+            // ratio alone can't tell "direction starved" from "magnitude zero".
+            // Surface the raw L2 norms here so HEALTH_DIAG can print both in
+            // scientific notation, distinguishing 1e-12 from exact zero.
+            // Cached at the TOP of `process_epoch_boundary` (same reason as
+            // the ratio above — eval-forward replay zeroes grad_buf).
+            let grad_dir_abs = grad_dir_abs_cached;
+            let grad_mag_abs = grad_mag_abs_cached;
 
             // Task 2.0 (+ extension) — per-component grad decomposition across
             // 9 measurement points: IQN / CQL / CQL-SAXPY / C51 / C51-budget-
@@ -2605,7 +2632,7 @@ impl DQNTrainer {
             };
 
             tracing::info!(
-                "HEALTH_DIAG[{}]: health={:.2} components [q_gap={:.2} q_var={:.2} atoms={:.2} grad_stable={:.2} ens_agree={:.2} grad_cos={:.2} spectral={:.2}] effective [cql_alpha={:.4} iqn_budget={:.2} cql_budget={:.2} c51_budget={:.2} tau={:.5} sarsa_tau={:.2} gamma={:.3} cf_ratio={:.2}] novels [distill={} barrier={:.3} plasticity={} ib={:.3} ensemble_collapse={:.2} contrarian={} meta_q_pred={:.2}] diag [sharpe_ema={:.3} action_entropy={:.2}] gems [g12_predictive={:.4}] mag [q_full={:.3} q_half={:.3} q_quarter={:.3} var_scale={:.3} kelly_f={:.3} avg_win_ratio={:.3} grad_ratio_mag_dir={:.4} dist_q={:.3} dist_h={:.3} dist_f={:.3}] grad_split_bwd [iqn={:.4} cql={:.4} c51={:.4} ens={:.4}] grad_split_aux [distill={:.4} rec={:.4} pred={:.4} cql_sx={:.4} c51_bs={:.4}] trail [fire_q={:.3} fire_h={:.3} fire_f={:.3} hold_q={:.2} hold_h={:.2} hold_f={:.2}] noisy [vsn_mag={:.3} vsn_dir={:.3} sigma_mag={:.4} sigma_dir={:.4} drift_mag={:.3} drift_dir={:.3}] eval_dist [eq={:.3} eh={:.3} ef={:.3}] reward_contrib [popart={:.3} cf={:.3} trail_r={:.3} micro={:.3} la={:.3}] controller [anti_lr={} tau={} gamma={} clip={} cql={} cost={} fire_frac={:.2}] explore [ent_mag={:.2} ent_dir={:.2} sigma_mean={:.4}]",
+                "HEALTH_DIAG[{}]: health={:.2} components [q_gap={:.2} q_var={:.2} atoms={:.2} grad_stable={:.2} ens_agree={:.2} grad_cos={:.2} spectral={:.2}] effective [cql_alpha={:.4} iqn_budget={:.2} cql_budget={:.2} c51_budget={:.2} tau={:.5} sarsa_tau={:.2} gamma={:.3} cf_ratio={:.2}] novels [distill={} barrier={:.3} plasticity={} ib={:.3} ensemble_collapse={:.2} contrarian={} meta_q_pred={:.2}] diag [sharpe_ema={:.3} action_entropy={:.2}] gems [g12_predictive={:.4}] mag [q_full={:.3} q_half={:.3} q_quarter={:.3} var_scale={:.3} kelly_f={:.3} avg_win_ratio={:.3} grad_ratio_mag_dir={:.4} dist_q={:.3} dist_h={:.3} dist_f={:.3}] grad_split_bwd [iqn={:.4} cql={:.4} c51={:.4} ens={:.4}] grad_split_aux [distill={:.4} rec={:.4} pred={:.4} cql_sx={:.4} c51_bs={:.4}] grad_abs [dir={:.6e} mag={:.6e}] trail [fire_q={:.3} fire_h={:.3} fire_f={:.3} hold_q={:.2} hold_h={:.2} hold_f={:.2}] noisy [vsn_mag={:.3} vsn_dir={:.3} sigma_mag={:.4} sigma_dir={:.4} drift_mag={:.3} drift_dir={:.3}] eval_dist [eq={:.3} eh={:.3} ef={:.3}] reward_contrib [popart={:.3} cf={:.3} trail_r={:.3} micro={:.3} la={:.3}] controller [anti_lr={} tau={} gamma={} clip={} cql={} cost={} fire_frac={:.2}] explore [ent_mag={:.2} ent_dir={:.2} sigma_mean={:.4}]",
                 epoch,
                 health_value,
                 self.learning_health.components.q_gap_norm,
@@ -2671,6 +2698,10 @@ impl DQNTrainer {
                 grad_mag_iqn, grad_mag_cql, grad_mag_c51, grad_mag_ens,
                 grad_mag_distill, grad_mag_rec, grad_mag_pred,
                 grad_mag_cql_sx, grad_mag_c51_bs,
+                // Task 2.0 confirmation — absolute per-branch grad L2 norms
+                // (dir, mag) in scientific notation. Disambiguates the
+                // `grad_ratio_mag_dir` threshold collapse (0.0 when dir < 1e-9).
+                grad_dir_abs, grad_mag_abs,
                 // Track 1 — trail (6 f32): fire_q/h/f, hold_q/h/f (H6 — Task 0.5).
                 // Real values from GPU experience collector's per-sample buffers.
                 trail_rates[0], trail_rates[1], trail_rates[2],