From d6eca73e52a2aeaa85300ff567ce29e02e2dcbdb Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Fri, 1 May 2026 17:11:19 +0200 Subject: [PATCH] =?UTF-8?q?feat(dqn):=20#212=20=E2=80=94=20intent-side=20m?= =?UTF-8?q?agnitude=20distribution=20HEALTH=5FDIAG=20metric?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds intent_dist_q/h/f alongside eval_dist_q/h/f to separate policy- learning quality from Kelly-enforcement reality. Per memory pearl project_magnitude_eval_collapse_kelly_capped.md: EVAL_DIST Quarter dominance is a downstream artefact of Kelly cap × warmup_floor × safety_multiplier math, NOT a policy-learning failure. The intent metric exposes the policy's pre-Kelly-cap chosen mag bucket so operators can distinguish "policy isn't learning Full" from "Kelly cap is suppressing Full" — the latter being an operationally-correct state during cold-start positions. Pure observability: no Kelly-math tweaks, no new tuned constants, no reward-bias mechanisms (all explicitly forbidden by the memory pearl). The intent_mag bucket comes straight from the factored action's mag_idx (0/1/2) which already maps 1:1 to Quarter/Half/Full buckets. The pre-cap mag_idx is already captured by the action-select kernel into intent_mag_buf and exposed via the trainer's pre-existing last_eval_intent_magnitude_dist host field — this commit only wires that signal into HEALTH_DIAG. Wired through (one coordinated commit per feedback_no_partial_refactor): - health_diag.rs: HealthDiagSnapshot gains intent_dist_q/h/f fields adjacent to eval_dist_*; size assertion bumped 147 → 150 fields. - health_diag_kernel.cu: 3 new WORD_INTENT_DIST_* slots at [77..80); every downstream WORD_* shifted by +3 in lockstep; WORD_TOTAL + static_assert bumped 147 → 150. (The slots are reserved identically to WORD_EVAL_DIST_* — both currently inert; HEALTH_DIAG GPU port Phases 2/3 will populate them in lockstep with their sibling slots.) - training_loop.rs: new adjacent tracing::info!() emitting "intent_dist [iq=... ih=... if=...]" from the existing host-side last_eval_intent_magnitude_dist field, immediately after the big HEALTH_DIAG line containing eval_dist [eq=... eh=... ef=...]. - docs/dqn-wire-up-audit.md: new top-of-file entry per Invariant 7. Behavior: zero change. Only adds observability. Verification: - cargo check -p ml --offline: clean. - cargo test -p ml --lib --offline -- health_diag: 3/3 pass. - cargo test -p ml --test sp4_producer_unit_tests --release --ignored: 16/16 GPU tests pass. Closes #212. --- crates/ml/src/cuda_pipeline/health_diag.rs | 22 ++++- .../src/cuda_pipeline/health_diag_kernel.cu | 94 ++++++++++--------- .../src/trainers/dqn/trainer/training_loop.rs | 19 ++++ docs/dqn-wire-up-audit.md | 2 + 4 files changed, 90 insertions(+), 47 deletions(-) diff --git a/crates/ml/src/cuda_pipeline/health_diag.rs b/crates/ml/src/cuda_pipeline/health_diag.rs index 0eea37eff..b942689cb 100644 --- a/crates/ml/src/cuda_pipeline/health_diag.rs +++ b/crates/ml/src/cuda_pipeline/health_diag.rs @@ -171,6 +171,20 @@ pub struct HealthDiagSnapshot { pub eval_dist_h: f32, pub eval_dist_f: f32, + // ── Intent per-magnitude distribution (3 floats) ───────────────────────── + /// Pure-policy magnitude bucket BEFORE Kelly cap clipping. Layout + /// `[Quarter, Half, Full]` in `[0, 1]`. Sourced from the trainer's + /// `last_eval_intent_magnitude_dist`, which reads `intent_mag_buf` + /// (the action-select kernel's pre-Kelly-cap mag_idx). Paired with + /// `eval_dist_*` so operators can distinguish "policy isn't learning + /// Full" (both metrics near zero) from "Kelly cap is suppressing + /// Full" (intent_dist_f ≫ eval_dist_f). Per memory pearl + /// `project_magnitude_eval_collapse_kelly_capped.md`: pure observability, + /// the Kelly cap math itself stays unchanged. + pub intent_dist_q: f32, + pub intent_dist_h: f32, + pub intent_dist_f: f32, + // ── Reward contributions (4 floats) ────────────────────────────────────── pub reward_contrib_popart: f32, pub reward_contrib_cf: f32, @@ -377,16 +391,16 @@ mod tests { // 8 health + 8 effective + 7 novels + 2 diag + 1 gem // + 10 mag + 4 grad_bwd + 5 grad_aux + 9 grad_trunk + 2 grad_abs // + 6 trail + 6 magstats + 6 noisy + 3 eval_dist - // + 4 reward_contrib + 7 controller + 3 explore + // + 3 intent_dist + 4 reward_contrib + 7 controller + 3 explore // + 16 val + 4 val_dir + 4 val_picked // + 6 reward_split + 4 aux + 10 aux_moe // + 12 action_counts - // = 8+8+7+2+1+10+4+5+9+2+6+6+6+3+4+7+3+16+4+4+6+4+10+12 = 147 fields - let expected_bytes = 147 * 4; + // = 8+8+7+2+1+10+4+5+9+2+6+6+6+3+3+4+7+3+16+4+4+6+4+10+12 = 150 fields + let expected_bytes = 150 * 4; assert_eq!( std::mem::size_of::(), expected_bytes, - "HealthDiagSnapshot must be {} bytes (147 × 4); update test if struct changes", + "HealthDiagSnapshot must be {} bytes (150 × 4); update test if struct changes", expected_bytes, ); } diff --git a/crates/ml/src/cuda_pipeline/health_diag_kernel.cu b/crates/ml/src/cuda_pipeline/health_diag_kernel.cu index 322a18888..e3a4e36e3 100644 --- a/crates/ml/src/cuda_pipeline/health_diag_kernel.cu +++ b/crates/ml/src/cuda_pipeline/health_diag_kernel.cu @@ -137,61 +137,69 @@ #define WORD_EVAL_DIST_H 75 #define WORD_EVAL_DIST_F 76 -/* Reward contributions (4 words, [77..81)). */ -#define WORD_REWARD_CONTRIB_POPART 77 -#define WORD_REWARD_CONTRIB_CF 78 -#define WORD_REWARD_CONTRIB_TRAIL 79 -#define WORD_REWARD_CONTRIB_MICRO 80 +/* Intent dist (3 words, [77..80)). Pure-policy magnitude bucket BEFORE + * Kelly cap clipping; sibling of eval_dist for separating "policy isn't + * learning Full" from "Kelly cap is suppressing Full". Per memory pearl + * `project_magnitude_eval_collapse_kelly_capped.md`. */ +#define WORD_INTENT_DIST_Q 77 +#define WORD_INTENT_DIST_H 78 +#define WORD_INTENT_DIST_F 79 -/* Controller fire bools + frac (7 words, [81..88)). */ -#define WORD_CTRL_FIRE_ANTI_LR 81 /* u32 */ -#define WORD_CTRL_FIRE_TAU 82 /* u32 */ -#define WORD_CTRL_FIRE_GAMMA 83 /* u32 */ -#define WORD_CTRL_FIRE_CLIP 84 /* u32 */ -#define WORD_CTRL_FIRE_CQL 85 /* u32 */ -#define WORD_CTRL_FIRE_COST 86 /* u32 */ -#define WORD_CTRL_FIRE_FRAC 87 +/* Reward contributions (4 words, [80..84)). */ +#define WORD_REWARD_CONTRIB_POPART 80 +#define WORD_REWARD_CONTRIB_CF 81 +#define WORD_REWARD_CONTRIB_TRAIL 82 +#define WORD_REWARD_CONTRIB_MICRO 83 -/* Explore (3 words, [88..91)). */ -#define WORD_EXPLORE_ENT_MAG 88 -#define WORD_EXPLORE_ENT_DIR 89 -#define WORD_EXPLORE_SIGMA_MEAN 90 +/* Controller fire bools + frac (7 words, [84..91)). */ +#define WORD_CTRL_FIRE_ANTI_LR 84 /* u32 */ +#define WORD_CTRL_FIRE_TAU 85 /* u32 */ +#define WORD_CTRL_FIRE_GAMMA 86 /* u32 */ +#define WORD_CTRL_FIRE_CLIP 87 /* u32 */ +#define WORD_CTRL_FIRE_CQL 88 /* u32 */ +#define WORD_CTRL_FIRE_COST 89 /* u32 */ +#define WORD_CTRL_FIRE_FRAC 90 -/* Validation block (16 words, [91..107)). */ -#define WORD_VAL_BASE 91 +/* Explore (3 words, [91..94)). */ +#define WORD_EXPLORE_ENT_MAG 91 +#define WORD_EXPLORE_ENT_DIR 92 +#define WORD_EXPLORE_SIGMA_MEAN 93 -/* Validation per-direction action distribution (4 words, [107..111)). */ -#define WORD_VAL_DIR_DIST_BASE 107 +/* Validation block (16 words, [94..110)). */ +#define WORD_VAL_BASE 94 -/* Validation per-direction picked distribution (4 words, [111..115)). */ -#define WORD_VAL_PICKED_DIR_DIST_BASE 111 +/* Validation per-direction action distribution (4 words, [110..114)). */ +#define WORD_VAL_DIR_DIST_BASE 110 -/* Reward split (6 words, [115..121)). */ -#define WORD_REWARD_SPLIT_POPART 115 -#define WORD_REWARD_SPLIT_CF 116 -#define WORD_REWARD_SPLIT_TRAIL 117 -#define WORD_REWARD_SPLIT_MICRO 118 -#define WORD_REWARD_SPLIT_OPP_COST 119 -#define WORD_REWARD_SPLIT_BONUS 120 +/* Validation per-direction picked distribution (4 words, [114..118)). */ +#define WORD_VAL_PICKED_DIR_DIST_BASE 114 -/* Aux block (4 words, [121..125)). */ -#define WORD_AUX_NEXT_BAR_MSE 121 -#define WORD_AUX_REGIME_CE 122 -#define WORD_AUX_WEIGHT 123 -#define WORD_AUX_LABEL_SCALE 124 +/* Reward split (6 words, [118..124)). */ +#define WORD_REWARD_SPLIT_POPART 118 +#define WORD_REWARD_SPLIT_CF 119 +#define WORD_REWARD_SPLIT_TRAIL 120 +#define WORD_REWARD_SPLIT_MICRO 121 +#define WORD_REWARD_SPLIT_OPP_COST 122 +#define WORD_REWARD_SPLIT_BONUS 123 -/* Aux MoE (10 words, [125..135) — 8 utils + ent + λ_eff). */ -#define WORD_AUX_MOE_UTIL_BASE 125 -#define WORD_AUX_MOE_ENT 133 -#define WORD_AUX_MOE_LAMBDA_EFF 134 +/* Aux block (4 words, [124..128)). */ +#define WORD_AUX_NEXT_BAR_MSE 124 +#define WORD_AUX_REGIME_CE 125 +#define WORD_AUX_WEIGHT 126 +#define WORD_AUX_LABEL_SCALE 127 -/* Action-count histogram (12 words, [135..147), all u32). */ -#define WORD_ACTION_COUNTS_BASE 135 +/* Aux MoE (10 words, [128..138) — 8 utils + ent + λ_eff). */ +#define WORD_AUX_MOE_UTIL_BASE 128 +#define WORD_AUX_MOE_ENT 136 +#define WORD_AUX_MOE_LAMBDA_EFF 137 -#define WORD_TOTAL 147 +/* Action-count histogram (12 words, [138..150), all u32). */ +#define WORD_ACTION_COUNTS_BASE 138 + +#define WORD_TOTAL 150 /* Pin the layout: matches `snapshot_size_is_stable` on the Rust side. */ -static_assert(WORD_TOTAL == 147, +static_assert(WORD_TOTAL == 150, "HealthDiagSnapshot layout drift — update kernel WORD_* and the " "Rust struct in lockstep, then bump snapshot_size_is_stable."); diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index 997a3324c..c8238f3d1 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -3846,6 +3846,25 @@ impl DQNTrainer { ent_mag, ent_dir, sigma_mean_overall, ); + // Task #212 — pure-policy magnitude intent distribution BEFORE + // Kelly cap clipping. Sourced from `last_eval_intent_magnitude_dist` + // (cached by `metrics.rs::compute_validation_loss` from + // `gpu_evaluator.read_eval_intent_magnitude_distribution`, which + // reads `intent_mag_buf` — the action-select kernel's pre-cap + // mag_idx). Paired with the `eval_dist [eq=...]` segment in the + // big info! line above so operators can distinguish "policy isn't + // learning Full" (both metrics near zero) from "Kelly cap is + // suppressing Full" (intent_dist_f ≫ eval_dist_f). Per memory + // pearl `project_magnitude_eval_collapse_kelly_capped.md`: pure + // observability — Kelly cap math itself is unchanged. + tracing::info!( + "HEALTH_DIAG[{}]: intent_dist [iq={:.3} ih={:.3} if={:.3}]", + epoch, + self.last_eval_intent_magnitude_dist[0], + self.last_eval_intent_magnitude_dist[1], + self.last_eval_intent_magnitude_dist[2], + ); + // C.2 Plan 3 Task 1 (spec §4.C.2): reward_split HEALTH_DIAG line. // Reads 6 ISV EMA slots updated by the GPU reward_component_ema kernel // launched just above. CPU-side code only reads; GPU wrote the values. diff --git a/docs/dqn-wire-up-audit.md b/docs/dqn-wire-up-audit.md index 5496194e1..ac20def36 100644 --- a/docs/dqn-wire-up-audit.md +++ b/docs/dqn-wire-up-audit.md @@ -2,6 +2,8 @@ **Status:** Populated during Plan 1 Task 6 (A.5 orphan audit). Updated on every commit per Invariant 7. +HEALTH_DIAG intent_dist diagnostic (#212, 2026-05-01): added `intent_dist_q/h/f` HEALTH_DIAG metric adjacent to the existing `eval_dist_q/h/f` so operators can split policy-learning quality from Kelly-enforcement reality. Per memory pearl `project_magnitude_eval_collapse_kelly_capped.md`: EVAL_DIST Quarter dominance is a downstream artefact of Kelly cap × warmup_floor × safety_multiplier math, NOT a policy-learning failure. The intent metric reads the policy's pre-Kelly-cap chosen mag bucket (`intent_mag_buf` written by the action-select kernel, exposed via the trainer's pre-existing `last_eval_intent_magnitude_dist` host field — no new compute, no new GPU paths). Pure observability — the Kelly cap math itself stays unchanged (it is already adaptive per recent commits 2c97e0436 / 0c9d1ee39 / 94157a8a6 / d9fee6ef8), no new tuned constants, no reward-bias mechanisms (all explicitly forbidden by the pearl). Three coordinated changes per `feedback_no_partial_refactor.md`: (1) `HealthDiagSnapshot` gains `intent_dist_q/h/f: f32` fields adjacent to `eval_dist_*`; size assertion bumped 147 → 150 fields. (2) `health_diag_kernel.cu` adds `WORD_INTENT_DIST_*` slots at [77..80); every downstream `WORD_*` shifted by +3 in lockstep; `WORD_TOTAL` + `static_assert` bumped 147 → 150. The slots are reserved identically to `WORD_EVAL_DIST_*` — both currently inert in the kernel; HEALTH_DIAG GPU port Phases 2/3 will populate them in lockstep with their sibling slots. (3) `training_loop.rs` adds an adjacent `tracing::info!()` emitting `"intent_dist [iq=… ih=… if=…]"` from `self.last_eval_intent_magnitude_dist` immediately after the big HEALTH_DIAG line containing `eval_dist [eq=… eh=… ef=…]`. Behavior: zero change. After this commit, HEALTH_DIAG emits BOTH metrics so downstream analysis can distinguish "policy is learning Full, Kelly cap is suppressing" (`intent_dist_f > 0.30` AND `eval_dist_f ≈ 0`, an operationally-correct cold-start state) from "policy genuinely isn't learning Full" (both metrics near zero, a policy-quality bug worth investigating). Touched: `cuda_pipeline/health_diag.rs`, `cuda_pipeline/health_diag_kernel.cu`, `trainers/dqn/trainer/training_loop.rs`. cargo check -p ml --lib clean (11 pre-existing warnings, none new); 3/3 health_diag layout tests pass; 16/16 sp4_producer_unit_tests GPU tests pass. No fingerprint change — separate mapped-pinned alloc, not part of param-tensor layout the fingerprint guards. + SP3 Mech 10 — ISV-driven post-trunk h_s2 activation clamp + slot 49 sentinel (2026-04-30): real root cause of the F1 step-1000 NaN identified by smoke-test-2xrxk on commit `0d7e27d61` (SP3 close-out v2 active). Trajectory: F0 Best Sharpe 45.47 (clean) → F1 NaN at step ~1000 → F2 first epoch reached Best Sharpe **56.70** (above the 55.87 baseline) before NaN epoch 2. Slot fire pattern at F1 NaN: `[6=grad_buf, 12=save_h_s2, 26=iqn_trunk_m, 32=bn_d_concat_buf, 36-38=trunk/value/branch_adam_m, 40-42=trunk/value/branch_adam_v]`. Crucially: slots 48 (iqn_weight_max), 39+43 (iqn_adam_m/v), 44-45 (trunk/heads weights) all stayed clean — disproving the SP3 close-out v2 hypothesis (IQN partial refactor). The actual cause: forward-pass GEMM accumulator overflow under Mech-9-clamped weights. With Mech 9 pinning weights at `100 × Q_ABS_REF = 5000` and trunk inner-dim K = 256, forward GEMM accumulators compound by `√K × |w|max ≈ 80000×` per layer; trunk depth ≥6 (encoder + VSN + GRN-blocks + bottleneck) hits `f32_max ≈ 3.4e38` somewhere. F0/F2 train clean because their gradients keep weights at natural scale (~0.5); F1's regime shift drives weights toward the clamp ceiling and the forward chain saturates. Mech 1+2 clamped TARGETS+ATOMS, Mech 9 clamped WEIGHTS, **nothing clamped ACTIVATIONS** — Mech 10 closes that gap. Mech 10 mechanism: after the trunk + temporal pipeline finalises `save_h_s2` in `submit_forward_ops_main` (end of "1c. Temporal pipeline" block — last writers are `mamba2_step` + `apply_regime_dropout`), the trainer launches `launch_clamp_finite_f32` on `save_h_s2` with bound `100 × ISV[H_S2_RMS_EMA_INDEX=96].max(1.0)`. Reuses the existing helper at `dqn_utility_kernels.cu:462` (`isfinite(v) ? clamp(v, ±max_abs) : 0.0f` — NaN→0, Inf→0, finite clamped) — no new kernel. ISV-driven via existing `H_S2_RMS_EMA_INDEX = 96` (already populated per Plan 4 Task 2c.3c.5; consumed by `mag_concat_qdir` adaptive scale per 2c.3c.6) — no new ISV slot, no hardcoded multipliers, consistent with `feedback_isv_for_adaptive_bounds.md`. ε on multiplier per SP1 pearl (`isv.max(1.0)`) — at fold boundary the EMA resets to 1.0 (neutral RMS) per `state_reset_registry.rs`, so the clamp floor is always at least `100 × 1 = 100`, never zero. SP1 diagnostic-ordering pearl honoured: the inline `check_nan_f32(slot 12)` fires BEFORE the clamp sanitises so the sentinel sees the original Inf/NaN before sanitization replaces it with 0; redundant with the slot 12 check inside `run_nan_checks_post_forward` (which runs later, post-clamp, on the sanitized buffer) — but the inline check at this site is the only one that captures the pre-clamp state. New diagnostic slot 49 = `h_s2_max_abs` at threshold `1e3 × h_s2_rms_ema_eff` mirrors the Mech 5 family pattern (clamp at 100×, diagnostic at 1000×) — sentinel that NEVER fires under normal Mech 10 operation. The fused NaN-check kernel (`dqn_nan_check_fused_f32_kernel`) gains a second ISV-derived float arg `h_s2_rms_ema_eff` distinct from `q_abs_ref_eff` (don't conflate the two ISV bases — `h_s2` lives in pre-readout activation space, not Q space). Buffer bumps: `nan_flags_buf` 49→50, metadata buffers (`nan_check_buf_ptrs`, `nan_check_buf_lens`) 25→26 entries, fused-kernel block count 25→26. Both name tables in `training_loop.rs` (`halt_nan` formatter + `halt_grad_collapse` formatter) extended to 50 entries with `"h_s2_max_abs"` at slot 49. `read_nan_flags()` return type 49→50 in both `GpuDqnTrainer` and `FusedTrainingCtx`. Per `feedback_no_partial_refactor.md`, Mech 10 + slot 49 land in the same commit (single coordinated migration). No new ISV slots, no new kernels, no graph-topology surprise (one extra launch added to the captured forward graph at the temporal-pipeline → loss-path boundary). Touched: `cuda_pipeline/dqn_utility_kernels.cu` (kernel signature + slot 25 branch), `cuda_pipeline/gpu_dqn_trainer.rs` (alloc 49→50, metadata 25→26, `populate_nan_check_meta` + slot 49 entry, fused-kernel `BLOCKS` 25→26 + `h_s2_rms_ema_eff` arg, `read_nan_flags` 49→50, Mech 10 clamp call at end of "1c. Temporal pipeline" in `submit_forward_ops_main`), `trainers/dqn/fused_training.rs` (`read_nan_flags` wrapper 49→50), `trainers/dqn/trainer/training_loop.rs` (both name tables 49→50 entries with `"h_s2_max_abs"` at slot 49), `docs/dqn-backward-nan-audit.md` (Mech 10 row in mechanism table + slot 49 row in slot-allocation table + ISV bindings note + full Mech 10 section). cargo check -p ml --lib clean (12 pre-existing warnings, none new). Validation deferred to one L40S smoke at this HEAD; expected F1 trains past step 1000 and slots 36-43 + slot 49 stay quiet. SP3 close-out v2 — Mech 9 to all 5 Adam kernels + IQN weight diag slot 48 (2026-04-30): real fix for the F1 step-3540 NaN. Commit `8956c2fb7` wired Mech 9 (post-Adam `|p_val|` clamp) into ONE Adam kernel (`dqn_adam_update_kernel`) out of FIVE in the codebase. The slot-26 GEMM (`apply_iqn_trunk_gradient` output) computes `grad_iqn @ W_iqn^T`; `W_iqn` lives in IQN's separate `online_params` buffer, updated by `iqn_adam_kernel` — never reached by Mech 9. Slots 44-45 only cover trunk + heads weights, leaving IQN's separate param buffer without a diagnostic. The chain: IQN weights drift via `iqn_adam_kernel` → finite gradient × non-finite weight → slot-26 NaN. Same partial-refactor class as Mech 4's missing GpuAttention reset (caught by B5 audit), corrected the same way per `feedback_no_partial_refactor.md`: every consumer of the contract migrates together. Extended Mech 9 to all four remaining Adam kernels — `iqn_adam_kernel` (`iqn_dual_head_kernel.cu`), `iql_adam_kernel` (`iql_value_kernel.cu`), `attn_adam_kernel` (`attention_backward_kernel.cu`, used by both `GpuAttention` and `GpuTlob` via the shared cubin), `curiosity_adam_step` (`curiosity_training_kernel.cu`). Same `if (weight_clamp_max_abs > 0.0f) p = fminf(fmaxf(p, -bound), bound)` pattern, last step before writeback. Same `100 × Q_ABS_REF.max(1.0)` ISV-driven bound (same slot 16, ε on multiplier per SP1 pearl). Each launch site computes the bound host-side and passes it as the trailing kernel arg, mirroring the existing dqn_adam_update_kernel pattern. IQN: `train_iqn_step_gpu` + `execute_training_pipeline` take new `weight_clamp_max_abs: f32` parameters; both fork-join arms in `fused_training.rs::submit_aux_ops` compute and pass the bound. IQL: `train_value_step` takes the bound; both `gpu_iql.train_value_step` (high-tau) and `gpu_iql_low.train_value_step` (low-tau) wired. Attention: `GpuAttention::adam_step` and `GpuTlob::adam_step` both take new `weight_clamp_max_abs: f32`; all 3 call sites in fused_training (parallel attn, sequential attn, TLOB Phase 6) wired. Curiosity: `GpuCuriosityTrainer::train_on_collector_buffers` and the inner `launch_adam_step` helper take the bound; `GpuExperienceCollector::train_curiosity_gpu` wraps and forwards; `training_loop.rs::collect_step` reads ISV from `fused_ctx` (curiosity collector doesn't own ISV) and passes through. DT launch in `decision_transformer.rs` keeps the 0.0 disable (offline-RL, outside SP3 scope). Added IQN-weight diagnostic slot 48 to plug the slot-26 GEMM blind spot: `nan_flags_buf` 48 → 49 (`stream.alloc_zeros::(49)`), fused-kernel block count 24 → 25 with new branch for absolute slot 48 = relative slot 24 at threshold `1e3 × q_abs_ref_eff` (matches slot 44-45 weight regime), metadata buffers `nan_check_buf_ptrs` / `nan_check_buf_lens` extended 24 → 25 with the new slot 48 entry populated by `populate_nan_check_meta` from `gpu_iqn.online_params_ptr()` + `online_params_len()` (new public accessors on `GpuIqnHead`; existing `cfg(test)` `online_params_slice()` accessor kept). Both name tables in `training_loop.rs` (the `halt_nan` formatter at line ~2042 and the `halt_grad_collapse` formatter at line ~2143) extended to 49 entries with `"iqn_weight_max"` at slot 48 — symmetry required because either path can format `nan_flags_buf`, and an out-of-range table would out-of-bounds index when slot 48 fires. `read_nan_flags()` return type bumped `[i32; 48] → [i32; 49]` on both `GpuDqnTrainer` and the `FusedTrainingCtx` wrapper. Audit doc `dqn-backward-nan-audit.md` Mech 5 slot table now spans 36-48 with the new row; Mech 9 row updated to span all 5 Adam kernels (file list expanded). No new ISV slots, no new mapped-pinned allocations beyond the metadata-buffer +1, no graph-topology change beyond the +1 block in the already-fused NaN check. Touched: `cuda_pipeline/iqn_dual_head_kernel.cu` (kernel signature + clamp), `cuda_pipeline/iql_value_kernel.cu` (kernel signature + clamp), `cuda_pipeline/attention_backward_kernel.cu` (kernel signature + clamp), `cuda_pipeline/curiosity_training_kernel.cu` (kernel signature + clamp), `cuda_pipeline/dqn_utility_kernels.cu` (fused NaN check: grid 24→25, slot 24 branch), `cuda_pipeline/gpu_iqn_head.rs` (new `online_params_ptr` / `online_params_len` accessors + `train_iqn_step_gpu` + `execute_training_pipeline` signature + Adam launch arg), `cuda_pipeline/gpu_iql_trainer.rs` (`train_value_step` signature + Adam launch arg), `cuda_pipeline/gpu_attention.rs` (`adam_step` signature + launch arg), `cuda_pipeline/gpu_tlob.rs` (`adam_step` signature + launch arg), `cuda_pipeline/gpu_curiosity_trainer.rs` (`launch_adam_step` + `train_on_collector_buffers` signatures + 4 launch args), `cuda_pipeline/gpu_experience_collector.rs` (`train_curiosity_gpu` signature wrapper), `cuda_pipeline/gpu_dqn_trainer.rs` (alloc 48→49, metadata 24→25, `populate_nan_check_meta` 2 new params + slot 48 entry, fused-kernel `BLOCKS` 24→25, `read_nan_flags` 48→49), `trainers/dqn/fused_training.rs` (Q_ABS_REF_INDEX import + 4 wiring sites + read_nan_flags type), `trainers/dqn/trainer/training_loop.rs` (curiosity ISV read + name table 49 entries), `docs/dqn-backward-nan-audit.md` (slot table + Mech 9 row). cargo check -p ml --lib clean (12 pre-existing warnings, none new). Validation deferred to one L40S smoke at this HEAD; expected F1 trains past step 3720 ceiling and slots 36-43 + new slot 48 stay quiet.