#![allow(unsafe_code)]

//! Fused CUDA DQN training — forward + loss + backward + Adam in 4 kernel launches.
//!
//! Replaces 2,100+ Candle kernel dispatches per training batch with 4 fused CUDA
//! kernels captured in a CUDA Graph for zero-overhead replay.
//!
//! ## Architecture
//!
//! All tensors are pre-allocated at construction (fixed shapes for CUDA Graph
//! compatibility). Weight pointers come from existing `DuelingWeightSet` +
//! `BranchingWeightSet` objects (no duplication). The only per-batch host work
//! is copying batch data into pre-allocated input buffers and reading back
//! the scalar loss + td_errors.
//!
//! ## CUDA Graphs (child graph architecture)
//!
//! Graph ownership lives in `FusedTrainingCtx` which captures 5 child sub-graphs
//! (spectral, forward, ddqn, aux, adam) composed into a single parent graph.
//! `GpuDqnTrainer` provides `submit_forward_ops_main()`, `submit_forward_ops_ddqn()`,
//! and `submit_adam_ops()` for child graph capture. An eval-only forward graph exec
//! is injected via `set_eval_forward_exec()` for deterministic Q-value computation.
//!
//! Per-step scalars (adam_step, tau, adaptive_clip) use pinned
//! device-mapped memory -- GPU reads via pointer, host writes directly,
//! no HtoD copies. Only batch input data needs explicit upload before replay.
//!
//! ## Kernel phases
//!
//! Forward and backward passes are handled by cuBLAS (`CublasForward` /
//! `CublasBackward`). The two NVRTC utility kernels from `dqn_utility_kernels.cu`
//! handle the optimizer step:
//!
//! 1. **Grad norm** (`dqn_grad_norm_kernel`): Gradient L2 norm via warp + block
//!    reduction + atomicAdd into a single output float.
//! 2. **Adam update** (`dqn_adam_update_kernel`): Reads completed norm, clips
//!    gradients, applies AdamW update over flattened parameter view.
//!
//! ## Parameter layout
//!
//! The flat parameter/gradient/moment buffers use the same layout as the CUDA
//! `GOFF_*` defines: 42 weight tensors concatenated in order (w_s1, b_s1, w_s2,
//! b_s2, ..., w_b3out, b_b3out, w_bn, b_bn, w_vsn1_0..w_vsn2_3, w_gate_0..b_gate_3).
//! See `compute_param_sizes()`.

use std::sync::Arc;

use cudarc::driver::{
    CudaFunction, CudaSlice, CudaStream, DevicePtrMut, LaunchConfig, PushKernelArg,
};
use tracing::info;

use cudarc::cublaslt::result as cublaslt_result;
use cudarc::cublaslt::sys as cublaslt_sys;

use crate::MLError;
use ml_dqn::{MOE_EXPERT_BOTTLENECK, MOE_GATE_HIDDEN, MOE_NUM_EXPERTS};
use super::gpu_attention::GpuAttention;
use super::gpu_weights::{DuelingWeightSet, BranchingWeightSet};
use super::batched_forward::{CublasForward, CublasGemmSet, f32_weight_ptrs_from_base};
use super::batched_backward::{CublasBackward, CublasBackwardSet, alloc_backward_scratch, raw_f32_ptr as bw_raw_f32_ptr};
use super::shared_cublas_handle::PerStreamCublasHandles;
use super::gpu_aux_heads::{
    AuxHeadsBackwardOps, AuxHeadsForwardOps, AuxTradeOutcomeBackwardOps,
    AuxTradeOutcomeForwardOps, AUX_HIDDEN_DIM, AUX_NEXT_BAR_K, AUX_OUTCOME_K, AUX_REGIME_K,
};
use super::gpu_health_diag::GpuHealthDiag;
use super::gpu_moe_head::GpuMoeHead;
use super::mapped_pinned::{MappedF32Buffer, MappedI32Buffer, MappedU32Buffer, MappedU64Buffer};
// SP4 Task A14: shared constants (engagement-counter buffer layout +
// per-Adam-kernel diag-slot allocations) used by `new()` constructor +
// the 5 Adam launch sites + the host-side Pearl C rate-deficit check.
use super::sp4_isv_slots::{
    SP4_PARAM_GROUP_COUNT, MAX_BLOCKS_PER_ADAM, SP4_ENGAGE_BUF_LEN,
    SP4_ENGAGE_OFFSET_DISABLED, SP4_NAN_FLAGS_END,
    SP4_DQN_ADAM_DIAG_SLOT, ParamGroup,
};

// ── Precompiled cubins (build.rs → include_bytes! → ZERO runtime nvcc) ──────
pub static DQN_UTILITY_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/dqn_utility_kernels.cubin"));
static EMA_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/ema_kernel.cubin"));
static RELU_MASK_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/relu_mask_kernel.cubin"));
static C51_LOSS_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/c51_loss_kernel.cubin"));
static C51_GRAD_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/c51_grad_kernel.cubin"));
static MSE_LOSS_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/mse_loss_kernel.cubin"));
static MSE_GRAD_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/mse_grad_kernel.cubin"));
/// Expected Q uses the compute_expected_q function from experience_kernels (single source of truth).
static EXPECTED_Q_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/experience_kernels.cubin"));
static Q_STATS_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/q_stats_kernel.cubin"));
static ENSEMBLE_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/ensemble_kernels.cubin"));
static CQL_GRAD_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/cql_grad_kernel.cubin"));
static MAMBA2_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/mamba2_temporal_kernel.cubin"));
pub(crate) static GRAPH_UTILITY_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/graph_utility_kernels.cubin"));
static GRAD_DECOMP_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/grad_decomp_kernel.cubin"));
/// SP7 Path A (2026-05-03): raw CQL gradient norm reading `cql_grad_scratch`
/// directly. Populates `grad_decomp_result_pinned[3..6]` (the previously
/// never-populated `cql` slot at element offset 3) with `‖raw_cql‖` over the
/// same trunk/dir/mag slices `grad_decomp_kernel` uses. The SP7
/// loss-balance controller reads this slot as a budget-independent CQL
/// reference; the prior `cql_sx` reference at offset 6 was budget-scaled
/// (cql_sx_norm = budget × raw_grad), creating a self-perpetuating
/// deadlock at small bootstrap budgets.
static CQL_RAW_NORM_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/cql_raw_norm_kernel.cubin"));
static BRANCH_GRAD_BALANCE_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/branch_grad_balance_kernel.cubin"));
/// Plan 1 Task 13: GPU-driven Polyak-EMA tau coefficient.
static TAU_UPDATE_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/tau_update_kernel.cubin"));
/// Plan 1 Task 14: GPU-driven effective exploration epsilon.
static EPSILON_UPDATE_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/epsilon_update_kernel.cubin"));
/// Plan 2 Task 3 D.2: GPU-driven per-branch effective gamma (replaces scalar gamma_update_kernel).
static PER_BRANCH_GAMMA_UPDATE_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/per_branch_gamma_update_kernel.cubin"));
/// Plan 1 Task 11: GPU-driven effective Kelly cap.
static KELLY_CAP_UPDATE_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/kelly_cap_update_kernel.cubin"));
/// Plan 1 Task 9: GPU-driven C51 atom position recompute (4-branch, single launch).
static ATOMS_UPDATE_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/atoms_update_kernel.cubin"));
static Q_QUANTILE_CUBIN:    &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/q_quantile_kernel.cubin"));
/// Plan 3 Task 1 C.2: per-component |reward| EMA into ISV[63..69).
pub(crate) static REWARD_COMPONENT_EMA_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/reward_component_ema_kernel.cubin"));
/// SP18 v2 Phase 0 Task 0.1 (2026-05-08): D-leg per-action reward
/// decomposition diagnostic. Reads `reward_components_per_sample`
/// + `actions_out` from the collector, bins per-direction
/// (DIR_HOLD / DIR_LONG / DIR_SHORT / DIR_FLAT), emits 5 stats per bin
/// (mean r_micro / r_opp_cost / r_popart / |reward| / fire_rate) into
/// a 20-float mapped-pinned `reward_decomp_diag_buf`. Block tree-
/// reduce — 4 blocks × 256 threads. Pure observability — no
/// production-path consumer in this commit. Plan: SP18 Phase 0 Task 0.1.
pub(crate) static REWARD_DECOMP_DIAG_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/reward_decomp_diag_kernel.cubin"));
/// SP18 v2 Phase 0 Task 0.2 (2026-05-08): B-leg TD-error magnitude
/// EMA producer. Single-block 256-thread kernel computing
/// mean(|td_errors[b]|) over [B] and EMA-blending into
/// ISV[TD_ERROR_MAG_EMA_INDEX=493] via Pearl-A first-observation
/// bootstrap + fixed α=0.4. Pure observability: pre-fix baseline for
/// the B-DD9 ratio gate. Plan: SP18 Phase 0 Task 0.2.
pub(crate) static TD_ERROR_MAG_EMA_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/td_error_mag_ema_kernel.cubin"));
/// Plan 3 Task 3 B.2: Flat→Positioned transition rate EMA into ISV[71].
pub(crate) static TRADE_RATE_EMA_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/trade_rate_ema_kernel.cubin"));
/// Plan 3 Task 4 B.4: per-batch readiness EMA + derived plan_threshold.
pub(crate) static PLAN_THRESHOLD_UPDATE_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/plan_threshold_update_kernel.cubin"));
/// Plan 3 Task 7 C.3: per-epoch train-vs-val state-distribution KL EMA +
/// adaptive Flat-trap escape amplifier (consumed by B.1/B.2 multipliers).
pub(crate) static STATE_KL_DIVERGENCE_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/state_kl_divergence_kernel.cubin"));
/// Plan 3 Task 8 B.3: GPU-only seeded warm-start scripted policies.
/// Replaces the network-Q action source during the seed phase. 4 policies
/// (uniform/momentum/mean-rev/vwap-deviation) deterministically mixed.
pub(crate) static SCRIPTED_POLICY_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/scripted_policy_kernel.cubin"));
/// Plan 3 Task 8 B.3: per-collect_experiences seed-phase progress counter
/// (ISV[83]) + adaptive seed-fraction EMA (ISV[84]).
pub(crate) static SEED_STEP_COUNTER_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/seed_step_counter_update_kernel.cubin"));
/// Plan 3 Task 9 C.5: CQL α ramp coupled to ISV[SEED_FRAC_EMA].
/// EMAs ISV[CQL_ALPHA_INDEX=48] toward `final × (1 - seed_frac)`.
pub(crate) static CQL_ALPHA_SEED_UPDATE_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/cql_alpha_seed_update_kernel.cubin"));
/// Plan 4 Task 5 Mode A (E.5): attention-focus interpretability EMA producer.
/// Single-thread cold-path kernel that EMA-updates ISV[VSN_MAG_EMA_INDEX=87],
/// ISV[VSN_DIR_EMA_INDEX=88], ISV[MAMBA2_RETENTION_EMA_INDEX=89] from 3 host
/// scalars passed by value. Diagnostic only.
pub(crate) static ATTENTION_FOCUS_EMA_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/attention_focus_ema_kernel.cubin"));

/// Plan 4 follow-up: target-drift EMA (replaces legacy CPU-DtoH
/// `per_branch_target_drift()`). 2-block GPU kernel computing
/// RMS(target − online) for mag + dir branches → ISV[92, 93].
pub(crate) static TARGET_DRIFT_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/target_drift_kernel.cubin"));

/// Plan 4 Task 2c.1: Gated Residual Network kernel module (forward + backward).
/// 8 kernels (`grn_elu_inplace`, `grn_glu_forward`,
/// `grn_residual_layernorm_forward`, `grn_layernorm_backward_dx`,
/// `grn_layernorm_backward_dgamma_dbeta_p1` / `_p2`, `grn_glu_backward`,
/// `grn_elu_backward`). Wired by Task 2c.2's `gpu_grn.rs` wrapper;
/// production callers added in 2c.3+4.
pub(crate) static GRN_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/grn_kernel.cubin"));

/// Plan 4 Task 2c.3c.5: per-batch RMS EMA of `save_h_s2` into ISV[96].
/// Single-block (256 threads, shmem-reduce, no atomicAdd) kernel launched
/// alongside `reward_component_ema` from `training_loop.rs`. Producer-only —
/// 2c.3c.6 wires the consumer in `mag_concat_qdir`'s adaptive-scale path.
static H_S2_RMS_EMA_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/h_s2_rms_ema_kernel.cubin"));

/// SP4 Layer A Task A5 (2026-04-30): first end-to-end SP4 producer cubin.
/// Single-block kernel that reads `denoise_target_q_buf`, computes
/// p99(|target_q|) via the header-only `sp4_histogram_p99<256>` device
/// function, and writes the per-step observation to
/// `producer_step_scratch_buf[0]`. The launcher
/// (`GpuDqnTrainer::launch_sp4_target_q_p99`) chains the GPU
/// `apply_pearls_ad_kernel` on the same stream (2026-05-01 GPU-Pearls
/// refactor), writing the new x_mean to ISV[TARGET_Q_BOUND_INDEX=131] +
/// Wiener state. No consumer wired yet — Mech 1's clamp keeps using
/// `10 × Q_ABS_REF.max(1.0)` until the SP4 consumer migration.
static SP4_TARGET_Q_P99_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/target_q_p99_kernel.cubin"));

/// SP4 Layer A Task A6 (2026-04-30): atom_pos_p99 × 4 branches cubin.
/// Single parameterised single-block kernel that reads ONE branch's slice
/// of `atom_positions_buf` (`[4 * num_atoms]`, branch `b` at offset
/// `b * num_atoms`, canonical layout per `atoms_update_kernel.cu`),
/// computes p99(|atom_positions[branch]|) via `sp4_histogram_p99<256>`,
/// writes the step observation to `producer_step_scratch_buf[scratch_idx]`.
/// The launcher (`GpuDqnTrainer::launch_sp4_atom_pos_p99_all_branches`)
/// loops `branch ∈ 0..4`, launching this kernel four times with distinct
/// `(branch_buf, scratch_idx ∈ 1..5)` pairs, then chains a single GPU
/// `apply_pearls_ad_kernel` launch (n_slots=4) on the same stream that
/// writes ISV[ATOM_POS_BOUND_BASE + branch ∈ {132, 133, 134, 135}] +
/// Wiener state in one device-side loop. 2026-05-01 GPU-Pearls refactor.
/// No consumer wired yet — Mech 2's atom-clamp still uses
/// `±10 × ISV[Q_ABS_REF=16].max(1.0)` (see `atoms_update_kernel.cu`).
static SP4_ATOM_POS_P99_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/atom_pos_p99_kernel.cubin"));

/// SP4 Layer A Task A7 (2026-04-30): Pearl B fused per-param-group oracle cubin.
/// Single-block kernel reads `(params, grads, adam_m, adam_v)` once and runs
/// 4-5 fused passes:
///   Pass A: WEIGHT_BOUND[group] = p99(|params|) via sp4_histogram_p99
///   Pass B: ADAM_M_BOUND[group] = p99(|adam_m|) via sp4_histogram_p99
///   Pass C: ADAM_V_BOUND[group] = p99(|adam_v|) via sp4_histogram_p99
///   Pass D: WD_RATE[group]      = |Σ w·g| / max(Σ w², EPS_DIV)
///   Pass E: L1_LAMBDA[trunk]    = (mean|g|/mean|w|) × entropy_deficit
///                                 (group 0 only, gated by l1_idx ≥ 0)
/// The launcher (`GpuDqnTrainer::launch_sp4_param_group_oracles_all_groups`)
/// loops `group ∈ 0..SP4_PARAM_GROUP_COUNT`, launching the kernel once per
/// group with the per-group `(params, grads, m, v, count, scratch slots)`
/// tuple, then chains the GPU `apply_pearls_ad_kernel` per output slot
/// (single-slot launches, 33 max — non-contiguous (scratch, ISV) pairs
/// per group prevent batching). 2026-05-01 GPU-Pearls refactor — same-
/// stream chaining, no host sync. Pearl B 4× memory-bandwidth reduction
/// vs four naive single-stat producer kernels: each buffer read once for
/// all 4-5 outputs. No consumer wired yet — Mech 9's clamp still uses
/// hardcoded `weight_clamp_max_abs` config args.
static SP4_PARAM_GROUP_ORACLE_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/param_group_oracle_kernel.cubin"));

/// SP4 Layer A Task A8 (2026-04-30): grad_norm_p99 producer cubin.
/// `grad_norm_buf` is a single-element f32 device buffer; p99 over 1 element
/// is the element itself, so the kernel reduces to a single-thread copy from
/// `grad_norm_buf[0]` into `producer_step_scratch_buf[scratch_idx]` with
/// `__threadfence_system()`. The launcher
/// (`GpuDqnTrainer::launch_sp4_grad_norm_p99`) chains the GPU
/// `apply_pearls_ad_kernel` on the same stream (2026-05-01 GPU-Pearls
/// refactor) which writes the new x_mean to ISV[GRAD_CLIP_BOUND_INDEX=168]
/// + Wiener state. No consumer wired yet — Mech 6's adaptive_clip upper-
/// bound consumer migrates in Layer B.
static SP4_GRAD_NORM_P99_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/grad_norm_p99_kernel.cubin"));

/// SP4 Layer A Task A9 (2026-04-30): h_s2_p99 producer cubin for
/// ISV[H_S2_BOUND=169]. Single-block, 256-thread kernel that reads
/// `save_h_s2 [B × SH2]` (the trunk-output activation surface populated each
/// step by the online forward), computes p99(|save_h_s2|) via the header-only
/// `sp4_histogram_p99<256>` device function, and writes the per-step
/// observation to `producer_step_scratch_buf[39]`. The launcher
/// (`GpuDqnTrainer::launch_sp4_h_s2_p99`) launches the GPU
/// `apply_pearls_ad_kernel` on the same stream to compute the new x_mean
/// and write back to ISV[H_S2_BOUND_INDEX=169] + Wiener state at offset
/// `(169-SP4_SLOT_BASE)*3 = 114`. NB: distinct from the existing
/// `h_s2_rms_ema_update` producer (slot 96) which tracks RMS — different
/// signal. Task A13 retrofitted that existing kernel to also use Pearls
/// A+D in a separate commit. No consumer wired yet — Mech 10's clamp
/// migrates in Layer B.
static SP4_H_S2_P99_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/h_s2_p99_kernel.cubin"));

/// SP4 Layer C #260 (2026-05-01): bw_d_h_s2_p99 producer cubin for
/// ISV[BW_D_H_S2_BOUND_INDEX=171]. Single-block, 256-thread kernel reading
/// `bw_d_h_s2 [B × SH2]` (the IQN trunk-backward d_h_s2 buffer post-DtoD
/// copy and post slot-35 NaN check) via `sp4_histogram_p99<256>`; output →
/// `producer_step_scratch_buf[69]`. The launcher
/// (`GpuDqnTrainer::launch_sp4_bw_d_h_s2_p99`) chains the GPU
/// `apply_pearls_ad_kernel` on the same stream which writes the new x_mean
/// to ISV[BW_D_H_S2_BOUND_INDEX=171] + Wiener state at offset
/// `(171 - SP4_SLOT_BASE) * 3 = 120`. Migrated from SP1-Phase-C
/// `1e6 × ISV[H_S2_RMS_EMA].max(1.0)` hardcoded multiplier per
/// `feedback_isv_for_adaptive_bounds`. Consumer at
/// `apply_iqn_trunk_gradient` (gpu_dqn_trainer.rs:~7615) reads
/// `ISV[171].max(EPS_CLAMP_FLOOR)` directly.
static SP4_BW_D_H_S2_P99_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/bw_d_h_s2_p99_kernel.cubin"));

/// SP4 Layer C #260 (2026-05-01): q_dir_grad_p99 producer cubin for
/// ISV[Q_DIR_GRAD_BOUND_INDEX=172]. Single-block, 256-thread kernel reading
/// the union of `d_value_logits` ∪ `d_adv_logits` via
/// `sp4_histogram_p99_multi<256>` with n_sub=2; output →
/// `producer_step_scratch_buf[70]`. The launcher
/// (`GpuDqnTrainer::launch_sp4_q_dir_grad_p99`) builds a 2-element
/// device-pointer table + 2-element count table in dedicated mapped-pinned
/// buffers (`q_dir_grad_subbuf_table_buf` / `q_dir_grad_subbuf_counts_buf`),
/// then chains the GPU `apply_pearls_ad_kernel` on the same stream which
/// writes the new x_mean to ISV[Q_DIR_GRAD_BOUND_INDEX] + Wiener state at
/// offset `(Q_DIR_GRAD_BOUND_INDEX - SP4_SLOT_BASE) * 3` floats. Migrated
/// from SP1-Phase-C `1e6 × ISV[Q_DIR_ABS_REF].max(1.0)` hardcoded
/// multiplier per `feedback_isv_for_adaptive_bounds`. Consumer at
/// `launch_cublas_backward_to` reads
/// `ISV[Q_DIR_GRAD_BOUND_INDEX].max(EPS_CLAMP_FLOOR)` directly.
static SP4_Q_DIR_GRAD_P99_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/q_dir_grad_p99_kernel.cubin"));

/// SP4 GPU-only Pearls A+D applicator cubin (2026-05-01). Loaded once by
/// `GpuDqnTrainer::new` and shared with the experience collector via the
/// `apply_pearls_ad_kernel_handle` accessor. Replaces the host-side
/// `apply_pearls_to_slot` helper per `feedback_no_cpu_compute_strict.md`;
/// the kernel's single-thread device-side loop runs on the producer's
/// stream, making the full Pearls A+D path graph-capture-compatible.
pub(crate) static APPLY_PEARLS_AD_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/apply_pearls_kernel.cubin"));

/// SP5 Task A1 (2026-05-01): q_branch_stats producer cubin.
/// Single-block 4-thread kernel reads `q_out_buf [B, total_actions=13]`
/// row-major (populated by `compute_expected_q` after each training step)
/// and computes per-branch (mean, max_abs_dev, var, entropy) for branches
/// Direction(4)/Magnitude(3)/Order(3)/Urgency(3).
/// Writes 16 floats to `producer_step_scratch_buf[71..87)` (4 per branch).
/// Shared signal source consumed by pearl_1_atom_kernel and the Pearl 3 / Pearl 2
/// consumers landing in later A-tasks. No atomicAdd (feedback_no_atomicadd —
/// single-thread per branch with independent writes).
static SP5_Q_BRANCH_STATS_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/q_branch_stats_kernel.cubin"));

/// SP5 Task A1 (2026-05-01): Pearl 1 atom-span controller cubin.
/// Single-block 4-thread kernel reads q_branch_stats scratch outputs and
/// per-branch `atoms_clip_count_buf [4 i32]`, computes per-branch
/// (v_center, v_half, headroom, clip_rate) via headroom controller
/// (TARGET_CLIP_RATE=0.01, HEADROOM_LR=0.01 — Invariant 1 anchors).
/// Writes 16 floats to `producer_step_scratch_buf[87..103)`.
/// The chained `apply_pearls_ad_kernel` (24 single-slot launches) then
/// smooths via Pearls A+D into ISV[174..190) + ISV[218..226).
/// Layer A only — no consumer reads these ISV slots yet.
static SP5_PEARL_1_ATOM_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/pearl_1_atom_kernel.cubin"));

/// SP5 Task A2 (2026-05-01): Pearl 3 per-branch NoisyNet sigma controller cubin.
/// Single-block 4-thread kernel. Reads ATOM_V_HALF[b] (Pearl 1, ISV[178..182))
/// + BRANCH_ENTROPY[b] (q_branch_stats, ISV[218..222)) + SIGMA_FRACTION[b]
/// (ISV[214..218), Pearl A sentinel 0 → bootstrap 0.1); applies entropy-deficit
/// controller (target=log(n_actions[b])×0.7, LR=0.005, envelope [0.001,0.5]);
/// writes (sigma[4], SF[4]) to producer_step_scratch_buf[103..111).
/// Chained apply_pearls_ad_kernel (8 single-slot launches) smooths via
/// Pearls A+D into ISV[210..218).
/// Layer A only — NoisyLinear consumer migration deferred to Layer B.
static SP5_PEARL_3_SIGMA_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/pearl_3_sigma_kernel.cubin"));

/// SP5 Task A3 + SP7 (2026-05-03): Pearl 2 IQN budget + flatness producer cubin.
/// Single-block 4-thread kernel. Reads Q_VAR_PER_BRANCH[b] (Pearl 1's
/// q_branch_stats output, ISV[222..226)) + NOISY_SIGMA[b] (Pearl 3 output,
/// ISV[210..214)). Computes per-branch flatness = clamp(var_q / (σ²+EPS_DIV),
/// 0, 1) and derives (budget_iqn, flatness) per branch. Writes 8 floats to
/// producer_step_scratch_buf[115..119, 127..131). Chained apply_pearls_ad_kernel
/// (8 single-slot launches) smooths via Pearls A+D into ISV[194..198, 206..210).
/// SP7: C51/CQL/ENS budget ownership transferred to loss_balance_controller_kernel.
/// Must run AFTER launch_sp5_pearl_1_atom (Q_VAR) AND
/// launch_sp5_pearl_3_sigma (NOISY_SIGMA).
static SP5_PEARL_2_BUDGET_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/pearl_2_budget_kernel.cubin"));

/// SP5 Task A4 (2026-05-01): auxiliary gradient cosine similarity cubin.
/// Single-block 8-thread kernel (one thread per SP4 param group). Reads
/// grad_buf + grad_prev_buf_per_group + group_param_offsets_buf [9 i32];
/// computes per-group cos_sim and l2_norm; writes to scratch_buf[131..147).
/// Also writes grad_curr → grad_prev_buf_writeback for next step's comparison.
/// No atomicAdd (feedback_no_atomicadd).
static SP5_GRAD_COSINE_SIM_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/grad_cosine_sim_kernel.cubin"));

/// SP5 Task A4 (2026-05-01): Pearl 4 per-group Adam β1/β2/ε cubin.
/// Single-block 8-thread kernel (one thread per param group). Reads
/// scratch_buf[131..147) (cosine_sim + l2_norm); computes β1/β2/ε within
/// structural envelopes (Invariant 1 anchors); writes to scratch_buf[147..171).
/// Chained apply_pearls_ad_kernel (24 launches, ALPHA_META=5e-4) smooths into
/// ISV[ADAM_BETA1_BASE..ADAM_EPS_BASE+8). No atomicAdd.
static SP5_PEARL_4_ADAM_HPARAMS_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/pearl_4_adam_hparams_kernel.cubin"));

/// SP5 Task A5 (2026-05-01): q_skew_kurtosis auxiliary cubin.
/// Single-block 4-thread kernel (one per branch). Reads save_q_online [B × 13]
/// row-major; computes per-branch (skew, excess kurtosis) via numerically-stable
/// two-pass central moments; writes 8 floats to scratch_buf[171..179)
/// (skew[4] at [171..175), ex_kurt[4] at [175..179)).
/// EPS_DIV=1e-12 Invariant 1 anchor (m2^1.5 / m2^2 denominator stability).
/// Envelope [-3,+3] / [-3,+30]. No atomicAdd (feedback_no_atomicadd).
static SP5_Q_SKEW_KURTOSIS_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/q_skew_kurtosis_kernel.cubin"));

/// SP5 Task A5 (2026-05-01): Pearl 5 per-branch IQN τ schedule cubin.
/// Single-block 4-thread kernel (one per branch). Reads per-branch skew from
/// scratch_buf[skew_idx_base..skew_idx_base+4) (q_skew_kurtosis output).
/// Computes 5-quantile τ schedule per branch: symmetric default
/// {0.05, 0.25, 0.5, 0.75, 0.95} shifted by skew × SKEW_SHIFT=0.05
/// (Invariant 1 anchor), clamped to structural envelope [0.01, 0.99].
/// Writes 20 floats to scratch_buf[179..199) (branch b, quantile q at
/// 179 + b*5 + q). Chained apply_pearls_ad_kernel (20 launches, ALPHA_META=1e-3)
/// smooths into ISV[IQN_TAU_BASE=250..270). No atomicAdd.
static SP5_PEARL_5_IQN_TAU_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/pearl_5_iqn_tau_kernel.cubin"));

/// SP5 Task A6 (2026-05-01): Pearl 6 cross-fold-persistent Kelly cap signals.
/// Single-block 6-thread kernel (one per output slot). Reads portfolio_state
/// [n_envs * PS_STRIDE] and current ISV[280..286); writes directly to ISV.
/// CROSS-FOLD-PERSISTENT: ISV[280..286) are EXEMPT from the fold-reset registry.
/// Does NOT use apply_pearls_ad_kernel: sentinel-bootstrap is incompatible with
/// cross-fold persistence (resetting to first-observation defeats the design).
/// In-kernel EWMA alpha=0.01 (Invariant 1 anchor). Cumulative-via-max() for
/// sample_count slot preserves cumulative trade count across window/fold resets.
/// Wiener offsets [525..543) that would naively map to these slots are unused.
static SP5_PEARL_6_KELLY_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/pearl_6_kelly_kernel.cubin"));

/// SP5 Task A7 (2026-05-01): Pearl 8 per-direction trail-stop distance.
/// 4-thread single-block kernel (Short=0, Hold=1, Long=2, Flat=3).
/// Reads features[bar_idx * market_dim + 9] (ATR_NORM column), denormalizes
/// via the canonical fxcache scheme (atr_abs = max(0.01, exp(atr_norm*16-7))).
/// Short and Long: trail_dist = atr_abs × ATR_TRAIL_FACTOR=2.0.
/// Hold and Flat: trail_dist = EPS_CLAMP_FLOOR=1.0.
/// Layer A simplification: Short and Long share the same ATR (same current
/// bar for both). Direction-conditional ATR deferred to Layer C.
/// Features dev pointer and bar_idx passed as args (not stored in trainer).
/// Followed by 4 apply_pearls_ad_kernel calls → ISV[TRAIL_DIST_PER_DIR_BASE=270..274).
static SP5_PEARL_8_TRAIL_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/pearl_8_trail_kernel.cubin"));

/// SP5 Task A8 (2026-05-01): Pearl 1-ext per-branch C51 num_atoms from per-branch v_half.
/// 4-thread single-block kernel (one per branch). Reads ISV[ATOM_V_HALF_BASE=178..182)
/// (populated by Pearl 1 in Task A1). Threshold cascade: v_half < 0.1 → 64 atoms (narrow
/// Q range, high resolution), v_half < 1.0 → 32 atoms (moderate), v_half ≥ 1.0 → 16 atoms
/// (wide Q range, modest resolution). Atom counts are powers of 2; total cap 4×64=256 atoms
/// is within the HW shared-memory budget the C51 backward kernel uses (Invariant 1 anchors).
/// Output smoothed by Pearls A+D; Layer B's atoms_update consumer rounds at consume time.
/// Writes 4 floats to scratch_buf[SCRATCH_PEARL_1_EXT_NUM_ATOMS=203..207).
/// Followed by 4 apply_pearls_ad_kernel calls → ISV[ATOM_NUM_ATOMS_BASE=274..278).
static SP5_PEARL_1_EXT_NUM_ATOMS_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/pearl_1_ext_num_atoms_kernel.cubin"));

/// SP5 Layer D Task D1 (rewrite, 2026-05-02): PnL aggregation producer kernel.
/// Single-block 256-thread shmem-reduce reads per-bar arrays (`step_returns`,
/// `done_flags`) plus already-reduced per-trade summary scalars (`n_trades`,
/// `sum_returns`, `sum_sq_returns`, `initial_capital`) and writes 4 aggregated
/// floats — log-space compounded total / per-trade mean / per-trade variance /
/// per-bar max drawdown with episode-boundary resets — to
/// scratch_buf[SCRATCH_PNL_AGG_BASE=207..211). Followed by 4
/// apply_pearls_ad_kernel calls → ISV[PNL_TOTAL_INDEX=286..290).
/// Reproduces `compute_epoch_financials` (financials.rs) bit-for-bit per
/// `feedback_no_cpu_compute_strict.md`. The original D1 kernel (commit
/// `5ee795f14`) was rewritten — it had been authored against per-trade arrays
/// that don't exist in production; the previous D4 agent surfaced that
/// blocker before any code changes. Additive in this commit — D4 wires the
/// consumer in the atomic Layer D refactor per `feedback_no_partial_refactor.md`.
static SP5_PNL_AGGREGATION_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/pnl_aggregation_kernel.cubin"));

/// SP5 Layer D Task D2 (2026-05-02): LearningHealth composition producer.
/// Single-block 1-thread arithmetic kernel takes 7 raw inputs by value
/// (q_gap, q_var, atom_util, grad_norm, ens_disagreement, grad_consistency,
/// spectral_gap), normalises each via smoothstep curves matching
/// `learning_health.rs::NormalizedComponents::from_raw` exactly, and
/// composes them into a scalar `health_score` via the spec Layer-1
/// weighted sum (`compose()`). Writes 4 scratch floats — composed score
/// + 3 normalised intermediates (q_gap_norm, q_var_norm,
/// grad_norm_norm = `grad_stable`) — to
/// scratch_buf[SCRATCH_HEALTH_COMP_BASE=211..215). Followed by 4
/// apply_pearls_ad_kernel calls → ISV[HEALTH_SCORE_INDEX=290..294).
/// Replaces the host-side composition in `learning_health.rs` invoked from
/// `training_loop.rs:2658-2745` per `feedback_no_cpu_compute_strict.md`.
/// Additive in this commit — D4 wires the consumer in the atomic
/// Layer D refactor per `feedback_no_partial_refactor.md`.
static SP5_HEALTH_COMPOSITION_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/health_composition_kernel.cubin"));

/// SP5 Layer D Task D3 (2026-05-02): Training metrics EMA producer cubin.
/// Single-block 3-thread arithmetic kernel reproducing the host-side
/// `training_sharpe_ema` (adaptive α), `max_dd_ema` (fixed α=0.1), and
/// `low_dd_ratio` (fixed α=0.15) updates at `training_loop.rs:5039-5113`
/// per `feedback_no_cpu_compute_strict.md`. Per-metric thread layout:
/// tid=0 sharpe (adaptive α with host-tracked `_initialized` sentinel),
/// tid=1 max_dd (fixed α=0.1 with prev<1e-12 sentinel), tid=2 low_dd_ratio
/// (fixed α=0.15 over a binary `is_low` derived from tid=1's *new*
/// max_dd_ema via __syncthreads()). Writes 3 floats to
/// scratch_buf[SCRATCH_TRAINING_METRICS_EMA_BASE=215..218). Followed by 3
/// apply_pearls_ad_kernel calls → ISV[TRAINING_SHARPE_EMA_INDEX=294..297).
/// Replaces the host-side updates that D4 atomically migrates per
/// `feedback_no_partial_refactor.md`. Additive in this commit.
static SP5_TRAINING_METRICS_EMA_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/training_metrics_ema_kernel.cubin"));

/// SP7 Task 5 (2026-05-03): loss-balance controller cubin.
/// 8-thread single-block kernel (2 loss heads × 4 branches). Reads three
/// 3-float views into `grad_decomp_result_dev_ptr` (IQN at element offset 0,
/// CQL_SX at offset 6, C51 at offset 9), FLATNESS_BASE, prior budgets, and
/// prior Wiener state from ISV; writes new_budget_cql[4] + new_budget_c51[4]
/// + diff_var/sample_var for each (6 groups × 4 branches = 24 floats) to
/// `producer_step_scratch_buf[218..242)`. Cold-start sentinel-aware.
/// Followed by 24 `apply_pearls_ad_kernel` launches → ISV[BUDGET_CQL_BASE,
/// BUDGET_C51_BASE, LB_DIFF_VAR_CQL_BASE, LB_SAMPLE_VAR_CQL_BASE,
/// LB_DIFF_VAR_C51_BASE, LB_SAMPLE_VAR_C51_BASE].
/// Producer-only — call site wires at T7 per `feedback_no_partial_refactor`.
static LOSS_BALANCE_CONTROLLER_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/loss_balance_controller_kernel.cubin"));
/// SP8 (Fix 36, 2026-05-03): GPU-only train_active_frac canary producer.
/// Single block, single thread; reads `monitoring_summary[5..17)` and writes
/// scratch[SCRATCH_TRAIN_ACTIVE_FRAC]. Downstream apply_pearls_ad_kernel
/// smooths into ISV[TRAIN_ACTIVE_FRAC_INDEX=321]. Replaces the host-side
/// computation at `training_loop.rs:3392-3396` per
/// `feedback_no_cpu_compute_strict.md`.
static TRAIN_ACTIVE_FRAC_COMPUTE_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/train_active_frac_compute_kernel.cubin"));
/// SP8 (Fix 36, 2026-05-03): ISV-driven adaptive MAX_BUDGET producer.
/// Single block, 8 threads (2 heads × 4 branches); reads
/// ISV[TRAIN_ACTIVE_FRAC_INDEX] and writes scratch[SCRATCH_LB_MAX_BUDGET_*].
/// Downstream apply_pearls_ad_kernel smooths into
/// ISV[LB_MAX_BUDGET_{CQL,C51}_BASE..+4). Replaces hardcoded
/// `MAX_BUDGET=1.0f` constant in `loss_balance_controller_kernel.cu` per
/// `pearl_controller_anchors_isv_driven.md`.
static LOSS_BALANCE_MAX_BUDGET_COMPUTE_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/loss_balance_max_budget_compute_kernel.cubin"));

/// SP9 (Fix 37, 2026-05-03): GPU producer for eval-side magnitude intent
/// distribution. Single block, 3 threads. Replaces the host-side
/// `read_eval_intent_magnitude_distribution()` DtoH path at
/// `gpu_backtest_evaluator.rs:1353` per `feedback_no_cpu_compute_strict.md`.
/// Downstream apply_pearls_ad_kernel (×3) smooths into
/// ISV[EVAL_DIST_Q/H/F_INDEX=336..339).
static SP9_EVAL_INTENT_DIST_COMPUTE_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/eval_intent_dist_compute_kernel.cubin"));

/// SP9 (Fix 37, 2026-05-03): intent vs eval Full-magnitude divergence
/// producer. Single block, 1 thread; reads `monitoring_summary[5..17)`
/// and `ISV[EVAL_DIST_F_INDEX]`; writes `intent_f / eval_f` to scratch.
/// Downstream apply_pearls_ad_kernel smooths into
/// ISV[INTENT_EVAL_DIVERGENCE_INDEX=332].
static SP9_INTENT_EVAL_DIVERGENCE_COMPUTE_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/intent_eval_divergence_compute_kernel.cubin"));

/// SP9 (Fix 37, 2026-05-03): EMA baseline of magnitude-branch Q-variance
/// for the self-relative `base_floor` ratio in the warmup-floor formula.
/// Single block, 1 thread; reads `ISV[FLATNESS_BASE+1=207]` and writes to
/// scratch. Downstream apply_pearls_ad_kernel smooths into
/// ISV[Q_VAR_MAG_EMA_INDEX=331].
static SP9_Q_VAR_MAG_EMA_COMPUTE_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/q_var_mag_ema_compute_kernel.cubin"));

/// SP9 (Fix 37, 2026-05-03): main Kelly cold-start warmup-floor producer.
/// Single block, 1 thread; reads 8 ISV inputs (q_var_mag, q_var_mag_ema,
/// kelly_sample_count, sample_count_target, intent_eval_divergence,
/// divergence_target, epoch_idx, temporal_target) and computes
///   base_floor × (1 − max(statistical_conf, behavioral_conf, temporal_conf))
/// per `pearl_cold_start_exit_signal_or.md`. Downstream apply_pearls_ad_kernel
/// smooths into ISV[KELLY_WARMUP_FLOOR_INDEX=330] which is consumed in
/// `unified_env_step_core` (`trade_physics.cuh`) as an adaptive max-floor
/// on the Kelly cap.
static SP9_KELLY_WARMUP_FLOOR_COMPUTE_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/kelly_warmup_floor_compute_kernel.cubin"));

/// SP11 Fix 39 (2026-05-04, Task A1): val-sharpe Δ + variance canary
/// producer. Single block, single thread; reads a 2-element mapped-pinned
/// `val_sharpe_history` ([prev_epoch, curr_epoch]) plus
/// ISV[VAL_SHARPE_DELTA_EMA_INDEX]; writes `(curr-prev)` and
/// `(delta - prev_delta_ema)^2` to scratch[SCRATCH_SP11_VAL_SHARPE_DELTA_BASE..+2).
/// Downstream `apply_pearls_ad_kernel` (n_slots=2) smooths into
/// ISV[VAL_SHARPE_DELTA_EMA_INDEX=350, VAL_SHARPE_VAR_EMA_INDEX=351].
static SP11_VAL_SHARPE_DELTA_COMPUTE_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/val_sharpe_delta_compute_kernel.cubin"));

/// SP11 Fix 39 (2026-05-04, Task A1): per-bar saboteur engagement canary
/// producer. Single block, 256 threads + grid-stride; reads
/// `saboteur_delta_reward_buf` populated by `experience_env_step` at the
/// saboteur perturbation site, computes per-bar engagement
/// (`|delta| > 0.01 × ISV[PNL_REWARD_MAGNITUDE_EMA_INDEX]`), block tree-
/// reduces to a fraction. Downstream `apply_pearls_ad_kernel` (n_slots=1)
/// smooths into ISV[SABOTEUR_ENGAGEMENT_RATE_INDEX=358].
static SP11_SABOTEUR_ENGAGEMENT_COMPUTE_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/saboteur_engagement_compute_kernel.cubin"));

/// SP11 Fix 39 (2026-05-04, Task A1): per-component reward-magnitude-ratio
/// canary producer. Single block, 6 threads; reads 6 contiguous component
/// magnitude EMAs from ISV[REWARD_POPART_EMA_INDEX..+6) (populated by
/// SP4's reward_component_ema_kernel), normalises to ratios, and mirrors
/// the popart magnitude into scratch[6] as a side-output. Downstream
/// `apply_pearls_ad_kernel` smooths the 6 ratios into
/// ISV[REWARD_COMPONENT_MAG_RATIO_BASE..+6) and the singleton mirror
/// into ISV[PNL_REWARD_MAGNITUDE_EMA_INDEX=359].
static SP11_MAG_RATIO_COMPUTE_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/reward_component_mag_ratio_compute_kernel.cubin"));

/// SP11 Fix 39 (2026-05-04, Task A2): reward-subsystem controller. Single
/// block, 10 threads. Reads 5 canary ISV slots [350..360) (val-sharpe Δ +
/// variance, 6 mag-ratios, saboteur engagement, PnL magnitude EMA) and
/// writes 10 outputs to `scratch[SCRATCH_SP11_CONTROLLER_BASE..+10)`.
/// Downstream `apply_pearls_ad_kernel` (Pearls A+D output smoothing per
/// spec §3.4.1, n_slots=10) smooths the controller outputs into the
/// 10 contiguous component weight + controller scalar slots at
/// ISV[340..350) — 6 weights + curiosity_pressure + saboteur_mult +
/// weight_floor + curiosity_bound.
static SP11_REWARD_SUBSYSTEM_CONTROLLER_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/reward_subsystem_controller_kernel.cubin"));

/// SP11 Fix 39 B1b fix-up (2026-05-04): popart-component-specific
/// magnitude EMA producer. Single block, BLOCK_SIZE=256; reads the
/// per-bar `r_popart` mapped-pinned buffer populated by
/// `experience_env_step` and block-tree-reduces mean(|x|) into
/// `scratch[SCRATCH_SP11_POPART_COMPONENT_EMA]`. Downstream
/// `apply_pearls_ad_kernel` (n_slots=1) smooths into
/// ISV[POPART_COMPONENT_MAG_EMA_INDEX=360]. Resolves the slot-63
/// PopArt-input vs popart-component-magnitude overload that B1b smoke
/// surfaced. Spec §4 amendment "Why slot 360".
static SP11_POPART_COMPONENT_EMA_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/popart_component_ema_kernel.cubin"));

/// SP11 Fix 39 (2026-05-04, Task A2): one-shot GPU init for the SimHash
/// projection matrix. Philox-driven (deterministic from a host-passed seed
/// derived from the SP11 salt) writes ±1 values into a 42×16 = 672-element
/// f32 buffer. Per `feedback_no_cpu_forwards.md` ("CPU is read-only"), the
/// projection matrix MUST be populated by a GPU kernel. Launched ONCE at
/// trainer construct; never re-initialised; never written from host. The
/// matrix outlives every fold (it's the hash function, not state) so it
/// has NO entry in the fold-reset registry.
static SP11_NOVELTY_SIMHASH_PROJ_INIT_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/novelty_simhash_proj_init_kernel.cubin"));

/// SP13 v3 Phase 0a P0a.T3 (2026-05-04): Hold-rate observer reducer.
///
/// Single-block tree-reduce kernel that decodes per-bar packed factored
/// `batch_actions[i]` (`dir = action_idx / (NUM_MAGNITUDES * NUM_ORD * NUM_URG)`)
/// and writes the resulting fraction `count(Hold) / batch_size ∈ [0, 1]`
/// to a 1-element mapped-pinned scratch. The chained
/// `apply_fixed_alpha_ema_kernel` (α=0.05, sentinel 0.0) blends the
/// per-step observation into `ISV[HOLD_RATE_OBSERVED_EMA_INDEX=382]`,
/// which feeds the SP16-P1 MIN_HOLD_TEMPERATURE producer at slot 460.
///
/// Loaded by `GpuExperienceCollector` (per-step on the action-select
/// stream) — the per-step training-time observation is collector-owned
/// because `batch_actions` lives there. The trainer carries no copy
/// (avoids a dead `CudaFunction` field per `feedback_no_stubs.md`); the
/// `pub(crate)` visibility lets the collector's constructor reach the
/// cubin without re-declaring it. See `hold_rate_observer_kernel.cu`
/// for kernel contract details.
pub(crate) static SP13_HOLD_RATE_OBSERVER_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/hold_rate_observer_kernel.cubin"));

/// SP13 Phase 0a (2026-05-04): aux-head directional-accuracy reducer.
/// Single-block tree-reduce kernel that scores the aux next-bar
/// regression head's per-bar prediction sign against the next-bar return
/// label sign and writes (dir_acc, pos_pred_frac, pos_label_frac) into a
/// 3-element mapped-pinned buffer. Consumed by
/// `GpuDqnTrainer::launch_aux_dir_acc_reduce` (the per-step launcher
/// wired by P0a.T4 — this static include lands the cubin handle for the
/// constructor in P0a.T2). Produces directional-accuracy signal feeding
/// the SP13 EMAs at ISV[373..375) (short/long EMAs, sentinel 0.5 per
/// `pearl_first_observation_bootstrap`). See
/// `aux_dir_acc_reduce_kernel.cu` for kernel contract details.
pub(crate) static SP13_AUX_DIR_ACC_REDUCE_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/aux_dir_acc_reduce_kernel.cubin"));

/// SP13 Phase 0a P0a.T4 (2026-05-04): fixed-α EMA applicator. Sibling
/// of `apply_pearls_kernel` for cases where Pearls A+D's Wiener-optimal
/// blend would collapse two EMAs of the same signal to identical
/// values (defeating the SP13 stagnation detector that compares
/// slot 373's α=0.3 EMA against slot 374's α=0.05 EMA). Per-thread
/// first-observation-sentinel bootstrap + fixed-α blend; caller passes
/// the slot's registry sentinel (0.5 for dir-acc, 0.0 for hold-rate).
/// Consumed by `GpuDqnTrainer::launch_apply_fixed_alpha_ema`. See
/// `apply_fixed_alpha_ema_kernel.cu` for kernel contract details.
pub(crate) static SP13_APPLY_FIXED_ALPHA_EMA_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/apply_fixed_alpha_ema_kernel.cubin"));

/// SP13 Phase 0a P0a.T4 (2026-05-04): aux-head per-bar prediction →
/// ISV[AUX_DIR_PREDICTION_INDEX=375] bounded scalar producer.
/// SP13 B1.1a (2026-05-05) rewrite: reads the K=2 softmax tile
/// (`aux_nb_softmax_buf [B, 2]`) and writes
/// `mean(softmax[:, 1] - softmax[:, 0])` ∈ [-1, +1] to the SHARED
/// ISV slot 375 (per-step overwrite — not an EMA). The bound is
/// structural (softmax components in [0, 1] sum to 1 ⇒ pairwise diff
/// in [-1, +1]) per `pearl_bounded_modifier_outputs_require_structural_activation`;
/// the runtime tanh squash retired. Cubin filename and ISV slot name
/// preserved for layout stability. Consumed by
/// `GpuDqnTrainer::launch_aux_pred_to_isv_tanh`. See
/// `aux_pred_to_isv_tanh_kernel.cu` for kernel contract details.
pub(crate) static SP13_AUX_PRED_TO_ISV_TANH_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/aux_pred_to_isv_tanh_kernel.cubin"));

/// SP14 B.3 (2026-05-05): per-step Q-head ↔ aux argmax disagreement EMA
/// producer. Reads online Q logits + aux softmax outputs; computes per-step
/// argmax-disagreement bool; updates a Wiener-optimal EMA into ISV[388].
/// Loaded from `q_disagreement_update_kernel.cubin`.
pub(crate) static SP14_Q_DISAGREEMENT_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/q_disagreement_update_kernel.cubin"));

/// SP22 H6 (2026-05-12): per-env p_up extractor. Copies
/// `aux_softmax[env, 1]` → `prev_aux_dir_prob[env]` after each aux forward
/// during rollout. The cache buffer is read at the next step's state
/// assembly to populate `state[AUX_DIR_PROB_INDEX = SL_PADDING_START + 0]`,
/// wiring the aux head's directional probability into policy STATE.
/// Loaded from `aux_softmax_to_per_env_kernel.cubin`. Consumed by the
/// collector (`GpuExperienceCollector::collect_experiences_gpu`); no
/// trainer-side consumer (this is a rollout-time bridge, not a training-
/// step kernel).
pub(crate) static SP22_AUX_SOFTMAX_TO_PER_ENV_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/aux_softmax_to_per_env_kernel.cubin"));

/// SP22 H6 Phase 3 α structural-prior init (atom-shift design 2026-05-13).
/// One-time GPU init kernel that writes the per-action prior
/// `[-0.5, 0.0, +0.5, 0.0]` for `[Short, Hold, Long, Flat]` into the
/// Adam-trained `w_aux_to_q_dir [4]` buffer at trainer construction.
/// NOT invoked inside any captured graph — pure init-time launch.
/// Loaded from `aux_w_prior_init_kernel.cubin`.
pub(crate) static SP22_AUX_W_PRIOR_INIT_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/aux_w_prior_init_kernel.cubin"));

/// SP22 H6 Phase 3 α Step 8 (2026-05-13): backward dW kernel for
/// `w_aux_to_q_dir [4]`. Reads scratch buffers from c51_loss_kernel
/// forward (`aux_target_a_dir` + `aux_proj_logdiff_dir`) and computes
/// per-action gradient via block tree-reduce (grid=(4,1,1),
/// block=(256,1,1)). Launched OUTSIDE captured graphs.
pub(crate) static SP22_C51_AUX_DW_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/c51_aux_dw_kernel.cubin"));

/// SP22 H6 Phase 3 α Step 11 (2026-05-13): Adam update kernel for
/// `w_aux_to_q_dir [4]`. Standard Adam with bias correction. 4 threads
/// single block. Launched once per training step OUTSIDE captured
/// graphs.
pub(crate) static SP22_ADAM_W_AUX_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/adam_w_aux_kernel.cubin"));

// SP22 H6 Phase 3 α SCALAR-BIAS DESIGN — DELETED 2026-05-13.
// The scalar-bias `Q_dir[b, a] += W[a] * state_121[b]` approach is
// mathematically ineffective in C51 distributional Q-learning (softmax-
// shift-invariance → W gradient = 0 → never trains). Replaced by the
// per-(b, a) atom-position shift design that threads `aux_atom_shift[b, a]
// = W[a] * state_121[b]` through compute_expected_q + c51_loss_kernel +
// c51_grad_kernel + mag_concat_qdir + (investigate quantile_q_select /
// iqn_dual_head). The dW + dstate gradient computations integrate into
// c51_grad_kernel's projection backward — no separate α forward/backward
// kernel needed. See audit doc Phase 3c entry + spec doc α section
// for the revised design.
//
// The .cu files `aux_to_q_dir_bias_kernel.cu` and
// `aux_to_q_dir_bias_backward_kernel.cu` are removed from build.rs
// registration in the same commit. The Adam-trained W buffer
// (`w_aux_to_q_dir`) + Adam moments + dW accumulator remain — they
// still hold the per-action W vector used by the atom-shift threading.

/// SP17 Task PP.3 (2026-05-08): pre-centering V/A mean diagnostic.
/// Single-block tree-reduce kernel reading the existing `on_v_logits_buf
/// [B, NA]` and `on_b_logits_buf [B, (B0+B1+B2+B3)*NA]` (BRANCH-MAJOR) and
/// emitting `[E[V], E[A_dir], E[A_mag], E[A_ord], E[A_urg]]` into a
/// `MappedF32Buffer<5>`. Cold-path (epoch boundary) launcher
/// `read_v_a_means_pre_centering()` is the single consumer; it surfaces the
/// V/A breakdown PRE-CENTERING via per-epoch HEALTH_DIAG so the SP17 Phase 0
/// kill criterion can observe whether V dominates the Q-magnitude budget.
/// Loaded from `v_a_means_diag_kernel.cubin`.
pub(crate) static SP17_V_A_MEANS_DIAG_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/v_a_means_diag_kernel.cubin"));

/// SP17 Phase 3.1 (2026-05-08): per-branch EMA of the over-actions
/// variance of `A_centered`. 4 blocks × 256 threads (one block per
/// branch). Cold-path (epoch boundary) producer for the HEALTH_DIAG
/// `dueling [a_var=...]` line; signals dueling-Q identifiability health
/// (Var → 0 = `all actions equivalent, V dominates` = pre-SP17 pathology
/// re-emerging). Block tree-reduce per `feedback_no_atomicadd`. Pearl-A
/// first-observation bootstrap on slots 474..478. Loaded from
/// `sp17_a_var_ema_kernel.cubin`. Consumed by
/// `read_a_var_ema_per_branch()`.
pub(crate) static SP17_A_VAR_EMA_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/sp17_a_var_ema_kernel.cubin"));

/// SP17 Phase 3.2 (2026-05-08): per-branch V_share EMA. 4 blocks × 256
/// threads. Cold-path producer for HEALTH_DIAG `dueling [v_share=...]`.
/// V_share = |E[V]| / (|E[V]| + |E[A_centered, picked]|) per branch,
/// picked = argmax_a Σ_z A_raw[i, a, z] (max-Q semantic). Pearl-A
/// bootstrap (sentinel 0.5) + α=WELFORD_ALPHA_MIN=0.4 + bilateral [0, 1]
/// clamp. Block tree-reduce per `feedback_no_atomicadd`. Loaded from
/// `sp17_v_share_kernel.cubin`. Consumed by
/// `read_v_share_per_branch()`.
pub(crate) static SP17_V_SHARE_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/sp17_v_share_kernel.cubin"));

/// SP17 Phase 3.2 (2026-05-08): adaptive advantage_clip_bound producer.
/// Single block × 256 threads. Cold-path producer for HEALTH_DIAG
/// `dueling [clip=K]`. Computes p99(|A_centered|) × 1.5 over the full
/// flat batch via `sp4_histogram_p99` (no atomicAdd per
/// `pearl_fused_per_group_statistics_oracle`); EMA-blends with α=0.01
/// + bilateral clamp [0.1, 100.0] per `pearl_symmetric_clamp_audit`.
/// Pearl-A bootstrap on sentinel 1.0. Observability-only in Phase 3 —
/// the actual clip wire-up is Phase 5 follow-up. Loaded from
/// `sp17_advantage_clip_bound_kernel.cubin`. Consumed by
/// `read_advantage_clip_bound()`.
pub(crate) static SP17_ADVANTAGE_CLIP_BOUND_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/sp17_advantage_clip_bound_kernel.cubin"));

// SP14 Layer C Phase C.1 (2026-05-08): α-machinery cubin statics deleted
// atomically with the alpha_grad_compute / gradient_hack_detect /
// sp14_scale_wire_col kernel files. SP14_ALPHA_GRAD_CUBIN,
// SP14_GRAD_HACK_CUBIN, SP14_SCALE_WIRE_COL_CUBIN removed per
// `feedback_no_legacy_aliases`. Coupling A (forward feature wire
// `dir_concat_qaux_kernel`) is preserved — its cubin static stays.

/// SP20 Phase 1.1 (2026-05-09): aux_logits p50 + std producer cubin.
/// Component 5 / Kernel 3 of the SP20 fused-producer chain. Single-
/// block, BLOCK=256 kernel reading `aux_logits [B, K=2]` and emitting
/// `[p50, std]` of the per-row peak-confidence signal `aux_conf =
/// max_c softmax(logits) - 1/2`. Loaded by the experience collector
/// for the per-rollout-step SP20 chain (Phase 1.4).
pub(crate) static SP20_STATS_COMPUTE_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/sp20_stats_compute_kernel.cubin"));

/// SP20 Phase 1.2 (2026-05-09): 8 Wiener-α EMA producer cubin.
/// Component 5 / Kernel 1 of the SP20 fused-producer chain. Single-
/// block, single-thread kernel updating 4 ISV slots + 4 internal
/// scratch slots from a device `SP20EmaInputs` struct (post Path C
/// kernel-arg refactor, 2026-05-09). Loaded by the experience
/// collector for the per-rollout-step SP20 chain (Phase 1.4).
pub(crate) static SP20_EMAS_COMPUTE_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/sp20_emas_compute_kernel.cubin"));

/// SP20 Phase 1.3 (2026-05-09): 6 derived ISV controller outputs
/// producer cubin. Component 5 / Kernel 2 of the SP20 fused-producer
/// chain. Single-block, single-thread kernel reading the EMAs Kernel
/// 1 wrote and deriving LOSS_CAP / N_STEP / AUX_CONF_THRESHOLD /
/// AUX_GATE_TEMP / TARGET_HOLD_PCT / HOLD_COST_SCALE per spec §4.5.
/// Loaded by the experience collector for the per-rollout-step SP20
/// chain (Phase 1.4).
pub(crate) static SP20_CONTROLLERS_COMPUTE_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/sp20_controllers_compute_kernel.cubin"));

/// SP20 Phase 1.4 (2026-05-09, Path C): per-env → SP20EmaInputs
/// aggregation kernel cubin. Bridges the experience collector's
/// per-sample `[N, L]` GPU buffers (`trade_close_per_sample`,
/// `step_ret_per_sample`, `hold_at_exit_per_sample`, `actions_out`)
/// to the EMA kernel's struct-input contract. Single-block, BLOCK=64
/// tree-reduce per `feedback_no_atomicadd`.
pub(crate) static SP20_AGGREGATE_INPUTS_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/sp20_aggregate_inputs_kernel.cubin"));

/// SP14 B.6 (2026-05-05): pre-SGEMM concat kernel for direction Q-head input.
/// Reads `h_s2 [B, SH2]` + `aux_softmax_diff [1]` and writes
/// `[h_s2 | softmax_diff] [B, SH2 + 1]` into `dir_qaux_concat_scratch`.
/// Coupling A (forward feature wire) — preserved through SP14 Layer C
/// per the plan: aux's directional softmax-diff continues to feed Q-head's
/// x_concat as a forward feature.
/// Loaded from `dir_concat_qaux_kernel.cubin`.
static SP14_DIR_CONCAT_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/dir_concat_qaux_kernel.cubin"));

/// SP15 Phase 1.1 (2026-05-06): unified per-bar sharpe. Single-block,
/// BLOCK=256 kernel computing mean/std/sharpe via 2-pass shared-memory
/// tree-reduce — no atomicAdd. Used by both train and val paths with
/// identical formula and identical window definition. Replaces the
/// SP14-era sharpe_ema (per-batch EMA) vs sharpe_annualised
/// (val × sqrt(525600)) split. Annualisation is the host-side caller's
/// responsibility (sharpe_annualised = out[2] × sqrt(N_bars_per_year)).
pub static SP15_SHARPE_PER_BAR_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/sharpe_per_bar_kernel.cubin"));

/// SP15 Phase 1.1 (2026-05-06): launcher for the unified per-bar sharpe
/// kernel. Free function (not a method on `GpuDqnTrainer`) so unit/oracle
/// tests can drive the kernel directly without the trainer struct;
/// production callers (train + val sharpe sites) invoke the same launcher.
///
/// Loads the cubin per-call: matches the precedent established by
/// `sp4_histogram_p99_test_kernel`'s standalone test wrapper, which is the
/// closest analog (tiny single-block kernel, infrequent caller). If a
/// hot-path caller emerges later it should cache the `CudaFunction` once
/// at construct time.
///
/// `pnl` and `out` are raw `CUdeviceptr`s — typically the `dev_ptr` field
/// of a `MappedF32Buffer`, but any device-resident f32 storage works.
/// `out` MUST point to at least 3 f32 slots: `[mean, std, sharpe]`.
pub fn launch_sp15_sharpe_per_bar(
    stream: &Arc<CudaStream>,
    pnl: cudarc::driver::sys::CUdeviceptr,
    n: i32,
    out: cudarc::driver::sys::CUdeviceptr,
) -> Result<(), MLError> {
    let module = stream
        .context()
        .load_cubin(SP15_SHARPE_PER_BAR_CUBIN.to_vec())
        .map_err(|e| MLError::ModelError(format!(
            "load sp15_sharpe_per_bar cubin: {e}"
        )))?;
    let kernel = module
        .load_function("sharpe_per_bar_kernel")
        .map_err(|e| MLError::ModelError(format!(
            "load sharpe_per_bar_kernel function: {e}"
        )))?;
    unsafe {
        stream
            .launch_builder(&kernel)
            .arg(&pnl)
            .arg(&n)
            .arg(&out)
            .launch(LaunchConfig {
                grid_dim: (1, 1, 1),
                block_dim: (256, 1, 1),
                shared_mem_bytes: 0,
            })
            .map_err(|e| MLError::ModelError(format!(
                "launch sharpe_per_bar_kernel: {e}"
            )))?;
    }
    Ok(())
}

/// SP15 Phase 1.2 (2026-05-06): cost-net sharpe cubin. Per spec §6.2
/// per-side cost semantics: commission_per_rt charged at close (rt_ind=1);
/// half-spread × |pos| charged at entry AND exit on side_ind events (sums
/// to one full spread per round-trip); ofi_lambda × |pos| × |ofi| same
/// per-side. Reads ofi_lambda from ISV[OFI_IMPACT_LAMBDA_INDEX=407];
/// emits mean cost-per-bar to ISV[COST_PER_BAR_AVG_INDEX=408]. Same
/// kernel reads LobBar fields from synthetic markets (Phase 2A) and real
/// fxcache LOB (prod) — dev/prod parity per Q3.
pub static SP15_COST_NET_SHARPE_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/cost_net_sharpe_kernel.cubin"));

/// SP15 Phase 1.2 (2026-05-06): launcher for the cost-net sharpe kernel.
/// Free function (matches `launch_sp15_sharpe_per_bar` precedent) so unit
/// / oracle tests can drive the kernel directly without the trainer struct;
/// production callers invoke the same launcher.
///
/// Per-side cost semantics (spec §6.2):
///   `cost_t = commission_per_rt × rt_ind[t]`
///          `+ half_spread[t] × |position[t]| × side_ind[t]`
///          `+ ofi_lambda × |position[t]| × |ofi[t]| × side_ind[t]`
///
/// All input streams are device f32 pointers (`pnl`, `half_spread`, `ofi`,
/// `side_ind`, `rt_ind`, `position`, `isv`, `out`). SP15 Wave 3b changed
/// `side_ind` / `rt_ind` from `unsigned int*` to `float*` (0.0/1.0
/// indicators) so the kernel chains directly with the f32 streams produced
/// by `sp15_position_history_derivation_kernel` — no u32 adapter buffer.
/// `out` MUST point to ≥3 f32 slots: `[mean_pnl_net, std, sharpe_net]`.
/// `isv` MUST be the ISV bus (≥443 f32 slots) — slot 407 is read for
/// `ofi_lambda`, slot 408 is written with `mean(cost_t)`.
///
/// SP15 Wave 5 follow-up (2026-05-07): pre-loaded `&CudaFunction`
/// parameter pattern, matches `5d63762ab` precedent for graph-safety +
/// cross-context-stable load. The previous on-demand `load_cubin` +
/// `load_function` per-call pattern caused `CUDA_ERROR_ILLEGAL_ADDRESS`
/// on hyperopt-trial child stream contexts (workflow `train-xggfc`,
/// trial 1 onwards). The kernel handle is now pre-loaded once in
/// `GpuBacktestEvaluator::new()` and passed through on every call.
#[allow(clippy::too_many_arguments)]
pub fn launch_sp15_cost_net_sharpe(
    stream: &Arc<CudaStream>,
    pnl: cudarc::driver::sys::CUdeviceptr,
    half_spread: cudarc::driver::sys::CUdeviceptr,
    ofi: cudarc::driver::sys::CUdeviceptr,
    side_ind: cudarc::driver::sys::CUdeviceptr,
    rt_ind: cudarc::driver::sys::CUdeviceptr,
    position: cudarc::driver::sys::CUdeviceptr,
    n: i32,
    commission_per_rt: f32,
    isv: cudarc::driver::sys::CUdeviceptr,
    out: cudarc::driver::sys::CUdeviceptr,
    kernel: &CudaFunction,
) -> Result<(), MLError> {
    unsafe {
        stream
            .launch_builder(kernel)
            .arg(&pnl)
            .arg(&half_spread)
            .arg(&ofi)
            .arg(&side_ind)
            .arg(&rt_ind)
            .arg(&position)
            .arg(&n)
            .arg(&commission_per_rt)
            .arg(&isv)
            .arg(&out)
            .launch(LaunchConfig {
                grid_dim: (1, 1, 1),
                block_dim: (256, 1, 1),
                shared_mem_bytes: 0,
            })
            .map_err(|e| MLError::ModelError(format!(
                "launch cost_net_sharpe_kernel: {e}"
            )))?;
    }
    Ok(())
}

/// SP15 Phase 1.3 (2026-05-06) + Phase 1.3.b-followup (2026-05-07):
/// per-env drawdown state tracking kernel.
///
/// READ-ONLY on the position state buffer's equity fields. Reads existing
/// PS_PEAK_EQUITY (slot 7) and PS_PREV_EQUITY (slot 9) from each env's
/// row (`pos_state[env * PORTFOLIO_STRIDE + …]`); does NOT write back to
/// those slots — `experience_env_step` is the canonical equity writer
/// (see `experience_kernels.cu:3473-3475`).
///
/// Phase 1.3.b-followup (Path A → Path B): the original Path A kernel
/// chose env-0 as the canonical DD observable and wrote 6 ISV scalar
/// slots directly. Path B redesigns to a per-env grid that writes a
/// per-env tile `dd_state_per_env[n_envs * 6]`; a separate
/// `dd_state_reduce_kernel` aggregates the tile into the 6 scalar ISV
/// slots [401..407) (mean across envs — HEALTH_DIAG diagnostic) plus
/// the new `DD_PERSISTENCE_MAX_INDEX = 443` slot (max across envs —
/// plasticity-injection trigger).
///
/// Per-env tile layout (offsets within each env's [6] sub-array):
///   tile[env*6 + 0] = DD_CURRENT          (max(0, (peak-equity)/peak))
///   tile[env*6 + 1] = DD_MAX              (running max in this fold)
///   tile[env*6 + 2] = DD_RECOVERY_BARS    (bars since last new HWM)
///   tile[env*6 + 3] = DD_PERSISTENCE      (twin counter to recovery)
///   tile[env*6 + 4] = CALMAR              (max(dd_max, 1e-4) — floor)
///   tile[env*6 + 5] = DD_PCT              (clip(curr/max(budget,1e-4), 0, 1))
///
/// The 1e-4 calmar floor eliminates the saturation-at-100 artifact seen
/// in the legacy host-side composer.
pub static SP15_DD_STATE_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/dd_state_kernel.cubin"));

/// SP15 Phase 1.3.b-followup (2026-05-07): launcher for the per-env
/// drawdown-state kernel. Per-env grid `[n_envs, 1, 1]` × `[1, 1, 1]`:
/// one thread per env (block) computes its own DD trajectory in a
/// state-machine fashion and writes 6 scalars to its slot in the
/// per-env tile. Free function (matches `launch_sp15_sharpe_per_bar`
/// and `launch_sp15_cost_net_sharpe` precedent) so unit/oracle tests
/// can drive the kernel directly without the trainer struct; production
/// callers invoke the same launcher.
///
/// Per-step semantics (spec §6.3, post-Phase-1.3.b-followup):
///   For each env in `[0, n_envs)`:
///     1. Read `prev_equity = pos_state[env * PORTFOLIO_STRIDE + PS_PREV_EQUITY]`
///        and `peak = pos_state[env * PORTFOLIO_STRIDE + PS_PEAK_EQUITY]`.
///     2. current_dd = max(0, (peak − prev_equity) / peak); write to
///        tile[env*6 + 0]. Update tile[env*6 + 1] = max(prior, current_dd).
///     3. If current_dd ≤ 0 (new HWM / pre-trade flat), zero
///        tile[env*6 + 2] / tile[env*6 + 3]; else increment both by 1.
///     4. dd_pct = clip(current_dd / max(dd_budget, 1e-4), 0, 1) →
///        tile[env*6 + 5]. tile[env*6 + 4] = max(dd_max, 1e-4) (floor).
///
/// The kernel does NOT write to the position state buffer — env_step is
/// the sole writer of PS_PREV_EQUITY / PS_PEAK_EQUITY. Wire callers
/// MUST launch this kernel AFTER the env_step launch on the same
/// stream, AND must launch `dd_state_reduce_kernel` after this kernel
/// to populate the ISV scalar slots from the tile (see
/// `gpu_experience_collector::launch_timestep_loop` step 5b).
///
/// `dd_budget` is the configured drawdown limit (e.g. 0.20 = 20%).
/// `n_envs` MUST be ≥ 1 (the reduction kernel divides by `n_envs`).
/// `dd_state_per_env` MUST point to at least `n_envs * 6` writable
/// f32 slots (the trainer's `sp15_dd_state_per_env` mapped-pinned
/// scratch buffer in production; a `MappedF32Buffer::new(n_envs * 6)`
/// in oracle tests). `pos_state` MUST point to a portfolio-state
/// buffer with at least `n_envs * PORTFOLIO_STRIDE` f32 slots.
/// SP15 Wave 5 follow-up (2026-05-07): pre-loaded `&CudaFunction`
/// parameter pattern, matches `5d63762ab` / `1396b62ec` precedent for
/// graph-safety + cross-context-stable load. The kernel handle is
/// pre-loaded once in `GpuExperienceCollector::new()` and passed
/// through on every per-rollout-step call. Per
/// `pearl_no_host_branches_in_captured_graph.md`: `load_cubin` /
/// `load_function` are host-side driver API calls that mutate driver
/// state and ARE NOT capturable inside `cuStreamBeginCapture`; doing
/// them per-call from the collector hot path was both a graph-capture
/// hazard and a CUDA_ERROR_ILLEGAL_ADDRESS hazard in subprocess child
/// contexts (documented failure mode in 1396b62ec).
pub fn launch_sp15_dd_state(
    stream: &Arc<CudaStream>,
    kernel: &CudaFunction,
    dd_budget: f32,
    n_envs: i32,
    dd_state_per_env: cudarc::driver::sys::CUdeviceptr,
    pos_state: cudarc::driver::sys::CUdeviceptr,
) -> Result<(), MLError> {
    let grid_x = n_envs.max(1) as u32;
    unsafe {
        stream
            .launch_builder(kernel)
            .arg(&dd_budget)
            .arg(&n_envs)
            .arg(&dd_state_per_env)
            .arg(&pos_state)
            .launch(LaunchConfig {
                grid_dim: (grid_x, 1, 1),
                block_dim: (1, 1, 1),
                shared_mem_bytes: 0,
            })
            .map_err(|e| MLError::ModelError(format!(
                "launch dd_state_kernel: {e}"
            )))?;
    }
    Ok(())
}

/// SP15 Phase 1.3.b-followup (2026-05-07): cubin slot for
/// `dd_state_reduce_kernel`.
///
/// Single-block tree-reduce that aggregates the per-env tile written by
/// `dd_state_kernel` into the 6 scalar ISV slots [401..407) (mean
/// across envs — smooth across-env summary for HEALTH_DIAG diagnostic
/// reporting) plus the new `DD_PERSISTENCE_MAX_INDEX = 443` slot (max
/// across envs — consumed by `plasticity_injection_kernel` so the
/// global advantage-head reset fires when ANY env's persistence
/// exceeds the threshold). BLOCK=256 with the strided initial
/// accumulation pattern handles n_envs up to production sizes (32768
/// envs on H100).
///
/// Aggregation rule rationale (per `feedback_isv_for_adaptive_bounds` —
/// the rule itself is structural, not adaptive):
///
///   * Mean for slots 401..407): preserves backward-compatible
///     "this is one DD scalar to look at" semantics for HEALTH_DIAG;
///     only the across-env aggregation rule changed (Path A:
///     env-0-canonical → Path B: mean across envs).
///   * Max for slot 443 (DD_PERSISTENCE_MAX): one set of advantage
///     weights → global firing semantics → ANY env exceeding the
///     threshold should arm the gate. Mean-aggregate would silently
///     gate the trigger when only a fraction of envs are in long DD.
///
/// Single-source-to-cubin convention (mirrors the SP15 1.3 / 1.4 / 3.X
/// pattern); single launcher (`launch_sp15_dd_state_reduce`).
pub static SP15_DD_STATE_REDUCE_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/dd_state_reduce_kernel.cubin"));

/// SP15 Phase 1.3.b-followup (2026-05-07): launcher for the DD state
/// reduction kernel. Single-block (`grid = (1, 1, 1)`, `block =
/// (256, 1, 1)`) tree-reduce — one launch per per-step `dd_state_kernel`
/// launch, on the same stream so CUDA serialises producer→consumer
/// without explicit event sync. `n_envs` MUST equal the value passed
/// to `launch_sp15_dd_state` for the same step (the per-env tile must
/// have been written by that earlier launch).
///
/// `dd_state_per_env` MUST point to at least `n_envs * 6` f32 slots
/// (read-only — the kernel only writes `isv`). `isv` MUST be the ISV
/// bus (≥SP15_SLOT_END=444 f32 slots — slots 401-406 + 443 written, and
/// slot 439 written when `dd_trajectory_per_env != 0`).
///
/// Phase 1.3.b-followup-B (2026-05-07): added `dd_trajectory_per_env`
/// parameter (nullable; 0 = skip the slot 439 mean-aggregate). When
/// provided, the kernel mean-aggregates the per-env trajectory tile
/// into ISV[DD_TRAJECTORY_DECREASING_INDEX=439] for HEALTH_DIAG
/// diagnostic preservation — production reads the per-env tile
/// directly via `per_insert_pa`'s per-transition lookup, but the
/// scalar slot remains for the existing diagnostic dashboard.
/// SP15 Wave 5 follow-up (2026-05-07): pre-loaded `&CudaFunction`
/// parameter pattern. See `launch_sp15_dd_state` doc-comment for the
/// graph-capture + child-context rationale.
pub fn launch_sp15_dd_state_reduce(
    stream: &Arc<CudaStream>,
    kernel: &CudaFunction,
    n_envs: i32,
    dd_state_per_env: cudarc::driver::sys::CUdeviceptr,
    dd_trajectory_per_env: cudarc::driver::sys::CUdeviceptr,
    isv: cudarc::driver::sys::CUdeviceptr,
) -> Result<(), MLError> {
    unsafe {
        stream
            .launch_builder(kernel)
            .arg(&n_envs)
            .arg(&dd_state_per_env)
            .arg(&dd_trajectory_per_env)
            .arg(&isv)
            .launch(LaunchConfig {
                grid_dim: (1, 1, 1),
                block_dim: (256, 1, 1),
                shared_mem_bytes: 0,
            })
            .map_err(|e| MLError::ModelError(format!(
                "launch dd_state_reduce_kernel: {e}"
            )))?;
    }
    Ok(())
}

/// SP15 Wave 4.1a (2026-05-06): launcher for the bottleneck-aware
/// dd_pct concat extension `bn_tanh_concat_dd_kernel`. Resolves the
/// kernel symbol from the existing `dqn_utility_kernels.cubin` (the
/// kernel was added next to `bn_tanh_concat_kernel` in the same source
/// file) — no separate cubin needed, multi-kernel-per-file pattern
/// matching `baseline_kernels.cu`.
///
/// LAYOUT FINGERPRINT BREAK: pre-SP15 checkpoints WILL NOT LOAD after
/// this commit (same as Phase 1.5 fingerprint marker
/// `TRUNK_INPUT_DD_PCT=sp15_phase_1_5`; the marker stays in
/// `layout_fingerprint_seed`). Greenfield OK per spec Q1.
///
/// Per spec §6.5 (Wave 4.1a correction): the standalone
/// `dd_pct_concat_kernel.cu` from Phase 1.5 was DELETED — it operated
/// on raw `[B, state_dim_padded=128]` state but production trunk
/// consumes `[B, s1_input_dim] = [B, bn_dim + portfolio_dim] = [B, 102]`
/// post-bottleneck. The bottleneck-aware extension folds the dd_pct
/// append into the same launch as the bn_tanh + portfolio concat;
/// output shape is `[B, concat_dd_dim] = [B, bn_dim + portfolio_dim + 1]`.
///
/// Wave 4.1a transient state: this launcher is currently ORPHAN —
/// production callers continue to use `bn_tanh_concat_kernel` /
/// `bn_tanh_concat_f32_kernel` until Wave 4.1b's atomic refactor flips
/// the 4 launch sites + grows `bn_concat_buf` / `tg_bn_concat_buf` /
/// `on_next_bn_concat_buf` / experience_collector's `bn_concat` by
/// `+B` floats and propagates `s1_input_dim+1` through GRN w_s1 +
/// cuBLAS gemm_caches. Acceptable per the documented Wave 3a/3b split
/// precedent (kernel + launcher + oracle test verify in isolation
/// first).
///
/// Contract:
///   `bn_hidden`:    device ptr to `[B, bn_dim]` f32 — IN/OUT (tanh
///                   applied in-place).
///   `states`:       device ptr to `[B, state_dim_padded]` f32 — source
///                   for portfolio passthrough columns.
///   `isv`:          device ptr to ISV bus (≥443 f32 slots) — reads
///                   slot 406 (DD_PCT_INDEX).
///   `concat_out`:   device ptr to `[B, concat_dd_dim]` f32 — caller-
///                   allocated, written by the kernel. Layout:
///                   `[bn_tanh | portfolio | dd_pct]`.
///   `batch_size`, `bn_dim`, `market_dim`, `state_dim_padded`,
///   `concat_dd_dim`: dimension scalars. `concat_dd_dim` MUST equal
///                   `bn_dim + (state_dim - market_dim) + 1`; the
///                   caller (Wave 4.1b) is responsible for sizing the
///                   buffer and computing the dim.
///
/// Grid: ceil(B * concat_dd_dim / 256), Block: 256.
///
/// SP15 Wave 4.1b OOB fix (2026-05-07): the kernel handle is now passed in
/// by the caller as a pre-loaded `&CudaFunction`, NOT resolved via
/// `load_cubin` + `load_function` on every call. The on-demand resolution
/// pattern is incompatible with CUDA Graph capture: `cuModuleLoadData` and
/// `cuModuleGetFunction` are HOST-side driver API calls that allocate
/// memory and mutate driver state, and they are NOT capturable inside a
/// `cuStreamBeginCapture` region. Calling them from `submit_forward_ops_main`
/// (which runs inside the `forward` child capture) caused a SEGV at exit
/// 139 — the captured-step path entered the loader while the stream was
/// already in capture mode, the cudarc wrapper raced with the driver's
/// internal capture tracking, and the host-side state corruption surfaced
/// as a segfault before `CAPTURE_PHASE_FORWARD_DONE` could print. Pre-
/// loading the handle in `compile_training_kernels` (mirroring the legacy
/// `bn_tanh_concat_kernel` field that Wave 4.1b removed) restores the
/// graph-safe contract: each child capture sees only kernel launches, not
/// driver-state mutations. See `pearl_no_host_branches_in_captured_graph`.
pub fn launch_sp15_bn_concat_dd(
    stream: &Arc<CudaStream>,
    kernel: &CudaFunction,
    bn_hidden: cudarc::driver::sys::CUdeviceptr,
    states: cudarc::driver::sys::CUdeviceptr,
    isv: cudarc::driver::sys::CUdeviceptr,
    concat_out: cudarc::driver::sys::CUdeviceptr,
    batch_size: i32,
    bn_dim: i32,
    market_dim: i32,
    state_dim_padded: i32,
    concat_dd_dim: i32,
) -> Result<(), MLError> {
    let total_elems = (batch_size as i64).saturating_mul(concat_dd_dim as i64);
    let blocks = (((total_elems as u64) + 255) / 256).max(1) as u32;
    unsafe {
        stream
            .launch_builder(kernel)
            .arg(&bn_hidden)
            .arg(&states)
            .arg(&isv)
            .arg(&concat_out)
            .arg(&batch_size)
            .arg(&bn_dim)
            .arg(&market_dim)
            .arg(&state_dim_padded)
            .arg(&concat_dd_dim)
            .launch(LaunchConfig {
                grid_dim: (blocks, 1, 1),
                block_dim: (256, 1, 1),
                shared_mem_bytes: 0,
            })
            .map_err(|e| MLError::ModelError(format!(
                "launch bn_tanh_concat_dd_kernel: {e}"
            )))?;
    }
    Ok(())
}

/// SP15 Phase 1.4 + Wave 3a (2026-05-06): cubin shared by all 4 constant-
/// policy counterfactual baselines (`baseline_buyhold_kernel`,
/// `baseline_hold_only_kernel`, `baseline_naive_momentum_kernel`,
/// `baseline_naive_reversion_kernel`). Per spec §6.4. Each baseline
/// computes its per-window `[mean, std, raw_sharpe]` by running a
/// constant-policy or last-bar-return-based action stream through a
/// 2-pass shared-memory tree-reduce (single-block BLOCK=256, no atomicAdd)
/// and writing to a caller-provided output buffer. Wave 3a contract
/// change: ISV-scalar writes to slots 409/410/412/416 are removed entirely
/// — per-window output is the correct shape for `WindowMetrics`
/// consumption, mirrors the 1.1.b sharpe_per_bar pattern. The four
/// launchers below all load this single cubin and resolve different
/// `get_function()` symbols — multi-kernel-per-file pattern per
/// `novelty_simhash_kernel.cu` precedent.
///
/// Trunk-shared baselines (random_dir_kelly slot 411, aux_only slot 413,
/// mag_quarter_fixed slot 414, trail_only slot 415) are out of scope for
/// this commit — they need partial-policy-forward access from the main
/// eval pass and are deferred to Task 1.4.b. Their slots stay at sentinel
/// 0.0 in the meantime.
///
/// Wave 3a transient state (2026-05-06): the 4 baseline launchers below
/// are currently ORPHAN — production callers in `GpuBacktestEvaluator`
/// (`Wave 3b`) invoke them per-window once that constructor signature
/// migration lands. Oracle tests in `sp15_phase1_oracle_tests.rs` drive
/// the launchers directly. Acceptable per the documented 3a/3b split.
pub static SP15_BASELINE_KERNELS_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/baseline_kernels.cubin"));

/// SP15 Phase 1.4 + Wave 3a (2026-05-06): launcher for the buyhold
/// counterfactual baseline. Free function (matches `launch_sp15_dd_state`
/// precedent) so unit/oracle tests can drive the kernel directly without
/// the trainer struct; production callers (eval-pass baseline launches
/// in Wave 3b's `GpuBacktestEvaluator::new`) invoke the same launcher.
///
/// Policy: position=+1 every bar (no exits). Writes per-window
/// `[mean, std, raw_sharpe]` to `out[0..3]`. `prices`, `half_spread`,
/// `ofi`, `out` are device f32 pointers; `n` is bar count. `out` MUST
/// point to ≥3 f32 slots — typically the `dev_ptr` of a `MappedF32Buffer`
/// of length `n_windows * 3`, with the launcher invoked sequentially per
/// window with the per-window slice (mirror of 1.1.b sharpe_per_bar
/// per-window launch sequence).
///
/// SP15 Wave 5 follow-up (2026-05-07): pre-loaded `&CudaFunction`
/// parameter pattern, matches `5d63762ab` precedent for graph-safety +
/// cross-context-stable load. The kernel handle is pre-loaded once in
/// `GpuBacktestEvaluator::new()` and passed through on every call.
pub fn launch_sp15_baseline_buyhold(
    stream: &Arc<CudaStream>,
    prices: cudarc::driver::sys::CUdeviceptr,
    half_spread: cudarc::driver::sys::CUdeviceptr,
    ofi: cudarc::driver::sys::CUdeviceptr,
    n: i32,
    commission_per_rt: f32,
    out: cudarc::driver::sys::CUdeviceptr,
    kernel: &CudaFunction,
) -> Result<(), MLError> {
    unsafe {
        stream
            .launch_builder(kernel)
            .arg(&prices)
            .arg(&half_spread)
            .arg(&ofi)
            .arg(&n)
            .arg(&commission_per_rt)
            .arg(&out)
            .launch(LaunchConfig {
                grid_dim: (1, 1, 1),
                block_dim: (256, 1, 1),
                shared_mem_bytes: 0,
            })
            .map_err(|e| MLError::ModelError(format!(
                "launch baseline_buyhold_kernel: {e}"
            )))?;
    }
    Ok(())
}

/// SP15 Phase 1.4 + Wave 3a (2026-05-06): launcher for the hold-only
/// counterfactual baseline. Policy: action=Hold every bar (no entry, no
/// exit). Writes per-window `[mean, std, raw_sharpe]` to `out[0..3]`.
/// On a strict no-trade trajectory `mean(pnl)=0` and `std(pnl)=0` so
/// the kernel's `(std > 0) ? mean / std : 0` guard emits sharpe=0, which
/// is the spec-mandated sentinel for an undefined sharpe.
///
/// SP15 Wave 5 follow-up (2026-05-07): pre-loaded `&CudaFunction`
/// parameter pattern, matches `5d63762ab` precedent for graph-safety +
/// cross-context-stable load. The kernel handle is pre-loaded once in
/// `GpuBacktestEvaluator::new()` and passed through on every call.
pub fn launch_sp15_baseline_hold_only(
    stream: &Arc<CudaStream>,
    prices: cudarc::driver::sys::CUdeviceptr,
    half_spread: cudarc::driver::sys::CUdeviceptr,
    ofi: cudarc::driver::sys::CUdeviceptr,
    n: i32,
    commission_per_rt: f32,
    out: cudarc::driver::sys::CUdeviceptr,
    kernel: &CudaFunction,
) -> Result<(), MLError> {
    unsafe {
        stream
            .launch_builder(kernel)
            .arg(&prices)
            .arg(&half_spread)
            .arg(&ofi)
            .arg(&n)
            .arg(&commission_per_rt)
            .arg(&out)
            .launch(LaunchConfig {
                grid_dim: (1, 1, 1),
                block_dim: (256, 1, 1),
                shared_mem_bytes: 0,
            })
            .map_err(|e| MLError::ModelError(format!(
                "launch baseline_hold_only_kernel: {e}"
            )))?;
    }
    Ok(())
}

/// SP15 Phase 1.4 + Wave 3a (2026-05-06): launcher for the naive-momentum
/// counterfactual baseline. Policy: `direction = sign(prices[i] −
/// prices[i-1])` with |position|=1. Writes per-window `[mean, std,
/// raw_sharpe]` to `out[0..3]`.
///
/// SP15 Wave 5 follow-up (2026-05-07): pre-loaded `&CudaFunction`
/// parameter pattern, matches `5d63762ab` precedent for graph-safety +
/// cross-context-stable load. The kernel handle is pre-loaded once in
/// `GpuBacktestEvaluator::new()` and passed through on every call.
pub fn launch_sp15_baseline_naive_momentum(
    stream: &Arc<CudaStream>,
    prices: cudarc::driver::sys::CUdeviceptr,
    half_spread: cudarc::driver::sys::CUdeviceptr,
    ofi: cudarc::driver::sys::CUdeviceptr,
    n: i32,
    commission_per_rt: f32,
    out: cudarc::driver::sys::CUdeviceptr,
    kernel: &CudaFunction,
) -> Result<(), MLError> {
    unsafe {
        stream
            .launch_builder(kernel)
            .arg(&prices)
            .arg(&half_spread)
            .arg(&ofi)
            .arg(&n)
            .arg(&commission_per_rt)
            .arg(&out)
            .launch(LaunchConfig {
                grid_dim: (1, 1, 1),
                block_dim: (256, 1, 1),
                shared_mem_bytes: 0,
            })
            .map_err(|e| MLError::ModelError(format!(
                "launch baseline_naive_momentum_kernel: {e}"
            )))?;
    }
    Ok(())
}

/// SP15 Phase 1.4 + Wave 3a (2026-05-06): launcher for the naive-reversion
/// counterfactual baseline. Policy: `direction = -sign(prices[i] −
/// prices[i-1])` with |position|=1 (mirror of naive_momentum). Writes
/// per-window `[mean, std, raw_sharpe]` to `out[0..3]`.
/// SP15 Wave 5 follow-up (2026-05-07): pre-loaded `&CudaFunction`
/// parameter pattern, matches `5d63762ab` precedent for graph-safety +
/// cross-context-stable load. The kernel handle is pre-loaded once in
/// `GpuBacktestEvaluator::new()` and passed through on every call.
pub fn launch_sp15_baseline_naive_reversion(
    stream: &Arc<CudaStream>,
    prices: cudarc::driver::sys::CUdeviceptr,
    half_spread: cudarc::driver::sys::CUdeviceptr,
    ofi: cudarc::driver::sys::CUdeviceptr,
    n: i32,
    commission_per_rt: f32,
    out: cudarc::driver::sys::CUdeviceptr,
    kernel: &CudaFunction,
) -> Result<(), MLError> {
    unsafe {
        stream
            .launch_builder(kernel)
            .arg(&prices)
            .arg(&half_spread)
            .arg(&ofi)
            .arg(&n)
            .arg(&commission_per_rt)
            .arg(&out)
            .launch(LaunchConfig {
                grid_dim: (1, 1, 1),
                block_dim: (256, 1, 1),
                shared_mem_bytes: 0,
            })
            .map_err(|e| MLError::ModelError(format!(
                "launch baseline_naive_reversion_kernel: {e}"
            )))?;
    }
    Ok(())
}

/// SP15 Wave 3a (2026-05-06): cubin for the post-loop position-history
/// derivation kernel (`sp15_position_history_derivation_kernel`). Reads
/// `actions_history_buf` (factored ints recorded by env_step) and emits
/// per-bar `position_history` / `side_ind` / `rt_ind` f32 streams
/// consumed by Wave 3b's cost-net sharpe path. Uses the new
/// `action_decoding_helpers.cuh` single-source-of-truth helper for the
/// action→direction→position mapping (mirrors
/// `trade_physics.cuh::decode_direction_4b` semantics so on-policy and
/// counterfactual paths agree on what each factored action means).
///
/// Wave 3a transient state: the launcher below is currently ORPHAN — the
/// production caller in `GpuBacktestEvaluator` lands in Wave 3b alongside
/// the cost-net sharpe wire-up. Oracle tests in
/// `sp15_phase1_oracle_tests.rs` drive the launcher directly. Acceptable
/// per the documented 3a/3b split.
pub static SP15_POSITION_HISTORY_DERIVATION_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/position_history_derivation_kernel.cubin"));

/// SP15 Wave 3a (2026-05-06): launcher for the position-history derivation
/// kernel. Free function (matches `launch_sp15_dd_state` /
/// `launch_sp15_baseline_*` precedent) so unit/oracle tests can drive
/// the kernel directly without the trainer struct.
///
/// `actions_history_buf` is the device pointer to `[n_windows * max_len]`
/// i32 factored actions (recorded by `experience_env_step` /
/// `backtest_env_step`; sentinel -1 marks unwritten / early-terminated
/// bars and is treated as "keep prev position"). The 3 output f32 streams
/// are caller-provided device pointers, each `[n_windows * max_len]` long.
///
/// Grid: `[n_windows, 1, 1]` × block `[1, 1, 1]`. The position state
/// machine is sequential per window (position[t] depends on position[t-1]),
/// so within-window parallelism is zero by construction; the across-
/// windows axis carries all the parallelism. No atomicAdd per
/// `feedback_no_atomicadd` (the kernel is a pure scan, not a reduction).
#[allow(clippy::too_many_arguments)]
pub fn launch_sp15_position_history_derivation(
    stream: &Arc<CudaStream>,
    actions_history_buf: cudarc::driver::sys::CUdeviceptr,
    n_windows: i32,
    max_len: i32,
    b1_size: i32,
    b2_size: i32,
    b3_size: i32,
    position_history_out: cudarc::driver::sys::CUdeviceptr,
    side_ind_out: cudarc::driver::sys::CUdeviceptr,
    rt_ind_out: cudarc::driver::sys::CUdeviceptr,
) -> Result<(), MLError> {
    let module = stream
        .context()
        .load_cubin(SP15_POSITION_HISTORY_DERIVATION_CUBIN.to_vec())
        .map_err(|e| MLError::ModelError(format!(
            "load sp15_position_history_derivation cubin: {e}"
        )))?;
    let kernel = module
        .load_function("sp15_position_history_derivation_kernel")
        .map_err(|e| MLError::ModelError(format!(
            "load sp15_position_history_derivation_kernel function: {e}"
        )))?;
    if n_windows <= 0 {
        return Ok(());
    }
    unsafe {
        stream
            .launch_builder(&kernel)
            .arg(&actions_history_buf)
            .arg(&n_windows)
            .arg(&max_len)
            .arg(&b1_size)
            .arg(&b2_size)
            .arg(&b3_size)
            .arg(&position_history_out)
            .arg(&side_ind_out)
            .arg(&rt_ind_out)
            .launch(LaunchConfig {
                grid_dim: (n_windows as u32, 1, 1),
                block_dim: (1, 1, 1),
                shared_mem_bytes: 0,
            })
            .map_err(|e| MLError::ModelError(format!(
                "launch sp15_position_history_derivation_kernel: {e}"
            )))?;
    }
    Ok(())
}

/// SP15 Wave 2 (2026-05-06): cubin for the per-step ISV-driven α
/// producer (`alpha_split_producer_kernel`). Replaces the pre-Wave-2
/// `SP15_R_QUALITY_DISCIPLINE_SPLIT_CUBIN` which housed both the
/// scalar composer and the producer; Wave 2 retains ONLY the producer
/// (the composer responsibility moves into the fused per-(i,t)
/// `compute_sp15_final_reward_kernel`).
///
/// Cold-start protocol:
///   • Constructor writes ALPHA_SPLIT_INDEX (slot 417) DIRECTLY to 0.5
///     (NOT derived from the formula, which would produce 0/0=0 from the
///     zero grad-norm EMAs at boot).
///   • The producer maintains the `sp15_alpha_warm_count` scratch buffer
///     ([1] f32 mapped-pinned, on the trainer struct). Per Q4 (Wave 2)
///     ownership: the producer kernel — launched once per step from
///     `gpu_experience_collector.rs` — is the SOLE owner of the
///     warm-count increment. While `*count < N_WARM=100` the producer
///     skips the formula write and leaves the slot at the
///     constructor-seeded 0.5 sentinel. Once `*count >= N_WARM` the
///     producer activates and writes
///     α = clamp(grad_norm_q / (grad_norm_q + grad_norm_d + ε), 0.05,
///     0.95) on each step.
///   • The fused composer reads ALPHA_SPLIT_INDEX directly each step, so
///     during warm-up it sees 0.5 (the sentinel); the producer's
///     α-formula activates ONLY after warm-count >= N_WARM.
pub static SP15_ALPHA_SPLIT_PRODUCER_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/alpha_split_producer_kernel.cubin"));

/// SP15 Wave 2 (2026-05-06): launcher for the ISV-driven α producer.
/// Reads grad-norm slots 418/419 + `sp15_alpha_warm_count` mapped-pinned
/// scratch buffer, writes ALPHA_SPLIT_INDEX (slot 417) clamped to
/// [0.05, 0.95] post-warmup, and increments the warm count by 1 each
/// call (Q4 ownership: this launcher fires ONCE per step, so the
/// counter advances by exactly 1; the fused per-(i,t) composer does
/// NOT increment — it would over-tick by 2*N*L per step).
///
/// Free function (matches `launch_sp15_dd_state` /
/// `launch_sp15_baseline_*` precedent) so unit/oracle tests can drive
/// the kernel directly without the trainer struct; production callers
/// in `gpu_experience_collector.rs::collect_experiences_gpu` invoke the
/// same launcher per step.
///
/// Single thread, single block.
/// SP15 Wave 5 follow-up (2026-05-07): pre-loaded `&CudaFunction`
/// parameter pattern. See `launch_sp15_dd_state` doc-comment for the
/// graph-capture + child-context rationale.
pub fn launch_sp15_alpha_split_producer(
    stream: &Arc<CudaStream>,
    kernel: &CudaFunction,
    isv: cudarc::driver::sys::CUdeviceptr,
    alpha_warm_count: cudarc::driver::sys::CUdeviceptr,
) -> Result<(), MLError> {
    unsafe {
        stream
            .launch_builder(kernel)
            .arg(&isv)
            .arg(&alpha_warm_count)
            .launch(LaunchConfig {
                grid_dim: (1, 1, 1),
                block_dim: (1, 1, 1),
                shared_mem_bytes: 0,
            })
            .map_err(|e| MLError::ModelError(format!(
                "launch alpha_split_producer_kernel: {e}"
            )))?;
    }
    Ok(())
}

/// SP15 Wave 2 (2026-05-06): cubin for the fused post-SP11 reward-axis
/// composer (`compute_sp15_final_reward_kernel`). Per-(i,t) parallel
/// over `[N*2*L]` slots; reads SP11's `out_rewards[idx]` in-place,
/// applies the full SP15 §8.2 / §9.2 four-stage composition (α-blend
/// split → DD asymmetric reward → quadratic DD penalty → SP12 v3
/// asymmetric cap via `state_layout.cuh` macros REWARD_NEG_CAP=-10 /
/// REWARD_POS_CAP=+5 per Q1 resolution), and writes the result back
/// to `out_rewards[idx]`.
///
/// Q3 — Early-return preservation: skips slots where
/// `slot_completed_normally[idx % (N*L)] == 0` (data-end at
/// experience_kernels.cu line 2142, blown-account at line 2203). Those
/// sentinel rewards (0.0f / -10.0f) pass through unmodified.
///
/// Q2 — Both on-policy and CF slots get the same DD-aware shaping:
/// `out_rewards[cf_off]` = `w_cf × r_cf` (SP11 controller weight already
/// applied) is composed via the same helpers as on-policy.
///
/// Subsumes the deleted `dd_penalty_kernel.cu` +
/// `dd_asymmetric_reward_kernel.cu` + the deleted scalar composer
/// `r_quality_discipline_split_kernel`; their bodies live as
/// `__device__` helpers in `sp15_reward_axis_helpers.cuh`.
pub static SP15_FINAL_REWARD_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/compute_sp15_final_reward_kernel.cubin"));

/// SP15 Wave 2 (2026-05-06): launcher for the fused post-SP11
/// reward-axis composer. Free function (matches `launch_sp15_dd_state`
/// precedent) so unit/oracle tests can drive the kernel directly
/// without the trainer struct; production callers in
/// `gpu_experience_collector.rs::collect_experiences_gpu` invoke the
/// same launcher per step (after experience_env_step + the α producer).
///
/// Contract:
///   `out_rewards`:                device ptr to `[N*2*L]` f32 row-major
///                                 (on-policy at `[0..N*L)`, CF at
///                                 `[N*L..2*N*L)`). Read-modify-written
///                                 in place — SP11 has already filled
///                                 the buffer.
///   `r_discipline_out`:           device ptr to `[N*L]` f32 — the
///                                 per-step `r_discipline` scalar
///                                 written by experience_env_step from
///                                 the REGRET_EMA ISV slot at the
///                                 finalisation point of normal reward
///                                 composition. CF slots reuse the
///                                 on-policy (i,t) value via index
///                                 `idx % (N*L)` per Q2 resolution.
///   `slot_completed_normally`:    device ptr to `[N*L]` i32 sentinel
///                                 flag — `1` if the slot completed the
///                                 normal reward composition path, `0`
///                                 if early-return (data-end / blown-
///                                 account). Slots with `0` are skipped
///                                 so SP11's sentinel rewards pass
///                                 through unmodified.
///   `isv`:                        device ptr to ISV bus
///                                 (≥SP15_SLOT_END=443 f32 slots).
///   `n`, `l`:                     dimension scalars; total threads =
///                                 `n * 2 * l`.
///
/// Grid: `ceil(n*2*l / 256)` blocks of 256 threads each. Each thread
/// processes one slot; no inter-thread communication.
///
/// SP15 Phase 1.3.b-followup (2026-05-07) — `dd_state_per_env` MUST
/// point to the per-env DD tile `[n * 6]` written by `dd_state_kernel`
/// earlier this step (the trainer's `sp15_dd_state_per_env` mapped-
/// pinned scratch buffer in production; a `MappedF32Buffer::new(n*6)`
/// in oracle tests). Each thread looks up its env's DD context via
/// `env_id = (idx % (N*L)) / L` so on-policy and CF slots at the same
/// (i,t) read the SAME per-env tile entry (one DD trajectory per env,
/// shared across slot kinds). The Path A `isv[DD_PCT_INDEX]` /
/// `isv[DD_CURRENT_INDEX]` reads in `sp15_dd_asymmetric_reward` /
/// `sp15_dd_penalty` migrated to this per-env lookup atomically per
/// `feedback_no_partial_refactor`.
/// SP15 Wave 5 follow-up (2026-05-07): pre-loaded `&CudaFunction`
/// parameter pattern. See `launch_sp15_dd_state` doc-comment for the
/// graph-capture + child-context rationale.
pub fn launch_sp15_final_reward(
    stream: &Arc<CudaStream>,
    kernel: &CudaFunction,
    out_rewards: cudarc::driver::sys::CUdeviceptr,
    r_discipline_out: cudarc::driver::sys::CUdeviceptr,
    slot_completed_normally: cudarc::driver::sys::CUdeviceptr,
    isv: cudarc::driver::sys::CUdeviceptr,
    dd_state_per_env: cudarc::driver::sys::CUdeviceptr,
    n: i32,
    l: i32,
) -> Result<(), MLError> {
    let total: i64 = (n as i64) * 2 * (l as i64);
    let blocks: u32 = (((total + 255) / 256).max(1)) as u32;
    unsafe {
        stream
            .launch_builder(kernel)
            .arg(&out_rewards)
            .arg(&r_discipline_out)
            .arg(&slot_completed_normally)
            .arg(&isv)
            .arg(&dd_state_per_env)
            .arg(&n)
            .arg(&l)
            .launch(LaunchConfig {
                grid_dim: (blocks, 1, 1),
                block_dim: (256, 1, 1),
                shared_mem_bytes: 0,
            })
            .map_err(|e| MLError::ModelError(format!(
                "launch compute_sp15_final_reward_kernel: {e}"
            )))?;
    }
    Ok(())
}

// SP15 Wave 2 (2026-05-06) — `SP15_DD_PENALTY_CUBIN` +
// `launch_sp15_dd_penalty` REMOVED. The Phase 3.3 standalone scalar
// kernel was scaffolding for a deferred consumer per
// `feedback_no_partial_refactor.md`. Wave 2's layered architecture
// replaces the deferred consumer with the
// `compute_sp15_final_reward_kernel` fused post-modifier; the quadratic
// DD penalty body now lives as a `__device__` inline `sp15_dd_penalty`
// helper in `sp15_reward_axis_helpers.cuh` consumed by the fused
// composer. ISV slots 420 (LAMBDA_DD) / 421 (DD_THRESHOLD) / 422
// (DD_PENALTY_GRAD_NORM) + state_reset_registry entries + dispatch
// arms remain (still read by the helper).

/// SP15 Phase 3.4 (2026-05-06): cubin for the `regret_signal_kernel`.
///
/// Per spec §8.2 (3.4): when policy holds AND a trade WOULD have been
/// profitable past `cost_t + trail_dist`,
/// `regret_bar = ideal_pnl_missed − cost_t`. Otherwise `regret_bar = 0`.
/// EMA tracked at REGRET_EMA (slot 423) via first-observation bootstrap
/// per `pearl_first_observation_bootstrap` + simple exponential decay
/// (α=0.05; Wiener-α adaptive swap per
/// `pearl_wiener_optimal_adaptive_alpha` is the documented follow-up).
/// Reads/writes ISV[REGRET_EMA_INDEX=423] (read-modify-write); writes a
/// single f32 `regret_bar` value to the caller-supplied output buffer.
///
/// Phase 3.4 lands kernel + launcher + 3 ISV constructor seeds + 3
/// state-reset registry entries + 3 dispatch arms only; the host-side
/// reward composer (`r_discipline -= LAMBDA_REGRET × REGRET_EMA`) is
/// deferred to a follow-up atomic commit per
/// `feedback_no_partial_refactor.md` (Phase 1.1-1.3 + 3.1-3.3
/// precedent).
pub static SP15_REGRET_SIGNAL_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/regret_signal_kernel.cubin"));

/// SP15 Phase 3.4 (2026-05-06): launcher for the regret-signal kernel.
/// Free function (matches `launch_sp15_dd_penalty` precedent — single
/// kernel, single cubin, single launcher) so unit/oracle tests can drive
/// the kernel directly without the trainer struct; production callers
/// invoke the same launcher.
///
/// `action` is the policy's discrete action (0 = Hold = regret-eligible;
/// any non-zero value treated as active and produces zero regret).
/// `ideal_pnl_missed` is the next-bar PnL the policy WOULD have captured
/// had it traded; `cost_t` is the same scalar shape as the Phase 1.2
/// `cost_per_bar` accumulator (commission + half-spread + ofi-impact);
/// `trail_dist` is the per-direction trail distance from the SP5 Pearl 8
/// trail-stop ISV signal. `isv` MUST be the ISV bus
/// (≥ SP15_SLOT_END=443 f32 slots — slot 423 is read-modify-written).
/// `regret_bar_out` MUST point to at least 1 writable f32 slot; the
/// kernel writes the scalar `regret_bar` per the gate semantics above.
/// Single thread, single block (mirrors `dd_penalty_kernel` — no
/// reduction).
pub fn launch_sp15_regret_signal(
    stream: &Arc<CudaStream>,
    action: i32,
    ideal_pnl_missed: f32,
    cost_t: f32,
    trail_dist: f32,
    isv: cudarc::driver::sys::CUdeviceptr,
    regret_bar_out: cudarc::driver::sys::CUdeviceptr,
) -> Result<(), MLError> {
    let module = stream
        .context()
        .load_cubin(SP15_REGRET_SIGNAL_CUBIN.to_vec())
        .map_err(|e| MLError::ModelError(format!(
            "load sp15_regret_signal cubin: {e}"
        )))?;
    let kernel = module
        .load_function("regret_signal_kernel")
        .map_err(|e| MLError::ModelError(format!(
            "load regret_signal_kernel function: {e}"
        )))?;
    unsafe {
        stream
            .launch_builder(&kernel)
            .arg(&action)
            .arg(&ideal_pnl_missed)
            .arg(&cost_t)
            .arg(&trail_dist)
            .arg(&isv)
            .arg(&regret_bar_out)
            .launch(LaunchConfig {
                grid_dim: (1, 1, 1),
                block_dim: (1, 1, 1),
                shared_mem_bytes: 0,
            })
            .map_err(|e| MLError::ModelError(format!(
                "launch regret_signal_kernel: {e}"
            )))?;
    }
    Ok(())
}

// SP15 Phase 3.5.b (2026-05-06): hold_floor moved INLINE into
// `experience_action_select` — the standalone `hold_floor_kernel.cu` +
// `SP15_HOLD_FLOOR_CUBIN` + `launch_sp15_hold_floor` were Phase 3.5
// scaffolding for a deferred consumer. Phase 3.5.b wires the consumer
// directly inside the production action-selection kernel reading the
// same ISV slots (426 / 427 / 428 / 429) and computing the same
// `hold_floor = α × σ(k × (entropy − ε₀))` as a single-line `__device__`
// expression on per-thread direction-policy entropy. Launching a kernel
// to write one f32 just to read it back was unnecessary; deletion is
// the correct application of `feedback_wire_everything_up.md` +
// `feedback_no_legacy_aliases.md`. ISV slots + state_reset_registry
// entries + dispatch arms remain — only the launch path is removed.

// SP15 Wave 2 (2026-05-06) — `SP15_DD_ASYMMETRIC_REWARD_CUBIN` +
// `launch_sp15_dd_asymmetric_reward` REMOVED. The Phase 3.5.2
// standalone scalar kernel was scaffolding for a deferred consumer per
// `feedback_no_partial_refactor.md`. Wave 2's layered architecture
// replaces the deferred consumer with the
// `compute_sp15_final_reward_kernel` fused post-modifier; the
// gain-only DD-recovery multiplier body now lives as a `__device__`
// inline `sp15_dd_asymmetric_reward` helper in
// `sp15_reward_axis_helpers.cuh` consumed by the fused composer. ISV
// slots 430 (DD_ASYMMETRY_LAMBDA) / 431 (R_GAIN_DD_BOOST) / 432
// (DD_DIST_VAR) + state_reset_registry entries + dispatch arms remain
// (still read or written by the helper). The diagnostic write to slot
// 431 happens inside the fused kernel from a single representative
// thread (on-policy slot at idx=0) to avoid concurrent writes.

/// SP15 Phase 3.5.3 (2026-05-06): cubin slot for the cooldown gate
/// `cooldown_kernel`.
///
/// Per spec §9.2 (3.5.3): after K consecutive losing trades, force Hold
/// for M bars. K and M are ISV-tracked (slots 433/434); initial
/// sentinels K=5, M=20 hardcoded in the trainer constructor. Counter
/// (COOLDOWN_BARS_REMAINING slot 435) decremented per bar — part of
/// state so the model can reason about it.
///
/// Per spec post-amendment-2 fix: streak counter (`consecutive_losses`)
/// updates only on trade-close events; per-bar non-close calls just
/// decrement the cooldown counter without touching the loss tracker.
///
/// MEDIAN_STREAK_LENGTH (slot 442) producer + ISV-driven K formula
/// reading the rolling median of consecutive-loss streak lengths via
/// the two-heap algorithm are documented Phase 3.5.3 follow-up
/// sub-tasks per `feedback_isv_for_adaptive_bounds.md`. ISV-driven M
/// from vol_normalizer time-to-mean-reversion is a parallel follow-up.
///
/// Phase 3.5.3 lands kernel + launcher + 4 ISV constructor seeds + 1
/// new `sp15_cooldown_consecutive_losses` scratch buffer + 5
/// state-reset registry entries + 5 dispatch arms only; the
/// action-selection wiring (force Hold while
/// `cooldown_remaining > 0`) is deferred to a follow-up atomic commit
/// per `feedback_no_partial_refactor.md` (Phase 1.1-1.5 + 3.1-3.5.2
/// precedent).
pub static SP15_COOLDOWN_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/cooldown_kernel.cubin"));

/// SP15 Phase 3.5.3 (2026-05-06): launcher for the cooldown gate
/// kernel. Free function (matches `launch_sp15_dd_penalty` /
/// `launch_sp15_regret_signal` / `launch_sp15_hold_floor` /
/// `launch_sp15_dd_asymmetric_reward` precedent — single kernel,
/// single cubin, single launcher) so unit/oracle tests can drive the
/// kernel directly without the trainer struct; production callers
/// invoke the same launcher.
///
/// `last_trade_outcome_loss` is 1 when the trade that just closed was
/// a loss, 0 otherwise (or when no trade closed). `trade_close_event`
/// is 1 on bars where a trade closed and 0 on per-bar non-close calls
/// — only trade-close events update the streak counter; non-close
/// calls just decrement `cooldown_remaining`. `isv` MUST be the ISV
/// bus (≥ `SP15_SLOT_END=443` f32 slots — slots 433/434 are read,
/// slot 435 is read-modify-written). `consecutive_losses` MUST point
/// to at least 1 writable f32 slot (the
/// `sp15_cooldown_consecutive_losses` mapped-pinned scratch buffer in
/// production; a fresh 1-element `MappedF32Buffer` in oracle tests).
/// Single thread, single block (mirrors `dd_penalty_kernel` /
/// `regret_signal_kernel` / `hold_floor_kernel` /
/// `dd_asymmetric_reward_kernel` — no reduction).
pub fn launch_sp15_cooldown(
    stream: &Arc<CudaStream>,
    last_trade_outcome_loss: i32,
    trade_close_event: i32,
    isv: cudarc::driver::sys::CUdeviceptr,
    consecutive_losses: cudarc::driver::sys::CUdeviceptr,
) -> Result<(), MLError> {
    let module = stream
        .context()
        .load_cubin(SP15_COOLDOWN_CUBIN.to_vec())
        .map_err(|e| MLError::ModelError(format!(
            "load sp15_cooldown cubin: {e}"
        )))?;
    let kernel = module
        .load_function("cooldown_kernel")
        .map_err(|e| MLError::ModelError(format!(
            "load cooldown_kernel function: {e}"
        )))?;
    unsafe {
        stream
            .launch_builder(&kernel)
            .arg(&last_trade_outcome_loss)
            .arg(&trade_close_event)
            .arg(&isv)
            .arg(&consecutive_losses)
            .launch(LaunchConfig {
                grid_dim: (1, 1, 1),
                block_dim: (1, 1, 1),
                shared_mem_bytes: 0,
            })
            .map_err(|e| MLError::ModelError(format!(
                "launch cooldown_kernel: {e}"
            )))?;
    }
    Ok(())
}

/// SP15 Phase 3.5.4 (2026-05-06) + Wave 4.2 / Phase 3.5.4.b
/// (2026-05-07): cubin slot for the plasticity injection trigger +
/// warm-up tracker + cuRAND Kaiming-He weight-reset kernel. TWO-STEP
/// recovery per spec §9.2 (3.5.4) post-amendment-2 fix:
///   1. Fire when `DD_PERSISTENCE > PLASTICITY_PERSISTENCE_THRESHOLD`
///      AND `PLASTICITY_FIRED_THIS_FOLD == 0` → set fired flag, set
///      warm-bars counter to M_warm (default 200), AND reset the
///      trailing 10% of advantage-head weights to Kaiming-He init
///      (`Normal(0, sqrt(2/fan_in))` sampled via cuRAND
///      `curand_normal()`; gain=√2 for ReLU/GLU activations).
///   2. Per-bar warm-bars decrement; consumer-wiring follow-up reads
///      `max(COOLDOWN_BARS_REMAINING, PLASTICITY_WARM_BARS_REMAINING)`
///      and forces Hold while > 0 (the OR-gate combining cooldown +
///      plasticity warm-up is the action-selection two-step).
///
/// Mirrors the Phase 3.3 `dd_penalty_kernel.cu` / 3.4
/// `regret_signal_kernel.cu` / 3.5 `hold_floor_kernel.cu` / 3.5.2
/// `dd_asymmetric_reward_kernel.cu` / 3.5.3 `cooldown_kernel.cu`
/// single-kernel-per-cubin pattern.
///
/// Phase 3.5.4 landed kernel + launcher + 3 ISV constructor seeds +
/// 3 state-reset registry entries + 3 dispatch arms with the weight-
/// reset deferred (kernel accepted `advantage_head_weights +
/// n_weights` but no-op'd via `(void)` cast). Wave 4.2 / Phase
/// 3.5.4.b lands the actual cuRAND Kaiming-He reset, extending the
/// launcher signature with `fan_in: i32` and `seed: u64` and
/// promoting the kernel grid from single-thread/single-block to a
/// 256-thread per-element grid for the weight write. The action-
/// selection two-step Flat + cooldown consumer wiring remains a
/// separate follow-up (Phase 3.5.4.c) per `feedback_no_partial_refactor.md`.
pub static SP15_PLASTICITY_INJECTION_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/plasticity_injection_kernel.cubin"));

/// SP15 Phase 3.5.4 (2026-05-06) + Wave 4.2 / Phase 3.5.4.b
/// (2026-05-07): launcher for the plasticity injection trigger +
/// warm-up tracker + cuRAND Kaiming-He weight-reset kernel. Free
/// function (matches `launch_sp15_dd_penalty` /
/// `launch_sp15_regret_signal` / `launch_sp15_hold_floor` /
/// `launch_sp15_dd_asymmetric_reward` / `launch_sp15_cooldown`
/// precedent — single kernel, single cubin, single launcher) so
/// unit/oracle tests can drive the kernel directly without the
/// trainer struct; production callers invoke the same launcher.
///
/// `isv` MUST be the ISV bus (≥ `SP15_SLOT_END=443` f32 slots — slot
/// 404 is read for DD_PERSISTENCE; slot 437 is read for
/// PLASTICITY_PERSISTENCE_THRESHOLD; slots 436 + 438 are read-modify-
/// written for PLASTICITY_FIRED_THIS_FOLD + PLASTICITY_WARM_BARS_
/// REMAINING). `advantage_head_weights` MUST point to at least
/// `n_weights` writable f32 slots — Wave 4.2 lands the actual
/// Kaiming-He reset of the trailing `n_weights / 10` slots when the
/// kernel fires.
///
/// `m_warm` is the warm-up bar count to set on fire (default 200 per
/// spec §9.2 (3.5.4)). `fan_in` is the input dimension of the
/// advantage head's last Linear layer (the Kaiming-He gain
/// denominator — `stddev = sqrt(2 / fan_in)`); production callers
/// derive it from the trainer's network config. `seed` is a
/// per-call deterministic value (e.g. derived from a trainer RNG
/// state or `SystemTime::now()` nanos) — the kernel re-initialises
/// per-thread `curandState`s from `(seed, sequence=tid, offset=0)`
/// every launch; the same `(seed, tid)` pair always yields the same
/// sample.
///
/// Grid: `[((n_weights / 10 + 255) / 256), 1, 1]` × `[256, 1, 1]`.
/// Block 0 / thread 0 owns the trigger + warm-bars decrement (single-
/// thread, mirrors `cooldown_kernel` / `dd_asymmetric_reward_kernel`
/// / `hold_floor_kernel`); the remaining threads race-free write the
/// per-element Kaiming-He samples (each thread writes a unique
/// `advantage_head_weights[reset_start + tid]`, no atomic, no
/// reduction). When `n_weights / 10 == 0` (oracle edge case) the
/// grid degenerates to `[1, 1, 1]` so the trigger still runs.
/// SP15 Wave 5 follow-up (2026-05-07): pre-loaded `&CudaFunction`
/// parameter pattern. See `launch_sp15_dd_state` doc-comment for the
/// graph-capture + child-context rationale.
pub fn launch_sp15_plasticity_injection(
    stream: &Arc<CudaStream>,
    kernel: &CudaFunction,
    isv: cudarc::driver::sys::CUdeviceptr,
    advantage_head_weights: cudarc::driver::sys::CUdeviceptr,
    n_weights: i32,
    m_warm: f32,
    fan_in: i32,
    seed: u64,
) -> Result<(), MLError> {
    // Grid sized so every weight in the trailing 10% gets a thread.
    // `block_dim = 256` matches the codebase's prevailing 1-D
    // element-wise launch shape. `grid_dim` always ≥ 1 so block 0
    // thread 0 always runs the trigger, even when `n_weights / 10 == 0`
    // (oracle edge case where the test passes a tiny weight buffer).
    let reset_count = (n_weights / 10).max(0);
    let grid_x = ((reset_count + 255) / 256).max(1) as u32;
    unsafe {
        stream
            .launch_builder(kernel)
            .arg(&isv)
            .arg(&advantage_head_weights)
            .arg(&n_weights)
            .arg(&m_warm)
            .arg(&fan_in)
            .arg(&seed)
            .launch(LaunchConfig {
                grid_dim: (grid_x, 1, 1),
                block_dim: (256, 1, 1),
                shared_mem_bytes: 0,
            })
            .map_err(|e| MLError::ModelError(format!(
                "launch plasticity_injection_kernel: {e}"
            )))?;
    }
    Ok(())
}

/// SP15 Phase 3.5.5 (2026-05-06): cubin slot for the recovery-curriculum
/// per-step DD_TRAJECTORY_DECREASING proxy kernel
/// (`dd_trajectory_decreasing_kernel`).
///
/// Per spec §9.2 (3.5.5) post-amendment-2 fix: replaces non-existent
/// episode-level metadata with a per-bar boolean computed from the
/// dd_pct trajectory:
///
///   trajectory(t) = 1.0  iff  dd_pct(t) < dd_pct(t-1)
///                            AND  dd_pct(t-1) > DD_TRAJECTORY_FLOOR
///
/// True when the transition is part of a recovery: drawdown shrinking
/// from a non-trivial level (above the floor).
///
/// PER sampling consumer (DEFERRED follow-up):
///
///   sampling_weight = base_priority × (1.0 + RECOVERY_OVERSAMPLE_WEIGHT × DD_TRAJECTORY_DECREASING_t)
///
/// DD_TRAJECTORY_FLOOR (slot 441) is ISV-driven (25th percentile of
/// running dd_pct distribution) — producer kernel is a documented
/// Phase 3.5.5 follow-up sub-task per `feedback_isv_for_adaptive_bounds.md`.
/// The 0.02 sentinel from the constructor / fold-reset arm is used until
/// the producer lands. RECOVERY_OVERSAMPLE_WEIGHT (slot 440) is also
/// ISV-driven (proportional to current dd_pct) — same follow-up
/// sub-task; the 2.0 sentinel is used until then.
///
/// Mirrors the Phase 3.3 `dd_penalty_kernel.cu` / 3.4
/// `regret_signal_kernel.cu` / 3.5 `hold_floor_kernel.cu` / 3.5.2
/// `dd_asymmetric_reward_kernel.cu` / 3.5.3 `cooldown_kernel.cu` / 3.5.4
/// `plasticity_injection_kernel.cu` single-kernel-per-cubin pattern.
///
/// Phase 3.5.5 lands kernel + launcher + 3 ISV constructor seeds + 1
/// new `sp15_dd_trajectory_prev_dd` scratch buffer + 4 state-reset
/// registry entries + 4 dispatch arms only; the PER sampling consumer
/// migration AND the ISV-driven 25th-percentile DD_TRAJECTORY_FLOOR
/// producer are deferred to follow-up atomic commits per
/// `feedback_no_partial_refactor.md` (matching Phase 1.1-1.5 + 3.1-3.5.4
/// precedent — kernel + launcher verify in isolation first via the
/// GPU oracle tests below).
pub static SP15_DD_TRAJECTORY_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/dd_trajectory_kernel.cubin"));

/// SP15 Phase 3.5.5 (2026-05-06): launcher for the recovery-curriculum
/// per-step DD_TRAJECTORY_DECREASING proxy kernel. Free function
/// (matches `launch_sp15_dd_penalty` / `launch_sp15_regret_signal` /
/// `launch_sp15_hold_floor` / `launch_sp15_dd_asymmetric_reward` /
/// `launch_sp15_cooldown` / `launch_sp15_plasticity_injection`
/// precedent — single kernel, single cubin, single launcher) so
/// unit/oracle tests can drive the kernel directly without the trainer
/// struct; production callers invoke the same launcher.
///
/// SP15 Phase 1.3.b-followup-B (2026-05-07) — per-env reshape. The
/// kernel now runs a per-env grid `[n_envs, 1, 1] × [1, 1, 1]` (one
/// thread per env, mirroring `dd_state_kernel`'s shape). Each thread
/// reads its env's DD_PCT from the shared `dd_state_per_env` tile
/// (offset 5 within the 6-stat sub-array, written by `dd_state_kernel`
/// in the launch immediately before this one on the same stream),
/// reads its persistent prev_dd_pct from the per-env scratch buffer
/// `prev_dd_pct_per_env`, and writes its trajectory boolean to the
/// per-env output tile `dd_trajectory_per_env`. The downstream
/// `per_insert_pa` reads the per-env output tile via insert-time
/// `env_id = (j % (n_envs * lookback)) / lookback` so each transition's
/// recovery boost reflects ITS env's recovery context. The reduction
/// kernel `dd_state_reduce_kernel` mean-aggregates the per-env
/// trajectory output into ISV[DD_TRAJECTORY_DECREASING_INDEX=439] for
/// HEALTH_DIAG diagnostic preservation.
///
/// `n_envs` MUST be ≥ 1. `dd_state_per_env` MUST point to at least
/// `n_envs * 6` readable f32 slots; `prev_dd_pct_per_env` and
/// `dd_trajectory_per_env` MUST each point to at least `n_envs`
/// writable f32 slots. `isv` MUST be the ISV bus (≥ `SP15_SLOT_END=444`
/// f32 slots — slot 441 is read for DD_TRAJECTORY_FLOOR).
/// SP15 Wave 5 follow-up (2026-05-07): pre-loaded `&CudaFunction`
/// parameter pattern. See `launch_sp15_dd_state` doc-comment for the
/// graph-capture + child-context rationale.
pub fn launch_sp15_dd_trajectory_decreasing(
    stream: &Arc<CudaStream>,
    kernel: &CudaFunction,
    n_envs: i32,
    dd_state_per_env: cudarc::driver::sys::CUdeviceptr,
    prev_dd_pct_per_env: cudarc::driver::sys::CUdeviceptr,
    dd_trajectory_per_env: cudarc::driver::sys::CUdeviceptr,
    isv: cudarc::driver::sys::CUdeviceptr,
) -> Result<(), MLError> {
    let grid_x = n_envs.max(1) as u32;
    unsafe {
        stream
            .launch_builder(kernel)
            .arg(&n_envs)
            .arg(&dd_state_per_env)
            .arg(&prev_dd_pct_per_env)
            .arg(&dd_trajectory_per_env)
            .arg(&isv)
            .launch(LaunchConfig {
                grid_dim: (grid_x, 1, 1),
                block_dim: (1, 1, 1),
                shared_mem_bytes: 0,
            })
            .map_err(|e| MLError::ModelError(format!(
                "launch dd_trajectory_decreasing_kernel: {e}"
            )))?;
    }
    Ok(())
}

/// SP11 Fix 39 (2026-05-04, Task A2): SimHash novelty signal — lookup +
/// update kernels sharing one cubin. Lookup reads
/// `1/sqrt(1+count)` ∈ [0, 1] for each (state, action) bucket; update
/// non-atomically increments the bucket count (race-tolerated per
/// `feedback_no_atomicadd.md`; under-counts bias novelty UPWARD, the safe
/// direction). 1M-slot hash table + 42×16 SimHash projection.
///
/// A2 builds + registers the cubin and provides Rust launchers; the
/// production wire-up to the replay path lives in B1 — A2 just owns the
/// reset registry contract end-to-end (the novelty_hash_buf reset arm
/// lands here alongside the buffer field).
static SP11_NOVELTY_SIMHASH_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/novelty_simhash_kernel.cubin"));

/// SP7 (2026-05-03): GPU dispatch for cql/c51 budget consumer + dev-ptr
/// SAXPY/scale variants. Consolidates three kernels into one cubin so they
/// share a CUmodule with the SP7 dispatch buffer. See
/// `consume_lb_budget_kernel.cu` for the per-kernel contract.
///
/// Architecture: the existing SAXPY/scale path took `alpha: f32` by value at
/// the call site, which baked the value into the kernel-arg buffer at
/// CUDA-Graph capture time. After SP7 introduced a controller writing to ISV
/// each step, the captured `aux_child` graph saw step-0 (FoldReset sentinel)
/// values frozen for every replay, defeating the controller. New kernels
/// read `alpha` from a device pointer (mapped-pinned) populated by
/// `lb_budget_dispatch` each step → capture-safe end-to-end.
static CONSUME_LB_BUDGET_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/consume_lb_budget_kernel.cubin"));

/// Plan 4 Task 3 (E.3): IQN multi-quantile diagnostic EMAs into ISV[99..103).
/// 4-block (256 threads/block, shmem-reduce, no atomicAdd) kernel launched
/// alongside `h_s2_rms_ema_update` from `training_loop.rs`. Reads the IQN
/// online forward's `save_q_online [TBA, B*Q]` surface and EMAs the mean
/// |Q| at τ ∈ {0.05, 0.25, 0.75, 0.95} into ISV[99..103) — the median
/// (τ=0.50) is intentionally skipped (already in the greedy-Q diagnostic).
/// Diagnostic only — producer-only in this commit.
static IQN_QUANTILE_EMA_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/iqn_quantile_ema_kernel.cubin"));

/// Plan 4 Task 1B-i / 1B-ii / 1B-iii: VSN feature-selection kernels
/// (forward + backward). The cubin is `pub(crate)` so `CublasGemmSet::new`
/// can load both the forward and backward kernel handles in 1B-iii (the
/// backward handle stays `#[allow(dead_code)]` until 1B-iv consumes it).
pub(crate) static VSN_FEATURE_SELECTION_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/vsn_feature_selection_kernel.cubin"));

/// Plan 4 Task 1B-i / 1B-iii: VSN per-group mask EMA producer.
/// Loaded by `GpuDqnTrainer::new` and launched per-step from
/// `training_loop.rs` next to `launch_h_s2_rms_ema`. Reads `vsn_mask_buf`
/// (saved by the online-on-states VSN forward) and EMA-updates
/// ISV[VSN_MASK_GROUP_0_EMA_INDEX..VSN_MASK_GROUP_5_EMA_INDEX) (slots
/// 105..111).
pub(crate) static VSN_MASK_EMA_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/vsn_mask_ema_kernel.cubin"));

/// Plan 4 Task 6 Commit A: aux-heads kernels (forward + backward + loss-
/// reduce + label-builder + per-tensor batch-reduce). 7 entry points:
/// `aux_next_bar_forward`, `aux_regime_forward`, `aux_next_bar_loss_reduce`,
/// `aux_regime_loss_reduce`, `aux_regime_label_from_states`,
/// `aux_next_bar_backward`, `aux_regime_backward`, `aux_param_grad_reduce`.
/// Loaded by `gpu_aux_heads::{AuxHeadsForwardOps, AuxHeadsBackwardOps}`.
/// `pub(crate)` so the orchestrator module in
/// `cuda_pipeline/gpu_aux_heads.rs` can `include_bytes!()` the same cubin
/// without a duplicate copy. Module is **additive** in this commit — ZERO
/// production callers; Commit B wires the forward/backward and training-
/// loop loss accumulation.
#[allow(dead_code)]
pub(crate) static AUX_HEADS_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/aux_heads_kernel.cubin"));

/// Plan 4 Task 6 Commit A: aux-heads loss EMA producer (single-thread,
/// single-block kernel mirroring `h_s2_rms_ema_kernel.cu`). Producer for
/// ISV[AUX_NEXT_BAR_MSE_EMA_INDEX] + ISV[AUX_REGIME_CE_EMA_INDEX]; consumer
/// wires in Commit B (HEALTH_DIAG aux-loss diagnostic line).
#[allow(dead_code)]
pub(crate) static AUX_HEADS_LOSS_EMA_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/aux_heads_loss_ema_kernel.cubin"));

/// SP22 H6 vNext Phase A2 (2026-05-13): trade-outcome label producer.
/// Per-env K=3 outcome classification at trade-close: reads
/// `trade_close_per_sample[i*L+t]` + `pnl_vs_target_at_close[env]` +
/// `pnl_vs_stop_at_close[env]` → writes `out_labels[env]` in
/// {-1 mask, 0=Profit, 1=Stop, 2=Timeout}. Loaded by
/// `gpu_aux_heads::AuxTradeOutcomeForwardOps`. Additive in this commit
/// — wire-up into the collector chain lands in Phase B (atomic).
#[allow(dead_code)]
pub(crate) static TRADE_OUTCOME_LABEL_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/trade_outcome_label_kernel.cubin"));

/// SP22 H6 vNext Phase A3 (2026-05-13): trade-outcome aux head forward.
/// K=3 softmax over `{Profit, Stop, Timeout}` mirroring
/// `aux_next_bar_forward`'s architecture (Linear → ELU → Linear → softmax)
/// at K=3 instead of K=2. Loaded by
/// `gpu_aux_heads::AuxTradeOutcomeForwardOps`.
#[allow(dead_code)]
pub(crate) static AUX_TRADE_OUTCOME_FORWARD_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/aux_trade_outcome_forward_kernel.cubin"));

/// SP22 H6 vNext Phase A4 (2026-05-13): trade-outcome sparse CE reduce.
/// Single-block shmem-tree reduce over the K=3 softmax tile with `-1`
/// mask handling. Sparse-label arithmetic (~1-5% valid bars). Loaded by
/// `gpu_aux_heads::AuxTradeOutcomeForwardOps`.
#[allow(dead_code)]
pub(crate) static AUX_TRADE_OUTCOME_LOSS_REDUCE_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/aux_trade_outcome_loss_reduce_kernel.cubin"));

/// SP22 H6 vNext Phase A5 (2026-05-14): trade-outcome aux head backward.
/// K=3 backward emitting per-sample partials (dW1, db1, dW2, db2,
/// dh_s2_aux). Mirrors `aux_next_bar_backward`'s structure at K=3.
/// Loaded by `gpu_aux_heads::AuxTradeOutcomeBackwardOps`. Reuses the
/// existing K-generic `aux_param_grad_reduce` for the per-sample →
/// final-grad batch-reduce step.
#[allow(dead_code)]
pub(crate) static AUX_TRADE_OUTCOME_BACKWARD_CUBIN: &[u8] =
    include_bytes!(concat!(env!("OUT_DIR"), "/aux_trade_outcome_backward_kernel.cubin"));

/// SP14 Layer C Phase C.3 (2026-05-08): aux trunk forward kernel cubin.
/// 3-layer Linear→ELU→Linear→ELU→Linear MLP forward — reads encoder
/// output, writes `h_s2_aux` plus saved-for-backward hidden activations.
/// Loaded by `gpu_aux_trunk::AuxTrunkForwardOps`. Module is additive in
/// this commit — wire-up into the collector chain lands in Phase C.5
/// (atomic). Backward kernel + Adam updates land in Phase C.4.
#[allow(dead_code)]
pub(crate) static AUX_TRUNK_FORWARD_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/aux_trunk_forward_kernel.cubin"));

/// SP14 Layer C Phase C.4 (2026-05-08): aux trunk backward cubin. Holds
/// three kernels — `aux_trunk_bwd_dh_pre`, `aux_trunk_bwd_dW_reduce`,
/// `aux_trunk_bwd_db_reduce` — that together propagate `dh_s2_aux`
/// through w3/w2/w1 to accumulate dW{1,2,3} + db{1,2,3}. Block-tree-
/// reduce only (no atomicAdd). CRITICAL: kernel set does NOT compute
/// `dx_in` — encoder boundary is the stop-gradient per locked design.
/// Loaded by `gpu_aux_trunk::AuxTrunkBackwardOps`. Module is additive
/// in this commit — wire-up into the collector backward chain + Adam
/// updates land in Phase C.5 (atomic).
#[allow(dead_code)]
pub(crate) static AUX_TRUNK_BACKWARD_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/aux_trunk_backward_kernel.cubin"));

/// SP14 Layer C Phase C.4b (2026-05-08): adaptive aux prediction horizon
/// producer cubin. Single-thread kernel that drives
/// `ISV[AUX_PRED_HORIZON_BARS_INDEX=450]` from
/// `ISV[AVG_WIN_HOLD_TIME_BARS_INDEX=451]` via Pearl-A first-observation
/// bootstrap + slow Wiener-α EMA. Loaded by
/// `gpu_aux_trunk::AuxHorizonUpdateOps`. Per-epoch boundary launch. See
/// `aux_horizon_update_kernel.cu` for the algorithm + plan §C.4b.
pub(crate) static AUX_HORIZON_UPDATE_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/aux_horizon_update_kernel.cubin"));

/// SP14 Layer C Phase C.4b (2026-05-08): avg winning hold time EMA
/// producer cubin. Single-block kernel that sweeps the per-epoch
/// `hold_at_exit_per_sample` + `trade_profitable_per_sample` buffers and
/// writes `ISV[AVG_WIN_HOLD_TIME_BARS_INDEX=451]` for downstream
/// consumption by `aux_horizon_update_kernel`. Loaded by
/// `gpu_aux_trunk::AvgWinHoldTimeUpdateOps`. Per-epoch boundary launch.
pub(crate) static AVG_WIN_HOLD_TIME_UPDATE_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/avg_win_hold_time_update_kernel.cubin"));

/// Class A P0-A (2026-05-08): adaptive REWARD_POS/NEG_CAP producer cubin.
/// Single-block kernel that sweeps the per-epoch `step_ret_per_sample` +
/// `trade_close_per_sample` buffers, computes a Welford p99 estimator of
/// realized winning returns × 1.5× safety factor → POS cap, and writes
/// both `ISV[REWARD_POS_CAP_ADAPTIVE_INDEX=452]` and
/// `ISV[REWARD_NEG_CAP_ADAPTIVE_INDEX=453]` (NEG = −2 × POS, Kahneman
/// asymmetry preserved at producer-time). Pearl-A first-observation
/// bootstrap; α=0.01 slow EMA thereafter. Loaded by
/// `gpu_aux_trunk::RewardCapUpdateOps`. Per-epoch boundary launch.
pub(crate) static REWARD_CAP_UPDATE_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/reward_cap_update_kernel.cubin"));

/// SP18 v2 Phase 3 (2026-05-09): adaptive HOLD_REWARD_POS/NEG_CAP producer
/// cubin. Single-block 256-thread kernel that sweeps the per-epoch
/// `step_ret_per_sample` + `trade_close_per_sample` buffers (same source
/// `reward_cap_update_kernel` reads from), filters by `(is_close &&
/// step_ret != 0)` to extract Long/Short close magnitudes (DD3=b),
/// computes a Welford `mean + Z_99 × sigma` p99 estimator with
/// `max(p99, max_observed)` conservative takeover × HOLD_REWARD_CAP_
/// SAFETY_FACTOR=1.5, applies Wiener-optimal α blend (Welford
/// accumulators in slots [487..493); floor at WELFORD_ALPHA_MIN=0.4 per
/// `pearl_wiener_alpha_floor_for_nonstationary`), and writes both
/// `ISV[HOLD_REWARD_POS_CAP_INDEX=483]` and
/// `ISV[HOLD_REWARD_NEG_CAP_INDEX=484]` (NEG = −2 × POS, DD5=b mirrored
/// asymmetry — same Kahneman 2:1 ratio as the position-side cap, single
/// source of truth at producer time per
/// `pearl_audit_unboundedness_for_implicit_asymmetry`). Pearl-A
/// first-observation bootstrap on POS (sentinels POS=5.0/NEG=-10.0
/// match SP14 P0-A pattern). Loaded by
/// `gpu_aux_trunk::HoldRewardCapUpdateOps`. Per-epoch boundary launch
/// right AFTER `launch_reward_cap_update`.
pub(crate) static HOLD_REWARD_CAP_UPDATE_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/hold_reward_cap_update_kernel.cubin"));

/// Class A P1-Producer (2026-05-08): adaptive Bayesian Kelly priors producer cubin.
/// Single-block 256-thread kernel that sweeps the per-fold-end
/// `portfolio_state[n_envs, PS_STRIDE]` buffer (same source the
/// `kelly_cap_update_kernel` reads from at the same boundary), aggregates
/// the realized PS_KELLY_{WIN_COUNT, LOSS_COUNT, SUM_WINS, SUM_LOSSES}
/// fields across envs, and slow-EMA-blends into ISV[454..458). Pearl-A
/// first-observation bootstrap (sentinels 2.0/2.0/0.01/0.01 match
/// pre-P1-Producer hardcoded values for bit-identical cold-start) +
/// α=0.005 slow EMA thereafter. Loaded by
/// `gpu_aux_trunk::KellyBayesianPriorsUpdateOps`. Per-epoch boundary launch
/// (RIGHT BEFORE `kelly_cap_update` so that kernel sees the freshly-blended
/// priors).
pub(crate) static KELLY_BAYESIAN_PRIORS_UPDATE_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/kelly_bayesian_priors_update_kernel.cubin"));

/// Class A audit-fix Batch 4-A (2026-05-08): adaptive DD saturation floor
/// producer cubin. Single-block 256-thread kernel that sweeps the
/// `dd_state_per_env[n_envs * 6]` tile (same source as
/// `compute_sp15_final_reward_kernel` reads from at the same boundary,
/// populated by `dd_state_kernel`), aggregates DD_MAX across envs (offset
/// 1 in each per-env tile), and slow-EMA-blends p75 estimator × 1.5 into
/// `ISV[DD_SATURATION_FLOOR_ADAPTIVE_INDEX=458]`. Replaces the hardcoded
/// `0.25f` saturation floor in `trade_physics.cuh::apply_margin_cap`
/// (the upper end of the linear position-size scaling ramp). Pearl-A
/// first-observation bootstrap (sentinel 0.25 matches pre-fix hardcoded
/// value for bit-identical cold-start) + α=0.01 slow EMA thereafter
/// (DD distribution is a slow-moving fold-volatility property — same
/// cadence as the P0-A REWARD_POS_CAP producer). Loaded by
/// `gpu_aux_trunk::DdSaturationFloorUpdateOps`. Per-epoch boundary launch.
pub(crate) static DD_SATURATION_FLOOR_UPDATE_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/dd_saturation_floor_update_kernel.cubin"));

/// Class A audit-fix Batch 4-B (2026-05-08, Item 4): adaptive
/// MIN_HOLD_TEMPERATURE producer cubin. Single-thread cold-path kernel
/// that reads `ISV[AUX_DIR_ACC_SHORT_EMA_INDEX=373]` (sp13 fast EMA of
/// binary directional accuracy), maps it to a [5, 50] temperature via
/// `temp = TEMP_MIN + (TEMP_MAX − TEMP_MIN) × confusion` where
/// `skill     = clamp((short_ema − 0.5)/0.5, 0, 1)` and
/// `confusion = 1 − skill`, and slow-EMA-blends into
/// `ISV[MIN_HOLD_TEMPERATURE_ADAPTIVE_INDEX=460]`. The `confusion`
/// inversion (vs. raw `skill`) gives the WR-plateau scenario its
/// designed exit ramp: dir_acc ≈ 0.5 → confusion=1 → temp HIGH
/// (permissive); dir_acc → 1.0 → confusion=0 → temp LOW (strict).
/// HIGH T = forgiving (deficit/(deficit+T) → 0); LOW T = sharp
/// (deficit/(deficit+T) → 1) per `compute_min_hold_penalty` at
/// trade_physics.cuh:575. Replaces the
/// deleted epoch-driven anneal schedule `T_end + (T_start − T_end) ×
/// exp(-epoch / decay)` (formerly state_layout.cuh::
/// MIN_HOLD_TEMPERATURE_{START, END, DECAY} +
/// training_loop.rs::min_hold_temperature_for_epoch). Pearl-A first-
/// observation bootstrap (sentinel 50.0 matches the deleted
/// MIN_HOLD_TEMPERATURE_START=50 for bit-identical cold-start) + α=0.05
/// mid-cadence EMA. Loaded by
/// `gpu_aux_trunk::MinHoldTemperatureUpdateOps`. Per-epoch boundary
/// launch (same cadence as P0-A REWARD_POS_CAP).
pub(crate) static MIN_HOLD_TEMPERATURE_UPDATE_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/min_hold_temperature_update_kernel.cubin"));

/// SP14 Layer C Phase C.6 (2026-05-08): h_s2_aux RMS EMA producer cubin.
/// Single-block 256-thread kernel computing `sqrt(mean(h_s2_aux²))` over
/// `h_s2_aux [B, SH2]` (aux trunk final output) and EMA-blending into
/// `ISV[H_S2_AUX_RMS_EMA_INDEX=449]`. Pearl-A first-observation bootstrap
/// embedded in kernel (sentinel 0.0 → replace); fixed α=0.05 EMA blend
/// thereafter. Slot 449 is outside the SP4/SP5 wiener buffer linear span
/// so the scratch+apply_pearls_ad_kernel path is not available. Loaded by
/// `gpu_aux_trunk::HS2AuxRmsEmaOps`. Per-collector-step launch (same
/// cadence as `aux_trunk_forward`).
pub(crate) static H_S2_AUX_RMS_EMA_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/h_s2_aux_rms_ema_kernel.cubin"));

/// Plan C Phase 2 follow-up A.2 (2026-04-29): q-drift rate ISV producer.
/// Single-thread single-block cold-path kernel mirroring
/// `moe_lambda_eff_kernel.cu` / `kelly_cap_update_kernel.cu`. Reads
/// ISV[Q_ABS_REF_INDEX] + ISV[Q_DIR_ABS_REF_INDEX] plus host-passed
/// `q_mean_curr` / `q_mean_prev` scalars; writes
/// ISV[Q_DRIFT_RATE_INDEX=129] = `clip(|Δq|/denom, 0, 4)`. Consumer:
/// `tau_update_kernel.cu` multiplies the cosine-scheduled `tau_base` by
/// `1/(1+ISV[129])` so the effective Polyak rate dampens monotonically as
/// q-drift rises.
static Q_DRIFT_RATE_EMA_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/q_drift_rate_ema_kernel.cubin"));

/// Plan C Phase 2 follow-up K (2026-04-29): adaptive fold-boundary warmup
/// signal. Single-block single-thread cold-path producer mirroring
/// `q_drift_rate_ema_kernel.cu` / `moe_lambda_eff_kernel.cu` shape. Reads
/// two grad-norm EMAs (fast α=0.1, slow α=0.001) plus a host-passed step
/// counter; writes ISV[FOLD_WARMUP_FACTOR_INDEX=130] = clamp(fast/slow, 0, 1).
/// Two consumers (lr_eff and clip_eff in `training_loop.rs`), both monotone.
static FOLD_WARMUP_FACTOR_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/fold_warmup_factor_kernel.cubin"));

/// SP4 Layer C close-out C1 redesigned (2026-05-01): GPU-only fast/slow
/// grad-norm EMA update kernel. Single-thread, single-block — replaces the
/// host-side arithmetic block in `update_adaptive_clip` per
/// `feedback_no_cpu_compute_strict`. Reads `grad_norm_buf[0]`, updates two
/// mapped-pinned EMA scalars consumed by `fold_warmup_factor_kernel`.
static UPDATE_GRAD_NORM_EMAS_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/update_grad_norm_emas_kernel.cubin"));

/// SP4 Layer C close-out C2 (2026-05-01): GPU-only IQN readiness gauge update
/// kernel. Single-thread, single-block — replaces the host-side arithmetic
/// block in `update_iqn_readiness` per `feedback_no_cpu_compute_strict`.
/// Takes the host-passed `iqn_loss` scalar (already from `read_total_loss`
/// mapped-pinned readback) and updates three coupled mapped-pinned scalars
/// (`iqn_loss_initial_pinned`, `iqn_loss_ema_pinned`, `iqn_readiness_pinned`)
/// in-place via their dev_ptrs. Cold-start sentinel `prev < 1e-12` preserves
/// the deleted host-side bootstrap branch.
static UPDATE_IQN_READINESS_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/update_iqn_readiness_kernel.cubin"));

/// SP4 Layer C close-out C3 (2026-05-01): GPU-only utilization EMA update
/// kernel. Single-thread, single-block — replaces the host-side arithmetic
/// block in `reduce_current_q_stats` per `feedback_no_cpu_compute_strict`.
/// Takes the host-passed `atom_util` scalar (from `host[6]` mapped-pinned
/// readback of `q_readback_pinned`) and updates `utilization_ema_pinned` +
/// `homeostatic_obs_pinned[1]` in lockstep. Cold-start sentinel `prev > 0.99`
/// preserves the deleted host-side bootstrap branch (constructor init=1.0).
static UPDATE_UTILIZATION_EMA_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/update_utilization_ema_kernel.cubin"));

/// SP4 Layer C close-out C4 (2026-05-01): GPU-only winsorized adaptive
/// grad-clip update kernel. Single-thread, single-block — replaces the
/// host-side scalar-reduction + EMA chain in `update_adaptive_clip` per
/// `feedback_no_cpu_compute_strict`. Migrates the ENTIRE chain (winsor +
/// cold-start sentinel + EMA + 2 clamps + ISV upper bound + mapped-pinned
/// write) per `feedback_no_partial_refactor` rather than splitting EMA-only
/// into the kernel and leaving the surrounding host scalar reductions.
static UPDATE_ADAPTIVE_CLIP_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/update_adaptive_clip_kernel.cubin"));

/// SP4 Layer C close-out follow-up (2026-05-01): GPU-only readiness-driven
/// homeostatic-target EMA kernel. Single-block, six threads (one per
/// homeostatic slot) — replaces the host-side EMA loop in
/// `calibrate_homeostatic_targets` per `feedback_no_cpu_compute_strict`.
/// Reads host-passed `readiness` scalar plus `homeostatic_obs_dev_ptr`
/// (mapped-pinned observations) and updates `homeostatic_targets_pinned[k]`
/// in-place via `homeostatic_targets_dev_ptr`. Thread 0 forces the Q-mean
/// invariant `targets[0] = 0.0` exactly as the deleted host post-loop did.
static CALIBRATE_HOMEOSTATIC_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/calibrate_homeostatic_kernel.cubin"));

/// Plan 4 Task 1B-ii: per-group VSN MLP hidden dim. `Linear_1[g]` is
/// `[VSN_HIDDEN_DIM, group_dim_g]`, `Linear_2[g]` is `[1, VSN_HIDDEN_DIM]`.
/// 16 chosen as a tractable expansion that fits comfortably against the
/// smallest group (`plan_isv` = 7 features); per-group params total
/// `VSN_HIDDEN_DIM*group_dim_g + VSN_HIDDEN_DIM + VSN_HIDDEN_DIM + 1`.
pub const VSN_HIDDEN_DIM: usize = 16;

/// Plan 4 Task 1B-iv-rc: VSN dW attenuation factor — global multiplier applied
/// to `vsn_d_gated_state_buf` IN-PLACE before each `vsn_backward` invocation
/// (one of 4 trunk-touching paths: main C51/MSE, IQN trunk, CQL, ensemble
/// diversity). The softmax-and-gate Jacobian is linear in d_gated_state, so
/// scaling the input scales every VSN dW/dB downstream uniformly across the
/// 24 VSN tensors at grad_buf[95..119).
///
/// Rationale (smoke regression analysis, 2026-04-25):
///   - 1B-iii baseline (VSN frozen at Xavier, no backward): geom-mean 71.24
///   - 1B-iv (VSN trains via main C51/MSE path only):       geom-mean 57.49
///   - 1B-iv-ext (VSN trains via all 4 paths):              geom-mean 36.36
///
/// More VSN gradient signal → worse smoke. Pattern is monotonic in the
/// "amount of trainable VSN signal", not in the "asymmetry" of coverage —
/// disproves the H3 (asymmetric coverage) hypothesis. Consistent with H1
/// (VSN-bottleneck coupling instability): VSN gates the bottleneck Linear's
/// input distribution; a strong trainable VSN gate shifts that distribution
/// faster than the bottleneck Linear can co-adapt, leading to a regime where
/// the GRN trunk receives an increasingly off-distribution embedding.
///
/// `VSN_DW_DAMP = 0.10` attenuates VSN gradient signal by 10× across all 4
/// paths uniformly. Constant (not a per-step warmup ramp) because per-step
/// variation would require pinned-device-mapped scalar plumbing and a kernel
/// variant that reads `*alpha_ptr` instead of the baked `alpha` literal —
/// out of scope for the rc commit. The constant is graph-capture-stable.
///
/// If smoke ≥ 71.24 with this dampening, we'll have established that the
/// regression IS gradient-magnitude driven and a follow-up commit can
/// elevate it to a per-step ramp tied to ISV[VSN_MASK_GROUP_*_EMA] drift
/// or to `c51_alpha`.
pub const VSN_DW_DAMP: f32 = 0.10;

/// Mamba2 temporal scan configuration.
const MAMBA2_HISTORY_K: usize = 8;    // Rolling history length
const MAMBA2_STATE_DIM: usize = 16;   // SSM state dimension
const OFI_EMBED_DIM: usize = 10;     // OFI embedding width appended to history rows
/// OFI embed MLP input width — mirrors `ml_core::state_layout::OFI_DIM` so the
/// MLP consumes every OFI slot persisted to fxcache (no silent [OFI_DIM..) gap).
const OFI_EMBED_IN: usize = ml_core::state_layout::OFI_DIM;
/// OFI embed MLP W tensor element count: [OFI_EMBED_DIM, OFI_EMBED_IN] row-major.
const OFI_EMBED_W_COUNT: usize = OFI_EMBED_DIM * OFI_EMBED_IN;
/// OFI embed MLP total parameter count: W[OFI_EMBED_W_COUNT] + b[OFI_EMBED_DIM].
const OFI_EMBED_TOTAL_PARAMS: usize = OFI_EMBED_W_COUNT + OFI_EMBED_DIM;

/// Introspective State Vector (ISV) configuration.
const ISV_K: usize = 4;   // Temporal ISV history length
/// ISV signal count.
///
/// Layout:
///   [0..7]   core training dynamics: q_drift, grad_norm_ema, td_err_ema,
///            ens_var_ema, ens_var_vel, reward_ema, atom_util_ema, loss_ema
///   [8..11]  regime awareness (ADX/CUSUM): adx_ema, regime_disagree,
///            regime_vel_ema, regime_stability
///   [12]     learning_health (written by separate kernel, consumed by c51/etc.)
///   [13..15] Task 2.X "make Full useful" — per-magnitude-bin Q-value EMAs.
///            [13] = Q_mean_ema(Quarter), [14] = Q_mean_ema(Half),
///            [15] = Q_mean_ema(Full). Populated by `q_mag_bin_means_reduce`
///            on the stats cadence, smoothed with tau=0.05. The C51 loss and
///            gradient kernels derive an adaptive bin weight from these
///            EMAs by computing the bias-vs-Full signature
///              collapse = max(0, max(Q_mean) − Q_mean(a1)) / max(|Q_ref|, eps)
///            applied ONLY to magnitude-branch samples (d==1). Self-disables
///            when Full becomes the argmax bin.
///   [16]     q_abs_ref_ema — max(|Q_mean_ema[k]|) across the 3 magnitude
///            bins. Scale reference for the collapse-fraction normaliser
///            (keeps the bin-weight formula scale-invariant as Q-values
///            grow over training).
///   [17..20] Task 2.Y "make direction branch useful at eval" — per-direction
///            Q-value EMAs. Layout mirrors [13..15]:
///              [17] = Q_mean_ema(Short), [18] = Q_mean_ema(Hold),
///              [19] = Q_mean_ema(Long),  [20] = Q_mean_ema(Flat).
///            Populated by `q_dir_bin_means_reduce` on the stats cadence,
///            smoothed with tau=0.05. The C51 loss / gradient kernels derive
///            an adaptive per-direction bin weight from these EMAs by the
///            same bias-vs-argmax composite applied to the magnitude axis,
///            with the direction-specific architectural shape
///              dir_bias_signal[k] = {1.0, 0.5, 0.0, 1.0} (Short/Hold/Long/Flat)
///            — tradable directions (Short/Long) are amplified; Flat is
///            never amplified (it is the already-over-estimated low-variance
///            bin); Hold sits between (position-preserving but not trading).
///   [21]     q_dir_abs_ref_ema — max(|Q_mean_ema[k]|) across the 4
///            direction bins. Scale reference for the direction-branch
///            collapse-fraction normaliser.
///
/// Slots [0..11] populated by `isv_signal_update` kernel and rotated through
/// `isv_history` for temporal decay in `isv_forward`. Slot [12] is now written
/// by `isv_signal_update` from `sharpe_ema` (slot 22) and NOT rotated into
/// history. Slots [13..16] written by `isv_signal_update` (added with the
/// Task 2.X adaptive magnitude fix) but NOT rotated into history — they are
/// already EMA-smoothed. Slots [17..21] likewise written by
/// `isv_signal_update` (added with the Task 2.Y direction-branch fix) and
/// NOT rotated into history. Slot [22] (`SHARPE_EMA_INDEX`) added by the
/// ISV-audit bundle (2026-04-23) — Rust host broadcasts `training_sharpe_ema`
/// each epoch; the kernel reads it to drive slot 12 (learning_health).
/// Network-facing ISV width: the length of the vector fed into `w_isv_fc1`
/// attention weights. **Never change without a checkpoint format migration** —
/// the w_isv_fc1 weight tensor is sized [16, ISV_NETWORK_DIM] at 368 floats.
const ISV_NETWORK_DIM: usize = 23;
/// Total ISV slot count including cross-kernel scratchpad slots that never
/// feed the attention path. Slots [23..31] carry per-branch Q-support
/// (center, half-width) for direction/magnitude/order/urgency — written by
/// `update_eval_v_range`, read by `adaptive_atom_positions`,
/// `warm_start_atom_positions`, and per-sample support tiling.
///
/// Grad-balance unification bundle (2026-04-23): slots [31..35] extend the
/// scratchpad for ISV-driven adaptive gradient balancing. Slots [31..35)
/// carry per-branch gradient-norm targets; slot [35] carries the
/// observed-spread-driven scale clamp limit. Written by
/// `grad_balance_isv_update` on-device (inside graph capture); read by
/// `branch_grad_rescale`. Replaces the pre-spec hardcoded `4 × median`
/// cap that was a no-op when three branches were co-elevated (see
/// feedback_isv_for_adaptive_bounds.md + task #92).
///
/// Slot [36] carries the IQL branch_scales per-sample safety floor —
/// bootstrapped at 0.1 in the constructor, read by `iql_per_branch_advantage`
/// to ensure no (sample, branch) pair collapses to zero gradient
/// contribution.
///
/// Slots [39..47) are the DQN v2 Plan 1 expansion (spec §4.C.6, 2026-04-24):
///   [39] EPOCH_IDX_INDEX       — CPU writes current epoch index at epoch boundary.
///   [40] TOTAL_EPOCHS_INDEX    — CPU writes total epoch count at constructor (static).
///   [41] EPSILON_EFF_INDEX     — GPU-written by epsilon_update kernel (Plan 1 Task 14).
///   [42] TAU_EFF_INDEX         — GPU-written by tau_update kernel (Plan 1 Task 13).
///
/// Slots [43..47) are the DQN v2 Plan 2 Task 3 D.2 per-branch gamma expansion (spec §4.D.2):
///   [43] GAMMA_DIR_EFF_INDEX   — GPU-written by per_branch_gamma_update kernel (dir horizon).
///   [44] GAMMA_MAG_EFF_INDEX   — GPU-written by per_branch_gamma_update kernel (mag horizon).
///   [45] GAMMA_ORD_EFF_INDEX   — GPU-written by per_branch_gamma_update kernel (ord horizon).
///   [46] GAMMA_URG_EFF_INDEX   — GPU-written by per_branch_gamma_update kernel (urg horizon).
///   Replaces the former scalar GAMMA_EFF_INDEX=43. Per-branch base/max:
///   direction [0.92, 0.99], magnitude [0.88, 0.95], order [0.85, 0.93],
///   urgency [0.80, 0.90]. GPU kernel reads V_CENTER/V_HALF + health and
///   diverges each branch's gamma from a uniform bootstrap.
///
/// Slots [47..50) continue the Plan 1 expansion (downstream shift +3 from D.2):
///   [47] KELLY_CAP_EFF_INDEX   — GPU-written by kelly_cap_update kernel (Plan 1 Task 11).
///   [48] CQL_ALPHA_INDEX       — constructor writes `config.cql_alpha`; CQL formula reads
///                                base from here instead of config field (Plan 1 Task 12).
///   [49] PLAN_THRESHOLD_INDEX  — constructor writes 0.5f; plan kernels read instead of
///                                hardcoded literal (Plan 1 Task 16).
///
/// Slots [50..58) are the DQN v2 Plan 2 Task 1 expansion (spec §4.C.1, 2026-04-24):
///   [50] Q_P05_DIR_INDEX — per-epoch 5th-percentile Q EMA for direction branch.
///   [51] Q_P05_MAG_INDEX — per-epoch 5th-percentile Q EMA for magnitude branch.
///   [52] Q_P05_ORD_INDEX — per-epoch 5th-percentile Q EMA for order branch.
///   [53] Q_P05_URG_INDEX — per-epoch 5th-percentile Q EMA for urgency branch.
///   [54] Q_P95_DIR_INDEX — per-epoch 95th-percentile Q EMA for direction branch.
///   [55] Q_P95_MAG_INDEX — per-epoch 95th-percentile Q EMA for magnitude branch.
///   [56] Q_P95_ORD_INDEX — per-epoch 95th-percentile Q EMA for order branch.
///   [57] Q_P95_URG_INDEX — per-epoch 95th-percentile Q EMA for urgency branch.
///
/// Slots [60..75) carry the D.8 TLOB diagnostic, C.2 reward-attribution EMAs,
/// B.2 trade-attempt novelty rate, and layout fingerprint.
///   [60] TLOB_REGIME_FOCUS_EMA_INDEX — mean-max attention weight EMA (CPU-written, per-epoch).
///   [63] REWARD_POPART_EMA_INDEX — mean |final reward| EMA (Plan 3 C.2).
///   [64] REWARD_CF_EMA_INDEX — mean |cf reward| EMA.
///   [65] REWARD_TRAIL_EMA_INDEX — mean |trail penalty| EMA.
///   [66] REWARD_MICRO_EMA_INDEX — mean |micro reward| EMA.
///   [67] REWARD_OPP_COST_EMA_INDEX — mean |opp_cost| EMA.
///   [68] REWARD_BONUS_EMA_INDEX — mean |bonus| EMA.
///   [71] TRADE_ATTEMPT_RATE_EMA_INDEX — Flat→Positioned transition rate EMA
///        (GPU-written by `trade_attempt_rate_ema_update`, Plan 3 Task 3 B.2).
///   [72] TRADE_TARGET_RATE_INDEX — CPU-frozen at epoch 5 from measured
///        TRADE_ATTEMPT_RATE_EMA; reference rate for novelty = max(0, 1 - attempt/target).
///   [75] READINESS_EMA_INDEX — per-batch mean readiness EMA (Plan 3 Task 4 B.4);
///        GPU-written by `plan_threshold_update`, drives derived
///        ISV[PLAN_THRESHOLD_INDEX=49] = max(0.1, 0.5 × ema). Producer-only
///        upgrade: consumers of slot 49 unchanged.
///   [78] STATE_KL_TRAIN_VAL_EMA_INDEX — per-epoch moment-match KL EMA between
///        train-state sample and val-state batch over the OFI block (Plan 3
///        Task 7 C.3). GPU-written by `state_kl_moment_match`. Higher value =
///        more train/val divergence.
///   [79] STATE_KL_AMPLIFICATION_INDEX — Flat-trap escape multiplier ∈ [1, 2]
///        smoothly tracking the trailing-EMA-of-self KL ratio. Multiplied into
///        B.1 opp_cost and B.2 bonus consumers in `experience_kernels.cu`.
///   [82] SEED_STEPS_TARGET_INDEX — Plan 3 Task 8 B.3: config replay_seed_steps target.
///        CPU constructor write from `config.replay_seed_steps`; constant for the run.
///   [83] SEED_STEPS_DONE_INDEX — Plan 3 Task 8 B.3: cumulative seed-phase steps
///        completed. GPU-incremented by `seed_step_counter_update` kernel after every
///        `collect_experiences_gpu`. Capped at TARGET to avoid overflow.
///   [84] SEED_FRAC_EMA_INDEX — Plan 3 Task 8 B.3: adaptive EMA of
///        `max(0, 1 - DONE/TARGET)`. Smoothly decays 1→0 as the seed phase completes.
///        Cold-start 1.0 (fully in seed phase). Consumed by Task 9 CQL α ramp kernel.
///   [87] VSN_MAG_EMA_INDEX — Plan 4 Task 5 Mode A (E.5 attention-focus): EMA of
///        magnitude-branch VSN projection-weight L2/abs mean. Diagnostic only.
///        Written by `attention_focus_ema_update` (cold-path single-thread kernel)
///        once per epoch boundary. Adaptive α matches Plan 3 Task 3 convention
///        (α_base × (1 + 0.5 × |clamp(sharpe, −2, 2)|)). FoldReset to 0.0.
///   [88] VSN_DIR_EMA_INDEX — same producer/contract as [87], direction branch.
///   [89] MAMBA2_RETENTION_EMA_INDEX — same producer/contract; EMA of per-batch
///        mean |mamba2_h_enriched| as a state-transition magnitude proxy.
///   [92] TARGET_DRIFT_MAG_EMA_INDEX — RMS(target - online) per-branch EMA;
///        replaces legacy CPU-DtoH `per_branch_target_drift()`. Written by
///        `target_drift_ema_update` GPU kernel. Magnitude branch.
///   [93] TARGET_DRIFT_DIR_EMA_INDEX — same kernel/contract; direction branch.
///   [96] H_S2_RMS_EMA_INDEX — EMA of RMS(save_h_s2) per batch. Producer-only
///        in this commit (no consumers). Wired in 2c.3c.6 to drive the
///        adaptive scale of `mag_concat_qdir`'s residual stack.
///   [99] IQN_Q_P05_EMA_INDEX — Plan 4 Task 3 (E.3) — EMA of the IQN τ=0.05
///        quantile averaged across (B × tba). Producer:
///        `iqn_quantile_ema_update` GPU kernel reading
///        `gpu_iqn.save_q_online` after each training step. Diagnostic only
///        (no consumer kernel reads slot [99..103) in this commit).
///   [100] IQN_Q_P25_EMA_INDEX — same producer/contract; τ=0.25.
///   [101] IQN_Q_P75_EMA_INDEX — same producer/contract; τ=0.75.
///   [102] IQN_Q_P95_EMA_INDEX — same producer/contract; τ=0.95.
///        Note: the median (τ=0.50) lives in the existing C51 / IQL greedy-Q
///        diagnostic — not duplicated here. Only the 4 off-median quantiles
///        are new.
///   [105] VSN_MASK_GROUP_0_EMA_INDEX — Plan 4 Task 1B-ii — `market` feature
///         group importance mask EMA. Producer: `vsn_mask_ema_update` GPU kernel
///         (single-block shmem reduction over per-sample mask buffer; α=0.05).
///         Launch site wired in 1B-iii. Cold-start `1.0/SL_NUM_FEATURE_GROUPS`
///         = 1/6 (uniform prior — no group preference at init). FoldReset → 1/6.
///   [106] VSN_MASK_GROUP_1_EMA_INDEX — `ofi` group, same producer/contract.
///   [107] VSN_MASK_GROUP_2_EMA_INDEX — `tlob` group, same producer/contract.
///   [108] VSN_MASK_GROUP_3_EMA_INDEX — `mtf` group, same producer/contract.
///   [109] VSN_MASK_GROUP_4_EMA_INDEX — `portfolio` group, same producer/contract.
///   [110] VSN_MASK_GROUP_5_EMA_INDEX — `plan_isv` group, same producer/contract.
///   [113] AUX_NEXT_BAR_MSE_EMA_INDEX — Plan 4 Task 6 Commit A — EMA of the
///         next-bar regression head's mean-batch MSE (α=0.05). Producer:
///         `aux_heads_loss_ema_update` GPU kernel (single-thread, single-block;
///         mirrors `h_s2_rms_ema_update`'s shape). Launch site wires in
///         Commit B alongside `launch_h_s2_rms_ema`. Cold-start 0.0 (no aux
///         forwards have happened yet). FoldReset → 0.0. Diagnostic only —
///         no consumer kernel reads slot [113] in either Commit A or B
///         (Commit B's HEALTH_DIAG emission is a CPU-side text consumer
///         only, not a kernel input).
///   [114] AUX_REGIME_CE_EMA_INDEX — Plan 4 Task 6 Commit A — EMA of the
///         regime classification head's mean-batch cross-entropy
///         (α=0.05). Same producer/contract as [113]; same FoldReset → 0.0.
///   [117] RETIRED — formerly `AUX_LABEL_SCALE_EMA_INDEX` (Plan 4 Task 6 /
///         Plan 5 Task 5 follow-up). Retired in SP13 B1.0 (2026-05-05) when
///         `aux_next_bar_loss_reduce` / `aux_next_bar_backward` flipped to
///         scale-free MSE: labels are z-normalised at the data layer so the
///         pre-B1.0 ISV-driven `(pred - label / scale)` residual reduces to
///         `(pred - label)` within rounding. The slot remains unused in the
///         ISV bus (layout fingerprint reserves it via the seed; do not
///         reuse without a fingerprint bump). B1.1 will replace MSE with CE
///         entirely, eliminating the formulation that made this slot useful.
///   [115] ISV_LAYOUT_FINGERPRINT_LO_INDEX — low 32 bits of layout fingerprint
///         (shifted 111→115 in Plan 4 Task 6 Commit A).
///   [116] ISV_LAYOUT_FINGERPRINT_HI_INDEX — high 32 bits of layout fingerprint
///         (shifted 112→116 in Plan 4 Task 6 Commit A).
/// Written by the constructor; checked at checkpoint load. Fail-fast only — no migration
/// path exists. See spec §4.A.2 and `LAYOUT_FINGERPRINT_CURRENT` for structural-hash rationale.
/// Size of the broadcast Internal Signal Vector (ISV) bus in floats.
///
/// The ISV is the kernel↔kernel signal bus: producers write into it, consumers
/// read from it, and `read_isv_signal_at` exposes a single slot to host code.
/// New slot ranges are appended at the tail (existing indices stay stable so
/// checkpoint layouts are forward-compatible — deleted slot ranges become
/// RESERVED gaps rather than being compacted; see `LAYOUT_FINGERPRINT_CURRENT`
/// + `layout_fingerprint_seed` for the structural-hash contract).
///
/// Slot index assignments live in `crate::cuda_pipeline::sp{5,13,14,15}_isv_slots`
/// (and the C-side mirrors in `state_layout.cuh`). Bump this constant in the
/// SAME commit that adds the new range, and update `layout_fingerprint_seed`
/// to register the new slot names.
pub(crate) const ISV_TOTAL_DIM: usize = 538;  // SP22 H6 Phase 3 (2026-05-13) adds 2 slots [536..538) for the aux→policy bypass: REWARD_AUX_ALIGN_EMA_INDEX (7th reward component EMA, anchor for SP11 controller's w_aux_align) and SP22_AUX_ALIGN_SCALE_INDEX (controller-emitted adaptive scale_β consumed by β producers at training and eval). SP21 Phase 7.5 added 8 slots [528..536) for CURRICULUM_WEIGHT_{0..8} (per_insert_pa per-segment priority boost); Phase 7 added 1 slot [527..528); Phase 5+6 added 2 slots [525..527); Phase 4 added 4 slots [521..525); Phase 3 added 1 slot [520..521)
/// Legacy alias preserved for call sites that haven't been audited for the
/// network-vs-total split. New code should pick `ISV_NETWORK_DIM` (for weight
/// tensor sizing) or `ISV_TOTAL_DIM` (for the broadcast bus buffer).
const ISV_DIM: usize = ISV_NETWORK_DIM;
pub const LEARNING_HEALTH_INDEX: usize = 12;
/// Task 2.X "make Full useful": ISV slots for per-magnitude-bin Q-mean EMAs.
/// Read by `c51_loss_batched` and `c51_grad_kernel` to compute the
/// bias-vs-Full signal that drives adaptive bin weighting.
pub const Q_MAG_MEAN_QUARTER_INDEX: usize = 13;
pub const Q_MAG_MEAN_HALF_INDEX: usize = 14;
pub const Q_MAG_MEAN_FULL_INDEX: usize = 15;
/// Task 2.X "make Full useful": absolute-Q-scale reference used to
/// normalise the bias signal into a scale-invariant collapse fraction.
pub const Q_ABS_REF_INDEX: usize = 16;
/// Task 2.Y "make direction branch useful at eval": ISV slots for
/// per-direction Q-mean EMAs. Read by `c51_loss_batched` and `c51_grad_kernel`
/// to compute the bias-vs-argmax-direction signal that drives the adaptive
/// direction-branch bin weighting. Layout matches the direction action
/// encoding (Short=0, Hold=1, Long=2, Flat=3).
pub const Q_DIR_MEAN_SHORT_INDEX: usize = 17;
pub const Q_DIR_MEAN_HOLD_INDEX: usize  = 18;
pub const Q_DIR_MEAN_LONG_INDEX: usize  = 19;
pub const Q_DIR_MEAN_FLAT_INDEX: usize  = 20;
/// Task 2.Y: absolute-Q-scale reference for the direction branch (mirror of
/// `Q_ABS_REF_INDEX` for magnitude). Keeps the direction-branch
/// collapse-fraction scale-invariant as Q-values grow over training.
pub const Q_DIR_ABS_REF_INDEX: usize = 21;
/// ISV-audit bundle (2026-04-23): slot carrying host-side `training_sharpe_ema`.
/// Broadcast each epoch by `training_loop.rs` via `write_isv_signal_at`, then
/// read on-GPU by `isv_signal_update` to drive slot 12 (learning_health).
/// Couples health directly to realised training outcomes, replacing the
/// component-aggregation formula whose Sharpe-correlation measured at
/// r=-0.765 on the train-mdh86 logs that triggered the audit.
pub const SHARPE_EMA_INDEX: usize = 22;
/// ISV v-range unification bundle (spec 2026-04-23): per-branch Q-support
/// centre + half-width, 2 slots per branch (direction=0, magnitude=1,
/// order=2, urgency=3). Written by `update_eval_v_range` from the per-branch
/// Q-stats EMA state; read by the `atoms_update` GPU kernel (`launch_atoms_update`,
/// Plan 1 Task 9) and `update_per_sample_support`
/// (loss kernel projection range). Bootstrap values at construction + fold
/// reset: `v_center = 0`, `v_half = (config.v_max - config.v_min) / 2` — at
/// epoch 1 this reproduces the pre-spec `[config.v_min, config.v_max]`
/// behaviour byte-for-byte across all four branches.
pub const V_CENTER_DIR_INDEX: usize = 23;
pub const V_HALF_DIR_INDEX:   usize = 24;
pub const V_CENTER_MAG_INDEX: usize = 25;
pub const V_HALF_MAG_INDEX:   usize = 26;
pub const V_CENTER_ORD_INDEX: usize = 27;
pub const V_HALF_ORD_INDEX:   usize = 28;
pub const V_CENTER_URG_INDEX: usize = 29;
pub const V_HALF_URG_INDEX:   usize = 30;
/// Grad-balance unification bundle (2026-04-23): per-branch adaptive gradient-
/// norm targets. Four slots, one per branch (dir/mag/ord/urg). Written on-GPU
/// by `grad_balance_isv_update` via adaptive-rate EMA toward the median of the
/// observed per-branch norms each step — so every branch's target tracks the
/// median, yielding equalisation. Read by `branch_grad_rescale` to compute
/// `scale[b] = clamp(target[b] / norm[b], 1/limit, limit)`.
///
/// Replaces the pre-spec `cap = 4 × median_branch_norm` formula, which became
/// a no-op once three branches co-elevated (observed grad_ratio_mag_dir
/// 48-868× in train-5wb4n). See `feedback_isv_for_adaptive_bounds.md`.
pub const GRAD_NORM_TARGET_DIR_INDEX: usize = 31;
pub const GRAD_NORM_TARGET_MAG_INDEX: usize = 32;
pub const GRAD_NORM_TARGET_ORD_INDEX: usize = 33;
pub const GRAD_NORM_TARGET_URG_INDEX: usize = 34;
/// Grad-balance unification bundle (2026-04-23): scale-clamp limit. Single
/// slot carrying the adaptive bound `limit` such that per-branch scale is
/// clamped to `[1/limit, limit]`. The limit tracks observed
/// `max_norm / min_norm` spread via adaptive EMA, so the clamp range grows
/// just enough to correct the actual asymmetry. Hard-capped at 1e4 as a
/// numerical-safety bound (not a tuning knob). Written on-GPU by
/// `grad_balance_isv_update`; read by `branch_grad_rescale`.
pub const GRAD_SCALE_LIMIT_INDEX: usize = 35;
/// IQL branch_scales per-sample floor (2026-04-23): the IQL advantage-
/// weighted `branch_scales[b, d]` can collapse to 0 on losing branches
/// when readiness → 1 (e.g. direction Hold/Flat samples where the branch
/// advantage equals zero). That zero multiplies the C51 gradient to zero
/// for that (sample, branch) pair, starving the branch of per-sample
/// learning signal. This floor, applied inside `iql_per_branch_advantage`,
/// ensures every sample always contributes at least `isv[36]` × gradient
/// to every branch. Bootstrapped at 0.1 — a numerical-safety bound
/// (10% of uniform ≈ 0.25) that prevents per-sample starvation without
/// meaningfully suppressing the IQL advantage weighting on winners.
/// The slot is addressable for potential adaptive updates but is not
/// actively EMA-tracked; treated as a safety floor per
/// `feedback_isv_for_adaptive_bounds.md` carve-out.
pub const IQL_BRANCH_SCALE_FLOOR_INDEX: usize = 36;

/// ISV slot [39] — current epoch index. CPU writes at each epoch boundary.
/// Zero at construction; updated by the epoch loop in a follow-up task.
pub const EPOCH_IDX_INDEX: usize = 39;
/// ISV slot [40] — total epoch count for this training run. CPU writes at
/// constructor from `config`; constant for the run's lifetime.
pub const TOTAL_EPOCHS_INDEX: usize = 40;
/// ISV slot [41] — effective epsilon written by the GPU epsilon adaptive
/// kernel (follow-up task). Starts at 0; kernel fills it each step.
pub const EPSILON_EFF_INDEX: usize = 41;
/// ISV slot [42] — effective tau written by the GPU tau adaptive kernel
/// (follow-up task). Starts at 0; kernel fills it each step.
pub const TAU_EFF_INDEX: usize = 42;
// ─── D.2 Per-branch gamma (Plan 2 Task 3, spec §4.D.2) ───────────────────────
// Replaces scalar GAMMA_EFF_INDEX=43 with per-branch γ. GPU kernel
// per_branch_gamma_update reads v-range + health; writes these 4 slots.
// Consumer kernels (c51_loss, iql_value) read per-branch γ from ISV
// per-sample based on which branch the sample's action belongs to.
// Direction runs ~0.99 (trend horizon); urgency runs ~0.80 (execution).
pub const GAMMA_DIR_EFF_INDEX: usize = 43;
pub const GAMMA_MAG_EFF_INDEX: usize = 44;
pub const GAMMA_ORD_EFF_INDEX: usize = 45;
pub const GAMMA_URG_EFF_INDEX: usize = 46;

/// ISV slot [47] — effective Kelly cap written by `kelly_cap_update` GPU kernel
/// (Plan 1 Task 11). Aggregates Kelly stats from portfolio_states across all envs.
/// Result: mean half-Kelly × health-coupled safety multiplier, written each epoch.
pub const KELLY_CAP_EFF_INDEX: usize = 47;
/// ISV slot [48] — CQL pessimism base coefficient. Constructor writes
/// `config.cql_alpha`; the CQL launch formula reads from here instead of
/// the config field (Plan 1 Task 12 / spec §4.C.6). Plan 3 B.3 may later
/// make this reactive by writing an adaptive value each epoch.
pub const CQL_ALPHA_INDEX: usize = 48;
/// ISV slot [49] — plan-MLP activation threshold. Constructor writes 0.5f
/// as a cold-start value; the GPU `plan_threshold_update` kernel
/// (Plan 3 Task 4 B.4) overwrites this slot at every experience-collection
/// epoch with `max(0.1, 0.5 × ISV[READINESS_EMA_INDEX=75])`. Plan kernels
/// (`experience_kernels.cu`, `backtest_plan_kernel.cu`) read from here
/// instead of the hardcoded literal (Plan 1 Task 16 / spec §4.C.6).
/// Producer-only upgrade — consumer code paths unchanged.
pub const PLAN_THRESHOLD_INDEX: usize = 49;

/// ISV slots [50..54) — C.1 Quantile-based atom support (Plan 2 Task 1, spec §4.C.1).
/// Per-branch 5th percentile of observed Q-values via adaptive-rate EMA.
/// Producer: q_quantile_reduce GPU kernel (cold-path per-epoch).
/// Consumer: update_eval_v_range, which computes
///   `half = max(|q_p95 - v_center|, |v_center - q_p5|)`
/// clamped to [min_half_floor, abs_half].
/// Bootstrap: q_p05 = v_min (matches cold-start atom range).
pub const Q_P05_DIR_INDEX: usize = 50;
pub const Q_P05_MAG_INDEX: usize = 51;
pub const Q_P05_ORD_INDEX: usize = 52;
pub const Q_P05_URG_INDEX: usize = 53;
/// ISV slots [54..58) — C.1 Quantile-based atom support (Plan 2 Task 1, spec §4.C.1).
/// Per-branch 95th percentile of observed Q-values via adaptive-rate EMA.
/// Producer: q_quantile_reduce GPU kernel (cold-path per-epoch).
/// Bootstrap: q_p95 = v_max (matches cold-start atom range).
pub const Q_P95_DIR_INDEX: usize = 54;
pub const Q_P95_MAG_INDEX: usize = 55;
pub const Q_P95_ORD_INDEX: usize = 56;
pub const Q_P95_URG_INDEX: usize = 57;

/// ISV slot [60] — TLOB regime focus diagnostic (D.8 Plan 2 Task 6C).
///
/// Mean of max-attention-weight across the batch, EMA-updated once per epoch
/// by the CPU monitor after calling `GpuTlob::mean_max_attention_weight`.
/// Range [0, 1]; bootstrap 0.0. Written by CPU (read_isv_write_isv at epoch boundary).
/// Not network-facing (index >= ISV_NETWORK_DIM=23).
pub const TLOB_REGIME_FOCUS_EMA_INDEX: usize = 60;

// ─── C.2 Reward-component attribution (Plan 3 Task 1, spec §4.C.2) ────────────
// Per-component mean |reward| EMA. Producer: reward_component_ema GPU kernel
// (per-step, single-block, REWARD_COMPONENT_COUNT threads). Consumer:
// HEALTH_DIAG reward_split line. Component indices match
// reward_components_per_sample layout in experience_kernels.cu.

/// Number of reward components in the C.2 attribution layout.
///
/// Used by:
///   - `reward_component_ema_kernel.cu` block-dim and per-thread index gate
///   - `gpu_experience_collector::launch_reward_component_ema_inplace` block-dim
///     and Pearls A+D loop range
///   - `state_reset_registry::reset_named_state` ISV slot range
///     `[REWARD_POPART_EMA_INDEX, REWARD_POPART_EMA_INDEX + REWARD_COMPONENT_COUNT)`
///
/// Single source of truth — bumping this value requires the kernel block-dim,
/// the launcher loop, the SP4 scratch slot reservation `63..63+N`, and the
/// reset dispatch arm to all migrate together (per `feedback_no_partial_refactor.md`).
pub const REWARD_COMPONENT_COUNT: usize = 6;

/// Total number of SP4 producer scratch slots — sized to hold one f32 step
/// observation per producer kernel writing into `producer_step_scratch_buf`.
///
/// Layout (single source of truth — extending this requires the launchers,
/// the Wiener-state buffer, and the SP4 producer unit tests to co-migrate
/// per `feedback_no_partial_refactor.md`):
///   - 40 SP4 Layer A producers (Tasks A5-A9): TARGET_Q_BOUND, ATOM_POS×4,
///     WEIGHT/ADAM_M/ADAM_V/WD_RATE×8, L1_LAMBDA, GRAD_CLIP, H_S2 — slots 0..40
///   - 29 Task-A13 retrofit producers — slots 40..69
///   - 2 Layer C #260 producers (2026-05-01) — slots 69..71:
///     69 = `bw_d_h_s2_p99` (single buffer)        → ISV[171]
///     70 = `q_dir_grad_p99`  (multi-sub-buffer)   → ISV[172]
///
/// Co-located with `SP4_WIENER_FLOATS_PER_SLOT` and `SP4_WIENER_TOTAL_FLOATS`
/// so the relationship `wiener_total = count × floats_per_slot` is visible at
/// declaration time. Re-exported via `cuda_pipeline::SP4_PRODUCER_COUNT`
/// for consumption by `crates/ml/tests/sp4_producer_unit_tests.rs`.
pub const SP4_PRODUCER_COUNT: usize = 71;

/// Per-slot Wiener-state float count (`sample_var`, `diff_var`, `x_lag`).
/// Matches `WienerState`'s flat layout in `sp4_wiener_ema.rs`.
pub const SP4_WIENER_FLOATS_PER_SLOT: usize = 3;

/// Total floats in the SP4 Wiener-state buffer (one triple per producer slot).
/// Equals `SP4_PRODUCER_COUNT × SP4_WIENER_FLOATS_PER_SLOT = 213`.
pub const SP4_WIENER_TOTAL_FLOATS: usize =
    SP4_PRODUCER_COUNT * SP4_WIENER_FLOATS_PER_SLOT;

// ── SP5 Task A1 buffer layout constants ───────────────────────────────────────

/// SP5 Task A1 + Layer D Task D1+D2+D3: combined wiener_state_buf total float count.
/// SP4 had 71 producers (213 floats); SP5 contributes 123 producer triples
/// (linear span from `SP5_SLOT_BASE=174` to `SP5_SLOT_END=297`, including the
/// 2-slot carve-out gap at 278..280; 123 × 3 = 369 floats). Combined =
/// (71 + 123) × 3 = 582 floats. Pre-D1 SP5 was 110 producers / 543 floats;
/// Task D1 added 4 unique slots (286..290) growing the linear span 112 → 116
/// (buffer 543 → 561). Task D2 added 4 more slots (290..294) growing the linear
/// span 116 → 120 (buffer 561 → 573). Task D3 adds 3 more slots (294..297)
/// growing the linear span 120 → 123 (buffer 573 → 582). Note:
/// SP5_PRODUCER_COUNT is the **linear-span** constant — see its docstring
/// for why it now diverges permanently from the unique-slot count (121 post-D3).
///
/// wiener_state_buf must grow to this size at construction for ALL SP5 slots
/// (including the new D1 PnL-aggregation, D2 health-composition, and D3
/// training-metrics-EMA blocks) to be in-bounds when `apply_pearls_ad_kernel`
/// writes their Wiener triples. The Pearl 6 Kelly block (slots 280..286) and
/// the carve-out gap (278..280) have reserved-but-unused wiener storage at
/// offsets [(280-174)*3+213..(286-174)*3+213) = [531..549) — Pearl 6 does
/// not call `launch_apply_pearls` (cross-fold persistent), so those 18 floats
/// stay zero-initialised. Layer D D1 slots 286..290 use offsets
/// [(286-174)*3+213..(290-174)*3+213) = [549..561). Layer D D2 slots 290..294
/// use offsets [(290-174)*3+213..(294-174)*3+213) = [561..573). Layer D D3
/// slots 294..297 use offsets [(294-174)*3+213..(297-174)*3+213) = [573..582).
pub const SP5_WIENER_TOTAL_FLOATS: usize =
    (SP4_PRODUCER_COUNT + crate::cuda_pipeline::sp5_isv_slots::SP5_PRODUCER_COUNT)
    * SP4_WIENER_FLOATS_PER_SLOT;

/// SP5 Task A1+A2+A3+A4: producer_step_scratch_buf total slot count after growth.
/// SP4 had 71 scratch slots [0..71); Task A1 adds:
///   16 floats for q_branch_stats output (SCRATCH_Q_STATS=71, 4 outputs × 4 branches)
///   16 floats for pearl_1_atom output  (SCRATCH_ATOM_OUT=87, 4 outputs × 4 branches)
/// Task A2 adds:
///    4 floats for pearl_3_sigma output  (SCRATCH_PEARL_3_SIGMA=103, sigma × 4 branches)
///    4 floats for pearl_3_sf output     (SCRATCH_PEARL_3_SF=107, sigma_fraction × 4 branches)
/// Task A3 adds:
///    8 floats for pearl_2_budget output (2 outputs × 4 branches; SP7 retired C51/CQL/ENS slots)
///     [111..115) reserved-for-future (T6 retired SCRATCH_PEARL_2_C51)
///     budget_iqn[4]  → SCRATCH_PEARL_2_IQN=115
///     [119..127) reserved-for-future (T6 retired SCRATCH_PEARL_2_CQL/_ENS)
///     flatness[4]    → SCRATCH_PEARL_2_FLATNESS=127
/// Task A4 adds:
///    8 floats for grad_cosine_sim output  (SCRATCH_PEARL_4_COSINE=131, cos_sim × 8 groups)
///    8 floats for grad_l2_norm output     (SCRATCH_PEARL_4_L2=139, l2_norm × 8 groups)
///    8 floats for pearl_4_beta1 output    (SCRATCH_PEARL_4_BETA1=147, β1 × 8 groups)
///    8 floats for pearl_4_beta2 output    (SCRATCH_PEARL_4_BETA2=155, β2 × 8 groups)
///    8 floats for pearl_4_eps output      (SCRATCH_PEARL_4_EPS=163, ε × 8 groups)
/// Task A5 adds:
///    4 floats for q_skew_kurtosis skew[4] output  (SCRATCH_PEARL_5_SKEW=171, skew × 4 branches)
///    4 floats for q_skew_kurtosis ex_kurt[4] output (SCRATCH_PEARL_5_EX_KURT=175, ex_kurt × 4 branches)
///   20 floats for pearl_5_iqn_tau τ output   (SCRATCH_PEARL_5_TAU=179, [4 branches × 5 quantiles] at [179..199))
/// Task A7 adds:
///    4 floats for pearl_8_trail output   (SCRATCH_PEARL_8_TRAIL=199, trail_dist × 4 directions at [199..203))
/// Task A8 adds:
///    4 floats for pearl_1_ext_num_atoms output  (SCRATCH_PEARL_1_EXT_NUM_ATOMS=203, num_atoms × 4 branches at [203..207))
/// Layer D Task D1 adds:
///    4 floats for pnl_aggregation output         (SCRATCH_PNL_AGG_BASE=207, total/mean/var/max_dd at [207..211))
/// Layer D Task D2 adds:
///    4 floats for health_composition output      (SCRATCH_HEALTH_COMP_BASE=211, score/q_gap_norm/q_var_norm/grad_norm_norm at [211..215))
/// Layer D Task D3 adds:
///    3 floats for training_metrics_ema output    (SCRATCH_TRAINING_METRICS_EMA_BASE=215, training_sharpe_ema/max_dd_ema/low_dd_ratio at [215..218))
/// SP7 Task 5 adds:
///    4 floats for LB new_budget_cql[4]     (SCRATCH_LB_BUDGET_CQL=218,  at [218..222))
///    4 floats for LB new_budget_c51[4]     (SCRATCH_LB_BUDGET_C51=222,  at [222..226))
///    4 floats for LB diff_var_cql[4]       (SCRATCH_LB_DIFF_VAR_CQL=226, at [226..230))
///    4 floats for LB sample_var_cql[4]     (SCRATCH_LB_SAMPLE_VAR_CQL=230, at [230..234))
///    4 floats for LB diff_var_c51[4]       (SCRATCH_LB_DIFF_VAR_C51=234, at [234..238))
///    4 floats for LB sample_var_c51[4]     (SCRATCH_LB_SAMPLE_VAR_C51=238, at [238..242))
/// SP7 activation-flag fix (2026-05-03) adds:
///    4 floats for LB cql_active[4]         (SCRATCH_LB_ACTIVE_CQL=242, at [242..246))
///    4 floats for LB c51_active[4]         (SCRATCH_LB_ACTIVE_C51=246, at [246..250))
/// SP8 (Fix 36, 2026-05-03) adds:
///    1 float  for train_active_frac        (SCRATCH_TRAIN_ACTIVE_FRAC=250, at [250..251))
///    4 floats for LB max_budget_cql[4]     (SCRATCH_LB_MAX_BUDGET_CQL=251, at [251..255))
///    4 floats for LB max_budget_c51[4]     (SCRATCH_LB_MAX_BUDGET_C51=255, at [255..259))
/// SP9 (Fix 37, 2026-05-03) adds:
///    1 float  for kelly_warmup_floor       (SCRATCH_SP9_KELLY_WARMUP_FLOOR=259, at [259..260))
///    1 float  for q_var_mag_ema            (SCRATCH_SP9_Q_VAR_MAG_EMA=260, at [260..261))
///    1 float  for intent_eval_divergence   (SCRATCH_SP9_INTENT_EVAL_DIVERGENCE=261, at [261..262))
///    3 floats for eval_dist[Q,H,F]         (SCRATCH_SP9_EVAL_DIST_BASE=262, at [262..265))
/// Fix 37.1 (2026-05-03): the 3 EMA target updaters that previously sat
/// at [262..265) (sample_count_target / divergence_target / temporal_target)
/// are deleted — their target ISV slots are constructor-written
/// Invariant-1 anchors per `feedback_isv_for_adaptive_bounds.md` and
/// require no scratch storage. The eval_dist base slides 265 → 262.
/// SP10 (Fix 38, 2026-05-03) adds:
///    1 float  for eval Thompson temp       (SCRATCH_SP10_THOMPSON_TEMP=265, at [265..266))
/// SP11 (Fix 39, 2026-05-04, A1) adds:
///    2 floats for val_sharpe_delta_compute      (SCRATCH_SP11_VAL_SHARPE_DELTA_BASE=266, [266..268))
///    1 float  for saboteur_engagement_compute   (SCRATCH_SP11_SABOTEUR_ENGAGEMENT=268, [268..269))
///    7 floats for reward_component_mag_ratio    (SCRATCH_SP11_MAG_RATIO_BASE=269, [269..276))
///        First 6 = component ratios → ISV[REWARD_COMPONENT_MAG_RATIO_BASE..+6);
///        7th    = popart EMA mirror → ISV[PNL_REWARD_MAGNITUDE_EMA_INDEX=359].
/// SP11 (Fix 39, 2026-05-04, A2) adds:
///   10 floats for reward_subsystem_controller   (SCRATCH_SP11_CONTROLLER_BASE=276, [276..286))
///        Layout matches ISV[340..350) one-for-one:
///          [276..282) = 6 component weights → ISV[340..346)
///          [282]      = curiosity_pressure   → ISV[346]
///          [283]      = saboteur_intensity   → ISV[347]
///          [284]      = reward_weight_floor  → ISV[348]
///          [285]      = curiosity_bound      → ISV[349]
///        Smoothed by `apply_pearls_ad_kernel` per spec §3.4.1 (Pearls A+D
///        applied to controller outputs). Slots 276..282 are contiguous in
///        scratch; the chained Pearls launch can cover them with `n_slots=6`
///        in one shot. Slots 282..286 each get a singleton Pearls launch
///        because the corresponding ISV slots [346..350) are also 4
///        contiguous singletons in the absolute ISV layout — actually they
///        ARE contiguous, but split into 1+1+1+1 launches in the launcher
///        body for explicit per-output traceability with the SP11 spec
///        table; functionally equivalent to a single n_slots=4 launch.
///        See `launch_sp11_reward_subsystem_controller` for the dispatch.
/// SP11 (Fix 39, 2026-05-04, B1b fix-up) adds:
///    1 float for popart_component_ema      (SCRATCH_SP11_POPART_COMPONENT_EMA=286, [286..287))
///         → ISV[POPART_COMPONENT_MAG_EMA_INDEX=360] via chained Pearls A+D singleton
/// SP11 (Fix 39, 2026-05-04, B1b smoke-recovery) adds:
///    1 float for popart_component_var      (SCRATCH_SP11_POPART_COMPONENT_VAR=287, [287..288))
///         → ISV[REWARD_COMPONENT_VAR_EMA_BASE=361] via chained Pearls A+D
///         (paired with the popart magnitude EMA to enable z-score normalisation
///          in the mag-ratio canary). Computed alongside the magnitude in the
///          extended `popart_component_ema_kernel` via Welford's online algorithm
///          (single-pass two-block-tree-reduce: first pass produces `mean(|x|)`,
///          second pass produces `var(|x|)` against that mean).
///    5 floats for reward_component_var     (SCRATCH_SP11_REWARD_COMPONENT_VAR_BASE=288, [288..293))
///         → ISV[REWARD_COMPONENT_VAR_EMA_BASE+1..+6) = ISV[362..367) via 5 chained
///         Pearls A+D singleton launches. Layout: [288]=cf var, [289]=trail var,
///         [290]=micro var, [291]=opp_cost var, [292]=bonus var. Computed by the
///         extended `reward_component_ema_kernel` via Welford's online algorithm
///         per non-popart component (component 0 / popart variance is sourced
///         from the `popart_component_ema_kernel` chain because slot 63 is the
///         pre-SP11 invariant total-reward magnitude EMA, not the popart-component
///         magnitude). Spec §4 amendment "Why z-score" (lines 564-619).
/// Combined: 259 + 6 + 1 + 10 + 10 + 1 + 1 + 5 = 293 scratch slots [0..293).
pub const SP5_SCRATCH_TOTAL: usize = 293;

/// SP5 Layer D Task D1 (rewrite, 2026-05-02): scratch index base for
/// `pnl_aggregation_update`.
/// Slots [207..211): aggregated PnL summary outputs in fixed order:
///   [207] = pnl_total  (log-space compounded total return:
///                       `exp(Σ ln(max(1+step_returns[i], 1e-10))) − 1`)
///   [208] = pnl_mean   (per-trade mean: `sum_returns / n_trades`)
///   [209] = pnl_var    (per-trade population variance:
///                       `max(0, sum_sq_returns/n_trades − mean²)`)
///   [210] = pnl_max_dd (per-bar equity-curve max drawdown over the last
///                       10K bars with episode-boundary resets, capped at 1.0)
/// Written by `pnl_aggregation_update`; consumed by `apply_pearls_ad_kernel` →
/// ISV[PNL_TOTAL_INDEX=286..PNL_MAX_DD_INDEX+1=290). Formula provenance:
/// `compute_epoch_financials` in `crates/ml/src/trainers/dqn/financials.rs`.
pub const SCRATCH_PNL_AGG_BASE: usize = 207;

/// SP5 Layer D Task D2 (2026-05-02): scratch index base for `health_composition_update`.
/// Slots [211..215): LearningHealth composition outputs in fixed order:
///   [211] = health_score    (composed Layer-1 score in [0,1])
///   [212] = q_gap_norm      (smoothstep(0.01, 0.5, q_gap_ema) ∈ [0,1])
///   [213] = q_var_norm      (smoothstep(0.001, 0.1, q_var_ema) ∈ [0,1])
///   [214] = grad_norm_norm  (1 − smoothstep(10, 100, grad_norm_ema) ∈ [0,1]
///                            — `grad_stable` in `learning_health.rs`)
/// Written by `health_composition_update`; consumed by `apply_pearls_ad_kernel`
/// → ISV[HEALTH_SCORE_INDEX=290..GRAD_NORM_NORM_INDEX+1=294). The other 4
/// normalised intermediates (`atom_util_norm`, `ens_agree`,
/// `grad_consistency_norm`, `spectral_gap_norm`) are computed in-register
/// inside the kernel, fed into the composed score, and discarded — they
/// can be promoted to scratch slots in a future Layer D extension without
/// a launcher signature change.
pub const SCRATCH_HEALTH_COMP_BASE: usize = 211;

/// SP5 Layer D Task D3 (2026-05-02): scratch index base for `training_metrics_ema_update`.
/// Slots [215..218): training-loop EMA outputs in fixed order:
///   [215] = training_sharpe_ema (adaptive α `clamp(0.05+0.3·|err|, 0, 0.5)`,
///                                host-tracked `_initialized` sentinel)
///   [216] = max_dd_ema           (fixed α=0.1, sentinel `prev<1e-12` ⇒
///                                 first-observation replacement)
///   [217] = low_dd_ratio         (fixed α=0.15 over binary
///                                 `is_low = (max_dd < new_max_dd_ema·0.5)
///                                 ? 1 : 0`; reads tid=1's *new* output via
///                                 __syncthreads(); no host-side sentinel,
///                                 constructor zero behaves correctly under
///                                 the EMA recurrence)
/// Written by `training_metrics_ema_update`; consumed by `apply_pearls_ad_kernel`
/// → ISV[TRAINING_SHARPE_EMA_INDEX=294..LOW_DD_RATIO_INDEX+1=297). All three
/// host-side EMA constants (0.05/0.3/0.5 adaptive bounds; 0.1; 0.15; 1e-12
/// sentinel) are Invariant 1 anchors per `feedback_no_quickfixes.md`,
/// migrated verbatim from `training_loop.rs:5039-5113`. Per
/// `feedback_trust_code_not_docs.md`: the third metric is `low_dd_ratio`
/// (not `gamma_blend` as the SP5 plan §D Task D3 hints — that field does
/// not exist in the codebase; the migration follows the actual host-side
/// EMA at `training_loop.rs:5052`).
pub const SCRATCH_TRAINING_METRICS_EMA_BASE: usize = 215;

/// SP7 Task 5 (2026-05-03): scratch index base for loss-balance controller
/// new_budget_cql[4]. Smoothed by apply_pearls_ad_kernel into
/// ISV[BUDGET_CQL_BASE..+4).
pub const SCRATCH_LB_BUDGET_CQL: usize = 218;     // 218..222
/// SP7: new_budget_c51[4] → ISV[BUDGET_C51_BASE..+4).
pub const SCRATCH_LB_BUDGET_C51: usize = SCRATCH_LB_BUDGET_CQL + 4;   // 222..226
/// SP7: diff_var_cql[4] → ISV[LB_DIFF_VAR_CQL_BASE..+4).
pub const SCRATCH_LB_DIFF_VAR_CQL: usize = SCRATCH_LB_BUDGET_CQL + 8;  // 226..230
/// SP7: sample_var_cql[4] → ISV[LB_SAMPLE_VAR_CQL_BASE..+4).
pub const SCRATCH_LB_SAMPLE_VAR_CQL: usize = SCRATCH_LB_BUDGET_CQL + 12; // 230..234
/// SP7: diff_var_c51[4] → ISV[LB_DIFF_VAR_C51_BASE..+4).
pub const SCRATCH_LB_DIFF_VAR_C51: usize = SCRATCH_LB_BUDGET_CQL + 16;  // 234..238
/// SP7: sample_var_c51[4] → ISV[LB_SAMPLE_VAR_C51_BASE..+4).
pub const SCRATCH_LB_SAMPLE_VAR_C51: usize = SCRATCH_LB_BUDGET_CQL + 20; // 238..242
/// SP7 activation-flag fix (2026-05-03): cql_active[4] → ISV[LB_CQL_ACTIVE_BASE..+4).
/// Kernel writes 1.0 in the active path and when prior was_active=1, 0.0 in the
/// genuine cold-start branch. apply_pearls_ad_kernel's `step_obs == 0.0` short-
/// circuit means writing 0.0 is a no-op (slot stays at its current value);
/// the activation flag is therefore monotonic per fold.
pub const SCRATCH_LB_ACTIVE_CQL: usize = SCRATCH_LB_BUDGET_CQL + 24;     // 242..246
/// SP7 activation-flag fix: c51_active[4] → ISV[LB_C51_ACTIVE_BASE..+4).
pub const SCRATCH_LB_ACTIVE_C51: usize = SCRATCH_LB_BUDGET_CQL + 28;     // 246..250

/// SP8 (Fix 36, 2026-05-03): scratch slot for train_active_frac canary
/// produced by `train_active_frac_compute_kernel`. Single float; downstream
/// `apply_pearls_ad_kernel` smooths into ISV[TRAIN_ACTIVE_FRAC_INDEX=321].
pub const SCRATCH_TRAIN_ACTIVE_FRAC: usize = 250;        // 250..251

/// SP8 (Fix 36, 2026-05-03): scratch base for per-(head, branch) MAX_BUDGET
/// cap produced by `loss_balance_max_budget_compute_kernel`. 4 floats per
/// head; downstream `apply_pearls_ad_kernel` smooths into
/// ISV[LB_MAX_BUDGET_CQL_BASE..+4) and ISV[LB_MAX_BUDGET_C51_BASE..+4).
pub const SCRATCH_LB_MAX_BUDGET_CQL: usize = 251;        // 251..255
pub const SCRATCH_LB_MAX_BUDGET_C51: usize = 255;        // 255..259

/// SP9 (Fix 37, 2026-05-03): scratch slots for Kelly cold-start warmup
/// floor + intent/eval divergence + eval_dist GPU migration. Each
/// producer kernel writes to its scratch slot; downstream
/// `apply_pearls_ad_kernel` smooths into the corresponding ISV slot via
/// Pearl A sentinel-bootstrap + Pearl D Wiener-α steady-state smoothing.
///
/// Fix 37.1 (2026-05-03): the 3 EMA target updater scratch slots
/// (sample_count_target / divergence_target / temporal_target) are
/// deleted — their target ISV slots are constructor-written
/// Invariant-1 anchors per `feedback_isv_for_adaptive_bounds.md` and
/// require no scratch storage. The eval_dist base slides 265 → 262.
pub const SCRATCH_SP9_KELLY_WARMUP_FLOOR:      usize = 259;  // → ISV[KELLY_WARMUP_FLOOR_INDEX=330]
pub const SCRATCH_SP9_Q_VAR_MAG_EMA:           usize = 260;  // → ISV[Q_VAR_MAG_EMA_INDEX=331]
pub const SCRATCH_SP9_INTENT_EVAL_DIVERGENCE:  usize = 261;  // → ISV[INTENT_EVAL_DIVERGENCE_INDEX=332]
pub const SCRATCH_SP9_EVAL_DIST_BASE:          usize = 262;  // → ISV[EVAL_DIST_Q/H/F_INDEX=336..339)

/// SP10 (Fix 38, 2026-05-03): scratch slot for the eval Thompson selector
/// temperature side-output of `intent_eval_divergence_compute_kernel`. Single
/// float; downstream `apply_pearls_ad_kernel` smooths into
/// ISV[EVAL_THOMPSON_TEMP_INDEX=339].
pub const SCRATCH_SP10_THOMPSON_TEMP:          usize = 265;  // → ISV[EVAL_THOMPSON_TEMP_INDEX=339]

/// SP11 (Fix 39, 2026-05-04, Task A1): scratch slots for the three canary
/// producers in the reward-as-controlled-subsystem chain. Each producer
/// writes to its scratch block; downstream `apply_pearls_ad_kernel`
/// smooths into the corresponding ISV slots in [350..360) via Pearl A
/// sentinel-bootstrap + Pearl D Wiener-α steady-state smoothing.
///
/// Spec: docs/superpowers/specs/2026-05-04-sp11-reward-as-controlled-subsystem.md
/// Plan: docs/superpowers/plans/2026-05-04-sp11-reward-as-controlled-subsystem.md
/// (Task A1 Steps 1-13).
pub const SCRATCH_SP11_VAL_SHARPE_DELTA_BASE:  usize = 266;  // [266..268) → ISV[VAL_SHARPE_DELTA_EMA_INDEX=350, VAL_SHARPE_VAR_EMA_INDEX=351]
pub const SCRATCH_SP11_SABOTEUR_ENGAGEMENT:    usize = 268;  // [268..269) → ISV[SABOTEUR_ENGAGEMENT_RATE_INDEX=358]
pub const SCRATCH_SP11_MAG_RATIO_BASE:         usize = 269;  // [269..276) — first 6 → ISV[REWARD_COMPONENT_MAG_RATIO_BASE..+6); 7th → ISV[PNL_REWARD_MAGNITUDE_EMA_INDEX=359]

/// SP11 (Fix 39, 2026-05-04, Task A2): scratch slots for the
/// `reward_subsystem_controller` producer. 10 contiguous floats
/// at [276..286) match the ISV[340..350) layout one-for-one:
///   [276..282) = 6 component weights (popart, cf, trail, micro,
///                opp_cost, bonus) → ISV[REWARD_*_WEIGHT_INDEX..+6)
///   [282]      = curiosity_pressure        → ISV[CURIOSITY_PRESSURE_INDEX=346]
///   [283]      = saboteur_intensity_mult   → ISV[SABOTEUR_INTENSITY_MULT_INDEX=347]
///   [284]      = reward_weight_floor       → ISV[REWARD_WEIGHT_FLOOR_INDEX=348]
///   [285]      = curiosity_bound           → ISV[CURIOSITY_BOUND_INDEX=349]
///
/// The chained `apply_pearls_ad_kernel` (Pearls A+D output smoothing per
/// spec §3.4.1) reads the entire 10-slot block in one launch with
/// `n_slots=10` since both scratch and ISV layouts are contiguous in
/// matched-stride order.
pub const SCRATCH_SP11_CONTROLLER_BASE:        usize = 276;  // [276..286) → ISV[340..350)

/// SP11 (Fix 39, 2026-05-04, B1b fix-up): scratch slot for the
/// `popart_component_ema` producer's mean(|x|) output. Single float at
/// [286..287) → ISV[POPART_COMPONENT_MAG_EMA_INDEX=360] via chained
/// Pearls A+D singleton launch. Resolves the slot-63 PopArt-input vs
/// popart-component-magnitude overload that B1b smoke surfaced.
pub const SCRATCH_SP11_POPART_COMPONENT_EMA:   usize = 286;  // [286..287) → ISV[POPART_COMPONENT_MAG_EMA_INDEX=360]

/// SP11 (Fix 39, 2026-05-04, B1b smoke-recovery): scratch slot for the
/// `popart_component_ema` producer's var(|x|) Welford output. Single
/// float at [287..288) → ISV[REWARD_COMPONENT_VAR_EMA_BASE=361] via
/// chained Pearls A+D singleton launch. Computed in the same single-
/// block kernel as the magnitude: a two-pass tree-reduce (mean → var)
/// against the just-computed mean. Pairs with the magnitude EMA to
/// drive z-score normalisation in `reward_component_mag_ratio_compute_kernel`.
pub const SCRATCH_SP11_POPART_COMPONENT_VAR:   usize = 287;  // [287..288) → ISV[REWARD_COMPONENT_VAR_EMA_BASE=361]

/// SP11 (Fix 39, 2026-05-04, B1b smoke-recovery): scratch base for the
/// 5 non-popart per-reward-component variance EMAs computed by the
/// extended `reward_component_ema_kernel`. Layout (matching component
/// indices c=1..5 in `reward_components_per_sample`):
///   [288] = cf       variance → ISV[REWARD_COMPONENT_VAR_EMA_BASE+1=362]
///   [289] = trail    variance → ISV[REWARD_COMPONENT_VAR_EMA_BASE+2=363]
///   [290] = micro    variance → ISV[REWARD_COMPONENT_VAR_EMA_BASE+3=364]
///   [291] = opp_cost variance → ISV[REWARD_COMPONENT_VAR_EMA_BASE+4=365]
///   [292] = bonus    variance → ISV[REWARD_COMPONENT_VAR_EMA_BASE+5=366]
///
/// c=0 (popart) variance lives at SCRATCH_SP11_POPART_COMPONENT_VAR
/// (slot 287) and is sourced from `popart_component_ema_kernel` because
/// `reward_components_per_sample[+0]` feeds slot 63 = pre-SP11
/// invariant total-reward magnitude EMA, NOT the popart-component
/// magnitude (see B1b fix-up rationale in `popart_component_ema_kernel.cu`).
/// Each variance is computed via two-pass Welford: first reduce produces
/// `mean(|r_c|)`, second reduce produces `var(|r_c|)` against the just-
/// computed mean. Smoothed per-slot by chained Pearls A+D singletons
/// (5 independent launches because the destination ISV slots 362..367
/// are contiguous but the sources occupy a contiguous scratch block —
/// the actual chained launcher uses `n_slots=5` in one shot since both
/// scratch and ISV layouts are matched-stride contiguous).
pub const SCRATCH_SP11_REWARD_COMPONENT_VAR_BASE: usize = 288;  // [288..293) → ISV[362..367)

/// SP5 Task A7: scratch index base for pearl_8_trail_update trail_dist[4] output block.
/// Slots [199..203): per-direction trail-stop distance (Short=0, Hold=1, Long=2, Flat=3).
/// Written by `pearl_8_trail_update`; consumed by apply_pearls_ad_kernel →
/// ISV[TRAIL_DIST_PER_DIR_BASE=270..274).
pub const SCRATCH_PEARL_8_TRAIL: usize = 199;

/// SP5 Task A8: scratch index base for pearl_1_ext_num_atoms_update num_atoms[4] output block.
/// Slots [203..207): per-branch C51 num_atoms discrete value {64, 32, 16} (as f32).
/// Written by `pearl_1_ext_num_atoms_update`; consumed by apply_pearls_ad_kernel →
/// ISV[ATOM_NUM_ATOMS_BASE=274..278). Pearls A+D smooths the discrete output during
/// transitions; Layer B's atoms_update consumer rounds to nearest valid count at consume time.
pub const SCRATCH_PEARL_1_EXT_NUM_ATOMS: usize = 203;

/// SP5 Task A5: scratch index base for q_skew_kurtosis skew[4] output block.
/// Slots [171..175): per-branch Q-distribution standardized skewness.
/// Written by `q_skew_kurtosis_update`; consumed by `pearl_5_iqn_tau_update`.
pub const SCRATCH_PEARL_5_SKEW: usize = 171;

/// SP5 Task A5: scratch index base for q_skew_kurtosis ex_kurt[4] output block.
/// Slots [175..179): per-branch Q-distribution excess kurtosis (clamped [-3, +30]).
/// Written by `q_skew_kurtosis_update`. Reserved for future heavy-tail-aware
/// τ schedule extension; not consumed by pearl_5_iqn_tau_update in this commit.
pub const SCRATCH_PEARL_5_EX_KURT: usize = 175;

/// SP5 Task A5: scratch index base for pearl_5_iqn_tau τ schedule output block.
/// Slots [179..199): per-branch × per-quantile τ values (4 branches × 5 quantiles).
/// Layout: branch b, quantile q → scratch[179 + b*5 + q].
/// Written by `pearl_5_iqn_tau_update`; consumed by apply_pearls_ad_kernel →
/// ISV[IQN_TAU_BASE=250..270).
pub const SCRATCH_PEARL_5_TAU: usize = 179;

/// SP5 Task A2: scratch index base for pearl_3_sigma sigma[4] output block.
/// Slots [103..107): per-branch NoisyNet sigma.
pub const SCRATCH_PEARL_3_SIGMA: usize = 103;

/// SP5 Task A2: scratch index base for pearl_3_sigma sigma_fraction[4] output block.
/// Slots [107..111): per-branch sigma fraction (entropy-deficit controller state).
pub const SCRATCH_PEARL_3_SF: usize = 107;

/// SP5 Task A3 + SP7 (2026-05-03): scratch index base for pearl_2_budget
/// budget_iqn[4] output block. C51/CQL/ENS scratch slots were retired in
/// SP7 — the loss-balance controller owns those budgets directly.
/// Producer scratch slots 111..115, 119..127 are now reserved-for-future.
const SCRATCH_PEARL_2_IQN: usize = 115;       // 115..119
/// SP5 Task A3: scratch index base for pearl_2_budget flatness[4] output
/// block.
const SCRATCH_PEARL_2_FLATNESS: usize = 127;  // 127..131

/// SP5 Task A4: scratch index base for grad_cosine_sim_kernel cosine_sim[8] output block.
/// Slots [131..139): per-param-group gradient direction cosine similarity vs previous step.
/// Written by `grad_cosine_sim_update`; consumed by `pearl_4_adam_hparams_update`.
pub const SCRATCH_PEARL_4_COSINE: usize = 131;

/// SP5 Task A4: scratch index base for grad_cosine_sim_kernel l2_norm[8] output block.
/// Slots [139..147): per-param-group L2 norm of current-step gradient.
/// Written by `grad_cosine_sim_update`; consumed by `pearl_4_adam_hparams_update`.
pub const SCRATCH_PEARL_4_L2: usize = 139;

/// SP5 Task A4: scratch index base for pearl_4_adam_hparams_kernel β1[8] output block.
/// Slots [147..155): per-param-group adaptive Adam β1 ∈ [0.85, 0.95].
/// Written by `pearl_4_adam_hparams_update`; consumed by apply_pearls_ad_kernel →
/// ISV[ADAM_BETA1_BASE..ADAM_BETA1_BASE+8).
pub const SCRATCH_PEARL_4_BETA1: usize = 147;

/// SP5 Task A4: scratch index base for pearl_4_adam_hparams_kernel β2[8] output block.
/// Slots [155..163): per-param-group adaptive Adam β2 ∈ [0.99, 0.9995].
/// Written by `pearl_4_adam_hparams_update`; consumed by apply_pearls_ad_kernel →
/// ISV[ADAM_BETA2_BASE..ADAM_BETA2_BASE+8).
pub const SCRATCH_PEARL_4_BETA2: usize = 155;

/// SP5 Task A4: scratch index base for pearl_4_adam_hparams_kernel ε[8] output block.
/// Slots [163..171): per-param-group adaptive Adam ε ∈ [1e-10, 1e-6].
/// Written by `pearl_4_adam_hparams_update`; consumed by apply_pearls_ad_kernel →
/// ISV[ADAM_EPS_BASE..ADAM_EPS_BASE+8).
pub const SCRATCH_PEARL_4_EPS: usize = 163;

/// SP5 Task A1: scratch index base for q_branch_stats 16-float output block.
/// Slots [71..87): mean, max_abs_dev, var, entropy × 4 branches.
/// SP4 slot 70 = q_dir_grad_p99; SP5 starts at 71 (no collision).
pub const SCRATCH_Q_STATS: usize = 71;

/// SP5 Task A1: scratch index base for pearl_1_atom 16-float output block.
/// Slots [87..103): v_center, v_half, headroom, clip_rate × 4 branches.
pub const SCRATCH_ATOM_OUT: usize = 87;

/// ISV slot [63] — mean |popart reward| EMA (final on-policy reward, pre-PopArt).
pub const REWARD_POPART_EMA_INDEX:   usize = 63;
/// ISV slot [64] — mean |counterfactual reward| EMA.
pub const REWARD_CF_EMA_INDEX:       usize = 64;
/// ISV slot [65] — mean |trail penalty| EMA (0.0 until Plan 3 B.1).
pub const REWARD_TRAIL_EMA_INDEX:    usize = 65;
/// ISV slot [66] — mean |micro reward| EMA (dense OFI signal, R5).
pub const REWARD_MICRO_EMA_INDEX:    usize = 66;
/// ISV slot [67] — mean |opp_cost reward| EMA (0.0 until Plan 3 B.1).
pub const REWARD_OPP_COST_EMA_INDEX: usize = 67;
/// ISV slot [68] — mean |trade-attempt/timing bonus| EMA (populated from Plan 3 B.2 rc[5]).
pub const REWARD_BONUS_EMA_INDEX:    usize = 68;

// ─── B.2 ISV-driven trade-attempt bonus (Plan 3 Task 3) ──────────────────────
/// ISV slot [71] — Flat→Positioned transition rate EMA.
/// GPU-written by `trade_attempt_rate_ema_update` kernel (single-block, per-epoch).
/// Adaptive EMA rate: `alpha = alpha_base * (1 + 0.5 * |clamp(sharpe, -2, 2)|)` —
/// higher-Sharpe epochs update the estimate faster. Consumer:
/// `experience_env_step` at the Flat→Positioned branch computes
/// `novelty = max(0, 1 - attempt/target)` and adds a bonus reward
/// `bonus = conviction * vol_proxy * novelty` to both the step reward and rc[5].
pub const TRADE_ATTEMPT_RATE_EMA_INDEX: usize = 71;
/// ISV slot [72] — reference rate for novelty computation (Plan 3 Task 3 B.2).
/// CPU-frozen at epoch 5 from the measured `TRADE_ATTEMPT_RATE_EMA` (floored at
/// 0.001 so novelty can't stick at 1.0). Pre-freeze the slot is 0.0 and the
/// bonus site in `experience_env_step` gates on `target_raw > 1e-6f`, so the
/// reward term is structurally inert until the freeze fires — no spurious
/// saturation from an unset target.
pub const TRADE_TARGET_RATE_INDEX: usize = 72;

/// ISV slot [75] — Plan 3 Task 4 B.4: per-batch mean readiness EMA.
/// Adaptive α = α_base × (1 + 0.5 × |clamp(sharpe, −2, 2)|). Consumer:
/// `plan_threshold_update` GPU kernel derives a threshold from this EMA
/// and writes it into PLAN_THRESHOLD_INDEX=49. Not a new threshold —
/// upgrades the producer of slot 49 from static constructor write to
/// GPU-driven adaptive.
pub const READINESS_EMA_INDEX: usize = 75;

/// ISV slot [78] — Plan 3 Task 7 C.3: per-epoch mean moment-match KL between
/// train-state sample and val-state batch (Gaussian per-dim KL, summed over
/// monitored OFI dims). Higher value = more train/val divergence. GPU-written
/// by `state_kl_moment_match`. Adaptive α = α_base × (1 + 0.5 × |clamp(sharpe,
/// −2, 2)|), α_base=0.05.
pub const STATE_KL_TRAIN_VAL_EMA_INDEX: usize = 78;
/// ISV slot [79] — Plan 3 Task 7 C.3: amplification multiplier ∈ [1, 2] for
/// Flat-trap escape rewards (B.1 opp_cost + B.2 bonus). Smoothly tracks
/// `1 + clamp(KL_ratio − 1, 0, 1)` where `KL_ratio = current_ema /
/// trailing_ema` (kernel-internal trailing-self pattern, no separate
/// threshold slot). Consumer multipliers in `experience_kernels.cu` use
/// `fmaxf(1.0, …)` to no-op against cold-start 0.
pub const STATE_KL_AMPLIFICATION_INDEX: usize = 79;

/// ISV slot [82] — Plan 3 Task 8 B.3: target number of seed-phase
/// experience-collection steps. CPU-written at constructor from
/// `config.replay_seed_steps` (default 100k). SchemaContract — never reset
/// after construction.
pub const SEED_STEPS_TARGET_INDEX: usize = 82;
/// ISV slot [83] — Plan 3 Task 8 B.3: cumulative seed-phase steps completed.
/// GPU-incremented by `seed_step_counter_update` per `collect_experiences_gpu`
/// call while DONE < TARGET. Capped at TARGET. Reset to 0 at fold boundary
/// (FoldReset).
pub const SEED_STEPS_DONE_INDEX: usize = 83;
/// ISV slot [84] — Plan 3 Task 8 B.3: seed-fraction adaptive EMA of
/// `max(0, 1 - DONE/TARGET)`. Cold-start 1.0 (fully in seed phase),
/// smoothly decays toward 0 as DONE approaches TARGET. Consumed by Task 9
/// CQL α ramp kernel. FoldReset (re-bootstrap to 1.0 each fold).
pub const SEED_FRAC_EMA_INDEX: usize = 84;

/// ─── E.5 Attention-focus interpretability (Plan 4 Task 5 Mode A) ────────────
///
/// Mode A is ISV-diagnostic-only. The scalars feeding these slots already
/// exist (vsn_mag/vsn_dir from `per_branch_vsn_mean()`; mamba2_retention is
/// the per-batch mean |mamba2_h_enriched| computed host-side via dtoh, mirror
/// of `per_branch_vsn_mean`'s cold-path pattern). Mode B (post-E.1 full VSN)
/// will tail-append 4 more per-feature-group slots; not in scope for this task.
/// ISV slot [87] — magnitude-branch value-head/VSN weight magnitude EMA.
pub const VSN_MAG_EMA_INDEX: usize = 87;
/// ISV slot [88] — direction-branch value-head/VSN weight magnitude EMA.
pub const VSN_DIR_EMA_INDEX: usize = 88;
/// ISV slot [89] — Mamba2 state-transition magnitude EMA (retention proxy).
pub const MAMBA2_RETENTION_EMA_INDEX: usize = 89;

// ─── Plan 4 follow-up: target-drift EMA (replaces legacy per_branch_target_drift) ───
/// ISV slot [92] — RMS(target_params − online_params) for the magnitude branch,
/// EMA-tracked. Replaces the legacy CPU-DtoH `per_branch_target_drift()` method
/// per `pearl_cold_path_no_exception_to_gpu_drives.md`. Written by
/// `target_drift_ema_update` GPU kernel (2-block multi-reduction). Diagnostic
/// only — surfaced in HEALTH_DIAG `noisy [drift_mag=…]`.
pub const TARGET_DRIFT_MAG_EMA_INDEX: usize = 92;
/// ISV slot [93] — same producer/contract as [92]; direction branch.
pub const TARGET_DRIFT_DIR_EMA_INDEX: usize = 93;

/// ISV slot [96] — Plan 4 Task 2c.3c.5: EMA of RMS(save_h_s2) per batch.
/// Producer: `h_s2_rms_ema_update` GPU kernel (single-block, 256 threads,
/// shmem-reduction, no atomicAdd). Cold-path-cadence — launched once per
/// experience-collection epoch alongside the other Plan 3/Plan 4 ISV producers
/// in `training_loop.rs`. EMA α = 0.05 (≈ 13-batch half-life). Cold-start
/// 1.0 (neutral RMS so first kernel fire EMAs measured RMS toward 1.0
/// rather than 0). FoldReset → 1.0.
///
/// Producer-only in this commit. 2c.3c.6 wires the consumer in
/// `mag_concat_qdir`'s adaptive-scale path (residual stack normalised by
/// the trunk-output RMS so the magnitude-branch decoder sees a scale-
/// invariant input regardless of the GRN trunk's drift).
pub const H_S2_RMS_EMA_INDEX: usize = 96;

/// ISV slots [99..103) — Plan 4 Task 3 (E.3): IQN multi-quantile diagnostic
/// EMAs for the four off-median fixed quantiles. The median (τ=0.50) is
/// already exposed via the existing greedy-Q diagnostics (no duplication).
///
/// Producer: `iqn_quantile_ema_update` GPU kernel (single-block, 256
/// threads, shmem-reduce, no atomicAdd). Reads
/// `GpuIqnHead::save_q_online [TBA, B*Q]` (col-major; TBA = sum of branch
/// sizes, Q = `IQN_NUM_QUANTILES = 5`) and EMA-updates each of the four
/// non-median quantile slices (τ ∈ {0.05, 0.25, 0.75, 0.95} → indices
/// {0, 1, 3, 4} per `FIXED_TAUS`) into ISV[99..103) at α matching the
/// other Plan 4 producers (cold-path-cadence, one launch per training
/// step alongside `h_s2_rms_ema_update`).
///
/// Cold-start: 0.0 (no IQN forwards have happened yet). FoldReset → 0.0.
/// Producer-only — diagnostic surface for HEALTH_DIAG / risk monitoring;
/// no consumer kernel reads these slots in this commit.
pub const IQN_Q_P05_EMA_INDEX: usize = 99;
pub const IQN_Q_P25_EMA_INDEX: usize = 100;
pub const IQN_Q_P75_EMA_INDEX: usize = 101;
pub const IQN_Q_P95_EMA_INDEX: usize = 102;

/// ISV slots [105..111) — Plan 4 Task 1B-ii: per-group VSN mask EMAs.
///
/// One slot per `FEATURE_GROUP_RANGES` entry (market / ofi / tlob / mtf /
/// portfolio / plan_isv). Producer: `vsn_mask_ema_update` GPU kernel
/// (single-block, 256 threads, shmem-reduction, no atomicAdd) — landed in
/// 1B-i, launch site wired in 1B-iii alongside the forward orchestrator that
/// produces the per-sample softmax mask. EMA α matches the rest of the
/// Plan 4 producer family (cold-path-cadence, one launch per training step
/// via `training_loop.rs`).
///
/// Cold-start: `1.0 / SL_NUM_FEATURE_GROUPS = 1/6` (uniform prior — no
/// group preference at init). The producer kernel overwrites these slots on
/// every fire, so the cold-start value matters only for consumers that read
/// the EMA before the first kernel launch (currently none — this commit is
/// producer/consumer-orchestrator-free; 1B-iii adds the launch).
/// FoldReset → `1/6`.
///
/// Producer-only after 1B-iii — diagnostic surface for HEALTH_DIAG /
/// per-group attention monitoring. No consumer kernel reads slots
/// [105..111) in this commit nor in 1B-iii (consumer attaches in a later
/// `attention_monitor` upgrade).
pub const VSN_MASK_GROUP_0_EMA_INDEX: usize = 105;
pub const VSN_MASK_GROUP_1_EMA_INDEX: usize = 106;
pub const VSN_MASK_GROUP_2_EMA_INDEX: usize = 107;
pub const VSN_MASK_GROUP_3_EMA_INDEX: usize = 108;
pub const VSN_MASK_GROUP_4_EMA_INDEX: usize = 109;
pub const VSN_MASK_GROUP_5_EMA_INDEX: usize = 110;

/// ISV slots [113..115) — Plan 4 Task 6 Commit A: aux-head loss EMAs.
///
/// Two slots: `[113] AUX_NEXT_BAR_MSE_EMA_INDEX` carries the EMA of the
/// next-bar regression head's batch-mean MSE; `[114] AUX_REGIME_CE_EMA_INDEX`
/// carries the EMA of the regime classification head's batch-mean
/// cross-entropy. Producer: `aux_heads_loss_ema_update` GPU kernel
/// (single-thread, single-block; mirrors `h_s2_rms_ema_update`'s footprint
/// — the work is two scalar EMAs, no reduction needed because the upstream
/// loss-reduce kernels emit scalars). EMA α matches the rest of the
/// Plan 4 producer family (cold-path-cadence, one launch per training
/// step alongside `h_s2_rms_ema_update`).
///
/// Cold-start: 0.0 (no aux forwards have happened yet — Commit B wires the
/// forwards). FoldReset → 0.0.
///
/// Producer-only after Commit B — the only diagnostic consumer is
/// HEALTH_DIAG's `aux[next_bar_mse=… regime_ce=…]` line (CPU-side text
/// emission, not a kernel input). No GPU kernel reads slots [113..115)
/// in either commit. Deliberately tail-appended after Plan 4 Task 1B-ii's
/// VSN slots so the ISV layout grows monotonically per
/// `feedback_no_partial_refactor.md` (every consumer of `ISV_TOTAL_DIM`
/// or the fingerprint shifts together in this commit).
pub const AUX_NEXT_BAR_MSE_EMA_INDEX: usize = 113;
pub const AUX_REGIME_CE_EMA_INDEX: usize = 114;

/// ISV slot [117] — RETIRED in SP13 B1.0 (2026-05-05).
///
/// Formerly `AUX_LABEL_SCALE_EMA_INDEX` — held the EMA of
/// `mean(|next_states[:, 0]|)` consumed by `aux_next_bar_loss_reduce` /
/// `aux_next_bar_backward` to divide the label by `max(scale, 1e-6)`
/// before the residual. Retired together with that division when the MSE
/// loss flipped to scale-free: labels are z-normalised at the data layer
/// so `label_scale_ema ≈ 1.0` empirically and `(pred - label * inv_scale)`
/// reduced to `(pred - label)` within rounding.
///
/// The slot remains unused in the ISV bus and is reserved by the layout
/// fingerprint seed — do NOT reuse it without a fingerprint bump (which
/// would invalidate every checkpoint stored under the post-B1.0
/// fingerprint). B1.1 will replace the MSE chain with CE entirely,
/// eliminating the formulation that made this slot useful.

/// Mixture-of-Experts per-expert utilization EMA, α=0.05.
/// 8 contiguous slots [118..126).
/// Producer: `moe_expert_util_ema_update` GPU kernel (cold-path cadence).
/// Reset value at fold boundary: 1.0 / 8.0 = 0.125 (uniform-gate initial).
pub const MOE_EXPERT_UTIL_EMA_BASE: usize = 118;
pub const MOE_EXPERT_UTIL_EMA_COUNT: usize = 8;

/// Entropy of batch-averaged gate distribution, EMA α=0.05.
/// Reset value at fold boundary: ln(8) ≈ 2.0794 (max entropy at uniform).
pub const MOE_GATE_ENTROPY_EMA_INDEX: usize = 126;

/// MoE adaptive load-balance λ controller output (2026-04-27).
///
/// `λ_eff = λ_floor + λ_max_extra × clamp((target − ent_ema)/target, 0, 1)`
/// where `target = entropy_target_frac × ln(K)`. Producer:
/// `moe_lambda_eff_update` GPU kernel (cold-path cadence, single-block
/// single-thread; runs each step after `moe_expert_util_ema_update` so
/// the entropy EMA fed to the controller is fresh for the NEXT step's
/// `moe_load_balance_loss` consumer). Consumer:
/// `moe_load_balance_loss` reads ISV[128] at runtime to scale the
/// per-expert squared-mean penalty.
///
/// Cold-start (constructor): `λ_floor` (= old static 0.01 default).
/// FoldReset: `λ_floor` (gate uniform-init again, deficit=0 ⇒ λ_floor).
///
/// Per `feedback_isv_for_adaptive_bounds.md` — adaptive bounds always
/// flow through ISV; no DtoH on hot path.
pub const MOE_LAMBDA_EFF_INDEX: usize = 128;

/// Plan C Phase 2 follow-up A.2 (2026-04-29): adaptive Polyak-tau drift signal.
///
/// Per-epoch normalised q-drift rate ∈ [0, 4]. Producer:
/// `q_drift_rate_ema_update` GPU kernel (cold-path cadence, single-block
/// single-thread). Reads host-passed `q_mean_curr`, `q_mean_prev` plus
/// ISV[Q_ABS_REF_INDEX] and ISV[Q_DIR_ABS_REF_INDEX] to compute
///   `drift_rate = |q_curr − q_prev| / max(|q_prev|, |Q|_mag + |Q|_dir, 1e-6)`
/// clipped to `[0, 4]`. Consumer: `tau_update_kernel.cu` multiplies the
/// cosine-scheduled `tau_base` by `1 / (1 + ISV[129])` so the effective
/// Polyak rate decays from `tau_base` (healthy run, drift≈0) down to
/// `tau_base / 5` (peak drift, dampening saturated).
///
/// Cold-start (constructor): 0.0 (no drift before any epochs have run).
/// FoldReset: 0.0 (drift is computed against `prev_epoch_q_mean` which
/// also resets to 0 at fold boundary, see A.1 in
/// `DQNTrainer::reset_for_fold`; the launch site gates on
/// `prev_epoch_q_mean.abs() > 1e-6` so the first epoch of a new fold
/// emits no drift).
///
/// Tail-appended at index 129 (one past the previous `ISV_TOTAL_DIM=129`
/// boundary, raising it to 130) per `feedback_no_partial_refactor.md`.
/// Layout fingerprint changes — checkpoint-incompatible per
/// `feedback_no_legacy_aliases.md`. The `4.0` upper clip is an
/// architectural drift-saturation bound (matches the kill-criterion's
/// 3.0× ratio threshold in `training_loop.rs`); the `1e-6` denom floor
/// is a numerical-stability bound (Invariant 1 carve-out per
/// `feedback_isv_for_adaptive_bounds.md`). Per
/// `feedback_no_quickfixes.md` — neither is a tuned constant.
pub const Q_DRIFT_RATE_INDEX: usize = 129;

/// ISV slot [130] — adaptive fold-boundary warmup factor ∈ [0, 1].
///
/// Plan C Phase 2 follow-up K (2026-04-29): single signal driving BOTH lr_eff
/// and clip_eff dampening at fold boundaries. Computed per-step by
/// `fold_warmup_factor_kernel.cu` from grad-norm stability:
///   `factor = current_grad_norm_ema / steady_state_grad_norm_ema`
/// clamped to `[0, 1]`. After fold reset (FoldReset: 0.0) the fast EMA recovers
/// from zero while the slow EMA tracks the cross-fold steady-state grad scale,
/// so the ratio rises 0 → 1 as gradients stabilise. Steady-state behaviour:
/// fast EMA matches slow EMA → factor = 1 → no dampening (healthy runs
/// unaffected — same monotone-only-dampens contract as A.2's
/// `Q_DRIFT_RATE_INDEX`).
///
/// Two consumers, both monotone (only dampen, never excite):
///   `lr_eff   = lr_base × max(MIN_WARMUP_LR_FRAC=0.05, factor)`
///   `clip_eff = clip_base × (MIN_CLIP_FRAC=0.1 + (1 - MIN_CLIP_FRAC) × factor)`
///
/// Cold-start (constructor): 0.0 (heavy damping until grad-norm EMAs warm up).
/// FoldReset: 0.0 (re-engage damping for the new fold's first steps; the slow
/// EMA persists across folds so the steady-state baseline stays meaningful).
///
/// Tail-appended at index 130 (one past the previous `ISV_TOTAL_DIM=130`
/// boundary, raising it to 131) per `feedback_no_partial_refactor.md`. Layout
/// fingerprint changes — checkpoint-incompatible per `feedback_no_legacy_aliases.md`.
/// The numerical-stability bounds `0.05` (lr floor frac) and `0.1` (clip floor
/// frac) are Invariant 1 carve-outs per `feedback_isv_for_adaptive_bounds.md`
/// — the warmup factor is the *adaptive* bound; consumers read ISV[130] at
/// runtime and compose it with their respective architectural floors.
pub const FOLD_WARMUP_FACTOR_INDEX: usize = 130;

/// ISV slot [115] — low 32 bits of the u64 layout fingerprint (stored as raw f32 bits).
///
/// Design note: this is NOT a version number. There is no ordered version space.
/// The fingerprint is a structural hash that automatically changes whenever any
/// slot is added, removed, reordered, or renamed — so there is no natural pairing
/// "v1 -> v2" and no place to write a migrator. Per spec §4.A.2 and
/// `feedback_no_legacy_aliases.md`: backward compat is structurally unwritable,
/// not merely discouraged.
///
/// Placement at the tail (not head) is mandated by slots [0] and [1] being
/// actively written by `isv_signal_update` (Q-drift and gradient-norm EMA).
/// Shifted 61→69 in Plan 3 Task 1, 69→73 in Plan 3 Task 3, 73→76 in
/// Plan 3 Task 4 B.4, 76→80 in Plan 3 Task 7 C.3, 80→85 in Plan 3 Task 8 B.3,
/// 85→90 in Plan 4 Task 5 Mode A, 90→94 in Plan 4 follow-up (target-drift
/// EMA replacing legacy CPU-DtoH path), 94→97 in Plan 4 Task 2c.3c.5
/// (H_S2_RMS_EMA producer slot), 97→103 in Plan 4 Task 3 E.3 (IQN
/// fixed-τ multi-quantile head, +4 diagnostic slots), 103→111 in
/// Plan 4 Task 1B-ii (VSN per-group mask EMA, +6 slots), 111→115 in
/// Plan 4 Task 6 Commit A (aux-head loss EMA, +2 slots — gap at [112]
/// preserved to keep the new slots bin-aligned with the existing
/// gap-anchored cadence).
pub const ISV_LAYOUT_FINGERPRINT_LO_INDEX: usize = 115;
/// ISV slot [116] — high 32 bits of the u64 layout fingerprint (stored as raw f32 bits).
/// Shifted 62→70 in Plan 3 Task 1, 70→74 in Plan 3 Task 3, 74→77 in Plan 3 Task 4 B.4,
/// 77→81 in Plan 3 Task 7 C.3, 81→86 in Plan 3 Task 8 B.3, 86→91 in Plan 4 Task 5,
/// 91→95 in Plan 4 follow-up, 95→98 in Plan 4 Task 2c.3c.5, 98→104 in Plan 4
/// Task 3 E.3, 104→112 in Plan 4 Task 1B-ii, 112→116 in Plan 4 Task 6 Commit A.
pub const ISV_LAYOUT_FINGERPRINT_HI_INDEX: usize = 116;
/// Canonical alias for the fingerprint slot (the low half).
pub const ISV_LAYOUT_FINGERPRINT_INDEX: usize = ISV_LAYOUT_FINGERPRINT_LO_INDEX;

/// FNV-1a 64-bit hash of a byte slice, usable as a `const fn`.
const fn fnv1a_64(bytes: &[u8]) -> u64 {
    const OFFSET: u64 = 0xcbf29ce484222325;
    const PRIME: u64 = 0x00000100000001b3;
    let mut h: u64 = OFFSET;
    let mut i = 0;
    while i < bytes.len() {
        h ^= bytes[i] as u64;
        h = h.wrapping_mul(PRIME);
        i += 1;
    }
    h
}

/// Source bytes for the layout fingerprint. Canonical format:
/// `b"<slot_name>=<index>;<slot_name>=<index>;...;ISV_TOTAL_DIM=<N>;\
///   PARAM_<TENSOR>=<idx>;...;PARAM_TOTAL_TENSORS=<M>"`.
///
/// When adding/removing a slot OR a param tensor, add/remove its entry here in
/// the SAME commit that changes the underlying list. The fingerprint
/// recomputes automatically. Order matches the constant/tensor declaration
/// order in this module.
///
/// Slots [0..12) have no named `pub const` (they are written directly via
/// literal indices in `isv_signal_update`). They are covered here by
/// `SLOT_<N>_<PURPOSE>` entries derived from the `ISV_TOTAL_DIM` docstring
/// and `experience_kernels.cu` write-site comments. Any renumber of these
/// slots changes the seed and invalidates the fingerprint.
///
/// Param-tensor section (Plan 4 Task 2b; reshuffled in 2c.3a): the 95 entries
/// `PARAM_<NAME>=<idx>` mirror `compute_param_sizes()` exactly. Names match
/// the in-code comments at each tensor index. Sizes are runtime-config-
/// dependent (`shared_h1` etc.) and intentionally NOT encoded — the const-fn
/// fingerprint covers **structural layout** (tensor names + positions).
/// Size mismatches between checkpoint and current binary are detected
/// separately by safetensors deserialization. Any structural reshuffle
/// (insert/delete/reorder/rename a tensor) changes the seed and triggers
/// fail-fast at checkpoint load.
const fn layout_fingerprint_seed() -> &'static [u8] {
    b"SLOT_0_Q_DRIFT=0;\
      SLOT_1_GRAD_NORM_EMA=1;\
      SLOT_2_TD_ERR_EMA=2;\
      SLOT_3_ENS_VAR_EMA=3;\
      SLOT_4_ENS_VAR_VEL=4;\
      SLOT_5_REWARD_EMA=5;\
      SLOT_6_ATOM_UTIL_EMA=6;\
      SLOT_7_LOSS_EMA=7;\
      SLOT_8_ADX_EMA=8;\
      SLOT_9_REGIME_DISAGREE=9;\
      SLOT_10_REGIME_VEL_EMA=10;\
      SLOT_11_REGIME_STABILITY=11;\
      LEARNING_HEALTH=12;\
      Q_MAG_MEAN_QUARTER=13;Q_MAG_MEAN_HALF=14;Q_MAG_MEAN_FULL=15;Q_ABS_REF=16;\
      Q_DIR_MEAN_SHORT=17;Q_DIR_MEAN_HOLD=18;Q_DIR_MEAN_LONG=19;Q_DIR_MEAN_FLAT=20;Q_DIR_ABS_REF=21;\
      SHARPE_EMA=22;\
      V_CENTER_DIR=23;V_HALF_DIR=24;V_CENTER_MAG=25;V_HALF_MAG=26;\
      V_CENTER_ORD=27;V_HALF_ORD=28;V_CENTER_URG=29;V_HALF_URG=30;\
      GRAD_NORM_TARGET_DIR=31;GRAD_NORM_TARGET_MAG=32;GRAD_NORM_TARGET_ORD=33;GRAD_NORM_TARGET_URG=34;\
      GRAD_SCALE_LIMIT=35;IQL_BRANCH_SCALE_FLOOR=36;\
      EPOCH_IDX=39;TOTAL_EPOCHS=40;\
      EPSILON_EFF=41;TAU_EFF=42;\
      GAMMA_DIR_EFF=43;GAMMA_MAG_EFF=44;GAMMA_ORD_EFF=45;GAMMA_URG_EFF=46;\
      KELLY_CAP_EFF=47;CQL_ALPHA=48;PLAN_THRESHOLD=49;\
      Q_P05_DIR=50;Q_P05_MAG=51;Q_P05_ORD=52;Q_P05_URG=53;\
      Q_P95_DIR=54;Q_P95_MAG=55;Q_P95_ORD=56;Q_P95_URG=57;\
      TLOB_REGIME_FOCUS_EMA=60;\
      REWARD_POPART_EMA=63;REWARD_CF_EMA=64;REWARD_TRAIL_EMA=65;\
      REWARD_MICRO_EMA=66;REWARD_OPP_COST_EMA=67;REWARD_BONUS_EMA=68;\
      TRADE_ATTEMPT_RATE_EMA=71;TRADE_TARGET_RATE=72;\
      READINESS_EMA=75;\
      STATE_KL_EMA=78;STATE_KL_AMP=79;\
      SEED_STEPS_TARGET=82;SEED_STEPS_DONE=83;SEED_FRAC_EMA=84;\
      VSN_MAG_EMA=87;VSN_DIR_EMA=88;MAMBA2_RETENTION_EMA=89;\
      TARGET_DRIFT_MAG_EMA=92;TARGET_DRIFT_DIR_EMA=93;\
      H_S2_RMS_EMA=96;\
      IQN_Q_P05_EMA=99;IQN_Q_P25_EMA=100;IQN_Q_P75_EMA=101;IQN_Q_P95_EMA=102;\
      VSN_MASK_GROUP_0_EMA=105;VSN_MASK_GROUP_1_EMA=106;VSN_MASK_GROUP_2_EMA=107;\
      VSN_MASK_GROUP_3_EMA=108;VSN_MASK_GROUP_4_EMA=109;VSN_MASK_GROUP_5_EMA=110;\
      AUX_NEXT_BAR_MSE_EMA=113;AUX_REGIME_CE_EMA=114;\
      ISV_LAYOUT_FINGERPRINT_LO=115;ISV_LAYOUT_FINGERPRINT_HI=116;\
      MOE_EXPERT_UTIL_EMA_BASE=118;MOE_EXPERT_UTIL_EMA_COUNT=8;\
      MOE_GATE_ENTROPY_EMA=126;\
      MOE_LAMBDA_EFF=128;\
      Q_DRIFT_RATE=129;\
      FOLD_WARMUP_FACTOR=130;\
      TARGET_Q_BOUND=131;\
      ATOM_POS_BOUND_BRANCH_0=132;ATOM_POS_BOUND_BRANCH_1=133;ATOM_POS_BOUND_BRANCH_2=134;ATOM_POS_BOUND_BRANCH_3=135;\
      WEIGHT_BOUND_GROUP_0=136;WEIGHT_BOUND_GROUP_1=137;WEIGHT_BOUND_GROUP_2=138;WEIGHT_BOUND_GROUP_3=139;\
      WEIGHT_BOUND_GROUP_4=140;WEIGHT_BOUND_GROUP_5=141;WEIGHT_BOUND_GROUP_6=142;WEIGHT_BOUND_GROUP_7=143;\
      ADAM_M_BOUND_GROUP_0=144;ADAM_M_BOUND_GROUP_1=145;ADAM_M_BOUND_GROUP_2=146;ADAM_M_BOUND_GROUP_3=147;\
      ADAM_M_BOUND_GROUP_4=148;ADAM_M_BOUND_GROUP_5=149;ADAM_M_BOUND_GROUP_6=150;ADAM_M_BOUND_GROUP_7=151;\
      ADAM_V_BOUND_GROUP_0=152;ADAM_V_BOUND_GROUP_1=153;ADAM_V_BOUND_GROUP_2=154;ADAM_V_BOUND_GROUP_3=155;\
      ADAM_V_BOUND_GROUP_4=156;ADAM_V_BOUND_GROUP_5=157;ADAM_V_BOUND_GROUP_6=158;ADAM_V_BOUND_GROUP_7=159;\
      WD_RATE_GROUP_0=160;WD_RATE_GROUP_1=161;WD_RATE_GROUP_2=162;WD_RATE_GROUP_3=163;\
      WD_RATE_GROUP_4=164;WD_RATE_GROUP_5=165;WD_RATE_GROUP_6=166;WD_RATE_GROUP_7=167;\
      GRAD_CLIP_BOUND=168;H_S2_BOUND=169;L1_LAMBDA_TRUNK=170;\
      BW_D_H_S2_BOUND=171;Q_DIR_GRAD_BOUND=172;\
      SP5_BASE=174;ATOM_V_CENTER_BASE=174;ATOM_V_HALF_BASE=178;ATOM_HEADROOM_BASE=182;\
      ATOM_CLIP_RATE_BASE=186;BUDGET_C51_BASE=190;BUDGET_IQN_BASE=194;BUDGET_CQL_BASE=198;\
      BUDGET_ENS_BASE=202;FLATNESS_BASE=206;NOISY_SIGMA_BASE=210;SIGMA_FRACTION_BASE=214;\
      BRANCH_ENTROPY_BASE=218;Q_VAR_PER_BRANCH_BASE=222;ADAM_BETA1_BASE=226;ADAM_BETA2_BASE=234;\
      ADAM_EPS_BASE=242;IQN_TAU_BASE=250;TRAIL_DIST_PER_DIR_BASE=270;ATOM_NUM_ATOMS_BASE=274;\
      KELLY_F_SMOOTH=280;CONVICTION_SMOOTH=281;TRADE_VAR_SMOOTH=282;\
      KELLY_SAMPLE_COUNT=283;WIN_RATE_SMOOTH=284;LOSS_RATE_SMOOTH=285;\
      PNL_TOTAL=286;PNL_MEAN=287;PNL_VAR=288;PNL_MAX_DD=289;\
      HEALTH_SCORE=290;Q_GAP_NORM=291;Q_VAR_NORM=292;GRAD_NORM_NORM=293;\
      TRAINING_SHARPE_EMA=294;MAX_DD_EMA=295;LOW_DD_RATIO=296;\
      LB_DIFF_VAR_CQL_BASE=297;LB_SAMPLE_VAR_CQL_BASE=301;\
      LB_DIFF_VAR_C51_BASE=305;LB_SAMPLE_VAR_C51_BASE=309;\
      LB_CQL_ACTIVE_BASE=313;LB_C51_ACTIVE_BASE=317;\
      TRAIN_ACTIVE_FRAC_INDEX=321;\
      LB_MAX_BUDGET_CQL_BASE=322;LB_MAX_BUDGET_C51_BASE=326;\
      KELLY_WARMUP_FLOOR=330;Q_VAR_MAG_EMA=331;\
      INTENT_EVAL_DIVERGENCE=332;KELLY_SAMPLE_COUNT_TARGET=333;\
      KELLY_DIVERGENCE_TARGET=334;KELLY_TEMPORAL_TARGET=335;\
      EVAL_DIST_Q=336;EVAL_DIST_H=337;EVAL_DIST_F=338;\
      EVAL_THOMPSON_TEMP=339;\
      REWARD_POPART_WEIGHT=340;REWARD_CF_WEIGHT=341;REWARD_TRAIL_WEIGHT=342;\
      REWARD_MICRO_WEIGHT=343;REWARD_OPP_COST_WEIGHT=344;REWARD_BONUS_WEIGHT=345;\
      CURIOSITY_PRESSURE=346;SABOTEUR_INTENSITY_MULT=347;\
      REWARD_WEIGHT_FLOOR=348;CURIOSITY_BOUND=349;\
      VAL_SHARPE_DELTA_EMA=350;VAL_SHARPE_VAR_EMA=351;\
      REWARD_COMPONENT_MAG_RATIO_BASE=352;\
      SABOTEUR_ENGAGEMENT_RATE=358;PNL_REWARD_MAGNITUDE_EMA=359;\
      POPART_COMPONENT_MAG_EMA=360;\
      REWARD_COMPONENT_VAR_EMA_BASE=361;\
      TARGET_DIR_ACC=372;AUX_DIR_ACC_SHORT_EMA=373;AUX_DIR_ACC_LONG_EMA=374;\
      AUX_DIR_PREDICTION=375;DIR_SKILL_BONUS_ALPHA=376;DIR_SKILL_BONUS_BETA=377;\
      LUCK_WIN_DISCOUNT=378;SKILL_BONUS_CAP_RATIO=379;\
      HOLD_COST=380;HOLD_RATE_TARGET=381;HOLD_RATE_OBSERVED_EMA=382;\
      Q_DISAGREEMENT_SHORT_EMA=383;Q_DISAGREEMENT_LONG_EMA=384;\
      Q_DISAGREEMENT_VARIANCE_EMA=389;\
      EGF_SCHMITT_HI=397;EGF_SCHMITT_LO=398;EGF_VAR_AUX_REF=399;EGF_VAR_Q_REF=400;\
      DD_CURRENT=401;DD_MAX=402;DD_RECOVERY_BARS=403;DD_PERSISTENCE=404;CALMAR=405;DD_PCT=406;\
      OFI_IMPACT_LAMBDA=407;COST_PER_BAR_AVG=408;\
      BASELINE_RANDOM_DIR_KELLY_SHARPE=411;\
      BASELINE_AUX_ONLY_SHARPE=413;BASELINE_MAG_QUARTER_FIXED_SHARPE=414;\
      BASELINE_TRAIL_ONLY_SHARPE=415;\
      ALPHA_SPLIT=417;GRAD_NORM_QUALITY=418;GRAD_NORM_DISCIPLINE=419;\
      LAMBDA_DD=420;DD_THRESHOLD=421;DD_PENALTY_GRAD_NORM=422;\
      REGRET_EMA=423;LAMBDA_REGRET=424;REGRET_GRAD_NORM=425;\
      HOLD_FLOOR_ALPHA=426;HOLD_FLOOR_K=427;HOLD_FLOOR_EPS0=428;ENTROPY_DIST_REF=429;\
      DD_ASYMMETRY_LAMBDA=430;R_GAIN_DD_BOOST=431;DD_DIST_VAR=432;\
      COOLDOWN_K_THRESHOLD=433;COOLDOWN_M_BARS=434;COOLDOWN_BARS_REMAINING=435;\
      PLASTICITY_FIRED_THIS_FOLD=436;PLASTICITY_PERSISTENCE_THRESHOLD=437;PLASTICITY_WARM_BARS_REMAINING=438;\
      DD_TRAJECTORY_DECREASING=439;RECOVERY_OVERSAMPLE_WEIGHT=440;\
      DD_TRAJECTORY_FLOOR=441;MEDIAN_STREAK_LENGTH=442;\
      DD_PERSISTENCE_MAX=443;\
      AUX_TRUNK_LR=444;AUX_TRUNK_BETA1=445;AUX_TRUNK_BETA2=446;\
      AUX_TRUNK_EPS=447;AUX_TRUNK_GRAD_CLIP=448;H_S2_AUX_RMS_EMA=449;\
      AUX_PRED_HORIZON_BARS=450;AVG_WIN_HOLD_TIME_BARS=451;\
      REWARD_POS_CAP_ADAPTIVE=452;REWARD_NEG_CAP_ADAPTIVE=453;\
      KELLY_PRIOR_WINS=454;KELLY_PRIOR_LOSSES=455;KELLY_PRIOR_SUM_WINS=456;KELLY_PRIOR_SUM_LOSSES=457;\
      DD_SATURATION_FLOOR_ADAPTIVE=458;\
      PLAN_THRESHOLD_FLOOR_ADAPTIVE=459;MIN_HOLD_TEMPERATURE_ADAPTIVE=460;\
      RESERVED_GAP_461_TO_468=SP18_P1_RETIRED;\
      MHT_TARGET_MEAN=468;MHT_TARGET_M2=469;MHT_DIFF_MEAN=470;\
      MHT_DIFF_M2=471;MHT_PREV_TARGET=472;MHT_SAMPLE_COUNT=473;\
      SLOT_474_A_VAR_EMA_DIR=474;\
      SLOT_475_A_VAR_EMA_MAG=475;\
      SLOT_476_A_VAR_EMA_ORD=476;\
      SLOT_477_A_VAR_EMA_URG=477;\
      SLOT_478_V_SHARE_DIR=478;\
      SLOT_479_V_SHARE_MAG=479;\
      SLOT_480_V_SHARE_ORD=480;\
      SLOT_481_V_SHARE_URG=481;\
      SLOT_482_ADVANTAGE_CLIP_BOUND=482;\
      SLOT_483_HOLD_REWARD_POS_CAP=483;\
      SLOT_484_HOLD_REWARD_NEG_CAP=484;\
      SLOT_485_HOLD_REWARD_DECOMP_DIAG=485;\
      SLOT_486_HOLD_OPP_COST_FIRE_RATE_EMA=486;\
      SLOT_487_HRC_TARGET_MEAN=487;\
      SLOT_488_HRC_TARGET_M2=488;\
      SLOT_489_HRC_DIFF_MEAN=489;\
      SLOT_490_HRC_DIFF_M2=490;\
      SLOT_491_HRC_PREV_TARGET=491;\
      SLOT_492_HRC_SAMPLE_COUNT=492;\
      SLOT_493_TD_ERROR_MAG_EMA=493;\
      SLOT_494_Q_NEXT_TARGET_P99=494;\
      SLOT_495_Q_NEXT_MINUS_REWARD_P99=495;\
      SLOT_496_V_SHARE_TREND_DIAG=496;\
      SLOT_497_POPART_RESET_FLAG=497;\
      SLOT_498_TDB_TARGET_MEAN=498;\
      SLOT_499_TDB_TARGET_M2=499;\
      SLOT_500_TDB_DIFF_MEAN=500;\
      SLOT_501_TDB_DIFF_M2=501;\
      SLOT_502_TDB_PREV_TARGET=502;\
      SLOT_503_TDB_SAMPLE_COUNT=503;\
      SLOT_504_SP18_B_LEG_RESERVED=504;\
      SLOT_505_SHRINK_ALPHA_ADAPTIVE=505;\
      SLOT_506_SHRINK_SIGMA_ADAPTIVE=506;\
      SLOT_507_REWARD_HORIZON_WEIGHT_1BAR=507;\
      SLOT_508_REWARD_HORIZON_WEIGHT_5BAR=508;\
      SLOT_509_REWARD_HORIZON_WEIGHT_30BAR=509;\
      SLOT_510_LOSS_CAP=510;\
      SLOT_511_ALPHA_EMA=511;\
      SLOT_512_WR_EMA=512;\
      SLOT_513_HOLD_COST_SCALE=513;\
      SLOT_514_TARGET_HOLD_PCT=514;\
      SLOT_515_HOLD_PCT_EMA=515;\
      SLOT_516_HOLD_REWARD_EMA=516;\
      SLOT_517_N_STEP=517;\
      SLOT_518_AUX_CONF_THRESHOLD=518;\
      SLOT_519_AUX_GATE_TEMP=519;\
      SLOT_520_Q_CORRECTION=520;\
      SLOT_521_BRANCH_LR_SCALE_DIR=521;\
      SLOT_522_BRANCH_LR_SCALE_MAG=522;\
      SLOT_523_BRANCH_LR_SCALE_ORDER=523;\
      SLOT_524_BRANCH_LR_SCALE_URGENCY=524;\
      SLOT_525_WINNER_CONCENTRATION=525;\
      SLOT_526_HINDSIGHT_MAGNITUDE=526;\
      SLOT_527_CURRICULUM_CONCENTRATION=527;\
      SLOT_528_CURRICULUM_WEIGHT_0=528;\
      SLOT_529_CURRICULUM_WEIGHT_1=529;\
      SLOT_530_CURRICULUM_WEIGHT_2=530;\
      SLOT_531_CURRICULUM_WEIGHT_3=531;\
      SLOT_532_CURRICULUM_WEIGHT_4=532;\
      SLOT_533_CURRICULUM_WEIGHT_5=533;\
      SLOT_534_CURRICULUM_WEIGHT_6=534;\
      SLOT_535_CURRICULUM_WEIGHT_7=535;\
      SLOT_536_REWARD_AUX_ALIGN_EMA=536;\
      SLOT_537_SP22_AUX_ALIGN_SCALE=537;\
      ISV_TOTAL_DIM=538;\
      SP19_PRODUCER_HARDCODED_HORIZON_BLEND=sp19_path_b;\
      SP14_C_AUX_TRUNK_CONTROL_PLANE=sp14_c_phase_1;\
      ALPHA_MACHINERY_DELETED=sp14_c_phase_1;\
      TRUNK_INPUT_DD_PCT=sp15_wave_4_1a;\
      DD_STATE_PER_ENV=sp15_phase_1_3_b_followup;\
      DD_TRAJECTORY_PER_ENV=sp15_phase_1_3_b_followup_B;\
      PARAM_W_A_H_S1=0;PARAM_B_A_H_S1=1;PARAM_W_B_H_S1=2;PARAM_B_B_H_S1=3;\
      PARAM_W_RESIDUAL_H_S1=4;PARAM_GAMMA_H_S1=5;PARAM_BETA_H_S1=6;\
      PARAM_W_A_H_S2=7;PARAM_B_A_H_S2=8;PARAM_W_B_H_S2=9;PARAM_B_B_H_S2=10;\
      PARAM_GAMMA_H_S2=11;PARAM_BETA_H_S2=12;\
      PARAM_W_V1=13;PARAM_B_V1=14;PARAM_W_V2=15;PARAM_B_V2=16;\
      PARAM_W_B0FC_AUX1=17;PARAM_B_B0FC=18;PARAM_W_B0OUT=19;PARAM_B_B0OUT=20;\
      PARAM_W_B1FC=21;PARAM_B_B1FC=22;PARAM_W_B1OUT=23;PARAM_B_B1OUT=24;\
      PARAM_W_B2FC=25;PARAM_B_B2FC=26;PARAM_W_B2OUT=27;PARAM_B_B2OUT=28;\
      PARAM_W_B3FC=29;PARAM_B_B3FC=30;PARAM_W_B3OUT=31;PARAM_B_B3OUT=32;\
      PARAM_W_BN=33;PARAM_B_BN=34;\
      PARAM_W_VSN1_0=35;PARAM_W_VSN2_0=36;\
      PARAM_W_VSN1_1=37;PARAM_W_VSN2_1=38;\
      PARAM_W_VSN1_2=39;PARAM_W_VSN2_2=40;\
      PARAM_W_VSN1_3=41;PARAM_W_VSN2_3=42;\
      PARAM_W_GATE_0=43;PARAM_B_GATE_0=44;\
      PARAM_W_GATE_1=45;PARAM_B_GATE_1=46;\
      PARAM_W_GATE_2=47;PARAM_B_GATE_2=48;\
      PARAM_W_GATE_3=49;PARAM_B_GATE_3=50;\
      PARAM_KAN_COEFF_0=51;PARAM_KAN_RESID_0=52;\
      PARAM_KAN_COEFF_1=53;PARAM_KAN_RESID_1=54;\
      PARAM_KAN_COEFF_2=55;PARAM_KAN_RESID_2=56;\
      PARAM_KAN_COEFF_3=57;PARAM_KAN_RESID_3=58;\
      PARAM_W_REGIME=59;PARAM_B_REGIME=60;\
      PARAM_SPACING_RAW_0=61;PARAM_SPACING_RAW_1=62;\
      PARAM_SPACING_RAW_2=63;PARAM_SPACING_RAW_3=64;\
      PARAM_W_V1_5BAR=65;PARAM_B_V1_5BAR=66;\
      PARAM_W_V2_5BAR=67;PARAM_B_V2_5BAR=68;\
      PARAM_W_V1_20BAR=69;PARAM_B_V1_20BAR=70;\
      PARAM_W_V2_20BAR=71;PARAM_B_V2_20BAR=72;\
      PARAM_W_RISK_FC=73;PARAM_B_RISK_FC=74;\
      PARAM_W_RISK_OUT=75;PARAM_B_RISK_OUT=76;\
      PARAM_W_ISV_FC1=77;PARAM_B_ISV_FC1=78;\
      PARAM_W_ISV_FC2=79;PARAM_B_ISV_FC2=80;\
      PARAM_W_ISV_GATE=81;PARAM_B_ISV_GATE=82;\
      PARAM_W_ISV_GAMMA=83;PARAM_B_ISV_GAMMA=84;\
      PARAM_W_CONF_FC=85;PARAM_B_CONF_FC=86;\
      PARAM_W_FEATURE_GATE=87;PARAM_B_FEATURE_GATE=88;\
      PARAM_W_TEMPORAL_ROUTE=89;PARAM_B_TEMPORAL_ROUTE=90;\
      PARAM_W_PLAN_FC=91;PARAM_B_PLAN_FC=92;\
      PARAM_W_PLAN_OUT=93;PARAM_B_PLAN_OUT=94;\
      PARAM_VSN_W1_G0=95;PARAM_VSN_B1_G0=96;PARAM_VSN_W2_G0=97;PARAM_VSN_B2_G0=98;\
      PARAM_VSN_W1_G1=99;PARAM_VSN_B1_G1=100;PARAM_VSN_W2_G1=101;PARAM_VSN_B2_G1=102;\
      PARAM_VSN_W1_G2=103;PARAM_VSN_B1_G2=104;PARAM_VSN_W2_G2=105;PARAM_VSN_B2_G2=106;\
      PARAM_VSN_W1_G3=107;PARAM_VSN_B1_G3=108;PARAM_VSN_W2_G3=109;PARAM_VSN_B2_G3=110;\
      PARAM_VSN_W1_G4=111;PARAM_VSN_B1_G4=112;PARAM_VSN_W2_G4=113;PARAM_VSN_B2_G4=114;\
      PARAM_VSN_W1_G5=115;PARAM_VSN_B1_G5=116;PARAM_VSN_W2_G5=117;PARAM_VSN_B2_G5=118;\
      PARAM_AUX_NB_W1=119;PARAM_AUX_NB_B1=120;PARAM_AUX_NB_W2_K2=121;PARAM_AUX_NB_B2_K2=122;\
      PARAM_AUX_RG_W1=123;PARAM_AUX_RG_B1=124;PARAM_AUX_RG_W2=125;PARAM_AUX_RG_B2=126;\
      PARAM_MOE_GATE_W1=127;PARAM_MOE_GATE_B1=128;PARAM_MOE_GATE_W2=129;PARAM_MOE_GATE_B2=130;\
      PARAM_MOE_EXPERT_0_W1=131;PARAM_MOE_EXPERT_0_B1=132;PARAM_MOE_EXPERT_0_W2=133;PARAM_MOE_EXPERT_0_B2=134;\
      PARAM_MOE_EXPERT_1_W1=135;PARAM_MOE_EXPERT_1_B1=136;PARAM_MOE_EXPERT_1_W2=137;PARAM_MOE_EXPERT_1_B2=138;\
      PARAM_MOE_EXPERT_2_W1=139;PARAM_MOE_EXPERT_2_B1=140;PARAM_MOE_EXPERT_2_W2=141;PARAM_MOE_EXPERT_2_B2=142;\
      PARAM_MOE_EXPERT_3_W1=143;PARAM_MOE_EXPERT_3_B1=144;PARAM_MOE_EXPERT_3_W2=145;PARAM_MOE_EXPERT_3_B2=146;\
      PARAM_MOE_EXPERT_4_W1=147;PARAM_MOE_EXPERT_4_B1=148;PARAM_MOE_EXPERT_4_W2=149;PARAM_MOE_EXPERT_4_B2=150;\
      PARAM_MOE_EXPERT_5_W1=151;PARAM_MOE_EXPERT_5_B1=152;PARAM_MOE_EXPERT_5_W2=153;PARAM_MOE_EXPERT_5_B2=154;\
      PARAM_MOE_EXPERT_6_W1=155;PARAM_MOE_EXPERT_6_B1=156;PARAM_MOE_EXPERT_6_W2=157;PARAM_MOE_EXPERT_6_B2=158;\
      PARAM_MOE_EXPERT_7_W1=159;PARAM_MOE_EXPERT_7_B1=160;PARAM_MOE_EXPERT_7_W2=161;PARAM_MOE_EXPERT_7_B2=162;\
      PARAM_TOTAL_TENSORS=163"
}

/// Compile-time layout fingerprint. Burned into the binary at build time.
/// Checked against the stored value at checkpoint load. Mismatch means retrain required.
pub const LAYOUT_FINGERPRINT_CURRENT: u64 = fnv1a_64(layout_fingerprint_seed());

const ISV_EMB_DIM: usize = 8; // ISV embedding output dimension (FC2 output, gate/gamma input)
/// First ISV tensor index in the flat param buffer.
/// ISV weights (77-88) are online-only — NOT synced to the target network.
/// Plan 4 Task 2c.3a: shifted from 68 to 77 (+9) by GRN trunk reshuffle that
/// replaced 4 trunk Linear tensors with 13 GRN tensors at the head of the
/// flat layout.
const FIRST_ISV_TENSOR: usize = 77;
const NEW_COMPONENT_WARMUP_STEPS: u32 = 500;

/// Homeostatic regularizer: number of observable signals.
/// Observables: [Q-mean, atom_util, branch_corr, trade_freq_normalized, Q-gap, Q-var]
const HOMEOSTATIC_N_OBS: usize = 6;

// ── Mamba2 cuBLAS GEMM descriptor ────────────────────────────────────────────

/// Cached cuBLAS GEMM descriptor for Mamba2 forward/backward projections.
struct Mamba2GemmDesc {
    matmul_desc: cublaslt_sys::cublasLtMatmulDesc_t,
    a_layout: cublaslt_sys::cublasLtMatrixLayout_t,
    b_layout: cublaslt_sys::cublasLtMatrixLayout_t,
    c_layout: cublaslt_sys::cublasLtMatrixLayout_t,
    d_layout: cublaslt_sys::cublasLtMatrixLayout_t,
    algo: cublaslt_sys::cublasLtMatmulAlgo_t,
}

unsafe impl Send for Mamba2GemmDesc {}
unsafe impl Sync for Mamba2GemmDesc {}

#[allow(clippy::too_many_arguments)]
fn create_mamba2_gemm_desc(
    lt_handle: cublaslt_sys::cublasLtHandle_t,
    transa: i32,
    transb: i32,
    m: usize,
    n: usize,
    k: usize,
    lda: usize,
    ldb: usize,
    ldc: usize,
    ws_size: usize,
    label: &str,
) -> Result<Mamba2GemmDesc, MLError> {
    let f32_type = cublaslt_sys::cudaDataType_t::CUDA_R_32F;
    let compute_type = cublaslt_sys::cublasComputeType_t::CUBLAS_COMPUTE_32F_FAST_TF32;

    unsafe {
        let matmul_desc = cublaslt_result::create_matmul_desc(compute_type, f32_type)
            .map_err(|e| MLError::ModelError(format!("mamba2 MatmulDescCreate {label}: {e:?}")))?;

        cublaslt_result::set_matmul_desc_attribute(
            matmul_desc,
            cublaslt_sys::cublasLtMatmulDescAttributes_t::CUBLASLT_MATMUL_DESC_TRANSA,
            &transa as *const i32 as *const std::ffi::c_void,
            std::mem::size_of::<i32>(),
        ).map_err(|e| MLError::ModelError(format!("mamba2 set TRANSA {label}: {e:?}")))?;

        cublaslt_result::set_matmul_desc_attribute(
            matmul_desc,
            cublaslt_sys::cublasLtMatmulDescAttributes_t::CUBLASLT_MATMUL_DESC_TRANSB,
            &transb as *const i32 as *const std::ffi::c_void,
            std::mem::size_of::<i32>(),
        ).map_err(|e| MLError::ModelError(format!("mamba2 set TRANSB {label}: {e:?}")))?;

        let (a_rows, a_cols, a_ld) = if transa == 1 { (k, m, lda) } else { (m, k, lda) };
        let a_layout = cublaslt_result::create_matrix_layout(
            f32_type, a_rows as u64, a_cols as u64, a_ld as i64,
        ).map_err(|e| MLError::ModelError(format!("mamba2 A layout {label}: {e:?}")))?;

        let (b_rows, b_cols, b_ld) = if transb == 1 { (n, k, ldb) } else { (k, n, ldb) };
        let b_layout = cublaslt_result::create_matrix_layout(
            f32_type, b_rows as u64, b_cols as u64, b_ld as i64,
        ).map_err(|e| MLError::ModelError(format!("mamba2 B layout {label}: {e:?}")))?;

        let c_layout = cublaslt_result::create_matrix_layout(
            f32_type, m as u64, n as u64, ldc as i64,
        ).map_err(|e| MLError::ModelError(format!("mamba2 C layout {label}: {e:?}")))?;

        let d_layout = cublaslt_result::create_matrix_layout(
            f32_type, m as u64, n as u64, ldc as i64,
        ).map_err(|e| MLError::ModelError(format!("mamba2 D layout {label}: {e:?}")))?;

        // ── Deterministic algorithm selection (Option B fix) ──
        // Replaces timing-based `cublasLtMatmulAlgoGetHeuristic` with a
        // hardware-stable AlgoGetIds → AlgoInit → AlgoCheck loop. TF32
        // compute type preserved.
        let shape = super::cublas_algo_deterministic::ShapeKey::new(
            transa, transb,
            m as i32, n as i32, k as i32,
            lda as i32, ldb as i32, ldc as i32,
            ws_size,
        );
        let heuristic = super::cublas_algo_deterministic::get_matmul_algo_f32_tf32(
            lt_handle, matmul_desc, a_layout, b_layout, c_layout, d_layout, shape,
        );
        let algo = heuristic.map_err(|e| MLError::ModelError(
            format!("mamba2 deterministic algo {label} (transa={transa},transb={transb},m={m},n={n},k={k}): {e}")
        ))?.algo;

        info!(%label, m, n, k, lda, ldb, ldc, transa, transb, "Mamba2 GEMM desc created (deterministic)");
        Ok(Mamba2GemmDesc { matmul_desc, a_layout, b_layout, c_layout, d_layout, algo })
    }
}

// ── Event tracking RAII guard ────────────────────────────────────────────────

/// RAII guard that disables cudarc event tracking on creation and
/// Event tracking is permanently disabled for CUDA graph compatibility
/// (see FusedTrainingCtx::new). This guard now only drains stale errors
/// on drop — it does NOT re-enable tracking.
pub(crate) struct EventTrackingGuard<'a> {
    ctx: &'a cudarc::driver::CudaContext,
}

impl<'a> EventTrackingGuard<'a> {
    pub(crate) fn new(ctx: &'a cudarc::driver::CudaContext) -> Self {
        unsafe { ctx.disable_event_tracking(); }
        Self { ctx }
    }
}

impl Drop for EventTrackingGuard<'_> {
    fn drop(&mut self) {
        // Do NOT re-enable event tracking — permanently disabled for
        // graph capture safety. Only drain stale errors.
        let _ = self.ctx.check_err();
    }
}

// ── CUDA Graph wrapper ──────────────────────────────────────────────────────


// ── SP4 Layer A Task A7 fix-up: per-aux-trainer buffer descriptors ─────────
//
// `launch_sp4_param_group_oracles_all_groups` reads `(params, grads, adam_m,
// adam_v)` once per group across 4-5 fused passes. The DQN-internal trunk /
// value / branches param-groups slice the main DQN params buffer that
// `GpuDqnTrainer` already owns. The aux trainers (IQN, IQL hi/lo, Attn,
// Curiosity) live on `FusedTrainingCtx`, so their device pointers must be
// threaded into the launcher; `Sp4ParamGroupBufs` captures one trainer's
// quartet, and `SP4AuxBuffers` aggregates the four aux trainers we *can*
// represent contiguously (Curiosity is the architectural hold-out — see
// `param_group_buffers` doc-comment).

/// SP4 Task A7 fix-up #2: one sub-buffer's `(params, grads, adam_m, adam_v)`
/// device pointers + element count. All four buffers must be the same length
/// (Adam state mirrors the params shape; `count` records that single shared
/// element count). For groups with a single contiguous params buffer (groups
/// 0-6) a `Sp4ParamGroupBufs` holds exactly one of these. For Curiosity
/// (group 7), four — one per `[w1, b1, w2, b2]` sub-tensor.
#[derive(Copy, Clone, Debug)]
pub struct Sp4SubBuffer {
    /// Raw device pointer to this sub-buffer's flat-f32 params slice.
    pub params_ptr: u64,
    /// Raw device pointer to the matched-shape reduced-gradient slice.
    pub grads_ptr: u64,
    /// Raw device pointer to the Adam first-moment slice.
    pub adam_m_ptr: u64,
    /// Raw device pointer to the Adam second-moment slice.
    pub adam_v_ptr: u64,
    /// Element count (f32 entries) shared by all four slices.
    pub count: usize,
}

/// SP4 Task A7 fix-up: per-param-group buffer descriptor for the Pearl B
/// oracle. Most groups (0-6) have a single contiguous flat-f32 params buffer
/// per signal type — `sub_buffers.len() == 1`. Curiosity (group 7) splits
/// its weights into 4 sub-buffers (`w1`, `b1`, `w2`, `b2`) each in its own
/// `CudaSlice<f32>`; `sub_buffers.len() == 4`. The oracle kernel iterates
/// sub-buffers within each pass, treating their union as a single logical
/// distribution for p99/WD_RATE/L1 computation.
///
/// Built host-side from each trainer's `params_ptr()` / `grads_ptr()` /
/// `adam_m_ptr()` / `adam_v_ptr()` accessors (or per-sub-buffer accessors
/// for Curiosity) and consumed by `param_group_buffers` /
/// `launch_sp4_param_group_oracles_all_groups`.
#[derive(Clone, Debug)]
pub struct Sp4ParamGroupBufs {
    /// Sub-buffers in matched-stride order: index `i` corresponds to the
    /// same logical sub-tensor across (params, grads, adam_m, adam_v).
    /// All four pointer arrays MUST be the same length (= `sub_buffers.len()`)
    /// and have matching `count` per index.
    pub sub_buffers: Vec<Sp4SubBuffer>,
}

impl Sp4ParamGroupBufs {
    /// Convenience constructor for groups with a single contiguous buffer
    /// per signal type (groups 0-6).
    pub fn single(
        params_ptr: u64,
        grads_ptr: u64,
        adam_m_ptr: u64,
        adam_v_ptr: u64,
        count: usize,
    ) -> Self {
        Sp4ParamGroupBufs {
            sub_buffers: vec![Sp4SubBuffer { params_ptr, grads_ptr, adam_m_ptr, adam_v_ptr, count }],
        }
    }

    /// Empty descriptor (no sub-buffers). Trips the launcher's `total_count==0`
    /// short-circuit so the kernel launch is silently skipped — used for
    /// optional aux trainers that aren't present in the current configuration.
    pub fn empty() -> Self {
        Sp4ParamGroupBufs { sub_buffers: Vec::new() }
    }

    /// Total element count across all sub-buffers (for kernel `total_count`
    /// arg + p99 normalisation).
    pub fn total_count(&self) -> usize {
        self.sub_buffers.iter().map(|s| s.count).sum()
    }
}

/// SP4 Task A7 fix-up #2: aux-trainer buffer pointers for the param-group
/// statistics oracle. `FusedTrainingCtx` builds this from its child
/// trainers (`gpu_iqn`, `gpu_iql`, `gpu_iql_low`, `gpu_attention`,
/// `gpu_curiosity`) and passes it to
/// `launch_sp4_param_group_oracles_all_groups`.
///
/// The mapping `aux_buffers.{field} → ParamGroup::{variant}` is fixed by
/// `param_group_buffers`:
///   - `iqn`       → group 3 (`ParamGroup::Iqn`)
///   - `iql_high`  → group 4 (`ParamGroup::IqlHigh`)
///   - `iql_low`   → group 5 (`ParamGroup::IqlLow`)
///   - `attn`      → group 6 (`ParamGroup::Attn`)
///   - `curiosity` → group 7 (`ParamGroup::Curiosity`) — multi-sub-buffer
///     descriptor (`Sp4ParamGroupBufs::sub_buffers.len() == 4` for
///     `[w1, b1, w2, b2]`); the kernel iterates them as one logical group.
///
/// All five aux trainers are *unconditional* in `FusedTrainingCtx` —
/// IQN/Attn used to be `Option<_>` for graceful-degrade reasons that
/// were resolved before this fix-up. If a future change reintroduces
/// optional aux trainers, the launcher would need a per-group "present"
/// flag; for now `aux_buffers.iqn`, `aux_buffers.iql_high`, etc. are
/// always live device pointers.
#[derive(Clone, Debug)]
pub struct SP4AuxBuffers {
    /// IQN online-network params + grads + Adam state.
    pub iqn: Sp4ParamGroupBufs,
    /// IQL τ-high V(s) params + grads + Adam state.
    pub iql_high: Sp4ParamGroupBufs,
    /// IQL τ-low V(s) params + grads + Adam state.
    pub iql_low: Sp4ParamGroupBufs,
    /// Multi-head attention params + grads + Adam state.
    pub attn: Sp4ParamGroupBufs,
    /// Curiosity forward-model params + grads + Adam state. Multi-sub-buffer
    /// descriptor: 4 entries for `[w1, b1, w2, b2]`. The oracle kernel
    /// treats the union as one logical group.
    pub curiosity: Sp4ParamGroupBufs,
}


// ── Configuration ───────────────────────────────────────────────────────────

/// Network dimensions and training hyperparameters for the fused CUDA trainer.
#[derive(Debug, Clone)]
pub struct GpuDqnTrainConfig {
    /// Input state dimension (e.g., 48 or 56 with OFI).
    pub state_dim: usize,
    /// First shared hidden layer width (default: 256).
    pub shared_h1: usize,
    /// Second shared hidden layer width (default: 256).
    pub shared_h2: usize,
    /// Value head hidden width (default: 128).
    pub value_h: usize,
    /// Advantage/branch head hidden width (default: 128).
    pub adv_h: usize,
    /// Number of C51 distributional atoms (default: 51).
    pub num_atoms: usize,
    /// C51 minimum support value (default: -50.0).
    pub v_min: f32,
    /// C51 maximum support value (default: 50.0).
    pub v_max: f32,
    /// Branch 0 (direction) action count (default: 3 — Short/Flat/Long).
    pub branch_0_size: usize,
    /// Branch 1 (magnitude) action count (default: 3 — 25%/50%/100%).
    pub branch_1_size: usize,
    /// Branch 2 (order) action count (default: 3).
    pub branch_2_size: usize,
    /// Branch 3 (urgency) action count (default: 3 — Patient/Normal/Aggressive).
    pub branch_3_size: usize,
    /// Training batch size (fixed for CUDA Graph capture).
    pub batch_size: usize,
    /// Discount factor for Bellman projection.
    pub gamma: f32,
    /// Learning rate for Adam optimizer.
    pub lr: f32,
    /// Adam β1 (first moment decay).
    pub beta1: f32,
    /// Adam β2 (second moment decay).
    pub beta2: f32,
    /// Adam ε (numerical stability).
    pub epsilon: f32,
    // SP4 Layer B: `weight_decay` field removed; per-group AdamW rates are
    // sourced from ISV[WD_RATE[group]] inside `launch_adam_update`'s 3-way
    // sub-launch loop (DqnTrunk / DqnValue / DqnBranches) and threaded
    // through to IQN/IQL/Attn/Curiosity launchers via per-launch
    // `weight_decay_value: f32` parameters.
    /// Maximum gradient L2 norm for clipping.
    pub max_grad_norm: f32,
    /// Spectral norm clipping target (σ_max). Constrains ||W||_σ ≤ σ_max.
    /// Default 3.0 (permits natural Xavier-init scaling, prevents explosion).
    /// Lower values = tighter constraint = more Q-value stability but less capacity.
    pub spectral_norm_sigma_max: f32,
    /// Spectral decoupling: L2 penalty on Q-value logit magnitudes.
    /// Pezeshki et al. 2021. Default: 0.01.
    pub spectral_decoupling_lambda: f32,
    /// N-step returns (default: 1). When > 1, the experience collector pre-computes
    /// R_n = sum(gamma^i * r_{t+i}) and the C51 Bellman uses gamma^n.
    pub n_steps: usize,
    /// IQN quantile loss weight relative to C51 (0.0 = C51 only)
    pub iqn_lambda: f32,
    /// Number of IQN quantile samples per forward pass (0 = IQN disabled)
    pub iqn_num_quantiles: usize,
    /// Cosine embedding dimension for IQN τ encoding
    pub iqn_embedding_dim: usize,
    /// Huber threshold for quantile loss
    pub iqn_kappa: f32,
    /// Entropy regularization coefficient for C51 distributions.
    /// Adds -coeff * H(p) to the loss, preventing distribution collapse to a single atom.
    /// 0.0 = disabled. Canonical default: 0.001.
    pub entropy_coefficient: f32,
    /// Number of epochs to use MSE loss on expected Q-values before switching to C51.
    /// MSE provides stronger gradient signal during early training.
    pub c51_warmup_epochs: usize,
    /// CQL regularization strength (0.0 = disabled, 0.1 = mild, 1.0 = full offline-RL).
    pub cql_alpha: f32,
    /// Curiosity Q-penalty lambda: scales gamma by 1/(1 + lambda * curiosity_error).
    /// 0.0 = disabled. Typical range 0.5-5.0. Higher = more aggressive penalization
    /// of Q-values in novel states (prevents overconfident extrapolation OOS).
    pub curiosity_q_penalty_lambda: f32,
    /// #18 Asymmetric drawdown loss weight. Extra penalty on Q-overestimation
    /// when portfolio is in drawdown. 0.0 = disabled.
    pub asymmetric_dd_weight: f32,
    /// #27 Ensemble disagreement penalty weight. Q_target -= weight * ensemble_std.
    /// 0.0 = disabled.
    pub ensemble_disagreement_penalty: f32,
    /// #21 Stochastic depth drop probability. 0.0 = disabled, 0.2 = 20% drop per layer.
    pub stochastic_depth_prob: f32,
    /// #34 Causal regularization weight.
    pub causal_weight: f64,
    /// #34 Run causal intervention every N training steps.
    pub causal_intervention_interval: usize,

    /// #20 Epoch at which to compute lottery ticket pruning mask.
    pub pruning_epoch: usize,
    /// #20 Fraction of weights to prune (0.7 = remove 70% smallest).
    pub pruning_fraction: f32,

    /// #31 Temporal causal bottleneck dimension. 0 = disabled (no bottleneck).
    /// 2 = maximum compression (14 market features → 2 abstract dimensions).
    pub bottleneck_dim: usize,
    /// Market feature dimension (always 42). Used by bottleneck to separate
    /// market features from portfolio+MTF+OFI features in the state vector.
    /// OFI features (8 dims, when enabled) bypass the bottleneck via portfolio_dim.
    pub market_dim: usize,
    /// Total number of training epochs for this run. Written to ISV[TOTAL_EPOCHS_INDEX]
    /// at constructor time so GPU kernels can compute epoch-fraction progress without
    /// a CPU→GPU round-trip. Default 0 means "unknown / single fold"; set this from
    /// `DQNHyperparameters::epochs` at construction.
    pub total_epochs: usize,

    /// Plan 3 Task 8 B.3: number of experience-collection samples that should be
    /// drawn from the GPU scripted-policy mixture before handing action selection
    /// back to the network. Written to ISV[SEED_STEPS_TARGET_INDEX=82] at
    /// constructor time. Default 100_000 (≈ first ~5 epochs at production
    /// batch_size×timesteps); set to 0 to disable the seed phase entirely
    /// (the dispatch branch in the launcher will see DONE >= TARGET on the
    /// very first call and route to the network kernel). Smoke configs may
    /// override to a smaller value (e.g. 10_000) so the seed→network transition
    /// is observable inside the smoke run.
    pub replay_seed_steps: u32,

    /// MoE load-balance loss adaptive controller — permanent floor weight.
    ///
    /// `λ_eff = moe_lambda_floor + moe_lambda_max_extra × deficit`, where
    /// `deficit ∈ [0, 1]` is the normalised entropy shortfall against
    /// `moe_entropy_target_frac × ln(K)`. The producer kernel
    /// `moe_lambda_eff_update` writes ISV[MOE_LAMBDA_EFF_INDEX=128] each
    /// step; consumer kernel `moe_load_balance_loss` reads it at runtime.
    ///
    /// Default 0.01 (matches the legacy static `moe_lambda` default — the
    /// old "anti-collapse only" baseline becomes the permanent minimum so
    /// the controller never weakens below the previous behaviour).
    pub moe_lambda_floor: f32,
    /// Maximum additional λ added when the gate is fully collapsed
    /// (`ent_ema = 0` ⇒ `deficit = 1`). Peak λ is `floor + max_extra`.
    /// Default 0.09 (so peak λ = 0.10, 10× boost over the floor).
    pub moe_lambda_max_extra: f32,
    /// Fraction of `ln(K)` treated as "diverse enough"; entropy at or above
    /// `entropy_target_frac × ln(K)` produces zero deficit (λ_eff = floor).
    /// Default 0.7 (target = 70% of ln(K) ≈ 1.456 for K=8).
    pub moe_entropy_target_frac: f32,
}

impl Default for GpuDqnTrainConfig {
    fn default() -> Self {
        Self {
            state_dim: 48,   // Actual aligned feature dim (40 market + 3 portfolio = 43, padded to 48)
            shared_h1: 256,
            shared_h2: 256,
            value_h: 128,
            adv_h: 128,
            num_atoms: 51,
            // v_min/v_max: tight support for 16× atom resolution (delta_z=0.6 vs 9.6)
            v_min: -15.0,
            v_max: 15.0,
            branch_0_size: 3,
            branch_1_size: 3,
            branch_2_size: 3,
            branch_3_size: 3,
            batch_size: 64,   // Smoke test default (production overrides via from_hyperparams)
            gamma: 0.99,
            n_steps: 1,
            lr: 3e-5,     // Conservative default for stable training
            beta1: 0.9,
            beta2: 0.999,
            epsilon: 1e-8,
            // SP4 Layer B: weight_decay field removed; sourced from ISV at runtime.
            max_grad_norm: 10000.0, // Safety-only — per-component clip removed, raw norms ~4000
            spectral_norm_sigma_max: 3.0,
            spectral_decoupling_lambda: 0.01,
            iqn_lambda: 0.25,
            iqn_num_quantiles: 0,
            iqn_embedding_dim: 64,
            iqn_kappa: 1.0,
            entropy_coefficient: 0.001, // Must be ≤ reward magnitude (~0.001). Old 0.01 was 10x reward → entropy dominated learning.
            c51_warmup_epochs: 5,
            cql_alpha: 0.1,
            curiosity_q_penalty_lambda: 0.0,
            asymmetric_dd_weight: 0.0,
            ensemble_disagreement_penalty: 0.0,
            stochastic_depth_prob: 0.0,
            pruning_epoch: 50,
            pruning_fraction: 0.7,
            causal_weight: 0.1,
            causal_intervention_interval: 50,
            bottleneck_dim: 16,
            market_dim: 42, // Default: 42 base features. Overridden to 50 when OFI (MBP-10) enabled.
            total_epochs: 0,
            replay_seed_steps: 100_000,
            moe_lambda_floor: 0.01,
            moe_lambda_max_extra: 0.09,
            moe_entropy_target_frac: 0.7,
        }
    }
}


/// Result of a single fused training step.
#[derive(Debug, Clone)]
pub struct FusedTrainResult {
    /// Batch-mean weighted loss (for logging).
    pub total_loss: f32,
    /// Per-sample TD errors for PER priority update.
    pub td_errors: Vec<f32>,
    /// Pre-clip gradient L2 norm (for monitoring).
    pub grad_norm: f32,
}

/// Q-value statistics computed entirely on GPU (7 scalars, 28-byte readback).
#[derive(Debug, Clone, Copy)]
pub struct QValueStatsResult {
    /// Average of per-sample max Q-values
    pub avg_max_q: f64,
    /// Global minimum Q-value
    pub q_min: f32,
    /// Global maximum Q-value
    pub q_max: f32,
    /// Mean of all Q-values
    pub q_mean: f32,
    /// Variance of all Q-values
    pub q_variance: f32,
    /// Normalized atom entropy [0,1]: 1.0 = uniform, 0.0 = point mass
    pub atom_entropy: f32,
    /// Atom utilization fraction [0,1]: fraction of atoms carrying >0.5/NA probability
    pub atom_utilization: f32,
}

/// Per-branch Q-value statistics — one 7-tuple per action branch (direction=0,
/// magnitude=1, order=2, urgency=3). Produced by `q_stats_per_branch_reduce`
/// kernel; consumed by `update_eval_v_range` to track per-branch EMA state
/// and write the 8 ISV v-range slots.
///
/// The per-branch atom-entropy / utilization slots in the kernel output are
/// left at 0 (per-branch atom_stats would require splitting the global
/// accumulator). `QValueStatsResult.atom_{entropy,utilization}` inherit 0
/// until that wiring lands — the v-range EMA updater doesn't use those two
/// fields, so the zero does not propagate into the ISV path.
#[derive(Debug, Clone, Copy)]
pub struct PerBranchQValueStats {
    pub per_branch: [QValueStatsResult; 4],
}

/// Scalar-only result from the fused GPU training path.
///
/// TD errors stay on GPU (`td_errors_buf`) — no readback. PER priority update
/// is done by a dedicated CUDA kernel that reads `td_errors_buf` directly.
/// Only 8 bytes DtoH per step (loss + grad_norm).
#[derive(Debug, Clone, Copy)]
pub struct FusedTrainScalars {
    /// Batch-mean weighted loss (for logging).
    pub total_loss: f32,
    /// Pre-clip gradient L2 norm (for monitoring).
    pub grad_norm: f32,
}

// ── Parameter layout ────────────────────────────────────────────────────────
//
// Matches GOFF_* / PARAM_* defines in dqn_training_kernel.cu exactly.
// 20 weight tensors in order: w_s1, b_s1, w_s2, b_s2, w_v1, b_v1, w_v2, b_v2,
// w_b0fc, b_b0fc, w_b0out, b_b0out, w_b1fc, b_b1fc, w_b1out, b_b1out,
// w_b2fc, b_b2fc, w_b2out, b_b2out.

/// #34 Causal intervention configuration.
#[derive(Clone)]
pub(crate) struct CausalInterventionConfig {
    pub weight: f32,
    pub interval: usize,
    pub market_dim: usize,
}

impl Default for CausalInterventionConfig {
    fn default() -> Self {
        Self { weight: 0.0, interval: 10, market_dim: 42 } // market_dim overridden at construction
    }
}

/// Number of weight tensors in the flat parameter buffer.
/// 20 original (DQN network) + 4 branch 3 (magnitude) + 2 bottleneck (w_bn, b_bn)
/// + 8 VSN bottleneck (R=16) + 8 GLU gate + 8 KAN spline (coeff+resid per branch)
/// + 2 regime branch gate (w_regime + b_regime) + 4 adaptive atom spacing
/// + 8 multi-horizon value heads (5-bar and 20-bar)
/// + 4 learned risk management (5th branch) = 68
/// + 10 ISV encoder/conditioning + 2 feature gate + 2 temporal routing = 82
/// + 4 trade plan head = 86 → indices [0..95).
/// + 24 Plan 4 Task 1B-ii VSN per-group MLP params (4 tensors × 6 groups —
///   w1_g/b1_g/w2_g/b2_g per `FEATURE_GROUP_RANGES` entry, hidden dim
///   `VSN_HIDDEN_DIM = 16`) → indices [95..119). Producer-only params in
///   this commit; the `vsn_softmax_and_gate_forward` consumer kernel and
///   the cuBLAS `Linear_1`/`Linear_2` wire-in land in 1B-iii.
/// + 8 Plan 4 Task 6 Commit A aux-head MLP params (2 heads × 4 tensors —
///   `aux_nb_{w1,b1,w2,b2}` for the next-bar regression head with K=1,
///   `aux_rg_{w1,b1,w2,b2}` for the regime classification head with K=5,
///   shared hidden dim `AUX_HIDDEN_DIM = 32`) → indices [119..127).
///   Producer-only params in this commit; the `aux_*_forward` consumer
///   kernels and the training-loop loss accumulation wire-in land in
///   Commit B. Adam SAXPY iterates `0..NUM_WEIGHT_TENSORS` so these
///   tensors will receive zero-gradient SAXPYs each step until Commit B
///   activates the backward — Adam keeps them at Xavier init meanwhile,
///   which is the intended dormant state.
/// + 36 MoE params (Phase 1 Task 1.6): 4 gate tensors + 8 experts × 4 tensors
///   → indices [127..163). Gate: gate_w1[SD,64], gate_b1[64], gate_w2[64,8],
///   gate_b2[8]. Per expert: w1[SH2,64], b1[64], w2[64,SH2], b2[SH2].
///   Allocated but not yet wired into forward/backward — Phase 3 wire-up.
/// SP22 H6 vNext Phase B1 (2026-05-14): bumped 163 → 167 to host the
/// trade-outcome aux head's 4 weight tensors at indices [163..167):
///   [163] aux_to_w1 [H=128, SH2=256]  Xavier
///   [164] aux_to_b1 [H=128]            zero
///   [165] aux_to_w2 [K=3, H=128]       Xavier
///   [166] aux_to_b2 [K=3]               zero
///
/// Adam m/v moment buffers and the SAXPY iteration pick up the new
/// tensors uniformly via `for i in 0..NUM_WEIGHT_TENSORS` loops — no
/// per-tensor Adam wiring needed.
///
/// Sized with Phase A3 SH2=256 (mirrors the K=2 next-bar head). The
/// spec's Phase B input concat (256 → 262 with `plan_params`) will
/// land later as a small change to `compute_param_sizes()` slot [163]
/// shape + the kernel-launch SH2 arg.
pub(crate) const NUM_WEIGHT_TENSORS: usize = 167;

/// Compute the size (element count) of each weight tensor.
///
/// Tensors 0-19: standard DQN network (shared trunk + value + branches 0-2).
/// Tensors 20-23: branch 3 (urgency) weights.
/// Tensors 24-25: temporal causal bottleneck (0 elements when bottleneck_dim=0).
/// Tensors 26-33: Variable Selection Network bottleneck (R=16).
/// Tensors 34-41: GLU gate weights and biases.
/// Tensors 42-49: KAN spline coefficients and residual weights (per branch).
/// Tensors 50-51: Regime branch gate (W_regime[4,4] + b_regime[4]).
/// Tensors 52-55: Adaptive atom spacing (spacing_raw per branch, NA each).
/// Tensors 56-63: Multi-horizon value heads (5-bar and 20-bar).
/// Tensors 64-67: Learned risk management (5th branch).
/// Tensors 68-77: Introspective State Vector (ISV) encoder + conditioning.
/// Tensors 78-79: ISV feature gate (per-feature h_s2 modulation).
/// Tensors 80-81: ISV temporal routing (per-feature temporal depth).
/// Tensors 82-85: Trade plan head (h_s2 → 6 plan parameters).
///
/// When bottleneck is active, w_s1 input dimension changes from state_dim to
/// (bottleneck_dim + portfolio_dim + 1) where portfolio_dim = state_dim - market_dim
/// and the trailing +1 column is the SP15 Wave 4.1b dd_pct broadcast (read from
/// ISV[DD_PCT_INDEX=406] by `bn_tanh_concat_dd_kernel`). The +1 is folded into
/// `s1_input_dim` so all consumers (compute_param_sizes, Xavier init, cuBLAS
/// gemm cache, encoder_backward_chain dX width, GRN w_s1 / w_residual_h_s1
/// fan-in) treat dd_pct as the (s1_input_dim)th feature column. The bn_concat
/// buffer is sized [B, bn_dim + portfolio_dim + 1] = [B, s1_input_dim] in the
/// bottleneck-on path.
pub(crate) fn compute_param_sizes(cfg: &GpuDqnTrainConfig) -> [usize; NUM_WEIGHT_TENSORS] {
    // When bottleneck is active, h_s1 input is [bottleneck_dim + portfolio_features + 1]
    // instead of [state_dim]. The trailing +1 column carries dd_pct from ISV (SP15
    // Wave 4.1b). portfolio_features = state_dim - market_dim (typically 6-8).
    let s1_input_dim = if cfg.bottleneck_dim > 0 {
        cfg.bottleneck_dim + (ml_core::state_layout::STATE_DIM.saturating_sub(cfg.market_dim)) + 1
    } else {
        ml_core::state_layout::STATE_DIM
    };
    let bn_dim = cfg.bottleneck_dim;
    let market_dim = cfg.market_dim;

    let mut sizes = [0usize; NUM_WEIGHT_TENSORS];
    let core: [usize; 95] = [
        // ── h_s1 GRN block (Plan 4 Task 2c.3a) ───────────────────────
        // GRN(x) = LayerNorm(Linear_residual(x) + GLU(Linear_b(ELU(Linear_a(x))))).
        // h_s1 maps SD → SH1 (dims differ → Linear_residual is a real GEMM).
        cfg.shared_h1 * s1_input_dim,                       // [0]  w_a_h_s1         [SH1, SD] (Linear_a)
        cfg.shared_h1,                                       // [1]  b_a_h_s1         [SH1]
        (2 * cfg.shared_h1) * cfg.shared_h1,                 // [2]  w_b_h_s1         [2*SH1, SH1] (Linear_b → GLU split)
        2 * cfg.shared_h1,                                   // [3]  b_b_h_s1         [2*SH1]
        cfg.shared_h1 * s1_input_dim,                        // [4]  w_residual_h_s1  [SH1, SD] (Linear_residual)
        cfg.shared_h1,                                       // [5]  gamma_h_s1       [SH1] (LN affine scale)
        cfg.shared_h1,                                       // [6]  beta_h_s1        [SH1] (LN affine shift)
        // ── h_s2 GRN block (Plan 4 Task 2c.3a) ───────────────────────
        // h_s2 maps SH1 → SH2 with SH1 == SH2 → identity residual, no Linear_residual.
        cfg.shared_h2 * cfg.shared_h1,                       // [7]  w_a_h_s2         [SH2, SH1]
        cfg.shared_h2,                                       // [8]  b_a_h_s2         [SH2]
        (2 * cfg.shared_h2) * cfg.shared_h2,                 // [9]  w_b_h_s2         [2*SH2, SH2]
        2 * cfg.shared_h2,                                   // [10] b_b_h_s2         [2*SH2]
        cfg.shared_h2,                                       // [11] gamma_h_s2       [SH2]
        cfg.shared_h2,                                       // [12] beta_h_s2        [SH2]
        cfg.value_h * cfg.shared_h2,                        // [13] w_v1
        cfg.value_h,                                         // [14] b_v1
        cfg.num_atoms * cfg.value_h,                        // [15] w_v2
        cfg.num_atoms,                                       // [16] b_v2
        cfg.adv_h * (cfg.shared_h2 + 1),                    // [17] w_b0fc (SP14 aux→Q wire: SH2+1 input — last col = aux_softmax_diff)
        cfg.adv_h,                                           // [18] b_b0fc
        cfg.branch_0_size * cfg.num_atoms * cfg.adv_h,     // [19] w_b0out
        cfg.branch_0_size * cfg.num_atoms,                  // [20] b_b0out
        cfg.adv_h * (cfg.shared_h2 + cfg.branch_0_size),    // [21] w_b1fc (direction-conditioned: SH2+b0 input)
        cfg.adv_h,                                           // [22] b_b1fc
        cfg.branch_1_size * cfg.num_atoms * cfg.adv_h,     // [23] w_b1out
        cfg.branch_1_size * cfg.num_atoms,                  // [24] b_b1out
        cfg.adv_h * (cfg.shared_h2 + 3),                     // [25] w_b2fc (OFI-conditioned)
        cfg.adv_h,                                           // [26] b_b2fc
        cfg.branch_2_size * cfg.num_atoms * cfg.adv_h,     // [27] w_b2out
        cfg.branch_2_size * cfg.num_atoms,                  // [28] b_b2out
        cfg.adv_h * (cfg.shared_h2 + 3),                   // [29] w_b3fc (OFI-conditioned)
        cfg.adv_h,                                           // [30] b_b3fc
        cfg.branch_3_size * cfg.num_atoms * cfg.adv_h,     // [31] w_b3out
        cfg.branch_3_size * cfg.num_atoms,                  // [32] b_b3out
        // #31 Temporal Causal Bottleneck (0 when disabled)
        bn_dim * market_dim,                                 // [33] w_bn [bottleneck_dim, market_dim]
        bn_dim,                                              // [34] b_bn [bottleneck_dim]
        // ── Variable Selection bottleneck (R=16) ──
        cfg.shared_h2 * 16,                                  // [35] w_vsn1_0 [SH2, R]
        16 * cfg.shared_h2,                                  // [36] w_vsn2_0 [R, SH2]
        cfg.shared_h2 * 16,                                  // [37] w_vsn1_1
        16 * cfg.shared_h2,                                  // [38] w_vsn2_1
        cfg.shared_h2 * 16,                                  // [39] w_vsn1_2
        16 * cfg.shared_h2,                                  // [40] w_vsn2_2
        cfg.shared_h2 * 16,                                  // [41] w_vsn1_3
        16 * cfg.shared_h2,                                  // [42] w_vsn2_3
        // ── GLU gate weights ──
        cfg.adv_h * cfg.shared_h2,                           // [43] w_gate_0 [AH, SH2]
        cfg.adv_h,                                           // [44] b_gate_0
        cfg.adv_h * (cfg.shared_h2 + cfg.branch_0_size),    // [45] w_gate_1 [AH, SH2+b0] (direction-conditioned)
        cfg.adv_h,                                           // [46] b_gate_1
        cfg.adv_h * (cfg.shared_h2 + 3),                     // [47] w_gate_2 (OFI-conditioned)
        cfg.adv_h,                                           // [48] b_gate_2
        cfg.adv_h * (cfg.shared_h2 + 3),                   // [49] w_gate_3 (OFI-conditioned)
        cfg.adv_h,                                           // [50] b_gate_3
        // ── KAN spline coefficients (8 bases per neuron) + residual weight ──
        cfg.adv_h * 8,                                       // [51] kan_coeff_0 [AH, 8]
        cfg.adv_h,                                           // [52] kan_resid_0 [AH]
        cfg.adv_h * 8,                                       // [53] kan_coeff_1 [AH, 8]
        cfg.adv_h,                                           // [54] kan_resid_1 [AH]
        cfg.adv_h * 8,                                       // [55] kan_coeff_2 [AH, 8]
        cfg.adv_h,                                           // [56] kan_resid_2 [AH]
        cfg.adv_h * 8,                                       // [57] kan_coeff_3 [AH, 8]
        cfg.adv_h,                                           // [58] kan_resid_3 [AH]
        // ── Regime branch gate ──
        4 * 4,                                               // [59] w_regime [4, 4]
        4,                                                   // [60] b_regime [4]
        // ── Adaptive atom positions (spacing_raw per branch) ──
        cfg.num_atoms,                                       // [61] spacing_raw_0 [NA] direction
        cfg.num_atoms,                                       // [62] spacing_raw_1 [NA] magnitude
        cfg.num_atoms,                                       // [63] spacing_raw_2 [NA] order
        cfg.num_atoms,                                       // [64] spacing_raw_3 [NA] urgency
        // ── Multi-horizon value heads (5-bar and 20-bar) ──
        cfg.value_h * cfg.shared_h2,                    // [65] w_v1_5bar [VH, SH2]
        cfg.value_h,                                     // [66] b_v1_5bar [VH]
        cfg.num_atoms * cfg.value_h,                    // [67] w_v2_5bar [NA, VH]
        cfg.num_atoms,                                   // [68] b_v2_5bar [NA]
        cfg.value_h * cfg.shared_h2,                    // [69] w_v1_20bar [VH, SH2]
        cfg.value_h,                                     // [70] b_v1_20bar [VH]
        cfg.num_atoms * cfg.value_h,                    // [71] w_v2_20bar [NA, VH]
        cfg.num_atoms,                                   // [72] b_v2_20bar [NA]
        // ── Learned risk management (5th branch) ──
        cfg.adv_h * (cfg.shared_h2 + 13),                // [73] w_risk_fc [AH, SH2+13]
        cfg.adv_h,                                       // [74] b_risk_fc [AH]
        cfg.adv_h,                                       // [75] w_risk_out [AH]
        1,                                               // [76] b_risk_out [1]
        // ── Introspective State Vector (ISV) encoder + conditioning ──
        16 * ISV_DIM,                                   // [77] w_isv_fc1 [16, ISV_DIM]
        16,                                              // [78] b_isv_fc1 [16]
        ISV_EMB_DIM * 16,                               // [79] w_isv_fc2 [EMB, 16]
        ISV_EMB_DIM,                                     // [80] b_isv_fc2 [EMB]
        4 * ISV_EMB_DIM,                                // [81] w_isv_gate [4, EMB]
        4,                                               // [82] b_isv_gate [4]
        ISV_EMB_DIM,                                     // [83] w_isv_gamma [EMB]
        1,                                               // [84] b_isv_gamma [1]
        cfg.shared_h2,                                   // [85] w_conf_fc [SH2]
        1,                                               // [86] b_conf_fc [1]
        // ── ISV Feature Gating (Phase 3) ──
        cfg.shared_h2 * ISV_EMB_DIM,                    // [87] w_feature_gate [SH2, 8]
        cfg.shared_h2,                                   // [88] b_feature_gate [SH2]
        // ── ISV Temporal Routing (Phase 3) ──
        cfg.shared_h2 * ISV_EMB_DIM,                    // [89] w_temporal_route [SH2, 8]
        cfg.shared_h2,                                   // [90] b_temporal_route [SH2]
        // ── Trade Plan Head ──
        cfg.adv_h * cfg.shared_h2,                      // [91] w_plan_fc [AH, SH2]
        cfg.adv_h,                                       // [92] b_plan_fc [AH]
        6 * cfg.adv_h,                                   // [93] w_plan_out [6, AH]
        6,                                                // [94] b_plan_out [6]
    ];
    sizes[..95].copy_from_slice(&core);

    // ── Plan 4 Task 1B-ii: VSN per-group MLP params (24 tensors) ───
    // 4 tensors per group × 6 groups (`FEATURE_GROUP_RANGES`).
    // Per-group total = VSN_HIDDEN_DIM * group_dim_g + VSN_HIDDEN_DIM (b1)
    //                 + VSN_HIDDEN_DIM (w2 flat) + 1 (b2)
    //                 = 16 * group_dim_g + 33.
    // Sum over groups (market=42, ofi=32, tlob=16, mtf=16, portfolio=8,
    // plan_isv=7) = 16*121 + 6*33 = 1936 + 198 = 2134 floats.
    // No production callers in this commit — params consumed by 1B-iii
    // (forward orchestrator + Linear_1/Linear_2 cuBLAS wire-in) and
    // 1B-iv (backward orchestrator).
    let group_ranges = ml_core::state_layout::FEATURE_GROUP_RANGES;
    let mut idx = 95;
    let mut g = 0;
    while g < ml_core::state_layout::SL_NUM_FEATURE_GROUPS {
        let group_dim = group_ranges[g].1 - group_ranges[g].0;
        sizes[idx]     = VSN_HIDDEN_DIM * group_dim; // vsn_w1_g{g} [VSN_HIDDEN_DIM, group_dim_g]
        sizes[idx + 1] = VSN_HIDDEN_DIM;             // vsn_b1_g{g} [VSN_HIDDEN_DIM]
        sizes[idx + 2] = VSN_HIDDEN_DIM;             // vsn_w2_g{g} [1, VSN_HIDDEN_DIM] (flat-stored)
        sizes[idx + 3] = 1;                          // vsn_b2_g{g} [1]
        idx += 4;
        g += 1;
    }
    debug_assert!(idx == 119,
        "compute_param_sizes: VSN tail fill ended at {} but expected 119 before aux-heads",
        idx);

    // ── Plan 4 Task 6 Commit A: aux-head MLP params (8 tensors) ───
    // Two heads: next-bar regression (K=1) + regime classification (K=5).
    // Each head is `Linear(SH2 → AUX_HIDDEN_DIM=32) → ELU → Linear(32 → K)`.
    // Per-head total = 32*SH2 + 32 + K*32 + K. Sum over both heads:
    // 2*32*SH2 + 2*32 + (2+5)*32 + (2+5) = 2*32*SH2 + 64 + 224 + 7
    //                                    = 2*32*SH2 + 295 floats.
    // SP13 B1.1a (2026-05-05): next-bar head K_out flipped 1 → 2;
    // [121] grew from `H` (= 32) to `K_NB * H` (= 64), [122] grew from `1`
    // to `K_NB` (= 2). Mirrors the regime row's expression for K=5.
    let sh2 = cfg.shared_h2;
    sizes[119] = crate::cuda_pipeline::gpu_aux_heads::AUX_HIDDEN_DIM * sh2; // [119] aux_nb_w1 [32, SH2]
    sizes[120] = crate::cuda_pipeline::gpu_aux_heads::AUX_HIDDEN_DIM;       // [120] aux_nb_b1 [32]
    sizes[121] = crate::cuda_pipeline::gpu_aux_heads::AUX_NEXT_BAR_K
                 * crate::cuda_pipeline::gpu_aux_heads::AUX_HIDDEN_DIM;     // [121] aux_nb_w2 [K_NB=2, 32]
    sizes[122] = crate::cuda_pipeline::gpu_aux_heads::AUX_NEXT_BAR_K;       // [122] aux_nb_b2 [K_NB=2]
    sizes[123] = crate::cuda_pipeline::gpu_aux_heads::AUX_HIDDEN_DIM * sh2; // [123] aux_rg_w1 [32, SH2]
    sizes[124] = crate::cuda_pipeline::gpu_aux_heads::AUX_HIDDEN_DIM;       // [124] aux_rg_b1 [32]
    sizes[125] = crate::cuda_pipeline::gpu_aux_heads::AUX_REGIME_K
                 * crate::cuda_pipeline::gpu_aux_heads::AUX_HIDDEN_DIM;     // [125] aux_rg_w2 [5, 32]
    sizes[126] = crate::cuda_pipeline::gpu_aux_heads::AUX_REGIME_K;         // [126] aux_rg_b2 [5]
    debug_assert!(
        {
            // VSN fill should have ended at 119 and aux-heads fill indices [119..127).
            // Check the last aux-head slot was written before proceeding to MoE.
            sizes[126] > 0 || crate::cuda_pipeline::gpu_aux_heads::AUX_REGIME_K == 0
        },
        "compute_param_sizes: aux-head tail sanity failed before MoE extension"
    );

    // ── Phase 1 Task 1.6: MoE params (36 tensors, indices [127..163)) ─────────
    // Gate subnetwork (4 tensors at [127..131)):
    //   gate_w1 [STATE_DIM, MOE_GATE_HIDDEN]   — zero-init so g(s) = uniform 1/K initially
    //   gate_b1 [MOE_GATE_HIDDEN]
    //   gate_w2 [MOE_GATE_HIDDEN, MOE_NUM_EXPERTS]
    //   gate_b2 [MOE_NUM_EXPERTS]
    // 8 experts × 4 tensors (4 per expert at [131..163)):
    //   expert_k_w1 [cfg.shared_h2, MOE_EXPERT_BOTTLENECK]  — Xavier init
    //   expert_k_b1 [MOE_EXPERT_BOTTLENECK]
    //   expert_k_w2 [MOE_EXPERT_BOTTLENECK, cfg.shared_h2]
    //   expert_k_b2 [cfg.shared_h2]
    // Not yet wired into forward/backward — Phase 3 wire-up.
    // Adam SAXPY iterates 0..NUM_WEIGHT_TENSORS so these tensors receive
    // zero-gradient SAXPYs until Phase 3 activates the backward chain,
    // keeping them at init state meanwhile (correct dormant behaviour).
    let sd = ml_core::state_layout::STATE_DIM;
    // Gate tensors [127..131)
    sizes[127] = sd * MOE_GATE_HIDDEN;                  // gate_w1  [SD, MOE_GATE_HIDDEN]
    sizes[128] = MOE_GATE_HIDDEN;                        // gate_b1  [MOE_GATE_HIDDEN]
    sizes[129] = MOE_GATE_HIDDEN * MOE_NUM_EXPERTS;     // gate_w2  [MOE_GATE_HIDDEN, K]
    sizes[130] = MOE_NUM_EXPERTS;                        // gate_b2  [MOE_NUM_EXPERTS]
    // Expert tensors [131..163) — 4 per expert, 8 experts
    {
        let sh2 = cfg.shared_h2;
        let btn = MOE_EXPERT_BOTTLENECK;
        let mut eidx = 131usize;
        let mut k = 0usize;
        while k < MOE_NUM_EXPERTS {
            sizes[eidx]     = sh2 * btn;  // expert_k_w1 [SH2, BTN]
            sizes[eidx + 1] = btn;         // expert_k_b1 [BTN]
            sizes[eidx + 2] = btn * sh2;  // expert_k_w2 [BTN, SH2]
            sizes[eidx + 3] = sh2;         // expert_k_b2 [SH2]
            eidx += 4;
            k += 1;
        }
        debug_assert!(eidx == 163,
            "compute_param_sizes: MoE expert fill ended at {} but expected 163",
            eidx);
    }

    // ── SP22 H6 vNext Phase B1 (2026-05-14): trade-outcome aux head ──
    // Architecture: `Linear(SH2 → AUX_HIDDEN_DIM) → ELU → Linear(H → K=3)`.
    // Same shape as `aux_nb_*` (K=2 sibling) but K_OUT = AUX_OUTCOME_K = 3.
    //
    // Indices: aux_to_w1=163, aux_to_b1=164, aux_to_w2=165, aux_to_b2=166.
    //
    // Per-tensor sizes (sh2 = cfg.shared_h2 = 256, h_dim = 128, k_to = 3):
    //   [163] aux_to_w1 [H, SH2]      → 128 × 256 = 32,768 floats
    //   [164] aux_to_b1 [H]           → 128 floats
    //   [165] aux_to_w2 [K=3, H]      → 3 × 128 = 384 floats
    //   [166] aux_to_b2 [K=3]          → 3 floats
    //
    // Total: 33,283 floats = ~133 KB. Adam m/v moments double this to
    // ~266 KB additional VRAM — negligible vs the existing aux trunk
    // tensor budget.
    //
    // SH2=256 mirrors the K=2 head exactly at this commit. The spec's
    // Phase B (input concat with plan_params 6-dim → SH2=262) will
    // re-shape slot [163] later: 128 × 262 = 33,536 floats (+128 from
    // the 6 extra input columns × 128 hidden lanes).
    {
        let h_dim = crate::cuda_pipeline::gpu_aux_heads::AUX_HIDDEN_DIM;
        let k_to  = crate::cuda_pipeline::gpu_aux_heads::AUX_OUTCOME_K;
        sizes[163] = h_dim * sh2;  // aux_to_w1 [H, SH2]
        sizes[164] = h_dim;        // aux_to_b1 [H]
        sizes[165] = k_to * h_dim; // aux_to_w2 [K=3, H]
        sizes[166] = k_to;         // aux_to_b2 [K=3]
    }

    debug_assert!(NUM_WEIGHT_TENSORS == 167,
        "compute_param_sizes: NUM_WEIGHT_TENSORS expected 167 after vNext extension, got {}",
        NUM_WEIGHT_TENSORS);

    sizes
}

/// Align element count to 4-element boundary (16 bytes for f32).
/// cublasLtMatmul FAST_TF32 requires 16-byte aligned buffer pointers.
#[inline]
pub(crate) fn align4(n: usize) -> usize {
    (n + 3) & !3
}

/// Total parameter count including 16-byte alignment padding per tensor.
pub(crate) fn compute_total_params(cfg: &GpuDqnTrainConfig) -> usize {
    compute_param_sizes(cfg).iter().map(|&s| align4(s)).sum()
}

/// Non-ISV parameter count (indices 0..FIRST_ISV_TENSOR).
/// EMA target sync is restricted to this range — ISV weights are online-only.
pub(crate) fn compute_non_isv_params(cfg: &GpuDqnTrainConfig) -> usize {
    compute_param_sizes(cfg)[..FIRST_ISV_TENSOR].iter().map(|&s| align4(s)).sum()
}

/// Cumulative byte offset to tensor `idx` in the padded flat buffer.
pub(crate) fn padded_byte_offset(param_sizes: &[usize], idx: usize) -> u64 {
    param_sizes[..idx].iter()
        .map(|&s| align4(s) * std::mem::size_of::<f32>())
        .sum::<usize>() as u64
}

// ── Mapped-pinned upload helpers ─────────────────────────────────────────────
// Per `feedback_no_htod_htoh_only_mapped_pinned.md`, HtoD memcpys are
// forbidden. These helpers stage CPU data through a transient mapped pinned
// buffer and DtoD-copy into a regular GPU-resident `CudaSlice<T>` — preserving
// the existing kernel-arg surface while eliminating the HtoD step.
// COLD path only: each helper allocates + frees a mapped buffer per call and
// stream-syncs to keep the staging buffer alive until the DtoD completes.

fn upload_via_mapped_f32(
    stream: &Arc<CudaStream>,
    n: usize,
    data: &[f32],
    label: &str,
) -> Result<CudaSlice<f32>, MLError> {
    use cudarc::driver::DevicePtrMut;
    assert!(data.len() <= n, "{label}: data.len {} > buf.len {n}", data.len());
    let mut buf = stream.alloc_zeros::<f32>(n)
        .map_err(|e| MLError::ModelError(format!("{label} alloc: {e}")))?;
    // Safety: a CUDA context is active on this thread (the stream was built
    // against it; the caller is mid-construction and holds it live).
    let staging = unsafe { MappedF32Buffer::new(n) }
        .map_err(|e| MLError::ModelError(format!("{label} staging alloc: {e}")))?;
    staging.write_from_slice(data);
    let n_bytes = n * std::mem::size_of::<f32>();
    {
        let (dst_ptr, _g) = buf.device_ptr_mut(stream);
        #[allow(unsafe_code)]
        unsafe {
            cudarc::driver::result::memcpy_dtod_async(
                dst_ptr, staging.dev_ptr, n_bytes, stream.cu_stream(),
            ).map_err(|e| MLError::ModelError(format!("{label} dtod: {e}")))?;
        }
    }
    // Sync so staging is safe to drop (DtoD completes before drop frees it).
    stream.synchronize()
        .map_err(|e| MLError::ModelError(format!("{label} dtod sync: {e}")))?;
    Ok(buf)
}

fn upload_via_mapped_i32(
    stream: &Arc<CudaStream>,
    n: usize,
    data: &[i32],
    label: &str,
) -> Result<CudaSlice<i32>, MLError> {
    use cudarc::driver::DevicePtrMut;
    assert!(data.len() <= n, "{label}: data.len {} > buf.len {n}", data.len());
    let mut buf = stream.alloc_zeros::<i32>(n)
        .map_err(|e| MLError::ModelError(format!("{label} alloc: {e}")))?;
    let staging = unsafe { MappedI32Buffer::new(n) }
        .map_err(|e| MLError::ModelError(format!("{label} staging alloc: {e}")))?;
    staging.write_from_slice(data);
    let n_bytes = n * std::mem::size_of::<i32>();
    {
        let (dst_ptr, _g) = buf.device_ptr_mut(stream);
        #[allow(unsafe_code)]
        unsafe {
            cudarc::driver::result::memcpy_dtod_async(
                dst_ptr, staging.dev_ptr, n_bytes, stream.cu_stream(),
            ).map_err(|e| MLError::ModelError(format!("{label} dtod: {e}")))?;
        }
    }
    stream.synchronize()
        .map_err(|e| MLError::ModelError(format!("{label} dtod sync: {e}")))?;
    Ok(buf)
}

fn upload_via_mapped_u32(
    stream: &Arc<CudaStream>,
    n: usize,
    data: &[u32],
    label: &str,
) -> Result<CudaSlice<u32>, MLError> {
    use cudarc::driver::DevicePtrMut;
    assert!(data.len() <= n, "{label}: data.len {} > buf.len {n}", data.len());
    let mut buf = stream.alloc_zeros::<u32>(n)
        .map_err(|e| MLError::ModelError(format!("{label} alloc: {e}")))?;
    let staging = unsafe { MappedU32Buffer::new(n) }
        .map_err(|e| MLError::ModelError(format!("{label} staging alloc: {e}")))?;
    staging.write_from_slice(data);
    let n_bytes = n * std::mem::size_of::<u32>();
    {
        let (dst_ptr, _g) = buf.device_ptr_mut(stream);
        #[allow(unsafe_code)]
        unsafe {
            cudarc::driver::result::memcpy_dtod_async(
                dst_ptr, staging.dev_ptr, n_bytes, stream.cu_stream(),
            ).map_err(|e| MLError::ModelError(format!("{label} dtod: {e}")))?;
        }
    }
    stream.synchronize()
        .map_err(|e| MLError::ModelError(format!("{label} dtod sync: {e}")))?;
    Ok(buf)
}

fn upload_via_mapped_u64(
    stream: &Arc<CudaStream>,
    n: usize,
    data: &[u64],
    label: &str,
) -> Result<CudaSlice<u64>, MLError> {
    use cudarc::driver::DevicePtrMut;
    assert!(data.len() <= n, "{label}: data.len {} > buf.len {n}", data.len());
    let mut buf = stream.alloc_zeros::<u64>(n)
        .map_err(|e| MLError::ModelError(format!("{label} alloc: {e}")))?;
    let staging = unsafe { MappedU64Buffer::new(n) }
        .map_err(|e| MLError::ModelError(format!("{label} staging alloc: {e}")))?;
    staging.write_from_slice(data);
    let n_bytes = n * std::mem::size_of::<u64>();
    {
        let (dst_ptr, _g) = buf.device_ptr_mut(stream);
        #[allow(unsafe_code)]
        unsafe {
            cudarc::driver::result::memcpy_dtod_async(
                dst_ptr, staging.dev_ptr, n_bytes, stream.cu_stream(),
            ).map_err(|e| MLError::ModelError(format!("{label} dtod: {e}")))?;
        }
    }
    stream.synchronize()
        .map_err(|e| MLError::ModelError(format!("{label} dtod sync: {e}")))?;
    Ok(buf)
}

/// Update an existing GPU-resident `CudaSlice<f32>` from a host slice via
/// mapped-pinned staging + DtoD copy. WARM path equivalent of
/// `upload_via_mapped_f32` for in-place updates of existing buffers.
fn update_via_mapped_f32(
    stream: &Arc<CudaStream>,
    dst: &mut CudaSlice<f32>,
    data: &[f32],
    label: &str,
) -> Result<(), MLError> {
    use cudarc::driver::DevicePtrMut;
    let n = data.len();
    assert!(dst.len() >= n, "{label}: dst.len {} < data.len {n}", dst.len());
    let staging = unsafe { MappedF32Buffer::new(n) }
        .map_err(|e| MLError::ModelError(format!("{label} staging alloc: {e}")))?;
    staging.write_from_slice(data);
    let n_bytes = n * std::mem::size_of::<f32>();
    let (dst_ptr, _g) = dst.device_ptr_mut(stream);
    #[allow(unsafe_code)]
    unsafe {
        cudarc::driver::result::memcpy_dtod_async(
            dst_ptr, staging.dev_ptr, n_bytes, stream.cu_stream(),
        ).map_err(|e| MLError::ModelError(format!("{label} dtod: {e}")))?;
    }
    stream.synchronize()
        .map_err(|e| MLError::ModelError(format!("{label} dtod sync: {e}")))?;
    Ok(())
}

/// Pre-resolved raw u64 CUDA device pointers for all GPU buffers.
/// Computed once at construction. Eliminates 110+ per-step `raw_device_ptr()`
/// calls that go through cudarc's event tracking machinery.
#[derive(Clone)]
struct CachedPtrs {
    /// f32 online weights — Adam writes, GemmEx reads (single buffer, no shadow).
    params_ptr: u64,
    /// f32 target weights — EMA writes, target forward reads (single buffer, no shadow).
    target_ptr: u64,
    grad_buf: u64,
    grad_norm_buf: u64,
    /// f32 Adam first moment.
    m_buf: u64,
    /// f32 Adam second moment.
    v_buf: u64,
    t_buf: u64,
    adaptive_clip_buf: u64,
    total_loss_buf: u64,
    mse_loss_buf: u64,
    cql_grad_scratch: u64,
    states_buf: u64,
    next_states_buf: u64,
    actions_buf: u64,
    rewards_buf: u64,
    dones_buf: u64,
    is_weights_buf: u64,
    save_h_s1: u64,
    save_h_s2: u64,
    save_h_v: u64,
    save_h_b0: u64,
    save_h_b1: u64,
    save_h_b2: u64,
    save_h_b3: u64,
    mag_concat_buf: u64,
    d_mag_concat_buf: u64,
    pub(crate) ord_concat_buf: u64,
    pub(crate) urg_concat_buf: u64,
    bw_d_h_s1: u64,
    bw_d_h_s2: u64,
    bw_d_h_v: u64,
    bw_d_h_b0: u64,
    bw_d_h_b1: u64,
    bw_d_h_b2: u64,
    bw_d_h_b3: u64,
    bw_d_glu_value: u64,
    bw_d_glu_gate: u64,
    iqn_trunk_m: u64,
    iqn_trunk_grad_norm: u64,
    td_errors_buf: u64,
    on_v_logits_buf: u64,
    on_b_logits_buf: u64,
    // Target network scratch buffers
    tg_h_s1_scratch: u64,
    tg_h_s2_buf: u64,
    tg_h_v_scratch: u64,
    tg_h_b0_scratch: u64,
    tg_h_b1_scratch: u64,
    tg_h_b2_scratch: u64,
    tg_h_b3_scratch: u64,
    tg_v_logits_buf: u64,
    tg_b_logits_buf: u64,
    // Double DQN online-on-next-states scratch buffers
    on_next_h_s1_scratch: u64,
    on_next_h_s2_scratch: u64,
    on_next_h_v_scratch: u64,
    on_next_h_b_scratch: u64,
    on_next_v_logits_buf: u64,
    on_next_b_logits_buf: u64,
}

// ── Main struct ─────────────────────────────────────────────────────────────

/// Fused CUDA DQN trainer — replaces Candle dispatch chain with 4 kernel launches.
///
/// All GPU buffers are pre-allocated at construction. The trainer borrows
/// weight sets from the caller (no weight duplication). Only batch input
/// data is uploaded per step; outputs (loss, td_errors, grad_norm) are downloaded.
///
/// On the first `train_step()`, the kernel sequence is captured into two CUDA
/// Graphs (`graph_forward` and `graph_adam`). Subsequent calls replay both
/// graphs for zero kernel-launch overhead, with an injection point between
/// them for auxiliary gradients (IQN, ensemble, attention).
#[allow(missing_debug_implementations)] // CudaSlice does not implement Debug
pub struct GpuDqnTrainer {
    config: GpuDqnTrainConfig,
    stream: Arc<CudaStream>,
    shared_cublas: Arc<PerStreamCublasHandles>,
    ptrs: CachedPtrs,

    // ── Compiled kernels ────────────────────────────────────────────
    // Dead kernels (forward_loss_kernel, forward_only_kernel, backward_kernel)
    // removed — replaced by cuBLAS forward/backward. See Task 2 for method cleanup.
    grad_norm_kernel: CudaFunction,
    grad_norm_finalize_kernel: CudaFunction,
    /// Separate finalize instance for adam_child — cannot share CUfunction across
    /// child graphs on Hopper (causes 3100ms replay from kernel state corruption).
    grad_norm_finalize_adam: CudaFunction,
    /// Separate finalize instance for post_aux_child — same Hopper CUfunction isolation.
    grad_norm_finalize_post_aux: CudaFunction,
    /// Separate instance of grad_norm for non-graph launches (outside CUDA graph).
    /// CUDA doesn't allow the same CUfunction to be launched both inside a captured
    /// graph and outside on the same stream.
    grad_norm_standalone: CudaFunction,
    /// Separate grad_norm instance for post_aux_child — cannot share CUfunction
    /// with forward_child's grad_norm_standalone on Hopper.
    grad_norm_standalone_post_aux: CudaFunction,
    adam_update_kernel: CudaFunction,
    /// Separate instances for post_aux_child — each from its own CUmodule.
    /// CUfunction sharing across child graphs corrupts kernel state on Hopper,
    /// causing 3100ms replay. Every child gets isolated CUfunction handles.
    adam_update_post_aux: CudaFunction,
    scale_f32_post_aux: CudaFunction,
    ema_kernel: CudaFunction,
    saxpy_kernel: CudaFunction,
    saxpy_f32_kernel: CudaFunction,
    /// Separate saxpy_f32 instance for adam_grad_child — cannot share CUfunction
    /// across child graphs (causes 3100ms replay on Hopper).
    saxpy_f32_adam_grad: CudaFunction,
    /// Separate saxpy_f32 instance for aux_child — same Hopper CUfunction isolation.
    saxpy_f32_aux: CudaFunction,
    /// Plan 4 Task 1B-iv-rc: separate `dqn_scale_f32_kernel` instance for
    /// `aux_child` — same Hopper CUfunction-per-child isolation as
    /// `saxpy_f32_aux`. Used to attenuate `vsn_d_gated_state_buf` by a
    /// constant `VSN_DW_DAMP` factor before each aux-path `vsn_backward`
    /// invocation, mirroring the main-path attenuation that uses
    /// `scale_f32_kernel` in `forward_child`. The constant is baked at
    /// graph capture time — no per-step variation, so no pinned-device
    /// scalar plumbing required.
    scale_f32_aux: CudaFunction,
    /// D1/N1: Distillation fused SAXPY kernel (loaded from aux_module for
    /// aux_child graph capture). Reads health from ISV pinned memory,
    /// computes alpha internally, performs `grad += alpha * (params − best)`
    /// in one pass. All parameters flow via stable device pointers so the
    /// graph capture locks in correct behaviour while per-epoch values
    /// (health via ISV, best snapshot via DtoD into stable buffer) vary.
    distill_saxpy_aux: CudaFunction,
    scale_f32_kernel: CudaFunction,
    zero_kernel: CudaFunction,
    regime_scale_kernel: CudaFunction,
    shrink_perturb_kernel: CudaFunction,
    /// Ungraphed variant of shrink_perturb — from a separate CUmodule.
    /// shrink_and_perturb() runs at epoch boundaries (after graph capture).
    /// On Hopper, launching ANY CUfunction from a CUmodule that also has
    /// graph-captured CUfunctions corrupts graph kernel state (3100ms replay).
    shrink_perturb_ungraphed: CudaFunction,
    /// Ungraphed variant of scale_f32 — from a separate CUmodule.
    /// scale_adam_momentum() runs at cosine LR warm restarts (after graph capture).
    /// scale_f32_kernel is captured in forward_child; launching from the same
    /// CUmodule ungraphed corrupts the child graph on Hopper.
    scale_f32_ungraphed: CudaFunction,
    relu_mask_kernel: CudaFunction,
    spectral_norm_batched_kernel: CudaFunction,
    /// Descriptor buffer for batched spectral norm: 12 matrices x 6 u64 = 72 elements.
    /// Each entry: [W_ptr, u_ptr, v_ptr, out_dim, in_dim, pad=0].
    /// Built once at construction (pointers into params_buf are stable).
    spectral_norm_descriptors: CudaSlice<u64>,
    clip_grad_kernel: CudaFunction,
    pad_states_kernel: CudaFunction,
    /// Spectral norm left singular vectors: [out_dim] per weight matrix (W_s1, W_s2)
    /// Mapped pinned (cuMemHostAlloc DEVICEMAP) — host-init at construction, kernel
    /// reads via dev_ptr. Avoids DtoD-via-pinned per
    /// feedback_no_htod_htoh_only_mapped_pinned. Stored in `spectral_norm_descriptors`
    /// as CUdeviceptr (= dev_ptr) so the batched power-iteration kernel reads them
    /// indirectly through the descriptor table.
    spec_u_s1: super::mapped_pinned::MappedF32Buffer,
    spec_v_s1: super::mapped_pinned::MappedF32Buffer,
    spec_u_s2: super::mapped_pinned::MappedF32Buffer,
    spec_v_s2: super::mapped_pinned::MappedF32Buffer,
    // Head spectral norm vectors (power iteration) — same mapped-pinned semantics as the trunk pair above.
    spec_u_v1: super::mapped_pinned::MappedF32Buffer, spec_v_v1: super::mapped_pinned::MappedF32Buffer,
    spec_u_v2: super::mapped_pinned::MappedF32Buffer, spec_v_v2: super::mapped_pinned::MappedF32Buffer,
    spec_u_a1: super::mapped_pinned::MappedF32Buffer, spec_v_a1: super::mapped_pinned::MappedF32Buffer,
    spec_u_a2: super::mapped_pinned::MappedF32Buffer, spec_v_a2: super::mapped_pinned::MappedF32Buffer,
    spec_u_bo1: super::mapped_pinned::MappedF32Buffer, spec_v_bo1: super::mapped_pinned::MappedF32Buffer,
    spec_u_bo2: super::mapped_pinned::MappedF32Buffer, spec_v_bo2: super::mapped_pinned::MappedF32Buffer,
    spec_u_bu1: super::mapped_pinned::MappedF32Buffer, spec_v_bu1: super::mapped_pinned::MappedF32Buffer,
    spec_u_bu2: super::mapped_pinned::MappedF32Buffer, spec_v_bu2: super::mapped_pinned::MappedF32Buffer,
    spec_u_bg1: super::mapped_pinned::MappedF32Buffer, spec_v_bg1: super::mapped_pinned::MappedF32Buffer,
    spec_u_bg2: super::mapped_pinned::MappedF32Buffer, spec_v_bg2: super::mapped_pinned::MappedF32Buffer,
    spec_u_bn: super::mapped_pinned::MappedF32Buffer, spec_v_bn: super::mapped_pinned::MappedF32Buffer,
    /// IQN trunk gradient scratch [trunk_params] (f32, receives backward dW output).
    iqn_trunk_m: CudaSlice<f32>,
    /// IQN trunk gradient Adam second moment [trunk_params].
    iqn_trunk_v: CudaSlice<f32>,
    /// IQN trunk Adam step counter.
    iqn_trunk_adam_step: i32,
    /// IQN trunk grad norm scratch [1] (f32 accumulator).
    iqn_trunk_grad_norm: CudaSlice<f32>,
    /// Scratch f32 [1] for IQN/ensemble finalize — prevents overwriting main grad_norm_buf.
    aux_norm_scratch: CudaSlice<f32>,
    /// Scratch f32 [B * SHARED_H2] for copying bw_d_h_s2 (f32) → attention scratch.
    /// Used for GpuAttention::backward input.
    attn_bw_scratch: CudaSlice<f32>,
    /// IQN trunk Adam step counter on device [1].
    iqn_trunk_t_buf: CudaSlice<i32>,
    /// Number of trunk parameters (w_s1 + b_s1 + w_s2 + b_s2).
    trunk_param_count: usize,


    // ── Batch input buffers (uploaded per step) ─────────────────────
    states_buf: CudaSlice<f32>,       // [B, STATE_DIM]
    next_states_buf: CudaSlice<f32>,  // [B, STATE_DIM]
    actions_buf: CudaSlice<i32>,      // [B]
    rewards_buf: CudaSlice<f32>,             // [B] f32
    /// 5-bar n-step reward buffer [B].
    rewards_5bar: CudaSlice<f32>,
    /// 20-bar n-step reward buffer [B].
    rewards_20bar: CudaSlice<f32>,
    dones_buf: CudaSlice<f32>,               // [B] f32
    is_weights_buf: CudaSlice<f32>,           // [B] f32

    // ── Activation save buffers (forward → backward) ────────────────
    save_h_s1: CudaSlice<f32>,        // [B, SHARED_H1]
    save_h_s2: CudaSlice<f32>,        // [B, SHARED_H2] — ISV feature-gated in-place
    save_h_v: CudaSlice<f32>,         // [B, VALUE_H]
    save_h_b0: CudaSlice<f32>,        // [B, ADV_H]
    save_h_b1: CudaSlice<f32>,        // [B, ADV_H]
    save_h_b2: CudaSlice<f32>,        // [B, ADV_H]
    save_h_b3: CudaSlice<f32>,        // [B, ADV_H]

    /// Magnitude branch concat input [B, SH2+branch_0_size]: [h_s2; Q_dir]
    /// for direction conditioning (4 directions: S/H/L/F).
    mag_concat_buf: CudaSlice<f32>,
    /// Backward scratch for magnitude concat dX [B, SH2+branch_0_size].
    d_mag_concat_buf: CudaSlice<f32>,
    /// Kernel: mag_concat_qdir — builds [h_s2; Q_dir] concat.
    mag_concat_kernel: CudaFunction,
    /// Kernel: strided_accumulate — extracts d_h_s2 from d_mag_concat in backward.
    strided_accumulate_kernel: CudaFunction,
    /// Kernel: strided_scatter — writes vsn_masked into first SH2 cols of mag_concat.
    strided_scatter_kernel: CudaFunction,
    /// Order branch concat input [B, SH2+3]: [vsn_masked; OFI_order] for microstructure conditioning.
    ord_concat_buf: CudaSlice<f32>,
    /// Backward scratch for order concat dX [B, SH2+3].
    d_ord_concat_buf: CudaSlice<f32>,
    /// Urgency branch concat input [B, SH2+3]: [vsn_masked; OFI_urgency] for microstructure conditioning.
    urg_concat_buf: CudaSlice<f32>,
    /// Backward scratch for urgency concat dX [B, SH2+3].
    d_urg_concat_buf: CudaSlice<f32>,
    /// Kernel: concat_ofi_features — builds [vsn_masked; OFI] for order/urgency branches.
    concat_ofi_kernel: CudaFunction,

    // ── Cross-Branch Q Attention ──
    q_attn_params: CudaSlice<f32>,    // [624]
    q_attn_adam_m: CudaSlice<f32>,     // [624]
    q_attn_adam_v: CudaSlice<f32>,     // [624]
    q_attn_adam_step: i32,
    q_coord_buf: CudaSlice<f32>,      // [B, 12]
    q_attn_kernel: CudaFunction,

    // ── Selectivity gate ──
    sel_params: CudaSlice<f32>,        // [SH2+1]
    sel_adam_m: CudaSlice<f32>,        // [SH2+1]
    sel_adam_v: CudaSlice<f32>,        // [SH2+1]
    sel_grad: CudaSlice<f32>,         // [SH2+1]
    sel_adam_step: i32,
    sel_out_buf: CudaSlice<f32>,      // [B]
    sel_fwd_kernel: CudaFunction,
    sel_compute_dz_kernel: CudaFunction,
    sel_dw_reduce_p1_kernel: CudaFunction,
    sel_dw_reduce_p2_kernel: CudaFunction,
    sel_d_z_buf: CudaSlice<f32>,
    sel_dw_partials_buf: CudaSlice<f32>,
    sel_num_blocks: u32,
    /// [1] L2 norm of sel_grad (written by grad_norm_finalize before sel Adam).
    sel_norm_buf: CudaSlice<f32>,     // [1]
    /// [1] partial sums scratch for sel grad_norm reduction (SH2+1 ≤ 256, so 1 block).
    sel_norm_partials: CudaSlice<f32>, // [1]
    /// [1] max grad norm for sel Adam clipping (fixed 1.0).
    /// Mapped pinned (cuMemHostAlloc DEVICEMAP) — host-init at construction;
    /// kernel reads via dev_ptr. Avoids DtoD-via-pinned per
    /// feedback_no_htod_htoh_only_mapped_pinned.
    sel_clip_buf: super::mapped_pinned::MappedF32Buffer,     // [1]
    /// Selectivity Adam step counter — pinned device-mapped (no per-step HtoD copy).
    sel_t_pinned: *mut i32,
    sel_t_dev_ptr: u64,

    /// SP15 Phase 3.1 (2026-05-06): warm-up counter for the
    /// r_quality + r_discipline split composer (per spec §8.2 (3.1)
    /// post-amendment-2 fix). Single-element mapped-pinned buffer
    /// initialised to 0.0 in the constructor and reset to 0.0 at fold
    /// boundary by the `sp15_alpha_warm_count` registry-arm dispatch
    /// (which calls `host_slice_mut().fill(0.0)` mirroring the
    /// `sp11_novelty_hash` pattern). Each per-step launch of
    /// `r_quality_discipline_split_kernel` increments the count by 1;
    /// once `*count >= N_WARM=100` the composer stops overriding α to
    /// 0.5 and `alpha_split_producer_kernel` activates from no-op state
    /// to write the formula's clamped output to ALPHA_SPLIT_INDEX. The
    /// counter is a producer-controlled scalar — the host writes 0.0
    /// at construction / fold reset (allowed under
    /// `feedback_no_htod_htoh_only_mapped_pinned.md`'s allowed-write
    /// rule, NOT a host-side compute) and reads it through the mapped
    /// page via `read_all()` only in tests.
    pub(crate) sp15_alpha_warm_count: super::mapped_pinned::MappedF32Buffer, // [1]

    /// SP15 Phase 3.5.3 (2026-05-06): consecutive-loss streak counter
    /// for the cooldown gate (per spec §9.2 (3.5.3) post-amendment-2
    /// fix). Single-element mapped-pinned f32 buffer initialised to
    /// 0.0 in the constructor and reset to 0.0 at fold boundary by the
    /// `sp15_cooldown_consecutive_losses` registry-arm dispatch (which
    /// calls `host_slice_mut().fill(0.0)` mirroring the Phase 3.1
    /// `sp15_alpha_warm_count` non-ISV mapped-pinned reset pattern).
    /// Read-modify-written by `cooldown_kernel`: `+= 1.0f` on
    /// trade-close losses, `= 0.0f` on trade-close wins (streak
    /// broken), untouched on non-trade-close bars (per-bar
    /// non-close calls just decrement the cooldown counter without
    /// touching the loss tracker per spec post-amendment-2 fix). f32
    /// representation is exact for any reachable streak length
    /// because the kernel only performs `+= 1.0f` increments and
    /// `= 0.0f` resets. The host writes 0.0 at construction / fold
    /// reset (allowed under
    /// `feedback_no_htod_htoh_only_mapped_pinned.md`'s allowed-write
    /// rule, NOT a host-side compute) and reads it through the mapped
    /// page via `read_all()` only in tests.
    pub(crate) sp15_cooldown_consecutive_losses: super::mapped_pinned::MappedF32Buffer, // [1]

    // SP15 Phase 3.5.5 (2026-05-06) → Phase 1.3.b-followup-B (2026-05-07):
    // the previous-bar dd_pct scratch buffer for the recovery-curriculum
    // DD_TRAJECTORY_DECREASING per-step proxy kernel was reshaped from
    // `[1]` (single-env scratch on the trainer struct) to `[n_envs]`
    // (per-env scratch on the experience collector) when the kernel
    // moved to a per-env grid. The trainer no longer owns this buffer —
    // it lives on `GpuExperienceCollector::sp15_dd_trajectory_prev_dd_per_env`
    // alongside the new `sp15_dd_trajectory_per_env` per-env output
    // tile. The reset arm `sp15_dd_trajectory_prev_dd` was renamed to
    // `sp15_dd_trajectory_prev_dd_per_env` and dispatches to the
    // collector's mapped-pinned tile (mirrors the
    // `sp15_dd_state_per_env` collector-owned reset pattern).

    // SP15 Phase 1.3.b-followup (2026-05-07): per-env DD state tile
    // `sp15_dd_state_per_env` is owned by `GpuExperienceCollector`
    // (which knows `alloc_episodes` = `n_envs`); the trainer does NOT
    // own a copy. The collector launches `dd_state_kernel` /
    // `dd_state_reduce_kernel` / `compute_sp15_final_reward_kernel`
    // back-to-back per step on the same stream, threading its own
    // tile pointer into all three. See
    // `gpu_experience_collector.rs::sp15_dd_state_per_env`.

    // ── Per-branch Q-gap EMA (trajectory backtracking state) ──
    /// [4] per-branch Q-gap EMA — device buffer (read-modify-write for backtracking restore).
    per_branch_q_gap_ema_buf: CudaSlice<f32>,
    /// Last computed per-branch Q-gaps from reduce_current_q_stats.
    last_per_branch_q_gaps: [f32; 4],
    /// Task 0.3 — cached magnitude-branch Q-value means `[q_full, q_half,
    /// q_quarter]`. Refreshed by `update_q_mag_means_cached` (cold-path dtoh
    /// at HEALTH_DIAG emit); read by `q_magnitude_bucket_means`. Initialised
    /// to zeros; the first epoch's emit sees the post-epoch-0 values.
    q_mag_means_cached: [f32; 3],
    /// SP16 Phase 0 — cached direction-branch Q-value means in HEALTH_DIAG
    /// field order `[q_hold, q_long, q_short, q_flat]`.
    ///
    /// Refreshed by `update_q_dir_means_cached` (cold-path dtoh at
    /// HEALTH_DIAG emit); read by `q_direction_action_means`. Initialised
    /// to zeros; the first epoch's emit sees the post-epoch-0 values.
    ///
    /// Diagnostic-only: feeds the structural-Q-bias-toward-Hold falsifier
    /// (per train-multi-seed-pfh9n post-mortem). Field order matches the
    /// emit format `q_by_action [hold long short flat]`; the kernel-side
    /// indexing is the canonical state_layout.cuh ordering DIR_SHORT=0,
    /// DIR_HOLD=1, DIR_LONG=2, DIR_FLAT=3 (see state_layout.cuh:123-126).
    q_dir_means_cached: [f32; 4],
    /// EMA of atom utilization [0,1] for adaptive entropy regularization.
    /// SP4 Layer C close-out C3 (2026-05-01) replaced the host-resident `f32`
    /// field with `utilization_ema_pinned` (mapped-pinned scalar updated GPU-
    /// side by `update_utilization_ema_kernel`) per
    /// `feedback_no_cpu_compute_strict`. Cold-start sentinel `prev > 0.99 ⇒
    /// assign obs directly` preserves the deleted host formula's bootstrap
    /// branch (constructor seeds the slot at 1.0 to mark "no observation
    /// yet"). FoldReset: 1.0 (back to bootstrap sentinel).
    utilization_ema_pinned: *mut f32,
    /// Device-mapped view of `utilization_ema_pinned`. Read+written in-place
    /// by `update_utilization_ema_kernel`; not consumed by any other GPU
    /// kernel via dev_ptr (`adaptive_entropy` host-side at line 20374 reads
    /// the host_ptr to derive a kernel arg; collector's `set_utilization_ema`
    /// reads the host_ptr to refresh its own cached scalar).
    utilization_ema_dev_ptr: u64,
    // ── VSN + GLU scratch ──
    vsn_masked_buf: CudaSlice<f32>,   // [B, SH2]
    vsn_kernel: CudaFunction,
    glu_gate_pre_buf: [CudaSlice<f32>; 4],  // 4 × [B, AH]
    glu_value_buf: [CudaSlice<f32>; 4],     // 4 × [B, AH]
    glu_combine_kernel: CudaFunction,
    glu_backward_kernel: CudaFunction,
    kan_gate_combine_kernel: CudaFunction,
    kan_gate_backward_kernel: CudaFunction,
    kan_grad_reduce_p1_kernel: CudaFunction,
    kan_grad_reduce_p2_kernel: CudaFunction,

    // ── Branch confidence routing (ISV gate × Q-value confidence) ──
    branch_confidence_routing_kernel: CudaFunction,
    /// Per-sample Q-gap for regime gate input [B].
    regime_q_gap_buf: CudaSlice<f32>,
    /// Atom utilization scalar for regime gate [1] — pinned device-mapped.
    regime_util_pinned: *mut f32,
    regime_util_dev_ptr: u64,

    // ── G9: Regime dropout on h_s2 ──
    /// Kernel: regime_dropout — regime-conditioned inverted dropout on trunk output.
    regime_dropout_kernel: CudaFunction,
    /// Epoch seed for regime dropout Philox PRNG (changes per epoch).
    regime_dropout_epoch_seed: i32,

    // ── G5: Epistemic-gated magnitude ──
    /// Kernel: epistemic_gate_magnitude — sigmoid-gates magnitude Q-values by ensemble variance.
    epistemic_gate_kernel: CudaFunction,
    /// Pinned device-mapped EMA of ensemble variance threshold [1].
    /// CPU writes via host ptr, GPU reads via dev_ptr. No HtoD copy.
    var_ema_pinned: *mut f32,
    var_ema_dev_ptr: u64,

    // G6 / G10 fields removed 2026-04-21: V7-gem measurement showed their
    // penalties were sub-noise (spectral_norm + NoisyNets already cover).
    // See commit 1961857c2 for the unwiring.

    // ── G12: Predictive coding auxiliary loss + backward ──
    /// Backward kernel: writes per-(sample, feature) gradient of the
    /// predictive_coding_loss into bw_d_h_s2 via plain += (no atomicAdd).
    predictive_coding_backward_kernel: CudaFunction,
    /// Kernel: predictive_coding_loss — self-supervised temporal smoothness on enriched trunk.
    predictive_coding_kernel: CudaFunction,
    /// Per-sample loss buffer [B] for predictive coding — reduced by c51_loss_reduce.
    predictive_per_sample_buf: CudaSlice<f32>,
    /// Scalar output buffer [1] for predictive coding MSE loss.
    predictive_loss_buf: CudaSlice<f32>,

    // ── G11: Q-value anchoring to flat ──
    /// Kernel: q_anchor_to_flat — anchors per-branch Q-values to neutral actions.
    q_anchor_kernel: CudaFunction,

    // ── Adaptive atom positions ──
    /// Precomputed adaptive atom positions [4, num_atoms] per branch.
    atom_positions_buf: CudaSlice<f32>,
    /// Kernel: adaptive_atom_positions — softmax spacing → cumulative positions.
    adaptive_atom_kernel: CudaFunction,
    /// Kernel: atom_position_gradient — entropy-based gradient for spacing_raw.
    atom_position_grad_kernel: CudaFunction,

    save_current_lp: CudaSlice<f32>,  // [B, NUM_BRANCHES(4), NUM_ATOMS]
    save_projected: CudaSlice<f32>,   // [B, NUM_BRANCHES(4), NUM_ATOMS]

    // ── Forward output buffers ──────────────────────────────────────
    per_sample_loss_buf: CudaSlice<f32>,  // [B]
    td_errors_buf: CudaSlice<f32>,        // [B]
    /// C51 loss accumulator [1] — pinned device-mapped. GPU writes via dev_ptr, CPU reads via host ptr.
    total_loss_pinned: *mut f32,
    total_loss_dev_ptr: u64,
    /// MSE loss accumulator [1] — pinned device-mapped. GPU writes via dev_ptr, CPU reads via host ptr.
    mse_loss_pinned: *mut f32,
    mse_loss_dev_ptr: u64,

    // ── Forward-only Q-value output ─────────────────────────────────
    q_out_buf: CudaSlice<f32>,            // [B, TOTAL_ACTIONS(13)] — 4 direction + 3 magnitude + 3 order + 3 urgency
    /// Snapshot buffers for td_errors + per_sample_loss during eval replay.
    /// Prevents graph_forward replay from corrupting PER priorities.
    eval_td_snapshot: CudaSlice<f32>,
    eval_loss_snapshot: CudaSlice<f32>,

    // ── Backward / Adam buffers (f32 master weights — GemmEx reads directly) ─
    grad_buf: CudaSlice<f32>,                // [TOTAL_PARAMS] f32 gradient accumulator
    /// Task 0.4 — pinned host buffer for per-branch grad-norm readback.
    /// Allocated once at construction sized to TOTAL_PARAMS f32. memcpy_dtoh
    /// into pinned memory runs at PCIe bandwidth (~0.5 ms for 4 MB) instead
    /// of pageable-host speed. Used only at epoch boundary, not in the
    /// training graph capture region.
    grad_readback_pinned_ptr: usize,         // host-visible f32 buffer
    grad_readback_pinned_capacity: usize,    // element count (TOTAL_PARAMS)
    /// Task 2.0 (+ Task 2.0 extension) — per-component magnitude-branch grad
    /// decomposition.
    ///
    /// Nine device-side scratch buffers sized to the combined length of
    /// branch 0 (direction, tensors 8..12) + branch 1 (magnitude, tensors
    /// 12..16). BEFORE each component's op fires (captured in the training
    /// graph), a `copy_f32` kernel snapshots `grad_buf` over the branch 0+1
    /// byte range into these slots. AFTER the component's op,
    /// `grad_component_delta_norm` computes `‖current−snapshot‖` on direction
    /// and magnitude slices and writes two f32 values into the pinned result
    /// slot at the component's 2-float offset.
    ///
    /// The C51 (post-backward) slot is paired with `grad_zero_ref_buf`
    /// (permanent zeros) because `grad_buf` is memset-zeroed at the start of
    /// every `submit_forward_ops_main` — there is no "pre-C51" grad to
    /// snapshot. Uniform subtraction keeps the kernel interface simple.
    ///
    /// Task 2.0 extension adds five measurement points in the aux-graph gap
    /// between CQL and Ens where Task 0.4's `grad_ratio_mag_dir=0` indicates
    /// the magnitude signal gets cancelled:
    ///   - `grad_snapshot_cql_sx`  — before `apply_cql_saxpy`
    ///   - `grad_snapshot_c51_bs`  — before `apply_c51_budget_scale`
    ///   - `grad_snapshot_distill` — before `apply_distillation_gradient`
    ///   - `grad_snapshot_rec`     — before `launch_recursive_confidence_backward`
    ///   - `grad_snapshot_pred`    — before `compute_predictive_coding_loss`
    ///
    /// Task 2.Z diagnostic extension: each snapshot now covers the TRUNK
    /// slice (tensors 0..4 — w_s1, b_s1, w_s2, b_s2) in addition to the
    /// branch-head slice. Layout per snapshot:
    ///   [0                 .. trunk_len)            = trunk
    ///   [trunk_len         .. trunk_len+dir_len)    = direction
    ///   [trunk_len+dir_len .. trunk_len+dir_len+mag_len) = magnitude
    /// The trunk slot exists to make IQN (`apply_iqn_trunk_gradient`) and
    /// Ens (`apply_ensemble_diversity_backward`) contributions measurable
    /// — both SAXPY exclusively into the trunk, which the prior branch-
    /// only pipeline reported as `0.0000` regardless of the real delta.
    ///
    /// Total device memory: 9 × (trunk+branch01)_elems × 4 bytes — a
    /// small bump on top of the pre-extension snapshot size.
    grad_snapshot_iqn: CudaSlice<f32>,       // [branch01_elems] — snapshot before IQN trunk grad
    grad_snapshot_cql: CudaSlice<f32>,       // [branch01_elems] — snapshot before CQL block (gradient+saxpy)
    grad_snapshot_cql_sx: CudaSlice<f32>,    // [branch01_elems] — snapshot before apply_cql_saxpy only
    grad_zero_ref_buf: CudaSlice<f32>,       // [branch01_elems] — permanent zeros; C51 snapshot substitute
    grad_snapshot_c51_bs: CudaSlice<f32>,    // [branch01_elems] — snapshot before apply_c51_budget_scale
    grad_snapshot_distill: CudaSlice<f32>,   // [branch01_elems] — snapshot before apply_distillation_gradient
    grad_snapshot_rec: CudaSlice<f32>,       // [branch01_elems] — snapshot before launch_recursive_confidence_backward
    grad_snapshot_pred: CudaSlice<f32>,      // [branch01_elems] — snapshot before compute_predictive_coding_loss
    grad_snapshot_ens: CudaSlice<f32>,       // [branch01_elems] — snapshot before ensemble diversity
    /// Pinned device-mapped 27-float result slot written by the reduction
    /// kernel. Layout (Task 2.Z diagnostic — 3 floats per component: mag,
    /// dir, trunk — preserving the original mag/dir order at [0]/[1] of
    /// each 3-float slot so pre-existing consumers continue to read
    /// unchanged offsets). Index order matches host-cache `[iqn, cql,
    /// cql_sx, c51, c51_bs, distill, rec, pred, ens]`:
    ///   [ 0]=iqn_mag      [ 1]=iqn_dir      [ 2]=iqn_trunk
    ///   [ 3]=cql_mag      [ 4]=cql_dir      [ 5]=cql_trunk
    ///   [ 6]=cql_sx_mag   [ 7]=cql_sx_dir   [ 8]=cql_sx_trunk
    ///   [ 9]=c51_mag      [10]=c51_dir      [11]=c51_trunk
    ///   [12]=c51_bs_mag   [13]=c51_bs_dir   [14]=c51_bs_trunk
    ///   [15]=distill_mag  [16]=distill_dir  [17]=distill_trunk
    ///   [18]=rec_mag      [19]=rec_dir      [20]=rec_trunk
    ///   [21]=pred_mag     [22]=pred_dir     [23]=pred_trunk
    ///   [24]=ens_mag      [25]=ens_dir      [26]=ens_trunk
    grad_decomp_result_pinned: *mut f32,
    grad_decomp_result_dev_ptr: u64,
    /// Cached grad-decomp slice metadata (element indices into `grad_buf`).
    /// `grad_decomp_trunk_start`/`trunk_len` cover trunk tensors 0..4
    /// (w_s1, b_s1, w_s2, b_s2) — Task 2.Z diagnostic addition.
    /// `grad_decomp_dir_start`/`dir_len` cover branch 0 tensors 8..12,
    /// `grad_decomp_mag_start`/`mag_len` cover branch 1 tensors 12..16.
    /// `grad_decomp_snapshot_len` = trunk_len + dir_len + mag_len.
    grad_decomp_trunk_start: i32,
    grad_decomp_trunk_len: i32,
    grad_decomp_dir_start: i32,
    grad_decomp_dir_len: i32,
    grad_decomp_mag_start: i32,
    grad_decomp_mag_len: i32,
    grad_decomp_snapshot_len: usize,
    /// Reduction kernel (one-block 256-thread launch, no atomics).
    grad_decomp_kernel: CudaFunction,
    /// SP7 Path A (2026-05-03) raw CQL norm kernel. Reads
    /// `cql_grad_scratch` directly (no snapshot pair) and writes 3 floats
    /// to `grad_decomp_result_pinned[3..6]` — the `cql` slot at element
    /// offset 3 that was never populated by the (defined-but-unused)
    /// `grad_decomp_launch_cql` snapshot pattern. Single block, 256
    /// threads, shared-memory tree reduction (no atomicAdd).
    cql_raw_norm_kernel: CudaFunction,
    /// Adaptive per-branch gradient-norm balancer — caps any branch whose
    /// L2 norm exceeds `num_branches × median_branch_norm` (see
    /// `branch_grad_balance_kernel.cu`).  `num_branches = 4` is the
    /// architectural factored-action axis count (direction, magnitude,
    /// order, urgency); the median is a per-step statistical reference,
    /// so the cap scales with the current training regime without tuned
    /// knobs.  Launched inside the `adam_grad_child` graph after all aux
    /// grad writes complete and BEFORE `compute_grad_norm_for_adam`, so
    /// Adam and the global clip observe the rebalanced gradient.
    branch_grad_balance_reduce: CudaFunction,
    /// ISV-driven producer kernel (2026-04-23): reads per-branch norms,
    /// applies adaptive-rate EMA to ISV slots `GRAD_NORM_TARGET_*` and
    /// `GRAD_SCALE_LIMIT_INDEX`. Runs between reduce and rescale.
    branch_grad_balance_isv_update: CudaFunction,
    branch_grad_balance_rescale: CudaFunction,
    /// `[4]` element offsets into `grad_buf` for branches 0-3
    /// (direction, magnitude, order, urgency).  Each branch covers the
    /// 4 contiguous tensors `(8+4b)..(12+4b)`: fc W, fc b, out W, out b.
    /// Stored device-side so both balancer kernels read them directly
    /// (graph-safe; values are written once at construction).
    branch_slice_starts_dev: CudaSlice<i32>,
    /// `[4]` element lengths (padded to align4) matching
    /// `branch_slice_starts_dev`.  Padding bytes in `grad_buf` are
    /// always zero (memset at step start; cuBLAS/aux kernels write only
    /// valid indices) so reading the padded range yields the same L2 as
    /// the unpadded range.
    branch_slice_lens_dev: CudaSlice<i32>,
    /// `[4]` scratch slot receiving per-branch L2 norms from the reduce
    /// kernel.  Consumed by the rescale kernel in the next launch.
    branch_grad_norms_dev: CudaSlice<f32>,
    /// `[4]` diagnostic slot — the scale each branch received on the
    /// last rescale launch (`1.0` when no scaling was needed).  Written
    /// by block (0, branch, 0) of `branch_grad_rescale`; readable at
    /// epoch boundary for HEALTH_DIAG auditing.
    branch_grad_scales_dev: CudaSlice<f32>,
    /// Maximum of the 4 branch slice lengths (element count).  Used to
    /// size the x-dimension of the rescale launch so every branch gets
    /// enough blocks to cover all its elements.
    branch_slice_max_len: usize,
    /// Plan 1 Task 13: GPU-driven Polyak-EMA tau. Cold-path (per-epoch).
    /// Reads ISV[39, 40, 12]; writes ISV[TAU_EFF_INDEX=42].
    tau_update_kernel: CudaFunction,
    /// Plan 1 Task 14: GPU-driven effective epsilon. Cold-path (per-epoch).
    /// Reads ISV[39, 40, 12]; writes ISV[EPSILON_EFF_INDEX=41].
    epsilon_update_kernel: CudaFunction,
    /// Plan 2 Task 3 D.2: GPU-driven per-branch effective gamma. Cold-path (per-epoch).
    /// Reads ISV[V_CENTER/V_HALF at 23..31] + ISV[12]; writes ISV[GAMMA_DIR..URG_EFF=43..47).
    per_branch_gamma_update_kernel: CudaFunction,
    /// D.2: per-branch gamma base values [4] on device. DIR=0.92, MAG=0.88, ORD=0.85, URG=0.80.
    per_branch_gamma_base_dev: CudaSlice<f32>,
    /// D.2: per-branch gamma max values [4] on device. DIR=0.99, MAG=0.95, ORD=0.93, URG=0.90.
    per_branch_gamma_max_dev: CudaSlice<f32>,
    /// Plan 1 Task 11: GPU-driven effective Kelly cap. Cold-path (per-epoch).
    /// Reads ISV[12] + portfolio_state[n_envs, PS_STRIDE]; writes ISV[KELLY_CAP_EFF_INDEX=44].
    kelly_cap_update_kernel: CudaFunction,
    /// Plan 1 Task 9: GPU-driven C51 atom position recompute. Cold-path (per-epoch + SGD step).
    /// Reads ISV v-range slots [23..31) + spacing_raw params; writes atom_positions_buf.
    atoms_update_kernel: CudaFunction,
    /// Plan 2 Task 1 C.1: GPU-driven per-branch Q P5/P95 percentile EMA update.
    /// Cold-path (per-epoch). Reads q_out_buf [N, total_actions]; for each branch,
    /// sorts per-sample max-Q via bitonic/quickselect, picks P5/P95, EMA into
    /// ISV[Q_P05_*=47..51) and ISV[Q_P95_*=51..55).
    q_quantile_kernel: CudaFunction,
    /// Action branch offsets in the flat [N, total_actions] Q-value buffer.
    /// [0]=dir_off=0, [1]=mag_off, [2]=ord_off, [3]=urg_off.
    /// Passed to q_quantile_reduce as the branch_offsets argument.
    q_quantile_branch_offsets_dev: CudaSlice<i32>,
    /// Action branch sizes in the flat [N, total_actions] Q-value buffer.
    /// [b] = number of actions for branch b (dir=4, mag=3, ord=3, urg=3 by default).
    /// Passed to q_quantile_reduce as the branch_sizes argument.
    q_quantile_branch_sizes_dev: CudaSlice<i32>,
    /// Host-side cached per-component norms populated at epoch boundary
    /// from the pinned result slot. Index order matches `grad_mag_*_ratio`
    /// accessors: `[iqn, cql, cql_sx, c51, c51_bs, distill, rec, pred, ens]`.
    grad_component_norms_mag: [f32; 9],
    grad_component_norms_dir: [f32; 9],
    /// Task 2.Z diagnostic — cached per-component TRUNK slice L2 norms
    /// populated by `refresh_grad_component_norms`. Index order matches
    /// mag/dir: `[iqn, cql, cql_sx, c51, c51_bs, distill, rec, pred, ens]`.
    /// Non-zero for components that actually write into trunk tensors 0..4
    /// (IQN via `apply_iqn_trunk_gradient`, Ens via
    /// `apply_ensemble_diversity_backward`); near-zero for components that
    /// write only to branch heads (C51, CQL, etc.).
    grad_component_norms_trunk: [f32; 9],
    params_buf: CudaSlice<f32>,              // [TOTAL_PARAMS] f32 master online parameters (Adam operates here)
    target_params_buf: CudaSlice<f32>,       // [TOTAL_PARAMS] f32 master target parameters (EMA operates here)
    m_buf: CudaSlice<f32>,                   // [TOTAL_PARAMS] Adam first moment (f32 for precision)
    v_buf: CudaSlice<f32>,                   // [TOTAL_PARAMS] Adam second moment (f32 for precision)
    /// Plan 4 Task 1B-iv (chain final fix): cached f32 element count of the
    /// VSN tensor slice `[95..119)` (= `(padded_byte_offset(119) −
    /// padded_byte_offset(95)) / size_of::<f32>()`). Used by the second
    /// `target_ema_update` Polyak launch that syncs target VSN params with
    /// online — without that second launch, target VSN stays at
    /// `alloc_zeros` while online VSN drifts toward measured per-group
    /// importances. Cached at construction so the per-epoch launch avoids
    /// recomputing `compute_param_sizes` + `padded_byte_offset`.
    vsn_param_total: usize,
    /// Plan 4 Task 1B-iv (chain final fix): cached byte offset from
    /// `params_buf.raw_ptr()` (and `target_params_buf.raw_ptr()`) to the
    /// start of the VSN tensor slice (= `padded_byte_offset(95)`). Used
    /// alongside `vsn_param_total` by the VSN-range Polyak EMA launch in
    /// `target_ema_update`.
    vsn_param_byte_offset: u64,
    /// Gradient L2 norm [1] — pinned device-mapped. Finalize kernel writes via dev_ptr, CPU reads via host ptr.
    grad_norm_pinned: *mut f32,
    grad_norm_dev_ptr: u64,
    grad_norm_partials: CudaSlice<f32>,                // [grad_norm_blocks] per-block partial sums
    grad_norm_blocks: usize,                           // number of blocks for grad_norm kernel
    cql_grad_scratch: CudaSlice<f32>,        // [TOTAL_PARAMS] f32 CQL gradient isolation buffer
    /// Weight decay mask [total_params]: 1.0 for trunk+value (indices 0-7), 0.0 for rest
    weight_decay_mask: CudaSlice<f32>,

    // ── Adam step counter + tau — pinned device-mapped, no HtoD copies ─
    /// Adam step counter — pinned device-mapped. GPU reads via device pointer.
    t_pinned: *mut i32,
    t_dev_ptr: u64,
    /// EMA tau — pinned device-mapped. Kernel reads via pointer, host writes directly.
    tau_pinned: *mut f32,
    tau_dev_ptr: u64,

    /// Per-sample, per-branch C51 support [B, 4, 3] stride-12 — pointer set by IQL trainer
    /// (Phase 2d). Consumed by c51_loss_batched / c51_grad_kernel / compute_expected_q /
    /// mag_concat_qdir / quantile_q_select, which index `sample_id * 12 + d * 3 + {0,1,2}`.
    per_sample_support_ptr: u64,
    /// Per-branch gradient scales [B, 4] — pointer set by IQL trainer.
    branch_scales_ptr: u64,
    /// Fixed wide v_range [2] for compute_expected_q argmax (not C51 loss).
    /// Pinned device-mapped: CPU writes v_min/v_max, GPU reads via dev_ptr. No HtoD copy.
    eval_v_range_pinned: *mut f32,
    eval_v_range_ptr: u64,
    /// Per-branch EMA-smoothed Q-stats for the ISV v-range update. Index
    /// corresponds to branch (direction=0, magnitude=1, order=2, urgency=3).
    /// Promoted from scalar to `[f32; 4]` as part of the ISV v-range
    /// unification — see `update_eval_v_range`.
    pub(crate) eval_q_mean_ema: [f32; 4],
    pub(crate) eval_q_std_ema: [f32; 4],
    eval_ema_initialized: [bool; 4],
    /// Adaptive IQN lambda readiness: 0=uncertain (suppress gradient), 1=converged (full weight).
    /// Cached host shadow for compatibility with non-host-pinned readers; the
    /// authoritative value lives in `iqn_readiness_pinned` (see below). After
    /// SP4 Layer C close-out C2 (2026-05-01) `update_iqn_readiness_kernel`
    /// writes the pinned slot directly; this field is kept in sync via the
    /// `iqn_readiness()` accessor reading through the pinned host_ptr.
    iqn_readiness: f32,
    /// IQN readiness — pinned device-mapped for CUDA graph. GPU reads via dev_ptr.
    /// SP4 Layer C close-out C2 (2026-05-01) made this the authoritative
    /// storage written by `update_iqn_readiness_kernel`; previously the host
    /// `iqn_readiness` field was the source of truth and the pinned slot a
    /// mirror per `feedback_no_cpu_compute_strict`.
    iqn_readiness_pinned: *mut f32,
    iqn_readiness_dev_ptr: u64,
    /// Streaming smoothed IQN loss — mapped-pinned scalar. SP4 Layer C
    /// close-out C2 (2026-05-01) replaced the host `iqn_loss_ema: f32` field
    /// with this mapped-pinned slot updated GPU-side by
    /// `update_iqn_readiness_kernel` per `feedback_no_cpu_compute_strict`.
    /// Cold-start sentinel `prev < 1e-12 ⇒ assign loss directly` preserves
    /// the deleted host-side formula's bootstrap branch exactly. FoldReset:
    /// 0.0 (the new fold's first call recaptures the bootstrap anchor).
    iqn_loss_ema_pinned: *mut f32,
    /// Device-mapped view of `iqn_loss_ema_pinned`. Read+written in-place by
    /// `update_iqn_readiness_kernel`; not consumed by any other GPU kernel
    /// (the gauge produced from it lives in `iqn_readiness_pinned`).
    iqn_loss_ema_dev_ptr: u64,
    /// Fold-anchor for the IQN improvement gauge — mapped-pinned scalar.
    /// SP4 Layer C close-out C2 (2026-05-01) replaced the host
    /// `iqn_loss_initial: f32` field with this mapped-pinned slot updated
    /// GPU-side by `update_iqn_readiness_kernel`. Captured once per fold on
    /// the first call where `prev < 1e-12` and never overwritten afterward
    /// (the kernel's bootstrap branch is the only writer of this field after
    /// fold reset). FoldReset: 0.0 so the next call rebootstraps the anchor.
    iqn_loss_initial_pinned: *mut f32,
    /// Device-mapped view of `iqn_loss_initial_pinned`. Read+conditionally-
    /// written in-place by `update_iqn_readiness_kernel`; not consumed by any
    /// other GPU kernel.
    iqn_loss_initial_dev_ptr: u64,

    /// Learning rate — pinned device-mapped host memory.
    /// CUDA graph captures the pointer (not the value). CPU writes new LR each epoch,
    /// kernel reads current value on replay. Zero graph recapture on cosine schedule.
    lr_pinned: *mut f32,
    /// Device pointer for lr_pinned (from cuMemHostGetDevicePointer).
    lr_dev_ptr: u64,
    /// Auxiliary LR (1e-4) for non-cosine Adam optimizers (selectivity, denoiser).
    /// Pinned device-mapped so it satisfies the const float* kernel signature.
    aux_lr_pinned: *mut f32,
    /// Device pointer for aux_lr_pinned.
    aux_lr_dev_ptr: u64,

    /// Adaptive gradient clip norm — pinned device-mapped host memory.
    /// GPU reads via device pointer (graph-safe), host writes directly (no HtoD copy).
    adaptive_clip_pinned: *mut f32,
    /// Device pointer for adaptive_clip_pinned (from cuMemHostGetDevicePointer).
    adaptive_clip_dev_ptr: u64,
    /// EMA of gradient L2 norm for adaptive clipping.
    /// SP4 Layer C close-out C4 (2026-05-01) replaced the host-resident `f32`
    /// field with `grad_norm_ema_pinned` (mapped-pinned scalar updated GPU-
    /// side by `update_adaptive_clip_kernel`) per `feedback_no_cpu_compute_strict`.
    /// Cold-start sentinel `prev <= 0.0 ⇒ assign clamped observation directly`
    /// preserves the deleted host formula's bootstrap branch. Stores the
    /// WINSORIZED grad-norm EMA (vs. fast/slow EMAs in C1 redesigned which
    /// store the RAW grad-norm) — mechanism intentional per the comment
    /// block at update_adaptive_clip line 22652-22657.
    grad_norm_ema_pinned: *mut f32,
    /// Device-mapped view of `grad_norm_ema_pinned`. Read+written in-place by
    /// `update_adaptive_clip_kernel`; not consumed by other GPU kernels via
    /// dev_ptr (the EMA itself is internal state; the kernel produces
    /// `adaptive_clip_pinned` as the actual consumed output).
    grad_norm_ema_dev_ptr: u64,
    /// SP4 Layer C close-out C4 (2026-05-01): mapped-pinned diagnostic slot
    /// for the GRAD_CLIP_OUTLIER warn log. `update_adaptive_clip_kernel`
    /// writes `(observed - clamped)` into this slot when winsorization
    /// triggers (delta > 0); the host reads it post-launch to decide
    /// whether to emit `tracing::warn!` and recovers the original observed
    /// value as `clamped + delta`. Mapped-pinned + `__threadfence_system()`
    /// guarantees the value is visible by the time the host accessor runs
    /// after the kernel launch returns.
    outlier_diag_pinned: *mut f32,
    /// Device-mapped view of `outlier_diag_pinned`. Written by
    /// `update_adaptive_clip_kernel`; read host-side after launch.
    outlier_diag_dev_ptr: u64,

    // ── Training state ──────────────────────────────────────────────
    pub(crate) adam_step: i32,
    total_params: usize,
    /// Non-ISV parameter count: EMA target sync restricted to this range.
    non_isv_params: usize,
    pub(crate) params_initialized: bool,
    pub(crate) target_params_initialized: bool,
    attention_initialized: bool,

    // ── CUDA Graphs ──────────────────────────────────────────────────
    // Graph ownership moved to FusedTrainingCtx (child graph architecture).
    // GpuDqnTrainer now only provides submit_*_ops() for capture and
    // ungraphed forward for eval. Eval forward graph exec is injected
    // from FusedTrainingCtx via set_eval_forward_exec().
    /// External eval-only forward graph exec (owned by FusedTrainingCtx).
    /// Used by replay_forward_for_q_values() for deterministic eval.
    eval_forward_exec: Option<cudarc::driver::sys::CUgraphExec>,
    /// Whether the eval forward exec has been set (graph captured).
    graphs_captured: bool,

    // ── Consolidated transfer buffers ─────────────────────────────
    /// Single staging buffer for batch upload consolidation.
    /// Layout: [states(B*SD) | next_states(B*SD) | rewards(B) | dones(B)]
    /// Actions (i32) and is_weights (f32) are uploaded separately.
    upload_staging_buf: CudaSlice<f32>,
    /// Total element count of the staging buffer.
    upload_staging_len: usize,

    /// Single readback buffer for loss download consolidation.
    /// Layout: [total_loss(1) | grad_norm(1) | td_errors(B)]
    /// One DtoH transfer replaces 3 separate PCIe round-trips.
    readback_buf: CudaSlice<f32>,
    /// Pre-allocated host-side readback vec (avoids per-step allocation).
    readback_host: Vec<f32>,

    /// Pre-allocated host-side staging vec for batch upload (avoids per-step malloc).
    /// Reused via `.clear()` + `.extend_from_slice()` each step.
    upload_staging_host: Vec<f32>,

    /// Scalar-only readback buffer: [total_loss(1) | grad_norm(1)] = 8 bytes.
    /// Used by `execute_train_scalars_only()` to avoid reading back td_errors.
    scalars_readback_buf: CudaSlice<f32>,
    scalars_readback_host: [f32; 2],

    /// Pinned host buffer [16 × f32] for async DtoH scalar readback.
    /// Layout:
    ///   [0]=loss, [1]=mse_loss, [2]=grad_norm_sq          (per-step, from graph_adam)
    ///   [3]=avg_max_q, [4]=q_min, [5]=q_max, [6]=q_mean, [7]=q_var,
    ///   [8]=atom_entropy, [9]=atom_utilization             (every 50 steps, 7 q_stats)
    ///   [10]=causal_mean_sens                              (every N steps)
    ///   [11]=ensemble_diversity_loss                       (epoch boundary)
    ///   [12..16]=reserved                                  (future use)
    /// Pinned memory enables true async cuMemcpyDtoHAsync without CPU blocking.
    readback_pinned: *mut f32,
    /// Whether there's an in-flight readback to collect
    readback_pending: bool,

    // ── Curiosity Q-penalty ─────────────────────────────────────────
    /// Per-sample curiosity prediction error buffer [B] (f32).
    /// Populated by curiosity cuBLAS GEMM pipeline before C51/MSE loss.
    /// Zero-filled when curiosity is disabled (lambda=0).
    curiosity_error_buf: CudaSlice<f32>,
    /// #18 Per-sample drawdown depths for asymmetric DD loss.
    /// Zero-filled when asymmetric_dd_weight=0.
    drawdown_depths_buf: CudaSlice<f32>,
    /// #18 Asymmetric DD loss weight (from config).
    asymmetric_dd_weight: f32,
    /// #20 Lottery ticket pruning mask [total_params] f32 (0=pruned, 1=kept).
    /// None until pruning epoch, then applied every training step.
    pruning_mask: Option<CudaSlice<f32>>,
    /// #20 Pruning mask kernel.
    pruning_mask_kernel: CudaFunction,
    /// #20 Pruning compute mask kernel.
    pruning_compute_kernel: CudaFunction,
    /// HER in-place goal relabel kernel (writes directly into padded staging buffers).
    pub(crate) her_inplace_kernel: CudaFunction,
    /// NaN detection kernels (GPU-side, no CPU readback per step).
    pub(crate) nan_check_f32_kernel: CudaFunction,
    /// Second NaN check kernel handle (formerly f32, now f32 — both check f32 buffers).
    pub(crate) nan_check_f32_kernel_b: CudaFunction,
    /// SP2: fused multi-buffer NaN check kernel handle
    /// (`dqn_nan_check_fused_f32_kernel` in `dqn_utility_kernels.cu`). One launch
    /// processes 12 backward-path slots (24-35) in 12 blocks. Reads `(ptr, len)`
    /// tables from `nan_check_buf_ptrs` / `nan_check_buf_lens` via the mapped
    /// device pointer; null-pointer entries (deferred slot 31, optional IQN
    /// slots when inactive, slots 33-35 inline elsewhere) no-op via the
    /// kernel's null-pointer guard.
    pub(crate) nan_check_fused_f32_kernel: CudaFunction,
    /// SP1 Phase C surgical fix: in-place finite clamp kernel
    /// (`dqn_clamp_finite_f32_kernel` in `dqn_utility_kernels.cu`). Replaces
    /// NaN/Inf with 0 and clamps finite |v| to ±max_abs. max_abs is sourced
    /// from dedicated SP4 ISV bound slots:
    ///   - IQN trunk path (`bw_d_h_s2`): ISV[BW_D_H_S2_BOUND_INDEX=171]
    ///     (Layer C #260, producer `bw_d_h_s2_p99`)
    ///   - Bottleneck Linear path (`d_value_logits` ∪ `d_adv_logits`):
    ///     ISV[Q_DIR_GRAD_BOUND_INDEX=172] (Layer C #260, producer
    ///     `q_dir_grad_p99` multi-sub-buffer n_sub=2)
    /// with `EPS_CLAMP_FLOOR=1.0` ε on cold-start (Invariant 1 carve-out).
    /// Used as pre-call sanitisation on cuBLAS GEMM backward inputs and
    /// post-call defence-in-depth on outputs at slots 26/32 per
    /// `docs/dqn-backward-nan-audit.md`.
    pub(crate) clamp_finite_f32_kernel: CudaFunction,
    /// SP1 Phase B: NaN flag buffer expanded from 24 → 48 slots. Slots 0-23
    /// are forward-path checks (existing); slots 24-35 are backward-path
    /// checks (new — wired by Task 4); slots 36-47 are reserved headroom
    /// for SP2 framework + SP3 observers. Reset per step, read at epoch
    /// boundary or on training-guard halts.
    pub(crate) nan_flags_buf: CudaSlice<i32>,

    /// SP4 Pearl D Wiener-EMA state for all 71 ISV-bound EMAs (40 SP4 +
    /// 29 Task-A13 retrofit + 2 Layer-C-#260 producers). Per slot, 3 floats
    /// laid out as `[sample_var, diff_var, x_lag]`. Mapped-pinned per
    /// `feedback_no_htod_htoh_only_mapped_pinned`. Reset to all-zeros at
    /// fold boundary (Pearl A sentinel via state-reset registry, Task A12;
    /// buffer grew 141→207 in Task A13.0 to cover the 29 retrofit producers,
    /// then 207→213 in Layer C #260 to cover bw_d_h_s2_p99 +
    /// q_dir_grad_p99). Indexed by ISV slot's offset from `SP4_SLOT_BASE`
    /// × 3 for SP4-block slots [131..173); retrofit producers (slots
    /// 40..69 in scratch) use scratch_idx × 3 directly.
    pub(crate) wiener_state_buf: MappedF32Buffer,

    /// SP4 Pearl C engagement counter — per (param-group, Adam-launch-block),
    /// counts threads that triggered the post-Adam clamp this step. Host
    /// reduces across blocks → engagement_rate; rate-deficit Wiener EMA
    /// detects post-clamp feedback-loop saturation and force-bumps the
    /// bound. Sized [SP4_PARAM_GROUP_COUNT × MAX_BLOCKS_PER_ADAM] = 8 × 256
    /// = 2048 ints. Mapped-pinned i32.
    pub(crate) clamp_engage_per_block_buf: MappedI32Buffer,

    /// SP4 Task A15: Pearl C rate-deficit Wiener-state buffer.
    /// `SP4_PARAM_GROUP_COUNT × 3 = 24` floats laid out as
    /// `[sample_var, diff_var, x_lag]` per param group. Tracks the
    /// engagement-rate-deficit (= engagement_rate - 0.01 theoretical p99
    /// baseline) via Pearls A+D over time. Mapped-pinned f32 — host-side
    /// `pearl_c_post_adam_engagement_check` reads/writes via host_ptr.
    /// Reset to all-zeros at fold boundary via the state-reset registry
    /// (Task A12). Layer A: this state is computed and tracked but not
    /// yet consumed — Layer B will read sustained-positive rate-deficit
    /// to force-bump `ISV[WEIGHT_BOUND[group]]`.
    pub(crate) pearl_c_rate_deficit_state_buf: MappedF32Buffer,

    /// SP4 Task A15: Pearl C rate-deficit EMA (host-only, one f32 per
    /// param group). Acts as the `prev_x_mean` for `pearls_ad_update`
    /// applied to engagement-rate deficits each Adam step. Pearls A+D
    /// state for those EMAs lives in `pearl_c_rate_deficit_state_buf`
    /// (mapped pinned). Host-only because rate-deficit reduce + EMA is
    /// post-Adam diagnostic, never read on device. Reset to all-zeros
    /// at fold boundary via the state-reset registry (Task A12).
    pub(crate) pearl_c_rate_deficit_ema: [f32; SP4_PARAM_GROUP_COUNT],

    /// SP4 producer step-observation scratch — each of the 71 producers
    /// (40 SP4 + 29 retrofit + 2 Layer-C-#260) writes its per-step
    /// `step_observation` to its dedicated slot here. The chained GPU
    /// `apply_pearls_ad_kernel` (2026-05-01 GPU-Pearls refactor) reads
    /// these step observations directly via `dev_ptr`, computes the new
    /// `x_mean` per slot, and writes to the corresponding ISV bound
    /// slot — same-stream, no host sync. Mapped-pinned f32 retained for
    /// cross-boundary debug inspection; the host-side `read_volatile`
    /// Phase 2 pattern was eliminated per
    /// `feedback_no_cpu_compute_strict.md`.
    ///
    /// Layout (post SP5 Task A1, total = 103 slots):
    ///   [0..40)  — SP4 producers (Tasks A5-A11)
    ///   [40]     — h_s2_rms_ema      → ISV[96]            (A13.0)
    ///   [41..43) — aux_heads_loss_ema → ISV[113],ISV[114]  (A13.1)
    ///   [43]     — RETIRED — formerly aux_label_scale_ema → ISV[117]
    ///                          (SP13 B1.0, 2026-05-05); slot reserved by
    ///                          SP4_PRODUCER_COUNT for layout stability —
    ///                          do not reuse without a fingerprint bump.
    ///   [44..52) — moe_expert_util_ema → ISV[118..126)     (A13.2)
    ///   [52]     — moe_gate_entropy_ema → ISV[126]         (A13.2)
    ///   [53..59) — vsn_mask_ema        → ISV[105..111)     (A13.3)
    ///   [59..63) — iqn_quantile_ema    → ISV[99..103]\med  (A13.4)
    ///   [63..69) — reward_component_ema → ISV[63..69)      (A13.5)
    ///   [69]     — bw_d_h_s2_p99      → ISV[171]           (Layer C #260)
    ///   [70]     — q_dir_grad_p99     → ISV[172]           (Layer C #260)
    ///   [71..87) — q_branch_stats output (4 per branch × 4 branches)  (SP5 A1)
    ///             [71+b*4+0]=mean, [71+b*4+1]=max_abs_dev, [71+b*4+2]=var, [71+b*4+3]=entropy
    ///   [87..103) — pearl_1_atom output (4 groups × 4 branches)        (SP5 A1)
    ///             [87..91)=v_center, [91..95)=v_half, [95..99)=headroom, [99..103)=clip_rate
    pub(crate) producer_step_scratch_buf: MappedF32Buffer,

    /// SP4 Task A7 fix-up #2: oracle sub-buffer device-pointer table.
    /// Mapped-pinned u64 buffer holding 4 contiguous K-wide ptr arrays
    /// (params, grads, adam_m, adam_v) where `K = SP4_ORACLE_TABLE_MAX_SUB`.
    /// Total: 4×K u64s. The kernel reads each ptr array starting at its
    /// own device-pointer offset (`oracle_subbuf_table_buf.dev_ptr +
    /// k_idx * K * 8`). Host populates entries [0..n_sub) once per group
    /// launch and the kernel iterates them via `n_sub` arg.
    pub(crate) oracle_subbuf_table_buf: MappedU64Buffer,

    /// SP4 Task A7 fix-up #2: oracle sub-buffer per-entry counts table.
    /// Companion to `oracle_subbuf_table_buf`. Mapped-pinned i32 buffer
    /// of `K = SP4_ORACLE_TABLE_MAX_SUB` entries. Host populates [0..n_sub)
    /// per group launch; the kernel reads via the device pointer as
    /// `const int*`.
    pub(crate) oracle_subbuf_counts_buf: MappedI32Buffer,

    /// SP4 Layer C #260 follow-up (2026-05-01): dedicated `q_dir_grad`
    /// sub-buffer device-pointer table — separate from
    /// `oracle_subbuf_table_buf` to eliminate cross-launcher temporal
    /// coupling. Mapped-pinned u64 buffer of exactly 2 entries
    /// (`d_value_logits.raw_ptr()`, `d_adv_logits.raw_ptr()`); the
    /// `q_dir_grad_p99_update` kernel always operates on n_sub=2.
    ///
    /// Cross-boundary data passage only — the host writes ptrs/counts,
    /// the kernel reads them. No formula on the host (no
    /// `feedback_no_cpu_compute_strict` violation). Populated by
    /// `launch_sp4_q_dir_grad_p99` via volatile writes once per launch.
    pub(crate) q_dir_grad_subbuf_table_buf: MappedU64Buffer,

    /// SP4 Layer C #260 follow-up (2026-05-01): companion to
    /// `q_dir_grad_subbuf_table_buf`. Mapped-pinned i32 buffer of exactly
    /// 2 entries holding the per-sub-buffer element counts
    /// (`d_value_logits.len()`, `d_adv_logits.len()`).
    pub(crate) q_dir_grad_subbuf_counts_buf: MappedI32Buffer,

    /// SP2: fused NaN-check buffer pointer table. 12 device pointers (u64) for
    /// slots 24-35. Host+device-visible mapped-pinned buffer; populated via
    /// host-side write (no HtoD copy) per
    /// `feedback_no_htod_htoh_only_mapped_pinned`. Populated ONCE in
    /// `populate_nan_check_meta` (slot pointers are stable across the
    /// trainer's lifetime). Slot 31 (deferred ensemble) holds 0 — the fused
    /// kernel skips that block.
    pub(crate) nan_check_buf_ptrs: MappedU64Buffer,

    /// SP2: fused NaN-check buffer length table. 12 i32 lengths for slots
    /// 24-35. Host+device-visible mapped-pinned buffer; populated via
    /// host-side write (no HtoD copy) alongside `nan_check_buf_ptrs`.
    pub(crate) nan_check_buf_lens: MappedI32Buffer,

    /// #20 Pruning epoch (epoch at which to compute the mask).
    pruning_epoch: usize,
    /// #20 Pruning fraction (0.7 = prune 70% of smallest weights).
    pruning_fraction: f32,

    /// #34 Causal intervention: scratch buffer for intervened states [B, state_dim_padded] f32.
    causal_states_scratch: CudaSlice<f32>,
    /// #34 Causal intervention: per-feature sensitivity [market_dim] f32.
    causal_sensitivity_buf: CudaSlice<f32>,
    /// #34 Causal intervention: Q-values from intervened forward [B, total_actions] f32.
    causal_q_scratch: CudaSlice<f32>,
    /// #34 Causal intervention config.
    pub(crate) causal_config: CausalInterventionConfig,
    /// #34 GPU-native causal intervention kernel.
    causal_intervene_kernel: CudaFunction,
    /// #34 GPU-native causal Q-delta reduction kernel.
    causal_reduce_kernel: CudaFunction,
    /// #34 GPU-native causal mean-reduce kernel (sensitivity → single scalar).
    causal_mean_reduce_kernel: CudaFunction,
    /// #34 Scratch buffer for causal mean reduction output (1 × f32).
    causal_mean_scratch: CudaSlice<f32>,

    /// Previous step's grad_buf snapshot for vaccine comparison [total_params] f32.
    prev_grad_buf: CudaSlice<f32>,

    /// D1/N1: Stable device buffer [total_params] f32 holding the current
    /// best snapshot for distillation. The CUDA Graph captures the pointer
    /// address (baked, stable). At epoch boundaries, when the snapshot ring
    /// accepts a new best, the ring's buffer is DtoD-copied into this one
    /// so every graph replay sees the freshest target without needing a
    /// graph re-capture. Initialized to a copy of `params_buf` so the
    /// initial `(params − best)` delta is zero — distillation is a no-op
    /// until a real snapshot overwrites the contents.
    distill_best_buf: CudaSlice<f32>,

    /// #32 Gradient vaccine: saved g_train gradient [total_params] f32.
    vaccine_grad_save: CudaSlice<f32>,
    /// #32 Gradient vaccine: dot(g_train, g_val) and |g_val|^2 result [2] f32.
    vaccine_dot_norm_buf: CudaSlice<f32>,
    vaccine_block_results: CudaSlice<f32>,  // [grad_norm_blocks * 2] partials for dot+norm
    /// #32 Gradient vaccine kernels.
    vaccine_dot_kernel: CudaFunction,
    vaccine_dot_finalize_kernel: CudaFunction,
    vaccine_project_kernel: CudaFunction,

    /// #31 Bottleneck hidden activation [B, bottleneck_dim] f32. None when bottleneck_dim=0.
    bn_hidden_buf: CudaSlice<f32>,
    /// #31 Bottleneck + portfolio concatenated [B, bottleneck_dim + portfolio_dim] f32.
    /// This replaces states_buf as input to h_s1 when bottleneck is active.
    bn_concat_buf: CudaSlice<f32>,
    /// Target-net mirror of `bn_hidden_buf` — fed by `tg_w_ptrs[33]` (target's
    /// w_bn) on `next_states_buf[market]`. Required so the target trunk sees
    /// the same compressed-features representation the online trunk does;
    /// without this the target encoder reads raw `next_states[0..s1_input_dim)`
    /// = `[market | ofi | tlob | mtf]` instead of `[bn_market_proj | portfolio]`,
    /// which silently mismatches the online encoder's input semantics.
    tg_bn_hidden_buf: CudaSlice<f32>,
    /// Target-net mirror of `bn_concat_buf`.
    tg_bn_concat_buf: CudaSlice<f32>,
    /// DDQN argmax-pass mirror — fed by **online** weights `on_w_ptrs[33]`
    /// on `next_states_buf[market]`. The DDQN argmax forward runs the online
    /// network on next_states, so its bottleneck input is online-w_bn @
    /// next_states (distinct from `tg_bn_hidden_buf` which uses target-w_bn).
    on_next_bn_hidden_buf: CudaSlice<f32>,
    /// DDQN argmax-pass mirror of `bn_concat_buf` (online-on-next_states).
    on_next_bn_concat_buf: CudaSlice<f32>,
    /// SP15 Wave 4.1b OOB fix (2026-05-07): pre-loaded handle for the
    /// bottleneck-aware `bn_tanh_concat_dd_kernel` (replaces the legacy
    /// `bn_tanh_concat_kernel` + `bn_tanh_concat_f32_kernel` fields with a
    /// single dd_pct-appending kernel). Wave 4.1b originally migrated the 3
    /// production trainer call sites (online forward, target forward, DDQN
    /// argmax) to a `launch_sp15_bn_concat_dd` free function that resolved
    /// the symbol via `load_cubin` + `load_function` ON EVERY CALL — that
    /// pattern is incompatible with CUDA Graph capture: those CUDA driver
    /// API calls allocate memory and mutate driver state, which is NOT
    /// capturable inside `cuStreamBeginCapture`. Calling the launcher from
    /// `submit_forward_ops_main` (graph-captured forward child) caused a
    /// SEGV at exit 139 — the loader raced with the capture-mode driver
    /// state and the host-side corruption surfaced as a segfault before
    /// `CAPTURE_PHASE_FORWARD_DONE` could print. The experience collector's
    /// equivalent caller (line 4127 of gpu_experience_collector.rs) was
    /// already pre-loading correctly via `bn_tanh_concat_fn` field — this
    /// field restores the same graph-safe pattern on the trainer side. All
    /// 3 trainer call sites consume this single handle (Hopper's
    /// "no-CUfunction-sharing-across-children" comment elsewhere refers to
    /// `dqn_utility_kernels` workhorse kernels like `dqn_saxpy_f32_kernel`
    /// that are invoked from MULTIPLE child captures — this kernel only
    /// runs in the forward + ddqn children, mirroring `bn_tanh_backward_kernel`'s
    /// single-handle pattern for the forward+backward chain).
    bn_tanh_concat_dd_kernel: CudaFunction,
    /// #31 Backward scratch: d_loss/d_bn_concat [B, concat_dim] f32.
    bn_d_concat_buf: CudaSlice<f32>,
    /// #31 Backward scratch: d_loss/d_bn [B, bn_dim] f32 (after tanh derivative).
    bn_d_hidden_buf: CudaSlice<f32>,
    /// #31 Bottleneck tanh backward kernel.
    bn_tanh_backward_kernel: CudaFunction,
    /// #31 Bottleneck bias gradient kernel.
    bn_bias_grad_kernel: CudaFunction,

    // ── Plan 4 Task 1B-iii: VSN feature-selection buffers ───────────────
    /// VSN-gated states for the online-on-states forward path. Same shape
    /// and stride as `states_buf` so all downstream consumers (bottleneck
    /// Linear, GRN encoder via bn_concat, OFI concat, mag_concat) read the
    /// gated state through unchanged ldb math. `[B, STATE_DIM_PADDED]`.
    vsn_gated_states_buf: CudaSlice<f32>,
    /// VSN-gated next_states for the target-on-next_states forward pass
    /// (uses `tg_w_ptrs` Polyak EMA copy of online VSN weights).
    /// `[B, STATE_DIM_PADDED]`.
    vsn_gated_next_states_buf: CudaSlice<f32>,
    /// VSN-gated next_states for the DDQN argmax pass (uses `on_w_ptrs`
    /// online VSN weights — DDQN argmax runs the online net on next_states).
    /// `[B, STATE_DIM_PADDED]`.
    vsn_gated_next_states_for_ddqn_buf: CudaSlice<f32>,
    /// Assembled per-group importance logits for the online-on-states pass
    /// `[B, SL_NUM_FEATURE_GROUPS]`. SAVED for 1B-iv backward.
    vsn_logits_buf: CudaSlice<f32>,
    /// Per-group softmax mask for the online-on-states pass
    /// `[B, SL_NUM_FEATURE_GROUPS]`. SAVED for both `vsn_mask_ema_update`
    /// (ISV producer, 1B-i) and 1B-iv backward.
    vsn_mask_buf: CudaSlice<f32>,
    /// Throwaway logits scratch for the target-on-next_states pass
    /// `[B, SL_NUM_FEATURE_GROUPS]`. Per-pass distinct buffer so the online
    /// pass's saved `vsn_logits_buf` isn't clobbered by the target pass.
    vsn_logits_target_scratch: CudaSlice<f32>,
    /// Throwaway mask scratch for the target-on-next_states pass
    /// `[B, SL_NUM_FEATURE_GROUPS]`. Per-pass distinct so the online pass's
    /// `vsn_mask_buf` isn't clobbered (the ISV producer reads
    /// `vsn_mask_buf` only).
    vsn_mask_target_scratch: CudaSlice<f32>,
    /// Throwaway logits scratch for the DDQN-online-on-next_states pass
    /// `[B, SL_NUM_FEATURE_GROUPS]`.
    vsn_logits_ddqn_scratch: CudaSlice<f32>,
    /// Throwaway mask scratch for the DDQN-online-on-next_states pass
    /// `[B, SL_NUM_FEATURE_GROUPS]`.
    vsn_mask_ddqn_scratch: CudaSlice<f32>,
    /// Per-group `Linear_1[g]` post-ReLU activations `[B, VSN_HIDDEN_DIM]`
    /// for the online-on-states pass — 6 disjoint buffers because they are
    /// SAVED for 1B-iv backward (each group's Linear_1 backward needs its
    /// own pre-ReLU/post-ReLU pair). Target / DDQN passes share a single
    /// scratch of the same shape (overwrite-per-group is fine — no save).
    vsn_save_h1_g0_buf: CudaSlice<f32>,
    vsn_save_h1_g1_buf: CudaSlice<f32>,
    vsn_save_h1_g2_buf: CudaSlice<f32>,
    vsn_save_h1_g3_buf: CudaSlice<f32>,
    vsn_save_h1_g4_buf: CudaSlice<f32>,
    vsn_save_h1_g5_buf: CudaSlice<f32>,
    /// Single shared scratch `[B, VSN_HIDDEN_DIM]` reused by target / DDQN
    /// passes' per-group Linear_1 outputs (no save — target/DDQN are
    /// inference-only). All 6 groups overwrite this same buffer in sequence.
    vsn_h1_inference_scratch: CudaSlice<f32>,
    /// Shared `[B]` scratch overwritten per group by the per-group `Linear_2[g]`
    /// output before it's scattered into a column of `vsn_logits_*_buf`.
    /// One buffer suffices across online/target/DDQN passes because each pass
    /// runs the per-group loop sequentially before the next pass starts.
    vsn_linear2_scratch_buf: CudaSlice<f32>,
    /// Device-side feature-group begin / end indices `[SL_NUM_FEATURE_GROUPS]`
    /// each, as i32 (matches the kernel's `int*` arg type). Populated once at
    /// construction from `FEATURE_GROUP_RANGES`; stable across all batches
    /// and all 3 forward passes.
    vsn_group_begins_buf: CudaSlice<i32>,
    vsn_group_ends_buf: CudaSlice<i32>,
    /// Plan 4 Task 1B-iv: VSN backward `d_logits` scratch `[B, num_groups]`.
    /// Output of `vsn_softmax_and_gate_backward`; consumed by per-group
    /// `Linear_2` cuBLAS backward via `strided_gather` + a tight per-group
    /// scratch (no strided cuBLAS variant required).
    vsn_d_logits_buf: CudaSlice<f32>,
    /// Plan 4 Task 1B-iv: VSN backward `d_state` scratch
    /// `[B, STATE_DIM_PADDED]`. Output of `vsn_softmax_and_gate_backward`
    /// (per-feature softmax-Jacobian + gate term); not propagated downstream
    /// because `state_buf` is not trainable. Allocated so the kernel has a
    /// valid write target; the value is then unused.
    vsn_d_state_buf: CudaSlice<f32>,
    /// Plan 4 Task 1B-iv: assembled VSN backward INPUT
    /// `d_vsn_gated_state[B, STATE_DIM_PADDED]`. Built from (a) cuBLAS
    /// bottleneck Linear backward dX into the market slice `[0..market_dim]`
    /// (row stride = `state_dim_padded` via `launch_dx_only_lda`) and
    /// (b) `vsn_d_gated_state_portfolio_pad_kernel` for the portfolio slice
    /// `[market_dim..STATE_DIM]` + zero pad `[STATE_DIM..STATE_DIM_PADDED]`.
    /// Consumed by `vsn_backward` as `d_gated_state` argument.
    vsn_d_gated_state_buf: CudaSlice<f32>,
    /// Plan 4 Task 1B-iv: per-group `d_logit_g` scratch `[B]`. The VSN
    /// backward extracts column `g` of `vsn_d_logits_buf [B, num_groups]`
    /// into this contiguous buffer (via `strided_gather`) so the per-group
    /// `Linear_2[g]` cuBLAS backward sees a tightly-packed `dY[B, 1]`. One
    /// shared buffer suffices because the per-group loop runs sequentially.
    vsn_d_logit_scratch_buf: CudaSlice<f32>,
    /// Plan 4 Task 1B-iv: per-group `d_h1_g` scratch `[B, VSN_HIDDEN_DIM=16]`.
    /// Output of per-group `Linear_2[g]` backward dX, then ReLU-mask
    /// in-place against the saved `vsn_save_h1_g{g}_buf`, then consumed as
    /// `dY` of per-group `Linear_1[g]` backward. One shared buffer because
    /// the per-group loop is sequential.
    vsn_d_h1_scratch_buf: CudaSlice<f32>,
    /// Plan 4 Task 1B-iv: gather kernel handle (`strided_gather` from
    /// `experience_kernels.cu`) used by `vsn_backward` to extract per-group
    /// `d_logit_g` columns from the assembled `d_logits[B, num_groups]`.
    /// Loaded from `EXPECTED_Q_CUBIN` next to `strided_scatter` in the
    /// constructor.
    vsn_logit_gather_kernel: CudaFunction,
    /// Plan 4 Task 1B-iv: portfolio-passthrough + pad kernel handle
    /// (`vsn_d_gated_state_portfolio_pad_kernel` from `dqn_utility_kernels.cu`)
    /// used by `launch_cublas_backward_to` to populate the portfolio slice +
    /// zero pad of `vsn_d_gated_state_buf` from `bn_d_concat_buf`. The
    /// market slice is overwritten by `launch_dx_only_lda` (bottleneck Linear
    /// backward) afterwards.
    vsn_d_gated_state_portfolio_kernel: CudaFunction,
    /// Plan 4 Task 1B-iv-ext: VSN dW scratch for the IQN-trunk auxiliary
    /// backward path (`apply_iqn_trunk_gradient`). Sized to exactly the
    /// padded VSN range [95..119) (24 tensors). The aux path runs the same
    /// VSN backward chain as the main path and writes per-group dW/dB into
    /// this scratch, then the trailing SAXPY (`saxpy_f32_aux`) accumulates
    /// `iqn_lambda * iqn_readiness * iqn_budget * scratch` into
    /// `grad_buf` over the same 24-tensor range. Independent buffer (not
    /// shared with `iqn_trunk_m`) because `iqn_trunk_m` is sized only to
    /// the 13 trunk tensors at indices [0..13) — extending it would leave
    /// 82 dead tensors of zeroed scratch between trunk and VSN, which the
    /// SAXPY can't skip without becoming non-contiguous.
    vsn_dw_iqn_aux_scratch: CudaSlice<f32>,
    /// Plan 4 Task 1B-iv-ext: VSN dW scratch for the ensemble-diversity
    /// auxiliary backward path (`apply_ensemble_diversity_backward`). Same
    /// shape and rationale as `vsn_dw_iqn_aux_scratch`; the trailing SAXPY
    /// scales by the caller-supplied diversity weight. The CQL aux path
    /// reuses `cql_grad_scratch` (which is sized `total_params` and SAXPY-
    /// covered end-to-end by `apply_cql_saxpy`) so it does not need a
    /// per-aux VSN scratch — VSN dW writes there land at offsets [95..119)
    /// and ride the existing CQL SAXPY into `grad_buf`.
    vsn_dw_ensemble_aux_scratch: CudaSlice<f32>,
    /// Plan 4 Task 1B-i / 1B-iii: VSN per-group mask EMA producer kernel.
    /// Single-block 256-thread shmem-reduction kernel reads `vsn_mask_buf`
    /// (saved by the online-on-states VSN forward) and EMA-updates
    /// `ISV[VSN_MASK_GROUP_0_EMA_INDEX..VSN_MASK_GROUP_5_EMA_INDEX]`
    /// (slots 105..111). Loaded from `VSN_MASK_EMA_CUBIN`. Consumer-side
    /// launch wired in `training_loop.rs` next to `launch_h_s2_rms_ema`.
    vsn_mask_ema_kernel: CudaFunction,

    // ── HEALTH_DIAG GPU port — Phase 2A (2026-04-28) ────────────────────────
    /// Orchestrator owning the mapped-pinned `HealthDiagSnapshot` + the
    /// `health_diag_*` kernel-family handles that fill it. Phase 2A landing
    /// holds only the `health_diag_isv_mirror` kernel (ISV signal-bus
    /// scalar-copy producer); phases 2B-2E append further kernels to the
    /// same orchestrator. Phase 3 wires the launch chain into the
    /// captured graph; Phase 4 deletes the CPU-side HEALTH_DIAG reductions
    /// and reads via `health_diag.snapshot().host_ref()` directly.
    /// `Option<>` so legacy non-CUDA test harnesses can construct trainers
    /// without a full CUDA context — production is always `Some`.
    health_diag: Option<GpuHealthDiag>,

    // ── Plan 4 Task 6 Commit B: aux-heads forward + backward orchestrators ──
    /// Forward orchestrator (next-bar regression + 5-class regime CE).
    /// Owns the 5 forward / loss-reduce / label-builder kernel handles + the
    /// `aux_heads_loss_ema_update` ISV producer handle. Construction loads
    /// from `AUX_HEADS_CUBIN` + `AUX_HEADS_LOSS_EMA_CUBIN`.
    aux_heads_fwd: AuxHeadsForwardOps,
    /// Backward orchestrator — 3 kernel handles (next_bar_backward,
    /// regime_backward, param_grad_reduce). Consumed by `aux_heads_backward`
    /// (called from `launch_cublas_backward_to` BEFORE `encoder_backward_chain`
    /// so the SAXPY accumulating `aux_dh_s2_*` into `bw_d_h_s2` lands before
    /// the trunk backward consumes that accumulator).
    aux_heads_bwd: AuxHeadsBackwardOps,
    /// `[B, AUX_HIDDEN_DIM]` saved post-ELU activation of the next-bar head's
    /// Linear_1 (consumed by `aux_next_bar_backward` for the dW1 / dh_s2 chain).
    aux_nb_hidden_buf: CudaSlice<f32>,
    /// `[B, AUX_NEXT_BAR_K=2]` saved logits — written by
    /// `aux_next_bar_forward` (B1.1a). Saved primarily for diagnostic
    /// readback / future consumers; backward reads the softmax tile
    /// instead of re-softmaxing the logits since the softmax is already
    /// materialised. Field renamed from `aux_nb_pred_buf` (regression
    /// scalar) to `aux_nb_logits_buf` (B1.1a logits) per
    /// `feedback_no_legacy_aliases.md`.
    aux_nb_logits_buf: CudaSlice<f32>,
    /// `[B, AUX_NEXT_BAR_K=2]` saved softmax tile — written by
    /// `aux_next_bar_forward` (B1.1a). Three downstream consumers read
    /// this tile:
    ///   - `aux_next_bar_loss_reduce`  (CE numerator)
    ///   - `aux_next_bar_backward`     (`d_logits = softmax - one_hot(label)`)
    ///   - `aux_dir_acc_reduce_kernel` (argmax-vs-label compare)
    ///   - `aux_pred_to_isv_tanh_kernel` (mean(softmax[1] - softmax[0]) → ISV[375])
    aux_nb_softmax_buf: CudaSlice<f32>,
    /// `[B] i32` per-sample direction label in `{-1, 0, 1}` (B1.1a):
    /// `-1` = mask/skip, `0` = down, `1` = up. **B1.1a is producer-less**
    /// — the buffer is `alloc_zeros`-initialised (every sample receives
    /// label 0 → "down") and stays at zero until B1.1b lands the
    /// producer kernel that computes labels from the price trajectory.
    /// "Model learns to predict class 0 (down) everywhere" is the known
    /// degraded training behavior between B1.1a and B1.1b.
    /// Consumed by `aux_next_bar_loss_reduce`, `aux_next_bar_backward`,
    /// AND `aux_dir_acc_reduce_kernel`.
    aux_nb_label_buf: CudaSlice<i32>,
    /// `[1]` scalar mean cross-entropy — written by
    /// `aux_next_bar_loss_reduce`, read by `aux_heads_loss_ema_update`
    /// (the ISV producer).
    aux_nb_loss_scalar_buf: CudaSlice<f32>,
    /// `[1]` scalar `B_valid` count (number of non-mask labels in the
    /// batch) — written by `aux_next_bar_loss_reduce`, read by
    /// `aux_next_bar_backward` so loss + backward share the same
    /// `1/B_valid` divisor (derivatives of the same scalar function).
    aux_nb_valid_count_buf: CudaSlice<f32>,
    /// `[B, AUX_HIDDEN_DIM]` saved post-ELU activation of the regime head's
    /// Linear_1 (consumed by `aux_regime_backward`).
    aux_rg_hidden_buf: CudaSlice<f32>,
    /// `[B, AUX_REGIME_K=5]` pre-softmax logits — written by
    /// `aux_regime_forward`, read by `aux_regime_loss_reduce` AND by
    /// `aux_regime_backward` (which re-softmaxes on the fly).
    aux_rg_logits_buf: CudaSlice<f32>,
    /// `[B] uint8` per-sample regime labels in `[0, AUX_REGIME_K)`. Produced
    /// per-step by `aux_regime_label_from_states` (reads
    /// `states_buf[:, OFI_START + 29]` = `regime_score` ∈ [0,1], bucketed
    /// into 5 uniform bins). Consumed by both `aux_regime_loss_reduce` and
    /// `aux_regime_backward`.
    aux_rg_label_buf: CudaSlice<u8>,
    /// `[1]` scalar mean cross-entropy — written by `aux_regime_loss_reduce`.
    aux_rg_loss_scalar_buf: CudaSlice<f32>,
    /// `[1]` scalar mean classification accuracy — written by
    /// `aux_regime_loss_reduce` alongside the CE loss. Currently unread by
    /// downstream consumers; kept as a real device buffer since the kernel's
    /// output ABI requires it.
    #[allow(dead_code)]
    aux_rg_correct_scalar_buf: CudaSlice<f32>,
    /// `[B, SH2]` per-sample dh_s2 emitted by `aux_next_bar_backward`. SAXPY-
    /// accumulated into `bw_d_h_s2` before `encoder_backward_chain` runs.
    aux_dh_s2_nb_buf: CudaSlice<f32>,
    /// `[B, SH2]` per-sample dh_s2 emitted by `aux_regime_backward`. Same
    /// SAXPY-accumulation pattern as `aux_dh_s2_nb_buf`.
    aux_dh_s2_rg_buf: CudaSlice<f32>,
    /// `[B, AUX_HIDDEN_DIM, SH2]` per-sample dW1 partial — next-bar head.
    aux_partial_nb_w1: CudaSlice<f32>,
    /// `[B, AUX_HIDDEN_DIM]` per-sample db1 partial — next-bar head.
    aux_partial_nb_b1: CudaSlice<f32>,
    /// `[B, AUX_NEXT_BAR_K, AUX_HIDDEN_DIM]` per-sample dW2 partial —
    /// next-bar head. SP13 B1.1a: K_NB flipped 1 → 2, so the partial
    /// alloc grew from `[B, H]` to `[B, K, H]` (mirrors regime head's
    /// `[B, K, H]` shape with K=5).
    aux_partial_nb_w2: CudaSlice<f32>,
    /// `[B, AUX_NEXT_BAR_K]` per-sample db2 partial — next-bar head.
    /// SP13 B1.1a: K_NB flipped 1 → 2, so the partial alloc grew from
    /// `[B]` to `[B, K]`. DEDICATED — does NOT alias `aux_nb_label_buf`.
    aux_partial_nb_b2: CudaSlice<f32>,
    /// `[B, AUX_HIDDEN_DIM, SH2]` per-sample dW1 partial — regime head.
    aux_partial_rg_w1: CudaSlice<f32>,
    /// `[B, AUX_HIDDEN_DIM]` per-sample db1 partial — regime head.
    aux_partial_rg_b1: CudaSlice<f32>,
    /// `[B, AUX_REGIME_K, AUX_HIDDEN_DIM]` per-sample dW2 partial — regime head.
    aux_partial_rg_w2: CudaSlice<f32>,
    /// `[B, AUX_REGIME_K]` per-sample db2 partial — regime head.
    aux_partial_rg_b2: CudaSlice<f32>,
    /// `[max_aux_param_len]` reusable per-tensor reduce target for
    /// `aux_param_grad_reduce`. Sized to the largest of the 8 aux tensors.
    aux_param_grad_final_buf: CudaSlice<f32>,

    // ── SP22 H6 vNext Phase B0+B2: trade-outcome aux head orchestrator + buffers ──
    //
    // Parallel to `aux_heads_fwd` / `aux_heads_bwd` + `aux_nb_*` buffers
    // but for the K=3 trade-outcome head. The trainer holds the forward
    // ops (forward + loss_reduce + label producer) AND the backward ops
    // (backward only — param_grad_reduce is reused from `aux_heads_bwd`).
    // Collector-side rollout buffers + ops handle land in Phase B3 wireup.
    //
    /// Phase B0 (2026-05-14): forward orchestrator for the K=3
    /// trade-outcome head. Holds the `aux_trade_outcome_forward`,
    /// `aux_trade_outcome_loss_reduce`, and `trade_outcome_label_kernel`
    /// handles. Phase B3 wires the launch chain into the trainer's
    /// replay-batch path; for B2 it's instantiated but unused.
    aux_to_fwd: AuxTradeOutcomeForwardOps,
    /// Phase B0 (2026-05-14): backward orchestrator for the K=3
    /// trade-outcome head. Holds the `aux_trade_outcome_backward`
    /// handle. The `aux_param_grad_reduce` K-generic reducer is reused
    /// from `aux_heads_bwd`.
    aux_to_bwd: AuxTradeOutcomeBackwardOps,
    /// `[B, AUX_HIDDEN_DIM]` saved post-ELU activation of the
    /// trade-outcome head's Linear_1. Consumed by
    /// `aux_trade_outcome_backward` for the dW1 / dh_s2_aux chain.
    aux_to_hidden_buf: CudaSlice<f32>,
    /// `[B, AUX_OUTCOME_K=3]` saved logits — written by
    /// `aux_trade_outcome_forward`. Kept for parity with the K=2 sibling
    /// (backward reads the softmax tile, not the logits — the softmax
    /// is already materialized in the forward).
    aux_to_logits_buf: CudaSlice<f32>,
    /// `[B, AUX_OUTCOME_K=3]` saved softmax tile — written by
    /// `aux_trade_outcome_forward`. Three downstream consumers will
    /// read this tile:
    ///   - `aux_trade_outcome_loss_reduce` (CE numerator)
    ///   - `aux_trade_outcome_backward`    (d_logits = softmax - one_hot)
    ///   - Phase C 3-slot state assembly producer (state[121..124])
    aux_to_softmax_buf: CudaSlice<f32>,
    /// `[B] i32` per-sample trade-outcome label in `{-1, 0, 1, 2}`:
    /// `-1` = mask (no trade close at this bar), `0` = Profit, `1` =
    /// Stop, `2` = Timeout. Sparse: most bars (~95-99%) are `-1`.
    /// Producer: `trade_outcome_label_kernel` per-step (reads
    /// `trade_close_per_sample[i*L+t]` and the save-for-backward
    /// `pnl_vs_target_at_close_per_env` / `pnl_vs_stop_at_close_per_env`
    /// from Phase A3). Direct-to-trainer PER gather will populate this
    /// on every replay sample step in Phase B3 (mirrors the `aux_nb_
    /// label_buf` direct-to-trainer pattern). Consumed by
    /// `aux_trade_outcome_loss_reduce` and `aux_trade_outcome_backward`.
    /// Cold-start `alloc_zeros` initialization yields label 0 (Profit)
    /// for every sample → degraded "model learns to predict Profit
    /// everywhere" until the PER gather wires up; same known-degraded
    /// behaviour the K=2 head went through between B1.1a and B1.1b.
    aux_to_label_buf: CudaSlice<i32>,
    /// `[1]` scalar mean cross-entropy — written by
    /// `aux_trade_outcome_loss_reduce`, will be read by Phase F's
    /// HEALTH_DIAG aux-outcome-CE EMA producer (parallel to
    /// `aux_heads_loss_ema_update`).
    aux_to_loss_scalar_buf: CudaSlice<f32>,
    /// `[1]` scalar `B_valid` count (number of non-mask labels in the
    /// batch) — written by `aux_trade_outcome_loss_reduce`, read by
    /// `aux_trade_outcome_backward` so loss + backward share the same
    /// `1/B_valid` divisor (derivatives of the same scalar function).
    /// Sparse-label arithmetic: typical `B_valid` ≈ 1-5% of nominal B.
    aux_to_valid_count_buf: CudaSlice<f32>,
    /// `[B, SH2]` per-sample dh_s2_aux emitted by
    /// `aux_trade_outcome_backward`. SAXPY-accumulated into the aux
    /// trunk's `dh_s2_aux_accum` before `aux_trunk_backward` runs.
    /// Encoder stop-grad enforced structurally by `aux_trunk_backward`
    /// (no `dx_in` output param), so Q's encoder is protected from aux
    /// contamination per SP14 Phase C.5b.
    aux_dh_s2_to_buf: CudaSlice<f32>,
    /// `[B, AUX_HIDDEN_DIM, SH2]` per-sample dW1 partial. Reduced along
    /// the batch dim by the K-generic `aux_param_grad_reduce` (reused
    /// from `aux_heads_bwd`) into the final dW1 written into the flat
    /// `params_buf` Adam slot [163].
    aux_partial_to_w1: CudaSlice<f32>,
    /// `[B, AUX_HIDDEN_DIM]` per-sample db1 partial. Reduced into the
    /// flat `params_buf` Adam slot [164].
    aux_partial_to_b1: CudaSlice<f32>,
    /// `[B, AUX_OUTCOME_K=3, AUX_HIDDEN_DIM]` per-sample dW2 partial.
    /// Reduced into the flat `params_buf` Adam slot [165].
    aux_partial_to_w2: CudaSlice<f32>,
    /// `[B, AUX_OUTCOME_K=3]` per-sample db2 partial. Reduced into the
    /// flat `params_buf` Adam slot [166].
    aux_partial_to_b2: CudaSlice<f32>,
    /// SP20 Phase 5 (2026-05-10): per-sample aux confidence at the SAMPLED
    /// state — `[batch_size]` f32, range `[0, 0.5]` (K=2 peak-softmax-above-
    /// uniform; 0.0 = uniform / no information sentinel; 0.5 = peak softmax
    /// at 1.0). Populated by direct-to-trainer PER gather from the replay
    /// buffer's `aux_conf` ring (filled by `experience_env_step` per (i, t)
    /// in Phase 3 Task 3.4). Wired in at trainer init via
    /// `GpuReplayBuffer::set_trainer_aux_conf_ptr(self.aux_conf_at_state_buf
    /// .raw_ptr())` so PER's `gather_f32_scalar` writes the SAMPLED bar's
    /// per-batch aux_conf directly into here on every step (mirrors the
    /// SP13 B1.1b `aux_nb_label_buf` direct-to-trainer pattern). Consumed
    /// by `c51_loss_batched`'s reward gate at the Bellman projection:
    /// `gate = sigmoid((aux_conf - ISV[AUX_CONF_THRESHOLD]) / ISV[AUX_GATE_TEMP])`,
    /// `r_used = gate × reward`. NULL-tolerant (gate=1.0 = no-op) at the
    /// kernel level so test scaffolds without a wired aux head still work.
    /// `alloc_zeros` cold-start: 0.0 sentinel → at threshold ≈ 0.10 and
    /// temp ≈ 0.05 the gate is `sigmoid(-2) ≈ 0.12` → reward is mostly
    /// suppressed pre-population — the "graceful degradation" semantic
    /// from the Phase 3 Task 3.4 audit doc spec §4.4. Once the PER
    /// gather populates it on the first sample step, the per-bar values
    /// from the producer drive the gate as designed.
    aux_conf_at_state_buf: CudaSlice<f32>,
    /// Per-step aux-loss weight, baked into the SAXPY alpha at graph-capture
    /// time. ISV-driven via `set_aux_weight`:
    ///   `aux_weight = clamp(0.1 × LEARNING_HEALTH × (1 - tanh(SHARPE_SCALE × sharpe_ema)), 0.05, 0.3)`.
    /// The clamp `[0.05, 0.3]` is a numerical-stability bound (Invariant 1
    /// carve-out per `feedback_isv_for_adaptive_bounds.md`) — `0.05` keeps
    /// the gradient signal alive when `learning_health == 0`, `0.3` caps the
    /// share so aux can never dominate the policy gradient.
    aux_weight: f32,

    // ── Phase 3: MoE forward/backward buffers + orchestrator ─────────────
    /// MoE kernel orchestrator (mixture fwd/bwd, load-balance, EMA, softmax).
    moe_head: GpuMoeHead,
    /// Gate sub-network layer-1 hidden activations [B, MOE_GATE_HIDDEN].
    /// Saved for backward (dW_gate_w1 = gate_h1^T @ d_gate_h1).
    moe_gate_h1_buf: CudaSlice<f32>,
    /// Gate pre-softmax logits [B, MOE_NUM_EXPERTS].
    /// Saved for softmax backward (needed by moe_softmax_backward).
    moe_gate_pre_buf: CudaSlice<f32>,
    /// Gate softmax output = gate weights g[B, MOE_NUM_EXPERTS].
    /// Saved for mixture forward + mixture backward + load-balance loss.
    moe_gate_softmax_buf: CudaSlice<f32>,
    /// 8 expert layer-1 hidden buffers, each [B, MOE_EXPERT_BOTTLENECK].
    /// Saved for backward (dW_ek_w1 = expert_h1_k^T @ d_expert_h1_k).
    moe_expert_h1_bufs: [CudaSlice<f32>; MOE_NUM_EXPERTS],
    /// Stacked expert outputs [K, B, MOE_SH2] (K=8, layout row-major).
    /// Saved for both mixture forward (direct input) and moe_dgate_reduce.
    moe_expert_outputs_buf: CudaSlice<f32>,
    /// d(loss)/d(expert_outputs) [K, B, MOE_SH2] from moe_mixture_backward.
    moe_de_k_buf: CudaSlice<f32>,
    /// d(loss)/d(gate_soft) [B, MOE_NUM_EXPERTS] from moe_dgate_reduce.
    moe_dg_buf: CudaSlice<f32>,
    /// d(loss)/d(gate_pre) [B, MOE_NUM_EXPERTS] from moe_softmax_backward.
    moe_dg_pre_buf: CudaSlice<f32>,
    /// Accumulated d(loss)/d(save_h_s1) from all 8 expert backward dX.
    /// After accumulation, DtoD-copied into bw_d_h_s2 for encoder_backward_chain.
    moe_dh_s1_scratch: CudaSlice<f32>,
    /// d(loss)/d(gate_h1) [B, MOE_GATE_HIDDEN] from gate_w2 backward dX.
    moe_gate_dh1_buf: CudaSlice<f32>,
    /// Per-expert load-balance loss [MOE_NUM_EXPERTS].
    moe_load_balance_loss_per_k: CudaSlice<f32>,
    /// Load-balance total loss scalar [1] — pinned device-mapped.
    /// GPU writes via dev_ptr, CPU reads host_ptr for optional logging.
    moe_load_balance_loss_total_pinned: *mut f32,
    /// Device pointer for moe_load_balance_loss_total_pinned.
    moe_load_balance_loss_total_dev_ptr: u64,
    /// Adaptive load-balance λ controller — permanent floor (= old static
    /// 0.01 default) read by the `moe_lambda_eff_update` GPU kernel.
    /// Per `pearl_blend_formulas_must_have_permanent_floor.md`, ISV[128]
    /// is bounded below by this value (`λ_eff ≥ moe_lambda_floor` always).
    moe_lambda_floor: f32,
    /// Maximum additional λ added when the gate is fully collapsed.
    /// Peak λ_eff is `moe_lambda_floor + moe_lambda_max_extra`.
    moe_lambda_max_extra: f32,
    /// Fraction of `ln(K)` treated as "diverse enough" — entropy ≥ target
    /// produces zero deficit (λ_eff = floor); entropy → 0 produces full
    /// deficit (λ_eff = floor + max_extra).
    moe_entropy_target_frac: f32,

    /// #21 Stochastic depth: per-layer scale buffer [3] (h_s1, h_s2, h_v).
    /// Written by GPU RNG kernel before each graph replay.
    stochastic_depth_scale_buf: CudaSlice<f32>,
    /// #21 Stochastic depth scale kernel function (applies scale to activations).
    stochastic_depth_kernel: CudaFunction,
    /// #21 Stochastic depth RNG kernel (generates scales on GPU, zero CPU RNG).
    stochastic_depth_rng_kernel: CudaFunction,
    /// #21 GPU-resident RNG state for stochastic depth [1] u32.
    stochastic_depth_rng_state: CudaSlice<u32>,
    /// #21 Stochastic depth drop probability (0.0=disabled, 0.2=20% drop).
    stochastic_depth_prob: f32,

    /// #27 Per-sample ensemble std for disagreement Q-penalty.
    /// Populated by `upload_ensemble_std()` before loss kernel launch.
    /// Zero-filled when ensemble is disabled.
    ensemble_std_buf: CudaSlice<f32>,
    /// #27 Ensemble disagreement penalty weight. Q_target -= weight * ensemble_std.
    ensemble_disagreement_weight: f32,
    /// Compiled curiosity element-wise kernel: build input vectors from states + action one-hot.
    curiosity_prepare_input_func: CudaFunction,
    /// Compiled curiosity element-wise kernel: in-place bias + LeakyReLU(0.01).
    curiosity_bias_leaky_relu_func: CudaFunction,
    /// Compiled curiosity element-wise kernel: bias + MSE vs next_states.
    curiosity_bias_mse_func: CudaFunction,
    /// Curiosity input buffer [B, CUR_INPUT] for cuBLAS GEMM 1 input.
    curiosity_input_buf: CudaSlice<f32>,
    /// Curiosity hidden buffer [B, CUR_HIDDEN] for cuBLAS GEMM 1 output / GEMM 2 input.
    curiosity_hidden_buf: CudaSlice<f32>,
    /// Curiosity prediction buffer [B, CUR_OUTPUT] for cuBLAS GEMM 2 output.
    curiosity_pred_buf: CudaSlice<f32>,
    /// Raw device pointers to curiosity weights (set via `set_curiosity_weights()`).
    /// These point into the `CuriosityWeightSet` owned by the experience collector.
    /// Values are updated in-place by the curiosity trainer — addresses are stable.
    /// u64::MAX = not set (curiosity disabled).
    curiosity_w1_ptr: u64,
    curiosity_b1_ptr: u64,
    curiosity_w2_ptr: u64,
    curiosity_b2_ptr: u64,

    // ── Batch staging buffers ──────────────────────────────────
    // Pre-allocated CudaSlice<f32> buffers for DtoD copy of tensors
    // from GpuBatch (states, next_states only).
    // Rewards, dones, weights are F32 in GpuBatch and use direct DtoD to F32 bufs.
    states_staging_buf: CudaSlice<f32>,      // [B * STATE_DIM]
    next_states_staging_buf: CudaSlice<f32>, // [B * STATE_DIM]

    // ── cuBLAS batched forward (Phase 2 — high-occupancy path) ───────
    /// cuBLAS forward context (handle + bias kernels).
    cublas_forward: CublasForward,
    /// Second cuBLAS forward context for Pass 3 (Double DQN).
    /// Uses the shared cuBLAS handle — runs sequentially on the main stream.
    cublas_forward_ddqn: CublasForward,

    // Scratch buffers for cuBLAS target network forward pass.
    // Online activations reuse the existing save_h_* buffers.
    // Target only needs h_s2 (for DDQN) + scratch for intermediate layers.
    /// Target trunk layer 1 scratch: [B, SH1]
    tg_h_s1_scratch: CudaSlice<f32>,
    /// Target trunk layer 2 output — kept for Double-DQN action selection: [B, SH2]
    tg_h_s2_buf: CudaSlice<f32>,
    /// Target value head hidden scratch: [B, VH]
    tg_h_v_scratch: CudaSlice<f32>,
    /// Target branch head hidden scratch — 4 separate buffers for multi-stream parallelism: [B, AH]
    tg_h_b0_scratch: CudaSlice<f32>,
    tg_h_b1_scratch: CudaSlice<f32>,
    tg_h_b2_scratch: CudaSlice<f32>,
    tg_h_b3_scratch: CudaSlice<f32>,
    /// Target value head logits: [B, NA] — f32
    tg_v_logits_buf: CudaSlice<f32>,
    /// Target branch logits: [B, (B0+B1+B2+B3)*NA] — f32
    tg_b_logits_buf: CudaSlice<f32>,

    /// Online value head logits (from cuBLAS pass on current states): [B, NA] — f32
    on_v_logits_buf: CudaSlice<f32>,
    /// Online branch logits (from cuBLAS pass on current states): [B, (B0+B1+B2+B3)*NA] — f32
    on_b_logits_buf: CudaSlice<f32>,

    /// Online-next value logits (Double DQN action selector on next_states): [B, NA] — f32
    on_next_v_logits_buf: CudaSlice<f32>,
    /// Online-next branch logits (Double DQN on next_states): [B, (B0+B1+B2+B3)*NA] — f32
    on_next_b_logits_buf: CudaSlice<f32>,
    /// Online-next scratch (reused): h_s1[B,SH1], h_s2[B,SH2], h_v[B,VH], h_b[B,AH]
    on_next_h_s1_scratch: CudaSlice<f32>,
    on_next_h_s2_scratch: CudaSlice<f32>,
    on_next_h_v_scratch: CudaSlice<f32>,
    on_next_h_b_scratch: CudaSlice<f32>,

    /// Compiled C51 loss kernel (standalone, replaces the fused forward+loss kernel).
    c51_loss_kernel: CudaFunction,
    /// Deterministic loss reduction kernel: sequential sum of per_sample_loss → total_loss.
    /// Shared by C51 loss, C51 mixup, and MSE loss (identical signature).
    c51_loss_reduce_kernel: CudaFunction,
    /// C51 loss gradient kernel: computes dL/d_logits for cuBLAS backward.
    c51_grad_kernel: CudaFunction,
    /// MSE loss kernel on expected Q-values (warmup before C51).
    mse_loss_kernel: CudaFunction,
    /// MSE loss gradient kernel through softmax expectation (warmup before C51).
    mse_grad_kernel: CudaFunction,
    /// C51 blend factor: 0.0 = pure MSE, 1.0 = pure C51.
    /// Ramps linearly over warmup epochs for smooth transition.
    c51_alpha: f32,
    /// Scratch buffers for blended loss (MSE grad stored here, then blended into d_value/d_adv)
    d_value_logits_mse: CudaSlice<f32>,
    d_adv_logits_mse: CudaSlice<f32>,
    /// Gradient w.r.t. value logits: [B, NA] — f32 for native atomicAdd
    d_value_logits_buf: CudaSlice<f32>,
    /// Gradient w.r.t. branch logits: [B, (B0+B1+B2+B3)*NA] — f32 for native atomicAdd
    d_adv_logits_buf: CudaSlice<f32>,
    /// Staging for backward pass value logits gradient
    d_value_logits: CudaSlice<f32>,
    /// Staging for backward pass advantage logits gradient
    d_adv_logits: CudaSlice<f32>,


    // ── cuBLAS batched backward (Phase 2 Task 2) ──────────────────────
    /// cuBLAS backward context (handle + ReLU mask + bias grad kernels).
    cublas_backward: CublasBackward,

    // Scratch buffers for inter-layer gradient propagation during cuBLAS backward.
    // Sized for the widest layer at each point in the network.
    // All f32 in the backward chain.
    /// Accumulated d_h_s2 from value head + all 4 branch heads: [B, SH2] — f32
    bw_d_h_s2: CudaSlice<f32>,
    /// Upstream gradient for shared layer 1: [B, SH1] — f32
    bw_d_h_s1: CudaSlice<f32>,
    /// Upstream gradient for value FC: [B, VH] — f32
    bw_d_h_v: CudaSlice<f32>,
    /// Branch FC upstream gradients (one per branch): [B, AH] — f32
    bw_d_h_b0: CudaSlice<f32>,
    bw_d_h_b1: CudaSlice<f32>,
    bw_d_h_b2: CudaSlice<f32>,
    bw_d_h_b3: CudaSlice<f32>,
    /// GLU backward scratch: d_value [B, AH] — f32 (reused across branches)
    bw_d_glu_value: CudaSlice<f32>,
    /// GLU backward scratch: d_gate_pre [B, AH] — f32 (reused across branches)
    bw_d_glu_gate: CudaSlice<f32>,

    // ── GRN trunk backward scratch (Plan 4 Task 2c.3c.3) ─────────────
    // 4 buffers per GRN block × 2 blocks (h_s1 + h_s2) = 8 total.
    // Consumed by `apply_grn_trunk_backward_raw` helper (Task 2c.3c.4).
    // Each pair of buffers per block:
    //   d_pre_LN       — phase1 input (d_glu_out + d_residual aliased) [B, hidden]
    //   d_linear_b_out — phase1 output for caller's Linear_b backward GEMM [B, 2*hidden]
    //   d_elu_out      — caller's Linear_b backward output, phase2 input [B, hidden]
    //   d_linear_a_out — phase2 output for caller's Linear_a backward GEMM [B, hidden]
    /// h_s2 GRN d_pre_LN scratch [B, SH2] — wired by Task 2c.3c.4
    #[allow(dead_code)]
    bw_grn_h_s2_d_pre_ln: CudaSlice<f32>,
    /// h_s2 GRN d_linear_b_out scratch [B, 2*SH2] — wired by Task 2c.3c.4
    #[allow(dead_code)]
    bw_grn_h_s2_d_linear_b_out: CudaSlice<f32>,
    /// h_s2 GRN d_elu_out scratch [B, SH2] — wired by Task 2c.3c.4
    #[allow(dead_code)]
    bw_grn_h_s2_d_elu_out: CudaSlice<f32>,
    /// h_s2 GRN d_linear_a_out scratch [B, SH2] — wired by Task 2c.3c.4
    #[allow(dead_code)]
    bw_grn_h_s2_d_linear_a_out: CudaSlice<f32>,
    /// h_s1 GRN d_pre_LN scratch [B, SH1] — wired by Task 2c.3c.4
    #[allow(dead_code)]
    bw_grn_h_s1_d_pre_ln: CudaSlice<f32>,
    /// h_s1 GRN d_linear_b_out scratch [B, 2*SH1] — wired by Task 2c.3c.4
    #[allow(dead_code)]
    bw_grn_h_s1_d_linear_b_out: CudaSlice<f32>,
    /// h_s1 GRN d_elu_out scratch [B, SH1] — wired by Task 2c.3c.4
    #[allow(dead_code)]
    bw_grn_h_s1_d_elu_out: CudaSlice<f32>,
    /// h_s1 GRN d_linear_a_out scratch [B, SH1] — wired by Task 2c.3c.4
    #[allow(dead_code)]
    bw_grn_h_s1_d_linear_a_out: CudaSlice<f32>,
    // ── Expected Q-value kernel (ad-hoc validation, not captured in CUDA Graph) ─
    /// Converts C51 value+advantage logits → expected Q-values (validation path).
    /// Input: on_v_logits_buf [B, NA], on_b_logits_buf [B, (B0+B1+B2+B3)*NA]
    /// Output: q_out_buf [B, B0+B1+B2+B3]
    expected_q_kernel: CudaFunction,
    /// GPU reduction: q_out_buf → 7 scalars [avg_max_q, q_min, q_max, q_mean, q_var, atom_entropy, atom_utilization]
    q_stats_kernel: CudaFunction,
    /// GPU buffer for Q-value statistics [7 floats]
    q_stats_buf: CudaSlice<f32>,
    /// Per-branch Q-stats reducer — writes 28 floats to `per_branch_q_stats_pinned`
    /// (4 × 7, branch-major). Runs alongside `q_stats_kernel`; feeds the
    /// ISV v-range per-branch EMA state in `update_eval_v_range`.
    q_stats_per_branch_kernel: CudaFunction,
    /// Pinned device-mapped readback for per-branch Q-stats [4 × 7 = 28 floats].
    /// GPU writes via `per_branch_q_stats_dev_ptr`; CPU reads host-side.
    per_branch_q_stats_pinned: *mut f32,
    per_branch_q_stats_dev_ptr: u64,
    /// GPU buffer for atom utilization accumulation [2 floats: sum_entropy, sum_utilized].
    /// Populated by `atom_stats_finalize` kernel (phase 2 of the deterministic reduction).
    atom_stats_buf: CudaSlice<f32>,
    /// Per-block partial sums for `compute_expected_q`'s atom-stat reduction.
    /// Phase 1 output: [max_blocks * 2] where block b writes [b*2+0]=entropy, [b*2+1]=util.
    /// Phase 2 (`atom_stats_finalize`) reduces these deterministically into atom_stats_buf.
    atom_stats_block_sums_buf: CudaSlice<f32>,
    /// Phase-2 finalize kernel for atom_stats — sequential reduction, bit-stable.
    atom_stats_finalize_kernel: CudaFunction,
    /// Pinned device-mapped readback for q_stats [7] + q_out sample 0 [total_actions] floats.
    /// GPU writes via q_readback_dev_ptr, CPU reads via q_readback_pinned — zero sync.
    q_readback_dev_ptr: u64,
    /// DtoH uses pinned DMA-capable destination for faster async transfer.
    q_readback_pinned: *mut f32,

    // ── CQL (Conservative Q-Learning) penalty kernel ─────────────────
    /// Computes CQL logit gradients: dCQL/d_value_logits and dCQL/d_adv_logits.
    /// Only used when `config.use_cql == true && config.cql_alpha > 0`.
    cql_logit_grad_kernel: Option<CudaFunction>,
    /// CQL scratch: value logit gradients [B, NA] — f32 for native atomicAdd
    cql_d_value_logits: CudaSlice<f32>,
    /// CQL scratch: advantage logit gradients [B, (B0+B1+B2+B3)*NA] — f32
    cql_d_adv_logits: CudaSlice<f32>,

    // ── F5/D2: Q-gap barrier gradient kernel ─────────────────────────
    /// Raises Q(argmax) and lowers Q(second_max) for the direction branch
    /// when q_gap < 0.05 * health. Piggybacks on the CQL d-logit buffers.
    /// Loaded from c51_loss_kernel.cubin ("barrier_gradient_direction").
    barrier_gradient_kernel: Option<CudaFunction>,

    // ── F4/D5: Information Bottleneck variance gradient kernel ───────
    /// Spreads Q values across direction-branch actions when population
    /// variance is below min_var (0.01). Piggybacks on the CQL d-logit
    /// buffers. Loaded from c51_loss_kernel.cubin ("ib_gradient_direction").
    ib_gradient_kernel: Option<CudaFunction>,

    /// v8: PopArt reward normalization kernel (running mean/variance).
    popart_normalize_kernel: CudaFunction,
    /// v8: Robust PopArt kernel (median/IQR normalization).
    popart_robust_kernel: CudaFunction,
    /// v8: PopArt running mean [1].
    popart_mean: CudaSlice<f32>,
    /// v8: PopArt running variance [1].
    popart_var: CudaSlice<f32>,
    /// v8: PopArt sample count [1].
    popart_count: CudaSlice<f32>,

    // ── Plan 4 Task 2c.3c.5: H_S2 RMS EMA producer ────────────────────
    /// Single-block kernel (256 threads, shmem-reduction, no atomicAdd). Reduces
    /// RMS = sqrt(sum_sq / (B*SH2)) over `save_h_s2 [B, SH2]` and EMA-updates
    /// ISV[H_S2_RMS_EMA_INDEX] with α = 0.05. Cold-path-cadence (one launch
    /// per training step, alongside `reward_component_ema`). Producer-only —
    /// 2c.3c.6 wires the consumer in `mag_concat_qdir`'s adaptive-scale path.
    /// Loaded from `h_s2_rms_ema_kernel.cubin`.
    h_s2_rms_ema_kernel: CudaFunction,

    // ── SP4 Layer A Task A5: target_q_p99 producer ─────────────────────
    /// SP4 first end-to-end producer kernel. Single-block kernel that reads
    /// `denoise_target_q_buf` (target-network Q-values [B, total_actions] flattened),
    /// computes p99(|target_q|) via the `sp4_histogram_p99<256>` header-only
    /// device function, writes the step observation to
    /// `producer_step_scratch_buf[0]`. Loaded from
    /// `SP4_TARGET_Q_P99_CUBIN`. Cold-path launch via
    /// `launch_sp4_target_q_p99` chains the GPU `apply_pearls_ad_kernel`
    /// on the same stream (2026-05-01 GPU-Pearls refactor). Tasks A6-A9
    /// mirror this kernel's shape + the launcher's Pearls A+D wire-up
    /// to populate the remaining 39 SP4 ISV bound slots.
    target_q_p99_update: CudaFunction,

    /// SP4 Layer A Task A6: atom_pos_p99 × 4 branches producer kernel.
    /// Single parameterised single-block kernel; one kernel handle, the
    /// launcher (`launch_sp4_atom_pos_p99_all_branches`) loops over 4
    /// branches with distinct slice pointer + scratch slot (slots 1..5),
    /// then chains a single GPU `apply_pearls_ad_kernel` launch
    /// (n_slots=4) on the same stream that writes ISV[ATOM_POS_BOUND_BASE
    /// + branch ∈ {132, 133, 134, 135}] in one device-side loop. Loaded
    /// from `SP4_ATOM_POS_P99_CUBIN`.
    atom_pos_p99_update: CudaFunction,

    /// SP4 Layer A Task A7: Pearl B fused per-param-group statistics oracle
    /// kernel. Single-block kernel reads `(params, grads, adam_m, adam_v)`
    /// once and runs 4 fused passes (5 for the trunk group):
    ///   Pass A: WEIGHT_BOUND[group] = p99(|params|)
    ///   Pass B: ADAM_M_BOUND[group] = p99(|adam_m|)
    ///   Pass C: ADAM_V_BOUND[group] = p99(|adam_v|)
    ///   Pass D: WD_RATE[group]      = |Σ w·g| / max(Σ w², EPS_DIV)
    ///   Pass E: L1_LAMBDA[trunk]    = (mean|g|/mean|w|) × entropy_deficit
    ///                                 (group 0 only, gated by l1_idx ≥ 0).
    /// Pearl B 4× memory-bandwidth reduction vs four naive single-stat
    /// producer kernels: each buffer read once for all 4-5 outputs. The
    /// launcher (`launch_sp4_param_group_oracles_all_groups`) loops over
    /// `SP4_PARAM_GROUP_COUNT=8` groups (DQN trunk/value/branches +
    /// IQN/IQL-hi/IQL-lo/Attn/Curiosity), one launch per group, then chains
    /// the GPU `apply_pearls_ad_kernel` per output slot (single-slot
    /// launches; non-contiguous (scratch, ISV) per group prevents batching).
    /// Loaded from `SP4_PARAM_GROUP_ORACLE_CUBIN`.
    param_group_oracle_update: CudaFunction,

    /// SP4 Layer A Task A8: grad_norm_p99 producer (degenerate single-element).
    /// `grad_norm_buf` is a single-element f32 device buffer (populated by
    /// `launch_grad_norm_finalize`). The kernel reduces to a single-thread
    /// copy of `grad_norm_buf[0]` → `producer_step_scratch_buf[scratch_idx=38]`
    /// with `__threadfence_system()`. The launcher
    /// (`launch_sp4_grad_norm_p99`) chains the GPU `apply_pearls_ad_kernel`
    /// on the same stream, writing back to ISV[GRAD_CLIP_BOUND_INDEX=168] +
    /// Wiener state at `wiener_state_buf[(168-SP4_SLOT_BASE)*3 = 111..114)`.
    /// Loaded from `SP4_GRAD_NORM_P99_CUBIN`.
    grad_norm_p99_update: CudaFunction,

    /// SP4 Layer A Task A9: h_s2_p99 producer for ISV[H_S2_BOUND=169].
    /// Single-block, 256-thread kernel reading `save_h_s2 [B × SH2]` (the
    /// trunk-output activation surface populated each step by the online
    /// forward), computes p99(|save_h_s2|) via the header-only
    /// `sp4_histogram_p99<256>` device function, writes the step observation
    /// to `producer_step_scratch_buf[39]` with `__threadfence_system()`.
    /// The launcher (`launch_sp4_h_s2_p99`) launches the GPU `apply_pearls_ad_kernel`
    /// on the same stream which writes back to ISV[H_S2_BOUND_INDEX=169] +
    /// Wiener state at `wiener_state_buf[(169-SP4_SLOT_BASE)*3 = 114..117)`.
    /// NB: distinct from `h_s2_rms_ema_update` (slot 96, RMS signal). Loaded
    /// from `SP4_H_S2_P99_CUBIN`.
    h_s2_p99_update: CudaFunction,

    /// SP4 Layer C #260 (2026-05-01): bw_d_h_s2_p99 producer kernel for
    /// ISV[BW_D_H_S2_BOUND_INDEX=171]. Single-block 256-thread histogram
    /// p99 kernel reading `bw_d_h_s2 [B × SH2]` (IQN trunk-backward d_h_s2
    /// buffer); writes step_p99 to `producer_step_scratch_buf[69]`. The
    /// launcher (`launch_sp4_bw_d_h_s2_p99`) chains the GPU
    /// `apply_pearls_ad_kernel` on the same stream — graph-capture-
    /// compatible by construction. Migrated from SP1-Phase-C
    /// `1e6 × ISV[H_S2_RMS_EMA].max(1.0)` per
    /// `feedback_isv_for_adaptive_bounds`. Loaded from
    /// `SP4_BW_D_H_S2_P99_CUBIN`.
    bw_d_h_s2_p99_update: CudaFunction,

    /// SP4 Layer C #260 (2026-05-01): q_dir_grad_p99 producer kernel for
    /// ISV[Q_DIR_GRAD_BOUND_INDEX]. Single-block 256-thread multi-sub-
    /// buffer histogram p99 kernel reading the union of `d_value_logits`
    /// ∪ `d_adv_logits` (n_sub=2); writes step_p99 to
    /// `producer_step_scratch_buf[70]`. The launcher
    /// (`launch_sp4_q_dir_grad_p99`) populates the 2-element pointer +
    /// count tables in dedicated `q_dir_grad_subbuf_table_buf` /
    /// `q_dir_grad_subbuf_counts_buf` (separate from
    /// `oracle_subbuf_table_buf` to eliminate cross-launcher temporal
    /// coupling), then chains the GPU `apply_pearls_ad_kernel` on the
    /// same stream — graph-capture-compatible by construction. Migrated
    /// from SP1-Phase-C `1e6 × ISV[Q_DIR_ABS_REF].max(1.0)` per
    /// `feedback_isv_for_adaptive_bounds`. Loaded from
    /// `SP4_Q_DIR_GRAD_P99_CUBIN`.
    q_dir_grad_p99_update: CudaFunction,

    /// SP4 GPU-only Pearls A+D applicator (2026-05-01). Single-thread
    /// device-side loop kernel that applies Pearls A+D to N consecutive
    /// ISV slots on the producer's stream. Consumed by every SP4 producer
    /// launcher (5 SP4 + 6 A13 retrofits = 11 launchers) plus the inline
    /// `aux_heads_forward` Step 2b call site. Replaces the pre-2026-05-01
    /// host-side `apply_pearls_to_slot` helper that broke CUDA Graph
    /// capture at `aux_label_scale_ema` mid-step (smoke `smoke-test-v9kjv`).
    /// Stream-ordered with the producer kernel that wrote `scratch_buf` —
    /// no host sync needed; graph-capture-compatible by construction.
    /// Loaded from `apply_pearls_kernel.cubin`.
    apply_pearls_ad_kernel: CudaFunction,

    // ── SP5 Task A1: per-branch Q-stats + Pearl 1 atom-span kernels ───
    /// SP5 Task A1 (2026-05-01): q_branch_stats producer kernel.
    /// Single-block 4-thread kernel (one thread per branch: Dir/Mag/Ord/Urg).
    /// Reads `q_out_buf [B, total_actions=13]` row-major, computes
    /// (mean, max_abs_dev, var, entropy) per branch, writes 16 floats to
    /// `producer_step_scratch_buf[71..87)`. __threadfence_system() at end
    /// ensures visibility to the chained `pearl_1_atom_kernel`.
    /// No atomicAdd (feedback_no_atomicadd — single thread per branch).
    /// Loaded from `q_branch_stats_kernel.cubin`.
    q_branch_stats_kernel: CudaFunction,
    /// SP5 Task A1 (2026-05-01): Pearl 1 atom-span controller kernel.
    /// Single-block 4-thread kernel (one thread per branch). Reads
    /// q_branch_stats scratch outputs + ATOM_HEADROOM_BASE from ISV +
    /// atoms_clip_count_buf [4 i32]; updates headroom via controller
    /// (TARGET_CLIP_RATE=0.01, HEADROOM_LR=0.01 — Invariant 1 anchors);
    /// writes 16 floats to `producer_step_scratch_buf[87..103)`.
    /// Chained `apply_pearls_ad_kernel` (24 single-slot launches) then
    /// smooths via Pearls A+D into ISV[174..190) + ISV[218..226).
    /// Loaded from `pearl_1_atom_kernel.cubin`.
    pearl_1_atom_kernel: CudaFunction,
    /// SP5 Task A2 (2026-05-01): Pearl 3 per-branch NoisyNet sigma controller kernel.
    /// Single-block 4-thread kernel (one thread per branch). Reads ATOM_V_HALF
    /// (Pearl 1, ISV[178..182)) + BRANCH_ENTROPY (ISV[218..222)) + SIGMA_FRACTION
    /// (ISV[214..218), Pearl A sentinel 0 → bootstrap 0.1); entropy-deficit
    /// controller (target = log(n_actions[b]) × 0.7, LR=0.005, envelope [0.001,0.5]);
    /// writes (sigma[4], SF[4]) to `producer_step_scratch_buf[103..111)`.
    /// Chained `apply_pearls_ad_kernel` (8 single-slot launches) then
    /// smooths via Pearls A+D into ISV[210..218).
    /// Layer A only — NoisyLinear consumer migration deferred to Layer B.
    /// No atomicAdd (feedback_no_atomicadd), no CPU compute (feedback_no_cpu_compute_strict).
    /// Loaded from `pearl_3_sigma_kernel.cubin`.
    pearl_3_sigma_kernel: CudaFunction,
    /// SP5 Task A3 + SP7 (2026-05-03): Pearl 2 IQN budget + flatness producer.
    /// Single-block 4-thread kernel (one thread per branch). Reads Q_VAR_PER_BRANCH
    /// (Pearl 1's q_branch_stats, ISV[222..226)) + NOISY_SIGMA (Pearl 3 output,
    /// ISV[210..214)); computes per-branch flatness = clamp(var_q / (σ²+EPS_DIV), 0, 1)
    /// and derives (budget_iqn, flatness) per branch. Writes 8 floats to
    /// `producer_step_scratch_buf[115..119, 127..131)` (IQN + flatness; slots
    /// 111..115 and 119..127 are reserved-for-future after T6 retirement).
    /// Chained `apply_pearls_ad_kernel` (8 single-slot launches) then smooths via
    /// Pearls A+D into ISV[BUDGET_IQN_BASE, FLATNESS_BASE].
    /// SP7: C51/CQL/ENS budget ownership transferred to loss_balance_controller_kernel.
    /// Kernel signature: 6 args (isv, q_var_base, sigma_base, scratch, iqn_idx, flatness_idx).
    /// Must run AFTER launch_sp5_pearl_1_atom (Q_VAR) AND
    /// launch_sp5_pearl_3_sigma (NOISY_SIGMA).
    /// No atomicAdd (feedback_no_atomicadd), no CPU compute (feedback_no_cpu_compute_strict).
    /// Loaded from `pearl_2_budget_kernel.cubin`.
    pearl_2_budget_kernel: CudaFunction,
    /// SP7 Task 5 (2026-05-03): loss-balance controller (CQL + C51 per-branch
    /// budget adapter). 8-thread single-block kernel (2 loss heads × 4 branches).
    /// Reads grad_decomp_result_dev_ptr (IQN@0, CQL_SX@6, C51@9), FLATNESS_BASE,
    /// prior budgets, prior Wiener state; writes 24 floats to
    /// `producer_step_scratch_buf[218..242)`. Followed by 24 apply_pearls_ad_kernel
    /// launches → ISV[BUDGET_CQL_BASE, BUDGET_C51_BASE, LB_*_VAR_{CQL,C51}_BASE].
    /// Producer-only — call site in training_loop.rs lands at T7 per
    /// `feedback_no_partial_refactor`. Loaded from
    /// `loss_balance_controller_kernel.cubin`.
    loss_balance_controller_kernel: CudaFunction,
    /// SP8 (Fix 36, 2026-05-03): GPU-only train_active_frac canary producer.
    /// Single-block, single-thread kernel reads
    /// `monitoring_summary[5..17)` (the 12-bin action_counts produced by
    /// `monitoring_reduce`) and writes 1 float to
    /// `scratch[SCRATCH_TRAIN_ACTIVE_FRAC=250]`. Followed by 1
    /// apply_pearls_ad_kernel launch → ISV[TRAIN_ACTIVE_FRAC_INDEX=321].
    /// Replaces the host-side compute at `training_loop.rs:3392-3396` per
    /// `feedback_no_cpu_compute_strict.md`. Loaded from
    /// `train_active_frac_compute_kernel.cubin`.
    train_active_frac_compute_kernel: CudaFunction,
    /// SP8 (Fix 36, 2026-05-03): ISV-driven adaptive MAX_BUDGET producer.
    /// Single-block, 8-thread kernel (2 heads × 4 branches) reads
    /// `ISV[TRAIN_ACTIVE_FRAC_INDEX]` and writes 8 floats to
    /// `scratch[SCRATCH_LB_MAX_BUDGET_CQL=251..259)`. Followed by 8
    /// apply_pearls_ad_kernel launches → ISV[LB_MAX_BUDGET_{CQL,C51}_BASE
    /// ..+4). Replaces the prior hardcoded `MAX_BUDGET=1.0f` constant in
    /// `loss_balance_controller_kernel.cu` per
    /// `pearl_controller_anchors_isv_driven.md`. Loaded from
    /// `loss_balance_max_budget_compute_kernel.cubin`.
    loss_balance_max_budget_compute_kernel: CudaFunction,
    /// SP9 (Fix 37, 2026-05-03): GPU producer for eval-side magnitude
    /// intent distribution. Replaces DtoH `read_eval_intent_magnitude_distribution()`.
    /// Single-block, 3-thread kernel (one per Q/H/F bin); writes 3 floats
    /// to `scratch[SCRATCH_SP9_EVAL_DIST_BASE=262..265)`. Followed by 3
    /// apply_pearls_ad_kernel launches → ISV[EVAL_DIST_Q/H/F_INDEX=336..339).
    /// Loaded from `eval_intent_dist_compute_kernel.cubin`.
    sp9_eval_intent_dist_compute_kernel: CudaFunction,
    /// SP9 (Fix 37, 2026-05-03): intent vs eval Full-magnitude divergence
    /// producer. Single-block, single-thread kernel; reads
    /// `monitoring_summary[5..17)` + `ISV[EVAL_DIST_F_INDEX]`; writes
    /// `intent_f / eval_f` to `scratch[SCRATCH_SP9_INTENT_EVAL_DIVERGENCE=261]`.
    /// Followed by 1 apply_pearls_ad_kernel launch →
    /// ISV[INTENT_EVAL_DIVERGENCE_INDEX=332]. Loaded from
    /// `intent_eval_divergence_compute_kernel.cubin`.
    sp9_intent_eval_divergence_compute_kernel: CudaFunction,
    /// SP9 (Fix 37, 2026-05-03): EMA baseline of magnitude-branch Q-variance.
    /// Single-block, single-thread; reads `ISV[FLATNESS_BASE+1=207]`; writes
    /// to `scratch[SCRATCH_SP9_Q_VAR_MAG_EMA=260]`. Followed by 1
    /// apply_pearls_ad_kernel launch → ISV[Q_VAR_MAG_EMA_INDEX=331].
    /// Loaded from `q_var_mag_ema_compute_kernel.cubin`.
    sp9_q_var_mag_ema_compute_kernel: CudaFunction,
    /// SP9 (Fix 37, 2026-05-03): main Kelly cold-start warmup-floor producer.
    /// Single-block, single-thread; reads 8 ISV inputs and computes
    ///   base_floor × (1 − max(statistical_conf, behavioral_conf, temporal_conf))
    /// per `pearl_cold_start_exit_signal_or.md`. Writes to
    /// `scratch[SCRATCH_SP9_KELLY_WARMUP_FLOOR=259]`. Followed by 1
    /// apply_pearls_ad_kernel launch → ISV[KELLY_WARMUP_FLOOR_INDEX=330],
    /// consumed in `unified_env_step_core` (`trade_physics.cuh`) as an
    /// adaptive max-floor on the Kelly cap. Loaded from
    /// `kelly_warmup_floor_compute_kernel.cubin`.
    sp9_kelly_warmup_floor_compute_kernel: CudaFunction,
    /// SP11 Fix 39 (2026-05-04, A1.1): val-sharpe Δ + variance canary
    /// producer. Single-block, single-thread kernel reads
    /// `val_sharpe_history_pinned` (mapped-pinned 2-element [prev, curr])
    /// + `ISV[VAL_SHARPE_DELTA_EMA_INDEX]`; writes raw delta and squared
    /// deviation to `scratch[SCRATCH_SP11_VAL_SHARPE_DELTA_BASE..+2)`.
    /// Followed by 1 apply_pearls_ad_kernel launch (n_slots=2) →
    /// ISV[VAL_SHARPE_DELTA_EMA_INDEX=350, VAL_SHARPE_VAR_EMA_INDEX=351].
    /// Loaded from `val_sharpe_delta_compute_kernel.cubin`.
    sp11_val_sharpe_delta_compute_kernel: CudaFunction,
    /// SP11 Fix 39 (2026-05-04, A1.2): per-bar saboteur engagement canary
    /// producer. Single-block, 256-thread kernel with grid-stride iteration
    /// over `saboteur_delta_reward_buf` (per-bar |Δreward| populated by
    /// `experience_env_step` at the saboteur perturbation site). Block tree-
    /// reduces engagement count, divides by n_bars; writes fraction to
    /// `scratch[SCRATCH_SP11_SABOTEUR_ENGAGEMENT=268]`. Followed by 1
    /// apply_pearls_ad_kernel launch → ISV[SABOTEUR_ENGAGEMENT_RATE_INDEX=358].
    /// Loaded from `saboteur_engagement_compute_kernel.cubin`.
    sp11_saboteur_engagement_compute_kernel: CudaFunction,
    /// SP11 Fix 39 (2026-05-04, A1.3 + B1b fix-up 2026-05-04):
    /// per-component reward-magnitude-ratio canary producer. Single-block,
    /// 6-thread kernel reads NON-CONTIGUOUS sources after the B1b fix-up:
    ///   axis 0 (popart)   = ISV[POPART_COMPONENT_MAG_EMA_INDEX = 360]
    ///   axis 1..5         = ISV[REWARD_POPART_EMA_INDEX + 1 .. + 6)
    ///                     (cf=64, trail=65, micro=66, opp_cost=67, bonus=68)
    /// Normalises to ratios and mirrors the popart-component magnitude
    /// into scratch[6] as the PnL signal scale (= ISV[359] post Pearls A+D).
    /// Writes 7 floats to `scratch[SCRATCH_SP11_MAG_RATIO_BASE..+7)`.
    /// Followed by 2 apply_pearls_ad_kernel launches:
    ///   (n_slots=6) → ISV[REWARD_COMPONENT_MAG_RATIO_BASE..+6)
    ///   (n_slots=1) → ISV[PNL_REWARD_MAGNITUDE_EMA_INDEX=359]
    /// Loaded from `reward_component_mag_ratio_compute_kernel.cubin`.
    sp11_mag_ratio_compute_kernel: CudaFunction,
    /// SP11 Fix 39 (2026-05-04, A1.1): mapped-pinned 2-element val-sharpe
    /// history buffer consumed by `sp11_val_sharpe_delta_compute_kernel`.
    /// Layout: `[prev_epoch_val_sharpe, curr_epoch_val_sharpe]`. The host
    /// writes the current epoch's val sharpe to slot[1] each val emit
    /// boundary in `training_loop.rs` (rotating slot[1] → slot[0] before
    /// the new write). Buffer is constructor-zero-initialised, which
    /// Pearl A's sentinel-bootstrap path consumes naturally on the first
    /// observation. The host write is a permitted CPU↔GPU path: it goes
    /// through mapped-pinned memory (no `htod_copy`), and writes a literal
    /// already-computed value rather than performing host-side compute,
    /// so it does NOT violate `feedback_no_htod_htoh_only_mapped_pinned`
    /// or `feedback_no_cpu_compute_strict`.
    pub(crate) val_sharpe_history_pinned: super::mapped_pinned::MappedF32Buffer,
    /// SP11 Fix 39 (2026-05-04, A1.2): per-bar saboteur Δreward GPU buffer
    /// populated by `experience_env_step` at the saboteur perturbation site.
    /// Length = `alloc_episodes × alloc_timesteps`; layout matches
    /// `total_reward_per_sample` (row-major [N, L]). Consumed by
    /// `saboteur_engagement_compute_kernel`. Pure GPU buffer (CudaSlice<f32>)
    /// — never read host-side, so the no-CudaSlice rule does not apply
    /// (it's a GPU-only path between two kernels). Owned by
    /// `gpu_experience_collector`; the trainer holds the device pointer
    /// (populated post-experience-collection) and the bar count.
    /// Per `feedback_wire_everything_up`: this field is initialised in
    /// the same commit that adds it; the consumer in `training_loop.rs`
    /// reads `self.gpu_experience_collector.saboteur_delta_reward_dev_ptr()`
    /// just before each per-step launch.
    pub(crate) sp11_saboteur_delta_reward_dev_ptr: u64,
    /// Length of `sp11_saboteur_delta_reward_dev_ptr`'s underlying buffer
    /// (alloc_episodes × alloc_timesteps). Cached on the trainer because
    /// the engagement kernel takes it as an i32 arg.
    pub(crate) sp11_saboteur_delta_reward_len: usize,
    /// SP11 Fix 39 B1b fix-up (2026-05-04): per-bar popart-component-
    /// specific reward magnitude buffer. Mapped-pinned f32, sized
    /// `MAX_BATCH_SIZE` for trainer-side use; the experience collector
    /// owns the main per-bar buffer (alloc_episodes × alloc_timesteps).
    /// The trainer field exists for the rare case where SP11 needs to
    /// stage popart-component values inside the per-step path; the
    /// canonical write site is `experience_env_step` operating on the
    /// collector's buffer (kernel-to-kernel dataflow). Spec §4 amendment
    /// "Why slot 360".
    ///
    /// Per `feedback_no_htod_htoh_only_mapped_pinned.md`: mapped-pinned
    /// (`MappedF32Buffer`), not `CudaSlice<f32>`. Initial value zero
    /// (constructor); Pearl A's sentinel-bootstrap consumes the zero on
    /// the first observation.
    pub(crate) popart_component_per_sample: super::mapped_pinned::MappedF32Buffer,
    /// SP11 Fix 39 B1b fix-up (2026-05-04): popart-component-specific
    /// magnitude EMA producer kernel. Reads
    /// `popart_component_per_sample_dev_ptr` (the collector's per-bar
    /// buffer) and writes mean(|x|) to
    /// `scratch[SCRATCH_SP11_POPART_COMPONENT_EMA]`. Followed by 1
    /// chained `apply_pearls_ad_kernel` launch (n_slots=1) →
    /// ISV[POPART_COMPONENT_MAG_EMA_INDEX=360]. Loaded from
    /// `popart_component_ema_kernel.cubin`. Resolves the slot-63 PopArt-
    /// input vs popart-component-magnitude overload that B1b smoke
    /// surfaced. Spec §4 amendment "Why slot 360".
    sp11_popart_component_ema_kernel: CudaFunction,
    /// SP11 Fix 39 B1b fix-up (2026-05-04): device pointer + bar count
    /// for the experience collector's `popart_component_per_sample`
    /// buffer. Wired in the same commit by
    /// `set_sp11_popart_component_buf` mirroring the saboteur Δreward
    /// pattern at `set_sp11_saboteur_delta_reward_buf`. Initial 0 / 0
    /// (kernel guarded — `launch_sp11_popart_component_ema` no-ops when
    /// unwired).
    pub(crate) sp11_popart_component_dev_ptr: u64,
    pub(crate) sp11_popart_component_len: usize,
    /// SP13 Phase 0a (2026-05-04): aux-head directional-accuracy
    /// reducer kernel. Single-block tree-reduce loaded from
    /// `aux_dir_acc_reduce_kernel.cubin`. SP13 B1.1a (2026-05-05) ABI
    /// flip: reads the aux next-bar K=2 softmax tile + i32 labels
    /// (replacing the regression-mode `(aux_pred f32, label_sign f32)`
    /// pair) and writes
    /// `(dir_acc, pos_pred_frac, pos_label_frac, n_down, n_up, n_skip)`
    /// to `aux_dir_acc_buf` (mapped-pinned, 6 floats — was 3). Producer
    /// is wired by P0a.T4; the kernel handle here lands the constructor
    /// edge per `feedback_no_partial_refactor.md` (P0a is one atomic
    /// commit). See the kernel source for the argmax prediction
    /// convention (`softmax[1] > softmax[0] ? 1 : 0`) and the
    /// all-mask-label sentinel (0.5, matching `DIR_ACC_EMA_SENTINEL`).
    aux_dir_acc_reduce: CudaFunction,
    /// SP13 Phase 0a (2026-05-04): mapped-pinned 6-element output
    /// buffer for `aux_dir_acc_reduce` (SP13 B1.1a grew 3 → 6 to
    /// expose per-class counts for HEALTH_DIAG). Layout
    /// `[dir_acc, pos_pred_frac, pos_label_frac, n_down, n_up, n_skip]`
    /// per the kernel contract. Per `feedback_no_htod_htoh_only_mapped_pinned.md`:
    /// every CPU↔GPU scalar read-back goes through `MappedF32Buffer`
    /// (host writes the launcher reads via `read_all()`'s volatile
    /// read after stream sync). Constructor-zero-initialised; the
    /// kernel overwrites all six slots on every launch (or first three
    /// to the sentinel 0.5 when the batch is empty / all-mask-label).
    pub(crate) aux_dir_acc_buf: super::mapped_pinned::MappedF32Buffer,
    /// SP13 Phase 0a P0a.T4 (2026-05-04): fixed-α EMA applicator
    /// kernel handle. Loaded from `apply_fixed_alpha_ema_kernel.cubin`.
    /// Consumed by `launch_apply_fixed_alpha_ema` for the SP13 dir-acc
    /// EMAs (slots 373/374 with α=0.3 / 0.05 and sentinel 0.5) and the
    /// hold-rate EMA (slot 382 with α=0.05 and sentinel 0.0). Sibling
    /// of `apply_pearls_ad_kernel`; see field declaration there for the
    /// Wiener-optimal counterpart. Fixed-α form needed because Pearls
    /// A+D collapse two EMAs of the same signal to identical values,
    /// defeating the stagnation detector that compares slot 373 vs 374.
    apply_fixed_alpha_ema_kernel: CudaFunction,
    /// SP13 Phase 0a P0a.T4 (2026-05-04): aux-head per-bar prediction
    /// → ISV[AUX_DIR_PREDICTION_INDEX=375] bounded scalar producer
    /// kernel handle. Loaded from `aux_pred_to_isv_tanh_kernel.cubin`.
    /// SP13 B1.1a (2026-05-05) rewrite: reads `aux_nb_softmax_buf [B, K=2]`
    /// and writes `mean(softmax[:, 1] - softmax[:, 0])` ∈ [-1, +1] to
    /// ISV[375]. Per-step state (not an EMA — slot is overwritten each
    /// launch); no FoldReset registry entry. Bound is structural (no
    /// runtime tanh). Consumed by `launch_aux_pred_to_isv_tanh`.
    aux_pred_to_isv_tanh_kernel: CudaFunction,
    // ── SP22 H6 Phase 3 α (2026-05-13): atom-position shift Adam-trained W ──
    /// SP22 H6 Phase 3 α — learnable weight vector [b0_size=4] f32.
    /// Adam-trained. Read by both the trainer (training-time atom-shift
    /// threading through compute_expected_q + c51_loss_kernel +
    /// c51_grad_kernel + mag_concat_qdir) and the collector (rollout-time
    /// atom-shift in compute_expected_q for action selection, via shared
    /// device pointer). Atom-shift formula:
    ///     `aux_atom_shift[b, a] = W[a] * state_121[b]`
    /// applied per atom of direction-branch action a as
    ///     `z_n_effective[b, a, n] = z_n_base + aux_atom_shift[b, a]`.
    /// **Initialization: structural prior** `[-0.5, 0.0, +0.5, 0.0]` for
    /// `[Short, Hold, Long, Flat]` — domain belief that aux conviction ×
    /// position-sign biases toward aligned trades. Adam refines from
    /// the prior via projection backward in c51_grad_kernel.
    pub(crate) w_aux_to_q_dir: cudarc::driver::CudaSlice<f32>,
    /// SP22 H6 Phase 3 α — Adam first moment (m) for `w_aux_to_q_dir`.
    /// Same shape [b0_size=4] f32; zero-init per Adam convention.
    adam_m_w_aux: cudarc::driver::CudaSlice<f32>,
    /// SP22 H6 Phase 3 α — Adam second moment (v) for `w_aux_to_q_dir`.
    adam_v_w_aux: cudarc::driver::CudaSlice<f32>,
    /// SP22 H6 Phase 3 α — gradient accumulator for `w_aux_to_q_dir`.
    /// Written by `c51_grad_kernel`'s projection backward (dW[a] =
    /// Σ_b state_121[b] * dz_shift[b, a] where dz_shift comes from
    /// dloss/dz_n_effective); read by the Adam-step update.
    /// [b0_size=4] f32.
    dw_aux_buf: cudarc::driver::CudaSlice<f32>,
    /// SP22 H6 Phase 3 α Step 8 — per-sample sampled target action for d == 0.
    /// Written by c51_loss_kernel forward at the Expected SARSA sampling
    /// site (`best_next_a` for dir branch). Read by `c51_aux_dw_kernel`
    /// to look up W[a*] for the target-side dW contribution.
    /// Shape `[batch_size]` i32. alloc_zeros default = 0 (Short action;
    /// harmless when aux_shift inactive since SP_b stays at 0 too).
    aux_target_a_dir_buf: cudarc::driver::CudaSlice<i32>,
    /// SP22 H6 Phase 3 α Step 8 — per-sample projection log-diff sum
    /// for d == 0. Written by c51_loss_kernel forward right after the
    /// Bellman projection completes:
    ///   `SP_b = Σ_n p_target_n × (current_lp[upper_n] - current_lp[lower_n])`
    /// Read by `c51_aux_dw_kernel` to compute
    ///   `dL/dΔ_online = SP_b / Δz`,  `dL/dΔ_target = -γ*(1-done)*dL/dΔ_online`
    /// Shape `[batch_size]` f32. alloc_zeros default = 0 → no gradient
    /// when aux_shift inactive.
    aux_proj_logdiff_dir_buf: cudarc::driver::CudaSlice<f32>,
    /// SP22 H6 Phase 3 α Step 8 — kernel handle for `c51_aux_dw_kernel`.
    c51_aux_dw_kernel: cudarc::driver::CudaFunction,
    /// SP22 H6 Phase 3 α Step 11 — kernel handle for `adam_w_aux_update`.
    adam_w_aux_kernel: cudarc::driver::CudaFunction,
    // ── SP14 Earned Gradient Flow kernels ─────────────────────────────────
    /// SP14 B.3 (2026-05-05): per-step Q-head ↔ aux argmax disagreement EMA
    /// producer. Reads online Q logits + aux softmax outputs; computes the
    /// per-step argmax-disagreement bool; updates a Wiener-optimal EMA into
    /// ISV[388]. Loaded from `q_disagreement_update_kernel.cubin`.
    sp14_q_disagreement_update_kernel: CudaFunction,
    // ── SP17 Task PP.3 (2026-05-08): pre-centering V/A mean diagnostic ────
    /// Single-block tree-reduce kernel emitting `[E[V], E[A_dir], E[A_mag],
    /// E[A_ord], E[A_urg]]` from the existing `on_v_logits_buf` +
    /// `on_b_logits_buf`. Cold-path (epoch boundary) producer for the
    /// HEALTH_DIAG `v_a_means` line — the diagnostic baseline that the
    /// SP17 Phase 0 kill criterion reads. Loaded from
    /// `v_a_means_diag_kernel.cubin`. Consumed by
    /// `read_v_a_means_pre_centering()`.
    sp17_v_a_means_diag_kernel: CudaFunction,
    /// 5-element mapped-pinned readback for the V/A means diagnostic.
    /// Layout: `[E[V], E[A_dir], E[A_mag], E[A_ord], E[A_urg]]`. The
    /// kernel writes via `dev_ptr` with `__threadfence_system()`; the
    /// host reads via `read_all()` after a stream sync. Per
    /// `feedback_no_htod_htoh_only_mapped_pinned`.
    sp17_v_a_means_diag_buf: super::mapped_pinned::MappedF32Buffer,
    // ── SP17 Phase 3.1 (2026-05-08): per-branch A_var EMA producer ────────
    /// 4-block × 256-thread kernel computing per-branch
    /// `Var(A_centered)` over (B × n_branch × NA) and writing the EMA
    /// into ISV[A_VAR_EMA_DIR/MAG/ORD/URG_INDEX] (slots 474..478).
    /// Pearl-A first-observation bootstrap on sentinel 0.0; α=0.4
    /// post-bootstrap per `pearl_wiener_alpha_floor_for_nonstationary`.
    /// Loaded from `sp17_a_var_ema_kernel.cubin`. Consumed by
    /// `read_a_var_ema_per_branch()`.
    sp17_a_var_ema_kernel: CudaFunction,
    // ── SP17 Phase 3.2 (2026-05-08): per-branch V_share producer ──────────
    /// 4-block × 256-thread kernel computing per-branch
    /// `|E[V]| / (|E[V]| + |E[A_centered, picked]|)` (picked = argmax E[Q])
    /// and EMA-blending into ISV[V_SHARE_DIR/MAG/ORD/URG_INDEX] (slots
    /// 478..482). Pearl-A bootstrap on sentinel 0.5; α=0.4 post-bootstrap.
    /// Block tree-reduce per `feedback_no_atomicadd`. Loaded from
    /// `sp17_v_share_kernel.cubin`. Consumed by
    /// `read_v_share_per_branch()`.
    sp17_v_share_kernel: CudaFunction,
    // ── SP17 Phase 3.2 (2026-05-08): adaptive advantage_clip_bound ────────
    /// Single-block × 256-thread kernel using `sp4_histogram_p99` to
    /// compute `p99(|A_centered|) × 1.5` over the full flat batch and
    /// EMA-blend (α=0.01) with bilateral clamp [0.1, 100.0] into
    /// ISV[ADVANTAGE_CLIP_BOUND_INDEX=482]. Pearl-A bootstrap on sentinel
    /// 1.0. Observability-only in Phase 3 — clip wire-up is Phase 5.
    /// Loaded from `sp17_advantage_clip_bound_kernel.cubin`. Consumed by
    /// `read_advantage_clip_bound()`.
    sp17_advantage_clip_bound_kernel: CudaFunction,
    /// SP17 Phase 3.2 scratch buffer for `sp17_advantage_clip_bound`. Sized
    /// to the maximum possible flat |A_centered| tensor: B × Σ_d b_d × NA.
    /// Mapped-pinned per `feedback_no_htod_htoh_only_mapped_pinned` (no
    /// host writes — kernel-owned). Lifetime matches the trainer; reset
    /// once at construct, written-then-read inside the kernel each launch.
    sp17_advantage_clip_scratch: super::mapped_pinned::MappedF32Buffer,

    // ── SP18 v2 Phase 0 Task 0.2 (2026-05-08): B-leg TD-error magnitude
    //    EMA producer kernel handle ─────────────────────────────────────
    /// Kernel handle for `td_error_mag_ema_update`. Reads
    /// `td_errors_buf [B]` (post-train-step, populated by `c51_loss` /
    /// `mse_loss` per backward configuration) and EMA-blends
    /// `mean(|td_errors[b]|)` into `ISV[TD_ERROR_MAG_EMA_INDEX=493]`
    /// via Pearl-A first-observation bootstrap + fixed α=0.4. Loaded
    /// from `td_error_mag_ema_kernel.cubin`. Consumed by
    /// `launch_sp18_td_error_mag_ema_update()` (cold-path post-train-
    /// step launch from `training_loop.rs`). Pure observability —
    /// pre-fix baseline for the B-DD9 ratio gate. Plan: SP18 Phase 0
    /// Task 0.2.
    sp18_td_error_mag_ema_kernel: CudaFunction,

    // ── SP18 v2 Phase 0 Task 0.1 (2026-05-08): D-leg per-action reward
    //    decomposition diagnostic buffer ──────────────────────────────────
    /// 20-float mapped-pinned diagnostic buffer for the SP18 D-leg
    /// per-action reward decomposition. Layout (row-major, 4×5):
    ///
    ///   row 0 = Hold,   row 1 = Long,   row 2 = Short,   row 3 = Flat
    ///   col 0 = mean(r_micro),     col 1 = mean(r_opp_cost),
    ///   col 2 = mean(r_popart),    col 3 = mean(|reward|),
    ///   col 4 = fire_rate
    ///
    /// Written by the collector's
    /// `launch_sp18_reward_decomp_diag(...)` kernel each per-step
    /// boundary, BEFORE `launch_reward_component_ema_inplace` zeros the
    /// source buffer. Consumed exclusively by the per-epoch HEALTH_DIAG
    /// emit at `training_loop.rs`. Pure observability — no production-
    /// path consumer in this commit. Plan: SP18 Phase 0 Task 0.1.
    pub(crate) sp18_reward_decomp_diag_buf: super::mapped_pinned::MappedF32Buffer,

    // SP14 Layer C Phase C.1 (2026-05-08): α-machinery struct fields deleted
    // atomically with the kernel files (alpha_grad_compute_kernel,
    // gradient_hack_detect_kernel, sp14_scale_wire_col_kernel) per
    // `feedback_no_legacy_aliases` and `feedback_no_partial_refactor`.
    // Coupling A's `sp14_dir_concat_qaux_kernel` is preserved below.
    /// SP14 B.6 (2026-05-05) + Phase C.1 (2026-05-08): pre-SGEMM concat
    /// kernel for direction Q-head input. Reads `h_s2 [B, SH2]` +
    /// `aux_softmax [B, K=2]` and writes `[h_s2 | softmax_diff]
    /// [B, SH2 + 1]` into `sp14_dir_qaux_concat_scratch`. Coupling A
    /// (forward feature wire) — preserved through SP14 Layer C: aux's
    /// directional softmax-diff continues to feed Q-head's x_concat as
    /// a forward feature. Loaded from `dir_concat_qaux_kernel.cubin`.
    sp14_dir_concat_qaux_kernel: CudaFunction,
    /// SP14 B.7 (2026-05-05) + Phase C.1 (2026-05-08): scratch buffer
    /// for direction Q-head input concat. Shape `[B, SH2 + 1]`. Written
    /// by `sp14_dir_concat_qaux_kernel` (Coupling A forward feature wire
    /// — preserved through Phase C.1) before the direction Q-head's
    /// first FC SGEMM consumes it. Column SH2 holds `aux_softmax_diff`
    /// (up_prob − down_prob) ∈ [-1, +1].
    sp14_dir_qaux_concat_scratch: CudaSlice<f32>,
    /// SP14 B.10 (2026-05-05) + Phase C.1 (2026-05-08): backward dX
    /// scratch for direction Q-head's first FC. Shape `[B, SH2 + 1]`.
    /// Written by `backward_branch_dx` (`d == 0` case) as the SGEMM
    /// `dY @ W^T` output. The first SH2 columns accumulate into
    /// `bw_d_h_s2` via `strided_accumulate` (mirroring the
    /// `d_mag_concat_buf` precedent for the magnitude branch's wider
    /// input); column SH2 is dropped (no propagation downstream).
    /// Phase C.1 deleted the `sp14_scale_wire_col_kernel` α-gate that
    /// previously scaled column SH2 — Coupling B is gone.
    sp14_d_dir_qaux_concat: CudaSlice<f32>,
    // SP14 Layer C Phase C.1 (2026-05-08): `sp14_scale_wire_col_kernel`
    // field deleted atomically with the kernel file. Coupling B (α-gated
    // backward wire) is gone; the wire column gradient flows unscaled
    // through `accumulate_d_h_s2_from_concat`, then SP14 Layer C wires
    // proper aux-trunk backward in Phase C.5. Pre-Phase C.5 the wire
    // column drops via the existing first-SH2-cols accumulator (column
    // SH2 is not propagated downstream, matching the legacy behavior
    // when ISV[393] sentinelled to 0.0).
    /// SP11 Fix 39 (2026-05-04, Task A2): reward-subsystem controller
    /// kernel. Single-block, 10-thread producer reading 5 canary ISV slots
    /// [350..360) and writing 10 floats to scratch[SCRATCH_SP11_CONTROLLER_BASE..+10).
    /// Followed by 1 chained `apply_pearls_ad_kernel` launch (n_slots=10)
    /// → ISV[340..350) (6 component weights + 4 controller scalars).
    /// Loaded from `reward_subsystem_controller_kernel.cubin`.
    sp11_reward_subsystem_controller_kernel: CudaFunction,
    /// SP11 Fix 39 (2026-05-04, Task A2): one-shot GPU init kernel for the
    /// 42×16 SimHash projection matrix. Philox-driven, deterministic from a
    /// host-passed seed derived from `SP11_PROJ_SEED_SALT`. Launched ONCE
    /// at trainer construct (see the constructor's allocation block) and
    /// never re-launched — the projection matrix is frozen for the run
    /// lifetime. Loaded from `novelty_simhash_proj_init_kernel.cubin`.
    sp11_novelty_simhash_proj_init_kernel: CudaFunction,
    /// SP11 Fix 39 (2026-05-04, Task A2): SimHash novelty lookup kernel.
    /// Reads (state, action) bucket count from the 1M-slot hash table and
    /// returns `1/sqrt(1+count)` ∈ [0, 1] per replay sample. Loaded from
    /// `novelty_simhash_kernel.cubin`.
    ///
    /// **A2 registers the kernel handle; B1 inlines the launch at the
    /// actual replay-time consumer site** per `feedback_no_partial_refactor.md`
    /// (atomic consumer migration). No A2-side launcher wrapper exists yet
    /// — B1 builds either an inline `stream.launch_builder(...)` or a
    /// purpose-fit wrapper at the same commit as the call site, matching
    /// the A0 deferral pattern for the novelty-hash registry entry.
    sp11_novelty_simhash_lookup_kernel: CudaFunction,
    /// SP11 Fix 39 (2026-05-04, Task A2): SimHash novelty update kernel.
    /// Race-tolerated non-atomic bucket increment per
    /// `feedback_no_atomicadd.md`; under-counts bias novelty UP (safe).
    /// **A2 registers the kernel handle; B1 inlines the launch at the
    /// replay-time consumer site** (same atomic-migration pattern as the
    /// lookup kernel above). Loaded from `novelty_simhash_kernel.cubin`
    /// (same cubin as the lookup kernel — both share the SimHash
    /// code-derivation device-inline).
    sp11_novelty_simhash_update_kernel: CudaFunction,
    /// SP11 Fix 39 (2026-05-04, Task A2): novelty visit-count hash table.
    /// 1M slots × f32 = 4 MB mapped-pinned. Bucket index =
    /// `(action_id × 65536 + simhash_code) & TABLE_MASK`. Reset at fold
    /// boundary via the `sp11_novelty_hash` registry entry's dispatch arm.
    /// The buffer is mapped-pinned even though the lookup/update kernels
    /// are pure GPU paths — the host-side reset is a `host_slice_mut().fill(0.0)`
    /// (CPU writing a literal value, NOT compute) per
    /// `feedback_no_htod_htoh_only_mapped_pinned.md`'s allowed-write rule.
    pub(crate) novelty_hash_buf: super::mapped_pinned::MappedF32Buffer,
    /// SP11 Fix 39 (2026-05-04, Task A2): SimHash projection matrix.
    /// 42 × 16 = 672 f32 mapped-pinned. Populated ONCE at trainer
    /// construct by `launch_novelty_simhash_proj_init` (Philox-driven from
    /// `SP11_PROJ_SEED_SALT` per `sp11_isv_slots.rs`). Never re-initialised;
    /// never written from host. Per `feedback_no_cpu_forwards.md` ("CPU is
    /// read-only"), the matrix MUST be populated on-device. The matrix
    /// outlives every fold (it's the hash function, not state) — it has
    /// NO entry in the fold-reset registry.
    pub(crate) novelty_simhash_proj: super::mapped_pinned::MappedF32Buffer,
    /// SP7 (2026-05-03): GPU dispatch kernel for the cql/c51 budget consumer.
    /// 8 threads (2 heads × 4 branches), single block. Reads ISV activation
    /// flag + controller budget per (head, branch); resolves
    /// `(active >= 0.5) ? max(ctrl, EPS_DIV) : bootstrap` on-device; writes
    /// trunk_mean + per-branch correction to `lb_budget_effective_buf` (10
    /// f32 mapped-pinned). Replaces the host-side if/else inside
    /// `compute_adaptive_budgets` that froze at CUDA-Graph capture time.
    /// Loaded from `consume_lb_budget_kernel.cubin`.
    lb_budget_dispatch_kernel: CudaFunction,
    /// SP7 (2026-05-03): SAXPY variant reading `alpha` from a device pointer.
    /// `y[i] += (*alpha_ptr) * x[i]`. Used by `apply_cql_saxpy` and per-branch
    /// variant when consuming `lb_budget_effective_buf` slots. Loaded from
    /// `consume_lb_budget_kernel.cubin` (separate CUmodule from
    /// `dqn_saxpy_f32_kernel` so existing scalar callers remain unchanged).
    saxpy_f32_dev_ptr_aux: CudaFunction,
    /// SP7 (2026-05-03): in-place scale variant reading `alpha` from a device
    /// pointer. `y[i] *= *alpha_ptr` (NaN-safe at alpha=0 — matches
    /// `dqn_scale_f32_kernel`). Used by `apply_c51_budget_scale` and
    /// per-branch variant when consuming `lb_budget_effective_buf` slots.
    /// Loaded from `consume_lb_budget_kernel.cubin`.
    scale_f32_dev_ptr_aux: CudaFunction,
    /// SP7 (2026-05-03): mapped-pinned scratch holding the GPU-resolved
    /// effective per-branch budgets for cql/c51. Layout (10 f32):
    ///   [0]  cql_trunk_mean                (consumed by `apply_c51_budget_scale`'s
    ///                                       sibling apply_cql_saxpy(trunk))
    ///   [1..5]  cql_correction[branch]    (consumed by `apply_cql_saxpy_branch`)
    ///   [5]  c51_trunk_mean               (consumed by `apply_c51_budget_scale`)
    ///   [6..10] c51_correction[branch]    (consumed by `apply_c51_budget_scale_branch`)
    /// Populated each step by `lb_budget_dispatch_kernel`; consumed by the
    /// dev-ptr SAXPY/scale kernels in the same captured `aux_child` graph.
    /// Mapped-pinned so HEALTH_DIAG can read host-side after stream sync —
    /// no DtoH copy.
    pub(crate) lb_budget_effective_buf: super::mapped_pinned::MappedF32Buffer,
    // ── SP5 Task A4: Pearl 4 per-group Adam β1/β2/ε kernels ─────────────
    /// SP5 Task A4 (2026-05-01): auxiliary gradient cosine similarity kernel.
    /// Single-block 8-thread kernel (one thread per SP4 param group).
    /// Reads `grad_buf [total_params]` + `grad_prev_buf_per_group [total_params]`
    /// + `group_param_offsets_buf [9]` (prefix sums in element units); computes
    /// per-group cos_sim = dot(g,g_prev)/(|g|×|g_prev|+EPS_COS) and l2_norm = |g|;
    /// writes to `producer_step_scratch_buf[131..147)`. Also writebacks g_curr →
    /// grad_prev_buf_per_group for the next step.
    /// No atomicAdd (feedback_no_atomicadd), no CPU compute (feedback_no_cpu_compute_strict).
    /// Loaded from `grad_cosine_sim_kernel.cubin`.
    grad_cosine_sim_kernel: CudaFunction,
    /// SP5 Task A4 (2026-05-01): Pearl 4 per-group Adam β1/β2/ε controller kernel.
    /// Single-block 8-thread kernel (one thread per param group). Reads per-group
    /// cosine_sim + l2_norm from scratch_buf[131..147); computes
    /// β1=0.85+0.10×stability ∈ [0.85,0.95],
    /// β2=0.99+0.0095×stability ∈ [0.99,0.9995],
    /// ε=clamp(grad_norm×1e-7, 1e-10, 1e-6); writes to scratch_buf[147..171).
    /// Chained apply_pearls_ad_kernel (24 single-slot launches, ALPHA_META=5e-4)
    /// smooths via Pearls A+D into ISV[ADAM_BETA1_BASE..ADAM_EPS_BASE+8).
    /// Structural envelopes (Invariant 1 anchors per spec line 88).
    /// No atomicAdd, no CPU compute.
    /// Loaded from `pearl_4_adam_hparams_kernel.cubin`.
    pearl_4_adam_hparams_kernel: CudaFunction,
    /// SP5 Task A5 (2026-05-01): q_skew_kurtosis auxiliary kernel.
    /// Single-block 4-thread kernel (one per branch). Reads save_q_online [B × 13]
    /// row-major; computes per-branch (skew, excess kurtosis) via two-pass central
    /// moments; writes 8 floats to scratch_buf[171..179) (skew[4] + ex_kurt[4]).
    /// Loaded from `q_skew_kurtosis_kernel.cubin`.
    q_skew_kurtosis_kernel: CudaFunction,
    /// SP5 Task A5 (2026-05-01): Pearl 5 per-branch IQN τ schedule controller kernel.
    /// Single-block 4-thread kernel (one per branch). Reads per-branch skew from
    /// scratch_buf[skew_idx_base..skew_idx_base+4); shifts symmetric default τ schedule
    /// {0.05, 0.25, 0.5, 0.75, 0.95} by skew × SKEW_SHIFT=0.05 (Invariant 1 anchor);
    /// clamps to [0.01, 0.99]; writes 20 floats to scratch_buf[179..199).
    /// Chained apply_pearls_ad_kernel (20 single-slot launches, ALPHA_META=1e-3) smooths
    /// into ISV[IQN_TAU_BASE=250..270). No atomicAdd, no consumer migration in this commit.
    /// Loaded from `pearl_5_iqn_tau_kernel.cubin`.
    pearl_5_iqn_tau_kernel: CudaFunction,
    /// SP5 Task A6 (2026-05-01): Pearl 6 cross-fold-persistent Kelly cap signals kernel.
    /// Single-block 6-thread kernel (one per ISV output slot [280..286)).
    /// Reads portfolio_state [n_envs * PS_STRIDE]; writes directly to ISV[280..286).
    /// CROSS-FOLD-PERSISTENT: these slots are EXEMPT from the fold-reset registry.
    /// In-kernel slow EWMA (alpha=0.01, Invariant 1 anchor); cumulative-via-max()
    /// for sample_count slot (s==3). No apply_pearls call; no atomicAdd.
    /// Loaded from `pearl_6_kelly_kernel.cubin`.
    pearl_6_kelly_kernel: CudaFunction,
    /// SP5 Task A7 (2026-05-01): Pearl 8 per-direction trail-stop distance kernel.
    /// 4-thread single-block (one per direction: Short=0, Hold=1, Long=2, Flat=3).
    /// Reads features[bar_idx * market_dim + 9] (ATR_NORM); denormalizes via
    /// canonical fxcache scheme; Short/Long get 2× ATR, Hold/Flat get floor 1.0.
    /// Writes 4 floats to scratch_buf[SCRATCH_PEARL_8_TRAIL=199..203).
    /// Chained apply_pearls_ad_kernel (4 launches, ALPHA_META=1e-3) →
    /// ISV[TRAIL_DIST_PER_DIR_BASE=270..274). In fold-reset registry (FoldReset).
    /// Loaded from `pearl_8_trail_kernel.cubin`.
    pearl_8_trail_kernel: CudaFunction,
    /// SP5 Task A8 (2026-05-01): Pearl 1-ext per-branch C51 num_atoms kernel.
    /// 4-thread single-block kernel. Reads ISV[ATOM_V_HALF_BASE=178..182) (Pearl 1, Task A1);
    /// maps to discrete {64, 32, 16} via threshold cascade (thresholds 0.1, 1.0 are Invariant 1
    /// anchors). Writes 4 floats to scratch_buf[SCRATCH_PEARL_1_EXT_NUM_ATOMS=203..207).
    /// Chained apply_pearls_ad_kernel (4 launches, ALPHA_META=1e-3) →
    /// ISV[ATOM_NUM_ATOMS_BASE=274..278). In fold-reset registry (FoldReset).
    /// Loaded from `pearl_1_ext_num_atoms_kernel.cubin`.
    pearl_1_ext_num_atoms_kernel: CudaFunction,
    /// SP5 Layer D Task D1 (rewrite, 2026-05-02): PnL aggregation kernel.
    /// Single-block 256-thread shmem-reduce kernel. Reads per-bar arrays
    /// (`step_returns`, `done_flags`) plus already-reduced per-trade summary
    /// scalars (`n_trades`, `sum_returns`, `sum_sq_returns`,
    /// `initial_capital`) and writes 4 aggregated floats — log-space compounded
    /// total / per-trade mean / per-trade variance / per-bar max drawdown with
    /// episode-boundary resets — to scratch_buf[SCRATCH_PNL_AGG_BASE=207..211).
    /// Chained apply_pearls_ad_kernel (4 launches, ALPHA_META=1e-3) →
    /// ISV[PNL_TOTAL_INDEX=286..PNL_MAX_DD_INDEX+1=290). In fold-reset registry
    /// (FoldReset; Layer D D1 outputs are NOT cross-fold persistent — unlike
    /// Pearl 6 Kelly stats — so they take the standard Pearl A sentinel path).
    /// Loaded from `pnl_aggregation_kernel.cubin`. Producer-only in this commit;
    /// D4 wires the consumer atomically. Reproduces `compute_epoch_financials`
    /// (financials.rs) bit-for-bit; supersedes the original D1 kernel (commit
    /// `5ee795f14`) which had been authored against per-trade arrays that
    /// don't exist in production.
    pnl_aggregation_kernel: CudaFunction,
    /// SP5 Layer D Task D2 (2026-05-02): LearningHealth composition producer.
    /// Single-block 1-thread arithmetic kernel takes 7 raw inputs by value
    /// (q_gap, q_var, atom_util, grad_norm, ens_disagreement,
    /// grad_consistency, spectral_gap), normalises each via smoothstep curves
    /// matching `learning_health.rs::NormalizedComponents::from_raw` exactly,
    /// composes them via the spec Layer-1 weighted sum, and writes 4 scratch
    /// floats — composed score + 3 normalised intermediates (q_gap_norm /
    /// q_var_norm / grad_norm_norm) — to scratch_buf[SCRATCH_HEALTH_COMP_BASE
    /// =211..215). Chained apply_pearls_ad_kernel (4 launches, ALPHA_META=1e-3)
    /// → ISV[HEALTH_SCORE_INDEX=290..GRAD_NORM_NORM_INDEX+1=294). In fold-reset
    /// registry (FoldReset; the underlying EMAs reset at fold boundary so the
    /// composition outputs reset together via the standard Pearl A sentinel
    /// path). Loaded from `health_composition_kernel.cubin`. Producer-only in
    /// this commit; D4 wires the consumer atomically.
    health_composition_kernel: CudaFunction,
    /// SP5 Layer D Task D3 (2026-05-02): training-metrics EMA producer.
    /// Single-block 3-thread arithmetic kernel (one thread per metric):
    /// tid=0 training_sharpe_ema (adaptive α `clamp(0.05+0.3·|err|, 0, 0.5)`,
    /// host-tracked `_initialized` sentinel), tid=1 max_dd_ema (fixed α=0.1,
    /// `prev<1e-12` sentinel), tid=2 low_dd_ratio (fixed α=0.15 over the
    /// binary `is_low` indicator derived from tid=1's *new* max_dd_ema via
    /// __syncthreads(); no host-side sentinel — constructor zero is
    /// recurrence-safe). Writes 3 scratch floats — training_sharpe_ema +
    /// max_dd_ema + low_dd_ratio — to scratch_buf[
    /// SCRATCH_TRAINING_METRICS_EMA_BASE=215..218). Chained
    /// apply_pearls_ad_kernel (3 launches, ALPHA_META=1e-3) →
    /// ISV[TRAINING_SHARPE_EMA_INDEX=294..LOW_DD_RATIO_INDEX+1=297). In
    /// fold-reset registry (FoldReset; the underlying host scalars
    /// `training_sharpe_ema` / `_initialized` / `max_dd_ema` /
    /// `low_dd_ratio` reset at fold boundary alongside the ISV slots via
    /// the standard Pearl A sentinel path). Loaded from
    /// `training_metrics_ema_kernel.cubin`. Producer-only in this commit;
    /// D4 wires the consumer atomically.
    training_metrics_ema_kernel: CudaFunction,
    /// SP5 Task A4 (2026-05-01): previous-step gradient buffer for cosine similarity.
    /// [total_params f32] mapped-pinned (feedback_no_htod_htoh_only_mapped_pinned).
    /// Zeroed at construction and at each fold boundary (Pearl A sentinel: cosine_sim=0
    /// on first step → β1/β2 start at envelope midpoints).
    pub(crate) grad_prev_buf_per_group: MappedF32Buffer,
    /// SP5 Task A4 (2026-05-01): param-group boundary prefix sums for cosine-sim kernel.
    /// [9 i32] with group g in element range [offsets[g], offsets[g+1]).
    /// Values are in f32-element units (not bytes): divide padded byte offsets by 4.
    /// Host-written once at construction; static for the lifetime of the trainer.
    pub(crate) group_param_offsets_buf: MappedI32Buffer,

    /// SP5 Task A1 (2026-05-01): per-branch clip-count buffer for Pearl 1
    /// headroom controller. [4 i32], one element per branch. Zero-initialized
    /// at construction; populated by `atoms_update_kernel` in Layer B.
    /// Until Layer B is wired, clip_rate=0 → headroom stays at bootstrap 6.0.
    /// Mapped-pinned i32 (feedback_no_htod_htoh_only_mapped_pinned).
    pub(crate) atoms_clip_count_buf: MappedI32Buffer,
    /// SP5 Task A1 (2026-05-01): static action-count layout descriptor.
    /// [4 i32] = {4, 3, 3, 3} (branches: Direction/Magnitude/Order/Urgency).
    /// Host-written once at construction; passed as `const int*` to
    /// q_branch_stats_kernel. Mapped-pinned i32.
    pub(crate) action_counts_buf: MappedI32Buffer,
    /// SP5 Task A1 (2026-05-01): static action-offset layout descriptor.
    /// [4 i32] = {0, 4, 7, 10} (cumulative offsets into [B, 13] q_out_buf).
    /// Host-written once at construction; passed as `const int*` to
    /// q_branch_stats_kernel. Mapped-pinned i32.
    pub(crate) action_offsets_buf: MappedI32Buffer,

    // ── Plan C Phase 2 follow-up A.2: q-drift rate ISV producer ───────
    /// Single-thread single-block cold-path kernel. Reads ISV[Q_ABS_REF=16] +
    /// ISV[Q_DIR_ABS_REF=21] plus host-passed `q_mean_curr` / `q_mean_prev`
    /// scalars; writes `ISV[Q_DRIFT_RATE_INDEX=129] = clip(|Δq|/denom, 0, 4)`
    /// at every epoch boundary. Consumer: `tau_update_kernel.cu` multiplies
    /// the cosine-scheduled `tau_base` by `1/(1+ISV[129])` so tau ranges
    /// `[tau_base/5, tau_base]` — monotone dampening under drift; healthy
    /// runs unaffected. Loaded from `q_drift_rate_ema_kernel.cubin`.
    q_drift_rate_ema_kernel: CudaFunction,

    // ── Plan C Phase 2 follow-up K: adaptive fold-boundary warmup ─────
    /// Single-thread single-block cold-path producer kernel. Reads the
    /// fast/slow grad-norm EMA mapped-pinned buffers and writes
    /// `ISV[FOLD_WARMUP_FACTOR_INDEX=130] = clamp(fast/slow, 0, 1)`.
    /// Consumers: `training_loop.rs` reads ISV[130] each step to derive
    /// `lr_eff = lr_base × max(0.05, factor)` (passed via `set_lr`) and
    /// `clip_eff = clip_base × (0.1 + 0.9 × factor)` (passed via the
    /// existing `update_adaptive_clip` pinned slot). Loaded from
    /// `fold_warmup_factor_kernel.cubin`.
    fold_warmup_factor_kernel: CudaFunction,
    /// SP4 Layer C close-out C1 redesigned (2026-05-01): GPU-only fast/slow
    /// grad-norm EMA update kernel. Single-thread, single-block. Replaces
    /// the host-side `(1-α) × prev + α × obs` arithmetic block in
    /// `update_adaptive_clip` per `feedback_no_cpu_compute_strict`. Chained
    /// on the producer's stream after `launch_grad_norm_finalize` so the
    /// freshly-written `grad_norm_buf[0]` is consumed without any host sync;
    /// updates `grad_norm_fast_ema_pinned` + `grad_norm_slow_ema_pinned`
    /// in-place via their mapped-pinned dev_ptrs. Loaded from
    /// `update_grad_norm_emas_kernel.cubin`.
    update_grad_norm_emas_kernel: CudaFunction,
    /// SP4 Layer C close-out C2 (2026-05-01): GPU-only IQN readiness gauge
    /// update kernel. Single-thread, single-block. Replaces the host-side
    /// arithmetic block in `update_iqn_readiness` per
    /// `feedback_no_cpu_compute_strict`. Takes the host-passed `iqn_loss`
    /// scalar (already from `read_total_loss` mapped-pinned readback) and
    /// updates the three coupled mapped-pinned scalars
    /// (`iqn_loss_initial_pinned`, `iqn_loss_ema_pinned`,
    /// `iqn_readiness_pinned`) in-place via their dev_ptrs. Loaded from
    /// `update_iqn_readiness_kernel.cubin`.
    update_iqn_readiness_kernel: CudaFunction,
    /// SP4 Layer C close-out C3 (2026-05-01): GPU-only utilization EMA
    /// update kernel. Single-thread, single-block. Replaces the host-side
    /// arithmetic block in `reduce_current_q_stats` per
    /// `feedback_no_cpu_compute_strict`. Takes the host-passed `atom_util`
    /// scalar (already from `host[6]` mapped-pinned readback) and updates
    /// `utilization_ema_pinned` + `homeostatic_obs_pinned[1]` mirror in
    /// lockstep. Loaded from `update_utilization_ema_kernel.cubin`.
    update_utilization_ema_kernel: CudaFunction,
    /// SP4 Layer C close-out C4 (2026-05-01): GPU-only winsorized adaptive
    /// grad-clip update kernel. Single-thread, single-block. Replaces the
    /// host-side scalar-reduction + EMA chain in `update_adaptive_clip`
    /// per `feedback_no_cpu_compute_strict`. Reads host-passed
    /// `observed_grad_norm` + four mapped-pinned scalars + ISV[168]; writes
    /// `adaptive_clip_pinned`, `grad_norm_ema_pinned`, `outlier_diag_pinned`.
    /// Loaded from `update_adaptive_clip_kernel.cubin`.
    update_adaptive_clip_kernel: CudaFunction,
    /// Mapped-pinned host pointer for the fast (α=0.1, ~10-step horizon)
    /// grad-norm EMA. Updated GPU-side by `update_grad_norm_emas_kernel`
    /// (chained on the producer's stream after `launch_grad_norm_finalize`)
    /// per training step — the same observation source feeds both this fast
    /// EMA and the existing `grad_norm_ema` field used by adaptive clip.
    /// SP4 Layer C close-out C1 redesigned (2026-05-01) migrated the EMA
    /// arithmetic from host-side to GPU per `feedback_no_cpu_compute_strict`.
    /// FoldReset: 0.0 (the new fold's first steps observe the recovery from
    /// zero).
    grad_norm_fast_ema_pinned: *mut f32,
    /// Device-mapped view of `grad_norm_fast_ema_pinned`. Consumed by
    /// `fold_warmup_factor_kernel` (numerator) and updated in-place by
    /// `update_grad_norm_emas_kernel`.
    grad_norm_fast_ema_dev_ptr: u64,
    /// Mapped-pinned host pointer for the slow (α=0.001, ~1000-step horizon)
    /// grad-norm EMA. Tracks the cross-fold steady-state grad scale —
    /// persists across folds so the warmup factor's denominator stays a
    /// meaningful long-window baseline. Updated GPU-side by
    /// `update_grad_norm_emas_kernel` per training step (SP4 Layer C
    /// close-out C1 redesigned, 2026-05-01). Constructor: 0.0 (cold-start;
    /// the `min_steps_for_ratio` gate in the kernel keeps the ratio at 1.0
    /// until enough samples have accumulated).
    grad_norm_slow_ema_pinned: *mut f32,
    /// Device-mapped view of `grad_norm_slow_ema_pinned`. Consumed by
    /// `fold_warmup_factor_kernel` (denominator) and updated in-place by
    /// `update_grad_norm_emas_kernel`.
    grad_norm_slow_ema_dev_ptr: u64,
    /// Number of times `update_adaptive_clip` has fed a finite grad-norm
    /// observation into the EMAs. The `fold_warmup_factor_update` kernel
    /// reads this as its bootstrap gate — until it crosses
    /// `WARMUP_FACTOR_MIN_STEPS`, the factor is forced to 1.0 (no damping).
    /// Survives across folds (the slow EMA also persists, so the gate
    /// only fires the very first time it warms up at training start).
    grad_norm_emas_step_count: u64,

    // ── Plan 4 Task 3 (E.3): IQN multi-quantile diagnostic EMA producer ──
    /// 4-block kernel (one per off-median quantile τ ∈ {0.05, 0.25, 0.75, 0.95})
    /// computing the mean |Q| over (B × TBA) at each fixed-τ position from
    /// `GpuIqnHead::save_q_online [TBA, B*Q]` and EMA-updating
    /// ISV[IQN_Q_P05_EMA_INDEX..IQN_Q_P95_EMA_INDEX] (slots 99..103). The
    /// median (τ=0.50) is intentionally skipped — already exposed via the
    /// existing greedy-Q diagnostic. Cold-path-cadence (one launch per
    /// training step, alongside `h_s2_rms_ema_update`). Producer-only —
    /// diagnostic surface for HEALTH_DIAG / risk monitoring.
    /// Loaded from `iqn_quantile_ema_kernel.cubin`.
    iqn_quantile_ema_kernel: CudaFunction,

    // ── Q-mean drift correction (Component 8) ──────────────────────────
    /// Phase 1 kernel: computes global mean of Q-values into q_mean_scratch (pinned).
    q_mean_reduce_kernel: CudaFunction,
    /// Phase 2 kernel: subtracts mean from all Q-values in-place.
    q_mean_subtract_kernel: CudaFunction,
    /// Scratch scalar [1] for q_mean_reduce output — pinned device-mapped.
    /// GPU writes via dev_ptr (kernel output), CPU reads via host_ptr (EMA update). Zero memcpy.
    q_mean_scratch_pinned: *mut f32,
    q_mean_scratch_dev_ptr: u64,
    /// Q-mean EMA for drift regularization — pinned device-mapped.
    q_mean_ema_pinned: *mut f32,
    q_mean_ema_dev_ptr: u64,

    // ── Cross-branch graph message passing ─────────────────────────────
    // Graph message passing params are fixed at initialization (domain-knowledge edges).
    // No gradient backward — the 4 edges encode structural dependencies (dir→mag, etc).
    // Training these would require a graph-level loss, deferred to future work.
    /// Edge parameters: 4 edges × 15 params (W_gate[6] + W_msg[9]) = 60.
    /// Fixed at near-identity initialization — not trained. The 4 edges encode
    /// structural dependencies (dir→mag, etc.) via domain knowledge; training
    /// would require a graph-level loss (deferred to future work).
    /// Mapped pinned (cuMemHostAlloc DEVICEMAP) — host-init at construction,
    /// kernel reads via dev_ptr. Avoids DtoD-via-pinned per
    /// feedback_no_htod_htoh_only_mapped_pinned.
    graph_params: super::mapped_pinned::MappedF32Buffer,
    /// branch_graph_message_pass kernel handle.
    graph_msg_kernel: CudaFunction,

    // ── Diffusion Q-refinement (2-step denoiser conditioned on Var[Q]) ─
    /// Flat params for 2 denoising steps: 2 × (W1[24,24]+b1[24]+W2[12,24]+b2[12]) = 2 × 900 = 1800.
    /// Mapped pinned (cuMemHostAlloc DEVICEMAP) — host-init at construction,
    /// then trained in place by the aux Adam path (kernel writes via dev_ptr).
    /// Avoids DtoD-via-pinned per feedback_no_htod_htoh_only_mapped_pinned.
    denoise_params: super::mapped_pinned::MappedF32Buffer,
    /// Adam first moment for denoise_params [1800].
    denoise_adam_m: CudaSlice<f32>,
    /// Adam second moment for denoise_params [1800].
    denoise_adam_v: CudaSlice<f32>,
    /// Adam step counter for denoise optimizer.
    denoise_adam_step: i32,
    /// Scratch buffer for ping-pong denoising [B, 12] (q_coord_buf is the other side).
    denoise_buf_a: CudaSlice<f32>,
    /// Unused ping-pong buffer [B, 12]. Retained for potential future K>2 denoising.
    denoise_buf_b: CudaSlice<f32>,
    /// Var[Q] buffer for training path [B, total_actions] — populated by
    /// compute_expected_q with stride `total_actions` (4+3+3+3 = 13 in the
    /// 4-direction factored layout). Denoiser consumers (q_denoise_step,
    /// q_denoise_backward, denoise_build_input) read only the first D=12 slots
    /// per sample but receive `var_stride = total_actions` to keep per-sample
    /// addressing consistent.
    q_var_buf_trainer: CudaSlice<f32>,
    /// q_denoise_step kernel handle.
    q_denoise_kernel: CudaFunction,
    /// q_denoise_backward kernel handle.
    q_denoise_backward_kernel: CudaFunction,
    /// Gradient accumulator for denoise_params [1800]. Zeroed before each backward.
    denoise_grad: CudaSlice<f32>,
    /// Target network Q-values [B, total_actions] for denoiser training signal.
    /// Sized at `total_actions = b0+b1+b2+b3` (= 13 for the 4-direction
    /// factored layout) to match the `compute_expected_q` writer stride.
    /// Denoiser consumers (q_denoise_backward, denoise_loss_grad) read only
    /// the first D=12 slots per sample via the `target_stride` parameter.
    denoise_target_q_buf: CudaSlice<f32>,
    /// Pre-denoise Q-values snapshot [B, 12] — saved before denoiser forward for backward.
    denoise_q_input_buf: CudaSlice<f32>,
    /// Grad norm partials for denoise Adam [1 block].
    denoise_norm_partials: CudaSlice<f32>,
    /// Grad norm output for denoise Adam [1].
    denoise_norm_buf: CudaSlice<f32>,
    /// Adaptive clip buffer for denoise Adam [1].
    denoise_clip_buf: CudaSlice<f32>,
    /// Pinned device-mapped step counter for denoise Adam.
    denoise_t_pinned: *mut i32,
    denoise_t_dev_ptr: u64,

    // ── Denoise cuBLAS backward pipeline ──────────────────────────────
    /// Col-major scratch [H=24, B] — denoiser MLP input for step 0.
    denoise_input0_buf: CudaSlice<f32>,
    /// Col-major scratch [H=24, B] — denoiser MLP input for step 1.
    denoise_input1_buf: CudaSlice<f32>,
    /// Col-major scratch [H=24, B] — pre-activation (with bias) for step 0.
    denoise_pre_h0_buf: CudaSlice<f32>,
    /// Col-major scratch [H=24, B] — pre-activation (with bias) for step 1.
    denoise_pre_h1_buf: CudaSlice<f32>,
    /// Col-major scratch [H=24, B] — SiLU output for step 0.
    denoise_h0_buf: CudaSlice<f32>,
    /// Col-major scratch [H=24, B] — SiLU output for step 1.
    denoise_h1_buf: CudaSlice<f32>,
    /// Row-major scratch [B, D=12] — Q values after step 0.
    denoise_q_step0_buf: CudaSlice<f32>,
    /// Col-major scratch [D=12, B] — loss gradient (upstream for backward).
    denoise_d_res_buf: CudaSlice<f32>,
    /// Col-major scratch [H=24, B] — d_h / d_pre scratch for backward.
    denoise_d_h_buf: CudaSlice<f32>,
    /// Scratch for bias gradient reduction phase 1 [num_blocks * max(H,D)].
    denoise_bias_grad_partials: CudaSlice<f32>,
    /// GEMM desc: forward W @ input, C[H=24, B] = W[H,H] @ X[H,B]. TRANSA=T, TRANSB=N.
    denoise_gemm_fwd_hh: Mamba2GemmDesc,
    /// GEMM desc: forward W2 @ h, C[D=12, B] = W2[D,H] @ h[H,B]. TRANSA=T, TRANSB=N.
    denoise_gemm_fwd_dh: Mamba2GemmDesc,
    /// GEMM desc: backward dW1 = input @ d_pre^T, output row-major [H,H].
    /// Actually computes dW1^T[H,H]_colmaj = input[H,B] @ d_pre[H,B]^T.
    /// TRANSA=N, TRANSB=T, m=H, n=H, k=B.
    denoise_gemm_dw1: Mamba2GemmDesc,
    /// GEMM desc: backward dW2 = h @ d_res^T, output row-major [D,H].
    /// Actually computes dW2^T[H,D]_colmaj = h[H,B] @ d_res[D,B]^T.
    /// TRANSA=N, TRANSB=T, m=H, n=D, k=B.
    denoise_gemm_dw2: Mamba2GemmDesc,
    /// GEMM desc: backward d_h = W2^T @ d_res, C[H=24, B] = W2^T[H,D] @ d_res[D,B].
    /// W2 stored row-major [D,H] = col-major [H,D]. TRANSA=N (no transpose), TRANSB=N.
    denoise_gemm_bwd_dh: Mamba2GemmDesc,
    /// Kernel handle: denoise_build_input.
    denoise_build_input_kernel: CudaFunction,
    /// Kernel handle: denoise_bias_silu.
    denoise_bias_silu_kernel: CudaFunction,
    /// Kernel handle: denoise_silu_bwd.
    denoise_silu_bwd_kernel: CudaFunction,
    /// Kernel handle: denoise_loss_grad.
    denoise_loss_grad_kernel: CudaFunction,
    /// Kernel handle: denoise_skip_connection.
    denoise_skip_conn_kernel: CudaFunction,
    /// Kernel handle: denoise_bias_grad_p1.
    denoise_bias_grad_p1_kernel: CudaFunction,
    /// Kernel handle: denoise_bias_grad_p2.
    denoise_bias_grad_p2_kernel: CudaFunction,

    // ── xLSTM temporal context (mLSTM cell) ────────────────────────────
    /// [528] 6 weight matrices × [11, 8]: W_k, W_v, W_q, W_i, W_f, W_o
    /// Mapped pinned (cuMemHostAlloc DEVICEMAP) — host-init at construction,
    /// kernel reads via dev_ptr and trains in place. Avoids DtoD-via-pinned
    /// per feedback_no_htod_htoh_only_mapped_pinned.
    qlstm_weights: super::mapped_pinned::MappedF32Buffer,
    /// [64] persistent 8×8 matrix memory (device state across steps)
    qlstm_c: CudaSlice<f32>,
    /// [8] persistent normalizer vector (device state across steps)
    qlstm_n: CudaSlice<f32>,
    /// [8] output context vector (written each qlstm_step call)
    qlstm_context: CudaSlice<f32>,
    /// [4] per-branch Q-gaps — pinned device-mapped. CPU writes, GPU reads via dev_ptr. No HtoD copy.
    per_branch_q_gaps_pinned: *mut f32,
    per_branch_q_gaps_dev_ptr: u64,
    /// qlstm_step kernel handle
    qlstm_step_kernel: CudaFunction,
    /// qlstm_train_step kernel handle — SGD update on prediction loss.
    qlstm_train_kernel: CudaFunction,
    /// Previous Q-mean — pinned device-mapped scalar. Updated after each qlstm_step.
    prev_q_mean_pinned: *mut f32,
    prev_q_mean_dev_ptr: u64,

    // ── Mamba2 temporal scan ──
    mamba2_h_history: CudaSlice<f32>,      // [B, K, SH2+OFI_EMBED_DIM] rolling history (widened for OFI)
    mamba2_h_enriched: CudaSlice<f32>,     // [B, SH2] output
    mamba2_update_kernel: CudaFunction,
    mamba2_copy_enriched_kernel: CudaFunction,
    mamba2_params: CudaSlice<f32>,         // W_A[(SH2+OFI)*SD] + W_B[(SH2+OFI)*SD] + W_C[SH2*SD]
    mamba2_grad: CudaSlice<f32>,           // gradient buffer (same layout as params)
    mamba2_adam_m: CudaSlice<f32>,         // Adam first moment (same size as params)
    mamba2_adam_v: CudaSlice<f32>,         // Adam second moment (same size as params)
    mamba2_adam_step: i32,                 // Adam step counter

    // ── Mamba2 cuBLAS projection buffers ──
    mamba2_a_proj: CudaSlice<f32>,         // [B*K, STATE_D] forward gate projection
    mamba2_b_proj: CudaSlice<f32>,         // [B*K, STATE_D] forward input projection
    mamba2_d_gate: CudaSlice<f32>,         // [B*K, STATE_D] backward gate gradient
    mamba2_d_x: CudaSlice<f32>,            // [B*K, STATE_D] backward input gradient
    mamba2_d_context: CudaSlice<f32>,      // [B, STATE_D] backward context (x_K per sample)
    mamba2_d_tw: CudaSlice<f32>,           // [B, SH2] scaled d_h_enriched * temporal_weight

    // ── Mamba2 cuBLAS GEMM descriptors ──
    mamba2_gemm_proj: Mamba2GemmDesc,      // fwd: W^T @ h, m=STATE_D, n=B*K, k=SH2+OFI
    mamba2_gemm_dw: Mamba2GemmDesc,        // bwd dW_A/dW_B: d^T @ h, m=STATE_D, n=SH2+OFI, k=B*K
    mamba2_gemm_dwc: Mamba2GemmDesc,       // bwd dW_C: xK^T @ dtw, m=STATE_D, n=SH2, k=B
    mamba2_gemm_dh: Mamba2GemmDesc,        // bwd d_h_history: W^T @ d, m=SH2+OFI, n=B*K, k=STATE_D

    // ── Mamba2 input gradient buffers ──
    d_h_history_buf: CudaSlice<f32>,       // [(SH2+OFI) * B * K] backward scratch
    d_ofi_embed_mamba2: CudaSlice<f32>,    // [OFI_EMBED_DIM * B] accumulated OFI embed gradient
    extract_ofi_embed_grad_kernel: CudaFunction,

    // ── Mamba2 cuBLAS scan kernels ──
    mamba2_scan_proj_fwd_kernel: CudaFunction,
    mamba2_scan_proj_bwd_kernel: CudaFunction,
    mamba2_scale_d_enriched_kernel: CudaFunction,

    /// Step counter for new component LR warmup (0 to 500).
    new_component_warmup_step: u32,

    // ── Multi-horizon value heads ──
    v_logits_5bar: CudaSlice<f32>,      // [B, NA]
    v_logits_20bar: CudaSlice<f32>,     // [B, NA]
    save_h_v_5bar: CudaSlice<f32>,      // [B, VH]
    save_h_v_20bar: CudaSlice<f32>,     // [B, VH]
    v_logits_blended: CudaSlice<f32>,   // [B, NA]

    // ── Homeostatic regularizer (G16) ──
    /// Compiled homeostatic_regularizer kernel from experience_kernels.cubin.
    homeostatic_kernel: CudaFunction,
    /// SP4 Layer C close-out follow-up (2026-05-01): GPU-only readiness-driven
    /// homeostatic-target EMA kernel. Single-block, six threads — replaces the
    /// host-side EMA loop in `calibrate_homeostatic_targets` per
    /// `feedback_no_cpu_compute_strict`. Loaded from
    /// `calibrate_homeostatic_kernel.cubin`.
    calibrate_homeostatic_kernel: CudaFunction,
    /// [6] observable signals — pinned device-mapped. GPU reads via dev_ptr.
    /// Slot 0 (q-mean) and slot 1 (atom-util) are written by GPU kernels;
    /// slots 2..5 may be written by host helpers (`update_homeostatic_observables`).
    homeostatic_obs_pinned: *mut f32,
    homeostatic_obs_dev_ptr: u64,
    /// [6] target set-points — pinned device-mapped. Updated GPU-side by
    /// `calibrate_homeostatic_kernel` (SP4 Layer C close-out follow-up,
    /// 2026-05-01); consumed by `homeostatic_kernel` via `homeostatic_targets_dev_ptr`.
    homeostatic_targets_pinned: *mut f32,
    homeostatic_targets_dev_ptr: u64,
    /// [6] per-observable penalty output (device buffer).
    homeostatic_penalties_buf: CudaSlice<f32>,
    /// [1] total penalty scalar output (device buffer).
    homeostatic_total_buf: CudaSlice<f32>,
    /// Whether calibration from early epochs is complete (epochs > 5).
    homeostatic_calibration_done: bool,

    // ── Learned risk management (5th branch) ──
    risk_forward_kernel: CudaFunction,
    risk_apply_kernel: CudaFunction,
    risk_backward_kernel: CudaFunction,
    risk_hidden_buf: CudaSlice<f32>,       // [B, AH]
    risk_budget_buf: CudaSlice<f32>,       // [B]
    cvar_alpha_buf: CudaSlice<f32>,        // [B] per-sample CVaR alpha
    commit_lambda_buf: CudaSlice<f32>,     // [B] per-sample commitment lambda
    q_mag_pre_risk: CudaSlice<f32>,        // [B, b1_size] saved for backward
    risk_grad_buf: CudaSlice<f32>,         // [risk_param_count]
    risk_adam_step: i32,

    // ── ISV scratch scalars (pinned device-mapped, written by GPU kernels) ──
    // Batch mean |TD-error| scratch. Written by the second
    // `c51_loss_reduce` launch at `launch_loss_reduce` (commit
    // 7f92fa242). Feeds ISV[2] TD-error EMA in `isv_signal_update`.
    td_error_scratch_pinned: *mut f32,
    td_error_scratch_dev_ptr: u64,
    // C51 Q-distribution variance scratch (ISV audit 2026-04-23: renamed
    // from `ensemble_var_scratch_pinned`). Written by a third
    // `c51_loss_reduce` launch at `launch_loss_reduce` that reduces
    // `q_var_buf_trainer` (size b*total_actions, atom-spread variance per
    // action). Feeds ISV[3] "C51 Q-variance EMA" and ISV[4] velocity in
    // `isv_signal_update`. NOT multi-head ensemble variance — the
    // original misnomer led to the signal being treated as a zeroed
    // dead path for the 20-epoch window analysed in the audit.
    q_var_scratch_pinned: *mut f32,
    q_var_scratch_dev_ptr: u64,
    reward_scratch_pinned: *mut f32,
    reward_scratch_dev_ptr: u64,
    atom_util_scratch_pinned: *mut f32,
    atom_util_scratch_dev_ptr: u64,

    // Task 2.X "make Full useful" — per-magnitude-bin Q-mean scratch arrays
    // written by `q_mag_bin_means_reduce` (runs with q_stats on the stats
    // cadence), read by isv_signal_update to drive the EMA slots [13..16].
    // The means scratch is sized for magnitude-branch capacity (MAX_MAG=4
    // in the kernel); the abs_ref scratch is a single scalar.
    q_mag_means_scratch_pinned: *mut f32,
    q_mag_means_scratch_dev_ptr: u64,
    q_abs_ref_scratch_pinned: *mut f32,
    q_abs_ref_scratch_dev_ptr: u64,
    q_mag_bin_means_reduce_kernel: CudaFunction,

    // Task 2.Y "make direction branch useful at eval" — per-direction-bin
    // Q-mean scratch arrays written by `q_dir_bin_means_reduce` (runs with
    // q_stats on the stats cadence), read by isv_signal_update to drive
    // the EMA slots [17..21]. The means scratch is sized for direction-branch
    // capacity (MAX_DIR=4 in the kernel); the abs_ref scratch is a scalar.
    q_dir_means_scratch_pinned: *mut f32,
    q_dir_means_scratch_dev_ptr: u64,
    q_dir_abs_ref_scratch_pinned: *mut f32,
    q_dir_abs_ref_scratch_dev_ptr: u64,
    q_dir_bin_means_reduce_kernel: CudaFunction,

    // ── ISV core buffers (pinned device-mapped, GPU read/write) ──
    // isv_signals_pinned is sized for ISV_TOTAL_DIM — see the
    // `const ISV_TOTAL_DIM` definition + `ISV_TOTAL_DIM=` line in the
    // layout-fingerprint docstring for the authoritative value. Slots
    // [23..31] carry per-branch Q-support centres/half-widths; [43..47)
    // are the D.2 per-branch gamma slots; [47..50) Plan 1 C.6 static-
    // config; [50..58) are the Plan 2 C.1 Q-quantile EMAs; [58..60)
    // layout fingerprint; SP4 [131..ISV_TOTAL_DIM) for Layer A/B/C
    // ISV-driven bounds. The network (w_isv_fc1) and history rotation
    // still consume only the first ISV_NETWORK_DIM slots; the tail is
    // broadcast-bus scratchpad only.
    isv_signals_pinned: *mut f32,        // [ISV_TOTAL_DIM]
    isv_signals_dev_ptr: u64,
    isv_history_pinned: *mut f32,        // [ISV_K * 12 = 48] — history rotates slots [0..11] only
    isv_history_dev_ptr: u64,
    isv_decay_pinned: *mut f32,          // [ISV_NETWORK_DIM = 23] learned decay weights
    isv_decay_dev_ptr: u64,
    lagged_td_error_pinned: *mut f32,    // [1] recursive confidence target
    lagged_td_error_dev_ptr: u64,
    isv_signal_update_kernel: CudaFunction,

    // ── ISV forward outputs ──
    isv_forward_kernel: CudaFunction,
    isv_feature_gate_kernel: CudaFunction,
    fill_gamma_buf_kernel: CudaFunction,
    isv_embedding_buf: CudaSlice<f32>,   // [ISV_EMB_DIM] = [8]
    branch_gate_buf: CudaSlice<f32>,     // [4]
    gamma_mod_buf: CudaSlice<f32>,       // [1]
    temporal_weight_buf: CudaSlice<f32>, // [SH2] per-feature temporal routing
    isv_temporal_route_kernel: CudaFunction,
    gamma_buf: CudaSlice<f32>,           // [B] per-sample effective gamma
    predicted_error_buf: CudaSlice<f32>, // [B] recursive confidence output

    // ── Recursive confidence kernels ──
    recursive_conf_fwd_kernel: CudaFunction,
    recursive_conf_bwd_kernel: CudaFunction,
    recursive_conf_reduce_kernel: CudaFunction,
    /// Per-block partial buffer for recursive_confidence_backward deterministic reduce.
    /// Shape: [max_conf_blocks, SH2 + 1].
    recursive_conf_partials: CudaSlice<f32>,
    max_conf_blocks: usize,

    // ── Trade plan head (cuBLAS 2-layer MLP) ──
    plan_params_buf: CudaSlice<f32>,          // [B, 6]
    trade_plan_hidden_buf: CudaSlice<f32>,    // [B, AH] scratch for layer 1 output
    trade_plan_pre_out_buf: CudaSlice<f32>,   // [B, 6] scratch for layer 2 pre-activation
    trade_plan_activate_kernel: CudaFunction, // bias + sigmoid/scaling activations
    plan_noise_kernel: CudaFunction,          // plan_noise_inject: ±5% temporal diversity

    // ── OFI embed MLP (OFI_EMBED_IN→OFI_EMBED_DIM) ──
    ofi_embed_input_buf: CudaSlice<f32>,          // [B, OFI_EMBED_IN] scratch for input extraction
    ofi_embed_output_buf: CudaSlice<f32>,         // [B, OFI_EMBED_DIM] embedding output
    ofi_embed_w: CudaSlice<f32>,                  // [OFI_EMBED_DIM, OFI_EMBED_IN] weight matrix (row-major)
    ofi_embed_b: CudaSlice<f32>,                  // [OFI_EMBED_DIM] bias
    ofi_embed_build_input_kernel: CudaFunction,
    // ── OFI embed MLP backward + Adam ──
    /// Contiguous param buffer [OFI_EMBED_TOTAL_PARAMS] = W[OFI_EMBED_W_COUNT] + b[OFI_EMBED_DIM] for Adam update.
    ofi_embed_params: CudaSlice<f32>,
    /// Weight gradient [OFI_EMBED_W_COUNT] (OFI_EMBED_DIM × OFI_EMBED_IN).
    ofi_embed_grad_w: CudaSlice<f32>,
    /// Bias gradient [OFI_EMBED_DIM].
    ofi_embed_grad_b: CudaSlice<f32>,
    /// Contiguous gradient buffer [OFI_EMBED_TOTAL_PARAMS] for Adam (assembled from grad_w + grad_b).
    ofi_embed_grad: CudaSlice<f32>,
    /// Combined gradient from Mamba2 + attention [OFI_EMBED_DIM * B].
    d_ofi_embed_combined: CudaSlice<f32>,
    /// Adam first moment [OFI_EMBED_TOTAL_PARAMS].
    ofi_embed_adam_m: CudaSlice<f32>,
    /// Adam second moment [OFI_EMBED_TOTAL_PARAMS].
    ofi_embed_adam_v: CudaSlice<f32>,
    /// Adam step counter.
    ofi_embed_adam_step: i32,
    /// Grad norm partials [1].
    ofi_embed_norm_partials: CudaSlice<f32>,
    /// Grad norm output [1].
    ofi_embed_norm_buf: CudaSlice<f32>,
    /// Adaptive gradient clip [1].
    ofi_embed_clip_buf: CudaSlice<f32>,
    /// Pinned device-mapped step counter for Adam.
    ofi_embed_t_pinned: *mut i32,
    ofi_embed_t_dev_ptr: u64,
    /// Bias grad reduce partial sums: [bias_num_blocks, OFI_EMBED_DIM].
    ofi_embed_bias_grad_partials: CudaSlice<f32>,

    // ── SP14 Layer C aux trunk params (Phase C.2, 2026-05-08) ──
    // 3-layer Linear→ELU→Linear→ELU→Linear MLP that produces `h_s2_aux`
    // for the auxiliary next-bar head. Mirrors GRN trunk's role for Q's
    // path but trained purely by aux loss (stop-grad at encoder boundary;
    // see plan §C.5). Allocated here per Phase C.2 — forward/backward
    // wire-up lands in Phases C.3-C.5 (`feedback_no_partial_refactor`
    // applies per-phase, not across phases). Topology:
    //   w1 [SH1, 256], b1 [256], w2 [256, 128], b2 [128],
    //   w3 [128, SH2], b3 [SH2]
    // SH1 = encoder output dim (`config.shared_h1`); SH2 = aux head's
    // expected input dim (`config.shared_h2`, matches `h_s2`'s shape).
    // Each weight tensor uses Kaiming-He fan_in init (`std=sqrt(2/fan_in)`)
    // for ELU activation. Biases zero-init.
    /// Aux trunk weight 1 [SH1 × 256] row-major: encoder_out → hidden1.
    aux_trunk_w1: CudaSlice<f32>,
    /// Aux trunk bias 1 [256].
    aux_trunk_b1: CudaSlice<f32>,
    /// Aux trunk weight 2 [256 × 128] row-major: hidden1 → hidden2.
    aux_trunk_w2: CudaSlice<f32>,
    /// Aux trunk bias 2 [128].
    aux_trunk_b2: CudaSlice<f32>,
    /// Aux trunk weight 3 [128 × SH2] row-major: hidden2 → h_s2_aux.
    aux_trunk_w3: CudaSlice<f32>,
    /// Aux trunk bias 3 [SH2] (matches aux head input dim).
    aux_trunk_b3: CudaSlice<f32>,
    /// Adam first moment for `aux_trunk_w1` [SH1 × 256].
    aux_trunk_w1_m: CudaSlice<f32>,
    /// Adam second moment for `aux_trunk_w1` [SH1 × 256].
    aux_trunk_w1_v: CudaSlice<f32>,
    /// Adam first moment for `aux_trunk_b1` [256].
    aux_trunk_b1_m: CudaSlice<f32>,
    /// Adam second moment for `aux_trunk_b1` [256].
    aux_trunk_b1_v: CudaSlice<f32>,
    /// Adam first moment for `aux_trunk_w2` [256 × 128].
    aux_trunk_w2_m: CudaSlice<f32>,
    /// Adam second moment for `aux_trunk_w2` [256 × 128].
    aux_trunk_w2_v: CudaSlice<f32>,
    /// Adam first moment for `aux_trunk_b2` [128].
    aux_trunk_b2_m: CudaSlice<f32>,
    /// Adam second moment for `aux_trunk_b2` [128].
    aux_trunk_b2_v: CudaSlice<f32>,
    /// Adam first moment for `aux_trunk_w3` [128 × SH2].
    aux_trunk_w3_m: CudaSlice<f32>,
    /// Adam second moment for `aux_trunk_w3` [128 × SH2].
    aux_trunk_w3_v: CudaSlice<f32>,
    /// Adam first moment for `aux_trunk_b3` [SH2].
    aux_trunk_b3_m: CudaSlice<f32>,
    /// Adam second moment for `aux_trunk_b3` [SH2].
    aux_trunk_b3_v: CudaSlice<f32>,

    // ── SP14 Layer C Phase C.5a saved-fwd + grad buffers (2026-05-08) ──
    // Allocated as DEAD CODE: these buffers are real device allocations
    // but no production caller reads / writes them yet. Phase C.5b
    // atomically wires them via the aux trunk forward (writes
    // h_s2_aux/h_aux1/h_aux2), aux heads backward (SAXPY-accumulates into
    // dh_s2_aux_accum), aux trunk backward (overwrites the 6 grad
    // tensors), and `launch_aux_trunk_adam_update` (consumes grads via
    // global L2-norm clip + per-tensor Adam step).
    //
    // 6-buffer choice (vs flat-buffer): mirrors C.2's separate-tensor
    // params/m/v allocation. Lets each Adam launch consume one
    // (param, grad, m, v) tuple of matching extent — no byte-offset
    // arithmetic. Per-tensor grad_norm partial sums into a unified
    // `aux_trunk_norm_partials` buffer at fixed offsets so the global
    // L2 norm spans all 6 tensors.
    /// `[B_max, SH2]` final aux trunk output (saved post-fwd; consumed by
    /// aux heads forward + backward in C.5b). Same shape contract as
    /// `aux_dh_s2_nb_buf` so the SAXPY accumulator type-matches.
    #[allow(dead_code)]
    h_s2_aux: CudaSlice<f32>,
    /// `[B_max, AUX_TRUNK_H1=256]` Layer-1 post-ELU activation. Consumed
    /// by `aux_trunk_bwd_dh_pre` (`ELU'(h_aux1)`) + `aux_trunk_bwd_dW_reduce`
    /// (Layer-2 `dW2 = h_aux1.T @ dh_aux2_pre`).
    #[allow(dead_code)]
    h_aux1: CudaSlice<f32>,
    /// `[B_max, AUX_TRUNK_H2=128]` Layer-2 post-ELU activation. Consumed
    /// by `aux_trunk_bwd_dh_pre` (`ELU'(h_aux2)`) + `aux_trunk_bwd_dW_reduce`
    /// (Layer-3 `dW3 = h_aux2.T @ d_logits`).
    #[allow(dead_code)]
    h_aux2: CudaSlice<f32>,
    /// `[B_max, SH2]` SAXPY accumulator for upstream gradient feeding the
    /// aux trunk's Layer-3 input (`dh_s2_aux`). C.5b reverts the zero-fill
    /// hack at `aux_next_bar_backward:599-638` + `aux_regime_backward:757-790`
    /// so the two aux head backwards SAXPY their per-sample dh into this
    /// buffer instead of the trunk's `bw_d_h_s2`. Consumed by aux trunk
    /// backward (`launch(... dh_s2_aux_in_ptr=accum_ptr ...)`).
    #[allow(dead_code)]
    dh_s2_aux_accum: CudaSlice<f32>,

    /// `[SH1 × 256]` aux trunk weight-1 gradient accumulator (overwritten
    /// per training step by `aux_trunk_bwd_dW_reduce`).
    #[allow(dead_code)]
    aux_trunk_w1_grad: CudaSlice<f32>,
    /// `[256]` aux trunk bias-1 gradient (overwritten by `aux_trunk_bwd_db_reduce`).
    #[allow(dead_code)]
    aux_trunk_b1_grad: CudaSlice<f32>,
    /// `[256 × 128]` aux trunk weight-2 gradient.
    #[allow(dead_code)]
    aux_trunk_w2_grad: CudaSlice<f32>,
    /// `[128]` aux trunk bias-2 gradient.
    #[allow(dead_code)]
    aux_trunk_b2_grad: CudaSlice<f32>,
    /// `[128 × SH2]` aux trunk weight-3 gradient.
    #[allow(dead_code)]
    aux_trunk_w3_grad: CudaSlice<f32>,
    /// `[SH2]` aux trunk bias-3 gradient.
    #[allow(dead_code)]
    aux_trunk_b3_grad: CudaSlice<f32>,

    /// SP14 Layer C Phase C.5a-fixup (2026-05-08): pre-activation gradient
    /// scratch for aux trunk Layer-1 (`dh_aux1_pre [B_max, AUX_TRUNK_H1=256]`).
    /// `aux_trunk_backward.launch` writes this scratch in its dh_pre kernel
    /// (overwrites; not accumulated) and reads it back in the dW_reduce
    /// kernel for `dW1 = x_in.T @ dh_aux1_pre` + `db1 = sum_b dh_aux1_pre`.
    /// Required by `gpu_aux_trunk.rs:266-267` — C.5a missed it. DEAD until
    /// C.5b atomically wires the aux trunk backward into the gradient chain.
    #[allow(dead_code)]
    dh_aux1_pre_scratch: CudaSlice<f32>,
    /// SP14 Layer C Phase C.5a-fixup: pre-activation gradient scratch for
    /// aux trunk Layer-2 (`dh_aux2_pre [B_max, AUX_TRUNK_H2=128]`). Same
    /// role + lifecycle as `dh_aux1_pre_scratch`; consumed by Layer-2/3
    /// matmul reductions in `aux_trunk_backward.launch`.
    #[allow(dead_code)]
    dh_aux2_pre_scratch: CudaSlice<f32>,

    /// `[num_blocks_total]` per-block partial sum-of-squares written by
    /// `dqn_grad_norm_kernel` across all 6 grad tensors (offsets matching
    /// `aux_trunk_grad_block_offsets`). Sized to the maximum total block
    /// count any of the 6 grads would produce.
    #[allow(dead_code)]
    aux_trunk_grad_norm_partials: CudaSlice<f32>,
    /// Per-tensor block offsets into `aux_trunk_grad_norm_partials`. The
    /// 7th entry is the total block count (passed as `num_blocks` to
    /// `dqn_grad_norm_finalize`). Captured at construction so the launcher
    /// has no host-side branching during its launches.
    #[allow(dead_code)]
    aux_trunk_grad_block_offsets: [usize; 7],
    /// `[1]` finalised L2 grad norm written by `dqn_grad_norm_finalize`.
    /// Read by 6 Adam launches as the `grad_norm_f32` arg.
    #[allow(dead_code)]
    aux_trunk_grad_norm_buf: CudaSlice<f32>,

    /// Pinned device-mapped grad-clip threshold. Host writes
    /// `ISV[AUX_TRUNK_GRAD_CLIP_INDEX=448]` per-step before launch; GPU
    /// reads via `aux_trunk_grad_clip_dev_ptr`. The Adam kernel's
    /// `max_grad_norm_buf` arg.
    #[allow(dead_code)]
    aux_trunk_grad_clip_pinned: *mut f32,
    /// Device pointer for `aux_trunk_grad_clip_pinned` (one
    /// `cuMemHostGetDevicePointer_v2` per construction; reusable across
    /// launches inside captured graphs).
    #[allow(dead_code)]
    aux_trunk_grad_clip_dev_ptr: u64,

    /// Pinned device-mapped LR. Host writes `ISV[AUX_TRUNK_LR_INDEX=444]`
    /// per-step. The Adam kernel's `lr_ptr` arg.
    #[allow(dead_code)]
    aux_trunk_lr_pinned: *mut f32,
    /// Device pointer for `aux_trunk_lr_pinned`.
    #[allow(dead_code)]
    aux_trunk_lr_dev_ptr: u64,

    /// Pinned device-mapped Adam step counter (i32). Incremented by host
    /// before each launch; the kernel's `t_ptr` arg.
    #[allow(dead_code)]
    aux_trunk_t_pinned: *mut i32,
    /// Device pointer for `aux_trunk_t_pinned`.
    #[allow(dead_code)]
    aux_trunk_t_dev_ptr: u64,
    /// Host-side mirror of the Adam step counter. Production caller in
    /// C.5b increments this and writes through to the pinned buffer.
    #[allow(dead_code)]
    aux_trunk_adam_step: i32,

    /// SP14 Layer C Phase C.3 (2026-05-08): aux trunk forward orchestrator.
    /// Owns the pre-loaded `CudaFunction` handle for `aux_trunk_forward`
    /// per `pearl_no_host_branches_in_captured_graph.md` — the launch
    /// path never touches `load_cubin` / `load_function`. Wire-up into
    /// the collector chain lands in Phase C.5 (atomic). Backward
    /// orchestrator + Adam updates land in Phase C.4. The `dead_code`
    /// allow is necessary because this commit is additive — production
    /// callers light up in C.5 per `feedback_no_partial_refactor`.
    #[allow(dead_code)]
    aux_trunk_forward_ops: super::gpu_aux_trunk::AuxTrunkForwardOps,

    /// SP14 Layer C Phase C.4 (2026-05-08): aux trunk backward
    /// orchestrator. Owns three pre-loaded `CudaFunction` handles
    /// (`aux_trunk_bwd_dh_pre` + `aux_trunk_bwd_dW_reduce` +
    /// `aux_trunk_bwd_db_reduce`). A single `launch()` call orchestrates
    /// the seven-launch backward pass over all three trunk layers
    /// without per-sample partial buffers (per-element block-tree-reduce
    /// instead). CRITICAL: kernel set does NOT compute or write `dx_in`
    /// — encoder boundary is the structural stop-gradient per locked
    /// design. Wire-up into the collector backward chain + Adam updates
    /// land in Phase C.5 (atomic). The `dead_code` allow is necessary
    /// because this commit is additive — production callers light up
    /// in C.5 per `feedback_no_partial_refactor`.
    #[allow(dead_code)]
    aux_trunk_backward_ops: super::gpu_aux_trunk::AuxTrunkBackwardOps,

    /// SP14 Layer C Phase C.4b (2026-05-08): adaptive aux prediction
    /// horizon producer. Drives `ISV[AUX_PRED_HORIZON_BARS_INDEX=450]`
    /// from `ISV[AVG_WIN_HOLD_TIME_BARS_INDEX=451]` via Pearl-A
    /// first-observation bootstrap + slow Wiener-α EMA. Single-thread
    /// kernel; per-epoch boundary launch. See `aux_horizon_update_kernel.cu`.
    aux_horizon_update_ops: super::gpu_aux_trunk::AuxHorizonUpdateOps,

    /// SP14 Layer C Phase C.4b (2026-05-08): avg winning hold time EMA
    /// producer. Sweeps the per-epoch `hold_at_exit_per_sample` +
    /// `trade_profitable_per_sample` buffers and writes
    /// `ISV[AVG_WIN_HOLD_TIME_BARS_INDEX=451]`. Block-tree-reduce
    /// (no atomicAdd); per-epoch boundary launch.
    avg_win_hold_time_update_ops: super::gpu_aux_trunk::AvgWinHoldTimeUpdateOps,

    /// Class A P0-A (2026-05-08): adaptive REWARD_POS/NEG_CAP producer.
    /// Sweeps the per-epoch `step_ret_per_sample` + `trade_close_per_sample`
    /// buffers, computes p99(winning_returns) × 1.5 → POS cap, NEG = −2 × POS,
    /// writes both `ISV[REWARD_POS_CAP_ADAPTIVE_INDEX=452]` and
    /// `ISV[REWARD_NEG_CAP_ADAPTIVE_INDEX=453]`. Block-tree-reduce
    /// (no atomicAdd); Pearl-A first-observation bootstrap; α=0.01 slow
    /// EMA. Per-epoch boundary launch — reward distribution is the
    /// foundation of training and shouldn't move fast.
    reward_cap_update_ops: super::gpu_aux_trunk::RewardCapUpdateOps,

    /// SP18 v2 Phase 3 (2026-05-09): adaptive HOLD_REWARD_POS/NEG_CAP
    /// producer Ops handle. Pre-loads the `CudaFunction` at construction
    /// per `pearl_no_host_branches_in_captured_graph.md`. Sweeps the
    /// per-epoch `step_ret_per_sample` + `trade_close_per_sample`
    /// buffers (same source `reward_cap_update_ops` reads from), filters
    /// by `(is_close && step_ret != 0)` to extract Long/Short close
    /// magnitudes per spec DD3=b, computes a Welford p99 estimator ×
    /// 1.5 safety factor → POS cap, derives Wiener-optimal α from
    /// Welford accumulators in slots [487..493), and writes both
    /// `ISV[HOLD_REWARD_POS_CAP_INDEX=483]` and
    /// `ISV[HOLD_REWARD_NEG_CAP_INDEX=484]` (NEG = −2 × POS, DD5=b
    /// mirrored asymmetry, same Kahneman 2:1 ratio as the position-
    /// side cap). Block-tree-reduce (no atomicAdd); Pearl-A first-
    /// observation bootstrap; Wiener-α floor at 0.4 per
    /// `pearl_wiener_alpha_floor_for_nonstationary` since the policy-
    /// realised distribution is intrinsically non-stationary as the
    /// policy adapts. Per-epoch boundary launch right AFTER
    /// `reward_cap_update_ops` so both cap producers share the same
    /// step_ret/trade_close source buffers; no shared state — they
    /// drive independent ISV slot pairs.
    hold_reward_cap_update_ops: super::gpu_aux_trunk::HoldRewardCapUpdateOps,

    /// Class A P1-Producer (2026-05-08): adaptive Bayesian Kelly priors
    /// producer. Sweeps `portfolio_state[n_envs, PS_STRIDE]` (same source
    /// `kelly_cap_update_kernel` reads), aggregates the realized
    /// PS_KELLY_{WIN_COUNT, LOSS_COUNT, SUM_WINS, SUM_LOSSES} fields
    /// across envs, and slow-EMA-blends into ISV[454..458). Pearl-A
    /// first-observation bootstrap (sentinels 2.0/2.0/0.01/0.01); α=0.005
    /// slow EMA. Replaces hardcoded `prior_wins/prior_losses/
    /// prior_sum_wins/prior_sum_losses` constants in
    /// `kelly_cap_update_kernel.cu` and
    /// `trade_physics.cuh::kelly_position_cap`. Per-epoch boundary launch
    /// (RIGHT BEFORE `kelly_cap_update` so that kernel sees the
    /// freshly-blended priors).
    kelly_bayesian_priors_update_ops: super::gpu_aux_trunk::KellyBayesianPriorsUpdateOps,

    /// Class A audit-fix Batch 4-A (2026-05-08): adaptive DD saturation
    /// floor producer Ops handle. Sweeps `dd_state_per_env[n_envs * 6]`,
    /// aggregates per-env DD_MAX (offset 1), computes Welford `mean +
    /// Z_75 × sigma` p75 estimator × 1.5 safety factor, slow-EMA blends
    /// into `ISV[DD_SATURATION_FLOOR_ADAPTIVE_INDEX=458]`. Replaces the
    /// hardcoded `0.25f` saturation floor in
    /// `trade_physics.cuh::apply_margin_cap`. Per-epoch boundary launch
    /// (mirrors P0-A REWARD_POS_CAP cadence).
    dd_saturation_floor_update_ops: super::gpu_aux_trunk::DdSaturationFloorUpdateOps,

    /// Class A audit-fix Batch 4-B (2026-05-08, Item 4): adaptive
    /// MIN_HOLD_TEMPERATURE producer Ops handle. Reads
    /// `ISV[AUX_DIR_ACC_SHORT_EMA_INDEX=373]` (sp13 fast EMA of binary
    /// directional accuracy), maps it to a [5, 50] temperature via
    /// `temp = TEMP_MIN + (TEMP_MAX − TEMP_MIN) × clamp((short_ema −
    /// 0.5)/0.5, 0, 1)`, and slow-EMA-blends into
    /// `ISV[MIN_HOLD_TEMPERATURE_ADAPTIVE_INDEX=460]`. Replaces the
    /// deleted epoch-driven anneal schedule
    /// `T_end + (T_start − T_end) × exp(-epoch / decay)`. Pearl-A
    /// first-observation bootstrap (sentinel 50.0 matches the deleted
    /// MIN_HOLD_TEMPERATURE_START=50 anchor for bit-identical
    /// cold-start under both regimes). α=0.05 mid-cadence EMA. Per-epoch
    /// boundary launch (same cadence as P0-A REWARD_POS_CAP).
    min_hold_temperature_update_ops: super::gpu_aux_trunk::MinHoldTemperatureUpdateOps,

    // ── Speculative inference cache ──
    /// Cached trunk output [B, SH2] from between-bar speculative forward.
    speculative_h_s2: CudaSlice<f32>,
    /// Last features [state_dim] used for speculation (host-side for L2 comparison).
    speculative_features: Vec<f32>,
    /// Whether the speculative cache contains a usable result.
    speculative_valid: bool,

    // ── GPU-side counter increment kernel ───────────────────────────
    /// Kernel: increment_step_counters — atomically increments all 8 Adam step
    /// counters and computes cosine-annealed tau on GPU, eliminating all host-side
    /// per-step writes from the critical path.
    increment_counters_kernel: CudaFunction,
    /// Graph-safe device-to-device f32 copy kernel.
    /// Replaces `memcpy_dtod_async` in graph-captured code paths (memcpy is NOT
    /// captured by CUDA Graph — only kernel launches are).
    copy_f32_kernel: CudaFunction,
    /// Initial tau for cosine annealing (written to tau_pinned at step 0).
    tau_init: f32,
    /// Final tau for cosine annealing (asymptotic value after tau_anneal_steps).
    tau_final: f32,
    /// Total number of steps over which tau is cosine-annealed.
    tau_anneal_steps: i32,

    /// Last effective cql_alpha applied (base × (1−regime_stability) × health).
    /// Populated by apply_cql_gradient. Initialized to 0.0.
    pub(crate) last_cql_alpha_eff: f32,

    /// B3/G4: Last health-scaled Expected SARSA tau factor (1.0 at full health, 6.0 at collapse).
    /// Populated by `launch_c51_loss`. Initialized to 0.0.
    pub(crate) last_sarsa_tau_factor: f32,

    /// B4/G5: Last adaptive gradient budget for IQN (0.10 + 0.30×health).
    /// Populated by `FusedTrainingCtx::compute_adaptive_budgets`. Default: health=1 → 0.40.
    pub(crate) last_iqn_budget_eff: f32,
    /// B4/G5: Adaptive gradient budget for ensemble (constant 0.05).
    /// Populated by `FusedTrainingCtx::compute_adaptive_budgets`.
    pub(crate) last_ens_budget_eff: f32,
    /// SP6 Pearl 2: per-branch budget arrays — set by compute_adaptive_budgets.
    /// Index order: dir=0, mag=1, ord=2, urg=3.
    ///
    /// SP7 (2026-05-03): the CQL/C51 caches were removed because the
    /// host-side dispatch they relied on was the source of the capture-time
    /// freeze (audit Fix 33). HEALTH_DIAG now reads the GPU-resolved values
    /// from `lb_budget_effective_buf` via `FusedTrainingCtx::last_{cql,c51}_budget_per_branch`.
    pub(crate) last_iqn_budget_per_branch: [f32; 4],
    pub(crate) last_ens_budget_per_branch: [f32; 4],

    /// D1/N1: Ring buffer of best-health weight snapshots for temporal self-distillation.
    pub(crate) q_snapshots: crate::cuda_pipeline::q_snapshot::SnapshotRing,
    /// D1/N1: Whether distillation gradient was applied in the most recent epoch boundary.
    pub(crate) last_distill_active: bool,
    /// F4/F5: Meta-Q collapse prediction cached from the DQN trainer each epoch
    /// boundary. Consumed by aux-op kernels to amplify barrier / IB gradient
    /// when imminent collapse is predicted (proactive defense).
    pub(crate) last_meta_q_pred: f32,

    /// F3: Rolling buffer of the last up-to-64 Q-value samples (each `total_actions` floats)
    /// read from q_readback_pinned[7..]. Used by `compute_q_spectral_gap` to estimate
    /// sigma_1/sigma_2 via Gram power iteration. Capped at 64 samples × 12 floats = 3 KB.
    pub(crate) q_sample_history: std::collections::VecDeque<Vec<f32>>,
}

impl GpuDqnTrainer {
    /// Reset Adam optimizer state for a new walk-forward fold.
    /// Zeroes momentum (m), variance (v), and step counter (t).
    /// Weights are preserved (warm-start for the new fold).
    pub fn reset_adam_state(&mut self) -> Result<(), MLError> {
        // R1 (post-bkdx5 + ISV-design correction): eliminated K's hardcoded
        // shrink ratios. K was a stepping-stone fix introduced before
        // fold_warmup_factor (commit 4ef1d8ebb) existed. fold_warmup_factor
        // already dampens lr+clip via ISV-driven ramp during the first ~50
        // post-reset steps, which IS the principled first-step protection.
        //
        // K's hardcoded shrink ratios (m×0.1, v×0.01) violated
        // feedback_adaptive_not_tuned (untracked tunable knobs) AND created
        // a downstream pathology: tiny v_hat denominator → oversized Adam
        // updates 50+ steps post-reset → trunk param overshoot → save_h_s2
        // NaN at F1 ~step 1745 (smoke-test-bkdx5 diagnostic).
        //
        // With K removed: Adam restarts at m=0, v=0 (standard fold reset);
        // fold_warmup_factor caps effective lr to MIN_WARMUP_LR_FRAC ≈ 0.05
        // of base for the first cold-start window → bounded effective updates
        // → no overshoot. The two mechanisms attacking the same failure mode
        // were over-engineered; warmup_factor subsumes K cleanly.
        //
        // `t_pinned` (Adam step counter) is still zeroed so bias correction
        // `1 - β^t` restarts cleanly for the new fold's first steps.
        self.stream.memset_zeros(&mut self.m_buf)
            .map_err(|e| MLError::ModelError(format!("m_buf reset: {e}")))?;
        self.stream.memset_zeros(&mut self.v_buf)
            .map_err(|e| MLError::ModelError(format!("v_buf reset: {e}")))?;
        unsafe { *self.t_pinned = 0; }
        self.adam_step = 0;

        // Also reset IQN trunk Adam state (currently unused but allocated).
        self.stream.memset_zeros(&mut self.iqn_trunk_m)
            .map_err(|e| MLError::ModelError(format!("reset iqn m: {e}")))?;
        self.stream.memset_zeros(&mut self.iqn_trunk_v)
            .map_err(|e| MLError::ModelError(format!("reset iqn v: {e}")))?;
        self.stream.memset_zeros(&mut self.iqn_trunk_t_buf)
            .map_err(|e| MLError::ModelError(format!("reset iqn t: {e}")))?;
        self.iqn_trunk_adam_step = 0;

        // Auxiliary Adam states — reset alongside the main Adam state so
        // every per-fold optimizer enters fold N+1 with fresh momentum.
        // Cross-Branch Q-Attention, selectivity gate, diffusion denoiser,
        // Mamba2 update, and OFI embed all carry their own (m, v, step)
        // tuple separate from the main `m_buf`/`v_buf`. Without this
        // reset the auxiliary optimizers enter fold N+1 with fold N's
        // momentum, producing oversized Adam steps in the first epochs
        // whose effect compounds through the corresponding backward path.
        // Closes the partial-refactor gap diagnosed in the fold-boundary
        // audit (companion to the IQN target sync + IQN Adam reset).
        self.stream.memset_zeros(&mut self.q_attn_adam_m)
            .map_err(|e| MLError::ModelError(format!("reset q_attn_adam_m: {e}")))?;
        self.stream.memset_zeros(&mut self.q_attn_adam_v)
            .map_err(|e| MLError::ModelError(format!("reset q_attn_adam_v: {e}")))?;
        self.q_attn_adam_step = 0;

        self.stream.memset_zeros(&mut self.sel_adam_m)
            .map_err(|e| MLError::ModelError(format!("reset sel_adam_m: {e}")))?;
        self.stream.memset_zeros(&mut self.sel_adam_v)
            .map_err(|e| MLError::ModelError(format!("reset sel_adam_v: {e}")))?;
        self.sel_adam_step = 0;

        self.stream.memset_zeros(&mut self.denoise_adam_m)
            .map_err(|e| MLError::ModelError(format!("reset denoise_adam_m: {e}")))?;
        self.stream.memset_zeros(&mut self.denoise_adam_v)
            .map_err(|e| MLError::ModelError(format!("reset denoise_adam_v: {e}")))?;
        self.denoise_adam_step = 0;

        self.stream.memset_zeros(&mut self.mamba2_adam_m)
            .map_err(|e| MLError::ModelError(format!("reset mamba2_adam_m: {e}")))?;
        self.stream.memset_zeros(&mut self.mamba2_adam_v)
            .map_err(|e| MLError::ModelError(format!("reset mamba2_adam_v: {e}")))?;
        self.mamba2_adam_step = 0;

        self.stream.memset_zeros(&mut self.ofi_embed_adam_m)
            .map_err(|e| MLError::ModelError(format!("reset ofi_embed_adam_m: {e}")))?;
        self.stream.memset_zeros(&mut self.ofi_embed_adam_v)
            .map_err(|e| MLError::ModelError(format!("reset ofi_embed_adam_v: {e}")))?;
        self.ofi_embed_adam_step = 0;

        // Reset PopArt running statistics — prevents fold 1's reward distribution
        // from contaminating fold 2's normalization.
        self.stream.memset_zeros(&mut self.popart_mean)
            .map_err(|e| MLError::ModelError(format!("reset popart_mean: {e}")))?;
        self.stream.memset_zeros(&mut self.popart_var)
            .map_err(|e| MLError::ModelError(format!("reset popart_var: {e}")))?;
        self.stream.memset_zeros(&mut self.popart_count)
            .map_err(|e| MLError::ModelError(format!("reset popart_count: {e}")))?;

        // Keep adaptive gradient clip EMA across folds — gradient scale is a property
        // of the model architecture and loss function, not the data window.
        // Resetting it would leave the first fold-2 epochs unprotected.

        tracing::info!("Adam optimizer + PopArt + grad_clip state reset for new fold");
        Ok(())
    }

    /// Reset Adam momentum (m/v) for branch weights only (indices 8-49).
    /// Trunk momentum (indices 0-7) preserved — carries valuable convergence history.
    /// Lighter than full rewind — tries to break Adam fixed point without changing weights.
    pub(crate) fn reset_branch_adam_momentum(&mut self) -> Result<(), MLError> {
        let param_sizes = compute_param_sizes(&self.config);
        // Start offset: index 8 (w_b0fc)
        let start_byte = padded_byte_offset(&param_sizes, 8);
        // End offset: after last branch weight (index 49, kan_resid_3)
        let end_byte = padded_byte_offset(&param_sizes, 50); // one past last
        let range_bytes = (end_byte - start_byte) as usize;

        // Zero the m_buf and v_buf ranges for branch weights only
        let m_ptr = self.m_buf.raw_ptr() + start_byte;
        let v_ptr = self.v_buf.raw_ptr() + start_byte;
        unsafe {
            cudarc::driver::sys::cuMemsetD32Async(m_ptr, 0, range_bytes / 4, self.stream.cu_stream());
            cudarc::driver::sys::cuMemsetD32Async(v_ptr, 0, range_bytes / 4, self.stream.cu_stream());
        }
        Ok(())
    }

    /// Scale Adam momentum buffers (m_buf) by `scale` — used for warm restart momentum decay.
    /// Preserves gradient direction but reduces magnitude to prevent Adam fixed point.
    pub(crate) fn scale_adam_momentum(&mut self, scale: f32) -> Result<(), MLError> {
        let cfg = {
            let blocks = ((self.total_params + 255) / 256) as u32;
            LaunchConfig { grid_dim: (blocks, 1, 1), block_dim: (256, 1, 1), shared_mem_bytes: 0 }
        };
        let n = self.total_params as i32;
        let m_ptr = self.m_buf.raw_ptr();
        // Use scale_f32_ungraphed — called at cosine LR warm restarts (after graph capture).
        // scale_f32_kernel is captured in forward_child; launching from the same CUmodule
        // ungraphed corrupts the child graph's kernel state on Hopper (3100ms replay).
        unsafe {
            self.stream.launch_builder(&self.scale_f32_ungraphed)
                .arg(&m_ptr)
                .arg(&scale)
                .arg(&n)
                .launch(cfg)
                .map_err(|e| MLError::ModelError(format!("scale_adam_momentum: {e}")))?;
        }
        Ok(())
    }

    /// Set learning rate for Adam optimizer.
    /// Writes to pinned device-mapped memory — no CUDA graph recapture needed.
    /// The CUDA graph captured lr_dev_ptr (the pointer), not the LR value.
    /// The kernel reads *lr_ptr at replay time, so the new value takes effect immediately.
    pub(crate) fn set_lr(&mut self, lr: f32) {
        self.config.lr = lr;
        unsafe { *self.lr_pinned = lr; }
        // NO graph invalidation — lr_dev_ptr is captured, value is read at replay time.
    }

    /// Plan 1 Task 9: GPU-driven atom position recompute (all 4 branches, single launch).
    ///
    /// Dispatches `atoms_update` kernel with grid=(4,1,1), block=(256,1,1).
    /// Each block handles one branch; reads ISV v-range slots [23..31) and
    /// spacing_raw params; writes atom_positions_buf in-place. Replaces the
    /// CPU-side loop over `adaptive_atom_positions` per branch. Must be called
    /// once per epoch boundary and after each `step_atom_positions` SGD update.
    pub(crate) fn launch_atoms_update(&self) -> Result<(), MLError> {
        let na = self.config.num_atoms;
        let param_sizes = compute_param_sizes(&self.config);
        let shmem = (na * 2) * std::mem::size_of::<f32>();
        let positions_out = self.atom_positions_buf.raw_ptr();
        let isv_dev_ptr   = self.isv_signals_dev_ptr;

        let sp_b0 = self.ptrs.params_ptr + padded_byte_offset(&param_sizes, 52);
        let sp_b1 = self.ptrs.params_ptr + padded_byte_offset(&param_sizes, 53);
        let sp_b2 = self.ptrs.params_ptr + padded_byte_offset(&param_sizes, 54);
        let sp_b3 = self.ptrs.params_ptr + padded_byte_offset(&param_sizes, 55);
        let na_i32 = na as i32;

        unsafe {
            self.stream
                .launch_builder(&self.atoms_update_kernel)
                .arg(&sp_b0)
                .arg(&sp_b1)
                .arg(&sp_b2)
                .arg(&sp_b3)
                .arg(&positions_out)
                .arg(&na_i32)
                .arg(&isv_dev_ptr)
                .launch(LaunchConfig {
                    grid_dim: (4, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: shmem as u32,
                })
                .map_err(|e| MLError::ModelError(format!("atoms_update launch: {e}")))?;
        }
        Ok(())
    }

    /// SGD update on adaptive atom spacing_raw parameters.
    /// Maximizes atom entropy (gradient ascent) — concentrates atoms where mass exists.
    /// Uses scale_f32_kernel to decay spacing_raw toward zero: softmax(0,...,0) = uniform.
    /// Called every 50 training steps after Q-stats.
    pub(crate) fn step_atom_positions(&mut self, step: usize) -> Result<(), MLError> {
        if step % 50 != 0 { return Ok(()); }

        let na = self.config.num_atoms;
        let param_sizes = compute_param_sizes(&self.config);
        let lr: f32 = 1e-3;
        let scale = 1.0_f32 - lr;
        let n = na as i32;
        let blocks = ((na as u32 + 255) / 256).max(1);

        for branch in 0..4_usize {
            let offset = padded_byte_offset(&param_sizes, 52 + branch);
            let param_ptr = self.ptrs.params_ptr + offset;
            // Decay spacing_raw toward zero → softmax→uniform (entropy maximization)
            // scale_f32_kernel: ptr[i] *= scale
            unsafe {
                self.stream.launch_builder(&self.scale_f32_kernel)
                    .arg(&param_ptr)
                    .arg(&scale)
                    .arg(&n)
                    .launch(LaunchConfig {
                        grid_dim: (blocks, 1, 1),
                        block_dim: (256, 1, 1),
                        shared_mem_bytes: 0,
                    })
                    .map_err(|e| MLError::ModelError(format!("atom_position_sgd branch {branch}: {e}")))?;
            }
        }
        // Recompute positions from updated spacing_raw via GPU atoms_update kernel.
        self.launch_atoms_update()?;
        Ok(())
    }

    // ── Homeostatic regularizer (G16) ────────────────────────────────────

    /// Update observables from current training state (CPU write to pinned).
    /// Slots: [0]=Q-mean, [1]=atom_util, [2..5] set by caller if available.
    ///
    /// SP4 Layer C close-out C3 (2026-05-01): slot [1] is now written by
    /// `update_utilization_ema_kernel` in lockstep with the EMA itself
    /// (see `launch_update_utilization_ema`). The host code retains
    /// responsibility only for slot [0] (Q-mean from
    /// `q_mean_scratch_pinned`); slots [2..5] remain the caller's
    /// responsibility.
    pub(crate) fn update_homeostatic_observables(&self) {
        unsafe {
            let obs = self.homeostatic_obs_pinned;
            *obs.add(0) = *self.q_mean_scratch_pinned;  // Q-mean
            // Slot [1] (atom utilization) — populated GPU-side by
            // `update_utilization_ema_kernel` to keep EMA storage and
            // homeostatic-observable mirror in lockstep without host compute.
            // branch_corr [2], trade_freq [3], Q-gap [4], Q-var [5]
            // are written by caller or left at previous value.
        }
    }

    /// Continuously adaptive homeostatic targets — driven by model readiness, not epoch.
    /// Low readiness (exploring): fast alpha=0.3 (targets track rapidly, model still discovering healthy range).
    /// High readiness (converged): slow alpha=0.01 (targets stable, only drift with gradual shifts).
    /// alpha = 0.3 * (1 - readiness) + 0.01 * readiness → smooth interpolation.
    /// Q-mean target always 0.0 (centered Q-values are the invariant).
    ///
    /// SP4 Layer C close-out follow-up (2026-05-01): the per-slot EMA arithmetic
    /// + Q-mean invariant assignment now run GPU-side via
    /// `calibrate_homeostatic_kernel` per `feedback_no_cpu_compute_strict`.
    /// Single-block, six threads — one per homeostatic slot. Reads host-passed
    /// `readiness` scalar (already-clamped IQN gauge) plus
    /// `homeostatic_obs_dev_ptr` (mapped-pinned observations); writes
    /// `homeostatic_targets_pinned[k]` in-place via
    /// `homeostatic_targets_dev_ptr`. `__threadfence_system()` after the
    /// writes guarantees PCIe-visibility for the homeostatic_regularizer
    /// kernel's subsequent dev_ptr reads. Caller-compat preserved: same α
    /// formula, same Q-mean=0 invariant, same call ordering — pure
    /// architectural fix.
    pub(crate) fn calibrate_homeostatic_targets(&mut self) {
        if let Err(e) = self.launch_calibrate_homeostatic() {
            tracing::warn!("calibrate_homeostatic_targets launch failed: {e}");
        }
    }

    /// SP4 Layer C close-out follow-up (2026-05-01): GPU-only readiness-driven
    /// homeostatic-target EMA launcher. Single-block, six threads — one thread
    /// per homeostatic slot. Caller-compat: the EMA storage retains its
    /// mapped-pinned shape so `homeostatic_kernel` continues to read
    /// `homeostatic_targets_dev_ptr` unchanged. Chained on the trainer's
    /// stream so the launch is graph-capture-compatible.
    fn launch_calibrate_homeostatic(&self) -> Result<(), MLError> {
        let readiness = self.iqn_readiness.clamp(0.0, 1.0);
        let obs_dev = self.homeostatic_obs_dev_ptr;
        let targets_dev = self.homeostatic_targets_dev_ptr;

        let cfg = LaunchConfig {
            grid_dim:         (1, 1, 1),
            block_dim:        (HOMEOSTATIC_N_OBS as u32, 1, 1),
            shared_mem_bytes: 0,
        };

        // Safety: kernel signature `(float, const float*, float*)` matches
        // the three args below; both dev_ptrs are mapped-pinned and valid for
        // the lifetime of `self`. HOMEOSTATIC_N_OBS=6 threads, one per slot.
        unsafe {
            self.stream
                .launch_builder(&self.calibrate_homeostatic_kernel)
                .arg(&readiness)
                .arg(&obs_dev)
                .arg(&targets_dev)
                .launch(cfg)
                .map_err(|e| MLError::ModelError(format!("calibrate_homeostatic_kernel: {e}")))?;
        }
        Ok(())
    }

    /// Run homeostatic regularizer kernel.
    /// Computes per-observable penalties and a clamped total penalty scalar.
    pub(crate) fn apply_homeostatic_regularization(&self) -> Result<(), MLError> {
        // Adaptive lambda: scale inversely with reward magnitude so penalty
        // is proportionally effective regardless of reward scale.
        // Trade-level reward (std~0.02) needs ~10× stronger lambda than per-bar (std~7.0).
        // lambda = 0.01 / max(reward_std_proxy, 0.01) where proxy = Q-value range / 10.
        let q_range = unsafe {
            let obs = self.homeostatic_obs_pinned;
            // Use atom utilization as a proxy for Q-value health (already tracked)
            // Simpler: use a fixed adaptive scale based on Q-gap (obs[4])
            let q_gap = (*obs.add(4)).abs().max(0.1);
            0.01_f32 / q_gap.min(10.0)
        };
        let lambda_base = q_range.clamp(0.005, 0.5);
        let budget_max = lambda_base * 50.0; // budget scales with lambda

        // Zero total_penalty before reduction
        let homeostatic_total_buf_ptr = self.homeostatic_total_buf.raw_ptr();
        let homeostatic_penalties_buf_ptr = self.homeostatic_penalties_buf.raw_ptr();
        unsafe {
            cudarc::driver::sys::cuMemsetD32Async(
                homeostatic_total_buf_ptr, 0, 1, self.stream.cu_stream(),
            );
        }
        unsafe {
            self.stream.launch_builder(&self.homeostatic_kernel)
                .arg(&self.homeostatic_obs_dev_ptr)
                .arg(&self.homeostatic_targets_dev_ptr)
                .arg(&homeostatic_penalties_buf_ptr)
                .arg(&homeostatic_total_buf_ptr)
                .arg(&(HOMEOSTATIC_N_OBS as i32))
                .arg(&lambda_base)
                .arg(&budget_max)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (1, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("homeostatic_regularizer: {e}")))?;
        }
        Ok(())
    }
}

impl Drop for GpuDqnTrainer {
    fn drop(&mut self) {
        // Synchronize stream BEFORE CudaSlice fields drop.
        unsafe { cudarc::driver::sys::cuStreamSynchronize(self.stream.cu_stream()); }
        // Destroy eval forward exec if set (not owned by cudarc).
        if let Some(exec) = self.eval_forward_exec.take() {
            unsafe { cudarc::driver::sys::cuGraphExecDestroy(exec); }
        }
        // Free pinned host memory
        if !self.readback_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.readback_pinned.cast()) };
        }
        if !self.lr_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.lr_pinned.cast()) };
        }
        if !self.aux_lr_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.aux_lr_pinned.cast()) };
        }
        if !self.adaptive_clip_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.adaptive_clip_pinned.cast()) };
        }
        // Plan C Phase 2 follow-up K (2026-04-29): grad-norm fast/slow EMA
        // mapped-pinned scalars feeding `fold_warmup_factor_update`.
        if !self.grad_norm_fast_ema_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.grad_norm_fast_ema_pinned.cast()) };
        }
        if !self.grad_norm_slow_ema_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.grad_norm_slow_ema_pinned.cast()) };
        }
        if !self.sel_t_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.sel_t_pinned.cast()) };
        }
        if !self.t_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.t_pinned.cast()) };
        }
        if !self.tau_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.tau_pinned.cast()) };
        }
        if !self.total_loss_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.total_loss_pinned.cast()) };
        }
        if !self.mse_loss_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.mse_loss_pinned.cast()) };
        }
        if !self.grad_norm_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.grad_norm_pinned.cast()) };
        }
        if !self.eval_v_range_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.eval_v_range_pinned.cast()) };
        }
        // Task 0.4 — pinned grad-readback buffer.
        if self.grad_readback_pinned_ptr != 0 {
            let _ = unsafe {
                cudarc::driver::result::free_host(self.grad_readback_pinned_ptr as *mut std::ffi::c_void)
            };
        }
        // Task 2.0 — pinned device-mapped grad-component result slot.
        if !self.grad_decomp_result_pinned.is_null() {
            let _ = unsafe {
                cudarc::driver::result::free_host(self.grad_decomp_result_pinned.cast())
            };
        }
        if !self.per_branch_q_gaps_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.per_branch_q_gaps_pinned.cast()) };
        }
        if !self.q_readback_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.q_readback_pinned.cast()) };
        }
        if !self.per_branch_q_stats_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.per_branch_q_stats_pinned.cast()) };
        }
        if !self.iqn_readiness_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.iqn_readiness_pinned.cast()) };
        }
        if !self.iqn_loss_initial_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.iqn_loss_initial_pinned.cast()) };
        }
        if !self.iqn_loss_ema_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.iqn_loss_ema_pinned.cast()) };
        }
        if !self.utilization_ema_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.utilization_ema_pinned.cast()) };
        }
        if !self.grad_norm_ema_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.grad_norm_ema_pinned.cast()) };
        }
        if !self.outlier_diag_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.outlier_diag_pinned.cast()) };
        }
        if !self.td_error_scratch_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.td_error_scratch_pinned.cast()) };
        }
        if !self.q_var_scratch_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.q_var_scratch_pinned.cast()) };
        }
        if !self.reward_scratch_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.reward_scratch_pinned.cast()) };
        }
        if !self.atom_util_scratch_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.atom_util_scratch_pinned.cast()) };
        }
        if !self.q_mag_means_scratch_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.q_mag_means_scratch_pinned.cast()) };
        }
        if !self.q_abs_ref_scratch_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.q_abs_ref_scratch_pinned.cast()) };
        }
        if !self.q_dir_means_scratch_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.q_dir_means_scratch_pinned.cast()) };
        }
        if !self.q_dir_abs_ref_scratch_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.q_dir_abs_ref_scratch_pinned.cast()) };
        }
        if !self.isv_signals_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.isv_signals_pinned.cast()) };
        }
        if !self.isv_history_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.isv_history_pinned.cast()) };
        }
        if !self.isv_decay_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.isv_decay_pinned.cast()) };
        }
        if !self.lagged_td_error_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.lagged_td_error_pinned.cast()) };
        }
        if !self.ofi_embed_t_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.ofi_embed_t_pinned.cast()) };
        }
        // SP14 Layer C Phase C.5a (2026-05-08): aux trunk Adam pinned ptrs.
        if !self.aux_trunk_grad_clip_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.aux_trunk_grad_clip_pinned.cast()) };
        }
        if !self.aux_trunk_lr_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.aux_trunk_lr_pinned.cast()) };
        }
        if !self.aux_trunk_t_pinned.is_null() {
            let _ = unsafe { cudarc::driver::result::free_host(self.aux_trunk_t_pinned.cast()) };
        }
    }
}

/// SP16 Phase 0 — pure-host helper that averages the four direction-branch
/// columns of a `[B, total_actions]` row-major Q-out buffer and returns the
/// per-action means in HEALTH_DIAG emit order `[q_hold, q_long, q_short, q_flat]`.
///
/// Factored out of `update_q_dir_means_cached` so the averaging logic can be
/// exercised by an oracle test (`sp14_oracle_tests::sp16_phase0_*`) without
/// constructing a full `GpuDqnTrainer` + dtoh harness. The production updater
/// calls this helper after the dtoh — the test seeds `host` directly with
/// known values and asserts the returned slot order matches the canonical
/// action enum.
///
/// Action-index mapping (canonical, see `state_layout.cuh:123-126`):
/// DIR_SHORT=0, DIR_HOLD=1, DIR_LONG=2, DIR_FLAT=3.
///
/// Returned slot order: `[hold, long, short, flat]` (same as the emit format).
///
/// Empty / degenerate input yields `[0.0; 4]`. `b0 < 4` is treated honestly:
/// out-of-range action indices fall back to slot 0 (matches the
/// `update_q_mag_means_cached` `b1.min(3)` precedent — preserves layout
/// invariants when the runtime config narrows a branch). Production always
/// has `b0 = NUM_DIRECTIONS = 4`.
pub fn compute_q_dir_means_from_host_buf(
    host: &[f32],
    b: usize,
    b0: usize,
    total_actions: usize,
) -> [f32; 4] {
    if b == 0 || b0 == 0 || total_actions == 0 {
        return [0.0_f32; 4];
    }
    if host.len() < b * total_actions {
        return [0.0_f32; 4];
    }
    // q_out_buf layout is [B, total_actions] row-major. Direction-branch
    // columns live at [0 .. b0]. Accumulate per-column in f64 to keep the
    // mean numerically clean across large batches, then cast to f32.
    let mut sums = [0.0_f64; 4];
    let cap = b0.min(4);
    for i in 0..b {
        let row_off = i * total_actions;
        for k in 0..cap {
            sums[k] += host[row_off + k] as f64;
        }
    }
    let inv_b = 1.0_f64 / b as f64;
    // Map kernel-side dir_idx → HEALTH_DIAG slot.
    //   dir_idx 0 = DIR_SHORT → slot 2
    //   dir_idx 1 = DIR_HOLD  → slot 0
    //   dir_idx 2 = DIR_LONG  → slot 1
    //   dir_idx 3 = DIR_FLAT  → slot 3
    let mean_short = (sums[0] * inv_b) as f32;
    let mean_hold  = (sums[1.min(b0 - 1)] * inv_b) as f32;
    let mean_long  = (sums[2.min(b0 - 1)] * inv_b) as f32;
    let mean_flat  = (sums[3.min(b0 - 1)] * inv_b) as f32;
    [mean_hold, mean_long, mean_short, mean_flat]
}

impl GpuDqnTrainer {
    /// Configured batch size (fixed at construction for CUDA Graph compatibility).
    pub fn batch_size(&self) -> usize {
        self.config.batch_size
    }

    pub fn eval_v_range_ptr(&self) -> u64 { self.eval_v_range_ptr }

    /// Device pointer to total_loss (pinned device-mapped).
    pub fn total_loss_dev_ptr(&self) -> u64 { self.total_loss_dev_ptr }
    /// Device pointer to mse_loss (pinned device-mapped).
    pub fn mse_loss_dev_ptr(&self) -> u64 { self.mse_loss_dev_ptr }
    /// Device pointer to grad_norm (pinned device-mapped).
    pub fn grad_norm_dev_ptr(&self) -> u64 { self.grad_norm_dev_ptr }

    /// Task 0.4 — per-action-branch L2 grad norm.
    ///
    /// Returns `[direction, magnitude, order, urgency]` L2 norms over each
    /// branch's 4 weight tensors (fc weights, fc bias, out weights, out bias).
    /// Branch 0 starts at param tensor index 8, each branch spans 4 indices.
    ///
    /// Implementation: single `memcpy_dtoh` of the full grad buffer into a
    /// **pinned** host slot allocated once at construction. Pinned host memory
    /// gives PCIe-bandwidth-limited transfer (~0.5 ms for ~4 MB) instead of
    /// pageable-memory copy. Per-branch norm reductions then run on host —
    /// 4 small slice-sum-of-squares loops, sub-microsecond each.
    ///
    /// Called once per epoch from HEALTH_DIAG — outside the training graph
    /// capture region, so the readback cost is acceptable.
    pub fn per_branch_grad_norms(&self) -> Result<[f32; 4], MLError> {
        if self.grad_readback_pinned_ptr == 0 {
            return Ok([0.0_f32; 4]);
        }
        // Read only the `total_params` prefix of grad_buf — the trailing
        // `cutlass_tile_pad` slots are GEMM padding, never hold param gradients,
        // and were never sized into `grad_readback_pinned_capacity`. The
        // original check compared `grad_buf.len()` (= total_params + tile_pad)
        // against the pinned capacity (= total_params) and returned Err — which
        // the fused proxy silently coerced to 0.0, so both
        // `grad_ratio_mag_dir` and `grad_norms_dir_mag_abs` reported zeros for
        // every epoch instead of real per-branch norms. Readback is bounded to
        // `total_params` which matches the pinned capacity exactly.
        let read_len = self.total_params;
        if read_len > self.grad_readback_pinned_capacity {
            return Err(MLError::ModelError(format!(
                "per_branch_grad_norms: total_params {} exceeds pinned readback capacity {}",
                read_len, self.grad_readback_pinned_capacity
            )));
        }
        // Stream-sync to ensure backward + Adam grad accumulation has completed
        // before reading. Acceptable cost — runs once per epoch.
        self.stream.synchronize().map_err(|e| {
            MLError::ModelError(format!("per_branch_grad_norms sync: {e}"))
        })?;
        // Reinterpret pinned host slot as &mut [f32].
        let host_slice: &mut [f32] = unsafe {
            std::slice::from_raw_parts_mut(self.grad_readback_pinned_ptr as *mut f32, read_len)
        };
        self.stream
            .memcpy_dtoh(&self.grad_buf.slice(..read_len), host_slice)
            .map_err(|e| MLError::ModelError(format!("per_branch_grad_norms dtoh: {e}")))?;
        // `memcpy_dtoh` into PINNED host memory is asynchronous w.r.t. the
        // host — cudarc calls `cuMemcpyDtoHAsync_v2` under the hood, which
        // queues a DMA on `self.stream` and returns before the transfer
        // completes. Without an explicit post-copy sync the host-side
        // sum-of-squares loop below races the DMA and reads a mix of
        // newly-transferred bytes and stale bytes left over from the
        // previous epoch's readback. That DMA-in-progress race is the
        // source of the residual `grad_abs[dir]` non-determinism: the
        // direction gradient itself is bit-identical across runs (same
        // CUDA kernels, same per-stream cuBLAS handles post-Option-C, same
        // `CUBLAS_WORKSPACE_CONFIG`), but the aggregate L2 accumulator
        // sums whatever bytes happen to already be in pinned memory when
        // the host-side loop reaches each element. Magnitude tolerates the
        // same race because mag gradients are ~300× larger than dir (the
        // trunk-to-mag-branch path carries most of the Q-loss signal), so
        // a few partially-updated bytes contribute a negligible fraction
        // of the L2 sum; dir's small magnitude makes it proportionally
        // more sensitive. Sync after the copy so both norms read the
        // final transferred bytes.
        self.stream.synchronize().map_err(|e| {
            MLError::ModelError(format!("per_branch_grad_norms post-dtoh sync: {e}"))
        })?;

        let param_sizes = compute_param_sizes(&self.config);
        let mut norms = [0.0_f32; 4];
        for branch in 0..4_usize {
            let first_tensor = 8 + branch * 4;       // 8, 12, 16, 20
            let last_tensor = first_tensor + 4;       // exclusive
            let mut sum_sq = 0.0_f64;
            for tensor_idx in first_tensor..last_tensor {
                let start_byte = padded_byte_offset(&param_sizes, tensor_idx);
                let start_idx = (start_byte as usize) / std::mem::size_of::<f32>();
                let len = param_sizes[tensor_idx];
                let end_idx = (start_idx + len).min(host_slice.len());
                for i in start_idx..end_idx {
                    let g = host_slice[i] as f64;
                    sum_sq += g * g;
                }
            }
            norms[branch] = sum_sq.sqrt() as f32;
        }
        Ok(norms)
    }

    /// Task 0.4 — direction grad norm divided by magnitude grad norm.
    /// Returns 0.0 if either norm is zero. H4 detection signal.
    pub fn grad_ratio_mag_dir(&self) -> Result<f32, MLError> {
        let n = self.per_branch_grad_norms()?;
        let dir = n[0];
        let mag = n[1];
        Ok(if dir > 1e-9 { mag / dir } else { 0.0 })
    }

    /// Absolute per-branch grad L2 norms, not ratios. Returns `(dir, mag)`.
    ///
    /// Task 2.0 confirmation accessor — `grad_ratio_mag_dir` collapses to 0.0
    /// whenever `dir < 1e-9`, which masks whether direction is starved
    /// (tiny but non-zero dir) vs magnitude is zero. Exposing the raw norms
    /// disambiguates the two. Zero here means genuine near-float-precision
    /// — don't coerce via threshold.
    pub fn grad_norms_dir_mag_abs(&self) -> Result<(f32, f32), MLError> {
        let n = self.per_branch_grad_norms()?;
        Ok((n[0], n[1]))  // dir, mag
    }

    /// Task 2.0 — in-graph DtoD snapshot of `grad_buf`'s trunk + branch 0+1
    /// slices into the caller-supplied scratch buffer. Uses the `copy_f32`
    /// kernel (graph-captured DtoD) because `cuMemcpyDtoDAsync` is NOT
    /// captured by CUDA Graph (see `graph_safe_copy_f32` doc comment).
    ///
    /// Two non-contiguous ranges in `grad_buf` are copied into one dst
    /// buffer laid out as `[trunk | dir | mag]` (matching the kernel's
    /// snapshot layout convention documented on
    /// `grad_component_delta_norm`):
    ///   * Trunk slice (tensors 0..4 — w_s1, b_s1, w_s2, b_s2) →
    ///     `dst[0 .. trunk_len)` (Task 2.Z diagnostic — captures IQN +
    ///     Ens contributions that SAXPY exclusively into the trunk).
    ///   * Branch slice (tensors 8..16 — direction + magnitude heads) →
    ///     `dst[trunk_len .. trunk_len + dir_len + mag_len)`. The branch
    ///     slice is a single contiguous element range in `grad_buf` —
    ///     tensors 12..16 (magnitude) directly follow 8..12 (direction)
    ///     in the flat parameter layout per `compute_param_sizes`.
    pub(crate) fn grad_decomp_snapshot(&self, dst: &CudaSlice<f32>, ctx: &str)
        -> Result<(), MLError>
    {
        let elem_size = std::mem::size_of::<f32>() as u64;
        // ── Trunk copy (tensors 0..4) → dst[0..trunk_len) ─────────────────
        {
            let src_ptr = self.ptrs.grad_buf + (self.grad_decomp_trunk_start as u64) * elem_size;
            let dst_ptr = dst.raw_ptr();
            let n_bytes = (self.grad_decomp_trunk_len as usize) * (elem_size as usize);
            self.graph_safe_copy_f32(dst_ptr, src_ptr, n_bytes, ctx)?;
        }
        // ── Branch copy (tensors 8..16, contiguous dir+mag) → dst[trunk..] ─
        {
            let src_ptr = self.ptrs.grad_buf + (self.grad_decomp_dir_start as u64) * elem_size;
            let dst_ptr = dst.raw_ptr() + (self.grad_decomp_trunk_len as u64) * elem_size;
            let branch_elems = (self.grad_decomp_dir_len + self.grad_decomp_mag_len) as usize;
            let n_bytes = branch_elems * (elem_size as usize);
            self.graph_safe_copy_f32(dst_ptr, src_ptr, n_bytes, ctx)?;
        }
        Ok(())
    }

    /// Task 2.0 — launch `grad_component_delta_norm` on a per-component
    /// snapshot, writing (mag_norm, dir_norm, trunk_norm) into the pinned
    /// result slot at the component's 3-float offset.
    ///
    /// `slot_offset_elems` is the element offset into the 27-float pinned
    /// slot — 0=iqn, 3=cql, 6=cql_sx, 9=c51, 12=c51_bs, 15=distill, 18=rec,
    /// 21=pred, 24=ens (Task 2.Z extension: 3 floats per component; matches
    /// pinned-layout convention documented on `grad_decomp_result_pinned`).
    pub(crate) fn launch_grad_decomp(
        &self,
        snapshot: &CudaSlice<f32>,
        slot_offset_elems: i32,
        ctx: &str,
    ) -> Result<(), MLError> {
        let current_ptr = self.ptrs.grad_buf;
        let snapshot_ptr = snapshot.raw_ptr();
        let result_ptr = self.grad_decomp_result_dev_ptr
            + (slot_offset_elems as u64) * (std::mem::size_of::<f32>() as u64);
        let cfg = LaunchConfig {
            grid_dim: (1, 1, 1),
            block_dim: (256, 1, 1),
            shared_mem_bytes: 0,
        };
        unsafe {
            self.stream.launch_builder(&self.grad_decomp_kernel)
                .arg(&current_ptr)
                .arg(&snapshot_ptr)
                .arg(&self.grad_decomp_trunk_start)
                .arg(&self.grad_decomp_trunk_len)
                .arg(&self.grad_decomp_dir_start)
                .arg(&self.grad_decomp_dir_len)
                .arg(&self.grad_decomp_mag_start)
                .arg(&self.grad_decomp_mag_len)
                .arg(&result_ptr)
                .launch(cfg)
                .map_err(|e| MLError::ModelError(format!("grad_decomp {ctx}: {e}")))?;
        }
        Ok(())
    }

    /// Task 2.0 — convenience: snapshot `grad_buf` into the component's
    /// snapshot buffer (captured DtoD via `copy_f32` kernel). Called BEFORE
    /// the component's backward / scaling op fires.
    pub(crate) fn grad_decomp_snapshot_iqn(&self) -> Result<(), MLError> {
        self.grad_decomp_snapshot(&self.grad_snapshot_iqn, "grad_snapshot_iqn")
    }
    pub(crate) fn grad_decomp_snapshot_cql(&self) -> Result<(), MLError> {
        self.grad_decomp_snapshot(&self.grad_snapshot_cql, "grad_snapshot_cql")
    }
    /// Task 2.0 extension — snapshot before `apply_cql_saxpy` (isolates the
    /// SAXPY step from the CQL-logit-gradient + cuBLAS-backward portion of
    /// `apply_cql_gradient`, which writes into `cql_grad_scratch` not
    /// `grad_buf`).
    pub(crate) fn grad_decomp_snapshot_cql_sx(&self) -> Result<(), MLError> {
        self.grad_decomp_snapshot(&self.grad_snapshot_cql_sx, "grad_snapshot_cql_sx")
    }
    /// Task 2.0 extension — snapshot before `apply_c51_budget_scale`. The
    /// scale op multiplies `grad_buf` in place by `c51_budget`, so delta =
    /// current − snapshot exposes the amplitude change the scale applied
    /// (negative-signed on both dir and mag components when c51_budget < 1).
    pub(crate) fn grad_decomp_snapshot_c51_bs(&self) -> Result<(), MLError> {
        self.grad_decomp_snapshot(&self.grad_snapshot_c51_bs, "grad_snapshot_c51_bs")
    }
    /// Task 2.0 extension — snapshot before `apply_distillation_gradient`.
    pub(crate) fn grad_decomp_snapshot_distill(&self) -> Result<(), MLError> {
        self.grad_decomp_snapshot(&self.grad_snapshot_distill, "grad_snapshot_distill")
    }
    /// Task 2.0 extension — snapshot before
    /// `launch_recursive_confidence_backward`.
    pub(crate) fn grad_decomp_snapshot_rec(&self) -> Result<(), MLError> {
        self.grad_decomp_snapshot(&self.grad_snapshot_rec, "grad_snapshot_rec")
    }
    /// Task 2.0 extension — snapshot before `compute_predictive_coding_loss`.
    pub(crate) fn grad_decomp_snapshot_pred(&self) -> Result<(), MLError> {
        self.grad_decomp_snapshot(&self.grad_snapshot_pred, "grad_snapshot_pred")
    }
    pub(crate) fn grad_decomp_snapshot_ens(&self) -> Result<(), MLError> {
        self.grad_decomp_snapshot(&self.grad_snapshot_ens, "grad_snapshot_ens")
    }

    /// Task 2.0 — convenience: launch the reduction kernel for each component
    /// with its pre-configured snapshot buffer and pinned-slot offset.
    ///
    /// Pinned-slot offsets (3 floats per component — mag, dir, trunk —
    /// Task 2.Z extension):
    ///   iqn    =  0, cql    =  3, cql_sx  =  6,
    ///   c51    =  9, c51_bs = 12, distill = 15,
    ///   rec    = 18, pred   = 21, ens     = 24.
    pub(crate) fn grad_decomp_launch_iqn(&self) -> Result<(), MLError> {
        self.launch_grad_decomp(&self.grad_snapshot_iqn, 0, "iqn")
    }
    pub(crate) fn grad_decomp_launch_cql(&self) -> Result<(), MLError> {
        self.launch_grad_decomp(&self.grad_snapshot_cql, 3, "cql")
    }
    pub(crate) fn grad_decomp_launch_cql_sx(&self) -> Result<(), MLError> {
        self.launch_grad_decomp(&self.grad_snapshot_cql_sx, 6, "cql_sx")
    }
    /// C51 uses `grad_zero_ref_buf` (permanent zeros) because `grad_buf` is
    /// zero before `submit_forward_ops_main` — there is no pre-C51 snapshot.
    /// Kernel computes `‖grad_buf − 0‖` on trunk + branch 0+1 slices.
    pub(crate) fn grad_decomp_launch_c51(&self) -> Result<(), MLError> {
        self.launch_grad_decomp(&self.grad_zero_ref_buf, 9, "c51")
    }
    pub(crate) fn grad_decomp_launch_c51_bs(&self) -> Result<(), MLError> {
        self.launch_grad_decomp(&self.grad_snapshot_c51_bs, 12, "c51_bs")
    }
    pub(crate) fn grad_decomp_launch_distill(&self) -> Result<(), MLError> {
        self.launch_grad_decomp(&self.grad_snapshot_distill, 15, "distill")
    }
    pub(crate) fn grad_decomp_launch_rec(&self) -> Result<(), MLError> {
        self.launch_grad_decomp(&self.grad_snapshot_rec, 18, "rec")
    }
    pub(crate) fn grad_decomp_launch_pred(&self) -> Result<(), MLError> {
        self.launch_grad_decomp(&self.grad_snapshot_pred, 21, "pred")
    }
    pub(crate) fn grad_decomp_launch_ens(&self) -> Result<(), MLError> {
        self.launch_grad_decomp(&self.grad_snapshot_ens, 24, "ens")
    }

    /// SP7 Path A (2026-05-03) — launch `cql_raw_norm_compute` reading
    /// `cql_grad_scratch` directly. Writes `‖raw_cql‖` over (mag, dir,
    /// trunk) slices to `grad_decomp_result_pinned[3..6]` — the `cql`
    /// slot at element offset 3.
    ///
    /// Why a dedicated launcher rather than reusing `launch_grad_decomp`
    /// with the `grad_decomp_launch_cql` snapshot: that pattern produces
    /// `‖grad_buf − snapshot_cql‖`, which is structurally always 0
    /// because `apply_cql_gradient` writes to `cql_grad_scratch` (a
    /// separate buffer), never to `grad_buf`. The previously-defined
    /// `grad_decomp_launch_cql` is unwired (no call site); this launcher
    /// supplants it with a real producer that the SP7 controller can
    /// read at element offset 3 as a budget-independent reference signal.
    ///
    /// Must run AFTER `apply_cql_gradient` populates `cql_grad_scratch`
    /// and BEFORE `apply_cql_saxpy` consumes it (the saxpy doesn't
    /// modify `cql_grad_scratch`, but ordering keeps the contract
    /// "raw norm reflects what gets SAXPYed this step" obvious).
    pub(crate) fn launch_cql_raw_norm(&self) -> Result<(), MLError> {
        let scratch_ptr = self.cql_grad_scratch.raw_ptr();
        let f32_size = std::mem::size_of::<f32>() as u64;
        // Offset 3 (cql slot) — was never populated by the unwired
        // `grad_decomp_launch_cql` snapshot pattern. Now produces the
        // raw-CQL-norm SP7 needs.
        let result_ptr = self.grad_decomp_result_dev_ptr + 3 * f32_size;
        let cfg = LaunchConfig {
            grid_dim: (1, 1, 1),
            block_dim: (256, 1, 1),
            shared_mem_bytes: 0,
        };
        unsafe {
            self.stream.launch_builder(&self.cql_raw_norm_kernel)
                .arg(&scratch_ptr)
                .arg(&self.grad_decomp_trunk_start)
                .arg(&self.grad_decomp_trunk_len)
                .arg(&self.grad_decomp_dir_start)
                .arg(&self.grad_decomp_dir_len)
                .arg(&self.grad_decomp_mag_start)
                .arg(&self.grad_decomp_mag_len)
                .arg(&result_ptr)
                .launch(cfg)
                .map_err(|e| MLError::ModelError(format!("cql_raw_norm_compute: {e}")))?;
        }
        Ok(())
    }

    /// Task 2.0 — populate the host-side `grad_component_norms_mag/_dir/
    /// _trunk` caches from the pinned result slot. Called at epoch
    /// boundary, before HEALTH_DIAG emission.
    ///
    /// Stream-syncs to ensure the final step's reduction kernel has completed
    /// (same pattern as `per_branch_grad_norms`). Zero-copy: the pinned
    /// memory is device-mapped, so the host read is a direct dereference of
    /// the last-step kernel output.
    ///
    /// Pinned layout (written by `grad_component_delta_norm`, 3 floats per
    /// component — mag, dir, trunk — Task 2.Z extension grows this from
    /// 18 → 27 floats to cover the trunk slice for all 9 measurement
    /// points):
    ///   [ 0]=iqn_mag      [ 1]=iqn_dir      [ 2]=iqn_trunk
    ///   [ 3]=cql_mag      [ 4]=cql_dir      [ 5]=cql_trunk
    ///   [ 6]=cql_sx_mag   [ 7]=cql_sx_dir   [ 8]=cql_sx_trunk
    ///   [ 9]=c51_mag      [10]=c51_dir      [11]=c51_trunk
    ///   [12]=c51_bs_mag   [13]=c51_bs_dir   [14]=c51_bs_trunk
    ///   [15]=distill_mag  [16]=distill_dir  [17]=distill_trunk
    ///   [18]=rec_mag      [19]=rec_dir      [20]=rec_trunk
    ///   [21]=pred_mag     [22]=pred_dir     [23]=pred_trunk
    ///   [24]=ens_mag      [25]=ens_dir      [26]=ens_trunk
    ///
    /// Host caches use `[iqn, cql, cql_sx, c51, c51_bs, distill, rec, pred,
    /// ens]` index order matching the `grad_mag_{iqn,cql,cql_sx,c51,c51_bs,
    /// distill,rec,pred,ens}_ratio` accessor families.
    pub fn refresh_grad_component_norms(&mut self) -> Result<(), MLError> {
        if self.grad_decomp_result_pinned.is_null() {
            return Err(MLError::ModelError(
                "refresh_grad_component_norms: pinned result slot is null".into()
            ));
        }
        self.stream.synchronize().map_err(|e| {
            MLError::ModelError(format!("grad_component_norms sync: {e}"))
        })?;
        let host_slice: &[f32] = unsafe {
            std::slice::from_raw_parts(self.grad_decomp_result_pinned, 27)
        };
        for comp in 0..9_usize {
            self.grad_component_norms_mag[comp]   = host_slice[3 * comp];
            self.grad_component_norms_dir[comp]   = host_slice[3 * comp + 1];
            self.grad_component_norms_trunk[comp] = host_slice[3 * comp + 2];
        }
        Ok(())
    }

    /// Task 2.0 — accessor for cached per-component magnitude norms
    /// populated by `refresh_grad_component_norms`. Index order:
    /// `[iqn, cql, cql_sx, c51, c51_bs, distill, rec, pred, ens]`.
    pub fn grad_component_norms_mag(&self) -> [f32; 9] { self.grad_component_norms_mag }
    /// Task 2.0 — accessor for cached per-component direction norms
    /// populated by `refresh_grad_component_norms`.
    pub fn grad_component_norms_dir(&self) -> [f32; 9] { self.grad_component_norms_dir }
    /// Task 2.Z diagnostic — accessor for cached per-component TRUNK
    /// (tensors 0..4 = w_s1, b_s1, w_s2, b_s2) L2 norms populated by
    /// `refresh_grad_component_norms`. Index order:
    /// `[iqn, cql, cql_sx, c51, c51_bs, distill, rec, pred, ens]`.
    /// Expected non-zero for IQN + Ens (both SAXPY into trunk); expected
    /// near-zero for C51/CQL/CQL-SX/C51-BS/distill/rec/pred (all write to
    /// branch heads only).
    pub fn grad_component_norms_trunk(&self) -> [f32; 9] { self.grad_component_norms_trunk }

    // Note: Legacy `per_branch_target_drift()` and `per_branch_vsn_mean()`
    // (CPU-DtoH + host-loop, Plan 1 era) were DELETED 2026-04-25 per
    // pearl_cold_path_no_exception_to_gpu_drives.md. Replaced by:
    //   - VSN mag/dir means: GPU-computed via `attention_focus_ema_kernel.cu`
    //     (Plan 4 Task 5 Mode A) → ISV[VSN_MAG_EMA_INDEX=87], ISV[88].
    //   - Target drift mag/dir RMS: GPU-computed via `target_drift_kernel.cu`
    //     → ISV[TARGET_DRIFT_MAG_EMA_INDEX=92], ISV[93].
    // HEALTH_DIAG `noisy [vsn_mag=…vsn_dir=…drift_mag=…drift_dir=…]` reads
    // these ISV slots via FusedTrainingCtx::read_isv_signal_at(...).
    // Order/urgency branches are not surfaced in HEALTH_DIAG; if they are
    // needed later, extend the kernel to add 2 more blocks.


    /// Plan 4 Task 5 Mode A (E.5 attention-focus): device pointer to the live
    /// `params_buf`. Used by `attention_focus_ema_update` so the kernel can
    /// reduce VSN-weight slices on-device. No DtoH.
    pub fn params_buf_device_ptr(&self) -> u64 {
        self.params_buf.raw_ptr()
    }

    /// Plan 4 Task 5 Mode A (E.5 attention-focus): expose the param-buffer
    /// slice indices for the magnitude / direction VSN-weight tensors. The
    /// `attention_focus_ema_update` GPU kernel reduces these slices on-device
    /// (no host DtoH, no CPU loop). Returns `(mag_start, mag_len, dir_start,
    /// dir_len)` in f32 element units (kernel takes them as `int` args).
    ///
    /// VSN tensor layout: tensors 26/27 are the dir branch, 28/29 are mag,
    /// 30/31 are order, 32/33 are urg (per `compute_param_sizes`). Each
    /// branch's two VSN tensors are adjacent in the flat params buffer
    /// modulo small alignment padding (which is benign zero-init data —
    /// the reduction's mean is only diagnostic).
    pub fn vsn_branch_slice_indices(&self) -> (usize, usize, usize, usize) {
        let param_sizes = compute_param_sizes(&self.config);
        let dir_start = padded_byte_offset(&param_sizes, 26) as usize / std::mem::size_of::<f32>();
        let dir_end   = padded_byte_offset(&param_sizes, 28) as usize / std::mem::size_of::<f32>();
        let mag_start = padded_byte_offset(&param_sizes, 28) as usize / std::mem::size_of::<f32>();
        let mag_end   = padded_byte_offset(&param_sizes, 30) as usize / std::mem::size_of::<f32>();
        (mag_start, mag_end.saturating_sub(mag_start),
         dir_start, dir_end.saturating_sub(dir_start))
    }

    /// Plan 4 follow-up (target-drift EMA replacing CPU-DtoH path): expose
    /// the per-branch param-buffer slice indices for mag (tensors 12..16) and
    /// dir (tensors 8..12). `target_drift_ema_update` GPU kernel reduces
    /// `RMS(target - online)` over each slice on-device. Returns
    /// `(mag_start, mag_len, dir_start, dir_len)` in f32 element units.
    pub fn branch_param_slice_indices(&self) -> (usize, usize, usize, usize) {
        let param_sizes = compute_param_sizes(&self.config);
        let dir_start = padded_byte_offset(&param_sizes, 8)  as usize / std::mem::size_of::<f32>();
        let dir_end   = padded_byte_offset(&param_sizes, 12) as usize / std::mem::size_of::<f32>();
        let mag_start = padded_byte_offset(&param_sizes, 12) as usize / std::mem::size_of::<f32>();
        let mag_end   = padded_byte_offset(&param_sizes, 16) as usize / std::mem::size_of::<f32>();
        (mag_start, mag_end.saturating_sub(mag_start),
         dir_start, dir_end.saturating_sub(dir_start))
    }

    /// Plan 4 follow-up: device pointer to `target_params_buf`. Used by
    /// `target_drift_ema_update` so the kernel can compute RMS(target -
    /// online) on-device. No DtoH.
    pub fn target_params_buf_device_ptr(&self) -> u64 {
        self.target_params_buf.raw_ptr()
    }

    /// Plan 4 Task 5 Mode A: device pointer + element count for the
    /// `mamba2_h_enriched` buffer the `attention_focus_ema_update` kernel
    /// reduces on-device. No DtoH; the kernel reads via the live cuBLAS
    /// alloc directly. Returns `(0, 0)` when the buffer is empty.
    pub fn mamba2_h_enriched_device(&self, batch_size: usize) -> (u64, usize) {
        let sh2 = self.config.shared_h2;
        let len = batch_size.saturating_mul(sh2).min(self.mamba2_h_enriched.len());
        if len == 0 {
            return (0, 0);
        }
        (self.mamba2_h_enriched.raw_ptr(), len)
    }

    /// Task 0.3 — per-magnitude-bucket Q-value mean (Full / Half / Quarter).
    ///
    /// Returns `[q_full, q_half, q_quarter]` = the mean Q-value predicted for
    /// the Full / Half / Quarter magnitude buckets respectively across the
    /// most-recent batch whose `q_out_buf` contents are live.
    ///
    /// Semantics: the kernel encodes magnitude with `mag_idx = 0` → Quarter,
    /// `1` → Half, `2` → Full (see action_counts layout comment in
    /// `training_loop.rs`). The returned slot order flips that, matching the
    /// HEALTH_DIAG `mag [q_full q_half q_quarter ...]` field order:
    ///
    /// * `[0] q_full`    = mean(q_out_buf[:, b0 + 2])
    /// * `[1] q_half`    = mean(q_out_buf[:, b0 + 1])
    /// * `[2] q_quarter` = mean(q_out_buf[:, b0 + 0])
    ///
    /// Read from the cached value populated by `update_q_mag_means_cached`
    /// (cold-path dtoh invoked once per epoch at HEALTH_DIAG emit).
    pub fn q_magnitude_bucket_means(&self) -> [f32; 3] {
        self.q_mag_means_cached
    }

    /// Task 0.3 — refresh the cached magnitude-branch Q-value means.
    ///
    /// Cold-path (epoch boundary) dtoh of the current `q_out_buf` contents.
    /// Reads `B * total_actions` floats (typically B=16384, total_actions=12
    /// → ~786 KB / epoch), extracts the 3 columns belonging to the magnitude
    /// branch at offsets `[b0 .. b0 + b1]`, averages each column over the
    /// batch, and stores into `q_mag_means_cached` as
    /// `[q_full, q_half, q_quarter]` (see `q_magnitude_bucket_means` for the
    /// full/half/quarter ↔ mag_idx mapping).
    ///
    /// Called at HEALTH_DIAG emit time only. Diagnostic path — dtoh cost is
    /// acceptable here but would be untenable per-step.
    pub fn update_q_mag_means_cached(&mut self) -> Result<(), MLError> {
        let b = self.config.batch_size;
        let b0 = self.config.branch_0_size;
        let b1 = self.config.branch_1_size;
        let total_actions = self.total_actions();
        if b == 0 || b1 == 0 || total_actions == 0 {
            self.q_mag_means_cached = [0.0_f32; 3];
            return Ok(());
        }
        let n_floats = b * total_actions;
        if self.q_out_buf.len() < n_floats {
            return Err(MLError::ModelError(format!(
                "update_q_mag_means_cached: q_out_buf len {} < B*total_actions {}",
                self.q_out_buf.len(),
                n_floats
            )));
        }
        let mut host = vec![0.0_f32; n_floats];
        self.stream
            .memcpy_dtoh(&self.q_out_buf.slice(0..n_floats), &mut host)
            .map_err(|e| MLError::ModelError(format!("dtoh q_out_buf for mag means: {e}")))?;
        // q_out_buf layout is [B, total_actions] row-major. Magnitude-branch
        // columns live at [b0 .. b0 + b1]. Accumulate per-column, divide by B.
        // b1 is expected to be 3 (Quarter/Half/Full); the loop generalises but
        // the returned array is always [f32; 3] — we clamp below.
        let mut sums = [0.0_f64; 3];
        for i in 0..b {
            let row_off = i * total_actions;
            for k in 0..b1.min(3) {
                sums[k] += host[row_off + b0 + k] as f64;
            }
        }
        let inv_b = 1.0_f64 / b as f64;
        // mag_idx 0 = Quarter, 1 = Half, 2 = Full. HEALTH_DIAG expects
        // [q_full, q_half, q_quarter] — flip mag_idx → slot.
        let mean_q = (sums[0] * inv_b) as f32;                 // Quarter
        let mean_h = (sums[1.min(b1 - 1)] * inv_b) as f32;     // Half
        let mean_f = (sums[2.min(b1 - 1)] * inv_b) as f32;     // Full
        self.q_mag_means_cached = [mean_f, mean_h, mean_q];
        Ok(())
    }

    /// SP16 Phase 0 — per-direction-action Q-value mean read accessor.
    ///
    /// Returns `[q_hold, q_long, q_short, q_flat]` = the mean Q-value
    /// predicted for the four direction-branch actions across the
    /// most-recent batch whose `q_out_buf` contents are live.
    ///
    /// Action-index mapping (canonical, see `state_layout.cuh:123-126`):
    /// DIR_SHORT=0, DIR_HOLD=1, DIR_LONG=2, DIR_FLAT=3. The returned slot
    /// order matches the HEALTH_DIAG `q_by_action [hold long short flat]`
    /// emit format (Hold first because the structural-Q-bias-toward-Hold
    /// hypothesis being falsified is about Hold's ascent).
    ///
    /// * `[0] q_hold`  = mean(q_out_buf[:, DIR_HOLD ])  = col 1
    /// * `[1] q_long`  = mean(q_out_buf[:, DIR_LONG ])  = col 2
    /// * `[2] q_short` = mean(q_out_buf[:, DIR_SHORT])  = col 0
    /// * `[3] q_flat`  = mean(q_out_buf[:, DIR_FLAT ])  = col 3
    ///
    /// Read from the cached value populated by `update_q_dir_means_cached`
    /// (cold-path dtoh invoked once per epoch at HEALTH_DIAG emit).
    pub fn q_direction_action_means(&self) -> [f32; 4] {
        self.q_dir_means_cached
    }

    /// SP16 Phase 0 — refresh the cached direction-branch Q-value means.
    ///
    /// Cold-path (epoch boundary) dtoh of the current `q_out_buf` contents.
    /// Reads `B * total_actions` floats, extracts the 4 columns belonging
    /// to the direction branch at offsets `[0 .. b0]`, averages each
    /// column over the batch, and stores into `q_dir_means_cached` as
    /// `[q_hold, q_long, q_short, q_flat]` (see `q_direction_action_means`
    /// for the action-idx ↔ slot mapping).
    ///
    /// Mirrors `update_q_mag_means_cached` for the magnitude branch — both
    /// share the same dtoh of `q_out_buf` per epoch (each is invoked
    /// independently at HEALTH_DIAG emit time; the two transfers can be
    /// batched in a future refactor but the diagnostic-path cost is
    /// acceptable).
    ///
    /// Called at HEALTH_DIAG emit time only. Diagnostic path — dtoh cost
    /// is acceptable here but would be untenable per-step.
    pub fn update_q_dir_means_cached(&mut self) -> Result<(), MLError> {
        let b = self.config.batch_size;
        let b0 = self.config.branch_0_size;
        let total_actions = self.total_actions();
        if b == 0 || b0 == 0 || total_actions == 0 {
            self.q_dir_means_cached = [0.0_f32; 4];
            return Ok(());
        }
        let n_floats = b * total_actions;
        if self.q_out_buf.len() < n_floats {
            return Err(MLError::ModelError(format!(
                "update_q_dir_means_cached: q_out_buf len {} < B*total_actions {}",
                self.q_out_buf.len(),
                n_floats
            )));
        }
        let mut host = vec![0.0_f32; n_floats];
        self.stream
            .memcpy_dtoh(&self.q_out_buf.slice(0..n_floats), &mut host)
            .map_err(|e| MLError::ModelError(format!("dtoh q_out_buf for dir means: {e}")))?;
        self.q_dir_means_cached = compute_q_dir_means_from_host_buf(&host, b, b0, total_actions);
        Ok(())
    }

    /// SP17 Task PP.3 (2026-05-08) — pre-centering V/A mean diagnostic readback.
    ///
    /// Cold-path (epoch boundary) launcher for `v_a_means_diag_kernel`. Reads
    /// the existing online value/advantage logit buffers and emits 5 floats
    /// into the mapped-pinned readback `sp17_v_a_means_diag_buf`:
    ///
    /// * `[0] E[V]`     — mean over `(B × NA)` of `on_v_logits_buf`
    /// * `[1] E[A_dir]` — mean over `(B × b0_size × NA)` of dir-branch
    ///                    advantage logits in `on_b_logits_buf`
    /// * `[2] E[A_mag]` — same for mag-branch
    /// * `[3] E[A_ord]` — same for ord-branch
    /// * `[4] E[A_urg]` — same for urg-branch
    ///
    /// Sets the diagnostic baseline for the SP17 Phase 0 kill criterion: if
    /// `E[V]` dominates the Q-magnitude budget while `E[A_*]` are small,
    /// mean-zero identifiability will help; if `E[A_*]` are already balanced
    /// and doing the work, dueling is a no-op and SP17 aborts.
    ///
    /// This is PRE-CENTERING — the kernel reads the same raw advantage
    /// logits that `compute_expected_q` consumes today. No mean subtraction
    /// is applied; the readout observes the natural decomposition produced
    /// by the cuBLAS forward.
    ///
    /// Single-block launch (no graph capture; this runs only on the
    /// HEALTH_DIAG cold path). Per `feedback_no_atomicadd.md` the kernel
    /// uses block tree-reduce; per `feedback_no_htod_htoh_only_mapped_pinned`
    /// the readback is mapped-pinned.
    pub fn read_v_a_means_pre_centering(&self) -> Result<[f32; 5], MLError> {
        let b = self.config.batch_size;
        let na = self.config.num_atoms;
        let b0 = self.config.branch_0_size;
        let b1 = self.config.branch_1_size;
        let b2 = self.config.branch_2_size;
        let b3 = self.config.branch_3_size;
        if b == 0 || na == 0 || (b0 + b1 + b2 + b3) == 0 {
            return Ok([0.0_f32; 5]);
        }
        let v_ptr = self.on_v_logits_buf.raw_ptr();
        let adv_ptr = self.on_b_logits_buf.raw_ptr();
        let out_ptr = self.sp17_v_a_means_diag_buf.dev_ptr;
        let b_i32 = b as i32;
        let na_i32 = na as i32;
        let b0_i32 = b0 as i32;
        let b1_i32 = b1 as i32;
        let b2_i32 = b2 as i32;
        let b3_i32 = b3 as i32;
        // Single block, BLOCK=256, dynamic shmem = 256 × sizeof(f32) = 1024
        // bytes (one float per thread for the per-reduction shared tile,
        // re-used across the 5 reductions). Matches the kernel's
        // `extern __shared__ float shmem[]` contract.
        let block_dim: u32 = 256;
        let shmem_bytes: u32 = block_dim * std::mem::size_of::<f32>() as u32;
        unsafe {
            self.stream
                .launch_builder(&self.sp17_v_a_means_diag_kernel)
                .arg(&v_ptr)
                .arg(&adv_ptr)
                .arg(&out_ptr)
                .arg(&b_i32)
                .arg(&na_i32)
                .arg(&b0_i32)
                .arg(&b1_i32)
                .arg(&b2_i32)
                .arg(&b3_i32)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (block_dim, 1, 1),
                    shared_mem_bytes: shmem_bytes,
                })
                .map_err(|e| MLError::ModelError(format!("v_a_means_diag launch: {e}")))?;
        }
        self.stream
            .synchronize()
            .map_err(|e| MLError::ModelError(format!("v_a_means_diag sync: {e}")))?;
        let host = self.sp17_v_a_means_diag_buf.read_all();
        Ok([host[0], host[1], host[2], host[3], host[4]])
    }

    /// SP17 Phase 3.1 (2026-05-08) — per-branch A_var EMA producer launch.
    ///
    /// Cold-path (epoch boundary) launcher for `sp17_a_var_ema_kernel`.
    /// Reads the same online advantage buffer (`on_b_logits_buf`) that
    /// `compute_expected_q` consumes; computes per-branch
    /// `Var(A_centered)` over (B × n_branch × NA) where
    /// `A_centered[i, a, z] = A[i, a, z] − (1/n_d) Σ_a' A[i, a', z]`
    /// and EMA-blends into ISV[A_VAR_EMA_*_INDEX] with Pearl-A
    /// first-observation bootstrap (sentinel
    /// `SENTINEL_A_VAR_EMA=0.0`) + α=`WELFORD_ALPHA_MIN=0.4` per
    /// `pearl_wiener_alpha_floor_for_nonstationary`.
    ///
    /// 4 blocks × 256 threads (one block per branch — dir/mag/ord/urg).
    /// Block tree-reduce per `feedback_no_atomicadd`. Returns Ok(()) on
    /// success; HEALTH_DIAG reads the ISV slots via
    /// `read_isv_signal_at(A_VAR_EMA_*_INDEX)` after a stream sync.
    ///
    /// Diagnostic only — does NOT modify any consumer path. Per
    /// `feedback_no_htod_htoh_only_mapped_pinned` the ISV bus is already
    /// mapped-pinned; this kernel writes via `dev_ptr +
    /// __threadfence_system()` and the host reads after sync.
    pub fn launch_a_var_ema_update(&self) -> Result<(), MLError> {
        use super::sp14_isv_slots::{
            A_VAR_EMA_DIR_INDEX, A_VAR_EMA_MAG_INDEX,
            A_VAR_EMA_ORD_INDEX, A_VAR_EMA_URG_INDEX,
            SENTINEL_A_VAR_EMA, WELFORD_ALPHA_MIN,
        };
        let b = self.config.batch_size;
        let na = self.config.num_atoms;
        let b0 = self.config.branch_0_size;
        let b1 = self.config.branch_1_size;
        let b2 = self.config.branch_2_size;
        let b3 = self.config.branch_3_size;
        if b == 0 || na == 0 || (b0 + b1 + b2 + b3) == 0 {
            return Ok(());
        }
        let adv_ptr = self.on_b_logits_buf.raw_ptr();
        let isv_ptr = self.isv_signals_dev_ptr;
        let b_i32 = b as i32;
        let na_i32 = na as i32;
        let b0_i32 = b0 as i32;
        let b1_i32 = b1 as i32;
        let b2_i32 = b2 as i32;
        let b3_i32 = b3 as i32;
        let var_idx_dir = A_VAR_EMA_DIR_INDEX as i32;
        let var_idx_mag = A_VAR_EMA_MAG_INDEX as i32;
        let var_idx_ord = A_VAR_EMA_ORD_INDEX as i32;
        let var_idx_urg = A_VAR_EMA_URG_INDEX as i32;
        let alpha: f32 = WELFORD_ALPHA_MIN;
        let sentinel: f32 = SENTINEL_A_VAR_EMA;
        // 4 blocks × 256 threads. Dynamic shmem = (BLOCK_SIZE + NA) × 4
        // bytes for the tree-reduce tile + per-atom mean cache.
        let block_dim: u32 = 256;
        let shmem_bytes: u32 =
            (block_dim + na as u32) * std::mem::size_of::<f32>() as u32;
        unsafe {
            self.stream
                .launch_builder(&self.sp17_a_var_ema_kernel)
                .arg(&adv_ptr)
                .arg(&isv_ptr)
                .arg(&b_i32)
                .arg(&na_i32)
                .arg(&b0_i32)
                .arg(&b1_i32)
                .arg(&b2_i32)
                .arg(&b3_i32)
                .arg(&var_idx_dir)
                .arg(&var_idx_mag)
                .arg(&var_idx_ord)
                .arg(&var_idx_urg)
                .arg(&alpha)
                .arg(&sentinel)
                .launch(LaunchConfig {
                    grid_dim: (4, 1, 1),
                    block_dim: (block_dim, 1, 1),
                    shared_mem_bytes: shmem_bytes,
                })
                .map_err(|e| MLError::ModelError(format!(
                    "sp17_a_var_ema_update launch: {e}"
                )))?;
        }
        Ok(())
    }

    /// SP17 Phase 3.1 (2026-05-08) — per-branch A_var EMA readback.
    ///
    /// Convenience wrapper: launches the producer, syncs, returns
    /// `[a_var_dir, a_var_mag, a_var_ord, a_var_urg]` from the ISV bus.
    /// Cold-path only (HEALTH_DIAG cadence).
    pub fn read_a_var_ema_per_branch(&self) -> Result<[f32; 4], MLError> {
        use super::sp14_isv_slots::{
            A_VAR_EMA_DIR_INDEX, A_VAR_EMA_MAG_INDEX,
            A_VAR_EMA_ORD_INDEX, A_VAR_EMA_URG_INDEX,
        };
        self.launch_a_var_ema_update()?;
        self.stream
            .synchronize()
            .map_err(|e| MLError::ModelError(format!("sp17_a_var_ema sync: {e}")))?;
        Ok([
            self.read_isv_signal_at(A_VAR_EMA_DIR_INDEX),
            self.read_isv_signal_at(A_VAR_EMA_MAG_INDEX),
            self.read_isv_signal_at(A_VAR_EMA_ORD_INDEX),
            self.read_isv_signal_at(A_VAR_EMA_URG_INDEX),
        ])
    }

    /// SP17 Phase 3.2 (2026-05-08) — per-branch V_share producer launch.
    ///
    /// Cold-path launcher for `sp17_v_share_kernel`. Reads
    /// `on_v_logits_buf` and `on_b_logits_buf`; computes per-branch
    /// `V_share = |E[V]| / (|E[V]| + |E[A_centered, picked]|)` where
    /// picked = argmax_a Σ_z A_raw[i, a, z] (max-Q semantic — tractable
    /// per-batch without depending on `actions_history_buf` which is
    /// collector-time state stale relative to the cuBLAS forward driving
    /// the on_*_logits buffers at HEALTH_DIAG cadence).
    ///
    /// Pearl-A bootstrap on sentinel `SENTINEL_V_SHARE=0.5` (cold-start
    /// healthy 50/50 split); α=`WELFORD_ALPHA_MIN=0.4` post-bootstrap.
    /// Bilateral [0, 1] clamp before ISV write per
    /// `pearl_symmetric_clamp_audit`. 4 blocks × 256 threads.
    pub fn launch_v_share_update(&self) -> Result<(), MLError> {
        use super::sp14_isv_slots::{
            V_SHARE_DIR_INDEX, V_SHARE_MAG_INDEX,
            V_SHARE_ORD_INDEX, V_SHARE_URG_INDEX,
            SENTINEL_V_SHARE, WELFORD_ALPHA_MIN,
        };
        let b = self.config.batch_size;
        let na = self.config.num_atoms;
        let b0 = self.config.branch_0_size;
        let b1 = self.config.branch_1_size;
        let b2 = self.config.branch_2_size;
        let b3 = self.config.branch_3_size;
        if b == 0 || na == 0 || (b0 + b1 + b2 + b3) == 0 {
            return Ok(());
        }
        let v_ptr = self.on_v_logits_buf.raw_ptr();
        let adv_ptr = self.on_b_logits_buf.raw_ptr();
        let isv_ptr = self.isv_signals_dev_ptr;
        let b_i32 = b as i32;
        let na_i32 = na as i32;
        let b0_i32 = b0 as i32;
        let b1_i32 = b1 as i32;
        let b2_i32 = b2 as i32;
        let b3_i32 = b3 as i32;
        let share_idx_dir = V_SHARE_DIR_INDEX as i32;
        let share_idx_mag = V_SHARE_MAG_INDEX as i32;
        let share_idx_ord = V_SHARE_ORD_INDEX as i32;
        let share_idx_urg = V_SHARE_URG_INDEX as i32;
        let alpha: f32 = WELFORD_ALPHA_MIN;
        let sentinel: f32 = SENTINEL_V_SHARE;
        // Dynamic shmem = (BLOCK_SIZE + NA + 1) × sizeof(f32) — tree-reduce
        // tile + per-atom mean cache + 1 float for E[V] broadcast.
        let block_dim: u32 = 256;
        let shmem_bytes: u32 =
            (block_dim + na as u32 + 1) * std::mem::size_of::<f32>() as u32;
        unsafe {
            self.stream
                .launch_builder(&self.sp17_v_share_kernel)
                .arg(&v_ptr)
                .arg(&adv_ptr)
                .arg(&isv_ptr)
                .arg(&b_i32)
                .arg(&na_i32)
                .arg(&b0_i32)
                .arg(&b1_i32)
                .arg(&b2_i32)
                .arg(&b3_i32)
                .arg(&share_idx_dir)
                .arg(&share_idx_mag)
                .arg(&share_idx_ord)
                .arg(&share_idx_urg)
                .arg(&alpha)
                .arg(&sentinel)
                .launch(LaunchConfig {
                    grid_dim: (4, 1, 1),
                    block_dim: (block_dim, 1, 1),
                    shared_mem_bytes: shmem_bytes,
                })
                .map_err(|e| MLError::ModelError(format!(
                    "sp17_v_share_update launch: {e}"
                )))?;
        }
        Ok(())
    }

    /// SP17 Phase 3.2 (2026-05-08) — per-branch V_share readback.
    /// Convenience wrapper: launches → syncs → reads ISV slots 478..482.
    pub fn read_v_share_per_branch(&self) -> Result<[f32; 4], MLError> {
        use super::sp14_isv_slots::{
            V_SHARE_DIR_INDEX, V_SHARE_MAG_INDEX,
            V_SHARE_ORD_INDEX, V_SHARE_URG_INDEX,
        };
        self.launch_v_share_update()?;
        self.stream
            .synchronize()
            .map_err(|e| MLError::ModelError(format!("sp17_v_share sync: {e}")))?;
        Ok([
            self.read_isv_signal_at(V_SHARE_DIR_INDEX),
            self.read_isv_signal_at(V_SHARE_MAG_INDEX),
            self.read_isv_signal_at(V_SHARE_ORD_INDEX),
            self.read_isv_signal_at(V_SHARE_URG_INDEX),
        ])
    }

    /// SP17 Phase 3.2 (2026-05-08) — adaptive advantage_clip_bound launch.
    ///
    /// Cold-path launcher for `sp17_advantage_clip_bound_kernel`. Computes
    /// `p99(|A_centered|) × ADVANTAGE_CLIP_SAFETY_FACTOR=1.5` over the
    /// full flat batch via `sp4_histogram_p99` (no atomicAdd per
    /// `pearl_fused_per_group_statistics_oracle`); EMA-blends into
    /// ISV[ADVANTAGE_CLIP_BOUND_INDEX=482] with α=`ADVANTAGE_CLIP_EMA_ALPHA=0.01`
    /// + bilateral clamp `[ADVANTAGE_CLIP_BOUND_MIN=0.1, _MAX=100.0]`
    /// per `pearl_symmetric_clamp_audit`. Pearl-A bootstrap on sentinel
    /// 1.0.
    ///
    /// Observability-only in Phase 3 — the actual clipping wire-up is
    /// Phase 5 follow-up. This launcher is the producer end of the
    /// chain that feeds the HEALTH_DIAG `dueling [clip=K]` line.
    ///
    /// Single block × 256 threads. Dynamic shmem = `(BLOCK / 32) × 256 × 4
    /// = 8192 bytes` for `sp4_histogram_p99`'s per-warp tile array.
    pub fn launch_advantage_clip_bound_update(&self) -> Result<(), MLError> {
        use super::sp14_isv_slots::{
            ADVANTAGE_CLIP_BOUND_INDEX,
            ADVANTAGE_CLIP_BOUND_MIN, ADVANTAGE_CLIP_BOUND_MAX,
            ADVANTAGE_CLIP_EMA_ALPHA, ADVANTAGE_CLIP_SAFETY_FACTOR,
            SENTINEL_ADVANTAGE_CLIP_BOUND,
        };
        let b = self.config.batch_size;
        let na = self.config.num_atoms;
        let b0 = self.config.branch_0_size;
        let b1 = self.config.branch_1_size;
        let b2 = self.config.branch_2_size;
        let b3 = self.config.branch_3_size;
        if b == 0 || na == 0 || (b0 + b1 + b2 + b3) == 0 {
            return Ok(());
        }
        let total = b * (b0 + b1 + b2 + b3) * na;
        let scratch_capacity = self.sp17_advantage_clip_scratch.len;
        if total > scratch_capacity {
            return Err(MLError::ModelError(format!(
                "sp17_advantage_clip_scratch too small: need {total}, have {scratch_capacity}"
            )));
        }
        let adv_ptr = self.on_b_logits_buf.raw_ptr();
        let scratch_ptr = self.sp17_advantage_clip_scratch.dev_ptr;
        let isv_ptr = self.isv_signals_dev_ptr;
        let scratch_cap_i32 = scratch_capacity as i32;
        let b_i32 = b as i32;
        let na_i32 = na as i32;
        let b0_i32 = b0 as i32;
        let b1_i32 = b1 as i32;
        let b2_i32 = b2 as i32;
        let b3_i32 = b3 as i32;
        let bound_idx = ADVANTAGE_CLIP_BOUND_INDEX as i32;
        let alpha: f32 = ADVANTAGE_CLIP_EMA_ALPHA;
        let safety: f32 = ADVANTAGE_CLIP_SAFETY_FACTOR;
        let bound_min: f32 = ADVANTAGE_CLIP_BOUND_MIN;
        let bound_max: f32 = ADVANTAGE_CLIP_BOUND_MAX;
        let sentinel: f32 = SENTINEL_ADVANTAGE_CLIP_BOUND;
        // Dynamic shmem for sp4_histogram_p99: (BLOCK / 32) × SP4_HIST_BINS
        // × sizeof(int) = 8 × 256 × 4 = 8192 bytes for BLOCK_SIZE=256.
        let block_dim: u32 = 256;
        let warps = block_dim / 32;
        let shmem_bytes: u32 = warps * 256 * std::mem::size_of::<i32>() as u32;
        unsafe {
            self.stream
                .launch_builder(&self.sp17_advantage_clip_bound_kernel)
                .arg(&adv_ptr)
                .arg(&scratch_ptr)
                .arg(&scratch_cap_i32)
                .arg(&isv_ptr)
                .arg(&b_i32)
                .arg(&na_i32)
                .arg(&b0_i32)
                .arg(&b1_i32)
                .arg(&b2_i32)
                .arg(&b3_i32)
                .arg(&bound_idx)
                .arg(&alpha)
                .arg(&safety)
                .arg(&bound_min)
                .arg(&bound_max)
                .arg(&sentinel)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (block_dim, 1, 1),
                    shared_mem_bytes: shmem_bytes,
                })
                .map_err(|e| MLError::ModelError(format!(
                    "sp17_advantage_clip_bound_update launch: {e}"
                )))?;
        }
        Ok(())
    }

    /// SP17 Phase 3.2 (2026-05-08) — adaptive advantage_clip_bound readback.
    /// Convenience wrapper: launches → syncs → reads ISV slot 482.
    pub fn read_advantage_clip_bound(&self) -> Result<f32, MLError> {
        use super::sp14_isv_slots::ADVANTAGE_CLIP_BOUND_INDEX;
        self.launch_advantage_clip_bound_update()?;
        self.stream
            .synchronize()
            .map_err(|e| MLError::ModelError(format!("sp17_advantage_clip_bound sync: {e}")))?;
        Ok(self.read_isv_signal_at(ADVANTAGE_CLIP_BOUND_INDEX))
    }

    /// SP18 v2 Phase 0 Task 0.2 (2026-05-08) — B-leg TD-error magnitude
    /// EMA producer launch.
    ///
    /// Cold-path (post-train-step) launcher for
    /// `td_error_mag_ema_update`. Reads the trainer-owned
    /// `td_errors_buf [B]` (populated by the C51/MSE loss kernel each
    /// training step, consumed in-place by `seg_tree_update` for PER
    /// priority recomputation), block tree-reduces
    /// `mean(|td_errors[b]|)`, applies Pearl-A first-observation
    /// bootstrap (sentinel 0.0 → REPLACE) and fixed α=0.4 EMA blend,
    /// and writes the result to `ISV[TD_ERROR_MAG_EMA_INDEX=493]`.
    ///
    /// Pure observability: pre-fix baseline for the B-DD9 ratio gate
    /// (`avg(|TD-error|) ratio post-fix / pre-fix ∈ [0.5, 5.0]`).
    /// Phase 0 does NOT yet use the Welford-derived α from the TDB_*
    /// accumulators in slots [498..504) — those are reserved for Phase
    /// 4 q_next_target Wiener-α blending.
    ///
    /// Returns Ok(()) on launch success; the host-side HEALTH_DIAG emit
    /// reads the slot via `read_isv_signal_at(TD_ERROR_MAG_EMA_INDEX)`
    /// after stream sync.
    pub fn launch_sp18_td_error_mag_ema_update(&self) -> Result<(), MLError> {
        use super::sp14_isv_slots::{
            TD_ERROR_MAG_EMA_INDEX, SENTINEL_TD_ERROR_MAG_EMA, WELFORD_ALPHA_MIN,
        };
        let b = self.config.batch_size;
        if b == 0 || self.isv_signals_dev_ptr == 0 {
            return Ok(());
        }
        let td_ptr = self.td_errors_buf.raw_ptr();
        let isv_ptr = self.isv_signals_dev_ptr;
        let b_i32 = b as i32;
        let ema_idx = TD_ERROR_MAG_EMA_INDEX as i32;
        let alpha: f32 = WELFORD_ALPHA_MIN;
        let sentinel: f32 = SENTINEL_TD_ERROR_MAG_EMA;
        // Single-block, 256 threads. Dynamic shmem = BLOCK_SIZE × f32.
        const BLOCK_DIM: u32 = 256;
        let shmem_bytes: u32 = BLOCK_DIM * std::mem::size_of::<f32>() as u32;
        unsafe {
            self.stream
                .launch_builder(&self.sp18_td_error_mag_ema_kernel)
                .arg(&td_ptr)
                .arg(&b_i32)
                .arg(&isv_ptr)
                .arg(&ema_idx)
                .arg(&alpha)
                .arg(&sentinel)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (BLOCK_DIM, 1, 1),
                    shared_mem_bytes: shmem_bytes,
                })
                .map_err(|e| MLError::ModelError(format!(
                    "sp18 td_error_mag_ema_update launch: {e}"
                )))?;
        }
        Ok(())
    }

    /// SP18 v2 Phase 0 Task 0.2 (2026-05-08) — B-leg TD-error magnitude
    /// EMA readback.
    ///
    /// Convenience wrapper: launches the producer, syncs the stream,
    /// returns the post-blend ISV slot 493 value. Cold-path only
    /// (HEALTH_DIAG cadence).
    pub fn read_sp18_td_error_mag_ema(&self) -> Result<f32, MLError> {
        use super::sp14_isv_slots::TD_ERROR_MAG_EMA_INDEX;
        self.launch_sp18_td_error_mag_ema_update()?;
        self.stream
            .synchronize()
            .map_err(|e| MLError::ModelError(format!(
                "sp18 td_error_mag_ema sync: {e}"
            )))?;
        Ok(self.read_isv_signal_at(TD_ERROR_MAG_EMA_INDEX))
    }

    /// SP18 v2 Phase 0 Task 0.1 (2026-05-08) — D-leg per-action reward
    /// decomposition diagnostic device pointer.
    ///
    /// Returns the mapped-pinned device pointer of the 20-float
    /// `sp18_reward_decomp_diag_buf` so the collector's
    /// `launch_sp18_reward_decomp_diag` can write into it. The host-side
    /// HEALTH_DIAG emit reads via `read_sp18_reward_decomp_diag()` after
    /// stream sync. Pure observability — caller is expected to be the
    /// per-step training loop wiring point.
    pub fn sp18_reward_decomp_diag_dev_ptr(&self) -> u64 {
        self.sp18_reward_decomp_diag_buf.dev_ptr
    }

    /// SP18 v2 Phase 0 Task 0.1 (2026-05-08) — D-leg per-action reward
    /// decomposition diagnostic readback.
    ///
    /// Returns the 20-float diagnostic block as a `[f32; 20]`, row-major
    /// 4 bins × 5 stats:
    ///
    ///     [Hold_micro, Hold_opp, Hold_popart, Hold_abs, Hold_fire,
    ///      Long_micro, Long_opp, Long_popart, Long_abs, Long_fire,
    ///      Short_micro, Short_opp, Short_popart, Short_abs, Short_fire,
    ///      Flat_micro, Flat_opp, Flat_popart, Flat_abs, Flat_fire]
    ///
    /// Reads the mapped-pinned host_ptr directly — caller MUST have
    /// stream-synced after the collector's launch (e.g. via the natural
    /// per-step boundary that already runs `cuStreamSynchronize` between
    /// collection and training). No HtoD/DtoH copy.
    ///
    /// On cold-start (before any kernel launch) returns the
    /// constructor-initialised zero block.
    pub fn read_sp18_reward_decomp_diag(&self) -> [f32; 20] {
        let v = self.sp18_reward_decomp_diag_buf.read_all();
        debug_assert_eq!(
            v.len(), 20,
            "sp18_reward_decomp_diag_buf size invariant: 4 bins × 5 stats"
        );
        let mut out = [0.0_f32; 20];
        for (i, x) in v.iter().take(20).enumerate() { out[i] = *x; }
        out
    }

    /// Update per-branch liquid tau modulation — GPU kernel (RK4/Euler adaptive ODE).
    ///
    /// Reads per_branch_q_gaps (pinned device-mapped) and qlstm_context from device.
    /// Update eval v_range from observed per-branch Q-value statistics.
    /// Called at epoch boundary after `reduce_current_q_stats_per_branch`.
    ///
    /// Per-branch path (ISV v-range unification, spec 2026-04-23):
    /// maintains 4 independent `(q_mean_ema, q_std_ema)` pairs and writes 8
    /// ISV slots (`V_CENTER_{branch}_INDEX`, `V_HALF_{branch}_INDEX`) — one
    /// pair of (centre, half) per branch. Consumers read through the ISV
    /// signal bus (`adaptive_atom_positions`, `warm_start_atom_positions`).
    ///
    /// Legacy global `eval_v_range_pinned[2]` is still updated using the
    /// direction-branch (branch 0) bounds so any consumer that has not yet
    /// migrated to the per-branch ISV bus sees the same value as the
    /// "dominant branch" behaviour prior to this change. Full removal of
    /// `eval_v_range_pinned` is deferred to the Phase-3 cleanup.
    pub fn update_eval_v_range(
        &mut self,
        per_branch_stats: &PerBranchQValueStats,
        per_branch_q_gaps: [f32; 4],
    ) {
        let v_min_f = self.config.v_min;
        let v_max_f = self.config.v_max;
        let v_range_full = (v_max_f - v_min_f).max(1e-6);
        // Prevents atom-grid collapse when the Q-distribution is narrow.
        // Spec 2026-04-23: min_half_floor = 0.1 * (v_max - v_min).
        let min_half_floor = 0.1_f32 * v_range_full;
        let abs_half = 0.5_f32 * v_range_full;

        for (branch_idx, stats) in per_branch_stats.per_branch.iter().enumerate() {
            let q_mean = stats.q_mean;
            let q_std = stats.q_variance.max(0.0).sqrt();
            let q_gap = per_branch_q_gaps[branch_idx].max(0.0);
            // ISV v-range diag: capture the branch's initialization state
            // BEFORE the EMA update so the log reflects which cold/warm path
            // produced the (center, half) being written.
            let initialized_before = self.eval_ema_initialized[branch_idx];

            // Adaptive-rate EMA with baseline proportional to Q-std.
            //
            // Cold-start seeding: the first observed `q_std` at epoch end is
            // dominated by the ±bootstrap atom spread (the scaffolding we set
            // in `construct` / `reset_eval_v_range_state`), NOT by the true
            // Q-distribution spread. Latching it makes `3*std_ema` exceed
            // `min_half_floor` and approach `abs_half`, so atoms stay wide
            // next epoch, the next `q_std` confirms that width, and the EMA
            // never escapes — Q saturates at ±abs_half.
            //
            // Seed conservatively so the initial `half` equals exactly
            // `min_half_floor`. The adaptive-rate EMA then relaxes upward
            // only if genuine Q-spread warrants it.
            if !self.eval_ema_initialized[branch_idx] {
                self.eval_q_mean_ema[branch_idx] = 0.0;
                self.eval_q_std_ema[branch_idx] = min_half_floor / 3.0;
                self.eval_ema_initialized[branch_idx] = true;
            } else {
                let baseline = self.eval_q_std_ema[branch_idx].max(0.001);
                let mean_err = (q_mean - self.eval_q_mean_ema[branch_idx]).abs();
                let std_err = (q_std - self.eval_q_std_ema[branch_idx]).abs();
                let alpha_mean = (mean_err / (mean_err + baseline)).clamp(0.01, 0.3);
                let alpha_std = (std_err / (std_err + baseline)).clamp(0.01, 0.3);
                self.eval_q_mean_ema[branch_idx] =
                    (1.0 - alpha_mean) * self.eval_q_mean_ema[branch_idx] + alpha_mean * q_mean;
                self.eval_q_std_ema[branch_idx] =
                    (1.0 - alpha_std) * self.eval_q_std_ema[branch_idx] + alpha_std * q_std;
            }

            // Plan 2 Task 1 C.1: quantile-based half-width (spec §4.C.1).
            // half = max(|q_p95 - v_center|, |v_center - q_p05|), clamped to
            // [min_half_floor, abs_half]. q_p05/q_p95 are the EMA-smoothed
            // percentiles written by q_quantile_reduce into ISV[47..55).
            // On cold-start (before q_quantile_reduce runs) the slots hold
            // v_min / v_max bootstraps, reproducing the pre-spec behaviour.
            let center = self.eval_q_mean_ema[branch_idx];
            let q_p05 = unsafe {
                *self.isv_signals_pinned.add(Q_P05_DIR_INDEX + branch_idx)
            };
            let q_p95 = unsafe {
                *self.isv_signals_pinned.add(Q_P95_DIR_INDEX + branch_idx)
            };
            let half_from_p95 = (q_p95 - center).abs();
            let half_from_p05 = (center - q_p05).abs();
            let half = half_from_p95.max(half_from_p05)
                .max(min_half_floor)
                .min(abs_half);
            // Centre clamp so the band always fits inside [v_min, v_max].
            let center = center.clamp(v_min_f + half, v_max_f - half);

            tracing::info!(
                target: "isv_vrange_diag",
                branch_idx, q_mean, q_std, q_gap, center, half,
                q_p05, q_p95,
                initialized = initialized_before,
                "update_eval_v_range branch"
            );

            unsafe {
                *self.isv_signals_pinned.add(V_CENTER_DIR_INDEX + 2 * branch_idx) = center;
                *self.isv_signals_pinned.add(V_CENTER_DIR_INDEX + 2 * branch_idx + 1) = half;
            }

            // Branch 0 (direction) drives the legacy scalar support — matches
            // the dominant-branch Q-scale the single-EMA path used.
            if branch_idx == 0 && !self.eval_v_range_pinned.is_null() {
                unsafe {
                    *self.eval_v_range_pinned = center - half;
                    *self.eval_v_range_pinned.add(1) = center + half;
                }
            }
        }

        let _ = per_branch_q_gaps; // already on device via per_branch_q_gaps_dev_ptr
    }
    pub fn set_per_sample_support_ptr(&mut self, ptr: u64) {
        self.per_sample_support_ptr = ptr;
    }
    pub fn set_branch_scales_ptr(&mut self, ptr: u64) {
        self.branch_scales_ptr = ptr;
    }

    /// Build [h_s2; Q_dir] concat buffer for magnitude branch conditioning.
    /// Must be called AFTER direction branch forward (branch 0 logits are ready).
    /// Build [source; Q_dir] concat buffer for magnitude branch conditioning.
    /// `source_ptr` is the [B, SH2] buffer to use as first columns — either
    /// save_h_s2 (before VSN) or vsn_masked_buf (after VSN).
    ///
    /// Plan 4 Task 2c.3c.6: kernel now consumes ISV[H_S2_RMS_EMA_INDEX]
    /// (populated each step by `launch_h_s2_rms_ema`, the producer wired in
    /// 2c.3c.5) so the Q_dir tail of the concat is adaptively rescaled to
    /// match the trunk-output RMS regardless of GRN drift across training.
    /// Two trailing kernel args added: ISV bus device pointer + slot index.
    pub(crate) fn launch_mag_concat_from(&self, source_ptr: u64, v_logits_ptr: u64, b_logits_ptr: u64) -> Result<(), MLError> {
        // Defaults to current-states for atom-shift (online forward path).
        // Target-side callers MUST use `launch_mag_concat_from_with_state`
        // and pass next_states_buf.
        self.launch_mag_concat_from_with_state(
            source_ptr,
            v_logits_ptr,
            b_logits_ptr,
            self.ptrs.states_buf,
        )
    }

    /// Same as `launch_mag_concat_from` but with explicit state buffer for
    /// SP22 H6 Phase 3 α atom-shift's `state_121` read. Online callers pass
    /// `ptrs.states_buf`; target callers pass `ptrs.next_states_buf`.
    pub(crate) fn launch_mag_concat_from_with_state(
        &self,
        source_ptr: u64,
        v_logits_ptr: u64,
        b_logits_ptr: u64,
        states_ptr: u64,
    ) -> Result<(), MLError> {
        // `isv_signals_dev_ptr` is unconditionally allocated by the constructor
        // via `cuMemHostGetDevicePointer_v2` on a `cuMemAllocHost_v2`-backed
        // pinned buffer (with its own `assert_eq!` on success) and is never
        // reassigned. Mirrors the 2c.3c.5 producer launcher's invariant —
        // silent-skip would mask a misconfiguration.
        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_mag_concat_from: isv_signals_dev_ptr must be allocated by constructor");
        let b = self.config.batch_size;
        let sh2 = self.config.shared_h2 as i32;
        let na = self.config.num_atoms as i32;
        let b0 = self.config.branch_0_size as i32;
        let n = b as i32;
        let blocks = ((b as u32 + 255) / 256).max(1);
        let h_s2_ptr = source_ptr;
        let concat_ptr = self.ptrs.mag_concat_buf;
        let support_ptr = self.per_sample_support_ptr;
        let isv_dev_ptr = self.isv_signals_dev_ptr;
        let isv_idx = H_S2_RMS_EMA_INDEX as i32;
        // SP22 H6 Phase 3 α atom-shift: W ptr + state buffer for state_121
        // read inside the kernel. NULL-safe in-kernel when either is NULL.
        let w_aux_ptr = self.w_aux_to_q_dir.raw_ptr();
        let aux_dir_prob_index = ml_core::state_layout::AUX_DIR_PROB_INDEX as i32;
        let state_dim_i32 = ml_core::state_layout::STATE_DIM as i32;
        unsafe {
            self.stream
                .launch_builder(&self.mag_concat_kernel)
                .arg(&h_s2_ptr)
                .arg(&v_logits_ptr)
                .arg(&b_logits_ptr)
                .arg(&concat_ptr)
                .arg(&n)
                .arg(&sh2)
                .arg(&na)
                .arg(&b0)
                .arg(&support_ptr)
                .arg(&isv_dev_ptr)              // Plan 4 Task 2c.3c.6
                .arg(&isv_idx)                  // Plan 4 Task 2c.3c.6
                .arg(&w_aux_ptr)
                .arg(&states_ptr)
                .arg(&aux_dir_prob_index)
                .arg(&state_dim_i32)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("mag_concat_qdir: {e}")))?;
        }
        Ok(())
    }

    /// SP14 Layer B Task B.9 (2026-05-05): pre-SGEMM direction-Q-head input
    /// concat. Reads `source_ptr [B, SH2]` (one of `save_h_s2` for online or
    /// `tg_h_s2_buf` for target) and `aux_nb_softmax_buf [B, 2]`, writes
    /// `[h_s2 | aux_softmax_diff] [B, SH2 + 1]` into
    /// `sp14_dir_qaux_concat_scratch`.
    ///
    /// Mirrors `launch_mag_concat_from` precedent (one-step-lag pattern):
    /// the aux-head forward pass that *produces* `aux_nb_softmax_buf` runs
    /// AFTER `forward_online_raw` in the per-step pipeline (line ~25526), so
    /// each forward consumes the PREVIOUS step's aux predictions. Step 0 sees
    /// the alloc_zeros-initial buffer (uniform 0.5/0.5 → diff = 0), then from
    /// step 1+ uses the prior step's aux output. Identical semantics to
    /// `launch_mag_concat_from`'s "uses the previous step's direction Q-values".
    ///
    /// Per `pearl_canary_input_freshness_launch_order`: producer (aux head)
    /// writes `aux_nb_softmax_buf` from the prior step; consumer (this concat
    /// kernel) reads it on the same stream BEFORE the direction Q-head SGEMM.
    /// Sequential same-stream submission enforces the dep — no host barrier
    /// needed and the order survives CUDA Graph capture.
    // SP14 Layer C Phase C.1 (2026-05-08): `launch_sp14_scale_wire_col`
    // (formerly EGF backward gate, B.10) deleted atomically with the
    // kernel file. Coupling B (α-gated backward wire) eliminated per
    // `feedback_no_legacy_aliases`. Phase C.5 will wire the proper
    // aux-trunk backward path with stop-gradient at the encoder boundary.

    pub(crate) fn launch_sp14_dir_concat_qaux(&self, source_ptr: u64) -> Result<(), MLError> {
        let b = self.config.batch_size;
        let sh2 = self.config.shared_h2 as i32;
        let b_i32 = b as i32;
        let total = b * (self.config.shared_h2 + 1);
        let blocks = ((total as u32 + 255) / 256).max(1);
        let h_s2_ptr = source_ptr;
        let aux_softmax_ptr = self.aux_nb_softmax_buf.raw_ptr();
        let out_ptr = self.sp14_dir_qaux_concat_scratch.raw_ptr();
        unsafe {
            self.stream
                .launch_builder(&self.sp14_dir_concat_qaux_kernel)
                .arg(&h_s2_ptr)
                .arg(&aux_softmax_ptr)
                .arg(&out_ptr)
                .arg(&b_i32)
                .arg(&sh2)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("dir_concat_qaux: {e}")))?;
        }
        Ok(())
    }

    /// SP14 Layer B Task B.11 (2026-05-05): per-step launcher for the
    /// EGF q_disagreement producer. Reads `aux_nb_softmax_buf [B, K=2]`
    /// (populated by the captured aux-head forward) and `q_out_buf
    /// [B, total_actions=13]` (populated by `populate_q_out` after the
    /// captured forward), runs the K=4↔K=2 mapped argmax-mismatch
    /// reduction, and updates `ISV[Q_DISAGREEMENT_SHORT_EMA=383]`,
    /// `ISV[Q_DISAGREEMENT_LONG_EMA=384]`, `ISV[Q_DISAGREEMENT_VARIANCE_EMA=389]`.
    ///
    /// Stride `total_actions=13` lets the kernel read the FIRST 4 columns
    /// of each row (direction Q-values) directly from `q_out_buf` without
    /// a stride-stripping copy. The kernel's K=4 loop bound is
    /// independent of stride.
    ///
    /// Launch contract (matches `q_disagreement_update_kernel.cu` header):
    /// single-block, 256 threads, dynamic shmem = `2 × 256 × sizeof(f32) = 2048`
    /// (numerator + count tiles, see kernel header).
    ///
    /// Producer-only — writes ISV slots [383, 384, 389]. Phase C.1
    /// (2026-05-08) deleted the EGF α-machinery consumer; the slots
    /// remain as HEALTH_DIAG diagnostics only.
    pub(crate) fn launch_sp14_q_disagreement_update(
        &self,
        alpha_short: f32,
        alpha_long: f32,
        alpha_var: f32,
    ) -> Result<(), MLError> {
        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_sp14_q_disagreement_update: isv_signals_dev_ptr must be allocated");
        let b_i32 = self.config.batch_size as i32;
        let q_stride = self.total_actions() as i32;
        let aux_softmax_ptr = self.aux_nb_softmax_buf.raw_ptr();
        let q_out_ptr = self.q_out_buf.raw_ptr();
        let isv_ptr = self.isv_signals_dev_ptr;
        // Block size 256 → smem = 2 × 256 × 4 = 2048 bytes (matches the
        // kernel's `extern __shared__ float smem[]` split into num_smem +
        // cnt_smem). Single block; the kernel's strided loop covers any
        // batch size on a single block.
        let block_dim: u32 = 256;
        let shmem_bytes: u32 = 2 * block_dim * std::mem::size_of::<f32>() as u32;
        unsafe {
            self.stream
                .launch_builder(&self.sp14_q_disagreement_update_kernel)
                .arg(&aux_softmax_ptr)
                .arg(&q_out_ptr)
                .arg(&isv_ptr)
                .arg(&b_i32)
                .arg(&q_stride)
                .arg(&alpha_short)
                .arg(&alpha_long)
                .arg(&alpha_var)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (block_dim, 1, 1),
                    shared_mem_bytes: shmem_bytes,
                })
                .map_err(|e| MLError::ModelError(format!("sp14 q_disagreement: {e}")))?;
        }
        Ok(())
    }

    // SP14 Layer C Phase C.1 (2026-05-08): `launch_sp14_alpha_grad_compute`
    // (formerly B.4 α_grad consumer, Schmitt + adaptive k + adaptive β
    // state machine) and `launch_sp14_gradient_hack_detect` (formerly
    // B.5 anti-mesa-optimization circuit breaker) deleted atomically
    // with the kernel files per `feedback_no_legacy_aliases` and
    // `feedback_no_partial_refactor`. Coupling B and the α-machinery
    // chain are gone; SP14 Layer C Phase C.5 wires the new aux-trunk
    // path. The surviving SP14 launcher is `launch_sp14_dir_concat_qaux`
    // (Coupling A, forward feature wire) and `launch_sp14_q_disagreement_update`
    // (HEALTH_DIAG diagnostic).

    /// Build OFI concat for order (d=2) or urgency (d=3) branch.
    ///
    /// Reads vsn_masked[B, SH2] and raw feature vector states[B, SD], appends 3 OFI
    /// features at ofi_base offset, producing [B, SH2+3] in ord_concat_buf / urg_concat_buf.
    ///
    /// For order (branch_idx=2):   OFI features at indices 66, 67, 68 (actual OFI location)
    /// For urgency (branch_idx=3): OFI features at indices 69, 70, 71
    pub(crate) fn launch_concat_ofi(&self, branch_idx: usize, states_ptr: u64) -> Result<(), MLError> {
        // OFI at dims 66-73 in the full state vector (was incorrectly 42/45 — hit portfolio features)
        let (concat_ptr, ofi_base) = match branch_idx {
            2 => (self.ptrs.ord_concat_buf, 66_i32),
            3 => (self.ptrs.urg_concat_buf, 69_i32),
            _ => return Ok(()),
        };
        let b = self.config.batch_size as i32;
        let sh2 = self.config.shared_h2 as i32;
        // states_buf has stride state_dim_padded (pad128), NOT state_dim
        let sd = ml_core::state_layout::STATE_DIM_PADDED as i32;
        let total = b * (sh2 + 3);
        let blocks = ((total as u32 + 255) / 256).max(1);
        let vsn_masked_ptr = self.vsn_masked_buf.raw_ptr();
        unsafe {
            self.stream
                .launch_builder(&self.concat_ofi_kernel)
                .arg(&concat_ptr)
                .arg(&vsn_masked_ptr)
                .arg(&states_ptr)
                .arg(&b)
                .arg(&sh2)
                .arg(&sd)
                .arg(&ofi_base)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("concat_ofi branch {branch_idx}: {e}")))?;
        }
        Ok(())
    }

    /// Run 2-head cross-branch Q attention over the 12 raw Q-values in q_out_buf.
    /// Writes coordinated Q-values to q_coord_buf [batch_size, 12].
    /// Must be called after compute_expected_q has populated q_out_buf.
    pub(crate) fn launch_q_attention(&self, batch_size: usize) -> Result<(), MLError> {
        let b = batch_size as i32;
        let blocks = ((batch_size as u32 + 255) / 256).max(1);
        let q_out_ptr = self.q_out_buf.raw_ptr();
        let q_coord_ptr = self.q_coord_buf.raw_ptr();
        let q_attn_params_ptr = self.q_attn_params.raw_ptr();
        unsafe {
            self.stream
                .launch_builder(&self.q_attn_kernel)
                .arg(&q_out_ptr)
                .arg(&q_coord_ptr)
                .arg(&q_attn_params_ptr)
                .arg(&b)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("cross_branch_q_attention: {e}")))?;
        }
        Ok(())
    }

    /// Raw device pointer to the coordinated Q-value buffer [batch_size, 12].
    /// Valid after `launch_q_attention` has been called.
    pub(crate) fn q_coord_buf_ptr(&self) -> u64 {
        self.q_coord_buf.raw_ptr()
    }

    /// Run cross-branch graph message passing on q_coord_buf [batch_size, 12] in-place.
    /// 4 directed edges: dir→mag, dir→ord, mag→urg, ord→urg.
    /// Must be called AFTER launch_q_attention.
    pub(crate) fn launch_graph_message_pass(&self, batch_size: usize) -> Result<(), MLError> {
        let b = batch_size as i32;
        let blocks = ((batch_size as u32 + 255) / 256).max(1);
        let q_ptr = self.q_coord_buf.raw_ptr();
        let params_ptr = self.graph_params.dev_ptr;
        unsafe {
            self.stream
                .launch_builder(&self.graph_msg_kernel)
                .arg(&q_ptr)
                .arg(&params_ptr)
                .arg(&b)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("graph_message_pass: {e}")))?;
        }
        Ok(())
    }

    /// Run 2-step diffusion denoising on q_coord_buf [batch_size, 12] conditioned on Var[Q].
    ///
    /// Each step: input = [Q_prev; sqrt(Var[Q]) * schedule] -> FC(24->24)-SiLU-FC(24->12) -> Q_next.
    /// Ping-pong with 2 steps (even): a->b (k=0), b->a (k=1). Result ends in buf_a = q_coord_buf.
    /// Using q_coord_buf directly as buf_a eliminates the initial DtoD copy.
    /// Must be called AFTER launch_graph_message_pass. Reads q_var_buf_trainer (populated by
    /// compute_expected_q with the training path's variance output).
    pub(crate) fn launch_q_denoise(&self, batch_size: usize) -> Result<(), MLError> {
        let b = batch_size as i32;
        let blocks = ((batch_size as u32 + 255) / 256).max(1);
        let var_ptr = self.q_var_buf_trainer.raw_ptr();
        let params_ptr = self.denoise_params.dev_ptr;
        // q_var_buf_trainer is sized [B, total_actions] to match the kernel's
        // per-action variance write stride in compute_expected_q. The denoiser
        // MLP only consumes the first D=12 slots per sample, but the launch
        // must pass the true buffer stride so successive samples align.
        let var_stride = self.total_actions() as i32;

        // Use q_coord_buf directly as buf_a (no initial copy needed).
        // With K=2 (even steps): k=0 a->b, k=1 b->a. Result in buf_a = q_coord_buf.
        let buf_a = self.q_coord_buf.raw_ptr();
        let buf_b = self.denoise_buf_a.raw_ptr(); // reuse denoise_buf_a as scratch

        // Layout per step: W1[24,24]=576 + b1[24]=24 + W2[12,24]=288 + b2[12]=12 = 900 floats.
        const STEP_PARAMS: usize = 900;
        // 2 ping-pong steps: k=0(a->b), k=1(b->a). After step 2, result is in buf_a = q_coord_buf.
        for k in 0..2_i32 {
            let (in_ptr, out_ptr) = if k % 2 == 0 { (buf_a, buf_b) } else { (buf_b, buf_a) };
            let step_off = k as usize * STEP_PARAMS;
            let w1_ptr = params_ptr + (step_off * std::mem::size_of::<f32>()) as u64;
            let b1_ptr = w1_ptr + (576 * std::mem::size_of::<f32>()) as u64;
            let w2_ptr = b1_ptr + (24 * std::mem::size_of::<f32>()) as u64;
            let b2_ptr = w2_ptr + (288 * std::mem::size_of::<f32>()) as u64;
            let step_k = k + 1_i32;
            let total_steps = 2_i32;
            unsafe {
                self.stream
                    .launch_builder(&self.q_denoise_kernel)
                    .arg(&in_ptr)
                    .arg(&out_ptr)
                    .arg(&var_ptr)
                    .arg(&w1_ptr)
                    .arg(&b1_ptr)
                    .arg(&w2_ptr)
                    .arg(&b2_ptr)
                    .arg(&b)
                    .arg(&var_stride)
                    .arg(&step_k)
                    .arg(&total_steps)
                    .launch(LaunchConfig {
                        grid_dim: (blocks, 1, 1),
                        block_dim: (256, 1, 1),
                        shared_mem_bytes: 0,
                    })
                    .map_err(|e| MLError::ModelError(format!("q_denoise_step k={}: {e}", k + 1)))?;
            }
        }

        // After 2 steps (even), result is in buf_a = q_coord_buf. No final copy needed.

        Ok(())
    }

    /// Center Q-values in q_out_buf by subtracting their global mean.
    ///
    /// Two-phase GPU operation (no CPU roundtrip):
    ///   Phase 1 (q_mean_reduce):    [B * total_actions] → mean scalar in q_mean_scratch_dev_ptr
    ///   Phase 2 (q_mean_subtract):  q_out_buf[i] -= q_mean_scratch[0]  in-place (pinned, zero copy)
    ///
    /// Prevents bootstrapping drift (Q-mean 0→+0.47 over 18 epochs observed in
    /// train-vpb4w). Must be called AFTER compute_expected_q, BEFORE launch_q_attention.
    pub(crate) fn launch_q_mean_center(&self, batch_size: usize) -> Result<(), MLError> {
        let total = (batch_size * self.total_actions()) as i32;
        let q_ptr = self.q_out_buf.raw_ptr();
        let scratch_ptr = self.q_mean_scratch_dev_ptr;
        // Phase 1: reduce all Q-values to their mean
        unsafe {
            self.stream.launch_builder(&self.q_mean_reduce_kernel)
                .arg(&q_ptr)
                .arg(&scratch_ptr)
                .arg(&total)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("q_mean_reduce: {e}")))?;
        }
        // Phase 2: subtract mean from every Q-value
        let blocks = ((total as u32 + 255) / 256).max(1);
        unsafe {
            self.stream.launch_builder(&self.q_mean_subtract_kernel)
                .arg(&q_ptr)
                .arg(&scratch_ptr)
                .arg(&total)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("q_mean_subtract: {e}")))?;
        }
        Ok(())
    }

    /// Update Q-mean EMA from q_mean_scratch (pinned device-mapped — zero memcpy).
    /// Takes &self because it only writes to raw pinned pointers, not Rust fields.
    pub(crate) fn update_q_mean_ema(&self) {
        let current_mean = unsafe { *self.q_mean_scratch_pinned };
        let alpha = 0.01_f32;
        let old = unsafe { *self.q_mean_ema_pinned };
        let new_ema = (1.0 - alpha) * old + alpha * current_mean;
        unsafe { *self.q_mean_ema_pinned = new_ema; }
    }

    /// Task 2.X "make Full useful" — launch the per-magnitude-bin Q-mean
    /// reducer over the current `q_out_buf`. Writes:
    ///   * `q_mag_means_scratch[0..mag_size]` = mean Q over batch for each
    ///     magnitude action (Quarter=0, Half=1, Full=2).
    ///   * `q_abs_ref_scratch[0]` = max(|mean Q|) across magnitude bins —
    ///     scale reference for the collapse-fraction normaliser.
    ///
    /// These feed ISV slots [13..16] (EMA) via `update_isv_signals`. The
    /// C51 loss / grad kernels then compute a bias-vs-Full signature
    /// `max(0, max(Q_mean) − Q_mean(a1)) / Q_abs_ref` — directly targets
    /// the "higher-variance bins systematically under-priced" bias cited
    /// at experience_kernels.cu:~904 without requiring an inter-branch
    /// reference (which failed during Task 2.X iteration 1 because the
    /// direction branch collapses at the same cadence as magnitude).
    ///
    /// Runs outside CUDA Graph capture on the stats cadence. Single-thread
    /// single-block, ~N * mag_size f32 loads — microseconds at typical B.
    pub(crate) fn launch_q_mag_bin_means_reduce(
        &self,
        batch_size: usize,
    ) -> Result<(), MLError> {
        let total_actions = self.total_actions() as i32;
        let n = batch_size as i32;
        let mag_off = self.config.branch_0_size as i32;
        let mag_size = self.config.branch_1_size as i32;
        let q_out_buf_ptr = self.q_out_buf.raw_ptr();
        unsafe {
            self.stream
                .launch_builder(&self.q_mag_bin_means_reduce_kernel)
                .arg(&q_out_buf_ptr)
                .arg(&self.q_mag_means_scratch_dev_ptr)
                .arg(&self.q_abs_ref_scratch_dev_ptr)
                .arg(&n)
                .arg(&total_actions)
                .arg(&mag_off)
                .arg(&mag_size)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (1, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("q_mag_bin_means_reduce: {e}")))?;
        }
        Ok(())
    }

    /// Task 2.Y "make direction branch useful at eval" — mirror of
    /// `launch_q_mag_bin_means_reduce` applied to the direction branch.
    /// Writes:
    ///   * `q_dir_means_scratch[0..dir_size]` = mean Q over batch for each
    ///     direction action (Short=0, Hold=1, Long=2, Flat=3).
    ///   * `q_dir_abs_ref_scratch[0]` = max(|mean Q|) across direction bins.
    ///
    /// These feed ISV slots [17..21] (EMA) via `update_isv_signals`. The
    /// C51 loss / grad kernels then compute the bias-vs-argmax-direction
    /// composite signal driving `get_direction_bin_weight`.
    ///
    /// Runs outside CUDA Graph capture on the stats cadence. Single-thread
    /// single-block, ~N * dir_size f32 loads — microseconds at typical B.
    pub(crate) fn launch_q_dir_bin_means_reduce(
        &self,
        batch_size: usize,
    ) -> Result<(), MLError> {
        let total_actions = self.total_actions() as i32;
        let n = batch_size as i32;
        /* Direction is the first branch in the factored-action layout —
         * offset 0, size = config.branch_0_size (4 for the 4-branch DQN). */
        let dir_off: i32 = 0;
        let dir_size = self.config.branch_0_size as i32;
        let q_out_buf_ptr = self.q_out_buf.raw_ptr();
        unsafe {
            self.stream
                .launch_builder(&self.q_dir_bin_means_reduce_kernel)
                .arg(&q_out_buf_ptr)
                .arg(&self.q_dir_means_scratch_dev_ptr)
                .arg(&self.q_dir_abs_ref_scratch_dev_ptr)
                .arg(&n)
                .arg(&total_actions)
                .arg(&dir_off)
                .arg(&dir_size)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (1, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("q_dir_bin_means_reduce: {e}")))?;
        }
        Ok(())
    }

    /// Pure GPU ISV signal update — reads all pinned pointers, updates ISV
    /// [ISV_DIM=23] + history [K,12]. Includes 4 regime awareness signals
    /// computed from ADX/CUSUM in the states buffer (batch-aggregated across
    /// all B rows; ISV audit 2026-04-23 replaced the prior sample-0 read
    /// that threw away 99.994% of the regime information). Slots [13..16]
    /// are the Task 2.X "make Full useful" fix — per-magnitude-bin Q-mean
    /// EMAs and absolute-scale reference, fed by `q_mag_bin_means_reduce`
    /// (launched inside `reduce_current_q_stats` just before this kernel).
    ///
    /// The `state_dim` kernel argument is passed as
    /// `ml_core::state_layout::STATE_DIM_PADDED` (128) — the actual stride
    /// of `states_buf`. The previous launch passed the logical
    /// `STATE_DIM` (104), which only produced correct values for row 0 by
    /// accident because both strides place sample 0 at the same byte
    /// offset. With the batch-aggregate fix all B rows must be strided
    /// correctly.
    ///
    /// Call after graph_adam replay and stream sync, before next graph_forward.
    pub(crate) fn update_isv_signals(&self) -> Result<(), MLError> {
        let k = ISV_K as i32;
        let state_dim = ml_core::state_layout::STATE_DIM_PADDED as i32;
        let batch_size_i32 = self.config.batch_size as i32;
        let mag_size_i32 = self.config.branch_1_size as i32;
        let dir_size_i32 = self.config.branch_0_size as i32;
        unsafe {
            self.stream.launch_builder(&self.isv_signal_update_kernel)
                .arg(&self.isv_signals_dev_ptr)
                .arg(&self.isv_history_dev_ptr)
                .arg(&self.lagged_td_error_dev_ptr)
                .arg(&self.q_mean_scratch_dev_ptr)
                .arg(&self.q_mean_ema_dev_ptr)
                .arg(&self.grad_norm_dev_ptr)
                .arg(&self.td_error_scratch_dev_ptr)
                .arg(&self.q_var_scratch_dev_ptr)
                .arg(&self.reward_scratch_dev_ptr)
                .arg(&self.atom_util_scratch_dev_ptr)
                .arg(&self.total_loss_dev_ptr)
                .arg(&k)
                .arg(&self.ptrs.states_buf)   // states for ADX/CUSUM regime signals
                .arg(&state_dim)
                .arg(&batch_size_i32)
                // Task 2.X "make Full useful" — per-magnitude-bin Q-mean
                // scratch array + absolute-scale reference scalar feed ISV
                // slots [13..15] (Q_mean per bin) and [16] (|Q| reference).
                .arg(&self.q_mag_means_scratch_dev_ptr)
                .arg(&self.q_abs_ref_scratch_dev_ptr)
                .arg(&mag_size_i32)
                // Task 2.Y "make direction branch useful at eval" —
                // per-direction-bin Q-mean scratch array + absolute-scale
                // reference scalar feed ISV slots [17..20] (Q_mean per bin)
                // and [21] (|Q| reference for direction).
                .arg(&self.q_dir_means_scratch_dev_ptr)
                .arg(&self.q_dir_abs_ref_scratch_dev_ptr)
                .arg(&dir_size_i32)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (1, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("isv_signal_update: {e}")))?;
        }
        Ok(())
    }

    /// ISV signal update — safe to call inside graph capture.
    /// Reads pinned device-mapped scalars (loss, grad_norm, Q-mean)
    /// written by Adam, updates ISV signal vector [12] + history.
    pub(crate) fn submit_isv_signal_update(&self) -> Result<(), MLError> {
        self.update_isv_signals()
    }

    /// ISV encoder MLP forward: temporal decay → MLP → branch gate + gamma mod.
    /// Reads ISV signals/history (pinned), ISV encoder weights from param buffer.
    /// Writes isv_embedding_buf [ISV_EMB_DIM=8], branch_gate_buf [4], gamma_mod_buf [1].
    pub(crate) fn launch_isv_forward(&self) -> Result<(), MLError> {
        let param_sizes = compute_param_sizes(&self.config);
        let w_fc1 = self.ptrs.params_ptr + padded_byte_offset(&param_sizes, 68);
        let b_fc1 = self.ptrs.params_ptr + padded_byte_offset(&param_sizes, 69);
        let w_fc2 = self.ptrs.params_ptr + padded_byte_offset(&param_sizes, 70);
        let b_fc2 = self.ptrs.params_ptr + padded_byte_offset(&param_sizes, 71);
        let w_gate = self.ptrs.params_ptr + padded_byte_offset(&param_sizes, 72);
        let b_gate = self.ptrs.params_ptr + padded_byte_offset(&param_sizes, 73);
        let w_gamma = self.ptrs.params_ptr + padded_byte_offset(&param_sizes, 74);
        let b_gamma = self.ptrs.params_ptr + padded_byte_offset(&param_sizes, 75);
        let k = ISV_K as i32;
        let isv_embedding_buf_ptr = self.isv_embedding_buf.raw_ptr();
        let branch_gate_buf_ptr = self.branch_gate_buf.raw_ptr();
        let gamma_mod_buf_ptr = self.gamma_mod_buf.raw_ptr();

        unsafe {
            self.stream.launch_builder(&self.isv_forward_kernel)
                .arg(&self.isv_signals_dev_ptr)
                .arg(&self.isv_history_dev_ptr)
                .arg(&self.isv_decay_dev_ptr)
                .arg(&w_fc1).arg(&b_fc1)
                .arg(&w_fc2).arg(&b_fc2)
                .arg(&w_gate).arg(&b_gate)
                .arg(&w_gamma).arg(&b_gamma)
                .arg(&isv_embedding_buf_ptr)
                .arg(&branch_gate_buf_ptr)
                .arg(&gamma_mod_buf_ptr)
                .arg(&k)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (1, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("isv_forward: {e}")))?;
        }
        Ok(())
    }

    /// ISV feature gate: modulate h_s2 features based on ISV embedding.
    /// h_s2[i,j] *= sigmoid(w_feature_gate[j,:] @ isv_embedding + b_feature_gate[j])
    /// Operates in-place on save_h_s2. Must be called AFTER launch_isv_forward.
    pub(crate) fn launch_isv_feature_gate(&self, batch_size: usize) -> Result<(), MLError> {
        let param_sizes = compute_param_sizes(&self.config);
        let w_gate = self.ptrs.params_ptr + padded_byte_offset(&param_sizes, 78);
        let b_gate = self.ptrs.params_ptr + padded_byte_offset(&param_sizes, 79);
        let blocks = ((batch_size as u32 + 255) / 256).max(1);
        let b_i32 = batch_size as i32;
        let sh2 = self.config.shared_h2 as i32;
        let save_h_s2_ptr = self.save_h_s2.raw_ptr();
        let isv_embedding_buf_ptr = self.isv_embedding_buf.raw_ptr();

        unsafe {
            self.stream.launch_builder(&self.isv_feature_gate_kernel)
                .arg(&save_h_s2_ptr)  // h_s2 in-place (WAW dep needed for Hopper graph replay)
                .arg(&isv_embedding_buf_ptr)
                .arg(&w_gate)
                .arg(&b_gate)
                .arg(&b_i32)
                .arg(&sh2)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("isv_feature_gate: {e}")))?;
        }
        Ok(())
    }

    /// ISV temporal routing: ISV embedding [8] → temporal_weight [SH2].
    /// Each feature gets a learned "how much history to use" weight via sigmoid.
    /// Must be called AFTER launch_isv_forward (produces embedding) and BEFORE mamba2_step.
    pub(crate) fn launch_isv_temporal_route(&self) -> Result<(), MLError> {
        let param_sizes = compute_param_sizes(&self.config);
        let w_route = self.ptrs.params_ptr + padded_byte_offset(&param_sizes, 80);
        let b_route = self.ptrs.params_ptr + padded_byte_offset(&param_sizes, 81);
        let sh2 = self.config.shared_h2 as i32;
        let isv_embedding_buf_ptr = self.isv_embedding_buf.raw_ptr();
        let temporal_weight_buf_ptr = self.temporal_weight_buf.raw_ptr();

        unsafe {
            self.stream.launch_builder(&self.isv_temporal_route_kernel)
                .arg(&isv_embedding_buf_ptr)
                .arg(&w_route)
                .arg(&b_route)
                .arg(&temporal_weight_buf_ptr)
                .arg(&sh2)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (1, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("isv_temporal_route: {e}")))?;
        }
        Ok(())
    }

    /// Recursive confidence forward: predict own TD-error from h_s2.
    /// h_s2 → sigmoid(w_conf @ h + b_conf) → predicted_error [B].
    pub(crate) fn launch_recursive_confidence_forward(&self, batch_size: usize) -> Result<(), MLError> {
        let param_sizes = compute_param_sizes(&self.config);
        let w_conf = self.ptrs.params_ptr + padded_byte_offset(&param_sizes, 76);
        let b_conf = self.ptrs.params_ptr + padded_byte_offset(&param_sizes, 77);
        let blocks = ((batch_size as u32 + 255) / 256).max(1);
        let b_i32 = batch_size as i32;
        let sh2 = self.config.shared_h2 as i32;
        let save_h_s2_ptr = self.save_h_s2.raw_ptr();
        let predicted_error_buf_ptr = self.predicted_error_buf.raw_ptr();

        unsafe {
            self.stream.launch_builder(&self.recursive_conf_fwd_kernel)
                .arg(&save_h_s2_ptr)
                .arg(&w_conf)
                .arg(&b_conf)
                .arg(&predicted_error_buf_ptr)
                .arg(&b_i32)
                .arg(&sh2)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("recursive_confidence_forward: {e}")))?;
        }
        Ok(())
    }

    /// Trade plan forward (cuBLAS 2-layer MLP): h_s2 → hidden[AH] (ReLU) → plan_params[B, 6].
    /// Output activations: target_bars[3-25], profit_target[0.05-1.5%],
    /// stop_loss[0.02-0.5%], scale_aggression[0-1], conviction[0-1], asymmetry[0.5-3.0].
    ///
    /// Decomposed into:
    ///   1. cuBLAS SGEMM: hidden[B, AH] = W_fc[AH, SH2] @ h_s2[B, SH2]^T  (via sgemm_f32)
    ///   2. Elementwise: bias + ReLU in-place on hidden
    ///   3. cuBLAS SGEMM: pre_out[B, 6] = W_out[6, AH] @ hidden[B, AH]^T
    ///   4. Elementwise: bias + sigmoid/scaling activations → plan_params
    pub(crate) fn launch_trade_plan_forward(&self, batch_size: usize) -> Result<(), MLError> {
        let param_sizes = compute_param_sizes(&self.config);
        let w_fc = self.ptrs.params_ptr + padded_byte_offset(&param_sizes, 82);
        let b_fc = self.ptrs.params_ptr + padded_byte_offset(&param_sizes, 83);
        let w_out = self.ptrs.params_ptr + padded_byte_offset(&param_sizes, 84);
        let b_out = self.ptrs.params_ptr + padded_byte_offset(&param_sizes, 85);
        let ah = self.config.adv_h;
        let sh2 = self.config.shared_h2;
        let save_h_s2_ptr = self.save_h_s2.raw_ptr();
        let hidden_ptr = self.trade_plan_hidden_buf.raw_ptr();
        let pre_out_ptr = self.trade_plan_pre_out_buf.raw_ptr();
        let plan_params_ptr = self.plan_params_buf.raw_ptr();

        // 1. cuBLAS GEMM: hidden[B, AH] = h_s2[B, SH2] @ W_fc^T
        //    sgemm_f32 handles row-major trick: W[AH, SH2] @ input[B, SH2] → output[B, AH]
        self.cublas_forward.sgemm_f32(
            &self.stream, w_fc, save_h_s2_ptr, hidden_ptr,
            ah, batch_size, sh2, "plan_l1",
        )?;

        // 2. Bias + ReLU in-place on hidden[B, AH]
        self.cublas_forward.launch_add_bias_relu_f32_raw(
            &self.stream, hidden_ptr, b_fc, ah, batch_size,
        )?;

        // 3. cuBLAS GEMM: pre_out[B, 6] = hidden[B, AH] @ W_out^T
        self.cublas_forward.sgemm_f32(
            &self.stream, w_out, hidden_ptr, pre_out_ptr,
            6, batch_size, ah, "plan_l2",
        )?;

        // 4. Bias + sigmoid/scaling activations → plan_params[B, 6]
        let blocks = ((batch_size as u32 + 255) / 256).max(1);
        let b_i32 = batch_size as i32;
        unsafe {
            self.stream.launch_builder(&self.trade_plan_activate_kernel)
                .arg(&pre_out_ptr)
                .arg(&b_out)
                .arg(&plan_params_ptr)
                .arg(&b_i32)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("trade_plan_activate: {e}")))?;
        }
        Ok(())
    }

    /// Plan noise injection: ±noise_scale multiplicative noise on plan_params [B, 6].
    /// Creates temporal ensemble diversity — the model sees slightly different plans
    /// each step, learning robust behaviour.  Deterministic via Philox hash on adam_step.
    pub(crate) fn launch_plan_noise_inject(&self, batch_size: usize) -> Result<(), MLError> {
        let blocks = ((batch_size as u32 + 255) / 256).max(1);
        let b_i32 = batch_size as i32;
        let step = self.adam_step;
        let noise_scale = 0.05_f32; // ±5%
        let plan_params_buf_ptr = self.plan_params_buf.raw_ptr();

        unsafe {
            self.stream.launch_builder(&self.plan_noise_kernel)
                .arg(&plan_params_buf_ptr)
                .arg(&b_i32)
                .arg(&step)
                .arg(&noise_scale)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("plan_noise_inject: {e}")))?;
        }
        Ok(())
    }

    /// OFI embedding MLP forward: extract OFI_EMBED_IN OFI features from
    /// states, project to OFI_EMBED_DIM.
    ///
    /// Input: states_buf [B, state_dim] → extracts the full OFI slice at
    /// [SL_OFI_START..SL_OFI_START+OFI_EMBED_IN).
    /// Output: ofi_embed_output_buf [B, OFI_EMBED_DIM] (bias + ReLU activated).
    pub(crate) fn launch_ofi_embed_forward(&self, batch_size: usize) -> Result<(), MLError> {
        let sd = ml_core::state_layout::STATE_DIM as i32;
        let b_i32 = batch_size as i32;

        // Step 1: Build input [B, OFI_EMBED_IN] from states_buf
        let states_ptr = self.states_buf.raw_ptr();
        let input_ptr = self.ofi_embed_input_buf.raw_ptr();
        let blocks = ((batch_size as u32 + 255) / 256).max(1);
        unsafe {
            self.stream.launch_builder(&self.ofi_embed_build_input_kernel)
                .arg(&states_ptr)
                .arg(&input_ptr)
                .arg(&b_i32)
                .arg(&sd)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("ofi_embed_build_input: {e}")))?;
        }

        // Step 2: cuBLAS SGEMM: output[B, OFI_EMBED_DIM] = input[B, OFI_EMBED_IN] @ W^T[OFI_EMBED_IN, OFI_EMBED_DIM]
        // sgemm_f32 uses row-major trick: W[OFI_EMBED_DIM, OFI_EMBED_IN] row-major,
        // input [B, OFI_EMBED_IN], output [B, OFI_EMBED_DIM]
        let w_ptr = self.ofi_embed_w.raw_ptr();
        let output_ptr = self.ofi_embed_output_buf.raw_ptr();
        self.cublas_forward.sgemm_f32(
            &self.stream, w_ptr, input_ptr, output_ptr,
            OFI_EMBED_DIM, batch_size, OFI_EMBED_IN, "ofi_embed_fwd",
        )?;

        // Step 3: Add bias + ReLU in-place on output[B, OFI_EMBED_DIM]
        let bias_ptr = self.ofi_embed_b.raw_ptr();
        self.cublas_forward.launch_add_bias_relu_f32_raw(
            &self.stream, output_ptr, bias_ptr, OFI_EMBED_DIM, batch_size,
        )?;

        Ok(())
    }

    /// OFI embed MLP backward pass.
    ///
    /// Accumulates gradients from two sources:
    /// - `d_ofi_embed_mamba2 [OFI_EMBED_DIM, B]` — from Mamba2 backward (self.d_ofi_embed_mamba2)
    /// - `d_ofi_embed_attn [OFI_EMBED_DIM, B]` — from attention backward (passed as raw pointer)
    ///
    /// Pipeline:
    ///   1. d_combined = d_mamba2 + d_attn (graph-safe copy + SAXPY via add kernel)
    ///   2. ReLU mask: d_combined *= (ofi_embed_output > 0)
    ///   3. dW[OFI_EMBED_DIM, OFI_EMBED_IN] = d_combined[OFI_EMBED_DIM, B] @ ofi_input[OFI_EMBED_IN, B]^T (cuBLAS backward)
    ///   4. db[OFI_EMBED_DIM] = sum_B(d_combined) (2-phase bias grad reduce)
    ///   5. Assemble contiguous grad[OFI_EMBED_TOTAL_PARAMS] = [grad_w[OFI_EMBED_W_COUNT]; grad_b[OFI_EMBED_DIM]]
    ///   6. Copy params W+b back to ofi_embed_w/ofi_embed_b (keep in sync)
    pub(crate) fn launch_ofi_embed_backward(&mut self, batch_size: usize, d_ofi_attn_ptr: Option<u64>) -> Result<(), MLError> {
        let b = batch_size;
        let n_embed = OFI_EMBED_DIM * b;

        // Step 1: d_combined = d_mamba2 [+ d_attn if attention is present]
        let d_mamba2_ptr = self.d_ofi_embed_mamba2.raw_ptr();
        let d_combined_ptr = self.d_ofi_embed_combined.raw_ptr();

        // Copy d_mamba2 → d_combined (graph-safe kernel copy)
        self.graph_safe_copy_f32(d_combined_ptr, d_mamba2_ptr, n_embed * 4, "d_ofi_mamba2→combined")?;

        // Add d_attn into d_combined if attention gradient is available
        if let Some(attn_ptr) = d_ofi_attn_ptr {
            // dqn_saxpy_kernel signature: (y, x, alpha, n) → y[i] += alpha * x[i]
            let alpha_one: f32 = 1.0;
            let n_i32 = n_embed as i32;
            unsafe {
                self.stream.launch_builder(&self.saxpy_kernel)
                    .arg(&d_combined_ptr)
                    .arg(&attn_ptr)
                    .arg(&alpha_one)
                    .arg(&n_i32)
                    .launch(LaunchConfig::for_num_elems(n_embed as u32))
                    .map_err(|e| MLError::ModelError(format!("ofi_embed saxpy d_attn: {e}")))?;
            }
        }

        // Step 2: ReLU mask — zero gradient where forward output was <= 0
        // ofi_embed_output_buf holds post-bias+ReLU activations from forward
        let act_ptr = self.ofi_embed_output_buf.raw_ptr();
        let n_relu = n_embed as i32;
        let blocks_relu = ((n_embed + 255) / 256) as u32;
        unsafe {
            self.stream
                .launch_builder(&self.relu_mask_kernel)
                .arg(&d_combined_ptr)
                .arg(&act_ptr)
                .arg(&n_relu)
                .launch(LaunchConfig {
                    grid_dim: (blocks_relu, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("ofi_embed relu_mask: {e}")))?;
        }

        // Step 3: dW[OFI_EMBED_DIM, OFI_EMBED_IN] = d_combined[OFI_EMBED_DIM, B] @ ofi_input[OFI_EMBED_IN, B]^T
        //
        // Forward used row-major W[OFI_EMBED_DIM, OFI_EMBED_IN], input[B, OFI_EMBED_IN], output[B, OFI_EMBED_DIM].
        // In col-major cuBLAS terms:
        //   d_combined is col-major [OFI_EMBED_DIM, B], ofi_input is col-major [OFI_EMBED_IN, B]
        //   dW_col[OFI_EMBED_DIM, OFI_EMBED_IN] = d_combined @ ofi_input^T
        //   → TRANSA=N, TRANSB=T, m=OFI_EMBED_DIM, n=OFI_EMBED_IN, k=B
        let grad_w_ptr = self.ofi_embed_grad_w.raw_ptr();
        let input_ptr = self.ofi_embed_input_buf.raw_ptr();
        {
            // Per-stream lt_handle (Option C determinism fix). Main stream — fast path.
            let (lt_handle, lt_ws_ptr, lt_ws_size) =
                self.shared_cublas.lt_for(&self.stream)?;
            let cu_stream = self.stream.cu_stream() as cublaslt_sys::cudaStream_t;
            let alpha: f32 = 1.0;
            let beta: f32 = 0.0;

            let f32_type = cublaslt_sys::cudaDataType_t::CUDA_R_32F;
            let compute_type = cublaslt_sys::cublasComputeType_t::CUBLAS_COMPUTE_32F_FAST_TF32;
            let op_n: i32 = 0; // CUBLAS_OP_N
            let op_t: i32 = 1; // CUBLAS_OP_T
            let m = OFI_EMBED_DIM;
            let n = OFI_EMBED_IN;
            let k = b;

            unsafe {
                let matmul_desc = cublaslt_result::create_matmul_desc(compute_type, f32_type)
                    .map_err(|e| MLError::ModelError(format!("ofi_embed dW matmul_desc: {e:?}")))?;
                cublaslt_result::set_matmul_desc_attribute(
                    matmul_desc,
                    cublaslt_sys::cublasLtMatmulDescAttributes_t::CUBLASLT_MATMUL_DESC_TRANSA,
                    &op_n as *const i32 as *const std::ffi::c_void,
                    std::mem::size_of::<i32>(),
                ).map_err(|e| MLError::ModelError(format!("ofi_embed dW TRANSA: {e:?}")))?;
                cublaslt_result::set_matmul_desc_attribute(
                    matmul_desc,
                    cublaslt_sys::cublasLtMatmulDescAttributes_t::CUBLASLT_MATMUL_DESC_TRANSB,
                    &op_t as *const i32 as *const std::ffi::c_void,
                    std::mem::size_of::<i32>(),
                ).map_err(|e| MLError::ModelError(format!("ofi_embed dW TRANSB: {e:?}")))?;

                // A = d_combined [OFI_EMBED_DIM, B] col-major, lda=OFI_EMBED_DIM
                let a_layout = cublaslt_result::create_matrix_layout(
                    f32_type, m as u64, k as u64, m as i64,
                ).map_err(|e| MLError::ModelError(format!("ofi_embed dW A layout: {e:?}")))?;
                // B = ofi_input [OFI_EMBED_IN, B] col-major (transposed), physical [OFI_EMBED_IN, B], ldb=OFI_EMBED_IN
                let b_layout = cublaslt_result::create_matrix_layout(
                    f32_type, n as u64, k as u64, n as i64,
                ).map_err(|e| MLError::ModelError(format!("ofi_embed dW B layout: {e:?}")))?;
                // C = dW [OFI_EMBED_DIM, OFI_EMBED_IN] col-major, ldc=OFI_EMBED_DIM
                let c_layout = cublaslt_result::create_matrix_layout(
                    f32_type, m as u64, n as u64, m as i64,
                ).map_err(|e| MLError::ModelError(format!("ofi_embed dW C layout: {e:?}")))?;

                cublaslt_sys::cublasLtMatmul(
                    lt_handle,
                    matmul_desc,
                    &alpha as *const f32 as *const std::ffi::c_void,
                    d_combined_ptr as *const std::ffi::c_void,
                    a_layout,
                    input_ptr as *const std::ffi::c_void,
                    b_layout,
                    &beta as *const f32 as *const std::ffi::c_void,
                    grad_w_ptr as *mut std::ffi::c_void,
                    c_layout,
                    grad_w_ptr as *mut std::ffi::c_void,
                    c_layout,  // D layout = C layout
                    std::ptr::null(),  // algo (NULL = heuristic)
                    lt_ws_ptr as *mut std::ffi::c_void,
                    lt_ws_size,
                    cu_stream,
                );

                // Cleanup descriptors
                cublaslt_sys::cublasLtMatmulDescDestroy(matmul_desc);
                cublaslt_sys::cublasLtMatrixLayoutDestroy(a_layout);
                cublaslt_sys::cublasLtMatrixLayoutDestroy(b_layout);
                cublaslt_sys::cublasLtMatrixLayoutDestroy(c_layout);
            }
        }

        // Step 4: db[OFI_EMBED_DIM] = sum_B(d_combined) — 2-phase bias grad reduce
        let bias_num_blocks = ((b + 255) / 256) as i32;
        let partials_ptr = self.ofi_embed_bias_grad_partials.raw_ptr();
        let grad_b_ptr = self.ofi_embed_grad_b.raw_ptr();
        let dim_i32 = OFI_EMBED_DIM as i32;
        unsafe {
            self.stream.launch_builder(&self.denoise_bias_grad_p1_kernel)
                .arg(&d_combined_ptr)
                .arg(&partials_ptr)
                .arg(&dim_i32)
                .arg(&(b as i32))
                .launch(LaunchConfig {
                    grid_dim: (bias_num_blocks as u32, OFI_EMBED_DIM as u32, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("ofi_embed_bias_grad_p1: {e}")))?;
        }
        unsafe {
            self.stream.launch_builder(&self.denoise_bias_grad_p2_kernel)
                .arg(&partials_ptr)
                .arg(&grad_b_ptr)
                .arg(&dim_i32)
                .arg(&bias_num_blocks)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("ofi_embed_bias_grad_p2: {e}")))?;
        }

        // Step 5: Assemble contiguous grad[OFI_EMBED_TOTAL_PARAMS]
        //         = [grad_w[OFI_EMBED_W_COUNT]; grad_b[OFI_EMBED_DIM]]
        let grad_ptr = self.ofi_embed_grad.raw_ptr();
        let w_bytes = OFI_EMBED_W_COUNT * std::mem::size_of::<f32>();
        let b_bytes = OFI_EMBED_DIM * std::mem::size_of::<f32>();
        self.graph_safe_copy_f32(grad_ptr, grad_w_ptr, w_bytes, "ofi_embed_grad_w→grad")?;
        self.graph_safe_copy_f32(grad_ptr + w_bytes as u64, grad_b_ptr, b_bytes, "ofi_embed_grad_b→grad")?;

        Ok(())
    }

    /// Adam optimizer step for ofi_embed_params [OFI_EMBED_TOTAL_PARAMS]. Same pattern as step_denoise_adam.
    pub(crate) fn step_ofi_embed_adam(&mut self) -> Result<(), MLError> {
        self.ofi_embed_adam_step += 1;
        let step_val = self.ofi_embed_adam_step;
        let n = OFI_EMBED_TOTAL_PARAMS as i32;

        // Write step counter to pinned device-mapped memory
        unsafe { *self.ofi_embed_t_pinned = step_val; }

        // Phase 1: grad_norm on ofi_embed_grad
        let grad_ptr = self.ofi_embed_grad.raw_ptr();
        let partials_ptr = self.ofi_embed_norm_partials.raw_ptr();
        unsafe {
            self.stream
                .launch_builder(&self.grad_norm_standalone_post_aux)
                .arg(&grad_ptr)
                .arg(&partials_ptr)
                .arg(&n)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("ofi_embed grad_norm: {e}")))?;
        }

        // Phase 2: grad_norm_finalize
        let norm_out_ptr = self.ofi_embed_norm_buf.raw_ptr();
        let nb = 1_i32;
        unsafe {
            self.stream
                .launch_builder(&self.grad_norm_finalize_post_aux)
                .arg(&partials_ptr)
                .arg(&norm_out_ptr)
                .arg(&nb)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("ofi_embed grad_norm_finalize: {e}")))?;
        }

        // Phase 3: Adam update on ofi_embed_params
        let aux_lr_ptr = self.aux_lr_dev_ptr;  // pinned device-mapped 1e-4
        let beta1: f32 = 0.9;
        let beta2: f32 = 0.999;
        let eps: f32 = 1e-8;
        let weight_decay: f32 = 0.0;
        let params_ptr = self.ofi_embed_params.raw_ptr();
        let m_ptr = self.ofi_embed_adam_m.raw_ptr();
        let v_ptr = self.ofi_embed_adam_v.raw_ptr();
        let clip_ptr = self.ofi_embed_clip_buf.raw_ptr();
        let t_ptr = self.ofi_embed_t_dev_ptr;
        let wd_mask_ptr = self.weight_decay_mask.raw_ptr();
        let l1_end_aux: i32 = 0;
        let l1_lambda_aux: f32 = 0.0;
        // SP4 Layer B: aux trainer (OFI embed) is outside the SP4 8-group
        // taxonomy — disable Mech 9 weight clamp here (mirrors DT pattern).
        // Setting `weight_clamp_max_abs = 0.0` triggers the kernel's
        // `if (bound > 0.0f)` short-circuit; Adam behaves as before for
        // ofi_embed without unbounded ISV-driven clamps that would
        // require an SP4 producer this aux head doesn't have.
        let weight_clamp_max_abs: f32 = 0.0_f32;
        // SP4 Task A14: aux trainer (OFI embed) is outside the SP4 8-group
        // taxonomy — disable Pearl C engagement counter for this launch.
        let _aux_nan_flags_null: u64 = 0;
        let _aux_diag_slot_disabled: i32 = -1;
        let _aux_engage_buf_null: u64 = 0;
        let _aux_engage_off_disabled: i32 = SP4_ENGAGE_OFFSET_DISABLED;
        let blocks = ((OFI_EMBED_TOTAL_PARAMS as u32 + 255) / 256).max(1);
        unsafe {
            self.stream
                .launch_builder(&self.adam_update_post_aux)
                .arg(&params_ptr)
                .arg(&grad_ptr)
                .arg(&m_ptr)
                .arg(&v_ptr)
                .arg(&norm_out_ptr)
                .arg(&aux_lr_ptr)
                .arg(&beta1)
                .arg(&beta2)
                .arg(&eps)
                .arg(&weight_decay)
                .arg(&clip_ptr)
                .arg(&t_ptr)
                .arg(&n)
                .arg(&wd_mask_ptr)
                .arg(&l1_end_aux)
                .arg(&l1_lambda_aux)
                .arg(&weight_clamp_max_abs) // SP3 Mech 9: post-Adam |p_val| clamp
                // SP4 Task A14: aux trainers (OFI embed) live outside the
                // SP4 8-group taxonomy — pass null nan_flags + disabled
                // engage_buf_offset so the kernel skips per-block writeback.
                .arg(&_aux_nan_flags_null)
                .arg(&_aux_diag_slot_disabled)
                .arg(&_aux_engage_buf_null)
                .arg(&_aux_engage_off_disabled)
                .arg(&1.0_f32) // SP21 Phase 4: lr_scale (aux trainer, non-branch = 1.0 no-op)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("ofi_embed Adam update: {e}")))?;
        }

        // Phase 4: Copy updated params back to ofi_embed_w and ofi_embed_b
        // so forward pass uses the latest weights.
        let w_dst = self.ofi_embed_w.raw_ptr();
        let b_dst = self.ofi_embed_b.raw_ptr();
        let w_bytes = OFI_EMBED_W_COUNT * std::mem::size_of::<f32>();
        let b_bytes = OFI_EMBED_DIM * std::mem::size_of::<f32>();
        self.graph_safe_copy_f32(w_dst, params_ptr, w_bytes, "ofi_embed_params→w")?;
        self.graph_safe_copy_f32(b_dst, params_ptr + w_bytes as u64, b_bytes, "ofi_embed_params→b")?;

        Ok(())
    }

    /// SP14 Layer C Phase C.5a (2026-05-08): aux trunk Adam optimizer step.
    ///
    /// **DEAD CODE.** No production caller invokes this in C.5a — the
    /// launcher exists so the build stays green while C.5b is the
    /// genuine atomic contract migration that wires the aux head
    /// backward → `dh_s2_aux_accum` → aux trunk backward → THIS launcher
    /// chain. Per `feedback_no_partial_refactor.md` the wire-up is one
    /// commit. Per `feedback_no_stubs.md` the body is fully real (no
    /// placeholder panic); a synthetic call from a unit test would
    /// execute end-to-end correctly given populated grad buffers.
    ///
    /// Reads ISV-driven hyper-parameters (no hardcoded constants):
    ///
    /// | ISV slot                         | Use                       |
    /// |----------------------------------|---------------------------|
    /// | `AUX_TRUNK_LR_INDEX = 444`       | Adam learning rate        |
    /// | `AUX_TRUNK_BETA1_INDEX = 445`    | Adam β1 (first moment)    |
    /// | `AUX_TRUNK_BETA2_INDEX = 446`    | Adam β2 (second moment)   |
    /// | `AUX_TRUNK_EPS_INDEX = 447`      | Adam ε (denom stability)  |
    /// | `AUX_TRUNK_GRAD_CLIP_INDEX = 448`| Global L2 clip threshold  |
    ///
    /// Mirrors the OFI embed Adam pattern (`step_ofi_embed_adam`) but
    /// over 6 separate (params, grad, m, v) tensor tuples instead of one
    /// flat buffer. The aux trunk lives outside the SP4 8-group
    /// taxonomy (Q/Target/CQL/Aux per-branch + auxiliaries) so a
    /// dedicated launcher is appropriate; SP4 Mech 9 weight clamp
    /// (`weight_clamp_max_abs`) and Pearl C engagement counter
    /// (`engage_buf_offset`) are disabled — same convention as OFI
    /// embed.
    ///
    /// Phase 1: per-tensor partial sum-of-squares into the unified
    /// `aux_trunk_grad_norm_partials` buffer at fixed offsets
    /// (`aux_trunk_grad_block_offsets[i..i+1]` slice covers tensor `i`'s
    /// blocks).
    ///
    /// Phase 2: a single `dqn_grad_norm_finalize` reduces all blocks to
    /// `aux_trunk_grad_norm_buf[0] = sqrt(sum_b partial_b)`.
    ///
    /// Phase 3: 6 `dqn_adam_update_kernel` launches, each consuming one
    /// (params, grad, m, v) tuple. The kernel's internal
    /// `clip_scale = (norm > clip) ? clip/norm : 1` divides by the
    /// SAME global norm (read by every launch), so clipping is
    /// consistent across tensors.
    ///
    /// `batch_size` is unused by this launcher (Adam is per-parameter,
    /// not per-batch). Kept in the signature so call-site code reads
    /// uniformly with other Adam launchers in this trainer.
    ///
    /// Per `pearl_no_host_branches_in_captured_graph.md` the launcher
    /// only touches pre-loaded `CudaFunction` handles
    /// (`grad_norm_standalone_post_aux`, `grad_norm_finalize_post_aux`,
    /// `adam_update_post_aux`) and pre-allocated buffers — no
    /// `load_cubin` / `load_function` on the hot path.
    #[allow(dead_code)]
    pub(crate) fn launch_aux_trunk_adam_update(
        &self,
        stream: &Arc<CudaStream>,
        _batch_size: i32,
    ) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp14_isv_slots::{
            AUX_TRUNK_BETA1_INDEX, AUX_TRUNK_BETA2_INDEX, AUX_TRUNK_EPS_INDEX,
        };

        // Adam hparams — ISV-driven with cold-start floors per
        // `pearl_first_observation_bootstrap.md` numerical-stability
        // carve-outs (β1≥0.5 avoids momentum inversion; β2≥0.9 prevents
        // exploding second-moment; ε≥1e-12 avoids div-by-zero).
        let beta1 = self
            .read_isv_signal_at(AUX_TRUNK_BETA1_INDEX)
            .max(0.5_f32)
            .min(0.9999_f32);
        let beta2 = self
            .read_isv_signal_at(AUX_TRUNK_BETA2_INDEX)
            .max(0.9_f32)
            .min(0.99999_f32);
        let epsilon = self.read_isv_signal_at(AUX_TRUNK_EPS_INDEX).max(1e-12_f32);
        let weight_decay: f32 = 0.0; // aux trunk is outside AdamW WD policy

        // Per-tensor (params, grad, m, v, extent) tuples in the same
        // order as `aux_trunk_grad_block_offsets`.
        let tensors: [(u64, u64, u64, u64, usize); 6] = [
            (
                self.aux_trunk_w1.raw_ptr(),
                self.aux_trunk_w1_grad.raw_ptr(),
                self.aux_trunk_w1_m.raw_ptr(),
                self.aux_trunk_w1_v.raw_ptr(),
                self.aux_trunk_w1.len(),
            ),
            (
                self.aux_trunk_b1.raw_ptr(),
                self.aux_trunk_b1_grad.raw_ptr(),
                self.aux_trunk_b1_m.raw_ptr(),
                self.aux_trunk_b1_v.raw_ptr(),
                self.aux_trunk_b1.len(),
            ),
            (
                self.aux_trunk_w2.raw_ptr(),
                self.aux_trunk_w2_grad.raw_ptr(),
                self.aux_trunk_w2_m.raw_ptr(),
                self.aux_trunk_w2_v.raw_ptr(),
                self.aux_trunk_w2.len(),
            ),
            (
                self.aux_trunk_b2.raw_ptr(),
                self.aux_trunk_b2_grad.raw_ptr(),
                self.aux_trunk_b2_m.raw_ptr(),
                self.aux_trunk_b2_v.raw_ptr(),
                self.aux_trunk_b2.len(),
            ),
            (
                self.aux_trunk_w3.raw_ptr(),
                self.aux_trunk_w3_grad.raw_ptr(),
                self.aux_trunk_w3_m.raw_ptr(),
                self.aux_trunk_w3_v.raw_ptr(),
                self.aux_trunk_w3.len(),
            ),
            (
                self.aux_trunk_b3.raw_ptr(),
                self.aux_trunk_b3_grad.raw_ptr(),
                self.aux_trunk_b3_m.raw_ptr(),
                self.aux_trunk_b3_v.raw_ptr(),
                self.aux_trunk_b3.len(),
            ),
        ];

        let partials_base = self.aux_trunk_grad_norm_partials.raw_ptr();
        let f32_sz = std::mem::size_of::<f32>() as u64;

        // ── Phase 1: per-tensor partial sum-of-squares ────────────────
        // Each launch writes `extent.div_ceil(256)` blocks into the
        // partials buffer at the per-tensor offset.
        for (i, &(_p, grad_ptr, _m, _v, extent)) in tensors.iter().enumerate() {
            let blocks = extent.div_ceil(256).max(1) as u32;
            let off = self.aux_trunk_grad_block_offsets[i] as u64;
            let slot_ptr = partials_base + off * f32_sz;
            let n_i32 = extent as i32;
            unsafe {
                stream
                    .launch_builder(&self.grad_norm_standalone_post_aux)
                    .arg(&grad_ptr)
                    .arg(&slot_ptr)
                    .arg(&n_i32)
                    .launch(LaunchConfig {
                        grid_dim: (blocks, 1, 1),
                        block_dim: (256, 1, 1),
                        shared_mem_bytes: 0,
                    })
                    .map_err(|e| {
                        MLError::ModelError(format!(
                            "aux_trunk grad_norm[tensor={}, blocks={}]: {e}",
                            i, blocks
                        ))
                    })?;
            }
        }

        // ── Phase 2: finalize across all 6 tensors' blocks ────────────
        let total_blocks = self.aux_trunk_grad_block_offsets[6] as i32;
        let norm_out_ptr = self.aux_trunk_grad_norm_buf.raw_ptr();
        unsafe {
            stream
                .launch_builder(&self.grad_norm_finalize_post_aux)
                .arg(&partials_base)
                .arg(&norm_out_ptr)
                .arg(&total_blocks)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| {
                    MLError::ModelError(format!("aux_trunk grad_norm_finalize: {e}"))
                })?;
        }

        // ── Phase 3: per-tensor Adam updates ──────────────────────────
        // All 6 launches consume the SAME `norm_out_ptr` so the kernel's
        // internal `clip_scale = (norm > max_grad_norm) ? max_grad_norm/norm : 1`
        // applies a globally consistent scale. `aux_trunk_grad_clip_dev_ptr`
        // is the ISV-driven threshold (host writes per-step).
        let lr_ptr = self.aux_trunk_lr_dev_ptr;
        let clip_ptr = self.aux_trunk_grad_clip_dev_ptr;
        let t_ptr = self.aux_trunk_t_dev_ptr;
        let wd_mask_ptr = self.weight_decay_mask.raw_ptr();
        let l1_end: i32 = 0;
        let l1_lambda: f32 = 0.0;
        let weight_clamp_max_abs: f32 = 0.0; // SP4 outside-taxonomy: disabled
        // SP4 Task A14: aux trunk lives outside the 8-group taxonomy —
        // pass null nan_flags + disabled engage offset (mirrors OFI embed).
        let nan_flags_null: u64 = 0;
        let diag_slot_disabled: i32 = -1;
        let engage_buf_null: u64 = 0;
        let engage_off_disabled: i32 = SP4_ENGAGE_OFFSET_DISABLED;

        for (i, &(params_ptr, grad_ptr, m_ptr, v_ptr, extent)) in tensors.iter().enumerate() {
            let n_i32 = extent as i32;
            let blocks = (extent.div_ceil(256)).max(1) as u32;
            unsafe {
                stream
                    .launch_builder(&self.adam_update_post_aux)
                    .arg(&params_ptr)
                    .arg(&grad_ptr)
                    .arg(&m_ptr)
                    .arg(&v_ptr)
                    .arg(&norm_out_ptr) // global L2 norm (shared across all 6)
                    .arg(&lr_ptr)
                    .arg(&beta1)
                    .arg(&beta2)
                    .arg(&epsilon)
                    .arg(&weight_decay)
                    .arg(&clip_ptr) // ISV-driven global clip threshold
                    .arg(&t_ptr)
                    .arg(&n_i32)
                    .arg(&wd_mask_ptr) // unused (wd=0); kernel still reads — pass real ptr
                    .arg(&l1_end)
                    .arg(&l1_lambda)
                    .arg(&weight_clamp_max_abs)
                    .arg(&nan_flags_null)
                    .arg(&diag_slot_disabled)
                    .arg(&engage_buf_null)
                    .arg(&engage_off_disabled)
                    .arg(&1.0_f32) // SP21 Phase 4: lr_scale (aux trunk Adam, non-branch = 1.0 no-op)
                    .launch(LaunchConfig {
                        grid_dim: (blocks, 1, 1),
                        block_dim: (256, 1, 1),
                        shared_mem_bytes: 0,
                    })
                    .map_err(|e| {
                        MLError::ModelError(format!(
                            "aux_trunk adam_update[tensor={}, n={}]: {e}",
                            i, extent
                        ))
                    })?;
            }
        }

        Ok(())
    }

    /// Recursive confidence backward: MSE loss gradient into trunk + conf weight gradients.
    /// Accumulates into grad_buf (same buffer Adam reads) and bw_d_h_s2 trunk gradient.
    pub(crate) fn launch_recursive_confidence_backward(&self, batch_size: usize) -> Result<(), MLError> {
        let param_sizes = compute_param_sizes(&self.config);
        let w_conf = self.ptrs.params_ptr + padded_byte_offset(&param_sizes, 76);
        // Gradient accumulators for w_conf and b_conf in main grad_buf
        let d_w_conf = self.ptrs.grad_buf + padded_byte_offset(&param_sizes, 76);
        let d_b_conf = self.ptrs.grad_buf + padded_byte_offset(&param_sizes, 77);

        let blocks = ((batch_size as u32 + 255) / 256).max(1);
        let b_i32 = batch_size as i32;
        let sh2 = self.config.shared_h2 as i32;
        let loss_weight = 0.01_f32;
        let save_h_s2_ptr = self.save_h_s2.raw_ptr();
        let predicted_error_buf_ptr = self.predicted_error_buf.raw_ptr();
        let recursive_conf_partials_ptr = self.recursive_conf_partials.raw_ptr();
        let bw_d_h_s2_ptr = self.bw_d_h_s2.raw_ptr();

        unsafe {
            // Stage 1: per-block partial reduction (no atomicAdd)
            self.stream.launch_builder(&self.recursive_conf_bwd_kernel)
                .arg(&save_h_s2_ptr)
                .arg(&predicted_error_buf_ptr)
                .arg(&self.lagged_td_error_dev_ptr)
                .arg(&w_conf)
                .arg(&recursive_conf_partials_ptr)
                .arg(&bw_d_h_s2_ptr)
                .arg(&b_i32)
                .arg(&sh2)
                .arg(&loss_weight)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("recursive_confidence_backward: {e}")))?;

            // Stage 2: deterministic reduce across block partials
            let num_blocks_i32 = blocks as i32;
            let total_params = sh2 + 1;
            let reduce_blocks = ((total_params as u32 + 255) / 256).max(1);
            self.stream.launch_builder(&self.recursive_conf_reduce_kernel)
                .arg(&recursive_conf_partials_ptr)
                .arg(&d_w_conf)
                .arg(&d_b_conf)
                .arg(&num_blocks_i32)
                .arg(&sh2)
                .launch(LaunchConfig {
                    grid_dim: (reduce_blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("recursive_confidence_reduce: {e}")))?;
        }
        Ok(())
    }

    /// Broadcast direction-branch gamma^n_steps * gamma_mod[0] into gamma_buf [B].
    /// Used by MSE loss (scalar γ path). C51 loss reads per-branch γ directly from ISV.
    pub(crate) fn fill_gamma_buf(&self) -> Result<(), MLError> {
        let base_gamma = self.read_isv_signal_at(GAMMA_DIR_EFF_INDEX).powi(self.config.n_steps as i32);
        let blocks = ((self.config.batch_size as u32 + 255) / 256).max(1);
        let b = self.config.batch_size as i32;
        let gamma_mod_ptr = self.gamma_mod_buf.raw_ptr();
        let gamma_buf_ptr = self.gamma_buf.raw_ptr();
        unsafe {
            self.stream.launch_builder(&self.fill_gamma_buf_kernel)
                .arg(&gamma_buf_ptr)
                .arg(&base_gamma)
                .arg(&gamma_mod_ptr)
                .arg(&b)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("fill_gamma_buf: {e}")))?;
        }
        Ok(())
    }

    /// LR scale factor for new components: min(1.0, step / 500).
    fn new_component_lr_scale(&self) -> f32 {
        (self.new_component_warmup_step as f32 / NEW_COMPONENT_WARMUP_STEPS as f32).min(1.0)
    }

    /// Increment warmup counter. Called after each training step.
    pub(crate) fn increment_warmup(&mut self) {
        self.new_component_warmup_step = self.new_component_warmup_step.saturating_add(1);
    }

    /// G11: Anchor Q-values to neutral actions per branch.
    ///
    /// Subtracts each branch's neutral action Q-value from all actions in that branch:
    ///   direction → Flat(1), magnitude → Small(0), order → Market(0), urgency → Normal(0).
    /// Focuses model capacity on alpha signal over doing nothing.
    pub(crate) fn apply_q_anchoring(&self, batch_size: usize) -> Result<(), MLError> {
        let q_out_ptr = self.q_out_buf.raw_ptr();
        let n = batch_size as i32;
        let b0 = self.config.branch_0_size as i32;
        let b1 = self.config.branch_1_size as i32;
        let b2 = self.config.branch_2_size as i32;
        let b3 = self.config.branch_3_size as i32;
        unsafe {
            self.stream.launch_builder(&self.q_anchor_kernel)
                .arg(&q_out_ptr)
                .arg(&n)
                .arg(&b0)
                .arg(&b1)
                .arg(&b2)
                .arg(&b3)
                .launch(LaunchConfig::for_num_elems(batch_size as u32))
                .map_err(|e| MLError::ModelError(format!("q_anchor_to_flat: {e}")))?;
        }
        Ok(())
    }

    /// Apply branch confidence routing: ISV gate × Q-value separation confidence.
    ///
    /// For each branch, computes confidence = sigmoid(5 * (Q_max - Q_mean)) and
    /// scales Q-values by isv_gate[d] * max(confidence, 0.3). Market regime info
    /// (ADX, CUSUM) still flows through the trunk → branches; ISV adds training
    /// dynamics awareness on top.
    pub(crate) fn apply_branch_confidence_routing(&self, batch_size: usize) -> Result<(), MLError> {
        let blocks = ((batch_size as u32 + 255) / 256).max(1);
        let q_out_ptr = self.q_out_buf.raw_ptr();
        let gate_ptr = self.branch_gate_buf.raw_ptr();
        let b_i32 = batch_size as i32;
        let b0 = self.config.branch_0_size as i32;
        let b1 = self.config.branch_1_size as i32;
        let b2 = self.config.branch_2_size as i32;
        let b3 = self.config.branch_3_size as i32;

        unsafe {
            self.stream.launch_builder(&self.branch_confidence_routing_kernel)
                .arg(&q_out_ptr)
                .arg(&gate_ptr)
                .arg(&b_i32)
                .arg(&b0).arg(&b1).arg(&b2).arg(&b3)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("branch_confidence_routing: {e}")))?;
        }
        Ok(())
    }

    /// Learned risk management: forward pass through risk branch.
    /// h_s2 → ReLU(w_risk_fc @ h_s2 + b_risk_fc) → sigmoid(w_risk_out @ hidden + b_risk_out) → R ∈ (0,1).
    pub(crate) fn risk_budget_forward(&self, batch_size: usize) -> Result<(), MLError> {
        let param_sizes = compute_param_sizes(&self.config);
        let w_fc_ptr = self.ptrs.params_ptr + padded_byte_offset(&param_sizes, 64);
        let b_fc_ptr = self.ptrs.params_ptr + padded_byte_offset(&param_sizes, 65);
        let w_out_ptr = self.ptrs.params_ptr + padded_byte_offset(&param_sizes, 66);
        let b_out_ptr = self.ptrs.params_ptr + padded_byte_offset(&param_sizes, 67);

        let save_h_s2_ptr = self.save_h_s2.raw_ptr();
        let predicted_error_buf_ptr = self.predicted_error_buf.raw_ptr();
        let risk_hidden_buf_ptr = self.risk_hidden_buf.raw_ptr();
        let risk_budget_buf_ptr = self.risk_budget_buf.raw_ptr();
        unsafe {
            self.stream.launch_builder(&self.risk_forward_kernel)
                .arg(&save_h_s2_ptr)
                .arg(&self.isv_signals_dev_ptr)    // ISV raw [8]
                .arg(&predicted_error_buf_ptr)     // predicted_error [B]
                .arg(&w_fc_ptr)
                .arg(&b_fc_ptr)
                .arg(&w_out_ptr)
                .arg(&b_out_ptr)
                .arg(&risk_hidden_buf_ptr)
                .arg(&risk_budget_buf_ptr)
                .arg(&(batch_size as i32))
                .arg(&(self.config.shared_h2 as i32))
                .arg(&(self.config.adv_h as i32))
                .launch(LaunchConfig {
                    // One block per sample, one thread per hidden neuron (AH)
                    grid_dim: (batch_size as u32, 1, 1),
                    block_dim: (self.config.adv_h as u32, 1, 1),
                    // Shared memory: input vector = (SH2 + 13) floats
                    shared_mem_bytes: ((self.config.shared_h2 + 13) * 4) as u32,
                })
                .map_err(|e| MLError::ModelError(format!("risk_budget_forward: {e}")))?;
        }
        Ok(())
    }

    /// Apply risk budget: scales magnitude Q-values (Full×R, Half×sqrt(R)),
    /// produces per-sample CVaR alpha and commitment lambda.
    pub(crate) fn apply_risk_budget(&self, batch_size: usize) -> Result<(), MLError> {
        let risk_budget_buf_ptr = self.risk_budget_buf.raw_ptr();
        let q_out_buf_ptr = self.q_out_buf.raw_ptr();
        let cvar_alpha_buf_ptr = self.cvar_alpha_buf.raw_ptr();
        let commit_lambda_buf_ptr = self.commit_lambda_buf.raw_ptr();
        unsafe {
            self.stream.launch_builder(&self.risk_apply_kernel)
                .arg(&risk_budget_buf_ptr)
                .arg(&q_out_buf_ptr)
                .arg(&cvar_alpha_buf_ptr)
                .arg(&commit_lambda_buf_ptr)
                .arg(&(batch_size as i32))
                .arg(&(self.config.branch_0_size as i32))
                .arg(&(self.config.branch_1_size as i32))
                .arg(&(self.config.branch_2_size as i32))
                .arg(&(self.config.branch_3_size as i32))
                .launch(LaunchConfig::for_num_elems(batch_size as u32))
                .map_err(|e| MLError::ModelError(format!("apply_risk_budget: {e}")))?;
        }
        Ok(())
    }

    /// Per-sample risk budget [B] — sigmoid output from learned risk branch.
    pub(crate) fn risk_budget_buf(&self) -> &CudaSlice<f32> {
        &self.risk_budget_buf
    }

    /// Per-sample CVaR alpha [B] — derived from risk budget for tail-risk optimization.
    pub(crate) fn cvar_alpha_buf(&self) -> &CudaSlice<f32> {
        &self.cvar_alpha_buf
    }

    /// Per-sample commitment lambda [B] — derived from risk budget for position sizing.
    pub(crate) fn commit_lambda_buf(&self) -> &CudaSlice<f32> {
        &self.commit_lambda_buf
    }

    /// Gentle weight decay for risk branch parameters (MVP training signal).
    ///
    /// Scales risk weights by `(1 - lr * 0.01)` each step, preventing R from
    /// collapsing to 0 or 1. Full BPTT backward deferred — the risk branch
    /// learns its initial representation from trunk gradients flowing through
    /// shared weights. Uses scale_f32_kernel (no memcpy, all device-side).
    pub(crate) fn step_risk_sgd(&mut self) -> Result<(), MLError> {
        let param_sizes = compute_param_sizes(&self.config);
        // Risk tensors are indices 64..67 (4 tensors).
        // Total aligned element count across all 4 risk tensors:
        let risk_aligned_count: usize = (64..68)
            .map(|i| align4(param_sizes[i]))
            .sum();

        let base_lr: f32 = 1e-4;
        let lr = base_lr * self.new_component_lr_scale();
        let decay = 1.0_f32 - lr * 0.01; // very gentle decay

        let risk_ptr = self.ptrs.params_ptr + padded_byte_offset(&param_sizes, 64);

        unsafe {
            self.stream.launch_builder(&self.scale_f32_post_aux)
                .arg(&risk_ptr)
                .arg(&decay)
                .arg(&(risk_aligned_count as i32))
                .launch(LaunchConfig::for_num_elems(risk_aligned_count as u32))
                .map_err(|e| MLError::ModelError(format!("risk_sgd decay: {e}")))?;
        }
        self.risk_adam_step += 1;
        Ok(())
    }

    /// Update regime gate atom utilization from latest Q-stats.
    pub fn set_regime_util(&mut self, util: f32) {
        unsafe { *self.regime_util_pinned = util; }
    }

    /// G9: Apply regime-conditioned dropout to h_s2 trunk output.
    /// Must be called AFTER mamba2_step and BEFORE compute_expected_q.
    pub(crate) fn apply_regime_dropout(&self, batch_size: usize, is_training: bool) -> Result<(), MLError> {
        let sh2 = self.config.shared_h2;
        let total = batch_size * sh2;
        let save_h_s2_ptr = self.save_h_s2.raw_ptr();
        unsafe {
            self.stream.launch_builder(&self.regime_dropout_kernel)
                .arg(&save_h_s2_ptr)
                .arg(&self.ptrs.states_buf)
                .arg(&(batch_size as i32))
                .arg(&(sh2 as i32))
                .arg(&(ml_core::state_layout::STATE_DIM as i32))
                .arg(&0.15_f32)
                .arg(&self.regime_dropout_epoch_seed)
                .arg(&(if is_training { 1_i32 } else { 0_i32 }))
                .launch(LaunchConfig::for_num_elems(total as u32))
                .map_err(|e| MLError::ModelError(format!("regime_dropout: {e}")))?;
        }
        Ok(())
    }

    /// G9: Set epoch seed for regime dropout Philox PRNG.
    pub fn set_regime_dropout_seed(&mut self, seed: i32) {
        self.regime_dropout_epoch_seed = seed;
    }

    /// G5: Apply epistemic gate to magnitude branch Q-values.
    /// Scales magnitude Q-values by sigmoid(5 * (var_mean - threshold)):
    /// high ensemble variance → conservative (small) magnitude bias.
    /// Must be called AFTER apply_branch_confidence_routing and BEFORE launch_q_attention.
    pub(crate) fn apply_epistemic_gate(&self, batch_size: usize) -> Result<(), MLError> {
        let ta = self.total_actions() as i32;
        let q_out_buf_ptr = self.q_out_buf.raw_ptr();
        let q_var_buf_trainer_ptr = self.q_var_buf_trainer.raw_ptr();
        unsafe {
            self.stream.launch_builder(&self.epistemic_gate_kernel)
                .arg(&q_out_buf_ptr)
                .arg(&q_var_buf_trainer_ptr)
                .arg(&self.var_ema_dev_ptr)
                .arg(&(batch_size as i32))
                .arg(&ta)
                .arg(&(self.config.branch_0_size as i32))
                .arg(&(self.config.branch_1_size as i32))
                .launch(LaunchConfig::for_num_elems(batch_size as u32))
                .map_err(|e| MLError::ModelError(format!("epistemic_gate: {e}")))?;
        }
        Ok(())
    }

    /// G5: Set the EMA threshold for epistemic gating (pinned device-mapped).
    pub fn set_var_ema(&self, val: f32) {
        unsafe { *self.var_ema_pinned = val; }
    }

    // G6 compute_branch_independence + G10 compute_temporal_consistency
    // deleted 2026-04-21 after V7-gem measurement confirmed both penalties
    // were sub-noise. See commit 1961857c2 (unwiring) and 61ab27ff3
    // (measurement data). G12 below remains — its backward IS active.

    /// G12: Predictive coding auxiliary loss + backward — temporal smoothness
    /// on the enriched trunk. MSE between consecutive h_s2 samples gives a
    /// noise-free self-supervised signal that flows back through the trunk.
    ///
    /// Three steps, all graph-safe (no memset, no atomicAdd):
    ///   1. Forward: per-sample MSE → predictive_per_sample_buf [B]
    ///   2. Reduce → predictive_loss_buf [1] (for monitoring readback)
    ///   3. Backward: gradient on h_s2 → ACCUMULATED into bw_d_h_s2
    ///
    /// The trunk backward (W_s2 → W_s1) reads bw_d_h_s2, so steps 1+3 effectively
    /// add a regularization term to the trunk gradient. lambda_pred = 0.1 keeps
    /// the predictive contribution small relative to the C51/IQN gradient.
    ///
    /// Caller MUST invoke this AFTER the main heads have written their share
    /// of bw_d_h_s2 (so we accumulate, not overwrite) and BEFORE the trunk
    /// backward GEMMs (so the trunk picks up the combined gradient).
    pub(crate) fn compute_predictive_coding_loss(&self, batch_size: usize) -> Result<(), MLError> {
        let sh2 = self.config.shared_h2 as i32;
        let save_h_s2_ptr = self.save_h_s2.raw_ptr();
        let predictive_per_sample_buf_ptr = self.predictive_per_sample_buf.raw_ptr();
        let lambda_pred = 0.1_f32;

        // Step 1: per-sample MSE loss
        unsafe {
            self.stream.launch_builder(&self.predictive_coding_kernel)
                .arg(&save_h_s2_ptr)
                .arg(&predictive_per_sample_buf_ptr)
                .arg(&(batch_size as i32))
                .arg(&sh2)
                .arg(&lambda_pred)
                .launch(LaunchConfig::for_num_elems(batch_size as u32))
                .map_err(|e| MLError::ModelError(format!("predictive_coding: {e}")))?;
        }
        // Step 2: deterministic reduce → predictive_loss_buf[0] (monitoring)
        let loss_ptr = self.predictive_loss_buf.raw_ptr();
        unsafe {
            self.stream
                .launch_builder(&self.c51_loss_reduce_kernel)
                .arg(&predictive_per_sample_buf_ptr)
                .arg(&loss_ptr)
                .arg(&(batch_size as i32))
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (1, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("predictive_coding reduce: {e}")))?;
        }
        // Step 3: backward → ACCUMULATE gradient into bw_d_h_s2 (one thread
        // per (sample, feature) cell, no atomic — unique writes).
        let bw_d_h_s2_ptr = self.bw_d_h_s2.raw_ptr();
        let total = (batch_size * (sh2 as usize)) as u32;
        unsafe {
            self.stream.launch_builder(&self.predictive_coding_backward_kernel)
                .arg(&save_h_s2_ptr)
                .arg(&bw_d_h_s2_ptr)
                .arg(&(batch_size as i32))
                .arg(&sh2)
                .arg(&lambda_pred)
                .launch(LaunchConfig::for_num_elems(total))
                .map_err(|e| MLError::ModelError(format!("predictive_coding_backward: {e}")))?;
        }
        Ok(())
    }

    /// Accumulate first SH2 columns of a concat dX buffer into d_h_s2 [B, SH2]
    /// from a wider buffer with `src_stride` columns. Magnitude callers pass
    /// `SH2 + branch_0_size` (direction-conditioned); order/urgency callers pass
    /// `SH2 + 3` (OFI-conditioned).
    pub(crate) fn accumulate_d_h_s2_from_concat(
        &self,
        d_concat_ptr: u64,
        d_h_s2_ptr: u64,
        batch: usize,
        src_stride: usize,
        beta: f32,
    ) -> Result<(), MLError> {
        let total = (batch * self.config.shared_h2) as i32;
        let blocks = ((total as u32 + 255) / 256).max(1);
        let sh2 = self.config.shared_h2 as i32;
        let src_stride_i32 = src_stride as i32;
        unsafe {
            self.stream
                .launch_builder(&self.strided_accumulate_kernel)
                .arg(&d_concat_ptr)
                .arg(&d_h_s2_ptr)
                .arg(&sh2)
                .arg(&src_stride_i32)
                .arg(&total)
                .arg(&beta)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("strided_accumulate: {e}")))?;
        }
        Ok(())
    }

    /// Update IQN readiness from the current IQN loss.
    /// Readiness ramps 0→1 as loss decreases from initial value.
    /// Uses adaptive-rate EMA (same pattern as eval_v_range).
    ///
    /// SP4 Layer C close-out C2 (2026-05-01): the EMA arithmetic +
    /// improvement-gauge formula now run GPU-side via
    /// `update_iqn_readiness_kernel` per `feedback_no_cpu_compute_strict`.
    /// The host signature is preserved (callers pass the loss scalar from
    /// `gpu_iqn.read_total_loss()` mapped-pinned readback); the kernel is
    /// launched on the trainer's stream and updates the three coupled
    /// mapped-pinned scalars (`iqn_loss_initial_pinned`, `iqn_loss_ema_pinned`,
    /// `iqn_readiness_pinned`) in-place via their dev_ptrs. The host
    /// `iqn_readiness` shadow field is refreshed from the pinned slot after
    /// launch so existing host-side accessors observe the same value (the
    /// kernel's `__threadfence_system()` ensures PCIe-visibility before the
    /// host_ptr deref).
    pub fn update_iqn_readiness(&mut self, iqn_loss: f32) {
        if let Err(e) = self.launch_update_iqn_readiness(iqn_loss) {
            tracing::warn!("update_iqn_readiness launch failed: {e}");
            return;
        }
        // Refresh host shadow from the pinned slot — kernel's
        // __threadfence_system() makes the write PCIe-visible to host_ptr
        // dereferences on this stream.
        unsafe {
            self.iqn_readiness = *self.iqn_readiness_pinned;
        }
    }

    /// SP4 Layer C close-out C2 (2026-05-01): GPU-only IQN readiness gauge
    /// update launcher.
    ///
    /// Replaces the host-side cold-start sentinel + adaptive-α EMA + improvement-
    /// gauge arithmetic in `update_iqn_readiness` per
    /// `feedback_no_cpu_compute_strict`. Single-thread, single-block — the
    /// kernel reads/writes three mapped-pinned scalars in-place. Caller-compat:
    /// the EMA buffers retain their mapped-pinned shape so c51_loss_kernel
    /// continues to read `iqn_readiness_pinned` via dev_ptr (CVaR α) unchanged,
    /// and the host-side accessors (`iqn_readiness()`, `iqn_loss_ema_value()`,
    /// `iqn_loss_initial_value()`) keep observing the same memory after the
    /// kernel's `__threadfence_system()`.
    fn launch_update_iqn_readiness(&self, iqn_loss: f32) -> Result<(), MLError> {
        let initial_dev = self.iqn_loss_initial_dev_ptr;
        let ema_dev     = self.iqn_loss_ema_dev_ptr;
        let readiness_dev = self.iqn_readiness_dev_ptr;

        let cfg = LaunchConfig {
            grid_dim:         (1, 1, 1),
            block_dim:        (1, 1, 1),
            shared_mem_bytes: 0,
        };

        // Safety: kernel signature `(float, float*, float*, float*)` matches
        // the four args below; the three dev_ptrs are mapped-pinned and valid
        // for the lifetime of `self`.
        unsafe {
            self.stream
                .launch_builder(&self.update_iqn_readiness_kernel)
                .arg(&iqn_loss)
                .arg(&initial_dev)
                .arg(&ema_dev)
                .arg(&readiness_dev)
                .launch(cfg)
                .map_err(|e| MLError::ModelError(format!("update_iqn_readiness: {e}")))?;
        }
        Ok(())
    }

    /// SP4 Layer C close-out C3 (2026-05-01): GPU-only utilization EMA
    /// update launcher.
    ///
    /// Replaces the host-side cold-start sentinel + adaptive-α EMA
    /// arithmetic at the bottom of `reduce_current_q_stats` per
    /// `feedback_no_cpu_compute_strict`. Single-thread, single-block — the
    /// kernel reads/writes `utilization_ema_pinned` and writes
    /// `homeostatic_obs_pinned[1]` as the slot-1 mirror in lockstep.
    /// Caller-compat: the EMA buffer retains its mapped-pinned shape so the
    /// `utilization_ema()` accessor reads through the host_ptr after the
    /// kernel's `__threadfence_system()`.
    fn launch_update_utilization_ema(&self, atom_util: f32) -> Result<(), MLError> {
        let util_dev = self.utilization_ema_dev_ptr;
        let homeo_dev = self.homeostatic_obs_dev_ptr;

        let cfg = LaunchConfig {
            grid_dim:         (1, 1, 1),
            block_dim:        (1, 1, 1),
            shared_mem_bytes: 0,
        };

        // Safety: kernel signature `(float, float*, float*)` matches the
        // three args; both dev_ptrs are mapped-pinned and valid for the
        // lifetime of `self`. The kernel writes only slot [1] of
        // `homeostatic_obs_pinned`; other slots are unaffected.
        unsafe {
            self.stream
                .launch_builder(&self.update_utilization_ema_kernel)
                .arg(&atom_util)
                .arg(&util_dev)
                .arg(&homeo_dev)
                .launch(cfg)
                .map_err(|e| MLError::ModelError(format!("update_utilization_ema: {e}")))?;
        }
        Ok(())
    }

    /// Raw pointer to the pinned host readback buffer [16 × f32].
    /// Used by `FusedTrainingCtx` for async diversity loss readback at offset 9.
    pub fn readback_pinned_ptr(&self) -> *mut f32 {
        self.readback_pinned
    }

    /// Current IQN readiness scalar [0, 1].
    /// 0 = uncertain/exploring, 1 = converged/exploiting.
    /// Used by the experience collector's quantile_q_select kernel.
    pub fn iqn_readiness(&self) -> f32 {
        self.iqn_readiness
    }

    /// Device pointer to IQN readiness [1] pinned — for plan enforcement gating in env_step.
    pub fn iqn_readiness_dev_ptr(&self) -> u64 { self.iqn_readiness_dev_ptr }

    /// Reference to saved h_s2 activations from the last forward pass.
    ///
    /// Shape: `[B, SHARED_H2]` — the shared trunk output for online states.
    /// Valid after `train_step()` or `forward_loss()` has been called.
    /// Used by the IQN dual-head to read trunk activations without re-computation.
    pub fn save_h_s2(&self) -> &CudaSlice<f32> {
        &self.save_h_s2
    }

    /// Target h_s2 from Pass 2 (target forward on next_states) — raw device pointer.
    /// Used by IQN to reuse pre-computed trunk embeddings instead of recomputing.
    pub fn tg_h_s2_ptr(&self) -> u64 {
        self.ptrs.tg_h_s2_buf
    }

    /// Raw device pointer to the online params flat buffer.
    pub fn params_buf_ptr(&self) -> u64 {
        self.params_buf.raw_ptr()
    }

    /// Raw device pointer to the target params flat buffer.
    pub fn target_params_buf_ptr(&self) -> u64 {
        self.target_params_buf.raw_ptr()
    }

    /// SP14 Layer C Phase C.5a-fixup (2026-05-08): raw device pointers to
    /// the aux trunk's 6 parameter tensors (`w1, b1, w2, b2, w3, b3`).
    ///
    /// Returned in the same order as the Adam launcher's per-tensor tuple
    /// list (`launch_aux_trunk_adam_update`). The collector calls the
    /// matching `set_trainer_aux_trunk_param_ptrs` setter once after the
    /// fused training context is created (cold-path, mirrors the existing
    /// `set_trainer_params_ptr` zero-copy contract — pointers are stable
    /// for the lifetime of the trainer's CudaSlice fields).
    ///
    /// C.5b consumes these ptrs to launch `aux_trunk_forward.launch(...)`
    /// from the collector's per-rollout-step body, reading the trainer's
    /// authoritative aux trunk weights without a DtoD copy. The accessor
    /// itself is dead in C.5a-fixup — populated but unread until C.5b
    /// flips the contract.
    pub fn aux_trunk_param_ptrs(&self) -> (u64, u64, u64, u64, u64, u64) {
        (
            self.aux_trunk_w1.raw_ptr(),
            self.aux_trunk_b1.raw_ptr(),
            self.aux_trunk_w2.raw_ptr(),
            self.aux_trunk_b2.raw_ptr(),
            self.aux_trunk_w3.raw_ptr(),
            self.aux_trunk_b3.raw_ptr(),
        )
    }

    /// SP15 Phase 3.5.4.c (2026-05-07): return the
    /// `(w_b0out_dev_ptr, n_weights, fan_in)` triple for the
    /// directional advantage-head last-Linear weight tensor (param-
    /// buffer tensor index 19). The collector's
    /// `set_sp15_plasticity_target()` consumes this triple to wire
    /// the per-step `launch_sp15_plasticity_injection` against the
    /// directional branch only.
    ///
    /// Layout (per `compute_param_sizes` and the row/col-dim pairs
    /// at `gpu_dqn_trainer.rs:29946`):
    ///   `w_b0out: [branch_0_size × num_atoms, adv_h]` row-major
    ///     (output_dim rows × input_dim columns).
    ///   `n_weights = branch_0_size × num_atoms × adv_h`
    ///     (default config: 4 × 51 × 128 = 26112).
    ///   `fan_in    = adv_h` (= input dim of the per-unit weight
    ///     vector — Kaiming-He gain `sqrt(2/fan_in)` is per-output-
    ///     unit variance; default config: 128).
    ///
    /// The Wave 4.2 audit doc previously documented `fan_in =
    /// adv_h × num_atoms = 6528`. That value is the total weight
    /// count of one row of branch outputs (`num_atoms` atoms per
    /// direction × `adv_h` inputs), not the input dim per unit; the
    /// Kaiming-He variance derivation uses input dim per unit, which
    /// is `adv_h`. The Phase 3.5.4.c audit-doc inline correction
    /// restores the right value (`128`) and the right stddev
    /// (`sqrt(2/128) ≈ 0.125`).
    pub fn sp15_w_b0out_target(&self) -> (u64, i32, i32) {
        let param_sizes = compute_param_sizes(&self.config);
        let byte_offset = padded_byte_offset(&param_sizes, 19);
        let dev_ptr = self.params_buf.raw_ptr() + byte_offset;
        let n_weights = (self.config.branch_0_size
            * self.config.num_atoms
            * self.config.adv_h) as i32;
        let fan_in = self.config.adv_h as i32;
        (dev_ptr, n_weights, fan_in)
    }

    /// Backward gradient w.r.t. h_s2 (trunk activation) — f32.
    pub fn bw_d_h_s2_buf(&self) -> &CudaSlice<f32> {
        &self.bw_d_h_s2
    }

    /// Copy `bw_d_h_s2` (f32) → attn_bw_scratch, returning a reference to the result.
    ///
    /// Uses `attn_bw_scratch` as the destination scratch.
    /// Needed for `GpuAttention::backward` input.
    pub fn bw_d_h_s2(&mut self) -> Result<&CudaSlice<f32>, MLError> {
        let n_bytes = self.config.batch_size * self.config.shared_h2 * std::mem::size_of::<f32>();
        let src = self.bw_d_h_s2.raw_ptr();
        let dst = self.attn_bw_scratch.raw_ptr();
        self.graph_safe_copy_f32(dst, src, n_bytes, "bw_d_h_s2→attn_scratch")?;
        Ok(&self.attn_bw_scratch)
    }

    /// Shared trunk hidden layer 2 dimension.
    pub fn shared_h2(&self) -> usize {
        self.config.shared_h2
    }

    /// Scratch buffer used by attention forward/backward — `[B, SHARED_H2]` f32.
    ///
    /// Exposed for parallel-stream attention: callers copy `save_h_s2`/`bw_d_h_s2`
    /// into this scratch on a dedicated stream, then pass it to `GpuAttention` methods.
    pub fn attn_bw_scratch(&self) -> &CudaSlice<f32> {
        &self.attn_bw_scratch
    }

    /// OFI embed MLP output buffer [B, 10] — used by attention forward as
    /// the additional input features concatenated to h_s2.
    pub fn ofi_embed_output(&self) -> &CudaSlice<f32> {
        &self.ofi_embed_output_buf
    }

    /// Graph-safe device-to-device copy using a CUDA kernel instead of `memcpy_dtod_async`.
    ///
    /// `memcpy_dtod_async` is NOT captured by CUDA Graph — only kernel launches are.
    /// Any DtoD copy inside a graph-captured code path must use this method.
    ///
    /// `n_bytes` is the total byte count (must be divisible by 4).
    /// Both `src` and `dst` are raw device pointers (u64).
    pub(crate) fn graph_safe_copy_f32(&self, dst: u64, src: u64, n_bytes: usize, ctx: &str) -> Result<(), MLError> {
        debug_assert_eq!(n_bytes % 4, 0, "graph_safe_copy_f32: n_bytes must be divisible by 4");
        let n_elems = (n_bytes / 4) as i32;
        unsafe {
            self.stream.launch_builder(&self.copy_f32_kernel)
                .arg(&src)
                .arg(&dst)
                .arg(&n_elems)
                .launch(LaunchConfig::for_num_elems(n_elems as u32))
                .map_err(|e| MLError::ModelError(format!("copy_f32 {ctx}: {e}")))?;
        }
        Ok(())
    }

    /// Graph-safe copy on a caller-specified stream (for multi-stream attention fork).
    /// Same as `graph_safe_copy_f32` but launches the kernel on `stream` instead of `self.stream`.
    pub(crate) fn graph_safe_copy_f32_on(
        &self,
        dst: u64,
        src: u64,
        n_bytes: usize,
        stream: &Arc<CudaStream>,
        ctx: &str,
    ) -> Result<(), MLError> {
        debug_assert_eq!(n_bytes % 4, 0, "graph_safe_copy_f32_on: n_bytes must be divisible by 4");
        let n_elems = (n_bytes / 4) as i32;
        unsafe {
            stream.launch_builder(&self.copy_f32_kernel)
                .arg(&src)
                .arg(&dst)
                .arg(&n_elems)
                .launch(LaunchConfig::for_num_elems(n_elems as u32))
                .map_err(|e| MLError::ModelError(format!("copy_f32 {ctx}: {e}")))?;
        }
        Ok(())
    }

    /// Run Mamba2 temporal scan: h_enriched = h_s2 + temporal_context(h_history).
    ///
    /// Uses cuBLAS GEMM for W_A/W_B projections (2 GEMMs) followed by a lightweight
    /// scan kernel that only loops over K=8 steps and STATE_D=16 per thread.
    /// The W_C output projection is done inline in the scan kernel (16 reads/thread).
    ///
    /// Previous: one thread per sample with inner SH2=256 matmul loops.
    /// New: cuBLAS GEMM (SH2=256 -> STATE_D=16) + scan grid=(B, ceil(SH2/256)).
    pub(crate) fn mamba2_forward(&self, batch_size: usize) -> Result<(), MLError> {
        let sh2 = self.config.shared_h2;
        let h_width = sh2 + OFI_EMBED_DIM;
        let param_ptr = self.mamba2_params.raw_ptr();
        let w_a_ptr = param_ptr;
        let w_b_ptr = param_ptr + (h_width * MAMBA2_STATE_DIM * 4) as u64;
        let w_c_ptr = param_ptr + (2 * h_width * MAMBA2_STATE_DIM * 4) as u64;

        // Per-stream lt_handle (Option C determinism fix). Main stream — fast path.
        let (lt_handle, lt_ws_ptr, lt_ws_size) =
            self.shared_cublas.lt_for(&self.stream)?;
        let cu_stream = self.stream.cu_stream() as cublaslt_sys::cudaStream_t;
        let alpha: f32 = 1.0;
        let beta: f32 = 0.0;

        // GEMM 1: A_proj[B*K, STATE_D] = h_history[B*K, SH2+OFI] @ W_A[SH2+OFI, STATE_D]
        // Col-major: C[STATE_D, B*K] = W_A_cm[STATE_D, SH2+OFI] @ h_cm[SH2+OFI, B*K]
        unsafe {
            cublaslt_sys::cublasLtMatmul(
                lt_handle,
                self.mamba2_gemm_proj.matmul_desc,
                &alpha as *const f32 as *const std::ffi::c_void,
                w_a_ptr as *const std::ffi::c_void,
                self.mamba2_gemm_proj.a_layout,
                self.mamba2_h_history.raw_ptr() as *const std::ffi::c_void,
                self.mamba2_gemm_proj.b_layout,
                &beta as *const f32 as *const std::ffi::c_void,
                self.mamba2_a_proj.raw_ptr() as *mut std::ffi::c_void,
                self.mamba2_gemm_proj.c_layout,
                self.mamba2_a_proj.raw_ptr() as *mut std::ffi::c_void,
                self.mamba2_gemm_proj.d_layout,
                &self.mamba2_gemm_proj.algo as *const _,
                lt_ws_ptr as *mut std::ffi::c_void,
                lt_ws_size,
                cu_stream,
            );
        }

        // GEMM 2: B_proj[B*K, STATE_D] = h_history[B*K, SH2+OFI] @ W_B[SH2+OFI, STATE_D]
        unsafe {
            cublaslt_sys::cublasLtMatmul(
                lt_handle,
                self.mamba2_gemm_proj.matmul_desc,
                &alpha as *const f32 as *const std::ffi::c_void,
                w_b_ptr as *const std::ffi::c_void,
                self.mamba2_gemm_proj.a_layout,
                self.mamba2_h_history.raw_ptr() as *const std::ffi::c_void,
                self.mamba2_gemm_proj.b_layout,
                &beta as *const f32 as *const std::ffi::c_void,
                self.mamba2_b_proj.raw_ptr() as *mut std::ffi::c_void,
                self.mamba2_gemm_proj.c_layout,
                self.mamba2_b_proj.raw_ptr() as *mut std::ffi::c_void,
                self.mamba2_gemm_proj.d_layout,
                &self.mamba2_gemm_proj.algo as *const _,
                lt_ws_ptr as *mut std::ffi::c_void,
                lt_ws_size,
                cu_stream,
            );
        }

        // Lightweight scan kernel: uses pre-projected A/B, reads W_C inline (16 per thread)
        // Grid: (B, ceil(SH2/256)), Block: 256
        let a_proj_ptr = self.mamba2_a_proj.raw_ptr();
        let b_proj_ptr = self.mamba2_b_proj.raw_ptr();
        let save_h_s2_ptr = self.save_h_s2.raw_ptr();
        let mamba2_h_enriched_ptr = self.mamba2_h_enriched.raw_ptr();
        let temporal_weight_buf_ptr = self.temporal_weight_buf.raw_ptr();
        let grid_y = ((sh2 + 255) / 256) as u32;

        unsafe {
            self.stream.launch_builder(&self.mamba2_scan_proj_fwd_kernel)
                .arg(&a_proj_ptr)
                .arg(&b_proj_ptr)
                .arg(&w_c_ptr)
                .arg(&save_h_s2_ptr)
                .arg(&mamba2_h_enriched_ptr)
                .arg(&(batch_size as i32))
                .arg(&(MAMBA2_HISTORY_K as i32))
                .arg(&(sh2 as i32))
                .arg(&(MAMBA2_STATE_DIM as i32))
                .arg(&self.isv_signals_dev_ptr)
                .arg(&temporal_weight_buf_ptr)
                .launch(LaunchConfig {
                    grid_dim: (batch_size as u32, grid_y, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("mamba2_scan_projected_fwd: {e}")))?;
        }
        Ok(())
    }

    /// Update rolling history: shift left, insert [h_s2; ofi_embed] at K-1.
    pub(crate) fn mamba2_update_history(&self, batch_size: usize) -> Result<(), MLError> {
        let sh2 = self.config.shared_h2;
        let embed_dim = OFI_EMBED_DIM as i32;
        let total_width = sh2 + OFI_EMBED_DIM;
        let total = batch_size * total_width;
        let mamba2_h_history_ptr = self.mamba2_h_history.raw_ptr();
        let save_h_s2_ptr = self.save_h_s2.raw_ptr();
        let ofi_embed_ptr = self.ofi_embed_output_buf.raw_ptr();

        unsafe {
            self.stream.launch_builder(&self.mamba2_update_kernel)
                .arg(&mamba2_h_history_ptr)
                .arg(&save_h_s2_ptr)
                .arg(&ofi_embed_ptr)
                .arg(&(batch_size as i32))
                .arg(&(MAMBA2_HISTORY_K as i32))
                .arg(&(sh2 as i32))
                .arg(&embed_dim)
                .launch(LaunchConfig::for_num_elems(total as u32))
                .map_err(|e| MLError::ModelError(format!("mamba2_update_history: {e}")))?;
        }
        Ok(())
    }

    /// Full Mamba2 step: update history then enrich h_s2.
    pub(crate) fn mamba2_step(&self, batch_size: usize) -> Result<(), MLError> {
        self.mamba2_update_history(batch_size)?;
        self.mamba2_forward(batch_size)?;
        // Graph-safe copy: enriched -> save_h_s2 so branch heads use enriched activations.
        // Uses a kernel instead of raw memcpy_dtod_async which is NOT captured in CUDA Graph.
        let n = (batch_size * self.config.shared_h2) as u32;
        let mamba2_h_enriched_ptr = self.mamba2_h_enriched.raw_ptr();
        let save_h_s2_ptr = self.save_h_s2.raw_ptr();
        unsafe {
            self.stream.launch_builder(&self.mamba2_copy_enriched_kernel)
                .arg(&mamba2_h_enriched_ptr)
                .arg(&save_h_s2_ptr)
                .arg(&(n as i32))
                .launch(LaunchConfig::for_num_elems(n))
                .map_err(|e| MLError::ModelError(format!("mamba2 enriched copy kernel: {e}")))?;
        }
        Ok(())
    }

    /// Mamba2 BPTT: compute gradients for W_A, W_B, W_C.
    ///
    /// Pipeline:
    ///   1. Lightweight reverse scan: grid=(B, ceil(STATE_D/32)), block=32
    ///      Reuses A_proj/B_proj from forward pass. Outputs d_gate, d_x, d_context.
    ///   2. Scale d_h_enriched by temporal_weight: mamba2_scale_d_enriched kernel
    ///   3. cuBLAS GEMM: dW_A = d_gate^T @ h_history
    ///   4. cuBLAS GEMM: dW_B = d_x^T @ h_history
    ///   5. cuBLAS GEMM: dW_C = (d_h_enriched*tw)^T @ x_K
    ///
    /// Previous: 12,288 threads each looping over B=8192 with inner SH2=256 loops.
    /// New: B*STATE_D threads with K=8 + SH2=256 inner loop, then 3 cuBLAS GEMMs.
    pub(crate) fn mamba2_backward(&mut self, batch_size: usize) -> Result<(), MLError> {
        let sh2 = self.config.shared_h2;
        let h_width = sh2 + OFI_EMBED_DIM;
        let param_ptr = self.mamba2_params.raw_ptr();
        let w_c_ptr = param_ptr + (2 * h_width * MAMBA2_STATE_DIM * 4) as u64;

        let grad_ptr = self.mamba2_grad.raw_ptr();
        let d_w_a = grad_ptr;
        let d_w_b = grad_ptr + (h_width * MAMBA2_STATE_DIM * 4) as u64;
        let d_w_c = grad_ptr + (2 * h_width * MAMBA2_STATE_DIM * 4) as u64;

        // Per-stream lt_handle (Option C determinism fix). Main stream — fast path.
        let (lt_handle, lt_ws_ptr, lt_ws_size) =
            self.shared_cublas.lt_for(&self.stream)?;
        let cu_stream = self.stream.cu_stream() as cublaslt_sys::cudaStream_t;
        let alpha: f32 = 1.0;
        let beta: f32 = 0.0;

        // Step 1: Lightweight reverse scan kernel
        // Reuses A_proj/B_proj saved from forward pass.
        let a_proj_ptr = self.mamba2_a_proj.raw_ptr();
        let b_proj_ptr = self.mamba2_b_proj.raw_ptr();
        let bw_d_h_s2_ptr = self.bw_d_h_s2.raw_ptr();
        let temporal_weight_buf_ptr = self.temporal_weight_buf.raw_ptr();
        let d_gate_ptr = self.mamba2_d_gate.raw_ptr();
        let d_x_ptr = self.mamba2_d_x.raw_ptr();
        let d_context_ptr = self.mamba2_d_context.raw_ptr();
        let grid_y_bwd = ((MAMBA2_STATE_DIM + 31) / 32) as u32;

        unsafe {
            self.stream.launch_builder(&self.mamba2_scan_proj_bwd_kernel)
                .arg(&a_proj_ptr)
                .arg(&b_proj_ptr)
                .arg(&bw_d_h_s2_ptr)
                .arg(&w_c_ptr)
                .arg(&temporal_weight_buf_ptr)
                .arg(&d_gate_ptr)
                .arg(&d_x_ptr)
                .arg(&d_context_ptr)
                .arg(&(batch_size as i32))
                .arg(&(MAMBA2_HISTORY_K as i32))
                .arg(&(sh2 as i32))
                .arg(&(MAMBA2_STATE_DIM as i32))
                .arg(&self.isv_signals_dev_ptr)
                .launch(LaunchConfig {
                    grid_dim: (batch_size as u32, grid_y_bwd, 1),
                    block_dim: (32, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("mamba2_scan_projected_bwd: {e}")))?;
        }

        // Step 2: Scale d_h_enriched by temporal_weight for dW_C GEMM
        let d_tw_ptr = self.mamba2_d_tw.raw_ptr();
        let total_bsh2 = (batch_size * sh2) as u32;
        unsafe {
            self.stream.launch_builder(&self.mamba2_scale_d_enriched_kernel)
                .arg(&bw_d_h_s2_ptr)
                .arg(&temporal_weight_buf_ptr)
                .arg(&d_tw_ptr)
                .arg(&(batch_size as i32))
                .arg(&(sh2 as i32))
                .launch(LaunchConfig::for_num_elems(total_bsh2))
                .map_err(|e| MLError::ModelError(format!("mamba2_scale_d_enriched: {e}")))?;
        }

        // Step 3: dW_A = d_gate^T @ h_history via cuBLAS
        let h_history_ptr = self.mamba2_h_history.raw_ptr();
        unsafe {
            cublaslt_sys::cublasLtMatmul(
                lt_handle,
                self.mamba2_gemm_dw.matmul_desc,
                &alpha as *const f32 as *const std::ffi::c_void,
                d_gate_ptr as *const std::ffi::c_void,
                self.mamba2_gemm_dw.a_layout,
                h_history_ptr as *const std::ffi::c_void,
                self.mamba2_gemm_dw.b_layout,
                &beta as *const f32 as *const std::ffi::c_void,
                d_w_a as *mut std::ffi::c_void,
                self.mamba2_gemm_dw.c_layout,
                d_w_a as *mut std::ffi::c_void,
                self.mamba2_gemm_dw.d_layout,
                &self.mamba2_gemm_dw.algo as *const _,
                lt_ws_ptr as *mut std::ffi::c_void,
                lt_ws_size,
                cu_stream,
            );
        }

        // Step 4: dW_B = d_x^T @ h_history via cuBLAS
        unsafe {
            cublaslt_sys::cublasLtMatmul(
                lt_handle,
                self.mamba2_gemm_dw.matmul_desc,
                &alpha as *const f32 as *const std::ffi::c_void,
                d_x_ptr as *const std::ffi::c_void,
                self.mamba2_gemm_dw.a_layout,
                h_history_ptr as *const std::ffi::c_void,
                self.mamba2_gemm_dw.b_layout,
                &beta as *const f32 as *const std::ffi::c_void,
                d_w_b as *mut std::ffi::c_void,
                self.mamba2_gemm_dw.c_layout,
                d_w_b as *mut std::ffi::c_void,
                self.mamba2_gemm_dw.d_layout,
                &self.mamba2_gemm_dw.algo as *const _,
                lt_ws_ptr as *mut std::ffi::c_void,
                lt_ws_size,
                cu_stream,
            );
        }

        // Step 5: dW_C = (d_h_enriched * tw)^T @ x_K via cuBLAS
        unsafe {
            cublaslt_sys::cublasLtMatmul(
                lt_handle,
                self.mamba2_gemm_dwc.matmul_desc,
                &alpha as *const f32 as *const std::ffi::c_void,
                d_context_ptr as *const std::ffi::c_void,
                self.mamba2_gemm_dwc.a_layout,
                d_tw_ptr as *const std::ffi::c_void,
                self.mamba2_gemm_dwc.b_layout,
                &beta as *const f32 as *const std::ffi::c_void,
                d_w_c as *mut std::ffi::c_void,
                self.mamba2_gemm_dwc.c_layout,
                d_w_c as *mut std::ffi::c_void,
                self.mamba2_gemm_dwc.d_layout,
                &self.mamba2_gemm_dwc.algo as *const _,
                lt_ws_ptr as *mut std::ffi::c_void,
                lt_ws_size,
                cu_stream,
            );
        }

        // Step 6: d_h_history = W_A^T @ d_gate + W_B^T @ d_x
        // This backpropagates through the W_A/W_B projections to get the input gradient.
        // W_A col-major is [STATE_D, h_width] with ld=STATE_D.
        // TRANSA=T gives [h_width, STATE_D], multiplied by d_gate[STATE_D, B*K].
        // Result: d_h_history[h_width, B*K] col-major.
        let w_a_ptr = param_ptr;
        let w_b_ptr = param_ptr + (h_width * MAMBA2_STATE_DIM * 4) as u64;
        let d_h_ptr = self.d_h_history_buf.raw_ptr();

        // GEMM 6a: d_h_history = W_A^T @ d_gate (beta=0, overwrite)
        unsafe {
            cublaslt_sys::cublasLtMatmul(
                lt_handle,
                self.mamba2_gemm_dh.matmul_desc,
                &alpha as *const f32 as *const std::ffi::c_void,
                w_a_ptr as *const std::ffi::c_void,
                self.mamba2_gemm_dh.a_layout,
                d_gate_ptr as *const std::ffi::c_void,
                self.mamba2_gemm_dh.b_layout,
                &beta as *const f32 as *const std::ffi::c_void,
                d_h_ptr as *mut std::ffi::c_void,
                self.mamba2_gemm_dh.c_layout,
                d_h_ptr as *mut std::ffi::c_void,
                self.mamba2_gemm_dh.d_layout,
                &self.mamba2_gemm_dh.algo as *const _,
                lt_ws_ptr as *mut std::ffi::c_void,
                lt_ws_size,
                cu_stream,
            );
        }

        // GEMM 6b: d_h_history += W_B^T @ d_x (beta=1, accumulate)
        let beta_one: f32 = 1.0;
        unsafe {
            cublaslt_sys::cublasLtMatmul(
                lt_handle,
                self.mamba2_gemm_dh.matmul_desc,
                &alpha as *const f32 as *const std::ffi::c_void,
                w_b_ptr as *const std::ffi::c_void,
                self.mamba2_gemm_dh.a_layout,
                d_x_ptr as *const std::ffi::c_void,
                self.mamba2_gemm_dh.b_layout,
                &beta_one as *const f32 as *const std::ffi::c_void,
                d_h_ptr as *mut std::ffi::c_void,
                self.mamba2_gemm_dh.c_layout,
                d_h_ptr as *mut std::ffi::c_void,
                self.mamba2_gemm_dh.d_layout,
                &self.mamba2_gemm_dh.algo as *const _,
                lt_ws_ptr as *mut std::ffi::c_void,
                lt_ws_size,
                cu_stream,
            );
        }

        // Step 7: Extract d_ofi_embed from d_h_history (last OFI_EMBED_DIM rows, sum over K)
        let d_ofi_ptr = self.d_ofi_embed_mamba2.raw_ptr();
        let sh2_i32 = sh2 as i32;
        let embed_dim_i32 = OFI_EMBED_DIM as i32;
        let b_i32 = batch_size as i32;
        let k_i32 = MAMBA2_HISTORY_K as i32;
        unsafe {
            self.stream.launch_builder(&self.extract_ofi_embed_grad_kernel)
                .arg(&d_h_ptr)
                .arg(&d_ofi_ptr)
                .arg(&b_i32)
                .arg(&k_i32)
                .arg(&sh2_i32)
                .arg(&embed_dim_i32)
                .launch(LaunchConfig {
                    grid_dim: (((batch_size + 255) / 256) as u32, OFI_EMBED_DIM as u32, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("extract_ofi_embed_grad: {e}")))?;
        }

        Ok(())
    }

    /// Standalone SGD step for Mamba2 parameters (separate from main params_buf Adam).
    /// params += -lr * grad  via dqn_saxpy_f32_kernel.
    pub(crate) fn step_mamba2_adam(&mut self) -> Result<(), MLError> {
        let h_width = self.config.shared_h2 + OFI_EMBED_DIM;
        let n = 2 * h_width * MAMBA2_STATE_DIM + self.config.shared_h2 * MAMBA2_STATE_DIM;
        self.mamba2_adam_step += 1;
        let base_lr: f32 = 1e-4;
        let lr = base_lr * self.new_component_lr_scale();

        // SGD: params += (-lr) * grad
        // Uses adam_grad_child-specific handle — saxpy_f32_kernel is captured in
        // forward_child and aux_child; sharing across children corrupts on Hopper.
        let mamba2_params_ptr = self.mamba2_params.raw_ptr();
        let mamba2_grad_ptr = self.mamba2_grad.raw_ptr();
        unsafe {
            self.stream.launch_builder(&self.saxpy_f32_adam_grad)
                .arg(&mamba2_params_ptr)
                .arg(&mamba2_grad_ptr)
                .arg(&(-lr))
                .arg(&(n as i32))
                .launch(LaunchConfig::for_num_elems(n as u32))
                .map_err(|e| MLError::ModelError(format!("mamba2 sgd: {e}")))?;
        }
        Ok(())
    }

    /// Reference to the Mamba2 OFI embed gradient buffer on GPU.
    ///
    /// Shape: `[OFI_EMBED_DIM, B]` col-major — accumulated d_ofi_embed from `mamba2_backward()`.
    /// Written by Step 7 (extract_ofi_embed_grad kernel) after d_h_history GEMMs.
    /// Read by Task 8 (OFI embed MLP backward) to accumulate with the attention gradient.
    pub(crate) fn d_ofi_embed_mamba2(&self) -> &CudaSlice<f32> {
        &self.d_ofi_embed_mamba2
    }

    /// Reference to the states buffer on GPU.
    ///
    /// Shape: `[B, STATE_DIM]` — contains the batch's states after `upload_batch_gpu()`.
    /// Used by IQL value network to read states without Candle tensor intermediaries.
    pub fn states_buf(&self) -> &CudaSlice<f32> {
        &self.states_buf
    }

    /// Mutable reference to the states buffer on GPU.
    ///
    /// Shape: `[B, STATE_DIM_PADDED]` row-major — TLOB forward writes TLOB features into
    /// `[SL_TLOB_START..SL_TLOB_START+SL_TLOB_DIM)` after state gather and before cuBLAS forward.
    pub(crate) fn states_buf_mut(&mut self) -> &mut CudaSlice<f32> {
        &mut self.states_buf
    }

    /// Reference to the bottleneck concat gradient buffer on GPU.
    ///
    /// Shape: `[B, concat_dim]` row-major — populated by `launch_cublas_backward` when
    /// bottleneck_dim > 0. TLOB backward reads the TLOB slice from this to compute weight gradients.
    pub(crate) fn bn_d_concat_buf(&self) -> &CudaSlice<f32> {
        &self.bn_d_concat_buf
    }

    /// Bottleneck concat dimension (bottleneck_dim + STATE_DIM - market_dim + 1).
    /// Used by TLOB backward to compute the slice offset for the TLOB gradient.
    /// The trailing +1 column is the SP15 Wave 4.1b dd_pct broadcast appended by
    /// `bn_tanh_concat_dd_kernel`; TLOB only reads the columns inside the
    /// portfolio slice (`[bn_dim + (TLOB_START − market_dim) ..]`), so the
    /// dd_pct column is naturally outside its window. The accessor reports the
    /// full row stride so the TLOB backward indexes `bn_d_concat_buf` correctly.
    pub(crate) fn bn_concat_dim(&self) -> usize {
        self.config.bottleneck_dim
            + (ml_core::state_layout::STATE_DIM.saturating_sub(self.config.market_dim))
            + 1
    }

    /// SP1 Phase B (slot 24): post-`c51_grad_kernel` value-stream gradient buffer
    /// `[B, NA]` (f32, atomicAdd target). Wired into `run_nan_checks_post_backward`
    /// (Task 4) as a symmetric orchestrator-side check; the launch-site local at
    /// `gpu_dqn_trainer.rs:17707` reads this same field. Per-slot semantics in
    /// `docs/dqn-backward-nan-audit.md` per-slot accessor table (slot 24 row).
    /// Distinct from the `d_value_logits` *staging* buffer at line 3131 — that one
    /// is unrelated to `c51_grad_kernel`.
    pub(crate) fn d_value_logits_buf_ptr(&self) -> u64 {
        self.d_value_logits_buf.raw_ptr()
    }

    /// SP1 Phase B (slot 25): post-`c51_grad_kernel` branch-advantage gradient
    /// buffer `[B, (b0+b1+b2+b3) × NA]` (f32, atomicAdd target). Companion to
    /// slot 24 — fires independently iff a per-branch path corrupted (e.g. a
    /// magnitude-only `inv_a_std` divide). Per-slot semantics in
    /// `docs/dqn-backward-nan-audit.md`. Distinct from the `d_adv_logits` staging
    /// buffer at line 3133 (unrelated to `c51_grad_kernel`).
    pub(crate) fn d_adv_logits_buf_ptr(&self) -> u64 {
        self.d_adv_logits_buf.raw_ptr()
    }

    /// SP1 Phase B (slot 29): CQL gradient buffer `[B, NA]` (f32) — written by
    /// `cql_logit_grad_kernel`. Wired into `run_nan_checks_post_backward` only if
    /// the slot is un-deferred from the audit's priority list; until then this
    /// accessor exists for symmetric coverage but the call site stays optional.
    /// Per-slot semantics in `docs/dqn-backward-nan-audit.md` (slot 29 row).
    pub(crate) fn cql_d_value_logits_ptr(&self) -> u64 {
        self.cql_d_value_logits.raw_ptr()
    }

    /// SP1 Phase B (slot 30): aux next-bar backward dh_s2 contribution `[B, SH2]`
    /// (f32) — produced by `aux_next_bar_backward`, SAXPY-accumulated into
    /// `bw_d_h_s2` before `encoder_backward_chain` runs. Wired into
    /// `run_nan_checks_post_backward` (Task 4). Companion buffer
    /// `aux_dh_s2_rg_buf` (regime variant) is intentionally NOT instrumented —
    /// the audit limits slot 30 to the next-bar variant; both share the same
    /// caller and shape so a single check provides sufficient diagnostic
    /// resolution. Per-slot semantics in `docs/dqn-backward-nan-audit.md`.
    pub(crate) fn aux_dh_s2_nb_buf_ptr(&self) -> u64 {
        self.aux_dh_s2_nb_buf.raw_ptr()
    }

    // ── SP3 slot 36-47 diagnostic accessors ──────────────────────────────
    //
    // These accessors expose existing buffers (Adam m/v, params_buf trunk vs
    // heads slices, denoise_target_q_buf, atom_positions_buf) for the SP3
    // Mech 5 threshold-check kernel (Task B6). Adam state for the DQN trunk
    // + value/branch heads is UNIFIED inside `m_buf` / `v_buf` (one buffer of
    // `total_params` floats covering all params at the same offsets as
    // `params_buf`). The trunk/value/branch m/v accessors return pointers
    // into the same buffer at the appropriate offset; the kernel
    // discriminates by slot index for threshold checks. IQN Adam state lives
    // separately on `GpuIqnHead` (slots 39/43) — see
    // `gpu_iqn_head.rs::adam_m_ptr` / `adam_v_ptr`.
    //
    // params_buf layout (per `compute_param_sizes`):
    //   trunk:  tensors [0..13)  (GRN h_s1 + h_s2 + gamma/beta)
    //   value:  tensors [13..17) (W_v1, B_v1, W_v2, B_v2)
    //   branch: tensors [17..33) (W/B_b{0..3}fc + W/B_b{0..3}out)
    //
    // Slot 45 (heads) covers value+branch concatenated (tensors 13..33).

    /// SP3 slot 36: device pointer to the trunk slice of the Adam first-moment
    /// buffer. The trunk corresponds to GRN tensors [0..13) (h_s1 + h_s2 + γ/β
    /// pairs). m_buf is unified across all params, so this returns
    /// `m_buf.raw_ptr()` (offset 0).
    pub(crate) fn trunk_adam_m_ptr(&self) -> u64 {
        self.m_buf.raw_ptr()
    }

    /// Element count of the trunk Adam-m slice (= `trunk_param_count`,
    /// computed as padded byte offset of tensor index 13 / sizeof(f32)).
    pub(crate) fn trunk_adam_m_len(&self) -> usize {
        self.trunk_param_count
    }

    /// SP3 slot 37: device pointer to the value-head slice of the Adam
    /// first-moment buffer. Value head = tensors [13..17) (W_v1, B_v1, W_v2,
    /// B_v2). Returns `m_buf.raw_ptr() + padded_byte_offset(13)`.
    pub(crate) fn value_adam_m_ptr(&self) -> u64 {
        let param_sizes = compute_param_sizes(&self.config);
        self.m_buf.raw_ptr() + padded_byte_offset(&param_sizes, 13)
    }

    /// Element count of the value-head Adam-m slice (tensors [13..17), padded).
    pub(crate) fn value_adam_m_len(&self) -> usize {
        let param_sizes = compute_param_sizes(&self.config);
        let start = padded_byte_offset(&param_sizes, 13) as usize;
        let end = padded_byte_offset(&param_sizes, 17) as usize;
        (end - start) / std::mem::size_of::<f32>()
    }

    /// SP3 slot 38: device pointer to the branch-head slice of the Adam
    /// first-moment buffer. Branch heads = tensors [17..33) (W/B_b{0..3}fc +
    /// W/B_b{0..3}out, concatenated dir+mag+ord+urg).
    pub(crate) fn branch_adam_m_ptr(&self) -> u64 {
        let param_sizes = compute_param_sizes(&self.config);
        self.m_buf.raw_ptr() + padded_byte_offset(&param_sizes, 17)
    }

    /// Element count of the branch-head Adam-m slice (tensors [17..33), padded).
    pub(crate) fn branch_adam_m_len(&self) -> usize {
        let param_sizes = compute_param_sizes(&self.config);
        let start = padded_byte_offset(&param_sizes, 17) as usize;
        let end = padded_byte_offset(&param_sizes, 33) as usize;
        (end - start) / std::mem::size_of::<f32>()
    }

    /// SP3 slot 40: device pointer to the trunk slice of the Adam
    /// second-moment buffer. Mirrors `trunk_adam_m_ptr` over `v_buf`.
    pub(crate) fn trunk_adam_v_ptr(&self) -> u64 {
        self.v_buf.raw_ptr()
    }

    /// Element count of the trunk Adam-v slice (= `trunk_param_count`).
    pub(crate) fn trunk_adam_v_len(&self) -> usize {
        self.trunk_param_count
    }

    /// SP3 slot 41: device pointer to the value-head slice of the Adam
    /// second-moment buffer. Mirrors `value_adam_m_ptr` over `v_buf`.
    pub(crate) fn value_adam_v_ptr(&self) -> u64 {
        let param_sizes = compute_param_sizes(&self.config);
        self.v_buf.raw_ptr() + padded_byte_offset(&param_sizes, 13)
    }

    /// Element count of the value-head Adam-v slice (tensors [13..17), padded).
    pub(crate) fn value_adam_v_len(&self) -> usize {
        self.value_adam_m_len()
    }

    /// SP3 slot 42: device pointer to the branch-head slice of the Adam
    /// second-moment buffer. Mirrors `branch_adam_m_ptr` over `v_buf`.
    pub(crate) fn branch_adam_v_ptr(&self) -> u64 {
        let param_sizes = compute_param_sizes(&self.config);
        self.v_buf.raw_ptr() + padded_byte_offset(&param_sizes, 17)
    }

    /// Element count of the branch-head Adam-v slice (tensors [17..33), padded).
    pub(crate) fn branch_adam_v_len(&self) -> usize {
        self.branch_adam_m_len()
    }

    /// SP3 slot 44: device pointer to the trunk slice of `params_buf` (the
    /// online weights backing the trunk forward path). Tensors [0..13).
    pub(crate) fn trunk_params_ptr(&self) -> u64 {
        self.params_buf.raw_ptr()
    }

    /// Element count of the trunk params slice (= `trunk_param_count`).
    pub(crate) fn trunk_params_len(&self) -> usize {
        self.trunk_param_count
    }

    /// SP3 slot 45: device pointer to the heads slice of `params_buf` (value +
    /// branch concatenated). Tensors [13..33). Used by the threshold-check
    /// kernel to scan head weights as a single contiguous range — the trunk
    /// vs heads split provides finer localisation when divergence appears.
    pub(crate) fn heads_params_ptr(&self) -> u64 {
        let param_sizes = compute_param_sizes(&self.config);
        self.params_buf.raw_ptr() + padded_byte_offset(&param_sizes, 13)
    }

    /// Element count of the heads params slice (tensors [13..33), padded).
    pub(crate) fn heads_params_len(&self) -> usize {
        let param_sizes = compute_param_sizes(&self.config);
        let start = padded_byte_offset(&param_sizes, 13) as usize;
        let end = padded_byte_offset(&param_sizes, 33) as usize;
        (end - start) / std::mem::size_of::<f32>()
    }

    /// SP3 slot 46: device pointer to `denoise_target_q_buf` `[B, total_actions]`,
    /// the target-network Q-values written by the C51 target-q compute pass via
    /// `compute_expected_q` (which strides by `total_actions = b0+b1+b2+b3 = 13`
    /// in the 4-direction factored layout). The threshold-check kernel scans
    /// the full buffer flat — all entries are valid Q-values, so widening from
    /// the legacy `b*12` exposure-layout sizing to `b*total_actions` is
    /// monotone (more samples in the histogram, no semantic change). Per-sample
    /// consumers (q_denoise_backward, denoise_loss_grad) receive
    /// `target_stride = total_actions` and read only the first D=12 entries —
    /// matching the cross-branch-attention narrowed denoiser pipeline.
    pub(crate) fn target_q_ptr(&self) -> u64 {
        self.denoise_target_q_buf.raw_ptr()
    }

    /// Element count of the target-q buffer (= `B * total_actions`, e.g. `B * 13`
    /// for the 4-direction factored layout). Used by SP4 p99 producer +
    /// SP3 threshold-check kernels for flat scans of the full buffer.
    pub(crate) fn target_q_len(&self) -> usize {
        self.denoise_target_q_buf.len()
    }

    /// SP3 slot 47: device pointer to `atom_positions_buf` `[4, num_atoms]`,
    /// the per-branch C51 adaptive atom positions written by the
    /// atom-positions ISV producer kernel. Threshold check guards against NaN
    /// or unsorted positions cascading through the distributional Q output.
    pub(crate) fn atom_positions_ptr(&self) -> u64 {
        self.atom_positions_buf.raw_ptr()
    }

    /// Element count of the atom-positions buffer (= `4 * num_atoms`).
    pub(crate) fn atom_positions_len(&self) -> usize {
        self.atom_positions_buf.len()
    }

    /// Reference to the next_states buffer on GPU.
    ///
    /// Shape: `[B, STATE_DIM]` — contains the batch's next states after `train_step()`.
    /// Used by the IQN dual-head to compute target h_s2 via a trunk forward kernel.
    pub fn next_states_buf(&self) -> &CudaSlice<f32> {
        &self.next_states_buf
    }

    /// Reference to the actions buffer on GPU.
    ///
    /// Shape: `[B]` i32 — flat actions from the batch. Valid after `train_step()` or
    /// `train_step_gpu()`. Used by IQN to decode flat → branch actions on GPU.
    pub fn actions_buf(&self) -> &CudaSlice<i32> {
        &self.actions_buf
    }

    /// Reference to the rewards buffer on GPU.
    ///
    /// Shape: `[B]` f32 — rewards from the batch. Valid after `train_step()` or
    /// `train_step_gpu()`. Used by IQN for Bellman target computation on GPU.
    pub fn rewards_buf(&self) -> &CudaSlice<f32> {
        &self.rewards_buf
    }

    /// Mutable reference to the rewards buffer on GPU (for in-place normalization).
    pub fn rewards_buf_mut(&mut self) -> &mut CudaSlice<f32> {
        &mut self.rewards_buf
    }

    /// Reference to the dones buffer on GPU.
    ///
    /// Shape: `[B]` f32 — done flags (0.0/1.0). Valid after `train_step()` or
    /// `train_step_gpu()`. Used by IQN for Bellman target masking on GPU.
    pub fn dones_buf(&self) -> &CudaSlice<f32> {
        &self.dones_buf
    }

    /// Reference to the training config.
    pub fn config(&self) -> &GpuDqnTrainConfig {
        &self.config
    }

    /// Raw device pointer to `grad_buf` for auxiliary gradient injection between graph phases.
    ///
    /// Mutable reference to grad_buf for test injection.
    #[cfg(test)]
    pub fn grad_buf_mut(&mut self) -> &mut CudaSlice<f32> {
        &mut self.grad_buf
    }

    /// Mutable reference to CQL gradient scratch for test injection.
    #[cfg(test)]
    pub fn cql_grad_scratch_mut(&mut self) -> &mut CudaSlice<f32> {
        &mut self.cql_grad_scratch
    }

    /// Total number of trainable parameters.
    pub fn total_params(&self) -> usize {
        self.total_params
    }

    /// SP4 Layer B: per-group param counts for the 3 DQN-internal groups
    /// (Trunk / Value / Branches). Mirrors the byte-offset slicing in
    /// `param_group_buffers` and `launch_adam_update`. Returned counts feed
    /// `pearl_c_post_adam_engagement_check` per group, replacing Layer A's
    /// "total_params under group 0 only" approximation.
    pub fn dqn_group_param_counts(&self) -> (usize, usize, usize) {
        let f32_sz = std::mem::size_of::<f32>() as u64;
        let param_sizes = compute_param_sizes(&self.config);
        let trunk = self.trunk_param_count;
        let value_off = padded_byte_offset(&param_sizes, 13);
        let value_end = padded_byte_offset(&param_sizes, 17);
        let value = ((value_end - value_off) / f32_sz) as usize;
        let branches_off = padded_byte_offset(&param_sizes, 17);
        let branches_end = padded_byte_offset(&param_sizes, 33);
        let branches = ((branches_end - branches_off) / f32_sz) as usize;
        (trunk, value, branches)
    }

    /// Apply IQN auxiliary gradient to the shared trunk via SAXPY into `grad_buf`.
    ///
    /// After `graph_forward` replay (C51 forward → backward) and IQN backward
    /// have both completed, this method backpropagates IQN's `d_h_s2` through
    /// the trunk's two FC layers and ADDs the resulting gradients to `grad_buf`
    /// (which already contains C51's gradients from `graph_forward`).
    ///
    /// The caller then replays `graph_adam` which sees the combined C51+IQN
    /// gradient and applies a single Adam update. This eliminates the momentum
    /// mismatch from having a separate IQN Adam optimizer.
    ///
    /// Steps:
    /// 1. Zero scratch buffer (`iqn_trunk_m`, repurposed as scratch)
    /// 2. GRN trunk backward (encoder_backward_chain) → trunk gradients in scratch
    /// 3. SAXPY: `grad_buf[trunk] += iqn_lambda * scratch[trunk]`
    pub fn apply_iqn_trunk_gradient(
        &mut self,
        iqn_d_h_s2_ptr: u64,
        _online_dueling: &mut DuelingWeightSet,
        iqn_budget: f32,
    ) -> Result<(), MLError> {
        // Plan 4 Task 2c.3c.4: Replaces the legacy ReLU+Linear→Linear chain
        // with the GRN trunk backward (encoder_backward_chain). Writes into
        // `iqn_trunk_m` (which now mirrors grad_buf's padded layout for the
        // first 13 tensors), then SAXPY into grad_buf. The `relu_mask` on
        // `bw_d_h_s2` is gone — h_s2 GRN ends in LayerNorm, not ReLU.
        let b = self.config.batch_size;
        let _eg = EventTrackingGuard::new(self.stream.context());
        let sh2 = self.config.shared_h2;
        let f32_size = std::mem::size_of::<f32>();

        // Total f32 element count in iqn_trunk_m, matching grad_buf's padded
        // layout for trunk tensors [0..13). See constructor (Task 2c.3c.4).
        let param_sizes = compute_param_sizes(&self.config);
        let trunk_grad_total: usize =
            (padded_byte_offset(&param_sizes, 13) as usize) / f32_size;

        // ── 1. Zero the scratch buffer (iqn_trunk_m) ──
        // encoder_backward_chain accumulates dW with beta=1 (cuBLAS dW GEMM)
        // and overwrites dgamma/dbeta/dB with beta=0 inside the bias-grad
        // and LN reduction kernels. The dW path requires zero init each call.
        {
            let scratch_base = self.ptrs.iqn_trunk_m;
            let n_bytes = trunk_grad_total * f32_size;
            unsafe {
                cudarc::driver::result::memset_d8_async(
                    scratch_base, 0u8, n_bytes, self.stream.cu_stream()
                ).map_err(|e| MLError::ModelError(format!("memset iqn scratch: {e}")))?;
            }
        }

        // ── 2. DtoD copy IQN d_h_s2 (f32) → bw_d_h_s2 (f32) ──
        // IQN d_h_s2 is the gradient flowing into the h_s2 GRN output (post-
        // LayerNorm). No ReLU mask — LayerNorm has no truncation derivative.
        {
            let n_bytes = b * sh2 * f32_size;
            self.graph_safe_copy_f32(self.ptrs.bw_d_h_s2, iqn_d_h_s2_ptr, n_bytes, "iqn_d_h_s2")?;
        }

        // SP1 Phase B (slot 35): bw_d_h_s2 snapshot AFTER the IQN DtoD
        // overwrite, BEFORE the IQN aux `encoder_backward_chain` consumes
        // it. Symmetric with slot 27 (which checks the SOURCE side
        // `iqn_d_h_s2_ptr`); both are needed because the DtoD might be a
        // graph-capture artefact (cudarc cuMemcpy in graph capture has
        // historic edge cases — see `bf16_session_2026-03-28.md` and
        // `session_2026-04-05_h100_hang.md`). If slot 27 is clean and
        // slot 35 fires, the DtoD itself is suspect; if slot 27 fires,
        // the IQN backward is the seed. Per-slot semantics in
        // `docs/dqn-backward-nan-audit.md` (slot 35 row).
        self.check_nan_f32(self.ptrs.bw_d_h_s2, b * sh2, 35)?;

        // SP4 Layer C #260 (2026-05-01): backward-grad sanitiser bound for
        // `bw_d_h_s2` is now ISV-driven via the dedicated
        // `BW_D_H_S2_BOUND_INDEX=171` slot. Producer
        // (`launch_sp4_bw_d_h_s2_p99`) measures p99(|bw_d_h_s2|) post-DtoD-
        // copy + post-NaN-check, applies Pearls A+D on the same stream, and
        // writes the smoothed bound to ISV[171]. Consumer reads the slot
        // directly with `EPS_CLAMP_FLOOR=1.0` ε on cold-start (Invariant 1
        // carve-out per `feedback_isv_for_adaptive_bounds`).
        //
        // Migrated from SP1-Phase-C `1e6 × ISV[H_S2_RMS_EMA].max(1.0)`
        // hardcoded multiplier — the last remaining `1e6 × isv` constant
        // in the cuBLAS backward-grad sanitiser path after Layer A/B
        // closed out the rest of SP4. The new bound is Pearls-smoothed
        // p99 of the actual gradient distribution, tighter than the
        // pre-existing 1e6 ceiling but appropriate-scale for the observed
        // values.
        //
        // Ordering invariant: the slot 35 NaN check (`check_nan_f32(...,
        // 35)`) fires BEFORE this clamp, so any NaN entry is captured in
        // the regression sentinel before sanitisation. Producer launches
        // BEFORE the consumer at this site.
        use crate::cuda_pipeline::sp4_isv_slots::BW_D_H_S2_BOUND_INDEX;
        use crate::cuda_pipeline::sp4_wiener_ema::EPS_CLAMP_FLOOR;
        self.launch_sp4_bw_d_h_s2_p99()?;
        let max_abs_input = self
            .read_isv_signal_at(BW_D_H_S2_BOUND_INDEX)
            .max(EPS_CLAMP_FLOOR);
        self.launch_clamp_finite_f32(self.ptrs.bw_d_h_s2, b * sh2, max_abs_input)?;

        // ── 3. GRN trunk backward (encoder_backward_chain) ──
        let w_ptrs = f32_weight_ptrs_from_base(self.ptrs.params_ptr, &param_sizes);
        let states_ptr = if self.config.bottleneck_dim > 0 {
            self.bn_concat_buf.raw_ptr()
        } else {
            bw_raw_f32_ptr(&self.states_buf, &self.stream)
        };
        let h_s1_save_ptr = bw_raw_f32_ptr(&self.save_h_s1, &self.stream);
        let d_h_s2 = self.ptrs.bw_d_h_s2;
        let d_h_s1 = bw_raw_f32_ptr(&self.bw_d_h_s1, &self.stream);
        let trunk_scratch_base = self.ptrs.iqn_trunk_m;

        let grn_h_s1_d_pre_ln_p      = self.bw_grn_h_s1_d_pre_ln.raw_ptr();
        let grn_h_s1_d_linear_b_out_p= self.bw_grn_h_s1_d_linear_b_out.raw_ptr();
        let grn_h_s1_d_elu_out_p     = self.bw_grn_h_s1_d_elu_out.raw_ptr();
        let grn_h_s1_d_linear_a_out_p= self.bw_grn_h_s1_d_linear_a_out.raw_ptr();
        let grn_h_s2_d_pre_ln_p      = self.bw_grn_h_s2_d_pre_ln.raw_ptr();
        let grn_h_s2_d_linear_b_out_p= self.bw_grn_h_s2_d_linear_b_out.raw_ptr();
        let grn_h_s2_d_elu_out_p     = self.bw_grn_h_s2_d_elu_out.raw_ptr();
        let grn_h_s2_d_linear_a_out_p= self.bw_grn_h_s2_d_linear_a_out.raw_ptr();

        // Plan 4 Task 1B-iv-ext: pass non-zero `s1_dx_output` so the chain
        // produces dL/d(bn_concat) into `bn_d_concat_buf` for the VSN
        // backward extension below. Reuses the existing scratch buffer
        // because the main backward path already consumed it (aux paths run
        // after main per `fused_training.rs` ordering); `encoder_backward_chain`
        // beta=0-overwrites the Linear_a dX path and beta=1-accumulates the
        // Linear_residual dX path into this slot.
        let bn_d_concat_for_iqn = if self.config.bottleneck_dim > 0 {
            self.bn_d_concat_buf.raw_ptr()
        } else {
            0u64
        };
        self.cublas_forward.encoder_backward_chain(
            &self.stream,
            &self.cublas_backward,
            states_ptr,
            &w_ptrs,
            trunk_scratch_base,
            &param_sizes,
            d_h_s2,
            d_h_s1,
            bn_d_concat_for_iqn, // 1B-iv-ext: bottleneck dX needed for VSN aux backward
            grn_h_s1_d_pre_ln_p,
            grn_h_s1_d_linear_b_out_p,
            grn_h_s1_d_elu_out_p,
            grn_h_s1_d_linear_a_out_p,
            grn_h_s2_d_pre_ln_p,
            grn_h_s2_d_linear_b_out_p,
            grn_h_s2_d_elu_out_p,
            grn_h_s2_d_linear_a_out_p,
            h_s1_save_ptr,
        )?;

        // SP1 Phase C surgical fix (slot 26 output sanitise): post-call
        // defence-in-depth — `encoder_backward_chain`'s cuBLAS Linear_a/
        // Linear_b/Linear_residual dW/dX/dB GEMMs accumulate onto
        // `iqn_trunk_m` (= `trunk_scratch_base`) with beta=1; an extreme
        // intermediate-product overflow can produce NaN/Inf even when the
        // input gradient was finite (residual 8% from
        // `session_2026-04-05_nan_investigation.md` cross-checked against
        // smoke-test-xvzgk slot 26). isfinite-or-zero + magnitude clamp
        // catches that residual class. Same `max_abs_input` bound as the
        // pre-call clamp because both inputs and outputs of the trunk GEMM
        // share the H_S2_RMS_EMA-driven magnitude scale.
        //
        // Inline slot 26 NaN check IMMEDIATELY BEFORE the clamp preserves
        // the regression sentinel: `run_nan_checks_post_backward`
        // (fused_training.rs:1540) runs AFTER this clamp, so without this
        // pre-clamp check the slot 26 flag would observe sanitised zeros
        // and never fire. Mirrors the same slot-35 NaN-check + clamp
        // pattern in `apply_iqn_trunk_gradient`.
        self.check_nan_f32(self.ptrs.iqn_trunk_m, trunk_grad_total, 26)?;
        self.launch_clamp_finite_f32(
            self.ptrs.iqn_trunk_m, trunk_grad_total, max_abs_input,
        )?;

        // ── 3b. Plan 4 Task 1B-iv-ext: VSN backward chain ─────────────────
        // Consumes the bn_d_concat just produced by encoder_backward_chain,
        // assembles `d_vsn_gated_state`, and writes 24 VSN dW/dB into
        // `vsn_dw_iqn_aux_scratch` at goff[95..119) — the scratch base is
        // anchored so `padded_byte_offset(95)` lands at scratch byte 0.
        // The trailing SAXPY (step 5) then mixes this into grad_buf[95..119)
        // with the same `iqn_lambda * iqn_readiness * iqn_budget` scale that
        // the trunk SAXPY uses. Closes the gradient-coverage gap from 1B-iv.
        if self.config.bottleneck_dim > 0 {
            let vsn_begin_off = padded_byte_offset(&param_sizes, 95);
            let vsn_aux_scratch_anchor =
                self.vsn_dw_iqn_aux_scratch.raw_ptr().wrapping_sub(vsn_begin_off);
            // Zero the scratch before write (vsn_backward writes dW with
            // beta=1 inside cuBLAS dW GEMMs). One memset per call —
            // vsn_aux_scratch_floats f32s.
            let vsn_scratch_bytes = self.vsn_dw_iqn_aux_scratch.len() * std::mem::size_of::<f32>();
            unsafe {
                cudarc::driver::result::memset_d8_async(
                    self.vsn_dw_iqn_aux_scratch.raw_ptr(),
                    0u8,
                    vsn_scratch_bytes,
                    self.stream.cu_stream(),
                ).map_err(|e| MLError::ModelError(format!(
                    "memset vsn_dw_iqn_aux_scratch: {e}"
                )))?;
            }
            Self::aux_bottleneck_vsn_backward_dispatch(
                &mut self.cublas_forward,
                &self.cublas_backward,
                &self.stream,
                &self.config,
                &self.bn_d_hidden_buf,
                &self.bn_hidden_buf,
                &self.states_buf,
                &self.vsn_d_gated_state_buf,
                &self.vsn_mask_buf,
                &self.vsn_d_logits_buf,
                &self.vsn_d_state_buf,
                &self.vsn_d_logit_scratch_buf,
                &self.vsn_d_h1_scratch_buf,
                &self.vsn_group_begins_buf,
                &self.vsn_group_ends_buf,
                &self.vsn_save_h1_g0_buf,
                &self.vsn_save_h1_g1_buf,
                &self.vsn_save_h1_g2_buf,
                &self.vsn_save_h1_g3_buf,
                &self.vsn_save_h1_g4_buf,
                &self.vsn_save_h1_g5_buf,
                &self.bn_tanh_backward_kernel,
                &self.vsn_d_gated_state_portfolio_kernel,
                &self.vsn_logit_gather_kernel,
                &self.scale_f32_aux,                  // 1B-iv-rc: VSN dW dilution (VSN_DW_DAMP)
                bn_d_concat_for_iqn,
                vsn_aux_scratch_anchor,
                &param_sizes,
                &w_ptrs,
            )?;
        }

        // ── 4. Plain SAXPY: grad_buf[trunk] += iqn_lambda * iqn_budget * scratch ──
        // Scale = config.iqn_lambda × ISV-adaptive iqn_budget. The previous
        // formula multiplied by `iqn_readiness` as a "loss-improvement gate",
        // but readiness initialises to 0.0 and only ramps up when iqn_loss_ema
        // drops below iqn_loss_initial — which cannot happen until IQN's
        // gradient reaches the trunk. Bootstrap deadlock: trunk_iqn=0.0000
        // for every observed epoch in the L40S validation runs, downstream
        // strangling direction-Q discrimination → val argmax glues to one
        // direction → 22-34 trades per 858k-bar window. iqn_budget already
        // throttles via the per-component budget controller (60% IQN /
        // ISV-driven), so readiness was an additive band-aid that became
        // load-bearing. Removed; readiness is retained on `self` solely
        // because the C51 loss kernel reuses iqn_readiness_dev_ptr as a CVaR
        // alpha pointer (see launch site near line 16227) — that is a
        // separate semantic overload tracked for follow-up.
        // Uses aux_child-specific handle — saxpy_f32_kernel is captured in
        // forward_child and a separate handle is needed here in aux_child.
        {
            let scratch_ptr = self.ptrs.iqn_trunk_m;
            let grad_ptr = self.ptrs.grad_buf;
            let scale = self.config.iqn_lambda * iqn_budget;
            let n_i32 = trunk_grad_total as i32;
            let blocks = ((trunk_grad_total + 255) / 256) as u32;
            unsafe {
                self.stream
                    .launch_builder(&self.saxpy_f32_aux)
                    .arg(&grad_ptr)
                    .arg(&scratch_ptr)
                    .arg(&scale)
                    .arg(&n_i32)
                    .launch(LaunchConfig {
                        grid_dim: (blocks, 1, 1),
                        block_dim: (256, 1, 1),
                        shared_mem_bytes: 0,
                    })
                    .map_err(|e| MLError::ModelError(format!("IQN trunk SAXPY: {e}")))?;
            }
        }

        // ── 5. Plan 4 Task 1B-iv-ext: VSN-range SAXPY ────────────────────
        // grad_buf[VSN range] += iqn_lambda * iqn_budget * vsn_dw_iqn_aux_scratch
        // Same scaling factor as the trunk SAXPY above (step 4) — both must
        // stay in lockstep so the per-component IQN budget distributes
        // identically across trunk and VSN tensors. iqn_readiness removed
        // for the same bootstrap-deadlock reason documented at the trunk
        // SAXPY above.
        if self.config.bottleneck_dim > 0 {
            let vsn_begin_off = padded_byte_offset(&param_sizes, 95);
            let vsn_count = self.vsn_dw_iqn_aux_scratch.len();
            let scratch_ptr = self.vsn_dw_iqn_aux_scratch.raw_ptr();
            let grad_vsn_ptr = self.ptrs.grad_buf + vsn_begin_off;
            let scale = self.config.iqn_lambda * iqn_budget;
            let n_i32 = vsn_count as i32;
            let blocks = ((vsn_count + 255) / 256) as u32;
            unsafe {
                self.stream
                    .launch_builder(&self.saxpy_f32_aux)
                    .arg(&grad_vsn_ptr)
                    .arg(&scratch_ptr)
                    .arg(&scale)
                    .arg(&n_i32)
                    .launch(LaunchConfig {
                        grid_dim: (blocks, 1, 1),
                        block_dim: (256, 1, 1),
                        shared_mem_bytes: 0,
                    })
                    .map_err(|e| MLError::ModelError(format!("IQN VSN SAXPY: {e}")))?;
            }
        }

        Ok(())
    }

    /// Apply ensemble diversity gradient through the value head into the trunk.
    ///
    /// Takes `d_logits` [B, NA] (the KL gradient w.r.t. head-0 value logits
    /// produced by the ensemble KL gradient kernel) and backpropagates through:
    ///   1. Value output layer:  d_logits [B, NA] × W_v2^T → d_h_v [B, VH]
    ///   2. ReLU mask:           d_h_v *= (save_h_v > 0)
    ///   3. Value FC layer:      d_h_v [B, VH]  × W_v1^T → d_h_s2 [B, SH2]
    ///   4. Trunk backward (shared layers 2 → 1) into scratch buffer
    ///   5. SAXPY: grad_buf[trunk] += scale * scratch[trunk]
    ///
    /// This mirrors `apply_iqn_trunk_gradient` but starts from value logits
    /// instead of h_s2. The `scale` parameter is `ensemble_diversity_weight`.
    ///
    /// All computation is GPU-only — no CPU readback in the gradient path.
    #[allow(clippy::too_many_arguments)]
    pub fn apply_ensemble_diversity_backward(
        &mut self,
        d_logits_ptr: u64,
        scale: f32,
    ) -> Result<(), MLError> {
        // Plan 4 Task 2c.3c.4: ensemble-diversity trunk path now mirrors the
        // GRN-aware structure — value head dX into d_h_s2 (LN, no ReLU mask),
        // then encoder_backward_chain into iqn_trunk_m, then SAXPY into
        // grad_buf. Value-head weight indices fixed: w_v1 4→13, w_v2 6→15.
        let b = self.config.batch_size;
        let _eg = EventTrackingGuard::new(self.stream.context());
        let sh2 = self.config.shared_h2;
        let vh = self.config.value_h;
        let na = self.config.num_atoms;
        let f32_size = std::mem::size_of::<f32>();

        let param_sizes = compute_param_sizes(&self.config);
        let trunk_grad_total: usize =
            (padded_byte_offset(&param_sizes, 13) as usize) / f32_size;

        // ── 1. Zero the scratch buffer (iqn_trunk_m) ──
        {
            let scratch_base = self.ptrs.iqn_trunk_m;
            let n_bytes = trunk_grad_total * f32_size;
            unsafe {
                cudarc::driver::result::memset_d8_async(
                    scratch_base, 0u8, n_bytes, self.stream.cu_stream()
                ).map_err(|e| MLError::ModelError(format!("memset ens scratch: {e}")))?;
            }
        }

        let w_ptrs = f32_weight_ptrs_from_base(self.ptrs.params_ptr, &param_sizes);

        // ── 2. Value output layer dX: d_logits → d_h_v ──
        // Plan 4 Task 2c.3c.4: w_v2 index 6 → 15 (post-GRN reshuffle).
        {
            let w_v2 = w_ptrs[15]; // W_v2 [NA, VH]
            let dx = self.ptrs.bw_d_h_v;
            self.cublas_backward.launch_dx_only(
                &self.stream, d_logits_ptr, w_v2, dx,
                na, vh, b, 0.0,
            )?;
        }

        // ── 3. ReLU mask on d_h_v (value FC head still uses ReLU) ──
        {
            let d_ptr = self.ptrs.bw_d_h_v;
            let act_ptr = self.ptrs.save_h_v;
            let n_relu = (b * vh) as i32;
            let blocks = ((b * vh + 255) / 256) as u32;
            unsafe {
                self.stream
                    .launch_builder(&self.relu_mask_kernel)
                    .arg(&d_ptr)
                    .arg(&act_ptr)
                    .arg(&n_relu)
                    .launch(LaunchConfig {
                        grid_dim: (blocks, 1, 1),
                        block_dim: (256, 1, 1),
                        shared_mem_bytes: 0,
                    })
                    .map_err(|e| MLError::ModelError(format!("relu_mask ens_h_v: {e}")))?;
            }
        }

        // ── 4. Value FC dX → d_h_s2 (no ReLU mask: trunk now ends in LN) ──
        // Plan 4 Task 2c.3c.4: w_v1 index 4 → 13 (post-GRN reshuffle).
        {
            let dy = self.ptrs.bw_d_h_v;
            let w_v1 = w_ptrs[13]; // W_v1 [VH, SH2]
            let dx = self.ptrs.bw_d_h_s2;
            self.cublas_backward.launch_dx_only(
                &self.stream, dy, w_v1, dx,
                vh, sh2, b, 0.0,
            )?;
        }

        // ── 5. GRN trunk backward (encoder_backward_chain) ──
        // h_s2's gradient flows from d_h_s2 (post-LN) into the trunk via the
        // GRN composition (no ReLU truncation).
        // Plan 4 Task 1B-iv-ext: pass non-zero `s1_dx_output` so the chain
        // produces dL/d(bn_concat) for the VSN backward chain step 5b
        // below. Reuses `bn_d_concat_buf` because the IQN aux path runs
        // before ensemble per `fused_training.rs` ordering and its consumer
        // (the IQN VSN backward) has already finished writing.
        let states_ptr = if self.config.bottleneck_dim > 0 {
            self.bn_concat_buf.raw_ptr()
        } else {
            bw_raw_f32_ptr(&self.states_buf, &self.stream)
        };
        let bn_d_concat_for_ens = if self.config.bottleneck_dim > 0 {
            self.bn_d_concat_buf.raw_ptr()
        } else {
            0u64
        };
        let h_s1_save_ptr = bw_raw_f32_ptr(&self.save_h_s1, &self.stream);
        let d_h_s2 = self.ptrs.bw_d_h_s2;
        let d_h_s1 = bw_raw_f32_ptr(&self.bw_d_h_s1, &self.stream);
        let trunk_scratch_base = self.ptrs.iqn_trunk_m;

        let grn_h_s1_d_pre_ln_p      = self.bw_grn_h_s1_d_pre_ln.raw_ptr();
        let grn_h_s1_d_linear_b_out_p= self.bw_grn_h_s1_d_linear_b_out.raw_ptr();
        let grn_h_s1_d_elu_out_p     = self.bw_grn_h_s1_d_elu_out.raw_ptr();
        let grn_h_s1_d_linear_a_out_p= self.bw_grn_h_s1_d_linear_a_out.raw_ptr();
        let grn_h_s2_d_pre_ln_p      = self.bw_grn_h_s2_d_pre_ln.raw_ptr();
        let grn_h_s2_d_linear_b_out_p= self.bw_grn_h_s2_d_linear_b_out.raw_ptr();
        let grn_h_s2_d_elu_out_p     = self.bw_grn_h_s2_d_elu_out.raw_ptr();
        let grn_h_s2_d_linear_a_out_p= self.bw_grn_h_s2_d_linear_a_out.raw_ptr();

        self.cublas_forward.encoder_backward_chain(
            &self.stream,
            &self.cublas_backward,
            states_ptr,
            &w_ptrs,
            trunk_scratch_base,
            &param_sizes,
            d_h_s2,
            d_h_s1,
            bn_d_concat_for_ens, // 1B-iv-ext: bottleneck dX needed for VSN aux backward
            grn_h_s1_d_pre_ln_p,
            grn_h_s1_d_linear_b_out_p,
            grn_h_s1_d_elu_out_p,
            grn_h_s1_d_linear_a_out_p,
            grn_h_s2_d_pre_ln_p,
            grn_h_s2_d_linear_b_out_p,
            grn_h_s2_d_elu_out_p,
            grn_h_s2_d_linear_a_out_p,
            h_s1_save_ptr,
        )?;

        // ── 5b. Plan 4 Task 1B-iv-ext: VSN backward chain ─────────────────
        // Mirrors the IQN aux path's VSN extension. Writes 24 dW/dB into
        // `vsn_dw_ensemble_aux_scratch`, then SAXPY's into grad_buf[VSN]
        // with the caller-supplied diversity weight (step 7).
        if self.config.bottleneck_dim > 0 {
            let vsn_begin_off = padded_byte_offset(&param_sizes, 95);
            let vsn_aux_scratch_anchor =
                self.vsn_dw_ensemble_aux_scratch.raw_ptr().wrapping_sub(vsn_begin_off);
            let vsn_scratch_bytes = self.vsn_dw_ensemble_aux_scratch.len() * std::mem::size_of::<f32>();
            unsafe {
                cudarc::driver::result::memset_d8_async(
                    self.vsn_dw_ensemble_aux_scratch.raw_ptr(),
                    0u8,
                    vsn_scratch_bytes,
                    self.stream.cu_stream(),
                ).map_err(|e| MLError::ModelError(format!(
                    "memset vsn_dw_ensemble_aux_scratch: {e}"
                )))?;
            }
            Self::aux_bottleneck_vsn_backward_dispatch(
                &mut self.cublas_forward,
                &self.cublas_backward,
                &self.stream,
                &self.config,
                &self.bn_d_hidden_buf,
                &self.bn_hidden_buf,
                &self.states_buf,
                &self.vsn_d_gated_state_buf,
                &self.vsn_mask_buf,
                &self.vsn_d_logits_buf,
                &self.vsn_d_state_buf,
                &self.vsn_d_logit_scratch_buf,
                &self.vsn_d_h1_scratch_buf,
                &self.vsn_group_begins_buf,
                &self.vsn_group_ends_buf,
                &self.vsn_save_h1_g0_buf,
                &self.vsn_save_h1_g1_buf,
                &self.vsn_save_h1_g2_buf,
                &self.vsn_save_h1_g3_buf,
                &self.vsn_save_h1_g4_buf,
                &self.vsn_save_h1_g5_buf,
                &self.bn_tanh_backward_kernel,
                &self.vsn_d_gated_state_portfolio_kernel,
                &self.vsn_logit_gather_kernel,
                &self.scale_f32_aux,                  // 1B-iv-rc: VSN dW dilution (VSN_DW_DAMP)
                bn_d_concat_for_ens,
                vsn_aux_scratch_anchor,
                &param_sizes,
                &w_ptrs,
            )?;
        }

        // ── 6. Plain SAXPY: grad_buf[trunk] += scale * scratch ──
        // No per-component clip — single global clip in Adam handles safety.
        // Uses aux_child-specific handle — saxpy_f32_kernel is captured in
        // forward_child and a separate handle is needed here in aux_child.
        {
            let scratch_ptr = self.ptrs.iqn_trunk_m;
            let grad_ptr = self.ptrs.grad_buf;
            let n_i32 = trunk_grad_total as i32;
            let blocks = ((trunk_grad_total + 255) / 256) as u32;
            unsafe {
                self.stream
                    .launch_builder(&self.saxpy_f32_aux)
                    .arg(&grad_ptr)
                    .arg(&scratch_ptr)
                    .arg(&scale)
                    .arg(&n_i32)
                    .launch(LaunchConfig {
                        grid_dim: (blocks, 1, 1),
                        block_dim: (256, 1, 1),
                        shared_mem_bytes: 0,
                    })
                    .map_err(|e| MLError::ModelError(format!("ensemble SAXPY: {e}")))?;
            }
        }

        // ── 7. Plan 4 Task 1B-iv-ext: VSN-range SAXPY ────────────────────
        // grad_buf[VSN range] += scale * vsn_dw_ensemble_aux_scratch
        // Same diversity weight as the trunk SAXPY above — closes the
        // gradient-coverage gap so VSN params learn from the ensemble KL
        // signal at the same effective amplitude as trunk weights do.
        if self.config.bottleneck_dim > 0 {
            let vsn_begin_off = padded_byte_offset(&param_sizes, 95);
            let vsn_count = self.vsn_dw_ensemble_aux_scratch.len();
            let scratch_ptr = self.vsn_dw_ensemble_aux_scratch.raw_ptr();
            let grad_vsn_ptr = self.ptrs.grad_buf + vsn_begin_off;
            let n_i32 = vsn_count as i32;
            let blocks = ((vsn_count + 255) / 256) as u32;
            unsafe {
                self.stream
                    .launch_builder(&self.saxpy_f32_aux)
                    .arg(&grad_vsn_ptr)
                    .arg(&scratch_ptr)
                    .arg(&scale)
                    .arg(&n_i32)
                    .launch(LaunchConfig {
                        grid_dim: (blocks, 1, 1),
                        block_dim: (256, 1, 1),
                        shared_mem_bytes: 0,
                    })
                    .map_err(|e| MLError::ModelError(format!("ensemble VSN SAXPY: {e}")))?;
            }
        }

        Ok(())
    }

    /// Apply regime-adaptive scaling to td_errors before PER priority update.
    ///
    /// Computes mean ADX/CUSUM from the batch states as the "current regime" proxy,
    /// then scales td_errors by Gaussian similarity: samples from similar regimes get
    /// higher priority, adapting replay to the current market conditions.
    ///
    /// GPU-only: reads from states_buf, writes to td_errors_buf.
    /// GPU-only regime-adaptive PER scaling. Reads target ADX/CUSUM from
    /// the first sample in states_buf directly in the kernel — zero CPU readback.
    pub fn regime_scale_td_errors(&self) -> Result<(), MLError> {
        let bs = self.config.batch_size as i32;
        let blocks = ((self.config.batch_size + 255) / 256) as u32;

        let _evt_guard = EventTrackingGuard::new(self.stream.context());

        let td_ptr = self.td_errors_buf.raw_ptr();
        let states_ptr = self.states_buf.raw_ptr();

        // Use padded state_dim as stride — states_buf rows are pad128(state_dim) wide.
        let state_dim_i32 = ml_core::state_layout::STATE_DIM_PADDED as i32;
        unsafe {
            self.stream
                .launch_builder(&self.regime_scale_kernel)
                .arg(&td_ptr)
                .arg(&states_ptr)
                .arg(&bs)
                .arg(&state_dim_i32)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("regime_scale_kernel: {e}")))?;
        }

        Ok(())
    }

    /// Launch `pad_states_kernel`: scatter contiguous `[batch, sd]` f32 states
    /// Launch HER in-place relabel kernel: writes goal columns + rewards directly
    /// into the trainer's padded staging buffers. Zero intermediate allocation.
    ///
    /// `src_next_states`: unpadded f32 from GpuBatch `[batch_size, state_dim]`
    /// `donor_indices`: GPU-resident i32 `[her_batch_size]`
    /// `normal_count`: first HER row offset in staging buffers
    /// `goal_dim`: number of goal columns to replace
    /// `state_dim`: unpadded state dimension
    pub fn launch_her_inplace_relabel(
        &self,
        src_next_states_ptr: u64,
        donor_indices_ptr: u64,
        normal_count: usize,
        her_batch_size: usize,
        goal_dim: usize,
        state_dim: usize,
    ) -> Result<(), MLError> {
        let padded_sd = ml_core::state_layout::STATE_DIM_PADDED;
        let offset_i32 = normal_count as i32;
        let goal_dim_i32 = goal_dim as i32;
        let src_stride_i32 = state_dim as i32;
        let dst_stride_i32 = padded_sd as i32;
        let state_dim_i32 = state_dim as i32;

        let states_ptr = self.states_buf.raw_ptr();
        let next_states_ptr = self.next_states_buf.raw_ptr();
        let rewards_ptr = self.rewards_buf.raw_ptr();

        let launch_cfg = cudarc::driver::LaunchConfig {
            grid_dim: (her_batch_size as u32, 1, 1),
            block_dim: (32, 1, 1),  // must match working SHA — graph_mega node structure on Hopper
            shared_mem_bytes: 0,
        };

        unsafe {
            self.stream
                .launch_builder(&self.her_inplace_kernel)
                .arg(&states_ptr)
                .arg(&next_states_ptr)
                .arg(&rewards_ptr)
                .arg(&src_next_states_ptr)
                .arg(&donor_indices_ptr)
                .arg(&offset_i32)
                .arg(&goal_dim_i32)
                .arg(&src_stride_i32)
                .arg(&dst_stride_i32)
                .arg(&state_dim_i32)
                .launch(launch_cfg)
                .map_err(|e| MLError::ModelError(format!("her_inplace_relabel launch: {e}")))?;
        }
        Ok(())
    }

    /// into a padded `[batch, padded_sd]` destination with zero-filled columns.
    ///
    /// This eliminates CUTLASS K-tile OOB reads in the first-layer GemmEx.
    pub(crate) fn launch_pad_states(
        &self,
        dst: u64,   // [batch, padded_sd]  — must be pre-allocated
        src: u64,   // [batch, sd]         — contiguous source
        batch: usize,
    ) -> Result<(), MLError> {
        let sd = ml_core::state_layout::STATE_DIM as i32;
        let padded_sd = ml_core::state_layout::STATE_DIM_PADDED as i32;
        let total = batch as i32 * padded_sd;
        let blocks = ((total as u32) + 255) / 256;

        unsafe {
            self.stream
                .launch_builder(&self.pad_states_kernel)
                .arg(&dst)
                .arg(&src)
                .arg(&(batch as i32))
                .arg(&sd)
                .arg(&padded_sd)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("pad_states_kernel: {e}")))?;
        }

        Ok(())
    }

    /// Apply CQL (Conservative Q-Learning) gradient injection.
    ///
    /// Computes the CQL penalty gradient w.r.t. the C51 logits and runs a
    /// second cuBLAS backward pass to produce parameter gradients, which are
    /// accumulated into `grad_buf` (beta=1.0) on top of the C51 gradients.
    ///
    /// `barrier_weight`: F5/D2 Q-gap barrier scale. When > 0, injects the
    /// barrier gradient into the CQL d-logit buffers AFTER the CQL kernel
    /// and BEFORE backward_full, so both contributions share one backward pass.
    /// Pass 0.0 to skip (no-op on the barrier path).
    ///
    /// Called between `graph_forward` and `graph_adam` in `FusedTrainingCtx`.
    /// Returns `true` if CQL was applied, `false` if skipped (disabled or no kernel).
    pub fn apply_cql_gradient(
        &mut self,
        barrier_weight: f32,
    ) -> Result<bool, MLError> {
        let _eg = EventTrackingGuard::new(self.stream.context());
        let cql_kernel = match &self.cql_logit_grad_kernel {
            Some(k) => k.clone(),
            None => return Ok(false),
        };
        // Read cql_alpha base from ISV[CQL_ALPHA_INDEX] (Plan 1 Task 12 / spec §4.C.6).
        // When ISV is null (smoke scale without ISV warmup), fall back to the config
        // field so smoke tests that skip ISV bootstrap still work.
        let cql_alpha_base = if self.isv_signals_pinned.is_null() {
            self.config.cql_alpha
        } else {
            unsafe { *self.isv_signals_pinned.add(CQL_ALPHA_INDEX) }
        };
        if cql_alpha_base <= 0.0 {
            return Ok(false);
        }

        let b = self.config.batch_size;
        let na = self.config.num_atoms;
        let b0 = self.config.branch_0_size;
        let b1 = self.config.branch_1_size;
        let b2 = self.config.branch_2_size;
        let b3 = self.config.branch_3_size;
        let total_actions = b0 + b1 + b2 + b3;
        let _total_branch_atoms = total_actions * na;

        let _evt_guard = EventTrackingGuard::new(self.stream.context());

        // Step 1: Compute CQL logit gradients via kernel
        let v_logits_ptr = self.on_v_logits_buf.raw_ptr();
        let adv_logits_ptr = self.on_b_logits_buf.raw_ptr();
        let actions_ptr = self.actions_buf.raw_ptr();
        let d_v_ptr = self.cql_d_value_logits.raw_ptr();
        let d_adv_ptr = self.cql_d_adv_logits.raw_ptr();

        // B1/G2: Adaptive cql_alpha.
        //  - health from ISV[LEARNING_HEALTH_INDEX]
        //  - regime_stability from ISV[11]
        //  - cql_alpha_eff = base × (1 − regime_stability) × health
        //    Collapse → 0 (CQL off); volatile+healthy → full; stable+healthy → 0.
        //
        // Task 2.5 Bug #7 — when `isv_signals_pinned` is null (smoke scale
        // without ISV warmup), the previous silent fallback of (0.5, 0.5)
        // gave `cql_alpha_eff = base × 0.5 × 0.5 = 0.25 × base` by degenerate
        // math, not by design — violating feedback_no_hiding.md. Fix: emit a
        // tracing::warn! once and gate the regime multiplier OFF (health=1.0,
        // regime=0.0 → cql_alpha = base × 1.0 × 1.0 = base, the scheduled
        // value). Full ISV warmup at smoke scale is a follow-up task.
        let cql_alpha = if self.isv_signals_pinned.is_null() {
            // Log once per-ctor lifetime so spam is bounded (this function
            // runs every step). The std::sync::Once pattern here is local
            // to this formula — constructors still populate isv_signals_pinned
            // at full scale so the warn only fires under smoke-scoped runs
            // that legitimately skip ISV warmup.
            static WARN_ONCE: std::sync::Once = std::sync::Once::new();
            WARN_ONCE.call_once(|| {
                tracing::warn!(
                    "cql_alpha regime gate: isv_signals_pinned is null — formula \
                     disabled, using scheduled cql_alpha (base × 1.0 × 1.0). \
                     Smoke-scale runs without ISV warmup expected; full-scale \
                     should never hit this path."
                );
            });
            cql_alpha_base
        } else {
            let (health, regime_stability) = unsafe {
                let h = (*self.isv_signals_pinned.add(LEARNING_HEALTH_INDEX)).clamp(0.0, 1.0);
                let s = (*self.isv_signals_pinned.add(11)).clamp(0.0, 1.0);
                (h, s)
            };
            cql_alpha_base * (1.0 - regime_stability) * health
        };

        // Cache for logging via FusedTrainingCtx / DQNTrainer.
        self.last_cql_alpha_eff = cql_alpha;

        let n_i32 = b as i32;
        let na_i32 = na as i32;
        let b0_i32 = b0 as i32;
        let b1_i32 = b1 as i32;
        let b2_i32 = b2 as i32;
        let b3_i32 = b3 as i32;
        let blocks = ((b + 255) / 256) as u32;
        unsafe {
            self.stream
                .launch_builder(&cql_kernel)
                .arg(&v_logits_ptr)
                .arg(&adv_logits_ptr)
                .arg(&actions_ptr)
                .arg(&d_v_ptr)
                .arg(&d_adv_ptr)
                .arg(&cql_alpha)
                .arg(&n_i32)
                .arg(&na_i32)
                .arg(&b0_i32)
                .arg(&b1_i32)
                .arg(&b2_i32)
                .arg(&b3_i32)
                .arg(&self.eval_v_range_ptr)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("cql_logit_grad_kernel: {e}")))?;
        }

        // F5/D2: Q-gap barrier gradient injection into the same d-logit buffers.
        // Runs AFTER the CQL kernel fills cql_d_* and BEFORE backward_full reads them,
        // so barrier contributes through the same backward pass with no extra cuBLAS call.
        // Inlined here (not a separate method call) to avoid double-borrow with _eg guard.
        if barrier_weight > 1e-6 {
            if let Some(ref barrier_kernel) = self.barrier_gradient_kernel.clone() {
                let adv_ptr2     = self.on_b_logits_buf.raw_ptr();
                let v_ptr2       = self.on_v_logits_buf.raw_ptr();
                let z_ptr2       = self.atom_positions_buf.raw_ptr();
                let d_adv_ptr2   = self.cql_d_adv_logits.raw_ptr();
                let d_v_ptr2     = self.cql_d_value_logits.raw_ptr();
                let isv_dev2     = self.isv_signals_dev_ptr;
                let b_i32        = b as i32;
                let na_i32_b     = na as i32;
                let b0_i32_b     = b0 as i32;
                let ta_i32_b     = total_actions as i32;
                let blocks_b     = blocks;
                unsafe {
                    let _ = self.stream
                        .launch_builder(barrier_kernel)
                        .arg(&adv_ptr2)
                        .arg(&v_ptr2)
                        .arg(&z_ptr2)
                        .arg(&isv_dev2)
                        .arg(&d_adv_ptr2)
                        .arg(&d_v_ptr2)
                        .arg(&b_i32)
                        .arg(&na_i32_b)
                        .arg(&b0_i32_b)
                        .arg(&ta_i32_b)
                        .arg(&barrier_weight)
                        .launch(LaunchConfig {
                            grid_dim: (blocks_b, 1, 1),
                            block_dim: (256, 1, 1),
                            shared_mem_bytes: 0,
                        });
                    // Non-fatal: log failure but continue with CQL backward.
                }
            }
        }

        // F4/D5: IB variance gradient — spreads Q values in the direction branch.
        // Runs after F5 barrier (both atomicAdd into same buffers, races are fine).
        // Zero-op when var_q >= 0.01 or health ≈ 1 (ib_weight ≈ 0).
        if let Some(ref ib_kernel) = self.ib_gradient_kernel.clone() {
            let adv_ptr_ib   = self.on_b_logits_buf.raw_ptr();
            let v_ptr_ib     = self.on_v_logits_buf.raw_ptr();
            let z_ptr_ib     = self.atom_positions_buf.raw_ptr();
            let isv_dev_ib   = self.isv_signals_dev_ptr;
            let d_adv_ptr_ib = self.cql_d_adv_logits.raw_ptr();
            let d_v_ptr_ib   = self.cql_d_value_logits.raw_ptr();
            let b_i32_ib     = b as i32;
            let na_i32_ib    = na as i32;
            let b0_i32_ib    = b0 as i32;
            let ta_i32_ib    = total_actions as i32;
            let blocks_ib    = blocks;
            let min_var      = 0.01_f32;
            unsafe {
                let _ = self.stream
                    .launch_builder(ib_kernel)
                    .arg(&adv_ptr_ib)
                    .arg(&v_ptr_ib)
                    .arg(&z_ptr_ib)
                    .arg(&isv_dev_ib)
                    .arg(&d_adv_ptr_ib)
                    .arg(&d_v_ptr_ib)
                    .arg(&b_i32_ib)
                    .arg(&na_i32_ib)
                    .arg(&b0_i32_ib)
                    .arg(&ta_i32_ib)
                    .arg(&min_var)
                    .launch(LaunchConfig {
                        grid_dim: (blocks_ib, 1, 1),
                        block_dim: (256, 1, 1),
                        shared_mem_bytes: 0,
                    });
                // Non-fatal: log failure but continue with CQL backward.
            }
        }

        // Step 2: SAXPY — add CQL logit gradients to d_value_logits_buf and d_adv_logits_buf
        // These are the same buffers that already hold C51's logit gradients.
        // After this, when we run cuBLAS backward, the backward pass sees
        // combined C51+CQL logit gradients and produces combined parameter gradients.
        // However, backward_full uses beta=1.0 accumulation, so calling it again
        // would ADD to grad_buf (doubling the C51 contribution).
        //
        // Instead, we SAXPY the CQL logit gradients into the d_logits buffers and
        // then rely on the fact that the NEXT graph_forward will use these updated
        // logit gradients. But that's for the next step, not this one.
        //
        // For the current step, the correct approach is to add CQL gradients
        // directly to grad_buf using a separate cuBLAS backward pass with
        // cql_d_value_logits and cql_d_adv_logits as input.
        // This is the same pattern as apply_iqn_trunk_gradient.

        // Extract f32 weight pointers for cuBLAS backward
        let param_sizes = compute_param_sizes(&self.config);
        let w_ptrs = f32_weight_ptrs_from_base(self.ptrs.params_ptr, &param_sizes);

        // CQL d_logits: pass f32 directly (same as main backward).
        let f32_size = std::mem::size_of::<f32>();
        let d_adv_ptrs = [
            d_adv_ptr,
            d_adv_ptr + (b * b0 * na * f32_size) as u64,
            d_adv_ptr + (b * (b0 + b1) * na * f32_size) as u64,
            d_adv_ptr + (b * (b0 + b1 + b2) * na * f32_size) as u64,
        ];

        // Saved activations from the forward pass (still valid)
        let states_ptr_fw = self.states_buf.raw_ptr();
        let h_s1_ptr = self.save_h_s1.raw_ptr();
        let h_s2_ptr = self.save_h_s2.raw_ptr();
        let h_v_ptr = self.save_h_v.raw_ptr();
        let h_b0_ptr = self.save_h_b0.raw_ptr();
        let h_b1_ptr = self.save_h_b1.raw_ptr();
        let h_b2_ptr = self.save_h_b2.raw_ptr();
        let h_b3_ptr = self.save_h_b3.raw_ptr();

        // Scratch buffers for inter-layer gradients (can reuse bw_d_h_* buffers)
        let scratch_d_h_s2 = self.bw_d_h_s2.raw_ptr();
        let scratch_d_h_s1 = self.bw_d_h_s1.raw_ptr();
        let scratch_d_h_v = self.bw_d_h_v.raw_ptr();
        let scratch_d_h_b0 = self.bw_d_h_b0.raw_ptr();
        let scratch_d_h_b1 = self.bw_d_h_b1.raw_ptr();
        let scratch_d_h_b2 = self.bw_d_h_b2.raw_ptr();
        let scratch_d_h_b3 = self.bw_d_h_b3.raw_ptr();

        // Zero CQL scratch buffer (backward_full uses beta=1.0 accumulation)
        // Raw cuMemsetD32Async — cudarc memset_zeros is NOT captured by CUDA Graph.
        unsafe {
            cudarc::driver::sys::cuMemsetD32Async(
                self.cql_grad_scratch.raw_ptr(), 0,
                self.cql_grad_scratch.len(), self.stream.cu_stream(),
            );
        }

        // GLU saved activation pointers
        let glu_gate_pre_ptrs = [
            self.glu_gate_pre_buf[0].raw_ptr(),
            self.glu_gate_pre_buf[1].raw_ptr(),
            self.glu_gate_pre_buf[2].raw_ptr(),
            self.glu_gate_pre_buf[3].raw_ptr(),
        ];
        let glu_value_ptrs = [
            self.glu_value_buf[0].raw_ptr(),
            self.glu_value_buf[1].raw_ptr(),
            self.glu_value_buf[2].raw_ptr(),
            self.glu_value_buf[3].raw_ptr(),
        ];

        // Run full backward pass with CQL logit gradients into ISOLATED scratch buffer.
        // Produces CQL parameter gradients WITHOUT mixing with C51's grad_buf.
        //
        // Phase H Site 2: pass the value-FC dReLU bit-mask written by the online
        // forward (same `save_h_v` source as the C51 backward, so the mask is
        // valid for the CQL backward as well — CQL reuses save_h_s2/save_h_v).
        // Pass 0 when the forward fused-AUX path is unavailable.
        let value_fc_relu_mask_cql = if self.cublas_forward.value_fc_relu_mask_available() {
            self.cublas_forward.value_fc_relu_mask_ptr()
        } else {
            0u64
        };
        // SP14 B.10: rebuild the ONLINE direction-Q-head concat scratch
        // [save_h_s2 | aux_softmax_diff] into `sp14_dir_qaux_concat_scratch`.
        // The forward pass overwrote it with the TARGET concat (line ~25817 in
        // `submit_forward_ops`), so the backward dW SGEMM would otherwise read
        // tg_h_s2 instead of save_h_s2 — wrong gradient. `aux_nb_softmax_buf`
        // is unchanged between forward and backward (CE loss writes a separate
        // `d_aux_softmax` scratch), so re-running the same concat with
        // `save_h_s2` yields the bit-identical online concat the forward
        // SGEMM consumed. Same one-step-lag semantic preserved.
        self.launch_sp14_dir_concat_qaux(self.ptrs.save_h_s2)?;
        // SP14 B.10: pre-zero `scratch_d_h_s2` because the direction branch
        // (d==0) no longer writes it directly with beta=0 — its dX now lands
        // in `d_dir_qaux_concat` and is gated + accumulated AFTER backward_full
        // returns. The value-FC dX accumulator inside backward_full uses
        // beta=1 and assumed scratch_d_h_s2 was pre-written by d==0; with the
        // direction branch redirected, we must establish the zero baseline
        // explicitly. Raw cuMemsetD32Async — captured by CUDA Graph (cudarc
        // memset_zeros is NOT graph-safe; matches the cql_grad_scratch
        // pre-backward-full zero pattern below).
        unsafe {
            cudarc::driver::sys::cuMemsetD32Async(
                scratch_d_h_s2, 0,
                self.config.batch_size * self.config.shared_h2,
                self.stream.cu_stream(),
            );
        }
        self.cublas_backward.backward_full(
            &self.stream,
            d_v_ptr,
            &d_adv_ptrs,
            states_ptr_fw,
            h_s1_ptr, h_s2_ptr, h_v_ptr,
            &[h_b0_ptr, h_b1_ptr, h_b2_ptr, h_b3_ptr],
            &w_ptrs,
            self.cql_grad_scratch.raw_ptr(),
            scratch_d_h_s2, scratch_d_h_s1, scratch_d_h_v,
            &[scratch_d_h_b0, scratch_d_h_b1, scratch_d_h_b2, scratch_d_h_b3],
            0, // CQL backward: no bottleneck dX needed (separate gradient budget)
            self.ptrs.mag_concat_buf,
            self.ptrs.d_mag_concat_buf,
            self.ptrs.ord_concat_buf,
            self.d_ord_concat_buf.raw_ptr(),
            self.ptrs.urg_concat_buf,
            self.d_urg_concat_buf.raw_ptr(),
            &glu_gate_pre_ptrs,
            &glu_value_ptrs,
            self.ptrs.bw_d_glu_value,
            self.ptrs.bw_d_glu_gate,
            value_fc_relu_mask_cql,
            // SP14 B.10: direction-Q-head EGF wire — saved forward concat
            // for dW + dX output buffer for col SH2 wire scaling.
            self.sp14_dir_qaux_concat_scratch.raw_ptr(),
            self.sp14_d_dir_qaux_concat.raw_ptr(),
        ).map_err(|e| MLError::ModelError(format!("CQL backward_full: {e}")))?;

        // SP14 Layer C Phase C.1 (2026-05-08): `launch_sp14_scale_wire_col`
        // call deleted atomically with the kernel file. Coupling B (α-gated
        // backward wire) is gone; the wire column gradient is dropped by the
        // first-SH2-cols accumulator below (column SH2 is not propagated
        // downstream, matching the pre-B.11 behavior when ISV[393]
        // sentinelled to 0.0). Phase C.5 wires the proper aux-trunk
        // backward path with stop-gradient at the encoder boundary.
        // Accumulate first SH2 columns of d_dir_qaux_concat into scratch_d_h_s2
        // with beta=1 (the value-FC dX inside backward_full already accumulated
        // its contribution; we ADD on top, NOT overwrite). Wire column (col
        // SH2) is NOT propagated downstream from here.
        self.accumulate_d_h_s2_from_concat(
            self.sp14_d_dir_qaux_concat.raw_ptr(),
            scratch_d_h_s2,
            self.config.batch_size,
            self.config.shared_h2 + 1,
            1.0,
        )?;

        // CQL: accumulate magnitude branch dX into scratch_d_h_s2
        if self.ptrs.mag_concat_buf != 0 {
            self.accumulate_d_h_s2_from_concat(
                self.ptrs.d_mag_concat_buf,
                scratch_d_h_s2,
                self.config.batch_size,
                self.config.shared_h2 + self.config.branch_0_size, // mag: direction-conditioned
                1.0, // beta=1: SP14 B.10 zeroed scratch_d_h_s2 + value-FC + dir-Q already accumulated
            )?;
        }
        if self.ptrs.ord_concat_buf != 0 {
            self.accumulate_d_h_s2_from_concat(
                self.d_ord_concat_buf.raw_ptr(),
                scratch_d_h_s2,
                self.config.batch_size,
                self.config.shared_h2 + 3, // ord: OFI-conditioned (3 OFI features)
                1.0,
            )?;
        }
        if self.ptrs.urg_concat_buf != 0 {
            self.accumulate_d_h_s2_from_concat(
                self.d_urg_concat_buf.raw_ptr(),
                scratch_d_h_s2,
                self.config.batch_size,
                self.config.shared_h2 + 3, // urg: OFI-conditioned (3 OFI features)
                1.0,
            )?;
        }

        // ── GRN trunk backward (Plan 4 Task 2c.3c.4, CQL path) ──
        // Mirrors the main backward path but writes trunk gradients into
        // `cql_grad_scratch` instead of `grad_buf`.
        // Plan 4 Task 1B-iv-ext: pass non-zero `s1_dx_output` so the chain
        // produces dL/d(bn_concat) for the VSN backward extension below.
        // The CQL aux path runs after the IQN aux path's VSN backward has
        // finished (per `fused_training.rs` ordering), so reusing
        // `bn_d_concat_buf` is safe.
        let grn_h_s1_d_pre_ln_cql      = self.bw_grn_h_s1_d_pre_ln.raw_ptr();
        let grn_h_s1_d_linear_b_out_cql= self.bw_grn_h_s1_d_linear_b_out.raw_ptr();
        let grn_h_s1_d_elu_out_cql     = self.bw_grn_h_s1_d_elu_out.raw_ptr();
        let grn_h_s1_d_linear_a_out_cql= self.bw_grn_h_s1_d_linear_a_out.raw_ptr();
        let grn_h_s2_d_pre_ln_cql      = self.bw_grn_h_s2_d_pre_ln.raw_ptr();
        let grn_h_s2_d_linear_b_out_cql= self.bw_grn_h_s2_d_linear_b_out.raw_ptr();
        let grn_h_s2_d_elu_out_cql     = self.bw_grn_h_s2_d_elu_out.raw_ptr();
        let grn_h_s2_d_linear_a_out_cql= self.bw_grn_h_s2_d_linear_a_out.raw_ptr();
        let cql_grad_scratch_base = self.cql_grad_scratch.raw_ptr();
        let cql_param_sizes = compute_param_sizes(&self.config);
        let cql_w_ptrs = f32_weight_ptrs_from_base(self.ptrs.params_ptr, &cql_param_sizes);
        let bn_d_concat_for_cql = if self.config.bottleneck_dim > 0 {
            self.bn_d_concat_buf.raw_ptr()
        } else {
            0u64
        };
        self.cublas_forward.encoder_backward_chain(
            &self.stream,
            &self.cublas_backward,
            states_ptr_fw,
            &cql_w_ptrs,
            cql_grad_scratch_base,
            &cql_param_sizes,
            scratch_d_h_s2,
            scratch_d_h_s1,
            bn_d_concat_for_cql, // 1B-iv-ext: bottleneck dX needed for VSN aux backward
            grn_h_s1_d_pre_ln_cql,
            grn_h_s1_d_linear_b_out_cql,
            grn_h_s1_d_elu_out_cql,
            grn_h_s1_d_linear_a_out_cql,
            grn_h_s2_d_pre_ln_cql,
            grn_h_s2_d_linear_b_out_cql,
            grn_h_s2_d_elu_out_cql,
            grn_h_s2_d_linear_a_out_cql,
            h_s1_ptr,
        )?;

        // ── Plan 4 Task 1B-iv-ext: VSN backward chain (CQL path) ─────────
        // CQL writes VSN dW directly into `cql_grad_scratch` at goff[95..119)
        // because the scratch is sized `total_params` and the trailing
        // `apply_cql_saxpy` already mixes the entire scratch into `grad_buf`
        // with the cql_budget scale. No per-aux VSN scratch needed — the
        // scratch slots [95..119) ride the existing CQL SAXPY.
        if self.config.bottleneck_dim > 0 {
            Self::aux_bottleneck_vsn_backward_dispatch(
                &mut self.cublas_forward,
                &self.cublas_backward,
                &self.stream,
                &self.config,
                &self.bn_d_hidden_buf,
                &self.bn_hidden_buf,
                &self.states_buf,
                &self.vsn_d_gated_state_buf,
                &self.vsn_mask_buf,
                &self.vsn_d_logits_buf,
                &self.vsn_d_state_buf,
                &self.vsn_d_logit_scratch_buf,
                &self.vsn_d_h1_scratch_buf,
                &self.vsn_group_begins_buf,
                &self.vsn_group_ends_buf,
                &self.vsn_save_h1_g0_buf,
                &self.vsn_save_h1_g1_buf,
                &self.vsn_save_h1_g2_buf,
                &self.vsn_save_h1_g3_buf,
                &self.vsn_save_h1_g4_buf,
                &self.vsn_save_h1_g5_buf,
                &self.bn_tanh_backward_kernel,
                &self.vsn_d_gated_state_portfolio_kernel,
                &self.vsn_logit_gather_kernel,
                &self.scale_f32_aux,                  // 1B-iv-rc: VSN dW dilution (VSN_DW_DAMP)
                bn_d_concat_for_cql,
                cql_grad_scratch_base,
                &cql_param_sizes,
                &cql_w_ptrs,
            )?;
        }

        Ok(true)
    }

    /// Whether CQL is enabled and the kernel was compiled.
    pub fn has_cql(&self) -> bool {
        if !self.cql_logit_grad_kernel.is_some() {
            return false;
        }
        // Read base from ISV[CQL_ALPHA_INDEX] when available; fall back to config.
        let base = if self.isv_signals_pinned.is_null() {
            self.config.cql_alpha
        } else {
            unsafe { *self.isv_signals_pinned.add(CQL_ALPHA_INDEX) }
        };
        base > 0.0
    }

    /// F8/G5 + SP7 (2026-05-03): Scale the C51-contributed portion of `grad_buf`
    /// by the adaptive c51_budget read from `lb_budget_effective_buf` at runtime.
    ///
    /// Must be called AFTER `submit_forward_ops_main` (which runs C51 backward and writes
    /// C51 gradients into `grad_buf`), and BEFORE any CQL/IQN SAXPY accumulations.
    ///
    /// The composite effect is:
    ///   grad = c51_budget × C51_grad + cql_budget × CQL_grad + iqn_budget × IQN_grad
    ///
    /// SP7 (2026-05-03): `c51_budget` is read on-device from `alpha_dev_ptr` (a u64
    /// device pointer to a single f32 — typically `&lb_budget_effective_buf[5]`,
    /// the c51_trunk_mean slot). Eliminates the capture-time freeze of the prior
    /// scalar-arg variant; see `consume_lb_budget_kernel.cu` and audit Fix 33.
    pub fn apply_c51_budget_scale(&mut self, alpha_dev_ptr: u64) -> Result<(), MLError> {
        // SP7 (2026-05-03): no host-side `if budget ≈ 1.0 { return; }` skip — the
        // value lives on-device and we can't read it without DtoH (forbidden mid-
        // capture). The kernel itself short-circuits the alpha=0.0 NaN-safety
        // clause; an alpha=1.0 multiply is a numerically-cheap no-op.
        let grad_ptr = self.ptrs.grad_buf;
        let n = self.total_params as i32;
        let blocks = ((self.total_params as u32 + 255) / 256) as u32;
        unsafe {
            self.stream
                .launch_builder(&self.scale_f32_dev_ptr_aux)
                .arg(&grad_ptr)
                .arg(&alpha_dev_ptr)
                .arg(&n)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("c51_budget_scale: {e}")))?;
        }
        Ok(())
    }

    /// SP6 Pearl 2 + SP7 (2026-05-03): scale only branch `branch_idx` parameter
    /// slice of grad_buf by the correction factor read from
    /// `lb_budget_effective_buf` at runtime.
    ///
    /// Called after `apply_c51_budget_scale(trunk_dev_ptr)` has scaled the entire
    /// grad_buf by c51_trunk_mean. The correction factor (loaded from
    /// `correction_dev_ptr`) is `branch_budget / trunk_mean`, so the branch slice
    /// ends up scaled by `branch_budget` rather than `trunk_mean`.
    ///
    /// Branch b owns tensors `(8+4b)..(12+4b)` in the padded flat layout.
    /// Padding bytes in `grad_buf` are always zero so scaling them is a no-op.
    pub fn apply_c51_budget_scale_branch(&mut self, branch_idx: usize, correction_dev_ptr: u64) -> Result<(), MLError> {
        // SP7 (2026-05-03): no host-side `if correction ≈ 1.0 { return; }` skip
        // for the same reason as the trunk variant — value lives on-device.
        let f32_size = std::mem::size_of::<f32>();
        let param_sizes = compute_param_sizes(&self.config);
        let first_tensor = 8 + branch_idx * 4;
        let start_bytes = padded_byte_offset(&param_sizes, first_tensor) as usize;
        let end_bytes   = padded_byte_offset(&param_sizes, first_tensor + 4) as usize;
        let start_elems = (start_bytes / f32_size) as i32;
        let len_elems   = ((end_bytes - start_bytes) / f32_size) as i32;
        if len_elems == 0 { return Ok(()); }
        let grad_slice_ptr = self.ptrs.grad_buf + (start_elems as u64) * (f32_size as u64);
        let blocks = ((len_elems as u32 + 255) / 256) as u32;
        unsafe {
            self.stream
                .launch_builder(&self.scale_f32_dev_ptr_aux)
                .arg(&grad_slice_ptr)
                .arg(&correction_dev_ptr)
                .arg(&len_elems)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!(
                    "c51_budget_scale_branch[{branch_idx}]: {e}"
                )))?;
        }
        Ok(())
    }

    /// Add CQL gradient scratch to grad_buf via plain SAXPY.
    ///
    /// Called after `apply_cql_gradient` populated `cql_grad_scratch`.
    /// No per-component clip — single global clip in Adam handles safety.
    ///
    /// SP7 (2026-05-03): `cql_budget` is read on-device from `alpha_dev_ptr` —
    /// typically `&lb_budget_effective_buf[0]`, the cql_trunk_mean slot.
    /// Eliminates the capture-time freeze of the prior scalar-arg variant.
    pub fn apply_cql_saxpy(&mut self, alpha_dev_ptr: u64) -> Result<(), MLError> {
        let total = self.total_params as i32;
        let blocks = self.grad_norm_blocks as u32;
        let scratch_ptr = self.ptrs.cql_grad_scratch;
        let grad_ptr = self.ptrs.grad_buf;

        // Plain SAXPY: grad_buf += (*alpha_dev_ptr) * cql_scratch
        // Uses aux_child-specific dev-ptr handle — saxpy_f32_kernel is captured
        // in forward_child and the scalar saxpy_f32_aux is reused by other
        // callers (distill, etc.). The dev-ptr variant is unique to SP7.
        unsafe {
            self.stream
                .launch_builder(&self.saxpy_f32_dev_ptr_aux)
                .arg(&grad_ptr)
                .arg(&scratch_ptr)
                .arg(&alpha_dev_ptr)
                .arg(&total)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("CQL SAXPY: {e}")))?;
        }

        Ok(())
    }

    /// SP6 Pearl 2 + SP7 (2026-05-03): SAXPY only branch `branch_idx` parameter
    /// slice:
    ///   grad_buf[branch_slice] += (*correction_dev_ptr) * cql_grad_scratch[branch_slice]
    ///
    /// Called after `apply_cql_saxpy(trunk_dev_ptr)` has performed the full-buffer
    /// SAXPY. The correction factor (loaded from `correction_dev_ptr`) is
    /// `branch_budget / trunk_mean`, so the branch slice ends up accumulating
    /// `branch_budget * cql_scratch` rather than `trunk_mean * cql_scratch`.
    ///
    /// Branch b owns tensors `(8+4b)..(12+4b)` in the padded flat layout.
    pub fn apply_cql_saxpy_branch(&mut self, branch_idx: usize, correction_dev_ptr: u64) -> Result<(), MLError> {
        let f32_size = std::mem::size_of::<f32>();
        let param_sizes = compute_param_sizes(&self.config);
        let first_tensor = 8 + branch_idx * 4;
        let start_bytes = padded_byte_offset(&param_sizes, first_tensor) as usize;
        let end_bytes   = padded_byte_offset(&param_sizes, first_tensor + 4) as usize;
        let start_elems = (start_bytes / f32_size) as i32;
        let len_elems   = ((end_bytes - start_bytes) / f32_size) as i32;
        if len_elems == 0 { return Ok(()); }
        let grad_ptr    = self.ptrs.grad_buf + (start_elems as u64) * (f32_size as u64);
        let scratch_ptr = self.ptrs.cql_grad_scratch + (start_elems as u64) * (f32_size as u64);
        let blocks = ((len_elems as u32 + 255) / 256) as u32;
        unsafe {
            self.stream
                .launch_builder(&self.saxpy_f32_dev_ptr_aux)
                .arg(&grad_ptr)
                .arg(&scratch_ptr)
                .arg(&correction_dev_ptr)
                .arg(&len_elems)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!(
                    "cql_saxpy_branch[{branch_idx}]: {e}"
                )))?;
        }
        Ok(())
    }

    /// F5/D2: Inject the Q-gap barrier gradient into cql_d_adv_logits and cql_d_value_logits
    /// (direction branch b0 only) via atomicAdd.
    ///
    /// Must be called AFTER `apply_cql_gradient` runs its CQL kernel (which fills those
    /// buffers) but BEFORE `backward_full` in the CQL path — therefore it must be called
    /// from inside `apply_cql_gradient` with the barrier_weight passed through.
    ///
    /// This is a separate entry point for calling directly when CQL is disabled,
    /// piggybacking on the same buffers for a standalone barrier-only backward pass.
    ///
    /// Returns Ok(false) if the kernel is not compiled. Returns Ok(true) on success.
    pub fn inject_barrier_into_cql_d_logits(&mut self, barrier_weight: f32) -> Result<bool, MLError> {
        if barrier_weight < 1e-6 { return Ok(false); }
        let kernel = match &self.barrier_gradient_kernel {
            Some(k) => k.clone(),
            None => return Ok(false),
        };
        let b = self.config.batch_size;
        let na = self.config.num_atoms;
        let b0 = self.config.branch_0_size;
        let b1 = self.config.branch_1_size;
        let b2 = self.config.branch_2_size;
        let b3 = self.config.branch_3_size;
        let total_actions = b0 + b1 + b2 + b3;
        let n_i32 = b as i32;
        let na_i32 = na as i32;
        let b0_i32 = b0 as i32;
        let ta_i32 = total_actions as i32;
        let blocks = ((b + 255) / 256) as u32;

        // adv_logits and cql_d_adv_logits are both [B, total_actions, num_atoms];
        // we only update the direction slice (first b0 action slots).
        let adv_ptr = self.on_b_logits_buf.raw_ptr();
        let v_ptr   = self.on_v_logits_buf.raw_ptr();
        // z_vals: use atom_positions_buf[0..num_atoms] (branch 0 adaptive positions).
        let z_ptr   = self.atom_positions_buf.raw_ptr();
        let d_adv_ptr = self.cql_d_adv_logits.raw_ptr();
        let d_v_ptr   = self.cql_d_value_logits.raw_ptr();

        unsafe {
            self.stream
                .launch_builder(&kernel)
                .arg(&adv_ptr)
                .arg(&v_ptr)
                .arg(&z_ptr)
                .arg(&self.isv_signals_dev_ptr)
                .arg(&d_adv_ptr)
                .arg(&d_v_ptr)
                .arg(&n_i32)
                .arg(&na_i32)
                .arg(&b0_i32)
                .arg(&ta_i32)
                .arg(&barrier_weight)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("barrier_gradient_direction: {e}")))?;
        }
        Ok(true)
    }

    /// v8: Normalize the internal rewards_buf in-place using PopArt.
    ///
    /// Convenience wrapper that avoids double-borrow by using `self.rewards_buf`
    /// directly. Called from `FusedTrainingCtx::run_full_step()`.
    pub fn normalize_rewards_popart_inplace(
        &mut self, n: usize,
    ) -> Result<(), MLError> {
        let _evt_guard = EventTrackingGuard::new(self.stream.context());
        let mean_ptr = self.popart_mean.raw_ptr();
        let var_ptr = self.popart_var.raw_ptr();
        let count_ptr = self.popart_count.raw_ptr();
        let rewards_ptr = self.rewards_buf.raw_ptr();
        let warmup = 10_i32;  // collect 10 batches of stats before normalizing (was 100, too long → C51 collapse)
        let n_i32 = n as i32;
        unsafe {
            self.stream.launch_builder(&self.popart_normalize_kernel)
                .arg(&rewards_ptr)
                .arg(&mean_ptr)
                .arg(&var_ptr)
                .arg(&count_ptr)
                .arg(&n_i32)
                .arg(&warmup)
                .launch(LaunchConfig { grid_dim: (1, 1, 1), block_dim: (256, 1, 1), shared_mem_bytes: 0 })
                .map_err(|e| MLError::ModelError(format!("popart_normalize_inplace: {e}")))?;
        }
        Ok(())
    }

    /// v8: Normalize rewards in-place using running mean/variance (PopArt).
    ///
    /// The kernel updates running statistics with Welford's algorithm and
    /// normalizes the reward buffer to zero-mean unit-variance. A warmup
    /// period (first 100 calls) accumulates statistics without normalizing.
    pub fn normalize_rewards_popart(
        &mut self, rewards: &mut CudaSlice<f32>, n: usize,
    ) -> Result<(), MLError> {
        let _evt_guard = EventTrackingGuard::new(self.stream.context());
        let mean_ptr = self.popart_mean.raw_ptr();
        let var_ptr = self.popart_var.raw_ptr();
        let count_ptr = self.popart_count.raw_ptr();
        let rewards_ptr = rewards.raw_ptr();
        let warmup = 10_i32;  // collect 10 batches of stats before normalizing (was 100, too long → C51 collapse)
        let n_i32 = n as i32;
        unsafe {
            self.stream.launch_builder(&self.popart_normalize_kernel)
                .arg(&rewards_ptr)
                .arg(&mean_ptr)
                .arg(&var_ptr)
                .arg(&count_ptr)
                .arg(&n_i32)
                .arg(&warmup)
                .launch(LaunchConfig { grid_dim: (1, 1, 1), block_dim: (256, 1, 1), shared_mem_bytes: 0 })
                .map_err(|e| MLError::ModelError(format!("popart_normalize: {e}")))?;
        }
        Ok(())
    }

    /// Normalize the internal rewards_buf in-place using pre-computed median and IQR.
    ///
    /// More robust than Welford mean/var for heavy-tailed bimodal distributions.
    /// Median/IQR are computed once per epoch from sorted experience rewards.
    /// Called instead of `normalize_rewards_popart_inplace` when `popart_robust = true`.
    pub fn normalize_rewards_robust(
        &self, n: usize, median: f32, iqr: f32,
    ) -> Result<(), MLError> {
        let rewards_ptr = self.rewards_buf.raw_ptr();
        let n_i32 = n as i32;
        unsafe {
            self.stream.launch_builder(&self.popart_robust_kernel)
                .arg(&rewards_ptr)
                .arg(&median)
                .arg(&iqr)
                .arg(&n_i32)
                .launch(LaunchConfig::for_num_elems(n as u32))
                .map_err(|e| MLError::ModelError(format!("popart_normalize_robust: {e}")))?;
        }
        Ok(())
    }

    // SP4 Layer A Task A13.5 (2026-05-01): the orphan
    // `launch_reward_component_ema` previously here was deleted per
    // `feedback_wire_everything_up.md`. The active launcher lives on
    // `GpuExperienceCollector::launch_reward_component_ema_inplace`,
    // which now takes the trainer's mapped-pinned Pearls A+D buffers
    // via `set_reward_component_pearls_buffers` (wired from
    // `FusedTrainingCtx::wire_reward_component_pearls_buffers`).
    // The trainer-side `reward_component_ema_kernel` field is also
    // deleted because no trainer launcher consumes it; the experience
    // collector loads its own copy from the same cubin.

    /// Plan 4 Task 2c.3c.5; SP4 Layer A Task A13.0 retrofit (2026-05-01):
    /// launch `h_s2_rms_ema_update` (single-block, 256-thread shmem-reduction
    /// kernel). Reduces RMS = sqrt(sum_sq / (B*SH2)) over the trainer's
    /// `save_h_s2 [B, SH2]` buffer and writes the step observation to
    /// `producer_step_scratch_buf[SCRATCH_IDX=40]`. Then launches the GPU
    /// `apply_pearls_ad_kernel` on the same stream — applies Pearls A+D
    /// to ISV[H_S2_RMS_EMA_INDEX] using
    /// `wiener_state_buf[40*3..40*3+3]` (slot 40 in the SP4 producer
    /// scratch / Wiener-state layout, first retrofit slot after the 40 SP4
    /// producers).
    ///
    /// α is derived adaptively from per-slot Pearls A+D Wiener state — see
    /// `sp4_wiener_ema::pearls_ad_update` (test-only reference for the GPU
    /// `apply_pearls_ad_kernel`).
    ///
    /// Producer-only — 2c.3c.6 wires the consumer in `mag_concat_qdir`'s
    /// adaptive-scale path. Cold-path-cadence: launched once per training step
    /// alongside `launch_reward_component_ema`. No atomicAdd, no host sync.
    /// Graph-capture-compatible by construction (2026-05-01 GPU-Pearls refactor).
    pub fn launch_h_s2_rms_ema(&self) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp4_wiener_ema::launch_apply_pearls;

        // `isv_signals_dev_ptr` / `isv_signals_pinned` are set by the constructor
        // via `cuMemHostGetDevicePointer_v2` on a `cuMemAllocHost_v2`-backed
        // pinned buffer and are never reassigned. Silent-skip on `== 0` would
        // mask a misconfiguration; `debug_assert!` flags it in dev builds.
        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_h_s2_rms_ema: isv_signals_dev_ptr must be allocated by constructor");

        // Task A13.0: scratch slot 40 — first retrofit slot.
        const SCRATCH_IDX: usize = 40;
        const BLOCK_DIM: u32 = 256;
        let smem_bytes = BLOCK_DIM * std::mem::size_of::<f32>() as u32;

        let h_s2_ptr = self.save_h_s2.raw_ptr();
        let b_i = self.config.batch_size as i32;
        let sh2_i = self.config.shared_h2 as i32;
        let scratch_dev = self.producer_step_scratch_buf.dev_ptr;
        let scratch_idx_arg: i32 = SCRATCH_IDX as i32;

        unsafe {
            self.stream.launch_builder(&self.h_s2_rms_ema_kernel)
                .arg(&h_s2_ptr)
                .arg(&b_i)
                .arg(&sh2_i)
                .arg(&scratch_dev)
                .arg(&scratch_idx_arg)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (BLOCK_DIM, 1, 1),
                    shared_mem_bytes: smem_bytes,
                })
                .map_err(|e| MLError::ModelError(format!("h_s2_rms_ema_update: {e}")))?;
        }

        // GPU Pearls A+D: same-stream ordering — kernel reads scratch[40]
        // after the producer's `__threadfence_system()`. Retrofit wiener
        // convention: offset = scratch_idx * 3 (slot 40 → 120).
        unsafe {
            launch_apply_pearls(
                &self.stream,
                &self.apply_pearls_ad_kernel,
                scratch_dev,
                SCRATCH_IDX as i32,
                self.isv_signals_dev_ptr,
                H_S2_RMS_EMA_INDEX as i32,
                self.wiener_state_buf.dev_ptr,
                (SCRATCH_IDX * 3) as i32,
                1,
                crate::cuda_pipeline::sp4_wiener_ema::ALPHA_META,
            )?;
        }
        Ok(())
    }

    /// SP4 helper: dispatch a single-buffer p99 producer + chained Pearls A+D.
    ///
    /// Used by the SP4 single-buffer histogram-p99 producers (target_q,
    /// h_s2, bw_d_h_s2). The kernel is expected to take exactly 4 args in
    /// this order: `(const float* buf, int count, float* scratch, int
    /// scratch_idx)`. Launch config is 1 block × 256 threads with the
    /// histogram contract's `(256/32) * 256 * 4 = 8192` bytes of shared
    /// memory. After the kernel writes its step p99 to
    /// `producer_step_scratch_buf[scratch_idx]`, this method chains the
    /// GPU `apply_pearls_ad_kernel` on the same stream — applies Pearls
    /// A+D to ISV[isv_idx] using `wiener_state_buf[(isv_idx -
    /// SP4_SLOT_BASE) * 3 .. +3]`. All Pearls A+D state is on-device; no
    /// host sync (graph-capture-compatible by construction).
    ///
    /// Multi-sub-buffer launchers (`launch_sp4_q_dir_grad_p99`,
    /// `launch_sp4_param_group_oracles_all_groups`) and shape-distinct
    /// producers (`launch_sp4_grad_norm_p99` with 1-thread blocks,
    /// `launch_sp4_atom_pos_p99_all_branches` with 4-iteration loop +
    /// batched n_slots=4 Pearls) keep their bespoke launch logic.
    pub(crate) fn launch_sp4_p99_producer_single_buf(
        &self,
        kernel: &CudaFunction,
        buf_ptr: u64,
        count: i32,
        scratch_idx: usize,
        isv_idx: usize,
        name_for_err: &'static str,
    ) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp4_isv_slots::SP4_SLOT_BASE;
        use crate::cuda_pipeline::sp4_wiener_ema::launch_apply_pearls;

        // Shared memory: 8 warps × 256 bins × sizeof(int) = 8192 bytes.
        const SHARED_BYTES: u32 = (256 / 32) * 256 * 4;

        let cfg = LaunchConfig {
            grid_dim:         (1, 1, 1),
            block_dim:        (256, 1, 1),
            shared_mem_bytes: SHARED_BYTES,
        };
        let scratch_dev = self.producer_step_scratch_buf.dev_ptr;
        let scratch_idx_arg: i32 = scratch_idx as i32;

        // Safety: kernel signature `(const float*, int, float*, int)`
        // matches the four args below; both device pointers are valid for
        // the lifetime of `self`.
        unsafe {
            self.stream
                .launch_builder(kernel)
                .arg(&buf_ptr)
                .arg(&count)
                .arg(&scratch_dev)
                .arg(&scratch_idx_arg)
                .launch(cfg)
                .map_err(|e| MLError::ModelError(format!("{name_for_err}: {e}")))?;
        }

        // GPU Pearls A+D: stream-ordered with the producer kernel, no host
        // sync (graph-capture-compatible). Wiener offset for SP4 producers:
        // `(isv_idx - SP4_SLOT_BASE) * 3`. Kernel-side `step_obs == 0`
        // short-circuit handles cold-start (before the producer's source
        // buffer has been populated).
        let isv_idx_arg: i32 = isv_idx as i32;
        let wiener_offset: i32 = ((isv_idx - SP4_SLOT_BASE) * SP4_WIENER_FLOATS_PER_SLOT) as i32;
        debug_assert!(
            (wiener_offset as usize + SP4_WIENER_FLOATS_PER_SLOT) <= SP4_WIENER_TOTAL_FLOATS,
            "wiener offset {wiener_offset}+{SP4_WIENER_FLOATS_PER_SLOT} exceeds \
             SP4_WIENER_TOTAL_FLOATS={SP4_WIENER_TOTAL_FLOATS}"
        );
        unsafe {
            launch_apply_pearls(
                &self.stream,
                &self.apply_pearls_ad_kernel,
                scratch_dev,
                scratch_idx_arg,
                self.isv_signals_dev_ptr,
                isv_idx_arg,
                self.wiener_state_buf.dev_ptr,
                wiener_offset,
                1,
                crate::cuda_pipeline::sp4_wiener_ema::ALPHA_META,
            )?;
        }
        Ok(())
    }

    /// SP4 Layer A Task A5: producer for ISV[TARGET_Q_BOUND_INDEX=131].
    ///
    /// Reads `denoise_target_q_buf` (target-network Q-values [B, total_actions]), launches
    /// the single-block `target_q_p99_update` kernel which computes
    /// p99(|target_q|) via the header-only `sp4_histogram_p99<256>` device
    /// function and writes the step observation to
    /// `producer_step_scratch_buf[SCRATCH_IDX=0]`. Then launches the GPU
    /// `apply_pearls_ad_kernel` on the same stream which applies Pearls A+D
    /// to ISV[131] + the per-slot Wiener state at
    /// `wiener_state_buf[(131-SP4_SLOT_BASE)*3..(131-SP4_SLOT_BASE)*3 + 3]`,
    /// in a single device-side loop iteration. Stream-ordered with the
    /// producer so no host sync is required (graph-capture-compatible).
    ///
    /// 2026-05-01 GPU-Pearls refactor — replaced the prior host-side
    /// `apply_pearls_to_slot` Phase 2 read/write per
    /// `feedback_no_cpu_compute_strict.md`.
    ///
    /// No consumer wired in this task — Mech 1's clamp still uses
    /// `10 × Q_ABS_REF.max(1.0)`. Behaviour unchanged; the ISV slot now
    /// holds an EMA but is not yet read.
    pub fn launch_sp4_target_q_p99(&self) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp4_isv_slots::TARGET_Q_BOUND_INDEX;

        // Producer-step-scratch slot assignment. Stable layout — Tasks A6-A11
        // extend with subsequent indices in producer-launch order:
        //   0      = TARGET_Q_BOUND
        //   1..5   = ATOM_POS_BOUND[0..4]
        //   5..29  = WEIGHT_BOUND[0..8] · ADAM_M_BOUND[0..8] · ADAM_V_BOUND[0..8]
        //   29..37 = WD_RATE[0..8]
        //   37     = L1_LAMBDA_TRUNK
        //   38     = GRAD_CLIP_BOUND
        //   39     = H_S2_BOUND
        //   ── Task A13 retrofit producers (Pearls A+D for existing EMA kernels) ──
        //   40     = H_S2_RMS_EMA           → ISV[96]            (A13.0)
        //   41..43 = AUX_HEADS_LOSS_EMA     → ISV[113],ISV[114]  (A13.1)
        //   43     = RETIRED (SP13 B1.0)    — formerly AUX_LABEL_SCALE_EMA
        //                                     → ISV[117]; slot reserved
        //                                     by SP4_PRODUCER_COUNT for
        //                                     layout stability.
        //   44..52 = MOE_EXPERT_UTIL_EMA    → ISV[118..126)      (A13.2)
        //   52     = MOE_GATE_ENTROPY_EMA   → ISV[126]           (A13.2)
        //   53..59 = VSN_MASK_EMA           → ISV[105..111)      (A13.3)
        //   59..63 = IQN_QUANTILE_EMA       → ISV[99..103]\med   (A13.4)
        //   63..69 = REWARD_COMPONENT_EMA   → ISV[63..69)        (A13.5)
        //   69     = BW_D_H_S2_BOUND         → ISV[171]           (Layer C #260)
        //   70     = Q_DIR_GRAD_BOUND        → ISV[172]           (Layer C #260)
        const SCRATCH_IDX: usize = 0;
        self.launch_sp4_p99_producer_single_buf(
            &self.target_q_p99_update,
            self.denoise_target_q_buf.raw_ptr(),
            self.denoise_target_q_buf.len() as i32,
            SCRATCH_IDX,
            TARGET_Q_BOUND_INDEX,
            "target_q_p99_update launch",
        )
    }

    /// SP4 Layer A Task A6: producer for ISV[ATOM_POS_BOUND[0..4]].
    ///
    /// Reads `atom_positions_buf` (`[4 * num_atoms]` flat, branch `b` at
    /// offset `b * num_atoms` per `atoms_update_kernel.cu`'s canonical
    /// `positions_out + (long long)branch * num_atoms` write contract).
    /// Loops `branch ∈ 0..SP4_BRANCH_COUNT`, launching the single parameterised
    /// `atom_pos_p99_update` kernel four times — each launch passes the
    /// per-branch slice pointer (`atom_positions_buf.raw_ptr() + branch *
    /// num_atoms * 4 bytes`), the per-branch element count (`num_atoms`),
    /// and a distinct producer-step-scratch slot (`SCRATCH_BASE + branch ∈
    /// {1, 2, 3, 4}`). Then chains a single GPU `apply_pearls_ad_kernel`
    /// launch (n_slots=4) on the same stream — scratch [1..5) → ISV
    /// [132..136) is contiguous so the device-side loop iterates all 4
    /// branches in lock-step, writing ISV + Wiener state. 2026-05-01
    /// GPU-Pearls refactor; graph-capture-compatible (no host sync).
    ///
    /// No consumer wired in this task — Mech 2's atom-position clamp in
    /// `atoms_update_kernel.cu` still uses `±10 × ISV[Q_ABS_REF=16].max(1.0)`.
    /// Behaviour unchanged from before this commit; the ISV slots now hold
    /// per-branch EMAs but are not yet read.
    pub fn launch_sp4_atom_pos_p99_all_branches(&self) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp4_isv_slots::{
            ATOM_POS_BOUND_BASE, SP4_BRANCH_COUNT, SP4_SLOT_BASE,
        };
        use crate::cuda_pipeline::sp4_wiener_ema::launch_apply_pearls;

        // Producer-step-scratch slot assignment (matches `launch_sp4_target_q_p99`'s
        // documented stable layout): slots 1..1+SP4_BRANCH_COUNT == 1..5 are
        // reserved for ATOM_POS_BOUND[0..4] in branch order.
        const SCRATCH_BASE: usize = 1;

        // Same shared-memory budget as `launch_sp4_target_q_p99` (and Task A4
        // wrapper test): 8 warps × 256 bins × sizeof(int) = 8192 bytes.
        const SHARED_BYTES: u32 = (256 / 32) * 256 * 4;

        let cfg = LaunchConfig {
            grid_dim:         (1, 1, 1),
            block_dim:        (256, 1, 1),
            shared_mem_bytes: SHARED_BYTES,
        };

        let num_atoms = self.config.num_atoms;
        let total_len = self.atom_positions_buf.len();
        // Sanity: layout is exactly [SP4_BRANCH_COUNT × num_atoms]. Guards
        // against a future config change to atom_positions_buf shape that
        // would invalidate this launcher's per-branch slice arithmetic.
        debug_assert_eq!(
            total_len, SP4_BRANCH_COUNT * num_atoms,
            "atom_positions_buf layout drift: expected {SP4_BRANCH_COUNT}×num_atoms={}, got {total_len}",
            SP4_BRANCH_COUNT * num_atoms,
        );

        let base_dev_ptr = self.atom_positions_buf.raw_ptr();
        let scratch_dev = self.producer_step_scratch_buf.dev_ptr;
        let count_per_branch_i32 = num_atoms as i32;
        let stride_bytes = (num_atoms * std::mem::size_of::<f32>()) as u64;

        // ── Phase 1: launch one kernel per branch on the same stream ──
        for branch in 0..SP4_BRANCH_COUNT {
            let branch_dev_ptr: u64 = base_dev_ptr + (branch as u64) * stride_bytes;
            let scratch_idx_arg: i32 = (SCRATCH_BASE + branch) as i32;

            // Safety: kernel signature `(const float*, int, float*, int)`
            // matches the four args below. `branch_dev_ptr` is within the
            // allocation [base, base + 4 × num_atoms × 4 bytes); the
            // debug_assert above guards the layout invariant.
            unsafe {
                self.stream
                    .launch_builder(&self.atom_pos_p99_update)
                    .arg(&branch_dev_ptr)
                    .arg(&count_per_branch_i32)
                    .arg(&scratch_dev)
                    .arg(&scratch_idx_arg)
                    .launch(cfg)
                    .map_err(|e| MLError::ModelError(format!(
                        "atom_pos_p99_update[branch={branch}] launch: {e}"
                    )))?;
            }
        }

        // ── Phase 2: GPU Pearls A+D for all 4 branches in one launch ──
        // Scratch [1..5) → ISV [132..136) is contiguous; wiener offsets
        // (132-131)*3 .. (136-131)*3 = [3..15) are contiguous too. The
        // `apply_pearls_ad_kernel`'s device-side loop iterates the 4 slots
        // in lock-step. Per-branch degenerate-zero (kernel hasn't run /
        // `recompute_atom_positions` not yet populated) is handled by
        // the kernel's `step_obs == 0.0f` short-circuit.
        debug_assert_eq!(ATOM_POS_BOUND_BASE, SP4_SLOT_BASE + 1,
            "atom_pos ISV layout drift — kernel-side wiener_offset arithmetic depends on it");
        unsafe {
            launch_apply_pearls(
                &self.stream,
                &self.apply_pearls_ad_kernel,
                scratch_dev,
                SCRATCH_BASE as i32,
                self.isv_signals_dev_ptr,
                ATOM_POS_BOUND_BASE as i32,
                self.wiener_state_buf.dev_ptr,
                ((ATOM_POS_BOUND_BASE - SP4_SLOT_BASE) * 3) as i32,
                SP4_BRANCH_COUNT as i32,
                crate::cuda_pipeline::sp4_wiener_ema::ALPHA_META,
            )?;
        }
        Ok(())
    }

    /// SP4 Layer A Task A7: cold-path producer for the Pearl B
    /// per-param-group statistics oracle.
    ///
    /// For each param-group `g ∈ 0..SP4_PARAM_GROUP_COUNT=8`, calls
    /// `param_group_buffers(g)` to resolve `(params, grads, m, v, count,
    /// k_in, h_dim)`, then launches `param_group_oracle_update` once with
    /// distinct producer-step-scratch slot indices into the documented
    /// stable layout:
    ///   `WEIGHT_BOUND[g]`  → scratch slot `5 + g`
    ///   `ADAM_M_BOUND[g]`  → scratch slot `13 + g`
    ///   `ADAM_V_BOUND[g]`  → scratch slot `21 + g`
    ///   `WD_RATE[g]`       → scratch slot `29 + g`
    ///   `L1_LAMBDA_TRUNK`  → scratch slot `37` (group 0 only; passed as
    ///                        `-1_i32` to the kernel for groups 1-7).
    ///
    /// After all per-group launches complete, chains the GPU
    /// `apply_pearls_ad_kernel` per output slot (single-slot launches —
    /// non-contiguous (scratch, ISV) per group prevents batching) on the
    /// same stream — 4 outputs per group plus L1_LAMBDA for group 0,
    /// totalling 33 max launches. 2026-05-01 GPU-Pearls refactor: all
    /// reads/writes are GPU-side via dev pointers, no host sync.
    ///
    /// **Pearl B 4× memory-bandwidth reduction** vs four naive single-stat
    /// producer kernels: the kernel reads each of `params`, `grads`,
    /// `adam_m`, `adam_v` exactly once across all 4-5 outputs.
    ///
    /// **No consumer wired in this task** — Mech 9's clamp still uses
    /// hardcoded `weight_clamp_max_abs = 100 × q_abs_ref_eff` config args;
    /// AdamW weight_decay and L1 unchanged. SP4 consumer migration follows
    /// once all producers (A5-A9) land.
    ///
    /// **Aux trainers wired via `SP4AuxBuffers`** (Task A7 fix-up + fix-up #2)
    /// — `param_group_buffers` consults the supplied `aux_buffers` for groups
    /// 3-7 (IQN, IQL-hi, IQL-lo, Attn, Curiosity) and the existing main-DQN
    /// param slicing for groups 0-2. Curiosity is described as a multi-sub-
    /// buffer descriptor (`Sp4ParamGroupBufs::sub_buffers.len() == 4` for
    /// `[w1, b1, w2, b2]`); the kernel iterates all sub-buffers per pass
    /// and treats the union as a single logical group for p99/WD_RATE.
    /// Pass E (L1 trunk lambda) is dispatched only for group 0 which is
    /// always single-sub-buffer.
    pub fn launch_sp4_param_group_oracles_all_groups(
        &self,
        aux_buffers: &SP4AuxBuffers,
    ) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp4_isv_slots::{
            adam_m_bound, adam_v_bound, weight_bound, wd_rate, ParamGroup,
            L1_LAMBDA_TRUNK_INDEX, SP4_PARAM_GROUP_COUNT, SP4_SLOT_BASE,
        };
        use crate::cuda_pipeline::sp4_wiener_ema::launch_apply_pearls;

        // Producer-step-scratch slot assignment (matches the stable layout
        // documented in `launch_sp4_target_q_p99` and extended for Task A7):
        //   slots 5..13   = WEIGHT_BOUND[0..8]
        //   slots 13..21  = ADAM_M_BOUND[0..8]
        //   slots 21..29  = ADAM_V_BOUND[0..8]
        //   slots 29..37  = WD_RATE[0..8]
        //   slot  37      = L1_LAMBDA_TRUNK (group 0 only)
        const SCRATCH_BASE_W:   usize = 5;
        const SCRATCH_BASE_M:   usize = 13;
        const SCRATCH_BASE_V:   usize = 21;
        const SCRATCH_BASE_WD:  usize = 29;
        const SCRATCH_L1_TRUNK: usize = 37;

        // Sub-buffer table layout: 4 ptr-arrays of length K each.
        // Must match `SP4_ORACLE_TABLE_MAX_SUB` in the constructor.
        const K_MAX: usize = 4;

        // Same shared-memory budget as the rest of the SP4 producer family
        // (sp4_histogram_p99 contract): 8 warps × 256 bins × sizeof(int).
        // Pass D's 4 sequential block-reduces reuse this same dynamic shmem
        // region as a `float[256]` accumulator buffer (256 × 4 bytes ≪ 8192).
        const SHARED_BYTES: u32 = (256 / 32) * 256 * 4;

        let cfg = LaunchConfig {
            grid_dim:         (1, 1, 1),
            block_dim:        (256, 1, 1),
            shared_mem_bytes: SHARED_BYTES,
        };

        let scratch_dev = self.producer_step_scratch_buf.dev_ptr;

        // Track which groups got a real launch — Phase 2 only post-processes
        // those (skipping any group whose buffers aren't present).
        let mut launched: [bool; SP4_PARAM_GROUP_COUNT] = [false; SP4_PARAM_GROUP_COUNT];

        // Device-pointer offsets into the persistent oracle sub-buffer table.
        // `oracle_subbuf_table_buf` is `4 × K_MAX` u64s (= 32 u64s = 128 B).
        // Each ptr-array slice starts `K_MAX × 8` bytes apart.
        let table_dev = self.oracle_subbuf_table_buf.dev_ptr;
        let stride_bytes = (K_MAX * std::mem::size_of::<u64>()) as u64;
        let params_ptrs_dev = table_dev;
        let grads_ptrs_dev  = table_dev + stride_bytes;
        let adam_m_ptrs_dev = table_dev + 2 * stride_bytes;
        let adam_v_ptrs_dev = table_dev + 3 * stride_bytes;
        let counts_dev      = self.oracle_subbuf_counts_buf.dev_ptr;

        // ── Phase 1: launch one kernel per group (when buffers available) ──
        for g_idx in 0..SP4_PARAM_GROUP_COUNT {
            let g = ParamGroup::ALL[g_idx];
            let (bufs, k_in, h_dim) = match self.param_group_buffers(g, aux_buffers) {
                Some(b) => b,
                None => continue,
            };
            let n_sub = bufs.sub_buffers.len();
            if n_sub == 0 { continue; }
            if n_sub > K_MAX {
                return Err(MLError::ModelError(format!(
                    "param_group_oracle[group={g_idx}]: n_sub={n_sub} exceeds \
                     SP4_ORACLE_TABLE_MAX_SUB={K_MAX} — extend the table buffers \
                     or reduce sub-buffer count",
                )));
            }
            let total_count = bufs.total_count();
            // Sanity: a zero-element union would yield degenerate p99 = 0
            // inside the kernel and short-circuit Pearls A+D below; cheaper
            // to skip the launch entirely.
            if total_count == 0 { continue; }

            // Populate the host-visible side of the mapped-pinned tables for
            // this group's sub-buffers. Layout matches the device-pointer
            // arithmetic above (4 ptr-arrays of length K_MAX, packed).
            //
            // Safety: `oracle_subbuf_table_buf.host_ptr` is valid for
            // `4*K_MAX` u64s; `oracle_subbuf_counts_buf.host_ptr` is valid
            // for K_MAX i32s. Volatile writes ensure the kernel-visible
            // mapping observes the freshest values once we hit the launch.
            unsafe {
                let table_host = self.oracle_subbuf_table_buf.host_ptr;
                let counts_host = self.oracle_subbuf_counts_buf.host_ptr;
                for (s, sb) in bufs.sub_buffers.iter().enumerate() {
                    std::ptr::write_volatile(table_host.add(0 * K_MAX + s), sb.params_ptr);
                    std::ptr::write_volatile(table_host.add(1 * K_MAX + s), sb.grads_ptr);
                    std::ptr::write_volatile(table_host.add(2 * K_MAX + s), sb.adam_m_ptr);
                    std::ptr::write_volatile(table_host.add(3 * K_MAX + s), sb.adam_v_ptr);
                    std::ptr::write_volatile(counts_host.add(s), sb.count as i32);
                }
                // Zero the unused tail entries so a bug that reads past
                // `n_sub` lands in known-safe territory (count=0 inner loop
                // is a no-op).
                for s in n_sub..K_MAX {
                    std::ptr::write_volatile(table_host.add(0 * K_MAX + s), 0u64);
                    std::ptr::write_volatile(table_host.add(1 * K_MAX + s), 0u64);
                    std::ptr::write_volatile(table_host.add(2 * K_MAX + s), 0u64);
                    std::ptr::write_volatile(table_host.add(3 * K_MAX + s), 0u64);
                    std::ptr::write_volatile(counts_host.add(s), 0i32);
                }
            }

            let n_sub_i32       = n_sub as i32;
            let total_count_i32 = total_count as i32;
            let k_in_i32        = k_in;
            let h_dim_i32       = h_dim;
            let weight_idx_i32  = (SCRATCH_BASE_W  + g_idx) as i32;
            let adam_m_idx_i32  = (SCRATCH_BASE_M  + g_idx) as i32;
            let adam_v_idx_i32  = (SCRATCH_BASE_V  + g_idx) as i32;
            let wd_rate_idx_i32 = (SCRATCH_BASE_WD + g_idx) as i32;
            let l1_idx_i32: i32 = if g_idx == 0 {
                SCRATCH_L1_TRUNK as i32
            } else {
                -1
            };

            // Safety: kernel signature `(const u64*, const u64*, const u64*,
            // const u64*, const int*, int, int, int, int, float*, int, int,
            // int, int, int)` matches the 15 args below; all device pointers
            // come from `param_group_buffers` (sub-buffers) and the
            // construction-time mapped-pinned tables (ptr arrays + counts).
            unsafe {
                self.stream
                    .launch_builder(&self.param_group_oracle_update)
                    .arg(&params_ptrs_dev)
                    .arg(&grads_ptrs_dev)
                    .arg(&adam_m_ptrs_dev)
                    .arg(&adam_v_ptrs_dev)
                    .arg(&counts_dev)
                    .arg(&n_sub_i32)
                    .arg(&total_count_i32)
                    .arg(&k_in_i32)
                    .arg(&h_dim_i32)
                    .arg(&scratch_dev)
                    .arg(&weight_idx_i32)
                    .arg(&adam_m_idx_i32)
                    .arg(&adam_v_idx_i32)
                    .arg(&wd_rate_idx_i32)
                    .arg(&l1_idx_i32)
                    .launch(cfg)
                    .map_err(|e| MLError::ModelError(format!(
                        "param_group_oracle_update[group={g_idx}] launch: {e}"
                    )))?;
            }
            // The kernel reads the table BEFORE the next host overwrite
            // because each launch is serialised on the same stream and the
            // next iteration's table population happens after kernel
            // dispatch. To avoid host stomping the table while the prior
            // launch is still reading, sync between launches.
            //
            // Cold-path producer; the per-launch sync cost is negligible vs
            // the kernel work. Without it, the next iteration's host writes
            // would race with the in-flight kernel's coalesced loads.
            self.stream.synchronize().map_err(|e| MLError::ModelError(format!(
                "param_group_oracle_update[group={g_idx}] inter-launch sync: {e}",
            )))?;
            launched[g_idx] = true;
        }

        // ── Phase 2: GPU Pearls A+D, per-group per-output single-slot launches ──
        // The Pearl B oracle's 4 outputs per group sit at NON-CONTIGUOUS
        // (scratch, ISV) pairs (W=5+g→136+g, M=13+g→144+g, V=21+g→152+g,
        // WD=29+g→160+g, L1=37→170 for g=0 only) — each output requires
        // its own `apply_pearls_ad_kernel` launch with n_slots=1. Per-group
        // gating by `launched[g_idx]` keeps the kernel's degenerate-zero
        // short-circuit off the hook for groups whose oracle never ran
        // (the scratch slot retains a stale value from the previous step).
        //
        // 33 single-slot launches max (8 groups × 4 outputs + 1 L1 for
        // group 0); each is a single-thread same-stream kernel ≪ 5µs and
        // fits fully inside CUDA Graph capture (the whole point of the
        // 2026-05-01 GPU-Pearls refactor — no host sync, no closure, no
        // mapped-pinned host_ptr reads).
        let apply_pearls_one = |isv_idx: usize, scratch_idx: usize| -> Result<(), MLError> {
            // SP4 wiener convention for slots inside [131, 171) (the SP4
            // contiguous block): offset = (isv_idx - SP4_SLOT_BASE) * 3.
            // Holds for W/M/V/WD (their ISV slots are consecutive across
            // groups), AND for L1_LAMBDA_TRUNK (ISV 170 → offset 117) —
            // even though scratch 37 ≠ isv − base, the ISV-keyed offset
            // is the canonical mapping.
            unsafe {
                launch_apply_pearls(
                    &self.stream,
                    &self.apply_pearls_ad_kernel,
                    scratch_dev,
                    scratch_idx as i32,
                    self.isv_signals_dev_ptr,
                    isv_idx as i32,
                    self.wiener_state_buf.dev_ptr,
                    ((isv_idx - SP4_SLOT_BASE) * 3) as i32,
                    1,
                    crate::cuda_pipeline::sp4_wiener_ema::ALPHA_META,
                )
            }
        };

        for g_idx in 0..SP4_PARAM_GROUP_COUNT {
            if !launched[g_idx] { continue; }
            apply_pearls_one(weight_bound(g_idx),  SCRATCH_BASE_W  + g_idx)?;
            apply_pearls_one(adam_m_bound(g_idx),  SCRATCH_BASE_M  + g_idx)?;
            apply_pearls_one(adam_v_bound(g_idx),  SCRATCH_BASE_V  + g_idx)?;
            apply_pearls_one(wd_rate(g_idx),       SCRATCH_BASE_WD + g_idx)?;
            if g_idx == 0 {
                apply_pearls_one(L1_LAMBDA_TRUNK_INDEX, SCRATCH_L1_TRUNK)?;
            }
        }
        Ok(())
    }

    /// SP4 Layer A Task A7 helper: resolve the per-param-group buffer
    /// descriptor for the Pearl B fused statistics oracle.
    ///
    /// Returns `Some((bufs, k_in, h_dim))` for every group; the descriptor
    /// holds 1 sub-buffer entry for groups 0-6 (single contiguous params
    /// buffer) and 4 entries for Curiosity (one per `[w1, b1, w2, b2]`
    /// sub-tensor). `None` reserved for forward-compat in case a trainer
    /// is unavailable in a future configuration.
    ///
    /// `k_in`/`h_dim` are non-zero only for `ParamGroup::DqnTrunk` (group 0)
    /// — the trunk Pass-E entropy-deficit computation needs to know the
    /// `[h_dim × k_in]` shape of the trunk-input weight tensor (`w_a_h_s1`,
    /// the GRN Linear_a at `param_sizes[0]`). Other groups pass 0 / 0 and
    /// the kernel skips Pass E via `l1_lambda_scratch_idx = -1`.
    ///
    /// For the 3 DQN-internal groups, slices are derived from
    /// `compute_param_sizes` + `padded_byte_offset`:
    ///   - **Trunk** (tensors `[0..13)`): GRN h_s1 + h_s2 + γ/β pairs. Same
    ///     range as the existing `trunk_adam_m_ptr` / `trunk_params_ptr`
    ///     accessors.
    ///   - **Value head** (tensors `[13..17)`): W_v1, B_v1, W_v2, B_v2.
    ///     Same range as `value_adam_m_ptr`.
    ///   - **Branch heads** (tensors `[17..33)`): W/B_b{0..3}fc +
    ///     W/B_b{0..3}out concatenated. Same range as `branch_adam_m_ptr`.
    ///
    /// For aux trainers (IQN, IQL-hi, IQL-lo, Attn, Curiosity) the caller
    /// supplies the descriptor via `aux_buffers`; `FusedTrainingCtx` builds
    /// that struct from each child trainer's device-pointer accessors.
    fn param_group_buffers(
        &self,
        group: crate::cuda_pipeline::sp4_isv_slots::ParamGroup,
        aux_buffers: &SP4AuxBuffers,
    ) -> Option<(Sp4ParamGroupBufs, i32, i32)> {
        use crate::cuda_pipeline::sp4_isv_slots::ParamGroup;

        let f32_sz = std::mem::size_of::<f32>() as u64;
        let param_sizes = compute_param_sizes(&self.config);

        match group {
            ParamGroup::DqnTrunk => {
                // Trunk slice = tensors [0..13). k_in = state_dim_padded
                // (the input dim of `w_a_h_s1`); h_dim = shared_h1 (the
                // output dim of `w_a_h_s1`). Per `compute_param_sizes`,
                // tensor [0] (`w_a_h_s1`) has size `shared_h1 × s1_input_dim`
                // — that's our [h_dim × k_in] row-major layout.
                let count = self.trunk_param_count;
                let params_ptr = self.params_buf.raw_ptr();
                let grads_ptr  = self.ptrs.grad_buf;
                let m_ptr      = self.m_buf.raw_ptr();
                let v_ptr      = self.v_buf.raw_ptr();
                let h_dim = self.config.shared_h1 as i32;
                let s1_input_dim = if self.config.shared_h1 > 0 {
                    (param_sizes[0] / self.config.shared_h1) as i32
                } else {
                    0
                };
                let k_in = s1_input_dim;
                Some((
                    Sp4ParamGroupBufs::single(params_ptr, grads_ptr, m_ptr, v_ptr, count),
                    k_in, h_dim,
                ))
            }
            ParamGroup::DqnValue => {
                let start = padded_byte_offset(&param_sizes, 13);
                let end   = padded_byte_offset(&param_sizes, 17);
                let count = ((end - start) / f32_sz) as usize;
                let params_ptr = self.params_buf.raw_ptr() + start;
                let grads_ptr  = self.ptrs.grad_buf       + start;
                let m_ptr      = self.m_buf.raw_ptr()      + start;
                let v_ptr      = self.v_buf.raw_ptr()      + start;
                Some((
                    Sp4ParamGroupBufs::single(params_ptr, grads_ptr, m_ptr, v_ptr, count),
                    0, 0,
                ))
            }
            ParamGroup::DqnBranches => {
                let start = padded_byte_offset(&param_sizes, 17);
                let end   = padded_byte_offset(&param_sizes, 33);
                let count = ((end - start) / f32_sz) as usize;
                let params_ptr = self.params_buf.raw_ptr() + start;
                let grads_ptr  = self.ptrs.grad_buf       + start;
                let m_ptr      = self.m_buf.raw_ptr()      + start;
                let v_ptr      = self.v_buf.raw_ptr()      + start;
                Some((
                    Sp4ParamGroupBufs::single(params_ptr, grads_ptr, m_ptr, v_ptr, count),
                    0, 0,
                ))
            }
            ParamGroup::Iqn       => Some((aux_buffers.iqn.clone(),       0, 0)),
            ParamGroup::IqlHigh   => Some((aux_buffers.iql_high.clone(),  0, 0)),
            ParamGroup::IqlLow    => Some((aux_buffers.iql_low.clone(),   0, 0)),
            ParamGroup::Attn      => Some((aux_buffers.attn.clone(),      0, 0)),
            ParamGroup::Curiosity => Some((aux_buffers.curiosity.clone(), 0, 0)),
        }
    }

    /// SP4 Layer A Task A8: producer for ISV[GRAD_CLIP_BOUND_INDEX=168].
    ///
    /// `grad_norm_buf` is a single-element f32 device buffer populated by
    /// `launch_grad_norm_finalize` with the current step's L2 grad norm.
    /// p99 over a single element is the element itself — the kernel
    /// (`grad_norm_p99_update`) reduces to a single-thread copy of
    /// `grad_norm_buf[0]` → `producer_step_scratch_buf[SCRATCH_IDX=38]` with
    /// `__threadfence_system()`. No histogram pass is needed; the launcher
    /// keeps the same shape as `launch_sp4_target_q_p99` for consistency
    /// across SP4 producers, and Mech 6's eventual Layer-B ISV read in the
    /// adaptive_clip path consumes ISV[168] directly.
    ///
    /// Chains the GPU `apply_pearls_ad_kernel` on the same stream
    /// (2026-05-01 GPU-Pearls refactor — graph-capture-compatible, no
    /// host sync) which writes the new `x_mean` to ISV[168] and the
    /// updated Wiener state at `wiener_state_buf[(168 - SP4_SLOT_BASE)
    /// * 3 = 111..114)`.
    ///
    /// **No consumer wired in this task** — Mech 6's `adaptive_clip` upper
    /// bound still uses the existing fast/slow grad-norm EMA path; SP4
    /// consumer migration follows once all producers (A5-A9) land.
    pub fn launch_sp4_grad_norm_p99(&self) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp4_isv_slots::{GRAD_CLIP_BOUND_INDEX, SP4_SLOT_BASE};
        use crate::cuda_pipeline::sp4_wiener_ema::launch_apply_pearls;

        // Producer-step-scratch slot 38 is reserved for GRAD_CLIP_BOUND per
        // the stable layout documented on `launch_sp4_target_q_p99`:
        //   0      = TARGET_Q_BOUND          (Task A5)
        //   1..5   = ATOM_POS_BOUND[0..4]    (Task A6)
        //   5..29  = WEIGHT/ADAM_M/ADAM_V × 8 (Task A7)
        //   29..37 = WD_RATE × 8             (Task A7)
        //   37     = L1_LAMBDA_TRUNK         (Task A7)
        //   38     = GRAD_CLIP_BOUND         (this task)
        //   39     = H_S2_BOUND              (Task A9)
        //   40..69 = retrofit existing 29 Task-A13 producers
        const SCRATCH_IDX: usize = 38;

        // Single-thread degenerate kernel — minimal launch config, no shared
        // memory. Mirrors the `q_drift_rate_ema_kernel.cu` /
        // `moe_lambda_eff_kernel.cu` cold-path single-thread footprint.
        let cfg = LaunchConfig {
            grid_dim:         (1, 1, 1),
            block_dim:        (1, 1, 1),
            shared_mem_bytes: 0,
        };

        let grad_norm_ptr = self.ptrs.grad_norm_buf;
        let scratch_dev = self.producer_step_scratch_buf.dev_ptr;
        let scratch_idx_arg: i32 = SCRATCH_IDX as i32;

        // Safety: kernel signature `(const float*, float*, int)` matches
        // the three args below; both device pointers are valid for the
        // lifetime of `self`.
        unsafe {
            self.stream
                .launch_builder(&self.grad_norm_p99_update)
                .arg(&grad_norm_ptr)
                .arg(&scratch_dev)
                .arg(&scratch_idx_arg)
                .launch(cfg)
                .map_err(|e| MLError::ModelError(format!("grad_norm_p99_update launch: {e}")))?;
        }

        // GPU Pearls A+D: stream-ordered with the producer kernel, no
        // host sync needed (graph-capture-compatible 2026-05-01 refactor).
        // Wiener offset for SP4 producer slot 168 → (168-131)*3 = 111.
        // Kernel-side `step_obs == 0` short-circuit handles cold-start
        // (before first backward populates `grad_norm_buf`).
        let isv_idx = GRAD_CLIP_BOUND_INDEX;
        unsafe {
            launch_apply_pearls(
                &self.stream,
                &self.apply_pearls_ad_kernel,
                scratch_dev,
                SCRATCH_IDX as i32,
                self.isv_signals_dev_ptr,
                isv_idx as i32,
                self.wiener_state_buf.dev_ptr,
                ((isv_idx - SP4_SLOT_BASE) * 3) as i32,
                1,
                crate::cuda_pipeline::sp4_wiener_ema::ALPHA_META,
            )?;
        }
        Ok(())
    }

    /// SP4 Layer A Task A9: cold-path producer for ISV[H_S2_BOUND_INDEX=169].
    ///
    /// Reads `save_h_s2 [B × SH2]` (the trunk-output activation surface,
    /// populated each step by the online forward — the same buffer
    /// `launch_h_s2_rms_ema` consumes for the slot-96 RMS signal), launches
    /// the single-block `h_s2_p99_update` kernel which computes
    /// p99(|save_h_s2|) via `sp4_histogram_p99<256>` and writes the step
    /// observation to `producer_step_scratch_buf[SCRATCH_IDX=39]`. Then
    /// chains the GPU `apply_pearls_ad_kernel` on the same stream
    /// (2026-05-01 GPU-Pearls refactor — graph-capture-compatible, no host
    /// sync) which writes the new `x_mean` to ISV[169] and the updated
    /// Wiener state at `wiener_state_buf[(169 - SP4_SLOT_BASE) * 3 =
    /// 114..117)`.
    ///
    /// **Distinct from `launch_h_s2_rms_ema`** (slot 96): that producer
    /// tracks RMS (a different signal). This producer tracks p99(|h_s2|).
    /// Task A13 retrofitted `h_s2_rms_ema_update` to also use Pearls A+D
    /// in a separate commit.
    ///
    /// **No consumer wired in this task** — Mech 10's `h_s2` clamp migrates
    /// in Layer B once all producers (A5-A9) land. Behaviour unchanged from
    /// before this commit.
    pub fn launch_sp4_h_s2_p99(&self) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp4_isv_slots::H_S2_BOUND_INDEX;

        // Producer-step-scratch slot 39 is reserved for H_S2_BOUND per the
        // stable layout documented on `launch_sp4_target_q_p99`.
        const SCRATCH_IDX: usize = 39;
        self.launch_sp4_p99_producer_single_buf(
            &self.h_s2_p99_update,
            self.save_h_s2.raw_ptr(),
            self.save_h_s2.len() as i32,
            SCRATCH_IDX,
            H_S2_BOUND_INDEX,
            "h_s2_p99_update launch",
        )
    }

    /// SP4 Layer C #260 (2026-05-01): producer for ISV[BW_D_H_S2_BOUND_INDEX=171].
    ///
    /// Reads `bw_d_h_s2 [B × SH2]` (the IQN trunk-backward d_h_s2 buffer
    /// post-DtoD copy from `iqn_d_h_s2_ptr` and post slot-35 NaN check),
    /// launches the single-block 256-thread `bw_d_h_s2_p99_update` kernel
    /// which computes p99(|bw_d_h_s2|) via `sp4_histogram_p99<256>` and
    /// writes the step observation to `producer_step_scratch_buf[SCRATCH_IDX=69]`
    /// with `__threadfence_system()`. Then chains the GPU
    /// `apply_pearls_ad_kernel` on the same stream — applies Pearls A+D
    /// to ISV[BW_D_H_S2_BOUND_INDEX=171] using
    /// `wiener_state_buf[(171 - SP4_SLOT_BASE) * 3 = 120 .. +3]`.
    /// Graph-capture-compatible by construction (no host sync; same-stream
    /// ordering with the producer kernel).
    ///
    /// **Migrated from SP1-Phase-C `1e6 × ISV[H_S2_RMS_EMA].max(1.0)`**
    /// hardcoded multiplier per `feedback_isv_for_adaptive_bounds`.
    /// Consumer at `apply_iqn_trunk_gradient` reads
    /// `ISV[BW_D_H_S2_BOUND_INDEX].max(EPS_CLAMP_FLOOR)` directly.
    ///
    /// Caller contract: invoke BEFORE the consumer's
    /// `launch_clamp_finite_f32(ptrs.bw_d_h_s2, ...)` so the bound is
    /// fresh; the producer launches AFTER the slot-35 NaN check
    /// (`check_nan_f32(..., 35)`) in `apply_iqn_trunk_gradient` (the NaN
    /// check captures the regression sentinel before the producer
    /// measures the clean distribution — same ordering invariant as the
    /// inline slot 24/25 checks in `launch_cublas_backward_to`).
    pub fn launch_sp4_bw_d_h_s2_p99(&self) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp4_isv_slots::BW_D_H_S2_BOUND_INDEX;

        // Layer C #260: scratch slot 69 — first Layer C slot post-Layer-A
        // retrofits (which occupy [40..69)).
        const SCRATCH_IDX: usize = 69;
        let b = self.config.batch_size;
        let sh2 = self.config.shared_h2;
        self.launch_sp4_p99_producer_single_buf(
            &self.bw_d_h_s2_p99_update,
            self.ptrs.bw_d_h_s2,
            (b * sh2) as i32,
            SCRATCH_IDX,
            BW_D_H_S2_BOUND_INDEX,
            "bw_d_h_s2_p99_update launch",
        )
    }

    /// SP4 Layer C #260 (2026-05-01): producer for ISV[Q_DIR_GRAD_BOUND_INDEX=172].
    ///
    /// Reads the union of `d_value_logits` ∪ `d_adv_logits` (both dueling-
    /// head logits gradient buffers) as one logical distribution, launches
    /// the single-block 256-thread `q_dir_grad_p99_update` kernel which
    /// computes p99(|union|) via `sp4_histogram_p99_multi<256>` with
    /// n_sub=2, and writes the step observation to
    /// `producer_step_scratch_buf[SCRATCH_IDX=70]` with
    /// `__threadfence_system()`. Then chains the GPU
    /// `apply_pearls_ad_kernel` on the same stream — applies Pearls A+D
    /// to ISV[Q_DIR_GRAD_BOUND_INDEX=172] using
    /// `wiener_state_buf[(172 - SP4_SLOT_BASE) * 3 = 123 .. +3]`.
    /// Graph-capture-compatible by construction (no host sync; same-stream
    /// ordering with the producer kernel).
    ///
    /// **Migrated from SP1-Phase-C `1e6 × ISV[Q_DIR_ABS_REF].max(1.0)`**
    /// hardcoded multiplier per `feedback_isv_for_adaptive_bounds`.
    /// Consumer at `launch_cublas_backward_to` reads
    /// `ISV[Q_DIR_GRAD_BOUND_INDEX].max(EPS_CLAMP_FLOOR)` directly. The
    /// same bound was previously applied to BOTH `d_value_logits` AND
    /// `d_adv_logits`, so the union p99 is the natural ISV-driven
    /// replacement (one bound, one ISV slot, one producer).
    ///
    /// Sub-buffer pointer table is the dedicated 2-entry mapped-pinned
    /// `q_dir_grad_subbuf_table_buf` + `q_dir_grad_subbuf_counts_buf`
    /// (allocated in the constructor). Separate from
    /// `oracle_subbuf_table_buf` to eliminate the previous implicit
    /// "must run before param_group_oracle" temporal coupling: each
    /// launcher now owns its own descriptor tables.
    ///
    /// Caller contract: invoke BEFORE the consumer's
    /// `launch_clamp_finite_f32(d_value_logits/d_adv_logits, ...)` so the
    /// bound is fresh; the producer launches AFTER the inline slot 24/25
    /// NaN checks (`check_nan_f32(..., 24)` + `check_nan_f32(..., 25)`)
    /// — same ordering invariant as site 1.
    pub fn launch_sp4_q_dir_grad_p99(&self) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp4_isv_slots::{Q_DIR_GRAD_BOUND_INDEX, SP4_SLOT_BASE};
        use crate::cuda_pipeline::sp4_wiener_ema::launch_apply_pearls;

        // Layer C #260: scratch slot 70 — second Layer C slot.
        const SCRATCH_IDX: usize = 70;

        // Shared memory: 8 warps × 256 bins × sizeof(int) = 8192 bytes.
        // Same contract as the other histogram-based SP4 producers.
        const SHARED_BYTES: u32 = (256 / 32) * 256 * 4;

        // q_dir_grad always operates on exactly 2 sub-buffers
        // (`d_value_logits` ∪ `d_adv_logits`); the descriptor tables
        // (`q_dir_grad_subbuf_table_buf` + `q_dir_grad_subbuf_counts_buf`)
        // are sized for exactly N_SUB entries — no spare tail to zero.
        const N_SUB: usize = 2;

        let cfg = LaunchConfig {
            grid_dim:         (1, 1, 1),
            block_dim:        (256, 1, 1),
            shared_mem_bytes: SHARED_BYTES,
        };

        // Populate the host-visible side of the dedicated mapped-pinned
        // tables. Layout is flat (no 4 × K_MAX ptr-array bookkeeping):
        // entries [0..N_SUB) hold the sub-buffer device pointers; the
        // count table holds the element counts at the same offsets.
        //
        // Safety: `q_dir_grad_subbuf_table_buf.host_ptr` is valid for
        // exactly N_SUB u64s; `q_dir_grad_subbuf_counts_buf.host_ptr` is
        // valid for exactly N_SUB i32s. Volatile writes ensure the
        // kernel-visible mapping observes the freshest values once we hit
        // the launch (mapped-pinned memory has device-visible coherence
        // after a `__threadfence_system()`-paired write/launch ordering,
        // which holds since the next op on the stream is the kernel
        // launch).
        let n_val = self.d_value_logits_buf.len();
        let n_adv = self.d_adv_logits_buf.len();
        let p_val = self.d_value_logits_buf.raw_ptr();
        let p_adv = self.d_adv_logits_buf.raw_ptr();
        unsafe {
            let table_host = self.q_dir_grad_subbuf_table_buf.host_ptr;
            let counts_host = self.q_dir_grad_subbuf_counts_buf.host_ptr;
            std::ptr::write_volatile(table_host.add(0), p_val);
            std::ptr::write_volatile(table_host.add(1), p_adv);
            std::ptr::write_volatile(counts_host.add(0), n_val as i32);
            std::ptr::write_volatile(counts_host.add(1), n_adv as i32);
        }

        // Device pointers into the dedicated tables — flat layout, no
        // K_MAX stride. No cross-launcher coupling: this launcher is the
        // sole owner of these descriptor buffers.
        let sub_buf_ptrs_dev = self.q_dir_grad_subbuf_table_buf.dev_ptr;
        let sub_counts_dev = self.q_dir_grad_subbuf_counts_buf.dev_ptr;
        let n_sub_i32: i32 = N_SUB as i32;
        let total_count_i32: i32 = (n_val + n_adv) as i32;
        let scratch_dev = self.producer_step_scratch_buf.dev_ptr;
        let scratch_idx_arg: i32 = SCRATCH_IDX as i32;

        // Safety: kernel signature
        //   `(const u64*, const i32*, i32, i32, float*, i32)`
        // matches the six args below; both device pointers (table buf,
        // counts buf) come from constructor-allocated mapped-pinned
        // buffers and are valid for the lifetime of `self`.
        unsafe {
            self.stream
                .launch_builder(&self.q_dir_grad_p99_update)
                .arg(&sub_buf_ptrs_dev)
                .arg(&sub_counts_dev)
                .arg(&n_sub_i32)
                .arg(&total_count_i32)
                .arg(&scratch_dev)
                .arg(&scratch_idx_arg)
                .launch(cfg)
                .map_err(|e| MLError::ModelError(format!("q_dir_grad_p99_update launch: {e}")))?;
        }

        // GPU Pearls A+D: stream-ordered with the producer kernel, no host
        // sync (graph-capture-compatible). Wiener offset for SP4 slot 172
        // → (172-131)*3 = 123. Kernel-side `step_obs == 0` short-circuit
        // handles degenerate-zero cases (uninitialised buffers).
        let isv_idx = Q_DIR_GRAD_BOUND_INDEX;
        unsafe {
            launch_apply_pearls(
                &self.stream,
                &self.apply_pearls_ad_kernel,
                scratch_dev,
                SCRATCH_IDX as i32,
                self.isv_signals_dev_ptr,
                isv_idx as i32,
                self.wiener_state_buf.dev_ptr,
                ((isv_idx - SP4_SLOT_BASE) * 3) as i32,
                1,
                crate::cuda_pipeline::sp4_wiener_ema::ALPHA_META,
            )?;
        }
        Ok(())
    }

    /// SP5 Task A1 (2026-05-01): launch the q_branch_stats + pearl_1_atom
    /// two-kernel chain, followed by 24 `apply_pearls_ad_kernel` launches
    /// (16 Pearl 1 slots + 8 shared-signal slots) to smooth step observations
    /// into ISV via Pearls A+D.
    ///
    /// Execution sequence (all on `self.stream`):
    ///   1. `q_branch_stats_update` (4-thread): reads `q_out_buf [B, 13]`
    ///      row-major, writes 16 floats to scratch[71..87).
    ///   2. `pearl_1_atom_update` (4-thread): reads scratch[71..87) + ISV
    ///      ATOM_HEADROOM_BASE + atoms_clip_count_buf; writes 16 floats to
    ///      scratch[87..103).
    ///   3. 24 × `apply_pearls_ad_kernel` (single-slot, n_slots=1): Pearls
    ///      A+D per ISV slot using the wiener_state_buf triples at offsets
    ///      SP4_PRODUCER_COUNT*3 + (slot-SP5_SLOT_BASE)*3 each.
    ///
    /// Layer A only — producers write ISV, no consumer reads these slots yet.
    /// Must be called AFTER `compute_expected_q` has populated `q_out_buf`
    /// and BEFORE the HEALTH_DIAG emit (same ordering invariant as SP4 launches).
    pub(crate) fn launch_sp5_pearl_1_atom(&self) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp4_wiener_ema::launch_apply_pearls;
        use crate::cuda_pipeline::sp5_isv_slots::{
            SP5_SLOT_BASE,
            ATOM_V_CENTER_BASE, ATOM_V_HALF_BASE, ATOM_HEADROOM_BASE, ATOM_CLIP_RATE_BASE,
            BRANCH_ENTROPY_BASE, Q_VAR_PER_BRANCH_BASE,
        };

        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_sp5_pearl_1_atom: isv_signals_dev_ptr must be allocated by constructor");

        let q_out_ptr  = self.q_out_buf.raw_ptr();
        let b_i32      = self.config.batch_size as i32;
        let scratch_dev = self.producer_step_scratch_buf.dev_ptr;
        let wiener_dev  = self.wiener_state_buf.dev_ptr;
        let isv_dev     = self.isv_signals_dev_ptr;

        let action_counts_dev  = self.action_counts_buf.dev_ptr;
        let action_offsets_dev = self.action_offsets_buf.dev_ptr;
        let clip_count_dev     = self.atoms_clip_count_buf.dev_ptr;

        // Step 1: q_branch_stats_update → scratch[71..87).
        // Kernel signature: (q_out_buf, batch_size, action_counts, action_offsets,
        //                    scratch_buf, scratch_idx_base)
        let scratch_q_stats_i32 = SCRATCH_Q_STATS as i32;
        unsafe {
            self.stream
                .launch_builder(&self.q_branch_stats_kernel)
                .arg(&q_out_ptr)
                .arg(&b_i32)
                .arg(&action_counts_dev)
                .arg(&action_offsets_dev)
                .arg(&scratch_dev)
                .arg(&scratch_q_stats_i32)
                .launch(LaunchConfig {
                    grid_dim:         (1, 1, 1),
                    block_dim:        (4, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("q_branch_stats_update: {e}")))?;
        }

        // Step 2: pearl_1_atom_update → scratch[87..103).
        // Reads scratch[71..87) (q_branch_stats output) + ISV ATOM_HEADROOM_BASE
        // + atoms_clip_count_buf.
        // Kernel signature: (scratch_buf, q_stats_idx_base, atoms_clip_count,
        //                    batch_size, scratch_out, v_center_idx_base,
        //                    v_half_idx_base, headroom_idx_base, clip_rate_idx_base,
        //                    isv_signals, isv_headroom_base)
        let v_center_base_i32    = SCRATCH_ATOM_OUT as i32;             // [87..91)
        let v_half_base_i32      = (SCRATCH_ATOM_OUT + 4) as i32;       // [91..95)
        let headroom_base_i32    = (SCRATCH_ATOM_OUT + 8) as i32;       // [95..99)
        let clip_rate_base_i32   = (SCRATCH_ATOM_OUT + 12) as i32;      // [99..103)
        let isv_headroom_base_i32 = ATOM_HEADROOM_BASE as i32;
        unsafe {
            self.stream
                .launch_builder(&self.pearl_1_atom_kernel)
                .arg(&scratch_dev)
                .arg(&scratch_q_stats_i32)
                .arg(&clip_count_dev)
                .arg(&action_counts_dev)
                .arg(&b_i32)
                .arg(&scratch_dev)
                .arg(&v_center_base_i32)
                .arg(&v_half_base_i32)
                .arg(&headroom_base_i32)
                .arg(&clip_rate_base_i32)
                .arg(&isv_dev)
                .arg(&isv_headroom_base_i32)
                .launch(LaunchConfig {
                    grid_dim:         (1, 1, 1),
                    block_dim:        (4, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("pearl_1_atom_update: {e}")))?;
        }

        // Step 3: apply_pearls_ad_kernel ×24 — one single-slot launch per
        // ISV output. All 24 slots share the same Wiener buffer indexed
        // by (SP4_PRODUCER_COUNT + (isv_slot - SP5_SLOT_BASE)) * 3.
        //
        // Pearl 1 atom outputs: v_center[4], v_half[4], headroom[4], clip_rate[4]
        //   → scratch [87..91), [91..95), [95..99), [99..103)
        //   → ISV    [174..178), [178..182), [182..186), [186..190)
        //
        // Shared signals: branch_entropy[4], q_var[4]
        //   → scratch [75..79) and [79..83) (q_branch_stats slots var+entropy)
        //   → ISV    [218..222) and [222..226)
        //
        // Note: q_branch_stats writes var at scratch[71+b*4+2] and entropy at
        // scratch[71+b*4+3]. The branch layout is NOT contiguous across branches
        // in the way ISV expects — we must use single-slot launches for each.
        let base_wiener_offset = SP4_PRODUCER_COUNT as i32 * 3; // 213

        // Pearl 1 outputs — 16 slots in 4 groups of 4:
        // v_center (ISV 174..178), v_half (178..182), headroom (182..186), clip_rate (186..190)
        for group_idx in 0..4_usize {
            let group_scratch_start = SCRATCH_ATOM_OUT + group_idx * 4;
            let group_isv_start = match group_idx {
                0 => ATOM_V_CENTER_BASE,
                1 => ATOM_V_HALF_BASE,
                2 => ATOM_HEADROOM_BASE,
                3 => ATOM_CLIP_RATE_BASE,
                _ => unreachable!(),
            };
            for b in 0..4_usize {
                let scratch_idx = (group_scratch_start + b) as i32;
                let isv_idx     = (group_isv_start + b) as i32;
                let wiener_off  = base_wiener_offset + (isv_idx - SP5_SLOT_BASE as i32) * 3;
                unsafe {
                    launch_apply_pearls(
                        &self.stream,
                        &self.apply_pearls_ad_kernel,
                        scratch_dev, scratch_idx,
                        isv_dev,     isv_idx,
                        wiener_dev,  wiener_off,
                        1,
                        crate::cuda_pipeline::sp4_wiener_ema::ALPHA_META,
                    )?;
                }
            }
        }

        // Shared signal outputs — 8 slots in 2 groups of 4:
        // branch_entropy (ISV 218..222), q_var (222..226)
        // q_branch_stats writes: branch b at scratch[71+b*4+3] (entropy) and scratch[71+b*4+2] (var)
        for b in 0..4_usize {
            // entropy → ISV BRANCH_ENTROPY_BASE + b
            let scratch_entropy = (SCRATCH_Q_STATS + b * 4 + 3) as i32;
            let isv_entropy     = (BRANCH_ENTROPY_BASE + b) as i32;
            let wiener_entropy  = base_wiener_offset + (isv_entropy - SP5_SLOT_BASE as i32) * 3;
            unsafe {
                launch_apply_pearls(
                    &self.stream,
                    &self.apply_pearls_ad_kernel,
                    scratch_dev, scratch_entropy,
                    isv_dev,     isv_entropy,
                    wiener_dev,  wiener_entropy,
                    1,
                    crate::cuda_pipeline::sp4_wiener_ema::ALPHA_META,
                )?;
            }
            // q_var → ISV Q_VAR_PER_BRANCH_BASE + b
            let scratch_var = (SCRATCH_Q_STATS + b * 4 + 2) as i32;
            let isv_var     = (Q_VAR_PER_BRANCH_BASE + b) as i32;
            let wiener_var  = base_wiener_offset + (isv_var - SP5_SLOT_BASE as i32) * 3;
            unsafe {
                launch_apply_pearls(
                    &self.stream,
                    &self.apply_pearls_ad_kernel,
                    scratch_dev, scratch_var,
                    isv_dev,     isv_var,
                    wiener_dev,  wiener_var,
                    1,
                    crate::cuda_pipeline::sp4_wiener_ema::ALPHA_META,
                )?;
            }
        }

        Ok(())
    }

    /// SP5 Task A2 (2026-05-01): Pearl 3 — per-branch NoisyNet sigma from
    /// per-branch v_half + entropy.
    ///
    /// Chained AFTER `launch_sp5_pearl_1_atom` (reads ATOM_V_HALF written by
    /// Pearl 1). Two-step sequence:
    ///   1. `pearl_3_sigma_update` (1×4 kernel) → scratch[103..111).
    ///   2. `apply_pearls_ad_kernel` × 8 (one per output slot):
    ///      sigma[4] → ISV[NOISY_SIGMA_BASE=210..214)
    ///      SF[4]    → ISV[SIGMA_FRACTION_BASE=214..218)
    ///
    /// Wiener offsets: (SP4_PRODUCER_COUNT=71 + (isv_slot − SP5_SLOT_BASE=174)) × 3.
    /// Layer A only — NoisyLinear consumer migration deferred to Layer B.
    pub(crate) fn launch_sp5_pearl_3_sigma(&self) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp4_wiener_ema::launch_apply_pearls;
        use crate::cuda_pipeline::sp5_isv_slots::{
            SP5_SLOT_BASE,
            ATOM_V_HALF_BASE, BRANCH_ENTROPY_BASE,
            NOISY_SIGMA_BASE, SIGMA_FRACTION_BASE,
        };

        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_sp5_pearl_3_sigma: isv_signals_dev_ptr must be allocated by constructor");

        let isv_dev     = self.isv_signals_dev_ptr;
        let scratch_dev = self.producer_step_scratch_buf.dev_ptr;
        let wiener_dev  = self.wiener_state_buf.dev_ptr;
        let action_counts_dev = self.action_counts_buf.dev_ptr;

        // Step 1: pearl_3_sigma_update → scratch[103..107) sigma + scratch[107..111) SF.
        // Kernel signature: (isv_signals, v_half_isv_base, branch_entropy_isv_base,
        //                    sigma_fraction_isv_base, action_counts,
        //                    scratch_out, sigma_idx_base, sigma_fraction_idx_base)
        let v_half_base_i32           = ATOM_V_HALF_BASE as i32;         // 178
        let branch_entropy_base_i32   = BRANCH_ENTROPY_BASE as i32;      // 218
        let sigma_fraction_base_i32   = SIGMA_FRACTION_BASE as i32;      // 214
        let sigma_idx_i32             = SCRATCH_PEARL_3_SIGMA as i32;    // 103
        let sf_idx_i32                = SCRATCH_PEARL_3_SF as i32;       // 107

        unsafe {
            self.stream
                .launch_builder(&self.pearl_3_sigma_kernel)
                .arg(&isv_dev)
                .arg(&v_half_base_i32)
                .arg(&branch_entropy_base_i32)
                .arg(&sigma_fraction_base_i32)
                .arg(&action_counts_dev)
                .arg(&scratch_dev)
                .arg(&sigma_idx_i32)
                .arg(&sf_idx_i32)
                .launch(LaunchConfig {
                    grid_dim:         (1, 1, 1),
                    block_dim:        (4, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("pearl_3_sigma_update: {e}")))?;
        }

        // Step 2: apply_pearls_ad_kernel × 8 — one single-slot launch per ISV output.
        // Wiener offset: (SP4_PRODUCER_COUNT + (isv_slot - SP5_SLOT_BASE)) * 3.
        //
        // sigma[4]  → scratch [103..107), ISV [210..214)  (NOISY_SIGMA_BASE)
        // SF[4]     → scratch [107..111), ISV [214..218)  (SIGMA_FRACTION_BASE)
        let base_wiener_offset = SP4_PRODUCER_COUNT as i32 * 3; // 213

        for b in 0..4_usize {
            // sigma → ISV NOISY_SIGMA_BASE + b
            let scratch_sigma = (SCRATCH_PEARL_3_SIGMA + b) as i32;
            let isv_sigma     = (NOISY_SIGMA_BASE + b) as i32;
            let wiener_sigma  = base_wiener_offset + (isv_sigma - SP5_SLOT_BASE as i32) * 3;
            unsafe {
                launch_apply_pearls(
                    &self.stream,
                    &self.apply_pearls_ad_kernel,
                    scratch_dev, scratch_sigma,
                    isv_dev,     isv_sigma,
                    wiener_dev,  wiener_sigma,
                    1,
                    crate::cuda_pipeline::sp4_wiener_ema::ALPHA_META,
                )?;
            }
            // SF → ISV SIGMA_FRACTION_BASE + b
            let scratch_sf = (SCRATCH_PEARL_3_SF + b) as i32;
            let isv_sf     = (SIGMA_FRACTION_BASE + b) as i32;
            let wiener_sf  = base_wiener_offset + (isv_sf - SP5_SLOT_BASE as i32) * 3;
            unsafe {
                launch_apply_pearls(
                    &self.stream,
                    &self.apply_pearls_ad_kernel,
                    scratch_dev, scratch_sf,
                    isv_dev,     isv_sf,
                    wiener_dev,  wiener_sf,
                    1,
                    crate::cuda_pipeline::sp4_wiener_ema::ALPHA_META,
                )?;
            }
        }

        Ok(())
    }

    /// SP5 Task A3 + SP7 (2026-05-03): Pearl 2 — IQN budget + flatness producer.
    ///
    /// Chained AFTER `launch_sp5_pearl_1_atom` (reads Q_VAR_PER_BRANCH written by
    /// Pearl 1's q_branch_stats) AND AFTER `launch_sp5_pearl_3_sigma` (reads
    /// NOISY_SIGMA written by Pearl 3). Two-step sequence:
    ///   1. `pearl_2_budget_update` (1×4 kernel, 6-arg signature) →
    ///      scratch[115..119) (IQN) + scratch[127..131) (flatness).
    ///   2. `apply_pearls_ad_kernel` × 8 (one per output slot):
    ///      budget_iqn[4] → ISV[BUDGET_IQN_BASE=194..198)
    ///      flatness[4]   → ISV[FLATNESS_BASE=206..210)
    ///
    /// SP7: C51/CQL/ENS budget smoothing removed — owned by
    /// `launch_loss_balance_controller`. Wiener offsets unchanged:
    /// (SP4_PRODUCER_COUNT=71 + (isv_slot − SP5_SLOT_BASE=174)) × 3.
    pub(crate) fn launch_sp5_pearl_2_budget(&self) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp4_wiener_ema::launch_apply_pearls;
        use crate::cuda_pipeline::sp5_isv_slots::{
            SP5_SLOT_BASE,
            BUDGET_IQN_BASE, FLATNESS_BASE,
            NOISY_SIGMA_BASE, Q_VAR_PER_BRANCH_BASE,
        };

        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_sp5_pearl_2_budget: isv_signals_dev_ptr must be allocated by constructor");

        let isv_dev     = self.isv_signals_dev_ptr;
        let scratch_dev = self.producer_step_scratch_buf.dev_ptr;
        let wiener_dev  = self.wiener_state_buf.dev_ptr;

        // SP7 (2026-05-03): kernel signature reduced to (isv, q_var, sigma,
        // scratch, iqn_idx, flatness_idx). The c51/cql/ens scratch slots
        // and apply_pearls smoothing for those slots are owned by the SP7
        // loss-balance controller; Pearl 2 only writes IQN + flatness now.
        let q_var_base_i32        = Q_VAR_PER_BRANCH_BASE as i32;   // 222
        let sigma_base_i32        = NOISY_SIGMA_BASE as i32;         // 210
        let iqn_base_i32          = SCRATCH_PEARL_2_IQN as i32;
        let flatness_base_i32     = SCRATCH_PEARL_2_FLATNESS as i32;

        unsafe {
            self.stream
                .launch_builder(&self.pearl_2_budget_kernel)
                .arg(&isv_dev)
                .arg(&q_var_base_i32)
                .arg(&sigma_base_i32)
                .arg(&scratch_dev)
                .arg(&iqn_base_i32)
                .arg(&flatness_base_i32)
                .launch(LaunchConfig {
                    grid_dim:         (1, 1, 1),
                    block_dim:        (4, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("pearl_2_budget_update: {e}")))?;
        }

        // SP7: smoothing chain reduced from 5 slot-blocks to 2 (IQN +
        // flatness). The SP7 controller owns the BUDGET_{C51,CQL,ENS}_BASE
        // smoothing chain.
        let base_wiener_offset = SP4_PRODUCER_COUNT as i32 * 3;     // 213

        for (isv_base, scratch_base) in [
            (BUDGET_IQN_BASE,  SCRATCH_PEARL_2_IQN),
            (FLATNESS_BASE,    SCRATCH_PEARL_2_FLATNESS),
        ] {
            for b in 0..4_usize {
                let scratch_idx = (scratch_base + b) as i32;
                let isv_idx     = (isv_base + b) as i32;
                let wiener_off  = base_wiener_offset + (isv_idx - SP5_SLOT_BASE as i32) * 3;
                unsafe {
                    launch_apply_pearls(
                        &self.stream,
                        &self.apply_pearls_ad_kernel,
                        scratch_dev, scratch_idx,
                        isv_dev,     isv_idx,
                        wiener_dev,  wiener_off,
                        1,
                        crate::cuda_pipeline::sp4_wiener_ema::ALPHA_META,
                    )?;
                }
            }
        }

        Ok(())
    }

    /// SP7 Task 5 (2026-05-03): Launch the loss-balance controller producer +
    /// apply_pearls_ad_kernel chain. Must run AFTER:
    ///   * `launch_sp5_pearl_2_budget`        (FLATNESS_BASE populated)
    ///   * `grad_decomp_launch_iqn`            (pinned slot at element offset 0)
    ///   * `grad_decomp_launch_cql_sx`         (pinned slot at element offset 6)
    ///   * `grad_decomp_launch_c51`            (pinned slot at element offset 9)
    ///
    /// Writes: scratch[SCRATCH_LB_*..]; `apply_pearls_ad_kernel` then smooths
    /// these into ISV[BUDGET_{CQL,C51}_BASE] and ISV[LB_*_VAR_{CQL,C51}_BASE].
    pub(crate) fn launch_loss_balance_controller(&self) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp4_wiener_ema::launch_apply_pearls;
        use crate::cuda_pipeline::sp5_isv_slots::{
            SP5_SLOT_BASE,
            BUDGET_CQL_BASE, BUDGET_C51_BASE, FLATNESS_BASE,
            LB_DIFF_VAR_CQL_BASE, LB_SAMPLE_VAR_CQL_BASE,
            LB_DIFF_VAR_C51_BASE, LB_SAMPLE_VAR_C51_BASE,
            LB_CQL_ACTIVE_BASE, LB_C51_ACTIVE_BASE,
            LB_MAX_BUDGET_CQL_BASE, LB_MAX_BUDGET_C51_BASE,
        };

        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_loss_balance_controller: isv_signals_dev_ptr must be allocated");
        debug_assert!(self.grad_decomp_result_dev_ptr != 0,
            "launch_loss_balance_controller: grad_decomp_result_dev_ptr must be allocated");

        let isv_dev     = self.isv_signals_dev_ptr;
        let scratch_dev = self.producer_step_scratch_buf.dev_ptr;
        let wiener_dev  = self.wiener_state_buf.dev_ptr;

        // grad_decomp pinned layout: 27 floats total, 9 components × 3 floats
        // each ([mag, dir, trunk]). Component element offsets per
        // launch_grad_decomp documentation: iqn=0, cql=3, cql_sx=6, c51=9.
        //
        // SP7 Path A (2026-05-03): the CQL reference now reads slot 3 (`cql`)
        // populated by `launch_cql_raw_norm` — a real producer that reads
        // `cql_grad_scratch` directly. Previously SP7 read slot 6 (`cql_sx`,
        // post-budget delta = budget × raw_grad), which created a self-
        // perpetuating deadlock at small bootstrap budgets: the controller
        // saw ≈0 norm → cold-started every step → never updated the budget.
        // Slot 3 is budget-independent — exactly the reference SP7 needs.
        // The historical "cql at offset 3" was zero-valued (`grad_decomp_launch_cql`
        // measured a delta on `grad_buf`, but `apply_cql_gradient` writes
        // to `cql_grad_scratch`); the unwired path is now fully replaced.
        let f32_size = std::mem::size_of::<f32>() as u64;
        let iqn_dev     = self.grad_decomp_result_dev_ptr + 0 * f32_size;
        let cql_dev     = self.grad_decomp_result_dev_ptr + 3 * f32_size;
        let c51_dev     = self.grad_decomp_result_dev_ptr + 9 * f32_size;

        // Step 1: producer kernel.
        let flatness_isv_base_i32           = FLATNESS_BASE as i32;
        let budget_cql_isv_base_i32         = BUDGET_CQL_BASE as i32;
        let budget_c51_isv_base_i32         = BUDGET_C51_BASE as i32;
        let diff_var_cql_isv_base_i32       = LB_DIFF_VAR_CQL_BASE as i32;
        let sample_var_cql_isv_base_i32     = LB_SAMPLE_VAR_CQL_BASE as i32;
        let diff_var_c51_isv_base_i32       = LB_DIFF_VAR_C51_BASE as i32;
        let sample_var_c51_isv_base_i32     = LB_SAMPLE_VAR_C51_BASE as i32;
        let active_cql_isv_base_i32         = LB_CQL_ACTIVE_BASE as i32;
        let active_c51_isv_base_i32         = LB_C51_ACTIVE_BASE as i32;
        // SP8 (Fix 36): per-(head, branch) MAX_BUDGET ISV bases. The cap
        // is signal-driven via `loss_balance_max_budget_compute_kernel`
        // from `train_active_frac` (Pearls A+D smoothed). Replaces the
        // prior hardcoded `MAX_BUDGET=1.0f` constant in the controller
        // kernel per `pearl_controller_anchors_isv_driven.md`.
        let max_budget_cql_isv_base_i32     = LB_MAX_BUDGET_CQL_BASE as i32;
        let max_budget_c51_isv_base_i32     = LB_MAX_BUDGET_C51_BASE as i32;
        let epoch_idx_isv_index_i32         = EPOCH_IDX_INDEX as i32;
        let sb_budget_cql_i32               = SCRATCH_LB_BUDGET_CQL as i32;
        let sb_budget_c51_i32               = SCRATCH_LB_BUDGET_C51 as i32;
        let sb_diff_var_cql_i32             = SCRATCH_LB_DIFF_VAR_CQL as i32;
        let sb_sample_var_cql_i32           = SCRATCH_LB_SAMPLE_VAR_CQL as i32;
        let sb_diff_var_c51_i32             = SCRATCH_LB_DIFF_VAR_C51 as i32;
        let sb_sample_var_c51_i32           = SCRATCH_LB_SAMPLE_VAR_C51 as i32;
        let sb_active_cql_i32               = SCRATCH_LB_ACTIVE_CQL as i32;
        let sb_active_c51_i32               = SCRATCH_LB_ACTIVE_C51 as i32;

        unsafe {
            self.stream
                .launch_builder(&self.loss_balance_controller_kernel)
                .arg(&iqn_dev)
                .arg(&cql_dev)
                .arg(&c51_dev)
                .arg(&isv_dev)
                .arg(&flatness_isv_base_i32)
                .arg(&budget_cql_isv_base_i32)
                .arg(&budget_c51_isv_base_i32)
                .arg(&diff_var_cql_isv_base_i32)
                .arg(&sample_var_cql_isv_base_i32)
                .arg(&diff_var_c51_isv_base_i32)
                .arg(&sample_var_c51_isv_base_i32)
                .arg(&active_cql_isv_base_i32)
                .arg(&active_c51_isv_base_i32)
                // SP8 (Fix 36): per-(head, branch) MAX_BUDGET ISV bases.
                .arg(&max_budget_cql_isv_base_i32)
                .arg(&max_budget_c51_isv_base_i32)
                .arg(&epoch_idx_isv_index_i32)
                .arg(&scratch_dev)
                .arg(&sb_budget_cql_i32)
                .arg(&sb_budget_c51_i32)
                .arg(&sb_diff_var_cql_i32)
                .arg(&sb_sample_var_cql_i32)
                .arg(&sb_diff_var_c51_i32)
                .arg(&sb_sample_var_c51_i32)
                .arg(&sb_active_cql_i32)
                .arg(&sb_active_c51_i32)
                .launch(LaunchConfig {
                    grid_dim:         (1, 1, 1),
                    block_dim:        (8, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("loss_balance_controller_update: {e}")))?;
        }

        // Step 2: apply_pearls_ad_kernel × 32 — one per ISV output slot
        // (8 slot-blocks × 4 branches). Wiener offset:
        // (SP4_PRODUCER_COUNT + (isv_slot - SP5_SLOT_BASE)) * 3.
        let base_wiener_offset = SP4_PRODUCER_COUNT as i32 * 3; // 213

        for (isv_base, scratch_base) in [
            (BUDGET_CQL_BASE,        SCRATCH_LB_BUDGET_CQL),
            (BUDGET_C51_BASE,        SCRATCH_LB_BUDGET_C51),
            (LB_DIFF_VAR_CQL_BASE,   SCRATCH_LB_DIFF_VAR_CQL),
            (LB_SAMPLE_VAR_CQL_BASE, SCRATCH_LB_SAMPLE_VAR_CQL),
            (LB_DIFF_VAR_C51_BASE,   SCRATCH_LB_DIFF_VAR_C51),
            (LB_SAMPLE_VAR_C51_BASE, SCRATCH_LB_SAMPLE_VAR_C51),
            (LB_CQL_ACTIVE_BASE,     SCRATCH_LB_ACTIVE_CQL),
            (LB_C51_ACTIVE_BASE,     SCRATCH_LB_ACTIVE_C51),
        ] {
            for b in 0..4_usize {
                let scratch_idx = (scratch_base + b) as i32;
                let isv_idx     = (isv_base + b) as i32;
                let wiener_off  = base_wiener_offset + (isv_idx - SP5_SLOT_BASE as i32) * 3;
                unsafe {
                    launch_apply_pearls(
                        &self.stream,
                        &self.apply_pearls_ad_kernel,
                        scratch_dev, scratch_idx,
                        isv_dev,     isv_idx,
                        wiener_dev,  wiener_off,
                        1,
                        crate::cuda_pipeline::sp4_wiener_ema::ALPHA_META,
                    )?;
                }
            }
        }

        Ok(())
    }

    /// SP14 Layer C Phase C.4b (2026-05-08): launch the aux prediction
    /// horizon producer chain at per-epoch boundary.
    ///
    /// Two sequential single-launch kernels:
    ///   1. `avg_win_hold_time_update` — sweeps per-sample
    ///      `hold_at_exit_per_sample` + `trade_profitable_per_sample`,
    ///      block-tree-reduces winning-trade hold-time mean, EMA-blends
    ///      into `ISV[AVG_WIN_HOLD_TIME_BARS_INDEX=451]` (Pearl-A
    ///      bootstrap then α=0.05).
    ///   2. `aux_horizon_update` — single-thread Pearl-A bootstrap +
    ///      slow Wiener-α (α=0.01 fixed) drives
    ///      `ISV[AUX_PRED_HORIZON_BARS_INDEX=450]` from the slot above.
    ///
    /// Per-epoch boundary launch — both signals are slow-moving and
    /// per-step launches would track sample noise. Caller passes the
    /// per-sample buffer pointers + length from the collector's
    /// `hold_at_exit_per_sample` / `trade_profitable_per_sample` slabs.
    ///
    /// Pearls applied:
    ///   - `pearl_first_observation_bootstrap.md`
    ///   - `pearl_wiener_optimal_adaptive_alpha.md`
    ///   - `pearl_no_host_branches_in_captured_graph.md`
    ///   - `feedback_no_atomicadd.md` (block-tree-reduce in shmem only).
    pub(crate) fn launch_aux_horizon_chain(
        &self,
        hold_at_exit_dev_ptr: u64,
        trade_profitable_dev_ptr: u64,
        total_samples: i32,
    ) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp14_isv_slots::{
            AUX_PRED_HORIZON_BARS_INDEX, AUX_PRED_HORIZON_BARS_MAX,
            AUX_PRED_HORIZON_BARS_MIN, AVG_WIN_HOLD_TIME_BARS_INDEX,
            SENTINEL_AUX_PRED_HORIZON_BARS, SENTINEL_AVG_WIN_HOLD_TIME_BARS,
        };

        debug_assert!(
            self.isv_signals_dev_ptr != 0,
            "launch_aux_horizon_chain: isv_signals_dev_ptr must be allocated"
        );

        // Step 1: avg winning hold time EMA producer.
        self.avg_win_hold_time_update_ops.launch(
            &self.stream,
            hold_at_exit_dev_ptr,
            trade_profitable_dev_ptr,
            total_samples,
            self.isv_signals_dev_ptr,
            AVG_WIN_HOLD_TIME_BARS_INDEX as i32,
            SENTINEL_AVG_WIN_HOLD_TIME_BARS,
            super::gpu_aux_trunk::AvgWinHoldTimeUpdateOps::ALPHA,
        )?;

        // Step 2: horizon producer (reads slot 451 from step 1).
        // No target-variance EMA in C.4b → pass -1 for var idx (kernel
        // falls back to fixed α=0.01).
        self.aux_horizon_update_ops.launch(
            &self.stream,
            self.isv_signals_dev_ptr,
            AUX_PRED_HORIZON_BARS_INDEX as i32,
            AVG_WIN_HOLD_TIME_BARS_INDEX as i32,
            -1,
            AUX_PRED_HORIZON_BARS_MIN,
            AUX_PRED_HORIZON_BARS_MAX,
            SENTINEL_AUX_PRED_HORIZON_BARS,
        )?;

        Ok(())
    }

    /// Class A P0-A (2026-05-08): launch the adaptive reward-cap producer
    /// at per-epoch boundary.
    ///
    /// Sweeps the per-epoch `step_ret_per_sample` + `trade_close_per_sample`
    /// buffers (populated by `unified_env_step_core` in
    /// `experience_kernels.cu`), computes a Welford `mean + Z_99 × sigma`
    /// p99 estimator over winning realized returns plus a
    /// `max(winning_returns)` conservative takeover, applies a 1.5×
    /// safety factor, clamps to dimensional bounds [1, 50], and writes
    /// both `ISV[REWARD_POS_CAP_ADAPTIVE_INDEX=452]` (POS cap) and
    /// `ISV[REWARD_NEG_CAP_ADAPTIVE_INDEX=453]` (NEG = −2 × POS,
    /// preserving Kahneman/Tversky 2:1 loss-aversion asymmetry per
    /// `pearl_audit_unboundedness_for_implicit_asymmetry`).
    ///
    /// Per-epoch boundary launch — reward distribution is the foundation
    /// of training and shouldn't move fast; per-step would track sample
    /// noise.
    ///
    /// Pearls applied:
    ///   - `pearl_first_observation_bootstrap.md` (sentinel = 5.0/-10.0)
    ///   - `pearl_no_host_branches_in_captured_graph.md`
    ///   - `feedback_no_atomicadd.md` (block-tree-reduce in shmem only)
    ///   - `pearl_symmetric_clamp_audit.md` (bilateral clamp on POS, NEG)
    ///   - `feedback_isv_for_adaptive_bounds.md` (POS bounds [1, 50] are
    ///     dimensional safety floors, not tuning)
    pub(crate) fn launch_reward_cap_update(
        &self,
        step_ret_dev_ptr: u64,
        trade_close_dev_ptr: u64,
        total_samples: i32,
    ) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp14_isv_slots::{
            REWARD_CAP_EMA_ALPHA, REWARD_CAP_SAFETY_FACTOR, REWARD_NEG_CAP_ADAPTIVE_INDEX,
            REWARD_NEG_TO_POS_RATIO, REWARD_POS_CAP_ADAPTIVE_INDEX, REWARD_POS_CAP_MAX,
            REWARD_POS_CAP_MIN, SENTINEL_REWARD_NEG_CAP, SENTINEL_REWARD_POS_CAP,
        };

        debug_assert!(
            self.isv_signals_dev_ptr != 0,
            "launch_reward_cap_update: isv_signals_dev_ptr must be allocated"
        );

        self.reward_cap_update_ops.launch(
            &self.stream,
            step_ret_dev_ptr,
            trade_close_dev_ptr,
            total_samples,
            self.isv_signals_dev_ptr,
            REWARD_POS_CAP_ADAPTIVE_INDEX as i32,
            REWARD_NEG_CAP_ADAPTIVE_INDEX as i32,
            SENTINEL_REWARD_POS_CAP,
            SENTINEL_REWARD_NEG_CAP,
            REWARD_CAP_SAFETY_FACTOR,
            REWARD_NEG_TO_POS_RATIO,
            REWARD_POS_CAP_MIN,
            REWARD_POS_CAP_MAX,
            REWARD_CAP_EMA_ALPHA,
        )?;

        Ok(())
    }

    /// SP18 v2 Phase 3 (2026-05-09): launch the adaptive HOLD_REWARD_POS/
    /// NEG_CAP producer. Sweeps the per-epoch `step_ret_per_sample` +
    /// `trade_close_per_sample` buffers (same source `launch_reward_cap_update`
    /// reads from), filters by `(is_close && step_ret != 0)` to extract
    /// Long/Short close magnitudes (per spec DD3=b — Hold/Flat closes
    /// don't generate non-zero step_ret per `experience_kernels.cu:2855`),
    /// computes a Welford `mean + Z_99 × sigma` p99 estimator ×
    /// `HOLD_REWARD_CAP_SAFETY_FACTOR=1.5`, applies Wiener-optimal α
    /// blend (Welford accumulators in slots [487..493); floor at
    /// `WELFORD_ALPHA_MIN=0.4` per
    /// `pearl_wiener_alpha_floor_for_nonstationary`), and writes both
    /// `ISV[HOLD_REWARD_POS_CAP_INDEX=483]` (POS cap) and
    /// `ISV[HOLD_REWARD_NEG_CAP_INDEX=484]` (NEG = −2 × POS, DD5=b
    /// mirrored asymmetry — same Kahneman 2:1 ratio as the position-
    /// side `launch_reward_cap_update`, single source of truth at
    /// producer time per
    /// `pearl_audit_unboundedness_for_implicit_asymmetry`).
    ///
    /// Per-epoch boundary launch — call AFTER `launch_reward_cap_update`
    /// so both cap producers share the same source buffers (no shared
    /// state; they drive independent ISV slot pairs at [452]/[453]
    /// and [483]/[484]).
    ///
    /// Pearls applied:
    ///   - `pearl_first_observation_bootstrap.md` (sentinel POS=5.0/
    ///     NEG=-10.0 match SP14 P0-A pattern for bit-identical cold-start)
    ///   - `pearl_no_host_branches_in_captured_graph.md`
    ///   - `feedback_no_atomicadd.md` (block-tree-reduce in shmem only)
    ///   - `pearl_symmetric_clamp_audit.md` (bilateral clamp on POS, NEG)
    ///   - `feedback_isv_for_adaptive_bounds.md` (POS bounds [0.5, 50.0]
    ///     are dimensional safety floors, not tuning)
    ///   - `pearl_wiener_optimal_adaptive_alpha.md` (Welford-derived α)
    ///   - `pearl_wiener_alpha_floor_for_nonstationary.md` (α ≥ 0.4)
    ///   - `pearl_fused_per_group_statistics_oracle.md` (sum/sumsq/count/
    ///     max accumulated in ONE fused reduction)
    pub(crate) fn launch_hold_reward_cap_update(
        &self,
        step_ret_dev_ptr: u64,
        trade_close_dev_ptr: u64,
        total_samples: i32,
    ) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp14_isv_slots::{
            HOLD_REWARD_CAP_SAFETY_FACTOR, HOLD_REWARD_NEG_CAP_INDEX,
            HOLD_REWARD_POS_CAP_INDEX, HOLD_REWARD_POS_CAP_MAX_BOUND,
            HOLD_REWARD_POS_CAP_MIN_BOUND, HRC_DIFF_M2_INDEX, HRC_DIFF_MEAN_INDEX,
            HRC_PREV_TARGET_INDEX, HRC_SAMPLE_COUNT_INDEX, HRC_TARGET_M2_INDEX,
            HRC_TARGET_MEAN_INDEX, REWARD_NEG_TO_POS_RATIO, SENTINEL_HOLD_REWARD_NEG_CAP,
            SENTINEL_HOLD_REWARD_POS_CAP,
        };

        debug_assert!(
            self.isv_signals_dev_ptr != 0,
            "launch_hold_reward_cap_update: isv_signals_dev_ptr must be allocated"
        );

        self.hold_reward_cap_update_ops.launch(
            &self.stream,
            step_ret_dev_ptr,
            trade_close_dev_ptr,
            total_samples,
            self.isv_signals_dev_ptr,
            HOLD_REWARD_POS_CAP_INDEX as i32,
            HOLD_REWARD_NEG_CAP_INDEX as i32,
            HRC_TARGET_MEAN_INDEX as i32,
            HRC_TARGET_M2_INDEX as i32,
            HRC_DIFF_MEAN_INDEX as i32,
            HRC_DIFF_M2_INDEX as i32,
            HRC_PREV_TARGET_INDEX as i32,
            HRC_SAMPLE_COUNT_INDEX as i32,
            SENTINEL_HOLD_REWARD_POS_CAP,
            SENTINEL_HOLD_REWARD_NEG_CAP,
            HOLD_REWARD_CAP_SAFETY_FACTOR,
            REWARD_NEG_TO_POS_RATIO,
            HOLD_REWARD_POS_CAP_MIN_BOUND,
            HOLD_REWARD_POS_CAP_MAX_BOUND,
        )?;

        Ok(())
    }

    /// Class A P1-Producer (2026-05-08): launch the adaptive Bayesian Kelly
    /// priors producer. Per-fold-end (per-epoch boundary in current
    /// scheduling) launch — Bayesian prior is a *prior belief* and should
    /// change slowly across folds (α=0.005 EMA).
    ///
    /// Aggregates realized PS_KELLY_{WIN_COUNT, LOSS_COUNT, SUM_WINS,
    /// SUM_LOSSES} across envs from the same `portfolio_state` buffer
    /// `kelly_cap_update_kernel` reads from at the same boundary and
    /// slow-EMA-blends into ISV[454..458). MUST be called RIGHT BEFORE
    /// `launch_kelly_cap_update` so that kernel sees the freshly-blended
    /// priors.
    ///
    /// Pearls applied:
    ///   - `pearl_first_observation_bootstrap.md` (sentinels 2.0/2.0/0.01/0.01)
    ///   - `pearl_no_host_branches_in_captured_graph.md`
    ///   - `feedback_no_atomicadd.md` (block-tree-reduce in shmem only)
    ///   - `pearl_symmetric_clamp_audit.md` (bilateral clamp)
    ///   - `feedback_isv_for_adaptive_bounds.md` (count [0.5, 100], sum
    ///     [0.001, 1.0] are dimensional safety floors, not tuning)
    ///   - `pearl_controller_anchors_isv_driven.md` (priors are
    ///     controller anchors)
    pub(crate) fn launch_kelly_bayesian_priors_update(
        &self,
        portfolio_state_dev_ptr: u64,
        n_envs: i32,
    ) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp14_isv_slots::{
            KELLY_PRIOR_COUNT_MAX, KELLY_PRIOR_COUNT_MIN, KELLY_PRIOR_EMA_ALPHA,
            KELLY_PRIOR_LOSSES_INDEX, KELLY_PRIOR_SUM_LOSSES_INDEX, KELLY_PRIOR_SUM_MAX,
            KELLY_PRIOR_SUM_MIN, KELLY_PRIOR_SUM_WINS_INDEX, KELLY_PRIOR_WINS_INDEX,
            SENTINEL_KELLY_PRIOR_LOSSES, SENTINEL_KELLY_PRIOR_SUM_LOSSES,
            SENTINEL_KELLY_PRIOR_SUM_WINS, SENTINEL_KELLY_PRIOR_WINS,
        };
        // PS_STRIDE from state_layout.cuh — matches the value used by
        // `launch_kelly_cap_update`. Grown 41→43 by Plan 3 D.4c.
        const PS_STRIDE: i32 = 43;

        debug_assert!(
            self.isv_signals_dev_ptr != 0,
            "launch_kelly_bayesian_priors_update: isv_signals_dev_ptr must be allocated"
        );

        self.kelly_bayesian_priors_update_ops.launch(
            &self.stream,
            portfolio_state_dev_ptr,
            n_envs,
            PS_STRIDE,
            self.isv_signals_dev_ptr,
            KELLY_PRIOR_WINS_INDEX as i32,
            KELLY_PRIOR_LOSSES_INDEX as i32,
            KELLY_PRIOR_SUM_WINS_INDEX as i32,
            KELLY_PRIOR_SUM_LOSSES_INDEX as i32,
            SENTINEL_KELLY_PRIOR_WINS,
            SENTINEL_KELLY_PRIOR_LOSSES,
            SENTINEL_KELLY_PRIOR_SUM_WINS,
            SENTINEL_KELLY_PRIOR_SUM_LOSSES,
            KELLY_PRIOR_COUNT_MIN,
            KELLY_PRIOR_COUNT_MAX,
            KELLY_PRIOR_SUM_MIN,
            KELLY_PRIOR_SUM_MAX,
            KELLY_PRIOR_EMA_ALPHA,
        )?;

        Ok(())
    }

    /// Class A audit-fix Batch 4-A (2026-05-08): launch the adaptive DD
    /// saturation floor producer. Per-epoch boundary launch — DD
    /// distribution is a slow-moving fold-volatility property and
    /// shouldn't track per-epoch noise (α=0.01 EMA mirrors P0-A
    /// REWARD_POS_CAP cadence).
    ///
    /// Aggregates per-env DD_MAX (offset 1 in the 6-wide
    /// `dd_state_per_env` tile populated by `dd_state_kernel`) across
    /// envs, computes Welford `mean + Z_75 × sigma` p75 estimator with
    /// `max(p75, mean)` robustness guard × 1.5 safety factor, and
    /// slow-EMA blends into ISV[458].
    ///
    /// Pearls applied:
    ///   - `pearl_first_observation_bootstrap.md` (sentinel = 0.25)
    ///   - `pearl_no_host_branches_in_captured_graph.md`
    ///   - `feedback_no_atomicadd.md` (block-tree-reduce in shmem only)
    ///   - `pearl_symmetric_clamp_audit.md` (bilateral clamp on target)
    ///   - `feedback_isv_for_adaptive_bounds.md` (bounds [0.10, 0.50]
    ///     are dimensional safety floors, not tuning)
    pub(crate) fn launch_dd_saturation_floor_update(
        &self,
        dd_state_per_env_ptr: u64,
        n_envs: i32,
    ) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp14_isv_slots::{
            DD_SATURATION_FLOOR_ADAPTIVE_INDEX, DD_SATURATION_FLOOR_EMA_ALPHA,
            DD_SATURATION_FLOOR_MAX, DD_SATURATION_FLOOR_MIN,
            DD_SATURATION_FLOOR_SAFETY_FACTOR, SENTINEL_DD_SATURATION_FLOOR,
        };
        // Per-env tile is 6 floats wide (mirrors dd_state_kernel layout).
        // DD_MAX sits at offset 1 (DD_CURRENT=0, DD_MAX=1, DD_RECOVERY_BARS=2,
        // DD_PERSISTENCE=3, CALMAR=4, DD_PCT=5).
        const DD_STATE_STRIDE: i32 = 6;
        const DD_MAX_OFF: i32 = 1;

        debug_assert!(
            self.isv_signals_dev_ptr != 0,
            "launch_dd_saturation_floor_update: isv_signals_dev_ptr must be allocated"
        );

        self.dd_saturation_floor_update_ops.launch(
            &self.stream,
            dd_state_per_env_ptr,
            n_envs,
            DD_STATE_STRIDE,
            DD_MAX_OFF,
            self.isv_signals_dev_ptr,
            DD_SATURATION_FLOOR_ADAPTIVE_INDEX as i32,
            SENTINEL_DD_SATURATION_FLOOR,
            DD_SATURATION_FLOOR_SAFETY_FACTOR,
            DD_SATURATION_FLOOR_MIN,
            DD_SATURATION_FLOOR_MAX,
            DD_SATURATION_FLOOR_EMA_ALPHA,
        )?;

        Ok(())
    }

    /// Class A audit-fix Batch 4-B (2026-05-08, Item 4): launch adaptive
    /// MIN_HOLD_TEMPERATURE producer.
    ///
    /// Reads `ISV[AUX_DIR_ACC_SHORT_EMA_INDEX=373]` (sp13 fast EMA of
    /// binary directional accuracy), maps it to a [5, 50] temperature
    /// via `temp = TEMP_MIN + (TEMP_MAX − TEMP_MIN) × clamp((short_ema
    /// − 0.5)/0.5, 0, 1)`, and slow-EMA-blends into
    /// `ISV[MIN_HOLD_TEMPERATURE_ADAPTIVE_INDEX=460]`. Replaces the
    /// deleted epoch-driven anneal schedule
    /// `T_end + (T_start − T_end) × exp(-epoch / decay)`.
    ///
    /// Pearls applied:
    ///   - `pearl_first_observation_bootstrap.md` (sentinel = 50.0,
    ///     matches the deleted MIN_HOLD_TEMPERATURE_START=50)
    ///   - `pearl_no_host_branches_in_captured_graph.md`
    ///   - `pearl_symmetric_clamp_audit.md` (bilateral clamp on
    ///     target_temp + post-blend)
    ///   - `feedback_isv_for_adaptive_bounds.md` (bounds [5, 50] are
    ///     dimensional safety floors, not tuning — matches the
    ///     original schedule range)
    ///   - `feedback_no_legacy_aliases.md` (the legacy schedule
    ///     constants and helper function are deleted atomically with
    ///     this wiring)
    pub(crate) fn launch_min_hold_temperature_update(
        &self,
    ) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp14_isv_slots::{
            MIN_HOLD_TEMPERATURE_ADAPTIVE_INDEX,
            MIN_HOLD_TEMPERATURE_MAX,
            MIN_HOLD_TEMPERATURE_MIN, SENTINEL_HOLD_RATE_OBSERVED,
            SENTINEL_MIN_HOLD_TEMPERATURE,
            MHT_TARGET_MEAN_INDEX, MHT_TARGET_M2_INDEX,
            MHT_DIFF_MEAN_INDEX, MHT_DIFF_M2_INDEX,
            MHT_PREV_TARGET_INDEX, MHT_SAMPLE_COUNT_INDEX,
        };
        use crate::cuda_pipeline::sp13_isv_slots::{
            HOLD_RATE_OBSERVED_EMA_INDEX, HOLD_RATE_TARGET_INDEX,
        };

        debug_assert!(
            self.isv_signals_dev_ptr != 0,
            "launch_min_hold_temperature_update: isv_signals_dev_ptr must be allocated"
        );

        // SP16 Phase 1 (revised, 2026-05-08): driving signal swapped
        // from AUX_DIR_ACC_SHORT_EMA (slot 373) to hold-rate overrun
        // (slots 382 + 381). Slot 373 stayed at sentinel 0.5 in Fold 1
        // and the kernel's early-return guard pinned slot 460 at 50
        // for the entire fold. Hold rates are measured per-epoch from
        // realised actions and survive fold reset cleanly.
        //
        // SP16 T3 (2026-05-08): hardcoded `α=0.05` removed — α is now
        // Wiener-optimal, derived from Welford accumulators in slots
        // [468..474). Per `pearl_wiener_optimal_adaptive_alpha`.
        self.min_hold_temperature_update_ops.launch(
            &self.stream,
            self.isv_signals_dev_ptr,
            MIN_HOLD_TEMPERATURE_ADAPTIVE_INDEX as i32,
            HOLD_RATE_OBSERVED_EMA_INDEX as i32,
            HOLD_RATE_TARGET_INDEX as i32,
            MHT_TARGET_MEAN_INDEX as i32,
            MHT_TARGET_M2_INDEX as i32,
            MHT_DIFF_MEAN_INDEX as i32,
            MHT_DIFF_M2_INDEX as i32,
            MHT_PREV_TARGET_INDEX as i32,
            MHT_SAMPLE_COUNT_INDEX as i32,
            SENTINEL_MIN_HOLD_TEMPERATURE,
            SENTINEL_HOLD_RATE_OBSERVED,
            MIN_HOLD_TEMPERATURE_MIN,
            MIN_HOLD_TEMPERATURE_MAX,
        )?;

        Ok(())
    }

    /// SP8 (Fix 36, 2026-05-03): launch GPU-only `train_active_frac` canary
    /// producer.
    ///
    /// Reads `monitoring_summary[5..17)` (the 12-bin action_counts written
    /// by `monitoring_reduce` — see `monitoring_kernel.cu` and
    /// `gpu_monitoring.rs`) and computes
    ///   active_frac = sum(bins[0..3] + bins[6..9]) / sum(bins[0..12])
    /// (Long+Short / total — matches the host-side formula at the deleted
    /// site `training_loop.rs:3392-3396`). Writes 1 float to
    /// `scratch[SCRATCH_TRAIN_ACTIVE_FRAC=250]`; chained `apply_pearls_ad_kernel`
    /// smooths via Pearls A+D into `ISV[TRAIN_ACTIVE_FRAC_INDEX=321]`.
    ///
    /// Pre-conditions:
    ///   - The monitoring `reduce()` kernel has launched on the same stream
    ///     (so `monitoring_summary` is populated this epoch).
    ///   - `isv_signals_dev_ptr` is non-null (constructor invariant).
    ///   - `scratch_dev` and `wiener_state_buf.dev_ptr` are non-null.
    ///
    /// Per `feedback_no_cpu_compute_strict.md` and Fix 36: replaces the
    /// CPU computation at the deleted site. Per
    /// `feedback_no_partial_refactor.md`: this launcher and the
    /// `loss_balance_max_budget_compute` launcher land in the same atomic
    /// commit as the consumer change in `loss_balance_controller_kernel.cu`.
    ///
    /// `monitoring_summary_dev_ptr` is the device pointer of the
    /// `summary_buf` `CudaSlice<f32>` owned by `GpuMonitoringReducer`. The
    /// trainer does not own the monitoring reducer — the caller threads
    /// the device pointer through, mirroring the way
    /// `launch_loss_balance_controller` reads from the
    /// `grad_decomp_result_dev_ptr` mapped-pinned buffer that the
    /// in-step backward writes (both pointers cross trainer boundaries
    /// without a HtoD copy per `feedback_no_htod_htoh_only_mapped_pinned`).
    pub(crate) fn launch_train_active_frac_compute(
        &self,
        monitoring_summary_dev_ptr: u64,
    ) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp4_wiener_ema::{launch_apply_pearls, ALPHA_META};
        use crate::cuda_pipeline::sp5_isv_slots::{
            SP5_SLOT_BASE, TRAIN_ACTIVE_FRAC_INDEX,
        };

        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_train_active_frac_compute: isv_signals_dev_ptr must be allocated");
        debug_assert!(monitoring_summary_dev_ptr != 0,
            "launch_train_active_frac_compute: monitoring_summary_dev_ptr must be valid");

        let isv_dev     = self.isv_signals_dev_ptr;
        let scratch_dev = self.producer_step_scratch_buf.dev_ptr;
        let wiener_dev  = self.wiener_state_buf.dev_ptr;

        let scratch_idx_i32: i32 = SCRATCH_TRAIN_ACTIVE_FRAC as i32;

        // Step 1: train_active_frac_compute — single-block, single-thread
        // reduction over 12 floats (cheap; the actual work is the cross-
        // boundary device-pointer arg threading).
        unsafe {
            self.stream
                .launch_builder(&self.train_active_frac_compute_kernel)
                .arg(&monitoring_summary_dev_ptr)
                .arg(&scratch_dev)
                .arg(&scratch_idx_i32)
                .launch(LaunchConfig {
                    grid_dim:         (1, 1, 1),
                    block_dim:        (1, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("train_active_frac_compute: {e}")))?;
        }

        // Step 2: 1 apply_pearls_ad_kernel call. Wiener offset:
        // (SP4_PRODUCER_COUNT + (isv_slot - SP5_SLOT_BASE)) * 3.
        let base_wiener_offset = SP4_PRODUCER_COUNT as i32 * 3;
        let isv_idx    = TRAIN_ACTIVE_FRAC_INDEX as i32;
        let wiener_off = base_wiener_offset + (isv_idx - SP5_SLOT_BASE as i32) * 3;
        unsafe {
            launch_apply_pearls(
                &self.stream,
                &self.apply_pearls_ad_kernel,
                scratch_dev, scratch_idx_i32,
                isv_dev,     isv_idx,
                wiener_dev,  wiener_off,
                1,
                ALPHA_META,
            )?;
        }

        Ok(())
    }

    /// SP8 (Fix 36, 2026-05-03): launch ISV-driven adaptive MAX_BUDGET
    /// producer.
    ///
    /// Reads `ISV[TRAIN_ACTIVE_FRAC_INDEX]` (Pearls A+D smoothed canary)
    /// and computes per-(head, branch) cap via linear interpolation:
    ///   new_cap = FLOOR + active_frac × (CEIL − FLOOR)
    /// where FLOOR=0.01 and CEIL=1.0 are Invariant 1 numerical-stability /
    /// structural-envelope anchors per `feedback_isv_for_adaptive_bounds.md`.
    /// Writes 8 floats (4 per head) to
    /// `scratch[SCRATCH_LB_MAX_BUDGET_CQL=251..255)` and
    /// `scratch[SCRATCH_LB_MAX_BUDGET_C51=255..259)`; chained
    /// `apply_pearls_ad_kernel` (8 launches) smooths into
    /// `ISV[LB_MAX_BUDGET_{CQL,C51}_BASE..+4)` via Pearl A sentinel-bootstrap
    /// + Pearl D Wiener-α steady-state smoothing.
    ///
    /// Replaces the prior hardcoded `MAX_BUDGET=1.0f` constant in
    /// `loss_balance_controller_kernel.cu` per
    /// `pearl_controller_anchors_isv_driven.md`.
    ///
    /// Pre-conditions:
    ///   - `launch_train_active_frac_compute` has run earlier in the same
    ///     epoch (so ISV[TRAIN_ACTIVE_FRAC_INDEX] is populated).
    ///   - `isv_signals_dev_ptr`, scratch_dev, wiener_dev all valid.
    ///
    /// Must run BEFORE `launch_loss_balance_controller` so the controller
    /// reads the freshly-computed cap on the same step.
    pub(crate) fn launch_max_budget_compute(&self) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp4_wiener_ema::{launch_apply_pearls, ALPHA_META};
        use crate::cuda_pipeline::sp5_isv_slots::{
            SP5_SLOT_BASE, TRAIN_ACTIVE_FRAC_INDEX,
            LB_MAX_BUDGET_CQL_BASE, LB_MAX_BUDGET_C51_BASE,
        };

        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_max_budget_compute: isv_signals_dev_ptr must be allocated");

        let isv_dev     = self.isv_signals_dev_ptr;
        let scratch_dev = self.producer_step_scratch_buf.dev_ptr;
        let wiener_dev  = self.wiener_state_buf.dev_ptr;

        let train_active_frac_isv_index_i32: i32 = TRAIN_ACTIVE_FRAC_INDEX as i32;
        let scratch_max_budget_cql_base_i32: i32 = SCRATCH_LB_MAX_BUDGET_CQL as i32;
        let scratch_max_budget_c51_base_i32: i32 = SCRATCH_LB_MAX_BUDGET_C51 as i32;

        // Step 1: loss_balance_max_budget_compute — single-block, 8 threads
        // (2 heads × 4 branches). Per-(head, branch) write of FLOOR +
        // active_frac × (CEIL − FLOOR).
        unsafe {
            self.stream
                .launch_builder(&self.loss_balance_max_budget_compute_kernel)
                .arg(&isv_dev)
                .arg(&train_active_frac_isv_index_i32)
                .arg(&scratch_dev)
                .arg(&scratch_max_budget_cql_base_i32)
                .arg(&scratch_max_budget_c51_base_i32)
                .launch(LaunchConfig {
                    grid_dim:         (1, 1, 1),
                    block_dim:        (8, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("loss_balance_max_budget_compute: {e}")))?;
        }

        // Step 2: 8 apply_pearls_ad_kernel calls (4 per head). Each
        // scratch[base+b] → ISV[isv_base+b] with its own Wiener triple.
        let base_wiener_offset = SP4_PRODUCER_COUNT as i32 * 3;
        for (isv_base, scratch_base) in [
            (LB_MAX_BUDGET_CQL_BASE, SCRATCH_LB_MAX_BUDGET_CQL),
            (LB_MAX_BUDGET_C51_BASE, SCRATCH_LB_MAX_BUDGET_C51),
        ] {
            for b in 0..4_usize {
                let scratch_idx = (scratch_base + b) as i32;
                let isv_idx     = (isv_base + b) as i32;
                let wiener_off  = base_wiener_offset + (isv_idx - SP5_SLOT_BASE as i32) * 3;
                unsafe {
                    launch_apply_pearls(
                        &self.stream,
                        &self.apply_pearls_ad_kernel,
                        scratch_dev, scratch_idx,
                        isv_dev,     isv_idx,
                        wiener_dev,  wiener_off,
                        1,
                        ALPHA_META,
                    )?;
                }
            }
        }

        Ok(())
    }

    /// SP9 (Fix 37, 2026-05-03): launch GPU producer for the eval-side
    /// magnitude intent distribution. Replaces the host-side DtoH path
    /// (`read_eval_intent_magnitude_distribution()` at
    /// `gpu_backtest_evaluator.rs:1353`) per `feedback_no_cpu_compute_strict`
    /// and `feedback_no_htod_htoh_only_mapped_pinned`.
    ///
    /// Reads `intent_mag_dev_ptr` (i32 device buffer, n_total elements —
    /// the evaluator's `intent_mag_buf` raw GPU buffer). Writes 3 floats
    /// (Q/H/F fractions) to `scratch[SCRATCH_SP9_EVAL_DIST_BASE..+3)`.
    /// Chained `apply_pearls_ad_kernel` (×3) smooths into
    /// ISV[EVAL_DIST_Q/H/F_INDEX=336..339).
    ///
    /// `intent_mag_dev_ptr` is the device pointer of the
    /// `gpu_backtest_evaluator::intent_mag_buf` `CudaSlice<i32>`. The
    /// trainer does not own the evaluator — caller threads through the
    /// device pointer (mirrors the `monitoring_summary_dev_ptr` pattern in
    /// `launch_train_active_frac_compute`). When `intent_mag_buf` is not
    /// allocated (n_total == 0), the kernel writes zeros — Pearl A's
    /// sentinel-bootstrap path will fire on the first non-empty call.
    pub(crate) fn launch_eval_intent_dist_compute(
        &self,
        intent_mag_dev_ptr: u64,
        n_total: i32,
    ) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp4_wiener_ema::{launch_apply_pearls, ALPHA_META};
        use crate::cuda_pipeline::sp5_isv_slots::{
            SP5_SLOT_BASE, EVAL_DIST_Q_INDEX,
        };

        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_eval_intent_dist_compute: isv_signals_dev_ptr must be allocated");

        let isv_dev     = self.isv_signals_dev_ptr;
        let scratch_dev = self.producer_step_scratch_buf.dev_ptr;
        let wiener_dev  = self.wiener_state_buf.dev_ptr;
        let scratch_base_i32: i32 = SCRATCH_SP9_EVAL_DIST_BASE as i32;

        // Step 1: 3-thread reduction kernel writes scratch[base..base+3).
        unsafe {
            self.stream
                .launch_builder(&self.sp9_eval_intent_dist_compute_kernel)
                .arg(&intent_mag_dev_ptr)
                .arg(&n_total)
                .arg(&scratch_dev)
                .arg(&scratch_base_i32)
                .launch(LaunchConfig {
                    grid_dim:         (1, 1, 1),
                    block_dim:        (3, 1, 1),
                    shared_mem_bytes: 4,
                })
                .map_err(|e| MLError::ModelError(format!("sp9 eval_intent_dist_compute: {e}")))?;
        }

        // Step 2: 3 apply_pearls_ad_kernel calls — one per Q/H/F bin.
        let base_wiener_offset = SP4_PRODUCER_COUNT as i32 * 3;
        for k in 0..3_usize {
            let scratch_idx = (SCRATCH_SP9_EVAL_DIST_BASE + k) as i32;
            let isv_idx     = (EVAL_DIST_Q_INDEX + k) as i32;
            let wiener_off  = base_wiener_offset + (isv_idx - SP5_SLOT_BASE as i32) * 3;
            unsafe {
                launch_apply_pearls(
                    &self.stream,
                    &self.apply_pearls_ad_kernel,
                    scratch_dev, scratch_idx,
                    isv_dev,     isv_idx,
                    wiener_dev,  wiener_off,
                    1,
                    ALPHA_META,
                )?;
            }
        }

        Ok(())
    }

    /// SP9 (Fix 37, 2026-05-03): launch GPU producer for intent vs eval
    /// Full-magnitude divergence — the behavioral-confidence axis input
    /// for the Kelly cold-start exit.
    ///
    /// SP10 (Fix 38, 2026-05-03): the same producer also writes the eval
    /// Thompson selector temperature derived from the divergence ratio
    /// (`temp = clamp(divergence/div_target, 0.5, 2.0)`). Two outputs in
    /// one kernel keeps the producer-and-consumer dependency tight per
    /// `pearl_engagement_rate_self_correction.md`'s "one signal, multiple
    /// consumers" pattern.
    ///
    /// Reads `monitoring_summary[5..17)` (12-bin action_counts) +
    /// `ISV[EVAL_DIST_F_INDEX]` + `ISV[KELLY_DIVERGENCE_TARGET_INDEX]`;
    /// writes
    ///   - `intent_f / eval_f` to
    ///     `scratch[SCRATCH_SP9_INTENT_EVAL_DIVERGENCE]` →
    ///     ISV[INTENT_EVAL_DIVERGENCE_INDEX=332]
    ///   - `clamp(divergence/div_target, 0.5, 2.0)` to
    ///     `scratch[SCRATCH_SP10_THOMPSON_TEMP]` →
    ///     ISV[EVAL_THOMPSON_TEMP_INDEX=339]
    /// Two chained `apply_pearls_ad_kernel` calls smooth each output into
    /// its respective ISV slot.
    ///
    /// Pre-conditions:
    ///   - The monitoring `reduce()` has launched on the same stream
    ///     (so `monitoring_summary[5..17)` is populated).
    ///   - `launch_eval_intent_dist_compute` has run earlier in the
    ///     current val window (so `ISV[EVAL_DIST_F_INDEX]` is populated).
    pub(crate) fn launch_intent_eval_divergence_compute(
        &self,
        monitoring_summary_dev_ptr: u64,
    ) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp4_wiener_ema::{launch_apply_pearls, ALPHA_META};
        use crate::cuda_pipeline::sp5_isv_slots::{
            SP5_SLOT_BASE, EVAL_DIST_F_INDEX, INTENT_EVAL_DIVERGENCE_INDEX,
            KELLY_DIVERGENCE_TARGET_INDEX, EVAL_THOMPSON_TEMP_INDEX,
        };

        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_intent_eval_divergence_compute: isv_signals_dev_ptr must be allocated");

        let isv_dev     = self.isv_signals_dev_ptr;
        let scratch_dev = self.producer_step_scratch_buf.dev_ptr;
        let wiener_dev  = self.wiener_state_buf.dev_ptr;

        let eval_dist_f_idx_i32:          i32 = EVAL_DIST_F_INDEX as i32;
        let divergence_target_idx_i32:    i32 = KELLY_DIVERGENCE_TARGET_INDEX as i32;
        let scratch_div_idx_i32:          i32 = SCRATCH_SP9_INTENT_EVAL_DIVERGENCE as i32;
        let scratch_temp_idx_i32:         i32 = SCRATCH_SP10_THOMPSON_TEMP as i32;

        unsafe {
            self.stream
                .launch_builder(&self.sp9_intent_eval_divergence_compute_kernel)
                .arg(&monitoring_summary_dev_ptr)
                .arg(&isv_dev)
                .arg(&eval_dist_f_idx_i32)
                .arg(&divergence_target_idx_i32)
                .arg(&scratch_dev)
                .arg(&scratch_div_idx_i32)
                .arg(&scratch_temp_idx_i32)
                .launch(LaunchConfig {
                    grid_dim:         (1, 1, 1),
                    block_dim:        (1, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("sp9 intent_eval_divergence_compute: {e}")))?;
        }

        let base_wiener_offset = SP4_PRODUCER_COUNT as i32 * 3;
        // Smooth divergence into ISV[332].
        {
            let isv_idx    = INTENT_EVAL_DIVERGENCE_INDEX as i32;
            let wiener_off = base_wiener_offset + (isv_idx - SP5_SLOT_BASE as i32) * 3;
            unsafe {
                launch_apply_pearls(
                    &self.stream,
                    &self.apply_pearls_ad_kernel,
                    scratch_dev, scratch_div_idx_i32,
                    isv_dev,     isv_idx,
                    wiener_dev,  wiener_off,
                    1,
                    ALPHA_META,
                )?;
            }
        }
        // SP10 (Fix 38): smooth Thompson temperature into ISV[339].
        {
            let isv_idx    = EVAL_THOMPSON_TEMP_INDEX as i32;
            let wiener_off = base_wiener_offset + (isv_idx - SP5_SLOT_BASE as i32) * 3;
            unsafe {
                launch_apply_pearls(
                    &self.stream,
                    &self.apply_pearls_ad_kernel,
                    scratch_dev, scratch_temp_idx_i32,
                    isv_dev,     isv_idx,
                    wiener_dev,  wiener_off,
                    1,
                    ALPHA_META,
                )?;
            }
        }

        Ok(())
    }

    /// SP9 (Fix 37, 2026-05-03): launch the four cold-path EMA producers
    /// (q_var_mag_ema) and the main warmup-floor producer.
    ///
    /// Both kernels are single-block, single-thread cold-path producers
    /// chained through `apply_pearls_ad_kernel`. The launch order matches
    /// the SP9 dependency DAG:
    ///   1. q_var_mag_ema_compute       — reads ISV[FLATNESS_BASE+1]
    ///   2. kelly_warmup_floor_compute  — reads 8 inputs (q_var_mag,
    ///                                    q_var_mag_ema, kelly_sample_count,
    ///                                    sample_count_target, intent_eval_divergence,
    ///                                    divergence_target, epoch_idx,
    ///                                    temporal_target). The 3 target
    ///                                    slots are constructor-written
    ///                                    Invariant-1 anchors per Fix 37.1
    ///                                    (`feedback_isv_for_adaptive_bounds.md`).
    ///
    /// Fix 37.1 (2026-05-03): the 3 EMA target updaters that previously sat
    /// between #1 and #2 (kelly_sample_count_target_ema /
    /// kelly_divergence_target_ema / kelly_temporal_target_ema) are deleted.
    /// Their target slots ISV[333..336) are now numerical-stability anchors
    /// — `KELLY_SAMPLE_COUNT_TARGET_INDEX = 100.0`,
    /// `KELLY_DIVERGENCE_TARGET_INDEX = 2.0`,
    /// `KELLY_TEMPORAL_TARGET_INDEX = 5.0` — written ONCE at trainer
    /// constructor time. Pearl A sentinel-bootstrap on the original EMA
    /// path saturated all three to the very first observation in epoch 1
    /// (`stat_count_tgt=419, div_tgt=70005, temp_tgt=1`), which made
    /// `current/target = 1.0`, `confidence = 1.0` and
    /// `floor = base × (1 − 1.0) = 0` — defeating the cold-start floor.
    /// A fixed denominator yields a meaningful target-relative ratio that
    /// rises monotonically as samples / divergence-stability / fold-time
    /// accumulate.
    ///
    /// Must run AFTER:
    ///   * `launch_intent_eval_divergence_compute` (so ISV[332] is fresh
    ///     before the warmup-floor kernel reads it)
    /// Must run BEFORE:
    ///   * any `unified_env_step_core` (which reads ISV[KELLY_WARMUP_FLOOR_INDEX])
    pub(crate) fn launch_sp9_kelly_warmup_floor(&self) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp4_wiener_ema::{launch_apply_pearls, ALPHA_META};
        use crate::cuda_pipeline::sp5_isv_slots::{
            SP5_SLOT_BASE,
            FLATNESS_BASE,
            KELLY_SAMPLE_COUNT_INDEX,
            INTENT_EVAL_DIVERGENCE_INDEX,
            Q_VAR_MAG_EMA_INDEX,
            KELLY_SAMPLE_COUNT_TARGET_INDEX,
            KELLY_DIVERGENCE_TARGET_INDEX,
            KELLY_TEMPORAL_TARGET_INDEX,
            KELLY_WARMUP_FLOOR_INDEX,
        };
        // EPOCH_IDX_INDEX is defined in this module — no extra import needed.

        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_sp9_kelly_warmup_floor: isv_signals_dev_ptr must be allocated");

        let isv_dev     = self.isv_signals_dev_ptr;
        let scratch_dev = self.producer_step_scratch_buf.dev_ptr;
        let wiener_dev  = self.wiener_state_buf.dev_ptr;
        let base_wiener_offset = SP4_PRODUCER_COUNT as i32 * 3;

        // Helper to launch a single-input EMA producer + its apply_pearls
        // chain. The ISV input slot is read by the kernel; the ISV output
        // slot receives Pearls A+D smoothing of scratch[scratch_idx].
        let launch_simple_ema = |kernel: &CudaFunction,
                                 input_isv_idx: usize,
                                 scratch_idx: usize,
                                 output_isv_idx: usize,
                                 label: &str|
         -> Result<(), MLError> {
            let input_idx_i32: i32 = input_isv_idx as i32;
            let scratch_idx_i32: i32 = scratch_idx as i32;
            unsafe {
                self.stream
                    .launch_builder(kernel)
                    .arg(&isv_dev)
                    .arg(&input_idx_i32)
                    .arg(&scratch_dev)
                    .arg(&scratch_idx_i32)
                    .launch(LaunchConfig {
                        grid_dim:         (1, 1, 1),
                        block_dim:        (1, 1, 1),
                        shared_mem_bytes: 0,
                    })
                    .map_err(|e| MLError::ModelError(format!("sp9 {label}: {e}")))?;
            }
            let isv_idx_i32 = output_isv_idx as i32;
            let wiener_off  = base_wiener_offset + (isv_idx_i32 - SP5_SLOT_BASE as i32) * 3;
            unsafe {
                launch_apply_pearls(
                    &self.stream,
                    &self.apply_pearls_ad_kernel,
                    scratch_dev, scratch_idx_i32,
                    isv_dev,     isv_idx_i32,
                    wiener_dev,  wiener_off,
                    1,
                    ALPHA_META,
                )?;
            }
            Ok(())
        };

        // 1. q_var_mag_ema (reads ISV[207] — magnitude flatness)
        launch_simple_ema(
            &self.sp9_q_var_mag_ema_compute_kernel,
            FLATNESS_BASE + 1,
            SCRATCH_SP9_Q_VAR_MAG_EMA,
            Q_VAR_MAG_EMA_INDEX,
            "q_var_mag_ema_compute",
        )?;

        // 2. kelly_warmup_floor_compute — combines all 8 inputs (target
        //    slots ISV[333..336) read directly from constructor-written
        //    Invariant-1 anchors per Fix 37.1; no producer chain).
        let q_var_mag_idx_i32:               i32 = (FLATNESS_BASE + 1) as i32;
        let q_var_mag_ema_idx_i32:           i32 = Q_VAR_MAG_EMA_INDEX as i32;
        let kelly_sample_count_idx_i32:      i32 = KELLY_SAMPLE_COUNT_INDEX as i32;
        let sample_count_target_idx_i32:     i32 = KELLY_SAMPLE_COUNT_TARGET_INDEX as i32;
        let intent_eval_divergence_idx_i32:  i32 = INTENT_EVAL_DIVERGENCE_INDEX as i32;
        let divergence_target_idx_i32:       i32 = KELLY_DIVERGENCE_TARGET_INDEX as i32;
        let epoch_idx_idx_i32:               i32 = EPOCH_IDX_INDEX as i32;
        let temporal_target_idx_i32:         i32 = KELLY_TEMPORAL_TARGET_INDEX as i32;
        let scratch_idx_i32:                 i32 = SCRATCH_SP9_KELLY_WARMUP_FLOOR as i32;

        unsafe {
            self.stream
                .launch_builder(&self.sp9_kelly_warmup_floor_compute_kernel)
                .arg(&isv_dev)
                .arg(&q_var_mag_idx_i32)
                .arg(&q_var_mag_ema_idx_i32)
                .arg(&kelly_sample_count_idx_i32)
                .arg(&sample_count_target_idx_i32)
                .arg(&intent_eval_divergence_idx_i32)
                .arg(&divergence_target_idx_i32)
                .arg(&epoch_idx_idx_i32)
                .arg(&temporal_target_idx_i32)
                .arg(&scratch_dev)
                .arg(&scratch_idx_i32)
                .launch(LaunchConfig {
                    grid_dim:         (1, 1, 1),
                    block_dim:        (1, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("sp9 kelly_warmup_floor_compute: {e}")))?;
        }

        let isv_idx_i32 = KELLY_WARMUP_FLOOR_INDEX as i32;
        let wiener_off  = base_wiener_offset + (isv_idx_i32 - SP5_SLOT_BASE as i32) * 3;
        unsafe {
            launch_apply_pearls(
                &self.stream,
                &self.apply_pearls_ad_kernel,
                scratch_dev, scratch_idx_i32,
                isv_dev,     isv_idx_i32,
                wiener_dev,  wiener_off,
                1,
                ALPHA_META,
            )?;
        }

        Ok(())
    }

    /// SP11 Fix 39 (2026-05-04, A1.1): set the GPU device pointer + bar
    /// count for the per-bar saboteur Δreward buffer populated by
    /// `experience_env_step`. The trainer caches this so
    /// `launch_sp11_saboteur_engagement_compute` has the consumer pointer
    /// available without re-querying the collector each step. Per
    /// `feedback_wire_everything_up`: the buffer is allocated by the
    /// collector and wired into the trainer in the same commit that adds
    /// it. Initial value is zero/zero (constructor); the training-loop
    /// init path calls this setter once after collector construction.
    pub(crate) fn set_sp11_saboteur_delta_reward_buf(
        &mut self,
        dev_ptr: u64,
        n_bars: usize,
    ) {
        self.sp11_saboteur_delta_reward_dev_ptr = dev_ptr;
        self.sp11_saboteur_delta_reward_len = n_bars;
    }

    /// SP11 Fix 39 B1b fix-up (2026-05-04): wire the experience collector's
    /// `popart_component_per_sample` device pointer into the trainer so
    /// `launch_sp11_popart_component_ema` reads the right buffer. Mirror
    /// of `set_sp11_saboteur_delta_reward_buf` — the collector owns the
    /// per-bar buffer (alloc_episodes × alloc_timesteps); the trainer
    /// caches the pointer + length for the per-step launcher path.
    /// Per `feedback_wire_everything_up`: this is wired in the same commit
    /// that adds the buffer + kernel.
    pub(crate) fn set_sp11_popart_component_buf(
        &mut self,
        dev_ptr: u64,
        n_bars: usize,
    ) {
        self.sp11_popart_component_dev_ptr = dev_ptr;
        self.sp11_popart_component_len = n_bars;
    }

    /// SP11 Fix 39 (2026-05-04, A1.1): launch the val-sharpe Δ + variance
    /// canary producer.
    ///
    /// Reads `val_sharpe_history_pinned` (mapped-pinned 2-element
    /// `[prev_epoch, curr_epoch]` written from `training_loop.rs` at the
    /// val emit boundary) plus `ISV[VAL_SHARPE_DELTA_EMA_INDEX]`; writes
    /// raw delta + squared deviation to
    /// `scratch[SCRATCH_SP11_VAL_SHARPE_DELTA_BASE..+2)`. Chained
    /// `apply_pearls_ad_kernel` (n_slots=2) smooths into
    /// ISV[VAL_SHARPE_DELTA_EMA_INDEX=350, VAL_SHARPE_VAR_EMA_INDEX=351].
    ///
    /// Layer A is additive — no consumer reads either canary slot in this
    /// commit. The controller producer (Layer A2) and reward-shaping
    /// consumers (Layer B) wire downstream.
    pub(crate) fn launch_sp11_val_sharpe_delta_compute(&self) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp4_wiener_ema::{launch_apply_pearls, ALPHA_META};
        use crate::cuda_pipeline::sp5_isv_slots::SP5_SLOT_BASE;
        use crate::cuda_pipeline::sp11_isv_slots::{
            VAL_SHARPE_DELTA_EMA_INDEX, VAL_SHARPE_VAR_EMA_INDEX,
        };

        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_sp11_val_sharpe_delta_compute: isv_signals_dev_ptr must be allocated");
        debug_assert_eq!(self.val_sharpe_history_pinned.len, 2,
            "launch_sp11_val_sharpe_delta_compute: val_sharpe_history_pinned must be 2 f32");

        let isv_dev     = self.isv_signals_dev_ptr;
        let scratch_dev = self.producer_step_scratch_buf.dev_ptr;
        let wiener_dev  = self.wiener_state_buf.dev_ptr;
        let history_dev = self.val_sharpe_history_pinned.dev_ptr;

        let scratch_base_i32: i32 = SCRATCH_SP11_VAL_SHARPE_DELTA_BASE as i32;
        let delta_ema_slot_i32: i32 = VAL_SHARPE_DELTA_EMA_INDEX as i32;
        let var_ema_slot_i32:   i32 = VAL_SHARPE_VAR_EMA_INDEX as i32;

        // Step 1: producer kernel writes scratch[base..base+2).
        // The kernel's scratch_out arg is offset by the base scratch index
        // via pointer arithmetic — the kernel writes to `scratch_out[0..2)`
        // relative to whatever pointer it receives. Pass the offset device
        // pointer rather than a base index parameter (keeps the kernel
        // signature minimal and matches the SP9 producer-launcher
        // pattern at `launch_intent_eval_divergence_compute`).
        let scratch_offset_bytes = (SCRATCH_SP11_VAL_SHARPE_DELTA_BASE as u64)
            * std::mem::size_of::<f32>() as u64;
        let scratch_out_dev: u64 = scratch_dev + scratch_offset_bytes;

        unsafe {
            self.stream
                .launch_builder(&self.sp11_val_sharpe_delta_compute_kernel)
                .arg(&history_dev)
                .arg(&scratch_out_dev)
                .arg(&isv_dev)
                .arg(&delta_ema_slot_i32)
                .arg(&var_ema_slot_i32)
                .launch(LaunchConfig {
                    grid_dim:         (1, 1, 1),
                    block_dim:        (1, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("sp11 val_sharpe_delta_compute: {e}")))?;
        }

        // Step 2: chained apply_pearls_ad_kernel (n_slots=2) — both
        // VAL_SHARPE_DELTA_EMA_INDEX and VAL_SHARPE_VAR_EMA_INDEX are
        // contiguous (350, 351) so a single n_slots=2 launch covers them.
        let base_wiener_offset = SP4_PRODUCER_COUNT as i32 * 3;
        let wiener_off = base_wiener_offset
            + (delta_ema_slot_i32 - SP5_SLOT_BASE as i32) * 3;
        unsafe {
            launch_apply_pearls(
                &self.stream,
                &self.apply_pearls_ad_kernel,
                scratch_dev, scratch_base_i32,
                isv_dev,     delta_ema_slot_i32,
                wiener_dev,  wiener_off,
                2,
                ALPHA_META,
            )?;
        }

        Ok(())
    }

    /// SP11 Fix 39 (2026-05-04, A1.2): launch the per-bar saboteur
    /// engagement canary producer.
    ///
    /// Reads `sp11_saboteur_delta_reward_dev_ptr` (a per-bar |Δreward|
    /// GPU buffer populated by `experience_env_step` at the saboteur
    /// perturbation site). Block tree-reduces engagement count over all
    /// `sp11_saboteur_delta_reward_len` bars, writes fraction to
    /// `scratch[SCRATCH_SP11_SABOTEUR_ENGAGEMENT]`. Chained
    /// `apply_pearls_ad_kernel` (n_slots=1) smooths into
    /// ISV[SABOTEUR_ENGAGEMENT_RATE_INDEX=358].
    ///
    /// Pre-condition: `set_sp11_saboteur_delta_reward_buf` has been called
    /// with the collector's saboteur Δreward dev ptr. If the saboteur is
    /// inactive on a given epoch (delta_reward stays zero), engagement
    /// reads zero and Pearl A's sentinel-bootstrap path holds the EMA
    /// at zero — no spurious activation.
    pub(crate) fn launch_sp11_saboteur_engagement_compute(&self) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp4_wiener_ema::{launch_apply_pearls, ALPHA_META};
        use crate::cuda_pipeline::sp5_isv_slots::SP5_SLOT_BASE;
        use crate::cuda_pipeline::sp11_isv_slots::{
            SABOTEUR_ENGAGEMENT_RATE_INDEX, PNL_REWARD_MAGNITUDE_EMA_INDEX,
        };

        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_sp11_saboteur_engagement_compute: isv_signals_dev_ptr must be allocated");

        // No-op when the producer hasn't been wired yet (e.g. early init
        // before the collector exists). This is NOT a stub return value
        // — it's a precondition guard that defers the launch until the
        // wire-up is complete. Once `set_sp11_saboteur_delta_reward_buf`
        // is called with a valid pointer, every subsequent step launches
        // the kernel for real.
        if self.sp11_saboteur_delta_reward_dev_ptr == 0
            || self.sp11_saboteur_delta_reward_len == 0
        {
            return Ok(());
        }

        let isv_dev      = self.isv_signals_dev_ptr;
        let scratch_dev  = self.producer_step_scratch_buf.dev_ptr;
        let wiener_dev   = self.wiener_state_buf.dev_ptr;
        let delta_dev    = self.sp11_saboteur_delta_reward_dev_ptr;

        let scratch_offset_bytes = (SCRATCH_SP11_SABOTEUR_ENGAGEMENT as u64)
            * std::mem::size_of::<f32>() as u64;
        let scratch_out_dev: u64 = scratch_dev + scratch_offset_bytes;
        let scratch_idx_i32: i32 = SCRATCH_SP11_SABOTEUR_ENGAGEMENT as i32;

        let n_bars_i32: i32 = i32::try_from(self.sp11_saboteur_delta_reward_len)
            .unwrap_or(i32::MAX);
        let pnl_mag_slot_i32: i32 = PNL_REWARD_MAGNITUDE_EMA_INDEX as i32;

        // Step 1: producer kernel — block tree-reduce, BLOCK_SIZE=256.
        // Shared memory = 256 ints = 1024 bytes.
        unsafe {
            self.stream
                .launch_builder(&self.sp11_saboteur_engagement_compute_kernel)
                .arg(&delta_dev)
                .arg(&scratch_out_dev)
                .arg(&isv_dev)
                .arg(&n_bars_i32)
                .arg(&pnl_mag_slot_i32)
                .launch(LaunchConfig {
                    grid_dim:         (1, 1, 1),
                    block_dim:        (256, 1, 1),
                    shared_mem_bytes: 256 * std::mem::size_of::<i32>() as u32,
                })
                .map_err(|e| MLError::ModelError(format!("sp11 saboteur_engagement_compute: {e}")))?;
        }

        // Step 2: chained apply_pearls_ad_kernel singleton →
        // ISV[SABOTEUR_ENGAGEMENT_RATE_INDEX=358].
        let base_wiener_offset = SP4_PRODUCER_COUNT as i32 * 3;
        let isv_idx_i32 = SABOTEUR_ENGAGEMENT_RATE_INDEX as i32;
        let wiener_off  = base_wiener_offset + (isv_idx_i32 - SP5_SLOT_BASE as i32) * 3;
        unsafe {
            launch_apply_pearls(
                &self.stream,
                &self.apply_pearls_ad_kernel,
                scratch_dev, scratch_idx_i32,
                isv_dev,     isv_idx_i32,
                wiener_dev,  wiener_off,
                1,
                ALPHA_META,
            )?;
        }

        Ok(())
    }

    /// SP11 Fix 39 (2026-05-04, A1.3 + B1b fix-up 2026-05-04): launch the
    /// per-component reward-magnitude-ratio canary producer.
    ///
    /// B1b fix-up: reads NON-CONTIGUOUS sources after the slot-360 split.
    ///   axis 0 (popart)   ← ISV[POPART_COMPONENT_MAG_EMA_INDEX = 360]
    ///   axis 1..5         ← ISV[REWARD_POPART_EMA_INDEX + 1 .. + 6)
    ///                       (cf=64, trail=65, micro=66, opp_cost=67, bonus=68)
    /// Normalises to ratios in `scratch[SCRATCH_SP11_MAG_RATIO_BASE..+6)`,
    /// and mirrors the popart-component magnitude into `scratch[base+6]`.
    /// Followed by two chained `apply_pearls_ad_kernel` launches:
    ///   1. n_slots=6 → ISV[REWARD_COMPONENT_MAG_RATIO_BASE..+6)
    ///   2. n_slots=1 → ISV[PNL_REWARD_MAGNITUDE_EMA_INDEX=359]
    /// (The ISV slots are non-contiguous: 352..358 plus 359, so two
    /// launches rather than one n_slots=7. The kernel still emits 7
    /// scratch values in one shot.)
    pub(crate) fn launch_sp11_mag_ratio_compute(&self) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp4_wiener_ema::{launch_apply_pearls, ALPHA_META};
        use crate::cuda_pipeline::sp5_isv_slots::SP5_SLOT_BASE;
        use crate::cuda_pipeline::sp11_isv_slots::{
            REWARD_COMPONENT_MAG_RATIO_BASE, REWARD_COMPONENT_MAG_RATIO_COUNT,
            PNL_REWARD_MAGNITUDE_EMA_INDEX, POPART_COMPONENT_MAG_EMA_INDEX,
            REWARD_COMPONENT_VAR_EMA_BASE,
        };

        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_sp11_mag_ratio_compute: isv_signals_dev_ptr must be allocated");

        let isv_dev     = self.isv_signals_dev_ptr;
        let scratch_dev = self.producer_step_scratch_buf.dev_ptr;
        let wiener_dev  = self.wiener_state_buf.dev_ptr;

        let scratch_offset_bytes = (SCRATCH_SP11_MAG_RATIO_BASE as u64)
            * std::mem::size_of::<f32>() as u64;
        let scratch_out_dev: u64 = scratch_dev + scratch_offset_bytes;

        // B1b fix-up: split popart axis (slot 360) from cf/trail/micro/
        // opp_cost/bonus axes (slots 64..68). Pre-fix the kernel read 6
        // contiguous slots starting at slot 63 — which was overloaded.
        let popart_specific_slot_i32: i32 = POPART_COMPONENT_MAG_EMA_INDEX as i32;
        let cf_others_base_slot_i32:  i32 = (REWARD_POPART_EMA_INDEX + 1) as i32;
        // B1b smoke-recovery (2026-05-04): per-component variance slots
        // for z-score normalisation. Popart variance lives at slot 361
        // (paired with mag at slot 360 — separate producer per the
        // slot-63 fix-up rationale); cf/trail/micro/opp_cost/bonus
        // variances live at slots 362..366 (paired with mags at 64..68).
        let popart_var_slot_i32:        i32 = REWARD_COMPONENT_VAR_EMA_BASE as i32;
        let cf_others_var_base_slot_i32: i32 = (REWARD_COMPONENT_VAR_EMA_BASE + 1) as i32;

        unsafe {
            self.stream
                .launch_builder(&self.sp11_mag_ratio_compute_kernel)
                .arg(&isv_dev)
                .arg(&scratch_out_dev)
                .arg(&popart_specific_slot_i32)
                .arg(&cf_others_base_slot_i32)
                .arg(&popart_var_slot_i32)
                .arg(&cf_others_var_base_slot_i32)
                .launch(LaunchConfig {
                    grid_dim:         (1, 1, 1),
                    block_dim:        (6, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("sp11 mag_ratio_compute: {e}")))?;
        }

        let base_wiener_offset = SP4_PRODUCER_COUNT as i32 * 3;

        // Pearls A+D #1: 6 contiguous slots → ISV[352..358).
        let ratios_isv_base_i32  = REWARD_COMPONENT_MAG_RATIO_BASE as i32;
        let ratios_scratch_idx_i32 = SCRATCH_SP11_MAG_RATIO_BASE as i32;
        let ratios_wiener_off    = base_wiener_offset
            + (ratios_isv_base_i32 - SP5_SLOT_BASE as i32) * 3;
        unsafe {
            launch_apply_pearls(
                &self.stream,
                &self.apply_pearls_ad_kernel,
                scratch_dev, ratios_scratch_idx_i32,
                isv_dev,     ratios_isv_base_i32,
                wiener_dev,  ratios_wiener_off,
                REWARD_COMPONENT_MAG_RATIO_COUNT as i32,
                ALPHA_META,
            )?;
        }

        // Pearls A+D #2: singleton mirror → ISV[PNL_REWARD_MAGNITUDE_EMA_INDEX=359].
        let mirror_isv_idx_i32     = PNL_REWARD_MAGNITUDE_EMA_INDEX as i32;
        let mirror_scratch_idx_i32 = (SCRATCH_SP11_MAG_RATIO_BASE
            + REWARD_COMPONENT_MAG_RATIO_COUNT) as i32;
        let mirror_wiener_off      = base_wiener_offset
            + (mirror_isv_idx_i32 - SP5_SLOT_BASE as i32) * 3;
        unsafe {
            launch_apply_pearls(
                &self.stream,
                &self.apply_pearls_ad_kernel,
                scratch_dev, mirror_scratch_idx_i32,
                isv_dev,     mirror_isv_idx_i32,
                wiener_dev,  mirror_wiener_off,
                1,
                ALPHA_META,
            )?;
        }

        Ok(())
    }

    /// SP11 Fix 39 B1b fix-up (2026-05-04): launch the popart-component-
    /// specific magnitude EMA producer + chained Pearls A+D singleton
    /// smoothing.
    ///
    /// Reads the experience collector's per-bar `popart_component_per_sample`
    /// buffer (wired in via `set_sp11_popart_component_buf`), computes
    /// mean(|x|) into `scratch[SCRATCH_SP11_POPART_COMPONENT_EMA]` via a
    /// single-block tree-reduce (BLOCK_SIZE=256), then chains a Pearls
    /// A+D singleton launch into ISV[POPART_COMPONENT_MAG_EMA_INDEX=360].
    ///
    /// Runs BEFORE `launch_sp11_mag_ratio_compute` in the per-step path
    /// so the mag-ratio canary's first read of slot 360 sees a populated
    /// value (or sentinel 0 on the very first epoch — Pearl A bootstraps
    /// from the first observation).
    ///
    /// No-op when the popart-component buffer hasn't been wired yet
    /// (early init before the collector exists). Mirrors the saboteur
    /// engagement launcher's NULL-guard pattern.
    pub(crate) fn launch_sp11_popart_component_ema(&self) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp4_wiener_ema::{launch_apply_pearls, ALPHA_META};
        use crate::cuda_pipeline::sp5_isv_slots::SP5_SLOT_BASE;
        use crate::cuda_pipeline::sp11_isv_slots::{
            POPART_COMPONENT_MAG_EMA_INDEX, REWARD_COMPONENT_VAR_EMA_BASE,
        };

        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_sp11_popart_component_ema: isv_signals_dev_ptr must be allocated");

        // Precondition guard — the popart-component buffer is wired by
        // `set_sp11_popart_component_buf` after collector construction.
        // Until then, no-op (NOT a stub return value per
        // `feedback_no_stubs.md` — a precondition guard that defers the
        // launch). Once wired, every subsequent step launches.
        if self.sp11_popart_component_dev_ptr == 0
            || self.sp11_popart_component_len == 0
        {
            return Ok(());
        }

        let isv_dev      = self.isv_signals_dev_ptr;
        let scratch_dev  = self.producer_step_scratch_buf.dev_ptr;
        let wiener_dev   = self.wiener_state_buf.dev_ptr;
        let popart_dev   = self.sp11_popart_component_dev_ptr;

        let scratch_offset_bytes = (SCRATCH_SP11_POPART_COMPONENT_EMA as u64)
            * std::mem::size_of::<f32>() as u64;
        let scratch_out_dev: u64 = scratch_dev + scratch_offset_bytes;
        let scratch_idx_i32: i32 = SCRATCH_SP11_POPART_COMPONENT_EMA as i32;

        let n_bars_i32: i32 = i32::try_from(self.sp11_popart_component_len)
            .unwrap_or(i32::MAX);

        // Step 1: producer kernel — block tree-reduce, BLOCK_SIZE=256.
        // Shared memory = 256 floats = 1024 bytes (matches the kernel's
        // `extern __shared__ float sdata[]` declaration). Writes mean(|x|)
        // to scratch_out[0] AND var(|x|) to scratch_out[1] via single-pass
        // Welford (B1b smoke-recovery, 2026-05-04 — spec §4 amendment
        // "Why z-score" lines 564-619).
        unsafe {
            self.stream
                .launch_builder(&self.sp11_popart_component_ema_kernel)
                .arg(&popart_dev)
                .arg(&scratch_out_dev)
                .arg(&n_bars_i32)
                .launch(LaunchConfig {
                    grid_dim:         (1, 1, 1),
                    block_dim:        (256, 1, 1),
                    shared_mem_bytes: 256 * std::mem::size_of::<f32>() as u32,
                })
                .map_err(|e| MLError::ModelError(format!("sp11 popart_component_ema: {e}")))?;
        }

        // Step 2: chained apply_pearls_ad_kernel singleton →
        // ISV[POPART_COMPONENT_MAG_EMA_INDEX=360] (mag).
        let base_wiener_offset = SP4_PRODUCER_COUNT as i32 * 3;
        let mag_isv_idx_i32 = POPART_COMPONENT_MAG_EMA_INDEX as i32;
        let mag_wiener_off  = base_wiener_offset + (mag_isv_idx_i32 - SP5_SLOT_BASE as i32) * 3;
        unsafe {
            launch_apply_pearls(
                &self.stream,
                &self.apply_pearls_ad_kernel,
                scratch_dev, scratch_idx_i32,
                isv_dev,     mag_isv_idx_i32,
                wiener_dev,  mag_wiener_off,
                1,
                ALPHA_META,
            )?;
        }

        // Step 3 (B1b smoke-recovery): chained singleton →
        // ISV[REWARD_COMPONENT_VAR_EMA_BASE=361] (popart variance).
        // Scratch source is the second float written by the kernel
        // (scratch_idx + 1).
        let var_isv_idx_i32 = REWARD_COMPONENT_VAR_EMA_BASE as i32;
        let var_wiener_off  = base_wiener_offset + (var_isv_idx_i32 - SP5_SLOT_BASE as i32) * 3;
        let var_scratch_idx_i32 = scratch_idx_i32 + 1;
        unsafe {
            launch_apply_pearls(
                &self.stream,
                &self.apply_pearls_ad_kernel,
                scratch_dev, var_scratch_idx_i32,
                isv_dev,     var_isv_idx_i32,
                wiener_dev,  var_wiener_off,
                1,
                ALPHA_META,
            )?;
        }

        Ok(())
    }

    /// SP13 Phase 0a (2026-05-04): launch the aux-head directional-
    /// accuracy reducer.
    ///
    /// SP13 B1.1a (2026-05-05): reads `aux_softmax [B, K]` (the K=2
    /// softmax tile produced inside the captured forward graph by
    /// `aux_next_bar_forward`) and `labels [B] i32` (`-1 / 0 / 1` —
    /// `-1` is producer mask sentinel; producer wires in B1.1b) and
    /// writes
    /// `(dir_acc, pos_pred_frac, pos_label_frac, n_down, n_up, n_skip)`
    /// into the 6-element output buffer at `out_6_dev`. Both softmax
    /// and labels live on the GPU; the 6-element output is the
    /// trainer's mapped-pinned `aux_dir_acc_buf` in production (the
    /// launcher is invoked with `aux_dir_acc_buf.dev_ptr` from
    /// `training_loop.rs` after the captured forward graph populates
    /// `aux_nb_softmax_buf` / `aux_nb_label_buf`). The launcher takes
    /// raw `u64` device pointers per Decision D /
    /// `feedback_no_htod_htoh_only_mapped_pinned.md` because the
    /// production source for `out_6_dev` is a `MappedF32Buffer` (which
    /// exposes only `dev_ptr: u64`).
    ///
    /// Single-block tree-reduce (BLOCK_SIZE=256) — six shared-memory
    /// int arrays (correct/pos_pred/pos_label/n_down/n_up/n_skip)
    /// reduce in lockstep per `feedback_no_atomicadd.md`. The launcher
    /// computes `shared_mem_bytes = 6 × bdim × sizeof(i32)` to match
    /// the kernel's `extern __shared__ int shared[]` declaration; passing
    /// a smaller value would make the kernel read past the dynamic
    /// shared-memory region and corrupt other data.
    ///
    /// Per `feedback_cudarc_f64_f32_abi.md`: `batch_size` and `k` are
    /// `i32` (match the kernel signature exactly — no implicit cast).
    pub(crate) fn launch_aux_dir_acc_reduce(
        &self,
        aux_softmax_dev: u64,
        labels_dev: u64,
        batch_size: i32,
        k: i32,
        out_6_dev: u64,
    ) -> Result<(), MLError> {
        let bdim: u32 = 256;
        let cfg = LaunchConfig {
            grid_dim:         (1, 1, 1),
            block_dim:        (bdim, 1, 1),
            shared_mem_bytes: 6 * bdim * std::mem::size_of::<i32>() as u32,
        };
        unsafe {
            self.stream
                .launch_builder(&self.aux_dir_acc_reduce)
                .arg(&aux_softmax_dev)
                .arg(&labels_dev)
                .arg(&batch_size)
                .arg(&k)
                .arg(&out_6_dev)
                .launch(cfg)
                .map_err(|e| MLError::ModelError(format!("sp13 aux_dir_acc_reduce launch: {e}")))?;
        }
        Ok(())
    }

    /// SP13 Phase 0a P0a.T4 (2026-05-04): launch the fixed-α EMA
    /// applicator.
    ///
    /// One launch updates `n` consecutive ISV slots starting at
    /// `isv_offset`, blending each with the matching `sample[i]` at
    /// fixed `alpha`. Caller passes the registry sentinel for the
    /// destination slots (`pearl_first_observation_bootstrap`):
    ///   - 0.5 for the SP13 dir-acc EMAs at slots 373 / 374 (random-
    ///     guessing baseline; sentinel means "no observation yet")
    ///   - 0.0 for the hold-rate EMA at slot 382 (empty-batch fallback)
    ///
    /// Sibling of `launch_apply_pearls` — needed because the Wiener-
    /// optimal blend collapses two EMAs of the same signal to identical
    /// values, defeating the SP13 stagnation detector at slot 374
    /// (compares against slot 373's faster EMA). The fixed-α form
    /// preserves the timescale separation that makes "improving vs
    /// stalled" diagnosable.
    ///
    /// Stream-ordered with the producer that wrote `sample_dev`. The
    /// producer kernel MUST issue `__threadfence_system()` before
    /// returning so this kernel's per-thread `sample[i]` read is
    /// visible.
    pub(crate) fn launch_apply_fixed_alpha_ema(
        &self,
        sample_dev: u64,
        n: i32,
        isv_offset: i32,
        alpha: f32,
        sentinel: f32,
    ) -> Result<(), MLError> {
        debug_assert!(n >= 0,
            "launch_apply_fixed_alpha_ema: n must be non-negative, got {n}");
        if n == 0 {
            return Ok(());
        }
        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_apply_fixed_alpha_ema: isv_signals_dev_ptr must be allocated by constructor");

        // 1D thread grid over the slot count. SP13 P0a always launches
        // with n=1 (single dir-acc scalar fed into both EMAs in
        // lockstep, hold-rate scalar into slot 382), but the kernel
        // generalises to n-slot blocks for future use without an
        // ABI change.
        let bdim: u32 = (n as u32).min(256).max(1);
        let grid: u32 = ((n as u32) + bdim - 1) / bdim;
        let cfg = LaunchConfig {
            grid_dim:         (grid, 1, 1),
            block_dim:        (bdim, 1, 1),
            shared_mem_bytes: 0,
        };
        let isv_dev = self.isv_signals_dev_ptr;
        unsafe {
            self.stream
                .launch_builder(&self.apply_fixed_alpha_ema_kernel)
                .arg(&sample_dev)
                .arg(&n)
                .arg(&isv_offset)
                .arg(&alpha)
                .arg(&sentinel)
                .arg(&isv_dev)
                .launch(cfg)
                .map_err(|e| MLError::ModelError(format!("sp13 apply_fixed_alpha_ema launch: {e}")))?;
        }
        Ok(())
    }

    /// SP13 Phase 0a P0a.T4 (2026-05-04): launch the aux-head
    /// per-bar prediction → ISV[AUX_DIR_PREDICTION_INDEX=375] bounded
    /// scalar producer.
    ///
    /// SP13 B1.1a (2026-05-05): reads `aux_softmax [B, K]` (production
    /// source: `aux_nb_softmax_buf`, populated by the captured forward
    /// graph's `aux_next_bar_forward`) and writes
    /// `mean(softmax[:, 1] - softmax[:, 0])` ∈ [-1, +1] to the SHARED
    /// ISV slot at `isv_slot_offset`. Per-step state, not an EMA —
    /// slot is overwritten each launch (no FoldReset registry entry,
    /// per the SP13 P0a registry comments). The bound is now structural
    /// (softmax components in [0, 1] sum to 1 ⇒ pairwise diff in
    /// [-1, +1]) per `pearl_bounded_modifier_outputs_require_structural_activation`;
    /// the runtime tanh squash is retired.
    ///
    /// Single-block tree-reduce (BLOCK_SIZE=256) — one shared-memory
    /// float array reduces in lockstep per `feedback_no_atomicadd.md`.
    /// `shared_mem_bytes = bdim × sizeof(f32)`.
    ///
    /// Stream-ordered with the producer that wrote `aux_softmax_dev`;
    /// same-stream contract mirrors `launch_aux_heads_loss_ema`.
    pub(crate) fn launch_aux_pred_to_isv_tanh(
        &self,
        aux_softmax_dev: u64,
        batch_size: i32,
        k: i32,
        isv_slot_offset: i32,
    ) -> Result<(), MLError> {
        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_aux_pred_to_isv_tanh: isv_signals_dev_ptr must be allocated by constructor");

        let bdim: u32 = 256;
        let cfg = LaunchConfig {
            grid_dim:         (1, 1, 1),
            block_dim:        (bdim, 1, 1),
            shared_mem_bytes: bdim * std::mem::size_of::<f32>() as u32,
        };
        let isv_dev = self.isv_signals_dev_ptr;
        unsafe {
            self.stream
                .launch_builder(&self.aux_pred_to_isv_tanh_kernel)
                .arg(&aux_softmax_dev)
                .arg(&batch_size)
                .arg(&k)
                .arg(&isv_slot_offset)
                .arg(&isv_dev)
                .launch(cfg)
                .map_err(|e| MLError::ModelError(format!("sp13 aux_pred_to_isv_tanh launch: {e}")))?;
        }
        Ok(())
    }

    /// SP13 Phase 0a P0a.T4 (2026-05-04): orchestrator for the per-step
    /// aux directional-accuracy producer chain.
    ///
    /// SP13 B1.1a (2026-05-05): reads softmax tile + i32 labels (replaces
    /// the regression-mode pred/label_f32 pair); dir-acc kernel emits
    /// 6 floats now (was 3 — added `n_down`/`n_up`/`n_skip` for HEALTH_DIAG
    /// observability of the producer-less B1.1a label distribution).
    ///
    /// Single-call API for `training_loop.rs` to fire the SP13 P0a
    /// per-step metrics:
    ///   1. `launch_aux_dir_acc_reduce` — reduce `aux_nb_softmax_buf` +
    ///      `aux_nb_label_buf` into `aux_dir_acc_buf [6]`
    ///      `(dir_acc, pos_pred_frac, pos_label_frac, n_down, n_up, n_skip)`.
    ///   2. `launch_apply_fixed_alpha_ema` — short EMA (α=0.3) into
    ///      ISV[AUX_DIR_ACC_SHORT_EMA_INDEX=373], sentinel 0.5.
    ///   3. `launch_apply_fixed_alpha_ema` — slow EMA (α=0.05) into
    ///      ISV[AUX_DIR_ACC_LONG_EMA_INDEX=374], sentinel 0.5.
    ///   4. `launch_aux_pred_to_isv_tanh` — mean(softmax[1]-softmax[0]) →
    ///      ISV[AUX_DIR_PREDICTION_INDEX=375].
    ///
    /// All launches are stream-ordered; the producer's
    /// `__threadfence_system()` orders against the consumer reads on
    /// the same stream so no host sync is needed between steps.
    /// Caller invokes once per training step BEFORE the HEALTH_DIAG
    /// snapshot reads slot 373/374/375 (matches the post-cascade-fix
    /// invariant from commit `a5f23b28f`).
    ///
    /// Pre-condition: the captured forward graph has run this step,
    /// so `aux_nb_softmax_buf` and `aux_nb_label_buf` are populated.
    /// Producer-only — no consumer kernel reads slots 373/374/375 in
    /// this commit; Phase 0b wires the controller and the Q-head
    /// input layer that reads slot 375.
    pub fn launch_sp13_aux_dir_metrics(
        &self,
    ) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp13_isv_slots::{
            AUX_DIR_ACC_SHORT_EMA_INDEX, AUX_DIR_ACC_LONG_EMA_INDEX,
            AUX_DIR_PREDICTION_INDEX,
            DIR_ACC_EMA_SENTINEL,
        };

        let aux_softmax_dev = self.aux_nb_softmax_buf.raw_ptr();
        let aux_label_dev   = self.aux_nb_label_buf.raw_ptr();
        let out_6_dev       = self.aux_dir_acc_buf.dev_ptr;
        let batch_size_i32  = self.config.batch_size as i32;
        let k_i32           = AUX_NEXT_BAR_K as i32;

        // 1. Reduce → mapped-pinned aux_dir_acc_buf [6]
        self.launch_aux_dir_acc_reduce(
            aux_softmax_dev, aux_label_dev, batch_size_i32, k_i32, out_6_dev,
        )?;

        // 2. Short EMA (α=0.3) — fast tracker for aux-w controller deficit term.
        //    n=1: only out_6[0] = dir_acc feeds the EMA; pos_pred/pos_label/
        //    n_down/n_up/n_skip slots are diagnostic-only and don't have ISV
        //    destinations.
        self.launch_apply_fixed_alpha_ema(
            out_6_dev,
            1,
            AUX_DIR_ACC_SHORT_EMA_INDEX as i32,
            0.3,
            DIR_ACC_EMA_SENTINEL,
        )?;

        // 3. Slow EMA (α=0.05) — stagnation detector comparator.
        self.launch_apply_fixed_alpha_ema(
            out_6_dev,
            1,
            AUX_DIR_ACC_LONG_EMA_INDEX as i32,
            0.05,
            DIR_ACC_EMA_SENTINEL,
        )?;

        // 4. mean(softmax[1] - softmax[0]) → ISV[375]. SHARED scalar
        //    (per-step overwrite, no EMA — see kernel header for layout
        //    rationale). Structurally bounded in [-1, +1] (no tanh).
        self.launch_aux_pred_to_isv_tanh(
            aux_softmax_dev,
            batch_size_i32,
            k_i32,
            AUX_DIR_PREDICTION_INDEX as i32,
        )?;

        Ok(())
    }

    /// SP11 Fix 39 (2026-05-04, Task A2): launch the reward-subsystem
    /// controller producer + chained Pearls A+D output smoothing.
    ///
    /// Single-block, 10-thread producer reads 5 canary slots from ISV
    /// [350..360) (val-sharpe Δ + variance, 6 mag-ratios, saboteur
    /// engagement, PnL magnitude EMA) and writes 10 outputs to
    /// `scratch[SCRATCH_SP11_CONTROLLER_BASE..+10)`. A single chained
    /// `apply_pearls_ad_kernel` launch with `n_slots=10` smooths into
    /// ISV[REWARD_POPART_WEIGHT_INDEX=340..CURIOSITY_BOUND_INDEX+1=350) —
    /// the controller's full output block. Both scratch and ISV layouts
    /// are contiguous and matched-stride, so the n_slots=10 single-launch
    /// path is correct (vs splitting into two contiguous-block launches
    /// the way `launch_sp11_mag_ratio_compute` does for its 6+1 split).
    ///
    /// Per spec §3.4.1 (output smoothing) the smoothed outputs are what
    /// downstream consumers (B1) read from ISV — not the raw scratch
    /// values. Pearl A's sentinel-bootstrap fires on the first per-fold
    /// observation; Pearl D's Wiener-α adapts smoothing strength to each
    /// output's own noise level (curiosity may be smoother than weights
    /// etc., adaptively).
    ///
    /// Pre-conditions (must be satisfied before this launch in the same
    /// stream): all 5 canary slots [350..360) are populated. In
    /// `training_loop.rs` the call site fires AFTER A1's three canary
    /// launches and BEFORE the HEALTH_DIAG emit so the controller reads
    /// fresh canary state and the operator sees current outputs.
    pub(crate) fn launch_sp11_reward_subsystem_controller(&self) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp4_wiener_ema::{launch_apply_pearls, ALPHA_META};
        use crate::cuda_pipeline::sp5_isv_slots::SP5_SLOT_BASE;
        use crate::cuda_pipeline::sp11_isv_slots::{
            REWARD_POPART_WEIGHT_INDEX,
            VAL_SHARPE_DELTA_EMA_INDEX, VAL_SHARPE_VAR_EMA_INDEX,
            REWARD_COMPONENT_MAG_RATIO_BASE,
            SABOTEUR_ENGAGEMENT_RATE_INDEX,
            PNL_REWARD_MAGNITUDE_EMA_INDEX,
        };

        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_sp11_reward_subsystem_controller: isv_signals_dev_ptr must be allocated");

        let isv_dev     = self.isv_signals_dev_ptr;
        let scratch_dev = self.producer_step_scratch_buf.dev_ptr;
        let wiener_dev  = self.wiener_state_buf.dev_ptr;

        // Producer writes scratch[SCRATCH_SP11_CONTROLLER_BASE..+10).
        // We pass the offset device pointer as the scratch arg so the
        // kernel's view starts at scratch_out[0] = scratch[base] — same
        // pointer-arithmetic pattern as A1's val_sharpe_delta launcher.
        let scratch_offset_bytes = (SCRATCH_SP11_CONTROLLER_BASE as u64)
            * std::mem::size_of::<f32>() as u64;
        let scratch_out_dev: u64 = scratch_dev + scratch_offset_bytes;

        let delta_ema_slot_i32:           i32 = VAL_SHARPE_DELTA_EMA_INDEX as i32;
        let var_ema_slot_i32:             i32 = VAL_SHARPE_VAR_EMA_INDEX as i32;
        let mag_ratio_base_slot_i32:      i32 = REWARD_COMPONENT_MAG_RATIO_BASE as i32;
        let saboteur_engagement_slot_i32: i32 = SABOTEUR_ENGAGEMENT_RATE_INDEX as i32;
        let pnl_mag_ema_slot_i32:         i32 = PNL_REWARD_MAGNITUDE_EMA_INDEX as i32;

        // Step 1: producer kernel — single block, 10 threads (one per output).
        unsafe {
            self.stream
                .launch_builder(&self.sp11_reward_subsystem_controller_kernel)
                .arg(&isv_dev)
                .arg(&scratch_out_dev)
                .arg(&delta_ema_slot_i32)
                .arg(&var_ema_slot_i32)
                .arg(&mag_ratio_base_slot_i32)
                .arg(&saboteur_engagement_slot_i32)
                .arg(&pnl_mag_ema_slot_i32)
                .launch(LaunchConfig {
                    grid_dim:         (1, 1, 1),
                    block_dim:        (10, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("sp11 reward_subsystem_controller: {e}")))?;
        }

        // Step 2: chained apply_pearls_ad_kernel (n_slots=10) per spec
        // §3.4.1. Both scratch [SCRATCH_SP11_CONTROLLER_BASE..+10) and
        // ISV [REWARD_POPART_WEIGHT_INDEX..+10) are contiguous and
        // matched-stride, so a single n_slots=10 launch covers them.
        //
        // Wiener-state offset for the first output slot:
        //   wiener_off = SP4_PRODUCER_COUNT*3 + (isv_slot - SP5_SLOT_BASE)*3
        // The applicator advances by 3 floats per slot internally.
        let base_wiener_offset = SP4_PRODUCER_COUNT as i32 * 3;
        let isv_idx_base_i32   = REWARD_POPART_WEIGHT_INDEX as i32;
        let scratch_idx_base_i32: i32 = SCRATCH_SP11_CONTROLLER_BASE as i32;
        let wiener_off = base_wiener_offset
            + (isv_idx_base_i32 - SP5_SLOT_BASE as i32) * 3;
        unsafe {
            launch_apply_pearls(
                &self.stream,
                &self.apply_pearls_ad_kernel,
                scratch_dev, scratch_idx_base_i32,
                isv_dev,     isv_idx_base_i32,
                wiener_dev,  wiener_off,
                10,
                ALPHA_META,
            )?;
        }

        Ok(())
    }

    /// SP7 (2026-05-03): launch the GPU dispatch kernel that resolves
    /// `(active >= 0.5) ? max(controller, EPS_DIV) : bootstrap` for every
    /// (head ∈ {CQL, C51}, branch ∈ [0..4)) on-device.
    ///
    /// Reads `ISV[LB_{CQL,C51}_ACTIVE_BASE+b]` (activation flag) and
    /// `ISV[BUDGET_{CQL,C51}_BASE+b]` (controller-derived budget); writes the
    /// trunk_mean + per-branch correction tuples to `lb_budget_effective_buf`
    /// (10 f32 mapped-pinned). Bootstrap constants are passed as kernel args
    /// (CQL=0.02, C51=0.05 — byte-identical to the prior host-side
    /// `CQL_BOOTSTRAP_BUDGET` / `C51_BOOTSTRAP_BUDGET` literals in
    /// `compute_adaptive_budgets` and to `loss_balance_controller_kernel.cu`'s
    /// `COLD_START_FLOOR_*` anchors per `feedback_isv_for_adaptive_bounds`).
    ///
    /// Must run BEFORE `apply_c51_budget_scale` / `apply_cql_saxpy` (and their
    /// per-branch variants) inside the same captured `aux_child` graph. The
    /// downstream SAXPY/scale kernels read `alpha` from a device pointer
    /// pointing into `lb_budget_effective_buf`, so every graph replay sees
    /// fresh values written by this kernel against the runtime ISV state —
    /// no capture-time freeze of the host-side if/else dispatch.
    pub(crate) fn launch_lb_budget_dispatch(&self) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp5_isv_slots::{
            BUDGET_CQL_BASE, BUDGET_C51_BASE,
            LB_CQL_ACTIVE_BASE, LB_C51_ACTIVE_BASE,
        };

        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_lb_budget_dispatch: isv_signals_dev_ptr must be allocated");
        debug_assert_eq!(self.lb_budget_effective_buf.len, 10,
            "launch_lb_budget_dispatch: lb_budget_effective_buf must be 10 f32");

        let isv_dev = self.isv_signals_dev_ptr;
        let out_dev = self.lb_budget_effective_buf.dev_ptr;

        let active_cql_isv_base_i32: i32 = LB_CQL_ACTIVE_BASE as i32;
        let active_c51_isv_base_i32: i32 = LB_C51_ACTIVE_BASE as i32;
        let budget_cql_isv_base_i32: i32 = BUDGET_CQL_BASE as i32;
        let budget_c51_isv_base_i32: i32 = BUDGET_C51_BASE as i32;
        // SP7 cold-start basis — must remain byte-identical to the prior host-
        // side literals in `compute_adaptive_budgets` (CQL_BOOTSTRAP_BUDGET=0.02
        // / C51_BOOTSTRAP_BUDGET=0.05). Passed as kernel args (not hardcoded
        // in kernel body) per `feedback_isv_for_adaptive_bounds`.
        let cql_bootstrap: f32 = 0.02_f32;
        let c51_bootstrap: f32 = 0.05_f32;

        unsafe {
            self.stream
                .launch_builder(&self.lb_budget_dispatch_kernel)
                .arg(&isv_dev)
                .arg(&active_cql_isv_base_i32)
                .arg(&active_c51_isv_base_i32)
                .arg(&budget_cql_isv_base_i32)
                .arg(&budget_c51_isv_base_i32)
                .arg(&cql_bootstrap)
                .arg(&c51_bootstrap)
                .arg(&out_dev)
                .launch(LaunchConfig {
                    grid_dim:         (1, 1, 1),
                    block_dim:        (8, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("lb_budget_dispatch: {e}")))?;
        }

        Ok(())
    }

    /// SP7 (2026-05-03): device pointer to the `cql_trunk_mean` slot in
    /// `lb_budget_effective_buf`. Pass into `apply_cql_saxpy` so the kernel
    /// reads the GPU-resolved budget at execution time (not capture time).
    #[inline]
    pub(crate) fn lb_budget_cql_trunk_dev_ptr(&self) -> u64 {
        // Slot 0 = cql_trunk_mean.
        self.lb_budget_effective_buf.dev_ptr
    }

    /// SP7 (2026-05-03): device pointer to the `cql_correction[branch]` slot.
    /// Pass into `apply_cql_saxpy_branch` so the kernel reads the per-branch
    /// correction at execution time.
    #[inline]
    pub(crate) fn lb_budget_cql_correction_dev_ptr(&self, branch: usize) -> u64 {
        debug_assert!(branch < 4, "branch must be in [0,4)");
        // Slots 1..5 = cql_correction[0..4].
        let f32_size = std::mem::size_of::<f32>() as u64;
        self.lb_budget_effective_buf.dev_ptr + (1 + branch as u64) * f32_size
    }

    /// SP7 (2026-05-03): device pointer to the `c51_trunk_mean` slot.
    #[inline]
    pub(crate) fn lb_budget_c51_trunk_dev_ptr(&self) -> u64 {
        // Slot 5 = c51_trunk_mean.
        let f32_size = std::mem::size_of::<f32>() as u64;
        self.lb_budget_effective_buf.dev_ptr + 5 * f32_size
    }

    /// SP7 (2026-05-03): device pointer to the `c51_correction[branch]` slot.
    #[inline]
    pub(crate) fn lb_budget_c51_correction_dev_ptr(&self, branch: usize) -> u64 {
        debug_assert!(branch < 4, "branch must be in [0,4)");
        // Slots 6..10 = c51_correction[0..4].
        let f32_size = std::mem::size_of::<f32>() as u64;
        self.lb_budget_effective_buf.dev_ptr + (6 + branch as u64) * f32_size
    }

    /// SP7 (2026-05-03): host-side read of the GPU-resolved per-branch
    /// effective budgets and trunk means. Returns `(cql_per_branch, c51_per_branch,
    /// cql_trunk, c51_trunk)`. Reconstructs per-branch effectives from the
    /// stored `(trunk, correction)` decomposition: `effective[b] = trunk × correction[b]`.
    ///
    /// Used by HEALTH_DIAG to surface the dispatched values that actually fed
    /// into SAXPY/scale this step. Caller MUST have `sync_all_streams()`'d the
    /// producing stream first — mapped-pinned coherence requires the kernel to
    /// have completed before host reads (otherwise reads return stale or
    /// in-flight values).
    pub(crate) fn read_lb_budget_effective(&self) -> ([f32; 4], [f32; 4], f32, f32) {
        // Safety: dev pointer and host pointer back the same physical memory
        // (cuMemHostAlloc DEVICEMAP); read_volatile defeats CPU cache.
        let host = self.lb_budget_effective_buf.host_ptr;
        let read = |offset: usize| -> f32 {
            unsafe { std::ptr::read_volatile(host.add(offset)) }
        };
        let cql_trunk = read(0);
        let cql_corr  = [read(1), read(2), read(3), read(4)];
        let c51_trunk = read(5);
        let c51_corr  = [read(6), read(7), read(8), read(9)];
        let cql_per_branch = [
            cql_trunk * cql_corr[0],
            cql_trunk * cql_corr[1],
            cql_trunk * cql_corr[2],
            cql_trunk * cql_corr[3],
        ];
        let c51_per_branch = [
            c51_trunk * c51_corr[0],
            c51_trunk * c51_corr[1],
            c51_trunk * c51_corr[2],
            c51_trunk * c51_corr[3],
        ];
        (cql_per_branch, c51_per_branch, cql_trunk, c51_trunk)
    }

    /// SP5 Task A4 (2026-05-01): launch the two-kernel Pearl 4 chain +
    /// 24 `apply_pearls_ad_kernel` launches to smooth β1/β2/ε into ISV.
    ///
    /// Execution sequence (all on `self.stream`):
    ///   1. `grad_cosine_sim_update` (8 threads, 1 block): reads
    ///      `grad_buf [total_params]` + `grad_prev_buf_per_group [total_params]`
    ///      + `group_param_offsets_buf [9]`; writes per-group cosine_sim[8]
    ///      to scratch[131..139) and l2_norm[8] to scratch[139..147); also
    ///      writes grad_curr → grad_prev_buf_per_group for the next step's comparison.
    ///   2. `pearl_4_adam_hparams_update` (8 threads, 1 block): reads
    ///      scratch[131..147); writes β1[8]→scratch[147..155),
    ///      β2[8]→scratch[155..163), ε[8]→scratch[163..171).
    ///   3. 24 `apply_pearls_ad_kernel` launches (ALPHA_META=5e-4, half the
    ///      SP4 default): smooths β1[8]/β2[8]/ε[8] via Pearls A+D into
    ///      ISV[ADAM_BETA1_BASE..ADAM_BETA1_BASE+8),
    ///      ISV[ADAM_BETA2_BASE..ADAM_BETA2_BASE+8),
    ///      ISV[ADAM_EPS_BASE..ADAM_EPS_BASE+8).
    ///
    /// Pearl 4 is independent of Pearls 1/2/3 outputs — depends only on
    /// grad_buf state. Can run in any order relative to other SP5 launchers.
    ///
    /// Structural envelopes (Invariant 1 anchors per spec line 88):
    ///   β1 ∈ [0.85, 0.95]; β2 ∈ [0.99, 0.9995]; ε ∈ [1e-10, 1e-6].
    /// Fold-boundary reset zeroes grad_prev_buf_per_group (Pearl A sentinel):
    /// first step cosine_sim=0 → β1=0.85, β2=0.99 (low-stability envelope).
    /// Layer A only — Adam optimizer consumer migration deferred to Layer B.
    pub(crate) fn launch_sp5_pearl_4_adam_hparams(&self) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp5_isv_slots::{
            ADAM_BETA1_BASE, ADAM_BETA2_BASE, ADAM_EPS_BASE, SP5_SLOT_BASE,
        };
        use crate::cuda_pipeline::sp4_wiener_ema::launch_apply_pearls;

        const ALPHA_META_PEARL_4: f32 = 5.0e-4_f32; // half SP4 default → slow β change rate

        let scratch_dev = self.producer_step_scratch_buf.dev_ptr;
        let isv_dev     = self.isv_signals_dev_ptr;
        let wiener_dev  = self.wiener_state_buf.dev_ptr;
        // Wiener base offset for SP5 slots: SP4_PRODUCER_COUNT * 3 Wiener slots
        // already consumed; SP5 slots start at offset (SP4_PRODUCER_COUNT * 3).
        let base_wiener_offset: i32 = (SP4_PRODUCER_COUNT * 3) as i32;

        let grad_buf_ptr        = self.ptrs.grad_buf;
        let grad_prev_ptr       = self.grad_prev_buf_per_group.dev_ptr;
        let group_offsets_ptr   = self.group_param_offsets_buf.dev_ptr;
        let cosine_idx_base     = SCRATCH_PEARL_4_COSINE as i32;  // 131
        let l2_norm_idx_base    = SCRATCH_PEARL_4_L2     as i32;  // 139
        let beta1_idx_base      = SCRATCH_PEARL_4_BETA1  as i32;  // 147
        let beta2_idx_base      = SCRATCH_PEARL_4_BETA2  as i32;  // 155
        let eps_idx_base        = SCRATCH_PEARL_4_EPS    as i32;  // 163

        // Step 1: grad_cosine_sim_update — 8 threads, 1 block.
        // Reads grad_buf + grad_prev_buf_per_group + group_param_offsets;
        // writes cosine_sim[8]→scratch[131..139) and l2_norm[8]→scratch[139..147).
        // Also writebacks grad_curr → grad_prev_buf_per_group.
        let cfg_8t = LaunchConfig {
            grid_dim:         (1, 1, 1),
            block_dim:        (8, 1, 1),
            shared_mem_bytes: 0,
        };
        unsafe {
            self.stream
                .launch_builder(&self.grad_cosine_sim_kernel)
                .arg(&grad_buf_ptr)
                .arg(&grad_prev_ptr)
                .arg(&group_offsets_ptr)
                .arg(&scratch_dev)
                .arg(&cosine_idx_base)
                .arg(&l2_norm_idx_base)
                .arg(&grad_prev_ptr)  // writeback = same buffer
                .launch(cfg_8t)
                .map_err(|e| MLError::ModelError(format!("grad_cosine_sim_update launch: {e}")))?;
        }

        // Step 2: pearl_4_adam_hparams_update — 8 threads, 1 block.
        // Reads scratch[131..147) (cosine_sim + l2_norm); writes
        // β1[8]→scratch[147..155), β2[8]→scratch[155..163), ε[8]→scratch[163..171).
        // Stream-ordered with Step 1 (same stream, no explicit sync needed).
        let cosine_ptr = scratch_dev + (SCRATCH_PEARL_4_COSINE * std::mem::size_of::<f32>()) as u64;
        let l2_ptr     = scratch_dev + (SCRATCH_PEARL_4_L2     * std::mem::size_of::<f32>()) as u64;
        unsafe {
            self.stream
                .launch_builder(&self.pearl_4_adam_hparams_kernel)
                .arg(&cosine_ptr)
                .arg(&l2_ptr)
                .arg(&scratch_dev)
                .arg(&beta1_idx_base)
                .arg(&beta2_idx_base)
                .arg(&eps_idx_base)
                .launch(cfg_8t)
                .map_err(|e| MLError::ModelError(format!("pearl_4_adam_hparams_update launch: {e}")))?;
        }

        // Step 3: 24 apply_pearls_ad_kernel launches (ALPHA_META=5e-4).
        // β1[8]: scratch[147..155) → ISV[ADAM_BETA1_BASE..ADAM_BETA1_BASE+8)
        for g in 0..8_usize {
            let scratch_idx = (SCRATCH_PEARL_4_BETA1 + g) as i32;
            let isv_idx     = (ADAM_BETA1_BASE + g) as i32;
            let wiener_off  = base_wiener_offset + (isv_idx - SP5_SLOT_BASE as i32) * 3;
            unsafe {
                launch_apply_pearls(
                    &self.stream,
                    &self.apply_pearls_ad_kernel,
                    scratch_dev, scratch_idx,
                    isv_dev,     isv_idx,
                    wiener_dev,  wiener_off,
                    1,
                    ALPHA_META_PEARL_4,
                )?;
            }
        }

        // β2[8]: scratch[155..163) → ISV[ADAM_BETA2_BASE..ADAM_BETA2_BASE+8)
        for g in 0..8_usize {
            let scratch_idx = (SCRATCH_PEARL_4_BETA2 + g) as i32;
            let isv_idx     = (ADAM_BETA2_BASE + g) as i32;
            let wiener_off  = base_wiener_offset + (isv_idx - SP5_SLOT_BASE as i32) * 3;
            unsafe {
                launch_apply_pearls(
                    &self.stream,
                    &self.apply_pearls_ad_kernel,
                    scratch_dev, scratch_idx,
                    isv_dev,     isv_idx,
                    wiener_dev,  wiener_off,
                    1,
                    ALPHA_META_PEARL_4,
                )?;
            }
        }

        // ε[8]: scratch[163..171) → ISV[ADAM_EPS_BASE..ADAM_EPS_BASE+8)
        for g in 0..8_usize {
            let scratch_idx = (SCRATCH_PEARL_4_EPS + g) as i32;
            let isv_idx     = (ADAM_EPS_BASE + g) as i32;
            let wiener_off  = base_wiener_offset + (isv_idx - SP5_SLOT_BASE as i32) * 3;
            unsafe {
                launch_apply_pearls(
                    &self.stream,
                    &self.apply_pearls_ad_kernel,
                    scratch_dev, scratch_idx,
                    isv_dev,     isv_idx,
                    wiener_dev,  wiener_off,
                    1,
                    ALPHA_META_PEARL_4,
                )?;
            }
        }

        Ok(())
    }

    /// SP5 Pearl 5: per-branch IQN τ schedule from per-branch Q-skew.
    ///
    /// Two-kernel chain on the producer's stream:
    ///   1. `q_skew_kurtosis_update` — computes per-branch (skew, ex_kurt) from
    ///      save_q_online ([B × 13] row-major); writes 8 floats to
    ///      scratch[171..179) (skew[4] at [171..175), ex_kurt[4] at [175..179)).
    ///   2. `pearl_5_iqn_tau_update` — derives 5-quantile τ schedule per branch
    ///      with skew-driven shift + structural-envelope clamp; writes 20 floats
    ///      to scratch[179..199) (branch b, quantile q at 179 + b*5 + q).
    ///
    /// 20 apply_pearls calls follow (4 branches × 5 quantiles), each chained on
    /// the same stream with ALPHA_META=1e-3 (default SP4/SP5 rate).
    ///
    /// Pearl 5 reads save_q_online (forward-pass output); independent of Pearls
    /// 1/2/3/4 ISV outputs.
    ///
    /// Layer A only — IQN τ-quantile-selection consumer migration deferred to
    /// Layer B.
    ///
    /// Wiener offset formula: `base_wiener_offset + (isv_slot - SP5_SLOT_BASE) * 3`
    /// where `base_wiener_offset = SP4_PRODUCER_COUNT * 3 = 213`.
    pub(crate) fn launch_sp5_pearl_5_iqn_tau(&self) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp5_isv_slots::{IQN_TAU_BASE, SP5_SLOT_BASE};
        use crate::cuda_pipeline::sp4_wiener_ema::{launch_apply_pearls, ALPHA_META};

        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_sp5_pearl_5_iqn_tau: isv_signals_dev_ptr must be allocated by constructor");

        let scratch_dev        = self.producer_step_scratch_buf.dev_ptr;
        let isv_dev            = self.isv_signals_dev_ptr;
        let wiener_dev         = self.wiener_state_buf.dev_ptr;
        let action_counts_dev  = self.action_counts_buf.dev_ptr;
        let action_offsets_dev = self.action_offsets_buf.dev_ptr;
        let q_out_ptr          = self.q_out_buf.raw_ptr();
        let b_i32              = self.config.batch_size as i32;

        // Wiener base offset for SP5 slots: SP4_PRODUCER_COUNT * 3 slots consumed.
        let base_wiener_offset: i32 = (SP4_PRODUCER_COUNT * 3) as i32;

        let skew_base_i32     = SCRATCH_PEARL_5_SKEW    as i32;  // 171
        let ex_kurt_base_i32  = SCRATCH_PEARL_5_EX_KURT as i32;  // 175
        let tau_base_i32      = SCRATCH_PEARL_5_TAU     as i32;  // 179

        let cfg_4t = LaunchConfig {
            grid_dim:         (1, 1, 1),
            block_dim:        (4, 1, 1),
            shared_mem_bytes: 0,
        };

        // Step 1: q_skew_kurtosis_update — reads save_q_online [B × 13];
        // writes skew[4] → scratch[171..175), ex_kurt[4] → scratch[175..179).
        unsafe {
            self.stream
                .launch_builder(&self.q_skew_kurtosis_kernel)
                .arg(&q_out_ptr)
                .arg(&b_i32)
                .arg(&action_counts_dev)
                .arg(&action_offsets_dev)
                .arg(&scratch_dev)
                .arg(&skew_base_i32)
                .arg(&ex_kurt_base_i32)
                .launch(cfg_4t)
                .map_err(|e| MLError::ModelError(format!("q_skew_kurtosis_update: {e}")))?;
        }

        // Step 2: pearl_5_iqn_tau_update — reads scratch[171..175) (skew);
        // writes τ[4 branches × 5 quantiles] → scratch[179..199).
        // Stream-ordered with Step 1 (same stream, no explicit sync needed).
        unsafe {
            self.stream
                .launch_builder(&self.pearl_5_iqn_tau_kernel)
                .arg(&scratch_dev)
                .arg(&skew_base_i32)
                .arg(&scratch_dev)
                .arg(&tau_base_i32)
                .launch(cfg_4t)
                .map_err(|e| MLError::ModelError(format!("pearl_5_iqn_tau_update: {e}")))?;
        }

        // Step 3: 20 apply_pearls calls (4 branches × 5 quantiles).
        // τ[b*5 + q]: scratch[179 + b*5 + q] → ISV[IQN_TAU_BASE + b*5 + q].
        for b in 0..4_usize {
            for q in 0..5_usize {
                let isv_slot    = (IQN_TAU_BASE + b * 5 + q) as i32;
                let scratch_idx = (SCRATCH_PEARL_5_TAU + b * 5 + q) as i32;
                let wiener_off  = base_wiener_offset + (isv_slot - SP5_SLOT_BASE as i32) * 3;
                unsafe {
                    launch_apply_pearls(
                        &self.stream,
                        &self.apply_pearls_ad_kernel,
                        scratch_dev, scratch_idx,
                        isv_dev,     isv_slot,
                        wiener_dev,  wiener_off,
                        1,
                        ALPHA_META,
                    )?;
                }
            }
        }

        Ok(())
    }

    /// SP5 Task A6: cross-fold-persistent Kelly cap signals.
    ///
    /// CROSS-FOLD-PERSISTENT — ISV[280..286) are EXEMPT from the fold-reset registry.
    /// The cold-start Quarter pathology (project_magnitude_eval_collapse_kelly_capped.md)
    /// is caused by portfolio_state having WindowReset lifecycle: every window boundary
    /// resets ps[14..18] (Kelly win/loss stats) to zero, forcing every new window to
    /// cold-start at low maturity. Pearl 6 maintains a cross-fold and cross-window
    /// representation so the maturity-driven cold-start cap does not re-engage.
    ///
    /// Investigation finding: portfolio_state is ResetCategory::WindowReset in
    /// state_reset_registry.rs (NOT FoldReset). This makes max() semantics for
    /// sample_count essential for both fold-level and window-level cold-start suppression.
    ///
    /// NO apply_pearls call: Pearls A+D's sentinel-bootstrap (zero ISV → first-observation
    /// replacement) is incompatible with cross-fold persistence — the sentinel would reset
    /// the slot if it were ever zeroed, defeating the whole design. In-kernel slow EWMA
    /// (alpha=0.01, Invariant 1 anchor) is used instead.
    ///
    /// Reads portfolio_state from the experience collector (same source as
    /// launch_kelly_cap_update). Launches on self.stream; portfolio_state is populated
    /// by the experience collector's stream before this epoch-boundary method is called
    /// (same ordering guarantee as launch_kelly_cap_update, called in training_loop.rs).
    pub(crate) fn launch_sp5_pearl_6_kelly(
        &self,
        portfolio_state_dev_ptr: u64,
        n_envs: i32,
    ) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp5_isv_slots::{
            KELLY_F_SMOOTH_INDEX, CONVICTION_SMOOTH_INDEX, TRADE_VAR_SMOOTH_INDEX,
            KELLY_SAMPLE_COUNT_INDEX, WIN_RATE_SMOOTH_INDEX, LOSS_RATE_SMOOTH_INDEX,
        };

        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_sp5_pearl_6_kelly: isv_signals_dev_ptr must be allocated");

        let isv_dev              = self.isv_signals_dev_ptr;
        let ps_stride_i32: i32   = 43; // PS_STRIDE from state_layout.cuh (grown 41→43 by Plan 3 D.4c)
        let kelly_f_idx_i32        = KELLY_F_SMOOTH_INDEX      as i32;
        let conviction_idx_i32     = CONVICTION_SMOOTH_INDEX   as i32;
        let trade_var_idx_i32      = TRADE_VAR_SMOOTH_INDEX     as i32;
        let sample_count_idx_i32   = KELLY_SAMPLE_COUNT_INDEX  as i32;
        let win_rate_idx_i32       = WIN_RATE_SMOOTH_INDEX      as i32;
        let loss_rate_idx_i32      = LOSS_RATE_SMOOTH_INDEX     as i32;

        unsafe {
            self.stream
                .launch_builder(&self.pearl_6_kelly_kernel)
                .arg(&portfolio_state_dev_ptr)
                .arg(&n_envs)
                .arg(&ps_stride_i32)
                .arg(&isv_dev)
                .arg(&kelly_f_idx_i32)
                .arg(&conviction_idx_i32)
                .arg(&trade_var_idx_i32)
                .arg(&sample_count_idx_i32)
                .arg(&win_rate_idx_i32)
                .arg(&loss_rate_idx_i32)
                .launch(LaunchConfig {
                    grid_dim:         (1, 1, 1),
                    block_dim:        (6, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("pearl_6_kelly_update: {e}")))?;
        }

        Ok(())
    }

    /// SP5 Task A7: launch `pearl_8_trail_update` — 4-thread single-block ISV producer.
    ///
    /// Reads the fold window's last bar ATR from `features[bar_idx * market_dim + 9]`
    /// (ATR_NORM column, canonical fxcache scheme). Writes 4 trail-distance floats to
    /// producer_step_scratch_buf[SCRATCH_PEARL_8_TRAIL=199..203), then chains 4
    /// apply_pearls_ad_kernel calls → ISV[TRAIL_DIST_PER_DIR_BASE=270..274).
    ///
    /// Short[270] and Long[272]: `atr_abs × ATR_TRAIL_FACTOR=2.0`.
    /// Hold[271] and Flat[273]: `EPS_CLAMP_FLOOR=1.0` (no trail-stop).
    ///
    /// Layer A simplification: Short and Long share the same current-bar ATR.
    /// Direction-conditional ATR tracking deferred to Layer C.
    ///
    /// Arguments:
    ///   - `features_dev_ptr`: u64 device pointer for the raw feature buffer
    ///     (same layout as `features_raw_cuda` uploaded by `init_from_fxcache`).
    ///   - `market_dim`: feature stride per bar (= `config.market_dim`, typically 48 w/ OFI).
    ///   - `bar_idx`: row index to read; caller passes `num_bars - 1` (last bar of fold).
    ///
    /// Per `feedback_no_cpu_compute_strict` — all arithmetic (ATR denorm, trail calc)
    /// on GPU. Per `feedback_no_atomicadd` — no atomicAdd; 4 independent thread writes.
    pub(crate) fn launch_sp5_pearl_8_trail(
        &self,
        features_dev_ptr: u64,
        market_dim: i32,
        bar_idx: i32,
    ) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp5_isv_slots::{TRAIL_DIST_PER_DIR_BASE, SP5_SLOT_BASE};
        use crate::cuda_pipeline::sp4_wiener_ema::{launch_apply_pearls, ALPHA_META};

        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_sp5_pearl_8_trail: isv_signals_dev_ptr must be allocated by constructor");

        let scratch_dev         = self.producer_step_scratch_buf.dev_ptr;
        let isv_dev             = self.isv_signals_dev_ptr;
        let wiener_dev          = self.wiener_state_buf.dev_ptr;
        let trail_idx_base_i32  = SCRATCH_PEARL_8_TRAIL as i32;

        // Wiener base offset for SP5 slots: SP4_PRODUCER_COUNT * 3 = 213.
        let base_wiener_offset: i32 = (SP4_PRODUCER_COUNT * 3) as i32;

        // Step 1: pearl_8_trail_update — reads features[bar_idx * market_dim + 9];
        // writes trail_dist[4 directions] → scratch[199..203).
        unsafe {
            self.stream
                .launch_builder(&self.pearl_8_trail_kernel)
                .arg(&features_dev_ptr)
                .arg(&bar_idx)
                .arg(&market_dim)
                .arg(&scratch_dev)
                .arg(&trail_idx_base_i32)
                .launch(LaunchConfig {
                    grid_dim:         (1, 1, 1),
                    block_dim:        (4, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("pearl_8_trail_update: {e}")))?;
        }

        // Step 2: 4 apply_pearls calls (one per direction).
        // direction d: scratch[199 + d] → ISV[TRAIL_DIST_PER_DIR_BASE + d].
        for d in 0..4_usize {
            let isv_slot    = (TRAIL_DIST_PER_DIR_BASE + d) as i32;
            let scratch_idx = (SCRATCH_PEARL_8_TRAIL + d) as i32;
            let wiener_off  = base_wiener_offset + (isv_slot - SP5_SLOT_BASE as i32) * 3;
            unsafe {
                launch_apply_pearls(
                    &self.stream,
                    &self.apply_pearls_ad_kernel,
                    scratch_dev, scratch_idx,
                    isv_dev,     isv_slot,
                    wiener_dev,  wiener_off,
                    1,
                    ALPHA_META,
                )?;
            }
        }

        Ok(())
    }

    /// SP5 Pearl 1-ext: per-branch C51 num_atoms from per-branch v_half (Pearl 1).
    ///
    /// Reads ATOM_V_HALF[178..182) (populated by Pearl 1 in A1), maps to discrete
    /// {64, 32, 16} via threshold cascade:
    ///   v_half < 0.1  → 64 atoms (narrow Q range, high resolution)
    ///   v_half < 1.0  → 32 atoms (moderate Q range)
    ///   v_half ≥ 1.0  → 16 atoms (wide Q range, modest resolution)
    /// Pearls A+D smooths the discrete output during transitions; Layer B's
    /// atoms_update consumer rounds to the nearest valid count at consume time.
    ///
    /// Single producer kernel + 4 apply_pearls calls (one per branch).
    /// Wiener offset formula: `SP4_PRODUCER_COUNT * 3 + (isv_slot - SP5_SLOT_BASE) * 3`.
    pub(crate) fn launch_sp5_pearl_1_ext_num_atoms(&self) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp5_isv_slots::{ATOM_V_HALF_BASE, ATOM_NUM_ATOMS_BASE, SP5_SLOT_BASE};
        use crate::cuda_pipeline::sp4_wiener_ema::{launch_apply_pearls, ALPHA_META};

        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_sp5_pearl_1_ext_num_atoms: isv_signals_dev_ptr must be allocated");

        let scratch_dev          = self.producer_step_scratch_buf.dev_ptr;
        let isv_dev              = self.isv_signals_dev_ptr;
        let wiener_dev           = self.wiener_state_buf.dev_ptr;
        let v_half_base_i32      = ATOM_V_HALF_BASE as i32;
        let num_atoms_base_i32   = SCRATCH_PEARL_1_EXT_NUM_ATOMS as i32;

        // Wiener base offset for SP5 slots: SP4_PRODUCER_COUNT * 3 = 213.
        let base_wiener_offset: i32 = (SP4_PRODUCER_COUNT * 3) as i32;

        // Step 1: pearl_1_ext_num_atoms_update — reads ISV[ATOM_V_HALF_BASE..+4];
        // writes num_atoms[4 branches] → scratch[203..207).
        unsafe {
            self.stream
                .launch_builder(&self.pearl_1_ext_num_atoms_kernel)
                .arg(&isv_dev)
                .arg(&v_half_base_i32)
                .arg(&scratch_dev)
                .arg(&num_atoms_base_i32)
                .launch(LaunchConfig {
                    grid_dim:         (1, 1, 1),
                    block_dim:        (4, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("pearl_1_ext_num_atoms_update: {e}")))?;
        }

        // Step 2: 4 apply_pearls calls (one per branch).
        // branch b: scratch[203 + b] → ISV[ATOM_NUM_ATOMS_BASE + b].
        for b in 0..4_usize {
            let isv_slot    = (ATOM_NUM_ATOMS_BASE + b) as i32;
            let scratch_idx = (SCRATCH_PEARL_1_EXT_NUM_ATOMS + b) as i32;
            let wiener_off  = base_wiener_offset + (isv_slot - SP5_SLOT_BASE as i32) * 3;
            unsafe {
                launch_apply_pearls(
                    &self.stream,
                    &self.apply_pearls_ad_kernel,
                    scratch_dev, scratch_idx,
                    isv_dev,     isv_slot,
                    wiener_dev,  wiener_off,
                    1,
                    ALPHA_META,
                )?;
            }
        }

        Ok(())
    }

    /// SP5 Layer D Task D1 (rewrite, 2026-05-02): launch `pnl_aggregation_update` —
    /// single-block 256-thread shmem-reduce producer for ISV[PNL_TOTAL_INDEX=286..PNL_MAX_DD_INDEX+1=290).
    ///
    /// **Rewrite supersedes the original D1 launcher.** The previous D4 agent
    /// found that the original kernel was authored against a brief assuming
    /// per-trade event arrays exist in production; production has per-bar
    /// `step_returns` + `done_flags` arrays plus already-aggregated summary
    /// scalars (`sum_returns`, `sum_sq_returns`, `n_trades`) on `TradeStats`.
    /// The kernel and launcher now consume the actual production data shape
    /// and reproduce `compute_epoch_financials` (financials.rs) bit-for-bit.
    ///
    /// Reads per-bar arrays — `step_returns[num_bars]`, `done_flags[num_bars]`
    /// — plus the already-reduced per-trade summary scalars and writes 4
    /// aggregated floats to scratch_buf[SCRATCH_PNL_AGG_BASE=207..211):
    ///   [207] pnl_total   `exp(Σ ln(max(1+r, 1e-10))) − 1` (log-space compounded)
    ///   [208] pnl_mean    `sum_returns / n_trades`         (per-trade)
    ///   [209] pnl_var     `max(0, sum_sq_returns/n_trades − mean²)` (per-trade)
    ///   [210] pnl_max_dd  per-bar equity walk over last 10K bars with episode
    ///                     resets at `done_flags > 0.5`, capped at 1.0
    ///
    /// Then chains 4 `apply_pearls_ad_kernel` calls smoothing through Pearls A+D
    /// into ISV[286..290) on the same stream (no host sync per
    /// `pearl_cold_path_no_exception_to_gpu_drives.md` and the SP5 GPU-only
    /// Pearls refactor).
    ///
    /// Empty-input contract: when `num_bars == 0` the kernel writes 0.0 to
    /// every output slot (no NaN/garbage); the apply_pearls chain still runs
    /// every step so the consumer (D4) sees a valid ISV value every step.
    /// `n_trades == 0` ⇒ mean and var both 0 (financials guards via
    /// `total_trades > 1`).
    ///
    /// Per `feedback_no_cpu_compute_strict.md` — replaces the host-side
    /// aggregation in `compute_epoch_financials` (`financials.rs:32-228`,
    /// invoked from `training_loop.rs ~4862-4916`).
    /// Per `feedback_no_atomicadd.md` — single-block tree-reduce; no atomicAdd.
    /// Per `feedback_no_partial_refactor.md` — D1 is additive only; D4 atomically
    /// migrates the host-side site to call this launcher in lockstep with D2/D3.
    /// Per `feedback_trust_code_not_docs.md` — formulas mirror the host oracle,
    /// not the original spec brief which spoke of per-trade arrays.
    ///
    /// Arguments:
    ///   - `step_returns`: mapped-pinned `[num_bars]` f32 — per-bar log/arith
    ///     returns matching `TradeStats::step_returns` (downcast f64 → f32 on
    ///     write).
    ///   - `done_flags`: mapped-pinned `[num_bars]` f32 — `1.0` = end of
    ///     episode (capital-floor reset boundary), `0.0` otherwise. Same length
    ///     as `step_returns`. Source: `TradeStats::done_flags`.
    ///   - `num_bars`: count of populated entries in the two per-bar arrays
    ///     (`<= step_returns.len`); the launcher does not validate this
    ///     against buffer length — caller's responsibility.
    ///   - `n_trades`: per-trade event count (`TradeStats::total_trades`)
    ///     used as the mean / variance denominator. Per-trade is the financials
    ///     oracle's choice; per-bar mean would change metric semantics.
    ///   - `sum_returns`: per-trade `Σ returns` already reduced on GPU by the
    ///     experience collector. f32 (downcast from f64 on write).
    ///   - `sum_sq_returns`: per-trade `Σ returns²` already reduced. f32.
    ///   - `initial_capital`: equity-curve starting capital for the max-DD walk.
    ///     Pass-through of the host `initial_capital` financials parameter
    ///     (default `100_000`).
    pub(crate) fn launch_sp5_pnl_aggregation(
        &self,
        step_returns:    &super::mapped_pinned::MappedF32Buffer,
        done_flags:      &super::mapped_pinned::MappedF32Buffer,
        num_bars:        i32,
        n_trades:        i32,
        sum_returns:     f32,
        sum_sq_returns:  f32,
        initial_capital: f32,
    ) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp5_isv_slots::{
            PNL_TOTAL_INDEX, PNL_MEAN_INDEX, PNL_VAR_INDEX, PNL_MAX_DD_INDEX, SP5_SLOT_BASE,
        };
        use crate::cuda_pipeline::sp4_wiener_ema::{launch_apply_pearls, ALPHA_META};

        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_sp5_pnl_aggregation: isv_signals_dev_ptr must be allocated by constructor");
        debug_assert!(num_bars >= 0,
            "launch_sp5_pnl_aggregation: num_bars must be non-negative, got {num_bars}");
        debug_assert!(n_trades >= 0,
            "launch_sp5_pnl_aggregation: n_trades must be non-negative, got {n_trades}");
        debug_assert!(step_returns.len == done_flags.len,
            "launch_sp5_pnl_aggregation: step_returns/done_flags length mismatch ({} vs {})",
            step_returns.len, done_flags.len);
        debug_assert!((num_bars as usize) <= step_returns.len,
            "launch_sp5_pnl_aggregation: num_bars={num_bars} exceeds step_returns.len={}",
            step_returns.len);

        let scratch_dev = self.producer_step_scratch_buf.dev_ptr;
        let isv_dev     = self.isv_signals_dev_ptr;
        let wiener_dev  = self.wiener_state_buf.dev_ptr;

        let returns_dev = step_returns.dev_ptr;
        let dones_dev   = done_flags.dev_ptr;

        let pnl_total_idx_i32:  i32 = (SCRATCH_PNL_AGG_BASE + 0) as i32;
        let pnl_mean_idx_i32:   i32 = (SCRATCH_PNL_AGG_BASE + 1) as i32;
        let pnl_var_idx_i32:    i32 = (SCRATCH_PNL_AGG_BASE + 2) as i32;
        let pnl_max_dd_idx_i32: i32 = (SCRATCH_PNL_AGG_BASE + 3) as i32;

        // Wiener base offset for SP5 slots: SP4_PRODUCER_COUNT * 3 = 213.
        let base_wiener_offset: i32 = (SP4_PRODUCER_COUNT * 3) as i32;

        // Step 1: pnl_aggregation_update — reads per-bar step_returns / done_flags
        // + already-reduced per-trade summary scalars; writes 4 aggregated
        // floats to scratch[SCRATCH_PNL_AGG_BASE..+4). 256 threads × 1 block;
        // shared mem = 256 × sizeof(f32) = 1024 bytes (single bank holding the
        // log-growth tree-reduce; the per-trade scalars don't need a bank).
        let block_dim:        u32 = 256;
        let shared_mem_bytes: u32 = block_dim * std::mem::size_of::<f32>() as u32;
        unsafe {
            self.stream
                .launch_builder(&self.pnl_aggregation_kernel)
                .arg(&returns_dev)
                .arg(&dones_dev)
                .arg(&num_bars)
                .arg(&n_trades)
                .arg(&sum_returns)
                .arg(&sum_sq_returns)
                .arg(&initial_capital)
                .arg(&scratch_dev)
                .arg(&pnl_total_idx_i32)
                .arg(&pnl_mean_idx_i32)
                .arg(&pnl_var_idx_i32)
                .arg(&pnl_max_dd_idx_i32)
                .launch(LaunchConfig {
                    grid_dim:         (1, 1, 1),
                    block_dim:        (block_dim, 1, 1),
                    shared_mem_bytes,
                })
                .map_err(|e| MLError::ModelError(format!("pnl_aggregation_update: {e}")))?;
        }

        // Step 2: 4 apply_pearls calls (one per output slot). Each scratch[207+k]
        // → ISV[286+k] with its own Wiener triple at offset
        // (base_wiener_offset + (isv_slot − SP5_SLOT_BASE) * 3).
        let isv_slots: [usize; 4] = [
            PNL_TOTAL_INDEX,
            PNL_MEAN_INDEX,
            PNL_VAR_INDEX,
            PNL_MAX_DD_INDEX,
        ];
        for k in 0..4_usize {
            let isv_slot    = isv_slots[k] as i32;
            let scratch_idx = (SCRATCH_PNL_AGG_BASE + k) as i32;
            let wiener_off  = base_wiener_offset + (isv_slot - SP5_SLOT_BASE as i32) * 3;
            unsafe {
                launch_apply_pearls(
                    &self.stream,
                    &self.apply_pearls_ad_kernel,
                    scratch_dev, scratch_idx,
                    isv_dev,     isv_slot,
                    wiener_dev,  wiener_off,
                    1,
                    ALPHA_META,
                )?;
            }
        }

        Ok(())
    }

    /// SP5 Layer D Task D2 (2026-05-02): launch `health_composition_update` —
    /// single-block 1-thread arithmetic producer for ISV[HEALTH_SCORE_INDEX=290..GRAD_NORM_NORM_INDEX+1=294).
    ///
    /// Reads 7 raw input signals as f32 scalars (passed by value; no ISV reads),
    /// reproduces `learning_health.rs::NormalizedComponents::from_raw` +
    /// `compose()` bit-for-bit on GPU, and writes 4 aggregated floats to
    /// scratch_buf[SCRATCH_HEALTH_COMP_BASE=211..215):
    ///   [211] composed health_score   (Layer-1 weighted sum, in [0,1])
    ///   [212] q_gap_norm              (smoothstep(0.01, 0.5, q_gap))
    ///   [213] q_var_norm              (smoothstep(0.001, 0.1, q_var))
    ///   [214] grad_norm_norm          (1 − smoothstep(10, 100, grad_norm))
    ///
    /// Then chains 4 `apply_pearls_ad_kernel` calls smoothing through Pearls
    /// A+D into ISV[290..294) on the same stream (no host sync per
    /// `pearl_cold_path_no_exception_to_gpu_drives.md` and the SP5 GPU-only
    /// Pearls refactor).
    ///
    /// Per `feedback_no_cpu_compute_strict.md` — replaces the host-side
    /// `LearningHealth` composition currently driven from
    /// `training_loop.rs:2658-2745`. Per `feedback_no_partial_refactor.md`
    /// — D2 is additive only; D4 atomically migrates the host-side site to
    /// call this launcher in lockstep with D1/D3.
    ///
    /// **Formula fidelity:** the kernel reproduces the host-side
    /// `NormalizedComponents::from_raw` + `NormalizedComponents::compose`
    /// chain verbatim — same edge constants, same weights, no algorithmic
    /// change. Only the EMA + warmup + clamp wrapper from
    /// `LearningHealth::update` is left to the host (and to the downstream
    /// apply_pearls smoothing); D2 is structurally additive.
    ///
    /// Arguments (all f32 by value, no mapped-pinned plumbing per the
    /// `learning_health.rs` pipeline shape):
    ///   - `q_gap`: per-branch Q-gap mean (host-aggregated EMA, 4 branches averaged).
    ///   - `q_var`: epoch q_max − q_min range proxy.
    ///   - `atom_util`: C51 atom utilisation fraction in [0,1].
    ///   - `grad_norm`: |avg_grad| from the epoch (positive scalar).
    ///   - `ens_disagreement`: dispersion across per-branch Q-gaps (range proxy).
    ///   - `grad_consistency`: Adam m-flat cosine similarity (or scalar fallback).
    ///   - `spectral_gap`: σ₁/σ₂ from rolling Gram + power iteration.
    pub(crate) fn launch_health_composition(
        &self,
        q_gap:            f32,
        q_var:            f32,
        atom_util:        f32,
        grad_norm:        f32,
        ens_disagreement: f32,
        grad_consistency: f32,
        spectral_gap:     f32,
    ) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp5_isv_slots::{
            HEALTH_SCORE_INDEX, Q_GAP_NORM_INDEX, Q_VAR_NORM_INDEX, GRAD_NORM_NORM_INDEX,
            SP5_SLOT_BASE,
        };
        use crate::cuda_pipeline::sp4_wiener_ema::{launch_apply_pearls, ALPHA_META};

        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_health_composition: isv_signals_dev_ptr must be allocated by constructor");

        let scratch_dev = self.producer_step_scratch_buf.dev_ptr;
        let isv_dev     = self.isv_signals_dev_ptr;
        let wiener_dev  = self.wiener_state_buf.dev_ptr;

        let health_score_idx_i32:   i32 = (SCRATCH_HEALTH_COMP_BASE + 0) as i32;
        let q_gap_norm_idx_i32:     i32 = (SCRATCH_HEALTH_COMP_BASE + 1) as i32;
        let q_var_norm_idx_i32:     i32 = (SCRATCH_HEALTH_COMP_BASE + 2) as i32;
        let grad_norm_norm_idx_i32: i32 = (SCRATCH_HEALTH_COMP_BASE + 3) as i32;

        // Wiener base offset for SP5 slots: SP4_PRODUCER_COUNT * 3 = 213.
        let base_wiener_offset: i32 = (SP4_PRODUCER_COUNT * 3) as i32;

        // Step 1: health_composition_update — single-block 1-thread; no shared
        // memory needed (composition is 7 reads / 7 smoothsteps / 7 multiply-
        // adds in registers).
        unsafe {
            self.stream
                .launch_builder(&self.health_composition_kernel)
                .arg(&q_gap)
                .arg(&q_var)
                .arg(&atom_util)
                .arg(&grad_norm)
                .arg(&ens_disagreement)
                .arg(&grad_consistency)
                .arg(&spectral_gap)
                .arg(&scratch_dev)
                .arg(&health_score_idx_i32)
                .arg(&q_gap_norm_idx_i32)
                .arg(&q_var_norm_idx_i32)
                .arg(&grad_norm_norm_idx_i32)
                .launch(LaunchConfig {
                    grid_dim:         (1, 1, 1),
                    block_dim:        (1, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("health_composition_update: {e}")))?;
        }

        // Step 2: 4 apply_pearls calls (one per output slot). Each
        // scratch[211+k] → ISV[290+k] with its own Wiener triple at offset
        // (base_wiener_offset + (isv_slot − SP5_SLOT_BASE) * 3).
        let isv_slots: [usize; 4] = [
            HEALTH_SCORE_INDEX,
            Q_GAP_NORM_INDEX,
            Q_VAR_NORM_INDEX,
            GRAD_NORM_NORM_INDEX,
        ];
        for k in 0..4_usize {
            let isv_slot    = isv_slots[k] as i32;
            let scratch_idx = (SCRATCH_HEALTH_COMP_BASE + k) as i32;
            let wiener_off  = base_wiener_offset + (isv_slot - SP5_SLOT_BASE as i32) * 3;
            unsafe {
                launch_apply_pearls(
                    &self.stream,
                    &self.apply_pearls_ad_kernel,
                    scratch_dev, scratch_idx,
                    isv_dev,     isv_slot,
                    wiener_dev,  wiener_off,
                    1,
                    ALPHA_META,
                )?;
            }
        }

        Ok(())
    }

    /// SP5 Layer D Task D3 (2026-05-02): launch `training_metrics_ema_update` —
    /// single-block 3-thread EMA producer for
    /// ISV[TRAINING_SHARPE_EMA_INDEX=294..LOW_DD_RATIO_INDEX+1=297).
    ///
    /// Reproduces the host-side EMA arithmetic block at
    /// `training_loop.rs:5039-5113` bit-for-bit on GPU per
    /// `feedback_no_cpu_compute_strict.md`. One thread per metric:
    ///
    ///   tid=0  training_sharpe_ema  α = clamp(0.05 + 0.3·|err|, 0, 0.5);
    ///                               sharpe_initialized==0 ⇒ replace.
    ///   tid=1  max_dd_ema           α = 0.1; prev<1e-12 ⇒ replace.
    ///   tid=2  low_dd_ratio         α = 0.15 over binary
    ///                               `is_low = (max_dd < new_max_dd_ema·0.5)
    ///                               ? 1 : 0`. Reads tid=1's *new* max_dd_ema
    ///                               via __syncthreads(); no host-side sentinel.
    ///
    /// All EMA constants (0.05, 0.3, 0.5, 0.1, 0.85, 0.15, 0.5 threshold,
    /// 1e-12 sentinel) are Invariant 1 anchors — deleted host formula
    /// literals migrated verbatim per `feedback_no_quickfixes.md` and
    /// `feedback_trust_code_not_docs.md` (the third metric is `low_dd_ratio`
    /// from `training_loop.rs:5052`, not the speculative `gamma_blend`
    /// the SP5 plan §D Task D3 mentions — that field does not exist in
    /// the codebase).
    ///
    /// Writes 3 floats to scratch_buf[SCRATCH_TRAINING_METRICS_EMA_BASE
    /// =215..218):
    ///   [215] new training_sharpe_ema
    ///   [216] new max_dd_ema
    ///   [217] new low_dd_ratio
    ///
    /// Then chains 3 `apply_pearls_ad_kernel` calls smoothing through
    /// Pearls A+D into ISV[294..297) on the same stream (no host sync per
    /// `pearl_cold_path_no_exception_to_gpu_drives.md` and the SP5 GPU-only
    /// Pearls refactor).
    ///
    /// Per `feedback_no_partial_refactor.md` — D3 is additive only; D4
    /// atomically migrates the host-side site to call this launcher in
    /// lockstep with D1/D2. The kernel takes the host-side prev EMA values
    /// and the `_initialized` flag by value in additive D3; D4 will lift
    /// these to mapped-pinned scalars so the host is no longer the source
    /// of truth.
    ///
    /// Arguments (all primitive scalars by value, no mapped-pinned plumbing
    /// per the additive-D3 design):
    ///   - `epoch_sharpe`: per-epoch Sharpe ratio observation (raw).
    ///   - `epoch_max_dd`: per-epoch max drawdown observation (raw, ∈ [0, 1]).
    ///   - `prev_sharpe_ema`: host-tracked previous training_sharpe_ema.
    ///   - `prev_max_dd_ema`: host-tracked previous max_dd_ema (cast to f32).
    ///   - `prev_low_dd_ratio`: host-tracked previous low_dd_ratio (cast to f32).
    ///   - `sharpe_initialized`: host-tracked sentinel flag; `false` ⇒ first
    ///     observation replaces the slot directly. Cast to `i32` at the
    ///     kernel boundary per `feedback_cudarc_f64_f32_abi.md` (cudarc's
    ///     `bool` ABI is unstable; `i32` round-trip is the safe pattern).
    pub(crate) fn launch_training_metrics_ema(
        &self,
        epoch_sharpe:       f32,
        epoch_max_dd:       f32,
        prev_sharpe_ema:    f32,
        prev_max_dd_ema:    f32,
        prev_low_dd_ratio:  f32,
        sharpe_initialized: bool,
    ) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp5_isv_slots::{
            TRAINING_SHARPE_EMA_INDEX, MAX_DD_EMA_INDEX, LOW_DD_RATIO_INDEX,
            SP5_SLOT_BASE,
        };
        use crate::cuda_pipeline::sp4_wiener_ema::{launch_apply_pearls, ALPHA_META};

        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_training_metrics_ema: isv_signals_dev_ptr must be allocated by constructor");

        let scratch_dev = self.producer_step_scratch_buf.dev_ptr;
        let isv_dev     = self.isv_signals_dev_ptr;
        let wiener_dev  = self.wiener_state_buf.dev_ptr;

        let sharpe_idx_i32:       i32 = (SCRATCH_TRAINING_METRICS_EMA_BASE + 0) as i32;
        let max_dd_idx_i32:       i32 = (SCRATCH_TRAINING_METRICS_EMA_BASE + 1) as i32;
        let low_dd_ratio_idx_i32: i32 = (SCRATCH_TRAINING_METRICS_EMA_BASE + 2) as i32;

        // i32 sentinel flag (cudarc `bool` ABI is unstable — cast to i32 here
        // matching the kernel's `int sharpe_initialized` parameter; per
        // `feedback_cudarc_f64_f32_abi.md` the launcher's primitive type
        // must match the kernel's declared C type exactly).
        let sharpe_initialized_i32: i32 = if sharpe_initialized { 1 } else { 0 };

        // Wiener base offset for SP5 slots: SP4_PRODUCER_COUNT * 3 = 213.
        let base_wiener_offset: i32 = (SP4_PRODUCER_COUNT * 3) as i32;

        // Step 1: training_metrics_ema_update — single-block 3-thread; no
        // dynamic shared memory (the kernel uses a single static __shared__
        // float for the tid=1 → tid=2 max_dd_ema handoff).
        unsafe {
            self.stream
                .launch_builder(&self.training_metrics_ema_kernel)
                .arg(&epoch_sharpe)
                .arg(&epoch_max_dd)
                .arg(&prev_sharpe_ema)
                .arg(&prev_max_dd_ema)
                .arg(&prev_low_dd_ratio)
                .arg(&sharpe_initialized_i32)
                .arg(&scratch_dev)
                .arg(&sharpe_idx_i32)
                .arg(&max_dd_idx_i32)
                .arg(&low_dd_ratio_idx_i32)
                .launch(LaunchConfig {
                    grid_dim:         (1, 1, 1),
                    block_dim:        (3, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("training_metrics_ema_update: {e}")))?;
        }

        // Step 2: 3 apply_pearls calls (one per output slot). Each
        // scratch[215+k] → ISV[294+k] with its own Wiener triple at offset
        // (base_wiener_offset + (isv_slot − SP5_SLOT_BASE) * 3). Contract
        // mirrors D1/D2 exactly.
        let isv_slots: [usize; 3] = [
            TRAINING_SHARPE_EMA_INDEX,
            MAX_DD_EMA_INDEX,
            LOW_DD_RATIO_INDEX,
        ];
        for k in 0..3_usize {
            let isv_slot    = isv_slots[k] as i32;
            let scratch_idx = (SCRATCH_TRAINING_METRICS_EMA_BASE + k) as i32;
            let wiener_off  = base_wiener_offset + (isv_slot - SP5_SLOT_BASE as i32) * 3;
            unsafe {
                launch_apply_pearls(
                    &self.stream,
                    &self.apply_pearls_ad_kernel,
                    scratch_dev, scratch_idx,
                    isv_dev,     isv_slot,
                    wiener_dev,  wiener_off,
                    1,
                    ALPHA_META,
                )?;
            }
        }

        Ok(())
    }

    /// Plan C Phase 2 follow-up A.2 (2026-04-29): launch `q_drift_rate_ema_update`
    /// — single-thread single-block ISV producer for ISV[Q_DRIFT_RATE_INDEX=129].
    ///
    /// Reads ISV[Q_ABS_REF_INDEX] + ISV[Q_DIR_ABS_REF_INDEX] plus the
    /// host-passed `q_mean_curr` / `q_mean_prev` scalars; writes
    /// `clip(|q_curr − q_prev| / max(|q_prev|, ISV[16]+ISV[21], 1e-6), 0, 4)`
    /// to ISV[129]. Consumer (`tau_update_kernel.cu`) multiplies the
    /// cosine-scheduled `tau_base` by `1 / (1 + ISV[129])` so the effective
    /// Polyak rate dampens monotonically as q-drift rises (range
    /// `[tau_base/5, tau_base]`).
    ///
    /// Cold-path cadence — invoked once per epoch boundary AFTER the per-
    /// step ISV producers (`launch_h_s2_rms_ema`, `launch_aux_heads_loss_ema`,
    /// etc.) but BEFORE the next epoch's `launch_tau_update` reads the slot.
    /// CPU-born inputs (`q_mean_curr`, `q_mean_prev`) are schedule values per
    /// spec §4.C.6 — passed by value, not via mapped-pinned plumbing.
    ///
    /// Per `pearl_cold_path_no_exception_to_gpu_drives.md` — even a cold-path
    /// scalar arithmetic stays on GPU when its inputs already live on GPU
    /// (ISV[16], ISV[21] both produced by `q_stats_kernel.cu`).
    pub fn launch_q_drift_rate_ema(
        &self,
        q_mean_curr: f32,
        q_mean_prev: f32,
    ) -> Result<(), MLError> {
        // Same invariant rationale as `launch_h_s2_rms_ema`. Loud failure
        // (debug_assert + kernel-launch error) preferred over silent skip.
        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_q_drift_rate_ema: isv_signals_dev_ptr must be allocated by constructor");
        let isv_dev_ptr = self.isv_signals_dev_ptr;
        let q_abs_ref_idx     = Q_ABS_REF_INDEX as i32;
        let q_dir_abs_ref_idx = Q_DIR_ABS_REF_INDEX as i32;
        let q_drift_rate_idx  = Q_DRIFT_RATE_INDEX as i32;
        unsafe {
            self.stream.launch_builder(&self.q_drift_rate_ema_kernel)
                .arg(&isv_dev_ptr)
                .arg(&q_abs_ref_idx)
                .arg(&q_dir_abs_ref_idx)
                .arg(&q_drift_rate_idx)
                .arg(&q_mean_curr)
                .arg(&q_mean_prev)
                .launch(LaunchConfig {
                    grid_dim:        (1, 1, 1),
                    block_dim:       (1, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("q_drift_rate_ema_update: {e}")))?;
        }
        Ok(())
    }

    /// Plan 4 Task 1B-iii: launch `vsn_mask_ema_update` (single-block,
    /// 256-thread shmem-reduction kernel; no atomicAdd). Reads
    /// `vsn_mask_buf [B, num_groups]` saved by the online-on-states VSN
    /// forward (this step's `vsn_forward` populated it) and EMA-updates the
    /// `num_groups` adjacent ISV slots in
    /// `[VSN_MASK_GROUP_0_EMA_INDEX, VSN_MASK_GROUP_0_EMA_INDEX + num_groups)`
    /// (slots 105..111) with the per-group batch mean.
    ///
    /// Producer-only — diagnostic surface for HEALTH_DIAG / VSN focus
    /// monitoring; no consumer kernel reads slots [105..111) in 1B-iii.
    /// Cold-path-cadence: launched once per training step alongside
    /// `launch_h_s2_rms_ema`. No atomicAdd, no DtoH. Shared memory = 256
    /// floats (matches `h_s2_rms_ema_kernel.cu`'s footprint exactly — the
    /// kernel reuses one 256-float scratchpad across `num_groups` separate
    /// reductions, see `vsn_mask_ema_kernel.cu` for the layout rationale).
    pub fn launch_vsn_mask_ema(&self) -> Result<(), MLError> {
        // SP4 Layer A Task A13.3 retrofit (2026-05-01); GPU-Pearls refactor
        // (2026-05-01): kernel writes per-group step observation to
        // `producer_step_scratch_buf[53..53+SL_NUM_FEATURE_GROUPS)`; the GPU
        // `apply_pearls_ad_kernel` (single launch, n_slots=6) writes
        // ISV[105..111) on the same stream — graph-capture-compatible.
        //
        // α is derived adaptively from per-slot Pearls A+D Wiener state — see
        // `sp4_wiener_ema::pearls_ad_update` (test-only reference).
        use crate::cuda_pipeline::sp4_wiener_ema::launch_apply_pearls;

        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_vsn_mask_ema: isv_signals_dev_ptr must be allocated by constructor");

        const SCRATCH_FIRST: usize = 53;
        let num_groups = ml_core::state_layout::SL_NUM_FEATURE_GROUPS;
        debug_assert_eq!(num_groups, 6,
            "SL_NUM_FEATURE_GROUPS must be 6 for the contiguous 6-slot scratch layout");

        let mask_ptr = self.vsn_mask_buf.raw_ptr();
        let b_i = self.config.batch_size as i32;
        let num_groups_i = num_groups as i32;
        let scratch_first_i = SCRATCH_FIRST as i32;
        let scratch_dev = self.producer_step_scratch_buf.dev_ptr;
        const BLOCK_DIM: u32 = 256;
        // 256 floats — same scratchpad size as h_s2_rms_ema; the kernel reuses
        // it across all 6 group reductions (see `vsn_mask_ema_kernel.cu`).
        let smem_bytes = BLOCK_DIM * std::mem::size_of::<f32>() as u32;
        unsafe {
            self.stream.launch_builder(&self.vsn_mask_ema_kernel)
                .arg(&mask_ptr)
                .arg(&b_i)
                .arg(&num_groups_i)
                .arg(&scratch_dev)
                .arg(&scratch_first_i)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (BLOCK_DIM, 1, 1),
                    shared_mem_bytes: smem_bytes,
                })
                .map_err(|e| MLError::ModelError(format!("vsn_mask_ema_update: {e}")))?;
        }

        // GPU Pearls A+D: scratch [53..59) → ISV [105..111) (both
        // contiguous). Retrofit wiener convention: offset = scratch_idx*3
        // → [159..177). VSN mask is softmax-normalised so a 0 step_obs
        // means the kernel hasn't run yet (cold-start) — handled by the
        // applicator's `step_obs == 0` skip.
        unsafe {
            launch_apply_pearls(
                &self.stream,
                &self.apply_pearls_ad_kernel,
                scratch_dev,
                SCRATCH_FIRST as i32,
                self.isv_signals_dev_ptr,
                VSN_MASK_GROUP_0_EMA_INDEX as i32,
                self.wiener_state_buf.dev_ptr,
                (SCRATCH_FIRST * 3) as i32,
                num_groups as i32,
                crate::cuda_pipeline::sp4_wiener_ema::ALPHA_META,
            )?;
        }
        Ok(())
    }

    /// Plan 4 Task 6 Commit B: set the per-step ISV-driven aux-loss weight.
    ///
    /// The per-step controller (`compute_aux_w_p0b` in
    /// `trainers/dqn/trainer/training_loop.rs`) computes `aux_weight` from the
    /// `target_dir_acc - short_ema` deficit and short/long EMA stagnation, in
    /// the design range `[base × floor_ratio, base × ceil_ratio]` =
    /// `[0.15, 1.5]`. The value is baked into kernel SAXPY alphas at
    /// graph-capture time (same staleness contract as `set_c51_alpha`). The
    /// clamp here is a defensive boundary guard using SP13 constants (Invariant
    /// 1 carve-out per `feedback_isv_for_adaptive_bounds.md`).
    ///
    /// SP14 Layer A (2026-05-05): lifted from SP11-era `[0.05, 0.3]`. The pre-
    /// SP14 cap silently masked the deficit-amplification term in
    /// `compute_aux_w_p0b` — controller computed `[0.15, 1.5]` but values
    /// > 0.3 got chopped, leaving fold 2 stagnation 45% below working value.
    pub fn set_aux_weight(&mut self, aux_weight: f32) {
        use crate::cuda_pipeline::sp13_isv_slots::{
            AUX_W_BASE, AUX_W_HARD_FLOOR_RATIO, AUX_W_HARD_CEIL_RATIO,
        };
        // Defensive re-clamp at the kernel boundary in case the caller forgot.
        self.aux_weight = aux_weight.clamp(
            AUX_W_BASE * AUX_W_HARD_FLOOR_RATIO,  // 0.15
            AUX_W_BASE * AUX_W_HARD_CEIL_RATIO,   // 1.5
        );
    }

    /// Plan 4 Task 6 Commit B: current aux-loss weight (the value baked into
    /// the most-recently-captured graph). HEALTH_DIAG reads this to surface
    /// the per-epoch `aux[w=…]` field.
    pub fn aux_weight(&self) -> f32 {
        self.aux_weight
    }

    /// Plan 4 Task 6 Commit B; SP4 Layer A Task A13.1 retrofit (2026-05-01):
    /// launch `aux_heads_loss_ema_update`, sync, then apply Pearls A+D
    /// host-side for both ISV[AUX_NEXT_BAR_MSE_EMA_INDEX=113] and
    /// ISV[AUX_REGIME_CE_EMA_INDEX=114].
    ///
    /// The single-thread single-block kernel writes both scalar loss values
    /// (from `aux_*_loss_reduce`) to `producer_step_scratch_buf[41..43)`
    /// (slots 41=next-bar, 42=regime). Wiener state at
    /// `wiener_state_buf[123..129)` (= scratch slots 41..43 × 3).
    ///
    /// α is derived adaptively from per-slot Pearls A+D Wiener state — see
    /// `sp4_wiener_ema::pearls_ad_update`.
    pub fn launch_aux_heads_loss_ema(&self) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp4_wiener_ema::launch_apply_pearls;

        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_aux_heads_loss_ema: isv_signals_dev_ptr must be allocated by constructor");

        const SCRATCH_IDX_NEXT_BAR: usize = 41;
        const SCRATCH_IDX_REGIME:   usize = 42;
        // Layout invariants: scratch is contiguous [41..43) and ISV is
        // contiguous [113..115); the GPU `apply_pearls_ad_kernel` iterates
        // both with the same stride.
        debug_assert_eq!(SCRATCH_IDX_REGIME, SCRATCH_IDX_NEXT_BAR + 1);
        debug_assert_eq!(AUX_REGIME_CE_EMA_INDEX, AUX_NEXT_BAR_MSE_EMA_INDEX + 1);

        let scratch_dev = self.producer_step_scratch_buf.dev_ptr;
        self.aux_heads_fwd.launch_loss_ema(
            &self.stream,
            self.aux_nb_loss_scalar_buf.raw_ptr(),
            self.aux_rg_loss_scalar_buf.raw_ptr(),
            scratch_dev,
            SCRATCH_IDX_NEXT_BAR as i32,
            SCRATCH_IDX_REGIME   as i32,
        )?;

        // GPU Pearls A+D: scratch [41..43) → ISV [113..115) (both
        // contiguous, single launch n_slots=2). Retrofit wiener convention:
        // offset = scratch_idx*3 → [123..129). Per-slot degenerate-zero
        // (kernel hasn't run / scalar reduction yielded 0) is handled by
        // the applicator's `step_obs == 0` skip.
        unsafe {
            launch_apply_pearls(
                &self.stream,
                &self.apply_pearls_ad_kernel,
                scratch_dev,
                SCRATCH_IDX_NEXT_BAR as i32,
                self.isv_signals_dev_ptr,
                AUX_NEXT_BAR_MSE_EMA_INDEX as i32,
                self.wiener_state_buf.dev_ptr,
                (SCRATCH_IDX_NEXT_BAR * 3) as i32,
                2,
                crate::cuda_pipeline::sp4_wiener_ema::ALPHA_META,
            )?;
        }
        Ok(())
    }

    /// HEALTH_DIAG GPU port — Phase 2A: launch `health_diag_isv_mirror`.
    ///
    /// Copies a curated set of ISV signal-bus slots (reward-component EMAs,
    /// VSN/drift focus EMAs, aux-head loss EMAs, MoE expert utilities, etc.)
    /// into the `HealthDiagSnapshot` mapped-pinned buffer. Cold-path-cadence
    /// (one launch per training step alongside `launch_h_s2_rms_ema`); Phase
    /// 3 wires the call site into the captured graph next to the existing
    /// producer family.
    ///
    /// Producer-only in this commit — no consumer reads `health_diag.snapshot()`
    /// yet. Phase 4's emit-path rewrite makes the snapshot the sole source of
    /// truth for the per-epoch HEALTH_DIAG line.
    ///
    /// ISV indices passed by-value from the trainer's compile-time constants
    /// so the kernel never embeds them — single source of truth lives at the
    /// top of this file (`REWARD_POPART_EMA_INDEX`, `VSN_MAG_EMA_INDEX`,
    /// `MOE_EXPERT_UTIL_EMA_BASE`, etc.).
    pub fn launch_health_diag_isv_mirror(&self) -> Result<(), MLError> {
        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_health_diag_isv_mirror: isv_signals_dev_ptr must be allocated by constructor");
        let hd = match self.health_diag.as_ref() {
            Some(h) => h,
            // Loud failure rather than silent skip — health_diag is allocated
            // unconditionally by the constructor, so None at this site is a
            // misconfiguration.
            None => return Err(MLError::ModelError(
                "launch_health_diag_isv_mirror: health_diag orchestrator not constructed".into()
            )),
        };
        hd.launch_isv_mirror(
            &self.stream,
            self.isv_signals_dev_ptr,
            REWARD_POPART_EMA_INDEX        as i32,
            REWARD_CF_EMA_INDEX            as i32,
            REWARD_TRAIL_EMA_INDEX         as i32,
            REWARD_MICRO_EMA_INDEX         as i32,
            REWARD_OPP_COST_EMA_INDEX      as i32,
            REWARD_BONUS_EMA_INDEX         as i32,
            VSN_MAG_EMA_INDEX              as i32,
            VSN_DIR_EMA_INDEX              as i32,
            TARGET_DRIFT_MAG_EMA_INDEX     as i32,
            TARGET_DRIFT_DIR_EMA_INDEX     as i32,
            AUX_NEXT_BAR_MSE_EMA_INDEX     as i32,
            AUX_REGIME_CE_EMA_INDEX        as i32,
            MOE_EXPERT_UTIL_EMA_BASE       as i32,
            MOE_GATE_ENTROPY_EMA_INDEX     as i32,
            MOE_LAMBDA_EFF_INDEX           as i32,
        )
    }

    /// Borrow the HEALTH_DIAG snapshot for read-only inspection. Returns
    /// `None` only when the orchestrator was not constructed (legacy
    /// non-CUDA test harness path; production always returns `Some`).
    /// Caller is responsible for stream-syncing prior kernel-family
    /// launches before treating the values as authoritative.
    #[inline]
    pub fn health_diag_snapshot(&self)
        -> Option<&super::health_diag::MappedHealthDiagSnapshot>
    {
        self.health_diag.as_ref().map(|h| h.snapshot())
    }

    // ── Phase 3 MoE methods ─────────────────────────────────────────────────

    /// Phase 3 T3.1–T3.3: MoE forward.
    ///
    /// Sequence:
    ///   1. Gate: state[B,SD] → Linear(SD→GH) + bias → ReLU → gate_h1[B,GH]
    ///   2. Gate: gate_h1[B,GH] → Linear(GH→K) + bias → gate_pre[B,K]
    ///   3. Gate: gate_pre → moe_row_softmax → gate_soft[B,K]  (saves for backward)
    ///   4. For k in [0, K): h_s1[B,SH2] → Linear(SH2→BTN)+bias → ReLU → expert_h1_k[B,BTN]
    ///                        expert_h1_k → Linear(BTN→SH2)+bias → expert_out_k[B,SH2]
    ///   5. moe_mixture_forward stacks expert_outs[K,B,SH2] × gate_soft[B,K] → h_s2_new[B,SH2]
    ///   6. DtoD copy h_s2_new → save_h_s2 (replaces the GRN-produced save_h_s2)
    ///
    /// Inputs: reads `save_h_s1` (h_s1 from GRN block 1), `states_buf` (for gate),
    ///         and online weight tensors [127..163).
    /// Output: overwrites `save_h_s2` with the MoE mixture.
    pub(crate) fn launch_moe_forward(&self) -> Result<(), MLError> {
        let b   = self.config.batch_size;
        let sh2 = self.config.shared_h2;
        let sd  = ml_core::state_layout::STATE_DIM;

        let param_sizes = compute_param_sizes(&self.config);
        let on_w_ptrs = f32_weight_ptrs_from_base(self.ptrs.params_ptr, &param_sizes);

        // Gate weight tensors [127..131)
        let gate_w1 = on_w_ptrs[127];  // [SD, GH]
        let gate_b1 = on_w_ptrs[128];  // [GH]
        let gate_w2 = on_w_ptrs[129];  // [GH, K]
        let gate_b2 = on_w_ptrs[130];  // [K]

        // Gate sub-network forward
        let gate_h1_ptr   = self.moe_gate_h1_buf.raw_ptr();
        let gate_pre_ptr  = self.moe_gate_pre_buf.raw_ptr();
        let gate_soft_ptr = self.moe_gate_softmax_buf.raw_ptr();
        let states_ptr    = self.ptrs.states_buf;

        // Step 1: state[B,SD] → Linear(SD→GH) → gate_h1[B,GH]
        self.cublas_forward.sgemm_f32(&self.stream, gate_w1, states_ptr, gate_h1_ptr,
            MOE_GATE_HIDDEN, b, sd, "moe_gate_layer1")?;
        self.cublas_forward.launch_add_bias_relu_f32_raw(
            &self.stream, gate_h1_ptr, gate_b1, MOE_GATE_HIDDEN, b)?;

        // Step 2: gate_h1[B,GH] → Linear(GH→K) → gate_pre[B,K]
        self.cublas_forward.sgemm_f32(&self.stream, gate_w2, gate_h1_ptr, gate_pre_ptr,
            MOE_NUM_EXPERTS, b, MOE_GATE_HIDDEN, "moe_gate_layer2")?;
        self.cublas_forward.launch_add_bias_f32_raw(
            &self.stream, gate_pre_ptr, gate_b2, MOE_NUM_EXPERTS, b)?;

        // Step 3: softmax → gate_soft[B,K]
        self.moe_head.launch_row_softmax(gate_pre_ptr, gate_soft_ptr, b, MOE_NUM_EXPERTS)?;

        // Steps 4: for each expert k: h_s1[B,SH2] → Linear(SH2→BTN)+ReLU → Linear(BTN→SH2)
        let h_s1_ptr = self.ptrs.save_h_s1;
        let expert_outputs_ptr = self.moe_expert_outputs_buf.raw_ptr();

        for k in 0..MOE_NUM_EXPERTS {
            let base_idx = 131 + k * 4;
            let ew1 = on_w_ptrs[base_idx];     // [SH2, BTN]
            let eb1 = on_w_ptrs[base_idx + 1]; // [BTN]
            let ew2 = on_w_ptrs[base_idx + 2]; // [BTN, SH2]
            let eb2 = on_w_ptrs[base_idx + 3]; // [SH2]

            let eh1_ptr  = self.moe_expert_h1_bufs[k].raw_ptr();
            // expert_out_k is the k-th [B, SH2] slice of moe_expert_outputs_buf
            let eout_ptr = expert_outputs_ptr + (k * b * sh2) as u64 * std::mem::size_of::<f32>() as u64;

            // h_s1[B,SH2] → Linear(SH2→BTN) → eh1[B,BTN]
            self.cublas_forward.sgemm_f32(&self.stream, ew1, h_s1_ptr, eh1_ptr,
                MOE_EXPERT_BOTTLENECK, b, sh2, &format!("moe_expert_{k}_layer1"))?;
            self.cublas_forward.launch_add_bias_relu_f32_raw(
                &self.stream, eh1_ptr, eb1, MOE_EXPERT_BOTTLENECK, b)?;

            // eh1[B,BTN] → Linear(BTN→SH2) → eout[B,SH2]
            self.cublas_forward.sgemm_f32(&self.stream, ew2, eh1_ptr, eout_ptr,
                sh2, b, MOE_EXPERT_BOTTLENECK, &format!("moe_expert_{k}_layer2"))?;
            self.cublas_forward.launch_add_bias_f32_raw(
                &self.stream, eout_ptr, eb2, sh2, b)?;
        }

        // Step 5: moe_mixture_forward: expert_outputs[K,B,SH2] × gate_soft[B,K] → h_s2_new[B,SH2]
        // Write directly into save_h_s2 — replaces the GRN output.
        self.moe_head.launch_mixture_forward(
            expert_outputs_ptr,
            gate_soft_ptr,
            self.ptrs.save_h_s2,
            b, MOE_NUM_EXPERTS, sh2,
        )?;

        Ok(())
    }

    /// Phase 3 T3.4: compute load-balance loss and SAXPY into total_loss_dev_ptr.
    ///
    /// Calls `moe_head.launch_load_balance_loss` which runs `moe_load_balance_loss`
    /// (K blocks × B reduction) + `moe_load_balance_reduce` (single thread sum) into
    /// `moe_load_balance_loss_total_dev_ptr`. Then SAXPYs the scalar (alpha=1.0)
    /// into `total_loss_dev_ptr` so it is included in the blended TD loss.
    ///
    /// λ is read inside the kernel from
    /// `ISV[MOE_LAMBDA_EFF_INDEX]`, populated each step by the
    /// `moe_lambda_eff_update` adaptive controller (see
    /// `launch_moe_lambda_eff_update`). No DtoH per
    /// `feedback_isv_for_adaptive_bounds.md`.
    pub(crate) fn launch_moe_load_balance_loss(&self) -> Result<(), MLError> {
        let b = self.config.batch_size;
        let gate_ptr        = self.moe_gate_softmax_buf.raw_ptr();
        let loss_per_k_ptr  = self.moe_load_balance_loss_per_k.raw_ptr();
        let loss_total_ptr  = self.moe_load_balance_loss_total_dev_ptr;

        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_moe_load_balance_loss: isv_signals_dev_ptr must be non-zero");

        // Compute per-expert load loss + reduce to scalar.
        // Kernel reads λ from ISV[MOE_LAMBDA_EFF_INDEX] at runtime.
        self.moe_head.launch_load_balance_loss(
            gate_ptr, loss_per_k_ptr, loss_total_ptr,
            self.isv_signals_dev_ptr,
            b, MOE_NUM_EXPERTS, MOE_LAMBDA_EFF_INDEX,
        )?;

        // SAXPY total_loss += 1.0 * lb_loss_total (both are device-mapped scalars).
        let n: i32 = 1;
        let alpha: f32 = 1.0;
        let total_loss_ptr = self.total_loss_dev_ptr;
        unsafe {
            self.stream
                .launch_builder(&self.saxpy_f32_kernel)
                .arg(&total_loss_ptr)
                .arg(&loss_total_ptr)
                .arg(&alpha)
                .arg(&n)
                .launch(LaunchConfig { grid_dim: (1, 1, 1), block_dim: (256, 1, 1), shared_mem_bytes: 0 })
                .map_err(|e| MLError::ModelError(format!("moe_lb_loss saxpy: {e}")))?;
        }
        Ok(())
    }

    /// Phase 3 T3.5; SP4 Layer A Task A13.2 retrofit (2026-05-01): launch
    /// `moe_expert_util_ema_update` step-observation producer, sync, then
    /// apply Pearls A+D host-side for all 9 ISV slots.
    ///
    /// Single-block single-thread cold-path kernel. Reads
    /// `moe_gate_softmax_buf [B, K]` and writes per-expert col_mean to
    /// `producer_step_scratch_buf[44..52)` (one slot per expert; K=8) plus
    /// gate Shannon entropy to slot 52. Host then applies Pearls A+D for
    /// each slot in turn:
    ///   - ISV[MOE_EXPERT_UTIL_EMA_BASE+k=118+k] for k in 0..K
    ///     ← Wiener offset (44+k)*3
    ///   - ISV[MOE_GATE_ENTROPY_EMA_INDEX=126]
    ///     ← Wiener offset 52*3=156
    ///
    /// α is derived adaptively from per-slot Pearls A+D Wiener state — see
    /// `sp4_wiener_ema::pearls_ad_update`.
    pub fn launch_moe_expert_util_ema(&self) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp4_wiener_ema::launch_apply_pearls;

        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_moe_expert_util_ema: isv_signals_dev_ptr must be non-zero");

        const SCRATCH_IDX_UTIL_BASE: usize = 44;
        const SCRATCH_IDX_ENTROPY:   usize = 52;
        debug_assert_eq!(MOE_NUM_EXPERTS, 8,
            "MOE_NUM_EXPERTS must be 8 for the contiguous 8-slot scratch layout");
        // Layout invariants: scratch [44..52) for utils + slot 52 for
        // entropy = contiguous [44..53); ISV [MOE_EXPERT_UTIL_EMA_BASE..
        // MOE_EXPERT_UTIL_EMA_BASE+8) for utils + MOE_GATE_ENTROPY_EMA_INDEX
        // = contiguous [118..127). Single GPU launch with n_slots=9.
        debug_assert_eq!(SCRATCH_IDX_ENTROPY, SCRATCH_IDX_UTIL_BASE + MOE_NUM_EXPERTS);
        debug_assert_eq!(MOE_GATE_ENTROPY_EMA_INDEX,
            MOE_EXPERT_UTIL_EMA_BASE + MOE_NUM_EXPERTS);

        let gate_ptr = self.moe_gate_softmax_buf.raw_ptr();
        let scratch_dev = self.producer_step_scratch_buf.dev_ptr;
        self.moe_head.launch_expert_util_ema(
            gate_ptr,
            scratch_dev,
            self.config.batch_size,
            MOE_NUM_EXPERTS,
            SCRATCH_IDX_UTIL_BASE,
            SCRATCH_IDX_ENTROPY,
        )?;

        // GPU Pearls A+D: scratch [44..53) → ISV [118..127), n_slots=9.
        // Retrofit wiener convention: offset = scratch_idx*3 → [132..159).
        // Cold-start (before MoE forward populates `moe_gate_softmax_buf`)
        // → kernel writes 0 → applicator's degenerate-zero skip preserves
        // the Pearl A sentinel for next step's first genuine observation.
        unsafe {
            launch_apply_pearls(
                &self.stream,
                &self.apply_pearls_ad_kernel,
                scratch_dev,
                SCRATCH_IDX_UTIL_BASE as i32,
                self.isv_signals_dev_ptr,
                MOE_EXPERT_UTIL_EMA_BASE as i32,
                self.wiener_state_buf.dev_ptr,
                (SCRATCH_IDX_UTIL_BASE * 3) as i32,
                (MOE_NUM_EXPERTS + 1) as i32,
                crate::cuda_pipeline::sp4_wiener_ema::ALPHA_META,
            )?;
        }

        Ok(())
    }

    /// Adaptive load-balance λ controller (2026-04-27).
    ///
    /// Single-block single-thread cold-path kernel. Reads
    /// ISV[MOE_GATE_ENTROPY_EMA_INDEX] (gate-entropy EMA from
    /// `moe_expert_util_ema_update`) and writes
    /// ISV[MOE_LAMBDA_EFF_INDEX] = `λ_floor + λ_max_extra × deficit`,
    /// where `deficit = clamp((target − ent_ema)/target, 0, 1)` and
    /// `target = moe_entropy_target_frac × ln(K)`.
    ///
    /// MUST be called AFTER `launch_moe_expert_util_ema` so the entropy
    /// EMA fed to the controller is fresh for the NEXT step's
    /// `moe_load_balance_loss` consumer.
    ///
    /// Per `pearl_blend_formulas_must_have_permanent_floor.md`,
    /// `λ_floor` is a permanent minimum (λ_eff never weakens below the
    /// old static 0.01 baseline).
    pub fn launch_moe_lambda_eff_update(&self) -> Result<(), MLError> {
        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_moe_lambda_eff_update: isv_signals_dev_ptr must be non-zero");
        self.moe_head.launch_lambda_eff_update(
            self.isv_signals_dev_ptr,
            MOE_LAMBDA_EFF_INDEX,
            MOE_GATE_ENTROPY_EMA_INDEX,
            self.moe_lambda_floor,
            self.moe_lambda_max_extra,
            self.moe_entropy_target_frac,
        )
    }

    /// Phase 3 T3.6: MoE backward.
    ///
    /// Sequence (reverse of forward):
    ///   1. moe_mixture_backward: dh_s2 × gate_soft → de_k[K,B,SH2]
    ///      + moe_dgate_reduce: dh_s2 × expert_outs → dg[B,K]
    ///   2. moe_softmax_backward: gate_soft × dg → dg_pre[B,K]
    ///   3. Gate layer 2 backward: dg_pre [B,K] → dW_gate_w2 (w_ptrs[129]), db_gate_b2 (w_ptrs[130])
    ///      + dX → moe_gate_dh1[B,GH]
    ///   4. Gate layer 1 backward (dX from step 3, gated by ReLU): apply relu mask,
    ///      → dW_gate_w1 (w_ptrs[127]), db_gate_b1 (w_ptrs[128])
    ///   5. For each expert k:
    ///      a. Expert layer 2 backward: de_k_slice[B,SH2] → dW_ew2, db_eb2 + dX → d_eh1_k[B,BTN]
    ///      b. Expert layer 1 backward (dX from 5a gated by ReLU): → dW_ew1, db_eb1 + dX → d_h_s1_k[B,SH2]
    ///   6. Accumulate 8× d_h_s1_k into moe_dh_s1_scratch[B,SH2]
    ///   7. DtoD-copy moe_dh_s1_scratch → bw_d_h_s2 (REPLACES existing, so encoder
    ///      backward chain sees the MoE-routed h_s1 gradient as its h_s2 input gradient)
    ///
    /// `grad_base`: base device pointer for parameter gradient accumulation.
    pub(crate) fn launch_moe_backward(&self, grad_base: u64) -> Result<(), MLError> {
        let b   = self.config.batch_size;
        let sh2 = self.config.shared_h2;
        let f32_size = std::mem::size_of::<f32>() as u64;

        let param_sizes = compute_param_sizes(&self.config);
        let w_ptrs = f32_weight_ptrs_from_base(self.ptrs.params_ptr, &param_sizes);

        // Gradient buffer base pointers — offset into grad_base by param position.
        // grad_base is laid out identically to the params buffer.
        let grad_ptrs = f32_weight_ptrs_from_base(grad_base, &param_sizes);

        let gate_soft_ptr   = self.moe_gate_softmax_buf.raw_ptr();
        let gate_h1_ptr     = self.moe_gate_h1_buf.raw_ptr();
        let expert_outs_ptr = self.moe_expert_outputs_buf.raw_ptr();
        let de_k_ptr        = self.moe_de_k_buf.raw_ptr();
        let dg_ptr          = self.moe_dg_buf.raw_ptr();
        let dg_pre_ptr      = self.moe_dg_pre_buf.raw_ptr();
        let gate_dh1_ptr    = self.moe_gate_dh1_buf.raw_ptr();
        let dh_s1_scratch   = self.moe_dh_s1_scratch.raw_ptr();

        // d_h_s2 holds dL/d(save_h_s2) accumulated by all prior backward ops
        // (branches, aux heads). This is the upstream gradient w.r.t. the MoE output.
        let d_h_s2_ptr = bw_raw_f32_ptr(&self.bw_d_h_s2, &self.stream);
        let h_s1_ptr   = self.ptrs.save_h_s1;

        // Step 1a: moe_mixture_backward — propagates d_h_s2 through the weighted sum to de_k.
        // de_k[k,b,c] = gate_soft[b,k] * d_h_s2[b,c]
        // Step 1b: moe_dgate_reduce — dg[b,k] = sum_c(d_h_s2[b,c] * expert_outs[k,b,c])
        self.moe_head.launch_mixture_backward(
            d_h_s2_ptr, gate_soft_ptr, expert_outs_ptr,
            de_k_ptr, dg_ptr,
            b, MOE_NUM_EXPERTS, sh2,
        )?;

        // Step 2: softmax backward — dg_pre[b,k] = gate_soft[b,k] * (dg[b,k] - dot(gate_soft[b,:], dg[b,:]))
        self.moe_head.launch_softmax_backward(gate_soft_ptr, dg_ptr, dg_pre_ptr, b, MOE_NUM_EXPERTS)?;

        // Step 3: gate layer 2 dW + dX
        // dg_pre[B,K] w.r.t. gate_h1[B,GH] through W_gate_w2[GH,K]
        self.cublas_backward.launch_dw_only(
            &self.stream,
            dg_pre_ptr, gate_h1_ptr,         // dy, x
            grad_ptrs[129], grad_ptrs[130],   // dW_gate_w2, db_gate_b2
            MOE_NUM_EXPERTS, MOE_GATE_HIDDEN, b,
        )?;
        // dX into gate_dh1 (beta=0: overwrite)
        self.cublas_backward.launch_dx_only(
            &self.stream,
            dg_pre_ptr, w_ptrs[129], gate_dh1_ptr,
            MOE_NUM_EXPERTS, MOE_GATE_HIDDEN, b, 0.0,
        )?;

        // Step 4: gate layer 1 dW — gated by ReLU of gate_h1.
        // gate_h1 holds the post-ReLU activation saved by forward; gate_dh1[i] = 0
        // wherever gate_h1[i] == 0 (i.e., the neuron was clamped). relu_mask
        // on cublas_backward implements dx[i] *= (activation[i] > 0).
        self.cublas_backward.relu_mask(
            &self.stream, gate_dh1_ptr, gate_h1_ptr, b * MOE_GATE_HIDDEN,
        )?;
        // dW_gate_w1, db_gate_b1 (gate layer 1 is state[B,SD] → gate_h1[B,GH])
        let states_ptr = self.ptrs.states_buf;
        self.cublas_backward.launch_dw_only(
            &self.stream,
            gate_dh1_ptr, states_ptr,          // dy, x
            grad_ptrs[127], grad_ptrs[128],    // dW_gate_w1, db_gate_b1
            MOE_GATE_HIDDEN, ml_core::state_layout::STATE_DIM, b,
        )?;

        // Step 5+6: expert backward — 8 experts, accumulate dX into dh_s1_scratch.
        //
        // Buffer aliasing strategy: after step 4 gate backward, gate_h1_ptr
        // (= moe_gate_h1_buf [B, GH]) is free. GH == BTN == 64, so gate_h1_ptr
        // is repurposed as d_eh1 scratch [B, BTN] for each expert in turn.
        // The per-expert saved activation (moe_expert_h1_bufs[k]) must NOT be
        // overwritten until after the ReLU mask is applied; the sequence is:
        //   a. dW_ew2 (reads eh1_saved as x — does NOT modify eh1_saved)
        //   b. dX_ew2 → gate_h1_ptr (d_eh1 scratch, beta=0; eh1_saved intact)
        //   c. relu_mask(d_eh1, eh1_saved)  (eh1_saved still valid here)
        //   d. dW_ew1 (reads eh1_saved? No — reads d_eh1 from gate_h1_ptr as dy)
        //   e. dX_ew1 → dh_s1_scratch (accumulate with beta 0 or 1)
        let d_eh1_scratch = gate_h1_ptr;  // repurpose gate_h1_ptr as [B, BTN] scratch
        for k in 0..MOE_NUM_EXPERTS {
            let base_idx = 131 + k * 4;
            let ew1 = w_ptrs[base_idx];      // [SH2, BTN]
            let ew2 = w_ptrs[base_idx + 2];  // [BTN, SH2]
            let g_ew1 = grad_ptrs[base_idx];
            let g_eb1 = grad_ptrs[base_idx + 1];
            let g_ew2 = grad_ptrs[base_idx + 2];
            let g_eb2 = grad_ptrs[base_idx + 3];

            let eh1_saved = self.moe_expert_h1_bufs[k].raw_ptr();  // post-ReLU, read-only here
            let dek_k     = de_k_ptr + (k * b * sh2) as u64 * f32_size;

            // a. Expert layer 2 dW: dek_k[B,SH2] × eh1_saved[B,BTN]^T → dW_ew2, db_eb2
            //    launch_dw_only reads eh1_saved as x; does NOT modify it.
            self.cublas_backward.launch_dw_only(
                &self.stream,
                dek_k, eh1_saved,      // dy, x
                g_ew2, g_eb2,          // dW_ew2, db_eb2
                sh2, MOE_EXPERT_BOTTLENECK, b,
            )?;
            // b. Expert layer 2 dX: dek_k × W_ew2^T → d_eh1 scratch (beta=0)
            self.cublas_backward.launch_dx_only(
                &self.stream,
                dek_k, ew2, d_eh1_scratch,
                sh2, MOE_EXPERT_BOTTLENECK, b, 0.0,
            )?;
            // c. ReLU mask: d_eh1[i] *= (eh1_saved[i] > 0)
            self.cublas_backward.relu_mask(
                &self.stream, d_eh1_scratch, eh1_saved, b * MOE_EXPERT_BOTTLENECK,
            )?;
            // d. Expert layer 1 dW: d_eh1[B,BTN] × h_s1[B,SH2]^T → dW_ew1, db_eb1
            self.cublas_backward.launch_dw_only(
                &self.stream,
                d_eh1_scratch, h_s1_ptr,   // dy=d_eh1, x=h_s1
                g_ew1, g_eb1,              // dW_ew1, db_eb1
                MOE_EXPERT_BOTTLENECK, sh2, b,
            )?;
            // e. Expert layer 1 dX → dh_s1_scratch (accumulate; beta=0 for k=0, 1 for k>0)
            let beta = if k == 0 { 0.0_f32 } else { 1.0_f32 };
            self.cublas_backward.launch_dx_only(
                &self.stream,
                d_eh1_scratch, ew1, dh_s1_scratch,
                MOE_EXPERT_BOTTLENECK, sh2, b, beta,
            )?;
        }

        // Step 7: DtoD-copy moe_dh_s1_scratch → bw_d_h_s2.
        // This REPLACES (not adds to) bw_d_h_s2 so encoder_backward_chain
        // receives the MoE-accumulated h_s1 gradient as its h_s2 input gradient.
        // The aux_heads_backward already wrote into bw_d_h_s2; we replace here
        // because the MoE mixture output IS save_h_s2 — the GRN h_s2 block was
        // replaced by MoE in forward, so its "d_h_s2" should be the MoE dX.
        let n_bytes = b * sh2 * std::mem::size_of::<f32>();
        self.graph_safe_copy_f32(d_h_s2_ptr, dh_s1_scratch, n_bytes, "moe_dh_s1→d_h_s2")?;

        Ok(())
    }

    /// Plan 4 Task 6 Commit B: aux-heads forward orchestrator.
    ///
    /// Runs the next-bar regression head + 5-class regime classification head
    /// off the trunk's just-produced `save_h_s2 [B, SH2]`. Sequence:
    ///   1. regime label builder: `states[:, OFI_START + 29]` → uint8 [B]
    ///   2. next-bar label gather: `next_states[:, 0]` → f32 [B] dedicated buf
    ///   3. next-bar forward: Linear → ELU → Linear → softmax[B, K=2]
    ///   4. regime forward:   Linear → ELU → Linear → logits[B, 5]
    ///   5. next-bar softmax CE reduce → loss_scalar[1] + valid_count[1]
    ///   6. regime CE reduce  → loss_scalar[1] + correct_scalar[1]
    ///
    /// SP13 B1.1a (2026-05-05): the next-bar head flipped from K=1 MSE
    /// regression to K=2 softmax CE classification. The pre-B1.1a step 2
    /// (strided_gather of `next_states[:, 0]` into a f32 label buffer)
    /// is retired; the producer kernel for the new i32 labels lands in
    /// B1.1b, so until then `aux_nb_label_buf` stays at its
    /// `alloc_zeros` value.
    ///
    /// Online-only — aux heads NEVER enter Bellman bootstrapping.
    pub(crate) fn aux_heads_forward(&self) -> Result<(), MLError> {
        let b   = self.config.batch_size;
        let sh2 = self.config.shared_h2;
        // SP14 Layer C Phase C.5b (2026-05-08): aux heads now read the
        // SEPARATE aux trunk's output `h_s2_aux` rather than Q's GRN trunk
        // output `save_h_s2`. The aux trunk forward (below) consumes
        // `save_h_s1` (encoder output) and writes `h_s2_aux`. Symmetric
        // contract change in `aux_heads_backward` + the kernel signature
        // rename `h_s2 → h_s2_aux` lock-in step.
        let h_s2_aux_ptr = self.h_s2_aux.raw_ptr();
        let states_ptr = self.ptrs.states_buf;
        let encoder_out_ptr = self.ptrs.save_h_s1;
        let encoder_out_dim = self.config.shared_h1;
        let aux_h1 = super::gpu_aux_trunk::AUX_TRUNK_H1;
        let aux_h2 = super::gpu_aux_trunk::AUX_TRUNK_H2;

        let param_sizes = compute_param_sizes(&self.config);
        let on_w_ptrs = f32_weight_ptrs_from_base(self.ptrs.params_ptr, &param_sizes);
        // Aux param tensors at indices [119..127): nb_{w1, b1, w2, b2} + rg_{w1, b1, w2, b2}.
        let nb_w1 = on_w_ptrs[119];
        let nb_b1 = on_w_ptrs[120];
        let nb_w2 = on_w_ptrs[121];
        let nb_b2 = on_w_ptrs[122];
        let rg_w1 = on_w_ptrs[123];
        let rg_b1 = on_w_ptrs[124];
        let rg_w2 = on_w_ptrs[125];
        let rg_b2 = on_w_ptrs[126];

        // Step 0: aux trunk forward — Linear→ELU→Linear→ELU→Linear MLP that
        // produces `h_s2_aux [B, SH2]` from the encoder's layer-1 output
        // `save_h_s1 [B, SH1]`. Runs BEFORE the regime label builder + per-
        // head forwards so both heads see the freshly-written `h_s2_aux`.
        // Aux trunk weights live in dedicated `aux_trunk_w{1,2,3}` /
        // `aux_trunk_b{1,2,3}` slabs (NOT in the flat `params_buf`); they
        // are trained by `launch_aux_trunk_adam_update` in the backward.
        self.aux_trunk_forward_ops.launch(
            &self.stream,
            encoder_out_ptr,
            self.aux_trunk_w1.raw_ptr(),
            self.aux_trunk_b1.raw_ptr(),
            self.aux_trunk_w2.raw_ptr(),
            self.aux_trunk_b2.raw_ptr(),
            self.aux_trunk_w3.raw_ptr(),
            self.aux_trunk_b3.raw_ptr(),
            self.h_aux1.raw_ptr(),
            self.h_aux2.raw_ptr(),
            h_s2_aux_ptr,
            b,
            encoder_out_dim,
            aux_h1,
            aux_h2,
            sh2,
        )?;

        let state_dim_padded = ml_core::state_layout::STATE_DIM_PADDED;
        let regime_score_idx = ml_core::state_layout::OFI_START + 29;

        // Step 1: regime-label builder. `regime_score` is at OFI offset 29
        // (= absolute index 71 = OFI_START + 29). It's a sigmoid-bounded
        // [0, 1] scalar per docs. The kernel discretises into K=5 uniform bins
        // with internal clamp.
        self.aux_heads_fwd.regime_label_from_states(
            &self.stream,
            states_ptr,
            b,
            state_dim_padded,
            regime_score_idx,
            AUX_REGIME_K,
            self.aux_rg_label_buf.raw_ptr(),
        )?;

        // SP13 B1.1a (2026-05-05): the previous regression-mode head
        // gathered `next_states[:, 0]` (= `log_return`) into a f32 label
        // buffer here. The B1.1a head is a 2-class softmax classifier
        // reading i32 direction labels in `{-1, 0, 1}`, and the producer
        // kernel that fills those labels lands in B1.1b. Until then,
        // `aux_nb_label_buf` stays at its `alloc_zeros` value (every
        // sample receives label 0 = "down") — the model converges on
        // "predict class 0 everywhere" until B1.1b lands the producer.
        // This is intentional, known, and documented per the B1.1a brief.
        // The strided_gather block + ISV[117] retirement comments are
        // both retired in this commit.

        // Step 3: next-bar direction-classification forward (SP13 B1.1a:
        // K=2 softmax). Writes hidden tile + logits tile + softmax tile.
        // SP14-C.5b: input is `h_s2_aux` (aux trunk output), NOT Q's `h_s2`.
        self.aux_heads_fwd.forward_next_bar(
            &self.stream,
            h_s2_aux_ptr,
            nb_w1, nb_b1, nb_w2, nb_b2,
            b, sh2, AUX_NEXT_BAR_K,
            self.aux_nb_hidden_buf.raw_ptr(),
            self.aux_nb_logits_buf.raw_ptr(),
            self.aux_nb_softmax_buf.raw_ptr(),
        )?;

        // Step 4: regime classification forward.
        // SP14-C.5b: input is `h_s2_aux` (aux trunk output), NOT Q's `h_s2`.
        self.aux_heads_fwd.forward_regime(
            &self.stream,
            h_s2_aux_ptr,
            rg_w1, rg_b1, rg_w2, rg_b2,
            b, sh2, AUX_REGIME_K,
            self.aux_rg_hidden_buf.raw_ptr(),
            self.aux_rg_logits_buf.raw_ptr(),
        )?;

        // Step 5: next-bar softmax CE reduce (SP13 B1.1a). Reads the
        // softmax tile + i32 labels; writes loss + B_valid scalars.
        self.aux_heads_fwd.next_bar_loss_reduce(
            &self.stream,
            self.aux_nb_softmax_buf.raw_ptr(),
            self.aux_nb_label_buf.raw_ptr(),
            b,
            AUX_NEXT_BAR_K,
            self.aux_nb_loss_scalar_buf.raw_ptr(),
            self.aux_nb_valid_count_buf.raw_ptr(),
        )?;

        // Step 6: regime CE reduce.
        self.aux_heads_fwd.regime_loss_reduce(
            &self.stream,
            self.aux_rg_logits_buf.raw_ptr(),
            self.aux_rg_label_buf.raw_ptr(),
            b,
            AUX_REGIME_K,
            self.aux_rg_loss_scalar_buf.raw_ptr(),
            self.aux_rg_correct_scalar_buf.raw_ptr(),
        )?;

        // ── SP22 H6 vNext Phase B4 (2026-05-14): trade-outcome head forward ──
        //
        // Steps 7 + 8: K=3 trade-outcome head forward + sparse CE reduce.
        // Mirrors steps 3 + 5 of the K=2 next-bar head exactly but reads
        // weights at flat-buffer indices [163..167) (Phase B1 additions)
        // and writes to the trainer's `aux_to_*` save-for-backward tiles
        // (Phase B2 additions).
        //
        // `aux_to_label_buf` is populated by the replay-buffer label
        // scatter (Phase B4b — deferred). In this commit the buffer stays
        // at `alloc_zeros` → every sample receives label 0 (Profit). The
        // model trains on "predict Profit everywhere" which is a
        // degraded but well-defined cold-start (no NaN). Mirrors the K=2
        // head's known-degraded state between B1.1a (forward landed) and
        // B1.1b (label producer wired). Once Phase B4b lands the
        // per-(i, t) label scatter, the head will train on the actual
        // sparse trade-outcome distribution.
        let to_w1 = on_w_ptrs[163];
        let to_b1 = on_w_ptrs[164];
        let to_w2 = on_w_ptrs[165];
        let to_b2 = on_w_ptrs[166];
        self.aux_to_fwd.forward(
            &self.stream,
            h_s2_aux_ptr,
            to_w1, to_b1, to_w2, to_b2,
            b, sh2, AUX_OUTCOME_K,
            self.aux_to_hidden_buf.raw_ptr(),
            self.aux_to_logits_buf.raw_ptr(),
            self.aux_to_softmax_buf.raw_ptr(),
        )?;
        self.aux_to_fwd.loss_reduce(
            &self.stream,
            self.aux_to_softmax_buf.raw_ptr(),
            self.aux_to_label_buf.raw_ptr(),
            b,
            AUX_OUTCOME_K,
            self.aux_to_loss_scalar_buf.raw_ptr(),
            self.aux_to_valid_count_buf.raw_ptr(),
        )?;

        Ok(())
    }

    /// Plan 4 Task 6 Commit B + SP14 Layer C Phase C.5b (2026-05-08):
    /// aux-heads + aux-trunk backward orchestrator.
    ///
    /// Runs both head backward kernels (next-bar + regime) over the saved
    /// forward state, then for each of the 8 aux head param tensors:
    ///   1. `aux_param_grad_reduce` collapses `partials [B, P]` into `final [P]`.
    ///   2. SAXPY `grad_buf[tensor_idx] += aux_weight * final` via the
    ///      existing `dqn_saxpy_f32_kernel`.
    /// Then SP14-C.5b: the per-sample `dh_s2_aux` from both heads is
    /// SAXPY-accumulated into `dh_s2_aux_accum [B, SH2]` (the AUX TRUNK's
    /// upstream-grad accumulator, NOT Q's `bw_d_h_s2`). Aux trunk backward
    /// then reads the accumulator and propagates gradient through w3/w2/w1
    /// + b3/b2/b1, terminating at the encoder boundary (structural stop-
    /// grad — kernel set has NO `dx_in` output, so Q's `bw_d_h_s2` is
    /// unaffected by aux loss). Finally, `launch_aux_trunk_adam_update`
    /// applies global L2-norm clip + per-tensor Adam updates over the 6
    /// aux trunk grad buffers.
    ///
    /// MUST run AFTER `backward_full` + concat-accumulators fill `bw_d_h_s2`
    /// AND BEFORE `encoder_backward_chain` consumes `bw_d_h_s2` (so the
    /// trunk dW/dB chain rule sees the un-augmented Q gradient).
    ///
    /// `grad_base` is the same value passed to `launch_cublas_backward_to` —
    /// usually `self.ptrs.grad_buf`, but the vaccine path (g_val computation)
    /// passes a scratch buffer here. The aux head SAXPY targets
    /// `grad_base[119..127)` at the same `padded_byte_offset` math the rest
    /// of the backward uses; the aux trunk Adam launcher writes its 6 grad
    /// tensors directly (separate slabs, NOT in `params_buf`/`grad_buf`).
    pub(crate) fn aux_heads_backward(&self, grad_base: u64) -> Result<(), MLError> {
        let b   = self.config.batch_size;
        let sh2 = self.config.shared_h2;
        // SP14 Layer C Phase C.5b (2026-05-08): aux heads now consume the
        // SEPARATE aux trunk's output `h_s2_aux`, and their per-sample
        // dh_s2 SAXPYs into `dh_s2_aux_accum` (NOT Q's `bw_d_h_s2`). The
        // aux trunk's backward kernel then reads `dh_s2_aux_accum` as
        // `dh_s2_aux_in_ptr` and propagates grad through the aux trunk's
        // own w1/w2/w3/b1/b2/b3 (terminating at the encoder boundary;
        // structural stop-grad enforced by missing `dx_in` output).
        let h_s2_aux_ptr = self.h_s2_aux.raw_ptr();
        let dh_s2_aux_accum_ptr = self.dh_s2_aux_accum.raw_ptr();
        let encoder_out_ptr = self.ptrs.save_h_s1;
        let encoder_out_dim = self.config.shared_h1;
        let aux_h1 = super::gpu_aux_trunk::AUX_TRUNK_H1;
        let aux_h2 = super::gpu_aux_trunk::AUX_TRUNK_H2;
        let aux_w = self.aux_weight;
        let aux_h = AUX_HIDDEN_DIM;
        let aux_kr = AUX_REGIME_K;
        let aux_knb = AUX_NEXT_BAR_K;

        let param_sizes = compute_param_sizes(&self.config);
        let on_w_ptrs = f32_weight_ptrs_from_base(self.ptrs.params_ptr, &param_sizes);
        let nb_w1 = on_w_ptrs[119];
        let nb_w2 = on_w_ptrs[121];
        let rg_w1 = on_w_ptrs[123];
        let rg_w2 = on_w_ptrs[125];

        // Run BOTH backward kernels. They WRITE to disjoint output buffers
        // (per-head partials + per-head dh_s2), so no inter-dependency beyond
        // the just-completed forward.
        // SP13 B1.1a (2026-05-05): softmax CE backward. Reads the saved
        // softmax tile + i32 labels + B_valid scalar (written by loss
        // reduce) so loss + gradient share the same `1/B_valid` divisor.
        // SP14-C.5b: input is `h_s2_aux` (aux trunk output). The dh_s2_aux
        // outputs are written to per-head buffers (overwrite, not
        // accumulate); the SAXPY-into-`dh_s2_aux_accum` happens below.
        self.aux_heads_bwd.backward_next_bar(
            &self.stream,
            h_s2_aux_ptr,
            nb_w1, nb_w2,
            self.aux_nb_hidden_buf.raw_ptr(),
            self.aux_nb_softmax_buf.raw_ptr(),
            self.aux_nb_label_buf.raw_ptr(),
            self.aux_nb_valid_count_buf.raw_ptr(),
            b, sh2, aux_knb,
            self.aux_partial_nb_w1.raw_ptr(),
            self.aux_partial_nb_b1.raw_ptr(),
            self.aux_partial_nb_w2.raw_ptr(),
            self.aux_partial_nb_b2.raw_ptr(),
            self.aux_dh_s2_nb_buf.raw_ptr(),
        )?;
        self.aux_heads_bwd.backward_regime(
            &self.stream,
            h_s2_aux_ptr,
            rg_w1, rg_w2,
            self.aux_rg_hidden_buf.raw_ptr(),
            self.aux_rg_logits_buf.raw_ptr(),
            self.aux_rg_label_buf.raw_ptr(),
            b, sh2, aux_kr,
            self.aux_partial_rg_w1.raw_ptr(),
            self.aux_partial_rg_b1.raw_ptr(),
            self.aux_partial_rg_w2.raw_ptr(),
            self.aux_partial_rg_b2.raw_ptr(),
            self.aux_dh_s2_rg_buf.raw_ptr(),
        )?;

        // ── SP22 H6 vNext Phase B4 (2026-05-14): trade-outcome backward ──
        //
        // K=3 backward — emits per-sample partials (dW1, db1, dW2, db2)
        // + per-sample dh_s2_aux. Sparse-label arithmetic: `valid_count`
        // saved by the loss reduce is small (~1-5% of B), so `inv_B =
        // 1/B_valid` is large → per-trade-close gradients have
        // proportionally higher magnitude. The downstream SAXPY scales
        // by `aux_weight` (same scalar as the K=2 sibling); Phase E may
        // need per-tensor LR tuning if magnitude balance becomes an
        // issue, but for B4 the unified aux_weight matches the K=2
        // sibling's pattern.
        //
        // Reads weights at indices [163, 165] (W1, W2) — same offset
        // arithmetic as the K=2 sibling at [119, 121].
        let to_w1_bwd = on_w_ptrs[163];
        let to_w2_bwd = on_w_ptrs[165];
        let aux_kto = AUX_OUTCOME_K;
        self.aux_to_bwd.backward(
            &self.stream,
            h_s2_aux_ptr,
            to_w1_bwd, to_w2_bwd,
            self.aux_to_hidden_buf.raw_ptr(),
            self.aux_to_softmax_buf.raw_ptr(),
            self.aux_to_label_buf.raw_ptr(),
            self.aux_to_valid_count_buf.raw_ptr(),
            b, sh2, aux_kto,
            self.aux_partial_to_w1.raw_ptr(),
            self.aux_partial_to_b1.raw_ptr(),
            self.aux_partial_to_w2.raw_ptr(),
            self.aux_partial_to_b2.raw_ptr(),
            self.aux_dh_s2_to_buf.raw_ptr(),
        )?;

        // Reduce + SAXPY each of the 8 aux param tensors. Tensor layout
        // (SP13 B1.1a; next-bar K_NB flipped 1 → 2):
        //   [119] aux_nb_w1 [H, SH2]      ← partial [B, H*SH2]
        //   [120] aux_nb_b1 [H]           ← partial [B, H]
        //   [121] aux_nb_w2 [K_NB=2, H]   ← partial [B, K_NB*H]
        //   [122] aux_nb_b2 [K_NB=2]      ← partial [B, K_NB]
        //   [123] aux_rg_w1 [H, SH2]
        //   [124] aux_rg_b1 [H]
        //   [125] aux_rg_w2 [K_RG=5, H]
        //   [126] aux_rg_b2 [K_RG=5]
        // Note: kernels emit (1/B_valid)*sum_b grad — `aux_param_grad_reduce`
        // sums per-sample partials, no extra division. SAXPY alpha is
        // `aux_weight` (small positive, [0.05, 0.3]).
        let final_ptr = self.aux_param_grad_final_buf.raw_ptr();
        // SP22 H6 vNext Phase B4 (2026-05-14): 4 new trade-outcome tensor
        // entries at indices [163..167) appended after the 8 K=2/K=5 entries.
        // SAXPY iteration uniformly scales all 12 by `aux_weight`.
        let aux_param_specs: [(usize, u64, usize); 12] = [
            (119, self.aux_partial_nb_w1.raw_ptr(), aux_h * sh2),
            (120, self.aux_partial_nb_b1.raw_ptr(), aux_h),
            (121, self.aux_partial_nb_w2.raw_ptr(), aux_knb * aux_h),
            (122, self.aux_partial_nb_b2.raw_ptr(), aux_knb),
            (123, self.aux_partial_rg_w1.raw_ptr(), aux_h * sh2),
            (124, self.aux_partial_rg_b1.raw_ptr(), aux_h),
            (125, self.aux_partial_rg_w2.raw_ptr(), aux_kr * aux_h),
            (126, self.aux_partial_rg_b2.raw_ptr(), aux_kr),
            // SP22 H6 vNext Phase B4: trade-outcome head (K=3) param grads.
            (163, self.aux_partial_to_w1.raw_ptr(), aux_h * sh2),
            (164, self.aux_partial_to_b1.raw_ptr(), aux_h),
            (165, self.aux_partial_to_w2.raw_ptr(), aux_kto * aux_h),
            (166, self.aux_partial_to_b2.raw_ptr(), aux_kto),
        ];
        for &(tensor_idx, partial_ptr, p_len) in &aux_param_specs {
            // Step (a): reduce per-sample partials [B, P] → final [P].
            self.aux_heads_bwd.param_grad_reduce(
                &self.stream,
                partial_ptr,
                b,
                p_len,
                final_ptr,
            )?;
            // Step (b): SAXPY into `grad_base[tensor_idx]`, scaled by aux_weight.
            // `grad_base[119..127)` is otherwise untouched by any earlier kernel
            // in this pipeline (Commit A's design: aux params are dormant
            // without this commit's wire-up). For the main backward call site
            // `grad_base == self.ptrs.grad_buf`; the vaccine path uses a
            // scratch buffer (`g_val`) but that sub-region also stays unwritten
            // for [119..127) — both code paths exercise the same aux backward.
            let dst_ptr = grad_base + padded_byte_offset(&param_sizes, tensor_idx);
            let n = p_len as i32;
            let blocks = ((p_len as u32 + 255) / 256).max(1);
            unsafe {
                self.stream
                    .launch_builder(&self.saxpy_f32_kernel)
                    .arg(&dst_ptr)
                    .arg(&final_ptr)
                    .arg(&aux_w)
                    .arg(&n)
                    .launch(LaunchConfig {
                        grid_dim: (blocks, 1, 1),
                        block_dim: (256, 1, 1),
                        shared_mem_bytes: 0,
                    })
                    .map_err(|e| MLError::ModelError(format!(
                        "aux_heads_backward saxpy tensor {tensor_idx}: {e}"
                    )))?;
            }
        }

        // SP14 Layer C Phase C.5b (2026-05-08): SAXPY both per-head
        // `aux_dh_s2_*_buf` buffers into the AUX TRUNK upstream-grad
        // accumulator `dh_s2_aux_accum` (NOT Q's `bw_d_h_s2`). The aux
        // trunk's backward kernel reads `dh_s2_aux_accum` as its
        // `dh_s2_aux_in_ptr` and propagates gradient through w3/w2/w1 +
        // b3/b2/b1, terminating at the encoder boundary (no `dx_in`
        // output param). Pre-zero the accumulator each step so we don't
        // carry forward last step's gradient — graph-safe via
        // `cuMemsetD32Async` (matches the d_h_s2 zero pattern in the
        // backward chain).
        let n_dh = (b * sh2) as i32;
        let blocks_dh = ((n_dh as u32 + 255) / 256).max(1);
        let dh_nb_ptr = self.aux_dh_s2_nb_buf.raw_ptr();
        let dh_rg_ptr = self.aux_dh_s2_rg_buf.raw_ptr();
        unsafe {
            // Pre-zero `dh_s2_aux_accum` so the SAXPY below establishes
            // the per-step value rather than accumulating across steps.
            cudarc::driver::sys::cuMemsetD32Async(
                dh_s2_aux_accum_ptr,
                0,
                (b * sh2),
                self.stream.cu_stream(),
            );
            self.stream
                .launch_builder(&self.saxpy_f32_kernel)
                .arg(&dh_s2_aux_accum_ptr)
                .arg(&dh_nb_ptr)
                .arg(&aux_w)
                .arg(&n_dh)
                .launch(LaunchConfig {
                    grid_dim: (blocks_dh, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!(
                    "aux_heads_backward saxpy dh_s2_aux next_bar: {e}"
                )))?;
            self.stream
                .launch_builder(&self.saxpy_f32_kernel)
                .arg(&dh_s2_aux_accum_ptr)
                .arg(&dh_rg_ptr)
                .arg(&aux_w)
                .arg(&n_dh)
                .launch(LaunchConfig {
                    grid_dim: (blocks_dh, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!(
                    "aux_heads_backward saxpy dh_s2_aux regime: {e}"
                )))?;
            // SP22 H6 vNext Phase B4 (2026-05-14): trade-outcome head's
            // per-sample dh_s2_aux. SAXPYs into the SAME `dh_s2_aux_accum`
            // buffer as the K=2 / K=5 heads — all three head's gradients
            // flow through the aux trunk to its own weights, NOT into Q's
            // encoder (structural stop-grad in `aux_trunk_backward`).
            // Same `aux_w` scaling as the K=2/K=5 SAXPYs above; Phase E
            // may differentiate per-head weights if magnitude balance
            // requires it.
            let dh_to_ptr = self.aux_dh_s2_to_buf.raw_ptr();
            self.stream
                .launch_builder(&self.saxpy_f32_kernel)
                .arg(&dh_s2_aux_accum_ptr)
                .arg(&dh_to_ptr)
                .arg(&aux_w)
                .arg(&n_dh)
                .launch(LaunchConfig {
                    grid_dim: (blocks_dh, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!(
                    "aux_heads_backward saxpy dh_s2_aux trade_outcome: {e}"
                )))?;
        }

        // SP14 Layer C Phase C.5b: aux trunk backward — propagates the
        // accumulated upstream gradient `dh_s2_aux_accum` through the
        // 3-layer aux trunk MLP, writing per-tensor gradients into the
        // 6 grad buffers (overwrite, not accumulate). Encoder boundary is
        // structural stop-grad: the kernel set has NO `dx_in` output, so
        // Q's encoder gradient is unaffected. Pearl
        // `pearl_no_host_branches_in_captured_graph.md` is preserved —
        // all kernels pre-loaded at construction; no host dispatch in
        // the captured region.
        self.aux_trunk_backward_ops.launch(
            &self.stream,
            dh_s2_aux_accum_ptr,
            encoder_out_ptr,
            self.h_aux1.raw_ptr(),
            self.h_aux2.raw_ptr(),
            self.aux_trunk_w2.raw_ptr(),
            self.aux_trunk_w3.raw_ptr(),
            self.dh_aux1_pre_scratch.raw_ptr(),
            self.dh_aux2_pre_scratch.raw_ptr(),
            self.aux_trunk_w1_grad.raw_ptr(),
            self.aux_trunk_b1_grad.raw_ptr(),
            self.aux_trunk_w2_grad.raw_ptr(),
            self.aux_trunk_b2_grad.raw_ptr(),
            self.aux_trunk_w3_grad.raw_ptr(),
            self.aux_trunk_b3_grad.raw_ptr(),
            b,
            encoder_out_dim,
            aux_h1,
            aux_h2,
            sh2,
        )?;

        // SP14 Layer C Phase C.5b: aux trunk Adam optimiser step —
        // consumes the 6 grad tensors via global L2-norm clip (single
        // norm spans all 6 tensors) + per-tensor Adam update. ISV-driven
        // hyperparams (LR, β1, β2, ε, grad_clip) read inside the launcher;
        // pinned LR/clip buffers are populated by the caller pre-capture
        // (host-write-through-mapped-pinned). The aux trunk step counter
        // `aux_trunk_t_dev_ptr` is incremented by `submit_counter_increments`
        // before this launch, ensuring Adam bias correction tracks step
        // count consistently across captured graph replays.
        self.launch_aux_trunk_adam_update(&self.stream, b as i32)?;

        Ok(())
    }

    /// Plan 4 Task 3 (E.3): launch `iqn_quantile_ema_update` (4-block,
    /// 256-thread shmem-reduction kernel). Reads the IQN online forward's
    /// per-quantile Q surface (`save_q_online [TBA, B*Q]`, populated by
    /// `GpuIqnHead::execute_training_pipeline` after each training step)
    /// and EMAs the mean |Q| at each off-median fixed-τ position
    /// {0.05, 0.25, 0.75, 0.95} into ISV[IQN_Q_P05_EMA_INDEX..=IQN_Q_P95_EMA_INDEX]
    /// (slots 99..103). Median (τ=0.50, idx=2) is intentionally skipped —
    /// already in the greedy-Q diagnostic.
    ///
    /// `save_q_online_ptr` / `tba` / `num_quantiles` are sourced from the
    /// `GpuIqnHead` accessors so the trainer doesn't have to duplicate the
    /// IQN config layout.  Producer-only — diagnostic surface for
    /// HEALTH_DIAG / risk monitoring; no consumer kernel reads slots
    /// [99..103) in this commit.
    ///
    /// No-op fast-return when `save_q_online_ptr == 0` (IQN not active in
    /// this trainer config — the Rust caller already gates on `gpu_iqn`'s
    /// `Option`, so a NULL here means an explicit programmer choice and
    /// returning silently keeps the caller's gating contract intact).
    pub fn launch_iqn_quantile_ema(
        &self,
        save_q_online_ptr: u64,
        tba: usize,
        num_quantiles: usize,
    ) -> Result<(), MLError> {
        // SP4 Layer A Task A13.4 retrofit (2026-05-01); GPU-Pearls refactor
        // (2026-05-01): kernel writes 4 step observations (mean |Q| at
        // p05/p25/p75/p95) to `producer_step_scratch_buf[59..63)`. The GPU
        // `apply_pearls_ad_kernel` (single launch, n_slots=4) maps each
        // contiguous scratch slot to ISV[99..103) and updates Wiener-EMA
        // state on the same stream — graph-capture-compatible.
        //
        // α is derived adaptively from per-slot Pearls A+D Wiener state — see
        // `sp4_wiener_ema::pearls_ad_update` (test-only reference).
        use crate::cuda_pipeline::sp4_wiener_ema::launch_apply_pearls;

        if save_q_online_ptr == 0 {
            return Ok(());
        }
        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_iqn_quantile_ema: isv_signals_dev_ptr must be allocated by constructor");
        debug_assert!(num_quantiles == 5,
            "launch_iqn_quantile_ema: Plan 4 Task 3 (E.3) pins IQN_NUM_QUANTILES = 5; \
             got num_quantiles={num_quantiles}");

        const SCRATCH_FIRST: usize = 59;
        // Layout invariant: ISV[99..103) = [P05, P25, P75, P95]
        // contiguous, mirroring the kernel's slot_offset 0..4.
        debug_assert_eq!(IQN_Q_P25_EMA_INDEX, IQN_Q_P05_EMA_INDEX + 1);
        debug_assert_eq!(IQN_Q_P75_EMA_INDEX, IQN_Q_P05_EMA_INDEX + 2);
        debug_assert_eq!(IQN_Q_P95_EMA_INDEX, IQN_Q_P05_EMA_INDEX + 3);

        let b_i = self.config.batch_size as i32;
        let q_i = num_quantiles as i32;
        let tba_i = tba as i32;
        let scratch_dev = self.producer_step_scratch_buf.dev_ptr;
        let scratch_first_i = SCRATCH_FIRST as i32;
        const BLOCK_DIM: u32 = 256;
        let smem_bytes = BLOCK_DIM * std::mem::size_of::<f32>() as u32;
        unsafe {
            self.stream.launch_builder(&self.iqn_quantile_ema_kernel)
                .arg(&save_q_online_ptr)
                .arg(&b_i)
                .arg(&q_i)
                .arg(&tba_i)
                .arg(&scratch_dev)
                .arg(&scratch_first_i)
                .launch(LaunchConfig {
                    grid_dim: (4, 1, 1),
                    block_dim: (BLOCK_DIM, 1, 1),
                    shared_mem_bytes: smem_bytes,
                })
                .map_err(|e| MLError::ModelError(format!("iqn_quantile_ema_update: {e}")))?;
        }

        // GPU Pearls A+D: scratch [59..63) → ISV [99..103), single launch
        // n_slots=4. Retrofit wiener convention: offset = scratch_idx*3 →
        // [177..189). Per-slot degenerate-zero (kernel hasn't run / cold
        // start before IQN forward populates `save_q_online`) is handled
        // by the applicator's `step_obs == 0` skip.
        unsafe {
            launch_apply_pearls(
                &self.stream,
                &self.apply_pearls_ad_kernel,
                scratch_dev,
                SCRATCH_FIRST as i32,
                self.isv_signals_dev_ptr,
                IQN_Q_P05_EMA_INDEX as i32,
                self.wiener_state_buf.dev_ptr,
                (SCRATCH_FIRST * 3) as i32,
                4,
                crate::cuda_pipeline::sp4_wiener_ema::ALPHA_META,
            )?;
        }
        Ok(())
    }

    /// Apply spectral normalization to all 10 weight matrices (trunk + 8 heads).
    ///
    /// One step of power iteration per call (standard practice — single step
    /// per training step converges in practice). Constrains ||W||_σ ≤ σ_max,
    /// bounding the network's Lipschitz constant and preventing Q-value explosion.
    ///
    /// Operates directly on `params_buf` via raw u64 offset pointers, eliminating
    /// the 10 D2D copies that previously synced per-layer weight slices back to
    /// params_buf. The spectral norm u/v vectors remain separate allocations.
    ///
    /// On first call (before `train_step_gpu` has flattened weights), this method
    /// flattens the weight sets into `params_buf` so the kernel has valid data.
    pub fn apply_spectral_norm(
        &mut self,
        online_dueling: &DuelingWeightSet,
        online_branching: &BranchingWeightSet,
    ) -> Result<(), MLError> {
        // Ensure params_buf is populated. On the first training step,
        // spectral norm runs before train_step_gpu's flatten call.
        if !self.params_initialized {
            self.flatten_online_weights(online_dueling, online_branching)?;
            self.params_initialized = true;
        }

        let _evt_guard = EventTrackingGuard::new(self.stream.context());
        let sigma_max = self.config.spectral_norm_sigma_max;
        let num_matrices = 13_i32;

        // Single batched launch: 12 blocks × 256 threads, one matrix per block.
        // Descriptor buffer was built at construction with stable pointers.
        let spectral_norm_descriptors_ptr = self.spectral_norm_descriptors.raw_ptr();
        unsafe {
            self.stream
                .launch_builder(&self.spectral_norm_batched_kernel)
                .arg(&spectral_norm_descriptors_ptr)
                .arg(&sigma_max)
                .arg(&num_matrices)
                .launch(LaunchConfig {
                    grid_dim: (num_matrices as u32, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0, // shmem[256] is static in the kernel
                })
                .map_err(|e| MLError::ModelError(format!("spectral_norm_batched: {e}")))?;
        }

        // Spectral norm wrote directly to params_buf — no shadow copy needed.

        Ok(())
    }

    /// Compute and return the current grad_buf L2 norm (synchronous readback).
    ///
    /// Used for per-component gradient diagnostics. Costs one stream sync +
    /// 4-byte DtoH transfer. Call sparingly (e.g. once per epoch or every N steps).
    pub fn read_grad_norm_sync(&mut self) -> Result<f32, MLError> {
        let _evt_guard = EventTrackingGuard::new(self.stream.context());

        // Two-phase grad_norm — no memset needed
        self.launch_grad_norm()?;
        self.launch_grad_norm_finalize()?;

        unsafe { cudarc::driver::sys::cuStreamSynchronize(self.stream.cu_stream()); }

        // grad_norm is pinned device-mapped — read directly from host pointer after sync.
        Ok(unsafe { *self.grad_norm_pinned })
    }

    /// Compute and return the current grad_buf L2 norm using the STANDALONE
    /// kernel handle (safe after mega-graph capture).
    ///
    /// `read_grad_norm_sync()` uses the graph-captured kernel handle which
    /// corrupts after `graph_mega` capture. This variant uses
    /// `grad_norm_standalone` — a separately loaded handle of the same kernel.
    ///
    /// Costs one stream sync + 4-byte DtoH. For diagnostic use only.
    pub fn read_grad_norm_standalone_sync(&mut self) -> Result<f32, MLError> {
        let _evt_guard = EventTrackingGuard::new(self.stream.context());

        self.launch_grad_norm_standalone()?;
        self.launch_grad_norm_finalize()?;

        unsafe { cudarc::driver::sys::cuStreamSynchronize(self.stream.cu_stream()); }

        // grad_norm is pinned device-mapped — read directly from host pointer after sync.
        Ok(unsafe { *self.grad_norm_pinned })
    }

    /// Apply GPU multi-head feature attention to `save_h_s2` (post-graph).
    ///
    /// Runs 4-head self-attention over the trunk output `h_s2 [B, SHARED_H2]`
    /// and writes the attended representation back into `save_h_s2`. The next
    /// CUDA Graph replay then uses the attended features as its trunk activation.
    ///
    /// This is a 1-step lag operation: attention modifies the buffer AFTER the
    /// current graph replay has already consumed the unmodified `save_h_s2`.
    /// The lag is acceptable — equivalent to async multi-task SGD.
    ///
    /// Must be called outside the CUDA Graph. Uses `EventTrackingGuard` for all
    /// buffer pointer extractions to avoid `CUDA_ERROR_INVALID_VALUE` from stale
    /// write events recorded during graph capture.
    pub fn apply_attention_forward(
        &mut self,
        attention: &mut GpuAttention,
        batch_size: usize,
    ) -> Result<(), MLError> {
        let sh2 = self.config.shared_h2;
        let n = (batch_size * sh2) as i32;

        // Disable cudarc event tracking — no sync needed, stream ordering
        // guarantees graph replay completed before these kernel launches.
        let _evt_guard = EventTrackingGuard::new(self.stream.context());

        // Copy save_h_s2 (f32) → attn_bw_scratch (f32) for attention forward
        let n_bytes = (n as usize) * std::mem::size_of::<f32>();
        let src_f32 = self.save_h_s2.raw_ptr();
        let dst_ptr = self.attn_bw_scratch.raw_ptr();
        self.graph_safe_copy_f32(dst_ptr, src_f32, n_bytes, "h_s2→attn_scratch")?;

        // Run attention forward with widened input: [D,B] states + [10,B] ofi_embed
        let attn_output = attention.forward(
            &self.attn_bw_scratch,
            &self.ofi_embed_output_buf,
            batch_size, None, None,
        )?;

        // Copy attention output (f32) back → save_h_s2 (f32)
        let attn_src = attn_output.raw_ptr();
        let save_dst = self.save_h_s2.raw_ptr();
        self.graph_safe_copy_f32(save_dst, attn_src, n_bytes, "attn_output→h_s2")?;

        Ok(())
    }

    /// Shrink-and-Perturb: prevent loss of plasticity between walk-forward windows.
    ///
    /// `params[i] = alpha * params[i] + (1 - alpha) * noise[i]`
    ///
    /// Where noise is drawn from N(0, sigma). This maintains the network's ability
    /// to learn new patterns across market regime shifts (Ash & Adams 2020).
    ///
    /// Called at walk-forward window boundaries (not per-step).
    /// Also invalidates the CUDA Graph since weights change outside the graph.
    pub fn shrink_and_perturb(
        &mut self,
        alpha: f32,
        sigma: f32,
    ) -> Result<(), MLError> {
        let n = self.total_params;
        let n_i32 = n as i32;
        let blocks = ((n + 255) / 256) as u32;
        let seed = (self.adam_step as u32).wrapping_mul(2654435761);

        // Protect ALL 4 branch heads from S&P: indices [8..24] in param_sizes.
        // (b0=direction, b1=magnitude, b2=order, b3=urgency — each has fc+out layers).
        // Branch heads encode the learned policy. Trunk + value head + bottleneck
        // are perturbed for generalization — trunk feature drift is acceptable because
        // branch heads adapt quickly (within 1-2 epochs on production data volumes).
        let param_sizes = compute_param_sizes(&self.config);
        let skip_start: i32 = (padded_byte_offset(&param_sizes, 8) / std::mem::size_of::<f32>() as u64) as i32;
        let skip_end: i32 = (padded_byte_offset(&param_sizes, 24) / std::mem::size_of::<f32>() as u64) as i32;

        // GPU-native: single kernel generates noise + blends in one pass on params_buf.
        // params_buf[i] = alpha * params_buf[i] + (1-alpha) * N(0, sigma)
        // Branch heads [skip_start, skip_end) are excluded from perturbation.
        // Use shrink_perturb_ungraphed — this runs AFTER graph capture (epoch boundary),
        // and shrink_perturb_kernel is in the same CUmodule as graphed kernels.
        // On Hopper, launching any CUfunction from a graphed CUmodule ungraphed
        // corrupts the child graph's kernel state (3100ms replay).
        let _evt_guard = EventTrackingGuard::new(self.stream.context());
        let params_ptr = self.params_buf.raw_ptr();
        unsafe {
            self.stream
                .launch_builder(&self.shrink_perturb_ungraphed)
                .arg(&params_ptr)
                .arg(&alpha)
                .arg(&sigma)
                .arg(&n_i32)
                .arg(&seed)
                .arg(&skip_start)
                .arg(&skip_end)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("shrink_perturb_kernel: {e}")))?;
        }
        drop(_evt_guard);

        // Signal graph re-capture needed — weights changed outside captured ops.
        self.graphs_captured = false;

        let skip_count = (skip_end - skip_start) as usize;
        info!(alpha, sigma, total_params = n, skip_count, "Shrink-and-Perturb applied (GPU-native, zero CPU, all branch heads protected)");
        Ok(())
    }

    /// Create a new fused DQN trainer with pre-allocated GPU buffers.
    ///
    /// Compiles all 4 kernels (forward+loss, backward, grad_norm, adam_update)
    /// from a single NVRTC compilation. All buffers are allocated once (fixed
    /// shapes for CUDA Graph compatibility).
    pub fn new(
        stream: Arc<CudaStream>,
        config: GpuDqnTrainConfig,
    ) -> Result<Self, MLError> {
        let b = config.batch_size;
        let total_params = compute_total_params(&config);
        let non_isv_params = compute_non_isv_params(&config);

        // Event tracking: kept ENABLED during normal ops (buffer allocation, DtoD).
        // DISABLED only during CUDA Graph capture (in capture_training_graphs).
        // Global disable pollutes the context for other tests/models.

        // Stack size for the training kernel: ~46KB/thread with NUM_ATOMS=51,
        // ~12KB/thread with NUM_ATOMS=11. DIST_SIZE(SCRATCH1_DIM=256) * 4 = 11KB
        // per array. Stack is set once in DQNTrainer::new() (64KB for all kernels).

        // ── Compile utility kernels (grad_norm, adam_update, etc.) ─
        let (grad_norm_kernel, grad_norm_finalize_kernel, grad_norm_finalize_adam, grad_norm_finalize_post_aux, adam_update_kernel, adam_update_post_aux, scale_f32_post_aux, saxpy_kernel, zero_kernel, regime_scale_kernel, shrink_perturb, _relu_mask_in_module, spectral_norm_kernel, clip_grad_kernel, pad_states_kernel, saxpy_f32_kernel, scale_f32_kernel, stochastic_depth_kernel, stochastic_depth_rng_kernel, bn_tanh_backward_kernel, bn_bias_grad_kernel, vaccine_dot_kernel, vaccine_project_kernel, vaccine_dot_finalize, causal_intervene_kernel_fn, causal_reduce_kernel_fn, causal_mean_reduce_kernel_fn, pruning_mask_kernel, pruning_compute_kernel, her_inplace_kernel, nan_check_f32_kernel, nan_check_f32_kernel_b, nan_check_fused_f32_kernel, clamp_finite_f32_kernel, popart_normalize_kernel, saxpy_f32_adam_grad, saxpy_f32_aux, scale_f32_aux, distill_saxpy_aux, grad_norm_standalone_post_aux, shrink_perturb_ungraphed, scale_f32_ungraphed, popart_robust_kernel, bn_tanh_concat_dd_kernel) =
            compile_training_kernels(&stream, &config)?;

        // Separate grad_norm instance for non-graph launches (outside CUDA graph).
        // CUDA graph captures CUfunction handles — the same handle can't be launched
        // both inside a replayed graph and outside on the same stream.
        let grad_norm_standalone = {
            let module = stream.context().load_cubin(DQN_UTILITY_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("grad_norm_standalone cubin: {e}")))?;
            module.load_function("dqn_grad_norm_kernel")
                .map_err(|e| MLError::ModelError(format!("grad_norm_standalone load: {e}")))?
        };

        // ── Compile EMA kernel (standalone — not captured in CUDA Graph) ──
        let ema_kernel = compile_ema_kernel(&stream)?;
        let relu_mask_standalone = compile_relu_mask_standalone(&stream)?;

        // ── Allocate batch input buffers ────────────────────────────
        // States buffers are padded to pad128(state_dim) per row so that
        // CUTLASS 128-element K-tiles in the first-layer GemmEx never read
        // past the row boundary.  The padding columns are kept at zero.
        let state_dim_padded = ml_core::state_layout::STATE_DIM_PADDED;
        let states_buf = alloc_f32(&stream, b * state_dim_padded, "states")?;
        let next_states_buf = alloc_f32(&stream, b * state_dim_padded, "next_states")?;
        let actions_buf = alloc_i32(&stream, b, "actions")?;
        let rewards_buf = stream.alloc_zeros::<f32>(b)
            .map_err(|e| MLError::ModelError(format!("alloc rewards f32: {e}")))?;
        let rewards_5bar = stream.alloc_zeros::<f32>(b)
            .map_err(|e| MLError::ModelError(format!("rewards_5bar alloc: {e}")))?;
        let rewards_20bar = stream.alloc_zeros::<f32>(b)
            .map_err(|e| MLError::ModelError(format!("rewards_20bar alloc: {e}")))?;
        let dones_buf = stream.alloc_zeros::<f32>(b)
            .map_err(|e| MLError::ModelError(format!("alloc dones f32: {e}")))?;
        let is_weights_buf = stream.alloc_zeros::<f32>(b)
            .map_err(|e| MLError::ModelError(format!("alloc is_weights f32: {e}")))?;

        // ── Allocate activation save buffers ────────────────────────
        // CUTLASS K-tile is 128 elements. When K < 128 (e.g. shared_h1=64),
        // the last column's tile overreads by (128-K) f32 values.
        // Pad each buffer by 128 elements to prevent compute-sanitizer OOB reports.
        let num_branches = 4;
        let kt = 128_usize; // CUTLASS K-tile padding
        let save_h_s1 = alloc_f32(&stream, b * config.shared_h1 + kt, "save_h_s1")?;
        let save_h_s2 = alloc_f32(&stream, b * config.shared_h2 + kt, "save_h_s2")?;
        let save_h_v = alloc_f32(&stream, b * config.value_h + kt, "save_h_v")?;
        let save_h_b0 = alloc_f32(&stream, b * config.adv_h + kt, "save_h_b0")?;
        let save_h_b1 = alloc_f32(&stream, b * config.adv_h + kt, "save_h_b1")?;
        let save_h_b2 = alloc_f32(&stream, b * config.adv_h + kt, "save_h_b2")?;
        let save_h_b3 = alloc_f32(&stream, b * config.adv_h + kt, "save_h_b3")?;
        // Magnitude branch is direction-conditioned (mag_concat_qdir kernel writes
        // SH2 + branch_0_size floats per state). Order/urgency branches are
        // OFI-conditioned (concat_ofi_features kernel writes SH2 + 3 floats per
        // state — exactly 3 OFI features per branch). These two strides differ
        // when branch_0_size != 3 (production: 4 = S/H/L/F).
        let mag_concat_dim = config.shared_h2 + config.branch_0_size;
        let ofi_concat_dim = config.shared_h2 + 3;
        let mag_concat_buf = alloc_f32(&stream, b * mag_concat_dim + kt, "mag_concat_buf")?;
        let d_mag_concat_buf = alloc_f32(&stream, b * mag_concat_dim + kt, "d_mag_concat_buf")?;
        let ord_concat_buf = alloc_f32(&stream, b * ofi_concat_dim + kt, "ord_concat_buf")?;
        let d_ord_concat_buf = alloc_f32(&stream, b * ofi_concat_dim + kt, "d_ord_concat_buf")?;
        let urg_concat_buf = alloc_f32(&stream, b * ofi_concat_dim + kt, "urg_concat_buf")?;
        let d_urg_concat_buf = alloc_f32(&stream, b * ofi_concat_dim + kt, "d_urg_concat_buf")?;
        let save_current_lp = alloc_f32(
            &stream,
            b * num_branches * config.num_atoms,
            "save_current_lp",
        )?;
        let save_projected = alloc_f32(
            &stream,
            b * num_branches * config.num_atoms,
            "save_projected",
        )?;

        // ── Allocate forward output buffers ─────────────────────────
        let per_sample_loss_buf = alloc_f32(&stream, b, "per_sample_loss")?;
        let td_errors_buf = alloc_f32(&stream, b, "td_errors")?;
        // total_loss + mse_loss — pinned device-mapped (zero-copy readback).
        let total_loss_pinned: *mut f32 = unsafe {
            let flags = cudarc::driver::sys::CU_MEMHOSTALLOC_DEVICEMAP;
            cudarc::driver::result::malloc_host(std::mem::size_of::<f32>(), flags)
                .map_err(|e| MLError::ModelError(format!("pinned total_loss alloc: {e}")))?
                as *mut f32
        };
        unsafe { *total_loss_pinned = 0.0; }
        let total_loss_dev_ptr = unsafe {
            let mut dp = 0u64;
            cudarc::driver::sys::cuMemHostGetDevicePointer_v2(&mut dp as *mut u64, total_loss_pinned.cast(), 0);
            dp
        };
        let mse_loss_pinned: *mut f32 = unsafe {
            let flags = cudarc::driver::sys::CU_MEMHOSTALLOC_DEVICEMAP;
            cudarc::driver::result::malloc_host(std::mem::size_of::<f32>(), flags)
                .map_err(|e| MLError::ModelError(format!("pinned mse_loss alloc: {e}")))?
                as *mut f32
        };
        unsafe { *mse_loss_pinned = 0.0; }
        let mse_loss_dev_ptr = unsafe {
            let mut dp = 0u64;
            cudarc::driver::sys::cuMemHostGetDevicePointer_v2(&mut dp as *mut u64, mse_loss_pinned.cast(), 0);
            dp
        };
        // iqn_readiness — pinned device-mapped (CVaR alpha for c51_loss_kernel).
        let iqn_readiness_pinned: *mut f32 = unsafe {
            let flags = cudarc::driver::sys::CU_MEMHOSTALLOC_DEVICEMAP;
            cudarc::driver::result::malloc_host(std::mem::size_of::<f32>(), flags)
                .map_err(|e| MLError::ModelError(format!("pinned iqn_readiness alloc: {e}")))?
                as *mut f32
        };
        unsafe { *iqn_readiness_pinned = 0.0; }
        let iqn_readiness_dev_ptr = unsafe {
            let mut dp = 0u64;
            cudarc::driver::sys::cuMemHostGetDevicePointer_v2(&mut dp as *mut u64, iqn_readiness_pinned.cast(), 0);
            dp
        };
        // SP4 Layer C close-out C2 (2026-05-01): mapped-pinned slots for the
        // IQN readiness gauge state (`iqn_loss_initial`, `iqn_loss_ema`).
        // Replace the prior host-resident `f32` fields per
        // `feedback_no_cpu_compute_strict`. Updated GPU-side by
        // `update_iqn_readiness_kernel` alongside `iqn_readiness_pinned`;
        // accessed host-side via the kept `iqn_loss_initial_value()` /
        // `iqn_loss_ema_value()` accessors that read the pinned slot directly.
        let iqn_loss_initial_pinned: *mut f32 = unsafe {
            let flags = cudarc::driver::sys::CU_MEMHOSTALLOC_DEVICEMAP;
            cudarc::driver::result::malloc_host(std::mem::size_of::<f32>(), flags)
                .map_err(|e| MLError::ModelError(format!("pinned iqn_loss_initial alloc: {e}")))?
                as *mut f32
        };
        unsafe { *iqn_loss_initial_pinned = 0.0; }
        let iqn_loss_initial_dev_ptr = unsafe {
            let mut dp = 0u64;
            cudarc::driver::sys::cuMemHostGetDevicePointer_v2(&mut dp as *mut u64, iqn_loss_initial_pinned.cast(), 0);
            dp
        };
        let iqn_loss_ema_pinned: *mut f32 = unsafe {
            let flags = cudarc::driver::sys::CU_MEMHOSTALLOC_DEVICEMAP;
            cudarc::driver::result::malloc_host(std::mem::size_of::<f32>(), flags)
                .map_err(|e| MLError::ModelError(format!("pinned iqn_loss_ema alloc: {e}")))?
                as *mut f32
        };
        unsafe { *iqn_loss_ema_pinned = 0.0; }
        let iqn_loss_ema_dev_ptr = unsafe {
            let mut dp = 0u64;
            cudarc::driver::sys::cuMemHostGetDevicePointer_v2(&mut dp as *mut u64, iqn_loss_ema_pinned.cast(), 0);
            dp
        };
        // SP4 Layer C close-out C3 (2026-05-01): mapped-pinned slot for the
        // atom utilization EMA. Replaces the prior host-resident
        // `utilization_ema: f32` field per `feedback_no_cpu_compute_strict`.
        // Updated GPU-side by `update_utilization_ema_kernel`. Constructor
        // initialises to 1.0 — the cold-start sentinel matching the deleted
        // host code's `if self.utilization_ema > 0.99 { assign obs }` branch.
        let utilization_ema_pinned: *mut f32 = unsafe {
            let flags = cudarc::driver::sys::CU_MEMHOSTALLOC_DEVICEMAP;
            cudarc::driver::result::malloc_host(std::mem::size_of::<f32>(), flags)
                .map_err(|e| MLError::ModelError(format!("pinned utilization_ema alloc: {e}")))?
                as *mut f32
        };
        unsafe { *utilization_ema_pinned = 1.0; }
        let utilization_ema_dev_ptr = unsafe {
            let mut dp = 0u64;
            cudarc::driver::sys::cuMemHostGetDevicePointer_v2(&mut dp as *mut u64, utilization_ema_pinned.cast(), 0);
            dp
        };
        // SP4 Layer C close-out C4 (2026-05-01): mapped-pinned slots for
        // `grad_norm_ema` (winsorized grad-norm EMA driving adaptive_clip)
        // and `outlier_diag` (winsor-trigger diagnostic). Replaces the
        // prior host-resident `grad_norm_ema: f32` field + the host-only
        // `tracing::warn!` decision per `feedback_no_cpu_compute_strict`.
        // grad_norm_ema_pinned init=0.0 — the cold-start sentinel that the
        // GPU kernel's `prev <= 0.0 ⇒ assign clamped` branch checks.
        let grad_norm_ema_pinned: *mut f32 = unsafe {
            let flags = cudarc::driver::sys::CU_MEMHOSTALLOC_DEVICEMAP;
            cudarc::driver::result::malloc_host(std::mem::size_of::<f32>(), flags)
                .map_err(|e| MLError::ModelError(format!("pinned grad_norm_ema alloc: {e}")))?
                as *mut f32
        };
        unsafe { *grad_norm_ema_pinned = 0.0; }
        let grad_norm_ema_dev_ptr = unsafe {
            let mut dp = 0u64;
            cudarc::driver::sys::cuMemHostGetDevicePointer_v2(&mut dp as *mut u64, grad_norm_ema_pinned.cast(), 0);
            dp
        };
        let outlier_diag_pinned: *mut f32 = unsafe {
            let flags = cudarc::driver::sys::CU_MEMHOSTALLOC_DEVICEMAP;
            cudarc::driver::result::malloc_host(std::mem::size_of::<f32>(), flags)
                .map_err(|e| MLError::ModelError(format!("pinned outlier_diag alloc: {e}")))?
                as *mut f32
        };
        unsafe { *outlier_diag_pinned = 0.0; }
        let outlier_diag_dev_ptr = unsafe {
            let mut dp = 0u64;
            cudarc::driver::sys::cuMemHostGetDevicePointer_v2(&mut dp as *mut u64, outlier_diag_pinned.cast(), 0);
            dp
        };
        let total_actions = config.branch_0_size + config.branch_1_size + config.branch_2_size + config.branch_3_size;
        let q_out_buf = alloc_f32(&stream, b * total_actions, "q_out")?;
        let eval_td_snapshot = alloc_f32(&stream, b, "eval_td_snapshot")?;
        let eval_loss_snapshot = alloc_f32(&stream, b, "eval_loss_snapshot")?;

        // ── Allocate backward / Adam buffers ────────────────────────
        // CUTLASS/cuBLAS read weight matrices in 32-row M-tiles. Output-layer
        // weights whose M dimension is not a multiple of 32 cause OOB reads
        // past the last row. Pad ALL flat parameter-sized buffers by
        // cutlass_tile_pad to prevent OOB without changing param_sizes,
        // offsets, or any GemmEx launch parameters. The padding is
        // zero-initialised and never updated by Adam (which operates on
        // exactly `total_params` elements).
        let cutlass_tile_pad = 32 * config.adv_h.max(config.value_h);
        let grad_buf = stream.alloc_zeros::<f32>(total_params + cutlass_tile_pad)
            .map_err(|e| MLError::ModelError(format!("alloc grad_buf f32: {e}")))?;
        // f32 master weights — Adam operates on these for full precision.
        let params_buf = stream.alloc_zeros::<f32>(total_params + cutlass_tile_pad)
            .map_err(|e| MLError::ModelError(format!("alloc params_buf f32: {e}")))?;
        let target_params_buf = stream.alloc_zeros::<f32>(total_params + cutlass_tile_pad)
            .map_err(|e| MLError::ModelError(format!("alloc target_params_buf f32: {e}")))?;
        let m_buf = stream.alloc_zeros::<f32>(total_params + cutlass_tile_pad)
            .map_err(|e| MLError::ModelError(format!("alloc adam_m f32: {e}")))?;
        let v_buf = stream.alloc_zeros::<f32>(total_params + cutlass_tile_pad)
            .map_err(|e| MLError::ModelError(format!("alloc adam_v f32: {e}")))?;

        // Plan 4 Task 1B-iv chain final fix: cache VSN tensor-slice byte offset
        // and float count for the second `target_ema_update` Polyak launch
        // that syncs target VSN params with online (without that launch
        // target VSN stays at `alloc_zeros` while online VSN trains).
        // VSN params themselves live in the shared `m_buf`/`v_buf` (main
        // Adam covers tensors `[0..NUM_WEIGHT_TENSORS)`); these cached
        // values are layout offsets only.
        let vsn_param_byte_offset: u64 = {
            let __vsn_sz = compute_param_sizes(&config);
            padded_byte_offset(&__vsn_sz, 95)
        };
        let vsn_param_total: usize = {
            let __vsn_sz = compute_param_sizes(&config);
            let begin_byte = padded_byte_offset(&__vsn_sz, 95) as usize;
            let end_byte   = padded_byte_offset(&__vsn_sz, 119) as usize;
            (end_byte - begin_byte) / std::mem::size_of::<f32>()
        };
        // grad_norm — pinned device-mapped (zero-copy readback).
        let grad_norm_pinned: *mut f32 = unsafe {
            let flags = cudarc::driver::sys::CU_MEMHOSTALLOC_DEVICEMAP;
            cudarc::driver::result::malloc_host(std::mem::size_of::<f32>(), flags)
                .map_err(|e| MLError::ModelError(format!("pinned grad_norm alloc: {e}")))?
                as *mut f32
        };
        unsafe { *grad_norm_pinned = 0.0; }
        let grad_norm_dev_ptr = unsafe {
            let mut dp = 0u64;
            cudarc::driver::sys::cuMemHostGetDevicePointer_v2(&mut dp as *mut u64, grad_norm_pinned.cast(), 0);
            dp
        };
        // Size for worst case: d_logits clipping uses val_blocks + adv_blocks
        // which can exceed total_params blocks at large batch sizes.
        let tba = config.branch_0_size + config.branch_1_size
            + config.branch_2_size + config.branch_3_size;
        let val_elems = b * pad32(config.num_atoms);
        let adv_elems = b * tba * config.num_atoms;
        let dlogit_blocks = (val_elems + 255) / 256 + (adv_elems + 255) / 256;
        let grad_norm_blocks = ((total_params + 255) / 256).max(dlogit_blocks);
        let grad_norm_partials = stream.alloc_zeros::<f32>(grad_norm_blocks)
            .map_err(|e| MLError::ModelError(format!("alloc grad_norm_partials: {e}")))?;
        let cql_grad_scratch = stream.alloc_zeros::<f32>(total_params + cutlass_tile_pad)
            .map_err(|e| MLError::ModelError(format!("alloc cql_grad_scratch f32: {e}")))?;

        // G1: Weight decay mask — 1.0 for trunk+value (indices 0-7), 0.0 for branch heads.
        // Allocated before CUDA Graph capture so pointer is stable across replays.
        // Mapped pinned staging eliminates HtoD per
        // `feedback_no_htod_htoh_only_mapped_pinned.md` — GPU reads via DtoD.
        let weight_decay_mask = {
            let param_sizes = compute_param_sizes(&config);
            let mut mask = vec![0.0_f32; total_params];
            let trunk_end: usize = (0..8).map(|i| align4(param_sizes[i])).sum();
            for i in 0..trunk_end {
                mask[i] = 1.0;
            }
            upload_via_mapped_f32(&stream, total_params, &mask, "wd_mask")?
        };

        // Task 0.4 — pinned host buffer for grad readback.
        // Plain pinned (NOT device-mapped) — written via memcpy_dtoh, not by GPU directly.
        // Sized to TOTAL_PARAMS f32. Allocated once, reused every epoch.
        let grad_readback_pinned_ptr: usize = unsafe {
            let bytes = total_params * std::mem::size_of::<f32>();
            cudarc::driver::result::malloc_host(bytes, 0)
                .map_err(|e| MLError::ModelError(format!("pinned grad_readback alloc ({} bytes): {e}", bytes)))?
                as usize
        };
        let grad_readback_pinned_capacity = total_params;

        // Task 2.0 — per-component grad-decomposition snapshots + pinned result slot.
        //
        // Task 2.Z diagnostic: each snapshot now covers three slices of
        // `grad_buf`:
        //   * Trunk (tensors 0..4 — w_s1, b_s1, w_s2, b_s2). Captures
        //     `apply_iqn_trunk_gradient` + `apply_ensemble_diversity_backward`
        //     SAXPY contributions that were structurally invisible to the
        //     prior branch-only measurement pipeline.
        //   * Direction (branch 0, tensors 8..12).
        //   * Magnitude (branch 1, tensors 12..16).
        //
        // Trunk tensors are at the start of the flat parameter layout
        // (offset 0). Direction and magnitude are a single contiguous
        // element range in `grad_buf`. Sizes are computed from
        // `compute_param_sizes` exactly like `per_branch_grad_norms`
        // (Task 0.4) so bounds stay in lock-step with the flat layout.
        let (grad_decomp_trunk_start, grad_decomp_trunk_len,
             grad_decomp_dir_start, grad_decomp_dir_len,
             grad_decomp_mag_start, grad_decomp_mag_len,
             grad_decomp_snapshot_len) = {
            let param_sizes = compute_param_sizes(&config);
            let trunk_start_byte = padded_byte_offset(&param_sizes, 0) as usize;
            let trunk_end_byte   = padded_byte_offset(&param_sizes, 4) as usize;
            let dir_start_byte   = padded_byte_offset(&param_sizes, 8) as usize;
            let mag_start_byte   = padded_byte_offset(&param_sizes, 12) as usize;
            let branch2_start_byte = padded_byte_offset(&param_sizes, 16) as usize;
            let elem_size = std::mem::size_of::<f32>();
            let trunk_start = trunk_start_byte / elem_size;
            let trunk_end   = trunk_end_byte / elem_size;
            let dir_start = dir_start_byte / elem_size;
            let mag_start = mag_start_byte / elem_size;
            let mag_end   = branch2_start_byte / elem_size;
            let trunk_len = trunk_end - trunk_start;
            let dir_len   = mag_start - dir_start;
            let mag_len   = mag_end - mag_start;
            let snap_len  = trunk_len + dir_len + mag_len;
            (trunk_start as i32, trunk_len as i32,
             dir_start as i32, dir_len as i32,
             mag_start as i32, mag_len as i32,
             snap_len)
        };
        let grad_snapshot_iqn = stream.alloc_zeros::<f32>(grad_decomp_snapshot_len)
            .map_err(|e| MLError::ModelError(format!("alloc grad_snapshot_iqn: {e}")))?;
        let grad_snapshot_cql = stream.alloc_zeros::<f32>(grad_decomp_snapshot_len)
            .map_err(|e| MLError::ModelError(format!("alloc grad_snapshot_cql: {e}")))?;
        // Task 2.0 extension — isolate `apply_cql_saxpy` (the point where the
        // CQL scratch is actually added into grad_buf with budget scaling).
        let grad_snapshot_cql_sx = stream.alloc_zeros::<f32>(grad_decomp_snapshot_len)
            .map_err(|e| MLError::ModelError(format!("alloc grad_snapshot_cql_sx: {e}")))?;
        // C51 fires BEFORE the aux phase and `grad_buf` is memset-zeroed at the
        // start of `submit_forward_ops_main` — no "pre-C51" grad exists to
        // snapshot. `grad_zero_ref_buf` is initialized to zeros and never
        // written again; the reduction kernel reads (current − 0) = current on
        // the C51 path.
        let grad_zero_ref_buf = stream.alloc_zeros::<f32>(grad_decomp_snapshot_len)
            .map_err(|e| MLError::ModelError(format!("alloc grad_zero_ref_buf: {e}")))?;
        // Task 2.0 extension — multiplicative scale + additive-writer snapshots.
        let grad_snapshot_c51_bs = stream.alloc_zeros::<f32>(grad_decomp_snapshot_len)
            .map_err(|e| MLError::ModelError(format!("alloc grad_snapshot_c51_bs: {e}")))?;
        let grad_snapshot_distill = stream.alloc_zeros::<f32>(grad_decomp_snapshot_len)
            .map_err(|e| MLError::ModelError(format!("alloc grad_snapshot_distill: {e}")))?;
        let grad_snapshot_rec = stream.alloc_zeros::<f32>(grad_decomp_snapshot_len)
            .map_err(|e| MLError::ModelError(format!("alloc grad_snapshot_rec: {e}")))?;
        let grad_snapshot_pred = stream.alloc_zeros::<f32>(grad_decomp_snapshot_len)
            .map_err(|e| MLError::ModelError(format!("alloc grad_snapshot_pred: {e}")))?;
        let grad_snapshot_ens = stream.alloc_zeros::<f32>(grad_decomp_snapshot_len)
            .map_err(|e| MLError::ModelError(format!("alloc grad_snapshot_ens: {e}")))?;

        // Pinned device-mapped 27-float result slot (Task 2.Z extension
        // grows from 18 → 27 floats — 3 floats per component × 9
        // components; the third slot is the trunk L2 norm). Device writes
        // via `grad_decomp_result_dev_ptr` (captured in graph); host reads
        // via `grad_decomp_result_pinned` at epoch boundary (zero-copy).
        let grad_decomp_result_pinned: *mut f32 = unsafe {
            let flags = cudarc::driver::sys::CU_MEMHOSTALLOC_DEVICEMAP;
            cudarc::driver::result::malloc_host(27 * std::mem::size_of::<f32>(), flags)
                .map_err(|e| MLError::ModelError(format!("pinned grad_decomp_result alloc: {e}")))?
                as *mut f32
        };
        unsafe {
            for i in 0..27 { *grad_decomp_result_pinned.add(i) = 0.0; }
        }
        let grad_decomp_result_dev_ptr = unsafe {
            let mut dp = 0u64;
            cudarc::driver::sys::cuMemHostGetDevicePointer_v2(
                &mut dp as *mut u64,
                grad_decomp_result_pinned.cast(),
                0,
            );
            dp
        };

        // Reduction kernel — loaded from a dedicated CUmodule so its CUfunction
        // is isolated from all other graphs (Hopper CUfunction-isolation rule).
        let grad_decomp_kernel = {
            let module = stream.context().load_cubin(GRAD_DECOMP_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("grad_decomp cubin load: {e}")))?;
            module.load_function("grad_component_delta_norm")
                .map_err(|e| MLError::ModelError(format!("grad_component_delta_norm load: {e}")))?
        };
        // SP7 Path A (2026-05-03) — raw CQL norm kernel. Same shape as
        // grad_decomp_kernel but reads `cql_grad_scratch` directly (no
        // snapshot pair). Loaded into its own CUmodule for the same
        // CUfunction-isolation reason.
        let cql_raw_norm_kernel = {
            let module = stream.context().load_cubin(CQL_RAW_NORM_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("cql_raw_norm cubin load: {e}")))?;
            module.load_function("cql_raw_norm_compute")
                .map_err(|e| MLError::ModelError(format!("cql_raw_norm_compute load: {e}")))?
        };
        info!(
            "GpuDqnTrainer: grad_decomp kernel loaded — snapshot_len={} trunk=[{}..+{}] dir=[{}..+{}] mag=[{}..+{}]",
            grad_decomp_snapshot_len,
            grad_decomp_trunk_start, grad_decomp_trunk_len,
            grad_decomp_dir_start, grad_decomp_dir_len,
            grad_decomp_mag_start, grad_decomp_mag_len,
        );

        // ── Adaptive per-branch gradient-norm balancer (Task: L40S grad imbalance fix) ──
        // Caps any of the 4 factored-action branches' weight-gradient L2 norm at
        //   `num_branches × median_branch_norm`
        // where `num_branches = 4` is architectural (direction, magnitude,
        // order, urgency — the action factorisation) and `median_branch_norm`
        // is a per-step statistical reference.  Scale-only-when-needed, no
        // tuned constants.
        let (branch_slice_starts_dev, branch_slice_lens_dev, branch_slice_max_len) = {
            let param_sizes = compute_param_sizes(&config);
            let elem_size = std::mem::size_of::<f32>();
            let mut starts = [0_i32; 4];
            let mut lens   = [0_i32; 4];
            let mut max_len = 0_usize;
            // Branch b owns tensors (8+4b)..(12+4b), which are contiguous in
            // the padded flat layout.  Reading the padded range is safe
            // because grad_buf is memset-zeroed at step start and writes
            // never touch padding bytes — so padding always contributes 0
            // to the L2 sum and no-ops under scalar multiplication.
            for b in 0..4_usize {
                let first = 8 + b * 4;
                let start_byte = padded_byte_offset(&param_sizes, first) as usize;
                let end_byte   = padded_byte_offset(&param_sizes, first + 4) as usize;
                let start = start_byte / elem_size;
                let end   = end_byte / elem_size;
                let len   = end - start;
                starts[b] = start as i32;
                lens[b]   = len as i32;
                if len > max_len { max_len = len; }
            }
            let starts_dev = upload_via_mapped_i32(&stream, 4, &starts, "branch_slice_starts_dev")?;
            let lens_dev = upload_via_mapped_i32(&stream, 4, &lens, "branch_slice_lens_dev")?;
            (starts_dev, lens_dev, max_len)
        };
        let branch_grad_norms_dev = stream.alloc_zeros::<f32>(4)
            .map_err(|e| MLError::ModelError(format!("alloc branch_grad_norms_dev: {e}")))?;
        let branch_grad_scales_dev = {
            // Initialised to 1.0 so HEALTH_DIAG reads a sensible "no-op"
            // value before the first rescale launch writes real scales.
            let ones = [1.0_f32; 4];
            upload_via_mapped_f32(&stream, 4, &ones, "branch_grad_scales_dev")?
        };
        // Load reduce + rescale kernels from a dedicated CUmodule so their
        // CUfunction handles are isolated from all other graphs (Hopper
        // CUfunction-isolation rule — shared handles corrupt captured
        // kernel state).
        let (branch_grad_balance_reduce, branch_grad_balance_isv_update, branch_grad_balance_rescale) = {
            let module = stream.context().load_cubin(BRANCH_GRAD_BALANCE_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("branch_grad_balance cubin load: {e}")))?;
            let reduce = module.load_function("branch_grad_norm_reduce")
                .map_err(|e| MLError::ModelError(format!("branch_grad_norm_reduce load: {e}")))?;
            let isv_update = module.load_function("grad_balance_isv_update")
                .map_err(|e| MLError::ModelError(format!("grad_balance_isv_update load: {e}")))?;
            let rescale = module.load_function("branch_grad_rescale")
                .map_err(|e| MLError::ModelError(format!("branch_grad_rescale load: {e}")))?;
            (reduce, isv_update, rescale)
        };
        info!(
            "GpuDqnTrainer: branch_grad_balance kernels loaded (ISV-driven) — max_slice_len={} num_branches=4 (architectural)",
            branch_slice_max_len,
        );

        // Plan 1 Task 13: load tau_update kernel (cold-path, per-epoch).
        let tau_update_kernel = {
            let module = stream.context().load_cubin(TAU_UPDATE_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("tau_update cubin load: {e}")))?;
            module.load_function("tau_update")
                .map_err(|e| MLError::ModelError(format!("tau_update load: {e}")))?
        };

        // Plan 1 Task 14: load epsilon_update kernel (cold-path, per-epoch).
        let epsilon_update_kernel = {
            let module = stream.context().load_cubin(EPSILON_UPDATE_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("epsilon_update cubin load: {e}")))?;
            module.load_function("epsilon_update")
                .map_err(|e| MLError::ModelError(format!("epsilon_update load: {e}")))?
        };

        // Plan 2 Task 3 D.2: load per_branch_gamma_update kernel (cold-path, per-epoch).
        let per_branch_gamma_update_kernel = {
            let module = stream.context().load_cubin(PER_BRANCH_GAMMA_UPDATE_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("per_branch_gamma_update cubin load: {e}")))?;
            module.load_function("per_branch_gamma_update")
                .map_err(|e| MLError::ModelError(format!("per_branch_gamma_update load: {e}")))?
        };
        // D.2: per-branch gamma base/max device arrays — allocated once at construction.
        let gamma_base_host: [f32; 4] = [0.92, 0.88, 0.85, 0.80]; // DIR, MAG, ORD, URG
        let gamma_max_host:  [f32; 4] = [0.99, 0.95, 0.93, 0.90]; // DIR, MAG, ORD, URG
        let per_branch_gamma_base_dev =
            upload_via_mapped_f32(&stream, 4, &gamma_base_host, "per_branch_gamma_base")?;
        let per_branch_gamma_max_dev =
            upload_via_mapped_f32(&stream, 4, &gamma_max_host, "per_branch_gamma_max")?;

        // Plan 1 Task 11: load kelly_cap_update kernel (cold-path, per-epoch).
        let kelly_cap_update_kernel = {
            let module = stream.context().load_cubin(KELLY_CAP_UPDATE_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("kelly_cap_update cubin load: {e}")))?;
            module.load_function("kelly_cap_update")
                .map_err(|e| MLError::ModelError(format!("kelly_cap_update load: {e}")))?
        };

        // Plan 1 Task 9: load atoms_update kernel (cold-path, per-epoch + SGD step).
        let atoms_update_kernel = {
            let module = stream.context().load_cubin(ATOMS_UPDATE_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("atoms_update cubin load: {e}")))?;
            module.load_function("atoms_update")
                .map_err(|e| MLError::ModelError(format!("atoms_update load: {e}")))?
        };

        // Plan 2 Task 1 C.1: load q_quantile_reduce kernel (cold-path, per-epoch).
        let q_quantile_kernel = {
            let module = stream.context().load_cubin(Q_QUANTILE_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("q_quantile cubin load: {e}")))?;
            module.load_function("q_quantile_reduce")
                .map_err(|e| MLError::ModelError(format!("q_quantile_reduce load: {e}")))?
        };

        // SP4 Task A13.5 (2026-05-01): the trainer-side
        // `reward_component_ema_kernel` loader was deleted along with the
        // orphan `launch_reward_component_ema`. The active kernel is loaded
        // by `GpuExperienceCollector` from the same cubin
        // (REWARD_COMPONENT_EMA_CUBIN) — the collector owns the launcher
        // (`launch_reward_component_ema_inplace`) and threads the trainer's
        // mapped-pinned Pearls A+D buffers via
        // `set_reward_component_pearls_buffers` (called from
        // `FusedTrainingCtx::wire_reward_component_pearls_buffers`).

        // Plan 4 Task 2c.3c.5: load h_s2_rms_ema kernel (cold-path, per-step).
        // Single-block 256-thread shmem-reduction kernel; no atomicAdd. Producer
        // for ISV[H_S2_RMS_EMA_INDEX]; consumer wired in 2c.3c.6.
        let h_s2_rms_ema_kernel = {
            let module = stream.context().load_cubin(H_S2_RMS_EMA_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("h_s2_rms_ema cubin load: {e}")))?;
            module.load_function("h_s2_rms_ema_update")
                .map_err(|e| MLError::ModelError(format!("h_s2_rms_ema_update load: {e}")))?
        };

        // SP4 Layer A Task A5: load target_q_p99 producer kernel.
        // Single-block kernel reading `denoise_target_q_buf`, writing
        // step_p99 to `producer_step_scratch_buf[0]`. Pearls A+D applied
        // via the GPU `apply_pearls_ad_kernel` chained on the same stream
        // (2026-05-01 GPU-Pearls refactor). Producer-only — no consumer
        // wired yet (Mech 1 still uses `10 × Q_ABS_REF.max(1.0)`).
        let target_q_p99_update = {
            let module = stream.context().load_cubin(SP4_TARGET_Q_P99_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp4 target_q_p99 cubin load: {e}")))?;
            module.load_function("target_q_p99_update")
                .map_err(|e| MLError::ModelError(format!("target_q_p99_update load: {e}")))?
        };

        // SP4 Layer A Task A6: load atom_pos_p99 producer kernel.
        // Single parameterised single-block kernel; launcher loops branches
        // 0..4 with distinct slice pointers + scratch slots (1..5), then
        // chains the GPU `apply_pearls_ad_kernel` (single launch n_slots=4)
        // on the same stream (2026-05-01 GPU-Pearls refactor).
        // Producer-only — no consumer wired yet (Mech 2 still uses
        // `±10 × ISV[Q_ABS_REF=16].max(1.0)` in `atoms_update_kernel.cu`).
        let atom_pos_p99_update = {
            let module = stream.context().load_cubin(SP4_ATOM_POS_P99_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp4 atom_pos_p99 cubin load: {e}")))?;
            module.load_function("atom_pos_p99_update")
                .map_err(|e| MLError::ModelError(format!("atom_pos_p99_update load: {e}")))?
        };

        // SP4 Layer A Task A7: load param_group_oracle producer kernel.
        // Single-block fused per-param-group oracle: reads
        // `(params, grads, adam_m, adam_v)` once and runs 4-5 sequential
        // passes producing WEIGHT/ADAM_M/ADAM_V p99 + WD_RATE + (group 0
        // only) L1_LAMBDA_TRUNK. Pearls A+D applied via the GPU
        // `apply_pearls_ad_kernel` chained per output slot (2026-05-01
        // GPU-Pearls refactor — single-slot launches; non-contiguous
        // (scratch, ISV) per group prevents batching). Producer-only —
        // no consumer wired yet (Mech 9 still uses hardcoded
        // `weight_clamp_max_abs` config args; weight-decay/L1 unchanged).
        let param_group_oracle_update = {
            let module = stream.context().load_cubin(SP4_PARAM_GROUP_ORACLE_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp4 param_group_oracle cubin load: {e}")))?;
            module.load_function("param_group_oracle_update")
                .map_err(|e| MLError::ModelError(format!("param_group_oracle_update load: {e}")))?
        };

        // SP4 Layer A Task A8: load grad_norm_p99 producer kernel.
        // Single-thread single-block kernel that copies `grad_norm_buf[0]`
        // (single-element scalar populated by `launch_grad_norm_finalize`) to
        // `producer_step_scratch_buf[38]` with `__threadfence_system()`.
        // p99 over a single element IS the element itself, hence the
        // degenerate-copy form (no histogram). Pearls A+D applied via the
        // GPU `apply_pearls_ad_kernel` chained on the same stream
        // (2026-05-01 GPU-Pearls refactor). Producer-only — Mech 6's
        // adaptive_clip upper bound consumer migrates in Layer B.
        let grad_norm_p99_update = {
            let module = stream.context().load_cubin(SP4_GRAD_NORM_P99_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp4 grad_norm_p99 cubin load: {e}")))?;
            module.load_function("grad_norm_p99_update")
                .map_err(|e| MLError::ModelError(format!("grad_norm_p99_update load: {e}")))?
        };

        // SP4 Layer A Task A9: load h_s2_p99 producer kernel (cold-path).
        // Single-block 256-thread kernel reading `save_h_s2 [B × SH2]` via
        // `sp4_histogram_p99<256>`, writing step_p99 to
        // `producer_step_scratch_buf[39]`. Pearls A+D applied via the GPU
        // `apply_pearls_ad_kernel` (2026-05-01 GPU-Pearls refactor).
        // Producer-only — Mech 10's clamp consumer migrates in Layer B.
        // NB: distinct from existing `h_s2_rms_ema_update` (slot 96, RMS
        // signal); Task A13 retrofitted that existing kernel separately.
        let h_s2_p99_update = {
            let module = stream.context().load_cubin(SP4_H_S2_P99_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp4 h_s2_p99 cubin load: {e}")))?;
            module.load_function("h_s2_p99_update")
                .map_err(|e| MLError::ModelError(format!("h_s2_p99_update load: {e}")))?
        };

        // SP4 Layer C #260: load bw_d_h_s2_p99 producer kernel (cold-path).
        // Single-block 256-thread kernel reading `bw_d_h_s2 [B × SH2]` via
        // `sp4_histogram_p99<256>`, writing step_p99 to
        // `producer_step_scratch_buf[69]`. Pearls A+D applied via the GPU
        // `apply_pearls_ad_kernel`. Replaces SP1-Phase-C
        // `1e6 × ISV[H_S2_RMS_EMA].max(1.0)` hardcoded multiplier in
        // `apply_iqn_trunk_gradient`.
        let bw_d_h_s2_p99_update = {
            let module = stream.context().load_cubin(SP4_BW_D_H_S2_P99_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp4 bw_d_h_s2_p99 cubin load: {e}")))?;
            module.load_function("bw_d_h_s2_p99_update")
                .map_err(|e| MLError::ModelError(format!("bw_d_h_s2_p99_update load: {e}")))?
        };

        // SP4 Layer C #260: load q_dir_grad_p99 producer kernel (cold-path).
        // Single-block 256-thread kernel reading the union of
        // `d_value_logits` ∪ `d_adv_logits` (n_sub=2) via
        // `sp4_histogram_p99_multi<256>`, writing step_p99 to
        // `producer_step_scratch_buf[70]`. Pearls A+D applied via the GPU
        // `apply_pearls_ad_kernel`. Replaces SP1-Phase-C
        // `1e6 × ISV[Q_DIR_ABS_REF].max(1.0)` hardcoded multiplier in
        // `launch_cublas_backward_to`.
        let q_dir_grad_p99_update = {
            let module = stream.context().load_cubin(SP4_Q_DIR_GRAD_P99_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp4 q_dir_grad_p99 cubin load: {e}")))?;
            module.load_function("q_dir_grad_p99_update")
                .map_err(|e| MLError::ModelError(format!("q_dir_grad_p99_update load: {e}")))?
        };

        // SP4 GPU-only Pearls A+D applicator (2026-05-01). Single-thread
        // device-side loop kernel that consumes scratch_buf step
        // observations and applies Pearls A+D in-place to ISV + Wiener
        // state. Replaces the host-side `apply_pearls_to_slot` helper
        // per `feedback_no_cpu_compute_strict.md`. Graph-capture-
        // compatible by construction (no host sync; same-stream ordering
        // with the producer kernel that wrote scratch_buf).
        let apply_pearls_ad_kernel = {
            let module = stream.context().load_cubin(APPLY_PEARLS_AD_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("apply_pearls_ad cubin load: {e}")))?;
            module.load_function("apply_pearls_ad_kernel")
                .map_err(|e| MLError::ModelError(format!("apply_pearls_ad_kernel load: {e}")))?
        };

        // SP5 Task A1 (2026-05-01): load q_branch_stats producer kernel.
        // Single-block 4-thread kernel (one thread per branch: Dir/Mag/Ord/Urg).
        // Reads q_out_buf [B, 13] row-major; writes 16 floats to scratch[71..87).
        let q_branch_stats_kernel = {
            let module = stream.context().load_cubin(SP5_Q_BRANCH_STATS_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp5 q_branch_stats cubin load: {e}")))?;
            module.load_function("q_branch_stats_update")
                .map_err(|e| MLError::ModelError(format!("q_branch_stats_update load: {e}")))?
        };

        // SP5 Task A1 (2026-05-01): load pearl_1_atom controller kernel.
        // Single-block 4-thread kernel; reads q_branch_stats scratch + ISV HEADROOM
        // + atoms_clip_count_buf; writes 16 floats to scratch[87..103).
        let pearl_1_atom_kernel = {
            let module = stream.context().load_cubin(SP5_PEARL_1_ATOM_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp5 pearl_1_atom cubin load: {e}")))?;
            module.load_function("pearl_1_atom_update")
                .map_err(|e| MLError::ModelError(format!("pearl_1_atom_update load: {e}")))?
        };

        // SP5 Task A2 (2026-05-01): load pearl_3_sigma NoisyNet sigma controller kernel.
        // Single-block 4-thread kernel; reads ATOM_V_HALF + BRANCH_ENTROPY + SIGMA_FRACTION
        // from ISV; writes (sigma[4], SF[4]) to scratch[103..111).
        let pearl_3_sigma_kernel = {
            let module = stream.context().load_cubin(SP5_PEARL_3_SIGMA_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp5 pearl_3_sigma cubin load: {e}")))?;
            module.load_function("pearl_3_sigma_update")
                .map_err(|e| MLError::ModelError(format!("pearl_3_sigma_update load: {e}")))?
        };

        // SP5 Task A3 + SP7 (2026-05-03): load pearl_2_budget IQN budget + flatness kernel.
        // Single-block 4-thread kernel; reads Q_VAR_PER_BRANCH (ISV[222..226)) +
        // NOISY_SIGMA (ISV[210..214)) from ISV; writes 8 floats to scratch[115..119, 127..131).
        // Must be invoked AFTER pearl_1_atom (Q_VAR) AND pearl_3_sigma (NOISY_SIGMA).
        let pearl_2_budget_kernel = {
            let module = stream.context().load_cubin(SP5_PEARL_2_BUDGET_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp5 pearl_2_budget cubin load: {e}")))?;
            module.load_function("pearl_2_budget_update")
                .map_err(|e| MLError::ModelError(format!("pearl_2_budget_update load: {e}")))?
        };

        // SP5 Task A1 (2026-05-01): allocate atoms_clip_count_buf [4 i32].
        // Zero-initialized; populated by atoms_update_kernel in Layer B.
        // Until Layer B wires, clip_rate=0 → headroom holds at 6.0 bootstrap.
        let atoms_clip_count_buf = unsafe { MappedI32Buffer::new(4) }
            .map_err(|e| MLError::ModelError(format!("SP5 atoms_clip_count_buf alloc: {e}")))?;

        // SP5 Task A1 (2026-05-01): static action layout descriptors.
        // action_counts: {4, 3, 3, 3} (Dir branches have 4 actions, others 3).
        // action_offsets: {0, 4, 7, 10} (cumulative: 0, 0+4=4, 4+3=7, 7+3=10).
        let action_counts_buf = unsafe { MappedI32Buffer::new(4) }
            .map_err(|e| MLError::ModelError(format!("SP5 action_counts_buf alloc: {e}")))?;
        action_counts_buf.write_from_slice(&[4_i32, 3, 3, 3]);

        let action_offsets_buf = unsafe { MappedI32Buffer::new(4) }
            .map_err(|e| MLError::ModelError(format!("SP5 action_offsets_buf alloc: {e}")))?;
        action_offsets_buf.write_from_slice(&[0_i32, 4, 7, 10]);

        // SP5 Task A4 (2026-05-01): load grad_cosine_sim_kernel (auxiliary
        // gradient cosine similarity). Single-block 8-thread, one thread per
        // SP4 param group. Reads grad_buf + grad_prev_buf_per_group.
        let grad_cosine_sim_kernel = {
            let module = stream.context().load_cubin(SP5_GRAD_COSINE_SIM_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp5 grad_cosine_sim cubin load: {e}")))?;
            module.load_function("grad_cosine_sim_update")
                .map_err(|e| MLError::ModelError(format!("grad_cosine_sim_update load: {e}")))?
        };

        // SP5 Task A4 (2026-05-01): load pearl_4_adam_hparams_kernel.
        // Single-block 8-thread, one thread per SP4 param group. Reads
        // cosine_sim + l2_norm from scratch; writes β1/β2/ε to scratch.
        let pearl_4_adam_hparams_kernel = {
            let module = stream.context().load_cubin(SP5_PEARL_4_ADAM_HPARAMS_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp5 pearl_4_adam_hparams cubin load: {e}")))?;
            module.load_function("pearl_4_adam_hparams_update")
                .map_err(|e| MLError::ModelError(format!("pearl_4_adam_hparams_update load: {e}")))?
        };

        // SP5 Task A5 (2026-05-01): load q_skew_kurtosis_kernel (auxiliary
        // per-branch Q-distribution skew + kurtosis reducer). Single-block
        // 4-thread kernel. Reads save_q_online [B × 13]; writes skew[4] →
        // scratch[171..175) and ex_kurt[4] → scratch[175..179).
        let q_skew_kurtosis_kernel = {
            let module = stream.context().load_cubin(SP5_Q_SKEW_KURTOSIS_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp5 q_skew_kurtosis cubin load: {e}")))?;
            module.load_function("q_skew_kurtosis_update")
                .map_err(|e| MLError::ModelError(format!("q_skew_kurtosis_update load: {e}")))?
        };

        // SP5 Task A5 (2026-05-01): load pearl_5_iqn_tau_kernel.
        // Single-block 4-thread kernel. Reads skew from scratch; writes
        // τ schedule [4 branches × 5 quantiles] → scratch[179..199).
        let pearl_5_iqn_tau_kernel = {
            let module = stream.context().load_cubin(SP5_PEARL_5_IQN_TAU_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp5 pearl_5_iqn_tau cubin load: {e}")))?;
            module.load_function("pearl_5_iqn_tau_update")
                .map_err(|e| MLError::ModelError(format!("pearl_5_iqn_tau_update load: {e}")))?
        };

        // SP5 Task A6 (2026-05-01): load pearl_6_kelly_kernel.
        // Single-block 6-thread kernel. Reads portfolio_state; writes
        // directly to ISV[280..286). CROSS-FOLD-PERSISTENT.
        // No apply_pearls — sentinel incompatible with cross-fold persistence.
        let pearl_6_kelly_kernel = {
            let module = stream.context().load_cubin(SP5_PEARL_6_KELLY_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp5 pearl_6_kelly cubin load: {e}")))?;
            module.load_function("pearl_6_kelly_update")
                .map_err(|e| MLError::ModelError(format!("pearl_6_kelly_update load: {e}")))?
        };

        // SP5 Task A7 (2026-05-01): load pearl_8_trail_kernel.
        // 4-thread single-block kernel. Reads features[bar_idx * market_dim + 9];
        // writes trail_dist[4 dirs] to scratch[SCRATCH_PEARL_8_TRAIL=199..203).
        // FoldReset: ISV[270..274) zeroed at fold boundary (Pearl A sentinel).
        let pearl_8_trail_kernel = {
            let module = stream.context().load_cubin(SP5_PEARL_8_TRAIL_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp5 pearl_8_trail cubin load: {e}")))?;
            module.load_function("pearl_8_trail_update")
                .map_err(|e| MLError::ModelError(format!("pearl_8_trail_update load: {e}")))?
        };

        // SP5 Task A8 (2026-05-01): load pearl_1_ext_num_atoms_kernel.
        // 4-thread single-block kernel. Reads ISV[ATOM_V_HALF_BASE=178..182) (Pearl 1, Task A1);
        // writes num_atoms[4 branches] to scratch[SCRATCH_PEARL_1_EXT_NUM_ATOMS=203..207).
        // FoldReset: ISV[274..278) zeroed at fold boundary (Pearl A sentinel).
        let pearl_1_ext_num_atoms_kernel = {
            let module = stream.context().load_cubin(SP5_PEARL_1_EXT_NUM_ATOMS_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp5 pearl_1_ext_num_atoms cubin load: {e}")))?;
            module.load_function("pearl_1_ext_num_atoms_update")
                .map_err(|e| MLError::ModelError(format!("pearl_1_ext_num_atoms_update load: {e}")))?
        };

        // SP5 Layer D Task D1 (2026-05-02): load pnl_aggregation_kernel.
        // Single-block 256-thread shmem-reduce kernel. Reads per-trade event
        // arrays; writes 4 aggregated floats (total/mean/var/max_dd) to
        // scratch[SCRATCH_PNL_AGG_BASE=207..211). FoldReset: ISV[286..290) zeroed
        // at fold boundary (Pearl A sentinel; Layer D D1 outputs are NOT cross-fold
        // persistent — they take the standard sentinel-bootstrap path).
        // Producer-only in D1; consumer wires atomically in D4 per
        // `feedback_no_partial_refactor.md`.
        let pnl_aggregation_kernel = {
            let module = stream.context().load_cubin(SP5_PNL_AGGREGATION_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp5 pnl_aggregation cubin load: {e}")))?;
            module.load_function("pnl_aggregation_update")
                .map_err(|e| MLError::ModelError(format!("pnl_aggregation_update load: {e}")))?
        };

        // SP5 Layer D Task D2 (2026-05-02): load health_composition_kernel.
        // Single-block 1-thread arithmetic kernel. Takes 7 raw inputs by value;
        // writes 4 floats (composed score + q_gap_norm + q_var_norm +
        // grad_norm_norm) to scratch[SCRATCH_HEALTH_COMP_BASE=211..215).
        // FoldReset: ISV[290..294) zeroed at fold boundary alongside the
        // host-side `health_ema` EMAs (Pearl A sentinel path). Producer-only
        // in D2; consumer wires atomically in D4 per
        // `feedback_no_partial_refactor.md`.
        let health_composition_kernel = {
            let module = stream.context().load_cubin(SP5_HEALTH_COMPOSITION_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp5 health_composition cubin load: {e}")))?;
            module.load_function("health_composition_update")
                .map_err(|e| MLError::ModelError(format!("health_composition_update load: {e}")))?
        };

        // SP5 Layer D Task D3 (2026-05-02): load training_metrics_ema_kernel.
        // Single-block 3-thread arithmetic kernel. Reads 5 host-passed f32
        // scalars (epoch_sharpe + epoch_max_dd raw observations, plus
        // prev_sharpe_ema / prev_max_dd_ema / prev_low_dd_ratio host-tracked
        // EMAs) and 1 i32 sentinel flag (sharpe_initialized); writes 3 floats
        // (sharpe/max_dd/low_dd_ratio) to scratch[SCRATCH_TRAINING_METRICS_EMA_BASE
        // =215..218). FoldReset: ISV[294..297) zeroed at fold boundary
        // alongside the host-side `training_sharpe_ema` / `_initialized` /
        // `max_dd_ema` / `low_dd_ratio` scalars (Pearl A sentinel path).
        // Producer-only in D3; consumer wires atomically in D4 per
        // `feedback_no_partial_refactor.md`.
        let training_metrics_ema_kernel = {
            let module = stream.context().load_cubin(SP5_TRAINING_METRICS_EMA_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp5 training_metrics_ema cubin load: {e}")))?;
            module.load_function("training_metrics_ema_update")
                .map_err(|e| MLError::ModelError(format!("training_metrics_ema_update load: {e}")))?
        };

        // SP7 Task 5 (2026-05-03): load loss_balance_controller_kernel.
        // 8-thread single-block kernel (2 loss heads × 4 branches). Reads three
        // 3-float views into grad_decomp_result_dev_ptr (IQN@0, CQL_SX@6, C51@9),
        // FLATNESS_BASE, prior budgets, and prior Wiener state from ISV; writes
        // 24 floats to scratch[SCRATCH_LB_BUDGET_CQL=218..242). Followed by 24
        // apply_pearls_ad_kernel launches → ISV[BUDGET_CQL_BASE, BUDGET_C51_BASE,
        // LB_*_VAR_{CQL,C51}_BASE]. Producer-only — call site lands at T7 per
        // feedback_no_partial_refactor.md.
        let loss_balance_controller_kernel = {
            let module = stream.context().load_cubin(LOSS_BALANCE_CONTROLLER_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("loss_balance_controller cubin load: {e}")))?;
            module.load_function("loss_balance_controller_update")
                .map_err(|e| MLError::ModelError(format!("loss_balance_controller_update load: {e}")))?
        };

        // SP8 (Fix 36, 2026-05-03): load train_active_frac_compute_kernel.
        // Single-block, single-thread producer that reduces
        // `monitoring_summary[5..17)` (the 12-bin action_counts from
        // monitoring_reduce) into a single fraction at scratch slot
        // SCRATCH_TRAIN_ACTIVE_FRAC. Pearls A+D smooth into
        // ISV[TRAIN_ACTIVE_FRAC_INDEX=321]. Replaces the host-side compute
        // at `training_loop.rs:3392-3396` per
        // `feedback_no_cpu_compute_strict.md`.
        let train_active_frac_compute_kernel = {
            let module = stream.context()
                .load_cubin(TRAIN_ACTIVE_FRAC_COMPUTE_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("train_active_frac_compute cubin load: {e}")))?;
            module.load_function("train_active_frac_compute")
                .map_err(|e| MLError::ModelError(format!("train_active_frac_compute load: {e}")))?
        };

        // SP8 (Fix 36, 2026-05-03): load loss_balance_max_budget_compute_kernel.
        // Single-block, 8-thread producer (2 heads × 4 branches) reads
        // ISV[TRAIN_ACTIVE_FRAC_INDEX] and computes per-(head, branch) cap
        // via linear interpolation FLOOR + active_frac × (CEIL − FLOOR);
        // 8 floats land in scratch[SCRATCH_LB_MAX_BUDGET_*]. Pearls A+D
        // smooth into ISV[LB_MAX_BUDGET_{CQL,C51}_BASE..+4). Replaces the
        // hardcoded `MAX_BUDGET = 1.0f` constant in
        // `loss_balance_controller_kernel.cu` per
        // `pearl_controller_anchors_isv_driven.md` and
        // `feedback_isv_for_adaptive_bounds.md`.
        let loss_balance_max_budget_compute_kernel = {
            let module = stream.context()
                .load_cubin(LOSS_BALANCE_MAX_BUDGET_COMPUTE_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("loss_balance_max_budget_compute cubin load: {e}")))?;
            module.load_function("loss_balance_max_budget_compute")
                .map_err(|e| MLError::ModelError(format!("loss_balance_max_budget_compute load: {e}")))?
        };

        // SP9 (Fix 37, 2026-05-03): load 7 SP9 producer kernels for the
        // Kelly cold-start warmup floor + EMA targets + intent/eval
        // divergence + eval_dist GPU migration. Each is a single-block
        // cold-path producer chained through apply_pearls_ad_kernel for
        // Pearl A sentinel-bootstrap + Pearl D Wiener-α steady-state
        // smoothing per `pearl_cold_start_exit_signal_or.md` and
        // `pearl_controller_anchors_isv_driven.md`.
        let sp9_eval_intent_dist_compute_kernel = {
            let module = stream.context()
                .load_cubin(SP9_EVAL_INTENT_DIST_COMPUTE_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp9 eval_intent_dist_compute cubin load: {e}")))?;
            module.load_function("eval_intent_dist_compute")
                .map_err(|e| MLError::ModelError(format!("sp9 eval_intent_dist_compute load: {e}")))?
        };
        let sp9_intent_eval_divergence_compute_kernel = {
            let module = stream.context()
                .load_cubin(SP9_INTENT_EVAL_DIVERGENCE_COMPUTE_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp9 intent_eval_divergence_compute cubin load: {e}")))?;
            module.load_function("intent_eval_divergence_compute")
                .map_err(|e| MLError::ModelError(format!("sp9 intent_eval_divergence_compute load: {e}")))?
        };
        let sp9_q_var_mag_ema_compute_kernel = {
            let module = stream.context()
                .load_cubin(SP9_Q_VAR_MAG_EMA_COMPUTE_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp9 q_var_mag_ema_compute cubin load: {e}")))?;
            module.load_function("q_var_mag_ema_compute")
                .map_err(|e| MLError::ModelError(format!("sp9 q_var_mag_ema_compute load: {e}")))?
        };
        let sp9_kelly_warmup_floor_compute_kernel = {
            let module = stream.context()
                .load_cubin(SP9_KELLY_WARMUP_FLOOR_COMPUTE_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp9 kelly_warmup_floor_compute cubin load: {e}")))?;
            module.load_function("kelly_warmup_floor_compute")
                .map_err(|e| MLError::ModelError(format!("sp9 kelly_warmup_floor_compute load: {e}")))?
        };

        // SP11 Fix 39 (2026-05-04, Task A1): load three canary producer
        // kernels for the reward-as-controlled-subsystem chain. Each is a
        // single-block producer chained through `apply_pearls_ad_kernel`
        // for Pearl A sentinel-bootstrap + Pearl D Wiener-α smoothing.
        // No consumer reads any of the canary slots [350..360) yet
        // (Layer A is additive); Layer B atomically migrates consumers.
        let sp11_val_sharpe_delta_compute_kernel = {
            let module = stream.context()
                .load_cubin(SP11_VAL_SHARPE_DELTA_COMPUTE_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp11 val_sharpe_delta_compute cubin load: {e}")))?;
            module.load_function("val_sharpe_delta_compute_kernel")
                .map_err(|e| MLError::ModelError(format!("sp11 val_sharpe_delta_compute load: {e}")))?
        };
        let sp11_saboteur_engagement_compute_kernel = {
            let module = stream.context()
                .load_cubin(SP11_SABOTEUR_ENGAGEMENT_COMPUTE_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp11 saboteur_engagement_compute cubin load: {e}")))?;
            module.load_function("saboteur_engagement_compute_kernel")
                .map_err(|e| MLError::ModelError(format!("sp11 saboteur_engagement_compute load: {e}")))?
        };
        let sp11_mag_ratio_compute_kernel = {
            let module = stream.context()
                .load_cubin(SP11_MAG_RATIO_COMPUTE_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp11 mag_ratio_compute cubin load: {e}")))?;
            module.load_function("reward_component_mag_ratio_compute_kernel")
                .map_err(|e| MLError::ModelError(format!("sp11 mag_ratio_compute load: {e}")))?
        };
        // SP11 Fix 39 B1b fix-up (2026-05-04): popart-component-specific
        // magnitude EMA producer kernel. Loaded alongside the other A1
        // canary producers — Pearls A+D output target is ISV[POPART_
        // COMPONENT_MAG_EMA_INDEX=360]. Resolves the slot-63 PopArt-input
        // vs popart-component-magnitude overload that B1b smoke surfaced.
        // Spec §4 amendment "Why slot 360".
        let sp11_popart_component_ema_kernel = {
            let module = stream.context()
                .load_cubin(SP11_POPART_COMPONENT_EMA_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp11 popart_component_ema cubin load: {e}")))?;
            module.load_function("popart_component_ema_kernel")
                .map_err(|e| MLError::ModelError(format!("sp11 popart_component_ema load: {e}")))?
        };
        // SP13 Phase 0a (2026-05-04): aux-head directional-accuracy
        // reducer kernel + 3-element mapped-pinned readback buffer.
        // Producer reads the aux next-bar regression-head prediction
        // tile + the per-bar sign-encoded next_bar_label tile and
        // writes (dir_acc, pos_pred_frac, pos_label_frac) to the
        // 3-float mapped-pinned buffer. P0a.T4 wires the per-step
        // launch + ISV[373..375) EMA producer; this constructor edge
        // is the cubin/kernel handle + buffer alloc — both atomic with
        // the rest of the P0a chain per `feedback_no_partial_refactor.md`.
        let aux_dir_acc_reduce = {
            let module = stream.context()
                .load_cubin(SP13_AUX_DIR_ACC_REDUCE_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp13 aux_dir_acc_reduce cubin load: {e}")))?;
            module.load_function("aux_dir_acc_reduce_kernel")
                .map_err(|e| MLError::ModelError(format!("sp13 aux_dir_acc_reduce load: {e}")))?
        };
        // SP13 B1.1a (2026-05-05): output grew 3 → 6 (added n_down/n_up/
        // n_skip for HEALTH_DIAG observability of the B1.1a producer-less
        // label distribution). Layout:
        //   [0] dir_acc, [1] pos_pred_frac, [2] pos_label_frac,
        //   [3] n_down,  [4] n_up,           [5] n_skip
        let aux_dir_acc_buf = unsafe {
            super::mapped_pinned::MappedF32Buffer::new(6)
        }.map_err(|e| MLError::ModelError(
            format!("SP13 aux_dir_acc_buf alloc (6 f32): {e}")
        ))?;
        // SP13 v3 P0a.T3 (2026-05-04): the per-step Hold-rate observer
        // chain lives on the `GpuExperienceCollector` stream — the
        // observer reads `batch_actions [N]` (collector-owned) and the
        // chained fixed-α EMA writes ISV[382]. The collector loads its
        // own copy of the cubins from `SP13_HOLD_RATE_OBSERVER_CUBIN` and
        // `SP13_APPLY_FIXED_ALPHA_EMA_CUBIN`, so the trainer carries no
        // hold-rate-specific kernel handle or scratch buffer (avoids a
        // dead `CudaFunction` field per `feedback_no_stubs.md`). Slot 382
        // feeds the SP16-P1 MIN_HOLD_TEMPERATURE producer at slot 460.
        // ISV[380] / ISV[381] sentinels are seeded by `seed_static_signals`
        // (Invariant-1 anchors); ISV[382] is FoldReset 0.0 via
        // `state_reset_registry`.

        // SP13 Phase 0a P0a.T4 (2026-05-04): fixed-α EMA applicator
        // kernel handle + aux-pred → ISV[375] tanh producer handle.
        // Both load from their cubins and stay live for the trainer's
        // lifetime. P0a.T4 wires the per-step launches in
        // `launch_sp13_aux_dir_metrics` and the hold-rate observer
        // chain in `gpu_experience_collector.rs`; this constructor edge
        // is the cubin/kernel handle, all-atomic with the rest of the
        // P0a chain per `feedback_no_partial_refactor.md`.
        let apply_fixed_alpha_ema_kernel = {
            let module = stream.context()
                .load_cubin(SP13_APPLY_FIXED_ALPHA_EMA_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp13 apply_fixed_alpha_ema cubin load: {e}")))?;
            module.load_function("apply_fixed_alpha_ema_kernel")
                .map_err(|e| MLError::ModelError(format!("sp13 apply_fixed_alpha_ema_kernel load: {e}")))?
        };
        let aux_pred_to_isv_tanh_kernel = {
            let module = stream.context()
                .load_cubin(SP13_AUX_PRED_TO_ISV_TANH_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp13 aux_pred_to_isv_tanh cubin load: {e}")))?;
            module.load_function("aux_pred_to_isv_tanh_kernel")
                .map_err(|e| MLError::ModelError(format!("sp13 aux_pred_to_isv_tanh_kernel load: {e}")))?
        };

        // SP22 H6 Phase 3 α (2026-05-13): post-encoder Q_dir bias.
        // Allocate the 4-element weight + 4-element Adam moments +
        // 4-element gradient accumulator. Load forward + backward
        // kernels. All buffers zero-init per `alloc_zeros`; Adam
        // moves W from cold-start zero based on the backward signal.
        let b0_size_usize: usize = 4;  // factored space b0_size = 4 (direction)
        let w_aux_to_q_dir = stream
            .alloc_zeros::<f32>(b0_size_usize)
            .map_err(|e| MLError::ModelError(format!(
                "sp22-h6-phase3 α: alloc w_aux_to_q_dir: {e}"
            )))?;
        let adam_m_w_aux = stream
            .alloc_zeros::<f32>(b0_size_usize)
            .map_err(|e| MLError::ModelError(format!(
                "sp22-h6-phase3 α: alloc adam_m_w_aux: {e}"
            )))?;
        let adam_v_w_aux = stream
            .alloc_zeros::<f32>(b0_size_usize)
            .map_err(|e| MLError::ModelError(format!(
                "sp22-h6-phase3 α: alloc adam_v_w_aux: {e}"
            )))?;
        let dw_aux_buf = stream
            .alloc_zeros::<f32>(b0_size_usize)
            .map_err(|e| MLError::ModelError(format!(
                "sp22-h6-phase3 α: alloc dw_aux_buf: {e}"
            )))?;
        // SP22 H6 Phase 3 α Step 8 (2026-05-13): per-sample scratch buffers
        // for the W gradient backward. aux_target_a_dir saves best_next_a
        // for d==0 from the forward; aux_proj_logdiff_dir saves the
        // per-sample SP scalar.
        let aux_target_a_dir_buf = stream
            .alloc_zeros::<i32>(config.batch_size)
            .map_err(|e| MLError::ModelError(format!(
                "sp22-h6-phase3 α Step 8: alloc aux_target_a_dir_buf: {e}"
            )))?;
        let aux_proj_logdiff_dir_buf = stream
            .alloc_zeros::<f32>(config.batch_size)
            .map_err(|e| MLError::ModelError(format!(
                "sp22-h6-phase3 α Step 8: alloc aux_proj_logdiff_dir_buf: {e}"
            )))?;
        // Load Step 8 + Step 11 kernels.
        let c51_aux_dw_kernel = {
            let module = stream.context()
                .load_cubin(SP22_C51_AUX_DW_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!(
                    "sp22-h6-phase3 α Step 8: c51_aux_dw cubin: {e}"
                )))?;
            module.load_function("c51_aux_dw_kernel")
                .map_err(|e| MLError::ModelError(format!(
                    "sp22-h6-phase3 α Step 8: c51_aux_dw_kernel load: {e}"
                )))?
        };
        let adam_w_aux_kernel = {
            let module = stream.context()
                .load_cubin(SP22_ADAM_W_AUX_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!(
                    "sp22-h6-phase3 α Step 11: adam_w_aux cubin: {e}"
                )))?;
            module.load_function("adam_w_aux_update")
                .map_err(|e| MLError::ModelError(format!(
                    "sp22-h6-phase3 α Step 11: adam_w_aux_update load: {e}"
                )))?
        };

        // SP22 H6 Phase 3 α (atom-shift design 2026-05-13): structural-prior
        // init for w_aux_to_q_dir. Writes per-action prior
        // `[-0.5, 0.0, +0.5, 0.0]` for `[Short, Hold, Long, Flat]`. The
        // Adam-trained W refines from this prior via c51_grad_kernel's
        // projection backward (Phase 3-final B9 Step 8). One-time
        // GPU init — NOT inside any captured graph.
        {
            let init_module = stream.context()
                .load_cubin(SP22_AUX_W_PRIOR_INIT_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!(
                    "sp22-h6-phase3 α: aux_w_prior_init cubin: {e}"
                )))?;
            let init_kernel = init_module
                .load_function("aux_w_prior_init")
                .map_err(|e| MLError::ModelError(format!(
                    "sp22-h6-phase3 α: aux_w_prior_init load: {e}"
                )))?;
            let w_ptr = w_aux_to_q_dir.raw_ptr();
            unsafe {
                stream
                    .launch_builder(&init_kernel)
                    .arg(&w_ptr)
                    .launch(LaunchConfig {
                        grid_dim: (1, 1, 1),
                        block_dim: (4, 1, 1),
                        shared_mem_bytes: 0,
                    })
                    .map_err(|e| MLError::ModelError(format!(
                        "sp22-h6-phase3 α: aux_w_prior_init launch: {e}"
                    )))?;
            }
        }

        // SP22 H6 Phase 3 α scalar-bias kernel loads — DELETED 2026-05-13.
        // Per the architectural revision (audit doc "Phase 3c — α
        // architectural finding"), the scalar-bias α design is replaced
        // by atom-position shift threading through compute_expected_q +
        // c51_loss_kernel + c51_grad_kernel + mag_concat_qdir. No
        // separate α forward / backward kernel is launched — the
        // gradient flows via c51_grad_kernel's existing projection
        // backward. The kernel handles (aux_to_q_dir_bias_kernel,
        // aux_to_q_dir_bias_backward_dw_kernel,
        // aux_to_q_dir_bias_backward_dstate_kernel) and the CUBIN
        // statics (SP22_AUX_TO_Q_DIR_BIAS_CUBIN,
        // SP22_AUX_TO_Q_DIR_BIAS_BWD_CUBIN) are deleted in this same
        // commit. The W + Adam moments + dW buffer survive — they
        // still hold the Adam-trained W vector used by atom-shift
        // threading. Phase 3-final implementation writes the
        // structural prior init `[-0.5, 0.0, +0.5, 0.0]` into
        // `w_aux_to_q_dir` after `alloc_zeros` above (via a small
        // fill-kernel launch or repurposed fill_f32 sequence).

        // SP14 EGF kernel loads (B.3–B.6).
        // Handles are held in struct fields but not launched until B.9/B.11;
        // per `feedback_no_partial_refactor.md` all 4 kernel handles +
        // scratch buffer land atomically in this single commit.
        let sp14_q_dis_module = stream.context()
            .load_cubin(SP14_Q_DISAGREEMENT_CUBIN.to_vec())
            .map_err(|e| MLError::ModelError(format!("sp14 q_dis cubin: {e}")))?;
        let sp14_q_disagreement_update_kernel = sp14_q_dis_module
            .load_function("q_disagreement_update_kernel")
            .map_err(|e| MLError::ModelError(format!("q_disagreement_update_kernel: {e}")))?;

        // SP17 Task PP.3 (2026-05-08): load V/A means diagnostic kernel +
        // allocate the 5-float mapped-pinned readback. Cold-path producer
        // launched at HEALTH_DIAG emit only — no graph capture, no per-step
        // cost. Per `feedback_no_htod_htoh_only_mapped_pinned`: readback uses
        // `MappedF32Buffer`; the kernel writes via `dev_ptr` with
        // `__threadfence_system()` and the host reads via `read_all()` after
        // a stream sync.
        let sp17_v_a_means_diag_module = stream.context()
            .load_cubin(SP17_V_A_MEANS_DIAG_CUBIN.to_vec())
            .map_err(|e| MLError::ModelError(format!("sp17 v_a_means_diag cubin: {e}")))?;
        let sp17_v_a_means_diag_kernel = sp17_v_a_means_diag_module
            .load_function("v_a_means_diag_kernel")
            .map_err(|e| MLError::ModelError(format!("v_a_means_diag_kernel: {e}")))?;
        let sp17_v_a_means_diag_buf = unsafe {
            super::mapped_pinned::MappedF32Buffer::new(5)
        }
        .map_err(|e| MLError::ModelError(format!("sp17_v_a_means_diag_buf alloc (5 f32): {e}")))?;

        // SP17 Phase 3.1 (2026-05-08): per-branch A_var EMA producer.
        // Cold-path (epoch boundary) — one launch per HEALTH_DIAG emit.
        // 4 blocks × 256 threads; block tree-reduce per
        // `feedback_no_atomicadd`. Pearl-A bootstrap on slots 474..478.
        let sp17_a_var_ema_module = stream.context()
            .load_cubin(SP17_A_VAR_EMA_CUBIN.to_vec())
            .map_err(|e| MLError::ModelError(format!("sp17 a_var_ema cubin: {e}")))?;
        let sp17_a_var_ema_kernel = sp17_a_var_ema_module
            .load_function("sp17_a_var_ema_update")
            .map_err(|e| MLError::ModelError(format!("sp17_a_var_ema_update: {e}")))?;

        // SP17 Phase 3.2 (2026-05-08): per-branch V_share producer.
        // Cold-path (epoch boundary). 4 blocks × 256 threads; Pearl-A
        // bootstrap on slots 478..482 (sentinel 0.5 cold-start). Bilateral
        // [0, 1] clamp before ISV write per `pearl_symmetric_clamp_audit`.
        let sp17_v_share_module = stream.context()
            .load_cubin(SP17_V_SHARE_CUBIN.to_vec())
            .map_err(|e| MLError::ModelError(format!("sp17 v_share cubin: {e}")))?;
        let sp17_v_share_kernel = sp17_v_share_module
            .load_function("sp17_v_share_update")
            .map_err(|e| MLError::ModelError(format!("sp17_v_share_update: {e}")))?;

        // SP17 Phase 3.2 (2026-05-08): adaptive advantage_clip_bound
        // producer + flat |A_centered| scratch buffer. Single block × 256
        // threads. Pearl-A bootstrap on slot 482 (sentinel 1.0). Uses
        // `sp4_histogram_p99` (block tree-reduce + per-warp tile binning,
        // no atomicAdd per `pearl_fused_per_group_statistics_oracle`).
        // Scratch sized to max possible flat |A_centered| =
        //   B × (b0 + b1 + b2 + b3) × NA  floats.
        let sp17_advantage_clip_bound_module = stream.context()
            .load_cubin(SP17_ADVANTAGE_CLIP_BOUND_CUBIN.to_vec())
            .map_err(|e| MLError::ModelError(format!("sp17 advantage_clip_bound cubin: {e}")))?;
        let sp17_advantage_clip_bound_kernel = sp17_advantage_clip_bound_module
            .load_function("sp17_advantage_clip_bound_update")
            .map_err(|e| MLError::ModelError(format!("sp17_advantage_clip_bound_update: {e}")))?;

        // SP18 v2 Phase 0 Task 0.2 (2026-05-08): B-leg TD-error
        // magnitude EMA producer load. Single block × 256 threads.
        // Pearl-A bootstrap on slot 493 (sentinel 0.0) + fixed
        // α=WELFORD_ALPHA_MIN=0.4. Reads `td_errors_buf [B]` post-
        // train-step. Pure observability.
        let sp18_td_error_mag_ema_module = stream.context()
            .load_cubin(TD_ERROR_MAG_EMA_CUBIN.to_vec())
            .map_err(|e| MLError::ModelError(format!(
                "sp18 td_error_mag_ema cubin: {e}")))?;
        let sp18_td_error_mag_ema_kernel = sp18_td_error_mag_ema_module
            .load_function("td_error_mag_ema_update")
            .map_err(|e| MLError::ModelError(format!(
                "td_error_mag_ema_update load: {e}")))?;
        let sp17_clip_scratch_capacity = config.batch_size
            .saturating_mul(
                config.branch_0_size + config.branch_1_size
                    + config.branch_2_size + config.branch_3_size
            )
            .saturating_mul(config.num_atoms);
        // Defensive minimum of 1 element so MappedF32Buffer::new() doesn't
        // fail on a zero-sized config (constructor invariants normally
        // prevent this; the saturating_mul above could in principle floor
        // to 0 in degenerate test configs).
        let sp17_clip_scratch_len = sp17_clip_scratch_capacity.max(1);
        let sp17_advantage_clip_scratch = unsafe {
            super::mapped_pinned::MappedF32Buffer::new(sp17_clip_scratch_len)
        }
        .map_err(|e| MLError::ModelError(format!(
            "sp17_advantage_clip_scratch alloc ({sp17_clip_scratch_len} f32): {e}"
        )))?;

        // SP18 v2 Phase 0 Task 0.1 (2026-05-08): D-leg per-action reward
        // decomposition diagnostic buffer. 4 bins × 5 stats = 20 floats
        // mapped-pinned. The collector's
        // `launch_sp18_reward_decomp_diag` writes per-step; the host-side
        // HEALTH_DIAG emit reads via the mapped host_ptr after stream
        // sync. Pure observability — no production-path consumer.
        const SP18_REWARD_DECOMP_DIAG_LEN: usize = 4 * 5;
        let sp18_reward_decomp_diag_buf = unsafe {
            super::mapped_pinned::MappedF32Buffer::new(SP18_REWARD_DECOMP_DIAG_LEN)
        }
        .map_err(|e| MLError::ModelError(format!(
            "sp18_reward_decomp_diag_buf alloc ({SP18_REWARD_DECOMP_DIAG_LEN} f32): {e}"
        )))?;
        // Initialise to 0.0 so the first HEALTH_DIAG read on cold-start
        // (before the collector has fired) is a deterministic zero block
        // rather than indeterminate page contents. Pearl-A first-
        // observation bootstrap fires inside the kernel itself when
        // n_samples > 0; this just zero-fills before the kernel ever runs.
        sp18_reward_decomp_diag_buf
            .write_from_slice(&vec![0.0_f32; SP18_REWARD_DECOMP_DIAG_LEN]);

        // SP14 Layer C Phase C.1 (2026-05-08): α-machinery cubin loads
        // (alpha_grad_compute, gradient_hack_detect, sp14_scale_wire_col)
        // deleted atomically with the kernel files. Coupling A
        // (`dir_concat_qaux`) load preserved below.
        let sp14_dir_concat_module = stream.context()
            .load_cubin(SP14_DIR_CONCAT_CUBIN.to_vec())
            .map_err(|e| MLError::ModelError(format!("sp14 dir_concat cubin: {e}")))?;
        let sp14_dir_concat_qaux_kernel = sp14_dir_concat_module
            .load_function("dir_concat_qaux_kernel")
            .map_err(|e| MLError::ModelError(format!("dir_concat_qaux_kernel: {e}")))?;

        // SP14 B.7: scratch buffer for direction Q-head input concat.
        // Shape [B, SH2 + 1]: one extra column per sample for the appended
        // aux_softmax_diff scalar (Coupling A forward feature wire).
        // `b` = config.batch_size (declared at the top of `new`);
        // `config.shared_h2` is the GRN trunk output dim. Phase C.1
        // removed the α-machinery scaling; the wire column flows through
        // unscaled and is dropped at the first-SH2-cols accumulator.
        let sp14_dir_qaux_concat_scratch = stream
            .alloc_zeros::<f32>(b * (config.shared_h2 + 1))
            .map_err(|e| MLError::ModelError(format!("sp14 dir_qaux scratch: {e}")))?;

        // SP14 B.10: backward dX scratch for the direction Q-head's first
        // FC. Shape `[B, SH2 + 1]`. Receives the SGEMM `dY @ W^T` output;
        // the first SH2 columns accumulate into `bw_d_h_s2` and column SH2
        // is dropped (no propagation downstream — the wire column gradient
        // does not back-flow to aux). Symmetric with the existing
        // `d_mag_concat_buf` (magnitude branch's `[B, SH2 + b0]` backward
        // scratch). Phase C.1 deleted the α-gate that previously scaled
        // column SH2 before the accumulator.
        let sp14_d_dir_qaux_concat = stream
            .alloc_zeros::<f32>(b * (config.shared_h2 + 1))
            .map_err(|e| MLError::ModelError(format!("sp14 d_dir_qaux scratch: {e}")))?;

        // SP11 Fix 39 B1b fix-up (2026-05-04): allocate trainer-side
        // popart-component-per-sample mapped-pinned placeholder buffer.
        // The canonical per-bar buffer is owned by the experience
        // collector (alloc_episodes × alloc_timesteps); the trainer
        // launcher reads via raw device pointer wired in by
        // `set_sp11_popart_component_buf` after collector construction.
        // The trainer-side field exists to satisfy the buffer-lifetime
        // contract on the trainer struct — it's a 1-element placeholder
        // that's never directly read. Initial value zero (constructor);
        // mapped-pinned per `feedback_no_htod_htoh_only_mapped_pinned`
        // (the rule applies even to placeholder allocations).
        let popart_component_per_sample = unsafe {
            super::mapped_pinned::MappedF32Buffer::new(1)
        }.map_err(|e| MLError::ModelError(
            format!("SP11 popart_component_per_sample alloc: {e}")
        ))?;

        // SP11 Fix 39 (2026-05-04, A1.1): allocate the val-sharpe history
        // mapped-pinned buffer. 2-element [prev_epoch, curr_epoch] — the
        // host writes the current val sharpe at every val emit boundary
        // in `training_loop.rs`, then rotates slot[1] into slot[0] before
        // the next observation. Constructor-zero-initialised, which Pearl
        // A's first-observation replacement consumes naturally.
        let val_sharpe_history_pinned = unsafe {
            super::mapped_pinned::MappedF32Buffer::new(2)
        }.map_err(|e| MLError::ModelError(
            format!("SP11 val_sharpe_history_pinned alloc: {e}")
        ))?;

        // SP11 Fix 39 (2026-05-04, Task A2): load reward-subsystem
        // controller + SimHash novelty kernels.
        //
        // Controller is the main A2 producer: 5 canaries → 10 outputs in
        // scratch[SCRATCH_SP11_CONTROLLER_BASE..+10) → Pearls A+D output
        // smoothing → ISV[340..350). SimHash kernels (proj-init, lookup,
        // update) build the novelty buffer infrastructure that B1 wires
        // into the replay path.
        let sp11_reward_subsystem_controller_kernel = {
            let module = stream.context()
                .load_cubin(SP11_REWARD_SUBSYSTEM_CONTROLLER_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp11 reward_subsystem_controller cubin load: {e}")))?;
            module.load_function("reward_subsystem_controller_kernel")
                .map_err(|e| MLError::ModelError(format!("sp11 reward_subsystem_controller load: {e}")))?
        };
        let sp11_novelty_simhash_proj_init_kernel = {
            let module = stream.context()
                .load_cubin(SP11_NOVELTY_SIMHASH_PROJ_INIT_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("sp11 novelty_simhash_proj_init cubin load: {e}")))?;
            module.load_function("novelty_simhash_proj_init_kernel")
                .map_err(|e| MLError::ModelError(format!("sp11 novelty_simhash_proj_init load: {e}")))?
        };
        // The lookup + update kernels share one cubin (one CUmodule).
        let sp11_novelty_simhash_module = stream.context()
            .load_cubin(SP11_NOVELTY_SIMHASH_CUBIN.to_vec())
            .map_err(|e| MLError::ModelError(format!("sp11 novelty_simhash cubin load: {e}")))?;
        let sp11_novelty_simhash_lookup_kernel = sp11_novelty_simhash_module
            .load_function("novelty_simhash_lookup_kernel")
            .map_err(|e| MLError::ModelError(format!("sp11 novelty_simhash_lookup load: {e}")))?;
        let sp11_novelty_simhash_update_kernel = sp11_novelty_simhash_module
            .load_function("novelty_simhash_update_kernel")
            .map_err(|e| MLError::ModelError(format!("sp11 novelty_simhash_update load: {e}")))?;

        // SP11 Fix 39 (2026-05-04, A2): allocate the novelty visit-count
        // hash table (1M slots × f32 = 4 MB mapped-pinned). Constructor-
        // zero-initialised by `MappedF32Buffer::new` (cuMemHostAlloc is
        // followed by `write_bytes(0, len)` per the buffer's invariants
        // doc) which is the desired cold-start state for the novelty
        // signal: every (state, action) bucket reads count=0 →
        // novelty=1/sqrt(1)=1.0. The fold-reset arm `sp11_novelty_hash`
        // re-zeros the buffer at fold boundaries.
        let novelty_hash_buf = unsafe {
            super::mapped_pinned::MappedF32Buffer::new(1usize << 20)
        }.map_err(|e| MLError::ModelError(
            format!("SP11 novelty_hash_buf alloc: {e}")
        ))?;

        // SP11 Fix 39 (2026-05-04, A2): allocate the SimHash projection
        // matrix (42 × 16 = 672 f32 mapped-pinned) and immediately launch
        // the GPU init kernel to populate it with deterministic ±1 values
        // derived from `SP11_PROJ_SEED_SALT`. Per `feedback_no_cpu_forwards.md`
        // ("CPU is read-only") + `feedback_no_cpu_compute_strict.md`, the
        // projection MUST be populated by a GPU kernel. The buffer is
        // mapped-pinned so the kernel can write through `dev_ptr` without
        // a separate device alloc — the host never reads or writes the
        // matrix after this point.
        //
        // Per `feedback_wire_everything_up.md`: the init kernel is wired
        // to the constructor that allocates the buffer. No orphan
        // additions. The matrix is frozen for the run lifetime; no
        // fold-reset entry exists for it (it's the hash function, not
        // state).
        let novelty_simhash_proj = unsafe {
            super::mapped_pinned::MappedF32Buffer::new(42 * 16)
        }.map_err(|e| MLError::ModelError(
            format!("SP11 novelty_simhash_proj alloc: {e}")
        ))?;
        {
            use crate::cuda_pipeline::sp11_isv_slots::SP11_PROJ_SEED_SALT;
            // Salt-only seed: the project does not have a global config.seed;
            // the SP11 spec just requires a deterministic seed reproducible
            // across runs. The salt itself is a stable 64-bit constant in
            // `sp11_isv_slots.rs` so every run gets the same projection
            // matrix → reproducible novelty hashing across reruns.
            let seed_u64: u64 = SP11_PROJ_SEED_SALT;
            // Philox-uniform takes an i32 seed — XOR-fold the high + low halves
            // rather than truncating, to preserve entropy from both halves of the
            // u64. Truncation (`seed_u64 as u32 as i32`) would silently discard
            // the upper 32 bits — a future reader switching to that pattern would
            // not be warned that it degrades seed diversity.
            let seed_i32: i32 = ((seed_u64 ^ (seed_u64 >> 32)) as u32) as i32;
            let proj_dev = novelty_simhash_proj.dev_ptr;
            let n_floats: i32 = (42 * 16) as i32;
            let block_dim: u32 = 32;
            let n_blocks: u32 = (672 + block_dim - 1) / block_dim; // 21 blocks
            unsafe {
                stream
                    .launch_builder(&sp11_novelty_simhash_proj_init_kernel)
                    .arg(&proj_dev)
                    .arg(&seed_i32)
                    .arg(&n_floats)
                    .launch(LaunchConfig {
                        grid_dim:         (n_blocks, 1, 1),
                        block_dim:        (block_dim, 1, 1),
                        shared_mem_bytes: 0,
                    })
                    .map_err(|e| MLError::ModelError(
                        format!("sp11 novelty_simhash_proj_init launch: {e}")
                    ))?;
            }
            // Sync so the matrix is fully populated before any subsequent
            // construct-time work (or the first lookup launch in B1) reads
            // it. One-shot init at construct → no hot-path penalty.
            stream.synchronize()
                .map_err(|e| MLError::ModelError(
                    format!("sp11 novelty_simhash_proj_init sync: {e}")
                ))?;
        }

        // SP7 (2026-05-03): load consume_lb_budget_kernel.cubin — three kernels
        // (lb_budget_dispatch + dqn_scale_f32_dev_ptr_kernel +
        // dqn_saxpy_f32_dev_ptr_kernel) sharing one CUmodule. Separate from
        // `aux_module` (which holds the scalar saxpy_f32/scale_f32 reused by
        // distill, VSN dW dilution, etc.) so existing scalar callers remain
        // unchanged. The dispatch kernel runs each step inside the captured
        // `aux_child` graph; the dev-ptr SAXPY/scale kernels are launched by
        // `apply_c51_budget_scale` / `apply_cql_saxpy` (and per-branch variants)
        // immediately after, reading `alpha` from `lb_budget_effective_buf`.
        let consume_lb_budget_module = stream.context().load_cubin(CONSUME_LB_BUDGET_CUBIN.to_vec())
            .map_err(|e| MLError::ModelError(format!("consume_lb_budget cubin load: {e}")))?;
        let lb_budget_dispatch_kernel = consume_lb_budget_module
            .load_function("lb_budget_dispatch")
            .map_err(|e| MLError::ModelError(format!("lb_budget_dispatch load: {e}")))?;
        let scale_f32_dev_ptr_aux = consume_lb_budget_module
            .load_function("dqn_scale_f32_dev_ptr_kernel")
            .map_err(|e| MLError::ModelError(format!("dqn_scale_f32_dev_ptr_kernel load: {e}")))?;
        let saxpy_f32_dev_ptr_aux = consume_lb_budget_module
            .load_function("dqn_saxpy_f32_dev_ptr_kernel")
            .map_err(|e| MLError::ModelError(format!("dqn_saxpy_f32_dev_ptr_kernel load: {e}")))?;
        // SP7 (2026-05-03): allocate the 10 f32 mapped-pinned scratch holding
        // GPU-resolved [cql_trunk, cql_corr×4, c51_trunk, c51_corr×4]. Zero-init
        // is fine — first dispatch-kernel launch overwrites the slots before
        // any consumer reads them. See `consume_lb_budget_kernel.cu` top-of-file
        // block for layout.
        let lb_budget_effective_buf = unsafe { MappedF32Buffer::new(10) }
            .map_err(|e| MLError::ModelError(format!("SP7 lb_budget_effective_buf alloc: {e}")))?;

        // SP5 Task A4 (2026-05-01): allocate grad_prev_buf_per_group [total_params f32].
        // Zeroed at construction; fold-boundary reset zeroes it again (Pearl A sentinel:
        // zero grad_prev → cosine_sim=0 on first step → β1/β2 start at envelope midpoints).
        // `total_params` was computed at constructor entry (line ~12337).
        let grad_prev_buf_per_group = unsafe { MappedF32Buffer::new(total_params) }
            .map_err(|e| MLError::ModelError(format!("SP5 grad_prev_buf_per_group alloc: {e}")))?;

        // SP5 Task A4 (2026-05-01): allocate group_param_offsets_buf [9 i32].
        // Prefix sums in f32-element units (not bytes): [offsets[0], ..., offsets[8]]
        // where group g covers [offsets[g], offsets[g+1]).
        // Groups match SP4_PARAM_GROUP_COUNT=8:
        //   0=DqnTrunk [0..trunk_param_count)
        //   1=DqnValue [trunk_param_count..trunk+value_count)
        //   2=DqnBranches [trunk+value..trunk+value+branch_count)
        //   3=Iqn, 4=IqlHigh, 5=IqlLow, 6=Attn, 7=Curiosity
        // For aux groups 3-7: they are separate grad buffers (not slices of grad_buf)
        // so their offsets are set to [total_params, total_params] — zero-width range
        // signals the kernel to skip those groups (thread loop produces dot=0,
        // norm=0, cosine_sim=0 → β1/β2 snap to stable envelope midpoints).
        // Only DqnTrunk/Value/Branches (groups 0-2) cover the contiguous main grad_buf.
        let param_sizes = compute_param_sizes(&config);
        let trunk_end   = (padded_byte_offset(&param_sizes, 13) / 4) as i32;
        let value_end   = (padded_byte_offset(&param_sizes, 17) / 4) as i32;
        let branch_end  = (padded_byte_offset(&param_sizes, 33) / 4) as i32;
        // Aux groups 3-7 sit outside the contiguous grad_buf slab → zero-width sentinel.
        let tp = total_params as i32;
        let group_param_offsets_buf = unsafe { MappedI32Buffer::new(9) }
            .map_err(|e| MLError::ModelError(format!("SP5 group_param_offsets_buf alloc: {e}")))?;
        group_param_offsets_buf.write_from_slice(&[
            0,          // group 0 DqnTrunk start
            trunk_end,  // group 1 DqnValue start / DqnTrunk end
            value_end,  // group 2 DqnBranches start / DqnValue end
            branch_end, // group 3 Iqn start / DqnBranches end
            tp,         // group 4 IqlHigh start — zero-width sentinel for aux groups
            tp,         // group 5 IqlLow start
            tp,         // group 6 Attn start
            tp,         // group 7 Curiosity start
            tp,         // sentinel: offsets[8] = end of group 7
        ]);

        // Plan C Phase 2 follow-up A.2: load q_drift_rate_ema kernel
        // (cold-path, per-epoch). Single-thread single-block ISV producer
        // for ISV[Q_DRIFT_RATE_INDEX=129]; consumer is `tau_update_kernel`.
        let q_drift_rate_ema_kernel = {
            let module = stream.context().load_cubin(Q_DRIFT_RATE_EMA_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("q_drift_rate_ema cubin load: {e}")))?;
            module.load_function("q_drift_rate_ema_update")
                .map_err(|e| MLError::ModelError(format!("q_drift_rate_ema_update load: {e}")))?
        };

        // Plan C Phase 2 follow-up K: load fold_warmup_factor kernel
        // (cold-path, per-step). Single-thread single-block ISV producer
        // for ISV[FOLD_WARMUP_FACTOR_INDEX=130]; consumers are the
        // `lr_eff` and `clip_eff` derivations in `training_loop.rs`.
        let fold_warmup_factor_kernel = {
            let module = stream.context().load_cubin(FOLD_WARMUP_FACTOR_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("fold_warmup_factor cubin load: {e}")))?;
            module.load_function("fold_warmup_factor_update")
                .map_err(|e| MLError::ModelError(format!("fold_warmup_factor_update load: {e}")))?
        };

        // SP4 Layer C close-out C1 redesigned (2026-05-01): load
        // update_grad_norm_emas kernel (cold-path, per-step). Single-thread
        // single-block — replaces the host-side EMA arithmetic block in
        // `update_adaptive_clip` per `feedback_no_cpu_compute_strict`. Reads
        // `grad_norm_buf[0]` and writes the fast/slow grad-norm EMA mapped-
        // pinned scalars consumed by `fold_warmup_factor_kernel`.
        let update_grad_norm_emas_kernel = {
            let module = stream.context().load_cubin(UPDATE_GRAD_NORM_EMAS_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("update_grad_norm_emas cubin load: {e}")))?;
            module.load_function("update_grad_norm_emas_kernel")
                .map_err(|e| MLError::ModelError(format!("update_grad_norm_emas_kernel load: {e}")))?
        };

        // SP4 Layer C close-out C2 (2026-05-01): load update_iqn_readiness
        // kernel. Single-thread, single-block — replaces the host-side EMA
        // arithmetic block in `update_iqn_readiness` per
        // `feedback_no_cpu_compute_strict`. Updates the three coupled
        // mapped-pinned scalars (`iqn_loss_initial_pinned`,
        // `iqn_loss_ema_pinned`, `iqn_readiness_pinned`) in-place.
        let update_iqn_readiness_kernel = {
            let module = stream.context().load_cubin(UPDATE_IQN_READINESS_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("update_iqn_readiness cubin load: {e}")))?;
            module.load_function("update_iqn_readiness_kernel")
                .map_err(|e| MLError::ModelError(format!("update_iqn_readiness_kernel load: {e}")))?
        };

        // SP4 Layer C close-out C3 (2026-05-01): load update_utilization_ema
        // kernel. Single-thread, single-block — replaces the host-side EMA
        // arithmetic block in `reduce_current_q_stats` per
        // `feedback_no_cpu_compute_strict`. Updates `utilization_ema_pinned`
        // + `homeostatic_obs_pinned[1]` mirror in lockstep.
        let update_utilization_ema_kernel = {
            let module = stream.context().load_cubin(UPDATE_UTILIZATION_EMA_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("update_utilization_ema cubin load: {e}")))?;
            module.load_function("update_utilization_ema_kernel")
                .map_err(|e| MLError::ModelError(format!("update_utilization_ema_kernel load: {e}")))?
        };

        // SP4 Layer C close-out C4 (2026-05-01): load update_adaptive_clip
        // kernel. Single-thread, single-block — replaces the host-side
        // scalar-reduction + EMA chain in `update_adaptive_clip` per
        // `feedback_no_cpu_compute_strict`. Updates the three coupled
        // mapped-pinned scalars (`adaptive_clip_pinned`,
        // `grad_norm_ema_pinned`, `outlier_diag_pinned`) plus reads
        // ISV[GRAD_CLIP_BOUND_INDEX].
        let update_adaptive_clip_kernel = {
            let module = stream.context().load_cubin(UPDATE_ADAPTIVE_CLIP_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("update_adaptive_clip cubin load: {e}")))?;
            module.load_function("update_adaptive_clip_kernel")
                .map_err(|e| MLError::ModelError(format!("update_adaptive_clip_kernel load: {e}")))?
        };

        // SP4 Layer C close-out follow-up (2026-05-01): load
        // calibrate_homeostatic kernel. Single-block, six threads — replaces
        // the host-side EMA loop in `calibrate_homeostatic_targets` per
        // `feedback_no_cpu_compute_strict`. Updates
        // `homeostatic_targets_pinned[k]` for k=1..5 with adaptive α
        // (`0.3 × (1 - readiness) + 0.01 × readiness`); thread 0 forces the
        // Q-mean invariant `targets[0] = 0.0`.
        let calibrate_homeostatic_kernel = {
            let module = stream.context().load_cubin(CALIBRATE_HOMEOSTATIC_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("calibrate_homeostatic cubin load: {e}")))?;
            module.load_function("calibrate_homeostatic_kernel")
                .map_err(|e| MLError::ModelError(format!("calibrate_homeostatic_kernel load: {e}")))?
        };

        // Plan 4 Task 3 (E.3): load iqn_quantile_ema kernel (cold-path, per-step).
        // 4-block kernel — one block per off-median fixed quantile in
        // FIXED_TAUS = [0.05, 0.25, 0.50, 0.75, 0.95]. Producer for
        // ISV[IQN_Q_P05_EMA_INDEX..IQN_Q_P95_EMA_INDEX] (slots 99..103).
        // Diagnostic only — no consumer kernel reads these slots in this commit.
        let iqn_quantile_ema_kernel = {
            let module = stream.context().load_cubin(IQN_QUANTILE_EMA_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("iqn_quantile_ema cubin load: {e}")))?;
            module.load_function("iqn_quantile_ema_update")
                .map_err(|e| MLError::ModelError(format!("iqn_quantile_ema_update load: {e}")))?
        };

        // Plan 2 Task 1 C.1: allocate branch-action offset/size device buffers for q_quantile_reduce.
        // Branch layout in q_out_buf: [dir | mag | ord | urg] row-major per sample.
        // dir occupies actions [0..b0), mag [b0..b0+b1), ord [b0+b1..b0+b1+b2),
        // urg [b0+b1+b2..total_actions).
        let (q_quantile_branch_offsets_dev, q_quantile_branch_sizes_dev) = {
            let b0 = config.branch_0_size as i32;
            let b1 = config.branch_1_size as i32;
            let b2 = config.branch_2_size as i32;
            let b3 = config.branch_3_size as i32;
            let offsets: [i32; 4] = [0, b0, b0 + b1, b0 + b1 + b2];
            let sizes:   [i32; 4] = [b0, b1, b2, b3];
            let off_dev = upload_via_mapped_i32(&stream, 4, &offsets, "q_quantile_branch_offsets")?;
            let sz_dev  = upload_via_mapped_i32(&stream, 4, &sizes,   "q_quantile_branch_sizes")?;
            (off_dev, sz_dev)
        };

        // eval_v_range — pinned device-mapped (zero-copy write from CPU).
        let eval_v_range_pinned: *mut f32 = unsafe {
            let flags = cudarc::driver::sys::CU_MEMHOSTALLOC_DEVICEMAP;
            cudarc::driver::result::malloc_host(2 * std::mem::size_of::<f32>(), flags)
                .map_err(|e| MLError::ModelError(format!("pinned eval_v_range alloc: {e}")))?
                as *mut f32
        };
        unsafe {
            *eval_v_range_pinned = config.v_min;
            *eval_v_range_pinned.add(1) = config.v_max;
        }
        let eval_v_range_ptr = unsafe {
            let mut dp = 0u64;
            cudarc::driver::sys::cuMemHostGetDevicePointer_v2(&mut dp as *mut u64, eval_v_range_pinned.cast(), 0);
            dp
        };
        // Adam step counter — pinned device-mapped (no HtoD copies).
        let t_pinned: *mut i32 = unsafe {
            let flags = cudarc::driver::sys::CU_MEMHOSTALLOC_DEVICEMAP;
            cudarc::driver::result::malloc_host(std::mem::size_of::<i32>(), flags)
                .map_err(|e| MLError::ModelError(format!("pinned t alloc: {e}")))?
                as *mut i32
        };
        unsafe { *t_pinned = 0; }
        let t_dev_ptr = unsafe {
            let mut dev_ptr: u64 = 0;
            cudarc::driver::sys::cuMemHostGetDevicePointer_v2(
                &mut dev_ptr as *mut u64,
                t_pinned.cast(),
                0,
            );
            dev_ptr
        };

        // EMA tau — pinned device-mapped (no HtoD copies).
        let tau_pinned: *mut f32 = unsafe {
            let flags = cudarc::driver::sys::CU_MEMHOSTALLOC_DEVICEMAP;
            cudarc::driver::result::malloc_host(std::mem::size_of::<f32>(), flags)
                .map_err(|e| MLError::ModelError(format!("pinned tau alloc: {e}")))?
                as *mut f32
        };
        unsafe { *tau_pinned = 0.0; }
        let tau_dev_ptr = unsafe {
            let mut dev_ptr: u64 = 0;
            cudarc::driver::sys::cuMemHostGetDevicePointer_v2(
                &mut dev_ptr as *mut u64,
                tau_pinned.cast(),
                0,
            );
            dev_ptr
        };

        // Learning rate — pinned device-mapped host memory.
        // CUDA graph captures the pointer (not the value). CPU writes new LR each epoch
        // via set_lr(); kernel reads current value on replay. Zero graph recapture overhead.
        let lr_pinned: *mut f32 = unsafe {
            let flags = cudarc::driver::sys::CU_MEMHOSTALLOC_DEVICEMAP;
            cudarc::driver::result::malloc_host(std::mem::size_of::<f32>(), flags)
                .map_err(|e| MLError::ModelError(format!("pinned lr alloc: {e}")))?
                as *mut f32
        };
        unsafe { *lr_pinned = config.lr; }
        let lr_dev_ptr = unsafe {
            let mut dev_ptr: u64 = 0;
            cudarc::driver::sys::cuMemHostGetDevicePointer_v2(
                &mut dev_ptr as *mut u64,
                lr_pinned.cast(),
                0,
            );
            dev_ptr
        };

        // Auxiliary LR (1e-4) for selectivity and denoiser Adam — fixed, not cosine-scheduled.
        // Pinned device-mapped so it satisfies the const float* kernel signature.
        let aux_lr_pinned: *mut f32 = unsafe {
            let flags = cudarc::driver::sys::CU_MEMHOSTALLOC_DEVICEMAP;
            cudarc::driver::result::malloc_host(std::mem::size_of::<f32>(), flags)
                .map_err(|e| MLError::ModelError(format!("pinned aux_lr alloc: {e}")))?
                as *mut f32
        };
        unsafe { *aux_lr_pinned = 1e-4_f32; }
        let aux_lr_dev_ptr = unsafe {
            let mut dev_ptr: u64 = 0;
            cudarc::driver::sys::cuMemHostGetDevicePointer_v2(
                &mut dev_ptr as *mut u64,
                aux_lr_pinned.cast(),
                0,
            );
            dev_ptr
        };

        // Adaptive gradient clip norm — pinned device-mapped host memory.
        // GPU reads via device pointer (zero-copy, no HtoD), host writes directly.
        let adaptive_clip_init = config.max_grad_norm;
        let adaptive_clip_pinned: *mut f32 = unsafe {
            let flags = cudarc::driver::sys::CU_MEMHOSTALLOC_DEVICEMAP;
            cudarc::driver::result::malloc_host(std::mem::size_of::<f32>(), flags)
                .map_err(|e| MLError::ModelError(format!("pinned adaptive_clip alloc: {e}")))?
                as *mut f32
        };
        unsafe { *adaptive_clip_pinned = adaptive_clip_init; }
        let adaptive_clip_dev_ptr = unsafe {
            let mut dev_ptr: u64 = 0;
            cudarc::driver::sys::cuMemHostGetDevicePointer_v2(
                &mut dev_ptr as *mut u64,
                adaptive_clip_pinned.cast(),
                0,
            );
            dev_ptr
        };

        // Plan C Phase 2 follow-up K (2026-04-29): grad-norm fast/slow EMA
        // mapped-pinned scalars feeding `fold_warmup_factor_update`. Both
        // are written by the host-side `update_adaptive_clip` (the same
        // observation source that maintains the legacy `grad_norm_ema`
        // field used by the existing adaptive clip). Mapped-pinned so the
        // kernel reads them via device-pointer with zero HtoD per
        // `feedback_no_htod_htoh_only_mapped_pinned.md`.
        //
        // FoldReset semantics (handled by `reset_named_state` arm
        // `isv_grad_norm_fast_ema`): fast → 0.0; slow persists across
        // folds (the cross-fold steady-state grad scale is the meaningful
        // baseline against which the new fold's recovery is measured —
        // resetting both would defeat the warmup signal entirely).
        let grad_norm_fast_ema_pinned: *mut f32 = unsafe {
            let flags = cudarc::driver::sys::CU_MEMHOSTALLOC_DEVICEMAP;
            cudarc::driver::result::malloc_host(std::mem::size_of::<f32>(), flags)
                .map_err(|e| MLError::ModelError(format!("pinned grad_norm_fast_ema alloc: {e}")))?
                as *mut f32
        };
        unsafe { *grad_norm_fast_ema_pinned = 0.0_f32; }
        let grad_norm_fast_ema_dev_ptr = unsafe {
            let mut dev_ptr: u64 = 0;
            cudarc::driver::sys::cuMemHostGetDevicePointer_v2(
                &mut dev_ptr as *mut u64,
                grad_norm_fast_ema_pinned.cast(),
                0,
            );
            dev_ptr
        };
        let grad_norm_slow_ema_pinned: *mut f32 = unsafe {
            let flags = cudarc::driver::sys::CU_MEMHOSTALLOC_DEVICEMAP;
            cudarc::driver::result::malloc_host(std::mem::size_of::<f32>(), flags)
                .map_err(|e| MLError::ModelError(format!("pinned grad_norm_slow_ema alloc: {e}")))?
                as *mut f32
        };
        unsafe { *grad_norm_slow_ema_pinned = 0.0_f32; }
        let grad_norm_slow_ema_dev_ptr = unsafe {
            let mut dev_ptr: u64 = 0;
            cudarc::driver::sys::cuMemHostGetDevicePointer_v2(
                &mut dev_ptr as *mut u64,
                grad_norm_slow_ema_pinned.cast(),
                0,
            );
            dev_ptr
        };

        // ── Allocate consolidated transfer buffers ─────────────────
        // Upload staging: states + next_states (f32)
        // Rewards/dones uploaded separately as f32
        // Actions uploaded separately as i32, is_weights separately as f32
        let upload_staging_len = b * ml_core::state_layout::STATE_DIM * 2; // 2*B*SD
        let upload_staging_buf = alloc_f32(&stream, upload_staging_len, "upload_staging")?;

        // Readback: total_loss(1) + mse_loss(1) + grad_norm(1) + td_errors(B)
        let readback_len = 3 + b;
        let readback_buf = alloc_f32(&stream, readback_len, "readback")?;
        let readback_host = vec![0.0_f32; readback_len];

        let upload_staging_host = vec![0.0_f32; upload_staging_len];

        // Scalar-only readback buffer
        let scalars_readback_buf = alloc_f32(&stream, 2, "scalars_readback")?;
        let scalars_readback_host = [0.0_f32; 2];

        // ── Allocate batch staging buffers ──────────────────────
        // DtoD from GpuBatch tensors (states, next_states only)
        // → these staging buffers → F32 trainer bufs.
        // Rewards, dones, weights are F32 in GpuBatch and skip staging.
        let states_staging_buf = alloc_f32(&stream, b * ml_core::state_layout::STATE_DIM, "states_staging")?;
        let next_states_staging_buf = alloc_f32(&stream, b * ml_core::state_layout::STATE_DIM, "next_states_staging")?;

        // ── cuBLAS target network scratch buffers ──────────────────
        // Target forward uses these instead of save_h_* (no grad saves needed).
        let tg_h_s1_scratch = alloc_f32(&stream, b * config.shared_h1 + kt, "tg_h_s1")?;
        let tg_h_s2_buf     = alloc_f32(&stream, b * config.shared_h2 + kt, "tg_h_s2")?;
        let tg_h_v_scratch  = alloc_f32(&stream, b * config.value_h + kt,   "tg_h_v")?;
        let tg_h_b0_scratch = alloc_f32(&stream, b * config.adv_h + kt,     "tg_h_b0")?;
        let tg_h_b1_scratch = alloc_f32(&stream, b * config.adv_h + kt,     "tg_h_b1")?;
        let tg_h_b2_scratch = alloc_f32(&stream, b * config.adv_h + kt,     "tg_h_b2")?;
        let tg_h_b3_scratch = alloc_f32(&stream, b * config.adv_h + kt,     "tg_h_b3")?;
        // Output logit buffers: F32 for GemmEx.
        // All layers are now f32.
        let tg_v_logits_buf = stream.alloc_zeros::<f32>(b * pad32(config.num_atoms))
            .map_err(|e| MLError::ModelError(format!("alloc tg_v_logits f32: {e}")))?;
        let total_branch_atoms =
            config.branch_0_size * config.num_atoms
            + config.branch_1_size * config.num_atoms
            + config.branch_2_size * config.num_atoms
            + config.branch_3_size * config.num_atoms;
        let tg_b_logits_buf = stream.alloc_zeros::<f32>(b * total_branch_atoms + 32 * 3)
            .map_err(|e| MLError::ModelError(format!("alloc tg_b_logits f32: {e}")))?;

        // ── cuBLAS online logit buffers (f32) ────────────────────────
        let on_v_logits_buf = stream.alloc_zeros::<f32>(b * pad32(config.num_atoms))
            .map_err(|e| MLError::ModelError(format!("alloc on_v_logits f32: {e}")))?;
        let on_b_logits_buf = stream.alloc_zeros::<f32>(b * total_branch_atoms + 32 * 3)
            .map_err(|e| MLError::ModelError(format!("alloc on_b_logits f32: {e}")))?;

        // ── cuBLAS online-on-next-states buffers (Double DQN action selector, f32) ──
        let on_next_v_logits_buf = stream.alloc_zeros::<f32>(b * pad32(config.num_atoms))
            .map_err(|e| MLError::ModelError(format!("alloc on_next_v_logits f32: {e}")))?;
        let on_next_b_logits_buf = stream.alloc_zeros::<f32>(b * total_branch_atoms + 32 * 3)
            .map_err(|e| MLError::ModelError(format!("alloc on_next_b_logits f32: {e}")))?;
        let on_next_h_s1_scratch = alloc_f32(&stream, b * config.shared_h1 + kt, "on_next_h_s1")?;
        let on_next_h_s2_scratch = alloc_f32(&stream, b * config.shared_h2 + kt, "on_next_h_s2")?;
        let on_next_h_v_scratch = alloc_f32(&stream, b * config.value_h + kt, "on_next_h_v")?;
        let on_next_h_b_scratch = alloc_f32(&stream, b * config.adv_h + kt, "on_next_h_b")?;

        // ── Compile standalone C51 loss + gradient kernels (required) ─
        let (c51_loss_kernel, c51_loss_reduce_kernel) = compile_c51_loss_kernel(&stream, &config)?;
        let c51_grad_kernel = compile_c51_grad_kernel(&stream, &config)?;
        // F5/D2: Q-gap barrier gradient kernel (from same c51_loss cubin).
        let barrier_gradient_kernel = load_barrier_gradient_kernel(&stream)?;
        // F4/D5: Information Bottleneck variance gradient kernel (from same c51_loss cubin).
        let ib_gradient_kernel = load_ib_gradient_kernel(&stream)?;
        info!("GpuDqnTrainer: c51_loss + c51_grad kernels compiled");

        // ── Compile MSE loss + gradient kernels (warmup before C51) ─
        let mse_loss_kernel = compile_mse_loss_kernel(&stream, &config)?;
        let mse_grad_kernel = compile_mse_grad_kernel(&stream, &config)?;
        info!("GpuDqnTrainer: mse_loss + mse_grad kernels compiled");

        // ── Compile expected Q-value + stats kernels (validation, not in CUDA Graph) ─
        let expected_q_kernel = compile_expected_q_kernel(&stream)?;
        let q_stats_kernel = compile_q_stats_kernel(&stream)?;
        // ISV v-range unification (spec 2026-04-23): per-branch Q-stats
        // reducer — emits one 7-tuple per action branch (4 × 7 = 28 floats).
        // Drives the per-branch EMA state in `update_eval_v_range` which in
        // turn writes the 8 ISV v-range slots.
        let q_stats_per_branch_kernel = compile_q_stats_per_branch_kernel(&stream)?;
        // Task 2.X "make Full useful" — per-magnitude-bin Q-mean reducer that
        // feeds ISV slots [13..16] on the stats cadence.
        let q_mag_bin_means_reduce_kernel = compile_q_mag_bin_means_kernel(&stream)?;
        let q_dir_bin_means_reduce_kernel = compile_q_dir_bin_means_kernel(&stream)?;
        let q_stats_buf = stream.alloc_zeros::<f32>(7)
            .map_err(|e| MLError::ModelError(format!("alloc q_stats_f32: {e}")))?;
        // Per-branch Q-stats readback — pinned device-mapped, 28 floats (4 × 7).
        // GPU writes, CPU reads without sync (one-step lag tolerated; the
        // update_eval_v_range path sees the latest value after the next
        // replay completes).
        let per_branch_q_stats_pinned: *mut f32 = unsafe {
            let flags = cudarc::driver::sys::CU_MEMHOSTALLOC_DEVICEMAP;
            cudarc::driver::result::malloc_host(28 * std::mem::size_of::<f32>(), flags)
                .map_err(|e| MLError::ModelError(format!("pinned per_branch_q_stats alloc: {e}")))?
                as *mut f32
        };
        unsafe { std::ptr::write_bytes(per_branch_q_stats_pinned, 0, 28); }
        let per_branch_q_stats_dev_ptr = unsafe {
            let mut dp = 0u64;
            cudarc::driver::sys::cuMemHostGetDevicePointer_v2(
                &mut dp as *mut u64,
                per_branch_q_stats_pinned.cast(),
                0,
            );
            dp
        };
        let atom_stats_buf = stream.alloc_zeros::<f32>(2)
            .map_err(|e| MLError::ModelError(format!("alloc atom_stats: {e}")))?;
        // Per-block partial sums for the atom-stat deterministic reduction.
        // Sized to max possible num_blocks for compute_expected_q (one thread
        // per sample, block_dim=256). For smoke (B=16) → 1 block; for
        // production (B=16384) → 64 blocks. Overallocate to a fixed cap.
        let atom_stats_max_blocks: usize = ((config.batch_size + 255) / 256).max(1);
        let atom_stats_block_sums_buf = stream.alloc_zeros::<f32>(atom_stats_max_blocks * 2)
            .map_err(|e| MLError::ModelError(format!("alloc atom_stats_block_sums: {e}")))?;
        // q_readback — pinned host buffer for q_stats[7] + q_out sample 0 [total_actions]
        let total_actions = config.branch_0_size + config.branch_1_size + config.branch_2_size + config.branch_3_size;
        let q_readback_size = 7 + total_actions;  // 7 stats + 13 Q-values = 20
        let q_readback_pinned: *mut f32 = unsafe {
            let flags = cudarc::driver::sys::CU_MEMHOSTALLOC_DEVICEMAP;
            cudarc::driver::result::malloc_host(q_readback_size * std::mem::size_of::<f32>(), flags)
                .map_err(|e| MLError::ModelError(format!("pinned q_readback alloc: {e}")))?
                as *mut f32
        };
        unsafe { std::ptr::write_bytes(q_readback_pinned, 0, 19); }
        let q_readback_dev_ptr = unsafe {
            let mut dp = 0u64;
            cudarc::driver::sys::cuMemHostGetDevicePointer_v2(&mut dp as *mut u64, q_readback_pinned.cast(), 0);
            dp
        };
        info!("GpuDqnTrainer: expected_q + q_stats kernels compiled");

        // ── Load mag_concat and strided_accumulate from experience_kernels cubin ──
        let exp_module_for_mag = stream.context()
            .load_cubin(EXPECTED_Q_CUBIN.to_vec())
            .map_err(|e| MLError::ModelError(format!("exp cubin for mag_concat: {e}")))?;
        let mag_concat_kernel = exp_module_for_mag.load_function("mag_concat_qdir")
            .map_err(|e| MLError::ModelError(format!("mag_concat_qdir load: {e}")))?;
        let strided_accumulate_kernel = exp_module_for_mag.load_function("strided_accumulate")
            .map_err(|e| MLError::ModelError(format!("strided_accumulate load: {e}")))?;
        let strided_scatter_kernel = exp_module_for_mag.load_function("strided_scatter")
            .map_err(|e| MLError::ModelError(format!("strided_scatter load: {e}")))?;
        let concat_ofi_kernel = exp_module_for_mag.load_function("concat_ofi_features")
            .map_err(|e| MLError::ModelError(format!("concat_ofi_features load: {e}")))?;
        let branch_confidence_routing_kernel = exp_module_for_mag.load_function("branch_confidence_routing")
            .map_err(|e| MLError::ModelError(format!("branch_confidence_routing load: {e}")))?;
        let adaptive_atom_kernel = exp_module_for_mag.load_function("adaptive_atom_positions")
            .map_err(|e| MLError::ModelError(format!("adaptive_atom_positions load: {e}")))?;
        let atom_stats_finalize_kernel = exp_module_for_mag.load_function("atom_stats_finalize")
            .map_err(|e| MLError::ModelError(format!("atom_stats_finalize load: {e}")))?;
        let atom_position_grad_kernel = exp_module_for_mag.load_function("atom_position_gradient")
            .map_err(|e| MLError::ModelError(format!("atom_position_gradient load: {e}")))?;
        let q_anchor_kernel = exp_module_for_mag.load_function("q_anchor_to_flat")
            .map_err(|e| MLError::ModelError(format!("q_anchor_to_flat load: {e}")))?;
        let regime_dropout_kernel = exp_module_for_mag.load_function("regime_dropout")
            .map_err(|e| MLError::ModelError(format!("regime_dropout load: {e}")))?;
        let epistemic_gate_kernel = exp_module_for_mag.load_function("epistemic_gate_magnitude")
            .map_err(|e| MLError::ModelError(format!("epistemic_gate_magnitude load: {e}")))?;
        // G6 / G10 kernel loads removed — .cu definitions deleted (sub-noise per V7).
        let predictive_coding_backward_kernel = exp_module_for_mag.load_function("predictive_coding_backward")
            .map_err(|e| MLError::ModelError(format!("predictive_coding_backward load: {e}")))?;
        let predictive_coding_kernel = exp_module_for_mag.load_function("predictive_coding_loss")
            .map_err(|e| MLError::ModelError(format!("predictive_coding_loss load: {e}")))?;
        let risk_forward_kernel = exp_module_for_mag.load_function("risk_budget_forward")
            .map_err(|e| MLError::ModelError(format!("risk_budget_forward load: {e}")))?;
        let risk_apply_kernel = exp_module_for_mag.load_function("apply_risk_budget")
            .map_err(|e| MLError::ModelError(format!("apply_risk_budget load: {e}")))?;
        let risk_backward_kernel = exp_module_for_mag.load_function("risk_budget_backward")
            .map_err(|e| MLError::ModelError(format!("risk_budget_backward load: {e}")))?;
        let isv_signal_update_kernel = exp_module_for_mag.load_function("isv_signal_update")
            .map_err(|e| MLError::ModelError(format!("isv_signal_update load: {e}")))?;
        let isv_forward_kernel = exp_module_for_mag.load_function("isv_forward")
            .map_err(|e| MLError::ModelError(format!("isv_forward load: {e}")))?;
        let isv_feature_gate_kernel = exp_module_for_mag.load_function("isv_feature_gate")
            .map_err(|e| MLError::ModelError(format!("isv_feature_gate load: {e}")))?;
        let fill_gamma_buf_kernel = exp_module_for_mag.load_function("fill_gamma_buf")
            .map_err(|e| MLError::ModelError(format!("fill_gamma_buf load: {e}")))?;
        let recursive_conf_fwd_kernel = exp_module_for_mag.load_function("recursive_confidence_forward")
            .map_err(|e| MLError::ModelError(format!("recursive_confidence_forward load: {e}")))?;
        let recursive_conf_bwd_kernel = exp_module_for_mag.load_function("recursive_confidence_backward")
            .map_err(|e| MLError::ModelError(format!("recursive_confidence_backward load: {e}")))?;
        let recursive_conf_reduce_kernel = exp_module_for_mag.load_function("recursive_confidence_reduce")
            .map_err(|e| MLError::ModelError(format!("recursive_confidence_reduce load: {e}")))?;
        let max_conf_blocks = ((b + 255) / 256).max(1);
        let recursive_conf_partials = stream.alloc_zeros::<f32>(max_conf_blocks * (config.shared_h2 + 1))
            .map_err(|e| MLError::ModelError(format!("recursive_conf_partials alloc: {e}")))?;
        let trade_plan_activate_kernel = exp_module_for_mag.load_function("trade_plan_activate")
            .map_err(|e| MLError::ModelError(format!("trade_plan_activate load: {e}")))?;
        let plan_noise_kernel = exp_module_for_mag.load_function("plan_noise_inject")
            .map_err(|e| MLError::ModelError(format!("plan_noise_inject load: {e}")))?;
        let ofi_embed_build_input_kernel = exp_module_for_mag.load_function("ofi_embed_build_input")
            .map_err(|e| MLError::ModelError(format!("ofi_embed_build_input load: {e}")))?;
        info!("GpuDqnTrainer: mag_concat + strided_accumulate/scatter + concat_ofi + regime_gate + adaptive_atom + atom_grad + q_anchor + regime_dropout + G5/G6/G10/G12 + risk_budget + isv_signal_update + isv_forward + fill_gamma_buf + trade_plan_activate + plan_noise + ofi_embed_build_input kernels loaded");

        // ── G5: Epistemic-gated magnitude — pinned var_ema threshold ─
        let (var_ema_pinned, var_ema_dev_ptr) = {
            let mut host_ptr: *mut std::ffi::c_void = std::ptr::null_mut();
            let mut dev_ptr_out: u64 = 0;
            unsafe {
                let rc = cudarc::driver::sys::cuMemAllocHost_v2(
                    &mut host_ptr,
                    std::mem::size_of::<f32>(),
                );
                assert_eq!(rc, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemAllocHost for var_ema");
                let rc2 = cudarc::driver::sys::cuMemHostGetDevicePointer_v2(
                    &mut dev_ptr_out,
                    host_ptr,
                    0,
                );
                assert_eq!(rc2, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemHostGetDevicePointer for var_ema");
                *(host_ptr as *mut f32) = 1.0; // Initialize to 1.0 (permissive threshold)
            }
            (host_ptr as *mut f32, dev_ptr_out)
        };

        // G6 branch_indep_penalty_buf + G10 temporal_{per_sample,penalty}_buf
        // allocs removed — their kernels were deleted after V7-gem measurement.

        // ── G12: Predictive coding auxiliary loss buffers ──
        let predictive_per_sample_buf = stream.alloc_zeros::<f32>(config.batch_size)
            .map_err(|e| MLError::ModelError(format!("alloc predictive_per_sample_buf: {e}")))?;
        let predictive_loss_buf = stream.alloc_zeros::<f32>(1)
            .map_err(|e| MLError::ModelError(format!("alloc predictive_loss_buf: {e}")))?;

        // ── Adaptive atom positions buffer ──────────────────────────
        let atom_positions_buf = stream.alloc_zeros::<f32>(4 * config.num_atoms)
            .map_err(|e| MLError::ModelError(format!("atom_positions alloc: {e}")))?;

        // ── Regime branch gate buffers ───────────────────────────────
        let regime_q_gap_buf = stream.alloc_zeros::<f32>(b)
            .map_err(|e| MLError::ModelError(format!("regime_q_gap alloc: {e}")))?;

        let (regime_util_pinned, regime_util_dev_ptr) = {
            let mut host_ptr: *mut std::ffi::c_void = std::ptr::null_mut();
            let mut dev_ptr_out: u64 = 0;
            unsafe {
                let rc = cudarc::driver::sys::cuMemAllocHost_v2(
                    &mut host_ptr,
                    std::mem::size_of::<f32>(),
                );
                assert_eq!(rc, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemAllocHost for regime_util");
                let rc2 = cudarc::driver::sys::cuMemHostGetDevicePointer_v2(
                    &mut dev_ptr_out,
                    host_ptr,
                    0,
                );
                assert_eq!(rc2, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemHostGetDevicePointer for regime_util");
                *(host_ptr as *mut f32) = 1.0; // Initialize to healthy utilization
            }
            (host_ptr as *mut f32, dev_ptr_out)
        };

        // ── Cross-Branch Q Attention buffers ──────────────────────────
        let q_attn_params = stream.alloc_zeros::<f32>(624)
            .map_err(|e| MLError::ModelError(format!("alloc q_attn_params: {e}")))?;
        let q_attn_adam_m = stream.alloc_zeros::<f32>(624)
            .map_err(|e| MLError::ModelError(format!("alloc q_attn_adam_m: {e}")))?;
        let q_attn_adam_v = stream.alloc_zeros::<f32>(624)
            .map_err(|e| MLError::ModelError(format!("alloc q_attn_adam_v: {e}")))?;
        let q_coord_buf = alloc_f32(&stream, b * 12, "q_coord_buf")?;

        // ── Selectivity gate buffers ──────────────────────────────────
        let sel_dim = config.shared_h2 + 1;
        let sel_params = alloc_f32(&stream, sel_dim, "sel_params")?;
        let sel_adam_m = stream.alloc_zeros::<f32>(sel_dim)
            .map_err(|e| MLError::ModelError(format!("alloc sel_adam_m: {e}")))?;
        let sel_adam_v = stream.alloc_zeros::<f32>(sel_dim)
            .map_err(|e| MLError::ModelError(format!("alloc sel_adam_v: {e}")))?;
        let sel_grad = stream.alloc_zeros::<f32>(sel_dim)
            .map_err(|e| MLError::ModelError(format!("alloc sel_grad: {e}")))?;
        let sel_out_buf = alloc_f32(&stream, b, "sel_out_buf")?;
        let sel_norm_buf = stream.alloc_zeros::<f32>(1)
            .map_err(|e| MLError::ModelError(format!("alloc sel_norm_buf: {e}")))?;
        let sel_norm_partials = stream.alloc_zeros::<f32>(1)
            .map_err(|e| MLError::ModelError(format!("alloc sel_norm_partials: {e}")))?;
        // Fixed clip norm of 1.0 — selectivity gate is a small sigmoid head, clip at 1.0 norm.
        // Mapped pinned: host-init via host_ptr; aux Adam-update kernel reads dev_ptr.
        // No DtoD copy per feedback_no_htod_htoh_only_mapped_pinned.
        let sel_clip_buf = unsafe { super::mapped_pinned::MappedF32Buffer::new(1) }
            .map_err(|e| MLError::ModelError(format!("sel_clip_buf alloc (1 f32): {e}")))?;
        sel_clip_buf.write_from_slice(&[1.0_f32]);
        // Selectivity Adam step counter — pinned device-mapped (no per-step HtoD copy)
        let sel_t_pinned: *mut i32 = unsafe {
            let flags = cudarc::driver::sys::CU_MEMHOSTALLOC_DEVICEMAP;
            cudarc::driver::result::malloc_host(std::mem::size_of::<i32>(), flags)
                .map_err(|e| MLError::ModelError(format!("pinned sel_t alloc: {e}")))?
                as *mut i32
        };
        unsafe { *sel_t_pinned = 0; }
        let sel_t_dev_ptr = unsafe {
            let mut dev_ptr: u64 = 0;
            cudarc::driver::sys::cuMemHostGetDevicePointer_v2(
                &mut dev_ptr as *mut u64,
                sel_t_pinned.cast(),
                0,
            );
            dev_ptr
        };

        // SP15 Phase 3.1 (2026-05-06): warm-up counter for the
        // r_quality + r_discipline split composer (per spec §8.2 (3.1)
        // post-amendment-2 fix). Single-element mapped-pinned buffer
        // initialised to 0.0 by `MappedF32Buffer::new`'s zero-init
        // contract — the host writes 0.0 at construction and again at
        // fold boundary via the registry-arm dispatch (allowed under
        // `feedback_no_htod_htoh_only_mapped_pinned.md`'s allowed-write
        // rule, NOT a host-side compute). The composer kernel
        // (`r_quality_discipline_split_kernel`) increments by 1 each
        // step; once `*count >= N_WARM=100` the composer stops
        // overriding α and the producer kernel
        // (`alpha_split_producer_kernel`) activates from no-op state.
        // Production wire-up of the per-step launches in
        // `training_loop.rs` is deferred to a follow-up commit per
        // `feedback_no_partial_refactor.md`.
        let sp15_alpha_warm_count = unsafe { super::mapped_pinned::MappedF32Buffer::new(1) }
            .map_err(|e| MLError::ModelError(format!(
                "sp15_alpha_warm_count alloc (1 f32): {e}"
            )))?;
        // `MappedF32Buffer::new` already zero-fills the host side; the
        // explicit write here makes the cold-start contract self-evident
        // at the call site (mirrors `sel_clip_buf.write_from_slice` for
        // its 1.0 fixed-clip default).
        sp15_alpha_warm_count.write_from_slice(&[0.0_f32]);

        // SP15 Phase 3.5.3 (2026-05-06): cooldown gate consecutive-loss
        // streak counter (per spec §9.2 (3.5.3) post-amendment-2 fix).
        // Single-element mapped-pinned f32 buffer initialised to 0.0
        // by `MappedF32Buffer::new`'s zero-init contract — the host
        // writes 0.0 at construction and again at fold boundary via
        // the registry-arm dispatch (allowed under
        // `feedback_no_htod_htoh_only_mapped_pinned.md`'s allowed-write
        // rule, NOT a host-side compute). The cooldown kernel
        // (`cooldown_kernel`) read-modify-writes this counter: `+=
        // 1.0f` on trade-close losses, `= 0.0f` on trade-close wins,
        // untouched on non-trade-close bars. Production wire-up of the
        // per-bar launches (forces Hold while
        // `cooldown_remaining > 0`) is deferred to a follow-up commit
        // per `feedback_no_partial_refactor.md`.
        let sp15_cooldown_consecutive_losses =
            unsafe { super::mapped_pinned::MappedF32Buffer::new(1) }
                .map_err(|e| MLError::ModelError(format!(
                    "sp15_cooldown_consecutive_losses alloc (1 f32): {e}"
                )))?;
        sp15_cooldown_consecutive_losses.write_from_slice(&[0.0_f32]);

        // SP15 Phase 3.5.5 (2026-05-06) → Phase 1.3.b-followup-B
        // (2026-05-07): the trainer no longer allocates a `[1]`
        // single-env prev_dd_pct scratch buffer here. The per-env
        // scratch + output tiles for the recovery-curriculum DD
        // trajectory live on `GpuExperienceCollector` (which knows
        // `alloc_episodes` = `n_envs`):
        //   * `sp15_dd_trajectory_prev_dd_per_env: MappedF32Buffer
        //     ([alloc_episodes])` — per-env persistent prev_dd_pct
        //     scratch (mirrors `sp15_dd_state_per_env` collector
        //     ownership pattern; the previous follow-up split was
        //     intentional pragmatic compromise).
        //   * `sp15_dd_trajectory_per_env: MappedF32Buffer
        //     ([alloc_episodes])` — per-env trajectory output tile,
        //     read by `per_insert_pa` per-transition via insert-time
        //     `env_id = (j % (n_envs * lookback)) / lookback` lookup.
        // Both buffers reset to zero at fold boundary via the
        // collector-owned dispatch arms `sp15_dd_trajectory_prev_dd_per_env`
        // and `sp15_dd_trajectory_per_env` in `training_loop.rs`.

        // ── Per-branch Q-gap EMA (trajectory backtracking state) ────────
        let per_branch_q_gap_ema_buf = stream.alloc_zeros::<f32>(4) // [4]
            .map_err(|e| MLError::ModelError(format!("alloc per_branch_q_gap_ema_buf: {e}")))?;

        // ── VSN + GLU scratch buffers ─────────────────────────────────
        let vsn_masked_buf = alloc_f32(&stream, b * config.shared_h2, "vsn_masked_buf")?;
        let glu_gate_pre_buf = [
            alloc_f32(&stream, b * config.adv_h, "glu_gate0")?,
            alloc_f32(&stream, b * config.adv_h, "glu_gate1")?,
            alloc_f32(&stream, b * config.adv_h, "glu_gate2")?,
            alloc_f32(&stream, b * config.adv_h, "glu_gate3")?,
        ];
        let glu_value_buf = [
            alloc_f32(&stream, b * config.adv_h, "glu_val0")?,
            alloc_f32(&stream, b * config.adv_h, "glu_val1")?,
            alloc_f32(&stream, b * config.adv_h, "glu_val2")?,
            alloc_f32(&stream, b * config.adv_h, "glu_val3")?,
        ];

        // ── Load Q-attn/selectivity/VSN/GLU kernels from experience_kernels cubin ──
        let cpbi_module = stream.context().load_cubin(EXPECTED_Q_CUBIN.to_vec())
            .map_err(|e| MLError::ModelError(format!("cpbi cubin: {e}")))?;
        let q_attn_kernel = cpbi_module.load_function("cross_branch_q_attention")
            .map_err(|e| MLError::ModelError(format!("cross_branch_q_attention load: {e}")))?;
        let sel_fwd_kernel = cpbi_module.load_function("selectivity_forward")
            .map_err(|e| MLError::ModelError(format!("selectivity_forward load: {e}")))?;
        let sel_compute_dz_kernel = cpbi_module.load_function("selectivity_compute_dz")
            .map_err(|e| MLError::ModelError(format!("selectivity_compute_dz load: {e}")))?;
        let sel_dw_reduce_p1_kernel = cpbi_module.load_function("selectivity_dw_reduce_p1")
            .map_err(|e| MLError::ModelError(format!("selectivity_dw_reduce_p1 load: {e}")))?;
        let sel_dw_reduce_p2_kernel = cpbi_module.load_function("selectivity_dw_reduce_p2")
            .map_err(|e| MLError::ModelError(format!("selectivity_dw_reduce_p2 load: {e}")))?;
        let sel_num_blocks = ((b + 255) / 256) as u32;
        let sel_d_z_buf = alloc_f32(&stream, b, "sel_d_z")?;
        let sel_dw_partials_buf = alloc_f32(&stream, sel_num_blocks as usize * (config.shared_h2 + 1), "sel_dw_partials")?;
        let vsn_kernel = cpbi_module.load_function("variable_select_bottleneck")
            .map_err(|e| MLError::ModelError(format!("variable_select_bottleneck load: {e}")))?;
        let glu_combine_kernel = cpbi_module.load_function("glu_combine")
            .map_err(|e| MLError::ModelError(format!("glu_combine load: {e}")))?;
        let glu_backward_kernel = cpbi_module.load_function("glu_backward")
            .map_err(|e| MLError::ModelError(format!("glu_backward load: {e}")))?;
        let kan_gate_combine_kernel = cpbi_module.load_function("kan_gate_combine")
            .map_err(|e| MLError::ModelError(format!("kan_gate_combine load: {e}")))?;
        let kan_gate_backward_kernel = cpbi_module.load_function("kan_gate_backward")
            .map_err(|e| MLError::ModelError(format!("kan_gate_backward load: {e}")))?;
        let kan_grad_reduce_p1_kernel = cpbi_module.load_function("kan_grad_reduce_p1")
            .map_err(|e| MLError::ModelError(format!("kan_grad_reduce_p1 load: {e}")))?;
        let kan_grad_reduce_p2_kernel = cpbi_module.load_function("kan_grad_reduce_p2")
            .map_err(|e| MLError::ModelError(format!("kan_grad_reduce_p2 load: {e}")))?;
        info!("GpuDqnTrainer: Q-attn + selectivity + VSN + GLU + KAN kernels loaded");

        // ── Compile CQL penalty kernel (if enabled) ──────────────────────
        let cql_logit_grad_kernel = if config.cql_alpha > 0.0 {
            match compile_cql_logit_grad_kernel(&stream) {
                Ok(k) => {
                    info!(cql_alpha = config.cql_alpha, "GpuDqnTrainer: CQL logit gradient kernel compiled");
                    Some(k)
                }
                Err(e) => {
                    tracing::warn!("CQL kernel compilation failed (non-fatal): {e}");
                    None
                }
            }
        } else {
            None
        };
        let cql_d_value_logits = stream.alloc_zeros::<f32>(b * pad32(config.num_atoms))
            .map_err(|e| MLError::ModelError(format!("alloc cql_d_value_logits f32: {e}")))?;
        let cql_d_adv_logits = stream.alloc_zeros::<f32>(b * total_branch_atoms + 32 * 3)
            .map_err(|e| MLError::ModelError(format!("alloc cql_d_adv_logits f32: {e}")))?;

        // ── Gradient output buffers (f32 for native atomicAdd) ─
        let d_value_logits_buf = stream.alloc_zeros::<f32>(b * pad32(config.num_atoms))
            .map_err(|e| MLError::ModelError(format!("alloc d_value_logits f32: {e}")))?;
        let d_adv_logits_buf = stream.alloc_zeros::<f32>(b * total_branch_atoms + 32 * 4)
            .map_err(|e| MLError::ModelError(format!("alloc d_adv_logits f32: {e}")))?;
        // Scratch buffers for blended MSE+C51 loss (MSE grad stored here, then blended)
        let d_value_logits_mse = stream.alloc_zeros::<f32>(b * pad32(config.num_atoms))
            .map_err(|e| MLError::ModelError(format!("alloc d_value_logits_mse f32: {e}")))?;
        let d_adv_logits_mse = stream.alloc_zeros::<f32>(b * total_branch_atoms + 32 * 4)
            .map_err(|e| MLError::ModelError(format!("alloc d_adv_logits_mse f32: {e}")))?;
        // Staging buffers for backward pass cuBLAS GEMM
        let d_value_logits = alloc_f32(&stream, b * pad32(config.num_atoms), "d_value_logits")?;
        let d_adv_logits = alloc_f32(&stream, b * total_branch_atoms + 32 * 4, "d_adv_logits")?;

        // ── Spectral normalization singular vectors ─────────────────
        // Initialize with random unit vectors for proper power iteration convergence.
        // Uniform 1/√n converges but slowly (aligned with no singular direction).
        // Random vectors almost surely have nonzero projection onto the leading
        // singular direction, giving fast convergence (typically 3-5 iterations).
        let mut rng_state = 0x5DEECE66Du64.wrapping_add(config.batch_size as u64);
        let rand_unit = |n: usize, rng: &mut u64| -> Vec<f32> {
            let mut v = Vec::with_capacity(n);
            for _ in 0..n {
                *rng = rng.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
                let u = (*rng >> 33) as f32 / (1u64 << 31) as f32 - 0.5;
                v.push(u);
            }
            let norm: f32 = v.iter().map(|x| x * x).sum::<f32>().sqrt().max(1e-8);
            for x in v.iter_mut() { *x /= norm; }
            v
        };
        let init_u_s1 = rand_unit(config.shared_h1, &mut rng_state);
        let init_v_s1 = rand_unit(ml_core::state_layout::STATE_DIM, &mut rng_state);
        let init_u_s2 = rand_unit(config.shared_h2, &mut rng_state);
        let init_v_s2 = rand_unit(config.shared_h1, &mut rng_state);

        // IQN trunk Adam state (separate from C51 Adam — prevents momentum mismatch)
        //
        // Plan 4 Task 2c.3c.4: post-GRN reshuffle the trunk owns 13 tensors at
        // indices [0..13) (h_s1: 7 tensors with Linear_residual; h_s2: 6 tensors
        // with identity residual). To keep the SAXPY-based mix-into-grad_buf
        // pattern simple, `iqn_trunk_m` mirrors `grad_buf`'s padded layout for
        // exactly those 13 tensors. The element count below = total padded f32
        // slots from index 0 up to (but excluding) tensor 13. Element-wise
        // SAXPY across this many floats writes to the same per-tensor offsets
        // in grad_buf because the layout is identical (padding inclusive).
        let __iqn_trunk_param_sizes = compute_param_sizes(&config);
        let trunk_params: usize =
            (padded_byte_offset(&__iqn_trunk_param_sizes, 13) as usize)
                / std::mem::size_of::<f32>();
        let iqn_trunk_m = stream.alloc_zeros::<f32>(trunk_params)
            .map_err(|e| MLError::ModelError(format!("alloc iqn_trunk_m f32: {e}")))?;
        let iqn_trunk_v = alloc_f32(&stream, trunk_params, "iqn_trunk_v")?;
        let iqn_trunk_grad_norm = stream.alloc_zeros::<f32>(1)
            .map_err(|e| MLError::ModelError(format!("alloc iqn_trunk_grad_norm f32: {e}")))?;
        // Scratch f32 for IQN/ensemble trunk finalize — prevents overwriting main grad_norm_buf.
        let aux_norm_scratch = alloc_f32(&stream, 1, "aux_norm_scratch")?;
        let attn_bw_scratch = alloc_f32(&stream, b * config.shared_h2, "attn_bw_scratch")?;
        let iqn_trunk_t_buf = alloc_i32(&stream, 1, "iqn_trunk_t_buf")?;
        // Mapped pinned: host-init via host_ptr; batched power-iteration kernel
        // reads via dev_ptr (indirected through `spectral_norm_descriptors`).
        // No DtoD copy per feedback_no_htod_htoh_only_mapped_pinned.
        let spec_u_s1 = unsafe { super::mapped_pinned::MappedF32Buffer::new(config.shared_h1) }
            .map_err(|e| MLError::ModelError(format!("spec_u_s1 alloc ({} f32): {e}", config.shared_h1)))?;
        let spec_v_s1 = unsafe { super::mapped_pinned::MappedF32Buffer::new(ml_core::state_layout::STATE_DIM) }
            .map_err(|e| MLError::ModelError(format!("spec_v_s1 alloc ({} f32): {e}", ml_core::state_layout::STATE_DIM)))?;
        let spec_u_s2 = unsafe { super::mapped_pinned::MappedF32Buffer::new(config.shared_h2) }
            .map_err(|e| MLError::ModelError(format!("spec_u_s2 alloc ({} f32): {e}", config.shared_h2)))?;
        let spec_v_s2 = unsafe { super::mapped_pinned::MappedF32Buffer::new(config.shared_h1) }
            .map_err(|e| MLError::ModelError(format!("spec_v_s2 alloc ({} f32): {e}", config.shared_h1)))?;
        spec_u_s1.write_from_slice(&init_u_s1);
        spec_v_s1.write_from_slice(&init_v_s1);
        spec_u_s2.write_from_slice(&init_u_s2);
        spec_v_s2.write_from_slice(&init_v_s2);

        // Head spectral norm vectors: value head, adv/branch heads
        let vh = config.value_h;
        let ah = config.adv_h;
        let na = config.num_atoms;
        let sh2 = config.shared_h2;
        let b0 = config.branch_0_size;
        let b1 = config.branch_1_size;
        let b2 = config.branch_2_size;

        // Mapped-pinned spectral-norm pair: host-init via host_ptr; the batched
        // power-iteration kernel reads dev_ptr (indirected through
        // `spectral_norm_descriptors`). No DtoD copy per
        // feedback_no_htod_htoh_only_mapped_pinned.
        macro_rules! alloc_spec_pair {
            ($u_name:ident, $v_name:ident, $u_n:expr, $v_n:expr, $lbl_u:literal, $lbl_v:literal) => {
                let $u_name = unsafe { super::mapped_pinned::MappedF32Buffer::new($u_n) }
                    .map_err(|e| MLError::ModelError(format!("{} alloc ({} f32): {e}", $lbl_u, $u_n)))?;
                let $v_name = unsafe { super::mapped_pinned::MappedF32Buffer::new($v_n) }
                    .map_err(|e| MLError::ModelError(format!("{} alloc ({} f32): {e}", $lbl_v, $v_n)))?;
                let init_u = rand_unit($u_n, &mut rng_state);
                let init_v = rand_unit($v_n, &mut rng_state);
                $u_name.write_from_slice(&init_u);
                $v_name.write_from_slice(&init_v);
            };
        }

        // W_v1 [value_h, shared_h2]
        alloc_spec_pair!(spec_u_v1, spec_v_v1, vh, sh2, "spec_u_v1", "spec_v_v1");
        // W_v2 [num_atoms, value_h]
        alloc_spec_pair!(spec_u_v2, spec_v_v2, na, vh, "spec_u_v2", "spec_v_v2");
        // W_a1 [adv_h, shared_h2 + 1] — SP14 aux→Q wire (input dim grew by 1)
        alloc_spec_pair!(spec_u_a1, spec_v_a1, ah, sh2 + 1, "spec_u_a1", "spec_v_a1");
        // W_a2 [b0*num_atoms, adv_h]
        alloc_spec_pair!(spec_u_a2, spec_v_a2, b0 * na, ah, "spec_u_a2", "spec_v_a2");
        // W_bo1 [adv_h, shared_h2]
        alloc_spec_pair!(spec_u_bo1, spec_v_bo1, ah, sh2, "spec_u_bo1", "spec_v_bo1");
        // W_bo2 [b1*num_atoms, adv_h]
        alloc_spec_pair!(spec_u_bo2, spec_v_bo2, b1 * na, ah, "spec_u_bo2", "spec_v_bo2");
        // W_bu1 [adv_h, shared_h2]
        alloc_spec_pair!(spec_u_bu1, spec_v_bu1, ah, sh2, "spec_u_bu1", "spec_v_bu1");
        // W_bu2 [b2*num_atoms, adv_h]
        alloc_spec_pair!(spec_u_bu2, spec_v_bu2, b2 * na, ah, "spec_u_bu2", "spec_v_bu2");
        // W_bg1 [adv_h, shared_h2] — branch 3 (urgency) FC
        let b3 = config.branch_3_size;
        alloc_spec_pair!(spec_u_bg1, spec_v_bg1, ah, sh2, "spec_u_bg1", "spec_v_bg1");
        // W_bg2 [b3*num_atoms, adv_h] — branch 3 (urgency) output
        alloc_spec_pair!(spec_u_bg2, spec_v_bg2, b3 * na, ah, "spec_u_bg2", "spec_v_bg2");
        // W_bn [bottleneck_dim, market_dim] — temporal causal bottleneck
        // CRITICAL: this was MISSING from spectral norm, causing unconstrained
        // backward amplification → exponential gradient norm growth.
        let bn_dim = config.bottleneck_dim.max(1);
        let md = config.market_dim.max(1);
        alloc_spec_pair!(spec_u_bn, spec_v_bn, bn_dim, md, "spec_u_bn", "spec_v_bn");

        // ── Build batched spectral norm descriptor buffer (13 matrices × 6 u64) ──
        // Pointers are stable: params_buf base and spec_u/v allocations never move.
        let spectral_norm_descriptors = {
            let param_sizes = compute_param_sizes(&config);
            let w_ptrs = f32_weight_ptrs_from_base(params_buf.raw_ptr(), &param_sizes);
            let sd_u64 = ml_core::state_layout::STATE_DIM as u64;
            let sh1_u64 = config.shared_h1 as u64;
            let sh2_u64 = config.shared_h2 as u64;
            let vh_u64 = config.value_h as u64;
            let ah_u64 = config.adv_h as u64;
            let na_u64 = config.num_atoms as u64;
            let b0_u64 = config.branch_0_size as u64;
            let b1_u64 = config.branch_1_size as u64;
            let b2_u64 = config.branch_2_size as u64;
            let b3_u64 = config.branch_3_size as u64;

            // 13 matrices, 6 u64 each: [W_ptr, u_ptr, v_ptr, out_dim, in_dim, pad]
            // Plan 4 Task 2c.3a: trunk W_s1/W_s2 replaced by GRN's
            // w_a_h_s1/w_a_h_s2 (Linear_a matrices) — shapes match exactly
            // (both [SH1, SD] and [SH2, SH1]), so the existing spectral-
            // norm constraint transfers cleanly to the GRN's first
            // Linear. Linear_b / Linear_residual are NOT yet spectral-
            // normed — Task 2c.3c will extend coverage.
            let bn_dim_u64 = config.bottleneck_dim.max(1) as u64;
            let md_u64 = config.market_dim.max(1) as u64;
            // Mapped-pinned spec_u/spec_v: dev_ptr is the CUdeviceptr (u64) the
            // batched power-iteration kernel reads. Same width as `raw_ptr()` on
            // CudaSlice — purely a structural rename, no semantic change.
            let host_desc: [u64; 78] = [
                // [0] w_a_h_s1 [shared_h1, state_dim] — GRN h_s1 Linear_a (was W_s1)
                w_ptrs[0], spec_u_s1.dev_ptr, spec_v_s1.dev_ptr, sh1_u64, sd_u64, 0,
                // [1] w_a_h_s2 [shared_h2, shared_h1] — GRN h_s2 Linear_a (was W_s2)
                w_ptrs[7], spec_u_s2.dev_ptr, spec_v_s2.dev_ptr, sh2_u64, sh1_u64, 0,
                // [2] W_v1 [value_h, shared_h2] — GOFF 13 (was 4)
                w_ptrs[13], spec_u_v1.dev_ptr, spec_v_v1.dev_ptr, vh_u64, sh2_u64, 0,
                // [3] W_v2 [num_atoms, value_h] — GOFF 15 (was 6)
                w_ptrs[15], spec_u_v2.dev_ptr, spec_v_v2.dev_ptr, na_u64, vh_u64, 0,
                // [4] W_a1 [adv_h, shared_h2 + 1] — GOFF 17 (was 8); SP14 aux→Q wire (input dim SH2+1)
                w_ptrs[17], spec_u_a1.dev_ptr, spec_v_a1.dev_ptr, ah_u64, sh2_u64 + 1, 0,
                // [5] W_a2 [b0*num_atoms, adv_h] — GOFF 19 (was 10)
                w_ptrs[19], spec_u_a2.dev_ptr, spec_v_a2.dev_ptr, b0_u64 * na_u64, ah_u64, 0,
                // [6] W_bo1 [adv_h, shared_h2] — GOFF 21 (was 12)
                w_ptrs[21], spec_u_bo1.dev_ptr, spec_v_bo1.dev_ptr, ah_u64, sh2_u64, 0,
                // [7] W_bo2 [b1*num_atoms, adv_h] — GOFF 23 (was 14)
                w_ptrs[23], spec_u_bo2.dev_ptr, spec_v_bo2.dev_ptr, b1_u64 * na_u64, ah_u64, 0,
                // [8] W_bu1 [adv_h, shared_h2] — GOFF 25 (was 16)
                w_ptrs[25], spec_u_bu1.dev_ptr, spec_v_bu1.dev_ptr, ah_u64, sh2_u64, 0,
                // [9] W_bu2 [b2*num_atoms, adv_h] — GOFF 27 (was 18)
                w_ptrs[27], spec_u_bu2.dev_ptr, spec_v_bu2.dev_ptr, b2_u64 * na_u64, ah_u64, 0,
                // [10] W_bg1 [adv_h, shared_h2] — GOFF 29 (was 20)
                w_ptrs[29], spec_u_bg1.dev_ptr, spec_v_bg1.dev_ptr, ah_u64, sh2_u64, 0,
                // [11] W_bg2 [b3*num_atoms, adv_h] — GOFF 31 (was 22)
                w_ptrs[31], spec_u_bg2.dev_ptr, spec_v_bg2.dev_ptr, b3_u64 * na_u64, ah_u64, 0,
                // [12] W_bn [bottleneck_dim, market_dim] — GOFF 33 (was 24)
                w_ptrs[33], spec_u_bn.dev_ptr, spec_v_bn.dev_ptr, bn_dim_u64, md_u64, 0,
            ];
            upload_via_mapped_u64(&stream, 78, &host_desc, "spectral_norm_descriptors")?
        };

        // ── Initialize shared cuBLAS handle (one handle for all forward/backward) ──
        let shared_cublas = Arc::new(PerStreamCublasHandles::new(&stream)?);

        // SP15 Wave 4.1b: the bottleneck-on s1_input_dim grows by +1 to carry
        // the dd_pct column appended by `bn_tanh_concat_dd_kernel`. Mirrors
        // `compute_param_sizes` exactly so the cuBLAS gemm cache shape keys
        // match the GRN w_s1 / w_residual_h_s1 tensor widths.
        let s1_input_dim = if config.bottleneck_dim > 0 {
            config.bottleneck_dim
                + ml_core::state_layout::STATE_DIM.saturating_sub(config.market_dim)
                + 1
        } else {
            ml_core::state_layout::STATE_DIM
        };
        let mut cublas_forward = CublasGemmSet::new(
            Arc::clone(&shared_cublas),
            config.batch_size,
            ml_core::state_layout::STATE_DIM,
            config.shared_h1,
            config.shared_h2,
            config.value_h,
            config.adv_h,
            config.num_atoms,
            config.branch_0_size,
            config.branch_1_size,
            config.branch_2_size,
            config.branch_3_size,
            s1_input_dim,
        )?;
        // Wire VSN bottleneck + KAN spline gating into branch forward paths
        cublas_forward.wire_vsn_glu(
            vsn_kernel.clone(),
            kan_gate_combine_kernel.clone(),
            strided_scatter_kernel.clone(),
            vsn_masked_buf.raw_ptr(),
            [
                glu_gate_pre_buf[0].raw_ptr(), glu_gate_pre_buf[1].raw_ptr(),
                glu_gate_pre_buf[2].raw_ptr(), glu_gate_pre_buf[3].raw_ptr(),
            ],
            [
                glu_value_buf[0].raw_ptr(), glu_value_buf[1].raw_ptr(),
                glu_value_buf[2].raw_ptr(), glu_value_buf[3].raw_ptr(),
            ],
            16, // VSN bottleneck rank R=16
        );
        cublas_forward.wire_ofi_concat(
            ord_concat_buf.raw_ptr(),
            urg_concat_buf.raw_ptr(),
        );
        // Plan 4 Task 1B-iii: wire the strided_scatter kernel handle reused by
        // `vsn_forward` to write per-group `Linear_2_g` outputs into columns
        // of `vsn_logits_*_buf`. Same kernel handle the trainer already uses
        // for VSN bottleneck wiring above.
        cublas_forward.wire_vsn_scatter(strided_scatter_kernel.clone());
        // Phase H Site 2: register the trainer's "main online save_h_v" pointer
        // so the value-FC forward only takes the RELU_AUX_BIAS (mask-writing)
        // path when the caller-supplied `h_v_ptr` matches `save_h_v` —
        // auxiliary forwards (causal-intervention, IQN, DDQN, ensemble) target
        // scratch h_v buffers and must NOT clobber the mask the backward reads.
        cublas_forward.wire_value_fc_save_h_v(save_h_v.raw_ptr());
        info!("GpuDqnTrainer: cuBLAS batched forward initialized (VSN+KAN+OFI wired, VSN feature-selection scatter wired)");

        // DDQN forward shares the same cuBLAS handle — runs sequentially
        // on the main stream for NVIDIA bit-wise reproducibility.
        let mut cublas_forward_ddqn = CublasGemmSet::new(
            Arc::clone(&shared_cublas),
            config.batch_size,
            ml_core::state_layout::STATE_DIM,
            config.shared_h1,
            config.shared_h2,
            config.value_h,
            config.adv_h,
            config.num_atoms,
            config.branch_0_size,
            config.branch_1_size,
            config.branch_2_size,
            config.branch_3_size,
            s1_input_dim,
        )?;
        // DDQN argmax pass also uses VSN+KAN — same kernels, same scratch buffers
        // (DDQN runs sequentially on main stream, no concurrent access).
        cublas_forward_ddqn.wire_vsn_glu(
            vsn_kernel.clone(),
            kan_gate_combine_kernel.clone(),
            strided_scatter_kernel.clone(),
            vsn_masked_buf.raw_ptr(),
            [
                glu_gate_pre_buf[0].raw_ptr(), glu_gate_pre_buf[1].raw_ptr(),
                glu_gate_pre_buf[2].raw_ptr(), glu_gate_pre_buf[3].raw_ptr(),
            ],
            [
                glu_value_buf[0].raw_ptr(), glu_value_buf[1].raw_ptr(),
                glu_value_buf[2].raw_ptr(), glu_value_buf[3].raw_ptr(),
            ],
            16,
        );
        cublas_forward_ddqn.wire_ofi_concat(
            ord_concat_buf.raw_ptr(),
            urg_concat_buf.raw_ptr(),
        );
        // Plan 4 Task 1B-iii: same scatter kernel for the DDQN-pass VSN forward.
        cublas_forward_ddqn.wire_vsn_scatter(strided_scatter_kernel.clone());
        info!("GpuDqnTrainer: cuBLAS batched forward (DDQN) initialized (VSN+KAN+OFI wired, VSN feature-selection scatter wired)");

        // ── Initialize cuBLAS backward context (required) ──────────
        let cublas_backward = CublasBackwardSet::new(
            Arc::clone(&shared_cublas), &config, kan_gate_backward_kernel.clone(),
            kan_grad_reduce_p1_kernel.clone(), kan_grad_reduce_p2_kernel.clone(),
            cublas_forward.branch_workspace_ptrs(),
        )?;
        info!("GpuDqnTrainer: cuBLAS batched backward initialized (KAN backward wired)");

        // ── Backward scratch buffers ────────────────────────────────
        // Pre-allocate inter-layer gradient buffers for the cuBLAS backward
        // pass.  These are separate from the activation saves used in forward.
        let (bw_d_h_s2, bw_d_h_s1, bw_d_h_v, bw_d_h_b0, bw_d_h_b1, bw_d_h_b2, bw_d_h_b3,
             bw_d_glu_value, bw_d_glu_gate) =
            alloc_backward_scratch(&stream, &config)
                .map_err(|e| MLError::ModelError(format!("backward scratch alloc: {e}")))?;

        // ── GRN trunk backward scratch (Plan 4 Task 2c.3c.3) ────────
        // 4 buffers per GRN block × 2 blocks = 8 buffers total. Consumed by
        // 2c.3c.4's `apply_grn_trunk_backward_raw` helper. Sizes mirror the
        // forward trunk hidden dims (SH1 / SH2) and use `config.batch_size`
        // as the batch dim, matching `alloc_backward_scratch` convention.
        // d_linear_b_out is [B, 2*hidden] because GLU's pre-activation has
        // 2× the hidden width (value path + gate_pre path concatenated).
        let bw_grn_h_s2_d_pre_ln = stream.alloc_zeros::<f32>(b * config.shared_h2)
            .map_err(|e| MLError::ModelError(format!("alloc bw_grn_h_s2_d_pre_ln: {e}")))?;
        let bw_grn_h_s2_d_linear_b_out = stream.alloc_zeros::<f32>(b * 2 * config.shared_h2)
            .map_err(|e| MLError::ModelError(format!("alloc bw_grn_h_s2_d_linear_b_out: {e}")))?;
        let bw_grn_h_s2_d_elu_out = stream.alloc_zeros::<f32>(b * config.shared_h2)
            .map_err(|e| MLError::ModelError(format!("alloc bw_grn_h_s2_d_elu_out: {e}")))?;
        let bw_grn_h_s2_d_linear_a_out = stream.alloc_zeros::<f32>(b * config.shared_h2)
            .map_err(|e| MLError::ModelError(format!("alloc bw_grn_h_s2_d_linear_a_out: {e}")))?;
        let bw_grn_h_s1_d_pre_ln = stream.alloc_zeros::<f32>(b * config.shared_h1)
            .map_err(|e| MLError::ModelError(format!("alloc bw_grn_h_s1_d_pre_ln: {e}")))?;
        let bw_grn_h_s1_d_linear_b_out = stream.alloc_zeros::<f32>(b * 2 * config.shared_h1)
            .map_err(|e| MLError::ModelError(format!("alloc bw_grn_h_s1_d_linear_b_out: {e}")))?;
        let bw_grn_h_s1_d_elu_out = stream.alloc_zeros::<f32>(b * config.shared_h1)
            .map_err(|e| MLError::ModelError(format!("alloc bw_grn_h_s1_d_elu_out: {e}")))?;
        let bw_grn_h_s1_d_linear_a_out = stream.alloc_zeros::<f32>(b * config.shared_h1)
            .map_err(|e| MLError::ModelError(format!("alloc bw_grn_h_s1_d_linear_a_out: {e}")))?;

        let batch_bytes = (b * ml_core::state_layout::STATE_DIM * 2
            + b * 4
            + b * config.shared_h1
            + b * config.shared_h2
            + b * config.value_h
            + b * config.adv_h * 4
            + b * num_branches * config.num_atoms * 2
            + b * 2 + 1)
            * std::mem::size_of::<f32>();
        // f32: grad + params + target_params + m + v (5 buffers, no shadow)
        let optim_bytes = total_params * 5 * std::mem::size_of::<f32>();

        info!(
            batch_size = b,
            state_dim = ml_core::state_layout::STATE_DIM,
            total_params,
            batch_alloc_mb = batch_bytes as f64 / (1024.0 * 1024.0),
            optim_alloc_mb = optim_bytes as f64 / (1024.0 * 1024.0),
            "GpuDqnTrainer: all buffers allocated, 4 kernels compiled"
        );

        let initial_c51_alpha = if config.c51_warmup_epochs > 0 { 0.0 } else { 1.0 };

        // ── Curiosity Q-penalty: inference-only kernel + error buffer ──
        let curiosity_error_buf = stream.alloc_zeros::<f32>(b)
            .map_err(|e| MLError::ModelError(format!("alloc curiosity_error_buf: {e}")))?;
        let drawdown_depths_buf = stream.alloc_zeros::<f32>(b)
            .map_err(|e| MLError::ModelError(format!("alloc drawdown_depths_buf: {e}")))?;
        let ensemble_std_buf = stream.alloc_zeros::<f32>(b)
            .map_err(|e| MLError::ModelError(format!("alloc ensemble_std_buf: {e}")))?;
        // #21 Stochastic depth: 3 layer scales + GPU RNG state
        let stochastic_depth_scale_buf =
            upload_via_mapped_f32(&stream, 3, &[1.0_f32, 1.0, 1.0], "stochastic_depth_scale")?;
        // Deterministic seed for reproducible stochastic depth masks
        let sd_seed: u32 = 0x5D5E_ED00;
        let stochastic_depth_rng_state =
            upload_via_mapped_u32(&stream, 1, &[sd_seed], "sd_rng_state")?;
        // Curiosity cuBLAS GEMM pipeline buffers:
        //   CUR_INPUT = market_dim + 3, CUR_HIDDEN = 128, CUR_OUTPUT = market_dim
        let cur_input = config.market_dim + 3;
        let cur_hidden: usize = 128;
        let cur_output = config.market_dim;
        let curiosity_input_buf = stream.alloc_zeros::<f32>(b * cur_input)
            .map_err(|e| MLError::ModelError(format!("alloc curiosity_input_buf: {e}")))?;
        let curiosity_hidden_buf = stream.alloc_zeros::<f32>(b * cur_hidden)
            .map_err(|e| MLError::ModelError(format!("alloc curiosity_hidden_buf: {e}")))?;
        let curiosity_pred_buf = stream.alloc_zeros::<f32>(b * cur_output)
            .map_err(|e| MLError::ModelError(format!("alloc curiosity_pred_buf: {e}")))?;

        let (curiosity_prepare_input_func, curiosity_bias_leaky_relu_func, curiosity_bias_mse_func) = {
            static CURIOSITY_INFERENCE_CUBIN: &[u8] = include_bytes!(
                concat!(env!("OUT_DIR"), "/curiosity_inference_kernel.cubin")
            );
            let module = stream.context().load_cubin(CURIOSITY_INFERENCE_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("curiosity_inference cubin: {e}")))?;
            let prepare = module.load_function("curiosity_prepare_input")
                .map_err(|e| MLError::ModelError(format!("curiosity_prepare_input load: {e}")))?;
            let bias_lrelu = module.load_function("curiosity_bias_leaky_relu")
                .map_err(|e| MLError::ModelError(format!("curiosity_bias_leaky_relu load: {e}")))?;
            let bias_mse = module.load_function("curiosity_bias_mse")
                .map_err(|e| MLError::ModelError(format!("curiosity_bias_mse load: {e}")))?;
            (prepare, bias_lrelu, bias_mse)
        };

        let ptrs = {
            let _evt_guard = EventTrackingGuard::new(stream.context());
            CachedPtrs {
                // f32 weights — Adam/EMA write, GemmEx reads (no shadow)
                params_ptr: params_buf.raw_ptr(),
                target_ptr: target_params_buf.raw_ptr(),
                grad_buf: grad_buf.raw_ptr(),
                grad_norm_buf: grad_norm_dev_ptr,
                m_buf: m_buf.raw_ptr(),
                v_buf: v_buf.raw_ptr(),
                t_buf: t_dev_ptr,
                adaptive_clip_buf: adaptive_clip_dev_ptr,
                total_loss_buf: total_loss_dev_ptr,
                mse_loss_buf: mse_loss_dev_ptr,
                cql_grad_scratch: cql_grad_scratch.raw_ptr(),
                states_buf: states_buf.raw_ptr(),
                next_states_buf: next_states_buf.raw_ptr(),
                actions_buf: actions_buf.raw_ptr(),
                rewards_buf: rewards_buf.raw_ptr(),
                dones_buf: dones_buf.raw_ptr(),
                is_weights_buf: is_weights_buf.raw_ptr(),
                save_h_s1: save_h_s1.raw_ptr(),
                save_h_s2: save_h_s2.raw_ptr(),
                save_h_v: save_h_v.raw_ptr(),
                save_h_b0: save_h_b0.raw_ptr(),
                save_h_b1: save_h_b1.raw_ptr(),
                save_h_b2: save_h_b2.raw_ptr(),
                save_h_b3: save_h_b3.raw_ptr(),
                mag_concat_buf: mag_concat_buf.raw_ptr(),
                d_mag_concat_buf: d_mag_concat_buf.raw_ptr(),
                ord_concat_buf: ord_concat_buf.raw_ptr(),
                urg_concat_buf: urg_concat_buf.raw_ptr(),
                bw_d_h_s1: bw_d_h_s1.raw_ptr(),
                bw_d_h_s2: bw_d_h_s2.raw_ptr(),
                bw_d_h_v: bw_d_h_v.raw_ptr(),
                bw_d_h_b0: bw_d_h_b0.raw_ptr(),
                bw_d_h_b1: bw_d_h_b1.raw_ptr(),
                bw_d_h_b2: bw_d_h_b2.raw_ptr(),
                bw_d_h_b3: bw_d_h_b3.raw_ptr(),
                bw_d_glu_value: bw_d_glu_value.raw_ptr(),
                bw_d_glu_gate: bw_d_glu_gate.raw_ptr(),
                iqn_trunk_m: iqn_trunk_m.raw_ptr(),
                iqn_trunk_grad_norm: iqn_trunk_grad_norm.raw_ptr(),
                td_errors_buf: td_errors_buf.raw_ptr(),
                on_v_logits_buf: on_v_logits_buf.raw_ptr(),
                on_b_logits_buf: on_b_logits_buf.raw_ptr(),
                tg_h_s1_scratch: tg_h_s1_scratch.raw_ptr(),
                tg_h_s2_buf: tg_h_s2_buf.raw_ptr(),
                tg_h_v_scratch: tg_h_v_scratch.raw_ptr(),
                tg_h_b0_scratch: tg_h_b0_scratch.raw_ptr(),
                tg_h_b1_scratch: tg_h_b1_scratch.raw_ptr(),
                tg_h_b2_scratch: tg_h_b2_scratch.raw_ptr(),
                tg_h_b3_scratch: tg_h_b3_scratch.raw_ptr(),
                tg_v_logits_buf: tg_v_logits_buf.raw_ptr(),
                tg_b_logits_buf: tg_b_logits_buf.raw_ptr(),
                on_next_h_s1_scratch: on_next_h_s1_scratch.raw_ptr(),
                on_next_h_s2_scratch: on_next_h_s2_scratch.raw_ptr(),
                on_next_h_v_scratch: on_next_h_v_scratch.raw_ptr(),
                on_next_h_b_scratch: on_next_h_b_scratch.raw_ptr(),
                on_next_v_logits_buf: on_next_v_logits_buf.raw_ptr(),
                on_next_b_logits_buf: on_next_b_logits_buf.raw_ptr(),
            }
        };

        // #34 Causal intervention buffers (always enabled — one production path)
        let causal_market_dim = config.market_dim;
        let causal_wt = config.causal_weight as f32;
        let causal_intv = config.causal_intervention_interval;
        let pad_sd = ml_core::state_layout::STATE_DIM_PADDED; // pad128
        // Causal intervention buffers — ALWAYS allocated (one production path)
        let _total_actions = config.branch_0_size + config.branch_1_size + config.branch_2_size + config.branch_3_size;
        let causal_states_scratch = stream.alloc_zeros::<f32>(b * pad_sd)
            .map_err(|e| MLError::ModelError(format!("alloc causal_states: {e}")))?;
        let causal_sensitivity_buf = stream.alloc_zeros::<f32>(causal_market_dim)
            .map_err(|e| MLError::ModelError(format!("alloc causal_sens: {e}")))?;
        let causal_mean_scratch = stream.alloc_zeros::<f32>(1)
            .map_err(|e| MLError::ModelError(format!("alloc causal_mean_scratch: {e}")))?;
        let causal_q_scratch = stream.alloc_zeros::<f32>(b * _total_actions)
            .map_err(|e| MLError::ModelError(format!("alloc causal_q: {e}")))?;

        // Previous-step gradient snapshot for vaccine comparison
        let prev_grad_buf = stream.alloc_zeros::<f32>(total_params + cutlass_tile_pad)
            .map_err(|e| MLError::ModelError(format!("alloc prev_grad_buf: {e}")))?;

        // D1/N1: Distillation best-snapshot mirror — stable buffer whose
        // pointer is baked into the CUDA Graph. Initialized to zeros so the
        // kernel's `(params − best)` delta equals `params` until the first
        // real snapshot DtoD-overwrites the contents. The alpha term
        // (0.1 × (1 − health) × DISTILL_PER_STEP_SCALE) naturally scales
        // this initial pull down, and training_loop copies params into it
        // at startup so the first few epochs see a strictly zero pull.
        let mut distill_best_buf = stream.alloc_zeros::<f32>(total_params + cutlass_tile_pad)
            .map_err(|e| MLError::ModelError(format!("alloc distill_best_buf: {e}")))?;
        // Initialize contents = params_buf so initial (params − best) = 0
        // and distillation is a strict no-op until a real snapshot overwrites.
        {
            use cudarc::driver::{DevicePtr, DevicePtrMut};
            let num_bytes = (total_params + cutlass_tile_pad) * std::mem::size_of::<f32>();
            let (src_ptr, _sg) = params_buf.device_ptr(&stream);
            let (dst_ptr, _dg) = distill_best_buf.device_ptr_mut(&stream);
            #[allow(unsafe_code)]
            unsafe {
                cudarc::driver::result::memcpy_dtod_async(
                    dst_ptr, src_ptr, num_bytes, stream.cu_stream(),
                ).map_err(|e| MLError::ModelError(format!("distill_best_buf init dtod: {e}")))?;
            }
        }

        // #32 Gradient vaccine buffers
        let vaccine_grad_save = stream.alloc_zeros::<f32>(total_params + cutlass_tile_pad)
            .map_err(|e| MLError::ModelError(format!("alloc vaccine_grad_save: {e}")))?;
        let vaccine_block_results = stream.alloc_zeros::<f32>(grad_norm_blocks * 2)
            .map_err(|e| MLError::ModelError(format!("alloc vaccine_block_results: {e}")))?;
        let vaccine_dot_norm_buf = stream.alloc_zeros::<f32>(2)
            .map_err(|e| MLError::ModelError(format!("alloc vaccine_dot_norm: {e}")))?;

        let dd_weight = config.asymmetric_dd_weight;
        let ens_weight = config.ensemble_disagreement_penalty;
        let sd_prob = config.stochastic_depth_prob;
        let prune_ep = config.pruning_epoch;
        let prune_frac = config.pruning_fraction;
        let bn_dim = config.bottleneck_dim;
        let market_dim = config.market_dim;

        // #31 Bottleneck buffers (None when bottleneck_dim=0)
        // Bottleneck buffers — always allocated (bottleneck_dim=2 by default)
        let portfolio_dim = ml_core::state_layout::STATE_DIM.saturating_sub(market_dim);
        let bn_alloc_dim = bn_dim.max(1); // at least 1 to avoid zero-size alloc
        let bn_hidden_buf = stream.alloc_zeros::<f32>(b * bn_alloc_dim)
            .map_err(|e| MLError::ModelError(format!("alloc bn_hidden: {e}")))?;
        // Pad concat buffers to 128-element stride (CUTLASS K-tile alignment).
        // Unpadded strides cause cuBLAS heuristic to pick algorithms that hang
        // in CUDA Graph replay on Hopper (sm_90). pad128 forces tile-aligned
        // algorithms that are graph-compatible.
        //
        // SP15 Wave 4.1b: trailing +1 column carries dd_pct (read from
        // ISV[DD_PCT_INDEX=406] by `bn_tanh_concat_dd_kernel`). Concat layout
        // is [bn_tanh | portfolio | dd_pct], `concat_dim = bn_dim +
        // portfolio_dim + 1`. All four buffers (online / target / DDQN
        // forward outputs + the backward dX accumulator) share the same
        // stride — the dd_pct column is a structural part of the trunk
        // input now, mirrored across all forward passes.
        let concat_dim = bn_alloc_dim + portfolio_dim + 1;
        let bn_concat_buf = stream.alloc_zeros::<f32>(b * concat_dim + kt)
            .map_err(|e| MLError::ModelError(format!("alloc bn_concat: {e}")))?;
        // Target-net mirrors of bn_hidden / bn_concat — same shapes; populated
        // each forward by `submit_forward_ops_target` from `next_states_buf`
        // and the target weights `tg_w_ptrs[33]` / `tg_w_ptrs[34]`. Same
        // K-tile padding rationale as the online buffers.
        let tg_bn_hidden_buf = stream.alloc_zeros::<f32>(b * bn_alloc_dim)
            .map_err(|e| MLError::ModelError(format!("alloc tg_bn_hidden: {e}")))?;
        let tg_bn_concat_buf = stream.alloc_zeros::<f32>(b * concat_dim + kt)
            .map_err(|e| MLError::ModelError(format!("alloc tg_bn_concat: {e}")))?;
        // DDQN argmax-pass mirrors — populated by `submit_forward_ops_ddqn`
        // using online weights on `next_states_buf`.
        let on_next_bn_hidden_buf = stream.alloc_zeros::<f32>(b * bn_alloc_dim)
            .map_err(|e| MLError::ModelError(format!("alloc on_next_bn_hidden: {e}")))?;
        let on_next_bn_concat_buf = stream.alloc_zeros::<f32>(b * concat_dim + kt)
            .map_err(|e| MLError::ModelError(format!("alloc on_next_bn_concat: {e}")))?;
        let bn_d_concat_buf = stream.alloc_zeros::<f32>(b * concat_dim + kt)
            .map_err(|e| MLError::ModelError(format!("alloc bn_d_concat: {e}")))?;
        let bn_d_hidden_buf = stream.alloc_zeros::<f32>(b * bn_alloc_dim)
            .map_err(|e| MLError::ModelError(format!("alloc bn_d_hidden: {e}")))?;

        // ── Plan 4 Task 1B-iii: VSN feature-selection buffers ────────────
        // 3 gated-state buffers (online/target/ddqn), 3 mask buffers (online
        // saved + 2 throwaways), 3 logits buffers (online saved + 2 throwaways),
        // 6 per-group h1 saves (online — saved for 1B-iv backward), 1 shared
        // h1 inference scratch (target/ddqn — overwrite-per-group is fine),
        // 1 shared linear2 scratch [B] (overwrite per group across all passes),
        // 2 device-side group_begins/ends i32 buffers populated from
        // `FEATURE_GROUP_RANGES` at construction.
        let state_dim_padded = ml_core::state_layout::STATE_DIM_PADDED;
        let num_groups = ml_core::state_layout::SL_NUM_FEATURE_GROUPS;
        let vsn_hidden = VSN_HIDDEN_DIM;
        let vsn_gated_states_buf = stream
            .alloc_zeros::<f32>(b * state_dim_padded)
            .map_err(|e| MLError::ModelError(format!("alloc vsn_gated_states: {e}")))?;
        let vsn_gated_next_states_buf = stream
            .alloc_zeros::<f32>(b * state_dim_padded)
            .map_err(|e| MLError::ModelError(format!("alloc vsn_gated_next_states: {e}")))?;
        let vsn_gated_next_states_for_ddqn_buf = stream
            .alloc_zeros::<f32>(b * state_dim_padded)
            .map_err(|e| MLError::ModelError(format!("alloc vsn_gated_next_states_for_ddqn: {e}")))?;
        let vsn_logits_buf = stream
            .alloc_zeros::<f32>(b * num_groups)
            .map_err(|e| MLError::ModelError(format!("alloc vsn_logits: {e}")))?;
        let vsn_mask_buf = stream
            .alloc_zeros::<f32>(b * num_groups)
            .map_err(|e| MLError::ModelError(format!("alloc vsn_mask: {e}")))?;
        let vsn_logits_target_scratch = stream
            .alloc_zeros::<f32>(b * num_groups)
            .map_err(|e| MLError::ModelError(format!("alloc vsn_logits_target_scratch: {e}")))?;
        let vsn_mask_target_scratch = stream
            .alloc_zeros::<f32>(b * num_groups)
            .map_err(|e| MLError::ModelError(format!("alloc vsn_mask_target_scratch: {e}")))?;
        let vsn_logits_ddqn_scratch = stream
            .alloc_zeros::<f32>(b * num_groups)
            .map_err(|e| MLError::ModelError(format!("alloc vsn_logits_ddqn_scratch: {e}")))?;
        let vsn_mask_ddqn_scratch = stream
            .alloc_zeros::<f32>(b * num_groups)
            .map_err(|e| MLError::ModelError(format!("alloc vsn_mask_ddqn_scratch: {e}")))?;
        let vsn_save_h1_g0_buf = stream
            .alloc_zeros::<f32>(b * vsn_hidden)
            .map_err(|e| MLError::ModelError(format!("alloc vsn_save_h1_g0: {e}")))?;
        let vsn_save_h1_g1_buf = stream
            .alloc_zeros::<f32>(b * vsn_hidden)
            .map_err(|e| MLError::ModelError(format!("alloc vsn_save_h1_g1: {e}")))?;
        let vsn_save_h1_g2_buf = stream
            .alloc_zeros::<f32>(b * vsn_hidden)
            .map_err(|e| MLError::ModelError(format!("alloc vsn_save_h1_g2: {e}")))?;
        let vsn_save_h1_g3_buf = stream
            .alloc_zeros::<f32>(b * vsn_hidden)
            .map_err(|e| MLError::ModelError(format!("alloc vsn_save_h1_g3: {e}")))?;
        let vsn_save_h1_g4_buf = stream
            .alloc_zeros::<f32>(b * vsn_hidden)
            .map_err(|e| MLError::ModelError(format!("alloc vsn_save_h1_g4: {e}")))?;
        let vsn_save_h1_g5_buf = stream
            .alloc_zeros::<f32>(b * vsn_hidden)
            .map_err(|e| MLError::ModelError(format!("alloc vsn_save_h1_g5: {e}")))?;
        let vsn_h1_inference_scratch = stream
            .alloc_zeros::<f32>(b * vsn_hidden)
            .map_err(|e| MLError::ModelError(format!("alloc vsn_h1_inference_scratch: {e}")))?;
        let vsn_linear2_scratch_buf = stream
            .alloc_zeros::<f32>(b)
            .map_err(|e| MLError::ModelError(format!("alloc vsn_linear2_scratch: {e}")))?;
        let (vsn_group_begins_buf, vsn_group_ends_buf) = {
            let mut host_begins = [0i32; ml_core::state_layout::SL_NUM_FEATURE_GROUPS];
            let mut host_ends   = [0i32; ml_core::state_layout::SL_NUM_FEATURE_GROUPS];
            for g in 0..ml_core::state_layout::SL_NUM_FEATURE_GROUPS {
                let (gb, ge) = ml_core::state_layout::FEATURE_GROUP_RANGES[g];
                host_begins[g] = gb as i32;
                host_ends[g]   = ge as i32;
            }
            let begins = upload_via_mapped_i32(&stream, num_groups, &host_begins, "vsn_group_begins")?;
            let ends   = upload_via_mapped_i32(&stream, num_groups, &host_ends,   "vsn_group_ends")?;
            (begins, ends)
        };
        // Plan 4 Task 1B-iv: VSN backward scratch buffers. The backward chain
        // unrolls per-group `Linear_2[g]` then `Linear_1[g]` cuBLAS GEMMs
        // sequentially, so each per-group dY/dX scratch is reused across the
        // 6 groups. The full `[B, num_groups]` `d_logits` is kept because the
        // softmax-and-gate backward kernel writes it as a single block; per-
        // group Linear_2 backward then gathers column `g` into the tight
        // `vsn_d_logit_scratch_buf`. The assembled `d_vsn_gated_state` is
        // built in `vsn_d_gated_state_buf` before `vsn_backward` consumes it.
        let vsn_d_logits_buf = stream
            .alloc_zeros::<f32>(b * num_groups)
            .map_err(|e| MLError::ModelError(format!("alloc vsn_d_logits: {e}")))?;
        let vsn_d_state_buf = stream
            .alloc_zeros::<f32>(b * state_dim_padded)
            .map_err(|e| MLError::ModelError(format!("alloc vsn_d_state: {e}")))?;
        let vsn_d_gated_state_buf = stream
            .alloc_zeros::<f32>(b * state_dim_padded)
            .map_err(|e| MLError::ModelError(format!("alloc vsn_d_gated_state: {e}")))?;
        let vsn_d_logit_scratch_buf = stream
            .alloc_zeros::<f32>(b)
            .map_err(|e| MLError::ModelError(format!("alloc vsn_d_logit_scratch: {e}")))?;
        let vsn_d_h1_scratch_buf = stream
            .alloc_zeros::<f32>(b * vsn_hidden)
            .map_err(|e| MLError::ModelError(format!("alloc vsn_d_h1_scratch: {e}")))?;
        // Plan 4 Task 1B-iv-ext: per-aux VSN dW scratches. The IQN-trunk and
        // ensemble-diversity auxiliary backward paths each need to write the
        // 24 VSN param-tensor gradients somewhere isolated from `grad_buf`
        // (so the SAXPY-with-scale pattern can scale only the auxiliary
        // contribution). `iqn_trunk_m` is sized to the 13 trunk tensors
        // [0..13) and lives at offset 0 — extending it to also cover the
        // VSN range [95..119) would leave 82 non-VSN tensors of dead bytes
        // between trunk and VSN, defeating the contiguous-SAXPY pattern.
        // Instead we allocate two scratches sized to exactly the padded-VSN
        // range and anchor them at byte offset `padded_byte_offset(95)`
        // (passing `grad_base = scratch.raw_ptr() - padded_byte_offset(95)`
        // makes per-tensor `padded_byte_offset(idx)` writes land at the
        // right local offset). The CQL aux path doesn't need its own scratch
        // because `cql_grad_scratch` is already sized `total_params` —
        // VSN dW writes at offsets [95..119) into `cql_grad_scratch` are
        // picked up by the existing `apply_cql_saxpy`'s full-buffer SAXPY.
        let vsn_aux_scratch_floats: usize = {
            let __vsn_param_sizes = compute_param_sizes(&config);
            let begin_byte = padded_byte_offset(&__vsn_param_sizes, 95) as usize;
            let end_byte   = padded_byte_offset(&__vsn_param_sizes, 119) as usize;
            (end_byte - begin_byte) / std::mem::size_of::<f32>()
        };
        let vsn_dw_iqn_aux_scratch = stream
            .alloc_zeros::<f32>(vsn_aux_scratch_floats)
            .map_err(|e| MLError::ModelError(format!("alloc vsn_dw_iqn_aux_scratch: {e}")))?;
        let vsn_dw_ensemble_aux_scratch = stream
            .alloc_zeros::<f32>(vsn_aux_scratch_floats)
            .map_err(|e| MLError::ModelError(format!("alloc vsn_dw_ensemble_aux_scratch: {e}")))?;
        // Plan 4 Task 1B-iv: load the column-gather kernel from
        // experience_kernels.cu (added next to strided_scatter). One thread
        // per (sample, output-column) pair extracts column `g` of
        // d_logits[B, num_groups] into a contiguous [B, 1] scratch consumed
        // by per-group cuBLAS Linear_2 backward.
        let vsn_logit_gather_kernel = exp_module_for_mag.load_function("strided_gather")
            .map_err(|e| MLError::ModelError(format!("strided_gather load: {e}")))?;
        // Plan 4 Task 1B-iv: load the portfolio-passthrough + pad kernel from
        // dqn_utility_kernels.cu (added next to bn_tanh_backward_kernel).
        // Populates `vsn_d_gated_state_buf[:, market_dim..STATE_DIM_PADDED]`
        // from `bn_d_concat_buf[:, bn_dim..]`; market slice is zeroed and
        // then overwritten by `launch_dx_only_lda` (bottleneck Linear bwd).
        let vsn_d_gated_state_portfolio_kernel = {
            let module = stream.context().load_cubin(DQN_UTILITY_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("dqn_util cubin for vsn_portfolio_pad: {e}")))?;
            module.load_function("vsn_d_gated_state_portfolio_pad_kernel")
                .map_err(|e| MLError::ModelError(format!("vsn_d_gated_state_portfolio_pad_kernel load: {e}")))?
        };
        // Plan 4 Task 1B-i / 1B-iii: load the VSN per-group mask EMA producer
        // kernel. Single-block 256-thread shmem-reduction kernel reads
        // `vsn_mask_buf` and EMA-updates ISV[VSN_MASK_GROUP_0_EMA..6).
        // Cold-path-cadence (one launch per training step alongside
        // `launch_h_s2_rms_ema`). No atomicAdd, no DtoH.
        let vsn_mask_ema_kernel = {
            let module = stream.context().load_cubin(VSN_MASK_EMA_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("vsn_mask_ema cubin load: {e}")))?;
            module.load_function("vsn_mask_ema_update")
                .map_err(|e| MLError::ModelError(format!("vsn_mask_ema_update load: {e}")))?
        };

        // ── HEALTH_DIAG GPU port — Phase 2A orchestrator construction ────────
        // Loads `health_diag_kernel.cubin`, allocates the mapped-pinned
        // `HealthDiagSnapshot`, and resolves the `health_diag_isv_mirror`
        // kernel handle. Phases 2B-2E append further kernel symbols to
        // the same orchestrator. Phase 3 wires the launch into the
        // captured graph; Phase 4 deletes the CPU-side reductions.
        // Construction must run after the CUDA context is active — same
        // as the other mapped-pinned producers in this constructor.
        // Safety: caller (constructor) holds an active CUDA context — the
        // very allocations above this line use it via `alloc_f32`,
        // `MappedF32Buffer::new`, etc.
        let health_diag = Some(unsafe { GpuHealthDiag::new(&stream)? });

        // ── Plan 4 Task 6 Commit B: aux-heads orchestrator + buffers ─────────
        // Construct the forward + backward orchestrators (loads 11 kernel
        // handles total from `AUX_HEADS_CUBIN` + `AUX_HEADS_LOSS_EMA_CUBIN`).
        // Buffers are sized off the runtime config (B, SH2, AUX_HIDDEN_DIM=32,
        // AUX_REGIME_K=5, AUX_NEXT_BAR_K=1) and zero-init'd so cold-start
        // launches are well-defined.
        let aux_heads_fwd = AuxHeadsForwardOps::new(&stream)?;
        let aux_heads_bwd = AuxHeadsBackwardOps::new(&stream)?;
        let aux_b   = b;
        let aux_sh2 = config.shared_h2;
        let aux_h   = AUX_HIDDEN_DIM;
        let aux_kr  = AUX_REGIME_K;
        let aux_knb = AUX_NEXT_BAR_K;
        let aux_nb_hidden_buf       = alloc_f32(&stream, aux_b * aux_h, "aux_nb_hidden_buf")?;
        // SP13 B1.1a (2026-05-05): next-bar head K_out flipped 1 → 2.
        // - `aux_nb_logits_buf` (renamed from `aux_nb_pred_buf`) holds the
        //   `[B, K]` logits saved by forward.
        // - `aux_nb_softmax_buf` (NEW) holds the `[B, K]` softmax tile
        //   read by 3 consumers (loss, backward, dir-acc, isv-tanh).
        // - `aux_nb_label_buf` flipped f32 → i32; producer wires in B1.1b
        //   so the buffer stays at `alloc_zeros` value (label 0 = "down")
        //   until then.
        // - `aux_nb_valid_count_buf` (NEW) carries `B_valid` from loss
        //   reduce to backward so they share the same `1/B_valid` divisor.
        let aux_nb_logits_buf       = alloc_f32(&stream, aux_b * aux_knb, "aux_nb_logits_buf")?;
        let aux_nb_softmax_buf      = alloc_f32(&stream, aux_b * aux_knb, "aux_nb_softmax_buf")?;
        let aux_nb_label_buf        = stream.alloc_zeros::<i32>(aux_b)
            .map_err(|e| MLError::ModelError(format!("alloc aux_nb_label_buf (B1.1a i32 flip): {e}")))?;
        let aux_nb_loss_scalar_buf  = alloc_f32(&stream, 1, "aux_nb_loss_scalar_buf")?;
        let aux_nb_valid_count_buf  = alloc_f32(&stream, 1, "aux_nb_valid_count_buf")?;
        let aux_rg_hidden_buf       = alloc_f32(&stream, aux_b * aux_h, "aux_rg_hidden_buf")?;
        let aux_rg_logits_buf       = alloc_f32(&stream, aux_b * aux_kr, "aux_rg_logits_buf")?;
        let aux_rg_label_buf        = stream.alloc_zeros::<u8>(aux_b)
            .map_err(|e| MLError::ModelError(format!("alloc aux_rg_label_buf: {e}")))?;
        let aux_rg_loss_scalar_buf  = alloc_f32(&stream, 1, "aux_rg_loss_scalar_buf")?;
        let aux_rg_correct_scalar_buf = alloc_f32(&stream, 1, "aux_rg_correct_scalar_buf")?;
        let aux_dh_s2_nb_buf        = alloc_f32(&stream, aux_b * aux_sh2, "aux_dh_s2_nb_buf")?;
        let aux_dh_s2_rg_buf        = alloc_f32(&stream, aux_b * aux_sh2, "aux_dh_s2_rg_buf")?;
        let aux_partial_nb_w1       = alloc_f32(&stream, aux_b * aux_h * aux_sh2, "aux_partial_nb_w1")?;
        let aux_partial_nb_b1       = alloc_f32(&stream, aux_b * aux_h, "aux_partial_nb_b1")?;
        // SP13 B1.1a: nb_w2 partial grew [B, H] → [B, K, H]; nb_b2 partial
        // grew [B] → [B, K]. Mirrors regime-head shapes with K_NB=2.
        let aux_partial_nb_w2       = alloc_f32(&stream, aux_b * aux_knb * aux_h, "aux_partial_nb_w2")?;
        let aux_partial_nb_b2       = alloc_f32(&stream, aux_b * aux_knb, "aux_partial_nb_b2")?;
        let aux_partial_rg_w1       = alloc_f32(&stream, aux_b * aux_h * aux_sh2, "aux_partial_rg_w1")?;
        let aux_partial_rg_b1       = alloc_f32(&stream, aux_b * aux_h, "aux_partial_rg_b1")?;
        let aux_partial_rg_w2       = alloc_f32(&stream, aux_b * aux_kr * aux_h, "aux_partial_rg_w2")?;
        let aux_partial_rg_b2       = alloc_f32(&stream, aux_b * aux_kr, "aux_partial_rg_b2")?;
        // SP13 B1.1a: include the K_NB-scaled tensor sizes in the max.
        // Both `aux_knb * aux_h` and `aux_knb` need coverage now.
        let max_aux_tensor_len = (aux_h * aux_sh2)
            .max(aux_kr * aux_h)
            .max(aux_knb * aux_h)
            .max(aux_h)
            .max(aux_kr)
            .max(aux_knb);
        let aux_param_grad_final_buf = alloc_f32(&stream, max_aux_tensor_len, "aux_param_grad_final_buf")?;

        // ── SP22 H6 vNext Phase B0+B2 (2026-05-14): trade-outcome head ──
        // orchestrator (load both fwd + bwd) + 11 buffers (mirror of
        // aux_nb_* pattern at K=3 instead of K=2).
        //
        // `aux_to_fwd` holds 3 kernel handles (forward, loss_reduce,
        // label producer); `aux_to_bwd` holds the backward kernel —
        // param_grad_reduce is reused from `aux_heads_bwd`.
        //
        // Buffer sizes (B=batch_size, H=128, K=3, SH2=256):
        //   aux_to_hidden_buf       [B, H]              = B × 128
        //   aux_to_logits_buf       [B, K=3]            = B × 3
        //   aux_to_softmax_buf      [B, K=3]            = B × 3
        //   aux_to_label_buf        [B] i32
        //   aux_to_loss_scalar_buf  [1]
        //   aux_to_valid_count_buf  [1]
        //   aux_dh_s2_to_buf        [B, SH2]            = B × 256
        //   aux_partial_to_w1       [B, H, SH2]         = B × 32,768 (largest)
        //   aux_partial_to_b1       [B, H]              = B × 128
        //   aux_partial_to_w2       [B, K=3, H]         = B × 384
        //   aux_partial_to_b2       [B, K=3]            = B × 3
        //
        // At B=batch_size=2048 and SH2=256: aux_partial_to_w1 dominates
        // at 2048 × 32,768 × 4 bytes = 256 MB. Matches the K=2 head's
        // partial size (same SH2, same H) — no new memory pressure.
        let aux_to_fwd = AuxTradeOutcomeForwardOps::new(&stream)?;
        let aux_to_bwd = AuxTradeOutcomeBackwardOps::new(&stream)?;
        let aux_kto = AUX_OUTCOME_K;
        let aux_to_hidden_buf       = alloc_f32(&stream, aux_b * aux_h, "aux_to_hidden_buf")?;
        let aux_to_logits_buf       = alloc_f32(&stream, aux_b * aux_kto, "aux_to_logits_buf")?;
        let aux_to_softmax_buf      = alloc_f32(&stream, aux_b * aux_kto, "aux_to_softmax_buf")?;
        let aux_to_label_buf        = stream.alloc_zeros::<i32>(aux_b)
            .map_err(|e| MLError::ModelError(format!(
                "alloc aux_to_label_buf (sp22-vnext B2 i32): {e}"
            )))?;
        let aux_to_loss_scalar_buf  = alloc_f32(&stream, 1, "aux_to_loss_scalar_buf")?;
        let aux_to_valid_count_buf  = alloc_f32(&stream, 1, "aux_to_valid_count_buf")?;
        let aux_dh_s2_to_buf        = alloc_f32(&stream, aux_b * aux_sh2, "aux_dh_s2_to_buf")?;
        let aux_partial_to_w1       = alloc_f32(&stream, aux_b * aux_h * aux_sh2, "aux_partial_to_w1")?;
        let aux_partial_to_b1       = alloc_f32(&stream, aux_b * aux_h, "aux_partial_to_b1")?;
        let aux_partial_to_w2       = alloc_f32(&stream, aux_b * aux_kto * aux_h, "aux_partial_to_w2")?;
        let aux_partial_to_b2       = alloc_f32(&stream, aux_b * aux_kto, "aux_partial_to_b2")?;
        // NOTE: aux_param_grad_final_buf is sized to handle the largest
        // tensor across BOTH K=2 (next_bar) and K=3 (trade_outcome)
        // heads — both heads' largest tensor is W1 [H, SH2] = 32,768
        // floats, identical. No re-sizing needed.
        // SP20 Phase 5: per-sample aux_conf at sampled state ([B] f32). PER's
        // direct-to-trainer `gather_f32_scalar` writes here on every sample
        // step once `set_trainer_aux_conf_ptr` is wired; until then, the
        // alloc_zeros 0.0 sentinel drives the c51_loss reward-gate to ~0
        // (graceful degradation per Phase 3 Task 3.4 audit doc spec §4.4).
        let aux_conf_at_state_buf  = alloc_f32(&stream, aux_b, "aux_conf_at_state_buf")?;
        // Initial aux_weight at the lower numerical-stability clamp (0.05).
        // training_loop's per-step `set_aux_weight` recomputes from ISV every
        // step. The first graph capture bakes 0.05 — an honest cold-start
        // value, not a stub.
        let aux_weight: f32 = 0.05;

        // ── Phase 3: MoE forward/backward orchestrator + working buffers ─────
        // GpuMoeHead loads `moe_kernels.cubin` and caches 8 kernel handles.
        // All working buffers are zero-initialised — the gate starts at zero
        // weights (uniform 1/K softmax), so cold-start EMA values are valid.
        let moe_head = GpuMoeHead::new(Arc::clone(&stream))?;
        let moe_sh2 = config.shared_h2;
        let moe_gate_h1_buf    = alloc_f32(&stream, b * MOE_GATE_HIDDEN, "moe_gate_h1_buf")?;
        let moe_gate_pre_buf   = alloc_f32(&stream, b * MOE_NUM_EXPERTS,  "moe_gate_pre_buf")?;
        let moe_gate_softmax_buf = alloc_f32(&stream, b * MOE_NUM_EXPERTS, "moe_gate_softmax_buf")?;
        // Allocate 8 expert hidden buffers (one per expert, each [B, BTN]).
        let moe_expert_h1_bufs = {
            let mut arr: [std::mem::MaybeUninit<CudaSlice<f32>>; MOE_NUM_EXPERTS] =
                unsafe { std::mem::MaybeUninit::uninit().assume_init() };
            for (ek, slot) in arr.iter_mut().enumerate() {
                let buf = alloc_f32(&stream, b * MOE_EXPERT_BOTTLENECK,
                    &format!("moe_expert_h1_{ek}"))?;
                slot.write(buf);
            }
            unsafe { std::mem::transmute::<_, [CudaSlice<f32>; MOE_NUM_EXPERTS]>(arr) }
        };
        let moe_expert_outputs_buf = alloc_f32(&stream, MOE_NUM_EXPERTS * b * moe_sh2, "moe_expert_outputs_buf")?;
        let moe_de_k_buf           = alloc_f32(&stream, MOE_NUM_EXPERTS * b * moe_sh2, "moe_de_k_buf")?;
        let moe_dg_buf             = alloc_f32(&stream, b * MOE_NUM_EXPERTS, "moe_dg_buf")?;
        let moe_dg_pre_buf         = alloc_f32(&stream, b * MOE_NUM_EXPERTS, "moe_dg_pre_buf")?;
        let moe_dh_s1_scratch      = alloc_f32(&stream, b * moe_sh2, "moe_dh_s1_scratch")?;
        let moe_gate_dh1_buf       = alloc_f32(&stream, b * MOE_GATE_HIDDEN, "moe_gate_dh1_buf")?;
        let moe_load_balance_loss_per_k = alloc_f32(&stream, MOE_NUM_EXPERTS, "moe_load_balance_loss_per_k")?;
        let (moe_load_balance_loss_total_pinned, moe_load_balance_loss_total_dev_ptr) = {
            let mut host_ptr: *mut std::ffi::c_void = std::ptr::null_mut();
            let mut dev_ptr_out: u64 = 0;
            unsafe {
                let rc = cudarc::driver::sys::cuMemAllocHost_v2(
                    &mut host_ptr as *mut *mut std::ffi::c_void,
                    std::mem::size_of::<f32>(),
                );
                if rc != cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS {
                    return Err(MLError::ModelError(format!("cuMemAllocHost moe_lb_total: {:?}", rc)));
                }
                let rc2 = cudarc::driver::sys::cuMemHostGetDevicePointer_v2(
                    &mut dev_ptr_out,
                    host_ptr,
                    0,
                );
                if rc2 != cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS {
                    return Err(MLError::ModelError(format!("cuMemHostGetDevicePointer moe_lb_total: {:?}", rc2)));
                }
                *(host_ptr as *mut f32) = 0.0f32;
            }
            (host_ptr as *mut f32, dev_ptr_out)
        };
        let moe_lambda_floor = config.moe_lambda_floor;
        let moe_lambda_max_extra = config.moe_lambda_max_extra;
        let moe_entropy_target_frac = config.moe_entropy_target_frac;
        tracing::info!(
            "GpuMoeHead initialized: K={} BTN={} GH={} SH2={} lambda_floor={:.4} lambda_max_extra={:.4} entropy_target_frac={:.3} (params [127..163), ISV[118..127), λ_eff ISV[128])",
            MOE_NUM_EXPERTS, MOE_EXPERT_BOTTLENECK, MOE_GATE_HIDDEN, moe_sh2,
            moe_lambda_floor, moe_lambda_max_extra, moe_entropy_target_frac,
        );

        tracing::info!(
            "AuxHeadsForwardOps initialized: K_nb={} K_rg={} aux_h={} max_aux_tensor_len={} (params [119..127), ISV[113..115))",
            aux_knb, aux_kr, aux_h, max_aux_tensor_len,
        );

        // SP1 Phase B: expanded 24 → 48 to make room for backward-path NaN
        // checks (slots 24-35) plus 12 reserved headroom slots (36-47) for
        // SP2 framework checker hooks + SP3 structural observers. SP3 close-
        // out v2 (slot 48): one additional slot for IQN online_params (the
        // separate weight buffer that drives the slot-26 cuBLAS GEMM and was
        // the F1 step-3540 NaN root). SP3 Mech 10 (slot 49): one additional
        // slot for post-trunk h_s2 activation max-abs sentinel — fires at
        // 1e3 × H_S2_RMS_EMA.max(1.0) and stays quiet under normal Mech 10
        // clamp operation (clamp at 100×, diagnostic at 1000×). SP4 Task
        // A14 (slots 50-54): per-Adam-kernel sticky engagement-overflow
        // flags (one per of dqn/iqn/iql/attn/curiosity Adam kernels).
        // Sized via SP4_NAN_FLAGS_END = 55. Buffer size reviewable by SP2
        // framework codification.
        let nan_flags_buf = stream.alloc_zeros::<i32>(SP4_NAN_FLAGS_END)
            .map_err(|e| MLError::ModelError(format!("nan_flags alloc: {e}")))?;

        // SP2 + SP3 Mech 5 + Mech 10: fused NaN/threshold-check (ptr, len) tables.
        // 26 entries each — slots 0-11 (= absolute flag idx 24-35) NaN-only
        // (SP2 Task A3) + slots 12-23 (= absolute 36-47) ISV-threshold
        // (SP3 Mech 5 Task B6) + slot 24 (= absolute 48) IQN online_params
        // (SP3 close-out v2) + slot 25 (= absolute 49) h_s2 activation
        // sentinel (SP3 Mech 10). Allocated as mapped-pinned
        // (`cuMemHostAlloc(DEVICEMAP|PORTABLE)`) per
        // `feedback_no_htod_htoh_only_mapped_pinned` — host-side writes are
        // visible to the kernel through the device-mapped pointer with zero
        // HtoD copies. Populated once via `populate_nan_check_meta` after all
        // backward-path buffers are constructed AND `gpu_iqn` is known. The
        // kernel reads `buf_ptrs[blockIdx.x]` and `buf_lens[blockIdx.x]` per
        // block; per-slot threshold is computed inline from two ISV-derived
        // launch args (`q_abs_ref_eff` for slots 12-24, `h_s2_rms_ema_eff`
        // for slot 25 — distinct ISV bases, no per-slot threshold buffer).
        //
        // Safety: a CUDA context is active on this thread (the GpuDqnTrainer
        // constructor runs within `FusedTrainingCtx::new` which has already
        // initialised CUDA via the shared cublas handle).
        let nan_check_buf_ptrs = unsafe { MappedU64Buffer::new(26) }
            .map_err(|e| MLError::ModelError(format!("nan_check_buf_ptrs alloc: {e}")))?;
        let nan_check_buf_lens = unsafe { MappedI32Buffer::new(26) }
            .map_err(|e| MLError::ModelError(format!("nan_check_buf_lens alloc: {e}")))?;

        // SP4 Layer A Task A2: three mapped-pinned buffers reserved for the
        // upcoming Pearls B/C/D wiring (Tasks A5-A18). Allocated now so the
        // struct layout is stable; no consumers read them yet — behaviour is
        // unchanged from before this allocation. Each `MappedF32Buffer::new` /
        // `MappedI32Buffer::new` zero-fills its host backing via
        // `ptr::write_bytes` at construction (acts as the Pearl A sentinel —
        // first-observation replacement on first producer launch). Per-fold
        // re-zero is added by Task A12 via the state-reset registry.
        // Per `feedback_no_htod_htoh_only_mapped_pinned`: mapped pinned
        // (`cuMemHostAlloc(DEVICEMAP|PORTABLE)`) is the only allowed CPU↔GPU
        // path for these state buffers.
        //
        // SP4_PRODUCER_COUNT / SP4_WIENER_FLOATS_PER_SLOT / SP4_WIENER_TOTAL_FLOATS
        // are module-level `pub(crate) const`s declared at the top of this
        // file so launch sites + tests share the single source of truth.
        // SP4 Task A14: SP4_PARAM_GROUP_COUNT, MAX_BLOCKS_PER_ADAM, and
        // SP4_ENGAGE_BUF_LEN are public constants in `sp4_isv_slots`.

        // SP5 Task A1 (2026-05-01): wiener_state_buf grows from SP4_WIENER_TOTAL_FLOATS (213)
        // to SP5_WIENER_TOTAL_FLOATS, sized via `SP5_PRODUCER_COUNT` (the wiener-buffer
        // linear-span constant — see SP5_PRODUCER_COUNT docstring for the linear-span
        // vs unique-slot distinction; Layer D D1 grew it 110→116, D2 grows it 116→120
        // so the buffer expanded 543→573 floats post-D2). All-zeros at allocation
        // (Pearl A sentinel contract).
        let wiener_state_buf = unsafe { MappedF32Buffer::new(SP5_WIENER_TOTAL_FLOATS) }
            .map_err(|e| MLError::ModelError(format!("SP4/SP5 wiener_state_buf alloc: {e}")))?;

        let clamp_engage_per_block_buf = unsafe { MappedI32Buffer::new(SP4_ENGAGE_BUF_LEN) }
            .map_err(|e| MLError::ModelError(format!("SP4 clamp_engage_per_block_buf alloc: {e}")))?;

        // SP5 (2026-05-01): producer_step_scratch_buf sized for SP4 + SP5 Layer A producer outputs.
        // Layout (each producer commit appends its output range; see audit doc for derivation):
        //   [0..71)    SP4 producers
        //   [71..87)   q_branch_stats         (Pearl 1 / A1 — shared signal source)
        //   [87..103)  pearl_1_atom_update    (Pearl 1 / A1)
        //   [103..107) pearl_3_sigma σ        (Pearl 3 / A2)
        //   [107..111) pearl_3_sigma SF       (Pearl 3 / A2)
        //   [111..131) pearl_2_budget         (Pearl 2 / A3 — iqn/flatness; SP7 retired c51/cql/ens)
        //   [131..147) pearl_4 cosine + l2    (auxiliary grad_cosine_sim, A4)
        //   [147..171) pearl_4 β1/β2/ε        (Pearl 4 / A4)
        //   [171..199) pearl_5 skew/ex_kurt + IQN τ schedule (A5)
        //   Pearl 6 (A6) writes directly to ISV — no scratch.
        //   [199..203) pearl_8 trail_dist     (Pearl 8 / A7)
        //   [203..207) pearl_1_ext num_atoms  (Pearl 1-ext / A8)
        //   [207..211) pnl_aggregation        (Layer D D1)
        //   [211..215) health_composition     (Layer D D2)
        //   [215..218) training_metrics_ema   (Layer D D3)
        //   [218..242) loss_balance_ctrl      (SP7 Task 5 — budget_cql/c51 + 4 Wiener vars)
        //   [242..250) loss_balance_active    (SP7 activation-flag fix — CQL/C51 active flags)
        //   [250..251) train_active_frac      (SP8 Fix 36 — canary scratch slot)
        //   [251..259) lb_max_budget_*        (SP8 Fix 36 — per-(head, branch) cap)
        //   [259..262) sp9 kelly + q_var_mag + intent_eval (SP9 Fix 37)
        //   [262..265) sp9 eval_dist Q/H/F    (SP9 Fix 37)
        //   [265..266) sp10 thompson temp     (SP10 Fix 38)
        //   [266..268) sp11 val_sharpe_delta + var (SP11 Fix 39 A1.1)
        //   [268..269) sp11 saboteur_engagement (SP11 Fix 39 A1.2)
        //   [269..276) sp11 mag_ratio (6 ratios + popart mirror) (SP11 Fix 39 A1.3)
        // Total: SP5_SCRATCH_TOTAL = 276. Audit doc records every commit's growth.
        let producer_step_scratch_buf = unsafe { MappedF32Buffer::new(SP5_SCRATCH_TOTAL) }
            .map_err(|e| MLError::ModelError(format!("SP4/SP5 producer_step_scratch_buf alloc: {e}")))?;

        // SP4 Task A15: dedicated Pearl C rate-deficit Wiener-state
        // buffer (8 groups × 3 floats: sample_var, diff_var, x_lag).
        // Separate from `wiener_state_buf` to keep SP4-bound producer
        // state isolated from Pearl C diagnostic state — clean reset
        // semantics + independent fold-boundary handling.
        const PEARL_C_RATE_DEFICIT_FLOATS: usize = SP4_PARAM_GROUP_COUNT * SP4_WIENER_FLOATS_PER_SLOT;
        let pearl_c_rate_deficit_state_buf =
            unsafe { MappedF32Buffer::new(PEARL_C_RATE_DEFICIT_FLOATS) }
                .map_err(|e| MLError::ModelError(format!(
                    "SP4 pearl_c_rate_deficit_state_buf alloc: {e}"
                )))?;

        // SP4 Task A7 fix-up #2: oracle sub-buffer descriptor tables.
        // `SP4_ORACLE_TABLE_MAX_SUB = 4` covers Curiosity's 4 sub-buffers
        // (groups 0-6 use only the first slot). Tables are reused across
        // all 8 per-step group launches — host overwrites entries [0..n_sub)
        // before each launch; the kernel reads them via the device pointer.
        // Allocated once at construction so per-step allocator pressure is
        // avoided. Shared across all launches because the launcher serialises
        // them on the same stream (the `__threadfence_system()` per pass +
        // single end-of-loop sync makes the table layout invisible to a
        // concurrent reader).
        const SP4_ORACLE_TABLE_MAX_SUB: usize = 4;
        let oracle_subbuf_table_buf =
            unsafe { MappedU64Buffer::new(4 * SP4_ORACLE_TABLE_MAX_SUB) }
                .map_err(|e| MLError::ModelError(format!(
                    "SP4 oracle_subbuf_table_buf alloc: {e}"
                )))?;
        let oracle_subbuf_counts_buf =
            unsafe { MappedI32Buffer::new(SP4_ORACLE_TABLE_MAX_SUB) }
                .map_err(|e| MLError::ModelError(format!(
                    "SP4 oracle_subbuf_counts_buf alloc: {e}"
                )))?;

        // SP4 Layer C #260 follow-up (2026-05-01): dedicated 2-entry
        // sub-buffer descriptor tables for `launch_sp4_q_dir_grad_p99`.
        // Allocated separately from `oracle_subbuf_table_buf` to eliminate
        // the implicit "must run before param_group_oracle" temporal
        // coupling — each launcher owns its own descriptor tables.
        // n_sub is fixed at 2 (`d_value_logits` ∪ `d_adv_logits`).
        const Q_DIR_GRAD_SUBBUF_COUNT: usize = 2;
        let q_dir_grad_subbuf_table_buf =
            unsafe { MappedU64Buffer::new(Q_DIR_GRAD_SUBBUF_COUNT) }
                .map_err(|e| MLError::ModelError(format!(
                    "SP4 q_dir_grad_subbuf_table_buf alloc: {e}"
                )))?;
        let q_dir_grad_subbuf_counts_buf =
            unsafe { MappedI32Buffer::new(Q_DIR_GRAD_SUBBUF_COUNT) }
                .map_err(|e| MLError::ModelError(format!(
                    "SP4 q_dir_grad_subbuf_counts_buf alloc: {e}"
                )))?;

        // v8: PopArt running statistics buffers
        let popart_mean = stream.alloc_zeros::<f32>(1)
            .map_err(|e| MLError::ModelError(format!("alloc popart_mean: {e}")))?;
        let popart_var = stream.alloc_zeros::<f32>(1)
            .map_err(|e| MLError::ModelError(format!("alloc popart_var: {e}")))?;
        let popart_count = stream.alloc_zeros::<f32>(1)
            .map_err(|e| MLError::ModelError(format!("alloc popart_count: {e}")))?;

        // ── Q-mean drift correction kernels (Component 8) ────────────────
        let cpbi_module_qmean = stream.context().load_cubin(EXPECTED_Q_CUBIN.to_vec())
            .map_err(|e| MLError::ModelError(format!("cpbi cubin (qmean): {e}")))?;
        let q_mean_reduce_kernel = cpbi_module_qmean.load_function("q_mean_reduce")
            .map_err(|e| MLError::ModelError(format!("q_mean_reduce load: {e}")))?;
        let q_mean_subtract_kernel = cpbi_module_qmean.load_function("q_mean_subtract")
            .map_err(|e| MLError::ModelError(format!("q_mean_subtract load: {e}")))?;
        let (q_mean_scratch_pinned, q_mean_scratch_dev_ptr) = {
            let mut host_ptr: *mut std::ffi::c_void = std::ptr::null_mut();
            let mut dev_ptr_out: u64 = 0;
            unsafe {
                let rc = cudarc::driver::sys::cuMemAllocHost_v2(
                    &mut host_ptr,
                    std::mem::size_of::<f32>(),
                );
                assert_eq!(rc, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemAllocHost for q_mean_scratch");
                let rc2 = cudarc::driver::sys::cuMemHostGetDevicePointer_v2(
                    &mut dev_ptr_out,
                    host_ptr,
                    0,
                );
                assert_eq!(rc2, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemHostGetDevicePointer for q_mean_scratch");
                *(host_ptr as *mut f32) = 0.0;
            }
            (host_ptr as *mut f32, dev_ptr_out)
        };

        let (q_mean_ema_pinned, q_mean_ema_dev_ptr) = {
            let mut host_ptr: *mut std::ffi::c_void = std::ptr::null_mut();
            let mut dev_ptr_out: u64 = 0;
            unsafe {
                let rc = cudarc::driver::sys::cuMemAllocHost_v2(
                    &mut host_ptr,
                    std::mem::size_of::<f32>(),
                );
                assert_eq!(rc, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemAllocHost for q_mean_ema");
                let rc2 = cudarc::driver::sys::cuMemHostGetDevicePointer_v2(
                    &mut dev_ptr_out,
                    host_ptr,
                    0,
                );
                assert_eq!(rc2, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemHostGetDevicePointer for q_mean_ema");
                *(host_ptr as *mut f32) = 0.0;
            }
            (host_ptr as *mut f32, dev_ptr_out)
        };

        // ── ISV scratch scalars (pinned device-mapped) ────────────────────
        let (td_error_scratch_pinned, td_error_scratch_dev_ptr) = {
            let mut host_ptr: *mut std::ffi::c_void = std::ptr::null_mut();
            let mut dev_ptr_out: u64 = 0;
            unsafe {
                let rc = cudarc::driver::sys::cuMemAllocHost_v2(
                    &mut host_ptr,
                    std::mem::size_of::<f32>(),
                );
                assert_eq!(rc, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemAllocHost for td_error_scratch");
                let rc2 = cudarc::driver::sys::cuMemHostGetDevicePointer_v2(
                    &mut dev_ptr_out,
                    host_ptr,
                    0,
                );
                assert_eq!(rc2, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemHostGetDevicePointer for td_error_scratch");
                *(host_ptr as *mut f32) = 0.0;
            }
            (host_ptr as *mut f32, dev_ptr_out)
        };

        let (q_var_scratch_pinned, q_var_scratch_dev_ptr) = {
            let mut host_ptr: *mut std::ffi::c_void = std::ptr::null_mut();
            let mut dev_ptr_out: u64 = 0;
            unsafe {
                let rc = cudarc::driver::sys::cuMemAllocHost_v2(
                    &mut host_ptr,
                    std::mem::size_of::<f32>(),
                );
                assert_eq!(rc, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemAllocHost for q_var_scratch");
                let rc2 = cudarc::driver::sys::cuMemHostGetDevicePointer_v2(
                    &mut dev_ptr_out,
                    host_ptr,
                    0,
                );
                assert_eq!(rc2, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemHostGetDevicePointer for q_var_scratch");
                *(host_ptr as *mut f32) = 0.0;
            }
            (host_ptr as *mut f32, dev_ptr_out)
        };

        let (reward_scratch_pinned, reward_scratch_dev_ptr) = {
            let mut host_ptr: *mut std::ffi::c_void = std::ptr::null_mut();
            let mut dev_ptr_out: u64 = 0;
            unsafe {
                let rc = cudarc::driver::sys::cuMemAllocHost_v2(
                    &mut host_ptr,
                    std::mem::size_of::<f32>(),
                );
                assert_eq!(rc, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemAllocHost for reward_scratch");
                let rc2 = cudarc::driver::sys::cuMemHostGetDevicePointer_v2(
                    &mut dev_ptr_out,
                    host_ptr,
                    0,
                );
                assert_eq!(rc2, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemHostGetDevicePointer for reward_scratch");
                *(host_ptr as *mut f32) = 0.0;
            }
            (host_ptr as *mut f32, dev_ptr_out)
        };

        let (atom_util_scratch_pinned, atom_util_scratch_dev_ptr) = {
            let mut host_ptr: *mut std::ffi::c_void = std::ptr::null_mut();
            let mut dev_ptr_out: u64 = 0;
            unsafe {
                let rc = cudarc::driver::sys::cuMemAllocHost_v2(
                    &mut host_ptr,
                    std::mem::size_of::<f32>(),
                );
                assert_eq!(rc, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemAllocHost for atom_util_scratch");
                let rc2 = cudarc::driver::sys::cuMemHostGetDevicePointer_v2(
                    &mut dev_ptr_out,
                    host_ptr,
                    0,
                );
                assert_eq!(rc2, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemHostGetDevicePointer for atom_util_scratch");
                *(host_ptr as *mut f32) = 0.0;
            }
            (host_ptr as *mut f32, dev_ptr_out)
        };

        // Task 2.X "make Full useful" — per-magnitude-bin Q-mean pinned
        // array (sized for up to 4 magnitude bins to match
        // MAX_MAGNITUDE_ACTIONS in experience_kernels.cu) plus the
        // absolute-Q-scale reference scalar. Written by
        // `q_mag_bin_means_reduce` (launched with q_stats); read by
        // isv_signal_update to populate ISV slots [13..16].
        let q_mag_means_slots: usize = 4;
        let (q_mag_means_scratch_pinned, q_mag_means_scratch_dev_ptr) = {
            let mut host_ptr: *mut std::ffi::c_void = std::ptr::null_mut();
            let mut dev_ptr_out: u64 = 0;
            unsafe {
                let rc = cudarc::driver::sys::cuMemAllocHost_v2(
                    &mut host_ptr,
                    q_mag_means_slots * std::mem::size_of::<f32>(),
                );
                assert_eq!(rc, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemAllocHost for q_mag_means_scratch");
                let rc2 = cudarc::driver::sys::cuMemHostGetDevicePointer_v2(
                    &mut dev_ptr_out,
                    host_ptr,
                    0,
                );
                assert_eq!(rc2, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemHostGetDevicePointer for q_mag_means_scratch");
                std::ptr::write_bytes(host_ptr as *mut u8, 0, q_mag_means_slots * std::mem::size_of::<f32>());
            }
            (host_ptr as *mut f32, dev_ptr_out)
        };

        let (q_abs_ref_scratch_pinned, q_abs_ref_scratch_dev_ptr) = {
            let mut host_ptr: *mut std::ffi::c_void = std::ptr::null_mut();
            let mut dev_ptr_out: u64 = 0;
            unsafe {
                let rc = cudarc::driver::sys::cuMemAllocHost_v2(
                    &mut host_ptr,
                    std::mem::size_of::<f32>(),
                );
                assert_eq!(rc, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemAllocHost for q_abs_ref_scratch");
                let rc2 = cudarc::driver::sys::cuMemHostGetDevicePointer_v2(
                    &mut dev_ptr_out,
                    host_ptr,
                    0,
                );
                assert_eq!(rc2, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemHostGetDevicePointer for q_abs_ref_scratch");
                *(host_ptr as *mut f32) = 0.0;
            }
            (host_ptr as *mut f32, dev_ptr_out)
        };

        // Task 2.Y "make direction branch useful at eval" — per-direction-bin
        // Q-mean pinned array (sized for the 4-branch direction head,
        // matching MAX_DIR=4 in q_dir_bin_means_reduce) plus the absolute-
        // Q-scale reference scalar. Written by `q_dir_bin_means_reduce`
        // (launched with q_stats); read by isv_signal_update to populate
        // ISV slots [17..21].
        let q_dir_means_slots: usize = 4;
        let (q_dir_means_scratch_pinned, q_dir_means_scratch_dev_ptr) = {
            let mut host_ptr: *mut std::ffi::c_void = std::ptr::null_mut();
            let mut dev_ptr_out: u64 = 0;
            unsafe {
                let rc = cudarc::driver::sys::cuMemAllocHost_v2(
                    &mut host_ptr,
                    q_dir_means_slots * std::mem::size_of::<f32>(),
                );
                assert_eq!(rc, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemAllocHost for q_dir_means_scratch");
                let rc2 = cudarc::driver::sys::cuMemHostGetDevicePointer_v2(
                    &mut dev_ptr_out,
                    host_ptr,
                    0,
                );
                assert_eq!(rc2, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemHostGetDevicePointer for q_dir_means_scratch");
                std::ptr::write_bytes(host_ptr as *mut u8, 0, q_dir_means_slots * std::mem::size_of::<f32>());
            }
            (host_ptr as *mut f32, dev_ptr_out)
        };

        let (q_dir_abs_ref_scratch_pinned, q_dir_abs_ref_scratch_dev_ptr) = {
            let mut host_ptr: *mut std::ffi::c_void = std::ptr::null_mut();
            let mut dev_ptr_out: u64 = 0;
            unsafe {
                let rc = cudarc::driver::sys::cuMemAllocHost_v2(
                    &mut host_ptr,
                    std::mem::size_of::<f32>(),
                );
                assert_eq!(rc, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemAllocHost for q_dir_abs_ref_scratch");
                let rc2 = cudarc::driver::sys::cuMemHostGetDevicePointer_v2(
                    &mut dev_ptr_out,
                    host_ptr,
                    0,
                );
                assert_eq!(rc2, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemHostGetDevicePointer for q_dir_abs_ref_scratch");
                *(host_ptr as *mut f32) = 0.0;
            }
            (host_ptr as *mut f32, dev_ptr_out)
        };

        // ── ISV core buffers (pinned device-mapped) ──────────────────────
        // Allocation uses `ISV_TOTAL_DIM` so all scratchpad slots [23..60) have
        // backing storage. The network-facing path still consumes only the first
        // `ISV_NETWORK_DIM` slots — the scratchpad tail is invisible to `w_isv_fc1`.
        // Slots [58..60) hold the layout fingerprint (tail placement because [0..2)
        // are actively written by `isv_signal_update`).
        let (isv_signals_pinned, isv_signals_dev_ptr) = {
            let num_bytes = ISV_TOTAL_DIM * std::mem::size_of::<f32>();
            let mut host_ptr: *mut std::ffi::c_void = std::ptr::null_mut();
            let mut dev_ptr_out: u64 = 0;
            unsafe {
                let rc = cudarc::driver::sys::cuMemAllocHost_v2(&mut host_ptr, num_bytes);
                assert_eq!(rc, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemAllocHost for isv_signals");
                let rc2 = cudarc::driver::sys::cuMemHostGetDevicePointer_v2(&mut dev_ptr_out, host_ptr, 0);
                assert_eq!(rc2, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemHostGetDevicePointer for isv_signals");
                std::ptr::write_bytes(host_ptr as *mut u8, 0, num_bytes);
                // Bootstrap the per-branch v-range slots to the config bounds so
                // that any consumer reaching for slots [23..31] before the first
                // `update_eval_v_range` call sees identical-to-config behaviour.
                // Half-width = (v_max - v_min) / 2, centre = 0 → v_min..v_max.
                let v_min_f = config.v_min as f32;
                let v_max_f = config.v_max as f32;
                let half_bootstrap = 0.5_f32 * (v_max_f - v_min_f);
                let sig_ptr = host_ptr as *mut f32;
                for b in 0..4usize {
                    *sig_ptr.add(23 + 2 * b)     = 0.0_f32;       // v_center
                    *sig_ptr.add(23 + 2 * b + 1) = half_bootstrap; // v_half
                }
                // Grad-balance unification bundle (2026-04-23): bootstrap
                // per-branch grad-norm targets and scale-clamp limit.
                // Targets start at 1.0 (neutral, no scaling until the first
                // on-GPU `grad_balance_isv_update` populates the EMA from
                // observed per-branch norms). Limit bootstraps at 2.0 —
                // allows up to 2× rescaling before any real spread has
                // been observed; EMA grows upward only if training
                // warrants. The 2.0 lower bound and 1e4 upper bound are
                // numerical-safety hard limits (not adaptive tuning).
                for b in 0..4usize {
                    *sig_ptr.add(GRAD_NORM_TARGET_DIR_INDEX + b) = 1.0_f32;
                }
                *sig_ptr.add(GRAD_SCALE_LIMIT_INDEX) = 2.0_f32;
                // IQL branch_scales per-sample floor (2026-04-23): bootstrap
                // at 0.1. Prevents (sample, branch) collapse to zero when
                // IQL readiness→1 and the losing branch has near-zero
                // advantage. Safety bound, not tuning.
                *sig_ptr.add(IQL_BRANCH_SCALE_FLOOR_INDEX) = 0.1_f32;
                // Plan 1 C.6 static-config slots (2026-04-24):
                // EPOCH_IDX starts at 0; epoch loop updates in a follow-up task.
                // EPSILON_EFF / TAU_EFF start at 0; GPU kernels fill them.
                // D.2: bootstrap all 4 per-branch γ slots to the same base value.
                // GPU kernel diverges them per-branch from here based on Q-stats.
                let gamma_base = config.gamma;
                for b in 0..4usize {
                    *sig_ptr.add(GAMMA_DIR_EFF_INDEX + b) = gamma_base;
                }
                *sig_ptr.add(TOTAL_EPOCHS_INDEX)     = config.total_epochs as f32;
                *sig_ptr.add(CQL_ALPHA_INDEX)        = config.cql_alpha;
                /* B.4 Plan 3 Task 4: Cold-start value; overwritten by
                 * `plan_threshold_update` kernel on the first experience-
                 * collection epoch. Kept so that consumers of slot 49 see a
                 * sensible 0.5 gate before the kernel's first fire (the
                 * kernel's derived `0.5 × ema` lands at the same 0.5 if the
                 * companion READINESS_EMA cold-start value is 1.0). */
                *sig_ptr.add(PLAN_THRESHOLD_INDEX)   = 0.5_f32;
                /* B.4 Plan 3 Task 4: cold-start readiness EMA. Set to 1.0
                 * so the first kernel fire — before any real readiness has
                 * been observed — derives `threshold = max(0.1, 0.5 × 1.0)
                 * = 0.5`, exactly matching the static default the consumers
                 * have always seen. The kernel then EMA-decays this toward
                 * the actual measured mean readiness over subsequent
                 * epochs. */
                *sig_ptr.add(READINESS_EMA_INDEX)    = 1.0_f32;
                /* C.3 Plan 3 Task 7: cold-start state-distribution KL slots.
                 * KL EMA at 0.0 — the first `state_kl_moment_match` fire EMAs
                 * the actual measured KL toward this with α≈0.05, so the
                 * trailing-self ratio is well-defined from epoch 2 onward.
                 * Amp at 1.0 — consumer multiplier in B.1/B.2 reads
                 * `fmaxf(1.0, ISV[79])`, so cold-start 1.0 is the no-op
                 * neutral element until the kernel populates a real value. */
                *sig_ptr.add(STATE_KL_TRAIN_VAL_EMA_INDEX) = 0.0_f32;
                *sig_ptr.add(STATE_KL_AMPLIFICATION_INDEX) = 1.0_f32;
                /* B.3 Plan 3 Task 8: GPU-only replay seed warm-start.
                 * Cold-start the seed-phase progress slots. TARGET is the
                 * static config (SchemaContract); DONE starts at 0 and the
                 * GPU `seed_step_counter_update` kernel increments it after
                 * every `collect_experiences_gpu`. SEED_FRAC_EMA cold-starts
                 * at 1.0 (fully in seed phase) so Task 9's CQL ramp sees
                 * `target = final × max(0, 1 - 1.0) = 0` until the seed
                 * phase actually decays — preserving the existing CQL=0
                 * pre-warmup behaviour byte-for-byte. */
                *sig_ptr.add(SEED_STEPS_TARGET_INDEX) = config.replay_seed_steps as f32;
                *sig_ptr.add(SEED_STEPS_DONE_INDEX)   = 0.0_f32;
                *sig_ptr.add(SEED_FRAC_EMA_INDEX)     = 1.0_f32;
                /* E.5 Plan 4 Task 5 Mode A: cold-start attention-focus EMAs at
                 * 0.0. The producer kernel `attention_focus_ema_update` runs
                 * once per epoch at the HEALTH_DIAG site; first fire EMAs the
                 * actual measured per-branch VSN-weight + Mamba2 retention
                 * scalars toward 0 with α≈0.05, so values ramp up cleanly
                 * from epoch 1 onward. Diagnostic only — no consumer reads
                 * these slots in Mode A. */
                *sig_ptr.add(VSN_MAG_EMA_INDEX)         = 0.0_f32;
                *sig_ptr.add(VSN_DIR_EMA_INDEX)         = 0.0_f32;
                *sig_ptr.add(MAMBA2_RETENTION_EMA_INDEX) = 0.0_f32;
                // Plan 4 follow-up: target-drift EMA cold-start.
                *sig_ptr.add(TARGET_DRIFT_MAG_EMA_INDEX) = 0.0_f32;
                *sig_ptr.add(TARGET_DRIFT_DIR_EMA_INDEX) = 0.0_f32;
                /* Plan 4 Task 2c.3c.5: H_S2 RMS EMA cold-start. Set to 1.0
                 * (neutral RMS) so the first kernel fire's α=0.05 EMA pulls
                 * the slot toward the measured RMS rather than collapsing
                 * toward 0. Half-life ≈ 13 batches (log(0.5)/log(0.95)).
                 * Producer-only in this commit; 2c.3c.6 wires the consumer
                 * in `mag_concat_qdir`'s adaptive-scale path. FoldReset → 1.0. */
                *sig_ptr.add(H_S2_RMS_EMA_INDEX) = 1.0_f32;
                /* Plan 4 Task 3 (E.3): cold-start the four IQN multi-quantile
                 * diagnostic EMAs at 0.0. The producer kernel
                 * `iqn_quantile_ema_update` runs each step alongside
                 * `h_s2_rms_ema_update`; first fire EMAs the measured
                 * per-quantile mean toward 0.0 with the same α as the other
                 * Plan 4 producers. Producer-only — no consumer kernel reads
                 * slots [99..103) in this commit. FoldReset → 0.0. */
                *sig_ptr.add(IQN_Q_P05_EMA_INDEX) = 0.0_f32;
                *sig_ptr.add(IQN_Q_P25_EMA_INDEX) = 0.0_f32;
                *sig_ptr.add(IQN_Q_P75_EMA_INDEX) = 0.0_f32;
                *sig_ptr.add(IQN_Q_P95_EMA_INDEX) = 0.0_f32;
                /* Plan 4 Task 1B-ii: VSN per-group mask-EMA cold-start.
                 * 1.0 / SL_NUM_FEATURE_GROUPS = 1/6 — uniform prior (no
                 * group preference at init). The producer kernel
                 * `vsn_mask_ema_update` (1B-i, launch site wired in
                 * 1B-iii) overwrites these slots on every fire; until
                 * 1B-iii lands, they stay at the uniform cold-start.
                 * FoldReset reapplies the same 1/6 at fold boundary so
                 * each fold begins with no inherited group bias. */
                let uniform_mask = 1.0_f32 / ml_core::state_layout::SL_NUM_FEATURE_GROUPS as f32;
                *sig_ptr.add(VSN_MASK_GROUP_0_EMA_INDEX) = uniform_mask;
                *sig_ptr.add(VSN_MASK_GROUP_1_EMA_INDEX) = uniform_mask;
                *sig_ptr.add(VSN_MASK_GROUP_2_EMA_INDEX) = uniform_mask;
                *sig_ptr.add(VSN_MASK_GROUP_3_EMA_INDEX) = uniform_mask;
                *sig_ptr.add(VSN_MASK_GROUP_4_EMA_INDEX) = uniform_mask;
                *sig_ptr.add(VSN_MASK_GROUP_5_EMA_INDEX) = uniform_mask;
                // Plan 2 C.1 Q-quantile bootstrap (2026-04-24):
                // q_p05 = v_min, q_p95 = v_max — matches the cold-start atom range
                // so the first update_eval_v_range call reads meaningful bounds.
                for b in 0..4usize {
                    *sig_ptr.add(Q_P05_DIR_INDEX + b) = v_min_f;
                    *sig_ptr.add(Q_P95_DIR_INDEX + b) = v_max_f;
                }
                /* Adaptive load-balance λ controller (2026-04-27):
                 * cold-start ISV[MOE_LAMBDA_EFF_INDEX=128] at the floor so
                 * the first `moe_load_balance_loss` consumer reads the
                 * legacy 0.01 baseline before the controller has fired.
                 * The producer (`moe_lambda_eff_update`) overwrites this
                 * each step from the entropy EMA. FoldReset reapplies the
                 * same floor (gate uniform-init ⇒ deficit=0 ⇒ λ_floor).
                 * Per `pearl_blend_formulas_must_have_permanent_floor.md`,
                 * λ_floor is a permanent minimum. */
                *sig_ptr.add(MOE_LAMBDA_EFF_INDEX) = config.moe_lambda_floor;
                /* Mixture-of-Experts cold-start (2026-04-27 follow-up):
                 * bootstrap util slots to uniform 1/K and entropy slot to
                 * ln(K) so the very first `moe_load_balance_loss` and
                 * `moe_lambda_eff_update` see consistent neutral state
                 * before `moe_expert_util_ema_update` fires. FoldReset
                 * reapplies these via the registry. */
                let inv_k = 1.0_f32 / MOE_EXPERT_UTIL_EMA_COUNT as f32;
                for k in 0..MOE_EXPERT_UTIL_EMA_COUNT {
                    *sig_ptr.add(MOE_EXPERT_UTIL_EMA_BASE + k) = inv_k;
                }
                *sig_ptr.add(MOE_GATE_ENTROPY_EMA_INDEX) =
                    (MOE_EXPERT_UTIL_EMA_COUNT as f32).ln();
                /* Plan C Phase 2 follow-up A.2 (2026-04-29):
                 * cold-start ISV[Q_DRIFT_RATE_INDEX=129] at 0.0 so the
                 * very first `tau_update` consumer (which multiplies
                 * tau_base by 1/(1+ISV[129])) sees a no-op dampening
                 * factor of 1.0 before the producer kernel
                 * `q_drift_rate_ema_update` fires. FoldReset reapplies the
                 * same 0.0 (no drift at the start of a new fold; the
                 * first epoch's `prev_epoch_q_mean.abs() > 1e-6` gate in
                 * `training_loop.rs` prevents the producer launch until
                 * a meaningful prev value exists). */
                *sig_ptr.add(Q_DRIFT_RATE_INDEX) = 0.0_f32;
                // SP9 Fix 37.1 (2026-05-03): the 3 Kelly cold-start exit-axis
                // targets are Invariant-1 numerical anchors per
                // `feedback_isv_for_adaptive_bounds.md`, not EMA-tracked.
                // The original Fix 37 routed each through a single-input
                // EMA producer + Pearl A sentinel-bootstrap + Pearl D
                // Wiener-α; smoke-test-wrwkz revealed this caused the
                // first observation to replace the sentinel directly,
                // making target == current_obs in epoch 1 (`stat_count_tgt
                // = 419`, `temp_tgt = 1`). Consequently `current/target =
                // 1.0`, `confidence = 1.0`, `floor = base × (1 − 1.0) = 0`
                // — defeating the cold-start mechanism entirely.
                //
                // The principled fix per the pearl: define what
                // "sufficient" means in absolute terms via Invariant-1
                // anchors. The target-relative ratio
                // `current_observation / fixed_anchor` then yields a
                // self-normalised confidence in [0, 1] that rises
                // monotonically as samples / divergence-stability /
                // fold-time accumulate.
                //
                //   ISV[KELLY_SAMPLE_COUNT_TARGET_INDEX = 333] = 100.0
                //     — minimum trade samples before statistical confidence
                //       releases the cap.
                //   ISV[KELLY_DIVERGENCE_TARGET_INDEX = 334] = 2.0
                //     — divergence ratio above which intent-vs-eval are
                //       considered diverged enough to keep the cap closed.
                //   ISV[KELLY_TEMPORAL_TARGET_INDEX = 335] = 5.0
                //     — minimum epochs in fold before temporal confidence
                //       releases the cap (safety net per Risk 3 in spec).
                //
                // FoldReset rewrites the same anchors (in `reset_named_state`)
                // since these are constants, not stateful EMAs — they must
                // never reach 0 sentinel between folds.
                use crate::cuda_pipeline::sp5_isv_slots::{
                    KELLY_SAMPLE_COUNT_TARGET_INDEX,
                    KELLY_DIVERGENCE_TARGET_INDEX,
                    KELLY_TEMPORAL_TARGET_INDEX,
                };
                *sig_ptr.add(KELLY_SAMPLE_COUNT_TARGET_INDEX) = 100.0_f32;
                *sig_ptr.add(KELLY_DIVERGENCE_TARGET_INDEX)   = 2.0_f32;
                *sig_ptr.add(KELLY_TEMPORAL_TARGET_INDEX)     = 5.0_f32;

                // SP13 directional-skill Invariant-1 anchors. Constants, not
                // stateful EMAs — must never reach 0 sentinel between folds
                // (FoldReset rewrites them in `reset_named_state`).
                use crate::cuda_pipeline::sp5_isv_slots::{
                    TARGET_DIR_ACC_INDEX, TARGET_DIR_ACC_DEFAULT,
                    DIR_SKILL_BONUS_ALPHA_INDEX, DIR_SKILL_BONUS_ALPHA_DEFAULT,
                    DIR_SKILL_BONUS_BETA_INDEX, DIR_SKILL_BONUS_BETA_DEFAULT,
                    LUCK_WIN_DISCOUNT_INDEX, LUCK_WIN_DISCOUNT_DEFAULT,
                    SKILL_BONUS_CAP_RATIO_INDEX, SKILL_BONUS_CAP_RATIO_DEFAULT,
                };
                *sig_ptr.add(TARGET_DIR_ACC_INDEX)        = TARGET_DIR_ACC_DEFAULT;
                *sig_ptr.add(DIR_SKILL_BONUS_ALPHA_INDEX) = DIR_SKILL_BONUS_ALPHA_DEFAULT;
                *sig_ptr.add(DIR_SKILL_BONUS_BETA_INDEX)  = DIR_SKILL_BONUS_BETA_DEFAULT;
                *sig_ptr.add(LUCK_WIN_DISCOUNT_INDEX)     = LUCK_WIN_DISCOUNT_DEFAULT;
                *sig_ptr.add(SKILL_BONUS_CAP_RATIO_INDEX) = SKILL_BONUS_CAP_RATIO_DEFAULT;

                // SP13 v3 P0a.T3 (2026-05-04): Hold-pricing observation
                // chain anchors (DD7=c). Slot 380 is a constructor-init-only
                // diagnostic anchor surfacing the static base cost — the SP18
                // structural Hold opportunity-cost reward (slots [483..493))
                // does NOT consume slot 380. HOLD_RATE_TARGET is a static MFT
                // default 0.20 — never reaches sentinel 0 between folds. The
                // observed-rate EMA at slot 382 is FoldReset (sentinel 0.0)
                // and registered separately.
                use crate::cuda_pipeline::sp5_isv_slots::{
                    HOLD_COST_INDEX, HOLD_COST_BASE,
                    HOLD_RATE_TARGET_INDEX, HOLD_RATE_TARGET_DEFAULT,
                };
                *sig_ptr.add(HOLD_COST_INDEX)        = HOLD_COST_BASE;
                *sig_ptr.add(HOLD_RATE_TARGET_INDEX) = HOLD_RATE_TARGET_DEFAULT;

                // SP15 Phase 1.2 (2026-05-06): OFI-impact lambda
                // (per spec §6.2). Constant Invariant-1 anchor — NOT
                // a stateful EMA — so constructor-write per
                // `feedback_isv_for_adaptive_bounds.md`. Per-fold ISV
                // refit may overwrite this in later phases; the
                // FoldReset arm rewrites the same default to keep the
                // cost kernel functional during cold-start of each
                // new fold (must never reach sentinel 0, which would
                // silently disable the OFI-impact term). The matching
                // COST_PER_BAR_AVG_INDEX=408 is a stateful kernel
                // output (FoldReset sentinel 0; first kernel launch
                // writes the true mean).
                use crate::cuda_pipeline::sp15_isv_slots::OFI_IMPACT_LAMBDA_INDEX;
                *sig_ptr.add(OFI_IMPACT_LAMBDA_INDEX) = 2.0e-4_f32;

                // SP15 Phase 3.1 (2026-05-06): r_quality + r_discipline
                // split slots (per spec §8.2 (3.1) post-amendment-2 fix).
                // ALPHA_SPLIT (slot 417) is initialized DIRECTLY to 0.5
                // — NOT derived from the formula
                // `α = grad_norm_q / (grad_norm_q + grad_norm_d + ε)` at
                // boot, which would produce 0/0=0 from the zero
                // grad-norm EMAs. The composer kernel
                // (`r_quality_discipline_split_kernel`) enforces this
                // 0.5 sentinel until `alpha_warm_count >= N_WARM=100`,
                // at which point the producer kernel
                // (`alpha_split_producer_kernel`) takes over and writes
                // the formula's clamped output here on each step. Per
                // `feedback_isv_for_adaptive_bounds.md` — the structural
                // 0.5 anchor is the cold-start absorber, the runtime
                // value is signal-driven. The matching grad-norm slots
                // (418, 419) are stateful EMA outputs, FoldReset
                // sentinel 0 so Pearl A bootstraps from the new fold's
                // first observation per
                // `pearl_first_observation_bootstrap.md`.
                use crate::cuda_pipeline::sp15_isv_slots::{
                    ALPHA_SPLIT_INDEX, GRAD_NORM_DISCIPLINE_INDEX,
                    GRAD_NORM_QUALITY_INDEX,
                };
                *sig_ptr.add(ALPHA_SPLIT_INDEX) = 0.5_f32;
                *sig_ptr.add(GRAD_NORM_QUALITY_INDEX) = 0.0_f32;
                *sig_ptr.add(GRAD_NORM_DISCIPLINE_INDEX) = 0.0_f32;

                // SP15 Phase 3.3 (2026-05-06): quadratic DD penalty anchors
                // (per spec §8.2 (3.3)). `penalty = λ_dd × max(0, dd_current
                // − dd_threshold)²`. λ_dd is initialised to 1.0 — the
                // structural cold-start absorber per
                // `feedback_isv_for_adaptive_bounds.md`; runtime adaptation
                // via grad-balance (a follow-up producer kernel) overwrites
                // this with a signal-driven value once the gradient-norm
                // EMA `DD_PENALTY_GRAD_NORM_INDEX=422` has accumulated
                // observations. dd_threshold is initialised to 0.05 (5%
                // drawdown trigger — the structural anchor per the spec).
                // DD_PENALTY_GRAD_NORM is FoldReset sentinel 0 so Pearl A
                // bootstraps from the new fold's first observation per
                // `pearl_first_observation_bootstrap.md`.
                use crate::cuda_pipeline::sp15_isv_slots::{
                    DD_PENALTY_GRAD_NORM_INDEX, DD_THRESHOLD_INDEX,
                    LAMBDA_DD_INDEX,
                };
                *sig_ptr.add(LAMBDA_DD_INDEX) = 1.0_f32;
                *sig_ptr.add(DD_THRESHOLD_INDEX) = 0.05_f32;
                *sig_ptr.add(DD_PENALTY_GRAD_NORM_INDEX) = 0.0_f32;

                // SP15 Phase 3.4 (2026-05-06): regret-signal anchors
                // (per spec §8.2 (3.4)). When policy holds AND a trade
                // WOULD have been profitable past `cost_t + trail_dist`,
                // `regret_bar = ideal_pnl_missed - cost_t`; the kernel
                // accumulates this into REGRET_EMA via first-observation
                // bootstrap (per `pearl_first_observation_bootstrap`) +
                // simple exponential decay α=0.05. The reward composer
                // subtracts `LAMBDA_REGRET × REGRET_EMA` from
                // r_discipline at the deferred follow-up consumer site.
                // REGRET_EMA is FoldReset sentinel 0 (Pearl A bootstrap).
                // LAMBDA_REGRET is initialised to 1.0 — the structural
                // cold-start absorber per
                // `feedback_isv_for_adaptive_bounds.md`; runtime
                // adaptation via grad-balance (a follow-up producer
                // kernel) overwrites this with a signal-driven value
                // once REGRET_GRAD_NORM accumulates observations.
                // REGRET_GRAD_NORM is FoldReset sentinel 0 (Pearl A
                // bootstrap on the new fold's first non-zero gradient
                // norm).
                use crate::cuda_pipeline::sp15_isv_slots::{
                    LAMBDA_REGRET_INDEX, REGRET_EMA_INDEX,
                    REGRET_GRAD_NORM_INDEX,
                };
                *sig_ptr.add(REGRET_EMA_INDEX) = 0.0_f32;
                *sig_ptr.add(LAMBDA_REGRET_INDEX) = 1.0_f32;
                *sig_ptr.add(REGRET_GRAD_NORM_INDEX) = 0.0_f32;

                // SP15 Phase 3.5 (2026-05-06): confidence-aware Hold
                // floor anchors (per spec §8.2 (3.5) post-amendment-2
                // fix). `hold_floor = α × σ(k × (entropy − ε₀))` —
                // action-selection level (NOT reward shaping). The
                // follow-up consumer adds `hold_floor` to Q_hold
                // pre-argmax/Thompson selection so Hold becomes
                // uncertainty expression rather than the distributional
                // default. α / k / ε₀ are ISV-tracked (slots 426 /
                // 427 / 428) — the signal-driven producers updating
                // these from rolling 95th percentile of |Q_dir| (NOT
                // running max — outlier-ratchet vulnerable per spec
                // second-review #6), running variance of entropy, and
                // 75th percentile of running entropy distribution are
                // documented Phase 3.5 follow-up sub-tasks. Until the
                // producers land the structural cold-start anchors
                // hold the Hold floor at a finite scale per
                // `feedback_isv_for_adaptive_bounds.md`:
                //   HOLD_FLOOR_ALPHA = 0.5  (sentinel — half a Q-unit
                //                            at peak σ)
                //   HOLD_FLOOR_K     = 10.0 (steep sigmoid — fast
                //                            saturation)
                //   HOLD_FLOOR_EPS0  = 1.0  (entropy threshold —
                //                            ~1 nat ≈ ln(e))
                //   ENTROPY_DIST_REF = 0.0  (Pearl A sentinel — first
                //                            non-zero observation from
                //                            the deferred ε₀ producer
                //                            replaces directly per
                //                            `pearl_first_observation_bootstrap`)
                use crate::cuda_pipeline::sp15_isv_slots::{
                    ENTROPY_DIST_REF_INDEX, HOLD_FLOOR_ALPHA_INDEX,
                    HOLD_FLOOR_EPS0_INDEX, HOLD_FLOOR_K_INDEX,
                };
                *sig_ptr.add(HOLD_FLOOR_ALPHA_INDEX) = 0.5_f32;
                *sig_ptr.add(HOLD_FLOOR_K_INDEX) = 10.0_f32;
                *sig_ptr.add(HOLD_FLOOR_EPS0_INDEX) = 1.0_f32;
                *sig_ptr.add(ENTROPY_DIST_REF_INDEX) = 0.0_f32;

                // SP15 Phase 3.5.2 (2026-05-06): asymmetric reward under
                // DD anchors (per spec §9.2 (3.5.2)). For gains
                // `r_adjusted = r × (1 + λ × dd_pct)`; for losses
                // unchanged. Applied BEFORE the SP12 v3 NEG/POS reward
                // cap clamp by the deferred reward-composer site.
                // λ_DD_ASYMMETRY is a structural cold-start anchor per
                // `feedback_isv_for_adaptive_bounds.md` — the runtime
                // value will be signal-driven from a follow-up producer
                // kernel that tracks the running variance of dd_pct
                // (`DD_DIST_VAR_INDEX=432`). The 0.5 anchor re-engages
                // on each new fold so the cold-start absorber holds λ at
                // a finite scale until the producer accumulates
                // observations. R_GAIN_DD_BOOST is the diagnostic slot
                // recording the most-recent boost factor (1.0 = no-op);
                // initial 1.0 sentinel makes the first HEALTH_DIAG
                // emission before any kernel launch read as "no boost
                // yet". DD_DIST_VAR is a stateful EMA (Pearl A
                // sentinel 0 — first observation replaces directly per
                // `pearl_first_observation_bootstrap.md`).
                use crate::cuda_pipeline::sp15_isv_slots::{
                    DD_ASYMMETRY_LAMBDA_INDEX, DD_DIST_VAR_INDEX,
                    R_GAIN_DD_BOOST_INDEX,
                };
                *sig_ptr.add(DD_ASYMMETRY_LAMBDA_INDEX) = 0.5_f32;
                *sig_ptr.add(R_GAIN_DD_BOOST_INDEX) = 1.0_f32;
                *sig_ptr.add(DD_DIST_VAR_INDEX) = 0.0_f32;

                // SP15 Phase 3.5.3 (2026-05-06): cooldown gate anchors
                // (per spec §9.2 (3.5.3)). After K consecutive losing
                // trades, force Hold for M bars. Counter
                // (COOLDOWN_BARS_REMAINING) decremented per bar — part
                // of state so the model can reason about it.
                //
                //   COOLDOWN_K_THRESHOLD = 5.0  (Invariant-1 anchor —
                //                                ISV-driven K via
                //                                MEDIAN_STREAK_LENGTH
                //                                producer is a
                //                                follow-up sub-task per
                //                                `feedback_isv_for_adaptive_bounds.md`)
                //   COOLDOWN_M_BARS      = 20.0 (Invariant-1 anchor —
                //                                ISV-driven M from
                //                                vol_normalizer
                //                                time-to-mean-reversion
                //                                is a parallel
                //                                follow-up)
                //   COOLDOWN_BARS_REMAINING = 0.0 (cold-start: not in
                //                                  cooldown)
                //   MEDIAN_STREAK_LENGTH = 0.0 (Pearl A sentinel — first
                //                               non-zero observation
                //                               from the deferred
                //                               two-heap median producer
                //                               replaces directly per
                //                               `pearl_first_observation_bootstrap`)
                use crate::cuda_pipeline::sp15_isv_slots::{
                    COOLDOWN_BARS_REMAINING_INDEX, COOLDOWN_K_THRESHOLD_INDEX,
                    COOLDOWN_M_BARS_INDEX, MEDIAN_STREAK_LENGTH_INDEX,
                };
                *sig_ptr.add(COOLDOWN_K_THRESHOLD_INDEX) = 5.0_f32;
                *sig_ptr.add(COOLDOWN_M_BARS_INDEX) = 20.0_f32;
                *sig_ptr.add(COOLDOWN_BARS_REMAINING_INDEX) = 0.0_f32;
                *sig_ptr.add(MEDIAN_STREAK_LENGTH_INDEX) = 0.0_f32;

                // SP15 Phase 3.5.4 (2026-05-06): plasticity injection
                // anchors (per spec §9.2 (3.5.4) post-amendment-2 fix).
                // TWO-STEP recovery — fire when
                // `DD_PERSISTENCE > PLASTICITY_PERSISTENCE_THRESHOLD`
                // AND `PLASTICITY_FIRED_THIS_FOLD == 0` → set fired
                // flag, set warm-bars counter to M_warm (default 200),
                // [DEFERRED: reset last 10% of advantage-head weights
                // to Kaiming-He init]. Per-bar warm-bars decrement;
                // consumer-wiring follow-up reads
                // max(COOLDOWN_BARS_REMAINING,
                // PLASTICITY_WARM_BARS_REMAINING) and forces Hold
                // while > 0 (the OR-gate combining cooldown +
                // plasticity warm-up is the action-selection two-step).
                //
                //   PLASTICITY_FIRED_THIS_FOLD = 0.0 (debounce flag —
                //                                    fires at most once
                //                                    per fold; reset
                //                                    to 0.0 at fold
                //                                    boundary re-arms
                //                                    the gate for the
                //                                    next fold)
                //   PLASTICITY_PERSISTENCE_THRESHOLD = 100.0 (Invariant-1
                //                                            anchor —
                //                                            ISV-driven
                //                                            from running
                //                                            mean of
                //                                            dd_persistence_bars
                //                                            is a
                //                                            follow-up
                //                                            sub-task per
                //                                            `feedback_isv_for_adaptive_bounds.md`;
                //                                            the 100.0
                //                                            anchor
                //                                            re-engages
                //                                            on each
                //                                            new fold)
                //   PLASTICITY_WARM_BARS_REMAINING = 0.0 (cold-start: no
                //                                        active warm-up;
                //                                        FoldReset
                //                                        sentinel 0 so
                //                                        the new fold
                //                                        starts with no
                //                                        active warm-up
                //                                        — leftover
                //                                        state from the
                //                                        previous fold
                //                                        would force
                //                                        Hold during
                //                                        the new fold's
                //                                        warmup)
                use crate::cuda_pipeline::sp15_isv_slots::{
                    PLASTICITY_FIRED_THIS_FOLD_INDEX,
                    PLASTICITY_PERSISTENCE_THRESHOLD_INDEX,
                    PLASTICITY_WARM_BARS_REMAINING_INDEX,
                };
                *sig_ptr.add(PLASTICITY_FIRED_THIS_FOLD_INDEX) = 0.0_f32;
                *sig_ptr.add(PLASTICITY_PERSISTENCE_THRESHOLD_INDEX) = 100.0_f32;
                *sig_ptr.add(PLASTICITY_WARM_BARS_REMAINING_INDEX) = 0.0_f32;

                // SP15 Phase 3.5.5 (2026-05-06): recovery curriculum in
                // PER — per-step DD_TRAJECTORY_DECREASING proxy anchors
                // (per spec §9.2 (3.5.5) post-amendment-2 fix).
                // Replaces non-existent episode-level metadata with a
                // per-bar boolean computed from the dd_pct trajectory:
                // `trajectory(t) = 1.0 iff dd_pct(t) < dd_pct(t-1) AND
                // dd_pct(t-1) > DD_TRAJECTORY_FLOOR`. PER sampling
                // consumer (deferred follow-up): `sampling_weight =
                // base × (1 + RECOVERY_OVERSAMPLE_WEIGHT ×
                // DD_TRAJECTORY_DECREASING)`.
                //
                //   DD_TRAJECTORY_DECREASING = 0.0 (cold-start: no
                //                                  observation yet —
                //                                  the kernel's first
                //                                  call after
                //                                  construction / fold
                //                                  reset has prev=0.0
                //                                  < floor=0.02 so the
                //                                  floor gate
                //                                  evaluates to false
                //                                  and trajectory
                //                                  stays 0)
                //   RECOVERY_OVERSAMPLE_WEIGHT = 2.0 (initial sentinel —
                //                                   ISV-driven from
                //                                   current dd_pct in
                //                                   follow-up sub-task
                //                                   per
                //                                   `feedback_isv_for_adaptive_bounds.md`;
                //                                   more DD now →
                //                                   higher weight for
                //                                   recovery samples;
                //                                   the 2.0 anchor
                //                                   re-engages on each
                //                                   new fold)
                //   DD_TRAJECTORY_FLOOR = 0.02 (initial sentinel —
                //                              ISV-driven 25th
                //                              percentile of running
                //                              dd_pct distribution in
                //                              follow-up sub-task per
                //                              `feedback_isv_for_adaptive_bounds.md`,
                //                              replacing the hardcoded
                //                              0.02 floor from the
                //                              original spec draft;
                //                              the 0.02 anchor
                //                              re-engages on each new
                //                              fold)
                use crate::cuda_pipeline::sp15_isv_slots::{
                    DD_TRAJECTORY_DECREASING_INDEX, DD_TRAJECTORY_FLOOR_INDEX,
                    RECOVERY_OVERSAMPLE_WEIGHT_INDEX,
                };
                *sig_ptr.add(DD_TRAJECTORY_DECREASING_INDEX) = 0.0_f32;
                *sig_ptr.add(RECOVERY_OVERSAMPLE_WEIGHT_INDEX) = 2.0_f32;
                *sig_ptr.add(DD_TRAJECTORY_FLOOR_INDEX) = 0.02_f32;

                // Layout fingerprint (ISV[58..60)). Compile-time structural hash of
                // the slot layout; checkpoint load fails-fast on mismatch.
                // Stored as a u64 split across two f32 lanes using raw bit-cast so
                // no precision is lost. See LAYOUT_FINGERPRINT_CURRENT docs for why
                // this is NOT a version number and why no migration path exists.
                let fp = LAYOUT_FINGERPRINT_CURRENT;
                *sig_ptr.add(ISV_LAYOUT_FINGERPRINT_LO_INDEX) = f32::from_bits(fp as u32);
                *sig_ptr.add(ISV_LAYOUT_FINGERPRINT_HI_INDEX) = f32::from_bits((fp >> 32) as u32);
            }
            (host_ptr as *mut f32, dev_ptr_out)
        };

        let (isv_history_pinned, isv_history_dev_ptr) = {
            let num_bytes = ISV_K * ISV_DIM * std::mem::size_of::<f32>();
            let mut host_ptr: *mut std::ffi::c_void = std::ptr::null_mut();
            let mut dev_ptr_out: u64 = 0;
            unsafe {
                let rc = cudarc::driver::sys::cuMemAllocHost_v2(&mut host_ptr, num_bytes);
                assert_eq!(rc, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemAllocHost for isv_history");
                let rc2 = cudarc::driver::sys::cuMemHostGetDevicePointer_v2(&mut dev_ptr_out, host_ptr, 0);
                assert_eq!(rc2, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemHostGetDevicePointer for isv_history");
                std::ptr::write_bytes(host_ptr as *mut u8, 0, num_bytes);
            }
            (host_ptr as *mut f32, dev_ptr_out)
        };

        let (isv_decay_pinned, isv_decay_dev_ptr) = {
            let num_bytes = ISV_DIM * std::mem::size_of::<f32>();
            let mut host_ptr: *mut std::ffi::c_void = std::ptr::null_mut();
            let mut dev_ptr_out: u64 = 0;
            unsafe {
                let rc = cudarc::driver::sys::cuMemAllocHost_v2(&mut host_ptr, num_bytes);
                assert_eq!(rc, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemAllocHost for isv_decay");
                let rc2 = cudarc::driver::sys::cuMemHostGetDevicePointer_v2(&mut dev_ptr_out, host_ptr, 0);
                assert_eq!(rc2, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemHostGetDevicePointer for isv_decay");
                std::ptr::write_bytes(host_ptr as *mut u8, 0, num_bytes);
            }
            (host_ptr as *mut f32, dev_ptr_out)
        };

        let (lagged_td_error_pinned, lagged_td_error_dev_ptr) = {
            let mut host_ptr: *mut std::ffi::c_void = std::ptr::null_mut();
            let mut dev_ptr_out: u64 = 0;
            unsafe {
                let rc = cudarc::driver::sys::cuMemAllocHost_v2(&mut host_ptr, std::mem::size_of::<f32>());
                assert_eq!(rc, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemAllocHost for lagged_td_error");
                let rc2 = cudarc::driver::sys::cuMemHostGetDevicePointer_v2(&mut dev_ptr_out, host_ptr, 0);
                assert_eq!(rc2, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemHostGetDevicePointer for lagged_td_error");
                *(host_ptr as *mut f32) = 0.0;
            }
            (host_ptr as *mut f32, dev_ptr_out)
        };

        // ── Cross-branch graph message passing ──────────────────────────
        let cpbi_module_graph = stream.context().load_cubin(EXPECTED_Q_CUBIN.to_vec())
            .map_err(|e| MLError::ModelError(format!("cpbi cubin (graph_msg): {e}")))?;
        let graph_msg_kernel = cpbi_module_graph.load_function("branch_graph_message_pass")
            .map_err(|e| MLError::ModelError(format!("branch_graph_message_pass load: {e}")))?;
        // Initialize graph_params: W_gate biases near zero, W_msg near identity (0.1 on diagonal).
        // Layout per edge (15 floats): W_gate[6] then W_msg[9 = 3×3 row-major].
        let mut graph_params_host = [0.0_f32; 60];
        for e in 0..4 {
            // W_msg diagonal entries: indices e*15+6, e*15+6+4, e*15+6+8  (row 0 col 0, row 1 col 1, row 2 col 2)
            graph_params_host[e * 15 + 6 + 0] = 0.1; // row 0, col 0
            graph_params_host[e * 15 + 6 + 4] = 0.1; // row 1, col 1
            graph_params_host[e * 15 + 6 + 8] = 0.1; // row 2, col 2
        }
        // Mapped pinned: host-init via host_ptr; branch_graph_message_pass kernel
        // reads dev_ptr. No DtoD copy per feedback_no_htod_htoh_only_mapped_pinned.
        let graph_params = unsafe { super::mapped_pinned::MappedF32Buffer::new(60) }
            .map_err(|e| MLError::ModelError(format!("graph_params alloc (60 f32): {e}")))?;
        graph_params.write_from_slice(&graph_params_host);
        info!("GpuDqnTrainer: branch_graph_message_pass kernel loaded (60 params, near-identity W_msg init)");

        // ── Diffusion Q-refinement ────────────────────────────────────────────────
        // 2 steps × (W1[24,24]=576 + b1[24]=24 + W2[12,24]=288 + b2[12]=12) = 2 × 900 = 1800 floats.
        const DENOISE_PARAMS_PER_STEP: usize = 576 + 24 + 288 + 12; // 900
        const DENOISE_TOTAL_PARAMS: usize = 2 * DENOISE_PARAMS_PER_STEP; // 1800
        let denoise_adam_m = stream.alloc_zeros::<f32>(DENOISE_TOTAL_PARAMS)
            .map_err(|e| MLError::ModelError(format!("alloc denoise_adam_m: {e}")))?;
        let denoise_adam_v = stream.alloc_zeros::<f32>(DENOISE_TOTAL_PARAMS)
            .map_err(|e| MLError::ModelError(format!("alloc denoise_adam_v: {e}")))?;
        let denoise_buf_a = alloc_f32(&stream, b * 12, "denoise_buf_a")?;
        let denoise_buf_b = alloc_f32(&stream, b * 12, "denoise_buf_b")?;
        // q_var_buf_trainer must match the per-action variance write stride in
        // compute_expected_q (`q_variance[i*total_actions + a]`). The legacy
        // 12-slot exposure layout (3+3+3+3) sized this at b*12, but with the
        // 4-direction factored layout (4+3+3+3 = 13) compute_expected_q writes
        // 13 floats per sample. Sized at b*total_actions to match — downstream
        // denoiser consumers receive `var_stride = total_actions` and read only
        // the first D=12 entries per sample (exposure layout). Compute-sanitizer
        // caught the b*12 sizing as 4 OOB writes at threads 60..63 (commit
        // log: see fix(dqn): compute_expected_q stride 12→total_actions).
        let q_var_total_actions = config.branch_0_size + config.branch_1_size + config.branch_2_size + config.branch_3_size;
        let q_var_buf_trainer = alloc_f32(&stream, b * q_var_total_actions, "q_var_buf_trainer")?;
        // Xavier uniform init for W1 and W2; biases stay zero.
        let mut denoise_params_host = vec![0.0_f32; DENOISE_TOTAL_PARAMS];
        {
            let mut rng = 0xDEAD_C0DE_u64.wrapping_add(DENOISE_TOTAL_PARAMS as u64);
            let mut off = 0usize;
            for _step in 0..2 {
                // W1 [24, 24]: fan_in=24, fan_out=24 → limit = sqrt(6/48)
                let w1_limit = (6.0_f64 / (24 + 24) as f64).sqrt() as f32;
                for w in &mut denoise_params_host[off..off + 576] {
                    rng = rng.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
                    let u = (rng >> 33) as f32 / (1u64 << 31) as f32 - 0.5;
                    *w = u * 2.0 * w1_limit;
                }
                off += 576;
                // b1 [24]: stays zero
                off += 24;
                // W2 [12, 24]: fan_in=24, fan_out=12 → limit = sqrt(6/36)
                let w2_limit = (6.0_f64 / (12 + 24) as f64).sqrt() as f32;
                for w in &mut denoise_params_host[off..off + 288] {
                    rng = rng.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
                    let u = (rng >> 33) as f32 / (1u64 << 31) as f32 - 0.5;
                    *w = u * 2.0 * w2_limit;
                }
                off += 288;
                // b2 [12]: stays zero
                off += 12;
            }
        }
        // Mapped pinned: host-init (Xavier) via host_ptr; q_denoise_step kernel
        // reads dev_ptr (and aux Adam writes back). No DtoD copy per
        // feedback_no_htod_htoh_only_mapped_pinned.
        let denoise_params = unsafe { super::mapped_pinned::MappedF32Buffer::new(DENOISE_TOTAL_PARAMS) }
            .map_err(|e| MLError::ModelError(format!("denoise_params alloc ({DENOISE_TOTAL_PARAMS} f32): {e}")))?;
        denoise_params.write_from_slice(&denoise_params_host);
        let cpbi_module_denoise = stream.context().load_cubin(EXPECTED_Q_CUBIN.to_vec())
            .map_err(|e| MLError::ModelError(format!("cpbi cubin (denoise): {e}")))?;
        let q_denoise_kernel = cpbi_module_denoise.load_function("q_denoise_step")
            .map_err(|e| MLError::ModelError(format!("q_denoise_step load: {e}")))?;
        let q_denoise_backward_kernel = cpbi_module_denoise.load_function("q_denoise_backward")
            .map_err(|e| MLError::ModelError(format!("q_denoise_backward load: {e}")))?;
        let denoise_grad = stream.alloc_zeros::<f32>(DENOISE_TOTAL_PARAMS)
            .map_err(|e| MLError::ModelError(format!("alloc denoise_grad: {e}")))?;
        // denoise_target_q_buf is written by `compute_expected_q` which strides
        // by `total_actions` (= 4+3+3+3 = 13 in the 4-direction factored layout).
        // The legacy `b*12` sizing was the 12-slot exposure layout (3+3+3+3) and
        // produced 4 OOB writes per batch (threads 60..63) under compute-sanitizer.
        // Sized at `b*total_actions` to match the kernel writer; downstream
        // denoiser consumers (q_denoise_backward, denoise_loss_grad) receive
        // `target_stride = total_actions` and read only the first D=12 entries
        // per sample. Same pattern as q_var_buf_trainer (see comment above
        // line 21629).
        // denoise_q_input_buf is a snapshot of q_coord_buf which stays at b*12
        // (the cross-branch attention output narrows 13→12 before the denoiser
        // consumes it), so its sizing is unchanged. The kernel input_stride
        // parameter still threads through for symmetry with target/refined.
        let q_var_total_actions_for_target = config.branch_0_size + config.branch_1_size + config.branch_2_size + config.branch_3_size;
        let denoise_target_q_buf = alloc_f32(&stream, b * q_var_total_actions_for_target, "denoise_target_q_buf")?;
        let denoise_q_input_buf = alloc_f32(&stream, b * 12, "denoise_q_input_buf")?;
        let denoise_norm_partials = stream.alloc_zeros::<f32>(1)
            .map_err(|e| MLError::ModelError(format!("alloc denoise_norm_partials: {e}")))?;
        let denoise_norm_buf = stream.alloc_zeros::<f32>(1)
            .map_err(|e| MLError::ModelError(format!("alloc denoise_norm_buf: {e}")))?;
        let denoise_clip_buf = stream.alloc_zeros::<f32>(1)
            .map_err(|e| MLError::ModelError(format!("alloc denoise_clip_buf: {e}")))?;
        // Pinned device-mapped step counter for denoise Adam
        let denoise_t_pinned: *mut i32 = unsafe {
            let flags = cudarc::driver::sys::CU_MEMHOSTALLOC_DEVICEMAP;
            cudarc::driver::result::malloc_host(std::mem::size_of::<i32>(), flags)
                .map_err(|e| MLError::ModelError(format!("pinned denoise_t alloc: {e}")))?
                as *mut i32
        };
        unsafe { *denoise_t_pinned = 0; }
        let denoise_t_dev_ptr = unsafe {
            let mut dp = 0u64;
            cudarc::driver::sys::cuMemHostGetDevicePointer_v2(&mut dp as *mut u64, denoise_t_pinned.cast(), 0);
            dp
        };
        info!("GpuDqnTrainer: q_denoise_backward kernel loaded (1800 params, denoiser training)");

        // ── Denoise cuBLAS backward pipeline ──────────────────────────────
        const DENOISE_D: usize = 12;
        const DENOISE_H: usize = 24;
        let denoise_input0_buf = alloc_f32(&stream, DENOISE_H * b, "denoise_input0")?;
        let denoise_input1_buf = alloc_f32(&stream, DENOISE_H * b, "denoise_input1")?;
        let denoise_pre_h0_buf = alloc_f32(&stream, DENOISE_H * b, "denoise_pre_h0")?;
        let denoise_pre_h1_buf = alloc_f32(&stream, DENOISE_H * b, "denoise_pre_h1")?;
        let denoise_h0_buf = alloc_f32(&stream, DENOISE_H * b, "denoise_h0")?;
        let denoise_h1_buf = alloc_f32(&stream, DENOISE_H * b, "denoise_h1")?;
        let denoise_q_step0_buf = alloc_f32(&stream, DENOISE_D * b, "denoise_q_step0")?;
        let denoise_d_res_buf = alloc_f32(&stream, DENOISE_D * b, "denoise_d_res")?;
        let denoise_d_h_buf = alloc_f32(&stream, DENOISE_H * b, "denoise_d_h")?;
        // Bias grad partials: ceil(B/256) blocks × max(H=24, D=12) = ceil(B/256)*24
        let denoise_bias_num_blocks = (b + 255) / 256;
        let denoise_bias_grad_partials = alloc_f32(&stream, denoise_bias_num_blocks * DENOISE_H, "denoise_bias_grad_partials")?;

        // Load denoise cuBLAS helper kernels from experience_kernels cubin
        let cpbi_module_denoise_cublas = stream.context().load_cubin(EXPECTED_Q_CUBIN.to_vec())
            .map_err(|e| MLError::ModelError(format!("cpbi cubin (denoise_cublas): {e}")))?;
        let denoise_build_input_kernel = cpbi_module_denoise_cublas.load_function("denoise_build_input")
            .map_err(|e| MLError::ModelError(format!("denoise_build_input load: {e}")))?;
        let denoise_bias_silu_kernel = cpbi_module_denoise_cublas.load_function("denoise_bias_silu")
            .map_err(|e| MLError::ModelError(format!("denoise_bias_silu load: {e}")))?;
        let denoise_silu_bwd_kernel = cpbi_module_denoise_cublas.load_function("denoise_silu_bwd")
            .map_err(|e| MLError::ModelError(format!("denoise_silu_bwd load: {e}")))?;
        let denoise_loss_grad_kernel = cpbi_module_denoise_cublas.load_function("denoise_loss_grad")
            .map_err(|e| MLError::ModelError(format!("denoise_loss_grad load: {e}")))?;
        let denoise_skip_conn_kernel = cpbi_module_denoise_cublas.load_function("denoise_skip_connection")
            .map_err(|e| MLError::ModelError(format!("denoise_skip_connection load: {e}")))?;
        let denoise_bias_grad_p1_kernel = cpbi_module_denoise_cublas.load_function("denoise_bias_grad_p1")
            .map_err(|e| MLError::ModelError(format!("denoise_bias_grad_p1 load: {e}")))?;
        let denoise_bias_grad_p2_kernel = cpbi_module_denoise_cublas.load_function("denoise_bias_grad_p2")
            .map_err(|e| MLError::ModelError(format!("denoise_bias_grad_p2 load: {e}")))?;

        // cuBLAS GEMM descriptors for denoise MLP backward
        let denoise_lt_handle = shared_cublas.lt_handle.0;
        let denoise_lt_ws_size = shared_cublas.lt_workspace_size;
        // W1 stored row-major [H=24, H=24]. cuBLAS sees it as col-major [H, H] = W1^T.
        // Forward: pre_h[H,B] = W1 @ input[H,B]. With W1 row-major = W1^T col-major:
        //   cuBLAS: C[H,B] = W1_cm^T[H,H] @ X_cm[H,B] → TRANSA=T, TRANSB=N
        let denoise_gemm_fwd_hh = create_mamba2_gemm_desc(
            denoise_lt_handle, 1, 0,  // TRANSA=T, TRANSB=N
            DENOISE_H, b, DENOISE_H,  // m=24, n=B, k=24
            DENOISE_H, DENOISE_H, DENOISE_H,  // lda=24, ldb=24, ldc=24
            denoise_lt_ws_size, "denoise_fwd_hh",
        )?;

        // W2 stored row-major [D=12, H=24]. cuBLAS sees it as col-major [H, D] = W2^T.
        // Forward: out[D,B] = W2 @ h[H,B]. With W2 row-major = W2^T col-major:
        //   cuBLAS: C[D,B] = W2_cm^T[D,H] @ h_cm[H,B] → TRANSA=T, TRANSB=N
        let denoise_gemm_fwd_dh = create_mamba2_gemm_desc(
            denoise_lt_handle, 1, 0,  // TRANSA=T, TRANSB=N
            DENOISE_D, b, DENOISE_H,  // m=12, n=B, k=24
            DENOISE_H, DENOISE_H, DENOISE_D,  // lda=24 (stored [H,D]->ld=H), ldb=24, ldc=12
            denoise_lt_ws_size, "denoise_fwd_dh",
        )?;

        // Backward dW1: we want row-major dW1[H, H].
        // dW1 = d_pre[H,B] @ input[H,B]^T in math (both col-major).
        // For correct row-major output: compute dW1^T col-major = input @ d_pre^T.
        // cuBLAS: C[H,H] = input[H,B] @ d_pre[H,B]^T -> TRANSA=N, TRANSB=T, m=H, n=H, k=B
        let denoise_gemm_dw1 = create_mamba2_gemm_desc(
            denoise_lt_handle, 0, 1,  // TRANSA=N, TRANSB=T
            DENOISE_H, DENOISE_H, b,  // m=24, n=24, k=B
            DENOISE_H, DENOISE_H, DENOISE_H,  // lda=24, ldb=24, ldc=24
            denoise_lt_ws_size, "denoise_bwd_dw1",
        )?;

        // Backward dW2: we want row-major dW2[D, H].
        // dW2 = d_res[D,B] @ h[H,B]^T in math.
        // For correct row-major output: compute dW2^T col-major = h @ d_res^T.
        // cuBLAS: C[H,D] = h[H,B] @ d_res[D,B]^T -> TRANSA=N, TRANSB=T, m=H, n=D, k=B
        let denoise_gemm_dw2 = create_mamba2_gemm_desc(
            denoise_lt_handle, 0, 1,  // TRANSA=N, TRANSB=T
            DENOISE_H, DENOISE_D, b,  // m=24, n=12, k=B
            DENOISE_H, DENOISE_D, DENOISE_H,  // lda=24, ldb=12, ldc=24
            denoise_lt_ws_size, "denoise_bwd_dw2",
        )?;

        // Backward d_h = W2^T @ d_res. W2 stored row-major [D,H] = col-major [H,D].
        // We need C[H,B] = W2^T[H,D] @ d_res[D,B].
        // W2 col-major has shape [H,D] with ld=H. No transpose needed.
        // cuBLAS: C[H,B] = A[H,D] @ B[D,B] -> TRANSA=N, TRANSB=N, m=H, n=B, k=D
        let denoise_gemm_bwd_dh = create_mamba2_gemm_desc(
            denoise_lt_handle, 0, 0,  // TRANSA=N, TRANSB=N
            DENOISE_H, b, DENOISE_D,  // m=24, n=B, k=12
            DENOISE_H, DENOISE_D, DENOISE_H,  // lda=24 (W2 col-major ld), ldb=12, ldc=24
            denoise_lt_ws_size, "denoise_bwd_dh",
        )?;

        info!(
            DENOISE_D, DENOISE_H, b,
            input_bytes = DENOISE_H * b * 4 * 2,
            "GpuDqnTrainer: Denoise cuBLAS backward initialized (10 scratch bufs, 5 GEMM descs, 7 kernels)"
        );

        // ── xLSTM temporal context (mLSTM cell, 528 params) ──────────────────────
        // 6 weight matrices × [11 input_dim, 8 head_dim] = 6 × 88 = 528 floats.
        // Xavier uniform: scale = sqrt(2 / (fan_in + fan_out)) = sqrt(2 / 19) ≈ 0.3244
        const QLSTM_INPUT_DIM: usize = 11;
        const QLSTM_HEAD_DIM: usize = 8;
        const QLSTM_WEIGHT_COUNT: usize = 6 * QLSTM_INPUT_DIM * QLSTM_HEAD_DIM; // 528
        let qlstm_xavier_scale = (2.0_f64 / (QLSTM_INPUT_DIM + QLSTM_HEAD_DIM) as f64).sqrt() as f32;
        let mut qlstm_weights_host = vec![0.0_f32; QLSTM_WEIGHT_COUNT];
        {
            let mut rng = 0xC0FFEE_DEAD_u64.wrapping_add(QLSTM_WEIGHT_COUNT as u64);
            for w in &mut qlstm_weights_host {
                rng = rng.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
                let u = (rng >> 33) as f32 / (1u64 << 31) as f32 - 0.5;
                *w = u * 2.0 * qlstm_xavier_scale;
            }
        }
        // Mapped pinned: host-init (Xavier uniform) via host_ptr; xLSTM kernels
        // (qlstm_step + qlstm_train_step) read dev_ptr and SGD-update in place.
        // No DtoD copy per feedback_no_htod_htoh_only_mapped_pinned.
        let qlstm_weights = unsafe { super::mapped_pinned::MappedF32Buffer::new(QLSTM_WEIGHT_COUNT) }
            .map_err(|e| MLError::ModelError(format!("qlstm_weights alloc ({QLSTM_WEIGHT_COUNT} f32): {e}")))?;
        qlstm_weights.write_from_slice(&qlstm_weights_host);
        let qlstm_c = stream.alloc_zeros::<f32>(QLSTM_HEAD_DIM * QLSTM_HEAD_DIM) // [64]
            .map_err(|e| MLError::ModelError(format!("alloc qlstm_c: {e}")))?;
        let qlstm_n = stream.alloc_zeros::<f32>(QLSTM_HEAD_DIM) // [8]
            .map_err(|e| MLError::ModelError(format!("alloc qlstm_n: {e}")))?;
        let qlstm_context = stream.alloc_zeros::<f32>(QLSTM_HEAD_DIM) // [8]
            .map_err(|e| MLError::ModelError(format!("alloc qlstm_context: {e}")))?;
        // per_branch_q_gaps — pinned device-mapped (zero-copy write from CPU).
        let per_branch_q_gaps_pinned: *mut f32 = unsafe {
            let flags = cudarc::driver::sys::CU_MEMHOSTALLOC_DEVICEMAP;
            cudarc::driver::result::malloc_host(4 * std::mem::size_of::<f32>(), flags)
                .map_err(|e| MLError::ModelError(format!("pinned per_branch_q_gaps alloc: {e}")))?
                as *mut f32
        };
        unsafe {
            std::ptr::write_bytes(per_branch_q_gaps_pinned, 0, 4);
        }
        let per_branch_q_gaps_dev_ptr = unsafe {
            let mut dp = 0u64;
            cudarc::driver::sys::cuMemHostGetDevicePointer_v2(&mut dp as *mut u64, per_branch_q_gaps_pinned.cast(), 0);
            dp
        };
        let cpbi_module_qlstm = stream.context().load_cubin(EXPECTED_Q_CUBIN.to_vec())
            .map_err(|e| MLError::ModelError(format!("cpbi cubin (qlstm): {e}")))?;
        let qlstm_step_kernel = cpbi_module_qlstm.load_function("qlstm_step")
            .map_err(|e| MLError::ModelError(format!("qlstm_step load: {e}")))?;
        let qlstm_train_kernel = cpbi_module_qlstm.load_function("qlstm_train_step")
            .map_err(|e| MLError::ModelError(format!("qlstm_train_step load: {e}")))?;
        // prev_q_mean — pinned device-mapped scalar for qlstm training signal.
        let prev_q_mean_pinned: *mut f32 = unsafe {
            let flags = cudarc::driver::sys::CU_MEMHOSTALLOC_DEVICEMAP;
            cudarc::driver::result::malloc_host(std::mem::size_of::<f32>(), flags)
                .map_err(|e| MLError::ModelError(format!("pinned prev_q_mean alloc: {e}")))?
                as *mut f32
        };
        unsafe { *prev_q_mean_pinned = 0.0; }
        let prev_q_mean_dev_ptr = unsafe {
            let mut dp = 0u64;
            cudarc::driver::sys::cuMemHostGetDevicePointer_v2(&mut dp as *mut u64, prev_q_mean_pinned.cast(), 0);
            dp
        };
        info!("GpuDqnTrainer: qlstm_step + qlstm_train_step kernels loaded (528 params, xLSTM mLSTM cell, scale={:.4})", qlstm_xavier_scale);

        // ── Mamba2 temporal scan: rolling history + selective SSM ──────────
        // W_A and W_B project from h_history width (SH2+OFI_EMBED_DIM) to STATE_D.
        // W_C projects from scan output (STATE_D) to SH2, so it stays at SH2 width.
        let sh2 = config.shared_h2;
        let h_width = sh2 + OFI_EMBED_DIM; // history row width: trunk + OFI embedding
        let wa_size = h_width * MAMBA2_STATE_DIM;
        let wb_size = h_width * MAMBA2_STATE_DIM;
        let wc_size = sh2 * MAMBA2_STATE_DIM;
        let mamba2_param_count = wa_size + wb_size + wc_size;
        let mamba2_h_history = stream.alloc_zeros::<f32>(b * MAMBA2_HISTORY_K * h_width)
            .map_err(|e| MLError::ModelError(format!("alloc mamba2_h_history: {e}")))?;
        let mamba2_h_enriched = stream.alloc_zeros::<f32>(b * sh2)
            .map_err(|e| MLError::ModelError(format!("alloc mamba2_h_enriched: {e}")))?;
        let mut mamba2_params = stream.alloc_zeros::<f32>(mamba2_param_count)
            .map_err(|e| MLError::ModelError(format!("alloc mamba2_params: {e}")))?;
        let mamba2_grad = stream.alloc_zeros::<f32>(mamba2_param_count)
            .map_err(|e| MLError::ModelError(format!("alloc mamba2_grad: {e}")))?;
        let mamba2_adam_m = stream.alloc_zeros::<f32>(mamba2_param_count)
            .map_err(|e| MLError::ModelError(format!("alloc mamba2_adam_m: {e}")))?;
        let mamba2_adam_v = stream.alloc_zeros::<f32>(mamba2_param_count)
            .map_err(|e| MLError::ModelError(format!("alloc mamba2_adam_v: {e}")))?;
        // Xavier init for mamba2_params (mapped pinned upload, no HtoD)
        {
            let scale = (2.0_f32 / (h_width + MAMBA2_STATE_DIM) as f32).sqrt();
            let init_data: Vec<f32> = (0..mamba2_param_count).map(|j| {
                let hash = (j as u32).wrapping_mul(2654435761).wrapping_add(0xDEADBEEF);
                let u = (hash as f32) / (u32::MAX as f32) * 2.0 - 1.0;
                u * scale
            }).collect();
            update_via_mapped_f32(&stream, &mut mamba2_params, &init_data, "mamba2_params_xavier")?;
        }
        let mamba2_module = stream.context().load_cubin(MAMBA2_CUBIN.to_vec())
            .map_err(|e| MLError::ModelError(format!("mamba2 cubin load: {e}")))?;
        let mamba2_update_kernel = mamba2_module.load_function("mamba2_update_history")
            .map_err(|e| MLError::ModelError(format!("mamba2_update_history load: {e}")))?;
        let mamba2_copy_enriched_kernel = mamba2_module.load_function("mamba2_copy_enriched")
            .map_err(|e| MLError::ModelError(format!("mamba2_copy_enriched load: {e}")))?;
        let isv_temporal_route_kernel = mamba2_module.load_function("isv_temporal_route")
            .map_err(|e| MLError::ModelError(format!("isv_temporal_route load: {e}")))?;
        // ── Mamba2 cuBLAS projection kernels ──
        let mamba2_scan_proj_fwd_kernel = mamba2_module.load_function("mamba2_scan_projected_fwd")
            .map_err(|e| MLError::ModelError(format!("mamba2_scan_projected_fwd load: {e}")))?;
        let mamba2_scan_proj_bwd_kernel = mamba2_module.load_function("mamba2_scan_projected_bwd")
            .map_err(|e| MLError::ModelError(format!("mamba2_scan_projected_bwd load: {e}")))?;
        let mamba2_scale_d_enriched_kernel = mamba2_module.load_function("mamba2_scale_d_enriched")
            .map_err(|e| MLError::ModelError(format!("mamba2_scale_d_enriched load: {e}")))?;

        // ── Mamba2 cuBLAS scratch buffers ──
        let bk = b * MAMBA2_HISTORY_K;
        let mamba2_a_proj = stream.alloc_zeros::<f32>(bk * MAMBA2_STATE_DIM)
            .map_err(|e| MLError::ModelError(format!("alloc mamba2_a_proj: {e}")))?;
        let mamba2_b_proj = stream.alloc_zeros::<f32>(bk * MAMBA2_STATE_DIM)
            .map_err(|e| MLError::ModelError(format!("alloc mamba2_b_proj: {e}")))?;
        let mamba2_d_gate = stream.alloc_zeros::<f32>(bk * MAMBA2_STATE_DIM)
            .map_err(|e| MLError::ModelError(format!("alloc mamba2_d_gate: {e}")))?;
        let mamba2_d_x = stream.alloc_zeros::<f32>(bk * MAMBA2_STATE_DIM)
            .map_err(|e| MLError::ModelError(format!("alloc mamba2_d_x: {e}")))?;
        let mamba2_d_context = stream.alloc_zeros::<f32>(b * MAMBA2_STATE_DIM)
            .map_err(|e| MLError::ModelError(format!("alloc mamba2_d_context: {e}")))?;
        let mamba2_d_tw = stream.alloc_zeros::<f32>(b * sh2)
            .map_err(|e| MLError::ModelError(format!("alloc mamba2_d_tw: {e}")))?;

        // ── Mamba2 cuBLAS GEMM descriptors ──
        // W_A/W_B are [SH2+OFI, STATE_D] row-major = [STATE_D, SH2+OFI] col-major with ld=STATE_D
        // h_history is [B*K, SH2+OFI] row-major = [SH2+OFI, B*K] col-major with ld=SH2+OFI
        // Forward: C[STATE_D, B*K] = W_cm[STATE_D, SH2+OFI] @ h_cm[SH2+OFI, B*K]
        let lt_handle = shared_cublas.lt_handle.0;
        let lt_ws_size = shared_cublas.lt_workspace_size;

        let mamba2_gemm_proj = create_mamba2_gemm_desc(
            lt_handle, 0, 0,
            MAMBA2_STATE_DIM, bk, h_width,
            MAMBA2_STATE_DIM, h_width, MAMBA2_STATE_DIM,
            lt_ws_size, "mamba2_fwd_proj",
        )?;

        // Backward dW_A/dW_B: C[STATE_D, SH2+OFI] = d_cm[STATE_D, B*K] @ h_cm^T[B*K, SH2+OFI]
        let mamba2_gemm_dw = create_mamba2_gemm_desc(
            lt_handle, 0, 1,
            MAMBA2_STATE_DIM, h_width, bk,
            MAMBA2_STATE_DIM, h_width, MAMBA2_STATE_DIM,
            lt_ws_size, "mamba2_bwd_dw",
        )?;

        // Backward dW_C: C[STATE_D, SH2] = xK_cm[STATE_D, B] @ dtw_cm^T[B, SH2]
        // W_C stays at SH2 width — it projects from scan output, not h_history
        let mamba2_gemm_dwc = create_mamba2_gemm_desc(
            lt_handle, 0, 1,
            MAMBA2_STATE_DIM, sh2, b,
            MAMBA2_STATE_DIM, sh2, MAMBA2_STATE_DIM,
            lt_ws_size, "mamba2_bwd_dwc",
        )?;

        // Backward d_h_history: C[h_width, B*K] = W_cm^T[h_width, STATE_D] @ d[STATE_D, B*K]
        // W_A/W_B col-major layout is [STATE_D, h_width] with ld=STATE_D.
        // TRANSA=T transposes to [h_width, STATE_D]. TRANSB=N on d[STATE_D, B*K].
        let mamba2_gemm_dh = create_mamba2_gemm_desc(
            lt_handle, 1, 0,
            h_width, bk, MAMBA2_STATE_DIM,
            MAMBA2_STATE_DIM, MAMBA2_STATE_DIM, h_width,
            lt_ws_size, "mamba2_bwd_dh",
        )?;

        // ── Mamba2 input gradient buffers ──
        let d_h_history_buf = stream.alloc_zeros::<f32>(h_width * bk)
            .map_err(|e| MLError::ModelError(format!("alloc d_h_history_buf: {e}")))?;
        let d_ofi_embed_mamba2 = stream.alloc_zeros::<f32>(OFI_EMBED_DIM * b)
            .map_err(|e| MLError::ModelError(format!("alloc d_ofi_embed_mamba2: {e}")))?;
        let extract_ofi_embed_grad_kernel = mamba2_module.load_function("extract_ofi_embed_grad")
            .map_err(|e| MLError::ModelError(format!("extract_ofi_embed_grad load: {e}")))?;

        info!(
            bk, state_dim = MAMBA2_STATE_DIM, sh2, h_width,
            a_proj_bytes = bk * MAMBA2_STATE_DIM * 4,
            "GpuDqnTrainer: Mamba2 cuBLAS projections initialized (6 scratch buffers, 3 GEMM descs)"
        );
        info!("GpuDqnTrainer: Mamba2 temporal scan loaded ({} params, K={}, state_dim={}, h_width={})", mamba2_param_count, MAMBA2_HISTORY_K, MAMBA2_STATE_DIM, h_width);

        // v8: Pessimistic Q-value initialization — shift value head bias to -0.1
        // Pessimistic Q-init REMOVED — incompatible with per-sample support.
        // Xavier init gives near-zero value head output, correct for adaptive C51.

        // ── Homeostatic regularizer (G16) ────────────────────────────────────
        let cpbi_module_homeo = stream.context().load_cubin(EXPECTED_Q_CUBIN.to_vec())
            .map_err(|e| MLError::ModelError(format!("cpbi cubin (homeostatic): {e}")))?;
        let homeostatic_kernel = cpbi_module_homeo.load_function("homeostatic_regularizer")
            .map_err(|e| MLError::ModelError(format!("homeostatic_regularizer load: {e}")))?;
        let homeostatic_penalties_buf = stream.alloc_zeros::<f32>(HOMEOSTATIC_N_OBS)
            .map_err(|e| MLError::ModelError(format!("alloc homeostatic_penalties: {e}")))?;
        let homeostatic_total_buf = stream.alloc_zeros::<f32>(1)
            .map_err(|e| MLError::ModelError(format!("alloc homeostatic_total: {e}")))?;

        // Pinned device-mapped: observables [6]
        let (homeostatic_obs_pinned, homeostatic_obs_dev_ptr) = {
            let mut host_ptr: *mut std::ffi::c_void = std::ptr::null_mut();
            let mut dev_ptr_out: u64 = 0;
            unsafe {
                let rc = cudarc::driver::sys::cuMemAllocHost_v2(
                    &mut host_ptr,
                    HOMEOSTATIC_N_OBS * std::mem::size_of::<f32>(),
                );
                assert_eq!(rc, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemAllocHost for homeostatic_obs");
                let rc2 = cudarc::driver::sys::cuMemHostGetDevicePointer_v2(
                    &mut dev_ptr_out,
                    host_ptr,
                    0,
                );
                assert_eq!(rc2, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemHostGetDevicePointer for homeostatic_obs");
                // Zero-initialize
                std::ptr::write_bytes(host_ptr as *mut f32, 0, HOMEOSTATIC_N_OBS);
            }
            (host_ptr as *mut f32, dev_ptr_out)
        };

        // Pinned device-mapped: targets [6]
        let (homeostatic_targets_pinned, homeostatic_targets_dev_ptr) = {
            let mut host_ptr: *mut std::ffi::c_void = std::ptr::null_mut();
            let mut dev_ptr_out: u64 = 0;
            unsafe {
                let rc = cudarc::driver::sys::cuMemAllocHost_v2(
                    &mut host_ptr,
                    HOMEOSTATIC_N_OBS * std::mem::size_of::<f32>(),
                );
                assert_eq!(rc, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemAllocHost for homeostatic_targets");
                let rc2 = cudarc::driver::sys::cuMemHostGetDevicePointer_v2(
                    &mut dev_ptr_out,
                    host_ptr,
                    0,
                );
                assert_eq!(rc2, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "cuMemHostGetDevicePointer for homeostatic_targets");
                // Sensible defaults before calibration: [Q-mean=0, atom_util=0.85, branch_corr=0.1, trade_freq=0, Q-gap=1.0, Q-var=0.5]
                let tgt = host_ptr as *mut f32;
                *tgt.add(0) = 0.0;
                *tgt.add(1) = 0.85;
                *tgt.add(2) = 0.1;
                *tgt.add(3) = 0.0;
                *tgt.add(4) = 1.0;
                *tgt.add(5) = 0.5;
            }
            (host_ptr as *mut f32, dev_ptr_out)
        };
        info!("GpuDqnTrainer: homeostatic_regularizer kernel loaded ({} observables, pinned device-mapped)", HOMEOSTATIC_N_OBS);

        // ── Multi-horizon value head buffers (5-bar and 20-bar) ──
        let v_logits_5bar = stream.alloc_zeros::<f32>(b * config.num_atoms)
            .map_err(|e| MLError::ModelError(format!("alloc v_logits_5bar: {e}")))?;
        let v_logits_20bar = stream.alloc_zeros::<f32>(b * config.num_atoms)
            .map_err(|e| MLError::ModelError(format!("alloc v_logits_20bar: {e}")))?;
        let save_h_v_5bar = stream.alloc_zeros::<f32>(b * config.value_h)
            .map_err(|e| MLError::ModelError(format!("alloc save_h_v_5bar: {e}")))?;
        let save_h_v_20bar = stream.alloc_zeros::<f32>(b * config.value_h)
            .map_err(|e| MLError::ModelError(format!("alloc save_h_v_20bar: {e}")))?;
        let v_logits_blended = stream.alloc_zeros::<f32>(b * config.num_atoms)
            .map_err(|e| MLError::ModelError(format!("alloc v_logits_blended: {e}")))?;
        info!("GpuDqnTrainer: multi-horizon value heads allocated (5-bar + 20-bar)");

        // ── Learned risk management buffers ──
        let risk_hidden_buf = stream.alloc_zeros::<f32>(b * config.adv_h)
            .map_err(|e| MLError::ModelError(format!("risk_hidden_buf alloc: {e}")))?;
        let risk_budget_buf = stream.alloc_zeros::<f32>(b)
            .map_err(|e| MLError::ModelError(format!("risk_budget_buf alloc: {e}")))?;
        let cvar_alpha_buf = stream.alloc_zeros::<f32>(b)
            .map_err(|e| MLError::ModelError(format!("cvar_alpha_buf alloc: {e}")))?;
        let commit_lambda_buf = stream.alloc_zeros::<f32>(b)
            .map_err(|e| MLError::ModelError(format!("commit_lambda_buf alloc: {e}")))?;
        let q_mag_pre_risk = stream.alloc_zeros::<f32>(b * config.branch_1_size)
            .map_err(|e| MLError::ModelError(format!("q_mag_pre_risk alloc: {e}")))?;
        let risk_param_count = config.adv_h * (config.shared_h2 + 13) + config.adv_h + config.adv_h + 1;
        let risk_grad_buf = stream.alloc_zeros::<f32>(risk_param_count)
            .map_err(|e| MLError::ModelError(format!("risk_grad_buf alloc: {e}")))?;

        // ── ISV forward output buffers ──
        let isv_embedding_buf = stream.alloc_zeros::<f32>(ISV_EMB_DIM)
            .map_err(|e| MLError::ModelError(format!("isv_embedding_buf alloc: {e}")))?;
        let branch_gate_buf = stream.alloc_zeros::<f32>(4)
            .map_err(|e| MLError::ModelError(format!("branch_gate_buf alloc: {e}")))?;
        let gamma_mod_buf = stream.alloc_zeros::<f32>(1)
            .map_err(|e| MLError::ModelError(format!("gamma_mod_buf alloc: {e}")))?;
        let temporal_weight_buf = stream.alloc_zeros::<f32>(sh2)
            .map_err(|e| MLError::ModelError(format!("temporal_weight_buf alloc: {e}")))?;
        let gamma_buf = stream.alloc_zeros::<f32>(b)
            .map_err(|e| MLError::ModelError(format!("gamma_buf alloc: {e}")))?;
        let predicted_error_buf = stream.alloc_zeros::<f32>(b)
            .map_err(|e| MLError::ModelError(format!("predicted_error_buf alloc: {e}")))?;
        let plan_params_buf = stream.alloc_zeros::<f32>(b * 6)
            .map_err(|e| MLError::ModelError(format!("plan_params_buf alloc: {e}")))?;
        let trade_plan_hidden_buf = alloc_f32(&stream, b * config.adv_h, "trade_plan_hidden")?;
        let trade_plan_pre_out_buf = alloc_f32(&stream, b * 6, "trade_plan_pre_out")?;

        // ── OFI embed MLP buffers (OFI_EMBED_IN → OFI_EMBED_DIM) ──
        let ofi_embed_input_buf = alloc_f32(&stream, b * OFI_EMBED_IN, "ofi_embed_input")?;
        let ofi_embed_output_buf = alloc_f32(&stream, b * OFI_EMBED_DIM, "ofi_embed_output")?;
        let ofi_embed_w = alloc_f32(&stream, OFI_EMBED_W_COUNT, "ofi_embed_w")?;
        // Zero-init OFI embed weights so Mamba2/attention start with clean h_s2.
        // The embed produces zeros at init → h_history = [h_s2; zeros] → no corruption.
        // Weights learn from zero as gradients flow back from Mamba2/attention.
        // (Previous Xavier init with scale ~0.27 injected noise that corrupted trunk features.)
        let ofi_embed_b = alloc_f32(&stream, OFI_EMBED_DIM, "ofi_embed_b")?; // zero-init bias
        info!(
            "GpuDqnTrainer: OFI embed MLP buffers allocated ({}→{}, {} params)",
            OFI_EMBED_IN, OFI_EMBED_DIM, OFI_EMBED_TOTAL_PARAMS,
        );

        // ── OFI embed MLP backward + Adam buffers ──
        // Contiguous params buffer: copy W then b for Adam
        let ofi_embed_params = alloc_f32(&stream, OFI_EMBED_TOTAL_PARAMS, "ofi_embed_params")?;
        {
            // Copy W[OFI_EMBED_W_COUNT] from ofi_embed_w into params[0..OFI_EMBED_W_COUNT]
            let w_src = ofi_embed_w.raw_ptr();
            let p_dst = ofi_embed_params.raw_ptr();
            let w_bytes = OFI_EMBED_W_COUNT * std::mem::size_of::<f32>();
            unsafe {
                cudarc::driver::sys::cuMemcpyDtoDAsync_v2(p_dst, w_src, w_bytes, stream.cu_stream());
            }
            // b[OFI_EMBED_DIM] is zero-init, tail of params already zeroed by alloc_zeros
        }
        let ofi_embed_grad_w = alloc_f32(&stream, OFI_EMBED_W_COUNT, "ofi_embed_grad_w")?;
        let ofi_embed_grad_b = alloc_f32(&stream, OFI_EMBED_DIM, "ofi_embed_grad_b")?;
        let ofi_embed_grad = alloc_f32(&stream, OFI_EMBED_TOTAL_PARAMS, "ofi_embed_grad")?;
        let d_ofi_embed_combined = alloc_f32(&stream, OFI_EMBED_DIM * b, "d_ofi_embed_combined")?;
        let ofi_embed_adam_m = alloc_f32(&stream, OFI_EMBED_TOTAL_PARAMS, "ofi_embed_adam_m")?;
        let ofi_embed_adam_v = alloc_f32(&stream, OFI_EMBED_TOTAL_PARAMS, "ofi_embed_adam_v")?;
        let ofi_embed_norm_partials = alloc_f32(&stream, 1, "ofi_embed_norm_partials")?;
        let ofi_embed_norm_buf = alloc_f32(&stream, 1, "ofi_embed_norm_buf")?;
        let ofi_embed_clip_buf = alloc_f32(&stream, 1, "ofi_embed_clip_buf")?;
        // Pinned device-mapped step counter for OFI embed Adam
        let ofi_embed_t_pinned: *mut i32 = unsafe {
            let flags = cudarc::driver::sys::CU_MEMHOSTALLOC_DEVICEMAP;
            cudarc::driver::result::malloc_host(std::mem::size_of::<i32>(), flags)
                .map_err(|e| MLError::ModelError(format!("pinned ofi_embed_t alloc: {e}")))?
                as *mut i32
        };
        unsafe { *ofi_embed_t_pinned = 0; }
        let ofi_embed_t_dev_ptr = unsafe {
            let mut dp = 0u64;
            cudarc::driver::sys::cuMemHostGetDevicePointer_v2(&mut dp as *mut u64, ofi_embed_t_pinned.cast(), 0);
            dp
        };
        let ofi_embed_bias_num_blocks = ((b + 255) / 256) as usize;
        let ofi_embed_bias_grad_partials = alloc_f32(&stream, ofi_embed_bias_num_blocks * OFI_EMBED_DIM, "ofi_embed_bias_grad_partials")?;
        info!("GpuDqnTrainer: OFI embed backward buffers allocated ({OFI_EMBED_TOTAL_PARAMS} params, Adam + bias reduce)");

        // ── SP14 Layer C aux trunk params + Adam state (Phase C.2, 2026-05-08) ──
        // 3-layer MLP (encoder_out_dim → 256 → 128 → SH2) for the auxiliary
        // next-bar head's separate representation pipeline. Allocation is
        // cold-path (constructor) only — forward/backward kernels land in
        // Phases C.3-C.5 per `feedback_no_partial_refactor` (the C.2 commit
        // is allocation-only by design; Phase C.5 wires every consumer in
        // one atomic commit). Adam m/v are zero-init via `alloc_zeros`.
        // Weight tensors get Kaiming-He init (`std = sqrt(2/fan_in)`)
        // sampled via Box-Muller from the same LCG family (mul/add
        // constants from `xavier_init_dqn_weights` on this file at line
        // ~30135 — matches denoise + spec-norm init pattern).
        let aux_trunk_h1: usize = 256;
        let aux_trunk_h2: usize = 128;
        let aux_trunk_in_dim = config.shared_h1; // encoder output dim (h_s1's output, GRN trunk's input)
        let aux_trunk_out_dim = config.shared_h2; // matches existing aux head input dim (h_s2 shape)
        let aux_trunk_w1_count = aux_trunk_in_dim * aux_trunk_h1;
        let aux_trunk_w2_count = aux_trunk_h1 * aux_trunk_h2;
        let aux_trunk_w3_count = aux_trunk_h2 * aux_trunk_out_dim;

        // Kaiming-He sampler: Box-Muller from LCG-uniform (cold path is fine,
        // matches existing `xavier_init_dqn_weights` LCG state-machine pattern).
        // PCG64-mul/add constants identical to `denoise_params_host` init above.
        fn aux_trunk_kaiming_he_fill(buf: &mut [f32], fan_in: usize, rng: &mut u64) {
            let stddev = (2.0_f64 / fan_in as f64).sqrt() as f32;
            let mut i = 0;
            while i + 1 < buf.len() {
                // LCG step ×2 → two uniforms in (0, 1].
                *rng = rng.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
                let u1_raw = ((*rng >> 33) as f32 / (1u64 << 31) as f32).max(f32::MIN_POSITIVE);
                *rng = rng.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
                let u2 = (*rng >> 33) as f32 / (1u64 << 31) as f32;
                // Box-Muller: z0/z1 ~ N(0, 1) independent.
                let r = (-2.0_f32 * u1_raw.ln()).sqrt();
                let theta = 2.0_f32 * std::f32::consts::PI * u2;
                let z0 = r * theta.cos();
                let z1 = r * theta.sin();
                buf[i]     = z0 * stddev;
                buf[i + 1] = z1 * stddev;
                i += 2;
            }
            // Tail (odd-length): single Box-Muller sample, take z0 only.
            if i < buf.len() {
                *rng = rng.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
                let u1_raw = ((*rng >> 33) as f32 / (1u64 << 31) as f32).max(f32::MIN_POSITIVE);
                *rng = rng.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
                let u2 = (*rng >> 33) as f32 / (1u64 << 31) as f32;
                let r = (-2.0_f32 * u1_raw.ln()).sqrt();
                let theta = 2.0_f32 * std::f32::consts::PI * u2;
                buf[i] = r * theta.cos() * stddev;
            }
        }

        // Seed: deterministic per-tensor (fan_in carried in seed so re-runs
        // with same config produce bit-identical init, matching the
        // `xavier_init_dqn_weights` convention of seed-from-total).
        let aux_trunk_seed_base = 0xA00B_5_C04u64
            .wrapping_add((aux_trunk_w1_count + aux_trunk_w2_count + aux_trunk_w3_count) as u64);

        let mut aux_trunk_w1_host = vec![0.0_f32; aux_trunk_w1_count];
        let mut rng_w1 = aux_trunk_seed_base.wrapping_add(1);
        aux_trunk_kaiming_he_fill(&mut aux_trunk_w1_host, aux_trunk_in_dim, &mut rng_w1);
        let aux_trunk_w1 = upload_via_mapped_f32(
            &stream, aux_trunk_w1_count, &aux_trunk_w1_host, "aux_trunk_w1",
        )?;
        let aux_trunk_b1 = alloc_f32(&stream, aux_trunk_h1, "aux_trunk_b1")?;
        let aux_trunk_w1_m = alloc_f32(&stream, aux_trunk_w1_count, "aux_trunk_w1_m")?;
        let aux_trunk_w1_v = alloc_f32(&stream, aux_trunk_w1_count, "aux_trunk_w1_v")?;
        let aux_trunk_b1_m = alloc_f32(&stream, aux_trunk_h1, "aux_trunk_b1_m")?;
        let aux_trunk_b1_v = alloc_f32(&stream, aux_trunk_h1, "aux_trunk_b1_v")?;

        let mut aux_trunk_w2_host = vec![0.0_f32; aux_trunk_w2_count];
        let mut rng_w2 = aux_trunk_seed_base.wrapping_add(2);
        aux_trunk_kaiming_he_fill(&mut aux_trunk_w2_host, aux_trunk_h1, &mut rng_w2);
        let aux_trunk_w2 = upload_via_mapped_f32(
            &stream, aux_trunk_w2_count, &aux_trunk_w2_host, "aux_trunk_w2",
        )?;
        let aux_trunk_b2 = alloc_f32(&stream, aux_trunk_h2, "aux_trunk_b2")?;
        let aux_trunk_w2_m = alloc_f32(&stream, aux_trunk_w2_count, "aux_trunk_w2_m")?;
        let aux_trunk_w2_v = alloc_f32(&stream, aux_trunk_w2_count, "aux_trunk_w2_v")?;
        let aux_trunk_b2_m = alloc_f32(&stream, aux_trunk_h2, "aux_trunk_b2_m")?;
        let aux_trunk_b2_v = alloc_f32(&stream, aux_trunk_h2, "aux_trunk_b2_v")?;

        let mut aux_trunk_w3_host = vec![0.0_f32; aux_trunk_w3_count];
        let mut rng_w3 = aux_trunk_seed_base.wrapping_add(3);
        aux_trunk_kaiming_he_fill(&mut aux_trunk_w3_host, aux_trunk_h2, &mut rng_w3);
        let aux_trunk_w3 = upload_via_mapped_f32(
            &stream, aux_trunk_w3_count, &aux_trunk_w3_host, "aux_trunk_w3",
        )?;
        let aux_trunk_b3 = alloc_f32(&stream, aux_trunk_out_dim, "aux_trunk_b3")?;
        let aux_trunk_w3_m = alloc_f32(&stream, aux_trunk_w3_count, "aux_trunk_w3_m")?;
        let aux_trunk_w3_v = alloc_f32(&stream, aux_trunk_w3_count, "aux_trunk_w3_v")?;
        let aux_trunk_b3_m = alloc_f32(&stream, aux_trunk_out_dim, "aux_trunk_b3_m")?;
        let aux_trunk_b3_v = alloc_f32(&stream, aux_trunk_out_dim, "aux_trunk_b3_v")?;

        let aux_trunk_total_params =
            aux_trunk_w1_count + aux_trunk_h1
            + aux_trunk_w2_count + aux_trunk_h2
            + aux_trunk_w3_count + aux_trunk_out_dim;
        info!(
            "SP14-C.2: aux trunk allocated ({}→{}→{}→{}, {} params, Kaiming-He w/ Box-Muller, biases zero, Adam m/v zero)",
            aux_trunk_in_dim, aux_trunk_h1, aux_trunk_h2, aux_trunk_out_dim, aux_trunk_total_params,
        );

        // ── SP14 Layer C Phase C.5a additive infrastructure (2026-05-08) ──
        // Saved-fwd buffers (h_s2_aux, h_aux1, h_aux2), upstream-grad
        // accumulator (dh_s2_aux_accum), 6 grad tensors mirroring C.2
        // params, plus grad-norm scratch + ISV-driven LR/clip pinned
        // pointers + Adam step counter. ALL DEAD until C.5b atomically
        // wires the aux head backward → accumulator → aux trunk backward
        // → Adam launcher chain. No production caller reads these yet.
        let h_s2_aux       = alloc_f32(&stream, b * aux_trunk_out_dim, "h_s2_aux")?;
        let h_aux1         = alloc_f32(&stream, b * aux_trunk_h1,      "h_aux1")?;
        let h_aux2         = alloc_f32(&stream, b * aux_trunk_h2,      "h_aux2")?;
        let dh_s2_aux_accum = alloc_f32(&stream, b * aux_trunk_out_dim, "dh_s2_aux_accum")?;

        let aux_trunk_w1_grad = alloc_f32(&stream, aux_trunk_w1_count, "aux_trunk_w1_grad")?;
        let aux_trunk_b1_grad = alloc_f32(&stream, aux_trunk_h1,       "aux_trunk_b1_grad")?;
        let aux_trunk_w2_grad = alloc_f32(&stream, aux_trunk_w2_count, "aux_trunk_w2_grad")?;
        let aux_trunk_b2_grad = alloc_f32(&stream, aux_trunk_h2,       "aux_trunk_b2_grad")?;
        let aux_trunk_w3_grad = alloc_f32(&stream, aux_trunk_w3_count, "aux_trunk_w3_grad")?;
        let aux_trunk_b3_grad = alloc_f32(&stream, aux_trunk_out_dim,  "aux_trunk_b3_grad")?;

        // SP14 Layer C Phase C.5a-fixup (2026-05-08): pre-activation gradient
        // scratch buffers required by `aux_trunk_backward.launch`
        // (`gpu_aux_trunk.rs:266-267`). Sized to `B_max × layer_h` so the
        // backward kernel can write `dh_pre` once and read it back for
        // dW/db reductions without aliasing. C.5a missed these. DEAD until
        // C.5b atomically wires the backward chain.
        let dh_aux1_pre_scratch = alloc_f32(&stream, b * aux_trunk_h1, "dh_aux1_pre_scratch")?;
        let dh_aux2_pre_scratch = alloc_f32(&stream, b * aux_trunk_h2, "dh_aux2_pre_scratch")?;

        // Per-tensor block counts for the dqn_grad_norm_kernel launches
        // (block_dim=256). One partial slot per (block × tensor) pair —
        // we lay them out in a contiguous buffer with fixed offsets so
        // the finalize kernel sees them as a single `[total_blocks]`
        // array. Order matches the 6 (param, grad, m, v) tuples consumed
        // by the Adam launcher: w1, b1, w2, b2, w3, b3.
        let aux_trunk_grad_extents: [usize; 6] = [
            aux_trunk_w1_count,
            aux_trunk_h1,
            aux_trunk_w2_count,
            aux_trunk_h2,
            aux_trunk_w3_count,
            aux_trunk_out_dim,
        ];
        let mut aux_trunk_grad_block_offsets: [usize; 7] = [0; 7];
        let mut total_grad_blocks: usize = 0;
        for (i, &ext) in aux_trunk_grad_extents.iter().enumerate() {
            aux_trunk_grad_block_offsets[i] = total_grad_blocks;
            let blocks = ext.div_ceil(256);
            total_grad_blocks += blocks;
        }
        aux_trunk_grad_block_offsets[6] = total_grad_blocks;
        let aux_trunk_grad_norm_partials =
            alloc_f32(&stream, total_grad_blocks.max(1), "aux_trunk_grad_norm_partials")?;
        let aux_trunk_grad_norm_buf =
            alloc_f32(&stream, 1, "aux_trunk_grad_norm_buf")?;

        // Pinned device-mapped grad-clip threshold. Host writes
        // ISV[AUX_TRUNK_GRAD_CLIP_INDEX] before each launch; the Adam
        // kernel reads via the device pointer. Cold-start value zero
        // means clip disabled (kernel's `(norm > clip) ? clip/norm : 1.0`
        // path; with clip=0 the second branch fires, dividing by zero —
        // C.5b is responsible for writing a real ISV value before the
        // first launch goes live).
        let aux_trunk_grad_clip_pinned: *mut f32 = unsafe {
            let flags = cudarc::driver::sys::CU_MEMHOSTALLOC_DEVICEMAP;
            cudarc::driver::result::malloc_host(std::mem::size_of::<f32>(), flags)
                .map_err(|e| MLError::ModelError(format!("pinned aux_trunk_grad_clip alloc: {e}")))?
                as *mut f32
        };
        unsafe { *aux_trunk_grad_clip_pinned = 0.0_f32; }
        let aux_trunk_grad_clip_dev_ptr = unsafe {
            let mut dp: u64 = 0;
            cudarc::driver::sys::cuMemHostGetDevicePointer_v2(
                &mut dp as *mut u64,
                aux_trunk_grad_clip_pinned.cast(),
                0,
            );
            dp
        };

        // Pinned device-mapped LR (host writes ISV[AUX_TRUNK_LR_INDEX]
        // each step). Cold-start zero — C.5b initialises before first
        // launch.
        let aux_trunk_lr_pinned: *mut f32 = unsafe {
            let flags = cudarc::driver::sys::CU_MEMHOSTALLOC_DEVICEMAP;
            cudarc::driver::result::malloc_host(std::mem::size_of::<f32>(), flags)
                .map_err(|e| MLError::ModelError(format!("pinned aux_trunk_lr alloc: {e}")))?
                as *mut f32
        };
        unsafe { *aux_trunk_lr_pinned = 0.0_f32; }
        let aux_trunk_lr_dev_ptr = unsafe {
            let mut dp: u64 = 0;
            cudarc::driver::sys::cuMemHostGetDevicePointer_v2(
                &mut dp as *mut u64,
                aux_trunk_lr_pinned.cast(),
                0,
            );
            dp
        };

        // Pinned device-mapped Adam step counter. Host increments before
        // each Adam launch; the kernel's bias-correction reads `*t_ptr`.
        let aux_trunk_t_pinned: *mut i32 = unsafe {
            let flags = cudarc::driver::sys::CU_MEMHOSTALLOC_DEVICEMAP;
            cudarc::driver::result::malloc_host(std::mem::size_of::<i32>(), flags)
                .map_err(|e| MLError::ModelError(format!("pinned aux_trunk_t alloc: {e}")))?
                as *mut i32
        };
        unsafe { *aux_trunk_t_pinned = 0; }
        let aux_trunk_t_dev_ptr = unsafe {
            let mut dp: u64 = 0;
            cudarc::driver::sys::cuMemHostGetDevicePointer_v2(
                &mut dp as *mut u64,
                aux_trunk_t_pinned.cast(),
                0,
            );
            dp
        };
        info!(
            "SP14-C.5a: aux trunk fwd/bwd buffers + Adam infra allocated \
             (h_s2_aux/h_aux1/h_aux2 + dh_s2_aux_accum, 6 grad tensors, \
             {} grad-norm partial slots, ISV-driven LR/clip + step counter; DEAD until C.5b)",
            total_grad_blocks
        );

        // SP14 Layer C Phase C.3 (2026-05-08): aux trunk forward orchestrator.
        // Pre-loads the `aux_trunk_forward` CudaFunction handle once at
        // construction (per `pearl_no_host_branches_in_captured_graph.md`).
        // Wire-up into the collector chain lands in Phase C.5 (atomic).
        let aux_trunk_forward_ops =
            super::gpu_aux_trunk::AuxTrunkForwardOps::new(&stream)?;

        // SP14 Layer C Phase C.4 (2026-05-08): aux trunk backward orchestrator.
        // Pre-loads three CudaFunction handles (`aux_trunk_bwd_dh_pre`,
        // `aux_trunk_bwd_dW_reduce`, `aux_trunk_bwd_db_reduce`) once at
        // construction (per `pearl_no_host_branches_in_captured_graph.md`).
        // CRITICAL: kernel set does NOT compute or write `dx_in` — encoder
        // boundary is the structural stop-gradient per locked design.
        // Wire-up into the collector backward chain + Adam updates lands
        // in Phase C.5 (atomic).
        let aux_trunk_backward_ops =
            super::gpu_aux_trunk::AuxTrunkBackwardOps::new(&stream)?;

        // SP14 Layer C Phase C.4b (2026-05-08): aux prediction horizon
        // producer + winning-hold-time EMA producer. Both pre-load their
        // `CudaFunction` handle at construction per
        // `pearl_no_host_branches_in_captured_graph.md`. Per-epoch
        // boundary launch (horizon is slow-moving — per-step would track
        // sample noise).
        let aux_horizon_update_ops =
            super::gpu_aux_trunk::AuxHorizonUpdateOps::new(&stream)?;
        let avg_win_hold_time_update_ops =
            super::gpu_aux_trunk::AvgWinHoldTimeUpdateOps::new(&stream)?;

        // Class A P0-A (2026-05-08): adaptive reward-cap producer. Pre-loads
        // the `CudaFunction` handle at construction per
        // `pearl_no_host_branches_in_captured_graph.md`. Per-epoch boundary
        // launch (slow-moving — reward distribution is the foundation of
        // training and shouldn't move fast).
        let reward_cap_update_ops =
            super::gpu_aux_trunk::RewardCapUpdateOps::new(&stream)?;

        // SP18 v2 Phase 3 (2026-05-09): adaptive HOLD_REWARD_POS/NEG_CAP
        // producer. Pre-loads the `CudaFunction` handle at construction
        // per `pearl_no_host_branches_in_captured_graph.md`. Per-epoch
        // boundary launch (Wiener-α blend with floor at 0.4 — non-
        // stationary control loop tracks the policy-realised Long/Short
        // close magnitude distribution as the policy adapts).
        let hold_reward_cap_update_ops =
            super::gpu_aux_trunk::HoldRewardCapUpdateOps::new(&stream)?;

        // Class A P1-Producer (2026-05-08): adaptive Bayesian Kelly priors
        // producer. Pre-loads the `CudaFunction` handle at construction per
        // `pearl_no_host_branches_in_captured_graph.md`. Per-fold-end (per-epoch
        // boundary in current scheduling) launch — Bayesian prior is a *prior
        // belief* and should change slowly across folds (α=0.005 EMA).
        let kelly_bayesian_priors_update_ops =
            super::gpu_aux_trunk::KellyBayesianPriorsUpdateOps::new(&stream)?;

        // Class A audit-fix Batch 4-A (2026-05-08): adaptive DD saturation
        // floor producer. Pre-loads the `CudaFunction` handle at
        // construction per `pearl_no_host_branches_in_captured_graph.md`.
        // Per-epoch boundary launch (slow-moving — DD distribution is a
        // fold-volatility property; α=0.01 EMA mirrors P0-A REWARD_POS_CAP).
        let dd_saturation_floor_update_ops =
            super::gpu_aux_trunk::DdSaturationFloorUpdateOps::new(&stream)?;

        // Class A audit-fix Batch 4-B (2026-05-08, Item 4): adaptive
        // MIN_HOLD_TEMPERATURE producer. Pre-loads the `CudaFunction`
        // handle at construction per
        // `pearl_no_host_branches_in_captured_graph.md`. Per-epoch
        // boundary launch (mid-cadence — temperature should track the
        // dir_acc skill EMA which itself updates per-step but with
        // α=0.05 to filter per-batch noise; same cadence as P0-A
        // REWARD_POS_CAP).
        let min_hold_temperature_update_ops =
            super::gpu_aux_trunk::MinHoldTemperatureUpdateOps::new(&stream)?;

        // ── Speculative inference cache ──
        let speculative_h_s2 = stream.alloc_zeros::<f32>(b * sh2)
            .map_err(|e| MLError::ModelError(format!("speculative_h_s2 alloc: {e}")))?;
        let speculative_features = vec![0.0_f32; ml_core::state_layout::STATE_DIM];

        // ── GPU-side counter increment kernel ──────────────────────
        // Load from a NEW CUmodule — MUST NOT share a CUmodule with any ungraphed
        // kernel, as this kernel will be captured in a child graph on Hopper.
        let increment_counters_kernel = {
            let graph_util_module = stream.context().load_cubin(GRAPH_UTILITY_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("graph_utility cubin load: {e}")))?;
            graph_util_module.load_function("increment_step_counters")
                .map_err(|e| MLError::ModelError(format!("increment_step_counters load: {e}")))?
        };
        info!("GpuDqnTrainer: increment_step_counters kernel loaded (GPU-side counter increments)");

        // Graph-safe copy kernel — loaded from a NEW CUmodule (Hopper CUfunction
        // isolation: each child graph needs its own CUmodule to avoid 3100ms replay
        // from kernel state corruption).
        let copy_f32_kernel = {
            let copy_module = stream.context().load_cubin(GRAPH_UTILITY_CUBIN.to_vec())
                .map_err(|e| MLError::ModelError(format!("copy_f32 cubin load: {e}")))?;
            copy_module.load_function("copy_f32")
                .map_err(|e| MLError::ModelError(format!("copy_f32 load: {e}")))?
        };
        info!("GpuDqnTrainer: copy_f32 kernel loaded (graph-safe DtoD copy)");

        // Tau cosine annealing parameters — defaults; caller overrides via set_tau_anneal_params.
        let tau_init = 0.005_f32;
        let tau_final = 0.001_f32;
        let tau_anneal_steps: i32 = 100_000;

        // D1/N1: clone Arc<CudaStream> before it is moved into the struct literal.
        let stream_for_snapshots = Arc::clone(&stream);

        let trainer = Self {
            config,
            stream,
            ptrs,
            grad_norm_kernel,
            grad_norm_finalize_kernel,
            grad_norm_finalize_adam,
            grad_norm_finalize_post_aux,
            grad_norm_standalone,
            grad_norm_standalone_post_aux,
            adam_update_kernel,
            adam_update_post_aux,
            scale_f32_post_aux,
            ema_kernel,
            saxpy_kernel,
            saxpy_f32_kernel,
            saxpy_f32_adam_grad,
            saxpy_f32_aux,
            scale_f32_aux,
            distill_saxpy_aux,
            scale_f32_kernel,
            zero_kernel,
            regime_scale_kernel,
            shrink_perturb_kernel: shrink_perturb,
            shrink_perturb_ungraphed,
            scale_f32_ungraphed,
            relu_mask_kernel: relu_mask_standalone,
            spectral_norm_batched_kernel: spectral_norm_kernel,
            spectral_norm_descriptors,
            clip_grad_kernel,
            pad_states_kernel,
            spec_u_s1,
            spec_v_s1,
            spec_u_s2,
            spec_v_s2,
            spec_u_v1, spec_v_v1,
            spec_u_v2, spec_v_v2,
            spec_u_a1, spec_v_a1,
            spec_u_a2, spec_v_a2,
            spec_u_bo1, spec_v_bo1,
            spec_u_bo2, spec_v_bo2,
            spec_u_bu1, spec_v_bu1,
            spec_u_bu2, spec_v_bu2,
            spec_u_bg1, spec_v_bg1,
            spec_u_bg2, spec_v_bg2,
            spec_u_bn, spec_v_bn,
            iqn_trunk_m,
            iqn_trunk_v,
            iqn_trunk_adam_step: 0,
            iqn_trunk_grad_norm,
            aux_norm_scratch,
            attn_bw_scratch,
            iqn_trunk_t_buf,
            trunk_param_count: trunk_params,
            states_buf,
            next_states_buf,
            actions_buf,
            rewards_buf,
            rewards_5bar,
            rewards_20bar,
            dones_buf,
            is_weights_buf,
            save_h_s1,
            save_h_s2,
            save_h_v,
            save_h_b0,
            save_h_b1,
            save_h_b2,
            save_h_b3,
            mag_concat_buf,
            d_mag_concat_buf,
            mag_concat_kernel,
            strided_accumulate_kernel,
            strided_scatter_kernel,
            ord_concat_buf,
            d_ord_concat_buf,
            urg_concat_buf,
            d_urg_concat_buf,
            concat_ofi_kernel,
            q_attn_params,
            q_attn_adam_m,
            q_attn_adam_v,
            q_attn_adam_step: 0,
            q_coord_buf,
            q_attn_kernel,
            sel_params,
            sel_adam_m,
            sel_adam_v,
            sel_grad,
            sel_adam_step: 0,
            sel_out_buf,
            sel_fwd_kernel,
            sel_compute_dz_kernel,
            sel_dw_reduce_p1_kernel,
            sel_dw_reduce_p2_kernel,
            sel_d_z_buf,
            sel_dw_partials_buf,
            sel_num_blocks,
            sel_norm_buf,
            sel_norm_partials,
            sel_clip_buf,
            sel_t_pinned,
            sel_t_dev_ptr,
            sp15_alpha_warm_count,
            sp15_cooldown_consecutive_losses,
            per_branch_q_gap_ema_buf,
            last_per_branch_q_gaps: [0.0; 4],
            q_mag_means_cached: [0.0_f32; 3],
            q_dir_means_cached: [0.0_f32; 4],
            utilization_ema_pinned,
            utilization_ema_dev_ptr,
            vsn_masked_buf,
            vsn_kernel,
            glu_gate_pre_buf,
            glu_value_buf,
            glu_combine_kernel,
            glu_backward_kernel,
            kan_gate_combine_kernel,
            kan_gate_backward_kernel,
            kan_grad_reduce_p1_kernel,
            kan_grad_reduce_p2_kernel,
            branch_confidence_routing_kernel,
            regime_q_gap_buf,
            regime_util_pinned,
            regime_util_dev_ptr,
            regime_dropout_kernel,
            regime_dropout_epoch_seed: 0,
            epistemic_gate_kernel,
            var_ema_pinned,
            var_ema_dev_ptr,
            predictive_coding_kernel,
            predictive_coding_backward_kernel,
            predictive_per_sample_buf,
            predictive_loss_buf,
            q_anchor_kernel,
            atom_positions_buf,
            adaptive_atom_kernel,
            atom_position_grad_kernel,
            save_current_lp,
            save_projected,
            per_sample_loss_buf,
            td_errors_buf,
            total_loss_pinned,
            total_loss_dev_ptr,
            mse_loss_pinned,
            mse_loss_dev_ptr,
            q_out_buf,
            eval_td_snapshot,
            eval_loss_snapshot,
            grad_buf,
            grad_readback_pinned_ptr,
            grad_readback_pinned_capacity,
            grad_snapshot_iqn,
            grad_snapshot_cql,
            grad_snapshot_cql_sx,
            grad_zero_ref_buf,
            grad_snapshot_c51_bs,
            grad_snapshot_distill,
            grad_snapshot_rec,
            grad_snapshot_pred,
            grad_snapshot_ens,
            grad_decomp_result_pinned,
            grad_decomp_result_dev_ptr,
            grad_decomp_trunk_start,
            grad_decomp_trunk_len,
            grad_decomp_dir_start,
            grad_decomp_dir_len,
            grad_decomp_mag_start,
            grad_decomp_mag_len,
            grad_decomp_snapshot_len,
            grad_decomp_kernel,
            cql_raw_norm_kernel,
            branch_grad_balance_reduce,
            branch_grad_balance_isv_update,
            branch_grad_balance_rescale,
            branch_slice_starts_dev,
            branch_slice_lens_dev,
            branch_grad_norms_dev,
            branch_grad_scales_dev,
            branch_slice_max_len,
            tau_update_kernel,
            epsilon_update_kernel,
            per_branch_gamma_update_kernel,
            per_branch_gamma_base_dev,
            per_branch_gamma_max_dev,
            kelly_cap_update_kernel,
            atoms_update_kernel,
            q_quantile_kernel,
            q_quantile_branch_offsets_dev,
            q_quantile_branch_sizes_dev,
            grad_component_norms_mag: [0.0_f32; 9],
            grad_component_norms_dir: [0.0_f32; 9],
            grad_component_norms_trunk: [0.0_f32; 9],
            params_buf,
            target_params_buf,
            m_buf,
            v_buf,
            vsn_param_total,
            vsn_param_byte_offset,
            grad_norm_pinned,
            grad_norm_dev_ptr,
            grad_norm_partials,
            grad_norm_blocks,
            cql_grad_scratch,
            weight_decay_mask,
            t_pinned,
            t_dev_ptr,
            tau_pinned,
            tau_dev_ptr,
            per_sample_support_ptr: 0,
            branch_scales_ptr: 0,
            eval_v_range_pinned,
            eval_v_range_ptr,
            eval_q_mean_ema: [0.0_f32; 4],
            eval_q_std_ema: [0.0_f32; 4],
            eval_ema_initialized: [false; 4],
            iqn_readiness: 0.0,
            iqn_readiness_pinned,
            iqn_readiness_dev_ptr,
            iqn_loss_ema_pinned,
            iqn_loss_ema_dev_ptr,
            iqn_loss_initial_pinned,
            iqn_loss_initial_dev_ptr,
            lr_pinned,
            lr_dev_ptr,
            aux_lr_pinned,
            aux_lr_dev_ptr,
            adaptive_clip_pinned,
            adaptive_clip_dev_ptr,
            grad_norm_ema_pinned,
            grad_norm_ema_dev_ptr,
            outlier_diag_pinned,
            outlier_diag_dev_ptr,
            adam_step: 0,
            total_params,
            non_isv_params,
            params_initialized: false,
            target_params_initialized: false,
            attention_initialized: false,
            eval_forward_exec: None,
            graphs_captured: false,
            upload_staging_buf,
            upload_staging_len,
            readback_buf,
            readback_host,
            upload_staging_host,
            scalars_readback_buf,
            scalars_readback_host,
            readback_pinned: {
                let flags = cudarc::driver::sys::CU_MEMHOSTALLOC_DEVICEMAP;
                unsafe {
                    cudarc::driver::result::malloc_host(16 * std::mem::size_of::<f32>(), flags)
                        .map_err(|e| MLError::ModelError(format!("pinned readback alloc: {e}")))?
                        as *mut f32
                }
            },
            readback_pending: false,
            states_staging_buf,
            next_states_staging_buf,
            shared_cublas,
            cublas_forward,
            cublas_forward_ddqn,
            tg_h_s1_scratch,
            tg_h_s2_buf,
            tg_h_v_scratch,
            tg_h_b0_scratch,
            tg_h_b1_scratch,
            tg_h_b2_scratch,
            tg_h_b3_scratch,
            tg_v_logits_buf,
            tg_b_logits_buf,
            on_v_logits_buf,
            on_b_logits_buf,
            on_next_v_logits_buf,
            on_next_b_logits_buf,
            on_next_h_s1_scratch,
            on_next_h_s2_scratch,
            on_next_h_v_scratch,
            on_next_h_b_scratch,
            c51_loss_kernel,
            c51_loss_reduce_kernel,
            c51_grad_kernel,
            mse_loss_kernel,
            mse_grad_kernel,
            c51_alpha: initial_c51_alpha,
            d_value_logits_buf,
            d_adv_logits_buf,
            d_value_logits,
            d_adv_logits,
            d_value_logits_mse,
            d_adv_logits_mse,
            cublas_backward,
            bw_d_h_s2,
            bw_d_h_s1,
            bw_d_h_v,
            bw_d_h_b0,
            bw_d_h_b1,
            bw_d_h_b2,
            bw_d_h_b3,
            bw_d_glu_value,
            bw_d_glu_gate,
            // GRN trunk backward scratch (Plan 4 Task 2c.3c.3) — wired by 2c.3c.4
            bw_grn_h_s2_d_pre_ln,
            bw_grn_h_s2_d_linear_b_out,
            bw_grn_h_s2_d_elu_out,
            bw_grn_h_s2_d_linear_a_out,
            bw_grn_h_s1_d_pre_ln,
            bw_grn_h_s1_d_linear_b_out,
            bw_grn_h_s1_d_elu_out,
            bw_grn_h_s1_d_linear_a_out,
            expected_q_kernel,
            q_stats_kernel,
            q_stats_per_branch_kernel,
            per_branch_q_stats_pinned,
            per_branch_q_stats_dev_ptr,
            q_mag_bin_means_reduce_kernel,
            q_mag_means_scratch_pinned,
            q_mag_means_scratch_dev_ptr,
            q_abs_ref_scratch_pinned,
            q_abs_ref_scratch_dev_ptr,
            q_dir_bin_means_reduce_kernel,
            q_dir_means_scratch_pinned,
            q_dir_means_scratch_dev_ptr,
            q_dir_abs_ref_scratch_pinned,
            q_dir_abs_ref_scratch_dev_ptr,
            q_stats_buf,
            atom_stats_buf,
            atom_stats_block_sums_buf,
            atom_stats_finalize_kernel,
            q_readback_pinned,
            q_readback_dev_ptr,
            cql_logit_grad_kernel,
            cql_d_value_logits,
            cql_d_adv_logits,
            barrier_gradient_kernel,
            ib_gradient_kernel,
            curiosity_error_buf,
            drawdown_depths_buf,
            asymmetric_dd_weight: dd_weight,
            causal_states_scratch,
            causal_sensitivity_buf,
            causal_q_scratch,
            pruning_mask: None, // Computed at pruning epoch
            pruning_mask_kernel,
            pruning_compute_kernel,
            her_inplace_kernel,
            nan_check_f32_kernel,
            nan_check_f32_kernel_b,
            nan_check_fused_f32_kernel,
            clamp_finite_f32_kernel,
            nan_flags_buf,
            wiener_state_buf,
            clamp_engage_per_block_buf,
            pearl_c_rate_deficit_state_buf,
            pearl_c_rate_deficit_ema: [0.0_f32; SP4_PARAM_GROUP_COUNT],
            producer_step_scratch_buf,
            oracle_subbuf_table_buf,
            oracle_subbuf_counts_buf,
            q_dir_grad_subbuf_table_buf,
            q_dir_grad_subbuf_counts_buf,
            nan_check_buf_ptrs,
            nan_check_buf_lens,
            pruning_epoch: prune_ep,
            pruning_fraction: prune_frac,
            causal_intervene_kernel: causal_intervene_kernel_fn,
            causal_reduce_kernel: causal_reduce_kernel_fn,
            causal_mean_reduce_kernel: causal_mean_reduce_kernel_fn,
            causal_mean_scratch,
            causal_config: CausalInterventionConfig {
                weight: causal_wt,
                interval: causal_intv,
                market_dim: causal_market_dim,
            },
            prev_grad_buf,
            distill_best_buf,
            vaccine_grad_save,
            vaccine_dot_norm_buf,
            vaccine_dot_kernel,
            vaccine_dot_finalize_kernel: vaccine_dot_finalize,
            vaccine_block_results,
            vaccine_project_kernel,
            bn_hidden_buf,
            bn_concat_buf,
            tg_bn_hidden_buf,
            tg_bn_concat_buf,
            on_next_bn_hidden_buf,
            on_next_bn_concat_buf,
            bn_d_concat_buf,
            bn_d_hidden_buf,
            // SP15 Wave 4.1b OOB fix (2026-05-07): pre-loaded handle, see
            // the field docstring in the struct definition.
            bn_tanh_concat_dd_kernel,
            bn_tanh_backward_kernel,
            bn_bias_grad_kernel,
            // ── Plan 4 Task 1B-iii: VSN feature-selection buffers ──────
            vsn_gated_states_buf,
            vsn_gated_next_states_buf,
            vsn_gated_next_states_for_ddqn_buf,
            vsn_logits_buf,
            vsn_mask_buf,
            vsn_logits_target_scratch,
            vsn_mask_target_scratch,
            vsn_logits_ddqn_scratch,
            vsn_mask_ddqn_scratch,
            vsn_save_h1_g0_buf,
            vsn_save_h1_g1_buf,
            vsn_save_h1_g2_buf,
            vsn_save_h1_g3_buf,
            vsn_save_h1_g4_buf,
            vsn_save_h1_g5_buf,
            vsn_h1_inference_scratch,
            vsn_linear2_scratch_buf,
            vsn_group_begins_buf,
            vsn_group_ends_buf,
            vsn_d_logits_buf,
            vsn_d_state_buf,
            vsn_d_gated_state_buf,
            vsn_d_logit_scratch_buf,
            vsn_d_h1_scratch_buf,
            vsn_logit_gather_kernel,
            vsn_d_gated_state_portfolio_kernel,
            vsn_dw_iqn_aux_scratch,
            vsn_dw_ensemble_aux_scratch,
            vsn_mask_ema_kernel,
            // HEALTH_DIAG GPU port — Phase 2A orchestrator (1 field; mirrors
            // aux_heads_fwd's positional placement next to the kernel-handle
            // block).
            health_diag,
            // Plan 4 Task 6 Commit B — aux-heads orchestrator + buffers (24 fields)
            aux_heads_fwd,
            aux_heads_bwd,
            aux_nb_hidden_buf,
            aux_nb_logits_buf,
            aux_nb_softmax_buf,
            aux_nb_label_buf,
            aux_nb_loss_scalar_buf,
            aux_nb_valid_count_buf,
            aux_rg_hidden_buf,
            aux_rg_logits_buf,
            aux_rg_label_buf,
            aux_rg_loss_scalar_buf,
            aux_rg_correct_scalar_buf,
            aux_dh_s2_nb_buf,
            aux_dh_s2_rg_buf,
            aux_partial_nb_w1,
            aux_partial_nb_b1,
            aux_partial_nb_w2,
            aux_partial_nb_b2,
            aux_partial_rg_w1,
            aux_partial_rg_b1,
            aux_partial_rg_w2,
            aux_partial_rg_b2,
            aux_param_grad_final_buf,
            // SP22 H6 vNext Phase B0+B2 (2026-05-14): trade-outcome head
            // orchestrator + 11 buffers. Mirrors aux_heads_fwd/bwd +
            // aux_nb_* / aux_partial_nb_* placement.
            aux_to_fwd,
            aux_to_bwd,
            aux_to_hidden_buf,
            aux_to_logits_buf,
            aux_to_softmax_buf,
            aux_to_label_buf,
            aux_to_loss_scalar_buf,
            aux_to_valid_count_buf,
            aux_dh_s2_to_buf,
            aux_partial_to_w1,
            aux_partial_to_b1,
            aux_partial_to_w2,
            aux_partial_to_b2,
            aux_conf_at_state_buf,
            aux_weight,
            stochastic_depth_scale_buf,
            stochastic_depth_kernel,
            stochastic_depth_rng_kernel,
            stochastic_depth_rng_state,
            stochastic_depth_prob: sd_prob,
            ensemble_std_buf,
            ensemble_disagreement_weight: ens_weight,
            curiosity_prepare_input_func,
            curiosity_bias_leaky_relu_func,
            curiosity_bias_mse_func,
            curiosity_input_buf,
            curiosity_hidden_buf,
            curiosity_pred_buf,
            curiosity_w1_ptr: u64::MAX,
            curiosity_b1_ptr: u64::MAX,
            curiosity_w2_ptr: u64::MAX,
            curiosity_b2_ptr: u64::MAX,
            popart_normalize_kernel,
            popart_robust_kernel,
            popart_mean,
            popart_var,
            popart_count,
            h_s2_rms_ema_kernel,
            target_q_p99_update,
            atom_pos_p99_update,
            param_group_oracle_update,
            grad_norm_p99_update,
            h_s2_p99_update,
            bw_d_h_s2_p99_update,
            q_dir_grad_p99_update,
            apply_pearls_ad_kernel,
            q_branch_stats_kernel,
            pearl_1_atom_kernel,
            pearl_3_sigma_kernel,
            pearl_2_budget_kernel,
            loss_balance_controller_kernel,
            train_active_frac_compute_kernel,
            loss_balance_max_budget_compute_kernel,
            sp9_eval_intent_dist_compute_kernel,
            sp9_intent_eval_divergence_compute_kernel,
            sp9_q_var_mag_ema_compute_kernel,
            sp9_kelly_warmup_floor_compute_kernel,
            sp11_val_sharpe_delta_compute_kernel,
            sp11_saboteur_engagement_compute_kernel,
            sp11_mag_ratio_compute_kernel,
            val_sharpe_history_pinned,
            // SP11 Fix 39: saboteur Δreward GPU buffer is owned by
            // `gpu_experience_collector`. The trainer caches the device
            // pointer + length; both default to 0 here and are wired in
            // via `set_sp11_saboteur_delta_reward_buf` after the collector
            // is constructed (see training_loop.rs init path).
            sp11_saboteur_delta_reward_dev_ptr: 0,
            sp11_saboteur_delta_reward_len:     0,
            // SP11 Fix 39 B1b fix-up (2026-05-04): popart-component-
            // specific magnitude EMA infrastructure. Kernel + trainer-side
            // mapped-pinned buffer, plus dev_ptr/len cached for the
            // canonical experience-collector buffer (wired in via
            // `set_sp11_popart_component_buf` after collector construction
            // — same pattern as the saboteur Δreward path above).
            popart_component_per_sample,
            sp11_popart_component_ema_kernel,
            sp11_popart_component_dev_ptr: 0,
            sp11_popart_component_len:     0,
            // SP13 Phase 0a (2026-05-04): aux-head directional-accuracy
            // reducer kernel + 3-element mapped-pinned readback buffer.
            // Both fields atomic with the SP13 ISV slot constants and
            // SP13 state-reset registry entries staged by P0a.T1; the
            // per-step launcher is wired by P0a.T4.
            aux_dir_acc_reduce,
            aux_dir_acc_buf,
            // SP13 Phase 0a P0a.T4 (2026-05-04): fixed-α EMA applicator +
            // aux-pred → ISV[375] tanh producer kernel handles. Wired into
            // `launch_sp13_aux_dir_metrics` (called per step from
            // `training_loop.rs`) and the hold-rate observer chain in
            // `gpu_experience_collector.rs` per-step action-select site.
            apply_fixed_alpha_ema_kernel,
            aux_pred_to_isv_tanh_kernel,
            // SP22 H6 Phase 3 α (2026-05-13 revised — atom-position shift).
            // 4-element Adam-trained W + Adam moments + grad accumulator.
            // No separate forward/backward kernel handles — the
            // scalar-bias design was replaced by atom-shift threading
            // through compute_expected_q + c51_loss_kernel + c51_grad_kernel
            // + mag_concat_qdir (Phase 3-final implementation). W is
            // initialized to the structural prior `[-0.5, 0.0, +0.5, 0.0]`
            // (Phase 3-final step in `new()`); Adam refines from there.
            w_aux_to_q_dir,
            adam_m_w_aux,
            adam_v_w_aux,
            dw_aux_buf,
            aux_target_a_dir_buf,
            aux_proj_logdiff_dir_buf,
            c51_aux_dw_kernel,
            adam_w_aux_kernel,
            // SP14 q_disagreement diagnostic + Coupling A forward feature
            // wire. Phase C.1 (2026-05-08) deleted the α-machinery struct
            // entries (sp14_alpha_grad_compute_kernel,
            // sp14_gradient_hack_detect_kernel, sp14_scale_wire_col_kernel)
            // atomically with the kernel files.
            sp14_q_disagreement_update_kernel,
            // SP17 Task PP.3 (2026-05-08): pre-centering V/A mean
            // diagnostic kernel + 5-float mapped-pinned readback. Cold-path
            // producer launched at HEALTH_DIAG emit only.
            sp17_v_a_means_diag_kernel,
            sp17_v_a_means_diag_buf,
            sp17_a_var_ema_kernel,
            sp17_v_share_kernel,
            sp17_advantage_clip_bound_kernel,
            sp17_advantage_clip_scratch,
            // SP18 v2 Phase 0 Task 0.2 (2026-05-08): B-leg TD-error
            // magnitude EMA producer kernel handle. Trainer-stream-
            // resident; launched from `launch_sp18_td_error_mag_ema_
            // update` post-train-step.
            sp18_td_error_mag_ema_kernel,
            // SP18 v2 Phase 0 Task 0.1 (2026-05-08): D-leg per-action
            // reward decomposition diagnostic buffer. Mapped-pinned 20
            // floats (4 bins × 5 stats). Written by collector kernel,
            // read host-side at HEALTH_DIAG cadence.
            sp18_reward_decomp_diag_buf,
            sp14_dir_concat_qaux_kernel,
            sp14_dir_qaux_concat_scratch,
            // SP14 backward dX scratch — column-SH2 wire is dropped at
            // the first-SH2-cols accumulator post-C.1 (no α-gate scaling).
            sp14_d_dir_qaux_concat,
            // SP11 Fix 39 (Task A2): controller + SimHash novelty kernels
            // and their backing buffers. Projection matrix was populated
            // on-device above by `launch_novelty_simhash_proj_init`; hash
            // table is constructor-zero and reset at fold boundary via
            // the `sp11_novelty_hash` registry arm.
            sp11_reward_subsystem_controller_kernel,
            sp11_novelty_simhash_proj_init_kernel,
            sp11_novelty_simhash_lookup_kernel,
            sp11_novelty_simhash_update_kernel,
            novelty_hash_buf,
            novelty_simhash_proj,
            lb_budget_dispatch_kernel,
            saxpy_f32_dev_ptr_aux,
            scale_f32_dev_ptr_aux,
            lb_budget_effective_buf,
            grad_cosine_sim_kernel,
            pearl_4_adam_hparams_kernel,
            q_skew_kurtosis_kernel,
            pearl_5_iqn_tau_kernel,
            pearl_6_kelly_kernel,
            pearl_8_trail_kernel,
            pearl_1_ext_num_atoms_kernel,
            pnl_aggregation_kernel,
            health_composition_kernel,
            training_metrics_ema_kernel,
            grad_prev_buf_per_group,
            group_param_offsets_buf,
            atoms_clip_count_buf,
            action_counts_buf,
            action_offsets_buf,
            q_drift_rate_ema_kernel,
            fold_warmup_factor_kernel,
            update_grad_norm_emas_kernel,
            update_iqn_readiness_kernel,
            update_utilization_ema_kernel,
            update_adaptive_clip_kernel,
            grad_norm_fast_ema_pinned,
            grad_norm_fast_ema_dev_ptr,
            grad_norm_slow_ema_pinned,
            grad_norm_slow_ema_dev_ptr,
            grad_norm_emas_step_count: 0,
            iqn_quantile_ema_kernel,
            q_mean_reduce_kernel,
            q_mean_subtract_kernel,
            q_mean_scratch_pinned,
            q_mean_scratch_dev_ptr,
            q_mean_ema_pinned,
            q_mean_ema_dev_ptr,
            graph_params,
            graph_msg_kernel,
            denoise_params,
            denoise_adam_m,
            denoise_adam_v,
            denoise_adam_step: 0,
            denoise_buf_a,
            denoise_buf_b,
            q_var_buf_trainer,
            q_denoise_kernel,
            q_denoise_backward_kernel,
            denoise_grad,
            denoise_target_q_buf,
            denoise_q_input_buf,
            denoise_norm_partials,
            denoise_norm_buf,
            denoise_clip_buf,
            denoise_t_pinned,
            denoise_t_dev_ptr,
            denoise_input0_buf,
            denoise_input1_buf,
            denoise_pre_h0_buf,
            denoise_pre_h1_buf,
            denoise_h0_buf,
            denoise_h1_buf,
            denoise_q_step0_buf,
            denoise_d_res_buf,
            denoise_d_h_buf,
            denoise_bias_grad_partials,
            denoise_gemm_fwd_hh,
            denoise_gemm_fwd_dh,
            denoise_gemm_dw1,
            denoise_gemm_dw2,
            denoise_gemm_bwd_dh,
            denoise_build_input_kernel,
            denoise_bias_silu_kernel,
            denoise_silu_bwd_kernel,
            denoise_loss_grad_kernel,
            denoise_skip_conn_kernel,
            denoise_bias_grad_p1_kernel,
            denoise_bias_grad_p2_kernel,
            qlstm_weights,
            qlstm_c,
            qlstm_n,
            qlstm_context,
            per_branch_q_gaps_pinned,
            per_branch_q_gaps_dev_ptr,
            qlstm_step_kernel,
            qlstm_train_kernel,
            prev_q_mean_pinned,
            prev_q_mean_dev_ptr,
            mamba2_h_history,
            mamba2_h_enriched,
            mamba2_update_kernel,
            mamba2_copy_enriched_kernel,
            mamba2_params,
            mamba2_grad,
            mamba2_adam_m,
            mamba2_adam_v,
            mamba2_adam_step: 0,
            mamba2_a_proj,
            mamba2_b_proj,
            mamba2_d_gate,
            mamba2_d_x,
            mamba2_d_context,
            mamba2_d_tw,
            mamba2_gemm_proj,
            mamba2_gemm_dw,
            mamba2_gemm_dwc,
            mamba2_gemm_dh,
            d_h_history_buf,
            d_ofi_embed_mamba2,
            extract_ofi_embed_grad_kernel,
            mamba2_scan_proj_fwd_kernel,
            mamba2_scan_proj_bwd_kernel,
            mamba2_scale_d_enriched_kernel,
            new_component_warmup_step: 0,
            v_logits_5bar,
            v_logits_20bar,
            save_h_v_5bar,
            save_h_v_20bar,
            v_logits_blended,
            homeostatic_kernel,
            calibrate_homeostatic_kernel,
            homeostatic_obs_pinned,
            homeostatic_obs_dev_ptr,
            homeostatic_targets_pinned,
            homeostatic_targets_dev_ptr,
            homeostatic_penalties_buf,
            homeostatic_total_buf,
            homeostatic_calibration_done: false,
            risk_forward_kernel,
            risk_apply_kernel,
            risk_backward_kernel,
            risk_hidden_buf,
            risk_budget_buf,
            cvar_alpha_buf,
            commit_lambda_buf,
            q_mag_pre_risk,
            risk_grad_buf,
            risk_adam_step: 0,
            td_error_scratch_pinned,
            td_error_scratch_dev_ptr,
            q_var_scratch_pinned,
            q_var_scratch_dev_ptr,
            reward_scratch_pinned,
            reward_scratch_dev_ptr,
            atom_util_scratch_pinned,
            atom_util_scratch_dev_ptr,
            isv_signals_pinned,
            isv_signals_dev_ptr,
            isv_history_pinned,
            isv_history_dev_ptr,
            isv_decay_pinned,
            isv_decay_dev_ptr,
            lagged_td_error_pinned,
            lagged_td_error_dev_ptr,
            isv_signal_update_kernel,
            isv_forward_kernel,
            isv_feature_gate_kernel,
            fill_gamma_buf_kernel,
            isv_embedding_buf,
            branch_gate_buf,
            gamma_mod_buf,
            temporal_weight_buf,
            isv_temporal_route_kernel,
            gamma_buf,
            predicted_error_buf,
            recursive_conf_fwd_kernel,
            recursive_conf_bwd_kernel,
            recursive_conf_reduce_kernel,
            recursive_conf_partials,
            max_conf_blocks,
            plan_params_buf,
            trade_plan_hidden_buf,
            trade_plan_pre_out_buf,
            trade_plan_activate_kernel,
            plan_noise_kernel,
            ofi_embed_input_buf,
            ofi_embed_output_buf,
            ofi_embed_w,
            ofi_embed_b,
            ofi_embed_build_input_kernel,
            ofi_embed_params,
            ofi_embed_grad_w,
            ofi_embed_grad_b,
            ofi_embed_grad,
            d_ofi_embed_combined,
            ofi_embed_adam_m,
            ofi_embed_adam_v,
            ofi_embed_adam_step: 0,
            ofi_embed_norm_partials,
            ofi_embed_norm_buf,
            ofi_embed_clip_buf,
            ofi_embed_t_pinned,
            ofi_embed_t_dev_ptr,
            ofi_embed_bias_grad_partials,
            // SP14 Layer C aux trunk params + Adam state (Phase C.2).
            aux_trunk_w1,
            aux_trunk_b1,
            aux_trunk_w2,
            aux_trunk_b2,
            aux_trunk_w3,
            aux_trunk_b3,
            aux_trunk_w1_m,
            aux_trunk_w1_v,
            aux_trunk_b1_m,
            aux_trunk_b1_v,
            aux_trunk_w2_m,
            aux_trunk_w2_v,
            aux_trunk_b2_m,
            aux_trunk_b2_v,
            aux_trunk_w3_m,
            aux_trunk_w3_v,
            aux_trunk_b3_m,
            aux_trunk_b3_v,
            // SP14 Layer C Phase C.5a (2026-05-08): saved-fwd + accumulator + grad buffers + Adam infra (DEAD).
            h_s2_aux,
            h_aux1,
            h_aux2,
            dh_s2_aux_accum,
            aux_trunk_w1_grad,
            aux_trunk_b1_grad,
            aux_trunk_w2_grad,
            aux_trunk_b2_grad,
            aux_trunk_w3_grad,
            aux_trunk_b3_grad,
            // SP14 Layer C Phase C.5a-fixup (2026-05-08): dh_pre scratch buffers (DEAD).
            dh_aux1_pre_scratch,
            dh_aux2_pre_scratch,
            aux_trunk_grad_norm_partials,
            aux_trunk_grad_block_offsets,
            aux_trunk_grad_norm_buf,
            aux_trunk_grad_clip_pinned,
            aux_trunk_grad_clip_dev_ptr,
            aux_trunk_lr_pinned,
            aux_trunk_lr_dev_ptr,
            aux_trunk_t_pinned,
            aux_trunk_t_dev_ptr,
            aux_trunk_adam_step: 0,
            // SP14 Layer C Phase C.3 (2026-05-08): aux trunk forward orchestrator.
            aux_trunk_forward_ops,
            // SP14 Layer C Phase C.4 (2026-05-08): aux trunk backward orchestrator.
            aux_trunk_backward_ops,
            // SP14 Layer C Phase C.4b (2026-05-08): aux horizon + winning-hold-time producers.
            aux_horizon_update_ops,
            avg_win_hold_time_update_ops,
            // Class A P0-A (2026-05-08): adaptive reward-cap producer.
            reward_cap_update_ops,
            // SP18 v2 Phase 3 (2026-05-09): adaptive Hold-reward-cap producer.
            hold_reward_cap_update_ops,
            // Class A P1-Producer (2026-05-08): adaptive Bayesian Kelly priors producer.
            kelly_bayesian_priors_update_ops,
            // Class A audit-fix Batch 4-A (2026-05-08): adaptive DD saturation floor producer.
            dd_saturation_floor_update_ops,
            // Class A audit-fix Batch 4-B (2026-05-08, Item 4): adaptive MIN_HOLD_TEMPERATURE producer.
            min_hold_temperature_update_ops,
            speculative_h_s2,
            speculative_features,
            speculative_valid: false,
            increment_counters_kernel,
            copy_f32_kernel,
            tau_init,
            tau_final,
            tau_anneal_steps,
            last_cql_alpha_eff: 0.0,
            last_sarsa_tau_factor: 0.0,
            last_iqn_budget_eff: 0.40,
            last_ens_budget_eff: 0.05,
            last_iqn_budget_per_branch: [0.40; 4],
            last_ens_budget_per_branch: [0.05; 4],
            q_snapshots: crate::cuda_pipeline::q_snapshot::SnapshotRing::new(
                stream_for_snapshots,
                total_params,
            ),
            last_distill_active: false,
            last_meta_q_pred: 0.5,
            moe_head,
            moe_gate_h1_buf,
            moe_gate_pre_buf,
            moe_gate_softmax_buf,
            moe_expert_h1_bufs,
            moe_expert_outputs_buf,
            moe_de_k_buf,
            moe_dg_buf,
            moe_dg_pre_buf,
            moe_dh_s1_scratch,
            moe_gate_dh1_buf,
            moe_load_balance_loss_per_k,
            moe_load_balance_loss_total_pinned,
            moe_load_balance_loss_total_dev_ptr,
            moe_lambda_floor,
            moe_lambda_max_extra,
            moe_entropy_target_frac,
            q_sample_history: std::collections::VecDeque::with_capacity(64),
        };
        // Self-check: verify the fingerprint we just wrote is readable and matches
        // the compile-time constant. This is a construction-time invariant check;
        // any restore path that loads persisted ISV state must also call this.
        trainer.check_layout_fingerprint()?;
        Ok(trainer)
    }

    /// Reference to the trainer's forked CudaStream.
    pub fn stream(&self) -> &Arc<CudaStream> {
        &self.stream
    }

    /// Reference to the flat f32 online parameter buffer.
    /// Used for GPU-native best-model snapshots (DtoD copy, no Candle).
    pub fn params(&self) -> &CudaSlice<f32> {
        &self.params_buf
    }

    /// Mutable reference to the flat f32 online parameter buffer.
    pub fn params_mut(&mut self) -> &mut CudaSlice<f32> {
        &mut self.params_buf
    }

    /// Reference to the flat f32 target parameter buffer.
    /// Used for GPU-native DtoD weight sync.
    pub fn target_params(&self) -> &CudaSlice<f32> {
        &self.target_params_buf
    }

    /// Mutable reference to the flat f32 target parameter buffer.
    pub fn target_params_mut(&mut self) -> &mut CudaSlice<f32> {
        &mut self.target_params_buf
    }

    /// Download online params from GPU to host (synchronous DtoH).
    ///
    /// `dst` is caller-supplied and may be pinned or pageable. cudarc's
    /// `memcpy_dtoh` forwards to `cuMemcpyDtoHAsync_v2`, which is asynchronous
    /// when the destination is pinned (the API queues the DMA on the stream and
    /// returns before the transfer completes). The pre-copy sync ensures all
    /// prior GPU work has landed; the post-copy sync ensures the DMA itself has
    /// completed so the caller sees final transferred bytes regardless of dst
    /// memory type. Matches the 5da434ab4 pattern (per_branch_grad_norms).
    pub fn download_params(&self, dst: &mut [f32]) -> Result<(), MLError> {
        unsafe { cudarc::driver::sys::cuStreamSynchronize(self.stream.cu_stream()); }
        self.stream.memcpy_dtoh(&self.params_buf, dst)
            .map_err(|e| MLError::ModelError(format!("download params: {e}")))?;
        self.stream.synchronize()
            .map_err(|e| MLError::ModelError(format!("download_params post-dtoh sync: {e}")))
    }

    /// Upload online params from host to GPU via mapped-pinned staging.
    /// No HtoD memcpy per `feedback_no_htod_htoh_only_mapped_pinned.md`.
    pub fn upload_params(&mut self, src: &[f32]) -> Result<(), MLError> {
        update_via_mapped_f32(&self.stream, &mut self.params_buf, src, "upload_params")
    }

    /// Download target params from GPU to host (synchronous DtoH).
    ///
    /// Same race-safety contract as `download_params`: post-copy sync ensures
    /// the caller's `dst` slice is fully populated when this returns, even if
    /// `dst` is pinned memory (where `cuMemcpyDtoHAsync_v2` is genuinely async).
    pub fn download_target_params(&self, dst: &mut [f32]) -> Result<(), MLError> {
        unsafe { cudarc::driver::sys::cuStreamSynchronize(self.stream.cu_stream()); }
        self.stream.memcpy_dtoh(&self.target_params_buf, dst)
            .map_err(|e| MLError::ModelError(format!("download target params: {e}")))?;
        self.stream.synchronize()
            .map_err(|e| MLError::ModelError(format!("download_target_params post-dtoh sync: {e}")))
    }

    /// Upload target params from host to GPU via mapped-pinned staging.
    /// No HtoD memcpy per `feedback_no_htod_htoh_only_mapped_pinned.md`.
    pub fn upload_target_params(&mut self, src: &[f32]) -> Result<(), MLError> {
        update_via_mapped_f32(&self.stream, &mut self.target_params_buf, src, "upload_target_params")
    }

    /// Speculative forward: pre-compute trunk from intermediate features.
    /// Called every ~5 seconds between bars with intermediate microstructure state.
    /// Stores h_s2 in cache for potential reuse at bar close.
    ///
    /// Current implementation: cache infrastructure only (features stored, ISV updated).
    /// Full cuBLAS trunk forward integration is deferred to a follow-up.
    pub fn speculative_forward(&mut self, features: &[f32]) -> Result<(), MLError> {
        if features.len() != ml_core::state_layout::STATE_DIM {
            return Err(MLError::ModelError(format!(
                "speculative_forward: expected {} features, got {}",
                ml_core::state_layout::STATE_DIM, features.len()
            )));
        }

        // Run ISV signal update (uses pinned scalars from last training step)
        self.update_isv_signals()?;

        // Run ISV forward (encoder MLP → branch gate + gamma mod)
        self.launch_isv_forward()?;

        // Cache the features for comparison at bar close
        self.speculative_features.clear();
        self.speculative_features.extend_from_slice(features);
        self.speculative_valid = true;

        Ok(())
    }

    /// Check if speculative cache is usable for the given bar-close features.
    /// Returns true if features are within 5% relative L2 distance of the cached speculation.
    pub fn check_speculative_cache(&self, actual_features: &[f32]) -> bool {
        if !self.speculative_valid || actual_features.len() != self.speculative_features.len() {
            return false;
        }

        // Compute relative L2 distance between speculative and actual features
        let mut sum_sq_diff = 0.0_f32;
        let mut sum_sq_actual = 0.0_f32;
        for (a, s) in actual_features.iter().zip(self.speculative_features.iter()) {
            sum_sq_diff += (a - s) * (a - s);
            sum_sq_actual += a * a;
        }

        let relative_diff = if sum_sq_actual > 1e-8 {
            (sum_sq_diff / sum_sq_actual).sqrt()
        } else {
            1.0 // Can't compare if actual is all zeros
        };

        // Use cache if features changed by less than 5%
        relative_diff < 0.05
    }

    /// Invalidate speculative cache (call at bar boundary or after training step).
    pub fn invalidate_speculative_cache(&mut self) {
        self.speculative_valid = false;
    }

    /// Clone params_buf to a new CudaSlice via DtoD copy — checkpoint stays on GPU, zero CPU.
    pub fn clone_params_gpu(&self) -> Result<CudaSlice<f32>, MLError> {
        let len = self.params_buf.len();
        let dst = self.stream.alloc_zeros::<f32>(len)
            .map_err(|e| MLError::ModelError(format!("alloc checkpoint params: {e}")))?;
        let n_bytes = len * std::mem::size_of::<f32>();
        unsafe {
            cudarc::driver::result::memcpy_dtod_async(
                dst.raw_ptr(), self.params_buf.raw_ptr(), n_bytes, self.stream.cu_stream(),
            ).map_err(|e| MLError::ModelError(format!("checkpoint params DtoD: {e}")))?;
        }
        Ok(dst)
    }

    /// Hard-copy online params into target params. At fold boundaries the online
    /// weights are shrink-and-perturb'd; the EMA target still holds the end-of-
    /// prior-fold values and would otherwise produce a large TD error gap in the
    /// first steps of the new fold, driving runaway gradients before Polyak
    /// averaging can close the divergence. DtoD copy — no CPU staging.
    pub fn sync_target_from_online(&mut self) -> Result<(), MLError> {
        let n_bytes = self.total_params * std::mem::size_of::<f32>();
        unsafe {
            cudarc::driver::result::memcpy_dtod_async(
                self.target_params_buf.raw_ptr(),
                self.params_buf.raw_ptr(),
                n_bytes,
                self.stream.cu_stream(),
            ).map_err(|e| MLError::ModelError(format!("sync_target_from_online DtoD: {e}")))?;
        }
        Ok(())
    }

    /// Clone target_params_buf to a new CudaSlice via DtoD copy — checkpoint stays on GPU, zero CPU.
    pub fn clone_target_params_gpu(&self) -> Result<CudaSlice<f32>, MLError> {
        let len = self.target_params_buf.len();
        let dst = self.stream.alloc_zeros::<f32>(len)
            .map_err(|e| MLError::ModelError(format!("alloc checkpoint target params: {e}")))?;
        let n_bytes = len * std::mem::size_of::<f32>();
        unsafe {
            cudarc::driver::result::memcpy_dtod_async(
                dst.raw_ptr(), self.target_params_buf.raw_ptr(), n_bytes, self.stream.cu_stream(),
            ).map_err(|e| MLError::ModelError(format!("checkpoint target params DtoD: {e}")))?;
        }
        Ok(dst)
    }

    /// Restore params_buf from a GPU checkpoint via DtoD copy — zero CPU involvement.
    pub fn restore_params_from_gpu(&mut self, src: &CudaSlice<f32>) -> Result<(), MLError> {
        let n_bytes = self.params_buf.len() * std::mem::size_of::<f32>();
        unsafe {
            cudarc::driver::result::memcpy_dtod_async(
                self.params_buf.raw_ptr(), src.raw_ptr(), n_bytes, self.stream.cu_stream(),
            ).map_err(|e| MLError::ModelError(format!("restore params DtoD: {e}")))?;
        }
        Ok(())
    }

    /// Restore target_params_buf from a GPU checkpoint via DtoD copy — zero CPU involvement.
    pub fn restore_target_params_from_gpu(&mut self, src: &CudaSlice<f32>) -> Result<(), MLError> {
        let n_bytes = self.target_params_buf.len() * std::mem::size_of::<f32>();
        unsafe {
            cudarc::driver::result::memcpy_dtod_async(
                self.target_params_buf.raw_ptr(), src.raw_ptr(), n_bytes, self.stream.cu_stream(),
            ).map_err(|e| MLError::ModelError(format!("restore target params DtoD: {e}")))?;
        }
        Ok(())
    }

    /// Read eval_q_mean_ema as the direction-branch scalar — matches the
    /// pre-unification single-EMA behaviour for callers that have not yet
    /// migrated to per-branch awareness.
    pub fn eval_q_mean_ema(&self) -> f32 { self.eval_q_mean_ema[0] }

    /// Read all four per-branch `q_mean_ema` values.
    pub fn eval_q_mean_ema_per_branch(&self) -> [f32; 4] { self.eval_q_mean_ema }

    /// ISV signals device pointer for adaptive hold enforcement in experience collector.
    pub fn isv_signals_dev_ptr(&self) -> u64 { self.isv_signals_dev_ptr }

    /// Read ISV regime signals from pinned host memory (zero-copy, no GPU sync needed).
    /// Returns (regime_stability [0,1], volatility [0,1]).
    pub fn read_isv_regime(&self) -> (f32, f32) {
        if self.isv_signals_pinned.is_null() { return (0.5, 0.5); }
        unsafe {
            let stability = *self.isv_signals_pinned.add(11);  // regime_stability
            let volatility = *self.isv_signals_pinned.add(2);   // volatility proxy
            (stability.clamp(0.0, 1.0), volatility.clamp(0.0, 1.0))
        }
    }

    /// Task 2.X "make Full useful" diagnostic — read the per-magnitude-bin
    /// Q-mean EMAs and absolute-scale reference from ISV (pinned, zero-copy).
    ///
    /// Returns `(q_mean_quarter, q_mean_half, q_mean_full, q_abs_ref)`. The
    /// C51 loss / gradient kernels use these to compute the bin weight:
    ///   max_mean = max(q_mean_*)
    ///   bias_gap = max(0, max_mean − q_mean[a1])
    ///   collapse_frac = min(1, bias_gap / max(q_abs_ref, 1e-6))
    ///   bin_weight = 1 + collapse_frac * (a1 + 1) / b1_size
    /// Returns all zeros if ISV is not allocated.
    pub fn read_isv_magnitude_bin_q_means(&self) -> (f32, f32, f32, f32) {
        if self.isv_signals_pinned.is_null() { return (0.0, 0.0, 0.0, 0.0); }
        unsafe {
            let q = *self.isv_signals_pinned.add(Q_MAG_MEAN_QUARTER_INDEX);
            let h = *self.isv_signals_pinned.add(Q_MAG_MEAN_HALF_INDEX);
            let f = *self.isv_signals_pinned.add(Q_MAG_MEAN_FULL_INDEX);
            let r = *self.isv_signals_pinned.add(Q_ABS_REF_INDEX);
            (q, h, f, r.max(0.0))
        }
    }

    /// Task 2.Y "make direction branch useful at eval" diagnostic — read the
    /// per-direction-bin Q-mean EMAs and absolute-scale reference from ISV
    /// (pinned, zero-copy). Layout mirrors `read_isv_magnitude_bin_q_means`.
    ///
    /// Returns `(q_short, q_hold, q_long, q_flat, q_dir_abs_ref)`. The C51
    /// loss / gradient kernels consume these via `get_direction_bin_weight`:
    ///   max_mean      = max(q_short, q_hold, q_long, q_flat)
    ///   bias_gap      = max(0, max_mean − q_mean[a0])
    ///   collapse_frac = min(1, bias_gap / max(q_dir_abs_ref, 1e-6) +
    ///                       (1 − learning_health))
    ///   bin_weight    = 1 + collapse_frac * dir_bias_signal(a0)
    ///
    /// Returns all zeros if ISV is not allocated.
    pub fn read_isv_direction_bin_q_means(&self) -> (f32, f32, f32, f32, f32) {
        if self.isv_signals_pinned.is_null() { return (0.0, 0.0, 0.0, 0.0, 0.0); }
        unsafe {
            let s = *self.isv_signals_pinned.add(Q_DIR_MEAN_SHORT_INDEX);
            let h = *self.isv_signals_pinned.add(Q_DIR_MEAN_HOLD_INDEX);
            let l = *self.isv_signals_pinned.add(Q_DIR_MEAN_LONG_INDEX);
            let f = *self.isv_signals_pinned.add(Q_DIR_MEAN_FLAT_INDEX);
            let r = *self.isv_signals_pinned.add(Q_DIR_ABS_REF_INDEX);
            (s, h, l, f, r.max(0.0))
        }
    }

    /// B4/G5: Read ISV health index and regime stability from pinned host memory.
    /// Returns (health [0,1], regime_stability [0,1]). Falls back to (0.5, 0.5) if
    /// pinned pointer is null.
    pub(crate) fn read_isv_health_and_regime(&self) -> (f32, f32) {
        if self.isv_signals_pinned.is_null() { return (0.5_f32, 0.5_f32); }
        unsafe {
            let h = (*self.isv_signals_pinned.add(LEARNING_HEALTH_INDEX)).clamp(0.0, 1.0);
            let s = (*self.isv_signals_pinned.add(11)).clamp(0.0, 1.0);
            (h, s)
        }
    }

    // ── D1/N1: Temporal self-distillation ─────────────────────────────────

    /// D1/N1: Whether distillation gradient was applied in the most recent epoch.
    pub fn last_distill_active(&self) -> bool {
        self.last_distill_active
    }

    /// F4/F5 temporal amplification hook — cached meta-Q collapse prediction.
    pub fn last_meta_q_pred(&self) -> f32 {
        self.last_meta_q_pred
    }

    /// Called each epoch boundary from the host training loop after meta_q.predict().
    pub fn set_meta_q_pred(&mut self, pred: f32) {
        self.last_meta_q_pred = pred.clamp(0.0, 1.0);
    }

    /// D1/N1: Snapshot current params_buf if q_gap qualifies.
    /// Avoids split-borrow between params_buf and q_snapshots.
    ///
    /// Gate is q_gap-only — health is stored as metadata but doesn't gate
    /// acceptance. See the `q_snapshot` module docstring for rationale.
    pub fn maybe_snapshot_params(&mut self, health: f32, q_gap: f32, epoch: u32) -> Result<bool, MLError> {
        use cudarc::driver::{DevicePtr, DevicePtrMut};

        let ring = &mut self.q_snapshots;

        // Track every observation (accept or reject) so the dynamic floor
        // reflects the true per-run q_gap distribution.
        ring.observe_q_gap(q_gap);

        // Dynamic q_gap floor: scales with the decaying peak q_gap observed
        // in this run. Absolute floor of 0.001 catches NaN/degenerate states.
        // No health gate — see module docstring.
        if q_gap < ring.dynamic_q_gap_floor() {
            return Ok(false);
        }
        if ring.snapshots.len() >= crate::cuda_pipeline::q_snapshot::MAX_SNAPSHOTS {
            let worst_q_gap = ring.snapshots.iter().map(|s| s.q_gap).fold(f32::INFINITY, f32::min);
            if q_gap <= worst_q_gap {
                return Ok(false);
            }
        }

        let n = ring.param_count();
        let num_bytes = n * std::mem::size_of::<f32>();
        let mut new_weights = unsafe { self.stream.alloc::<f32>(n) }
            .map_err(|e| MLError::ModelError(format!("snapshot alloc: {e}")))?;
        {
            let (src_ptr, _sg) = self.params_buf.device_ptr(&self.stream);
            let (dst_ptr, _dg) = new_weights.device_ptr_mut(&self.stream);
            #[allow(unsafe_code)]
            unsafe {
                cudarc::driver::result::memcpy_dtod_async(
                    dst_ptr, src_ptr, num_bytes, self.stream.cu_stream(),
                ).map_err(|e| MLError::ModelError(format!("snapshot dtod: {e}")))?;
            }
        }

        let ring = &mut self.q_snapshots;
        if ring.snapshots.len() >= crate::cuda_pipeline::q_snapshot::MAX_SNAPSHOTS {
            let worst_idx = ring.snapshots.iter()
                .enumerate()
                .min_by(|(_, a), (_, b)| {
                    // NaN-safe: NaN q_gaps compare as Equal instead of panicking.
                    a.q_gap.partial_cmp(&b.q_gap).unwrap_or(std::cmp::Ordering::Equal)
                })
                .map(|(i, _)| i)
                .expect("MAX_SNAPSHOTS >= 1 and len >= MAX_SNAPSHOTS, so at least one snapshot exists");
            ring.snapshots.swap_remove(worst_idx);
        }

        ring.snapshots.push(crate::cuda_pipeline::q_snapshot::QSnapshot {
            weights: new_weights,
            health,
            q_gap,
            epoch,
        });

        // Mirror the (possibly-new) best snapshot into the stable
        // `distill_best_buf` the graph reads. The ring's best may have
        // changed (either because this snapshot is the new best, or
        // because capacity eviction moved the "best" to a different
        // index). Unconditional mirror avoids having to track which
        // snapshot is best across the capacity-eviction branch.
        self.mirror_best_snapshot_to_distill_buf()?;

        Ok(true)
    }

    /// D1/N1: Apply distillation gradient — pulls current weights toward the
    /// best snapshot. Launched **per training step** from the aux-op phase
    /// (between graph_forward's grad_buf zero-memset-and-fill and graph_adam's
    /// consumption), so the contribution actually reaches Adam. Prior
    /// epoch-boundary placement wrote to grad_buf *after* the last step —
    /// the next step's graph_forward zeroed it before Adam could consume it,
    /// making the whole mechanism a silent no-op.
    ///
    /// Fully GPU-native:
    ///  - `grad_buf`, `params`, `distill_best_buf`: stable device pointers,
    ///    baked into the CUDA Graph at capture time. Pointer addresses are
    ///    stable; their *contents* can vary between replays.
    ///  - `isv_signals_dev_ptr`: pinned device-mapped, the kernel reads
    ///    ISV[LEARNING_HEALTH_INDEX=12] to compute alpha on-device. The ISV
    ///    update kernel writes health from the GPU side — no CPU path.
    ///  - `alpha = 0.1 × (1 − health) × DISTILL_PER_STEP_SCALE` (computed in
    ///    the kernel itself, see `dqn_distill_saxpy_kernel`).
    ///
    /// No CPU→device writes on any path. The initial `distill_best_buf` is
    /// a copy of `params_buf` (done at construction via DtoD), so the initial
    /// `(params − best)` delta is zero and the kernel is a strict no-op until
    /// `mirror_best_snapshot_to_distill_buf` (called by
    /// `maybe_snapshot_params` when a new best lands) overwrites the contents.
    pub fn apply_distillation_gradient(&mut self) -> Result<(), MLError> {
        let grad_ptr = self.ptrs.grad_buf;
        let params_ptr = self.ptrs.params_ptr;
        let best_ptr = self.distill_best_buf.raw_ptr();
        let isv_ptr = self.isv_signals_dev_ptr;
        let n = self.total_params as i32;
        let blocks = self.grad_norm_blocks as u32;

        unsafe {
            self.stream
                .launch_builder(&self.distill_saxpy_aux)
                .arg(&grad_ptr)
                .arg(&params_ptr)
                .arg(&best_ptr)
                .arg(&isv_ptr)
                .arg(&n)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("distill saxpy kernel: {e}")))?;
        }

        // The mechanism is "active" whenever the kernel fires — which is
        // always, post-refactor. Actual effect depends on (1 − health) and
        // whether `distill_best_buf` has been populated with a real snapshot
        // (which HEALTH_DIAG can't easily introspect, but q_gap trajectory
        // reveals). This flag is kept for log-line backward compatibility.
        self.last_distill_active = true;
        Ok(())
    }

    /// D1/N1: Copy the ring's current best snapshot into the stable
    /// `distill_best_buf` that the distillation kernel reads. DtoD async on
    /// the training stream (cold path — fires only when a new best is
    /// accepted at epoch boundary, not per step). The buffer pointer stays
    /// stable so graph capture is unaffected; only its contents change.
    fn mirror_best_snapshot_to_distill_buf(&mut self) -> Result<(), MLError> {
        use cudarc::driver::DevicePtrMut;
        let best_ptr = match self.q_snapshots.best_raw_ptr() {
            Some(p) => p,
            None => return Ok(()),
        };
        let num_bytes = self.total_params * std::mem::size_of::<f32>();
        let (dst_ptr, _dg) = self.distill_best_buf.device_ptr_mut(&self.stream);
        #[allow(unsafe_code)]
        unsafe {
            cudarc::driver::result::memcpy_dtod_async(
                dst_ptr, best_ptr, num_bytes, self.stream.cu_stream(),
            ).map_err(|e| MLError::ModelError(format!("distill_best_buf dtod: {e}")))?;
        }
        Ok(())
    }

    // ── A3: LearningHealth signal writers / readers ───────────────────────

    /// Write a scalar into the ISV pinned buffer at the given index.
    /// No-op if the pinned pointer is null or index >= ISV_TOTAL_DIM.
    /// Accepts any slot in the full bus [0..ISV_TOTAL_DIM), including the
    /// scratchpad tail [ISV_NETWORK_DIM..ISV_TOTAL_DIM).
    pub fn write_isv_signal_at(&self, index: usize, value: f32) {
        if self.isv_signals_pinned.is_null() { return; }
        if index >= ISV_TOTAL_DIM { return; }
        unsafe { *self.isv_signals_pinned.add(index) = value; }
    }

    /// Read a scalar from the ISV pinned buffer at the given index.
    ///
    /// `isv_signals_pinned` is unconditionally allocated by the constructor
    /// (`cuMemAllocHost_v2` with `assert_eq!` on success), so non-null is an
    /// invariant. `index` is always a named constant (e.g. `GAMMA_DIR_EFF_INDEX`)
    /// drawn from the ISV slot table, so `index < ISV_TOTAL_DIM` is also a
    /// compile-time invariant. `debug_assert!` flags violations in dev builds;
    /// in release, the unsafe deref reflects the broken precondition rather
    /// than silently returning a misleading 0.0 (which would compete with
    /// real-zero ISV values in downstream consumers).
    pub fn read_isv_signal_at(&self, index: usize) -> f32 {
        debug_assert!(!self.isv_signals_pinned.is_null(),
            "read_isv_signal_at: isv_signals_pinned must be allocated by constructor");
        debug_assert!(index < ISV_TOTAL_DIM,
            "read_isv_signal_at: index {} out of bounds (ISV_TOTAL_DIM={})", index, ISV_TOTAL_DIM);
        unsafe { *self.isv_signals_pinned.add(index) }
    }

    /// SP4 Layer B helper: read both Adam bounds
    /// (`weight_clamp_max_abs`, `weight_decay_value`) for a given param
    /// group from ISV slots in one call. Replaces the verbose 2-line
    /// `read_isv_signal_at(weight_bound(...)).max(EPS_CLAMP_FLOOR);
    ///  read_isv_signal_at(wd_rate(...)).max(0.0)` pattern repeated at
    /// every per-group Adam launcher (DQN main, IQN, IQL high+low,
    /// Attention, TLOB).
    ///
    /// `weight_clamp_max_abs` enforces the SP1 cold-start ε floor
    /// (`EPS_CLAMP_FLOOR = 1.0`) on the post-Adam |p| bound. The
    /// returned value is always ≥ EPS_CLAMP_FLOOR; cold-start ISV reads
    /// (which can be 0.0 before the producer has run) are saturated to
    /// the floor so the kernel never silently disables clamping.
    ///
    /// `weight_decay_value` enforces the AdamW non-negativity contract
    /// (`max(0.0)`) — the kernel applies `wd * p` decoupled from Adam
    /// state, so a negative wd would create a positive feedback loop.
    /// Returns `0.0` for cold-start.
    #[inline]
    pub fn read_group_adam_bounds(
        &self,
        group: crate::cuda_pipeline::sp4_isv_slots::ParamGroup,
    ) -> (f32, f32) {
        use crate::cuda_pipeline::sp4_isv_slots::{weight_bound, wd_rate};
        use crate::cuda_pipeline::sp4_wiener_ema::EPS_CLAMP_FLOOR;
        let clamp = self.read_isv_signal_at(weight_bound(group.idx())).max(EPS_CLAMP_FLOOR);
        let wd    = self.read_isv_signal_at(wd_rate(group.idx())).max(0.0);
        (clamp, wd)
    }

    /// Read the ISV layout fingerprint from pinned memory and compare to the
    /// compile-time `LAYOUT_FINGERPRINT_CURRENT`.
    ///
    /// Returns `Err` on mismatch. NEVER returns `Ok` on mismatch — that would
    /// permit silent backward compat, which this design forbids. Treat a null
    /// ISV pointer as fingerprint `0`, which never matches any real fingerprint.
    ///
    /// Called at construction (after writing) to self-verify. Must also be called
    /// by any restore path that loads persisted ISV state from disk.
    pub fn check_layout_fingerprint(&self) -> Result<(), MLError> {
        let (lo_bits, hi_bits) = if self.isv_signals_pinned.is_null() {
            (0u32, 0u32)
        } else if ISV_TOTAL_DIM < ISV_LAYOUT_FINGERPRINT_HI_INDEX + 1 {
            (0u32, 0u32)
        } else {
            unsafe {
                let lo = f32::to_bits(*self.isv_signals_pinned.add(ISV_LAYOUT_FINGERPRINT_LO_INDEX));
                let hi = f32::to_bits(*self.isv_signals_pinned.add(ISV_LAYOUT_FINGERPRINT_HI_INDEX));
                (lo, hi)
            }
        };
        let fp_stored: u64 = (lo_bits as u64) | ((hi_bits as u64) << 32);
        if fp_stored != LAYOUT_FINGERPRINT_CURRENT {
            return Err(MLError::ModelError(format!(
                "ISV layout fingerprint mismatch: stored 0x{:016x}, current code 0x{:016x}. \
                 Checkpoint layout does not match current code - retrain required. \
                 (This is a structural fingerprint, not a version number. There is no migration path.)",
                fp_stored, LAYOUT_FINGERPRINT_CURRENT
            )));
        }
        Ok(())
    }

    /// Read atom utilization from q_readback_pinned[6] (one-step lag, written by
    /// reduce_current_q_stats).
    ///
    /// `q_readback_pinned` is unconditionally allocated by the constructor and
    /// never reassigned, so non-null is an invariant. `debug_assert!` catches
    /// any future regression that breaks the invariant.
    pub fn read_atom_utilization(&self) -> f32 {
        debug_assert!(!self.q_readback_pinned.is_null(),
            "read_atom_utilization: q_readback_pinned must be allocated by constructor");
        unsafe { (*self.q_readback_pinned.add(6)).clamp(0.0, 1.0) }
    }

    /// F2: Read a representative sample of Adam m-state across all 5 component
    /// buffers. Takes up to `max_per_component` elements from each, concatenates
    /// in a stable order for cross-epoch vector cosine similarity. Epoch-boundary
    /// call only (issues a synchronous DtoH).
    pub fn read_adam_m_flat_sample(&self, max_per_component: usize) -> Result<Vec<f32>, MLError> {
        let components: &[(&CudaSlice<f32>, &str)] = &[
            (&self.q_attn_adam_m, "q_attn"),
            (&self.sel_adam_m, "sel"),
            (&self.denoise_adam_m, "denoise"),
            (&self.mamba2_adam_m, "mamba2"),
            (&self.ofi_embed_adam_m, "ofi_embed"),
        ];
        let mut out = Vec::with_capacity(components.len() * max_per_component);
        for (buf, name) in components {
            let take = max_per_component.min(buf.len());
            if take == 0 { continue; }
            let view = buf.slice(0..take);
            let mut host = vec![0.0f32; take];
            super::dtoh_f32(&self.stream, &view, &mut host)
                .map_err(|e| MLError::ModelError(format!("read_adam_m_flat_sample {name}: {e}")))?;
            out.extend_from_slice(&host);
        }
        Ok(out)
    }

    /// F3: Spectral gap — sigma_1 / sigma_2 of the rolling Q-sample matrix.
    /// High ratio = rank-1 collapse (one direction dominates); ~1.0 = balanced.
    ///
    /// Maintains a host-side rolling buffer of the last 64 Q-value samples
    /// (each `total_actions` floats from `q_readback_pinned[7..]`). Once ≥ 8
    /// samples are available, computes sigma_1/sigma_2 via power iteration on
    /// the n_cols × n_cols Gram matrix X^T X (matrix is tiny — instant on host).
    /// Falls back to single-sample max/min ratio when the buffer is too small.
    ///
    /// The downstream `NormalizedComponents::from_raw` thresholds are calibrated
    /// for this behaviour (`spectral_gap_norm` drops toward 0 as this value rises).
    pub fn compute_q_spectral_gap(&mut self) -> f32 {
        // `q_readback_pinned` is constructor-allocated and never reassigned;
        // `total_actions()` is sum of branch sizes which the trainer config
        // requires to be non-zero. Invariants enforced by debug_assert.
        debug_assert!(!self.q_readback_pinned.is_null(),
            "compute_q_spectral_gap: q_readback_pinned must be allocated by constructor");
        let n_cols = self.total_actions();
        debug_assert!(n_cols > 0,
            "compute_q_spectral_gap: total_actions={} (config must specify at least one action)", n_cols);
        let mut sample = Vec::with_capacity(n_cols);
        let mut nonfinite_seen = false;
        unsafe {
            let base = self.q_readback_pinned.add(7);
            for i in 0..n_cols {
                let v = *base.add(i);
                if !v.is_finite() {
                    nonfinite_seen = true;
                    sample.push(0.0); // placeholder — escape hatch below returns collapse-signal
                } else {
                    sample.push(v);
                }
            }
        }
        if nonfinite_seen {
            // Non-finite Q-values indicate training instability (NaN/Inf
            // gradient propagation). Surface this as a collapse signal
            // (high spectral gap, downstream calibration's "rank-1
            // collapse" sentinel) rather than the misleading 1.0 = "balanced"
            // that earlier code returned. Logged at warn so HEALTH_DIAG
            // operators see the failure mode without searching for it.
            tracing::warn!(
                target = "ml::trainers::dqn::gpu_dqn_trainer",
                "compute_q_spectral_gap: non-finite Q-value(s) in readback — surfacing collapse sentinel 100.0"
            );
            return 100.0; // ok: domain encoding — same sentinel as lambda_2/sigma2 degenerate cases below
        }

        // Push into rolling history (capacity 64).
        if self.q_sample_history.len() >= 64 { self.q_sample_history.pop_front(); }
        self.q_sample_history.push_back(sample);

        if self.q_sample_history.len() < 8 {
            // Not enough samples yet — fall back to coarse max/min ratio of the
            // latest sample (pre-F3 behaviour). The `push_back` above guarantees
            // history length ≥ 1, so `back()` returns Some; `expect` documents
            // that invariant for any future reader who refactors the push.
            let latest = self.q_sample_history.back()
                .expect("q_sample_history just received push_back; back() cannot be None");
            let mut max = f32::NEG_INFINITY;
            let mut min = f32::INFINITY;
            for &v in latest {
                if v.is_finite() {
                    if v > max { max = v; }
                    if v < min { min = v; }
                }
            }
            let range = (max - min).abs();
            return if range < 1e-6 { 100.0 } else { 1.0 + range.recip() };
        }

        // Build centered X matrix (B × n_cols), compute column means first.
        let b = self.q_sample_history.len();
        let mut means = vec![0.0f32; n_cols];
        for s in &self.q_sample_history {
            for j in 0..n_cols { means[j] += s[j]; }
        }
        for m in &mut means { *m /= b as f32; }

        // Gram matrix G = X^T X, an n_cols × n_cols symmetric PSD matrix.
        let mut gram = vec![0.0f32; n_cols * n_cols];
        for s in &self.q_sample_history {
            for i in 0..n_cols {
                let ci = s[i] - means[i];
                for j in 0..n_cols {
                    let cj = s[j] - means[j];
                    gram[i * n_cols + j] += ci * cj;
                }
            }
        }

        // Power iteration for largest eigenvalue (lambda_1 = sigma_1^2).
        let lambda_1 = power_iteration_largest(&gram, n_cols, 40);
        // Deflate and compute second-largest.
        let deflated = deflate_rank_one(&gram, n_cols, lambda_1);
        let lambda_2 = power_iteration_largest(&deflated, n_cols, 40);

        // Domain encoding: lambda_2 ≈ 0 (and equivalently sigma2 ≈ 0) means the
        // deflated Gram matrix has effectively rank ≤ 1 — sigma_1 / sigma_2
        // diverges. Returning the calibrated collapse sentinel 100.0 lets
        // `NormalizedComponents::from_raw` map this to its highest collapse
        // score without producing an Inf/NaN.
        if lambda_2 < 1e-12 { return 100.0; } // ok: domain encoding — rank-1 collapse sentinel (deflated Gram is singular)
        let sigma1 = lambda_1.max(0.0).sqrt();
        let sigma2 = lambda_2.max(0.0).sqrt();
        if sigma2 < 1e-6 { return 100.0; } // ok: domain encoding — rank-1 collapse sentinel (sigma_2 below numerical floor)
        sigma1 / sigma2
    }

    /// Read eval_q_std_ema as the direction-branch scalar — matches the
    /// pre-unification single-EMA behaviour for callers that have not yet
    /// migrated to per-branch awareness.
    pub fn eval_q_std_ema(&self) -> f32 { self.eval_q_std_ema[0] }

    /// Read all four per-branch `q_std_ema` values.
    pub fn eval_q_std_ema_per_branch(&self) -> [f32; 4] { self.eval_q_std_ema }

    // G6 branch_indep_loss_value + G10 temporal_loss_value removed —
    // their buffers were deallocated after V7-gem sub-noise verdict.

    /// G12: Read predictive_coding loss scalar (sync DtoH; epoch-boundary only).
    /// Same pattern as G6/G10. Used for HEALTH_DIAG monitoring.
    pub fn predictive_loss_value(&self) -> f32 {
        unsafe { cudarc::driver::sys::cuStreamSynchronize(self.stream.cu_stream()); }
        let mut v = 0.0_f32;
        unsafe {
            cudarc::driver::sys::cuMemcpyDtoH_v2(
                std::ptr::addr_of_mut!(v).cast(),
                self.predictive_loss_buf.raw_ptr(),
                std::mem::size_of::<f32>(),
            );
        }
        v
    }

    /// SP5 Layer D Task D4 (2026-05-02): cold-path stream synchronisation
    /// helper for ISV-slot read-back after a producer kernel chain.
    ///
    /// Called once per epoch boundary by the D4 atomic wiring of D1/D2/D3
    /// to ensure the apply_pearls writes to ISV[286..297) are visible to
    /// `read_isv_signal_at` (which reads via the mapped pinned host_ptr
    /// alias). Mapped pinned coherence requires a stream-fence boundary
    /// between kernel writes (via `dev_ptr`) and host reads (via
    /// `host_ptr`); the host-side `isv_signals_pinned` is only safe to
    /// dereference after this sync.
    ///
    /// This is the same stream sync pattern as `per_branch_q_gap_ema`
    /// below — once-per-epoch DtoH/host-read sites are exempt from the
    /// no-sync rule (per `feedback_no_cpu_compute_strict.md`'s
    /// host-side-compute carve-out for cold-path metrics that drive
    /// downstream HEALTH_DIAG / smoke-test asserts). Producer launches
    /// remain on the same training stream — no event-based fence is
    /// needed because every kernel that subsequently reads from ISV via
    /// `dev_ptr` runs on the same stream and sees the same FIFO order.
    pub fn synchronize_isv_stream(&self) {
        unsafe { cudarc::driver::sys::cuStreamSynchronize(self.stream.cu_stream()); }
    }

    /// Read per_branch_q_gap_ema — synchronous DtoH for trajectory backtracking.
    pub fn per_branch_q_gap_ema(&self) -> [f32; 4] {
        unsafe { cudarc::driver::sys::cuStreamSynchronize(self.stream.cu_stream()); }
        let mut v = [0.0_f32; 4];
        unsafe {
            cudarc::driver::sys::cuMemcpyDtoH_v2(
                v.as_mut_ptr().cast(),
                self.per_branch_q_gap_ema_buf.raw_ptr(),
                4 * std::mem::size_of::<f32>(),
            );
        }
        v
    }

    /// Set eval_q_mean_ema for all four branches (trajectory backtracking
    /// restore). Scalar setter kept for backwards compatibility — broadcasts
    /// the same value to every branch. Callers that snapshot per-branch state
    /// should use `set_eval_q_mean_ema_per_branch`.
    pub fn set_eval_q_mean_ema(&mut self, v: f32) {
        self.eval_q_mean_ema = [v; 4];
        self.eval_ema_initialized = [v != 0.0; 4];
    }

    /// Set eval_q_std_ema for all four branches (trajectory backtracking
    /// restore). Scalar setter kept for backwards compatibility — broadcasts
    /// the same value to every branch. Callers that snapshot per-branch state
    /// should use `set_eval_q_std_ema_per_branch`.
    pub fn set_eval_q_std_ema(&mut self, v: f32) {
        self.eval_q_std_ema = [v; 4];
    }

    /// Per-branch setter for trajectory backtracking restore.
    pub fn set_eval_q_mean_ema_per_branch(&mut self, v: [f32; 4]) {
        self.eval_q_mean_ema = v;
        // Treat any non-zero value as evidence the EMA was observed before.
        for b in 0..4 {
            self.eval_ema_initialized[b] = v[b] != 0.0 || self.eval_ema_initialized[b];
        }
    }

    /// Per-branch setter for trajectory backtracking restore.
    pub fn set_eval_q_std_ema_per_branch(&mut self, v: [f32; 4]) {
        self.eval_q_std_ema = v;
    }

    /// Reset the adaptive C51 v_range state at a fold boundary.
    ///
    /// Clears the per-branch `eval_q_mean_ema` / `eval_q_std_ema` arrays and
    /// resets both the legacy `eval_v_range_pinned[2]` buffer and the 8
    /// per-branch ISV slots (23..30) back to bootstrap values so that fold
    /// N+1 starts with atoms spanning the full `[config.v_min, config.v_max]`
    /// range on every branch. Without this, Fold N+1 inherits Fold N's final
    /// tight atom support — if the new fold's Q-distribution doesn't fit the
    /// stale range, TD errors explode and training NaN's out (observed:
    /// train-92xbj Fold 1 Epoch 12, grad_norm=inf → NaN loss at step 5).
    ///
    /// The first call to `update_eval_v_range` after this reset will
    /// reinitialise each branch's EMA from the new fold's first observed
    /// `(q_mean, q_std)`.
    pub fn reset_eval_v_range_state(&mut self) {
        self.eval_q_mean_ema = [0.0_f32; 4];
        self.eval_q_std_ema = [0.0_f32; 4];
        self.eval_ema_initialized = [false; 4];
        // Restore the pinned device-mapped legacy range to the theoretical
        // bounds so the experience-collection kernels running before the
        // next update_eval_v_range call see the wide safe support.
        if !self.eval_v_range_pinned.is_null() {
            unsafe {
                *self.eval_v_range_pinned = self.config.v_min;
                *self.eval_v_range_pinned.add(1) = self.config.v_max;
            }
        }
        // Bootstrap the 8 per-branch ISV slots to centre=0, half=abs_half.
        // Matches the construction-time bootstrap — epoch 1 of the next fold
        // sees `[config.v_min, config.v_max]` on every branch.
        if !self.isv_signals_pinned.is_null() {
            let v_min_f = self.config.v_min;
            let v_max_f = self.config.v_max;
            let half_bootstrap = 0.5_f32 * (v_max_f - v_min_f);
            unsafe {
                for b in 0..4usize {
                    *self.isv_signals_pinned.add(V_CENTER_DIR_INDEX + 2 * b) = 0.0_f32;
                    *self.isv_signals_pinned.add(V_CENTER_DIR_INDEX + 2 * b + 1) = half_bootstrap;
                }
            }
        }
    }

    /// Reset IQN readiness gating state at fold boundary.
    ///
    /// `update_iqn_readiness` (gpu_dqn_trainer.rs:5804) is a streaming
    /// improvement gauge: `iqn_readiness = (initial - ema) / initial`.
    /// `iqn_loss_initial` is captured once on the first call where it is
    /// still ~0 and never resets — across folds it stays pinned to the
    /// fold-0 epoch-1 IQN loss. After fold-boundary shrink-and-perturb +
    /// IQN target sync the new fold's first IQN losses can land far
    /// from the fold-0 baseline (different reward distribution post-
    /// adversarial regime, fresh online↔target alignment), so the
    /// readiness fraction is computed against a stale anchor — either
    /// pinning readiness near 1 (over-confident, full IQN gradient
    /// weight on a still-recovering head) or near 0 (under-weighting a
    /// converged head).
    ///
    /// **Fix**: zero `iqn_loss_initial` so the next `update_iqn_readiness`
    /// call captures fold N+1's first batch as the new baseline; zero
    /// the EMA + the readiness scalar; write 0.0 through the device-
    /// mapped pinned slot so any in-flight kernel reads the reset value
    /// without an HtoD copy. `update_iqn_readiness` already handles the
    /// `iqn_loss_initial < 1e-12` case as the bootstrap branch.
    ///
    /// SP4 Layer C close-out C2 (2026-05-01): `iqn_loss_initial` and
    /// `iqn_loss_ema` storage migrated to mapped-pinned slots updated by
    /// `update_iqn_readiness_kernel`; the fold-boundary reset writes 0.0
    /// through their host_ptrs so the next kernel launch (from the new
    /// fold's first `update_iqn_readiness` call) sees `prev_initial < 1e-12`
    /// and re-enters the bootstrap branch.
    pub fn reset_iqn_readiness_state(&mut self) {
        self.iqn_readiness = 0.0;
        if !self.iqn_readiness_pinned.is_null() {
            // Pinned device-mapped: writing the host slot propagates
            // immediately to GPU via the mapping; no HtoD copy issued.
            unsafe { *self.iqn_readiness_pinned = 0.0; }
        }
        if !self.iqn_loss_initial_pinned.is_null() {
            unsafe { *self.iqn_loss_initial_pinned = 0.0; }
        }
        if !self.iqn_loss_ema_pinned.is_null() {
            unsafe { *self.iqn_loss_ema_pinned = 0.0; }
        }
    }

    /// Read the streaming smoothed IQN loss from the mapped-pinned slot.
    /// SP4 Layer C close-out C2 (2026-05-01) accessor — the pinned slot is
    /// the authoritative storage; the kernel's `__threadfence_system()`
    /// guarantees PCIe-visibility before this read sees a fresh value.
    pub fn iqn_loss_ema_value(&self) -> f32 {
        if self.iqn_loss_ema_pinned.is_null() { 0.0 }
        else { unsafe { *self.iqn_loss_ema_pinned } }
    }

    /// Read the IQN loss fold-anchor from the mapped-pinned slot. Same
    /// rationale as `iqn_loss_ema_value()`.
    pub fn iqn_loss_initial_value(&self) -> f32 {
        if self.iqn_loss_initial_pinned.is_null() { 0.0 }
        else { unsafe { *self.iqn_loss_initial_pinned } }
    }

    /// Set per_branch_q_gap_ema — HtoD for trajectory backtracking restore.
    pub fn set_per_branch_q_gap_ema(&mut self, v: [f32; 4]) {
        unsafe {
            cudarc::driver::sys::cuMemcpyHtoDAsync_v2(
                self.per_branch_q_gap_ema_buf.raw_ptr(),
                v.as_ptr().cast(),
                4 * std::mem::size_of::<f32>(),
                self.stream.cu_stream(),
            );
        }
    }

    /// Read adam_step counter.
    pub fn adam_step_val(&self) -> i32 { self.adam_step }

    // ═══════════════════════════════════════════════════════════════════

    /// GPU-direct training step — no CPU roundtrip.
    ///
    /// Takes Candle GPU tensors from `GpuBatch` and copies them to trainer
    /// buffers via DtoD memcpy (skipping the GPU→CPU `.to_vec1()` → CPU→GPU
    /// `memcpy_htod` roundtrip of `train_step()`).
    ///
    /// Same CUDA Graph execution and readback as `train_step()`.
    #[allow(clippy::too_many_arguments)]
    pub fn train_step_gpu(
        &mut self,
        gpu_batch: &crate::dqn::replay_buffer_type::GpuBatch,
        online_dueling: &DuelingWeightSet,
        online_branching: &BranchingWeightSet,
        _target_dueling: &DuelingWeightSet,
        _target_branching: &BranchingWeightSet,
    ) -> Result<FusedTrainScalars, MLError> {
        // ── First call: flatten weights ──────────────────────────────
        if !self.params_initialized {
            self.flatten_online_weights(online_dueling, online_branching)?;
            self.params_initialized = true;
        }

        // Initialize target = online via flat buffer DtoD copy.
        // Replaces per-layer copy_weights_from at DQN construction.
        if !self.target_params_initialized {
            let n_bytes = self.total_params * std::mem::size_of::<f32>();
            let src = self.params_buf.raw_ptr();
            let dst = self.target_params_buf.raw_ptr();
            unsafe {
                cudarc::driver::result::memcpy_dtod_async(dst, src, n_bytes, self.stream.cu_stream())
                    .map_err(|e| MLError::ModelError(format!("init target DtoD: {e}")))?;
            }
            self.target_params_initialized = true;
        }

        // ── GPU-direct upload (DtoD, no CPU staging) ─────────────────
        self.upload_batch_gpu(gpu_batch)?;

        // Forward ops submitted ungraphed — graph capture is managed by
        // FusedTrainingCtx's child graph architecture.
        self.adam_step += 1;
        // Pinned device-mapped: host write is visible to GPU (no HtoD copy).
        unsafe { *self.t_pinned = self.adam_step; }
        self.update_stochastic_depth_mask()?;
        self.submit_forward_ops_main()?;
        self.submit_forward_ops_ddqn()?;
        Ok(FusedTrainScalars { total_loss: 0.0, grad_norm: 0.0 })
    }

    /// Replay graph_adam + readback scalars. Call AFTER injecting auxiliary
    /// gradients into grad_buf (IQN, attention, ensemble).

    /// Adaptive per-branch gradient-norm balancer.
    ///
    /// For each factored-action branch `d ∈ {0..4}` (direction, magnitude,
    /// order, urgency) computes the branch's weight-gradient L2 norm,
    /// and scales the branch's gradient down to the cap
    ///
    ///   `cap = num_branches × median(branch_norms)`
    ///
    /// whenever its norm exceeds the cap.  Branches within the cap pass
    /// through unchanged.
    ///
    /// Why this shape is fully adaptive with no tuned knobs:
    ///   * `num_branches = 4` is the architectural count of factored
    ///     action axes.  Not a hyperparameter.
    ///   * `median(branch_norms)` is a per-step statistical reference
    ///     that tracks the current gradient regime (Q-magnitude, loss
    ///     blend, ISV-driven scales).
    ///   * Their product gives the defensible bound: "no branch may
    ///     dominate by more than `num_branches` × the median" — i.e.
    ///     no branch can carry more L2 mass than all other branches
    ///     would, evaluated at the median.
    ///
    /// Fixes the pathology observed on L40S (train-mdh86, epochs 0-6):
    /// `grad_ratio_mag_dir ≈ 1.5e4-2.6e4×` for the warm-up, then a
    /// single-step collapse to ~100× at epoch 7 that destabilised
    /// learning (Sharpe +34 → -67).  Symmetric capping prevents the
    /// swing at both ends.
    ///
    /// Insertion point: inside the `adam_grad_child` graph, AFTER all
    /// aux grad writes complete and BEFORE `compute_grad_norm_for_adam`
    /// — so Adam's global clip and the Adam update both observe the
    /// rebalanced gradient.  Two kernel launches; no atomicAdd; no host
    /// syncs; no dynamic allocations — safe to capture.
    pub(crate) fn launch_branch_grad_balance(&self) -> Result<(), MLError> {
        let grad_ptr     = self.ptrs.grad_buf;
        let starts_ptr   = self.branch_slice_starts_dev.raw_ptr();
        let lens_ptr     = self.branch_slice_lens_dev.raw_ptr();
        let norms_ptr    = self.branch_grad_norms_dev.raw_ptr();
        let scales_ptr   = self.branch_grad_scales_dev.raw_ptr();
        let isv_ptr      = self.isv_signals_dev_ptr;
        let target_base_idx: i32 = GRAD_NORM_TARGET_DIR_INDEX as i32;
        let limit_idx: i32 = GRAD_SCALE_LIMIT_INDEX as i32;

        // ── Pass 1: per-branch L2 reduction (grid=(4,), block=(256,)) ─
        unsafe {
            self.stream
                .launch_builder(&self.branch_grad_balance_reduce)
                .arg(&grad_ptr)
                .arg(&starts_ptr)
                .arg(&lens_ptr)
                .arg(&norms_ptr)
                .launch(LaunchConfig {
                    grid_dim: (4, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(
                    format!("branch_grad_norm_reduce launch: {e}")
                ))?;
        }

        // ── Pass 2: ISV-driven producer (grid=(1,), block=(32,)) ──────
        // Reads per-branch norms, applies adaptive-rate EMA to the ISV
        // targets (slots 31..34) and scale clamp limit (slot 35). Must
        // run AFTER reduce and BEFORE rescale so rescale reads the
        // updated targets. Graph-safe: fixed grid/block, no atomicAdd,
        // single producer path.
        unsafe {
            self.stream
                .launch_builder(&self.branch_grad_balance_isv_update)
                .arg(&norms_ptr)
                .arg(&isv_ptr)
                .arg(&target_base_idx)
                .arg(&limit_idx)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (32, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(
                    format!("grad_balance_isv_update launch: {e}")
                ))?;
        }

        // ── Pass 3: rescale (grid=(blocks_x, 4,), block=(256,)) ───────
        // blocks_x covers the largest branch; shorter branches early-exit
        // via the `idx < len` guard in the kernel. Each branch reads
        // `isv_signals[GRAD_NORM_TARGET_DIR_INDEX + branch]` and
        // `isv_signals[GRAD_SCALE_LIMIT_INDEX]` to compute its scale as
        // `clamp(target / norm, 1/limit, limit)`. Equalises branches to
        // the adaptive median; bounded by observed-spread-driven limit.
        let blocks_x = ((self.branch_slice_max_len as u32 + 255) / 256).max(1);
        unsafe {
            self.stream
                .launch_builder(&self.branch_grad_balance_rescale)
                .arg(&grad_ptr)
                .arg(&starts_ptr)
                .arg(&lens_ptr)
                .arg(&norms_ptr)
                .arg(&isv_ptr)
                .arg(&target_base_idx)
                .arg(&limit_idx)
                .arg(&scales_ptr)
                .launch(LaunchConfig {
                    grid_dim: (blocks_x, 4, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(
                    format!("branch_grad_rescale launch: {e}")
                ))?;
        }

        Ok(())
    }

    /// Plan 1 Task 13: GPU-driven tau update.
    ///
    /// Single-thread cold-path kernel: reads ISV[EPOCH_IDX=39, TOTAL_EPOCHS=40,
    /// LEARNING_HEALTH=12]; writes ISV[TAU_EFF_INDEX=42] with cosine schedule +
    /// health-coupled floor. Must be called once per epoch boundary BEFORE any
    /// tau consumer reads ISV[TAU_EFF_INDEX].
    pub(crate) fn launch_tau_update(&self, tau_base: f32, tau_final: f32) -> Result<(), MLError> {
        let isv_ptr = self.isv_signals_dev_ptr;
        let isv_out = isv_ptr; // kernel writes to the same ISV bus (offset 42)
        unsafe {
            self.stream
                .launch_builder(&self.tau_update_kernel)
                .arg(&isv_ptr)
                .arg(&isv_out)
                .arg(&tau_base)
                .arg(&tau_final)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (1, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("tau_update launch: {e}")))?;
        }
        Ok(())
    }

    /// Plan 1 Task 14: GPU-driven epsilon update.
    ///
    /// Single-thread cold-path kernel: reads ISV[EPOCH_IDX=39, TOTAL_EPOCHS=40,
    /// LEARNING_HEALTH=12]; writes ISV[EPSILON_EFF_INDEX=41] with cosine schedule
    /// + health-coupled boost. Must be called once per epoch boundary BEFORE
    /// any epsilon consumer reads ISV[EPSILON_EFF_INDEX].
    pub(crate) fn launch_epsilon_update(&self, eps_start: f32, eps_end: f32) -> Result<(), MLError> {
        let isv_ptr = self.isv_signals_dev_ptr;
        let isv_out = isv_ptr; // kernel writes to the same ISV bus (offset 41)
        unsafe {
            self.stream
                .launch_builder(&self.epsilon_update_kernel)
                .arg(&isv_ptr)
                .arg(&isv_out)
                .arg(&eps_start)
                .arg(&eps_end)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (1, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("epsilon_update launch: {e}")))?;
        }
        Ok(())
    }

    /// Plan 2 Task 3 D.2: GPU-driven per-branch gamma update.
    ///
    /// 4-thread cold-path kernel (one thread per branch): reads per-branch
    /// V_CENTER/V_HALF from ISV[23..31) and LEARNING_HEALTH from ISV[12];
    /// writes ISV[43..47) = [GAMMA_DIR, GAMMA_MAG, GAMMA_ORD, GAMMA_URG].
    /// Must be called once per epoch boundary BEFORE any gamma consumer reads
    /// the per-branch slots. GPU-drives invariant (§4.C.6): no CPU-side γ.
    pub(crate) fn launch_per_branch_gamma_update(&self) -> Result<(), MLError> {
        let isv_ptr = self.isv_signals_dev_ptr;
        let isv_out = isv_ptr; // in-place: kernel writes to same ISV bus
        let base_ptr = self.per_branch_gamma_base_dev.raw_ptr();
        let max_ptr  = self.per_branch_gamma_max_dev.raw_ptr();
        unsafe {
            self.stream
                .launch_builder(&self.per_branch_gamma_update_kernel)
                .arg(&isv_ptr)
                .arg(&isv_out)
                .arg(&base_ptr)
                .arg(&max_ptr)
                .launch(LaunchConfig {
                    grid_dim:        (1, 1, 1),
                    block_dim:       (4, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("per_branch_gamma_update launch: {e}")))?;
        }
        Ok(())
    }

    /// Plan 1 Task 11: GPU-driven Kelly cap update.
    ///
    /// Single-thread cold-path kernel: reads ISV[LEARNING_HEALTH_INDEX=12] and
    /// aggregates Kelly win/loss stats from `portfolio_state[n_envs, PS_STRIDE]`;
    /// writes ISV[KELLY_CAP_EFF_INDEX=44] with the mean half-Kelly fraction
    /// multiplied by a health-coupled safety multiplier (0.5 + 0.5*health).
    /// Must be called once per epoch boundary BEFORE any kelly_cap consumer
    /// reads ISV[KELLY_CAP_EFF_INDEX].
    pub(crate) fn launch_kelly_cap_update(
        &self,
        portfolio_state_dev_ptr: u64,
        n_envs: i32,
    ) -> Result<(), MLError> {
        let isv_ptr = self.isv_signals_dev_ptr;
        let isv_out = isv_ptr; // kernel writes to the same ISV bus (offset 44)
        let ps_stride: i32 = 43; // PS_STRIDE from state_layout.cuh (grown 41→43 by Plan 3 D.4c)
        unsafe {
            self.stream
                .launch_builder(&self.kelly_cap_update_kernel)
                .arg(&isv_ptr)
                .arg(&isv_out)
                .arg(&portfolio_state_dev_ptr)
                .arg(&n_envs)
                .arg(&ps_stride)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (1, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("kelly_cap_update launch: {e}")))?;
        }
        Ok(())
    }

    /// Plan 2 Task 1 C.1: GPU-driven per-branch Q P5/P95 percentile EMA update.
    ///
    /// Single-block cold-path kernel (grid=(4,1,1), block=(1,1,1)): for each branch,
    /// collects per-sample max-Q-over-actions, sorts via bitonic/quickselect, picks the
    /// 5th and 95th percentile, and EMA-blends into ISV[Q_P05_*=47..51) and
    /// ISV[Q_P95_*=51..55). Must be called once per epoch boundary AFTER
    /// `reduce_current_q_stats` (which calls `populate_q_out`) and BEFORE
    /// `update_eval_v_range` reads the percentile slots.
    ///
    /// ema_alpha: adaptive smoothing rate ∈ [0.01, 0.5]. A rate of 0.1 is a
    /// reasonable default (10% of each new epoch's estimate).
    pub(crate) fn launch_q_quantile_reduce(&self, ema_alpha: f32) -> Result<(), MLError> {
        let q_out_ptr    = self.q_out_buf.raw_ptr();
        let n_samples    = self.config.batch_size as i32;
        let total_acts   = self.total_actions() as i32;
        let off_ptr      = self.q_quantile_branch_offsets_dev.raw_ptr();
        let sz_ptr       = self.q_quantile_branch_sizes_dev.raw_ptr();
        let isv_ptr      = self.isv_signals_dev_ptr;
        let isv_out      = isv_ptr; // writes back to same pinned bus (reads prev before writing)
        unsafe {
            self.stream
                .launch_builder(&self.q_quantile_kernel)
                .arg(&q_out_ptr)
                .arg(&n_samples)
                .arg(&total_acts)
                .arg(&off_ptr)
                .arg(&sz_ptr)
                .arg(&isv_ptr)
                .arg(&isv_out)
                .arg(&ema_alpha)
                .launch(LaunchConfig {
                    grid_dim:  (4, 1, 1), // one block per branch
                    block_dim: (1, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("q_quantile_reduce launch: {e}")))?;
        }
        Ok(())
    }

    /// Read back the per-branch scales applied on the most recent
    /// `launch_branch_grad_balance` call — `[dir, mag, ord, urg]`.
    /// Values in `[0, 1]`; `1.0` means the branch passed through
    /// unchanged; `< 1.0` means the branch exceeded
    /// `num_branches × median(branch_norms)` and was capped to that
    /// ratio.
    ///
    /// Epoch-boundary cold path (stream-syncs once, same pattern as
    /// `per_branch_grad_norms`).  Useful for HEALTH_DIAG auditing — a
    /// scale of `~1.0/K` on the magnitude branch confirms the cap
    /// fired at `num_branches × median / K × norm_mag = cap`.
    pub fn branch_grad_scales(&self) -> Result<[f32; 4], MLError> {
        self.stream.synchronize().map_err(|e| {
            MLError::ModelError(format!("branch_grad_scales sync: {e}"))
        })?;
        let mut host = [1.0_f32; 4];
        self.stream
            .memcpy_dtoh(&self.branch_grad_scales_dev.slice(..4), &mut host)
            .map_err(|e| MLError::ModelError(format!("branch_grad_scales dtoh: {e}")))?;
        self.stream.synchronize().map_err(|e| {
            MLError::ModelError(format!("branch_grad_scales post-dtoh sync: {e}"))
        })?;
        Ok(host)
    }

    /// Read back the per-branch gradient L2 norms computed by the most
    /// recent `launch_branch_grad_balance` reduce pass — `[dir, mag,
    /// ord, urg]`.  These are the norms the rescale pass observed
    /// BEFORE capping; compare against `branch_grad_scales` to
    /// reconstruct the post-cap ratio (`norm × scale ≤ cap`).
    pub fn branch_grad_norms_device(&self) -> Result<[f32; 4], MLError> {
        self.stream.synchronize().map_err(|e| {
            MLError::ModelError(format!("branch_grad_norms_device sync: {e}"))
        })?;
        let mut host = [0.0_f32; 4];
        self.stream
            .memcpy_dtoh(&self.branch_grad_norms_dev.slice(..4), &mut host)
            .map_err(|e| MLError::ModelError(format!("branch_grad_norms_device dtoh: {e}")))?;
        self.stream.synchronize().map_err(|e| {
            MLError::ModelError(format!("branch_grad_norms_device post-dtoh sync: {e}"))
        })?;
        Ok(host)
    }

    /// Compute gradient norm for adam_child graph capture.
    /// Uses grad_norm_kernel (NOT standalone) — the standalone is already captured
    /// in forward_child (d_logits clipping). Sharing the same CUfunction across
    /// two captured graphs corrupts kernel state on Hopper, causing 3120ms replays.
    pub fn compute_grad_norm_for_adam(&mut self) -> Result<(), MLError> {
        self.launch_grad_norm()?;
        // Use adam-specific finalize instance — grad_norm_finalize_kernel is already
        // captured in forward_child (d_logits clipping). Sharing CUfunction across
        // child graphs corrupts kernel state on Hopper → 3100ms replay.
        let partials_ptr = self.grad_norm_partials.raw_ptr();
        let buf_ptr = self.grad_norm_dev_ptr;
        let nb = self.grad_norm_blocks as i32;
        unsafe {
            self.stream
                .launch_builder(&self.grad_norm_finalize_adam)
                .arg(&partials_ptr)
                .arg(&buf_ptr)
                .arg(&nb)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("grad_norm_finalize_adam: {e}")))?;
        }
        Ok(())
    }

    /// Readback scalars from pinned device-mapped memory. No graph replay --
    /// graph replay is handled by FusedTrainingCtx parent graph launch.
    /// Reads the PREVIOUS step's values (double-buffered via GPU→host latency).
    pub fn readback_training_scalars(&mut self) -> Result<FusedTrainScalars, MLError> {
        let (loss, mse_loss, grad_norm) = unsafe {
            (*self.total_loss_pinned, *self.mse_loss_pinned, *self.grad_norm_pinned)
        };
        let blended = Self::blend_loss(self.c51_alpha, mse_loss, loss);
        let prev_scalars = FusedTrainScalars {
            total_loss: blended,
            grad_norm,
        };
        self.scalars_readback_host = [prev_scalars.total_loss, prev_scalars.grad_norm];
        Ok(prev_scalars)
    }

    /// Flush the last in-flight readback at epoch end — non-blocking.
    ///
    /// Reads directly from the pinned host buffer without synchronizing.
    /// The 12-byte async DtoH from the previous step completes in <1 µs,
    /// while thousands of GPU kernel launches elapse between the last
    /// `replay_adam_and_readback()` and epoch end, so the data is guaranteed
    /// to have landed. The training guard reads loss/grad_norm from GPU
    /// buffers directly — these CPU scalars are monitoring-only.
    pub fn flush_readback(&mut self) -> Result<FusedTrainScalars, MLError> {
        self.readback_pending = false;
        // Read directly from pinned device-mapped host memory — no DtoH needed.
        // The last forward pass wrote these values to host DRAM via PCIe; by epoch
        // end, thousands of kernel launches have elapsed, guaranteeing visibility.
        let (loss, mse_loss, grad_norm) = unsafe {
            (*self.total_loss_pinned, *self.mse_loss_pinned, *self.grad_norm_pinned)
        };
        let blended = Self::blend_loss(self.c51_alpha, mse_loss, loss);
        Ok(FusedTrainScalars {
            total_loss: blended,
            grad_norm,
        })
    }

    /// Phase 2: reduce block partials → L2 norm.
    /// Single block, 256 threads — reduces grad_norm_partials[0..grad_norm_blocks].
    /// Writes L2 norm (with sqrt) into grad_norm pinned device-mapped buffer.
    fn launch_grad_norm_finalize(&self) -> Result<(), MLError> {
        let partials_ptr = self.grad_norm_partials.raw_ptr();
        let buf_ptr = self.grad_norm_dev_ptr;
        let nb = self.grad_norm_blocks as i32;
        unsafe {
            self.stream
                .launch_builder(&self.grad_norm_finalize_kernel)
                .arg(&partials_ptr)
                .arg(&buf_ptr)
                .arg(&nb)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("grad_norm_finalize: {e}")))?;
        }
        Ok(())
    }

    /// Sync stream and read back loss + grad_norm from GPU.
    /// Called ONLY at epoch boundary — never per-step.
    pub fn readback_scalars_sync(&mut self) -> Result<FusedTrainScalars, MLError> {
        // Launch finalize to convert float sum-of-squares → f32 L2 norm
        self.launch_grad_norm_finalize()?;
        unsafe {
            cudarc::driver::sys::cuStreamSynchronize(self.stream.cu_stream());
        }
        // After sync, pinned device-mapped host memory is coherent — read directly.
        let c51_loss = unsafe { *self.total_loss_pinned };
        let mse_loss = unsafe { *self.mse_loss_pinned };
        let norm_val = unsafe { *self.grad_norm_pinned };
        let blended = Self::blend_loss(self.c51_alpha, mse_loss, c51_loss);
        Ok(FusedTrainScalars {
            total_loss: blended,
            grad_norm: norm_val,
        })
    }

    /// Common post-upload execution: Adam step → CUDA Graph → readback.
    ///
    /// Shared by `train_step()` (CPU upload path) and `train_step_gpu()`
    /// (GPU-direct DtoD path). Everything after batch data lands in the
    /// trainer's CudaSlice buffers is identical.
    ///
    /// Ungraphed execution path (graph capture managed by FusedTrainingCtx).
    fn execute_train_and_readback(
        &mut self,
        online_dueling: &DuelingWeightSet,
        online_branching: &BranchingWeightSet,
    ) -> Result<FusedTrainResult, MLError> {
        let b = self.config.batch_size;

        // ── Update Adam step counter (pinned device-mapped, no HtoD) ───
        self.adam_step += 1;
        unsafe { *self.t_pinned = self.adam_step; }

        // ── Execute training step ungraphed (graph capture in FusedTrainingCtx) ──
        self.update_stochastic_depth_mask()?;
        self.submit_forward_ops_main()?;
        self.submit_forward_ops_ddqn()?;
        // Adaptive per-branch gradient-norm balancer — caps each branch's
        // weight-gradient L2 norm at `num_branches × median_branch_norm`.
        // Runs AFTER all aux grad writes complete and BEFORE the global
        // grad_norm reduction so Adam's clip and the update both see the
        // rebalanced gradient.  (No aux path in this ungraphed fallback —
        // but the method is the sole guard against the L40S pathology so
        // it must be unconditional on every step, graphed or not.)
        self.launch_branch_grad_balance()?;
        self.compute_grad_norm_for_adam()?;
        self.submit_adam_ops(online_dueling, online_branching)?;

        // ── Readback: td_errors from GPU, scalars from pinned host memory ──
        // loss/mse_loss/grad_norm are pinned device-mapped — GPU already wrote
        // to host DRAM. Only td_errors[B] needs DtoH transfer.
        let readback_base = self.readback_buf.raw_ptr();
        let f32_bytes = std::mem::size_of::<f32>();

        // td_errors → readback[3..3+B]  (only td_errors needs DtoD → DtoH)
        let td_src = self.td_errors_buf.raw_ptr();
        let td_bytes = b * f32_bytes;
        dtod_copy_async(readback_base + (3 * f32_bytes) as u64, td_src, td_bytes, &self.stream, 3, "readback_gather")?;

        // ── Single DtoH transfer (only td_errors portion matters) ─────
        super::dtoh_f32(&self.stream, &self.readback_buf, &mut self.readback_host)?;

        // ── Unpack: scalars from pinned host, td_errors from readback ─
        let (c51_loss, mse_loss, grad_norm_sq) = unsafe {
            (*self.total_loss_pinned, *self.mse_loss_pinned, *self.grad_norm_pinned)
        };
        let td_errors = self.readback_host[3..3 + b].to_vec();
        let blended = Self::blend_loss(self.c51_alpha, mse_loss, c51_loss);

        Ok(FusedTrainResult {
            total_loss: blended,
            td_errors,
            grad_norm: grad_norm_sq.sqrt(),
        })
    }

    /// #20 Apply lottery ticket pruning mask after Adam step.
    /// When mask is Some, zeroes pruned weights in params_buf.
    pub fn apply_pruning_mask(&mut self) -> Result<(), MLError> {
        if let Some(ref mask) = self.pruning_mask {
            let tp = self.total_params as i32;
            let blocks = ((tp as u32 + 255) / 256) as u32;
            let params_ptr = self.ptrs.params_ptr;
            let mask_ptr = mask.raw_ptr();
            unsafe {
                self.stream
                    .launch_builder(&self.pruning_mask_kernel)
                    .arg(&params_ptr)
                    .arg(&params_ptr)
                    .arg(&mask_ptr)
                    .arg(&tp)
                    .launch(LaunchConfig {
                        grid_dim: (blocks, 1, 1),
                        block_dim: (256, 1, 1),
                        shared_mem_bytes: 0,
                    })
                    .map_err(|e| MLError::ModelError(format!("pruning mask apply: {e}")))?;
            }
        }
        Ok(())
    }

    /// #20 Compute the pruning mask at the designated epoch.
    /// Reads all f32 weights, finds the magnitude threshold at the pruning_fraction
    /// percentile, and creates a binary mask.
    pub fn compute_pruning_mask(&mut self, epoch: usize) -> Result<(), MLError> {
        if epoch != self.pruning_epoch {
            return Ok(());
        }

        let tp = self.total_params;

        // Read all weight magnitudes to host (one-time at pruning epoch)
        let mut host_params = vec![0.0_f32; tp];
        self.stream.synchronize()
            .map_err(|e| MLError::ModelError(format!("pruning sync: {e}")))?;
        unsafe {
            cudarc::driver::sys::cuMemcpyDtoH_v2(
                host_params.as_mut_ptr().cast(),
                self.ptrs.params_ptr,
                tp * std::mem::size_of::<f32>(),
            );
        }

        // Sort by magnitude to find threshold
        let mut magnitudes: Vec<f32> = host_params.iter().map(|w| w.abs()).collect();
        magnitudes.sort_unstable_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
        let prune_idx = ((tp as f32 * self.pruning_fraction) as usize).min(tp - 1);
        let threshold = magnitudes[prune_idx];

        tracing::info!(
            epoch, threshold, prune_idx, total_params = tp,
            fraction = self.pruning_fraction,
            "LOTTERY TICKET: computing pruning mask"
        );

        // Compute mask on GPU using threshold
        let mut mask = self.stream.alloc_zeros::<f32>(tp)
            .map_err(|e| MLError::ModelError(format!("alloc pruning mask: {e}")))?;
        let tp_i32 = tp as i32;
        let blocks = ((tp as u32 + 255) / 256) as u32;
        unsafe {
            self.stream
                .launch_builder(&self.pruning_compute_kernel)
                .arg(&mut mask)
                .arg(&self.ptrs.params_ptr)
                .arg(&threshold)
                .arg(&tp_i32)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("pruning compute mask: {e}")))?;
        }

        self.pruning_mask = Some(mask);

        // Signal graph re-capture needed (mask changes the training step).
        self.graphs_captured = false;

        Ok(())
    }

    /// #34 Causal intervention: compute per-feature sensitivity on GPU.
    /// Zero CPU-side computation — all interventions, forward passes, and
    /// reductions happen via CUDA kernels.
    ///
    /// For each feature k: GPU kernel zeros feature k in scratch states,
    /// cuBLAS forward produces intervened Q-values, GPU reduction kernel
    /// computes |Q_original - Q_intervened|² and accumulates sensitivity.
    ///
    /// Returns mean sensitivity (read back once at the end, not per-feature).
    pub fn run_causal_intervention(&mut self, step: usize) -> Result<f32, MLError> {
        // Causal intervention: runs every N steps (skip before graph capture)
        if !self.graphs_captured || step % self.causal_config.interval != 0 {
            return Ok(0.0);
        }

        let b = self.config.batch_size;
        let market_dim = self.causal_config.market_dim;
        let pad_sd = ml_core::state_layout::STATE_DIM_PADDED;
        let na = self.config.num_atoms;
        let val_size = (b * na) as i32;

        // Zero sensitivity accumulator
        self.stream.memset_zeros(&mut self.causal_sensitivity_buf)
            .map_err(|e| MLError::ModelError(format!("zero causal_sens: {e}")))?;

        let param_sizes = compute_param_sizes(&self.config);
        let on_w_ptrs = f32_weight_ptrs_from_base(self.ptrs.params_ptr, &param_sizes);
        let states_ptr = self.ptrs.states_buf;
        let scratch_ptr = self.causal_states_scratch.raw_ptr();
        let sensitivity_ptr = self.causal_sensitivity_buf.raw_ptr();

        let b_i32 = b as i32;
        let pad_sd_i32 = pad_sd as i32;
        let blocks = ((b as u32 + 255) / 256) as u32;

        // Scratch activation buffers (reuse — causal runs AFTER graph forward)
        let h_s1 = self.ptrs.save_h_s1;
        let h_s2 = self.ptrs.save_h_s2;
        let h_v = self.ptrs.save_h_v;
        let h_b0 = self.ptrs.save_h_b0;
        let h_b1 = self.ptrs.save_h_b1;
        let h_b2 = self.ptrs.save_h_b2;
        let h_b3 = self.ptrs.save_h_b3;

        for k in 0..market_dim.min(14) {
            let k_i32 = k as i32;

            // Step 1+2: GPU kernel copies states and zeros feature k
            unsafe {
                self.stream
                    .launch_builder(&self.causal_intervene_kernel)
                    .arg(&states_ptr)
                    .arg(&scratch_ptr)
                    .arg(&k_i32)
                    .arg(&pad_sd_i32)
                    .arg(&b_i32)
                    .launch(LaunchConfig {
                        grid_dim: (blocks, 1, 1),
                        block_dim: (256, 1, 1),
                        shared_mem_bytes: 0,
                    })
                    .map_err(|e| MLError::ModelError(format!(
                        "causal_intervene k={k}: {e}"
                    )))?;
            }

            // Step 3: Forward pass on intervened states (cuBLAS, non-graph)
            self.cublas_forward.forward_online_raw(
                &self.stream,
                scratch_ptr,
                &on_w_ptrs,
                h_s1, h_s2, h_v, h_b0, h_b1, h_b2, h_b3,
                self.ptrs.on_next_v_logits_buf,
                self.ptrs.on_next_b_logits_buf,
                0u64, // mag_concat_ptr: causal intervention uses unconditioned forward
                0u64, // dir_qaux_concat_ptr: SP14 EGF disabled — causal sensitivity
                      // reads ONLY value logits (`on_next_v_logits_buf`), never branch
                      // logits. The direction Q-head SGEMM still executes against
                      // B.8-widened `w_b0fc` weights with K=SH2 fallback (residual
                      // numerical garbage in unread `on_next_b_logits_buf`).
            )?;

            // Step 4: GPU reduction — |Q_orig - Q_interv|² → sensitivity[k]
            unsafe {
                self.stream
                    .launch_builder(&self.causal_reduce_kernel)
                    .arg(&self.ptrs.on_v_logits_buf)
                    .arg(&self.ptrs.on_next_v_logits_buf)
                    .arg(&sensitivity_ptr)
                    .arg(&k_i32)
                    .arg(&val_size)
                    .launch(LaunchConfig {
                        grid_dim: (1, 1, 1),
                        block_dim: (256, 1, 1),
                        shared_mem_bytes: 0,
                    })
                    .map_err(|e| MLError::ModelError(format!(
                        "causal_reduce k={k}: {e}"
                    )))?;
            }
        }

        // GPU-side mean reduction + async DtoH to pinned buffer
        // causal_mean_reduce kernel fully overwrites out[0] via direct assignment
        // (single-block, thread 0 only, no atomicAdd) — no memset needed.
        let n_features_i32 = market_dim.min(14) as i32;
        let causal_sensitivity_buf_ptr = self.causal_sensitivity_buf.raw_ptr();
        let causal_mean_scratch_ptr = self.causal_mean_scratch.raw_ptr();
        unsafe {
            self.stream
                .launch_builder(&self.causal_mean_reduce_kernel)
                .arg(&causal_sensitivity_buf_ptr)
                .arg(&causal_mean_scratch_ptr)
                .arg(&n_features_i32)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("causal_mean_reduce: {e}")))?;
        }

        // Async DtoH: 1 f32 from scratch → pinned buffer offset 8
        unsafe {
            cudarc::driver::sys::cuMemcpyDtoHAsync_v2(
                self.readback_pinned.add(10).cast(),
                causal_mean_scratch_ptr,
                std::mem::size_of::<f32>(),
                self.stream.cu_stream(),
            );
        }

        // Return previous result (double-buffered — current arrives by next call)
        let prev = unsafe { *self.readback_pinned.add(10) };
        Ok(prev)
    }

    /// Unconditional causal intervention for child graph capture.
    /// Removes the `graphs_captured` and `step % interval` guards.
    pub(crate) fn run_causal_intervention_unconditional(&mut self) -> Result<(), MLError> {
        let b = self.config.batch_size;
        let market_dim = self.causal_config.market_dim;
        let pad_sd = ml_core::state_layout::STATE_DIM_PADDED;
        let na = self.config.num_atoms;
        let val_size = (b * na) as i32;

        // Zero sensitivity accumulator
        // Raw cuMemsetD32Async — cudarc memset_zeros is NOT captured by CUDA Graph.
        unsafe {
            cudarc::driver::sys::cuMemsetD32Async(
                self.causal_sensitivity_buf.raw_ptr(), 0,
                self.causal_sensitivity_buf.len(), self.stream.cu_stream(),
            );
        }

        let param_sizes = compute_param_sizes(&self.config);
        let on_w_ptrs = f32_weight_ptrs_from_base(self.ptrs.params_ptr, &param_sizes);
        let states_ptr = self.ptrs.states_buf;
        let scratch_ptr = self.causal_states_scratch.raw_ptr();
        let sensitivity_ptr = self.causal_sensitivity_buf.raw_ptr();

        let b_i32 = b as i32;
        let pad_sd_i32 = pad_sd as i32;
        let blocks = ((b as u32 + 255) / 256) as u32;

        // Scratch activation buffers (reuse — causal runs AFTER graph forward)
        let h_s1 = self.ptrs.save_h_s1;
        let h_s2 = self.ptrs.save_h_s2;
        let h_v = self.ptrs.save_h_v;
        let h_b0 = self.ptrs.save_h_b0;
        let h_b1 = self.ptrs.save_h_b1;
        let h_b2 = self.ptrs.save_h_b2;
        let h_b3 = self.ptrs.save_h_b3;

        for k in 0..market_dim.min(14) {
            let k_i32 = k as i32;

            // Step 1+2: GPU kernel copies states and zeros feature k
            unsafe {
                self.stream
                    .launch_builder(&self.causal_intervene_kernel)
                    .arg(&states_ptr)
                    .arg(&scratch_ptr)
                    .arg(&k_i32)
                    .arg(&pad_sd_i32)
                    .arg(&b_i32)
                    .launch(LaunchConfig {
                        grid_dim: (blocks, 1, 1),
                        block_dim: (256, 1, 1),
                        shared_mem_bytes: 0,
                    })
                    .map_err(|e| MLError::ModelError(format!(
                        "causal_intervene_uncond k={k}: {e}"
                    )))?;
            }

            // Step 3: Forward pass on intervened states (cuBLAS, non-graph)
            self.cublas_forward.forward_online_raw(
                &self.stream,
                scratch_ptr,
                &on_w_ptrs,
                h_s1, h_s2, h_v, h_b0, h_b1, h_b2, h_b3,
                self.ptrs.on_next_v_logits_buf,
                self.ptrs.on_next_b_logits_buf,
                0u64, // mag_concat_ptr: causal intervention uses unconditioned forward
                0u64, // dir_qaux_concat_ptr: SP14 EGF disabled — causal sensitivity
                      // reads ONLY value logits (`on_next_v_logits_buf`), never branch
                      // logits. The direction Q-head SGEMM still executes against
                      // B.8-widened `w_b0fc` weights with K=SH2 fallback (residual
                      // numerical garbage in unread `on_next_b_logits_buf`).
            )?;

            // Step 4: GPU reduction — |Q_orig - Q_interv|² → sensitivity[k]
            unsafe {
                self.stream
                    .launch_builder(&self.causal_reduce_kernel)
                    .arg(&self.ptrs.on_v_logits_buf)
                    .arg(&self.ptrs.on_next_v_logits_buf)
                    .arg(&sensitivity_ptr)
                    .arg(&k_i32)
                    .arg(&val_size)
                    .launch(LaunchConfig {
                        grid_dim: (1, 1, 1),
                        block_dim: (256, 1, 1),
                        shared_mem_bytes: 0,
                    })
                    .map_err(|e| MLError::ModelError(format!(
                        "causal_reduce_uncond k={k}: {e}"
                    )))?;
            }
        }

        // GPU-side mean reduction into causal_mean_scratch (device-only; result stays on GPU).
        let n_features_i32 = market_dim.min(14) as i32;
        let causal_sensitivity_buf_ptr = self.causal_sensitivity_buf.raw_ptr();
        let causal_mean_scratch_ptr = self.causal_mean_scratch.raw_ptr();
        unsafe {
            self.stream
                .launch_builder(&self.causal_mean_reduce_kernel)
                .arg(&causal_sensitivity_buf_ptr)
                .arg(&causal_mean_scratch_ptr)
                .arg(&n_features_i32)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("causal_mean_reduce_uncond: {e}")))?;
        }

        Ok(())
    }

    /// Vaccine using previous step's gradients (no separate batch needed).
    /// Compares current grad_buf direction against prev_grad_buf.
    /// Projects out conflicting gradient components to reduce catastrophic
    /// forgetting between consecutive training steps.
    pub(crate) fn apply_gradient_vaccine_from_prev(&self) -> Result<(), MLError> {
        let tp = self.total_params;
        let tp_i32 = tp as i32;
        let g_train_ptr = self.ptrs.grad_buf;
        let g_prev_ptr = self.prev_grad_buf.raw_ptr();

        // Reuse vaccine_block_results and vaccine_dot_norm_buf as scratch
        let blocks = self.grad_norm_blocks as u32;
        let nb = blocks as i32;
        let block_res_ptr = self.vaccine_block_results.raw_ptr();
        let dot_norm_ptr = self.vaccine_dot_norm_buf.raw_ptr();

        // Phase 1: per-block dot(g_train, g_prev) + ||g_prev||²
        unsafe {
            self.stream
                .launch_builder(&self.vaccine_dot_kernel)
                .arg(&g_train_ptr)
                .arg(&g_prev_ptr)
                .arg(&block_res_ptr)
                .arg(&tp_i32)
                .launch(LaunchConfig { grid_dim: (blocks, 1, 1), block_dim: (256, 1, 1), shared_mem_bytes: 0 })
                .map_err(|e| MLError::ModelError(format!("vaccine_prev dot phase1: {e}")))?;
        }

        // Phase 2: finalize dot product + norm
        unsafe {
            self.stream
                .launch_builder(&self.vaccine_dot_finalize_kernel)
                .arg(&block_res_ptr)
                .arg(&dot_norm_ptr)
                .arg(&nb)
                .launch(LaunchConfig { grid_dim: (1, 1, 1), block_dim: (256, 1, 1), shared_mem_bytes: 0 })
                .map_err(|e| MLError::ModelError(format!("vaccine_prev dot phase2: {e}")))?;
        }

        // Phase 3: project g_train in-place (only when dot < 0)
        unsafe {
            self.stream
                .launch_builder(&self.vaccine_project_kernel)
                .arg(&g_train_ptr)
                .arg(&g_prev_ptr)
                .arg(&dot_norm_ptr)
                .arg(&tp_i32)
                .launch(LaunchConfig { grid_dim: (blocks, 1, 1), block_dim: (256, 1, 1), shared_mem_bytes: 0 })
                .map_err(|e| MLError::ModelError(format!("vaccine_prev project: {e}")))?;
        }

        Ok(())
    }

    /// #32 Gradient Vaccine: project out overfitting gradient components.
    ///
    /// After the main forward+backward fills `grad_buf` with g_train:
    /// 1. Swap grad_buf ↔ vaccine_grad_save (zero-copy pointer swap)
    /// 2. Zero grad_buf (now scratch for g_val)
    /// 3. Upload vaccine batch + run forward+backward → grad_buf = g_val
    /// 4. Dot product: dot(vaccine_grad_save=g_train, grad_buf=g_val), |g_val|²
    /// 5. If dot < 0: g_train -= (dot/|g_val|²) * g_val (project in-place)
    /// 6. Swap back → grad_buf = projected g_train (zero-copy)
    ///
    /// Zero-copy: g_train stays in ptrs.grad_buf, g_val is computed into
    /// vaccine_grad_save scratch. Projection modifies ptrs.grad_buf in-place.
    /// No CudaSlice swaps — single pointer source of truth throughout.
    pub fn apply_gradient_vaccine(
        &mut self,
        vaccine_batch: &crate::dqn::replay_buffer_type::GpuBatch,
    ) -> Result<(), MLError> {
        if !self.graphs_captured {
            return Ok(());
        }
        let tp = self.total_params;
        let g_train_ptr = self.ptrs.grad_buf;                 // training grads — stays put
        let g_val_ptr = self.vaccine_grad_save.raw_ptr();      // scratch for vaccine grads

        // Step 1: Zero scratch + d_logits for vaccine pass
        self.stream.memset_zeros(&mut self.vaccine_grad_save)
            .map_err(|e| MLError::ModelError(format!("vaccine zero scratch: {e}")))?;
        self.stream.memset_zeros(&mut self.d_value_logits_buf)
            .map_err(|e| MLError::ModelError(format!("vaccine zero d_val: {e}")))?;
        self.stream.memset_zeros(&mut self.d_adv_logits_buf)
            .map_err(|e| MLError::ModelError(format!("vaccine zero d_adv: {e}")))?;

        // Step 2: Forward+backward on vaccine batch → g_val into scratch
        self.upload_batch_gpu(vaccine_batch)?;
        self.launch_cublas_forward()?;
        self.launch_curiosity_inference()?;
        self.stream.memset_zeros(&mut self.d_value_logits_mse)
            .map_err(|e| MLError::ModelError(format!("vaccine zero mse: {e}")))?;
        self.stream.memset_zeros(&mut self.d_adv_logits_mse)
            .map_err(|e| MLError::ModelError(format!("vaccine zero mse2: {e}")))?;
        self.launch_mse_loss()?;
        self.launch_loss_reduce(self.mse_loss_dev_ptr)?;
        self.launch_mse_grad_to_scratch()?;
        self.fill_gamma_buf()?;
        self.launch_c51_loss()?;
        self.launch_loss_reduce(self.total_loss_dev_ptr)?;
        self.launch_c51_grad()?;
        // Backward writes g_val to scratch (not grad_buf)
        self.launch_cublas_backward_to(g_val_ptr)?;

        // Step 3: dot(g_train, g_val) + ||g_val||² — two-phase reduction
        let tp_i32 = tp as i32;
        let blocks = self.grad_norm_blocks as u32;
        let nb = blocks as i32;
        let block_res_ptr = self.vaccine_block_results.raw_ptr();
        let dot_norm_ptr = self.vaccine_dot_norm_buf.raw_ptr();
        unsafe {
            self.stream
                .launch_builder(&self.vaccine_dot_kernel)
                .arg(&g_train_ptr)
                .arg(&g_val_ptr)
                .arg(&block_res_ptr)
                .arg(&tp_i32)
                .launch(LaunchConfig { grid_dim: (blocks, 1, 1), block_dim: (256, 1, 1), shared_mem_bytes: 0 })
                .map_err(|e| MLError::ModelError(format!("vaccine dot phase1: {e}")))?;
            self.stream
                .launch_builder(&self.vaccine_dot_finalize_kernel)
                .arg(&block_res_ptr)
                .arg(&dot_norm_ptr)
                .arg(&nb)
                .launch(LaunchConfig { grid_dim: (1, 1, 1), block_dim: (256, 1, 1), shared_mem_bytes: 0 })
                .map_err(|e| MLError::ModelError(format!("vaccine dot phase2: {e}")))?;
        }

        // Step 4: Project g_train in-place (only when dot < 0)
        unsafe {
            self.stream
                .launch_builder(&self.vaccine_project_kernel)
                .arg(&g_train_ptr)
                .arg(&g_val_ptr)
                .arg(&dot_norm_ptr)
                .arg(&tp_i32)
                .launch(LaunchConfig { grid_dim: (blocks, 1, 1), block_dim: (256, 1, 1), shared_mem_bytes: 0 })
                .map_err(|e| MLError::ModelError(format!("vaccine project: {e}")))?;
        }

        Ok(())
    }

    /// #21 Stochastic depth: generate per-layer drop/keep scales on GPU.
    /// Uses GPU-resident LCG RNG — zero CPU-side randomness or HtoD transfers.
    /// Must be called BEFORE replay_forward() each training step.
    /// CUDA Graphs capture buffer addresses, not contents — this is safe.
    ///
    /// NOTE: grid=(1,1,1), block=(1,1,1) — standalone RNG that writes 3 floats.
    /// Not a finalizer after a reduction — cannot be fused into anything else.
    /// Remaining single-block kernels (causal_reduce, causal_mean_reduce, q_stats)
    /// are called at intervention-interval or epoch-boundary, not per-step.
    pub fn update_stochastic_depth_mask(&mut self) -> Result<(), MLError> {
        // Stochastic depth always active — one production path
        let p = self.stochastic_depth_prob;
        let scale_ptr = self.stochastic_depth_scale_buf.raw_ptr();
        let rng_ptr = self.stochastic_depth_rng_state.raw_ptr();
        unsafe {
            self.stream
                .launch_builder(&self.stochastic_depth_rng_kernel)
                .arg(&scale_ptr)
                .arg(&rng_ptr)
                .arg(&p)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (1, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("stochastic_depth_rng: {e}")))?;
        }
        Ok(())
    }

    /// Set the eval forward graph exec handle (owned by FusedTrainingCtx).
    /// Used by replay_forward_for_q_values() for deterministic eval.
    /// The exec handle is NOT destroyed by GpuDqnTrainer -- ownership stays
    /// with FusedTrainingCtx.
    pub fn set_eval_forward_exec(&mut self, exec: cudarc::driver::sys::CUgraphExec) {
        // Destroy previous eval exec if any (shouldn't happen, but defensive).
        if let Some(old) = self.eval_forward_exec.take() {
            unsafe { cudarc::driver::sys::cuGraphExecDestroy(old); }
        }
        self.eval_forward_exec = Some(exec);
        self.graphs_captured = true;
    }

    /// Mark graphs as captured (called by FusedTrainingCtx after parent graph compose).
    pub fn mark_graphs_captured(&mut self) {
        self.graphs_captured = true;
    }

    /// Whether graphs have been captured.
    pub fn graphs_captured(&self) -> bool {
        self.graphs_captured
    }

    /// Scalar-only readback path: only total_loss + grad_norm (8 bytes DtoH).
    ///
    /// TD errors stay on GPU in `td_errors_buf` — PER priority update is done
    /// by `seg_tree_update` which reads them directly via device pointer.
    fn execute_train_scalars_only(
        &mut self,
        online_dueling: &DuelingWeightSet,
        online_branching: &BranchingWeightSet,
    ) -> Result<FusedTrainScalars, MLError> {
        // ── Update Adam step counter (pinned device-mapped, no HtoD) ───
        self.adam_step += 1;
        unsafe { *self.t_pinned = self.adam_step; }

        // ── Execute training step ungraphed (graph capture in FusedTrainingCtx) ──
        self.submit_forward_ops_main()?;
        self.submit_forward_ops_ddqn()?;
        // Adaptive per-branch gradient-norm balancer — see the detailed
        // rationale on `launch_branch_grad_balance`.  Kept unconditional
        // so no code path can skip the cap.
        self.launch_branch_grad_balance()?;
        self.compute_grad_norm_for_adam()?;
        self.submit_adam_ops(online_dueling, online_branching)?;

        // ── Sync then read pinned device-mapped host memory (no DtoH copy) ──
        unsafe {
            cudarc::driver::sys::cuStreamSynchronize(self.stream.cu_stream());
        }
        let c51_loss = unsafe { *self.total_loss_pinned };
        let mse_loss = unsafe { *self.mse_loss_pinned };
        let norm_val = unsafe { *self.grad_norm_pinned };
        let blended = Self::blend_loss(self.c51_alpha, mse_loss, c51_loss);
        self.scalars_readback_host = [blended, norm_val];

        Ok(FusedTrainScalars {
            total_loss: self.scalars_readback_host[0],
            grad_norm: self.scalars_readback_host[1].sqrt(),
        })
    }


    /// Reference to the td_errors buffer on GPU.
    ///
    /// Shape: `[B]` f32 — per-sample TD errors from the last training step.
    /// Valid after `train_step_gpu()` or `train_step()` has been called.
    /// Used by the PER priority update kernel to avoid DtoH readback.
    pub fn td_errors_buf(&self) -> &CudaSlice<f32> {
        &self.td_errors_buf
    }
    pub fn td_errors_buf_mut(&mut self) -> &mut CudaSlice<f32> {
        &mut self.td_errors_buf
    }

    /// Run pre-forward NaN checks: f32 params_buf (flag 4) + params_ptr alias (flag 5).
    /// Detects if previous step's Adam corrupted the weights.
    ///
    /// Does NOT reset flags — flags persist across steps so the FIRST buffer to go
    /// NaN remains visible at host-readback time. Caller resets flags only at fold
    /// boundary via `reset_nan_flags()`.
    pub fn run_nan_checks_pre_forward(&mut self) -> Result<(), MLError> {
        let tp = self.total_params;
        // Flag 4: f32 params_buf
        self.check_nan_f32(self.params_buf.raw_ptr(), tp, 4)?;
        // Flag 5: params_ptr (same buffer, second check for redundancy)
        self.check_nan_f32_b(self.ptrs.params_ptr, tp, 5)?;
        Ok(())
    }

    /// Run all post-forward NaN checks (GPU-side, no CPU sync).
    /// Checks cuBLAS output logits, MSE intermediates, gradients, params, and
    /// loss-component buffers (C51 target dist, MoE gate softmax, aux losses).
    ///
    /// SP1 Phase B 48-slot layout (high-level summary; per-slot accessor +
    /// semantics live in `docs/dqn-backward-nan-audit.md` per-slot accessor
    /// table — that audit is the source of truth):
    ///   0-23  forward-path (this method populates 0-3, 6-12, 16-20; slots
    ///         4-5 are populated by `run_nan_checks_pre_forward`; 13-15 and
    ///         21-23 are reserved headroom).
    ///   24-35 backward-path (wired by Task 4 — see
    ///         `run_nan_checks_post_backward` for slots 24-30 + 32 with
    ///         stable post-backward state, plus three inline `check_nan_f32`
    ///         calls during backward orchestration for slots 33/34/35
    ///         (`bw_d_h_s2` post-main / post-aux / post-IQN snapshots
    ///         localising the entry point of NaN poisoning into the
    ///         backward chain). Slot 31 (ensemble_d_logits_buf) DEFERRED —
    ///         owner is on `FusedDqnTraining` (different struct); will be
    ///         un-deferred when ensemble Phase B saxpy guards are verified.
    ///   36-47 reserved headroom (SP2/SP3).
    pub fn run_nan_checks_post_forward(&mut self, batch_size: usize) -> Result<(), MLError> {
        let b = batch_size;
        let na = self.config.num_atoms;
        let b0 = self.config.branch_0_size;
        let b1 = self.config.branch_1_size;
        let b2 = self.config.branch_2_size;
        let b3 = self.config.branch_3_size;
        // Don't reset — pre-forward already set flags 4-5, we add 0-3 + 6-11.
        // Flag 0: states_buf (f32 padded — if states have NaN, everything downstream does)
        let pad_sd = ml_core::state_layout::STATE_DIM_PADDED;
        self.check_nan_f32(self.states_buf.raw_ptr(), b * pad_sd, 0)?;
        // Flag 1: on_v_logits (f32 cuBLAS forward output — value stream logits)
        self.check_nan_f32(self.on_v_logits_buf.raw_ptr(), b * na, 1)?;
        // Flag 2: on_b_logits (f32 cuBLAS forward output — branch advantage logits)
        self.check_nan_f32(self.on_b_logits_buf.raw_ptr(), b * (b0 + b1 + b2 + b3) * na, 2)?;
        // Flag 3: mse_loss_buf [1] (MSE loss scalar)
        self.check_nan_f32(self.mse_loss_dev_ptr, 1, 3)?;
        // Flag 6: grad_buf (cuBLAS backward output) — use ptrs for consistency
        self.check_nan_f32(self.ptrs.grad_buf, self.total_params, 6)?;
        // Flag 7: save_current_lp (C51 online softmax probs — f32)
        self.check_nan_f32_b(self.save_current_lp.raw_ptr(), b * 4 * na, 7)?;
        // Flag 8: save_projected (C51 target distribution — most NaN-prone under
        // adversarial reward stress; categorical projection of TD targets onto atoms)
        self.check_nan_f32(self.save_projected.raw_ptr(), b * 4 * na, 8)?;
        // Flag 9: moe_gate_softmax (gate distribution over 8 experts — softmax
        // saturation under post-S&P weights produces NaN that propagates into the
        // MoE mixture forward pass)
        self.check_nan_f32(
            self.moe_gate_softmax_buf.raw_ptr(),
            b * MOE_NUM_EXPERTS,
            9,
        )?;
        // Flag 10: aux_nb_loss_scalar [1] (aux next-bar MSE loss)
        self.check_nan_f32(self.aux_nb_loss_scalar_buf.raw_ptr(), 1, 10)?;
        // Flag 11: aux_rg_loss_scalar [1] (aux regime CE loss)
        self.check_nan_f32(self.aux_rg_loss_scalar_buf.raw_ptr(), 1, 11)?;
        // Flag 12: save_h_s2 (post-trunk activation, input to value FC + branch FC).
        // Diagnostic differentiator: if `on_b_logits` (flag 2) is NaN but
        // `save_h_s2` is finite, the branch GEMM itself overflowed; if both are
        // NaN, the corruption is upstream in the trunk encoder.
        let sh2 = self.config.shared_h2;
        self.check_nan_f32(self.save_h_s2.raw_ptr(), b * sh2, 12)?;

        // P (post-bkdx5): GRN h_s2 sub-stage NaN checks (slots 16-20). Drills
        // into the trunk-encoder pathway to locate which sub-step of the GRN
        // forward composition (Linear_a → ELU → Linear_b → GLU → residual+LN)
        // produces NaN first when save_h_s2 (slot 12) goes NaN.
        //
        // Pragmatic instrumentation scope: only h_s2 (the stage that actually
        // failed at F1 ~step 1745 in smoke-test-bkdx5). Slot 16/17/18/19/20
        // map to elu_post / linear_b_out / sigmoid_b / ln_rstd / ln_normed of
        // the h_s2 GRN block — see the index map in this method's docstring.
        let grn_h_s2 = self.cublas_forward.grn_h_s2_online();
        let elu_post_p   = grn_h_s2.elu_post_ptr();
        let lin_b_out_p  = self.cublas_forward.grn_h_s2_linear_b_ptr();
        let sigmoid_b_p  = grn_h_s2.sigmoid_b_ptr();
        let ln_rstd_p    = grn_h_s2.ln_rstd_ptr();
        let ln_normed_p  = grn_h_s2.ln_normed_ptr();
        self.check_nan_f32(elu_post_p,  b * sh2,     16)?;
        self.check_nan_f32(lin_b_out_p, b * 2 * sh2, 17)?;
        self.check_nan_f32(sigmoid_b_p, b * sh2,     18)?;
        self.check_nan_f32(ln_rstd_p,   b,           19)?;
        self.check_nan_f32(ln_normed_p, b * sh2,     20)?;
        Ok(())
    }

    /// SP1 Phase B (Task 4): NaN checks on backward-path kernel outputs that
    /// have a stable post-backward state. Runs after the entire backward
    /// orchestration has completed (main backward chain + IQN aux + CQL +
    /// aux heads + bottleneck) but before Adam consumes `grad_buf`.
    ///
    /// Index → buffer mapping (mirrors training_loop.rs name table; per-slot
    /// semantics + length expressions in `docs/dqn-backward-nan-audit.md`
    /// per-slot accessor table at the end of that file):
    ///   24 d_value_logits_buf       — post-c51_grad value gradient
    ///   25 d_adv_logits_buf         — post-c51_grad branch advantage
    ///   26 iqn_trunk_m              — apply_iqn_trunk_gradient cuBLAS bwd output
    ///   27 iqn_d_h_s2_buf           — IQN backward dh_s2 (caller-supplied)
    ///   28 d_branch_logits_buf      — production IQN backward (iqn_quantile_huber_loss)
    ///   29 cql_d_value_logits       — CQL gradient output
    ///   30 aux_dh_s2_nb_buf         — Aux next-bar backward dh_s2
    ///   32 bn_d_concat_buf          — Bottleneck Linear backward dy
    ///
    /// Slots 33/34/35 (`bw_d_h_s2` at three orchestration phases) are NOT
    /// checked here — they fire inline during backward orchestration to
    /// localize the NaN entry point. See `launch_cublas_backward_to`
    /// (slots 33, 34) and `apply_iqn_trunk_gradient` (slot 35).
    ///
    /// Slot 31 (`ensemble_d_logits_buf`) is DEFERRED per Task 2 commit
    /// `387335e2b` — owner is `FusedDqnTraining` (different struct) and the
    /// ensemble Phase B saxpy guards must be verified before un-deferring.
    ///
    /// SP2 (Task A4): collapses 8 individual `check_nan_f32` launches into a
    /// single fused launch covering all 12 backward-path slots (24-35) in
    /// 12 blocks. The `(ptr, len)` table is populated once at construction
    /// via `populate_nan_check_meta` (called from `fused_training.rs` after
    /// `gpu_iqn` is known) — Option<u64> nullity for slots 27/28 (IQN
    /// inactive) and slot 31 (deferred ensemble) and slots 33-35 (inline
    /// at backward orchestration phases) is baked into the metadata buffer
    /// as `(0, 0)` entries; the fused kernel's null-pointer guard skips
    /// them without per-step Rust branching.
    ///
    /// Inline checks for slots 33/34/35 (bw_d_h_s2 multi-point snapshots)
    /// continue to fire at backward orchestration call sites
    /// (`launch_cublas_backward_to`, `apply_iqn_trunk_gradient`) — those
    /// provide localization across the 3 backward phases (post-main,
    /// post-aux, post-iqn) and are not subsumed by the single fused launch.
    ///
    /// Fires every training step inside the `post_aux` captured child
    /// graph; flags accumulate within fold and reset on fold boundary via
    /// `reset_nan_flags()`.
    pub fn run_nan_checks_post_backward(&mut self) -> Result<(), MLError> {
        self.launch_nan_check_fused_f32()
    }

    /// Launch GPU-side NaN check on an f32 buffer. Writes 1 to nan_flags_buf[flag_idx] if NaN found.
    /// No CPU sync — stays entirely on GPU. Call reset_nan_flags() before a batch of checks.
    pub fn check_nan_f32(&self, buf_ptr: u64, n: usize, flag_idx: i32) -> Result<(), MLError> {
        let n_i32 = n as i32;
        let blocks = ((n + 255) / 256) as u32;
        let flags_ptr = self.nan_flags_buf.raw_ptr();
        unsafe {
            self.stream
                .launch_builder(&self.nan_check_f32_kernel)
                .arg(&buf_ptr).arg(&n_i32).arg(&flags_ptr).arg(&flag_idx)
                .launch(LaunchConfig { grid_dim: (blocks.max(1), 1, 1), block_dim: (256, 1, 1), shared_mem_bytes: 0 })
                .map_err(|e| MLError::ModelError(format!("nan_check_f32[{flag_idx}]: {e}")))?;
        }
        Ok(())
    }

    /// Launch GPU-side NaN check on an f32 buffer (secondary handle).
    pub fn check_nan_f32_b(&self, buf_ptr: u64, n: usize, flag_idx: i32) -> Result<(), MLError> {
        let n_i32 = n as i32;
        let blocks = ((n + 255) / 256) as u32;
        let flags_ptr = self.nan_flags_buf.raw_ptr();
        unsafe {
            self.stream
                .launch_builder(&self.nan_check_f32_kernel)
                .arg(&buf_ptr).arg(&n_i32).arg(&flags_ptr).arg(&flag_idx)
                .launch(LaunchConfig { grid_dim: (blocks.max(1), 1, 1), block_dim: (256, 1, 1), shared_mem_bytes: 0 })
                .map_err(|e| MLError::ModelError(format!("nan_check_f32_b[{flag_idx}]: {e}")))?;
        }
        Ok(())
    }

    /// SP2 + SP3 Mech 5: populate the fused NaN/threshold-check `(ptr, len)`
    /// tables. Called once during construction after all backward-path
    /// buffers are allocated AND `gpu_iqn` is known (its accessors feed slot
    /// 27/28 entries — and now slots 39/43 IQN Adam m/v — when IQN is active;
    /// null/0 when None — the kernel's null-pointer guard skips them).
    ///
    /// Slot index → buffer mapping (mirrors the per-slot accessor table in
    /// `docs/dqn-backward-nan-audit.md` and the `run_nan_checks_post_backward`
    /// per-slot launches):
    ///
    ///   SP2 Task A3 — NaN-only checks (slots 24-35, threshold = +inf):
    ///     0 (slot 24) d_value_logits_buf      — len = `d_value_logits_buf.len()`
    ///     1 (slot 25) d_adv_logits_buf        — len = `d_adv_logits_buf.len()`
    ///     2 (slot 26) iqn_trunk_m             — len = `trunk_param_count`
    ///     3 (slot 27) iqn d_h_s2 buf          — len = b * sh2 (or 0 if IQN inactive)
    ///     4 (slot 28) iqn d_branch_logits_buf — len = total_branch_actions * b * q (or 0)
    ///     5 (slot 29) cql_d_value_logits      — len = `cql_d_value_logits.len()`
    ///     6 (slot 30) aux_dh_s2_nb_buf        — len = `aux_dh_s2_nb_buf.len()`
    ///     7 (slot 31) ensemble (deferred)     — ptr=0, len=0 → kernel skips block
    ///     8 (slot 32) bn_d_concat_buf         — len = `bn_d_concat_buf().len()`
    ///     9-11 (slots 33-35) bw_d_h_s2 inline — ptr=0, len=0 → fused kernel skips
    ///         (inline checks fire at backward orchestration phases — see
    ///          `launch_cublas_backward_to` slots 33/34 and
    ///          `apply_iqn_trunk_gradient` slot 35).
    ///
    ///   SP3 Mech 5 (Task B6) — ISV-threshold checks (slots 36-47):
    ///    12 (slot 36) trunk_adam_m   — Adam m max-abs ≥ 100 × q_abs_ref_eff
    ///    13 (slot 37) value_adam_m   — same threshold class
    ///    14 (slot 38) branch_adam_m  — same threshold class
    ///    15 (slot 39) iqn_adam_m     — caller-supplied; 0/null when IQN inactive
    ///    16 (slot 40) trunk_adam_v   — Adam v max-abs ≥ 1e6 × q_abs_ref_eff²
    ///    17 (slot 41) value_adam_v   — same threshold class
    ///    18 (slot 42) branch_adam_v  — same threshold class
    ///    19 (slot 43) iqn_adam_v     — caller-supplied; 0/null when IQN inactive
    ///    20 (slot 44) trunk_params   — weight max-abs ≥ 1e3 × q_abs_ref_eff
    ///    21 (slot 45) heads_params   — same threshold class
    ///    22 (slot 46) target_q       — post-clip ≥ 95 × q_abs_ref_eff
    ///    23 (slot 47) atom_positions — span max ≥ 190 × q_abs_ref_eff
    ///
    ///   SP3 close-out v2 — slot 48 (IQN online_params):
    ///    24 (slot 48) iqn_online_params — weight max-abs ≥ 1e3 × q_abs_ref_eff
    ///                                     (same regime as slots 44-45;
    ///                                     captures the slot-26 GEMM blind
    ///                                     spot that hid the F1 step-3540 NaN)
    ///
    ///   SP3 Mech 10 — slot 49 (post-trunk h_s2 activation sentinel):
    ///    25 (slot 49) save_h_s2 — max-abs ≥ 1e3 × ISV[H_S2_RMS_EMA_INDEX].max(1)
    ///                              (regression sentinel for Mech 10's
    ///                               activation clamp; mirrors Mech 5 family
    ///                               clamp-at-100×/diagnostic-at-1000× pattern;
    ///                               distinct ISV base from slots 36-48 because
    ///                               h_s2 lives in activation space, not Q space)
    ///
    /// Mapped-pinned host write — no HtoD copy. The kernel reads the same
    /// memory through the device-mapped pointer (`dev_ptr`) once the launch
    /// stream barrier ensures coherence.
    pub(crate) fn populate_nan_check_meta(
        &mut self,
        b: usize,
        sh2: usize,
        iqn_d_h_s2_ptr: Option<u64>,
        iqn_d_branch_logits_buf_ptr: Option<u64>,
        iqn_q: Option<usize>,
        // SP3 Mech 5 additions: IQN Adam m/v ptrs + lens for slots 39/43.
        // None when IQN is inactive (`gpu_iqn = None`); in that case slot
        // 39/43 entries become (0, 0) and the kernel's null-pointer guard
        // skips them — symmetric with slots 27/28.
        iqn_adam_m_ptr: Option<u64>,
        iqn_adam_m_len: Option<usize>,
        iqn_adam_v_ptr: Option<u64>,
        iqn_adam_v_len: Option<usize>,
        // SP3 close-out v2 addition: IQN online_params ptr + len for slot 48
        // (the diagnostic blind spot that hid the F1 step-3540 NaN — IQN's
        // separate weight buffer drives the slot-26 cuBLAS GEMM in
        // `apply_iqn_trunk_gradient`, never reached by slots 44-45). Same
        // None=inactive convention as the slot 27/28/39/43 entries.
        iqn_online_params_ptr: Option<u64>,
        iqn_online_params_len: Option<usize>,
    ) -> Result<(), MLError> {
        // Slot-28 length expression: `tba × b × q` per
        // `run_nan_checks_post_backward` slot 28. tba = b0+b1+b2+b3.
        let tba = self.config.branch_0_size
            + self.config.branch_1_size
            + self.config.branch_2_size
            + self.config.branch_3_size;
        let bn_d_concat_len = self.bn_d_concat_buf().len() as i32;
        let trunk_param_count = self.trunk_param_count as i32;

        // SP3 Mech 5 — slot 36-47 accessors capture once to keep `entries`
        // construction free of repeated `compute_param_sizes` calls.
        let trunk_adam_m_ptr = self.trunk_adam_m_ptr();
        let trunk_adam_m_len = self.trunk_adam_m_len() as i32;
        let value_adam_m_ptr = self.value_adam_m_ptr();
        let value_adam_m_len = self.value_adam_m_len() as i32;
        let branch_adam_m_ptr = self.branch_adam_m_ptr();
        let branch_adam_m_len = self.branch_adam_m_len() as i32;
        let trunk_adam_v_ptr = self.trunk_adam_v_ptr();
        let trunk_adam_v_len = self.trunk_adam_v_len() as i32;
        let value_adam_v_ptr = self.value_adam_v_ptr();
        let value_adam_v_len = self.value_adam_v_len() as i32;
        let branch_adam_v_ptr = self.branch_adam_v_ptr();
        let branch_adam_v_len = self.branch_adam_v_len() as i32;
        let trunk_params_ptr = self.trunk_params_ptr();
        let trunk_params_len = self.trunk_params_len() as i32;
        let heads_params_ptr = self.heads_params_ptr();
        let heads_params_len = self.heads_params_len() as i32;
        let target_q_ptr = self.target_q_ptr();
        let target_q_len = self.target_q_len() as i32;
        let atom_positions_ptr = self.atom_positions_ptr();
        let atom_positions_len = self.atom_positions_len() as i32;

        // SP3 Mech 10 — slot 49 accessor for post-trunk h_s2 activation
        // sentinel. Same buffer as slot 12's NaN check (`save_h_s2`); the
        // sentinel only fires at 1e3 × H_S2_RMS_EMA.max(1) so it stays quiet
        // under normal Mech 10 clamp operation (clamp at 100×).
        let save_h_s2_ptr = self.save_h_s2.raw_ptr();
        let save_h_s2_len = self.save_h_s2.len() as i32;

        let entries: [(u64, i32); 26] = [
            // SP2 Task A3 — slot 24 — post-c51_grad value gradient
            (self.d_value_logits_buf_ptr(), self.d_value_logits_buf.len() as i32),
            // slot 25 — post-c51_grad branch advantage
            (self.d_adv_logits_buf_ptr(), self.d_adv_logits_buf.len() as i32),
            // slot 26 — IQN trunk cuBLAS bwd output
            (self.ptrs.iqn_trunk_m, trunk_param_count),
            // slot 27 — IQN dh_s2 (caller-supplied; 0 when IQN inactive)
            (
                iqn_d_h_s2_ptr.unwrap_or(0),
                if iqn_d_h_s2_ptr.is_some() { (b * sh2) as i32 } else { 0 },
            ),
            // slot 28 — IQN d_branch_logits_buf
            (
                iqn_d_branch_logits_buf_ptr.unwrap_or(0),
                match (iqn_d_branch_logits_buf_ptr, iqn_q) {
                    (Some(_), Some(q)) => (tba * b * q.max(1)) as i32,
                    _ => 0,
                },
            ),
            // slot 29 — CQL gradient buffer
            (self.cql_d_value_logits_ptr(), self.cql_d_value_logits.len() as i32),
            // slot 30 — aux next-bar dh_s2
            (self.aux_dh_s2_nb_buf_ptr(), self.aux_dh_s2_nb_buf.len() as i32),
            // slot 31 — deferred ensemble (owned by FusedDqnTraining); null entry → kernel skip
            (0, 0),
            // slot 32 — bottleneck Linear backward dy
            (self.bn_d_concat_buf().raw_ptr(), bn_d_concat_len),
            // slots 33-35 — bw_d_h_s2 inline checks fire separately at backward
            // orchestration phases (post-main / post-aux / post-iqn). Fused
            // kernel skips these via null-pointer guard.
            (0, 0),
            (0, 0),
            (0, 0),
            // SP3 Mech 5 — slot 36 — trunk Adam m
            (trunk_adam_m_ptr, trunk_adam_m_len),
            // slot 37 — value-head Adam m
            (value_adam_m_ptr, value_adam_m_len),
            // slot 38 — branch-head Adam m
            (branch_adam_m_ptr, branch_adam_m_len),
            // slot 39 — IQN Adam m (caller-supplied; 0 when IQN inactive)
            (
                iqn_adam_m_ptr.unwrap_or(0),
                iqn_adam_m_len.unwrap_or(0) as i32,
            ),
            // slot 40 — trunk Adam v
            (trunk_adam_v_ptr, trunk_adam_v_len),
            // slot 41 — value-head Adam v
            (value_adam_v_ptr, value_adam_v_len),
            // slot 42 — branch-head Adam v
            (branch_adam_v_ptr, branch_adam_v_len),
            // slot 43 — IQN Adam v (caller-supplied; 0 when IQN inactive)
            (
                iqn_adam_v_ptr.unwrap_or(0),
                iqn_adam_v_len.unwrap_or(0) as i32,
            ),
            // slot 44 — trunk weight slice
            (trunk_params_ptr, trunk_params_len),
            // slot 45 — heads weight slice
            (heads_params_ptr, heads_params_len),
            // slot 46 — target_q post-clip
            (target_q_ptr, target_q_len),
            // slot 47 — atom_positions
            (atom_positions_ptr, atom_positions_len),
            // SP3 close-out v2 — slot 48 — IQN online_params (caller-supplied;
            // 0 when IQN inactive — null-pointer guard skips block).
            (
                iqn_online_params_ptr.unwrap_or(0),
                iqn_online_params_len.unwrap_or(0) as i32,
            ),
            // SP3 Mech 10 — slot 49 — post-trunk h_s2 activation sentinel.
            // Same buffer as slot 12 (NaN-only check). Threshold is computed
            // inline in the fused kernel from `h_s2_rms_ema_eff` (distinct
            // from `q_abs_ref_eff` — h_s2 lives in activation space).
            (save_h_s2_ptr, save_h_s2_len),
        ];

        // Host-side write through the mapped-pinned pages — kernel sees the
        // values via dev_ptr after the next stream-sync barrier (no HtoD copy
        // required, per `feedback_no_htod_htoh_only_mapped_pinned`).
        let ptrs_view: [u64; 26] = std::array::from_fn(|i| entries[i].0);
        let lens_view: [i32; 26] = std::array::from_fn(|i| entries[i].1);
        self.nan_check_buf_ptrs.write_from_slice(&ptrs_view);
        self.nan_check_buf_lens.write_from_slice(&lens_view);
        Ok(())
    }

    /// SP2 + SP3 Mech 5 + SP3 close-out v2 + SP3 Mech 10: launch the fused
    /// NaN/threshold-check kernel.
    /// Single launch processes all 26 backward-path slots (24-49) in 26
    /// blocks, in parallel. Slots with null pointer (deferred slot 31,
    /// optional IQN slots 27/28/39/43/48 when inactive, slots 33-35 inline
    /// elsewhere) no-op via the kernel's null-pointer guard. Sticky-flag
    /// semantics preserved (kernel only writes 1, never clears).
    ///
    /// Slots 0-11 (= absolute 24-35) are NaN-only checks (threshold = +inf
    /// trivially false). Slots 12-23 (= absolute 36-47) are SP3 Mech 5
    /// ISV-threshold checks. Slot 24 (= absolute 48) is SP3 close-out v2 —
    /// IQN online_params, threshold = 1e3 × q_abs_ref_eff (same regime as
    /// slots 44-45). Slot 25 (= absolute 49) is SP3 Mech 10 — post-trunk
    /// h_s2 activation sentinel, threshold = 1e3 × h_s2_rms_ema_eff (distinct
    /// ISV base — h_s2 lives in activation space, not Q space). Thresholds
    /// computed inline per-slot from two ISV-derived args (no per-slot
    /// threshold buffer, no per-step HtoD).
    ///
    /// Replaces the 8-launch sequence in `run_nan_checks_post_backward`.
    pub(crate) fn launch_nan_check_fused_f32(&mut self) -> Result<(), MLError> {
        let nan_flags_ptr = self.nan_flags_buf.raw_ptr();
        let buf_ptrs_dev = self.nan_check_buf_ptrs.dev_ptr;
        let buf_lens_dev = self.nan_check_buf_lens.dev_ptr;
        // SP4 Layer B: per-slot threshold sourced directly from the SP4 ISV
        // bound for that slot. Kernel reads `isv_signals[SP4_*_BOUND[...]]`
        // inline — no host-side multiplier, no per-step HtoD copy. The
        // mapped-pinned ISV buffer is graph-capture-safe via its dev_ptr.
        let isv_signals_dev_ptr: u64 = self.isv_signals_dev_ptr;
        const BLOCKS: u32 = 26;
        const THREADS: u32 = 256;
        const BASE_FLAG_IDX: i32 = 24;
        let cfg = LaunchConfig {
            grid_dim: (BLOCKS, 1, 1),
            block_dim: (THREADS, 1, 1),
            shared_mem_bytes: 0,
        };
        unsafe {
            self.stream
                .launch_builder(&self.nan_check_fused_f32_kernel)
                .arg(&buf_ptrs_dev)
                .arg(&buf_lens_dev)
                .arg(&isv_signals_dev_ptr)
                .arg(&BASE_FLAG_IDX)
                .arg(&nan_flags_ptr)
                .launch(cfg)
                .map_err(|e| MLError::ModelError(format!("nan_check_fused_f32: {e}")))?;
        }
        Ok(())
    }

    /// Zero the NaN flags buffer (call before a batch of check_nan calls).
    pub fn reset_nan_flags(&mut self) -> Result<(), MLError> {
        self.stream.memset_zeros(&mut self.nan_flags_buf)
            .map_err(|e| MLError::ModelError(format!("nan_flags reset: {e}")))
    }

    /// SP1 Phase C surgical fix: launch in-place finite clamp on `buf_ptr`.
    ///
    /// Replaces NaN/Inf with 0 and clamps finite |v| to ±max_abs (host-side
    /// `f32`). max_abs is sourced from per-call-site SP4 ISV-driven bounds
    /// (`ISV[BW_D_H_S2_BOUND_INDEX]` for the IQN trunk path,
    /// `ISV[Q_DIR_GRAD_BOUND_INDEX]` for the Bottleneck Linear path), each
    /// driven by a per-step p99 producer (`launch_sp4_bw_d_h_s2_p99` /
    /// `launch_sp4_q_dir_grad_p99`) chained on the producer's stream. An
    /// `EPS_CLAMP_FLOOR` is applied at the call site for uninitialised
    /// ISV state (Invariant 1 carve-out per
    /// `feedback_isv_for_adaptive_bounds`).
    ///
    /// Pattern matches the existing `isfinite`-guard precedent in
    /// `iqn_backward_per_sample` (line 612) and `iqn_adam_kernel` (line 873).
    /// The kernel is in-place — no DtoD/HtoD/HtoH copies (per
    /// `feedback_no_htod_htoh_only_mapped_pinned`). Graph-safe — both call
    /// sites land inside captured children (forward_child for slot 32,
    /// aux_child for slot 26) and the kernel uses only stream-bound
    /// `launch_builder` like `check_nan_f32`.
    pub fn launch_clamp_finite_f32(
        &self,
        buf_ptr: u64,
        n: usize,
        max_abs: f32,
    ) -> Result<(), MLError> {
        let n_i32 = n as i32;
        let blocks = ((n + 255) / 256) as u32;
        unsafe {
            self.stream
                .launch_builder(&self.clamp_finite_f32_kernel)
                .arg(&buf_ptr).arg(&n_i32).arg(&max_abs)
                .launch(LaunchConfig {
                    grid_dim: (blocks.max(1), 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("clamp_finite_f32 launch: {e}")))?;
        }
        Ok(())
    }

    /// Read PopArt running variance from GPU (single scalar DtoH).
    pub fn read_popart_var(&self, out: &mut [f32; 1]) -> Result<(), MLError> {
        self.stream.memcpy_dtoh(&self.popart_var, out)
            .map_err(|e| MLError::ModelError(format!("PopArt var readback: {e}")))?;
        Ok(())
    }

    /// Read NaN flags back to CPU (synchronizes stream). Returns [50] flags.
    /// Index → buffer mapping is documented in `run_nan_checks_post_forward`.
    /// SP1 Phase B: slots 24-35 reserved for backward-path checks (wired by
    /// Task 4); slots 36-47 SP3 Mech 5 ISV-threshold checks; slot 48 SP3
    /// close-out v2 IQN online_params weight diagnostic; slot 49 SP3 Mech 10
    /// post-trunk h_s2 activation max-abs sentinel (1e3 × H_S2_RMS_EMA.max(1)).
    pub fn read_nan_flags(&self) -> Result<[i32; 50], MLError> {
        let mut host = [0_i32; 50];
        self.stream.memcpy_dtoh(&self.nan_flags_buf, &mut host)
            .map_err(|e| MLError::ModelError(format!("nan_flags read: {e}")))?;
        Ok(host)
    }

    /// Copy f32 source buffer → td_errors_buf (f32→f32 DtoD).
    /// Uses graph-safe copy kernel (called from aux child graph).
    pub fn cast_f32_to_td_errors(&self, src: &CudaSlice<f32>) -> Result<(), MLError> {
        let n_bytes = self.config.batch_size * std::mem::size_of::<f32>();
        let src_ptr = src.raw_ptr();
        let dst_ptr = self.td_errors_buf.raw_ptr();
        self.graph_safe_copy_f32(dst_ptr, src_ptr, n_bytes, "td_errors")?;
        Ok(())
    }

    /// Reference to the online value logits buffer from the last cuBLAS forward.
    ///
    /// Shape: `[B, num_atoms]` f32. Used by ensemble heads to get head-0 C51 logits
    /// for diversity loss computation without re-running the forward pass.
    pub fn on_v_logits_buf(&self) -> &CudaSlice<f32> {
        &self.on_v_logits_buf
    }

    /// Forward pass for 5-bar and 20-bar value heads.
    /// Called from submit_post_aux_ops (captured in post_aux child graph).
    pub(crate) fn multi_horizon_value_forward(&self, batch_size: usize) -> Result<(), MLError> {
        // For now, d2d copy 1-bar logits as degenerate blend.
        // Full GEMM wiring deferred until the cuBLAS handle sharing is resolved.
        let n_bytes = batch_size * self.config.num_atoms * std::mem::size_of::<f32>();
        self.graph_safe_copy_f32(
            self.v_logits_blended.raw_ptr(),
            self.on_v_logits_buf.raw_ptr(),
            n_bytes,
            "v_logits_blend",
        )?;
        Ok(())
    }

    /// Reference to the target h_v scratch buffer (reused for ensemble head intermediate).
    ///
    /// Shape: `[B, VALUE_H]`. Safe to reuse between the CUDA Graph step and the next
    /// train_step call (not used during the post-graph ensemble forward phase).
    pub fn tg_h_v_scratch_ptr(&self) -> &CudaSlice<f32> {
        &self.tg_h_v_scratch
    }

    /// Run value head forward for an ensemble extra head.
    ///
    /// h_s2 → W_v1 → ReLU → W_v2 → v_logits
    ///
    /// Uses shared trunk activations (save_h_s2) with per-head value weights.
    /// Writes v_logits to `out_ptr` (offset into ensemble_logits_buf).
    pub fn forward_value_head_for_ensemble(
        &self,
        head_w_v1: u64, head_b_v1: u64,
        head_w_v2: u64, head_b_v2: u64,
        v_logits_out: u64,
        batch_size: usize,
    ) -> Result<(), MLError> {
        let h_s2_ptr = self.save_h_s2.raw_ptr();
        let h_v_ptr = self.tg_h_v_scratch.raw_ptr();
        self.cublas_forward.forward_value_head(
            &self.stream,
            h_s2_ptr, head_w_v1, head_b_v1, head_w_v2, head_b_v2,
            h_v_ptr, v_logits_out, batch_size,
        )
    }

    /// Total number of per-branch actions (BRANCH_0 + BRANCH_1 + BRANCH_2).
    // NOTE: forward_loss(), forward_only_q(), launch_forward_only(), launch_forward_loss(),
    // and launch_backward() were removed here. They depended on forward_loss_kernel,
    // forward_only_kernel, and backward_kernel which are dead code (replaced by cuBLAS).
    // See Task 2 for full cleanup context.
    pub fn total_actions(&self) -> usize {
        self.config.branch_0_size + self.config.branch_1_size + self.config.branch_2_size + self.config.branch_3_size
    }

    /// Reference to the Q-value output buffer.
    ///
    /// Shape: `[B, total_actions]`. Valid after `populate_q_out()` or `replay_forward_for_q_values()`.
    pub fn q_out_buf(&self) -> &CudaSlice<f32> {
        &self.q_out_buf
    }

    /// Convert current logits → Q-values in q_out_buf.
    ///
    /// Runs compute_expected_q on the logits already in on_v_logits_buf / on_b_logits_buf
    /// (populated by graph_forward replay). Does NOT replay any graph — just the
    /// lightweight expected_q kernel.
    ///
    /// Used by IQL to get Q(s, a_taken) for expectile regression target.
    pub fn populate_q_out(&self, batch_size: usize) -> Result<&CudaSlice<f32>, MLError> {
        let n = batch_size as i32;
        let na = self.config.num_atoms as i32;
        let b0 = self.config.branch_0_size as i32;
        let b1 = self.config.branch_1_size as i32;
        let b2 = self.config.branch_2_size as i32;
        let b3 = self.config.branch_3_size as i32;
        let block_dim = 256_u32;
        let grid_dim = ((batch_size as u32 + block_dim - 1) / block_dim).max(1);
        let q_out_ptr = self.q_out_buf.raw_ptr();
        let atom_stats_ptr = self.atom_stats_buf.raw_ptr();
        let atom_stats_block_sums_ptr = self.atom_stats_block_sums_buf.raw_ptr();
        let q_var_ptr = self.q_var_buf_trainer.raw_ptr();
        let atom_positions_buf_ptr = self.atom_positions_buf.raw_ptr();
        // SP22 H6 Phase 3 α atom-shift: pass W + current states + AUX_DIR_PROB_INDEX
        // so the kernel shifts direction-branch atom positions by W[a] * state_121.
        // Online forward → use current states_buf.
        let w_aux_ptr = self.w_aux_to_q_dir.raw_ptr();
        let aux_dir_prob_index = ml_core::state_layout::AUX_DIR_PROB_INDEX as i32;
        let state_dim_i32 = ml_core::state_layout::STATE_DIM as i32;
        // Phase 1: per-block sums (deterministic, no atomic) → block_sums
        unsafe {
            self.stream
                .launch_builder(&self.expected_q_kernel)
                .arg(&self.ptrs.on_v_logits_buf)
                .arg(&self.ptrs.on_b_logits_buf)
                .arg(&q_out_ptr)
                .arg(&n)
                .arg(&na)
                .arg(&b0)
                .arg(&b1)
                .arg(&b2)
                .arg(&b3)
                .arg(&self.per_sample_support_ptr)
                .arg(&atom_stats_block_sums_ptr)
                .arg(&q_var_ptr)
                .arg(&atom_positions_buf_ptr)
                .arg(&w_aux_ptr)
                .arg(&self.ptrs.states_buf)
                .arg(&aux_dir_prob_index)
                .arg(&state_dim_i32)
                .launch(LaunchConfig {
                    grid_dim: (grid_dim, 1, 1),
                    block_dim: (block_dim, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("populate_q_out expected_q: {e}")))?;
        }
        // Phase 2: sequential reduce block_sums → atom_stats (deterministic)
        let num_blocks_i32 = grid_dim as i32;
        unsafe {
            self.stream
                .launch_builder(&self.atom_stats_finalize_kernel)
                .arg(&atom_stats_block_sums_ptr)
                .arg(&atom_stats_ptr)
                .arg(&num_blocks_i32)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (1, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("populate_q_out atom_stats_finalize: {e}")))?;
        }
        Ok(&self.q_out_buf)
    }

    /// Run the cuBLAS forward pass without graph (used during graph capture).
    fn replay_forward_ungraphed(&mut self) -> Result<(), MLError> {
        let param_sizes = compute_param_sizes(&self.config);
        let on_w_ptrs = f32_weight_ptrs_from_base(self.ptrs.params_ptr, &param_sizes);
        // Build mag_concat from previous step's logits (one-step lag)
        self.launch_mag_concat_from(self.ptrs.save_h_s2, self.ptrs.on_v_logits_buf, self.ptrs.on_b_logits_buf)?;
        // SP14 B.9: rebuild dir_qaux concat from previous step's aux softmax
        // (one-step lag, mirrors mag_concat). Required because B.8 widened
        // `w_b0fc` to K=SH2+1 — the SGEMM consumer needs a [B, SH2+1] input.
        self.launch_sp14_dir_concat_qaux(self.ptrs.save_h_s2)?;
        self.launch_concat_ofi(2, self.ptrs.states_buf)?;
        self.launch_concat_ofi(3, self.ptrs.states_buf)?;
        self.cublas_forward.forward_online_raw(
            &self.stream, self.ptrs.states_buf, &on_w_ptrs,
            self.ptrs.save_h_s1, self.ptrs.save_h_s2, self.ptrs.save_h_v,
            self.ptrs.save_h_b0, self.ptrs.save_h_b1, self.ptrs.save_h_b2, self.ptrs.save_h_b3,
            self.ptrs.on_v_logits_buf, self.ptrs.on_b_logits_buf,
            self.ptrs.mag_concat_buf,
            self.sp14_dir_qaux_concat_scratch.raw_ptr(),
        )
    }

    /// Replay the TRAINING graph_forward for deterministic Q-value computation.
    ///
    /// Replays the EXACT SAME CUDA Graph used for training. This guarantees
    /// bit-identical cublasLtMatmul results because it replays the kernel
    /// launch configuration captured at step 0 — same tiling, same reduction
    /// order, same everything.
    ///
    /// The graph includes the full forward + loss + grad + backward pipeline.
    /// The loss/grad/backward writes go to scratch buffers that are overwritten
    /// on the next training step — harmless side-effects.
    ///
    /// States must be in `states_buf` (via `launch_pad_states`) before calling.
    /// After replay, logits are in on_v_logits_buf / on_b_logits_buf.
    /// Launches compute_expected_q to convert logits → Q-values in q_out_buf.
    pub fn replay_forward_for_q_values(
        &mut self,
        batch_size: usize,
    ) -> Result<&CudaSlice<f32>, MLError> {
        // Replay eval forward graph for deterministic Q-value computation.
        // Uses the eval_forward_exec injected by FusedTrainingCtx (captures
        // the same forward ops as the training child graph).
        if let Some(exec) = self.eval_forward_exec {
            // forward graph writes td_errors + per_sample_loss which PER reads.
            // Save before replay, restore after to prevent PER corruption.
            let b = self.config.batch_size;
            let f32_sz = std::mem::size_of::<f32>();
            let td_ptr = self.td_errors_buf.raw_ptr();
            let loss_ptr = self.per_sample_loss_buf.raw_ptr();
            let snap_td = self.eval_td_snapshot.raw_ptr();
            let snap_loss = self.eval_loss_snapshot.raw_ptr();
            unsafe {
                cudarc::driver::result::memcpy_dtod_async(snap_td, td_ptr, b * f32_sz, self.stream.cu_stream())
                    .map_err(|e| MLError::ModelError(format!("eval save td_errors: {e}")))?;
                cudarc::driver::result::memcpy_dtod_async(snap_loss, loss_ptr, b * f32_sz, self.stream.cu_stream())
                    .map_err(|e| MLError::ModelError(format!("eval save per_sample_loss: {e}")))?;
            }

            let result = unsafe { cudarc::driver::sys::cuGraphLaunch(exec, self.stream.cu_stream()) };
            if result != cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS {
                return Err(MLError::ModelError(format!("eval forward graph launch: {result:?}")));
            }

            // Restore -- undo PER-visible writes
            unsafe {
                cudarc::driver::result::memcpy_dtod_async(td_ptr, snap_td, b * f32_sz, self.stream.cu_stream())
                    .map_err(|e| MLError::ModelError(format!("eval restore td_errors: {e}")))?;
                cudarc::driver::result::memcpy_dtod_async(loss_ptr, snap_loss, b * f32_sz, self.stream.cu_stream())
                    .map_err(|e| MLError::ModelError(format!("eval restore per_sample_loss: {e}")))?;
            }
        } else {
            // Pre-step-0: no graph captured. Run ungraphed forward.
            let param_sizes = compute_param_sizes(&self.config);
            let on_w_ptrs = f32_weight_ptrs_from_base(self.ptrs.params_ptr, &param_sizes);
            // Build mag_concat from previous step's logits (zeros at step 0)
            self.launch_mag_concat_from(self.ptrs.save_h_s2, self.ptrs.on_v_logits_buf, self.ptrs.on_b_logits_buf)?;
            // SP14 B.9: rebuild dir_qaux concat (zeros at step 0; one-step lag thereafter).
            self.launch_sp14_dir_concat_qaux(self.ptrs.save_h_s2)?;
            self.launch_concat_ofi(2, self.ptrs.states_buf)?;
            self.launch_concat_ofi(3, self.ptrs.states_buf)?;
            self.cublas_forward.forward_online_raw(
                &self.stream, self.ptrs.states_buf, &on_w_ptrs,
                self.ptrs.save_h_s1, self.ptrs.save_h_s2, self.ptrs.save_h_v,
                self.ptrs.save_h_b0, self.ptrs.save_h_b1, self.ptrs.save_h_b2, self.ptrs.save_h_b3,
                self.ptrs.on_v_logits_buf, self.ptrs.on_b_logits_buf,
                self.ptrs.mag_concat_buf,
                self.sp14_dir_qaux_concat_scratch.raw_ptr(),
            )?;
        }

        // Convert logits → expected Q-values (graph-captured: NULL atom_stats/q_var)
        let n = batch_size as i32;
        let na = self.config.num_atoms as i32;
        let b0 = self.config.branch_0_size as i32;
        let b1 = self.config.branch_1_size as i32;
        let b2 = self.config.branch_2_size as i32;
        let b3 = self.config.branch_3_size as i32;
        let block_dim = 256_u32;
        let grid_dim = ((batch_size as u32 + block_dim - 1) / block_dim).max(1);
        let q_out_ptr = self.q_out_buf.raw_ptr();
        let null_atom_stats = 0u64;
        let null_q_var = 0u64;
        let atom_positions_buf_ptr = self.atom_positions_buf.raw_ptr();
        // SP22 H6 Phase 3 α atom-shift: online replay → current states.
        let w_aux_ptr = self.w_aux_to_q_dir.raw_ptr();
        let aux_dir_prob_index = ml_core::state_layout::AUX_DIR_PROB_INDEX as i32;
        let state_dim_i32 = ml_core::state_layout::STATE_DIM as i32;
        unsafe {
            self.stream
                .launch_builder(&self.expected_q_kernel)
                .arg(&self.ptrs.on_v_logits_buf)
                .arg(&self.ptrs.on_b_logits_buf)
                .arg(&q_out_ptr)
                .arg(&n)
                .arg(&na)
                .arg(&b0)
                .arg(&b1)
                .arg(&b2)
                .arg(&b3)
                .arg(&self.per_sample_support_ptr)
                .arg(&null_atom_stats)
                .arg(&null_q_var)
                .arg(&atom_positions_buf_ptr)
                .arg(&w_aux_ptr)
                .arg(&self.ptrs.states_buf)
                .arg(&aux_dir_prob_index)
                .arg(&state_dim_i32)
                .launch(LaunchConfig {
                    grid_dim: (grid_dim, 1, 1),
                    block_dim: (block_dim, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("replay expected_q: {e}")))?;
        }
        Ok(&self.q_out_buf)
    }

    /// Compute Q-value statistics entirely on GPU — zero CPU reduction.
    ///
    /// Runs cuBLAS forward + expected_q + q_stats_kernel → returns 7 scalars:
    /// `(avg_max_q, q_min, q_max, q_mean, q_variance, atom_entropy, atom_utilization)`.
    /// Only 28 bytes (7 f32) are read back to host.
    /// Compute Q-value statistics by running a fresh forward pass on the given states.
    /// Runs AFTER Adam update — evaluates the UPDATED model's Q-values.
    pub fn compute_q_stats(
        &mut self,
        states: &CudaSlice<f32>,
        batch_size: usize,
    ) -> Result<QValueStatsResult, MLError> {
        {
            let _eg = EventTrackingGuard::new(self.stream.context());
        } // drain stale errors, then drop guard before mutable borrow

        // Pad states into trainer's states_buf (graphed forward reads from there)
        self.launch_pad_states(
            self.ptrs.states_buf,
            states.raw_ptr(),
            batch_size,
        )?;

        self.replay_forward_for_q_values(batch_size)?;

        // q_stats_reduce kernel fully overwrites all 7 output floats via direct
        // assignment (single-thread, no atomicAdd) — no memset needed.

        let total_actions = self.total_actions() as i32;
        let n = batch_size as i32;
        let num_atoms = self.config.num_atoms as i32;
        let q_out_buf_ptr = self.q_out_buf.raw_ptr();
        let atom_stats_buf_ptr = self.atom_stats_buf.raw_ptr();
        let q_stats_buf_ptr = self.q_stats_buf.raw_ptr();

        unsafe {
            self.stream
                .launch_builder(&self.q_stats_kernel)
                .arg(&q_out_buf_ptr)
                .arg(&atom_stats_buf_ptr)
                .arg(&q_stats_buf_ptr)
                .arg(&n)
                .arg(&total_actions)
                .arg(&num_atoms)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("q_stats_kernel: {e}")))?;
        }

        // Single 28-byte readback: [avg_max_q, q_min, q_max, q_mean, q_var, atom_entropy, atom_utilization]
        let mut host = [0.0_f32; 7];
        self.stream.memcpy_dtoh(&self.q_stats_buf, &mut host)
            .map_err(|e| MLError::ModelError(format!("q_stats DtoH: {e}")))?;

        Ok(QValueStatsResult {
            avg_max_q: host[0] as f64,
            q_min: host[1],
            q_max: host[2],
            q_mean: host[3],
            q_variance: host[4],
            atom_entropy: host[5],
            atom_utilization: host[6],
        })
    }

    /// Reduce Q-value stats from the CURRENT training batch's q_out_buf.
    ///
    /// No extra forward pass — reuses the Q-values already computed by the
    /// CUDA graph's forward pass. Call AFTER `replay_graph_forward()`.
    ///
    /// Uses async DtoH readback via CUDA events to avoid stalling the GPU.
    /// Synchronous readback — computes Q-stats and returns current values.
    /// Called every 50 training steps. Cost: ~5µs sync + 28B DtoH.
    pub fn reduce_current_q_stats(&mut self) -> Result<QValueStatsResult, MLError> {
        let _eg = EventTrackingGuard::new(self.stream.context());

        // Re-run compute_expected_q to populate atom_stats_buf (entropy + utilization).
        // The graph replay computes Q-values but passes NULL atom_stats (can't atomicAdd in graph).
        // This extra launch runs outside the graph, only every 50 steps — negligible cost.
        let batch_size = self.config.batch_size;
        self.populate_q_out(batch_size)?;

        let n = self.config.batch_size as i32;
        let total_actions = self.total_actions() as i32;
        let num_atoms = self.config.num_atoms as i32;

        // Stats reduction → pinned device-mapped buffer (zero sync, one-step lag).
        // q_stats_kernel writes 7 floats to q_readback_dev_ptr[0..7].
        // CPU reads from q_readback_pinned — previous step's values.
        let stats_dev_ptr = self.q_readback_dev_ptr;
        let q_out_buf_ptr = self.q_out_buf.raw_ptr();
        let atom_stats_buf_ptr = self.atom_stats_buf.raw_ptr();
        unsafe {
            self.stream
                .launch_builder(&self.q_stats_kernel)
                .arg(&q_out_buf_ptr)
                .arg(&atom_stats_buf_ptr)
                .arg(&stats_dev_ptr)
                .arg(&n)
                .arg(&total_actions)
                .arg(&num_atoms)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("q_stats_kernel: {e}")))?;
        }
        // Copy q_out sample 0 (total_actions floats) to pinned buffer [7..] for per-branch Q-gap.
        let q_out_dst = self.q_readback_dev_ptr + 7 * std::mem::size_of::<f32>() as u64;
        unsafe {
            cudarc::driver::result::memcpy_dtod_async(
                q_out_dst, self.q_out_buf.raw_ptr(),
                total_actions as usize * std::mem::size_of::<f32>(), self.stream.cu_stream(),
            ).map_err(|e| MLError::ModelError(format!("q_out sample0 DtoD: {e}")))?;
        }
        // No sync — one-step lag is fine for monitoring. populate_q_out + q_stats_reduce
        // launched async; pinned memory becomes visible by the NEXT epoch's call.
        // Epoch 1 reads zeros (expected), epoch 2+ reads real atom stats.

        self.update_q_mean_ema();

        // Read from pinned buffer — host[0..7] = q_stats, host[7..19] = q_out sample 0
        let host: [f32; 7] = unsafe {
            let mut h = [0.0_f32; 7];
            std::ptr::copy_nonoverlapping(self.q_readback_pinned, h.as_mut_ptr(), 7);
            h
        };
        let total_actions = self.total_actions() as usize;
        let mut q_vals = vec![0.0_f32; total_actions];
        unsafe {
            std::ptr::copy_nonoverlapping(self.q_readback_pinned.add(7), q_vals.as_mut_ptr(), total_actions);
        };

        // Per-branch Q-gap: first sample's Q-values.
        // Branch layout: [dir(4), mag(3), ord(3), urg(3)] = 13 total.
        let branch_sizes = [
            self.config.branch_0_size, self.config.branch_1_size,
            self.config.branch_2_size, self.config.branch_3_size,
        ];
        let mut offset = 0;
        for d in 0..4 {
            let bs = branch_sizes[d];
            let slice = &q_vals[offset..offset + bs];
            let max_q = slice.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
            let mean_q = slice.iter().sum::<f32>() / bs as f32;
            self.last_per_branch_q_gaps[d] = (max_q - mean_q).max(0.0);
            offset += bs;
        }

        // Write per-branch Q-gaps to pinned device-mapped buffer — no HtoD copy.
        // GPU reads via dev_ptr (qlstm_step kernel).
        unsafe {
            std::ptr::copy_nonoverlapping(
                self.last_per_branch_q_gaps.as_ptr(),
                self.per_branch_q_gaps_pinned,
                4,
            );
        }

        // ISV: write atom utilization to pinned scratch for GPU-side ISV signal update
        unsafe { *self.atom_util_scratch_pinned = host[6].clamp(0.0, 1.0); }
        // ISV: write Q-mean as reward proxy to pinned scratch
        unsafe { *self.reward_scratch_pinned = host[3]; }

        // Task 2.X "make Full useful" — per-magnitude-bin Q-mean reducer
        // writes the 3 per-bin Q-means + absolute-scale reference to
        // pinned scratch. Runs on the same stream; isv_signal_update
        // reads the values directly via pinned device pointers (no sync).
        let mag_means_batch_size = self.config.batch_size;
        self.launch_q_mag_bin_means_reduce(mag_means_batch_size)?;

        // Task 2.Y "make direction branch useful at eval" — per-direction
        // Q-mean reducer (mirror of the magnitude reducer one axis up).
        // Writes the 4 per-bin Q-means + absolute-scale reference to pinned
        // scratch. Runs on the same stream; isv_signal_update reads the
        // values directly via pinned device pointers (no sync).
        self.launch_q_dir_bin_means_reduce(mag_means_batch_size)?;

        // ISV signal update — pure GPU, reads all pinned scalars, updates ISV vector
        self.update_isv_signals()?;

        // xLSTM temporal context: update matrix memory and write context[8] on GPU.
        self.launch_qlstm_step()?;

        // xLSTM training: prediction loss on context[0] vs prev_q_mean, SGD update.
        self.launch_qlstm_train_step()?;

        let result = QValueStatsResult {
            avg_max_q: host[0] as f64,
            q_min: host[1],
            q_max: host[2],
            q_mean: host[3],
            q_variance: host[4],
            atom_entropy: host[5],
            atom_utilization: host[6],
        };

        // Adaptive-rate EMA for atom utilization — SP4 Layer C close-out C3
        // (2026-05-01): the cold-start sentinel + adaptive-α EMA arithmetic
        // now run GPU-side via `update_utilization_ema_kernel` per
        // `feedback_no_cpu_compute_strict`. The kernel updates
        // `utilization_ema_pinned` (the field's new mapped-pinned storage)
        // and writes the same value into `homeostatic_obs_pinned[1]`,
        // taking over the slot-1 mirror that the host
        // `update_homeostatic_observables` previously maintained.
        if let Err(e) = self.launch_update_utilization_ema(result.atom_utilization) {
            tracing::warn!("update_utilization_ema launch failed: {e}");
        }

        // G16: Homeostatic regularizer — write observables and launch penalty kernel.
        self.update_homeostatic_observables();
        self.apply_homeostatic_regularization()?;

        // Drop EventTrackingGuard before mutable borrow in step_atom_positions.
        drop(_eg);

        // Adaptive atom position SGD: decay spacing_raw toward uniform every 50 steps.
        self.step_atom_positions(self.adam_step as usize)?;

        Ok(result)
    }

    /// Reduce current `q_out_buf` to per-branch Q-stats (4 × 7 floats).
    ///
    /// Launches `q_stats_per_branch_reduce` — slices the flat Q-output by
    /// branch and computes `[avg_max_q, q_min, q_max, q_mean, q_var,
    /// atom_entropy(=0), atom_utilization(=0)]` per branch. Drives the
    /// `update_eval_v_range` per-branch EMA state and the 8 ISV v-range
    /// slots (spec 2026-04-23).
    ///
    /// Non-destructive to `q_stats_buf` / `q_readback_pinned` — uses a
    /// separate pinned readback. Intended to be called alongside
    /// `reduce_current_q_stats` at epoch boundary; both share the same
    /// `q_out_buf` population so there is no extra forward pass.
    pub fn reduce_current_q_stats_per_branch(&mut self)
        -> Result<PerBranchQValueStats, MLError>
    {
        let _eg = EventTrackingGuard::new(self.stream.context());

        let total_actions = self.total_actions() as i32;
        let n = self.config.batch_size as i32;
        let b0 = self.config.branch_0_size as i32;
        let b1 = self.config.branch_1_size as i32;
        let b2 = self.config.branch_2_size as i32;
        let b3 = self.config.branch_3_size as i32;
        let b0_off: i32 = 0;
        let b1_off: i32 = b0;
        let b2_off: i32 = b0 + b1;
        let b3_off: i32 = b0 + b1 + b2;

        let q_out_buf_ptr = self.q_out_buf.raw_ptr();
        let out_ptr = self.per_branch_q_stats_dev_ptr;

        unsafe {
            self.stream
                .launch_builder(&self.q_stats_per_branch_kernel)
                .arg(&q_out_buf_ptr)
                .arg(&out_ptr)
                .arg(&n)
                .arg(&total_actions)
                .arg(&b0_off).arg(&b0)
                .arg(&b1_off).arg(&b1)
                .arg(&b2_off).arg(&b2)
                .arg(&b3_off).arg(&b3)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (1, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("q_stats_per_branch_reduce: {e}")))?;
        }

        // Synchronous read from pinned host buffer. The kernel is launched on
        // the same stream as the C51 graph, so a single stream synchronize
        // guarantees the pinned bytes reflect the current launch — matches the
        // legacy `compute_q_stats` cadence.
        unsafe { cudarc::driver::sys::cuStreamSynchronize(self.stream.cu_stream()); }

        let mut host = [0.0_f32; 28];
        unsafe {
            std::ptr::copy_nonoverlapping(
                self.per_branch_q_stats_pinned, host.as_mut_ptr(), 28,
            );
        }

        let mut per_branch = [QValueStatsResult {
            avg_max_q: 0.0,
            q_min: 0.0,
            q_max: 0.0,
            q_mean: 0.0,
            q_variance: 0.0,
            atom_entropy: 0.0,
            atom_utilization: 0.0,
        }; 4];
        for b in 0..4_usize {
            let base = b * 7;
            per_branch[b] = QValueStatsResult {
                avg_max_q: host[base] as f64,
                q_min: host[base + 1],
                q_max: host[base + 2],
                q_mean: host[base + 3],
                q_variance: host[base + 4],
                atom_entropy: host[base + 5],
                atom_utilization: host[base + 6],
            };
        }
        Ok(PerBranchQValueStats { per_branch })
    }

    /// Launch the xLSTM mLSTM step: reads q_stats_buf[7] + per_branch_q_gaps[4] (pinned),
    /// updates persistent matrix memory C[8,8] and normalizer n[8],
    /// writes context[8] output. Runs in a single thread (grid=1, block=1).
    /// Called every 50 training steps from reduce_current_q_stats.
    pub(crate) fn launch_qlstm_step(&self) -> Result<(), MLError> {
        let q_stats_ptr = self.q_stats_buf.raw_ptr();
        let gaps_ptr = self.per_branch_q_gaps_dev_ptr;
        let c_ptr = self.qlstm_c.raw_ptr();
        let n_ptr = self.qlstm_n.raw_ptr();
        let ctx_ptr = self.qlstm_context.raw_ptr();
        let w_ptr = self.qlstm_weights.dev_ptr;
        let id = 11_i32;
        let hd = 8_i32;
        unsafe {
            self.stream.launch_builder(&self.qlstm_step_kernel)
                .arg(&q_stats_ptr)
                .arg(&gaps_ptr)
                .arg(&c_ptr)
                .arg(&n_ptr)
                .arg(&ctx_ptr)
                .arg(&w_ptr)
                .arg(&id)
                .arg(&hd)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (1, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("qlstm_step: {e}")))?;
        }
        Ok(())
    }

    /// Launch xLSTM training step: prediction loss on context[0] vs prev_q_mean.
    /// SGD update in-place at LR=1e-4. Runs after qlstm_step.
    /// Then updates prev_q_mean from current q_stats[3] (q_mean slot).
    /// Takes &self: raw pointers (prev_q_mean_pinned) are mutated via unsafe.
    pub(crate) fn launch_qlstm_train_step(&self) -> Result<(), MLError> {
        let w_ptr = self.qlstm_weights.dev_ptr;
        let ctx_ptr = self.qlstm_context.raw_ptr();
        let c_ptr = self.qlstm_c.raw_ptr();
        let n_ptr = self.qlstm_n.raw_ptr();
        let q_stats_ptr = self.q_stats_buf.raw_ptr();
        let gaps_ptr = self.per_branch_q_gaps_dev_ptr;
        let prev_q_mean = unsafe { *self.prev_q_mean_pinned };
        let lr: f32 = 1e-4;
        let id = 11_i32;
        let hd = 8_i32;
        unsafe {
            self.stream.launch_builder(&self.qlstm_train_kernel)
                .arg(&w_ptr)
                .arg(&ctx_ptr)
                .arg(&c_ptr)
                .arg(&n_ptr)
                .arg(&q_stats_ptr)
                .arg(&gaps_ptr)
                .arg(&prev_q_mean)
                .arg(&lr)
                .arg(&id)
                .arg(&hd)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (1, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("qlstm_train_step: {e}")))?;
        }
        // Update prev_q_mean for next step from q_stats[3] (q_mean slot).
        // q_stats_buf[3] was read back synchronously above in reduce_current_q_stats.
        // The readback buffer at q_readback_pinned[3] contains the current q_mean.
        let current_q_mean = unsafe { *self.q_readback_pinned.add(3) };
        unsafe { *self.prev_q_mean_pinned = current_q_mean; }
        Ok(())
    }

    /// Snapshot pre-denoise Q-values from q_coord_buf into denoise_q_input_buf.
    /// Must be called BEFORE launch_q_denoise (which modifies q_coord_buf in-place).
    pub(crate) fn snapshot_pre_denoise_q(&self, batch_size: usize) -> Result<(), MLError> {
        let n_bytes = batch_size * 12 * std::mem::size_of::<f32>();
        let src = self.q_coord_buf.raw_ptr();
        let dst = self.denoise_q_input_buf.raw_ptr();
        self.graph_safe_copy_f32(dst, src, n_bytes, "pre_denoise_q")?;
        Ok(())
    }

    /// Compute target network Q-values [B, total_actions] into denoise_target_q_buf.
    /// Uses compute_expected_q on target logits (tg_v_logits_buf, tg_b_logits_buf).
    /// `compute_expected_q` writes `total_actions = b0+b1+b2+b3 = 13` floats per
    /// sample (4-direction factored layout); the buffer is sized to match.
    /// Per-sample consumers thread `target_stride = total_actions` to read only
    /// the first D=12 entries.
    pub(crate) fn compute_denoise_target_q(&self, batch_size: usize) -> Result<(), MLError> {
        let n = batch_size as i32;
        let na = self.config.num_atoms as i32;
        let b0 = self.config.branch_0_size as i32;
        let b1 = self.config.branch_1_size as i32;
        let b2 = self.config.branch_2_size as i32;
        let b3 = self.config.branch_3_size as i32;
        let blocks = ((batch_size as u32 + 255) / 256).max(1);
        let q_out_ptr = self.denoise_target_q_buf.raw_ptr();
        let null_atom_stats = 0u64;
        let null_q_var = 0u64;
        let atom_positions_buf_ptr = self.atom_positions_buf.raw_ptr();
        // SP22 H6 Phase 3 α atom-shift: target Q evaluated on s' → next_states_buf.
        // W is shared between online and target (no separate target W: aux head
        // is online-only — the atom-shift on target Q uses the SAME W and reads
        // next-state aux p_up that was written into next_states_buf[121] by the
        // state-assembly pipeline).
        let w_aux_ptr = self.w_aux_to_q_dir.raw_ptr();
        let aux_dir_prob_index = ml_core::state_layout::AUX_DIR_PROB_INDEX as i32;
        let state_dim_i32 = ml_core::state_layout::STATE_DIM as i32;
        unsafe {
            self.stream
                .launch_builder(&self.expected_q_kernel)
                .arg(&self.ptrs.tg_v_logits_buf)
                .arg(&self.ptrs.tg_b_logits_buf)
                .arg(&q_out_ptr)
                .arg(&n)
                .arg(&na)
                .arg(&b0)
                .arg(&b1)
                .arg(&b2)
                .arg(&b3)
                .arg(&self.per_sample_support_ptr)
                .arg(&null_atom_stats)
                .arg(&null_q_var)
                .arg(&atom_positions_buf_ptr)
                .arg(&w_aux_ptr)
                .arg(&self.ptrs.next_states_buf)
                .arg(&aux_dir_prob_index)
                .arg(&state_dim_i32)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("compute_denoise_target_q expected_q: {e}")))?;
        }

        // SP4 Layer B (Mech 1): clip target_q to ±ISV[TARGET_Q_BOUND=131].
        // Hard clamp via dqn_clamp_finite_f32_kernel (reused from SP1).
        // Single point — all downstream losses (C51 / IQN / MSE) consume the
        // same denoise_target_q_buf, so one clip covers every consumer.
        // SP4 producer `launch_sp4_target_q_p99` (Layer A Task A5) tracks
        // p99(|target_q|) into the ISV bound directly; the multiplier-driven
        // bound from SP3 (`10 × Q_ABS_REF.max(1.0)`) is replaced with this
        // adaptive p99 bound per `feedback_no_quickfixes` and
        // `feedback_isv_for_adaptive_bounds`. EPS_CLAMP_FLOOR=1.0 ε on the
        // bound itself (SP1 ε-floor pearl) handles cold-start ISV (= 0).
        use crate::cuda_pipeline::sp4_isv_slots::TARGET_Q_BOUND_INDEX;
        use crate::cuda_pipeline::sp4_wiener_ema::EPS_CLAMP_FLOOR;
        let max_abs_target_q = self.read_isv_signal_at(TARGET_Q_BOUND_INDEX).max(EPS_CLAMP_FLOOR);
        self.launch_clamp_finite_f32(
            self.denoise_target_q_buf.raw_ptr(),
            self.denoise_target_q_buf.len(),
            max_abs_target_q,
        )?;
        Ok(())
    }

    /// Backward pass through the diffusion denoiser.
    /// Computes MSE gradient of Q_refined vs Q_target and backprops through
    /// the 2 FC-SiLU-FC steps. Deterministic: one thread per weight, plain writes.
    pub(crate) fn launch_q_denoise_backward(&mut self, batch_size: usize) -> Result<(), MLError> {
        let b = batch_size as i32;
        // Grid: one thread per weight element (1800 total = 2 steps × 900)
        let blocks = ((1800_u32 + 255) / 256).max(1);
        let q_refined_ptr = self.q_coord_buf.raw_ptr();
        let q_target_ptr = self.denoise_target_q_buf.raw_ptr();
        let params_ptr = self.denoise_params.dev_ptr;
        let d_params_ptr = self.denoise_grad.raw_ptr();
        let q_input_ptr = self.denoise_q_input_buf.raw_ptr();
        let var_ptr = self.q_var_buf_trainer.raw_ptr();
        // Per-sample strides for the four [B, ?] inputs. The denoiser MLP only
        // reads the first D=12 slots per sample, but each upstream buffer may
        // be sized wider — the kernel needs the true stride to align per-sample
        // addressing.
        //
        //   q_var_buf_trainer:    [B, total_actions=13]  (compute_expected_q writes)
        //   denoise_target_q_buf: [B, total_actions=13]  (compute_expected_q writes)
        //   q_coord_buf:          [B, 12]                (cross-branch attention output)
        //   denoise_q_input_buf:  [B, 12]                (snapshot of q_coord_buf)
        let var_stride = self.total_actions() as i32;
        let target_stride = self.total_actions() as i32;
        let refined_stride = 12_i32;
        let input_stride = 12_i32;
        unsafe {
            self.stream
                .launch_builder(&self.q_denoise_backward_kernel)
                .arg(&q_refined_ptr)
                .arg(&q_target_ptr)
                .arg(&params_ptr)
                .arg(&d_params_ptr)
                .arg(&q_input_ptr)
                .arg(&var_ptr)
                .arg(&b)
                .arg(&var_stride)
                .arg(&refined_stride)
                .arg(&target_stride)
                .arg(&input_stride)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("q_denoise_backward: {e}")))?;
        }
        Ok(())
    }

    /// cuBLAS-accelerated backward through the 2-step diffusion denoiser MLP.
    ///
    /// Replaces the old `q_denoise_backward` kernel (1800 threads × B=8192 inner loops)
    /// with batched cuBLAS GEMMs + lightweight elementwise kernels.
    ///
    /// Pipeline:
    ///   Forward replay (save activations):
    ///     1. Build input0 [H=24, B] from Q_input + var_q
    ///     2. cuBLAS: pre_h0 = W1_0 @ input0
    ///     3. Bias + SiLU → h0, save pre_h0
    ///     4. cuBLAS: out0 = W2_0 @ h0
    ///     5. Skip connection → q_step0 = Q_input + 0.1*(out0 + b2_0)
    ///     6. Build input1 [H, B] from q_step0 + var_q
    ///     7. cuBLAS: pre_h1 = W1_1 @ input1
    ///     8. Bias + SiLU → h1, save pre_h1
    ///     9. cuBLAS: out1 = W2_1 @ h1 (out1 col-major scratch in d_res_buf)
    ///   Backward:
    ///     10. denoise_loss_grad → d_res [D, B]
    ///     11-16. Step 1 backward: dW2_1, db2_1, d_h1, d_pre1, dW1_1, db1_1
    ///     17-22. Step 0 backward: dW2_0, db2_0, d_h0, d_pre0, dW1_0, db1_0
    pub(crate) fn launch_q_denoise_backward_cublas(&mut self, batch_size: usize) -> Result<(), MLError> {
        const D: usize = 12;
        const H: usize = 24;
        const STEP_PARAMS: usize = 900; // W1[576] + b1[24] + W2[288] + b2[12]
        let b = batch_size;

        // Per-stream lt_handle (Option C determinism fix). Main stream — fast path.
        let (lt_handle, lt_ws_ptr, lt_ws_size) =
            self.shared_cublas.lt_for(&self.stream)?;
        let cu_stream = self.stream.cu_stream() as cublaslt_sys::cudaStream_t;
        let alpha: f32 = 1.0;
        let beta: f32 = 0.0;

        let params_ptr = self.denoise_params.dev_ptr;
        let d_params_ptr = self.denoise_grad.raw_ptr();
        let q_input_ptr = self.denoise_q_input_buf.raw_ptr();
        let var_ptr = self.q_var_buf_trainer.raw_ptr();
        let q_refined_ptr = self.q_coord_buf.raw_ptr();
        let q_target_ptr = self.denoise_target_q_buf.raw_ptr();

        // Weight pointers for step 0
        let w1_0_ptr = params_ptr;
        let b1_0_ptr = params_ptr + (576 * 4) as u64;
        let w2_0_ptr = params_ptr + (600 * 4) as u64;
        let b2_0_ptr = params_ptr + (888 * 4) as u64;
        // Weight pointers for step 1
        let w1_1_ptr = params_ptr + (STEP_PARAMS * 4) as u64;
        let b1_1_ptr = params_ptr + ((STEP_PARAMS + 576) * 4) as u64;
        let w2_1_ptr = params_ptr + ((STEP_PARAMS + 600) * 4) as u64;
        let _b2_1_ptr = params_ptr + ((STEP_PARAMS + 888) * 4) as u64;

        // Gradient pointers for step 0
        let dw1_0_ptr = d_params_ptr;
        let db1_0_ptr = d_params_ptr + (576 * 4) as u64;
        let dw2_0_ptr = d_params_ptr + (600 * 4) as u64;
        let db2_0_ptr = d_params_ptr + (888 * 4) as u64;
        // Gradient pointers for step 1
        let dw1_1_ptr = d_params_ptr + (STEP_PARAMS * 4) as u64;
        let db1_1_ptr = d_params_ptr + ((STEP_PARAMS + 576) * 4) as u64;
        let dw2_1_ptr = d_params_ptr + ((STEP_PARAMS + 600) * 4) as u64;
        let db2_1_ptr = d_params_ptr + ((STEP_PARAMS + 888) * 4) as u64;

        let blocks_b = ((b as u32 + 255) / 256).max(1);
        let bias_num_blocks = ((b + 255) / 256) as i32;

        // ═══════════════════════════════════════════════════════
        // FORWARD REPLAY — save activations for backward
        // ═══════════════════════════════════════════════════════

        // Step 0: schedule = 1.0 - 1/2 = 0.5
        let schedule_0: f32 = 0.5;
        let d_val = D as i32;
        // q_var_buf_trainer is sized [B, total_actions]; denoiser only consumes
        // the first D=12 variance slots, but stride is needed for alignment.
        let var_stride = self.total_actions() as i32;
        let input0_ptr = self.denoise_input0_buf.raw_ptr();
        let pre_h0_ptr = self.denoise_pre_h0_buf.raw_ptr();
        let h0_ptr = self.denoise_h0_buf.raw_ptr();

        // 1. Build input0 [H=24, B] from Q_input + var_q
        unsafe {
            self.stream.launch_builder(&self.denoise_build_input_kernel)
                .arg(&q_input_ptr)
                .arg(&var_ptr)
                .arg(&input0_ptr)
                .arg(&(b as i32))
                .arg(&d_val)
                .arg(&var_stride)
                .arg(&schedule_0)
                .launch(LaunchConfig {
                    grid_dim: (blocks_b, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("denoise_build_input step0: {e}")))?;
        }

        // 2. cuBLAS: pre_h0 = W1_0 @ input0. W1_0 row-major [H,H] → col-major [H,H]^T.
        //    C[H,B] = W1^T_cm @ input_cm → TRANSA=T
        unsafe {
            cublaslt_sys::cublasLtMatmul(
                lt_handle,
                self.denoise_gemm_fwd_hh.matmul_desc,
                &alpha as *const f32 as *const std::ffi::c_void,
                w1_0_ptr as *const std::ffi::c_void,
                self.denoise_gemm_fwd_hh.a_layout,
                input0_ptr as *const std::ffi::c_void,
                self.denoise_gemm_fwd_hh.b_layout,
                &beta as *const f32 as *const std::ffi::c_void,
                pre_h0_ptr as *mut std::ffi::c_void,
                self.denoise_gemm_fwd_hh.c_layout,
                pre_h0_ptr as *mut std::ffi::c_void,
                self.denoise_gemm_fwd_hh.d_layout,
                &self.denoise_gemm_fwd_hh.algo as *const _,
                lt_ws_ptr as *mut std::ffi::c_void,
                lt_ws_size,
                cu_stream,
            );
        }

        // 3. Bias + SiLU: pre_h0 += b1_0, h0 = SiLU(pre_h0). Saves pre_h0 with bias.
        let hb_total = (H * b) as i32;
        let h_dim = H as i32;
        unsafe {
            self.stream.launch_builder(&self.denoise_bias_silu_kernel)
                .arg(&pre_h0_ptr)
                .arg(&h0_ptr)
                .arg(&b1_0_ptr)
                .arg(&h_dim)
                .arg(&hb_total)
                .launch(LaunchConfig {
                    grid_dim: (((H * b) as u32 + 255) / 256, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("denoise_bias_silu step0: {e}")))?;
        }

        // 4. cuBLAS: out0 = W2_0 @ h0. W2_0 row-major [D,H] → TRANSA=T.
        //    Use denoise_d_res_buf as scratch for out0 [D, B] col-major.
        let out0_ptr = self.denoise_d_res_buf.raw_ptr(); // reuse d_res_buf for fwd scratch
        unsafe {
            cublaslt_sys::cublasLtMatmul(
                lt_handle,
                self.denoise_gemm_fwd_dh.matmul_desc,
                &alpha as *const f32 as *const std::ffi::c_void,
                w2_0_ptr as *const std::ffi::c_void,
                self.denoise_gemm_fwd_dh.a_layout,
                h0_ptr as *const std::ffi::c_void,
                self.denoise_gemm_fwd_dh.b_layout,
                &beta as *const f32 as *const std::ffi::c_void,
                out0_ptr as *mut std::ffi::c_void,
                self.denoise_gemm_fwd_dh.c_layout,
                out0_ptr as *mut std::ffi::c_void,
                self.denoise_gemm_fwd_dh.d_layout,
                &self.denoise_gemm_fwd_dh.algo as *const _,
                lt_ws_ptr as *mut std::ffi::c_void,
                lt_ws_size,
                cu_stream,
            );
        }

        // 5. Skip connection: q_step0 = Q_input + 0.1 * (out0 + b2_0)
        let q_step0_ptr = self.denoise_q_step0_buf.raw_ptr();
        let scale_01: f32 = 0.1;
        unsafe {
            self.stream.launch_builder(&self.denoise_skip_conn_kernel)
                .arg(&q_input_ptr)
                .arg(&out0_ptr)
                .arg(&b2_0_ptr)
                .arg(&q_step0_ptr)
                .arg(&(b as i32))
                .arg(&d_val)
                .arg(&scale_01)
                .launch(LaunchConfig {
                    grid_dim: (((D * b) as u32 + 255) / 256, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("denoise_skip_connection step0: {e}")))?;
        }

        // Step 1: schedule = 1.0 - 2/2 = 0.0
        let schedule_1: f32 = 0.0;
        let input1_ptr = self.denoise_input1_buf.raw_ptr();
        let pre_h1_ptr = self.denoise_pre_h1_buf.raw_ptr();
        let h1_ptr = self.denoise_h1_buf.raw_ptr();

        // 6. Build input1 from q_step0 + var_q
        unsafe {
            self.stream.launch_builder(&self.denoise_build_input_kernel)
                .arg(&q_step0_ptr)
                .arg(&var_ptr)
                .arg(&input1_ptr)
                .arg(&(b as i32))
                .arg(&d_val)
                .arg(&var_stride)
                .arg(&schedule_1)
                .launch(LaunchConfig {
                    grid_dim: (blocks_b, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("denoise_build_input step1: {e}")))?;
        }

        // 7. cuBLAS: pre_h1 = W1_1 @ input1
        unsafe {
            cublaslt_sys::cublasLtMatmul(
                lt_handle,
                self.denoise_gemm_fwd_hh.matmul_desc,
                &alpha as *const f32 as *const std::ffi::c_void,
                w1_1_ptr as *const std::ffi::c_void,
                self.denoise_gemm_fwd_hh.a_layout,
                input1_ptr as *const std::ffi::c_void,
                self.denoise_gemm_fwd_hh.b_layout,
                &beta as *const f32 as *const std::ffi::c_void,
                pre_h1_ptr as *mut std::ffi::c_void,
                self.denoise_gemm_fwd_hh.c_layout,
                pre_h1_ptr as *mut std::ffi::c_void,
                self.denoise_gemm_fwd_hh.d_layout,
                &self.denoise_gemm_fwd_hh.algo as *const _,
                lt_ws_ptr as *mut std::ffi::c_void,
                lt_ws_size,
                cu_stream,
            );
        }

        // 8. Bias + SiLU for step 1
        unsafe {
            self.stream.launch_builder(&self.denoise_bias_silu_kernel)
                .arg(&pre_h1_ptr)
                .arg(&h1_ptr)
                .arg(&b1_1_ptr)
                .arg(&h_dim)
                .arg(&hb_total)
                .launch(LaunchConfig {
                    grid_dim: (((H * b) as u32 + 255) / 256, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("denoise_bias_silu step1: {e}")))?;
        }

        // 9. cuBLAS: out1 = W2_1 @ h1 (we don't need out1 explicitly — Q_refined is already
        //    computed in the forward pass. But we do need h1 saved for backward.)
        //    Skip materializing out1 — it's redundant. Q_refined = q_step0 + 0.1*(out1+b2_1)
        //    is already in q_coord_buf from the forward pass.

        // ═══════════════════════════════════════════════════════
        // BACKWARD
        // ═══════════════════════════════════════════════════════

        // 10. Loss gradient: d_res = 0.1 * 2 * (Q_refined - Q_target) / B.
        // Q_refined = q_coord_buf [B, 12] (post-cross-branch-attention).
        // Q_target = denoise_target_q_buf [B, total_actions=13] (post-compute_expected_q).
        // Kernel reads only the first D=12 entries from each per sample but
        // needs per-sample strides to handle the asymmetric layout.
        let inv_b: f32 = 1.0 / b as f32;
        let d_res_ptr = self.denoise_d_res_buf.raw_ptr();
        let refined_stride: i32 = 12;
        let target_stride: i32 = self.total_actions() as i32;
        unsafe {
            self.stream.launch_builder(&self.denoise_loss_grad_kernel)
                .arg(&q_refined_ptr)
                .arg(&q_target_ptr)
                .arg(&d_res_ptr)
                .arg(&(b as i32))
                .arg(&d_val)
                .arg(&refined_stride)
                .arg(&target_stride)
                .arg(&inv_b)
                .launch(LaunchConfig {
                    grid_dim: (((D * b) as u32 + 255) / 256, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("denoise_loss_grad: {e}")))?;
        }

        let partials_ptr = self.denoise_bias_grad_partials.raw_ptr();
        let d_h_ptr = self.denoise_d_h_buf.raw_ptr();

        // ── Step 1 backward ──────────────────────────────────────────

        // 11. dW2_1 = h1 @ d_res^T → output at dw2_1_ptr (row-major [D,H])
        //     cuBLAS: C[H,D]_colmaj = h1[H,B] @ d_res[D,B]^T. Col-major [H,D] = row-major [D,H].
        unsafe {
            cublaslt_sys::cublasLtMatmul(
                lt_handle,
                self.denoise_gemm_dw2.matmul_desc,
                &alpha as *const f32 as *const std::ffi::c_void,
                h1_ptr as *const std::ffi::c_void,
                self.denoise_gemm_dw2.a_layout,
                d_res_ptr as *const std::ffi::c_void,
                self.denoise_gemm_dw2.b_layout,
                &beta as *const f32 as *const std::ffi::c_void,
                dw2_1_ptr as *mut std::ffi::c_void,
                self.denoise_gemm_dw2.c_layout,
                dw2_1_ptr as *mut std::ffi::c_void,
                self.denoise_gemm_dw2.d_layout,
                &self.denoise_gemm_dw2.algo as *const _,
                lt_ws_ptr as *mut std::ffi::c_void,
                lt_ws_size,
                cu_stream,
            );
        }

        // 12. db2_1 = sum_b(d_res[d, b]) — bias grad reduce for D=12
        let d_dim = D as i32;
        unsafe {
            self.stream.launch_builder(&self.denoise_bias_grad_p1_kernel)
                .arg(&d_res_ptr)
                .arg(&partials_ptr)
                .arg(&d_dim)
                .arg(&(b as i32))
                .launch(LaunchConfig {
                    grid_dim: (bias_num_blocks as u32, D as u32, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("denoise_bias_grad_p1 db2_1: {e}")))?;
        }
        unsafe {
            self.stream.launch_builder(&self.denoise_bias_grad_p2_kernel)
                .arg(&partials_ptr)
                .arg(&db2_1_ptr)
                .arg(&d_dim)
                .arg(&bias_num_blocks)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("denoise_bias_grad_p2 db2_1: {e}")))?;
        }

        // 13. d_h1 = W2_1^T @ d_res. W2_1 row-major [D,H] = col-major [H,D].
        //     cuBLAS: C[H,B] = W2_cm[H,D] @ d_res[D,B] → TRANSA=N, TRANSB=N
        unsafe {
            cublaslt_sys::cublasLtMatmul(
                lt_handle,
                self.denoise_gemm_bwd_dh.matmul_desc,
                &alpha as *const f32 as *const std::ffi::c_void,
                w2_1_ptr as *const std::ffi::c_void,
                self.denoise_gemm_bwd_dh.a_layout,
                d_res_ptr as *const std::ffi::c_void,
                self.denoise_gemm_bwd_dh.b_layout,
                &beta as *const f32 as *const std::ffi::c_void,
                d_h_ptr as *mut std::ffi::c_void,
                self.denoise_gemm_bwd_dh.c_layout,
                d_h_ptr as *mut std::ffi::c_void,
                self.denoise_gemm_bwd_dh.d_layout,
                &self.denoise_gemm_bwd_dh.algo as *const _,
                lt_ws_ptr as *mut std::ffi::c_void,
                lt_ws_size,
                cu_stream,
            );
        }

        // 14. SiLU backward: d_pre1 = d_h1 * SiLU'(pre_h1). Write into d_h_buf (reuse).
        //     Actually we need d_pre in a separate buffer since d_h is consumed.
        //     We can write d_pre over pre_h1 since pre_h1 won't be needed after this.
        unsafe {
            self.stream.launch_builder(&self.denoise_silu_bwd_kernel)
                .arg(&d_h_ptr)
                .arg(&pre_h1_ptr)
                .arg(&d_h_ptr)   // output d_pre1 into d_h_buf (d_h no longer needed)
                .arg(&hb_total)
                .launch(LaunchConfig {
                    grid_dim: (((H * b) as u32 + 255) / 256, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("denoise_silu_bwd step1: {e}")))?;
        }
        // d_pre1 is now in d_h_buf

        // 15. dW1_1 = input1 @ d_pre1^T → output at dw1_1_ptr (row-major [H,H])
        //     cuBLAS: C[H,H]_colmaj = input1[H,B] @ d_pre1[H,B]^T
        unsafe {
            cublaslt_sys::cublasLtMatmul(
                lt_handle,
                self.denoise_gemm_dw1.matmul_desc,
                &alpha as *const f32 as *const std::ffi::c_void,
                input1_ptr as *const std::ffi::c_void,
                self.denoise_gemm_dw1.a_layout,
                d_h_ptr as *const std::ffi::c_void,   // d_pre1
                self.denoise_gemm_dw1.b_layout,
                &beta as *const f32 as *const std::ffi::c_void,
                dw1_1_ptr as *mut std::ffi::c_void,
                self.denoise_gemm_dw1.c_layout,
                dw1_1_ptr as *mut std::ffi::c_void,
                self.denoise_gemm_dw1.d_layout,
                &self.denoise_gemm_dw1.algo as *const _,
                lt_ws_ptr as *mut std::ffi::c_void,
                lt_ws_size,
                cu_stream,
            );
        }

        // 16. db1_1 = sum_b(d_pre1) — bias grad reduce for H=24
        unsafe {
            self.stream.launch_builder(&self.denoise_bias_grad_p1_kernel)
                .arg(&d_h_ptr)   // d_pre1
                .arg(&partials_ptr)
                .arg(&h_dim)
                .arg(&(b as i32))
                .launch(LaunchConfig {
                    grid_dim: (bias_num_blocks as u32, H as u32, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("denoise_bias_grad_p1 db1_1: {e}")))?;
        }
        unsafe {
            self.stream.launch_builder(&self.denoise_bias_grad_p2_kernel)
                .arg(&partials_ptr)
                .arg(&db1_1_ptr)
                .arg(&h_dim)
                .arg(&bias_num_blocks)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("denoise_bias_grad_p2 db1_1: {e}")))?;
        }

        // ── Step 0 backward ──────────────────────────────────────────
        // d_res is the same (skip connection passes gradient through unchanged)

        // 17. dW2_0 = h0 @ d_res^T
        unsafe {
            cublaslt_sys::cublasLtMatmul(
                lt_handle,
                self.denoise_gemm_dw2.matmul_desc,
                &alpha as *const f32 as *const std::ffi::c_void,
                h0_ptr as *const std::ffi::c_void,
                self.denoise_gemm_dw2.a_layout,
                d_res_ptr as *const std::ffi::c_void,
                self.denoise_gemm_dw2.b_layout,
                &beta as *const f32 as *const std::ffi::c_void,
                dw2_0_ptr as *mut std::ffi::c_void,
                self.denoise_gemm_dw2.c_layout,
                dw2_0_ptr as *mut std::ffi::c_void,
                self.denoise_gemm_dw2.d_layout,
                &self.denoise_gemm_dw2.algo as *const _,
                lt_ws_ptr as *mut std::ffi::c_void,
                lt_ws_size,
                cu_stream,
            );
        }

        // 18. db2_0 = sum_b(d_res)
        unsafe {
            self.stream.launch_builder(&self.denoise_bias_grad_p1_kernel)
                .arg(&d_res_ptr)
                .arg(&partials_ptr)
                .arg(&d_dim)
                .arg(&(b as i32))
                .launch(LaunchConfig {
                    grid_dim: (bias_num_blocks as u32, D as u32, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("denoise_bias_grad_p1 db2_0: {e}")))?;
        }
        unsafe {
            self.stream.launch_builder(&self.denoise_bias_grad_p2_kernel)
                .arg(&partials_ptr)
                .arg(&db2_0_ptr)
                .arg(&d_dim)
                .arg(&bias_num_blocks)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("denoise_bias_grad_p2 db2_0: {e}")))?;
        }

        // 19. d_h0 = W2_0^T @ d_res
        unsafe {
            cublaslt_sys::cublasLtMatmul(
                lt_handle,
                self.denoise_gemm_bwd_dh.matmul_desc,
                &alpha as *const f32 as *const std::ffi::c_void,
                w2_0_ptr as *const std::ffi::c_void,
                self.denoise_gemm_bwd_dh.a_layout,
                d_res_ptr as *const std::ffi::c_void,
                self.denoise_gemm_bwd_dh.b_layout,
                &beta as *const f32 as *const std::ffi::c_void,
                d_h_ptr as *mut std::ffi::c_void,
                self.denoise_gemm_bwd_dh.c_layout,
                d_h_ptr as *mut std::ffi::c_void,
                self.denoise_gemm_bwd_dh.d_layout,
                &self.denoise_gemm_bwd_dh.algo as *const _,
                lt_ws_ptr as *mut std::ffi::c_void,
                lt_ws_size,
                cu_stream,
            );
        }

        // 20. SiLU backward step 0: d_pre0 = d_h0 * SiLU'(pre_h0)
        unsafe {
            self.stream.launch_builder(&self.denoise_silu_bwd_kernel)
                .arg(&d_h_ptr)
                .arg(&pre_h0_ptr)
                .arg(&d_h_ptr)
                .arg(&hb_total)
                .launch(LaunchConfig {
                    grid_dim: (((H * b) as u32 + 255) / 256, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("denoise_silu_bwd step0: {e}")))?;
        }

        // 21. dW1_0 = input0 @ d_pre0^T
        unsafe {
            cublaslt_sys::cublasLtMatmul(
                lt_handle,
                self.denoise_gemm_dw1.matmul_desc,
                &alpha as *const f32 as *const std::ffi::c_void,
                input0_ptr as *const std::ffi::c_void,
                self.denoise_gemm_dw1.a_layout,
                d_h_ptr as *const std::ffi::c_void,   // d_pre0
                self.denoise_gemm_dw1.b_layout,
                &beta as *const f32 as *const std::ffi::c_void,
                dw1_0_ptr as *mut std::ffi::c_void,
                self.denoise_gemm_dw1.c_layout,
                dw1_0_ptr as *mut std::ffi::c_void,
                self.denoise_gemm_dw1.d_layout,
                &self.denoise_gemm_dw1.algo as *const _,
                lt_ws_ptr as *mut std::ffi::c_void,
                lt_ws_size,
                cu_stream,
            );
        }

        // 22. db1_0 = sum_b(d_pre0)
        unsafe {
            self.stream.launch_builder(&self.denoise_bias_grad_p1_kernel)
                .arg(&d_h_ptr)   // d_pre0
                .arg(&partials_ptr)
                .arg(&h_dim)
                .arg(&(b as i32))
                .launch(LaunchConfig {
                    grid_dim: (bias_num_blocks as u32, H as u32, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("denoise_bias_grad_p1 db1_0: {e}")))?;
        }
        unsafe {
            self.stream.launch_builder(&self.denoise_bias_grad_p2_kernel)
                .arg(&partials_ptr)
                .arg(&db1_0_ptr)
                .arg(&h_dim)
                .arg(&bias_num_blocks)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("denoise_bias_grad_p2 db1_0: {e}")))?;
        }

        Ok(())
    }

    /// Adam optimizer step for denoise_params. Same pattern as step_selectivity_adam.
    pub(crate) fn step_denoise_adam(&mut self) -> Result<(), MLError> {
        self.denoise_adam_step += 1;
        let step_val = self.denoise_adam_step;
        const DENOISE_TOTAL_PARAMS: usize = 1800;
        let n = DENOISE_TOTAL_PARAMS as i32;

        // Write step counter to pinned device-mapped memory
        unsafe { *self.denoise_t_pinned = step_val; }

        // Phase 1: grad_norm on denoise_grad
        // Uses post_aux_child-specific handle — grad_norm_standalone is captured in
        // forward_child (d_logits clipping). Sharing across children corrupts on Hopper.
        let grad_ptr = self.denoise_grad.raw_ptr();
        let partials_ptr = self.denoise_norm_partials.raw_ptr();
        unsafe {
            self.stream
                .launch_builder(&self.grad_norm_standalone_post_aux)
                .arg(&grad_ptr)
                .arg(&partials_ptr)
                .arg(&n)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("denoise grad_norm: {e}")))?;
        }

        // Phase 2: grad_norm_finalize
        let norm_out_ptr = self.denoise_norm_buf.raw_ptr();
        let nb = 1_i32;
        unsafe {
            self.stream
                .launch_builder(&self.grad_norm_finalize_post_aux)
                .arg(&partials_ptr)
                .arg(&norm_out_ptr)
                .arg(&nb)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("denoise grad_norm_finalize: {e}")))?;
        }

        // Phase 3: Adam update
        let aux_lr_ptr = self.aux_lr_dev_ptr;  // pinned device-mapped 1e-4
        let beta1: f32 = 0.9;
        let beta2: f32 = 0.999;
        let eps: f32 = 1e-8;
        let weight_decay: f32 = 0.0;
        let params_ptr = self.denoise_params.dev_ptr;
        let m_ptr = self.denoise_adam_m.raw_ptr();
        let v_ptr = self.denoise_adam_v.raw_ptr();
        let clip_ptr = self.denoise_clip_buf.raw_ptr();
        let t_ptr = self.denoise_t_dev_ptr;
        // Reuse main mask buffer as dummy — weight_decay=0.0 nullifies all mask values.
        // l1_end=0 + l1_lambda=0.0 disables L1 for auxiliary heads.
        let wd_mask_ptr = self.weight_decay_mask.raw_ptr();
        let l1_end_aux: i32 = 0;
        let l1_lambda_aux: f32 = 0.0;
        // SP4 Layer B: aux trainer (denoise) is outside the SP4 8-group
        // taxonomy — disable Mech 9 weight clamp here (mirrors DT pattern).
        let weight_clamp_max_abs: f32 = 0.0_f32;
        // SP4 Task A14: aux trainer (denoise) is outside the SP4 8-group
        // taxonomy — Pearl C engagement counter disabled for this launch.
        let _aux_nan_flags_null: u64 = 0;
        let _aux_diag_slot_disabled: i32 = -1;
        let _aux_engage_buf_null: u64 = 0;
        let _aux_engage_off_disabled: i32 = SP4_ENGAGE_OFFSET_DISABLED;
        let blocks = ((DENOISE_TOTAL_PARAMS as u32 + 255) / 256).max(1);
        unsafe {
            self.stream
                .launch_builder(&self.adam_update_post_aux)
                .arg(&params_ptr)
                .arg(&grad_ptr)
                .arg(&m_ptr)
                .arg(&v_ptr)
                .arg(&norm_out_ptr)
                .arg(&aux_lr_ptr)
                .arg(&beta1)
                .arg(&beta2)
                .arg(&eps)
                .arg(&weight_decay)
                .arg(&clip_ptr)
                .arg(&t_ptr)
                .arg(&n)
                .arg(&wd_mask_ptr)
                .arg(&l1_end_aux)
                .arg(&l1_lambda_aux)
                .arg(&weight_clamp_max_abs) // SP3 Mech 9: post-Adam |p_val| clamp
                // SP4 Task A14: disabled — denoise outside SP4 group taxonomy.
                .arg(&_aux_nan_flags_null)
                .arg(&_aux_diag_slot_disabled)
                .arg(&_aux_engage_buf_null)
                .arg(&_aux_engage_off_disabled)
                .arg(&1.0_f32) // SP21 Phase 4: lr_scale (denoise aux trainer, non-branch = 1.0 no-op)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("denoise Adam update: {e}")))?;
        }
        Ok(())
    }

    /// Last computed per-branch Q-gaps from reduce_current_q_stats.
    pub fn get_per_branch_q_gaps(&self) -> [f32; 4] { self.last_per_branch_q_gaps }

    /// EMA of atom utilization [0,1]. Updated GPU-side by
    /// `update_utilization_ema_kernel` per training step (SP4 Layer C
    /// close-out C3, 2026-05-01) — host accessor reads through the
    /// mapped-pinned host_ptr after the kernel's `__threadfence_system()`.
    pub fn utilization_ema(&self) -> f32 {
        if self.utilization_ema_pinned.is_null() { 1.0 }
        else { unsafe { *self.utilization_ema_pinned } }
    }

    /// Flush the last in-flight Q-stats readback at epoch end — non-blocking.
    ///
    /// Reads directly from the pinned host buffer without synchronizing.
    /// The 28-byte async DtoH (7 × f32) completes in <1 µs while thousands
    /// of GPU kernel launches elapse between the last Q-stats computation
    // ═══════════════════════════════════════════════════════════════════
    // CUDA Graph capture and invalidation
    // ═══════════════════════════════════════════════════════════════════

    /// Capture two CUDA Graphs: forward (zero→backward) and adam (grad_norm→unflatten).
    ///
    /// Called on the first `train_step()` or after `invalidate_training_graph()`.
    /// The split allows external code to inject auxiliary gradients (IQN, attention,
    /// ensemble) into `grad_buf` between the two graph replays.
    ///
    /// Graph A (`graph_forward`): zero → cuBLAS forward → C51 loss → C51 grad → cuBLAS backward
    /// Graph B (`graph_adam`):    grad_norm → Adam → unflatten (42 d2d copies)
    /// Submit main-stream forward ops for graph capture (Pass 1 + 2 + loss + grad + backward).
    ///
    /// Contains everything that runs on `self.stream`:
    /// - Zero accumulators
    /// - Pass 1: online forward on states + stochastic depth
    /// - Pass 2: target forward on next_states
    /// - Curiosity inference
    /// - MSE + C51 loss/gradient blend
    /// - f32 d_logits copy
    /// - cuBLAS backward
    ///
    /// Does NOT contain Pass 3 (Double DQN) or any event/set_stream ops.
    /// Pass 3 is submitted separately via `submit_forward_ops_ddqn()`.
    pub(crate) fn submit_forward_ops_main(&mut self) -> Result<(), MLError> {
        // ── Zero accumulators (all REQUIRED — deterministic reduce / beta=1.0 accumulation) ─
        // total_loss + mse_loss are pinned device-mapped — zero via cuMemsetD32Async on dev_ptr.
        unsafe {
            cudarc::driver::sys::cuMemsetD32Async(
                self.total_loss_dev_ptr, 0, 1, self.stream.cu_stream(),
            );
            cudarc::driver::sys::cuMemsetD32Async(
                self.mse_loss_dev_ptr, 0, 1, self.stream.cu_stream(),
            );
        }
        // grad_buf: backward_full uses beta=1.0 GEMM accumulation — zero via ptrs
        unsafe {
            cudarc::driver::sys::cuMemsetD32Async(
                self.ptrs.grad_buf,
                0,
                self.total_params,
                self.stream.cu_stream(),
            );
        }
        // d_value/adv_logits: c51_grad + mse_grad kernels write directly (no atomicAdd)
        // Raw cuMemsetD32Async — cudarc memset_zeros is NOT captured by CUDA Graph.
        unsafe {
            cudarc::driver::sys::cuMemsetD32Async(
                self.d_value_logits_buf.raw_ptr(), 0,
                self.d_value_logits_buf.len(), self.stream.cu_stream(),
            );
            cudarc::driver::sys::cuMemsetD32Async(
                self.d_adv_logits_buf.raw_ptr(), 0,
                self.d_adv_logits_buf.len(), self.stream.cu_stream(),
            );
        }

        // ── 1. Forward (cuBLAS SGEMM — Pass 1 + 2, no Pass 3) ──────────
        self.launch_cublas_forward()?;

        // ── 1b. ISV + plan heads (inside graph capture — matches working SHA) ──
        {
            let batch_size = self.config.batch_size;
            self.launch_isv_forward()?;
            self.launch_isv_feature_gate(batch_size)?;
            self.launch_recursive_confidence_forward(batch_size)?;
            self.launch_trade_plan_forward(batch_size)?;
            // OFI embedding: extract [raw_ofi; delta_ofi; book_aggression; log_duration] → MLP → 10-dim
            self.launch_ofi_embed_forward(batch_size)?;
        }

        // ── 1c. Temporal pipeline (enriches h_s2 before loss) ──
        {
            let batch_size = self.config.batch_size;
            self.mamba2_step(batch_size)?;
            self.compute_predictive_coding_loss(batch_size)?;
            self.apply_regime_dropout(batch_size, true)?;
            self.launch_isv_temporal_route()?;
            self.risk_budget_forward(batch_size)?;
        }

        // ── 1d. SP3 Mech 10: ISV-driven post-trunk activation clamp ──
        //
        // The F1 step-1000 NaN cascade (smoke-test-2xrxk on commit 0d7e27d61)
        // root-caused to forward-pass GEMM accumulator overflow under
        // Mech-9-clamped weights. With trunk weights pinned at 100 ×
        // Q_ABS_REF=5000 and K=256 inner-dim, accumulators compound by
        // ~80000× per layer; with trunk depth ≥6 (encoder + VSN + GRN
        // blocks + bottleneck) the chain hits f32_max ≈ 3.4e38. The slot 12
        // (`save_h_s2`) firing was the smoking gun — h_s2 itself goes
        // non-finite mid-forward. Mech 1+2 clamp targets+atoms; Mech 9 clamps
        // weights; Mech 10 closes the gap by clamping h_s2 immediately after
        // the trunk + temporal pipeline finalises it, BEFORE any downstream
        // consumer (loss path, IQN, value head, replay save, eval-action
        // select) reads the buffer.
        //
        // SP4 Layer B (Mech 10): bound = ISV[H_S2_BOUND=169]. Producer
        // `launch_sp4_h_s2_p99` (Layer A Task A9) tracks p99(|save_h_s2|)
        // through Pearls A+D into ISV[169] directly — no magnitude
        // multiplier, no separate RMS scaling. The EPS_CLAMP_FLOOR ε on
        // the bound itself preserves SP1 cold-start convention.
        //
        // Diagnostic ordering (SP1 pearl): the inline NaN check on slot 12
        // fires BEFORE the clamp sanitises, so the sentinel sees the original
        // Inf/NaN before sanitization replaces it with 0. This is redundant
        // with the slot 12 check inside `run_nan_checks_post_forward` (which
        // runs later, post-clamp, on the sanitized buffer) — but the inline
        // check at this site is the only one that captures the pre-clamp
        // state.
        //
        // Sanitiser: `launch_clamp_finite_f32` (dqn_utility_kernels.cu:462)
        // does `isfinite(v) ? clamp(v, ±max_abs) : 0.0f`. NaN→0, Inf→0,
        // finite→clamped. NO new kernel — same helper used by Mech 9 sites.
        // The clamp adds a single kernel launch in the captured graph
        // (expected — captured graph topology grows by one node).
        {
            use crate::cuda_pipeline::sp4_isv_slots::H_S2_BOUND_INDEX;
            use crate::cuda_pipeline::sp4_wiener_ema::EPS_CLAMP_FLOOR;
            self.check_nan_f32(self.save_h_s2.raw_ptr(), self.save_h_s2.len(), 12)?;
            let max_abs_h_s2 = self.read_isv_signal_at(H_S2_BOUND_INDEX).max(EPS_CLAMP_FLOOR);
            self.launch_clamp_finite_f32(
                self.save_h_s2.raw_ptr(),
                self.save_h_s2.len(),
                max_abs_h_s2,
            )?;
        }

        // ── 2+3. Loss + gradient (blended MSE + C51 via c51_alpha ramp) ─
        self.launch_curiosity_inference()?;

        // MSE path → scratch buffers (REQUIRED: mse_grad_kernel uses atomicAdd)
        // Raw cuMemsetD32Async — cudarc memset_zeros is NOT captured by CUDA Graph.
        unsafe {
            cudarc::driver::sys::cuMemsetD32Async(
                self.d_value_logits_mse.raw_ptr(), 0,
                self.d_value_logits_mse.len(), self.stream.cu_stream(),
            );
            cudarc::driver::sys::cuMemsetD32Async(
                self.d_adv_logits_mse.raw_ptr(), 0,
                self.d_adv_logits_mse.len(), self.stream.cu_stream(),
            );
        }
        self.launch_mse_loss()?;
        self.launch_loss_reduce(self.mse_loss_dev_ptr)?;
        self.launch_mse_grad_to_scratch()?;

        // C51 path → main buffers (already zeroed above)
        self.fill_gamma_buf()?;
        self.launch_c51_loss()?;
        self.launch_loss_reduce(self.total_loss_dev_ptr)?;
        self.launch_c51_grad()?;

        // ── Phase 3 T3.4: MoE load-balance auxiliary loss ────────────────
        // Computes λ·K·Σ_k(mean_b g[b,k])² and SAXPYs the scalar into
        // total_loss_dev_ptr. Runs after C51 loss so it adds on top of the
        // already-accumulated TD loss. Does NOT generate a direct gradient
        // (the gradient flows through the gate backward in launch_moe_backward).
        self.launch_moe_load_balance_loss()?;

        // Blend: main = α * C51 + (1-α) * MSE
        {
            let alpha = self.c51_alpha;
            let na = self.config.num_atoms;
            let b = self.config.batch_size;
            let b0 = self.config.branch_0_size;
            let b1 = self.config.branch_1_size;
            let b2 = self.config.branch_2_size;
            let b3 = self.config.branch_3_size;
            let n_val = (b * pad32(na)) as i32;
            let n_adv = (b * (b0 + b1 + b2 + b3) * na + 32 * 4) as i32;
            let scale_mse = 1.0 - alpha;
            let val_blocks = ((n_val as usize + 255) / 256) as u32;
            let adv_blocks = ((n_adv as usize + 255) / 256) as u32;
            let cfg_val = LaunchConfig { grid_dim: (val_blocks, 1, 1), block_dim: (256, 1, 1), shared_mem_bytes: 0 };
            let cfg_adv = LaunchConfig { grid_dim: (adv_blocks, 1, 1), block_dim: (256, 1, 1), shared_mem_bytes: 0 };

            let val_ptr = self.d_value_logits_buf.raw_ptr();
            let val_mse_ptr = self.d_value_logits_mse.raw_ptr();
            let adv_ptr = self.d_adv_logits_buf.raw_ptr();
            let adv_mse_ptr = self.d_adv_logits_mse.raw_ptr();

            unsafe {
                self.stream.launch_builder(&self.scale_f32_kernel)
                    .arg(&val_ptr)
                    .arg(&alpha).arg(&n_val)
                    .launch(cfg_val).map_err(|e| MLError::ModelError(format!("scale c51 val: {e}")))?;
                self.stream.launch_builder(&self.saxpy_f32_kernel)
                    .arg(&val_ptr).arg(&val_mse_ptr)
                    .arg(&scale_mse).arg(&n_val)
                    .launch(cfg_val).map_err(|e| MLError::ModelError(format!("blend mse val: {e}")))?;
                self.stream.launch_builder(&self.scale_f32_kernel)
                    .arg(&adv_ptr)
                    .arg(&alpha).arg(&n_adv)
                    .launch(cfg_adv).map_err(|e| MLError::ModelError(format!("scale c51 adv: {e}")))?;
                self.stream.launch_builder(&self.saxpy_f32_kernel)
                    .arg(&adv_ptr).arg(&adv_mse_ptr)
                    .arg(&scale_mse).arg(&n_adv)
                    .launch(cfg_adv).map_err(|e| MLError::ModelError(format!("blend mse adv: {e}")))?;
            }
        }

        // ── 3.5. Clamp d_logits norm before backward to prevent amplification.
        // The backward GEMM multiplies d_logits by W^T at each layer. If d_logits
        // are large (from PER IS-weight extremes, C51 edge atoms, or fold transition),
        // the amplification through 4+ layers can produce exponentially growing
        // grad_buf norms (17→8134 over 15 epochs). Clamping here bounds the entire
        // backward chain. Threshold 5.0 ≈ 3× typical d_logits norm (~1.7).
        {
            let b = self.config.batch_size;
            let na = self.config.num_atoms;
            let tba = (self.config.branch_0_size + self.config.branch_1_size
                + self.config.branch_2_size + self.config.branch_3_size) * na;
            let n_val = (b * pad32(na)) as i32;
            let n_adv = (b * tba) as i32;
            let max_d_logit_norm = 5.0_f32;

            // Two-phase norm on d_value_logits (reuse grad_norm infrastructure)
            let val_ptr = self.d_value_logits_buf.raw_ptr();
            let adv_ptr = self.d_adv_logits_buf.raw_ptr();
            let partials_ptr = self.grad_norm_partials.raw_ptr();
            let norm_ptr = self.aux_norm_scratch.raw_ptr();
            // Compute d_value norm (phase 1)
            let val_blocks = ((n_val as u32 + 255) / 256) as u32;
            unsafe {
                self.stream.launch_builder(&self.grad_norm_standalone)
                    .arg(&val_ptr).arg(&partials_ptr).arg(&n_val)
                    .launch(LaunchConfig { grid_dim: (val_blocks, 1, 1), block_dim: (256, 1, 1), shared_mem_bytes: 0 })
                    .map_err(|e| MLError::ModelError(format!("d_logits norm val: {e}")))?;
            }
            // Append d_adv partials after d_val partials
            let adv_blocks = ((n_adv as u32 + 255) / 256) as u32;
            let adv_off = (val_blocks as usize * std::mem::size_of::<f32>()) as u64;
            unsafe {
                self.stream.launch_builder(&self.grad_norm_standalone)
                    .arg(&adv_ptr).arg(&(partials_ptr + adv_off)).arg(&n_adv)
                    .launch(LaunchConfig { grid_dim: (adv_blocks, 1, 1), block_dim: (256, 1, 1), shared_mem_bytes: 0 })
                    .map_err(|e| MLError::ModelError(format!("d_logits norm adv: {e}")))?;
            }
            // Finalize: reduce all partials → single L2 norm in aux_norm_scratch
            let nb = (val_blocks + adv_blocks) as i32;
            unsafe {
                self.stream.launch_builder(&self.grad_norm_finalize_kernel)
                    .arg(&partials_ptr).arg(&norm_ptr).arg(&nb)
                    .launch(LaunchConfig { grid_dim: (1, 1, 1), block_dim: (256, 1, 1), shared_mem_bytes: 0 })
                    .map_err(|e| MLError::ModelError(format!("d_logits norm fin: {e}")))?;
            }
            // Clip both buffers using the combined norm (reuse clip_grad_kernel)
            unsafe {
                self.stream.launch_builder(&self.clip_grad_kernel)
                    .arg(&val_ptr).arg(&norm_ptr).arg(&max_d_logit_norm).arg(&n_val)
                    .launch(LaunchConfig { grid_dim: (val_blocks, 1, 1), block_dim: (256, 1, 1), shared_mem_bytes: 0 })
                    .map_err(|e| MLError::ModelError(format!("clip d_val: {e}")))?;
                self.stream.launch_builder(&self.clip_grad_kernel)
                    .arg(&adv_ptr).arg(&norm_ptr).arg(&max_d_logit_norm).arg(&n_adv)
                    .launch(LaunchConfig { grid_dim: (adv_blocks, 1, 1), block_dim: (256, 1, 1), shared_mem_bytes: 0 })
                    .map_err(|e| MLError::ModelError(format!("clip d_adv: {e}")))?;
            }
        }

        // ── 4. Backward (cuBLAS SGEMM, chain rule through layers) ─
        self.launch_cublas_backward()?;

        Ok(())
    }

    /// Submit loss computation + gradient ops (everything between forward and backward).
    /// Extracted from submit_forward_ops_main for sub-graph timing.
    pub(crate) fn submit_loss_and_grad_ops(&mut self) -> Result<(), MLError> {
        // Zero accumulators — pinned device-mapped, use cuMemsetD32Async on dev_ptr.
        unsafe {
            cudarc::driver::sys::cuMemsetD32Async(
                self.total_loss_dev_ptr, 0, 1, self.stream.cu_stream(),
            );
            cudarc::driver::sys::cuMemsetD32Async(
                self.mse_loss_dev_ptr, 0, 1, self.stream.cu_stream(),
            );
        }
        unsafe {
            cudarc::driver::sys::cuMemsetD32Async(
                self.ptrs.grad_buf,
                0,
                self.total_params,
                self.stream.cu_stream(),
            );
        }
        self.stream.memset_zeros(&mut self.d_value_logits_buf)
            .map_err(|e| MLError::ModelError(format!("zero d_value_logits: {e}")))?;
        self.stream.memset_zeros(&mut self.d_adv_logits_buf)
            .map_err(|e| MLError::ModelError(format!("zero d_adv_logits: {e}")))?;

        self.launch_curiosity_inference()?;

        // MSE path
        self.stream.memset_zeros(&mut self.d_value_logits_mse)
            .map_err(|e| MLError::ModelError(format!("zero d_value_mse: {e}")))?;
        self.stream.memset_zeros(&mut self.d_adv_logits_mse)
            .map_err(|e| MLError::ModelError(format!("zero d_adv_mse: {e}")))?;
        self.launch_mse_loss()?;
        self.launch_loss_reduce(self.mse_loss_dev_ptr)?;
        self.launch_mse_grad_to_scratch()?;

        // C51 path
        self.fill_gamma_buf()?;
        self.launch_c51_loss()?;
        self.launch_loss_reduce(self.total_loss_dev_ptr)?;
        self.launch_c51_grad()?;

        // Blend: main = α * C51 + (1-α) * MSE
        {
            let alpha = self.c51_alpha;
            let na = self.config.num_atoms;
            let b = self.config.batch_size;
            let b0 = self.config.branch_0_size;
            let b1 = self.config.branch_1_size;
            let b2 = self.config.branch_2_size;
            let b3 = self.config.branch_3_size;
            let n_val = (b * pad32(na)) as i32;
            let n_adv = (b * (b0 + b1 + b2 + b3) * na + 32 * 4) as i32;
            let scale_mse = 1.0 - alpha;
            let val_blocks = ((n_val as usize + 255) / 256) as u32;
            let adv_blocks = ((n_adv as usize + 255) / 256) as u32;
            let cfg_val = LaunchConfig { grid_dim: (val_blocks, 1, 1), block_dim: (256, 1, 1), shared_mem_bytes: 0 };
            let cfg_adv = LaunchConfig { grid_dim: (adv_blocks, 1, 1), block_dim: (256, 1, 1), shared_mem_bytes: 0 };

            let val_ptr = self.d_value_logits_buf.raw_ptr();
            let val_mse_ptr = self.d_value_logits_mse.raw_ptr();
            let adv_ptr = self.d_adv_logits_buf.raw_ptr();
            let adv_mse_ptr = self.d_adv_logits_mse.raw_ptr();

            unsafe {
                self.stream.launch_builder(&self.scale_f32_kernel)
                    .arg(&val_ptr).arg(&alpha).arg(&n_val)
                    .launch(cfg_val).map_err(|e| MLError::ModelError(format!("scale c51 val: {e}")))?;
                self.stream.launch_builder(&self.saxpy_f32_kernel)
                    .arg(&val_ptr).arg(&val_mse_ptr).arg(&scale_mse).arg(&n_val)
                    .launch(cfg_val).map_err(|e| MLError::ModelError(format!("blend mse val: {e}")))?;
                self.stream.launch_builder(&self.scale_f32_kernel)
                    .arg(&adv_ptr).arg(&alpha).arg(&n_adv)
                    .launch(cfg_adv).map_err(|e| MLError::ModelError(format!("scale c51 adv: {e}")))?;
                self.stream.launch_builder(&self.saxpy_f32_kernel)
                    .arg(&adv_ptr).arg(&adv_mse_ptr).arg(&scale_mse).arg(&n_adv)
                    .launch(cfg_adv).map_err(|e| MLError::ModelError(format!("blend mse adv: {e}")))?;
            }
        }

        Ok(())
    }

    /// Submit Pass 3 (Double DQN online forward on next_states) on the main stream.
    /// Uses its own CublasGemmSet but shares the cuBLAS handle via PerStreamCublasHandles.
    pub(crate) fn submit_forward_ops_ddqn(&mut self) -> Result<(), MLError> {
        let param_sizes = compute_param_sizes(&self.config);
        let on_w_ptrs = f32_weight_ptrs_from_base(self.ptrs.params_ptr, &param_sizes);

        // ── Plan 4 Task 1B-iii: VSN feature-selection (DDQN-online-on-next_states) ──
        // DDQN argmax uses the ONLINE network on next_states, so this VSN
        // forward uses `on_w_ptrs` (the same VSN weights as the online-on-states
        // pass in `launch_cublas_forward`). Throwaway logits/mask scratches —
        // the producer-only `vsn_mask_ema_update` reads `vsn_mask_buf` (online
        // pass) only. h1 outputs share `vsn_h1_inference_scratch` (DDQN is
        // inference-only — no save-for-backward).
        let ddqn_h1_inf_scratch_ptrs: [u64; ml_core::state_layout::SL_NUM_FEATURE_GROUPS] = {
            let p = self.vsn_h1_inference_scratch.raw_ptr();
            [p, p, p, p, p, p]
        };
        let on_next_states_for_bn = self.vsn_gated_next_states_for_ddqn_buf.raw_ptr();
        self.cublas_forward_ddqn.vsn_forward(
            &self.stream,
            self.ptrs.next_states_buf,
            on_next_states_for_bn,
            &on_w_ptrs,
            self.vsn_logits_ddqn_scratch.raw_ptr(),
            self.vsn_mask_ddqn_scratch.raw_ptr(),
            &ddqn_h1_inf_scratch_ptrs,
            self.vsn_linear2_scratch_buf.raw_ptr(),
            self.vsn_group_begins_buf.raw_ptr(),
            self.vsn_group_ends_buf.raw_ptr(),
            self.config.batch_size,
        )?;

        // Bottleneck for DDQN argmax pass: online weights on next_states.
        // Mirrors the online-on-states block in `launch_cublas_forward` and
        // the target-on-next_states block; without this the DDQN argmax
        // encoder reads raw `next_states[0..s1_input_dim)` and produces
        // argmax actions inconsistent with the online net's training-time
        // input semantics.
        // Plan 4 Task 1B-iii: input is `on_next_states_for_bn` (= vsn_gated_next_states_for_ddqn_buf).
        let on_next_s1_input_ptr = if self.config.bottleneck_dim > 0 {
            let bn_dim = self.config.bottleneck_dim;
            let b = self.config.batch_size;
            let market_dim = self.config.market_dim;
            let portfolio_dim = ml_core::state_layout::STATE_DIM.saturating_sub(market_dim);
            // SP15 Wave 4.1b: trailing +1 column = dd_pct (broadcast from
            // ISV[DD_PCT_INDEX=406] by `bn_tanh_concat_dd_kernel`).
            let concat_dim = bn_dim + portfolio_dim + 1;
            let bn_hidden_ptr = self.on_next_bn_hidden_buf.raw_ptr();
            let state_dim_padded_usize = self.cublas_forward.state_dim_padded;
            self.cublas_forward.sgemm_f32_ldb(
                &self.stream,
                on_w_ptrs[33],                  // online w_bn
                on_next_states_for_bn,          // VSN-gated next_states
                bn_hidden_ptr,
                bn_dim, b, market_dim,
                state_dim_padded_usize,
                "ddqn_h_bn",
            )?;
            self.cublas_forward.launch_add_bias_f32_raw(
                &self.stream, bn_hidden_ptr, on_w_ptrs[34], bn_dim, b,
            )?;
            let concat_ptr = self.on_next_bn_concat_buf.raw_ptr();
            let state_dim_padded_i32 = state_dim_padded_usize as i32;
            let bn_dim_i32 = bn_dim as i32;
            let market_dim_i32 = market_dim as i32;
            let concat_dim_i32 = concat_dim as i32;
            let b_i32 = b as i32;
            // SP15 Wave 4.1b: launch the dd_pct-aware concat kernel. Reads
            // `isv[DD_PCT_INDEX=406]` and broadcasts it across batch as the
            // (concat_dim − 1)th column. ISV bus is the same pointer all the
            // other ISV consumers use; experience_collector populates it via
            // `launch_sp15_dd_state` before the trainer's forward graph runs,
            // so by the time this kernel reads slot 406 the value is fresh.
            launch_sp15_bn_concat_dd(
                &self.stream,
                &self.bn_tanh_concat_dd_kernel,
                bn_hidden_ptr,
                on_next_states_for_bn,
                self.isv_signals_dev_ptr,
                concat_ptr,
                b_i32,
                bn_dim_i32,
                market_dim_i32,
                state_dim_padded_i32,
                concat_dim_i32,
            )?;
            concat_ptr
        } else {
            // No bottleneck: pass VSN-gated next_states directly to DDQN encoder.
            on_next_states_for_bn
        };

        self.cublas_forward_ddqn.forward_online_raw(
            &self.stream, on_next_s1_input_ptr, &on_w_ptrs,
            self.ptrs.on_next_h_s1_scratch, self.ptrs.on_next_h_s2_scratch,
            self.ptrs.on_next_h_v_scratch,
            self.ptrs.on_next_h_b_scratch, self.ptrs.on_next_h_b_scratch,
            self.ptrs.on_next_h_b_scratch, self.ptrs.on_next_h_b_scratch,
            self.ptrs.on_next_v_logits_buf, self.ptrs.on_next_b_logits_buf,
            0u64, // mag_concat_ptr: DDQN argmax pass — no magnitude conditioning
            0u64, // dir_qaux_concat_ptr: SP14 EGF disabled — DDQN argmax runs on
                  // next_states (no aux head was forwarded for next_states); the
                  // online direction head's SGEMM falls back to K=SH2 against
                  // B.8-widened `w_b0fc`. The argmax over branch_0 (direction)
                  // is consumed downstream in target evaluation (line ~25741); the
                  // resulting one-step bias on direction-argmax is the residual
                  // accepted by the spec for diagnostic-only paths.
        )?;
        Ok(())
    }


    /// SP22 H6 Phase 3 α Step 8 (2026-05-13): dW kernel launcher.
    /// Reads scratch buffers populated by c51_loss_kernel forward
    /// (aux_target_a_dir + aux_proj_logdiff_dir for d == 0 only) and
    /// writes dw_aux_buf[4] via per-action block tree-reduce. No
    /// atomicAdd per pearl_no_atomicadd.
    ///
    /// Graph-capture-safe: all kernel args are device pointers (the
    /// scratch buffers / params buffer / state buffers are stable across
    /// captured replays; their CONTENTS update each step via the captured
    /// forward and the externally-written batch upload).
    fn launch_c51_aux_dw(&self) -> Result<(), MLError> {
        let b = self.config.batch_size as i32;
        let b0 = self.config.branch_0_size as i32;
        let b1 = self.config.branch_1_size as i32;
        let b2 = self.config.branch_2_size as i32;
        let b3 = self.config.branch_3_size as i32;
        let aux_dir_prob_index = ml_core::state_layout::AUX_DIR_PROB_INDEX as i32;
        let state_dim_i32 = ml_core::state_layout::STATE_DIM as i32;

        let aux_target_a_ptr = self.aux_target_a_dir_buf.raw_ptr();
        let aux_proj_logdiff_ptr = self.aux_proj_logdiff_dir_buf.raw_ptr();
        let actions_ptr = self.ptrs.actions_buf;
        let is_weights_ptr = self.ptrs.is_weights_buf;
        let dones_ptr = self.ptrs.dones_buf;
        let gamma_buf_ptr = self.gamma_buf.raw_ptr();
        let per_sample_support_ptr = self.per_sample_support_ptr;
        let states_ptr = self.ptrs.states_buf;
        let next_states_ptr = self.ptrs.next_states_buf;
        let dw_aux_ptr = self.dw_aux_buf.raw_ptr();

        unsafe {
            self.stream
                .launch_builder(&self.c51_aux_dw_kernel)
                .arg(&aux_target_a_ptr)
                .arg(&aux_proj_logdiff_ptr)
                .arg(&actions_ptr)
                .arg(&is_weights_ptr)
                .arg(&dones_ptr)
                .arg(&gamma_buf_ptr)
                .arg(&per_sample_support_ptr)
                .arg(&states_ptr)
                .arg(&next_states_ptr)
                .arg(&dw_aux_ptr)
                .arg(&b)
                .arg(&b0)
                .arg(&b1)
                .arg(&b2)
                .arg(&b3)
                .arg(&aux_dir_prob_index)
                .arg(&state_dim_i32)
                .launch(LaunchConfig {
                    grid_dim: (4, 1, 1),    // one block per W index (b0_size=4)
                    block_dim: (256, 1, 1), // tree-reduce across batch
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!(
                    "sp22-h6-phase3 α Step 8: c51_aux_dw_kernel launch: {e}"
                )))?;
        }
        Ok(())
    }

    /// SP22 H6 Phase 3 α Step 11 (2026-05-13): Adam-update launcher for
    /// `w_aux_to_q_dir [4]`. Reads `dw_aux_buf` (written by
    /// `launch_c51_aux_dw`) and updates W in-place using Adam moments.
    ///
    /// Graph-capture-safe: lr and step counter are read via
    /// device-mapped pinned pointers (matching the main Adam pattern).
    /// beta1/beta2/eps are constants from sp5_isv_slots — host-stable.
    fn launch_adam_w_aux(&self) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp5_isv_slots::{
            ADAM_BETA1_BASE, ADAM_BETA2_BASE, ADAM_EPS_BASE,
        };

        let w_ptr = self.w_aux_to_q_dir.raw_ptr();
        let dw_ptr = self.dw_aux_buf.raw_ptr();
        let m_ptr = self.adam_m_w_aux.raw_ptr();
        let v_ptr = self.adam_v_w_aux.raw_ptr();
        let lr_ptr = self.lr_dev_ptr;
        let t_ptr = self.ptrs.t_buf;
        let beta1 = ADAM_BETA1_BASE as f32;
        let beta2 = ADAM_BETA2_BASE as f32;
        let eps = ADAM_EPS_BASE as f32;

        unsafe {
            self.stream
                .launch_builder(&self.adam_w_aux_kernel)
                .arg(&w_ptr)
                .arg(&dw_ptr)
                .arg(&m_ptr)
                .arg(&v_ptr)
                .arg(&lr_ptr)
                .arg(&beta1)
                .arg(&beta2)
                .arg(&eps)
                .arg(&t_ptr)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (4, 1, 1),  // one thread per W slot
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!(
                    "sp22-h6-phase3 α Step 11: adam_w_aux_kernel launch: {e}"
                )))?;
        }
        Ok(())
    }

    /// Submit the optimizer phase ops to the stream (used by adam_child capture).
    ///
    /// Steps: Adam → unflatten.
    /// Reads `grad_buf` which may contain combined C51 + IQN + ensemble gradients.
    /// grad_norm is computed by FusedTrainingCtx before adam_child.
    pub(crate) fn submit_adam_ops(
        &mut self,
        online_d: &DuelingWeightSet,
        online_b: &BranchingWeightSet,
    ) -> Result<(), MLError> {

        // ── 6. Adam update (f32 master weights) ─────────────────
        self.launch_adam_update()?;

        // ── 6a. SP22 H6 Phase 3 α Step 8 + Step 11 (2026-05-13) ──
        // RESTORED after bisect cut #1 confirmed backward (this pair) was
        // the NaN source. Defensive guards added:
        //   - c51_aux_dw_kernel: clamp NaN/inf dW total to 0 at write
        //   - adam_w_aux_kernel: skip update if any input is non-finite
        //   - compute_expected_q: guard W reads with isfinite (already
        //     guards state_121 reads)
        // Together these prevent any NaN propagation through atom-shift
        // regardless of where the upstream NaN originated.
        self.launch_c51_aux_dw()?;
        self.launch_adam_w_aux()?;

        // ── 6.5. Snapshot grad_buf → prev_grad_buf for next step's vaccine comparison ──
        // Graph-safe: submit_adam_ops is captured in adam_update child graph.
        self.graph_safe_copy_f32(
            self.prev_grad_buf.raw_ptr(),
            self.ptrs.grad_buf,
            self.total_params * std::mem::size_of::<f32>(),
            "grad→prev_grad",
        )?;

        // ── 7. Unflatten: params_buf → individual f32 weight tensors ─
        self.unflatten_online_weights(online_d, online_b)?;

        Ok(())
    }

    /// Submit post-aux ops for child graph capture (unconditional, every step).
    /// Selectivity fwd+bwd+adam, denoise fwd+bwd+adam, risk SGD, multi-horizon value.
    pub(crate) fn submit_post_aux_ops(&mut self, batch_size: usize) -> Result<(), MLError> {
        // Selectivity: forward + backward + adam
        self.launch_selectivity_forward(batch_size)?;
        self.launch_selectivity_backward(batch_size, 1.0)?;
        self.step_selectivity_adam()?;

        // Denoise: target + backward + adam
        self.compute_denoise_target_q(batch_size)?;
        self.launch_q_denoise_backward_cublas(batch_size)?;
        self.step_denoise_adam()?;

        // Risk SGD (uses scale_f32_kernel)
        self.step_risk_sgd()?;

        // Multi-horizon value forward (degenerate DtoD copy)
        self.multi_horizon_value_forward(batch_size)?;

        // NaN diagnostic — runs every step, flags persist across steps so the
        // first buffer to go NaN remains visible when host-side guard halts.
        // Reset is host-side at fold boundary (`reset_nan_flags()` from
        // `reset_for_fold`). Pre-forward checks (flags 4-5) check params_buf
        // which is updated by adam; checking here = checking the params Adam
        // just produced (== params the NEXT step's forward will use).
        self.run_nan_checks_pre_forward()?;
        self.run_nan_checks_post_forward(batch_size)?;

        Ok(())
    }

    /// Submit maintenance ops for child graph capture (unconditional, every step).
    /// Causal intervention + gradient vaccine using prev_grad_buf.
    pub(crate) fn submit_maintenance_ops(&mut self) -> Result<(), MLError> {
        // Causal intervention (unconditional — no step % interval guard)
        self.run_causal_intervention_unconditional()?;

        // Vaccine: compare current grad_buf against prev_grad_buf
        self.apply_gradient_vaccine_from_prev()?;

        Ok(())
    }

    /// Signal that graph re-capture is needed.
    ///
    /// Call after:
    /// - Target network EMA update (target weight pointers may change)
    /// - Learning rate schedule change (lr is baked into the graph)
    /// - Any external weight modification
    ///
    /// The actual graph ownership is in FusedTrainingCtx. This flag tells
    /// FusedTrainingCtx to re-capture on next step.
    pub fn invalidate_training_graph(&mut self) {
        self.graphs_captured = false;
        self.params_initialized = false;
    }


    /// Set the C51 blend factor directly for gradual ramp.
    ///
    /// `alpha = 0.0`: pure MSE loss (warmup).
    /// `alpha = 1.0`: pure C51 distributional loss (converged).
    /// Intermediate values blend: `grad = alpha*C51 + (1-alpha)*MSE`.
    ///
    /// Set the C51 blend alpha. No graph invalidation — c51_alpha is only used
    /// at graph capture time. If c51_warmup_epochs is 0, alpha is constant and
    /// the captured blend is correct for the entire training run. graph_forward
    /// persists across folds for deterministic eval replay.
    pub fn set_c51_alpha(&mut self, alpha: f32) {
        self.c51_alpha = alpha.clamp(0.0, 1.0);
    }

    /// Blend MSE and C51 loss values with NaN-safe alpha guard.
    ///
    /// During warmup (alpha near 0), C51 loss may be NaN from random logits.
    /// IEEE 754: `0.0 * NaN = NaN`, so we must guard against it.
    /// When alpha < epsilon: return pure MSE (ignore potentially-NaN C51).
    /// When alpha > 1-epsilon: return pure C51 (ignore MSE).
    /// Otherwise: linear blend `(1-alpha)*mse + alpha*c51`.
    #[inline]
    fn blend_loss(alpha: f32, mse_loss: f32, c51_loss: f32) -> f32 {
        if alpha < 1e-6 {
            mse_loss
        } else if alpha > 1.0 - 1e-6 {
            c51_loss
        } else {
            (1.0 - alpha) * mse_loss + alpha * c51_loss
        }
    }


    /// Current C51 blend factor (0.0 = pure MSE, 1.0 = pure C51).
    pub fn c51_alpha(&self) -> f32 {
        self.c51_alpha
    }

    /// Invalidate cached state after external weight modifications.
    ///
    /// Call after target network EMA update or any manual weight change.
    /// Forces re-flatten on the next `train_step()`.
    pub fn invalidate_params(&mut self) {
        self.invalidate_training_graph();
    }


    /// Upload batch data from GPU tensors — pure CudaSlice, zero Candle temporaries.
    ///
    /// GpuBatch layout from GpuReplayBuffer:
    /// - states/next_states: F32 (stored in F32 ring buffer)
    /// - rewards/dones: F32 GpuTensor
    /// - weights: CudaSlice<f32> (IS-weights)
    /// - actions: U32 (stored in U32 ring buffer)
    ///
    /// No Candle `to_dtype()` temporaries — eliminates Drop/event conflicts with forked stream.
    pub(crate) fn upload_batch_gpu(
        &mut self,
        gpu_batch: &crate::dqn::replay_buffer_type::GpuBatch,
    ) -> Result<(), MLError> {
        let b = gpu_batch.batch_size;

        let f32_size = std::mem::size_of::<f32>();

        // States + next_states: contiguous [B, SD] in GpuBatch → padded [B, pad128(SD)]
        self.launch_pad_states(
            self.states_buf.raw_ptr(),
            gpu_batch.states_ptr,
            b,
        )?;
        self.launch_pad_states(
            self.next_states_buf.raw_ptr(),
            gpu_batch.next_states_ptr,
            b,
        )?;

        // Rewards, dones: f32 DtoD
        dtod_copy_async(
            self.rewards_buf.raw_ptr(),
            gpu_batch.rewards_ptr,
            b * f32_size, &self.stream, 2, "rewards",
        )?;
        dtod_copy_async(
            self.dones_buf.raw_ptr(),
            gpu_batch.dones_ptr,
            b * f32_size, &self.stream, 3, "dones",
        )?;

        // IS-weights: f32 DtoD
        dtod_copy_async(
            self.is_weights_buf.raw_ptr(),
            gpu_batch.weights_ptr,
            b * f32_size, &self.stream, 4, "is_weights",
        )?;

        // Actions: CudaSlice<i32> → i32 buf (async DtoD, zero CPU sync)
        dtod_copy_async(
            self.actions_buf.raw_ptr(), gpu_batch.actions_ptr,
            b * std::mem::size_of::<i32>(), &self.stream, 4, "actions",
        )?;

        Ok(())
    }

    // ═══════════════════════════════════════════════════════════════════
    // Kernel launch methods
    // ═══════════════════════════════════════════════════════════════════

    /// Launch the cuBLAS batched forward pass (online + target networks).
    ///
    /// Replaces `launch_forward_loss` for the forward phase only.
    /// The C51 distributional loss is still computed by the existing NVRTC kernel
    /// which reads the cuBLAS output buffers (`on_v_logits_buf`, `on_b_logits_buf`,
    /// `tg_v_logits_buf`, `tg_b_logits_buf`) together with the saved activations.
    ///
    /// This raises GPU occupancy from ~1.56% to >60% on H100 by replacing the
    /// 1-warp-per-sample shared-memory-bound kernel with cuBLAS DGEMM which
    /// uses tensor cores and efficiently tiles across the full SM.
    ///
    /// Returns `Ok(false)` if cuBLAS is not initialized.
    pub(crate) fn launch_cublas_forward(&mut self) -> Result<(), MLError> {
        // Compute 20 raw device pointers from CachedPtrs (no device_ptr calls — graph-safe).
        let param_sizes = compute_param_sizes(&self.config);
        let on_w_ptrs = f32_weight_ptrs_from_base(self.ptrs.params_ptr, &param_sizes);
        let tg_w_ptrs = f32_weight_ptrs_from_base(self.ptrs.target_ptr, &param_sizes);

        // ── Plan 4 Task 1B-iii: VSN feature-selection (online-on-states) ──
        // Per-group importance gating runs BEFORE the bottleneck Linear so all
        // downstream consumers (bottleneck Linear, GRN encoder via bn_concat,
        // OFI concat for branches 2/3, mag_concat reading save_h_s2 — h_s2 is
        // post-VSN through the trunk) read the gated state through unchanged
        // ldb math (vsn_gated_states_buf has identical [B, STATE_DIM_PADDED]
        // shape and stride as states_buf). The online pass SAVES vsn_logits /
        // vsn_mask / per-group h1 buffers for 1B-iv backward; producer for the
        // ISV[VSN_MASK_GROUP_*_EMA] slots reads vsn_mask_buf only.
        let on_h1_save_ptrs: [u64; ml_core::state_layout::SL_NUM_FEATURE_GROUPS] = [
            self.vsn_save_h1_g0_buf.raw_ptr(),
            self.vsn_save_h1_g1_buf.raw_ptr(),
            self.vsn_save_h1_g2_buf.raw_ptr(),
            self.vsn_save_h1_g3_buf.raw_ptr(),
            self.vsn_save_h1_g4_buf.raw_ptr(),
            self.vsn_save_h1_g5_buf.raw_ptr(),
        ];
        let states_for_bn = self.vsn_gated_states_buf.raw_ptr();
        self.cublas_forward.vsn_forward(
            &self.stream,
            self.ptrs.states_buf,
            states_for_bn,
            &on_w_ptrs,
            self.vsn_logits_buf.raw_ptr(),
            self.vsn_mask_buf.raw_ptr(),
            &on_h1_save_ptrs,
            self.vsn_linear2_scratch_buf.raw_ptr(),
            self.vsn_group_begins_buf.raw_ptr(),
            self.vsn_group_ends_buf.raw_ptr(),
            self.config.batch_size,
        )?;

        // ── #31 Bottleneck: compress market features → 2D before Q-network ──
        // When bottleneck_dim > 0:
        //   1. GemmEx: states[B, market_dim] @ w_bn[bn_dim, market_dim]^T → h_bn[B, bn_dim]
        //   2. Tanh + concat: [tanh(h_bn), portfolio_features] → bn_concat[B, bn_dim + portfolio_dim]
        //   3. Pass bn_concat as "states" to h_s1 (which expects [B, s1_input_dim])
        // Plan 4 Task 1B-iii: input is `states_for_bn` (= vsn_gated_states_buf),
        // not the raw `states_buf` — the VSN forward gates per-group features
        // before bottleneck consumes them.
        let s1_input_ptr = if self.config.bottleneck_dim > 0 {
            let bn_dim = self.config.bottleneck_dim;
            let b = self.config.batch_size;
            let market_dim = self.config.market_dim;
            let portfolio_dim = ml_core::state_layout::STATE_DIM.saturating_sub(market_dim);
            // SP15 Wave 4.1b: trailing +1 column = dd_pct (broadcast from
            // ISV[DD_PCT_INDEX=406] by `bn_tanh_concat_dd_kernel`).
            let concat_dim = bn_dim + portfolio_dim + 1;

            // Bottleneck GEMM: states[B, market_dim] @ w_bn^T → h_bn[B, bn_dim]
            // w_bn is at on_w_ptrs[33], b_bn is at on_w_ptrs[34] (post Plan 4
            // Task 2c.3a, the GRN trunk reshuffle moved every legacy
            // tensor-index ≥ 4 by +9 — these two runtime sites were missed in
            // that migration; 24/25 in the post-2c.3a layout point at b_b1out
            // / w_b2fc which would silently corrupt the bottleneck weights).
            let bn_hidden_ptr = self.bn_hidden_buf.raw_ptr();
            let state_dim_padded_usize = self.cublas_forward.state_dim_padded;
            self.cublas_forward.sgemm_f32_ldb(
                &self.stream,
                on_w_ptrs[33],         // w_bn [bn_dim, market_dim]
                states_for_bn,         // VSN-gated states [B, state_dim_padded]
                bn_hidden_ptr,         // h_bn [B, bn_dim]
                bn_dim, b, market_dim,
                state_dim_padded_usize, // ldb = padded state_dim
                "h_bn",
            )?;
            // Add bias (on_w_ptrs[34]) — no ReLU, tanh applied by concat kernel
            self.cublas_forward.launch_add_bias_f32_raw(
                &self.stream, bn_hidden_ptr, on_w_ptrs[34], bn_dim, b,
            )?;

            // Tanh + concat + dd_pct: [tanh(h_bn), portfolio_from_states,
            // isv[DD_PCT_INDEX]] → bn_concat. Plan 4 Task 1B-iii: portfolio
            // passthrough source is the gated state — keeps bottleneck and
            // portfolio halves in sync.
            // SP15 Wave 4.1b: dd_pct broadcast comes from the same ISV bus
            // every other consumer reads; experience_collector's
            // `launch_sp15_dd_state` populates ISV[406] in the per-step env
            // loop, so the trunk forward sees a fresh value here.
            let concat_ptr = self.bn_concat_buf.raw_ptr();
            let state_dim_padded_i32 = state_dim_padded_usize as i32;
            let bn_dim_i32 = bn_dim as i32;
            let market_dim_i32 = market_dim as i32;
            let concat_dim_i32 = concat_dim as i32;
            let b_i32 = b as i32;

            launch_sp15_bn_concat_dd(
                &self.stream,
                &self.bn_tanh_concat_dd_kernel,
                bn_hidden_ptr,
                states_for_bn,
                self.isv_signals_dev_ptr,
                concat_ptr,
                b_i32,
                bn_dim_i32,
                market_dim_i32,
                state_dim_padded_i32,
                concat_dim_i32,
            )?;
            concat_ptr
        } else {
            // No bottleneck: bypass directly to the gated state. Downstream
            // GRN encoder reads `s1_input_ptr` and receives VSN-gated features.
            states_for_bn
        };

        // ── Build mag_concat from previous step's direction logits (one-step lag).
        // At step 0, logit buffers are zero-init → mag_concat = [h_s2; 0,0,0].
        // From step 1+, uses the previous step's direction Q-values.
        // This call is captured in the CUDA graph — on replay it reads the logits
        // from the previous graph execution (still in the buffer).
        self.launch_mag_concat_from(self.ptrs.save_h_s2, self.ptrs.on_v_logits_buf, self.ptrs.on_b_logits_buf)?;
        // ── SP14 B.9 (2026-05-05): direction Q-head input concat (one-step lag)
        // [save_h_s2 | aux_softmax_diff] → sp14_dir_qaux_concat_scratch [B, SH2+1].
        // Mirrors mag_concat semantics: aux_heads_forward runs AFTER this forward
        // (line ~25526), so this consumes the PREVIOUS step's aux_nb_softmax_buf.
        // Step 0 sees alloc_zeros (uniform 0.5/0.5 → diff = 0); step 1+ sees the
        // prior step's aux next-bar predictions. Closes the latent SGEMM K-mismatch
        // left by B.8 (the direction Q-head's `w_b0fc` weight grew SH2 → SH2+1
        // but its consumer kept K=SH2 until this commit).
        self.launch_sp14_dir_concat_qaux(self.ptrs.save_h_s2)?;
        // ── Build OFI concat for order (d=2) and urgency (d=3) branches (one-step lag).
        // Plan 4 Task 1B-iii: OFI features sit inside the VSN-gated `ofi`
        // group (indices 42..74) so the order/urgency branches see the gated
        // values — pass `states_for_bn` (= vsn_gated_states_buf) instead of
        // raw `states_buf` to keep the VSN gate active across all branch
        // input paths. The `vsn_masked` portion is updated inside
        // `launch_vsn_glu_branch` after the trunk's VSN runs.
        self.launch_concat_ofi(2, states_for_bn)?;
        self.launch_concat_ofi(3, states_for_bn)?;

        // ── Pass 1: Online forward on STATES (graph-safe: CachedPtrs, no device_ptr)
        // When bottleneck is active, s1_input_ptr = bn_concat (compressed features).
        // When disabled, s1_input_ptr = states_buf (original features).
        // SP14 B.9: pass `sp14_dir_qaux_concat_scratch` so the direction Q-head's
        // first FC SGEMM consumes [B, SH2+1] (matching B.8's widened `w_b0fc`).
        self.cublas_forward.forward_online_raw(
            &self.stream, s1_input_ptr, &on_w_ptrs,
            self.ptrs.save_h_s1, self.ptrs.save_h_s2, self.ptrs.save_h_v,
            self.ptrs.save_h_b0, self.ptrs.save_h_b1, self.ptrs.save_h_b2, self.ptrs.save_h_b3,
            self.ptrs.on_v_logits_buf, self.ptrs.on_b_logits_buf,
            self.ptrs.mag_concat_buf,
            self.sp14_dir_qaux_concat_scratch.raw_ptr(),
        )?;

        // ── Phase 3 T3.1–T3.3: MoE forward ─────────────────────────────────
        // Runs AFTER forward_online_raw (which produces save_h_s1 and the
        // pre-MoE save_h_s2) and BEFORE aux_heads_forward (which reads
        // save_h_s2). Overwrites save_h_s2 with the MoE mixture output
        // so all downstream consumers (aux heads, stochastic depth, loss
        // kernels) see the MoE-enriched representation.
        self.launch_moe_forward()?;

        // ── Plan 4 Task 6 Commit B: aux-heads forward (online-only) ──
        // Runs the next-bar regression + 5-class regime CE heads off the
        // freshly-saved `save_h_s2` activation. MUST run BEFORE stochastic
        // depth (which scales `save_h_s2` in-place); the aux backward
        // re-reads `save_h_s2` from the SAME buffer in
        // `launch_cublas_backward_to`, so the aux forward + backward see the
        // pre-dropout activation symmetrically. Stochastic depth still
        // applies to the trunk's downstream branch consumers; aux heads are
        // an off-branch supervised loss family per spec §4.E.6.
        self.aux_heads_forward()?;

        // ── #21 Stochastic depth: scale hidden activations by per-layer mask ──
        // Only applied to Pass 1 (online on current states). Target and Double-DQN
        // passes use full network (no stochastic depth at inference).
        // Stochastic depth always active
        {
            let b = self.config.batch_size;
            let scale_ptr = self.stochastic_depth_scale_buf.raw_ptr();
            let layers: [(u64, usize); 3] = [
                (self.ptrs.save_h_s1, self.config.shared_h1),  // layer 0: first shared hidden
                (self.ptrs.save_h_s2, self.config.shared_h2),  // layer 1: second shared hidden
                (self.ptrs.save_h_v,  self.config.value_h),    // layer 2: value head hidden
            ];
            for (layer_idx, &(buf_ptr, dim)) in layers.iter().enumerate() {
                let n = (b * dim) as i32;
                let blocks = ((n as u32 + 255) / 256) as u32;
                let li = layer_idx as i32;
                unsafe {
                    self.stream
                        .launch_builder(&self.stochastic_depth_kernel)
                        .arg(&buf_ptr)
                        .arg(&scale_ptr)
                        .arg(&li)
                        .arg(&n)
                        .launch(LaunchConfig {
                            grid_dim: (blocks, 1, 1),
                            block_dim: (256, 1, 1),
                            shared_mem_bytes: 0,
                        })
                        .map_err(|e| MLError::ModelError(format!(
                            "stochastic_depth layer {layer_idx}: {e}"
                        )))?;
                }
            }
        }

        // ── Pass 2: Target forward on NEXT_STATES — main stream (graph-safe)
        // No event sync needed — Pass 3 (Double DQN) is submitted separately
        // via submit_forward_ops_ddqn() and captured on its own stream/graph.
        // Build target mag_concat from previous step's target logits (one-step lag).
        // Reuses the same mag_concat_buf — target forward runs after online forward.
        //
        // SP22 H6 Phase 3 α atom-shift: target-side mag_concat reads
        // `next_state_121` for the per-action shift `eq_a += W[a]*next_state_121`.
        // Online callers use launch_mag_concat_from (defaults to states_buf).
        self.launch_mag_concat_from_with_state(
            self.ptrs.save_h_s2,
            self.ptrs.tg_v_logits_buf,
            self.ptrs.tg_b_logits_buf,
            self.ptrs.next_states_buf,
        )?;

        // ── Plan 4 Task 1B-iii: VSN feature-selection (target-on-next_states) ──
        // Uses `tg_w_ptrs` (Polyak EMA copy of the online VSN weights at
        // indices [95..119) — the existing target-EMA path covers all VSN
        // tensors automatically because `m_buf`/`v_buf`/`target_params_buf`
        // are sized to `total_params + cutlass_tile_pad` and the Polyak loop
        // iterates the full param length). Writes throwaway scratch logits +
        // mask buffers — the producer-only `vsn_mask_ema_update` ISV kernel
        // reads `vsn_mask_buf` (online pass) only. The 6 per-group h1
        // outputs share `vsn_h1_inference_scratch` (overwrite-per-group is
        // fine since target is inference-only — no save-for-backward).
        let tg_h1_inf_scratch_ptrs: [u64; ml_core::state_layout::SL_NUM_FEATURE_GROUPS] = {
            let p = self.vsn_h1_inference_scratch.raw_ptr();
            [p, p, p, p, p, p]
        };
        let tg_states_for_bn = self.vsn_gated_next_states_buf.raw_ptr();
        self.cublas_forward.vsn_forward(
            &self.stream,
            self.ptrs.next_states_buf,
            tg_states_for_bn,
            &tg_w_ptrs,
            self.vsn_logits_target_scratch.raw_ptr(),
            self.vsn_mask_target_scratch.raw_ptr(),
            &tg_h1_inf_scratch_ptrs,
            self.vsn_linear2_scratch_buf.raw_ptr(),
            self.vsn_group_begins_buf.raw_ptr(),
            self.vsn_group_ends_buf.raw_ptr(),
            self.config.batch_size,
        )?;

        // OFI concat for target forward (branches 2 and 3) — reuses ord/urg concat buffers
        // with VSN-gated next_states features (target forward sees gated next_states).
        self.launch_concat_ofi(2, tg_states_for_bn)?;
        self.launch_concat_ofi(3, tg_states_for_bn)?;

        // Target-net bottleneck: mirror of the online block above. Without this,
        // the target encoder reads raw `next_states_buf[0..s1_input_dim)` =
        // `[market | ofi | tlob | mtf]` while the online encoder reads
        // `[bn_market_proj | portfolio]` from `bn_concat`. The two paths must
        // share encoder-input semantics for the TD target Q-values to be
        // comparable to the online Q-values. Uses target weights
        // `tg_w_ptrs[33]` / `[34]` (Polyak EMA of online's `w_bn` / `b_bn`).
        // Plan 4 Task 1B-iii: input is `tg_states_for_bn` (= vsn_gated_next_states_buf).
        let tg_s1_input_ptr = if self.config.bottleneck_dim > 0 {
            let bn_dim = self.config.bottleneck_dim;
            let b = self.config.batch_size;
            let market_dim = self.config.market_dim;
            let portfolio_dim = ml_core::state_layout::STATE_DIM.saturating_sub(market_dim);
            // SP15 Wave 4.1b: trailing +1 column = dd_pct (broadcast from
            // ISV[DD_PCT_INDEX=406] by `bn_tanh_concat_dd_kernel`). Target
            // forward reads the SAME ISV bus the online forward read on the
            // current step, so both networks see the same dd_pct context for
            // their respective state inputs (next_states for target, states
            // for online). dd_pct is a global episode statistic — not
            // per-state — so the broadcast value is correct for both.
            let concat_dim = bn_dim + portfolio_dim + 1;
            let tg_bn_hidden_ptr = self.tg_bn_hidden_buf.raw_ptr();
            let state_dim_padded_usize = self.cublas_forward.state_dim_padded;
            self.cublas_forward.sgemm_f32_ldb(
                &self.stream,
                tg_w_ptrs[33],                  // tg_w_bn [bn_dim, market_dim]
                tg_states_for_bn,               // VSN-gated next_states [B, state_dim_padded]
                tg_bn_hidden_ptr,               // tg_h_bn [B, bn_dim]
                bn_dim, b, market_dim,
                state_dim_padded_usize,
                "tg_h_bn",
            )?;
            self.cublas_forward.launch_add_bias_f32_raw(
                &self.stream, tg_bn_hidden_ptr, tg_w_ptrs[34], bn_dim, b,
            )?;
            let tg_concat_ptr = self.tg_bn_concat_buf.raw_ptr();
            let state_dim_padded_i32 = state_dim_padded_usize as i32;
            let bn_dim_i32 = bn_dim as i32;
            let market_dim_i32 = market_dim as i32;
            let concat_dim_i32 = concat_dim as i32;
            let b_i32 = b as i32;
            launch_sp15_bn_concat_dd(
                &self.stream,
                &self.bn_tanh_concat_dd_kernel,
                tg_bn_hidden_ptr,
                tg_states_for_bn,
                self.isv_signals_dev_ptr,
                tg_concat_ptr,
                b_i32,
                bn_dim_i32,
                market_dim_i32,
                state_dim_padded_i32,
                concat_dim_i32,
            )?;
            tg_concat_ptr
        } else {
            // No bottleneck: pass VSN-gated next_states directly to target encoder.
            tg_states_for_bn
        };

        // ── SP14 B.9 (2026-05-05): target direction Q-head input concat
        // (one-step lag). Rebuilds [tg_h_s2 | aux_softmax_diff] from THIS step's
        // target trunk output (`tg_h_s2_buf`, just produced by the target encoder
        // above) and the same one-step-lagged `aux_nb_softmax_buf` the online
        // forward used. Aux head only runs on current states (online), so the
        // target sees the same lagged aux signal — symmetric one-step lag
        // semantic across online/target. Reuses the same
        // `sp14_dir_qaux_concat_scratch` buffer; the online forward consumed it
        // earlier in the captured graph (branch streams join back to main before
        // this point via `branch_done_events`), so the overwrite is safe.
        self.launch_sp14_dir_concat_qaux(self.ptrs.tg_h_s2_buf)?;
        self.cublas_forward.forward_target_raw(
            &self.stream, tg_s1_input_ptr, &tg_w_ptrs,
            self.ptrs.tg_h_s1_scratch, self.ptrs.tg_h_s2_buf,
            self.ptrs.tg_h_v_scratch,
            self.ptrs.tg_h_b0_scratch, self.ptrs.tg_h_b1_scratch,
            self.ptrs.tg_h_b2_scratch, self.ptrs.tg_h_b3_scratch,
            self.ptrs.tg_v_logits_buf, self.ptrs.tg_b_logits_buf,
            self.ptrs.mag_concat_buf,
            self.sp14_dir_qaux_concat_scratch.raw_ptr(),
        )?;

        Ok(())
    }

    /// Set curiosity weight raw pointers from the experience collector's `CuriosityWeightSet`.
    ///
    /// Must be called once after the experience collector is initialized. The pointers
    /// are stable (same CudaSlice addresses) — only values change via in-place update.
    /// After this call, the curiosity inference kernel can be captured in the CUDA Graph.
    pub fn set_curiosity_weights(
        &mut self,
        w1: &CudaSlice<f32>,
        b1: &CudaSlice<f32>,
        w2: &CudaSlice<f32>,
        b2: &CudaSlice<f32>,
    ) {
        self.curiosity_w1_ptr = w1.raw_ptr();
        self.curiosity_b1_ptr = b1.raw_ptr();
        self.curiosity_w2_ptr = w2.raw_ptr();
        self.curiosity_b2_ptr = b2.raw_ptr();
        // Signal graph re-capture so next capture includes curiosity inference.
        self.graphs_captured = false;
    }

    /// Launch curiosity inference-only kernel on the current training batch.
    ///
    /// Computes per-sample MSE prediction error from the curiosity forward model
    /// and writes to `curiosity_error_buf`. Called inside the CUDA Graph capture
    /// sequence, right before the loss kernel (C51 or MSE).
    ///
    /// Requires `set_curiosity_weights()` to have been called first.
    fn launch_curiosity_inference(&self) -> Result<(), MLError> {
        if self.config.curiosity_q_penalty_lambda <= 0.0 || self.curiosity_w1_ptr == u64::MAX {
            return Ok(()); // Penalty disabled or weights not set — error buffer stays zero
        }
        let b = self.config.batch_size;
        let sd = ml_core::state_layout::STATE_DIM as i32;
        let n = b as i32;
        let cur_input = self.config.market_dim + 3;
        let cur_hidden: usize = 128;
        let cur_output = self.config.market_dim;
        let curiosity_input_buf_ptr = self.curiosity_input_buf.raw_ptr();
        let curiosity_hidden_buf_ptr = self.curiosity_hidden_buf.raw_ptr();
        let curiosity_pred_buf_ptr = self.curiosity_pred_buf.raw_ptr();
        let curiosity_error_buf_ptr = self.curiosity_error_buf.raw_ptr();

        // Step 1: Prepare input — build [B, CUR_INPUT] from states + action one-hot
        let blocks_b = ((b + 255) / 256) as u32;
        unsafe {
            self.stream
                .launch_builder(&self.curiosity_prepare_input_func)
                .arg(&self.ptrs.states_buf)
                .arg(&self.ptrs.actions_buf)
                .arg(&curiosity_input_buf_ptr)
                .arg(&n)
                .arg(&sd)
                .launch(cudarc::driver::LaunchConfig {
                    grid_dim: (blocks_b, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("curiosity_prepare_input launch: {e}")))?;
        }

        // Step 2: cuBLAS GEMM 1 — hidden[B, CUR_HIDDEN] = input[B, CUR_INPUT] @ W1^T
        self.cublas_forward.sgemm_f32(
            &self.stream,
            self.curiosity_w1_ptr,
            curiosity_input_buf_ptr,
            curiosity_hidden_buf_ptr,
            cur_hidden, b, cur_input,
            "cur_gemm1",
        )?;

        // Step 3: Bias + LeakyReLU on hidden
        let total_hidden = (b * cur_hidden) as i32;
        let cur_hidden_i32 = cur_hidden as i32;
        let blocks_hidden = ((b * cur_hidden + 255) / 256) as u32;
        unsafe {
            self.stream
                .launch_builder(&self.curiosity_bias_leaky_relu_func)
                .arg(&curiosity_hidden_buf_ptr)
                .arg(&self.curiosity_b1_ptr)
                .arg(&cur_hidden_i32)
                .arg(&total_hidden)
                .launch(cudarc::driver::LaunchConfig {
                    grid_dim: (blocks_hidden, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("curiosity_bias_leaky_relu launch: {e}")))?;
        }

        // Step 4: cuBLAS GEMM 2 — pred[B, CUR_OUTPUT] = hidden[B, CUR_HIDDEN] @ W2^T
        self.cublas_forward.sgemm_f32(
            &self.stream,
            self.curiosity_w2_ptr,
            curiosity_hidden_buf_ptr,
            curiosity_pred_buf_ptr,
            cur_output, b, cur_hidden,
            "cur_gemm2",
        )?;

        // Step 5: Bias + MSE — add b2, per-sample MSE vs next_states, clamp to 10.0
        unsafe {
            self.stream
                .launch_builder(&self.curiosity_bias_mse_func)
                .arg(&curiosity_pred_buf_ptr)
                .arg(&self.curiosity_b2_ptr)
                .arg(&self.ptrs.next_states_buf)
                .arg(&curiosity_error_buf_ptr)
                .arg(&n)
                .arg(&sd)
                .launch(cudarc::driver::LaunchConfig {
                    grid_dim: (blocks_b, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("curiosity_bias_mse launch: {e}")))?;
        }
        Ok(())
    }

    /// Launch the standalone C51 loss kernel on pre-computed cuBLAS logit outputs.
    ///
    /// Reads: on_v_logits, on_b_logits, tg_v_logits, tg_b_logits,
    ///        on_next_v_logits, on_next_b_logits (Double DQN).
    /// Writes: per_sample_loss, td_errors, total_loss, save_current_lp, save_projected.
    fn launch_c51_loss(&mut self) -> Result<(), MLError> {
        let kernel = &self.c51_loss_kernel;

        let b = self.config.batch_size;
        let na = self.config.num_atoms;
        let b0 = self.config.branch_0_size;
        let b1 = self.config.branch_1_size;
        let b2 = self.config.branch_2_size;
        let b3 = self.config.branch_3_size;

        // Extract raw u64 pointers for CudaSlice fields (eliminates cudarc overhead)
        let on_v_logits_buf_ptr = self.on_v_logits_buf.raw_ptr();
        let tg_v_logits_buf_ptr = self.tg_v_logits_buf.raw_ptr();
        let on_next_v_logits_buf_ptr = self.on_next_v_logits_buf.raw_ptr();
        let actions_buf_ptr = self.actions_buf.raw_ptr();
        let rewards_buf_ptr = self.rewards_buf.raw_ptr();
        let dones_buf_ptr = self.dones_buf.raw_ptr();
        let is_weights_buf_ptr = self.is_weights_buf.raw_ptr();
        let per_sample_loss_buf_ptr = self.per_sample_loss_buf.raw_ptr();
        let td_errors_buf_ptr = self.td_errors_buf.raw_ptr();
        let save_current_lp_ptr = self.save_current_lp.raw_ptr();
        let save_projected_ptr = self.save_projected.raw_ptr();
        let curiosity_error_buf_ptr = self.curiosity_error_buf.raw_ptr();
        let drawdown_depths_buf_ptr = self.drawdown_depths_buf.raw_ptr();
        let ensemble_std_buf_ptr = self.ensemble_std_buf.raw_ptr();
        let atom_positions_buf_ptr = self.atom_positions_buf.raw_ptr();
        let cvar_alpha_buf_ptr = self.cvar_alpha_buf.raw_ptr();

        // Extract raw pointers for online branch logits (contiguous [B, (B0+B1+B2+B3)*NA], f32)
        let f32_sz = std::mem::size_of::<f32>();
        let on_b_base = self.on_b_logits_buf.raw_ptr();
        let on_b0_ptr = on_b_base;
        let on_b1_ptr = on_b_base + (b * b0 * na * f32_sz) as u64;
        let on_b2_ptr = on_b1_ptr + (b * b1 * na * f32_sz) as u64;
        let on_b3_ptr = on_b2_ptr + (b * b2 * na * f32_sz) as u64;

        // Target branch logits (f32)
        let tg_b_base = self.tg_b_logits_buf.raw_ptr();
        let tg_b0_ptr = tg_b_base;
        let tg_b1_ptr = tg_b_base + (b * b0 * na * f32_sz) as u64;
        let tg_b2_ptr = tg_b1_ptr + (b * b1 * na * f32_sz) as u64;
        let tg_b3_ptr = tg_b2_ptr + (b * b2 * na * f32_sz) as u64;

        // Online-next branch logits (Double DQN action selector, f32)
        let on_next_b_base = self.on_next_b_logits_buf.raw_ptr();
        let on_next_b0_ptr = on_next_b_base;
        let on_next_b1_ptr = on_next_b_base + (b * b0 * na * f32_sz) as u64;
        let on_next_b2_ptr = on_next_b1_ptr + (b * b1 * na * f32_sz) as u64;
        let on_next_b3_ptr = on_next_b2_ptr + (b * b2 * na * f32_sz) as u64;

        // N-step returns: gamma_buf [B] is pre-filled by fill_gamma_buf kernel
        // (base_gamma^n * gamma_mod[0] per sample).
        let gamma_buf_ptr = self.gamma_buf.raw_ptr();
        let batch_i32 = b as i32;
        let na_i32 = na as i32;
        let b0_i32 = b0 as i32;
        let b1_i32 = b1 as i32;
        let b2_i32 = b2 as i32;
        let b3_i32 = b3 as i32;

        // Shared memory: float arrays (support + val + adv + lp + proj + current_lp + reduce)
        // Float (4 bytes/elem) for numerically stable softmax/projection
        let max_branch = b0.max(b1).max(b2).max(b3);
        let shmem_floats = na + na + max_branch * na + na + na + na + 8;
        let shmem_bytes = (shmem_floats * std::mem::size_of::<f32>()) as u32;

        // B3/G4: mirror the kernel's tau_health_factor on host for logging.
        let current_health = if self.isv_signals_pinned.is_null() {
            0.5_f32
        } else {
            unsafe { (*self.isv_signals_pinned.add(LEARNING_HEALTH_INDEX)).clamp(0.0, 1.0) }
        };
        self.last_sarsa_tau_factor = 1.0 + 5.0 * (1.0 - current_health);

        // Kernel expects f32 scalars for these lambdas; cast config f64 so
        // cudarc writes 4 bytes into the 4-byte arg slot instead of 8 (see
        // recompute_atom_positions for the same ABI trap).
        let curiosity_lambda_f32 = self.config.curiosity_q_penalty_lambda as f32;
        let spectral_lambda_f32 = self.config.spectral_decoupling_lambda as f32;

        unsafe {
            self.stream
                .launch_builder(kernel)
                // ── Online on current states (5, f32 logits) ──
                .arg(&on_v_logits_buf_ptr)
                .arg(&on_b0_ptr)
                .arg(&on_b1_ptr)
                .arg(&on_b2_ptr)
                .arg(&on_b3_ptr)
                // ── Target on next_states (5, f32 logits) ──
                .arg(&tg_v_logits_buf_ptr)
                .arg(&tg_b0_ptr)
                .arg(&tg_b1_ptr)
                .arg(&tg_b2_ptr)
                .arg(&tg_b3_ptr)
                // ── Online-next on next_states (Double DQN, 5, f32 logits) ──
                .arg(&on_next_v_logits_buf_ptr)
                .arg(&on_next_b0_ptr)
                .arg(&on_next_b1_ptr)
                .arg(&on_next_b2_ptr)
                .arg(&on_next_b3_ptr)
                // ── Batch data (4) ──
                .arg(&actions_buf_ptr)
                .arg(&rewards_buf_ptr)
                .arg(&dones_buf_ptr)
                .arg(&is_weights_buf_ptr)
                // ── Outputs (3) ──
                .arg(&per_sample_loss_buf_ptr)
                .arg(&td_errors_buf_ptr)
                .arg(&self.total_loss_dev_ptr)
                // ── Saved for backward (2) ──
                .arg(&save_current_lp_ptr)
                .arg(&save_projected_ptr)
                // ── Curiosity Q-penalty (2) ──
                .arg(&curiosity_error_buf_ptr)
                .arg(&curiosity_lambda_f32)
                // ── Config (8 — per_sample_support replaces v_min+v_max) ──
                .arg(&gamma_buf_ptr)
                .arg(&batch_i32)
                .arg(&na_i32)
                .arg(&self.per_sample_support_ptr)
                .arg(&b0_i32)
                .arg(&b1_i32)
                .arg(&b2_i32)
                .arg(&b3_i32)
                // ── #18 Asymmetric DD loss (2) ──
                .arg(&drawdown_depths_buf_ptr)
                .arg(&self.asymmetric_dd_weight)
                // ── #27 Ensemble disagreement (2) ──
                .arg(&ensemble_std_buf_ptr)
                .arg(&self.ensemble_disagreement_weight)
                // ── Spectral decoupling (1) ──
                .arg(&spectral_lambda_f32)
                // ── CVaR alpha from IQN readiness ──
                .arg(&self.iqn_readiness_dev_ptr)
                // ── Adam step counter for stochastic Expected SARSA ──
                .arg(&self.ptrs.t_buf)
                // ── Adaptive atom positions ──
                .arg(&atom_positions_buf_ptr)
                // ── Per-sample CVaR alpha from learned risk branch ──
                .arg(&cvar_alpha_buf_ptr)
                // ── B3/G4: ISV signals for health-scaled Expected SARSA temperature ──
                .arg(&self.isv_signals_dev_ptr)
                // ── SP20 Phase 5: per-sample aux_conf at sampled state ──
                // PER's `gather_f32_scalar` writes here every step (when
                // `set_trainer_aux_conf_ptr` is wired at trainer init);
                // c51 reads it for the reward gate at the Bellman target.
                // NULL-tolerant in-kernel: gate=1.0 → identity if either
                // aux_conf or isv_signals is NULL.
                .arg(&self.aux_conf_at_state_buf)
                // ── SP22 H6 Phase 3 α atom-shift (2026-05-13) ──
                // W [b0_size=4] structural prior (Step 5 init); state +
                // next_state buffers for state_121 / next_state_121
                // reads; AUX_DIR_PROB_INDEX + STATE_DIM constants.
                // NULL-safe in-kernel: any NULL → shift collapses to 0.
                .arg(&self.w_aux_to_q_dir.raw_ptr())
                .arg(&self.ptrs.states_buf)
                .arg(&self.ptrs.next_states_buf)
                .arg(&(ml_core::state_layout::AUX_DIR_PROB_INDEX as i32))
                .arg(&(ml_core::state_layout::STATE_DIM as i32))
                // ── SP22 H6 Phase 3 α Step 8 backward scratch (2026-05-13) ──
                // Per-sample scratch outputs for the W gradient kernel:
                //   aux_target_a_dir[B]   — best_next_a for d==0 per sample
                //   aux_proj_logdiff_dir[B] — projection log-diff sum per sample
                // Both populated by c51_loss_kernel forward d==0 path;
                // consumed by c51_aux_dw_kernel post-loss.
                .arg(&self.aux_target_a_dir_buf.raw_ptr())
                .arg(&self.aux_proj_logdiff_dir_buf.raw_ptr())
                .launch(LaunchConfig {
                    grid_dim: (b as u32, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: shmem_bytes,
                })
                .map_err(|e| MLError::ModelError(format!("c51_loss_batched launch: {e}")))?;
        }

        Ok(())
    }

    /// Deterministic loss reduction: sequential sum of per_sample_loss → total_loss / batch_size.
    /// Grid=(1,1,1), Block=(1,1,1). Must stay single-thread inside graph_mega on Hopper.
    ///
    /// Also emits two additional pinned scratch scalars on the same stream:
    ///   * Second launch writes batch mean |TD-error| into
    ///     `td_error_scratch_dev_ptr` (reducing `td_errors_buf` [B]).
    ///     Feeds ISV[2] (TD-error EMA in `isv_signal_update`).
    ///   * Third launch writes batch mean C51 Q-distribution variance into
    ///     `q_var_scratch_dev_ptr` (reducing `q_var_buf_trainer`
    ///     [B*total_actions]). Feeds ISV[3] and ISV[4] (variance EMA +
    ///     velocity in `isv_signal_update`). Added per ISV audit 2026-04-23
    ///     which identified slot 3 as having no writer and reading
    ///     constant-zero into the ISV encoder MLP.
    ///
    /// The reducer kernel divides by its `n` argument, so we pass
    /// `b * total_actions` for the q_var reduction to get the full-tensor mean.
    fn launch_loss_reduce(&self, total_loss_dev_ptr: u64) -> Result<(), MLError> {
        let b = self.config.batch_size as i32;
        let b_qvar = (self.config.batch_size * self.total_actions()) as i32;
        let per_sample_loss_buf_ptr = self.per_sample_loss_buf.raw_ptr();
        let td_errors_buf_ptr = self.td_errors_buf.raw_ptr();
        let q_var_buf_trainer_ptr = self.q_var_buf_trainer.raw_ptr();
        unsafe {
            self.stream
                .launch_builder(&self.c51_loss_reduce_kernel)
                .arg(&per_sample_loss_buf_ptr)
                .arg(&total_loss_dev_ptr)
                .arg(&b)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (1, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("c51_loss_reduce launch: {e}")))?;
            /* Second launch: batch mean |TD-error| into the ISV scratch.
             * Generic mean-reduction kernel, reused verbatim. */
            self.stream
                .launch_builder(&self.c51_loss_reduce_kernel)
                .arg(&td_errors_buf_ptr)
                .arg(&self.td_error_scratch_dev_ptr)
                .arg(&b)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (1, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("c51_td_error_reduce launch: {e}")))?;
            /* Third launch: batch mean C51 Q-distribution variance into the
             * ISV scratch. Feeds slot 3 EMA + slot 4 velocity. Same generic
             * reducer; n=b*12 reduces the full per-action variance tensor. */
            self.stream
                .launch_builder(&self.c51_loss_reduce_kernel)
                .arg(&q_var_buf_trainer_ptr)
                .arg(&self.q_var_scratch_dev_ptr)
                .arg(&b_qvar)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (1, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("c51_q_var_reduce launch: {e}")))?;
        }
        Ok(())
    }

    /// Launch C51 loss gradient kernel: save_current_lp, save_projected → dL/d_logits.
    ///
    /// Computes dL/d_combined = is_weights * (exp(current_lp) - projected) per branch,
    /// then routes through dueling to produce d_value_logits and d_adv_logits.
    fn launch_c51_grad(&self) -> Result<(), MLError> {
        let b = self.config.batch_size;
        let na = self.config.num_atoms;
        let b0 = self.config.branch_0_size;
        let b1 = self.config.branch_1_size;
        let b2 = self.config.branch_2_size;
        let b3 = self.config.branch_3_size;
        let total_branch_atoms = (b0 + b1 + b2 + b3) * na;

        let batch_i32 = b as i32;
        let na_i32 = na as i32;
        let b0_i32 = b0 as i32;
        let b1_i32 = b1 as i32;
        let b2_i32 = b2 as i32;
        let b3_i32 = b3 as i32;
        let total_branch_atoms_i32 = total_branch_atoms as i32;
        let base_entropy = self.config.entropy_coefficient;
        // SP4 Layer C close-out C3 (2026-05-01): `utilization_ema` storage
        // migrated to mapped-pinned slot updated GPU-side; host accessor
        // reads via host_ptr (not host compute on the EMA — purely a scalar
        // pass-through to derive a kernel-arg multiplier per the rule's
        // "mapped-pinned for cross-boundary data passage is allowed; only
        // running formulas on host is forbidden" carve-out). The adaptive-
        // entropy formula here is a host-side branch-and-multiply on a
        // mapped-pinned-read scalar to derive a single by-value kernel arg
        // — this is data-routing into the kernel, not running an EMA on the
        // host. The actual `entropy_coeff` consumer is the c51_grad kernel
        // launch below.
        let util = self.utilization_ema();
        let adaptive_entropy = if util < 0.5 {
            base_entropy * (1.0 - util / 0.5)
        } else {
            0.0
        };
        let entropy_coeff = base_entropy + adaptive_entropy;

        // Grid = B*NA (one thread per (sample, atom), loops over 4 branches).
        // Zero atomicAdd — fully deterministic gradient computation.
        let blocks = ((b * na + 255) / 256) as u32;
        let save_current_lp_ptr = self.save_current_lp.raw_ptr();
        let save_projected_ptr = self.save_projected.raw_ptr();
        let is_weights_buf_ptr = self.is_weights_buf.raw_ptr();
        let actions_buf_ptr = self.actions_buf.raw_ptr();
        let d_value_logits_buf_ptr = self.d_value_logits_buf.raw_ptr();
        let d_adv_logits_buf_ptr = self.d_adv_logits_buf.raw_ptr();
        let atom_positions_buf_ptr = self.atom_positions_buf.raw_ptr();

        // Backward-standardization fix (2026-04-23): thread the magnitude-
        // branch online advantage logits pointer through to the backward
        // kernel so it can recompute `a_std` at each (b, j) and apply the
        // same `1 / (a_std + 1e-6)` factor the forward applied. The layout
        // matches the forward: f32 stride `[B, b1_size, num_atoms]`. Offset
        // base = on_b_logits_buf + B * b0_size * num_atoms * 4 (b0 slice).
        let f32_sz = std::mem::size_of::<f32>();
        let on_b_base = self.on_b_logits_buf.raw_ptr();
        let on_adv_b1_ptr: u64 = on_b_base + (b * self.config.branch_0_size * na * f32_sz) as u64;

        unsafe {
            self.stream
                .launch_builder(&self.c51_grad_kernel)
                .arg(&save_current_lp_ptr)
                .arg(&save_projected_ptr)
                .arg(&is_weights_buf_ptr)
                .arg(&actions_buf_ptr)
                .arg(&d_value_logits_buf_ptr)
                .arg(&d_adv_logits_buf_ptr)
                .arg(&batch_i32)
                .arg(&na_i32)
                .arg(&b0_i32)
                .arg(&b1_i32)
                .arg(&b2_i32)
                .arg(&b3_i32)
                .arg(&total_branch_atoms_i32)
                .arg(&entropy_coeff)
                .arg(&self.branch_scales_ptr)
                .arg(&self.per_sample_support_ptr)
                .arg(&atom_positions_buf_ptr)
                .arg(&self.q_mean_ema_dev_ptr) // unused; preserves param-count graph node structure on Hopper
                // Task 2.X adaptive magnitude fix — ISV signal bus; kernel reads
                // slots [13] (q_mag_spread_ema) and [14] (q_dir_spread_ema) to
                // derive the magnitude-branch adaptive bin weight.
                .arg(&self.isv_signals_dev_ptr)
                // Backward-standardization fix (2026-04-23): magnitude-branch
                // advantage logits for per-(b, j) a_std recomputation in the
                // d==1 dueling-grad block. Matches forward layout.
                .arg(&on_adv_b1_ptr)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("c51_grad_kernel launch: {e}")))?;
        }
        Ok(())
    }

    /// Launch the MSE loss kernel on expected Q-values (warmup before C51).
    ///
    /// Computes MSE loss between online E[Q] and Bellman target E[Q].
    /// Writes: per_sample_loss, td_errors, total_loss, save_current_lp (online softmax probs),
    /// save_projected (repurposed: stores online E[Q] per branch for the grad kernel).
    fn launch_mse_loss(&self) -> Result<(), MLError> {
        let kernel = &self.mse_loss_kernel;

        let b = self.config.batch_size;
        let na = self.config.num_atoms;
        let b0 = self.config.branch_0_size;
        let b1 = self.config.branch_1_size;
        let b2 = self.config.branch_2_size;
        let b3 = self.config.branch_3_size;

        // Extract raw u64 pointers for CudaSlice fields (eliminates cudarc overhead)
        let on_v_logits_buf_ptr = self.on_v_logits_buf.raw_ptr();
        let tg_v_logits_buf_ptr = self.tg_v_logits_buf.raw_ptr();
        let on_next_v_logits_buf_ptr = self.on_next_v_logits_buf.raw_ptr();
        let actions_buf_ptr = self.actions_buf.raw_ptr();
        let rewards_buf_ptr = self.rewards_buf.raw_ptr();
        let dones_buf_ptr = self.dones_buf.raw_ptr();
        let is_weights_buf_ptr = self.is_weights_buf.raw_ptr();
        let per_sample_loss_buf_ptr = self.per_sample_loss_buf.raw_ptr();
        let td_errors_buf_ptr = self.td_errors_buf.raw_ptr();
        let save_current_lp_ptr = self.save_current_lp.raw_ptr();
        let save_projected_ptr = self.save_projected.raw_ptr();
        let curiosity_error_buf_ptr = self.curiosity_error_buf.raw_ptr();
        let drawdown_depths_buf_ptr = self.drawdown_depths_buf.raw_ptr();
        let ensemble_std_buf_ptr = self.ensemble_std_buf.raw_ptr();

        // Extract raw pointers for online branch logits (contiguous [B, (B0+B1+B2+B3)*NA], f32)
        let f32_sz = std::mem::size_of::<f32>();
        let on_b_base = self.on_b_logits_buf.raw_ptr();
        let on_b0_ptr = on_b_base;
        let on_b1_ptr = on_b_base + (b * b0 * na * f32_sz) as u64;
        let on_b2_ptr = on_b1_ptr + (b * b1 * na * f32_sz) as u64;
        let on_b3_ptr = on_b2_ptr + (b * b2 * na * f32_sz) as u64;

        // Target branch logits (f32)
        let tg_b_base = self.tg_b_logits_buf.raw_ptr();
        let tg_b0_ptr = tg_b_base;
        let tg_b1_ptr = tg_b_base + (b * b0 * na * f32_sz) as u64;
        let tg_b2_ptr = tg_b1_ptr + (b * b1 * na * f32_sz) as u64;
        let tg_b3_ptr = tg_b2_ptr + (b * b2 * na * f32_sz) as u64;

        // Online-next branch logits (Double DQN action selector, f32)
        let on_next_b_base = self.on_next_b_logits_buf.raw_ptr();
        let on_next_b0_ptr = on_next_b_base;
        let on_next_b1_ptr = on_next_b_base + (b * b0 * na * f32_sz) as u64;
        let on_next_b2_ptr = on_next_b1_ptr + (b * b1 * na * f32_sz) as u64;
        let on_next_b3_ptr = on_next_b2_ptr + (b * b2 * na * f32_sz) as u64;

        // N-step returns: use direction-branch gamma^n for the Bellman projection.
        // MSE loss uses a scalar gamma; direction branch is the primary actor.
        let gamma = self.read_isv_signal_at(GAMMA_DIR_EFF_INDEX).powi(self.config.n_steps as i32);
        let batch_i32 = b as i32;
        let na_i32 = na as i32;
        let b0_i32 = b0 as i32;
        let b1_i32 = b1 as i32;
        let b2_i32 = b2 as i32;
        let b3_i32 = b3 as i32;

        // Shared memory: float arrays (support + val + adv + lp + proj + current_lp + reduce)
        // Float (4 bytes/elem) for numerically stable softmax
        let max_branch = b0.max(b1).max(b2).max(b3);
        let shmem_floats = na + na + max_branch * na + na + na + na + 8;
        let shmem_bytes = (shmem_floats * std::mem::size_of::<f32>()) as u32;

        // Kernel expects f32 scalars for these lambdas; cast config f64 so
        // cudarc writes 4 bytes into the 4-byte arg slot instead of 8 (see
        // recompute_atom_positions for the same ABI trap).
        let curiosity_lambda_f32 = self.config.curiosity_q_penalty_lambda as f32;
        let spectral_lambda_f32 = self.config.spectral_decoupling_lambda as f32;

        // SP21 T2.2 Phase 3 (2026-05-11): E1 q_correction Bellman offset.
        // Read from `ISV[Q_CORRECTION_INDEX=520]` host-side (set by the
        // enrichment phase after each val pass). Cold-start = 0.0
        // sentinel → kernel applies no shift. c51_loss_batched reads
        // slot 520 from its existing `isv_signals` arg; this scalar is
        // the mse-side mirror so blended `(1-α)·MSE + α·C51` sees
        // identical Q-target shifts (`feedback_no_partial_refactor`
        // — symmetric application).
        let q_correction = {
            use crate::cuda_pipeline::sp21_isv_slots::Q_CORRECTION_INDEX;
            self.read_isv_signal_at(Q_CORRECTION_INDEX)
        };

        unsafe {
            self.stream
                .launch_builder(kernel)
                // ── Online on current states (5) ──
                .arg(&on_v_logits_buf_ptr)
                .arg(&on_b0_ptr)
                .arg(&on_b1_ptr)
                .arg(&on_b2_ptr)
                .arg(&on_b3_ptr)
                // ── Target on next_states (5) ──
                .arg(&tg_v_logits_buf_ptr)
                .arg(&tg_b0_ptr)
                .arg(&tg_b1_ptr)
                .arg(&tg_b2_ptr)
                .arg(&tg_b3_ptr)
                // ── Online-next on next_states (Double DQN, 5) ──
                .arg(&on_next_v_logits_buf_ptr)
                .arg(&on_next_b0_ptr)
                .arg(&on_next_b1_ptr)
                .arg(&on_next_b2_ptr)
                .arg(&on_next_b3_ptr)
                // ── Batch data (4) ──
                .arg(&actions_buf_ptr)
                .arg(&rewards_buf_ptr)
                .arg(&dones_buf_ptr)
                .arg(&is_weights_buf_ptr)
                // ── Outputs (3) ──
                .arg(&per_sample_loss_buf_ptr)
                .arg(&td_errors_buf_ptr)
                .arg(&self.mse_loss_dev_ptr)     // MSE writes to separate accumulator (not total_loss)
                // ── Saved for backward (2) ── repurposed: save_current_lp = softmax probs,
                // save_projected = per-branch E[Q] values (4 floats per sample per branch)
                .arg(&save_current_lp_ptr)
                .arg(&save_projected_ptr)
                // ── Curiosity Q-penalty (2) ──
                .arg(&curiosity_error_buf_ptr)
                .arg(&curiosity_lambda_f32)
                // ── Config (8 — per_sample_support replaces v_min+v_max) ──
                .arg(&gamma)
                .arg(&batch_i32)
                .arg(&na_i32)
                .arg(&self.per_sample_support_ptr)
                .arg(&b0_i32)
                .arg(&b1_i32)
                .arg(&b2_i32)
                .arg(&b3_i32)
                // ── #18 Asymmetric DD loss (2) ──
                .arg(&drawdown_depths_buf_ptr)
                .arg(&self.asymmetric_dd_weight)
                // ── #27 Ensemble disagreement (2) ──
                .arg(&ensemble_std_buf_ptr)
                .arg(&self.ensemble_disagreement_weight)
                // ── Spectral decoupling (1) ──
                .arg(&spectral_lambda_f32)
                // ── SP21 T2.2 Phase 3: E1 q_correction Bellman offset (1) ──
                .arg(&q_correction)
                .launch(LaunchConfig {
                    grid_dim: (b as u32, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: shmem_bytes,
                })
                .map_err(|e| MLError::ModelError(format!("mse_loss_batched launch: {e}")))?;
        }

        Ok(())
    }

    /// Launch MSE loss gradient kernel through softmax expectation.
    ///
    /// Reads save_current_lp (softmax probs) and save_projected (expected Q + td_error),
    /// computes d_logit_j = td_error * is_weight * p_j * (z_j - E[Q]) per branch,
    /// then routes through dueling to produce d_value_logits and d_adv_logits.
    fn launch_mse_grad(&self) -> Result<(), MLError> {
        self.launch_mse_grad_inner(&self.d_value_logits_buf, &self.d_adv_logits_buf)
    }

    /// Launch MSE grad to scratch buffers (for blended MSE+C51 loss path).
    ///
    /// Identical to `launch_mse_grad()` but writes to `d_value_logits_mse` /
    /// `d_adv_logits_mse` instead of the main gradient buffers. The caller then
    /// SAXPY-blends scratch into main: main = alpha*C51 + (1-alpha)*MSE.
    fn launch_mse_grad_to_scratch(&self) -> Result<(), MLError> {
        self.launch_mse_grad_inner(&self.d_value_logits_mse, &self.d_adv_logits_mse)
    }

    /// Shared implementation for MSE gradient kernel launch.
    ///
    /// Writes gradient outputs to the provided destination buffers.
    fn launch_mse_grad_inner(
        &self,
        d_value_dst: &CudaSlice<f32>,
        d_adv_dst: &CudaSlice<f32>,
    ) -> Result<(), MLError> {
        let b = self.config.batch_size;
        let na = self.config.num_atoms;
        let b0 = self.config.branch_0_size;
        let b1 = self.config.branch_1_size;
        let b2 = self.config.branch_2_size;
        let b3 = self.config.branch_3_size;
        let total_branch_atoms = (b0 + b1 + b2 + b3) * na;

        let batch_i32 = b as i32;
        let na_i32 = na as i32;
        let b0_i32 = b0 as i32;
        let b1_i32 = b1 as i32;
        let b2_i32 = b2 as i32;
        let b3_i32 = b3 as i32;
        let total_branch_atoms_i32 = total_branch_atoms as i32;
        let save_current_lp_ptr = self.save_current_lp.raw_ptr();
        let save_projected_ptr = self.save_projected.raw_ptr();
        let is_weights_buf_ptr = self.is_weights_buf.raw_ptr();
        let actions_buf_ptr = self.actions_buf.raw_ptr();
        let d_value_dst_ptr = d_value_dst.raw_ptr();
        let d_adv_dst_ptr = d_adv_dst.raw_ptr();

        // Grid = B*NA (deterministic: one thread per (sample, atom), loops 4 branches)
        let blocks = ((b * na + 255) / 256) as u32;

        unsafe {
            self.stream
                .launch_builder(&self.mse_grad_kernel)
                .arg(&save_current_lp_ptr)
                .arg(&save_projected_ptr)
                .arg(&is_weights_buf_ptr)
                .arg(&actions_buf_ptr)
                .arg(&d_value_dst_ptr)
                .arg(&d_adv_dst_ptr)
                .arg(&batch_i32)
                .arg(&na_i32)
                .arg(&b0_i32)
                .arg(&b1_i32)
                .arg(&b2_i32)
                .arg(&b3_i32)
                .arg(&total_branch_atoms_i32)
                .arg(&self.eval_v_range_ptr)
                .arg(&self.branch_scales_ptr)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("mse_grad_kernel launch: {e}")))?;
        }
        Ok(())
    }

    /// Copy f32 d_logits → staging buffers for cuBLAS backward GEMM.
    ///
    /// The gradient kernels write to f32 buffers (native atomicAdd, no overflow).
    /// This method copies the f32 gradients to the staging buffers (now f32→f32 identity).
    fn cast_d_logits_to_f32_staging(&self) -> Result<(), MLError> {
        let na = self.config.num_atoms;
        let b = self.config.batch_size;
        let b0 = self.config.branch_0_size;
        let b1 = self.config.branch_1_size;
        let b2 = self.config.branch_2_size;
        let b3 = self.config.branch_3_size;

        let n_val_bytes = b * pad32(na) * std::mem::size_of::<f32>();
        let n_adv_bytes = (b * (b0 + b1 + b2 + b3) * na + 32 * 4) * std::mem::size_of::<f32>();

        let val_src = self.d_value_logits_buf.raw_ptr();
        let val_dst = self.d_value_logits.raw_ptr();
        let adv_src = self.d_adv_logits_buf.raw_ptr();
        let adv_dst = self.d_adv_logits.raw_ptr();

        self.graph_safe_copy_f32(val_dst, val_src, n_val_bytes, "d_value_logits")?;
        self.graph_safe_copy_f32(adv_dst, adv_src, n_adv_bytes, "d_adv_logits")?;
        Ok(())
    }

    /// cuBLAS backward pass: chain rule through all layers.
    ///
    /// Reads dL/d_logits from f32 buffers (`d_value_logits_buf` and
    /// `d_adv_logits_buf`) directly. The entire backward
    /// chain stays in f32 to preserve gradient precision at large batch sizes.
    /// Accumulates weight gradients into `grad_buf`.
    pub(crate) fn launch_cublas_backward(&mut self) -> Result<(), MLError> {
        self.launch_cublas_backward_to(self.ptrs.grad_buf)
    }

    /// Plan 4 Task 1B-iv-ext: aux-path VSN backward chain.
    ///
    /// Closes the gradient-coverage gap left by 1B-iv (which only wired VSN
    /// backward at the main C51/MSE path). Each of the three auxiliary
    /// trunk-touching backward paths — `apply_iqn_trunk_gradient`,
    /// `apply_ensemble_diversity_backward`, and the CQL backward block —
    /// invokes this helper after `encoder_backward_chain` produces
    /// `bn_d_concat_buf` (= dL/d(bn_concat)). The helper then runs the same
    /// chain the main backward already runs, but with `vsn_dw_grad_base`
    /// pointing at the caller's per-aux VSN scratch (or directly at
    /// `cql_grad_scratch` for CQL):
    ///
    ///   1. tanh' kernel: `bn_d_hidden = bn_d_concat[:, :bn_dim] * (1 - tanh^2)`
    ///   2. portfolio passthrough + zero pad: assemble
    ///      `vsn_d_gated_state[:, market_dim..STATE_DIM_PADDED)` from
    ///      `bn_d_concat[:, bn_dim..]`. Market slice is zeroed so cuBLAS
    ///      step 3 can `beta=0`-overwrite it.
    ///   3. `launch_dx_only_lda`: `vsn_d_gated_state[:, :market_dim] =
    ///      bn_d_hidden @ W_bn` with row stride `state_dim_padded`.
    ///   4. `vsn_backward`: writes 24 dW/dB into `vsn_dw_grad_base + offsets
    ///      [95..119)` per the layout from 1B-ii.
    ///
    /// Step is a strict subset of the main-path post-1B-iv chain; the only
    /// omitted step is the bottleneck Linear's own dW/dB (indices 33/34).
    /// Bottleneck-Linear weight gradients stay at C51/MSE-only coverage in
    /// this commit — extending them would need a non-contiguous SAXPY
    /// (trunk[0..13) + bn[33..35) + VSN[95..119)), which is out of scope.
    /// VSN's coverage gap is the one identified as causing the 1B-iv smoke
    /// regression vs 1B-iii baseline.
    ///
    /// Reuses existing scratch buffers — `bn_d_hidden_buf`,
    /// `vsn_d_gated_state_buf`, `vsn_d_logits_buf`, `vsn_d_state_buf`,
    /// `vsn_d_logit_scratch_buf`, `vsn_d_h1_scratch_buf` — because the four
    /// backward paths run sequentially on the main stream (main → IQN aux →
    /// CQL → ensemble aux per `fused_training.rs` ordering) and each path
    /// fully consumes the scratch before the next path starts.
    ///
    /// `vsn_dw_grad_base` is the byte address that `padded_byte_offset(95)`
    /// is added to inside `vsn_backward`. For an aux scratch sized exactly
    /// to the VSN range, pass `scratch.raw_ptr() - padded_byte_offset(95)`
    /// so VSN tensor 95 lands at scratch offset 0. For CQL whose scratch
    /// covers all 119 tensors, pass the scratch base directly.
    /// Internal entry point that delegates to a free function operating
    /// on disjoint field borrows. The caller's `EventTrackingGuard` already
    /// borrows `self.stream` immutably, so this entry point destructures
    /// `self` into individual field references — that way the borrow
    /// checker sees that `self.cublas_forward` (mutable) and `self.stream`
    /// (immutable) are non-overlapping.
    #[allow(clippy::too_many_arguments)]
    fn aux_bottleneck_vsn_backward_dispatch(
        cublas_forward: &mut super::batched_forward::CublasForward,
        cublas_backward: &super::batched_backward::CublasBackwardSet,
        stream: &Arc<cudarc::driver::CudaStream>,
        config: &GpuDqnTrainConfig,
        bn_d_hidden_buf: &CudaSlice<f32>,
        bn_hidden_buf: &CudaSlice<f32>,
        states_buf: &CudaSlice<f32>,
        vsn_d_gated_state_buf: &CudaSlice<f32>,
        vsn_mask_buf: &CudaSlice<f32>,
        vsn_d_logits_buf: &CudaSlice<f32>,
        vsn_d_state_buf: &CudaSlice<f32>,
        vsn_d_logit_scratch_buf: &CudaSlice<f32>,
        vsn_d_h1_scratch_buf: &CudaSlice<f32>,
        vsn_group_begins_buf: &CudaSlice<i32>,
        vsn_group_ends_buf: &CudaSlice<i32>,
        vsn_save_h1_g0_buf: &CudaSlice<f32>,
        vsn_save_h1_g1_buf: &CudaSlice<f32>,
        vsn_save_h1_g2_buf: &CudaSlice<f32>,
        vsn_save_h1_g3_buf: &CudaSlice<f32>,
        vsn_save_h1_g4_buf: &CudaSlice<f32>,
        vsn_save_h1_g5_buf: &CudaSlice<f32>,
        bn_tanh_backward_kernel: &CudaFunction,
        vsn_d_gated_state_portfolio_kernel: &CudaFunction,
        vsn_logit_gather_kernel: &CudaFunction,
        scale_f32_aux_kernel: &CudaFunction,
        bn_d_concat_ptr: u64,
        vsn_dw_grad_base: u64,
        param_sizes: &[usize],
        w_ptrs: &[u64; NUM_WEIGHT_TENSORS],
    ) -> Result<(), MLError> {
        // Aux paths only run a VSN backward when the bottleneck is active —
        // otherwise the bottleneck Linear chain rule is a no-op and there's
        // no `bn_d_concat` to consume.
        if config.bottleneck_dim == 0 {
            return Ok(());
        }

        let bn_dim = config.bottleneck_dim;
        let market_dim = config.market_dim;
        let b = config.batch_size;
        let portfolio_dim = ml_core::state_layout::STATE_DIM.saturating_sub(market_dim);
        // SP15 Wave 4.1b: bn_d_concat_buf row stride is the full forward
        // concat_dim including the dd_pct column — encoder_backward_chain's
        // Linear_a/Linear_residual dX writes 103 columns. The
        // bn_tanh_backward_kernel reads only the bn_dim slice [0..bn_dim) and
        // the vsn_d_gated_state_portfolio_pad_kernel reads only the
        // portfolio slice [bn_dim..bn_dim+portfolio_dim) — both correctly
        // skip the dd_pct gradient column at index bn_dim+portfolio_dim
        // because dd_pct sources from the ISV bus (no learnable input feeds
        // it back). The discarded column's gradient is harmless: its only
        // effect would be to update a downstream variable, which doesn't
        // exist.
        let concat_dim = bn_dim + portfolio_dim + 1;
        let state_dim = ml_core::state_layout::STATE_DIM;
        let state_dim_padded = ml_core::state_layout::STATE_DIM_PADDED;

        let d_bn_ptr = bn_d_hidden_buf.raw_ptr();
        let bn_hidden_ptr = bn_hidden_buf.raw_ptr();
        let raw_states_ptr = states_buf.raw_ptr();
        let d_vsn_gated_state_ptr = vsn_d_gated_state_buf.raw_ptr();

        let b_i32 = b as i32;
        let bn_dim_i32 = bn_dim as i32;
        let concat_dim_i32 = concat_dim as i32;

        // Step 1: tanh' chain rule into bn_d_hidden.
        let total_relu = (b * bn_dim) as i32;
        let blocks_relu = ((total_relu as u32 + 255) / 256) as u32;
        unsafe {
            stream
                .launch_builder(bn_tanh_backward_kernel)
                .arg(&d_bn_ptr)
                .arg(&bn_d_concat_ptr)
                .arg(&bn_hidden_ptr)
                .arg(&b_i32)
                .arg(&bn_dim_i32)
                .arg(&concat_dim_i32)
                .launch(LaunchConfig {
                    grid_dim: (blocks_relu, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!(
                    "aux_bottleneck_vsn_backward bn_tanh_backward: {e}"
                )))?;
        }

        // Step 2: portfolio passthrough + zero pad — populate
        // d_vsn_gated_state[:, market_dim..STATE_DIM_PADDED) from
        // bn_d_concat[:, bn_dim..]; zero market slice for cuBLAS step 3.
        let market_dim_i32 = market_dim as i32;
        let state_dim_i32 = state_dim as i32;
        let state_dim_padded_i32 = state_dim_padded as i32;
        let pad_total = (b * state_dim_padded) as i32;
        let pad_blocks = ((pad_total as u32 + 255) / 256) as u32;
        unsafe {
            stream
                .launch_builder(vsn_d_gated_state_portfolio_kernel)
                .arg(&d_vsn_gated_state_ptr)
                .arg(&bn_d_concat_ptr)
                .arg(&b_i32)
                .arg(&market_dim_i32)
                .arg(&bn_dim_i32)
                .arg(&concat_dim_i32)
                .arg(&state_dim_i32)
                .arg(&state_dim_padded_i32)
                .launch(LaunchConfig {
                    grid_dim: (pad_blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!(
                    "aux_bottleneck_vsn_backward portfolio_pad: {e}"
                )))?;
        }

        // Step 3: bottleneck Linear backward dX into market slice with
        // x_lda = state_dim_padded. beta=0 overwrites the zeroed market
        // slice from step 2. Bottleneck Linear's own dW/dB is intentionally
        // skipped here (out of scope for 1B-iv-ext — see fn doc).
        let w_bn_ptr = w_ptrs[33];
        cublas_backward.launch_dx_only_lda(
            stream,
            d_bn_ptr,
            w_bn_ptr,
            d_vsn_gated_state_ptr,
            bn_dim,
            market_dim,
            state_dim_padded,
            b,
            0.0_f32,
        )?;

        // Plan 4 Task 1B-iv-rc: attenuate aux-path VSN dW signal by
        // `VSN_DW_DAMP`. The IN-PLACE scaler runs on `vsn_d_gated_state_buf`
        // BEFORE `vsn_backward` consumes it; because the softmax-and-gate
        // Jacobian + downstream Linear_2 / Linear_1 backward GEMMs are all
        // linear in `d_gated_state`, scaling the input uniformly attenuates
        // every VSN dW/dB write that follows (including writes into the
        // per-aux scratch + writes directly into `cql_grad_scratch[95..119)`).
        // Trunk SAXPYs do NOT route through this dispatcher — main-path
        // VSN backward (`launch_cublas_backward_to`) uses the same
        // `scale_f32_kernel` with `VSN_DW_DAMP` for symmetric attenuation.
        {
            let n = (b * state_dim_padded) as i32;
            let blocks = ((n as u32 + 255) / 256).max(1);
            let alpha = VSN_DW_DAMP;
            unsafe {
                stream
                    .launch_builder(scale_f32_aux_kernel)
                    .arg(&d_vsn_gated_state_ptr)
                    .arg(&alpha)
                    .arg(&n)
                    .launch(LaunchConfig {
                        grid_dim: (blocks, 1, 1),
                        block_dim: (256, 1, 1),
                        shared_mem_bytes: 0,
                    })
                    .map_err(|e| MLError::ModelError(format!(
                        "vsn dW aux attenuate: {e}"
                    )))?;
            }
        }

        // Step 4: VSN backward — writes 24 dW/dB at goff[95..119) of
        // `vsn_dw_grad_base`.
        let h1_save_ptrs: [u64; ml_core::state_layout::SL_NUM_FEATURE_GROUPS] = [
            vsn_save_h1_g0_buf.raw_ptr(),
            vsn_save_h1_g1_buf.raw_ptr(),
            vsn_save_h1_g2_buf.raw_ptr(),
            vsn_save_h1_g3_buf.raw_ptr(),
            vsn_save_h1_g4_buf.raw_ptr(),
            vsn_save_h1_g5_buf.raw_ptr(),
        ];
        cublas_forward.vsn_backward(
            stream,
            cublas_backward,
            d_vsn_gated_state_ptr,
            raw_states_ptr,
            vsn_mask_buf.raw_ptr(),
            &h1_save_ptrs,
            w_ptrs,
            vsn_dw_grad_base,
            param_sizes,
            vsn_d_logits_buf.raw_ptr(),
            vsn_d_state_buf.raw_ptr(),
            vsn_d_logit_scratch_buf.raw_ptr(),
            vsn_d_h1_scratch_buf.raw_ptr(),
            vsn_group_begins_buf.raw_ptr(),
            vsn_group_ends_buf.raw_ptr(),
            vsn_logit_gather_kernel,
            b,
        )?;

        Ok(())
    }

    /// Backward pass writing weight gradients to an arbitrary output buffer.
    /// Used by the vaccine to write g_val to scratch without touching grad_buf.
    ///
    /// Plan 4 Task 2c.3c.4: takes `&mut self` because the GRN trunk-backward
    /// chain (`encoder_backward_chain`) borrows `&mut self.cublas_forward`
    /// (the GRN block's `backward_raw_phase1`/`backward_raw_phase2` need to
    /// scribble per-block partial-reduction scratch).
    fn launch_cublas_backward_to(&mut self, grad_base: u64) -> Result<(), MLError> {
        let bw = &self.cublas_backward;

        let param_sizes = compute_param_sizes(&self.config);
        let w_ptrs = f32_weight_ptrs_from_base(self.ptrs.params_ptr, &param_sizes);

        // #31 Bottleneck: backward pass input is bn_concat, not raw states
        let states_ptr = if self.config.bottleneck_dim > 0 {
            self.bn_concat_buf.raw_ptr()
        } else {
            bw_raw_f32_ptr(&self.states_buf, &self.stream)
        };
        let raw_states_ptr = bw_raw_f32_ptr(&self.states_buf, &self.stream);
        let h_s1_ptr = bw_raw_f32_ptr(&self.save_h_s1, &self.stream);
        let h_s2_ptr = bw_raw_f32_ptr(&self.save_h_s2, &self.stream);
        let h_v_ptr  = bw_raw_f32_ptr(&self.save_h_v,  &self.stream);
        let h_b0_ptr = bw_raw_f32_ptr(&self.save_h_b0, &self.stream);
        let h_b1_ptr = bw_raw_f32_ptr(&self.save_h_b1, &self.stream);
        let h_b2_ptr = bw_raw_f32_ptr(&self.save_h_b2, &self.stream);
        let h_b3_ptr = bw_raw_f32_ptr(&self.save_h_b3, &self.stream);

        let d_h_s2_ptr = bw_raw_f32_ptr(&self.bw_d_h_s2, &self.stream);
        let d_h_s1_ptr = bw_raw_f32_ptr(&self.bw_d_h_s1, &self.stream);
        let d_h_v_ptr  = bw_raw_f32_ptr(&self.bw_d_h_v,  &self.stream);
        let d_h_b0_ptr = bw_raw_f32_ptr(&self.bw_d_h_b0, &self.stream);
        let d_h_b1_ptr = bw_raw_f32_ptr(&self.bw_d_h_b1, &self.stream);
        let d_h_b2_ptr = bw_raw_f32_ptr(&self.bw_d_h_b2, &self.stream);
        let d_h_b3_ptr = bw_raw_f32_ptr(&self.bw_d_h_b3, &self.stream);

        // dL/d_logits: f32 directly.
        let d_value_logits_ptr = self.d_value_logits_buf.raw_ptr();
        let d_adv_logits_ptr = self.d_adv_logits_buf.raw_ptr();

        let na = self.config.num_atoms;
        let f32_size = std::mem::size_of::<f32>() as u64;
        let d_adv0 = d_adv_logits_ptr;
        let d_adv1 = d_adv0 + (self.config.batch_size * self.config.branch_0_size * na) as u64 * f32_size;
        let d_adv2 = d_adv1 + (self.config.batch_size * self.config.branch_1_size * na) as u64 * f32_size;
        let d_adv3 = d_adv2 + (self.config.batch_size * self.config.branch_2_size * na) as u64 * f32_size;

        // #31 Bottleneck: when active, compute dX for s1 (flows into bottleneck backward)
        let s1_dx_output = if self.config.bottleneck_dim > 0 {
            self.bn_d_concat_buf.raw_ptr()
        } else {
            0u64  // no dX needed — states not trainable
        };

        // GLU saved activation pointers
        let glu_gate_pre_ptrs = [
            self.glu_gate_pre_buf[0].raw_ptr(),
            self.glu_gate_pre_buf[1].raw_ptr(),
            self.glu_gate_pre_buf[2].raw_ptr(),
            self.glu_gate_pre_buf[3].raw_ptr(),
        ];
        let glu_value_ptrs = [
            self.glu_value_buf[0].raw_ptr(),
            self.glu_value_buf[1].raw_ptr(),
            self.glu_value_buf[2].raw_ptr(),
            self.glu_value_buf[3].raw_ptr(),
        ];

        // Phase H Site 2: pass the value-FC dReLU bit-mask written by the
        // online forward via `RELU_AUX_BIAS`. Backward consumes via
        // `DRELU_BGRAD` to fuse the standalone relu_mask + bias_grad reduce
        // into the value-output dX GEMM (saves 2 launches per backward).
        // Pass 0 when the forward fused-AUX path is unavailable (the mask
        // buffer is stale in that case) — backward will fall back to the
        // standalone-kernel sequence.
        let value_fc_relu_mask = if self.cublas_forward.value_fc_relu_mask_available() {
            self.cublas_forward.value_fc_relu_mask_ptr()
        } else {
            0u64
        };

        // SP4 Layer C #260 (2026-05-01): backward-grad sanitiser bound for
        // `d_value_logits ∪ d_adv_logits` is now ISV-driven via the
        // dedicated `Q_DIR_GRAD_BOUND_INDEX=172` slot. Producer
        // (`launch_sp4_q_dir_grad_p99`) measures p99(|union of both
        // buffers|) via `sp4_histogram_p99_multi<256>` with n_sub=2,
        // applies Pearls A+D on the same stream, and writes the smoothed
        // bound to ISV[172]. Consumer reads the slot directly with
        // `EPS_CLAMP_FLOOR=1.0` ε on cold-start (Invariant 1 carve-out per
        // `feedback_isv_for_adaptive_bounds`).
        //
        // Migrated from SP1-Phase-C `1e6 × ISV[Q_DIR_ABS_REF].max(1.0)`
        // hardcoded multiplier. The same bound was previously applied to
        // BOTH d_value_logits AND d_adv_logits, so the union p99 is the
        // natural ISV-driven replacement (one bound, one ISV slot, one
        // producer). The new bound is Pearls-smoothed p99 of the actual
        // joint gradient distribution, tighter than the pre-existing 1e6
        // ceiling but appropriate-scale for the observed values.
        //
        // Inline slot 24/25 NaN checks IMMEDIATELY BEFORE each clamp
        // preserve the regression sentinel: `run_nan_checks_post_backward`
        // (fused_training.rs:1540) runs AFTER these clamps, so without
        // these pre-clamp checks the slot 24/25 flags would observe
        // sanitised zeros and never fire. Producer launches BEFORE the
        // consumer's clamp_finite calls, AFTER the slot 24/25 NaN checks
        // (same ordering invariant as site 1: NaN-check → producer →
        // consumer-clamp).
        //
        // Why the existing 5.0 L2 norm-clip is still in play: the L2
        // norm-clip (gpu_dqn_trainer.rs:16827-16868, max_d_logit_norm =
        // 5.0_f32 + clip_grad_kernel) is a per-buffer-norm aggregate
        // operation; this kernel is a per-element magnitude clamp — they
        // catch different failure modes (norm = aggregate; clamp_finite =
        // per-element NaN/Inf/extreme), so both stay.
        use crate::cuda_pipeline::sp4_isv_slots::Q_DIR_GRAD_BOUND_INDEX;
        use crate::cuda_pipeline::sp4_wiener_ema::EPS_CLAMP_FLOOR;
        let n_val_clamp = self.d_value_logits_buf.len();
        let n_adv_clamp = self.d_adv_logits_buf.len();
        self.check_nan_f32(d_value_logits_ptr, n_val_clamp, 24)?;
        self.check_nan_f32(d_adv_logits_ptr, n_adv_clamp, 25)?;
        // Producer reads BOTH sub-buffers via the union p99; launch AFTER
        // both NaN checks so the regression sentinel captures any NaN entry
        // before the producer measures the clean distribution.
        self.launch_sp4_q_dir_grad_p99()?;
        let max_abs_q = self
            .read_isv_signal_at(Q_DIR_GRAD_BOUND_INDEX)
            .max(EPS_CLAMP_FLOOR);
        self.launch_clamp_finite_f32(d_value_logits_ptr, n_val_clamp, max_abs_q)?;
        self.launch_clamp_finite_f32(d_adv_logits_ptr, n_adv_clamp, max_abs_q)?;

        // SP14 B.10: rebuild the ONLINE direction-Q-head concat scratch
        // [save_h_s2 | aux_softmax_diff] into `sp14_dir_qaux_concat_scratch`.
        // The forward pass overwrote it with the TARGET concat (line ~25817 in
        // `submit_forward_ops`), so the backward dW SGEMM would otherwise read
        // tg_h_s2 instead of save_h_s2 — wrong gradient. `aux_nb_softmax_buf`
        // is unchanged between forward and backward (CE loss writes a separate
        // `d_aux_softmax` scratch), so re-running the same concat with
        // `save_h_s2` yields the bit-identical online concat the forward
        // SGEMM consumed. Same one-step-lag semantic preserved.
        self.launch_sp14_dir_concat_qaux(self.ptrs.save_h_s2)?;
        // SP14 B.10: pre-zero `d_h_s2` because the direction branch (d==0) no
        // longer writes it directly with beta=0 — its dX now lands in
        // `d_dir_qaux_concat` and is gated + accumulated AFTER backward_full
        // returns. The value-FC dX accumulator inside backward_full uses
        // beta=1 and assumed d_h_s2 was pre-written by d==0; with the
        // direction branch redirected, we must establish the zero baseline
        // explicitly. Raw cuMemsetD32Async — captured by CUDA Graph (cudarc
        // memset_zeros is NOT graph-safe; matches the cql_grad_scratch
        // pre-backward-full zero pattern in the CQL path).
        unsafe {
            cudarc::driver::sys::cuMemsetD32Async(
                d_h_s2_ptr, 0,
                self.config.batch_size * self.config.shared_h2,
                self.stream.cu_stream(),
            );
        }
        bw.backward_full(
            &self.stream,
            d_value_logits_ptr,
            &[d_adv0, d_adv1, d_adv2, d_adv3],
            states_ptr,
            h_s1_ptr,
            h_s2_ptr,
            h_v_ptr,
            &[h_b0_ptr, h_b1_ptr, h_b2_ptr, h_b3_ptr],
            &w_ptrs,
            grad_base,
            d_h_s2_ptr,
            d_h_s1_ptr,
            d_h_v_ptr,
            &[d_h_b0_ptr, d_h_b1_ptr, d_h_b2_ptr, d_h_b3_ptr],
            s1_dx_output,
            self.ptrs.mag_concat_buf,
            self.ptrs.d_mag_concat_buf,
            self.ptrs.ord_concat_buf,
            self.d_ord_concat_buf.raw_ptr(),
            self.ptrs.urg_concat_buf,
            self.d_urg_concat_buf.raw_ptr(),
            &glu_gate_pre_ptrs,
            &glu_value_ptrs,
            self.ptrs.bw_d_glu_value,
            self.ptrs.bw_d_glu_gate,
            value_fc_relu_mask,
            // SP14 B.10: direction-Q-head EGF wire — saved forward concat
            // for dW + dX output buffer for col SH2 wire scaling.
            self.sp14_dir_qaux_concat_scratch.raw_ptr(),
            self.sp14_d_dir_qaux_concat.raw_ptr(),
        )?;

        // SP14 Layer C Phase C.1 (2026-05-08): `launch_sp14_scale_wire_col`
        // call deleted atomically with the kernel file. Coupling B (α-gated
        // backward wire) is gone; the wire column gradient is dropped by the
        // first-SH2-cols accumulator below (column SH2 is not propagated
        // downstream, matching the pre-B.11 behavior when ISV[393]
        // sentinelled to 0.0). Phase C.5 wires the proper aux-trunk
        // backward path with stop-gradient at the encoder boundary.
        // Accumulate first SH2 columns of d_dir_qaux_concat into d_h_s2 with
        // beta=1 (the value-FC dX inside backward_full already accumulated its
        // contribution; we ADD on top, NOT overwrite). Wire column (col SH2)
        // is NOT propagated downstream from here.
        self.accumulate_d_h_s2_from_concat(
            self.sp14_d_dir_qaux_concat.raw_ptr(),
            d_h_s2_ptr,
            self.config.batch_size,
            self.config.shared_h2 + 1,
            1.0,
        )?;

        // Accumulate magnitude branch dX (first SH2 columns) into d_h_s2
        if self.ptrs.mag_concat_buf != 0 {
            self.accumulate_d_h_s2_from_concat(
                self.ptrs.d_mag_concat_buf,
                d_h_s2_ptr,
                self.config.batch_size,
                self.config.shared_h2 + self.config.branch_0_size, // mag: direction-conditioned
                1.0, // beta=1: SP14 B.10 zeroed d_h_s2 + value-FC + dir-Q already accumulated
            )?;
        }
        // Accumulate order branch dX (first SH2 columns) into d_h_s2
        if self.ptrs.ord_concat_buf != 0 {
            self.accumulate_d_h_s2_from_concat(
                self.d_ord_concat_buf.raw_ptr(),
                d_h_s2_ptr,
                self.config.batch_size,
                self.config.shared_h2 + 3, // ord: OFI-conditioned (3 OFI features)
                1.0,
            )?;
        }
        // Accumulate urgency branch dX (first SH2 columns) into d_h_s2
        if self.ptrs.urg_concat_buf != 0 {
            self.accumulate_d_h_s2_from_concat(
                self.d_urg_concat_buf.raw_ptr(),
                d_h_s2_ptr,
                self.config.batch_size,
                self.config.shared_h2 + 3, // urg: OFI-conditioned (3 OFI features)
                1.0,
            )?;
        }

        // SP1 Phase B (slot 33): bw_d_h_s2 snapshot AFTER backward_full +
        // branch concat accumulations, BEFORE aux_heads_backward SAXPY. The
        // buffer holds the "main path" gradient — branches 0/1/2/3 accumulated
        // via `accumulate_d_h_s2_from_concat` (beta=0 for branch 0, beta=1
        // thereafter) plus the value-FC contribution via `backward_full`.
        // Localises whether the main path corrupted the buffer before any
        // aux-head SAXPY runs. Per-slot semantics in
        // `docs/dqn-backward-nan-audit.md` (slot 33 row).
        {
            let b_post = self.config.batch_size;
            let sh2_post = self.config.shared_h2;
            self.check_nan_f32(self.ptrs.bw_d_h_s2, b_post * sh2_post, 33)?;
        }

        // ── SP14 Layer C Phase C.5b (2026-05-08): aux trunk Adam pre-capture ──
        // Host-write through mapped-pinned to populate Adam hyperparams +
        // step counter for the captured aux trunk Adam launch (inside
        // `aux_heads_backward` below). The captured graph reads pointers,
        // not values — these host writes are picked up at replay time.
        //
        // Per `pearl_no_host_branches_in_captured_graph.md` we cannot do
        // these host writes from inside the captured region, but
        // `launch_cublas_backward_to` is `&mut self` and runs OUTSIDE the
        // captured graph (the captured chain is each `cuGraphLaunch`
        // replay; this Rust function runs each per-step host driver).
        //
        // ISV[AUX_TRUNK_LR_INDEX=444] / ISV[AUX_TRUNK_GRAD_CLIP_INDEX=448]
        // are populated by the StateResetRegistry at fold boundaries
        // (`AUX_TRUNK_LR_DEFAULT=1e-4` / `AUX_TRUNK_GRAD_CLIP_DEFAULT=1.0`)
        // and unchanged within a fold; reading once per step lets future
        // adaptive controllers drive these slots without re-wiring.
        {
            use crate::cuda_pipeline::sp14_isv_slots::{
                AUX_TRUNK_GRAD_CLIP_INDEX, AUX_TRUNK_LR_INDEX,
            };
            let aux_lr = self.read_isv_signal_at(AUX_TRUNK_LR_INDEX);
            let aux_clip = self.read_isv_signal_at(AUX_TRUNK_GRAD_CLIP_INDEX);
            // Step counter is host-tracked (mirrors `step_ofi_embed_adam`
            // pattern): increment then write through to the pinned buffer.
            // The captured Adam kernel reads `*t_dev_ptr` at replay time.
            self.aux_trunk_adam_step += 1;
            unsafe {
                *self.aux_trunk_lr_pinned = aux_lr;
                *self.aux_trunk_grad_clip_pinned = aux_clip;
                *self.aux_trunk_t_pinned = self.aux_trunk_adam_step;
            }
        }

        // ── Plan 4 Task 6 Commit B: aux-heads backward ──
        // Runs the per-head backward kernels, reduces 8 per-tensor partials
        // into final dW/dB, SAXPYs them into `grad_base[119..127)` with
        // alpha = `aux_weight`. SP14-C.5b: SAXPYs both per-head dh_s2_aux
        // buffers into `dh_s2_aux_accum` (NOT `bw_d_h_s2`); aux trunk
        // backward propagates that gradient to the aux trunk's own
        // params; aux trunk Adam launches at the end of the chain.
        // Q's `bw_d_h_s2` is unaffected by aux loss — encoder boundary
        // structural stop-grad enforced by `aux_trunk_backward`.
        self.aux_heads_backward(grad_base)?;

        // SP1 Phase B (slot 34): bw_d_h_s2 snapshot AFTER aux_heads_backward
        // SAXPYed `aux_dh_s2_nb_buf` + `aux_dh_s2_rg_buf` into the trunk
        // accumulator. Compared against slot 33: any NEW NaN here came from
        // the aux-head SAXPY path (the saxpy kernel currently has no
        // isfinite guard — Phase C proposal). Per-slot semantics in
        // `docs/dqn-backward-nan-audit.md` (slot 34 row).
        {
            let b_post = self.config.batch_size;
            let sh2_post = self.config.shared_h2;
            self.check_nan_f32(self.ptrs.bw_d_h_s2, b_post * sh2_post, 34)?;
        }

        // ── Phase 3 T3.6: MoE backward ─────────────────────────────────
        // Runs AFTER aux_heads_backward (which SAXPYed into bw_d_h_s2) and
        // BEFORE encoder_backward_chain. Computes dW/dB for MoE params
        // [127..163) and accumulates expert dX → moe_dh_s1_scratch, then
        // DtoD-copies moe_dh_s1_scratch into bw_d_h_s2 so the GRN encoder
        // chain sees the MoE-routed gradient as its h_s2 input gradient.
        self.launch_moe_backward(grad_base)?;

        // ── GRN trunk backward (Plan 4 Task 2c.3c.4) ──
        // d_h_s2_ptr now holds the fully-accumulated gradient flowing into
        // the h_s2 GRN output. encoder_backward_chain unrolls both GRN
        // blocks symmetrically with `encoder_forward_only`, writing dW/dB
        // into goff[0..13) of `grad_base` and producing the trunk input
        // gradient at `s1_dx_output` (which `bn_d_concat_buf` aliases when
        // bottleneck is active).
        self.cublas_forward.encoder_backward_chain(
            &self.stream,
            bw,
            states_ptr,
            &w_ptrs,
            grad_base,
            &param_sizes,
            d_h_s2_ptr,
            d_h_s1_ptr,
            s1_dx_output,
            self.bw_grn_h_s1_d_pre_ln.raw_ptr(),
            self.bw_grn_h_s1_d_linear_b_out.raw_ptr(),
            self.bw_grn_h_s1_d_elu_out.raw_ptr(),
            self.bw_grn_h_s1_d_linear_a_out.raw_ptr(),
            self.bw_grn_h_s2_d_pre_ln.raw_ptr(),
            self.bw_grn_h_s2_d_linear_b_out.raw_ptr(),
            self.bw_grn_h_s2_d_elu_out.raw_ptr(),
            self.bw_grn_h_s2_d_linear_a_out.raw_ptr(),
            h_s1_ptr,
        )?;

        // SP1 Phase C surgical fix (slot 32 output sanitise): post-call
        // defence-in-depth on `bn_d_concat_buf` (= `s1_dx_output` when
        // bottleneck active). The trunk-encoder backward writes here via
        // Linear_a (beta=0) + Linear_residual (beta=1) cuBLAS GEMMs whose
        // intermediate-product accumulator can overflow even on finite
        // inputs at extreme F1-ep2 magnitudes (smoke-test-xvzgk slot 32
        // signature). Reuses `max_abs_q` from the slot 24/25 pre-call
        // clamp because the same Q-magnitude reference scale bounds both
        // ends of the dueling backward chain. Skipped when bottleneck is
        // disabled (`bn_d_concat_buf` is unused / `s1_dx_output == 0`).
        //
        // Inline slot 32 NaN check IMMEDIATELY BEFORE the clamp preserves
        // the regression sentinel: `run_nan_checks_post_backward`
        // (fused_training.rs:1540) runs AFTER this clamp, so without this
        // pre-clamp check the slot 32 flag would observe sanitised zeros
        // and never fire. Mirrors the same slot-35 NaN-check + clamp
        // pattern in `apply_iqn_trunk_gradient`.
        if self.config.bottleneck_dim > 0 {
            let bn_d_concat_len = self.bn_d_concat_buf.len();
            let bn_d_concat_ptr = self.bn_d_concat_buf.raw_ptr();
            self.check_nan_f32(bn_d_concat_ptr, bn_d_concat_len, 32)?;
            self.launch_clamp_finite_f32(bn_d_concat_ptr, bn_d_concat_len, max_abs_q)?;
        }

        // ── #31 Bottleneck backward: chain rule through tanh + GEMM ──
        // After backward_full, d_bn_concat_buf has dL/d(bn_concat) [B, concat_dim] f32.
        // Chain rule (post-1B-iii — bottleneck Linear's forward consumes
        // VSN-gated states, so its backward dW is computed against the same
        // gated input; 1B-iv adds the dX path that flows the gradient through
        // the bottleneck Linear into d_VSN_gated_state[market], which is then
        // assembled with the portfolio passthrough to drive `vsn_backward`):
        //   dL/d_bn   = dL/d_concat[:, :bn_dim] * tanh'(bn_raw)
        //   dW_bn     = dL/d_bn^T @ vsn_gated_states[:, :market_dim]
        //   db_bn     = sum(dL/d_bn, dim=0)
        //   dX_bn     = dL/d_bn @ W_bn   (writes into d_vsn_gated_state[:, :market_dim])
        if self.config.bottleneck_dim > 0 {
            let bn_dim = self.config.bottleneck_dim;
            let market_dim = self.config.market_dim;
            let b = self.config.batch_size;
            let portfolio_dim = ml_core::state_layout::STATE_DIM.saturating_sub(market_dim);
            // SP15 Wave 4.1b: bn_d_concat_buf row stride is the full forward
            // concat_dim including the dd_pct column. Same rationale as the
            // aux_bottleneck_vsn_backward_dispatch site — bn_tanh_backward
            // reads [0..bn_dim) and the portfolio_pad reads
            // [bn_dim..bn_dim+portfolio_dim); the dd_pct column at
            // bn_dim+portfolio_dim is correctly skipped because dd_pct has no
            // learnable input source.
            let concat_dim = bn_dim + portfolio_dim + 1;
            let state_dim = ml_core::state_layout::STATE_DIM;
            let state_dim_padded = ml_core::state_layout::STATE_DIM_PADDED;

            let d_concat_ptr = self.bn_d_concat_buf.raw_ptr();
            let d_bn_ptr = self.bn_d_hidden_buf.raw_ptr();
            let bn_hidden_ptr = self.bn_hidden_buf.raw_ptr();
            // 1B-iii forward feeds the bottleneck Linear with VSN-gated states.
            // Backward dW must use the SAME X — pre-1B-iv this was incorrectly
            // `raw_states_ptr`, which made dW_bn ≈ 6× the correct magnitude
            // when the cold-start VSN mask is uniform-1/6 (Adam normalises but
            // the gradient direction is still distorted because the gate is a
            // multiplicative scale, not a constant). Now that VSN trains, the
            // mismatch would diverge over time — fixed in lockstep with the
            // VSN backward extension below.
            let vsn_gated_states_ptr = self.vsn_gated_states_buf.raw_ptr();

            // Step 1: Tanh derivative — d_bn = d_concat[:, :bn_dim] * (1 - tanh^2)
            let total = (b * bn_dim) as i32;
            let blocks = ((total as u32 + 255) / 256) as u32;
            let b_i32 = b as i32;
            let bn_dim_i32 = bn_dim as i32;
            let concat_dim_i32 = concat_dim as i32;
            {
                unsafe {
                    self.stream
                        .launch_builder(&self.bn_tanh_backward_kernel)
                        .arg(&d_bn_ptr)
                        .arg(&d_concat_ptr)
                        .arg(&bn_hidden_ptr)
                        .arg(&b_i32)
                        .arg(&bn_dim_i32)
                        .arg(&concat_dim_i32)
                        .launch(LaunchConfig {
                            grid_dim: (blocks, 1, 1),
                            block_dim: (256, 1, 1),
                            shared_mem_bytes: 0,
                        })
                        .map_err(|e| MLError::ModelError(format!(
                            "bn_tanh_backward: {e}"
                        )))?;
                }
            }

            // Step 2: dW_bn[bn_dim, market_dim] += d_bn^T @ vsn_gated_states[:, :market_dim]
            // Grad offsets for w_bn (tensor 33) / b_bn (tensor 34) — see the
            // matching forward-site comment for the post-2c.3a +9 shift that
            // these two sites were missing.
            let goff_w_bn = padded_byte_offset(&param_sizes, 33);
            let goff_b_bn = padded_byte_offset(&param_sizes, 34);

            // dW_bn = d_bn^T[bn_dim, B] @ vsn_gated_states[B, market_dim_padded]
            // Use launch_dw_only (d_bn=dY f32, X=gated states, grad=dW f32)
            bw.launch_dw_only(
                &self.stream,
                d_bn_ptr,                            // dY [B, bn_dim] f32
                vsn_gated_states_ptr,                // X = VSN-gated states [B, state_dim_padded]
                grad_base + goff_w_bn,               // dW [bn_dim, market_dim] f32
                grad_base + goff_b_bn,               // db [bn_dim] f32
                bn_dim,                              // out_dim
                market_dim,                          // in_dim
                b,
            )?;

            // ── Plan 4 Task 1B-iv: VSN backward chain extension ──
            // Final step of the backward chain. Reads the bottleneck Linear
            // backward dX (computed below) + the portfolio passthrough +
            // padding zero into a single assembled `d_vsn_gated_state`, then
            // calls `vsn_backward` to propagate that gradient through the 6
            // per-group VSN MLPs. Output: 24 dW/dB entries written into
            // grad_buf[95..119) — these are the slots Adam consumes for
            // VSN params (w_a/b_a/w_b/b_b per group, 4 tensors × 6 groups).
            //
            // The d_state output of the softmax-and-gate backward is
            // computed but discarded (state buf not trainable); allocating
            // a scratch keeps the kernel signature happy.
            let d_vsn_gated_state_ptr = self.vsn_d_gated_state_buf.raw_ptr();

            // Step 3a: portfolio passthrough + zero pad. Populates the
            // portfolio slice [market_dim..STATE_DIM] of d_vsn_gated_state
            // from d_concat[:, bn_dim..]; zeros the market slice (cuBLAS
            // will overwrite it next) and the padding tail [STATE_DIM,
            // STATE_DIM_PADDED). One thread per (sample, padded-index).
            let market_dim_i32 = market_dim as i32;
            let state_dim_i32 = state_dim as i32;
            let state_dim_padded_i32 = state_dim_padded as i32;
            let pad_total = (b * state_dim_padded) as i32;
            let pad_blocks = ((pad_total as u32 + 255) / 256) as u32;
            unsafe {
                self.stream
                    .launch_builder(&self.vsn_d_gated_state_portfolio_kernel)
                    .arg(&d_vsn_gated_state_ptr)
                    .arg(&d_concat_ptr)
                    .arg(&b_i32)
                    .arg(&market_dim_i32)
                    .arg(&bn_dim_i32)
                    .arg(&concat_dim_i32)
                    .arg(&state_dim_i32)
                    .arg(&state_dim_padded_i32)
                    .launch(LaunchConfig {
                        grid_dim: (pad_blocks, 1, 1),
                        block_dim: (256, 1, 1),
                        shared_mem_bytes: 0,
                    })
                    .map_err(|e| MLError::ModelError(format!(
                        "vsn_d_gated_state_portfolio_pad: {e}"
                    )))?;
            }

            // Step 3b: bottleneck Linear backward dX into the market slice
            // of d_vsn_gated_state with row stride state_dim_padded.
            //   dX_bn[B, market_dim]@stride=state_dim_padded
            //   = d_bn[B, bn_dim] @ W_bn[bn_dim, market_dim]
            // beta=0 overwrites the zero-initialised market slice from step 3a.
            let w_bn_ptr = w_ptrs[33];
            bw.launch_dx_only_lda(
                &self.stream,
                d_bn_ptr,                            // dY [B, bn_dim]
                w_bn_ptr,                             // W  [bn_dim, market_dim]
                d_vsn_gated_state_ptr,                // dX written at offset 0 with x_lda=state_dim_padded
                bn_dim,
                market_dim,
                state_dim_padded,                     // x_lda — write into market slice of wide buffer
                b,
                0.0_f32,
            )?;

            // Plan 4 Task 1B-iv-rc: attenuate VSN dW signal by VSN_DW_DAMP.
            // Scales `d_vsn_gated_state` IN-PLACE; the softmax-and-gate
            // Jacobian + downstream Linear_2 / Linear_1 backward GEMMs are
            // all linear in this input, so a single scale here uniformly
            // attenuates the 24 VSN dW/dB writes that hit grad_buf[95..119).
            // Uses `scale_f32_kernel` (forward_child capture) — captured
            // alongside the rest of `submit_forward_ops_main`. Constant
            // alpha is graph-capture-stable. See `VSN_DW_DAMP` doc for
            // rationale.
            {
                let n = (self.config.batch_size
                    * ml_core::state_layout::STATE_DIM_PADDED) as i32;
                let blocks = ((n as u32 + 255) / 256).max(1);
                let alpha = VSN_DW_DAMP;
                unsafe {
                    self.stream
                        .launch_builder(&self.scale_f32_kernel)
                        .arg(&d_vsn_gated_state_ptr)
                        .arg(&alpha)
                        .arg(&n)
                        .launch(LaunchConfig {
                            grid_dim: (blocks, 1, 1),
                            block_dim: (256, 1, 1),
                            shared_mem_bytes: 0,
                        })
                        .map_err(|e| MLError::ModelError(format!(
                            "vsn dW main attenuate: {e}"
                        )))?;
                }
            }

            // Step 3c: VSN backward — writes 24 dW/dB into grad_buf[95..119).
            let h1_save_ptrs: [u64; ml_core::state_layout::SL_NUM_FEATURE_GROUPS] = [
                self.vsn_save_h1_g0_buf.raw_ptr(),
                self.vsn_save_h1_g1_buf.raw_ptr(),
                self.vsn_save_h1_g2_buf.raw_ptr(),
                self.vsn_save_h1_g3_buf.raw_ptr(),
                self.vsn_save_h1_g4_buf.raw_ptr(),
                self.vsn_save_h1_g5_buf.raw_ptr(),
            ];
            self.cublas_forward.vsn_backward(
                &self.stream,
                bw,
                d_vsn_gated_state_ptr,                  // assembled d_VSN_gated_state input
                raw_states_ptr,                          // saved state (raw — VSN forward read this)
                self.vsn_mask_buf.raw_ptr(),             // saved mask from online VSN forward
                &h1_save_ptrs,
                &w_ptrs,
                grad_base,
                &param_sizes,
                self.vsn_d_logits_buf.raw_ptr(),
                self.vsn_d_state_buf.raw_ptr(),
                self.vsn_d_logit_scratch_buf.raw_ptr(),
                self.vsn_d_h1_scratch_buf.raw_ptr(),
                self.vsn_group_begins_buf.raw_ptr(),
                self.vsn_group_ends_buf.raw_ptr(),
                &self.vsn_logit_gather_kernel,
                b,
            )?;
        }

        Ok(())
    }

    /// Launch the gradient L2 norm reduction kernel.
    ///
    /// Argument order matches `dqn_grad_norm_kernel` in `dqn_training_kernel.cu`:
    /// grads, out_grad_norm, total_params = 3 args.
    ///
    /// Phase 1: each block writes its partial sum to grad_norm_partials[blockIdx.x].
    /// No memset needed — each block overwrites its slot.
    fn launch_grad_norm(&self) -> Result<(), MLError> {
        let tp = self.total_params as i32;
        let blocks = self.grad_norm_blocks as u32;

        let grad_ptr = self.ptrs.grad_buf;
        let partials_ptr = self.grad_norm_partials.raw_ptr();

        unsafe {
            self.stream
                .launch_builder(&self.grad_norm_kernel)
                .arg(&grad_ptr)
                .arg(&partials_ptr)
                .arg(&tp)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| {
                    MLError::ModelError(format!("dqn_grad_norm_kernel launch: {e}"))
                })?;
        }

        Ok(())
    }

    /// Launch grad norm using the STANDALONE kernel handle (not the captured one).
    /// Must be used outside CUDA Graph context.
    /// After graph_mega capture, the captured CUfunction handle cannot be launched
    /// outside the graph — CUDA silently corrupts kernel state.
    fn launch_grad_norm_standalone(&self) -> Result<(), MLError> {
        let tp = self.total_params as i32;
        let blocks = self.grad_norm_blocks as u32;

        let grad_ptr = self.ptrs.grad_buf;
        let partials_ptr = self.grad_norm_partials.raw_ptr();

        unsafe {
            self.stream
                .launch_builder(&self.grad_norm_standalone)
                .arg(&grad_ptr)
                .arg(&partials_ptr)
                .arg(&tp)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| {
                    MLError::ModelError(format!("dqn_grad_norm_standalone launch: {e}"))
                })?;
        }

        Ok(())
    }

    /// Launch the Adam update kernel with correct gradient clipping.
    ///
    /// Argument order matches `dqn_adam_update_kernel` in `dqn_utility_kernels.cu`:
    /// params, grads, m, v, grad_norm_sq, lr, beta1, beta2, epsilon,
    /// weight_decay, max_grad_norm, t_ptr, total_params,
    /// weight_decay_mask, l1_end, l1_lambda, weight_clamp_max_abs,
    /// nan_flags_buf, diag_slot, clamp_engage_per_block_buf, engage_buf_offset
    /// = 21 args.
    ///
    /// Must be launched AFTER `launch_grad_norm` so that `grad_norm_buf`
    /// contains the completed sum of squares (no race condition).
    /// `t_ptr` is a device buffer (not scalar) so the CUDA Graph can be
    /// replayed with an updated step counter.
    ///
    /// **SP4 Layer B (2026-05-01)**: split into 4 sub-launches over the
    /// 3-group SP4 taxonomy + a "trunk-extras" tail. The first 3
    /// (DqnTrunk / DqnValue / DqnBranches) cover tensors `[0..33)` with
    /// per-group ISV-driven `WEIGHT_BOUND[group]` clamp and `WD_RATE[group]`
    /// AdamW rate. L1 lambda (`ISV[L1_LAMBDA_TRUNK_INDEX]`) applies to
    /// the trunk only; value/branches pass `l1_end=0 + l1_lambda=0`.
    /// Per-group split also resolves the Layer A Pearl C engagement-
    /// counter accounting deferral — each sub-launch writes per-block
    /// counts at its own group offset.
    ///
    /// **Layer B fix-up (2026-05-01)**: the 4th sub-launch covers the
    /// tail tensors `[33..163)` (bottleneck, VSN bottleneck, GLU gates,
    /// KAN, regime gate, spacing, multi-horizon value heads, risk, ISV
    /// encoder, VSN per-group MLP, aux heads, MoE) — collectively
    /// "trunk-extras". The original 3-way split silently no-op'd Adam
    /// for these ~70% of weight tensors (grads computed, m/v allocated,
    /// no parameter step). The fix-up co-locates trunk-extras under
    /// `ParamGroup::DqnTrunk` for the WEIGHT_BOUND / WD_RATE contract
    /// (these tensors share the trunk's gradient regime: pre-bottleneck,
    /// attention, encoder paths) and uses `SP4_ENGAGE_OFFSET_DISABLED`
    /// for Pearl C engagement (rolls into trunk's accounting since they
    /// share bounds). A future task may introduce dedicated SP4
    /// producers for sub-trunk regions (VSN, MoE, etc.) once warranted
    /// by signal analysis. Per `feedback_no_partial_refactor`: trunk-
    /// extras' single (pragmatic) consumer ships with the launch fix in
    /// this commit.
    ///
    /// All parameters `[0..total_params)` are covered across the 4
    /// sub-launches:
    ///   - trunk     = `params[0..trunk_param_count]` (tensors [0..13))
    ///   - value     = byte-slice for tensors [13..17)
    ///   - branches  = byte-slice for tensors [17..33)
    ///   - trunk-tail = byte-slice for tensors [33..163)
    /// A coverage debug_assert at the bottom of the function verifies
    /// `trunk + value + branches + trunk_extras == total_params` to
    /// prevent regressions of this kind.
    ///
    /// Per `feedback_no_quickfixes`: NOT a max/min-of-3 single-launch
    /// shortcut. Per `feedback_no_partial_refactor`: every consumer of
    /// the SP4 contract migrates in this same commit.
    fn launch_adam_update(&self) -> Result<(), MLError> {
        use crate::cuda_pipeline::sp4_isv_slots::{
            ParamGroup as SP4Group, L1_LAMBDA_TRUNK_INDEX, SP4_ENGAGE_OFFSET_DISABLED,
            SP4_ENGAGE_OFFSET_BRANCH_DIR, SP4_ENGAGE_OFFSET_BRANCH_MAG,
            SP4_ENGAGE_OFFSET_BRANCH_ORDER, SP4_ENGAGE_OFFSET_BRANCH_URGENCY,
        };
        use crate::cuda_pipeline::sp5_isv_slots::{
            ADAM_BETA1_BASE, ADAM_BETA2_BASE, ADAM_EPS_BASE,
        };

        let lr_ptr = self.lr_dev_ptr;  // pinned device-mapped — graph-safe, value read at replay
        let adaptive_clip_ptr = self.ptrs.adaptive_clip_buf;
        let norm_ptr = self.ptrs.grad_norm_buf;
        let t_ptr = self.ptrs.t_buf;

        // G1+G8: weight_decay_mask + L1 proximal on w_s1.
        let wd_mask_base = self.weight_decay_mask.raw_ptr();
        let param_sizes = compute_param_sizes(&self.config);
        let trunk_l1_end: i32 = align4(param_sizes[0]) as i32;  // end of w_s1

        // SP4 Layer B (Mech 9 + Pearl C): per-group sub-launches replace the
        // single fused launch. Each group reads its own ISV-driven weight
        // clamp (`ISV[WEIGHT_BOUND[group]]`) and AdamW weight_decay
        // (`ISV[WD_RATE[group]]`); L1 lambda is read from
        // `ISV[L1_LAMBDA_TRUNK_INDEX]` for the trunk only (groups 1+2+
        // trunk-extras pass 0). The split also resolves Layer A's Pearl C
        // engagement-counter limitation (engagement was previously
        // accounted under group 0 only — A14/A15 deferral). Each
        // SP4-canonical sub-launch writes per-block counts at distinct
        // offsets `group_idx × MAX_BLOCKS_PER_ADAM`; trunk-extras passes
        // `SP4_ENGAGE_OFFSET_DISABLED` because it shares trunk's
        // WEIGHT_BOUND / WD_RATE bounds (Pearl C engagement is a per-
        // bound diagnostic, not a per-launch one).
        let nan_flags_ptr = self.nan_flags_buf.raw_ptr();
        let diag_slot: i32 = SP4_DQN_ADAM_DIAG_SLOT;
        let engage_buf_ptr: u64 = self.clamp_engage_per_block_buf.dev_ptr;

        // Per-group descriptor: (ParamGroup, byte_offset, count, engage_offset).
        // Mirrors `param_group_buffers` for the DQN-internal groups exactly
        // for groups 0/1/2; the trunk-extras tail extends through end of
        // params buffer. We compute byte-offsets here directly to avoid
        // threading `SP4AuxBuffers` through this launcher (DQN-internal
        // groups never need aux pointers).
        let f32_sz = std::mem::size_of::<f32>() as u64;
        let trunk_count = self.trunk_param_count;
        let trunk_byte_off: u64 = 0;
        let value_byte_off = padded_byte_offset(&param_sizes, 13);
        let value_byte_end = padded_byte_offset(&param_sizes, 17);
        let value_count = ((value_byte_end - value_byte_off) / f32_sz) as usize;

        // SP21 T2.2 Phase 4 (2026-05-11): per-branch byte ranges within the
        // canonical DqnBranches range [17..33). Each branch occupies 4 param
        // tensors (W_FC, B_FC, W_OUT, B_OUT). Splitting the single
        // DqnBranches sub-launch into 4 per-branch sub-launches lets each
        // branch consume its own E4 LR scale from
        // ISV[BRANCH_LR_SCALE_{DIR,MAG,ORDER,URGENCY}_INDEX]. Coverage
        // invariant unchanged — total span is still [17..33) without gaps.
        let b_dir_off     = padded_byte_offset(&param_sizes, 17);
        let b_dir_end     = padded_byte_offset(&param_sizes, 21);
        let b_mag_off     = b_dir_end;
        let b_mag_end     = padded_byte_offset(&param_sizes, 25);
        let b_order_off   = b_mag_end;
        let b_order_end   = padded_byte_offset(&param_sizes, 29);
        let b_urgency_off = b_order_end;
        let b_urgency_end = padded_byte_offset(&param_sizes, 33);
        let b_dir_count     = ((b_dir_end     - b_dir_off)     / f32_sz) as usize;
        let b_mag_count     = ((b_mag_end     - b_mag_off)     / f32_sz) as usize;
        let b_order_count   = ((b_order_end   - b_order_off)   / f32_sz) as usize;
        let b_urgency_count = ((b_urgency_end - b_urgency_off) / f32_sz) as usize;
        let branches_count = b_dir_count + b_mag_count + b_order_count + b_urgency_count;

        // Trunk-extras tail: tensors [33..163). Spans from end of branches
        // to end of params buffer.
        let trunk_extras_byte_off = padded_byte_offset(&param_sizes, 33);
        let trunk_extras_count = self.total_params
            .saturating_sub((trunk_extras_byte_off / f32_sz) as usize);

        // SP21 T2.2 Phase 4 (2026-05-11): read per-branch LR scales from
        // ISV[521..525). Cold-start sentinel (slot at 0.0 before E4
        // first emits) is floored at 1.0 so the cold-start Adam update
        // doesn't degenerate to lr=0. Once E4 writes a real value
        // (range [0.5, 2.0] per its `compute_branch_lr_scale` clamp),
        // the floor is a no-op. Order matches branch indexing convention
        // (0=Dir, 1=Mag, 2=Order, 3=Urgency).
        let read_branch_lr_scale = |branch_idx: usize| -> f32 {
            use crate::cuda_pipeline::sp21_isv_slots::branch_lr_scale_index;
            let v = self.read_isv_signal_at(branch_lr_scale_index(branch_idx));
            // Bootstrap floor: ISV at sentinel 0.0 → 1.0 (no-op).
            // Per `pearl_first_observation_bootstrap` discipline.
            if v.abs() <= 1e-6 { 1.0 } else { v }
        };
        let lr_scale_dir     = read_branch_lr_scale(0);
        let lr_scale_mag     = read_branch_lr_scale(1);
        let lr_scale_order   = read_branch_lr_scale(2);
        let lr_scale_urgency = read_branch_lr_scale(3);

        // (group_tag, byte_off, count, engage_offset, lr_scale). All
        // non-branch entries pass lr_scale=1.0 (no-op for trunk / value
        // / trunk-extras / aux trainers).
        //
        // SP21 Phase 4.5 (2026-05-11): the 4 branch sub-launches now have
        // their OWN distinct engage_offsets within `clamp_engage_per_block_buf`
        // (Dir reuses canonical DqnBranches slot 2; Mag/Order/Urgency get
        // extra slots 11/12/13 immediately after Curiosity's tail at 10).
        // `pearl_c_post_adam_engagement_check(DqnBranches, ...)` aggregates
        // all 4 sub-ranges into a single per-group rate-deficit EMA —
        // semantically equivalent to the pre-Phase-4 single-launch
        // tracking, and follows the same pattern Curiosity uses (4
        // sub-launches W1/b1/W2/b2 sharing one Pearl C bucket).
        let trunk_engage_off:    i32 = (SP4Group::DqnTrunk.idx()    * MAX_BLOCKS_PER_ADAM) as i32;
        let value_engage_off:    i32 = (SP4Group::DqnValue.idx()    * MAX_BLOCKS_PER_ADAM) as i32;
        let groups: [(SP4Group, u64, usize, i32, f32); 7] = [
            (SP4Group::DqnTrunk,    trunk_byte_off,         trunk_count,         trunk_engage_off,                  1.0),
            (SP4Group::DqnValue,    value_byte_off,         value_count,         value_engage_off,                  1.0),
            // SP21 Phase 4: 4 per-branch sub-launches with per-branch
            // lr_scale; Phase 4.5: per-branch Pearl C engagement offsets.
            (SP4Group::DqnBranches, b_dir_off,              b_dir_count,         SP4_ENGAGE_OFFSET_BRANCH_DIR,      lr_scale_dir),
            (SP4Group::DqnBranches, b_mag_off,              b_mag_count,         SP4_ENGAGE_OFFSET_BRANCH_MAG,      lr_scale_mag),
            (SP4Group::DqnBranches, b_order_off,            b_order_count,       SP4_ENGAGE_OFFSET_BRANCH_ORDER,    lr_scale_order),
            (SP4Group::DqnBranches, b_urgency_off,          b_urgency_count,     SP4_ENGAGE_OFFSET_BRANCH_URGENCY,  lr_scale_urgency),
            // Trunk-extras tail — shares DqnTrunk bounds; Pearl C disabled
            // for this sub-launch (engagement folds into trunk's
            // accounting since the same WEIGHT_BOUND drives both clamps).
            (SP4Group::DqnTrunk,    trunk_extras_byte_off,  trunk_extras_count,  SP4_ENGAGE_OFFSET_DISABLED,        1.0),
        ];

        // Coverage invariant: every of the 163 weight tensors must be
        // covered by exactly one of the 7 sub-launches above. Layer B
        // fix-up: prevents regression of the silent ~70% Adam freeze
        // that the original 3-way split introduced. SP21 Phase 4
        // expanded from 4 to 7 sub-launches (DqnBranches split into 4
        // per-branch); the coverage sum remains
        // trunk + value + branches_total + trunk_extras.
        debug_assert_eq!(
            trunk_count + value_count + branches_count + trunk_extras_count,
            self.total_params,
            "SP21 Phase 4 7-way Adam coverage must span all 163 weight tensors \
             (got trunk={} + value={} + branches_total={} (dir={}+mag={}+order={}+urgency={}) + trunk_extras={} = {}, expected total_params={})",
            trunk_count, value_count, branches_count,
            b_dir_count, b_mag_count, b_order_count, b_urgency_count,
            trunk_extras_count,
            trunk_count + value_count + branches_count + trunk_extras_count,
            self.total_params
        );

        // Safety: argument order matches the extern "C" kernel signature exactly.
        // grad_norm_buf contains L2 norm from launch_grad_norm_finalize().
        for (group, byte_off, count, engage_buf_offset, lr_scale) in groups {
            if count == 0 {
                continue;
            }

            // SP5 Layer B (Pearl 4): per-group Adam β1/β2/ε from ISV[226+g],
            // ISV[234+g], ISV[242+g].  Floors are Invariant 1 carve-outs
            // (numerical-stability): β1≥0.5 avoids momentum inversion,
            // β2≥0.9 prevents exploding second-moment estimates, ε≥1e-12
            // avoids division-by-zero.  Cold-start ISV reads 0 — floors fire.
            let g = group.idx();
            let beta1   = self.read_isv_signal_at(ADAM_BETA1_BASE + g).max(0.5_f32).min(0.9999_f32);
            let beta2   = self.read_isv_signal_at(ADAM_BETA2_BASE + g).max(0.9_f32).min(0.99999_f32);
            let epsilon = self.read_isv_signal_at(ADAM_EPS_BASE   + g).max(1e-12_f32);

            // Per-group ISV-driven post-Adam weight clamp + AdamW
            // weight_decay rate via the SP4 Layer B helper. Trunk-extras
            // sub-launch reuses DqnTrunk's bounds (its `group` tag).
            let (weight_clamp_max_abs, weight_decay_value) =
                self.read_group_adam_bounds(group);

            // L1 proximal step is configured for the trunk only (G8 boundary
            // at end of `w_s1`). Value/branches/trunk-extras pass
            // `l1_end=0 + l1_lambda=0` which deactivates the in-kernel L1
            // path. Trunk-extras tensors [33..163) are well past `w_s1`'s
            // boundary and don't get L1 — only the canonical trunk
            // sub-launch (byte_off == 0) sets L1.
            let (l1_end_arg, l1_lambda_arg): (i32, f32) = if byte_off == 0 {
                let l1_lambda = self.read_isv_signal_at(L1_LAMBDA_TRUNK_INDEX).max(0.0);
                (trunk_l1_end, l1_lambda)
            } else {
                (0_i32, 0.0_f32)
            };

            // Sub-buffer pointers via byte-offset — same arithmetic as
            // `param_group_buffers`.
            let params_ptr = self.ptrs.params_ptr + byte_off;
            let grad_ptr   = self.ptrs.grad_buf   + byte_off;
            let m_ptr      = self.m_buf.raw_ptr() + byte_off;
            let v_ptr      = self.v_buf.raw_ptr() + byte_off;
            // wd_mask is a per-param byte-aligned buffer matching params layout.
            let wd_mask_ptr = wd_mask_base + byte_off;

            let count_i32 = count as i32;
            let blocks = ((count + 255) / 256) as u32;
            debug_assert!(
                (blocks as usize) <= MAX_BLOCKS_PER_ADAM,
                "dqn_adam[{:?} byte_off={}] blocks={} exceeds MAX_BLOCKS_PER_ADAM={} — Pearl C \
                 clamp_engage_per_block_buf cannot accommodate; grow MAX_BLOCKS_PER_ADAM \
                 or chunk the launch.",
                group, byte_off, blocks, MAX_BLOCKS_PER_ADAM
            );
            let cfg = LaunchConfig {
                grid_dim: (blocks, 1, 1),
                block_dim: (256, 1, 1),
                shared_mem_bytes: 0,
            };

            unsafe {
                self.stream
                    .launch_builder(&self.adam_update_kernel)
                    .arg(&params_ptr)
                    .arg(&grad_ptr)
                    .arg(&m_ptr)
                    .arg(&v_ptr)
                    .arg(&norm_ptr)
                    .arg(&lr_ptr)
                    .arg(&beta1)
                    .arg(&beta2)
                    .arg(&epsilon)
                    .arg(&weight_decay_value)   // SP4 Layer B: per-group ISV-driven
                    .arg(&adaptive_clip_ptr) // device buffer — EMA-based adaptive clip
                    .arg(&t_ptr)             // device pointer — not baked scalar
                    .arg(&count_i32)
                    .arg(&wd_mask_ptr)       // G1: per-param weight decay mask (sub-slice)
                    .arg(&l1_end_arg)        // G8: L1 boundary (trunk only)
                    .arg(&l1_lambda_arg)     // G8: SP4 Layer B: ISV-driven for trunk; 0 elsewhere
                    .arg(&weight_clamp_max_abs) // SP4 Layer B: per-group ISV-driven
                    .arg(&nan_flags_ptr)        // SP4 A14: sticky engage flag store
                    .arg(&diag_slot)            // SP4 A14: diag slot (50 for dqn_adam)
                    .arg(&engage_buf_ptr)       // SP4 A14: per-block engagement buf
                    .arg(&engage_buf_offset)    // SP4 Layer B: per-group offset; SP4_ENGAGE_OFFSET_DISABLED for trunk-extras tail
                    .arg(&lr_scale)             // SP21 Phase 4: per-branch LR scale (1.0 for non-branch groups)
                    .launch(cfg)
                    .map_err(|e| {
                        MLError::ModelError(format!(
                            "dqn_adam_update_kernel[{:?} byte_off={}] launch: {e}",
                            group, byte_off
                        ))
                    })?;
            }
        }

        Ok(())
    }

    /// SP4 Task A15: host-side Pearl C engagement-rate-deficit check.
    ///
    /// Reads per-block engagement counts from
    /// `clamp_engage_per_block_buf` (mapped pinned, zero-copy host
    /// access), reduces to a single rate, computes the deficit relative
    /// to the theoretical p99 baseline (1%), then applies Pearls A + D
    /// (`pearls_ad_update`) to maintain the rate-deficit EMA per param
    /// group. For Layer A this method only updates the diagnostic EMA
    /// and emits a `tracing::debug!` line when the EMA exceeds 0.005
    /// (sustained over-engagement). Layer B will read the EMA and
    /// force-bump `ISV[WEIGHT_BOUND[group]]`.
    ///
    /// Curiosity reads four sub-launch ranges
    /// (`SP4_ENGAGE_OFFSET_CURIOSITY_{W1,B1,W2,B2}`) so the four
    /// internal Adam updates' counts are summed into a single group
    /// engagement rate. All other groups read the single contiguous
    /// `[group_idx × MAX_BLOCKS_PER_ADAM, +MAX_BLOCKS_PER_ADAM)` range.
    ///
    /// **Layer B (2026-05-01) — split sub-launches landed.** The main
    /// DQN Adam now runs as 3 SP4-canonical sub-launches (Trunk / Value
    /// / Branches), each writing per-block engagement counts at its own
    /// group offset. `pearl_c_post_adam_engagement_check(group, count)`
    /// reads the dedicated range for any of the 8 SP4 groups directly.
    /// The Layer A "groups 1/2 accounted under group 0" limitation is
    /// resolved by the same Layer B atomic commit that introduces the
    /// `WEIGHT_BOUND[group]` consumer migration.
    ///
    /// **Layer B fix-up (2026-05-01)** — the 4th sub-launch (trunk-
    /// extras tail covering tensors `[33..163)`) is tagged DqnTrunk for
    /// the WEIGHT_BOUND/WD_RATE bounds contract but uses
    /// `SP4_ENGAGE_OFFSET_DISABLED` for Pearl C. Engagement for those
    /// tensors **folds into the canonical DqnTrunk count** (same bound
    /// drives both clamps; Pearl C is a per-bound diagnostic). Callers
    /// passing `ParamGroup::DqnTrunk` here only see the canonical
    /// trunk's engagement counts in the buffer; trunk-extras' clamp
    /// engagements do not affect the rate-deficit EMA. This keeps
    /// engagement accounting faithful to the WEIGHT_BOUND[DqnTrunk]
    /// contract Pearl C is regulating.
    pub(crate) fn pearl_c_post_adam_engagement_check(
        &mut self,
        group: ParamGroup,
        param_count: usize,
    ) -> Result<(), MLError> {
        use super::sp4_wiener_ema::{pearls_ad_update, WienerState};

        if param_count == 0 {
            return Ok(());
        }

        let group_idx = group.idx();
        // Curiosity sums 4 sub-launch ranges (W1/b1/W2/b2). SP21 Phase 4.5
        // (2026-05-11): DqnBranches now also sums 4 sub-launch ranges
        // (Dir/Mag/Order/Urgency) — Dir at canonical group_idx=2 offset,
        // Mag/Order/Urgency at extra slots 11/12/13. All other groups
        // read a single contiguous range starting at
        // `group_idx × MAX_BLOCKS_PER_ADAM`.
        let block_offsets: &[usize] = match group {
            ParamGroup::Curiosity => &[
                7 * MAX_BLOCKS_PER_ADAM,   // W1
                8 * MAX_BLOCKS_PER_ADAM,   // b1
                9 * MAX_BLOCKS_PER_ADAM,   // W2
                10 * MAX_BLOCKS_PER_ADAM,  // b2
            ],
            ParamGroup::DqnBranches => &[
                2 * MAX_BLOCKS_PER_ADAM,   // Dir (canonical DqnBranches slot)
                11 * MAX_BLOCKS_PER_ADAM,  // Mag (extra)
                12 * MAX_BLOCKS_PER_ADAM,  // Order (extra)
                13 * MAX_BLOCKS_PER_ADAM,  // Urgency (extra)
            ],
            _ => &[
                /* group_idx * MAX_BLOCKS_PER_ADAM — fall back to
                 * runtime computation in the loop body. */
                0
            ],
        };

        // Reduce per-block counts → total engagement count.
        let mut total_engage: i64 = 0;
        // Safety: `clamp_engage_per_block_buf.host_ptr` is a mapped-pinned
        // i32 buffer of `SP4_ENGAGE_BUF_LEN` elements — valid for the
        // lifetime of the trainer. Reads are zero-copy via the device-
        // mapped page; the kernel writes were ordered by Adam-launch
        // synchronization upstream of this call.
        let multi_range = matches!(group, ParamGroup::Curiosity | ParamGroup::DqnBranches);
        unsafe {
            let p = self.clamp_engage_per_block_buf.host_ptr as *const i32;
            if multi_range {
                for &base in block_offsets {
                    for b in 0..MAX_BLOCKS_PER_ADAM {
                        total_engage += *p.add(base + b) as i64;
                    }
                }
            } else {
                let base = group_idx * MAX_BLOCKS_PER_ADAM;
                for b in 0..MAX_BLOCKS_PER_ADAM {
                    total_engage += *p.add(base + b) as i64;
                }
            }
        }

        let engagement_rate = (total_engage as f32) / (param_count as f32);
        let rate_deficit = engagement_rate - 0.01_f32;

        // Apply Pearls A + D to the rate-deficit time series. The EMA
        // surrogate (`prev_x_mean`) lives in `pearl_c_rate_deficit_ema`
        // (host-only), and the (sample_var, diff_var, x_lag) Wiener
        // triple lives in `pearl_c_rate_deficit_state_buf` at offset
        // `group_idx * 3`.
        let prev_x_mean = self.pearl_c_rate_deficit_ema[group_idx];
        let mut state = unsafe {
            let p = self.pearl_c_rate_deficit_state_buf.host_ptr as *const f32;
            let off = group_idx * 3;
            WienerState {
                sample_var: *p.add(off),
                diff_var:   *p.add(off + 1),
                x_lag:      *p.add(off + 2),
            }
        };
        let new_x_mean = pearls_ad_update(prev_x_mean, &mut state, rate_deficit);
        self.pearl_c_rate_deficit_ema[group_idx] = new_x_mean;
        unsafe {
            let p = self.pearl_c_rate_deficit_state_buf.host_ptr as *mut f32;
            let off = group_idx * 3;
            *p.add(off)     = state.sample_var;
            *p.add(off + 1) = state.diff_var;
            *p.add(off + 2) = state.x_lag;
        }

        // Layer A: log only on sustained over-engagement. Layer B will
        // mutate `ISV[WEIGHT_BOUND[group]]` instead.
        if new_x_mean > 0.005_f32 {
            tracing::debug!(
                target: "sp4.pearl_c",
                group = ?group,
                param_count,
                engagement_rate,
                rate_deficit,
                rate_deficit_ema = new_x_mean,
                "SP4 Pearl C: sustained engagement-rate deficit \
                 (>0.005 EMA) — Layer A diagnostic; Layer B will \
                 force-bump WEIGHT_BOUND."
            );
        }

        // Zero per-block counts post-read so next step starts clean.
        // Curiosity + DqnBranches (SP21 Phase 4.5) zero all 4 sub-ranges;
        // others zero one range.
        unsafe {
            let p = self.clamp_engage_per_block_buf.host_ptr as *mut i32;
            if multi_range {
                for &base in block_offsets {
                    std::ptr::write_bytes(p.add(base), 0, MAX_BLOCKS_PER_ADAM);
                }
            } else {
                let base = group_idx * MAX_BLOCKS_PER_ADAM;
                std::ptr::write_bytes(p.add(base), 0, MAX_BLOCKS_PER_ADAM);
            }
        }

        Ok(())
    }

    // ═══════════════════════════════════════════════════════════════════
    // Weight flattening / unflattening (GPU d2d — zero host roundtrip)
    // ═══════════════════════════════════════════════════════════════════

    /// Copy 24 individual f32 weight tensors into the flat `params_buf`.
    ///
    /// Order matches the GOFF_* layout in `dqn_training_kernel.cu`.
    /// Pure device-to-device copies via `cuMemcpyDtoDAsync` — zero host roundtrip.
    /// Indices 24-25 (bottleneck) are not backed by individual CudaSlice fields
    /// and are left untouched in the flat buffer.
    pub(crate) fn flatten_online_weights(
        &self,
        online_d: &DuelingWeightSet,
        online_b: &BranchingWeightSet,
    ) -> Result<(), MLError> {
        let sizes = compute_param_sizes(&self.config);
        // Flatten into params_buf (GemmEx + Adam both read/write this).
        let dst_base = self.params_buf.raw_ptr();

        // Per-tensor byte offsets across the GRN-expanded layout.  The same
        // prefix-sum `f32_weight_ptrs_from_base` and `from_flat_buffer` use,
        // ensuring all consumers agree on a single canonical layout.
        let mut byte_offsets = [0u64; NUM_WEIGHT_TENSORS];
        {
            let mut acc: u64 = 0;
            for i in 0..NUM_WEIGHT_TENSORS {
                byte_offsets[i] = acc;
                acc += (align4(sizes[i]) * std::mem::size_of::<f32>()) as u64;
            }
        }

        // Ordered as the 12 DWS fields followed by the 12 BWS fields. Each
        // entry pairs (source pointer, source length, GRN-layout index).
        let ptrs: [(u64, usize, usize); 24] = [
            (online_d.w_s1, online_d.w_s1_len, super::gpu_weights::DUELING_FLAT_INDICES[0]),
            (online_d.b_s1, online_d.b_s1_len, super::gpu_weights::DUELING_FLAT_INDICES[1]),
            (online_d.w_s2, online_d.w_s2_len, super::gpu_weights::DUELING_FLAT_INDICES[2]),
            (online_d.b_s2, online_d.b_s2_len, super::gpu_weights::DUELING_FLAT_INDICES[3]),
            (online_d.w_v1, online_d.w_v1_len, super::gpu_weights::DUELING_FLAT_INDICES[4]),
            (online_d.b_v1, online_d.b_v1_len, super::gpu_weights::DUELING_FLAT_INDICES[5]),
            (online_d.w_v2, online_d.w_v2_len, super::gpu_weights::DUELING_FLAT_INDICES[6]),
            (online_d.b_v2, online_d.b_v2_len, super::gpu_weights::DUELING_FLAT_INDICES[7]),
            (online_d.w_a1, online_d.w_a1_len, super::gpu_weights::DUELING_FLAT_INDICES[8]),
            (online_d.b_a1, online_d.b_a1_len, super::gpu_weights::DUELING_FLAT_INDICES[9]),
            (online_d.w_a2, online_d.w_a2_len, super::gpu_weights::DUELING_FLAT_INDICES[10]),
            (online_d.b_a2, online_d.b_a2_len, super::gpu_weights::DUELING_FLAT_INDICES[11]),
            (online_b.w_bo1, online_b.w_bo1_len, super::gpu_weights::BRANCHING_FLAT_INDICES[0]),
            (online_b.b_bo1, online_b.b_bo1_len, super::gpu_weights::BRANCHING_FLAT_INDICES[1]),
            (online_b.w_bo2, online_b.w_bo2_len, super::gpu_weights::BRANCHING_FLAT_INDICES[2]),
            (online_b.b_bo2, online_b.b_bo2_len, super::gpu_weights::BRANCHING_FLAT_INDICES[3]),
            (online_b.w_bu1, online_b.w_bu1_len, super::gpu_weights::BRANCHING_FLAT_INDICES[4]),
            (online_b.b_bu1, online_b.b_bu1_len, super::gpu_weights::BRANCHING_FLAT_INDICES[5]),
            (online_b.w_bu2, online_b.w_bu2_len, super::gpu_weights::BRANCHING_FLAT_INDICES[6]),
            (online_b.b_bu2, online_b.b_bu2_len, super::gpu_weights::BRANCHING_FLAT_INDICES[7]),
            (online_b.w_bg1, online_b.w_bg1_len, super::gpu_weights::BRANCHING_FLAT_INDICES[8]),
            (online_b.b_bg1, online_b.b_bg1_len, super::gpu_weights::BRANCHING_FLAT_INDICES[9]),
            (online_b.w_bg2, online_b.w_bg2_len, super::gpu_weights::BRANCHING_FLAT_INDICES[10]),
            (online_b.b_bg2, online_b.b_bg2_len, super::gpu_weights::BRANCHING_FLAT_INDICES[11]),
        ];

        let weight_names = [
            "w_s1", "b_s1", "w_s2", "b_s2",
            "w_v1", "b_v1", "w_v2", "b_v2",
            "w_a1", "b_a1", "w_a2", "b_a2",
            "w_bo1", "b_bo1", "w_bo2", "b_bo2",
            "w_bu1", "b_bu1", "w_bu2", "b_bu2",
            "w_bg1", "b_bg1", "w_bg2", "b_bg2",
        ];

        for (i, &(src, actual, layout_idx)) in ptrs.iter().enumerate() {
            if actual != sizes[layout_idx] {
                return Err(MLError::ModelError(format!(
                    "flatten[{}] ({}) layout_idx={} size mismatch: src has {} elems, expected {}",
                    i, weight_names[i], layout_idx, actual, sizes[layout_idx],
                )));
            }
            let num_bytes = sizes[layout_idx] * std::mem::size_of::<f32>();
            dtod_copy_async(dst_base + byte_offsets[layout_idx], src, num_bytes, &self.stream, i, "flatten")?;
        }

        Ok(())
    }

    /// No-op when weight sets are already pointer views into params_buf.
    ///
    /// When DuelingWeightSet/BranchingWeightSet point directly into params_buf
    /// (created via `from_flat_buffer()`), unflatten is a no-op — the pointers
    /// already reference the correct regions. When the weight sets point at
    /// separate allocations (experience collector, ensemble), this copies the
    /// flat buffer content back to the individual regions.
    pub(crate) fn unflatten_online_weights(
        &self,
        online_d: &DuelingWeightSet,
        online_b: &BranchingWeightSet,
    ) -> Result<(), MLError> {
        let sizes = compute_param_sizes(&self.config);
        let src_base = self.params_buf.raw_ptr();

        // Check if the weight set points directly into params_buf (zero-copy path).
        // If w_s1 points at the start of params_buf, all pointers are views — skip D2D.
        if online_d.w_s1 == src_base {
            return Ok(());
        }

        // Per-tensor byte offsets across the GRN-expanded layout (matches
        // `flatten_online_weights` and `from_flat_buffer`).
        let mut byte_offsets = [0u64; NUM_WEIGHT_TENSORS];
        {
            let mut acc: u64 = 0;
            for i in 0..NUM_WEIGHT_TENSORS {
                byte_offsets[i] = acc;
                acc += (align4(sizes[i]) * std::mem::size_of::<f32>()) as u64;
            }
        }

        // Ordered as the 12 DWS fields followed by the 12 BWS fields, paired
        // with their canonical layout index.
        let ptrs: [(u64, usize); 24] = [
            (online_d.w_s1, super::gpu_weights::DUELING_FLAT_INDICES[0]),
            (online_d.b_s1, super::gpu_weights::DUELING_FLAT_INDICES[1]),
            (online_d.w_s2, super::gpu_weights::DUELING_FLAT_INDICES[2]),
            (online_d.b_s2, super::gpu_weights::DUELING_FLAT_INDICES[3]),
            (online_d.w_v1, super::gpu_weights::DUELING_FLAT_INDICES[4]),
            (online_d.b_v1, super::gpu_weights::DUELING_FLAT_INDICES[5]),
            (online_d.w_v2, super::gpu_weights::DUELING_FLAT_INDICES[6]),
            (online_d.b_v2, super::gpu_weights::DUELING_FLAT_INDICES[7]),
            (online_d.w_a1, super::gpu_weights::DUELING_FLAT_INDICES[8]),
            (online_d.b_a1, super::gpu_weights::DUELING_FLAT_INDICES[9]),
            (online_d.w_a2, super::gpu_weights::DUELING_FLAT_INDICES[10]),
            (online_d.b_a2, super::gpu_weights::DUELING_FLAT_INDICES[11]),
            (online_b.w_bo1, super::gpu_weights::BRANCHING_FLAT_INDICES[0]),
            (online_b.b_bo1, super::gpu_weights::BRANCHING_FLAT_INDICES[1]),
            (online_b.w_bo2, super::gpu_weights::BRANCHING_FLAT_INDICES[2]),
            (online_b.b_bo2, super::gpu_weights::BRANCHING_FLAT_INDICES[3]),
            (online_b.w_bu1, super::gpu_weights::BRANCHING_FLAT_INDICES[4]),
            (online_b.b_bu1, super::gpu_weights::BRANCHING_FLAT_INDICES[5]),
            (online_b.w_bu2, super::gpu_weights::BRANCHING_FLAT_INDICES[6]),
            (online_b.b_bu2, super::gpu_weights::BRANCHING_FLAT_INDICES[7]),
            (online_b.w_bg1, super::gpu_weights::BRANCHING_FLAT_INDICES[8]),
            (online_b.b_bg1, super::gpu_weights::BRANCHING_FLAT_INDICES[9]),
            (online_b.w_bg2, super::gpu_weights::BRANCHING_FLAT_INDICES[10]),
            (online_b.b_bg2, super::gpu_weights::BRANCHING_FLAT_INDICES[11]),
        ];

        for (i, &(dst, layout_idx)) in ptrs.iter().enumerate() {
            let num_bytes = sizes[layout_idx] * std::mem::size_of::<f32>();
            dtod_copy_async(dst, src_base + byte_offsets[layout_idx], num_bytes, &self.stream, i, "unflatten")?;
        }

        Ok(())
    }

    // ═══════════════════════════════════════════════════════════════════
    // Xavier initialization (directly into flat buffer)
    // ═══════════════════════════════════════════════════════════════════

    /// Xavier-uniform initialize `params_buf` and `target_params_buf` on CPU, then HtoD.
    ///
    /// Uses `compute_param_sizes()` for correct bottleneck-aware tensor dimensions
    /// (s1_input_dim = bottleneck_dim + portfolio_dim, NOT state_dim). Each weight
    /// matrix gets Xavier uniform init; biases are zero. The flat buffer layout
    /// matches the GOFF_* defines exactly, with `align4()` padding per tensor.
    ///
    /// Both params_buf and target_params_buf receive identical weights — the EMA
    /// kernel will diverge them during training. ISV weights (indices 68-79) in
    /// target_params_buf are inert: EMA skips them, and the target forward never
    /// reads them.
    pub(crate) fn xavier_init_params_buf(&mut self) -> Result<(), MLError> {
        let cfg = &self.config;
        let sizes = compute_param_sizes(cfg);
        let total = self.total_params;

        // Compute s1_input_dim matching compute_param_sizes — SP15 Wave 4.1b
        // bumps the bottleneck-on path by +1 for the dd_pct column. Xavier
        // sees the full (s1_input_dim, shared_h1) fan and initialises every
        // column uniformly, including the new dd_pct column at index
        // `bn_dim + portfolio_dim`. The dd_pct value at runtime is bounded
        // ∈ [0, 1] (Wave 1.3 contract), matching Xavier's small-magnitude
        // assumption — no special-case zero-init needed (cf. SP14 zeroed the
        // aux_softmax_diff column of w_b0fc because that signal can be ±1
        // which Xavier assumes is zero-mean; dd_pct is non-negative but
        // similarly small-magnitude, so the gradient flows symmetrically
        // from day 0).
        let s1_input_dim = if cfg.bottleneck_dim > 0 {
            cfg.bottleneck_dim + ml_core::state_layout::STATE_DIM.saturating_sub(cfg.market_dim) + 1
        } else {
            ml_core::state_layout::STATE_DIM
        };

        // (fan_out, fan_in) for weight tensors; (0, 0) for biases.
        // Weight matrices: limit = sqrt(6 / (fan_in + fan_out))
        // Plan 4 Task 2c.3a: trunk reshuffled — 4 Linear tensors removed,
        // 13 GRN tensors inserted (h_s1 GRN: Linear_a, Linear_b, Linear_residual,
        // gamma, beta — 7 tensors; h_s2 GRN: Linear_a, Linear_b, gamma, beta —
        // 6 tensors since SH1==SH2 makes Linear_residual identity). Every
        // index ≥ 4 in the OLD layout shifts +9 in the NEW layout.
        let mut fan_dims: [(usize, usize); NUM_WEIGHT_TENSORS] = [(0, 0); NUM_WEIGHT_TENSORS];
        let core_fan: [(usize, usize); 95] = [
            // ── h_s1 GRN ──
            (cfg.shared_h1, s1_input_dim),                           // [0]  w_a_h_s1(Kaiming/Xavier)
            (0, 0),                                                   // [1]  b_a_h_s1 (zero)
            (2 * cfg.shared_h1, cfg.shared_h1),                      // [2]  w_b_h_s1 (Xavier)
            (0, 0),                                                   // [3]  b_b_h_s1 (zero)
            (cfg.shared_h1, s1_input_dim),                           // [4]  w_residual_h_s1 (Xavier)
            (0, 0),                                                   // [5]  gamma_h_s1 (special init: 1.0)
            (0, 0),                                                   // [6]  beta_h_s1 (zero)
            // ── h_s2 GRN ──
            (cfg.shared_h2, cfg.shared_h1),                          // [7]  w_a_h_s2 (Xavier)
            (0, 0),                                                   // [8]  b_a_h_s2 (zero)
            (2 * cfg.shared_h2, cfg.shared_h2),                      // [9]  w_b_h_s2 (Xavier)
            (0, 0),                                                   // [10] b_b_h_s2 (zero)
            (0, 0),                                                   // [11] gamma_h_s2 (special init: 1.0)
            (0, 0),                                                   // [12] beta_h_s2 (zero)
            (cfg.value_h, cfg.shared_h2),                            // [13] w_v1
            (0, 0),                                                   // [14] b_v1
            (cfg.num_atoms, cfg.value_h),                            // [15] w_v2
            (0, 0),                                                   // [16] b_v2
            (cfg.adv_h, cfg.shared_h2 + 1),                          // [17] w_b0fc (SP14 aux→Q wire: SH2+1 input)
            (0, 0),                                                   // [18] b_b0fc
            (cfg.branch_0_size * cfg.num_atoms, cfg.adv_h),         // [19] w_b0out
            (0, 0),                                                   // [20] b_b0out
            (cfg.adv_h, cfg.shared_h2),                              // [21] w_b1fc
            (0, 0),                                                   // [22] b_b1fc
            (cfg.branch_1_size * cfg.num_atoms, cfg.adv_h),         // [23] w_b1out
            (0, 0),                                                   // [24] b_b1out
            (cfg.adv_h, cfg.shared_h2 + 3),                          // [25] w_b2fc (OFI-conditioned: SH2+3 input)
            (0, 0),                                                   // [26] b_b2fc
            (cfg.branch_2_size * cfg.num_atoms, cfg.adv_h),         // [27] w_b2out
            (0, 0),                                                   // [28] b_b2out
            (cfg.adv_h, cfg.shared_h2 + 3),                         // [29] w_b3fc (OFI-conditioned: SH2+3 input)
            (0, 0),                                                   // [30] b_b3fc
            (cfg.branch_3_size * cfg.num_atoms, cfg.adv_h),         // [31] w_b3out
            (0, 0),                                                   // [32] b_b3out
            (cfg.bottleneck_dim, cfg.market_dim),                    // [33] w_bn
            (0, 0),                                                   // [34] b_bn
            (cfg.shared_h2, 16),                                      // [35] w_vsn1_0 (Xavier)
            (0, 0),                                                   // [36] w_vsn2_0 (ZERO — uniform softmax)
            (cfg.shared_h2, 16),                                      // [37] w_vsn1_1
            (0, 0),                                                   // [38] w_vsn2_1
            (cfg.shared_h2, 16),                                      // [39] w_vsn1_2
            (0, 0),                                                   // [40] w_vsn2_2
            (cfg.shared_h2, 16),                                      // [41] w_vsn1_3
            (0, 0),                                                   // [42] w_vsn2_3
            (0, 0),                                                   // [43] w_gate_0 (ZERO — sigmoid=0.5)
            (0, 0),                                                   // [44] b_gate_0
            (0, 0),                                                   // [45] w_gate_1
            (0, 0),                                                   // [46] b_gate_1
            (0, 0),                                                   // [47] w_gate_2
            (0, 0),                                                   // [48] b_gate_2
            (0, 0),                                                   // [49] w_gate_3
            (0, 0),                                                   // [50] b_gate_3
            (0, 0),                                                   // [51] kan_coeff_0 — special init (sigmoid approx)
            (0, 0),                                                   // [52] kan_resid_0 — zero
            (0, 0),                                                   // [53] kan_coeff_1
            (0, 0),                                                   // [54] kan_resid_1
            (0, 0),                                                   // [55] kan_coeff_2
            (0, 0),                                                   // [56] kan_resid_2
            (0, 0),                                                   // [57] kan_coeff_3
            (0, 0),                                                   // [58] kan_resid_3
            (4, 4),                                                   // [59] w_regime (Xavier)
            (0, 0),                                                   // [60] b_regime (zero)
            (0, 0),                                                   // [61] spacing_raw_0 (zero = uniform)
            (0, 0),                                                   // [62] spacing_raw_1
            (0, 0),                                                   // [63] spacing_raw_2
            (0, 0),                                                   // [64] spacing_raw_3
            (cfg.value_h, cfg.shared_h2),                            // [65] w_v1_5bar (Xavier)
            (0, 0),                                                   // [66] b_v1_5bar
            (cfg.num_atoms, cfg.value_h),                            // [67] w_v2_5bar (Xavier)
            (0, 0),                                                   // [68] b_v2_5bar
            (cfg.value_h, cfg.shared_h2),                            // [69] w_v1_20bar (Xavier)
            (0, 0),                                                   // [70] b_v1_20bar
            (cfg.num_atoms, cfg.value_h),                            // [71] w_v2_20bar (Xavier)
            (0, 0),                                                   // [72] b_v2_20bar
            // ── Learned risk management (5th branch) ──
            (cfg.adv_h, cfg.shared_h2 + 13),                           // [73] w_risk_fc (Xavier, SH2+13 input)
            (0, 0),                                                   // [74] b_risk_fc (zero)
            (0, 0),                                                   // [75] w_risk_out (special init below)
            (0, 0),                                                   // [76] b_risk_out (zero → sigmoid(0)=0.5)
            // ── ISV encoder + conditioning ──
            (16, ISV_DIM),                                            // [77] w_isv_fc1 (Xavier, ISV_DIM input)
            (0, 0),                                                   // [78] b_isv_fc1 (zero)
            (ISV_EMB_DIM, 16),                                        // [79] w_isv_fc2 (Xavier, EMB output)
            (0, 0),                                                   // [80] b_isv_fc2 (zero)
            (4, ISV_EMB_DIM),                                         // [81] w_isv_gate (Xavier, EMB input)
            (0, 0),                                                   // [82] b_isv_gate (zero)
            (0, 0),                                                   // [83] w_isv_gamma (zero → sigmoid(0)=0.5)
            (0, 0),                                                   // [84] b_isv_gamma (zero)
            (1, cfg.shared_h2),                                       // [85] w_conf_fc (Xavier)
            (0, 0),                                                   // [86] b_conf_fc (zero)
            // ── ISV Feature Gating (Phase 3) ──
            (cfg.shared_h2, ISV_EMB_DIM),                             // [87] w_feature_gate (Xavier)
            (0, 0),                                                   // [88] b_feature_gate (special init: 2.0)
            // ── ISV Temporal Routing (Phase 3) ──
            (cfg.shared_h2, ISV_EMB_DIM),                             // [89] w_temporal_route (Xavier)
            (0, 0),                                                   // [90] b_temporal_route (special init: 2.0)
            // ── Trade Plan Head ──
            (cfg.adv_h, cfg.shared_h2),                               // [91] w_plan_fc (Xavier)
            (0, 0),                                                   // [92] b_plan_fc (zero)
            (6, cfg.adv_h),                                           // [93] w_plan_out (Xavier)
            (0, 0),                                                   // [94] b_plan_out (zero)
        ];
        fan_dims[..95].copy_from_slice(&core_fan);

        // ── Plan 4 Task 1B-ii: VSN per-group MLP fan dims (24 tensors) ─
        // Xavier on weights (`w1_g`, `w2_g`); zero on biases (`b1_g`, `b2_g`).
        // Initial logits ≈ 0 → softmax mask ≈ uniform 1/SL_NUM_FEATURE_GROUPS
        // per group at every sample (cold-start neutral, matches the ISV
        // mask-EMA cold-start of 1/6).
        {
            let group_ranges = ml_core::state_layout::FEATURE_GROUP_RANGES;
            let mut g = 0;
            let mut idx = 95;
            while g < ml_core::state_layout::SL_NUM_FEATURE_GROUPS {
                let group_dim = group_ranges[g].1 - group_ranges[g].0;
                fan_dims[idx]     = (VSN_HIDDEN_DIM, group_dim);   // w1_g [VSN_HIDDEN_DIM, group_dim_g] (Xavier)
                fan_dims[idx + 1] = (0, 0);                        // b1_g (zero)
                fan_dims[idx + 2] = (1, VSN_HIDDEN_DIM);           // w2_g [1, VSN_HIDDEN_DIM] (Xavier)
                fan_dims[idx + 3] = (0, 0);                        // b2_g (zero)
                idx += 4;
                g += 1;
            }
        }

        // ── Plan 4 Task 6 Commit A: aux-head fan dims (8 tensors at [119..127)) ─
        // Both heads: `Linear(SH2 → AUX_HIDDEN_DIM=32) → ELU → Linear(32 → K)`.
        // Xavier on weight matrices (uses standard `sqrt(6/(fan_in+fan_out))`);
        // zero on biases. Cold-start: predictions and logits ≈ 0 → next-bar
        // pred ≈ 0 (matches the typical centred next-bar return), regime
        // logits ≈ uniform softmax (no class preference).
        // Indices: aux_nb_w1=119, aux_nb_b1=120, aux_nb_w2=121, aux_nb_b2=122,
        //          aux_rg_w1=123, aux_rg_b1=124, aux_rg_w2=125, aux_rg_b2=126.
        {
            let h_dim = crate::cuda_pipeline::gpu_aux_heads::AUX_HIDDEN_DIM;
            let k_rg  = crate::cuda_pipeline::gpu_aux_heads::AUX_REGIME_K;
            // Next-bar head (K=1).
            fan_dims[119] = (h_dim, cfg.shared_h2);   // aux_nb_w1 [32, SH2] (Xavier)
            fan_dims[120] = (0, 0);                   // aux_nb_b1 (zero)
            fan_dims[121] = (1, h_dim);               // aux_nb_w2 [1, 32] (Xavier)
            fan_dims[122] = (0, 0);                   // aux_nb_b2 (zero)
            // Regime head (K=5).
            fan_dims[123] = (h_dim, cfg.shared_h2);   // aux_rg_w1 [32, SH2] (Xavier)
            fan_dims[124] = (0, 0);                   // aux_rg_b1 (zero)
            fan_dims[125] = (k_rg, h_dim);            // aux_rg_w2 [5, 32] (Xavier)
            fan_dims[126] = (0, 0);                   // aux_rg_b2 (zero)
        }

        // ── SP22 H6 vNext Phase B1 (2026-05-14): trade-outcome head fan dims ──
        // Same shape pattern as aux_nb_* (K=2 sibling) — Xavier on W
        // matrices, zero on biases. Cold-start: logits ≈ 0 → softmax ≈
        // uniform 1/3 → no class preference for {Profit, Stop, Timeout}.
        // The label producer's sparse-label semantics (most bars masked)
        // mean the head trains only on actual trade-close events, so
        // cold-start neutrality is the correct starting point per
        // pearl_first_observation_bootstrap.
        //
        // Indices: aux_to_w1=163, aux_to_b1=164, aux_to_w2=165, aux_to_b2=166.
        {
            let h_dim = crate::cuda_pipeline::gpu_aux_heads::AUX_HIDDEN_DIM;
            let k_to  = crate::cuda_pipeline::gpu_aux_heads::AUX_OUTCOME_K;
            fan_dims[163] = (h_dim, cfg.shared_h2);   // aux_to_w1 [H, SH2] (Xavier)
            fan_dims[164] = (0, 0);                   // aux_to_b1 (zero)
            fan_dims[165] = (k_to, h_dim);            // aux_to_w2 [K=3, H] (Xavier)
            fan_dims[166] = (0, 0);                   // aux_to_b2 (zero)
        }

        // ── Phase 1 Task 1.6: MoE fan dims (36 tensors at [127..163)) ─────────
        // Gate: zero-init (fan_out=0,fan_in=0) so g(s) = uniform 1/K at cold start.
        // Experts: Xavier init on w1/w2 for initial differentiation; zero on biases.
        {
            let sd = ml_core::state_layout::STATE_DIM;
            let btn = MOE_EXPERT_BOTTLENECK;
            let sh2 = cfg.shared_h2;
            // Gate tensors [127..131) — zero-init: softmax of zero-logits = uniform 1/K
            fan_dims[127] = (0, 0); // gate_w1 [SD, MOE_GATE_HIDDEN] (zero-init per spec)
            fan_dims[128] = (0, 0); // gate_b1 [MOE_GATE_HIDDEN]
            fan_dims[129] = (0, 0); // gate_w2 [MOE_GATE_HIDDEN, K] (zero-init per spec)
            fan_dims[130] = (0, 0); // gate_b2 [MOE_NUM_EXPERTS]
            // Expert tensors [131..163) — Xavier on w1/w2, zero on b1/b2
            let _ = sd; // used indirectly via gate tensors above
            let mut eidx = 131usize;
            let mut k = 0usize;
            while k < MOE_NUM_EXPERTS {
                fan_dims[eidx]     = (btn, sh2);  // expert_k_w1 [SH2→BTN] (Xavier fan_out=BTN,fan_in=SH2)
                fan_dims[eidx + 1] = (0, 0);       // expert_k_b1 [BTN] (zero)
                fan_dims[eidx + 2] = (sh2, btn);  // expert_k_w2 [BTN→SH2] (Xavier fan_out=SH2,fan_in=BTN)
                fan_dims[eidx + 3] = (0, 0);       // expert_k_b2 [SH2] (zero)
                eidx += 4;
                k += 1;
            }
        }

        // Build flat host buffer: Xavier init for weights, zeros for biases + padding.
        let cutlass_tile_pad = 32 * cfg.adv_h.max(cfg.value_h);
        let buf_len = total + cutlass_tile_pad;
        let mut host_buf = vec![0.0_f32; buf_len];
        // LCG PRNG (same family as spectral norm init in this file)
        let mut rng = 0xCAFE_BABE_u64.wrapping_add(total as u64);
        let mut offset = 0usize;

        for i in 0..NUM_WEIGHT_TENSORS {
            let (fan_out, fan_in) = fan_dims[i];
            let n = sizes[i];

            if fan_out > 0 && fan_in > 0 {
                // Xavier uniform: U[-limit, limit]
                let limit = (6.0_f64 / (fan_in + fan_out) as f64).sqrt() as f32;
                for w in &mut host_buf[offset..offset + n] {
                    rng = rng.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
                    let u = (rng >> 33) as f32 / (1u64 << 31) as f32 - 0.5;
                    *w = u * 2.0 * limit;
                }
            }
            // Biases and zero-size tensors stay zero from vec init
            offset += align4(n);
        }

        // ── Zero the last 3 OFI columns of w_b2fc [25] and w_b3fc [29] ──
        // xavier_init fills the full tensor (adv_h × (shared_h2+3)) but the OFI
        // columns should start at zero so the network learns to use them gradually.
        // (gate weights w_gate_2/w_gate_3 are already zero via (0,0) fan_dims.)
        for &fc_idx in &[25_usize, 29_usize] {
            let base: usize = sizes[..fc_idx].iter().map(|&s| align4(s)).sum();
            let sh2 = cfg.shared_h2;
            let sh2p3 = sh2 + 3;
            let ah = cfg.adv_h;
            // w_b2fc/w_b3fc layout: [AH, SH2+3] row-major.
            // Zero columns [SH2, SH2+1, SH2+2] for all AH rows.
            for row in 0..ah {
                for col in sh2..sh2p3 {
                    let idx = base + row * sh2p3 + col;
                    if idx < host_buf.len() {
                        host_buf[idx] = 0.0_f32;
                    }
                }
            }
        }

        // ── SP14 forward wire: zero the last (aux_softmax_diff) column of w_b0fc [17] ──
        // xavier_init fills the full tensor (adv_h × (shared_h2 + 1)). The new
        // column at index SH2 carries `softmax_up - softmax_down` from the aux
        // head; starting it at zero means the model initially ignores the new
        // input and learns to use it through gradient descent. Xavier-init on
        // this column would inject day-0 noise that the trunk would have to
        // denoise — we let the EGF gate decide when the signal is trustworthy.
        // (Mirrors the OFI column zeroing for w_b2fc/w_b3fc above.)
        {
            let fc_idx = 17_usize;
            let base: usize = sizes[..fc_idx].iter().map(|&s| align4(s)).sum();
            let sh2 = cfg.shared_h2;
            let sh2p1 = sh2 + 1;
            let ah = cfg.adv_h;
            // w_b0fc layout: [AH, SH2+1] row-major. Zero column [SH2] for all AH rows.
            for row in 0..ah {
                let idx = base + row * sh2p1 + sh2;
                if idx < host_buf.len() {
                    host_buf[idx] = 0.0_f32;
                }
            }
        }

        // ── KAN spline coefficient init: uniform 0.125 (flat 0.5 gate ≈ sigmoid(0)) ──
        // Indices 51, 53, 55, 57 are kan_coeff_d [AH, 8]. Fill with 1/8 so the
        // spline sums to 0.5 regardless of input (mimics sigmoid(0) = 0.5).
        // Residual weights (52, 54, 56, 58) stay zero.
        for &coeff_idx in &[51_usize, 53, 55, 57] {
            let coeff_offset: usize = sizes[..coeff_idx].iter().map(|&s| align4(s)).sum();
            let n_coeff = sizes[coeff_idx];
            for w in &mut host_buf[coeff_offset..coeff_offset + n_coeff] {
                *w = 0.125_f32;
            }
        }

        // ── GRN LayerNorm γ init: gamma_h_s1 [5], gamma_h_s2 [11] → 1.0 ──
        // Plan 4 Task 2c.3a. β stays at zero (init via vec! default).
        for &gamma_idx in &[5_usize, 11_usize] {
            let gamma_offset: usize = sizes[..gamma_idx].iter().map(|&s| align4(s)).sum();
            let n_gamma = sizes[gamma_idx];
            for w in &mut host_buf[gamma_offset..gamma_offset + n_gamma] {
                *w = 1.0_f32;
            }
        }

        // Feature gate bias: init to 2.0 for near-pass-through (sigmoid(2)≈0.88)
        {
            let b_gate_offset: usize = sizes[..88].iter().map(|&s| align4(s)).sum();
            for j in 0..cfg.shared_h2 {
                host_buf[b_gate_offset + j] = 2.0;
            }
        }

        // Temporal route bias: init to 2.0 (sigmoid(2)≈0.88 → mostly use temporal context)
        {
            let b_route_offset: usize = sizes[..90].iter().map(|&s| align4(s)).sum();
            for j in 0..cfg.shared_h2 {
                host_buf[b_route_offset + j] = 2.0;
            }
        }

        debug_assert_eq!(offset, total, "xavier_init: offset mismatch with total_params");

        // HtoD to both params_buf and target_params_buf (same initial weights).
        // Use raw cuMemcpyHtoD because params_buf.len() includes cutlass_tile_pad
        // but host_buf is sized to match exactly.
        let n_bytes = buf_len * std::mem::size_of::<f32>();
        let result = unsafe {
            cudarc::driver::sys::cuMemcpyHtoDAsync_v2(
                self.params_buf.raw_ptr(),
                host_buf.as_ptr().cast(),
                n_bytes,
                self.stream.cu_stream(),
            )
        };
        if result != cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS {
            return Err(MLError::ModelError(format!("xavier_init params_buf HtoD: {result:?}")));
        }
        let result = unsafe {
            cudarc::driver::sys::cuMemcpyHtoDAsync_v2(
                self.target_params_buf.raw_ptr(),
                host_buf.as_ptr().cast(),
                n_bytes,
                self.stream.cu_stream(),
            )
        };
        if result != cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS {
            return Err(MLError::ModelError(format!("xavier_init target_params_buf HtoD: {result:?}")));
        }

        self.params_initialized = true;
        self.target_params_initialized = true;

        info!(total_params = total, s1_input_dim, "Xavier-initialized params_buf and target_params_buf");
        Ok(())
    }

    // ═══════════════════════════════��═══════════════════════════════════
    // GPU-native Polyak EMA target update
    // ═══════════════════════════════════════════════════════════════════

    /// Copy 24 individual f32 target weight tensors into the flat `target_params_buf`.
    ///
    /// Same GOFF_* layout as `flatten_online_weights()`.
    /// Pure device-to-device copies -- zero host roundtrip.
    /// Indices 24-25 (bottleneck) are not backed by individual CudaSlice fields.
    fn flatten_target_weights(
        &self,
        target_d: &DuelingWeightSet,
        target_b: &BranchingWeightSet,
    ) -> Result<(), MLError> {
        let sizes = compute_param_sizes(&self.config);
        let dst_base = self.target_params_buf.raw_ptr();

        // If target already points into target_params_buf, no-op.
        if target_d.w_s1 == dst_base {
            return Ok(());
        }

        // Per-tensor byte offsets across the GRN-expanded layout.
        let mut byte_offsets = [0u64; NUM_WEIGHT_TENSORS];
        {
            let mut acc: u64 = 0;
            for i in 0..NUM_WEIGHT_TENSORS {
                byte_offsets[i] = acc;
                acc += (align4(sizes[i]) * std::mem::size_of::<f32>()) as u64;
            }
        }

        let ptrs: [(u64, usize); 24] = [
            (target_d.w_s1, super::gpu_weights::DUELING_FLAT_INDICES[0]),
            (target_d.b_s1, super::gpu_weights::DUELING_FLAT_INDICES[1]),
            (target_d.w_s2, super::gpu_weights::DUELING_FLAT_INDICES[2]),
            (target_d.b_s2, super::gpu_weights::DUELING_FLAT_INDICES[3]),
            (target_d.w_v1, super::gpu_weights::DUELING_FLAT_INDICES[4]),
            (target_d.b_v1, super::gpu_weights::DUELING_FLAT_INDICES[5]),
            (target_d.w_v2, super::gpu_weights::DUELING_FLAT_INDICES[6]),
            (target_d.b_v2, super::gpu_weights::DUELING_FLAT_INDICES[7]),
            (target_d.w_a1, super::gpu_weights::DUELING_FLAT_INDICES[8]),
            (target_d.b_a1, super::gpu_weights::DUELING_FLAT_INDICES[9]),
            (target_d.w_a2, super::gpu_weights::DUELING_FLAT_INDICES[10]),
            (target_d.b_a2, super::gpu_weights::DUELING_FLAT_INDICES[11]),
            (target_b.w_bo1, super::gpu_weights::BRANCHING_FLAT_INDICES[0]),
            (target_b.b_bo1, super::gpu_weights::BRANCHING_FLAT_INDICES[1]),
            (target_b.w_bo2, super::gpu_weights::BRANCHING_FLAT_INDICES[2]),
            (target_b.b_bo2, super::gpu_weights::BRANCHING_FLAT_INDICES[3]),
            (target_b.w_bu1, super::gpu_weights::BRANCHING_FLAT_INDICES[4]),
            (target_b.b_bu1, super::gpu_weights::BRANCHING_FLAT_INDICES[5]),
            (target_b.w_bu2, super::gpu_weights::BRANCHING_FLAT_INDICES[6]),
            (target_b.b_bu2, super::gpu_weights::BRANCHING_FLAT_INDICES[7]),
            (target_b.w_bg1, super::gpu_weights::BRANCHING_FLAT_INDICES[8]),
            (target_b.b_bg1, super::gpu_weights::BRANCHING_FLAT_INDICES[9]),
            (target_b.w_bg2, super::gpu_weights::BRANCHING_FLAT_INDICES[10]),
            (target_b.b_bg2, super::gpu_weights::BRANCHING_FLAT_INDICES[11]),
        ];

        // Sync stream to catch any deferred errors from graph replay
        let sync_r = unsafe { cudarc::driver::sys::cuStreamSynchronize(self.stream.cu_stream()) };
        if sync_r != cudarc::driver::sys::CUresult::CUDA_SUCCESS {
            return Err(MLError::ModelError(format!("flatten_target pre-sync: {sync_r:?}")));
        }
        for (i, &(src, layout_idx)) in ptrs.iter().enumerate() {
            let num_bytes = sizes[layout_idx] * std::mem::size_of::<f32>();
            let dst = dst_base + byte_offsets[layout_idx];
            dtod_copy_async(dst, src, num_bytes, &self.stream, i, "flatten_target")?;
        }

        Ok(())
    }

    /// Scatter flat target_params_buf back to individual target weight pointers.
    ///
    /// No-op when target weight set points directly into target_params_buf.
    fn unflatten_target_weights(
        &self,
        target_d: &DuelingWeightSet,
        target_b: &BranchingWeightSet,
    ) -> Result<(), MLError> {
        let sizes = compute_param_sizes(&self.config);
        let src_base = self.target_params_buf.raw_ptr();

        // No-op if target already points into target_params_buf.
        if target_d.w_s1 == src_base {
            return Ok(());
        }

        // Per-tensor byte offsets across the GRN-expanded layout.
        let mut byte_offsets = [0u64; NUM_WEIGHT_TENSORS];
        {
            let mut acc: u64 = 0;
            for i in 0..NUM_WEIGHT_TENSORS {
                byte_offsets[i] = acc;
                acc += (align4(sizes[i]) * std::mem::size_of::<f32>()) as u64;
            }
        }

        let ptrs: [(u64, usize); 24] = [
            (target_d.w_s1, super::gpu_weights::DUELING_FLAT_INDICES[0]),
            (target_d.b_s1, super::gpu_weights::DUELING_FLAT_INDICES[1]),
            (target_d.w_s2, super::gpu_weights::DUELING_FLAT_INDICES[2]),
            (target_d.b_s2, super::gpu_weights::DUELING_FLAT_INDICES[3]),
            (target_d.w_v1, super::gpu_weights::DUELING_FLAT_INDICES[4]),
            (target_d.b_v1, super::gpu_weights::DUELING_FLAT_INDICES[5]),
            (target_d.w_v2, super::gpu_weights::DUELING_FLAT_INDICES[6]),
            (target_d.b_v2, super::gpu_weights::DUELING_FLAT_INDICES[7]),
            (target_d.w_a1, super::gpu_weights::DUELING_FLAT_INDICES[8]),
            (target_d.b_a1, super::gpu_weights::DUELING_FLAT_INDICES[9]),
            (target_d.w_a2, super::gpu_weights::DUELING_FLAT_INDICES[10]),
            (target_d.b_a2, super::gpu_weights::DUELING_FLAT_INDICES[11]),
            (target_b.w_bo1, super::gpu_weights::BRANCHING_FLAT_INDICES[0]),
            (target_b.b_bo1, super::gpu_weights::BRANCHING_FLAT_INDICES[1]),
            (target_b.w_bo2, super::gpu_weights::BRANCHING_FLAT_INDICES[2]),
            (target_b.b_bo2, super::gpu_weights::BRANCHING_FLAT_INDICES[3]),
            (target_b.w_bu1, super::gpu_weights::BRANCHING_FLAT_INDICES[4]),
            (target_b.b_bu1, super::gpu_weights::BRANCHING_FLAT_INDICES[5]),
            (target_b.w_bu2, super::gpu_weights::BRANCHING_FLAT_INDICES[6]),
            (target_b.b_bu2, super::gpu_weights::BRANCHING_FLAT_INDICES[7]),
            (target_b.w_bg1, super::gpu_weights::BRANCHING_FLAT_INDICES[8]),
            (target_b.b_bg1, super::gpu_weights::BRANCHING_FLAT_INDICES[9]),
            (target_b.w_bg2, super::gpu_weights::BRANCHING_FLAT_INDICES[10]),
            (target_b.b_bg2, super::gpu_weights::BRANCHING_FLAT_INDICES[11]),
        ];

        for (i, &(dst, layout_idx)) in ptrs.iter().enumerate() {
            let num_bytes = sizes[layout_idx] * std::mem::size_of::<f32>();
            dtod_copy_async(dst, src_base + byte_offsets[layout_idx], num_bytes, &self.stream, i, "unflatten_target")?;
        }

        Ok(())
    }

    // ═══════════════════════════════════════════════════════════════════
    // GPU-native Polyak EMA target update (fused single kernel)
    // ═══════════════════════════════════════════════════════════════════

    /// GPU-native Polyak EMA: `target[i] = (1-tau)*target[i] + tau*online[i]`
    ///
    /// Fused two-kernel update over the flat parameter buffer:
    ///   1. Main launch covers `[0..non_isv_params)` (= padded f32 sum of
    ///      tensors `[0..FIRST_ISV_TENSOR=77)` — trunk + value/advantage
    ///      heads + branch heads + GLU + bottleneck weights).
    ///   2. VSN launch covers `[vsn_floats_offset..total_params)` (= padded
    ///      f32 sum of VSN tensors `[95..119)`). VSN params receive real
    ///      gradients via the 1B-iv backward chain, so target VSN must
    ///      track online VSN through the same Polyak EMA — without this
    ///      second launch the target VSN slice stays at `alloc_zeros` for
    ///      the entire training run, producing a constant uniform 1/6
    ///      mask in Bellman-target VSN forward while online VSN drifts
    ///      toward the measured per-group importances.
    ///
    /// ISV scalars at `[FIRST_ISV_TENSOR..95)` are online-only by design
    /// (`_eff` slots / EMA producers — target uses neutral defaults
    /// instead of ISV-modulated values).
    ///
    /// On first call, both launches use `tau=1.0` so target = online (no
    /// separate DtoD copy needed). After the launches, scatters the updated
    /// flat target weights back to individual tensors via
    /// `unflatten_target_weights()` — note this only covers the 24
    /// `DuelingWeightSet`+`BranchingWeightSet` tensors, since target VSN
    /// weights are accessed via direct pointers into `target_params_buf`
    /// (`tg_w_ptrs = f32_weight_ptrs_from_base(target_ptr, &param_sizes)`)
    /// rather than separate weight tensors.
    ///
    /// Runs OUTSIDE the captured CUDA Graph -- device pointers are stable so
    /// the graph stays valid.
    pub fn target_ema_update(
        &mut self,
        _online_d: &DuelingWeightSet,
        _online_b: &BranchingWeightSet,
        target_d: &DuelingWeightSet,
        target_b: &BranchingWeightSet,
        tau: f32,
    ) -> Result<(), MLError> {
        // Disable cudarc event tracking for the duration of this method.
        // No cuStreamSynchronize — stream ordering is sufficient.
        // The previous hang was caused by the sync memcpy_htod for adam_step
        // (now async), not by missing sync here.
        let _eg = EventTrackingGuard::new(self.stream.context());

        // First call: target_params_buf is zeros. Use tau=1.0 so the EMA kernel
        // computes target = 0*zeros + 1*online = online. No separate DtoD copy needed.
        let effective_tau = if !self.target_params_initialized {
            self.target_params_initialized = true;
            1.0_f32
        } else {
            tau
        };

        // Main EMA covers non-ISV params (indices `[0..FIRST_ISV_TENSOR=77)`).
        // ISV weight slots `[77..95)` are online-only — the target network
        // uses neutral defaults (base_gamma, uniform gates) instead of
        // ISV-modulated values. VSN tensors `[95..119)` are handled by a
        // dedicated second launch below — see the chain-final-fix block.
        let n = self.non_isv_params as i32;
        let blocks = ((self.non_isv_params + 255) / 256) as u32;
        let launch_cfg = LaunchConfig {
            grid_dim: (blocks, 1, 1),
            block_dim: (256, 1, 1),
            shared_mem_bytes: 0,
        };

        // EMA on f32 master weights: target = (1-tau)*target + tau*online
        // Pinned device-mapped: host write is visible to GPU (no HtoD copy).
        unsafe { *self.tau_pinned = effective_tau; }
        let t_ptr = self.target_params_buf.raw_ptr();
        let o_ptr = self.params_buf.raw_ptr();
        let tau_ptr = self.tau_dev_ptr;
        unsafe {
            self.stream
                .launch_builder(&self.ema_kernel)
                .arg(&t_ptr)
                .arg(&o_ptr)
                .arg(&tau_ptr)
                .arg(&n)
                .launch(launch_cfg)
                .map_err(|e| {
                    MLError::ModelError(format!("dqn_ema_kernel flat launch: {e}"))
                })?;
        }

        // Plan 4 Task 1B-iv chain final fix: VSN range Polyak EMA target sync.
        //
        // The main EMA above stops at `non_isv_params` (= padded f32 sum of
        // tensors `[0..FIRST_ISV_TENSOR=77)`). The 24 VSN param tensors at
        // `[95..119)` (added by 1B-ii, trained from real gradients by the
        // 1B-iv backward chain) are NOT covered by that launch — without
        // this second launch, `target_params_buf[vsn_floats_offset..]`
        // stays at `alloc_zeros` (≡ zero VSN logits → softmax(0) ≡
        // uniform 1/6 mask) for the entire training run, while the online
        // VSN's mask drifts toward measured per-group importances. The
        // resulting Bellman-target divergence collapses fold 2 magnitude
        // training and pinned F0 best Sharpe at 21.14 across the four
        // 1B-iv-rc/rc2/rc3 remedy attempts that all chased downstream
        // symptoms (gradient scale, Adam variance, kernel sync).
        //
        // ISV scalars at `[FIRST_ISV_TENSOR..95)` remain online-only by
        // design — `_eff` slots (gamma_*_eff, epsilon_eff, etc.) are
        // ISV-modulated runtime values that the target uses neutral
        // defaults for, not a learned parameter that target should track.
        // Only the VSN tensor slice [95..119) is a learned parameter
        // family that requires target tracking.
        let vsn_count = self.vsn_param_total as i32;
        let vsn_blocks = ((self.vsn_param_total + 255) / 256) as u32;
        let vsn_cfg = LaunchConfig {
            grid_dim: (vsn_blocks, 1, 1),
            block_dim: (256, 1, 1),
            shared_mem_bytes: 0,
        };
        let vsn_t_ptr = self.target_params_buf.raw_ptr() + self.vsn_param_byte_offset;
        let vsn_o_ptr = self.params_buf.raw_ptr() + self.vsn_param_byte_offset;
        unsafe {
            self.stream
                .launch_builder(&self.ema_kernel)
                .arg(&vsn_t_ptr)
                .arg(&vsn_o_ptr)
                .arg(&tau_ptr)             // SAME tau as main EMA (effective_tau)
                .arg(&vsn_count)
                .launch(vsn_cfg)
                .map_err(|e| {
                    MLError::ModelError(format!("dqn_ema_kernel vsn launch: {e}"))
                })?;
        }

        // Scatter flat f32 target buffer back to individual weight tensors
        self.unflatten_target_weights(target_d, target_b)?;

        Ok(())
    }

    // ── Adaptive gradient clipping ──────────────────────────────────

    /// Update EMA-based adaptive gradient clip norm.
    /// Writes directly to pinned device-mapped memory — GPU reads via pointer, no HtoD copy.
    pub fn update_adaptive_clip(&mut self, observed_grad_norm: f32) {
        if !observed_grad_norm.is_finite() || observed_grad_norm <= 0.0 {
            return;
        }

        // SP4 Layer C close-out C4 (2026-05-01): the host-side scalar-
        // reduction + EMA arithmetic chain (winsor + cold-start sentinel +
        // EMA + 2 clamps + ISV upper-bound) now runs GPU-side via
        // `update_adaptive_clip_kernel` per `feedback_no_cpu_compute_strict`.
        // The full chain — winsor (1) + cold-start sentinel + EMA (3) +
        // scalar reduction (4) + ISV upper-bound clamp (5) + mapped-pinned
        // write (6) — was migrated coherently per
        // `feedback_no_partial_refactor` rather than splitting the EMA into
        // the kernel and leaving the surrounding scalar reductions on host.
        // Per `pearl_cold_path_no_exception_to_gpu_drives.md`: the cold-
        // path scalar arithmetic stays on GPU because all inputs (the
        // winsor's `prev_clip`, the EMA's `prev_ema`, the upper-bound's ISV
        // slot) already live in mapped-pinned host memory the device reads
        // directly.
        //
        // Constants (legacy values, preserved exactly per
        // `feedback_no_quickfixes.md`):
        //   EMA_BETA=0.95           — fixed-α (cross-fold steady-state
        //                              behaviour the Mech 6 design depends
        //                              on; Pearls A+D would defeat the
        //                              winsor→EMA→clip chain's stability
        //                              guarantees).
        //   CLIP_MULTIPLIER=2.0     — clip threshold = 2 × steady-state
        //                              grad-norm (legacy value; same
        //                              "headroom over the EMA" the SP4
        //                              Layer B Mech 6 anchor uses).
        //   MIN_CLIP=1.0            — ε-floor preventing cold-start
        //                              `EMA × multiplier ≈ 0` from pinning
        //                              the clip threshold at 0 (the actual
        //                              clip's `min(grad, threshold)` would
        //                              zero every gradient).
        //   GRAD_CLIP_OUTLIER_K=100 — winsor multiplier (Plan C Phase 2
        //                              follow-up N rationale at the deleted
        //                              host code's comment block).
        //   EPS_CLAMP_FLOOR         — SP4 ε-floor for the ISV upper-bound
        //                              slot (cold-start ISV reads as 0
        //                              must not collapse the bound).
        //
        // Outlier-warn diagnostic: the kernel writes
        // `outlier_diag_pinned[0] = observed - clamped` so this host-side
        // post-launch readback can replicate the original
        // `tracing::warn!("GRAD_CLIP_OUTLIER: clamping ...")` log without
        // running scalar arithmetic on the host. The mapped-pinned slot
        // is `__threadfence_system()`-visible by the time the launcher
        // returns.
        use crate::cuda_pipeline::sp4_isv_slots::GRAD_CLIP_BOUND_INDEX;
        use crate::cuda_pipeline::sp4_wiener_ema::EPS_CLAMP_FLOOR;

        // Snapshot the prev_clip BEFORE the kernel overwrites
        // `adaptive_clip_pinned[0]` — needed only for the outlier warn log
        // (host log formatting is data-routing, not compute on GPU values).
        let prev_clip_for_log = unsafe { *self.adaptive_clip_pinned };

        const EMA_BETA: f32 = 0.95;
        const CLIP_MULTIPLIER: f32 = 2.0;
        const MIN_CLIP: f32 = 1.0;
        const GRAD_CLIP_OUTLIER_K: f32 = 100.0;

        if let Err(e) = self.launch_update_adaptive_clip(
            observed_grad_norm,
            EMA_BETA,
            CLIP_MULTIPLIER,
            MIN_CLIP,
            GRAD_CLIP_OUTLIER_K,
            EPS_CLAMP_FLOOR,
            GRAD_CLIP_BOUND_INDEX as i32,
        ) {
            tracing::warn!("update_adaptive_clip launch failed: {e}");
            return;
        }

        // Post-launch outlier-warn — read the diagnostic slot the kernel
        // populated. Mapped-pinned + `__threadfence_system()` ⇒ the value
        // is visible without any explicit host sync.
        let delta = unsafe { *self.outlier_diag_pinned };
        if delta > 0.0 {
            let clamped = observed_grad_norm - delta;
            tracing::warn!(
                "GRAD_CLIP_OUTLIER: clamping grad_norm {:.4e} → {:.4e} (prev_clip={:.4e}, K={})",
                observed_grad_norm, clamped, prev_clip_for_log, GRAD_CLIP_OUTLIER_K
            );
        }

        // Plan C Phase 2 follow-up K (2026-04-29) + SP4 Layer C close-out
        // C1 redesigned (2026-05-01): feed the SAME observation into the
        // fast/slow grad-norm EMAs that drive the fold-boundary warmup
        // factor `fold_warmup_factor_update`. Constants are numerical-
        // stability bounds per `feedback_isv_for_adaptive_bounds.md` — fast
        // α=0.1 is the architectural ~10-step horizon (recover from fold
        // reset in roughly the time the existing adaptive clip's EMA takes
        // to recapture grad scale); slow α=0.001 is the ~1000-step horizon
        // (cross-fold steady-state baseline). Both are STRUCTURAL (preserve
        // direction / amortise scale across folds), not tuned.
        //
        // Per N's decision (above): fast/slow EMAs use the RAW grad-norm
        // observation (`grad_norm_buf[0]`), not the winsorized sample —
        // they're a stability signal where outliers should transiently
        // spike fast EMA and dampen the warmup factor on the next step.
        //
        // The EMA arithmetic itself runs GPU-side via
        // `launch_update_grad_norm_emas` per `feedback_no_cpu_compute_strict`
        // (any compute — EMA, reduction, mean — belongs on GPU regardless
        // of frequency). The host-side `observed_grad_norm` parameter is
        // retained only as the early-return guard above (≤ 0.0 / non-finite
        // ⇒ skip the EMA update); the kernel reads the same scalar from
        // `grad_norm_buf[0]` (populated earlier in the same stream by
        // `launch_grad_norm_finalize`, fully written by the time
        // `readback_scalars_sync` produced `observed_grad_norm` host-side).
        //
        // Launch-error handling mirrors the other per-step ISV producers
        // (`launch_h_s2_rms_ema`, `launch_fold_warmup_factor` at
        // `training_loop.rs:3450/3464`): warn-and-continue rather than
        // propagating, since `update_adaptive_clip`'s `()` return type is
        // a stable contract with multiple callers and a kernel-launch
        // failure here is non-fatal (the previous EMA values stay valid;
        // `fold_warmup_factor_kernel` still produces a valid factor on the
        // next step's still-fresh EMAs).
        if let Err(e) = self.launch_update_grad_norm_emas() {
            tracing::warn!("update_grad_norm_emas launch failed: {e}");
        }
        self.grad_norm_emas_step_count = self.grad_norm_emas_step_count.saturating_add(1);
    }

    /// SP4 Layer C close-out C1 redesigned (2026-05-01): GPU-only fast/slow
    /// grad-norm EMA update launcher.
    ///
    /// Replaces the host-side `(1-α) × prev + α × obs` arithmetic in
    /// `update_adaptive_clip` per `feedback_no_cpu_compute_strict`. Runs in
    /// the same stream as `launch_grad_norm_finalize`, so the freshly-
    /// written `grad_norm_buf[0]` is consumed without any host sync —
    /// sequential same-stream ordering provides the producer→consumer
    /// happens-before. Caller-compat: the EMA buffers retain their mapped-
    /// pinned shape so `fold_warmup_factor_kernel` continues to read them
    /// via dev_ptr unchanged, and the host-side accessors
    /// (`grad_norm_fast_ema_value` / `grad_norm_slow_ema_value`) keep
    /// observing the same memory after the kernel's
    /// `__threadfence_system()`.
    ///
    /// Cold-start sentinel (`prev ≤ 0.0` ⇒ assign obs directly) preserves
    /// the deleted host-side formula's semantics exactly. FAST_ALPHA /
    /// SLOW_ALPHA constants mirror the architectural ~10-step / ~1000-step
    /// horizons that the existing fold-warmup-factor design depends on.
    /// SP4 Layer C close-out C4 (2026-05-01): GPU-only winsorized adaptive
    /// grad-clip update launcher.
    ///
    /// Replaces the host-side scalar-reduction + EMA arithmetic chain in
    /// `update_adaptive_clip` per `feedback_no_cpu_compute_strict`. The
    /// kernel runs on the trainer's stream (all four mapped-pinned slots
    /// are owned by the trainer), reads the host-passed `observed_grad_norm`
    /// + four mapped-pinned scalars + ISV[GRAD_CLIP_BOUND_INDEX], and
    /// writes the three output mapped-pinned scalars (adaptive_clip,
    /// grad_norm_ema, outlier_diag) in lockstep with `__threadfence_system()`.
    fn launch_update_adaptive_clip(
        &self,
        observed_grad_norm: f32,
        ema_beta: f32,
        clip_multiplier: f32,
        min_clip: f32,
        outlier_k: f32,
        eps_clamp_floor: f32,
        grad_clip_bound_idx: i32,
    ) -> Result<(), MLError> {
        let clip_dev = self.adaptive_clip_dev_ptr;
        let ema_dev = self.grad_norm_ema_dev_ptr;
        let outlier_dev = self.outlier_diag_dev_ptr;
        let isv_ptr = self.isv_signals_dev_ptr;

        let cfg = LaunchConfig {
            grid_dim:         (1, 1, 1),
            block_dim:        (1, 1, 1),
            shared_mem_bytes: 0,
        };

        // Safety: kernel signature
        //   (float, float, float, float, float, float,
        //    float*, float*, const float*, int, float*)
        // matches the eleven args below; the three writeable dev_ptrs are
        // mapped-pinned scalars and `isv_signals_dev_ptr` is the ISV bus
        // pointer (read-only here).
        unsafe {
            self.stream
                .launch_builder(&self.update_adaptive_clip_kernel)
                .arg(&observed_grad_norm)
                .arg(&ema_beta)
                .arg(&clip_multiplier)
                .arg(&min_clip)
                .arg(&outlier_k)
                .arg(&eps_clamp_floor)
                .arg(&clip_dev)
                .arg(&ema_dev)
                .arg(&isv_ptr)
                .arg(&grad_clip_bound_idx)
                .arg(&outlier_dev)
                .launch(cfg)
                .map_err(|e| MLError::ModelError(format!("update_adaptive_clip: {e}")))?;
        }
        Ok(())
    }

    fn launch_update_grad_norm_emas(&self) -> Result<(), MLError> {
        const FAST_ALPHA: f32 = 0.1;   // ~10-step horizon
        const SLOW_ALPHA: f32 = 0.001; // ~1000-step horizon (cross-fold steady baseline)

        let grad_norm_ptr = self.ptrs.grad_norm_buf;
        let fast_dev_ptr  = self.grad_norm_fast_ema_dev_ptr;
        let slow_dev_ptr  = self.grad_norm_slow_ema_dev_ptr;

        let cfg = LaunchConfig {
            grid_dim:         (1, 1, 1),
            block_dim:        (1, 1, 1),
            shared_mem_bytes: 0,
        };

        // Safety: kernel signature `(const float*, float*, float*, float, float)`
        // matches the five args below; `grad_norm_buf` is the device pointer
        // for the L2-norm scalar populated by `launch_grad_norm_finalize`,
        // and `grad_norm_fast/slow_ema_dev_ptr` are mapped-pinned dev_ptrs
        // valid for the lifetime of `self`.
        unsafe {
            self.stream
                .launch_builder(&self.update_grad_norm_emas_kernel)
                .arg(&grad_norm_ptr)
                .arg(&fast_dev_ptr)
                .arg(&slow_dev_ptr)
                .arg(&FAST_ALPHA)
                .arg(&SLOW_ALPHA)
                .launch(cfg)
                .map_err(|e| MLError::ModelError(format!("update_grad_norm_emas: {e}")))?;
        }
        Ok(())
    }

    /// Plan C Phase 2 follow-up K (2026-04-29): launch
    /// `fold_warmup_factor_update` — single-thread single-block ISV
    /// producer for ISV[FOLD_WARMUP_FACTOR_INDEX=130]. Reads the fast/slow
    /// grad-norm EMA mapped-pinned scalars (filled by
    /// `update_adaptive_clip` per training step) plus a host-passed step
    /// counter (gates the bootstrap `factor=1` window) and writes
    /// `clamp(fast/slow, 0, 1)`. Cold-path-cadence — invoked once per
    /// training step alongside the other ISV producers
    /// (`launch_h_s2_rms_ema`, `launch_iqn_quantile_ema`, etc.).
    ///
    /// Per `pearl_cold_path_no_exception_to_gpu_drives.md`: even a cold-
    /// path scalar arithmetic stays on GPU when its EMA inputs already
    /// live in mapped-pinned host memory the device can read directly.
    pub fn launch_fold_warmup_factor(&self) -> Result<(), MLError> {
        // Same invariant rationale as `launch_q_drift_rate_ema` /
        // `launch_h_s2_rms_ema`: loud failure (debug_assert + kernel-launch
        // error) preferred over silent skip.
        debug_assert!(self.isv_signals_dev_ptr != 0,
            "launch_fold_warmup_factor: isv_signals_dev_ptr must be allocated by constructor");
        let isv_dev_ptr  = self.isv_signals_dev_ptr;
        let fast_dev_ptr = self.grad_norm_fast_ema_dev_ptr;
        let slow_dev_ptr = self.grad_norm_slow_ema_dev_ptr;
        let warmup_idx   = FOLD_WARMUP_FACTOR_INDEX as i32;
        // `WARMUP_FACTOR_MIN_STEPS` is a numerical-stability bound (Invariant 1
        // carve-out per `feedback_isv_for_adaptive_bounds.md`) — until the
        // EMAs have absorbed at least this many samples the slow EMA is
        // effectively cold-start (within ~10× of the first observation),
        // making the ratio meaningless. 200 ≈ 2τ for the fast EMA so
        // both EMAs have meaningful state. Saturating-cast to i32 caps at
        // i32::MAX, which is fine: once we exceed the cap the gate is
        // permanently passed.
        const WARMUP_FACTOR_MIN_STEPS: i32 = 200;
        let steps_observed_i32: i32 = self.grad_norm_emas_step_count.min(i32::MAX as u64) as i32;
        unsafe {
            self.stream.launch_builder(&self.fold_warmup_factor_kernel)
                .arg(&isv_dev_ptr)
                .arg(&fast_dev_ptr)
                .arg(&slow_dev_ptr)
                .arg(&warmup_idx)
                .arg(&WARMUP_FACTOR_MIN_STEPS)
                .arg(&steps_observed_i32)
                .launch(LaunchConfig {
                    grid_dim:        (1, 1, 1),
                    block_dim:       (1, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("fold_warmup_factor_update: {e}")))?;
        }
        Ok(())
    }

    /// Plan C Phase 2 follow-up K (2026-04-29): reset the FAST grad-norm
    /// EMA at fold boundary. The SLOW EMA persists — it tracks the
    /// cross-fold steady-state grad scale that the warmup factor's
    /// denominator compares against. Resetting only the fast EMA causes
    /// the ratio to start at 0 (fast=0, slow≈baseline) → factor = 0 →
    /// heavy damping for the new fold's first steps. Step counter is NOT
    /// reset — once the warmup gate has been crossed (typical after the
    /// first fold), subsequent folds skip the cold-start guard and rely
    /// purely on the EMA ratio.
    pub fn reset_fast_grad_norm_ema(&mut self) {
        unsafe { *self.grad_norm_fast_ema_pinned = 0.0_f32; }
    }

    /// SP4 Layer A Task A12 (2026-04-30): bulk-zero the Pearl D Wiener-EMA
    /// state at fold boundary.
    ///
    /// `wiener_state_buf` is a 213-float mapped-pinned buffer holding all
    /// 71 producers' Wiener triples `[sample_var, diff_var, x_lag]` (40 SP4
    /// + 29 Task-A13 retrofit + 2 Layer-C-#260 producers). Pearl A's
    /// first-observation sentinel requires both `prev_x_mean` (the ISV
    /// bound slot) AND `state.x_lag` (Wiener triple offset 2) to be exactly
    /// 0 when a producer first fires in a new fold. Without this reset the
    /// new fold's first producer launch would EMA against fold-N's stale
    /// Wiener state — the cross-fold anchor staleness Mech 8's reverted
    /// slow_ema reset was trying to fix imperfectly. Companion to the SP4
    /// ISV bound slots `state_reset_registry.rs::sp4_*_bound*` entries —
    /// per `feedback_no_partial_refactor.md`, both halves of the sentinel
    /// contract reset in lockstep.
    ///
    /// Constructor zero-init via `MappedF32Buffer::new`'s
    /// `ptr::write_bytes(host_ptr, 0, len)` is the cold-start; this method
    /// re-applies the same byte-zero at every subsequent fold boundary.
    pub fn reset_sp4_wiener_state(&mut self) {
        // Safety: `host_ptr` is a mapped-pinned `*mut f32` of length
        // `self.wiener_state_buf.len` returned by `cuMemHostAlloc`.
        // f32 zero == bytewise-zero per IEEE-754 +0.0 representation, so
        // `write_bytes(.., 0, ..)` produces 213 valid f32 zeros.
        unsafe {
            std::ptr::write_bytes(
                self.wiener_state_buf.host_ptr as *mut u8,
                0u8,
                self.wiener_state_buf.len * std::mem::size_of::<f32>(),
            );
        }
    }

    /// SP4 Layer A Task A12 (2026-04-30): bulk-zero the Pearl C
    /// engagement-counter buffer at fold boundary.
    ///
    /// `clamp_engage_per_block_buf` is a 2816-int mapped-pinned buffer
    /// (`(SP4_PARAM_GROUP_COUNT + 3 curiosity sub-launches) ×
    /// MAX_BLOCKS_PER_ADAM = 11 × 256`) counting post-Adam clamp
    /// triggers per (param-group, Adam-launch-block). Host reduces
    /// across blocks → engagement_rate; rate-deficit Wiener EMA detects
    /// post-clamp feedback-loop saturation. The buffer length grew
    /// 2048→2816 in Task A14/A15 to give Curiosity's 4 sub-launches
    /// (W1/b1/W2/b2) distinct offset slots avoiding per-block index
    /// collisions. At fold boundary all counters reset to 0 so the new
    /// fold's engagement-rate tracking restarts from a clean baseline
    /// (no inherited fold-N saturation state biasing the new fold's
    /// first per-step rate readings).
    ///
    /// Constructor zero-init via `MappedI32Buffer::new` is the cold-start;
    /// this method re-applies the same byte-zero at every subsequent fold
    /// boundary.
    pub fn reset_sp4_clamp_engage_counters(&mut self) {
        // Safety: `host_ptr` is a mapped-pinned `*mut i32` of length
        // `self.clamp_engage_per_block_buf.len` returned by `cuMemHostAlloc`.
        // i32 zero == bytewise-zero, so `write_bytes(.., 0, ..)` produces
        // SP4_ENGAGE_BUF_LEN=2816 valid i32 zeros.
        unsafe {
            std::ptr::write_bytes(
                self.clamp_engage_per_block_buf.host_ptr as *mut u8,
                0u8,
                self.clamp_engage_per_block_buf.len * std::mem::size_of::<i32>(),
            );
        }
    }

    /// SP4 Task A15: bulk-zero the Pearl C rate-deficit Wiener-state
    /// buffer at fold boundary.
    ///
    /// `pearl_c_rate_deficit_state_buf` is a 24-float mapped-pinned
    /// buffer = `SP4_PARAM_GROUP_COUNT × {sample_var, diff_var, x_lag}`.
    /// Reset to all-zeros so Pearl A's first-observation sentinel fires
    /// on the new fold's first engagement-rate-deficit observation
    /// (`prev_x_mean == 0 AND state.x_lag == 0` → first-observation
    /// replacement). Companion to `reset_sp4_pearl_c_rate_deficit_ema`
    /// — both halves of Pearl A's sentinel contract reset in lockstep.
    pub fn reset_sp4_pearl_c_rate_deficit_state(&mut self) {
        unsafe {
            std::ptr::write_bytes(
                self.pearl_c_rate_deficit_state_buf.host_ptr as *mut u8,
                0u8,
                self.pearl_c_rate_deficit_state_buf.len * std::mem::size_of::<f32>(),
            );
        }
    }

    /// SP4 Task A15: zero the host-only Pearl C rate-deficit EMA at
    /// fold boundary. Companion to
    /// `reset_sp4_pearl_c_rate_deficit_state` — both halves of Pearl
    /// A's sentinel contract reset together.
    pub fn reset_sp4_pearl_c_rate_deficit_ema(&mut self) {
        self.pearl_c_rate_deficit_ema = [0.0_f32; SP4_PARAM_GROUP_COUNT];
    }

    /// SP5 Task A4 (2026-05-01): bulk-zero the Pearl 4 previous-step
    /// gradient buffer at fold boundary.
    ///
    /// `grad_prev_buf_per_group` is a `total_params`-element mapped-pinned
    /// f32 buffer holding the previous step's gradient direction for the
    /// per-group cosine-similarity computation in `grad_cosine_sim_kernel`.
    /// At fold boundary, zeroing this buffer makes the new fold's first
    /// `grad_cosine_sim_update` see `cos_sim = 0/(norm_curr×0+EPS) = 0`,
    /// which Pearl 4's clamp(0,1) maps to the low-stability envelope
    /// floor — clean cold-start of the per-group Adam β1/β2 hyperparam
    /// adaptation. Companion to the sp5_adam_beta1/beta2/eps ISV resets:
    /// both halves of Pearl A's sentinel contract reset in lockstep.
    pub fn reset_sp5_grad_prev_buf(&mut self) {
        // Safety: `host_ptr` is a mapped-pinned `*mut f32` of length
        // `self.grad_prev_buf_per_group.len` returned by `cuMemHostAlloc`.
        // f32 zero == bytewise-zero (IEEE-754 +0.0).
        unsafe {
            std::ptr::write_bytes(
                self.grad_prev_buf_per_group.host_ptr as *mut u8,
                0u8,
                self.grad_prev_buf_per_group.len * std::mem::size_of::<f32>(),
            );
        }
    }

    /// SP4 Task A14: device pointer to the trainer's `nan_flags_buf`
    /// (mapped via `CudaSlice::raw_ptr()`). Used by aux trainers (IQN,
    /// IQL hi/lo, Attn, Curiosity) to share the Pearl C sticky-flag
    /// store via `set_pearl_c_buffers`.
    pub fn nan_flags_buf_ptr(&self) -> u64 {
        self.nan_flags_buf.raw_ptr()
    }

    /// SP4 Task A14: device pointer to the trainer's
    /// `clamp_engage_per_block_buf` (mapped pinned). Used by aux
    /// trainers to share the Pearl C per-block engagement counter
    /// buffer via `set_pearl_c_buffers`.
    pub fn clamp_engage_per_block_buf_dev_ptr(&self) -> u64 {
        self.clamp_engage_per_block_buf.dev_ptr
    }

    /// SP4 Task A13.5 (2026-05-01): mapped-pinned host pointer for
    /// `wiener_state_buf`. Retained post-2026-05-01 GPU-Pearls refactor
    /// for the cross-boundary `debug_assert!` validity check; the GPU
    /// `apply_pearls_ad_kernel` operates on the dev pointer only.
    pub fn wiener_state_buf_host_ptr(&self) -> *mut f32 {
        self.wiener_state_buf.host_ptr
    }

    /// SP4 GPU-Pearls refactor (2026-05-01): mapped-pinned device pointer
    /// for `wiener_state_buf`. The GPU `apply_pearls_ad_kernel` reads/writes
    /// per-slot `[sample_var, diff_var, x_lag]` triples directly via this
    /// pointer; same underlying allocation as the host pointer (zero-copy
    /// `cuMemHostAlloc(... DEVICEMAP)`).
    pub fn wiener_state_buf_dev_ptr(&self) -> u64 {
        self.wiener_state_buf.dev_ptr
    }

    /// SP4 Task A13.5 (2026-05-01): mapped-pinned device pointer for
    /// `producer_step_scratch_buf`. Used by `GpuExperienceCollector`
    /// to launch the Pearls-A+D-retrofitted `reward_component_ema`
    /// kernel writing slots [63..69).
    pub fn producer_step_scratch_buf_dev_ptr(&self) -> u64 {
        self.producer_step_scratch_buf.dev_ptr
    }

    /// SP4 Task A13.5 (2026-05-01): mapped-pinned host pointer for
    /// `producer_step_scratch_buf`. Used by `GpuExperienceCollector`
    /// to read step observations after the kernel + sync (slots [63..69)
    /// for the 6 reward components).
    pub fn producer_step_scratch_buf_host_ptr(&self) -> *mut f32 {
        self.producer_step_scratch_buf.host_ptr
    }

    /// SP4 Task A13.5 (2026-05-01): mapped-pinned host pointer for
    /// `isv_signals_pinned`. Used by `GpuExperienceCollector` to read
    /// `prev_x_mean` and write `new_x_mean` for ISV[REWARD_POPART_EMA_INDEX..+6)
    /// during the host-side Pearls A+D update.
    pub fn isv_signals_pinned_ptr(&self) -> *mut f32 {
        self.isv_signals_pinned
    }

    /// Read-only accessor for diagnostics / HEALTH_DIAG mirror.
    pub fn grad_norm_fast_ema_value(&self) -> f32 { unsafe { *self.grad_norm_fast_ema_pinned } }
    /// Read-only accessor for diagnostics / HEALTH_DIAG mirror.
    pub fn grad_norm_slow_ema_value(&self) -> f32 { unsafe { *self.grad_norm_slow_ema_pinned } }

    pub fn adaptive_clip_value(&self) -> f32 { unsafe { *self.adaptive_clip_pinned } }
    /// SP4 Layer C close-out C4 (2026-05-01): the winsorized grad-norm EMA
    /// is now stored in `grad_norm_ema_pinned` (mapped-pinned). Accessor
    /// reads via host_ptr after the kernel's `__threadfence_system()`.
    pub fn grad_norm_ema_value(&self) -> f32 {
        if self.grad_norm_ema_pinned.is_null() { 0.0 }
        else { unsafe { *self.grad_norm_ema_pinned } }
    }

    /// Plan C T11 follow-up — F1 ep2 explosion diagnostic (2026-04-29).
    /// Read-only accessor over the pinned C51 (total) loss. Same pinned
    /// device-mapped slot the readback path reads at the end of each step.
    /// Diagnostic-only; remove once root cause is identified.
    pub fn c51_loss_pinned_value(&self) -> f32 { unsafe { *self.total_loss_pinned } }
    /// Plan C T11 follow-up — F1 ep2 explosion diagnostic (2026-04-29).
    /// Read-only accessor over the pinned MSE warmup loss. Same pinned
    /// device-mapped slot the readback path reads at the end of each step.
    /// Diagnostic-only; remove once root cause is identified.
    pub fn mse_loss_pinned_value(&self) -> f32 { unsafe { *self.mse_loss_pinned } }
    /// Plan C T11 follow-up — F1 ep2 explosion diagnostic (2026-04-29).
    /// Borrow the trainer's owned CUDA stream so cold-path diagnostic readers
    /// can synchronise before raw `cuMemcpyDtoH_v2`. The stream is otherwise
    /// private; this accessor exists solely for the FOLD_EXPLOSION_DIAG
    /// atom-range read in `fused_training.rs`. Diagnostic-only; remove once
    /// root cause is identified.
    pub fn stream_for_diag(&self) -> &Arc<cudarc::driver::CudaStream> { &self.stream }

    /// Plan C Phase 2 follow-up K (2026-04-29): write a warmup-factor-scaled
    /// clip value directly to the pinned device-mapped slot. Mirrors `set_lr`'s
    /// zero-graph-recapture contract (the kernel reads `*adaptive_clip_pinned`
    /// at replay, so a mapped-pinned write is GPU-visible immediately on the
    /// next kernel launch). Used by the per-step warmup-factor consumer in
    /// `training_loop.rs` to compose `clip_eff = clip_base × (0.1 + 0.9 × factor)`
    /// on top of the EMA-derived baseline that `update_adaptive_clip` just
    /// wrote.
    pub fn set_active_clip(&mut self, clip: f32) {
        if clip.is_finite() && clip > 0.0 {
            unsafe { *self.adaptive_clip_pinned = clip; }
        }
    }

    /// Write the EMA tau value directly to pinned device-mapped memory.
    /// GPU reads the updated value on next kernel launch — no HtoD copy needed.
    pub(crate) fn set_tau_value(&mut self, tau: f32) {
        unsafe { *self.tau_pinned = tau; }
    }

    /// Access the shared cuBLAS handle (for passing to new CublasGemmSet instances).
    pub(crate) fn shared_cublas(&self) -> &Arc<PerStreamCublasHandles> {
        &self.shared_cublas
    }

    /// Synchronize the main stream. Used before graph capture
    /// to ensure all prior work is complete.
    pub(crate) fn sync_all_streams(&self) -> Result<(), MLError> {
        self.stream.synchronize()
            .map_err(|e| MLError::ModelError(format!("main stream sync: {e}")))?;
        Ok(())
    }

    /// Increment Adam step counter and async-upload to device buffer.
    /// Must be called OUTSIDE the captured graph (before replay).
    pub(crate) fn adam_step_async(&mut self) {
        self.adam_step += 1;
        // Pinned device-mapped: host write is visible to GPU (no HtoD copy).
        unsafe { *self.t_pinned = self.adam_step; }
    }

    /// Set tau cosine-annealing parameters for GPU-side kernel computation.
    /// Called once after construction (e.g. from DQNTrainer::new).
    pub(crate) fn set_tau_anneal_params(&mut self, tau_init: f32, tau_final: f32, tau_anneal_steps: i32) {
        self.tau_init = tau_init;
        self.tau_final = tau_final;
        self.tau_anneal_steps = tau_anneal_steps;
    }

    /// Submit `increment_step_counters` kernel — increments all 8 Adam step counters
    /// and computes cosine-annealed tau on GPU. Eliminates all host-side per-step writes.
    ///
    /// Must be called once per training step, before graph replay. The kernel is a
    /// single-thread (1,1,1)×(1,1,1) launch captured in a child graph node.
    ///
    /// `rng_step_ptr` is the PER RNG step counter device pointer.
    pub(crate) fn submit_counter_increments(
        &self,
        iql_t_ptr: u64,
        iql_low_t_ptr: u64,
        iqn_t_ptr: u64,
        attn_t_ptr: u64,
        rng_step_ptr: u64,
    ) -> Result<(), MLError> {
        unsafe {
            self.stream
                .launch_builder(&self.increment_counters_kernel)
                .arg(&self.t_dev_ptr)
                .arg(&self.sel_t_dev_ptr)
                .arg(&self.denoise_t_dev_ptr)
                .arg(&iql_t_ptr)
                .arg(&iql_low_t_ptr)
                .arg(&iqn_t_ptr)
                .arg(&attn_t_ptr)
                .arg(&rng_step_ptr)
                .arg(&self.tau_dev_ptr)
                .arg(&self.tau_init)
                .arg(&self.tau_final)
                .arg(&self.tau_anneal_steps)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (1, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("increment_step_counters: {e}")))?;
        }
        Ok(())
    }

    /// Raw pointer to the main stream.
    pub(crate) fn main_cu_stream(&self) -> cudarc::driver::sys::CUstream {
        self.stream.cu_stream()
    }

    /// Raw pointer to states_buf for pad_states kernel.
    pub(crate) fn states_buf_ptr(&self) -> u64 { self.states_buf.raw_ptr() }

    /// Raw pointer to next_states_buf for direct-to-trainer gather.
    pub(crate) fn next_states_buf_ptr(&self) -> u64 { self.next_states_buf.raw_ptr() }

    /// Raw pointer to actions_buf for direct-to-trainer gather.
    pub(crate) fn actions_buf_ptr(&self) -> u64 { self.actions_buf.raw_ptr() }

    /// Raw pointer to rewards_buf for direct-to-trainer gather.
    pub(crate) fn rewards_buf_ptr(&self) -> u64 { self.rewards_buf.raw_ptr() }

    /// Raw pointer to dones_buf for direct-to-trainer gather.
    pub(crate) fn dones_buf_ptr(&self) -> u64 { self.dones_buf.raw_ptr() }

    /// Raw pointer to is_weights_buf for direct-to-trainer gather.
    pub(crate) fn is_weights_buf_ptr(&self) -> u64 { self.is_weights_buf.raw_ptr() }

    /// SP13 Layer B Commit B1.1b (2026-05-05): raw pointer to
    /// `aux_nb_label_buf` for direct-to-trainer gather. Mirrors the existing
    /// 6 trainer-buf accessors (states / next_states / actions / rewards /
    /// dones / is_weights). The buffer is `CudaSlice<i32>` (B1.1a struct
    /// flip from f32) — the K=2 softmax CE consumer
    /// (`aux_next_bar_loss_reduce` + `aux_next_bar_backward` +
    /// `aux_dir_acc_reduce_kernel` + `aux_pred_to_isv_tanh_kernel`) reads
    /// it as i32. The PER `gather_i32_scalar` writes the per-batch sample
    /// of the aux_sign_labels ring (filled by `aux_sign_label_kernel.cu`
    /// in `gpu_experience_collector.rs`) into here on every step.
    pub(crate) fn aux_nb_label_buf_ptr(&self) -> u64 { self.aux_nb_label_buf.raw_ptr() }

    /// SP20 Phase 5 (2026-05-10): raw pointer to `aux_conf_at_state_buf`
    /// for direct-to-trainer PER gather. Mirrors the SP13 B1.1b
    /// `aux_nb_label_buf_ptr` direct-to-trainer pattern. The buffer is
    /// `CudaSlice<f32>` `[batch_size]`; the c51 reward-gate consumer
    /// (`c51_loss_batched`) reads it as f32 to compute
    /// `gate = sigmoid((aux_conf - ISV[AUX_CONF_THRESHOLD]) / ISV[AUX_GATE_TEMP])`.
    /// `GpuReplayBuffer::set_trainer_aux_conf_ptr` is invoked once at
    /// trainer init in `training_loop.rs` so PER's `gather_f32_scalar`
    /// writes the SAMPLED bar's aux_conf into here on every step.
    pub(crate) fn aux_conf_at_state_buf_ptr(&self) -> u64 { self.aux_conf_at_state_buf.raw_ptr() }

    /// SP22 H6 vNext Phase B4b-2 (2026-05-14): raw pointer to
    /// `aux_to_label_buf` for direct-to-trainer PER gather. Mirrors the
    /// SP13 B1.1b `aux_nb_label_buf_ptr` direct-to-trainer pattern. The
    /// buffer is `CudaSlice<i32>` `[batch_size]`; the K=3 trade-outcome
    /// sparse CE loss reduce consumer (`aux_trade_outcome_loss_reduce`)
    /// reads `{-1, 0, 1, 2}` labels — `-1` is the mask sentinel that
    /// must survive signed (i32, not u32).
    /// `GpuReplayBuffer::set_trainer_aux_outcome_labels_ptr` is invoked
    /// once at trainer init in `training_loop.rs` so PER's
    /// `gather_i32_scalar` writes the SAMPLED bar's label into here on
    /// every step.
    pub(crate) fn aux_to_label_buf_ptr(&self) -> u64 { self.aux_to_label_buf.raw_ptr() }

    /// Padded state dimension (128-byte aligned) for direct-to-trainer gather.
    pub(crate) fn state_dim_padded(&self) -> usize { ml_core::state_layout::STATE_DIM_PADDED }

    /// Raw pointer to plan_params_buf [B, 6] for trade plan integration.
    pub fn plan_params_buf_ptr(&self) -> u64 { self.plan_params_buf.raw_ptr() }

    /// Raw pointer to on_b_logits_buf
    /// [batch_size, (B0+B1+B2+B3) * num_atoms] f32, branch-major.
    /// Populated by the most recent `replay_forward_for_q_values()` /
    /// graphed forward; consumed by Thompson direction sampling in
    /// `experience_action_select` (Plan C T2 amendment, 2026-04-29).
    /// The direction-branch slice is the first
    /// `batch_size * B0 * num_atoms` floats (Branch 0).
    pub fn on_b_logits_buf_ptr(&self) -> u64 { self.on_b_logits_buf.raw_ptr() }

    /// Raw pointer to `on_v_logits_buf [B, num_atoms]` f32, the V-head C51
    /// distributional logits row written by the graphed forward pass.
    /// Consumed by:
    ///   * SP17 Commit C Thompson direction selection in
    ///     `experience_action_select` (samples from softmax(V + A_centered)).
    ///   * `compute_expected_q` + `quantile_q_select` + `mag_concat_qdir`
    ///     (already wired via `on_v_logits_buf` directly in the trainer).
    /// Stable across forward calls — overwritten by each
    /// `replay_forward_for_q_values` invocation.
    pub fn on_v_logits_buf_ptr(&self) -> u64 { self.on_v_logits_buf.raw_ptr() }

    /// Raw pointer to atom_positions_buf [B0, num_atoms] f32, the C51
    /// adaptive (non-linear) atom positions used by all branches. Stable
    /// across forward calls — written by the atom-positions ISV producer
    /// kernel during training.
    pub fn atom_positions_buf_ptr(&self) -> u64 { self.atom_positions_buf.raw_ptr() }

    /// Raw pointer to the per-sample C51 support buffer
    /// [max_batch_size, 4, 3] stride-12 (v_min, v_max, delta_z) per branch
    /// per sample. Owned by the GpuIqlTrainer and registered via
    /// `set_per_sample_support_ptr`; this accessor returns whatever the
    /// caller registered. Stable across forward calls.
    pub fn per_sample_support_ptr_get(&self) -> u64 { self.per_sample_support_ptr }

    /// F32 states buffer ref for cold-path Q-value computation.
    pub(crate) fn states_buf_f32(&self) -> &CudaSlice<f32> { &self.states_buf }

    /// Compute Q-stats using the internal states_buf (already populated by launch_pad_states).
    pub(crate) fn compute_q_stats_internal(&mut self, batch_size: usize) -> Result<QValueStatsResult, MLError> {
        let states = &self.states_buf as *const CudaSlice<f32>;
        unsafe { self.compute_q_stats(&*states, batch_size) }
    }

    // ── Selectivity gate methods ─────────────────────────────────────────────

    /// Launch selectivity_forward kernel: sigmoid(dot(W_sel, h_s2) + b_sel) per sample.
    ///
    /// Reads `save_h_s2` [B, SH2], writes `sel_out_buf` [B].
    /// Must be called after the DQN forward pass (graph_forward / graph_mega replay).
    pub(crate) fn launch_selectivity_forward(&self, batch_size: usize) -> Result<(), MLError> {
        let b = batch_size as i32;
        let sh2 = self.config.shared_h2 as i32;
        let blocks = ((batch_size as u32 + 255) / 256).max(1);
        let h_s2_ptr = self.ptrs.save_h_s2;
        let sel_out_ptr = self.sel_out_buf.raw_ptr();
        let sel_params_ptr = self.sel_params.raw_ptr();
        unsafe {
            self.stream
                .launch_builder(&self.sel_fwd_kernel)
                .arg(&h_s2_ptr)
                .arg(&sel_out_ptr)
                .arg(&sel_params_ptr)
                .arg(&b)
                .arg(&sh2)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("selectivity_forward: {e}")))?;
        }
        Ok(())
    }

    /// Launch 3-phase selectivity backward: compute d_z, then dW reduce (p1 + p2).
    ///
    /// Phase 1: d_z[i] = sel_out[i] * (1 - sel_out[i]) * (loss[i] / mean_loss)  [B threads]
    /// Phase 2: partial sums of d_z[i] * h_s2[i,j] over batch               [num_blocks x (SH2+1) threads]
    /// Phase 3: sum partials into sel_grad[j]                                [(SH2+1) threads]
    /// Writes `sel_grad` [SH2+1] — deterministic, no atomicAdd.
    pub(crate) fn launch_selectivity_backward(&mut self, batch_size: usize, mean_loss: f32) -> Result<(), MLError> {
        let b = batch_size as i32;
        let sh2 = self.config.shared_h2 as i32;
        let total_params = (self.config.shared_h2 + 1) as i32;

        // Step 1: compute d_z
        let sel_out_ptr = self.sel_out_buf.raw_ptr();
        let loss_ptr = self.per_sample_loss_buf.raw_ptr();
        let d_z_ptr = self.sel_d_z_buf.raw_ptr();
        unsafe {
            self.stream
                .launch_builder(&self.sel_compute_dz_kernel)
                .arg(&sel_out_ptr)
                .arg(&loss_ptr)
                .arg(&d_z_ptr)
                .arg(&mean_loss)
                .arg(&b)
                .launch(LaunchConfig::for_num_elems(batch_size as u32))
                .map_err(|e| MLError::ModelError(format!("selectivity_compute_dz: {e}")))?;
        }

        // Step 2: dW reduce phase 1
        let h_s2_ptr = self.ptrs.save_h_s2;
        let partials_ptr = self.sel_dw_partials_buf.raw_ptr();
        let num_blocks = self.sel_num_blocks;
        unsafe {
            self.stream
                .launch_builder(&self.sel_dw_reduce_p1_kernel)
                .arg(&d_z_ptr)
                .arg(&h_s2_ptr)
                .arg(&partials_ptr)
                .arg(&b)
                .arg(&sh2)
                .launch(LaunchConfig {
                    grid_dim: (num_blocks, total_params as u32, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 256 * 4,
                })
                .map_err(|e| MLError::ModelError(format!("selectivity_dw_reduce_p1: {e}")))?;
        }

        // Step 3: dW reduce phase 2
        let grad_ptr = self.sel_grad.raw_ptr();
        let num_blocks_i32 = num_blocks as i32;
        let p2_blocks = ((total_params as u32 + 255) / 256).max(1);
        unsafe {
            self.stream
                .launch_builder(&self.sel_dw_reduce_p2_kernel)
                .arg(&partials_ptr)
                .arg(&grad_ptr)
                .arg(&total_params)
                .arg(&num_blocks_i32)
                .launch(LaunchConfig {
                    grid_dim: (p2_blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("selectivity_dw_reduce_p2: {e}")))?;
        }
        Ok(())
    }

    /// Run Adam update on selectivity gate parameters.
    ///
    /// Computes L2 grad norm from sel_grad, then runs dqn_adam_update_kernel at LR=1e-4.
    /// Increments sel_adam_step and writes to pinned device-mapped sel_t_pinned.
    pub(crate) fn step_selectivity_adam(&mut self) -> Result<(), MLError> {
        self.sel_adam_step += 1;
        let step_val = self.sel_adam_step;
        let sel_dim = self.config.shared_h2 + 1;
        let sel_n = sel_dim as i32;

        // Write step counter to pinned device-mapped memory (no HtoD copy needed)
        unsafe { *self.sel_t_pinned = step_val; }

        // Phase 1: grad_norm_kernel on sel_grad → sel_norm_partials [1 block].
        // Uses post_aux_child-specific handle — grad_norm_standalone is captured in
        // forward_child (d_logits clipping). Sharing across children corrupts on Hopper.
        let grad_ptr = self.sel_grad.raw_ptr();
        let partials_ptr = self.sel_norm_partials.raw_ptr();
        unsafe {
            self.stream
                .launch_builder(&self.grad_norm_standalone_post_aux)
                .arg(&grad_ptr)
                .arg(&partials_ptr)
                .arg(&sel_n)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("sel grad_norm kernel: {e}")))?;
        }

        // Phase 2: grad_norm_finalize → sel_norm_buf [1].
        let norm_out_ptr = self.sel_norm_buf.raw_ptr();
        let nb = 1_i32;
        unsafe {
            self.stream
                .launch_builder(&self.grad_norm_finalize_post_aux)
                .arg(&partials_ptr)
                .arg(&norm_out_ptr)
                .arg(&nb)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("sel grad_norm_finalize: {e}")))?;
        }

        // Phase 3: Adam update on sel_params.
        // LR=1e-4 via pinned device-mapped pointer — satisfies const float* kernel signature.
        let aux_lr_ptr = self.aux_lr_dev_ptr;
        let beta1: f32 = 0.9;
        let beta2: f32 = 0.999;
        let eps: f32 = 1e-8;
        let weight_decay: f32 = 0.0;
        let params_ptr = self.sel_params.raw_ptr();
        let m_ptr = self.sel_adam_m.raw_ptr();
        let v_ptr = self.sel_adam_v.raw_ptr();
        let clip_ptr = self.sel_clip_buf.dev_ptr;
        let t_ptr = self.sel_t_dev_ptr;
        // Reuse main mask buffer as dummy — weight_decay=0.0 nullifies all mask values.
        let wd_mask_ptr = self.weight_decay_mask.raw_ptr();
        let l1_end_aux: i32 = 0;
        let l1_lambda_aux: f32 = 0.0;
        // SP4 Layer B: aux trainer (sel/recursive_conf) is outside the SP4
        // 8-group taxonomy — disable Mech 9 weight clamp here (mirrors DT pattern).
        let weight_clamp_max_abs: f32 = 0.0_f32;
        // SP4 Task A14: aux trainer (sel/recursive_conf) is outside the SP4
        // 8-group taxonomy — Pearl C engagement counter disabled.
        let _aux_nan_flags_null: u64 = 0;
        let _aux_diag_slot_disabled: i32 = -1;
        let _aux_engage_buf_null: u64 = 0;
        let _aux_engage_off_disabled: i32 = SP4_ENGAGE_OFFSET_DISABLED;
        let blocks = ((sel_dim as u32 + 255) / 256).max(1);
        unsafe {
            self.stream
                .launch_builder(&self.adam_update_post_aux)
                .arg(&params_ptr)
                .arg(&grad_ptr)
                .arg(&m_ptr)
                .arg(&v_ptr)
                .arg(&norm_out_ptr)
                .arg(&aux_lr_ptr)
                .arg(&beta1)
                .arg(&beta2)
                .arg(&eps)
                .arg(&weight_decay)
                .arg(&clip_ptr)
                .arg(&t_ptr)
                .arg(&sel_n)
                .arg(&wd_mask_ptr)
                .arg(&l1_end_aux)
                .arg(&l1_lambda_aux)
                .arg(&weight_clamp_max_abs) // SP3 Mech 9: post-Adam |p_val| clamp
                // SP4 Task A14: disabled — sel/recursive_conf outside SP4 group taxonomy.
                .arg(&_aux_nan_flags_null)
                .arg(&_aux_diag_slot_disabled)
                .arg(&_aux_engage_buf_null)
                .arg(&_aux_engage_off_disabled)
                .arg(&1.0_f32) // SP21 Phase 4: lr_scale (sel/recursive_conf aux trainer, non-branch = 1.0 no-op)
                .launch(LaunchConfig {
                    grid_dim: (blocks, 1, 1),
                    block_dim: (256, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("sel Adam update: {e}")))?;
        }
        Ok(())
    }
}

// ── F3: Spectral-gap power-iteration helpers ─────────────────────────────────

/// F3: Largest eigenvalue of a symmetric matrix via power iteration.
/// `mat` is stored row-major, size `n × n`. Runs `max_iters` iterations.
/// Numerically stable: renormalizes each step.
fn power_iteration_largest(mat: &[f32], n: usize, max_iters: usize) -> f32 {
    debug_assert!(n > 0,
        "power_iteration_largest: empty matrix (n=0); spectral_gap caller must guard");
    let mut v = vec![1.0f32 / (n as f32).sqrt(); n];
    let mut lambda = 0.0f32;
    for _ in 0..max_iters {
        let mut av = vec![0.0f32; n];
        for i in 0..n {
            for j in 0..n { av[i] += mat[i * n + j] * v[j]; }
        }
        let norm = av.iter().map(|x| x * x).sum::<f32>().sqrt();
        if norm < 1e-20 { return 0.0; } // ok: domain encoding — zero matrix has zero eigenvalue (norm vanishes only when M·v ≈ 0 for the iterate)
        let inv = 1.0 / norm;
        for i in 0..n { v[i] = av[i] * inv; }
        // Rayleigh quotient
        let mut new_lambda = 0.0f32;
        let mut mv = vec![0.0f32; n];
        for i in 0..n {
            for j in 0..n { mv[i] += mat[i * n + j] * v[j]; }
        }
        for i in 0..n { new_lambda += v[i] * mv[i]; }
        if (new_lambda - lambda).abs() < 1e-9 { return new_lambda; }
        lambda = new_lambda;
    }
    lambda
}

/// F3: Deflate a symmetric matrix by removing its dominant eigenpair.
/// Given largest eigenvalue `lambda_1` (approximately), returns mat - lambda_1 * v_1 v_1^T
/// where v_1 is extracted via one more power iteration.
fn deflate_rank_one(mat: &[f32], n: usize, lambda_1: f32) -> Vec<f32> {
    // Recover v_1 by one more round of power iteration (cheap, n=12).
    let mut v = vec![1.0f32 / (n as f32).sqrt(); n];
    for _ in 0..40 {
        let mut av = vec![0.0f32; n];
        for i in 0..n {
            for j in 0..n { av[i] += mat[i * n + j] * v[j]; }
        }
        let norm = av.iter().map(|x| x * x).sum::<f32>().sqrt();
        if norm < 1e-20 { break; }
        let inv = 1.0 / norm;
        for i in 0..n { v[i] = av[i] * inv; }
    }
    let mut out = mat.to_vec();
    for i in 0..n {
        for j in 0..n {
            out[i * n + j] -= lambda_1 * v[i] * v[j];
        }
    }
    out
}

// ── Compilation ─────────────────────────────────────────────────────────────

/// Load the utility kernels from precompiled cubin (ZERO runtime nvcc).
///
/// The cubin is produced by build.rs from `common_device_functions.cuh` +
/// `dqn_utility_kernels.cu`. Contains: grad_norm, adam_update, saxpy,
/// spectral_norm_batched, clip_grad.
///
/// Returns `(grad_norm, grad_norm_finalize, adam_update, saxpy, zero, regime_scale, shrink_perturb, relu_mask, spectral_norm_batched, clip_grad, ...)`.
fn compile_training_kernels(
    stream: &Arc<CudaStream>,
    config: &GpuDqnTrainConfig,
) -> Result<(CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction), MLError> {
    info!(
        state_dim = ml_core::state_layout::STATE_DIM,
        total_params = compute_total_params(config),
        "GpuDqnTrainer: loading precompiled utility kernels (ZERO runtime nvcc)"
    );

    let context = stream.context();
    let module = context.load_cubin(DQN_UTILITY_CUBIN.to_vec()).map_err(|e| {
        MLError::ModelError(format!("dqn_utility cubin load: {e}"))
    })?;

    let grad_norm = module.load_function("dqn_grad_norm_kernel")
        .map_err(|e| MLError::ModelError(format!("dqn_grad_norm_kernel load: {e}")))?;
    let grad_norm_finalize = module.load_function("dqn_grad_norm_finalize")
        .map_err(|e| MLError::ModelError(format!("dqn_grad_norm_finalize load: {e}")))?;
    // Separate finalize instance for adam_child — MUST be from a different CUmodule
    // to avoid CUfunction sharing across child graphs (causes 3100ms replay on Hopper).
    let adam_finalize_module = context.load_cubin(DQN_UTILITY_CUBIN.to_vec())
        .map_err(|e| MLError::ModelError(format!("adam finalize cubin load: {e}")))?;
    let grad_norm_finalize_adam = adam_finalize_module.load_function("dqn_grad_norm_finalize")
        .map_err(|e| MLError::ModelError(format!("adam grad_norm_finalize load: {e}")))?;
    // Separate CUmodule for post_aux_child — adam_update + grad_norm_finalize
    // must NOT share CUfunction handles with adam_child or forward_child on Hopper.
    let post_aux_module = context.load_cubin(DQN_UTILITY_CUBIN.to_vec())
        .map_err(|e| MLError::ModelError(format!("post_aux cubin load: {e}")))?;
    let grad_norm_finalize_post_aux = post_aux_module.load_function("dqn_grad_norm_finalize")
        .map_err(|e| MLError::ModelError(format!("post_aux grad_norm_finalize load: {e}")))?;
    let adam_update_post_aux = post_aux_module.load_function("dqn_adam_update_kernel")
        .map_err(|e| MLError::ModelError(format!("post_aux adam_update load: {e}")))?;
    let scale_f32_post_aux = post_aux_module.load_function("dqn_scale_f32_kernel")
        .map_err(|e| MLError::ModelError(format!("post_aux scale_f32 load: {e}")))?;
    // Separate grad_norm instance for post_aux_child — forward_child already uses
    // grad_norm_standalone for d_logits clipping. Sharing across children corrupts
    // CUfunction state on Hopper → 3100ms replay.
    let grad_norm_standalone_post_aux = post_aux_module.load_function("dqn_grad_norm_kernel")
        .map_err(|e| MLError::ModelError(format!("post_aux grad_norm_standalone load: {e}")))?;
    // Separate CUmodule for adam_grad_child — saxpy_f32 is used in forward_child and
    // aux_child, so adam_grad_child needs its own isolated handle.
    let adam_grad_module = context.load_cubin(DQN_UTILITY_CUBIN.to_vec())
        .map_err(|e| MLError::ModelError(format!("adam_grad cubin load: {e}")))?;
    let saxpy_f32_adam_grad = adam_grad_module.load_function("dqn_saxpy_f32_kernel")
        .map_err(|e| MLError::ModelError(format!("adam_grad saxpy_f32 load: {e}")))?;
    // Separate CUmodule for aux_child — saxpy_f32 is used in forward_child and
    // adam_grad_child, so aux_child needs its own isolated handle.
    let aux_module = context.load_cubin(DQN_UTILITY_CUBIN.to_vec())
        .map_err(|e| MLError::ModelError(format!("aux cubin load: {e}")))?;
    let saxpy_f32_aux = aux_module.load_function("dqn_saxpy_f32_kernel")
        .map_err(|e| MLError::ModelError(format!("aux saxpy_f32 load: {e}")))?;
    // Plan 4 Task 1B-iv-rc: VSN dW attenuation handle for aux_child capture.
    // Loaded from the same `aux_module` as `saxpy_f32_aux` so it shares the
    // aux_child graph boundary; mirrors the main-path `scale_f32_kernel`
    // (loaded from `module` for forward_child) used by the main VSN backward.
    let scale_f32_aux = aux_module.load_function("dqn_scale_f32_kernel")
        .map_err(|e| MLError::ModelError(format!("aux scale_f32 load: {e}")))?;
    // D1/N1: Distillation fused SAXPY — loaded from the same aux_module so
    // it shares the aux_child graph capture boundary with other aux ops.
    let distill_saxpy_aux = aux_module.load_function("dqn_distill_saxpy_kernel")
        .map_err(|e| MLError::ModelError(format!("aux distill_saxpy load: {e}")))?;
    let adam_update = module.load_function("dqn_adam_update_kernel")
        .map_err(|e| MLError::ModelError(format!("dqn_adam_update_kernel load: {e}")))?;
    // f32_to_bf16_kernel / bf16_to_f32_kernel DELETED — pure f32 pipeline.
    let saxpy = module.load_function("dqn_saxpy_kernel")
        .map_err(|e| MLError::ModelError(format!("dqn_saxpy_kernel load: {e}")))?;
    let saxpy_f32 = module.load_function("dqn_saxpy_f32_kernel")
        .map_err(|e| MLError::ModelError(format!("dqn_saxpy_f32_kernel load: {e}")))?;
    let scale_f32 = module.load_function("dqn_scale_f32_kernel")
        .map_err(|e| MLError::ModelError(format!("dqn_scale_f32_kernel load: {e}")))?;
    let zero = module.load_function("dqn_zero_kernel")
        .map_err(|e| MLError::ModelError(format!("dqn_zero_kernel load: {e}")))?;
    let regime_scale = module.load_function("dqn_regime_scale_kernel")
        .map_err(|e| MLError::ModelError(format!("dqn_regime_scale_kernel load: {e}")))?;
    let shrink_perturb = module.load_function("dqn_shrink_perturb_kernel")
        .map_err(|e| MLError::ModelError(format!("dqn_shrink_perturb_kernel load: {e}")))?;
    let _relu_mask_from_module = module.load_function("dqn_relu_mask_kernel")
        .map_err(|e| MLError::ModelError(format!("dqn_relu_mask_kernel load: {e}")))?;
    let spectral_norm = module.load_function("spectral_norm_batched")
        .map_err(|e| MLError::ModelError(format!("spectral_norm_batched load: {e}")))?;
    let clip_grad = module.load_function("dqn_clip_grad_kernel")
        .map_err(|e| MLError::ModelError(format!("dqn_clip_grad_kernel load: {e}")))?;
    // Ungraphed kernels MUST be from a separate CUmodule — on Hopper, launching ANY
    // CUfunction from a CUmodule that also has graph-captured CUfunctions corrupts
    // the graph's kernel state (3100ms replay). All kernels launched outside graph
    // capture (per-step or at epoch boundaries) must use handles from this module.
    let ungraphed_module = context.load_cubin(DQN_UTILITY_CUBIN.to_vec())
        .map_err(|e| MLError::ModelError(format!("ungraphed utility cubin load: {e}")))?;
    let pad_states = ungraphed_module.load_function("pad_states_kernel")
        .map_err(|e| MLError::ModelError(format!("pad_states_kernel load: {e}")))?;
    let stochastic_depth_rng = ungraphed_module.load_function("stochastic_depth_rng")
        .map_err(|e| MLError::ModelError(format!("stochastic_depth_rng load: {e}")))?;
    // shrink_and_perturb() is called at epoch boundaries (after graph capture).
    // dqn_shrink_perturb_kernel lives in the same CUmodule as other graphed kernels
    // (scale_f32, saxpy_f32, spectral_norm, adam_update, etc.) — so it MUST be
    // loaded from ungraphed_module to prevent Hopper CUmodule corruption.
    let shrink_perturb_ungraphed = ungraphed_module.load_function("dqn_shrink_perturb_kernel")
        .map_err(|e| MLError::ModelError(format!("shrink_perturb_ungraphed load: {e}")))?;
    // scale_adam_momentum() is called at cosine LR warm restarts (after graph capture).
    // dqn_scale_f32_kernel is captured in forward_child — same CUmodule as above.
    let scale_f32_ungraphed = ungraphed_module.load_function("dqn_scale_f32_kernel")
        .map_err(|e| MLError::ModelError(format!("scale_f32_ungraphed load: {e}")))?;
    // stochastic_depth_scale is GRAPHED (inside forward_child via apply_regime_dropout)
    // — stays in the main module.
    let stochastic_depth = module.load_function("stochastic_depth_scale")
        .map_err(|e| MLError::ModelError(format!("stochastic_depth_scale load: {e}")))?;
    let pruning_mask_fn = module.load_function("lottery_ticket_mask")
        .map_err(|e| MLError::ModelError(format!("lottery_ticket_mask load: {e}")))?;
    let pruning_compute_fn = module.load_function("lottery_ticket_compute_mask")
        .map_err(|e| MLError::ModelError(format!("lottery_ticket_compute_mask load: {e}")))?;
    let causal_intervene = module.load_function("causal_intervene_feature")
        .map_err(|e| MLError::ModelError(format!("causal_intervene load: {e}")))?;
    let causal_reduce = module.load_function("causal_q_delta_reduce")
        .map_err(|e| MLError::ModelError(format!("causal_reduce load: {e}")))?;
    let causal_mean_reduce = module.load_function("causal_mean_reduce")
        .map_err(|e| MLError::ModelError(format!("causal_mean_reduce load: {e}")))?;
    let vaccine_dot = module.load_function("gradient_dot_and_norm")
        .map_err(|e| MLError::ModelError(format!("gradient_dot_and_norm load: {e}")))?;
    let vaccine_dot_finalize = module.load_function("gradient_dot_norm_finalize")
        .map_err(|e| MLError::ModelError(format!("gradient_dot_norm_finalize load: {e}")))?;
    let vaccine_project = module.load_function("gradient_project")
        .map_err(|e| MLError::ModelError(format!("gradient_project load: {e}")))?;
    let bn_tanh_bw = module.load_function("bn_tanh_backward_kernel")
        .map_err(|e| MLError::ModelError(format!("bn_tanh_backward_kernel load: {e}")))?;
    let bn_bias_grad = module.load_function("bn_bias_grad_kernel")
        .map_err(|e| MLError::ModelError(format!("bn_bias_grad_kernel load: {e}")))?;
    // SP15 Wave 4.1b OOB fix (2026-05-07): pre-load the dd_pct-appending
    // bottleneck concat kernel from the same `module` as `bn_tanh_bw` /
    // `bn_bias_grad` (forward-child captured replay group). The 3 trainer
    // production callers (online forward, target forward, DDQN argmax)
    // consume this handle via `&self.bn_tanh_concat_dd_kernel`; resolving
    // the symbol on-demand inside the captured forward child caused a
    // SEGV (cf. the field-level docstring in the struct definition).
    let bn_tanh_concat_dd = module.load_function("bn_tanh_concat_dd_kernel")
        .map_err(|e| MLError::ModelError(format!("bn_tanh_concat_dd_kernel load: {e}")))?;

    let her_inplace = module.load_function("her_inplace_relabel")
        .map_err(|e| MLError::ModelError(format!("her_inplace_relabel load: {e}")))?;
    let nan_check_f32 = module.load_function("dqn_nan_check_f32")
        .map_err(|e| MLError::ModelError(format!("dqn_nan_check_f32 load: {e}")))?;
    let nan_check_f32_b = module.load_function("dqn_nan_check_f32_b")
        .map_err(|e| MLError::ModelError(format!("dqn_nan_check_f32_b load: {e}")))?;
    // SP2: fused multi-buffer NaN check kernel. Single launch processes 12
    // backward-path slots (24-35) in 12 blocks. Replaces 8 individual
    // dqn_nan_check_f32 launches in `run_nan_checks_post_backward` (Task A4
    // wires the call site). Loaded from the same `module` as the per-buffer
    // variant — both belong to the post_aux_child captured replay group.
    let nan_check_fused_f32 = module.load_function("dqn_nan_check_fused_f32_kernel")
        .map_err(|e| MLError::ModelError(format!("dqn_nan_check_fused_f32_kernel load: {e}")))?;
    // SP1 Phase C surgical fix: in-place finite clamp kernel (replaces NaN/Inf
    // with 0; clamps finite |v| to ±max_abs from caller). Loaded from the same
    // `module` as the NaN check kernels because both are launched inside
    // `submit_forward_ops_main` (graph-captured forward_child) and
    // `apply_iqn_trunk_gradient` (aux_child) — sharing the module is correct
    // because they belong to the same captured replay group.
    let clamp_finite_f32 = module.load_function("dqn_clamp_finite_f32_kernel")
        .map_err(|e| MLError::ModelError(format!("dqn_clamp_finite_f32_kernel load: {e}")))?;

    let popart_normalize = module.load_function("popart_normalize_kernel")
        .map_err(|e| MLError::ModelError(format!("popart_normalize_kernel load: {e}")))?;
    let popart_robust = ungraphed_module.load_function("popart_normalize_robust")
        .map_err(|e| MLError::ModelError(format!("popart_normalize_robust load: {e}")))?;

    info!("GpuDqnTrainer: 40 utility kernels loaded from precompiled cubin (5 CUmodules)");
    Ok((grad_norm, grad_norm_finalize, grad_norm_finalize_adam, grad_norm_finalize_post_aux, adam_update, adam_update_post_aux, scale_f32_post_aux, saxpy, zero, regime_scale, shrink_perturb, _relu_mask_from_module, spectral_norm, clip_grad, pad_states, saxpy_f32, scale_f32, stochastic_depth, stochastic_depth_rng, bn_tanh_bw, bn_bias_grad, vaccine_dot, vaccine_project, vaccine_dot_finalize, causal_intervene, causal_reduce, causal_mean_reduce, pruning_mask_fn, pruning_compute_fn, her_inplace, nan_check_f32, nan_check_f32_b, nan_check_fused_f32, clamp_finite_f32, popart_normalize, saxpy_f32_adam_grad, saxpy_f32_aux, scale_f32_aux, distill_saxpy_aux, grad_norm_standalone_post_aux, shrink_perturb_ungraphed, scale_f32_ungraphed, popart_robust, bn_tanh_concat_dd))
}

/// Load the standalone Polyak EMA kernel from precompiled cubin.
///
/// `ema_kernel(target, online, tau, n)`: `target[i] = (1-tau)*target[i] + tau*online[i]`
///
/// This kernel is NOT captured in the CUDA Graph -- it runs after graph replay
/// to blend online weights into the target network in-place.
fn compile_ema_kernel(stream: &Arc<CudaStream>) -> Result<CudaFunction, MLError> {
    let context = stream.context();
    let module = context.load_cubin(EMA_CUBIN.to_vec()).map_err(|e| {
        MLError::ModelError(format!("ema cubin load: {e}"))
    })?;
    module.load_function("dqn_ema_kernel").map_err(|e| {
        MLError::ModelError(format!("dqn_ema_kernel load: {e}"))
    })
}

/// Load standalone ReLU mask kernel from precompiled cubin.
fn compile_relu_mask_standalone(stream: &Arc<CudaStream>) -> Result<CudaFunction, MLError> {
    let context = stream.context();
    let module = context.load_cubin(RELU_MASK_CUBIN.to_vec()).map_err(|e| {
        MLError::ModelError(format!("relu_mask cubin load: {e}"))
    })?;
    module.load_function("relu_mask_standalone").map_err(|e| {
        MLError::ModelError(format!("relu_mask_standalone load: {e}"))
    })
}

/// Load the standalone C51 distributional loss kernel from precompiled cubin.
///
/// This kernel operates on pre-computed forward pass outputs (value + advantage logits)
/// from cuBLAS, computing the C51 cross-entropy loss with Bellman projection.
/// Grid: (batch_size, 1, 1), Block: (256, 1, 1), Shmem: ~2 KB/block.
///
/// All variable constants (num_atoms, v_min, v_max, branch sizes) are passed
/// as runtime kernel parameters — no NVRTC compilation needed.
fn compile_c51_loss_kernel(
    stream: &Arc<CudaStream>,
    _config: &GpuDqnTrainConfig,
) -> Result<(CudaFunction, CudaFunction), MLError> {
    let context = stream.context();
    let module = context.load_cubin(C51_LOSS_CUBIN.to_vec())
        .map_err(|e| MLError::ModelError(format!("c51_loss cubin load: {e}")))?;
    let loss = module.load_function("c51_loss_batched")
        .map_err(|e| MLError::ModelError(format!("c51_loss_batched load: {e}")))?;
    let reduce = module.load_function("c51_loss_reduce")
        .map_err(|e| MLError::ModelError(format!("c51_loss_reduce load: {e}")))?;
    Ok((loss, reduce))
}

/// Load the F5/D2 Q-gap barrier gradient kernel from the c51_loss cubin.
///
/// Returns Ok(None) gracefully if the symbol is not found (older cubin cached).
fn load_barrier_gradient_kernel(
    stream: &Arc<CudaStream>,
) -> Result<Option<CudaFunction>, MLError> {
    let context = stream.context();
    let module = context.load_cubin(C51_LOSS_CUBIN.to_vec())
        .map_err(|e| MLError::ModelError(format!("c51_loss cubin load (barrier): {e}")))?;
    match module.load_function("barrier_gradient_direction") {
        Ok(f) => Ok(Some(f)),
        Err(e) => {
            tracing::warn!("barrier_gradient_direction not found in cubin (non-fatal): {e}");
            Ok(None)
        }
    }
}

/// Load the F4/D5 Information Bottleneck variance gradient kernel from the c51_loss cubin.
///
/// Returns Ok(None) gracefully if the symbol is not found (older cubin cached).
fn load_ib_gradient_kernel(
    stream: &Arc<CudaStream>,
) -> Result<Option<CudaFunction>, MLError> {
    let context = stream.context();
    let module = context.load_cubin(C51_LOSS_CUBIN.to_vec())
        .map_err(|e| MLError::ModelError(format!("c51_loss cubin load (ib): {e}")))?;
    match module.load_function("ib_gradient_direction") {
        Ok(f) => Ok(Some(f)),
        Err(e) => {
            tracing::warn!("ib_gradient_direction not found in cubin (non-fatal): {e}");
            Ok(None)
        }
    }
}

/// Load the C51 loss gradient kernel from precompiled cubin.
///
/// entropy_coeff is now passed as a runtime kernel parameter (not #define).
fn compile_c51_grad_kernel(
    stream: &Arc<CudaStream>,
    _config: &GpuDqnTrainConfig,
) -> Result<CudaFunction, MLError> {
    let context = stream.context();
    let module = context.load_cubin(C51_GRAD_CUBIN.to_vec())
        .map_err(|e| MLError::ModelError(format!("c51_grad cubin load: {e}")))?;
    module.load_function("c51_grad_kernel")
        .map_err(|e| MLError::ModelError(format!("c51_grad_kernel load: {e}")))
}

/// Load the MSE loss kernel from precompiled cubin.
///
/// All variable constants (num_atoms, v_min, v_max, branch sizes) are passed
/// as runtime kernel parameters — no NVRTC compilation needed.
fn compile_mse_loss_kernel(
    stream: &Arc<CudaStream>,
    _config: &GpuDqnTrainConfig,
) -> Result<CudaFunction, MLError> {
    let context = stream.context();
    let module = context.load_cubin(MSE_LOSS_CUBIN.to_vec())
        .map_err(|e| MLError::ModelError(format!("mse_loss cubin load: {e}")))?;
    module.load_function("mse_loss_batched")
        .map_err(|e| MLError::ModelError(format!("mse_loss_batched load: {e}")))
}

/// Load the MSE loss gradient kernel from precompiled cubin.
/// v_min/v_max are now runtime kernel parameters (not compile-time #define).
fn compile_mse_grad_kernel(
    stream: &Arc<CudaStream>,
    _config: &GpuDqnTrainConfig,
) -> Result<CudaFunction, MLError> {
    let context = stream.context();
    let module = context.load_cubin(MSE_GRAD_CUBIN.to_vec())
        .map_err(|e| MLError::ModelError(format!("mse_grad cubin load: {e}")))?;
    module.load_function("mse_grad_kernel")
        .map_err(|e| MLError::ModelError(format!("mse_grad_kernel load: {e}")))
}

/// Load the expected Q kernel from precompiled cubin.
fn compile_expected_q_kernel(
    stream: &Arc<CudaStream>,
) -> Result<CudaFunction, MLError> {
    let context = stream.context();
    let module = context.load_cubin(EXPECTED_Q_CUBIN.to_vec())
        .map_err(|e| MLError::ModelError(format!("expected_q cubin load: {e}")))?;
    module.load_function("compute_expected_q")
        .map_err(|e| MLError::ModelError(format!("compute_expected_q load: {e}")))
}

/// Load the Q-value statistics reduction kernel from precompiled cubin.
fn compile_q_stats_kernel(
    stream: &Arc<CudaStream>,
) -> Result<CudaFunction, MLError> {
    let context = stream.context();
    let module = context.load_cubin(Q_STATS_CUBIN.to_vec())
        .map_err(|e| MLError::ModelError(format!("q_stats cubin load: {e}")))?;
    module.load_function("q_stats_reduce")
        .map_err(|e| MLError::ModelError(format!("q_stats_reduce load: {e}")))
}

/// Load the per-branch Q-value statistics reducer.
///
/// Emits one 7-tuple per branch (direction, magnitude, order, urgency) for the
/// ISV v-range per-branch EMA update path. See `q_stats_kernel.cu` for the
/// `q_stats_per_branch_reduce` body. Runs alongside the legacy global
/// `q_stats_reduce` — no semantic conflict, both single-threaded deterministic.
fn compile_q_stats_per_branch_kernel(
    stream: &Arc<CudaStream>,
) -> Result<CudaFunction, MLError> {
    let context = stream.context();
    let module = context.load_cubin(Q_STATS_CUBIN.to_vec())
        .map_err(|e| MLError::ModelError(format!("q_stats cubin load: {e}")))?;
    module.load_function("q_stats_per_branch_reduce")
        .map_err(|e| MLError::ModelError(format!("q_stats_per_branch_reduce load: {e}")))
}

/// Load the per-magnitude-bin Q-mean reducer from the q_stats cubin.
///
/// Task 2.X "make Full useful": this kernel computes batch-mean Q-values
/// per magnitude-bin (Quarter/Half/Full) plus an absolute-scale reference,
/// feeding the four ISV slots [13..16] that drive the adaptive C51
/// magnitude bin weight. See q_stats_kernel.cu for the kernel body.
fn compile_q_mag_bin_means_kernel(
    stream: &Arc<CudaStream>,
) -> Result<CudaFunction, MLError> {
    let context = stream.context();
    let module = context.load_cubin(Q_STATS_CUBIN.to_vec())
        .map_err(|e| MLError::ModelError(format!("q_mag_bin_means cubin load: {e}")))?;
    module.load_function("q_mag_bin_means_reduce")
        .map_err(|e| MLError::ModelError(format!("q_mag_bin_means_reduce load: {e}")))
}

/// Load the per-direction-bin Q-mean reducer from the q_stats cubin.
///
/// Task 2.Y "make direction branch useful at eval": this kernel computes
/// batch-mean Q-values per direction-bin (Short/Hold/Long/Flat) plus an
/// absolute-scale reference, feeding ISV slots [17..21] that drive the
/// adaptive C51 direction bin weight. See q_stats_kernel.cu for the kernel
/// body (q_dir_bin_means_reduce).
fn compile_q_dir_bin_means_kernel(
    stream: &Arc<CudaStream>,
) -> Result<CudaFunction, MLError> {
    let context = stream.context();
    let module = context.load_cubin(Q_STATS_CUBIN.to_vec())
        .map_err(|e| MLError::ModelError(format!("q_dir_bin_means cubin load: {e}")))?;
    module.load_function("q_dir_bin_means_reduce")
        .map_err(|e| MLError::ModelError(format!("q_dir_bin_means_reduce load: {e}")))
}

// ── Shared memory sizing ────────────────────────────────────────────────────

/// Query the hardware's max shared memory per block via cuDeviceGetAttribute.
/// Returns usable bytes (floored at 48KB default).
///
/// Uses `CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK_OPTIN` to get the
/// true hardware ceiling. Uses 80% to leave headroom for kernel state.
/// Falls back to 48KB if the query fails.
pub(crate) fn query_max_shmem_bytes() -> usize {
    use cudarc::driver::sys::{cuDeviceGetAttribute, CUdevice_attribute};
    let mut max_shmem: i32 = 49152;
    let result = unsafe {
        cuDeviceGetAttribute(
            &mut max_shmem,
            CUdevice_attribute::CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK_OPTIN,
            0,
        )
    };
    if result != cudarc::driver::sys::CUresult::CUDA_SUCCESS {
        tracing::warn!("cuDeviceGetAttribute(MAX_SHARED_MEMORY_PER_BLOCK_OPTIN) failed: {result:?}");
    }
    // A100 reports 164KB, H100 reports 228KB — these genuinely support opt-in.
    // Consumer GPUs (RTX 3050: 100KB, RTX 4090: 100KB) report > 48KB but
    // fail with CUDA_ERROR_ILLEGAL_ADDRESS on dynamic shmem > 48KB with
    // some driver versions. Use 48KB safe default for < 164KB.
    let usable = if max_shmem >= 164_000 {
        (max_shmem as usize) * 80 / 100
    } else {
        49152
    };
    usable.max(49152)
}

// ── Device-to-device copy helpers ────────────────────────────────────────────

/// Extract raw CUDA device pointer (CUdeviceptr = u64) from a CudaSlice.
///
// raw_device_ptr wrappers removed — use CudaSlice::raw_ptr() directly (no event tracking).

/// Async device-to-device memcpy with error context.
pub(crate) fn dtod_copy_async(
    dst: u64,
    src: u64,
    num_bytes: usize,
    stream: &Arc<CudaStream>,
    idx: usize,
    op: &str,
) -> Result<(), MLError> {
    // Safety: caller guarantees src/dst are valid device pointers within
    // the same CUDA context, num_bytes ≤ allocation size of both regions,
    // and stream belongs to the same context.
    unsafe {
        cudarc::driver::result::memcpy_dtod_async(dst, src, num_bytes, stream.cu_stream())
            .map_err(|e| MLError::ModelError(format!("DtoD {op}[{idx}]: {e}")))?;
    }
    Ok(())
}

// ── Candle tensor → CudaSlice DtoD copy helpers ────────────────────────────

/// DtoD copy from a CudaSlice<f32> to a pre-allocated CudaSlice<f32>.
fn dtod_from_slice_f32(
    src: &CudaSlice<f32>,
    dst: &CudaSlice<f32>,
    num_elements: usize,
    stream: &Arc<CudaStream>,
    ctx: &str,
) -> Result<(), MLError> {
    let src_ptr = src.raw_ptr();
    let dst_ptr = dst.raw_ptr();
    dtod_copy_async(dst_ptr, src_ptr, num_elements * std::mem::size_of::<f32>(), stream, 0, ctx)
}

/// DtoD copy from a CudaSlice<u32> to a CudaSlice<i32>.
///
/// U32 and I32 are bit-compatible for action indices in [0, 44].
fn dtod_from_u32_to_i32(
    src: &CudaSlice<u32>,
    dst: &CudaSlice<i32>,
    num_elements: usize,
    stream: &Arc<CudaStream>,
    ctx: &str,
) -> Result<(), MLError> {
    let src_ptr = src.raw_ptr();
    let dst_ptr = dst.raw_ptr();
    dtod_copy_async(dst_ptr, src_ptr, num_elements * std::mem::size_of::<u32>(), stream, 0, ctx)
}

// ── Allocation helpers ──────────────────────────────────────────────────────

/// Round up to the next multiple of 32.
///
/// cuBLAS CUTLASS kernels (e.g. `cutlass_80_wmma_tensorop_f32_s161616gemm_f32_32x32_128x2_tn_align8`)
/// read in 32-element M-tiles. When the M dimension (num_atoms = 51) is not a
/// multiple of 32 the kernel reads past the logical buffer end. Padding the
/// **allocation** to a 32-element boundary prevents OOB reads without changing
/// the GemmEx logical dimensions or any kernel launch parameters.
#[inline(always)]
fn pad32(n: usize) -> usize {
    (n + 31) & !31
}

/// Round up to the next multiple of 128 (CUTLASS K-tile alignment).
///
/// cuBLAS CUTLASS kernels use 128-element K-tiles for TF32 GemmEx.
/// When the K dimension (state_dim) is not a multiple of 128, CUTLASS
/// reads past the row boundary of the B-matrix.  Padding each row of
/// the states buffer to `pad128(state_dim)` with zeros eliminates OOB
/// reads without changing the GEMM logical K dimension.
#[inline(always)]
fn pad128(n: usize) -> usize {
    (n + 127) & !127
}

fn alloc_f32(
    stream: &Arc<CudaStream>,
    n: usize,
    _name: &str,
) -> Result<CudaSlice<f32>, MLError> {
    stream.alloc_zeros::<f32>(n).map_err(|e| {
        MLError::ModelError(format!("GpuDqnTrainer alloc): {e}"))
    })
}

fn alloc_i32(
    stream: &Arc<CudaStream>,
    n: usize,
    name: &str,
) -> Result<CudaSlice<i32>, MLError> {
    stream.alloc_zeros::<i32>(n).map_err(|e| {
        MLError::ModelError(format!("GpuDqnTrainer alloc {name} ({n} i32): {e}"))
    })
}

fn alloc_u16(
    stream: &Arc<CudaStream>,
    n: usize,
    name: &str,
) -> Result<CudaSlice<u16>, MLError> {
    stream.alloc_zeros::<u16>(n).map_err(|e| {
        MLError::ModelError(format!("GpuDqnTrainer alloc {name} ({n} u16): {e}"))
    })
}

/// DtoD copy from a CudaSlice<f32> staging buffer to a pre-allocated CudaSlice<f32>.
fn dtod_from_staging(
    src: &CudaSlice<f32>,
    dst: &CudaSlice<f32>,
    num_elements: usize,
    stream: &Arc<CudaStream>,
    ctx: &str,
) -> Result<(), MLError> {
    let src_ptr = src.raw_ptr();
    let dst_ptr = dst.raw_ptr();
    dtod_copy_async(dst_ptr, src_ptr, num_elements * std::mem::size_of::<f32>(), stream, 0, ctx)
}

/// Load ensemble aggregate + diversity kernels from precompiled cubin.
///
/// Returns just the aggregate kernel; the diagnostic-only diversity-loss
/// kernel was removed (its result was never read — see ensemble_kernels.cu
/// header for the full story). The KL *gradient* kernel
/// (`ensemble_kl_gradient_kernel`) is loaded separately in
/// `fused_training.rs` since that one DOES feed back into training.
pub(crate) fn compile_ensemble_kernels(
    stream: &Arc<CudaStream>,
    _state_dim: usize,
) -> Result<CudaFunction, MLError> {
    let context = stream.context();
    let module = context.load_cubin(ENSEMBLE_CUBIN.to_vec())
        .map_err(|e| MLError::ModelError(format!("ensemble cubin load: {e}")))?;

    let aggregate = module
        .load_function("ensemble_aggregate_kernel")
        .map_err(|e| MLError::ModelError(format!("ensemble_aggregate_kernel load: {e}")))?;

    Ok(aggregate)
}

/// Launch the f32 copy CUDA kernel: copies f32 CudaSlice<f32> → f32 CudaSlice<f32>.
///
/// Grid: ceil(n/256), Block: 256. Kernel signature: `copy_kernel_b(src, dst, n)`.
/// Zero Candle tensor involvement — pure cudarc kernel launch.
pub(crate) fn launch_f32_copy(
    kernel: &CudaFunction,
    src: &CudaSlice<f32>,
    dst: &CudaSlice<f32>,
    n: usize,
    stream: &Arc<CudaStream>,
    ctx: &str,
) -> Result<(), MLError> {
    let n_i32 = n as i32;
    let launch_cfg = LaunchConfig {
        grid_dim: (((n + 255) / 256) as u32, 1, 1),
        block_dim: (256, 1, 1),
        shared_mem_bytes: 0,
    };

    // Safety: both src and dst are CudaSlice<f32>, valid GPU allocations on the
    // same context. Kernel reads src[0..n] and writes dst[0..n].
    unsafe {
        stream
            .launch_builder(kernel)
            .arg(src)
            .arg(dst)
            .arg(&n_i32)
            .launch(launch_cfg)
            .map_err(|e| MLError::ModelError(format!("f32_copy {ctx}: {e}")))?;
    }
    Ok(())
}

/// Compile the CQL (Conservative Q-Learning) logit gradient kernel.
///
/// Computes dCQL/d_value_logits and dCQL/d_adv_logits for the Branching Dueling
/// C51 architecture. The CQL penalty per branch is:
///   `penalty = alpha * (logsumexp_a(Q(a)) - Q(a_taken))`
///
/// The gradient flows through the softmax expectation:
///   `dCQL/d_logit[a,j] = alpha * dCQL_dQ[a] * p[j] * (z[j] - Q[a])`
/// where `dCQL_dQ[a] = softmax_Q[a] - I(a == a_taken)` and `p[j] = softmax(combined_logits)[j]`.
///
/// One thread per sample. Iterates over the 3 branches (exposure, order, urgency).
fn compile_cql_logit_grad_kernel(
    stream: &Arc<CudaStream>,
) -> Result<CudaFunction, MLError> {
    let context = stream.context();
    let module = context.load_cubin(CQL_GRAD_CUBIN.to_vec())
        .map_err(|e| MLError::ModelError(format!("cql_grad cubin load: {e}")))?;
    module.load_function("cql_logit_grad_kernel")
        .map_err(|e| MLError::ModelError(format!("cql_logit_grad_kernel load: {e}")))
}

// ---------------------------------------------------------------------------
// Test-only accessors: Plan 2 Task 2 (D.1 Mamba2 backward validation)
// ---------------------------------------------------------------------------
#[cfg(test)]
impl GpuDqnTrainer {
    /// Frobenius norm of the Mamba2 weight gradient buffer.
    ///
    /// Synchronises the CUDA stream, copies `mamba2_grad` (length =
    /// `2*(SH2+OFI_EMBED_DIM)*STATE_D + SH2*STATE_D`) from device to host,
    /// and returns `sqrt(sum(x^2))`.  Returns 0.0 if the buffer is empty.
    ///
    /// Called by the `mamba2_backward_gradients_propagate` smoke test to
    /// confirm the backward kernel is not silently producing zeros.
    pub(crate) fn mamba2_grad_norm_for_test(&self) -> Result<f32, MLError> {
        let n = self.mamba2_grad.len();
        if n == 0 { return Ok(0.0); }
        self.stream.synchronize()
            .map_err(|e| MLError::ModelError(format!("mamba2_grad_norm sync: {e}")))?;
        let mut host = vec![0.0_f32; n];
        self.stream.memcpy_dtoh(&self.mamba2_grad, &mut host)
            .map_err(|e| MLError::ModelError(format!("mamba2_grad dtoh: {e}")))?;
        self.stream.synchronize()
            .map_err(|e| MLError::ModelError(format!("mamba2_grad_norm post-dtoh sync: {e}")))?;
        let norm: f32 = host.iter().map(|x| x * x).sum::<f32>().sqrt();
        Ok(norm)
    }

    /// Launch `mamba2_scan_projected_bwd` on caller-supplied device buffers.
    ///
    /// Used by `mamba2_backward_grad_check` to exercise the kernel directly
    /// with synthetic data, independent of the CUDA graph and training loop.
    ///
    /// # Arguments
    /// * `a_proj`        – `[B*K*STATE_D]` gate projection from forward
    /// * `b_proj`        – `[B*K*STATE_D]` input projection from forward
    /// * `d_h_enriched`  – `[B*SH2]` upstream gradient
    /// * `w_c`           – `[SH2*STATE_D]` W_C weight
    /// * `d_gate`        – `[B*K*STATE_D]` output: gate gradient (must be pre-allocated)
    /// * `d_x_out`       – `[B*K*STATE_D]` output: input gradient (must be pre-allocated)
    /// * `d_context`     – `[B*STATE_D]` output: x_K for W_C grad GEMM (must be pre-allocated)
    /// * `b`, `k`, `sh2`, `state_d` – dimensions (must match allocation sizes)
    pub(crate) fn launch_mamba2_bwd_for_test(
        &self,
        a_proj:       &CudaSlice<f32>,
        b_proj:       &CudaSlice<f32>,
        d_h_enriched: &CudaSlice<f32>,
        w_c:          &CudaSlice<f32>,
        d_gate:       &mut CudaSlice<f32>,
        d_x_out:      &mut CudaSlice<f32>,
        d_context:    &mut CudaSlice<f32>,
        b: usize,
        k: usize,
        sh2: usize,
        state_d: usize,
    ) -> Result<(), MLError> {
        let a_ptr = a_proj.raw_ptr();
        let b_ptr = b_proj.raw_ptr();
        let dh_ptr = d_h_enriched.raw_ptr();
        let wc_ptr = w_c.raw_ptr();
        let null_tw: u64 = 0; // temporal_weight = NULL (pass 1.0 scaling)
        let dg_ptr = d_gate.raw_ptr();
        let dx_ptr = d_x_out.raw_ptr();
        let dc_ptr = d_context.raw_ptr();
        let null_isv: u64 = 0; // isv_signals = NULL (stability = 1.0)
        let grid_y = ((state_d + 31) / 32) as u32;
        unsafe {
            self.stream.launch_builder(&self.mamba2_scan_proj_bwd_kernel)
                .arg(&a_ptr)
                .arg(&b_ptr)
                .arg(&dh_ptr)
                .arg(&wc_ptr)
                .arg(&null_tw)
                .arg(&dg_ptr)
                .arg(&dx_ptr)
                .arg(&dc_ptr)
                .arg(&(b as i32))
                .arg(&(k as i32))
                .arg(&(sh2 as i32))
                .arg(&(state_d as i32))
                .arg(&null_isv)
                .launch(cudarc::driver::LaunchConfig {
                    grid_dim: (b as u32, grid_y, 1),
                    block_dim: (32, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("mamba2_bwd test launch: {e}")))?;
        }
        self.stream.synchronize()
            .map_err(|e| MLError::ModelError(format!("mamba2_bwd test sync: {e}")))?;
        Ok(())
    }
}