foxhunt/crates/ml/src/cuda_pipeline/gpu_aux_trunk.rs

#![allow(unsafe_code)] // Required for CUDA kernel launches.
// Allow non-snake-case for `dW{1,2,3}_ptr` / `dW_reduce_kernel` /
// `launch_dW_reduce` — the math notation `dW` is the universally
// recognised symbol for weight gradient (matches the kernel parameter
// names in `aux_trunk_backward_kernel.cu`). Snake-casing them
// (`d_w1_ptr`, `d_w_reduce_kernel`) loses the math correspondence and
// hurts readability when paired with the kernel source.
#![allow(non_snake_case)]

//! `gpu_aux_trunk` — SP14 Layer C Phase C.3 (2026-05-08).
//!
//! Rust wrapper for the auxiliary trunk forward kernel
//! (`aux_trunk_forward_kernel.cu`). Owns the pre-loaded `CudaFunction`
//! handle so launches in the captured graph never touch host-side cubin
//! load paths (per `pearl_no_host_branches_in_captured_graph.md`).
//!
//! # Architecture
//!
//! 3-layer Linear→ELU→Linear→ELU→Linear MLP parallel to Q's GRN trunk.
//! Both trunks read the shared encoder output (`x_in [B, ENCODER_OUT_DIM]`);
//! Q trunk produces `h_s2`, aux trunk produces `h_s2_aux`. Aux backward
//! (Phase C.4) terminates at the encoder boundary — Q-loss is the sole
//! shaping force on the encoder per the locked design decision in
//! `2026-05-07-sp14-layer-c-separate-aux-trunk.md`.
//!
//! Saved-for-backward outputs (consumed by aux trunk backward in C.4):
//!
//!   - `h_aux1 [B, H1]`       — Layer-1 post-ELU activation
//!   - `h_aux2 [B, H2]`       — Layer-2 post-ELU activation
//!   - `h_s2_aux [B, AUX_HIDDEN_DIM]` — final linear output (no activation)
//!
//! Production topology (verified C.2):
//!   - `ENCODER_OUT_DIM = config.shared_h1 = 256`
//!   - `H1 = AUX_TRUNK_H1 = 256`
//!   - `H2 = AUX_TRUNK_H2 = 128` (mild bottleneck)
//!   - `AUX_HIDDEN_DIM = config.shared_h2 = 256` (matches aux head input dim;
//!     internal aux head hidden layer enlarged 32 → 128 on 2026-05-13;
//!     initial 32 → 256 attempt OOM'd L40S — dialed back to 128)
//!
//! # Pearls applied
//!
//! - `pearl_no_host_branches_in_captured_graph.md` — `CudaFunction`
//!   pre-loaded once at construction; no `load_cubin` / `load_function`
//!   on the launch path.
//! - `feedback_no_atomicadd.md` — forward kernel writes only per-(b, j)
//!   outputs; no contention across blocks.
//! - `feedback_no_stubs.md` — full launcher, no placeholder body.

use std::sync::Arc;

use cudarc::driver::{CudaFunction, CudaStream, LaunchConfig, PushKernelArg};

use crate::MLError;

use super::gpu_dqn_trainer::{
    AUX_HORIZON_UPDATE_CUBIN, AUX_TRUNK_BACKWARD_CUBIN, AUX_TRUNK_FORWARD_CUBIN,
    AVG_WIN_HOLD_TIME_UPDATE_CUBIN, DD_SATURATION_FLOOR_UPDATE_CUBIN,
    HOLD_REWARD_CAP_UPDATE_CUBIN,
    H_S2_AUX_RMS_EMA_CUBIN,
    KELLY_BAYESIAN_PRIORS_UPDATE_CUBIN, MIN_HOLD_TEMPERATURE_UPDATE_CUBIN,
    REWARD_CAP_UPDATE_CUBIN,
};

/// Hidden width of the aux trunk's first internal layer (Linear_1 → ELU
/// output). Mirrors the encoder output dimension so Layer-1 acts as a
/// representation-mixing identity-shaped projection.
pub(crate) const AUX_TRUNK_H1: usize = 256;

/// Hidden width of the aux trunk's second internal layer (Linear_2 → ELU
/// output). Smaller than H1 for a mild bottleneck — encourages the aux
/// trunk to compress its representation before lifting back up to
/// `AUX_HIDDEN_DIM` in Layer 3.
///
/// **Kept at 128 (2026-05-13)**: initial enlargement to 256 alongside
/// `AUX_HIDDEN_DIM 32 → 256` OOM'd L40S — trunk dW partials grew too.
/// Kept at 128 for memory budget; head's 4× enlargement to 128 is the
/// primary capacity uplift.
pub(crate) const AUX_TRUNK_H2: usize = 128;

/// Block dim used by the per-sample forward kernel. Mirrors
/// `aux_heads_kernel.cu`'s `AUX_BLOCK = 256`.
const AUX_TRUNK_BLOCK: u32 = 256;

/// Forward orchestrator — owns the single `CudaFunction` for
/// `aux_trunk_forward`. Backward kernel + ops land in Phase C.4.
///
/// Dropping this struct releases the underlying `CudaFunction` handle
/// via cudarc's RAII; no explicit teardown required.
#[allow(missing_debug_implementations)]
pub(crate) struct AuxTrunkForwardOps {
    /// Pre-loaded forward kernel handle (loaded once at construction;
    /// never re-loaded on the launch path).
    forward_kernel: CudaFunction,
}

impl AuxTrunkForwardOps {
    /// Load the forward kernel handle from the precompiled cubin.
    /// Mirrors `GrnBlock::new`'s loader pattern.
    pub(crate) fn new(stream: &Arc<CudaStream>) -> Result<Self, MLError> {
        let context = stream.context();
        let module = context
            .load_cubin(AUX_TRUNK_FORWARD_CUBIN.to_vec())
            .map_err(|e| MLError::ModelError(format!("aux_trunk_forward cubin load: {e}")))?;
        let forward_kernel = module
            .load_function("aux_trunk_forward")
            .map_err(|e| MLError::ModelError(format!("aux_trunk_forward load: {e}")))?;
        Ok(Self { forward_kernel })
    }

    /// Launch `aux_trunk_forward`: 3-layer Linear→ELU→Linear→ELU→Linear MLP.
    ///
    /// Caller-owned buffers (raw `u64` device pointers for graph-capture
    /// safety; mirrors `gpu_grn.rs::forward_raw` and `gpu_aux_heads.rs`):
    ///   * `x_in_ptr`         — `[B, ENCODER_OUT_DIM]` row-major (encoder output).
    ///   * `w1_ptr`/`b1_ptr`  — `[ENCODER_OUT_DIM, H1]` / `[H1]`.
    ///   * `w2_ptr`/`b2_ptr`  — `[H1, H2]` / `[H2]`.
    ///   * `w3_ptr`/`b3_ptr`  — `[H2, AUX_HIDDEN_DIM]` / `[AUX_HIDDEN_DIM]`.
    ///   * `h_aux1_out_ptr`   — `[B, H1]`  SAVED post-ELU Layer-1 (consumed by C.4 backward).
    ///   * `h_aux2_out_ptr`   — `[B, H2]`  SAVED post-ELU Layer-2 (consumed by C.4 backward).
    ///   * `h_s2_aux_out_ptr` — `[B, AUX_HIDDEN_DIM]` final aux trunk output.
    ///
    /// `h1` / `h2` / `aux_hidden_dim` are passed as runtime args for
    /// kernel-signature stability (a future bump won't require an ABI
    /// change). `encoder_out_dim` is the encoder's actual output width
    /// (typically `config.shared_h1 = 256`).
    ///
    /// Block: `AUX_TRUNK_BLOCK = 256` threads.
    /// Grid: `B` blocks (one block per batch row).
    /// Shared mem: `(H1 + H2) * sizeof(f32)` bytes — Layer-1 cache for
    /// Layer-2 matmul + Layer-2 cache for Layer-3 matmul.
    #[allow(clippy::too_many_arguments)]
    pub(crate) fn launch(
        &self,
        stream: &Arc<CudaStream>,
        x_in_ptr: u64,
        w1_ptr: u64,
        b1_ptr: u64,
        w2_ptr: u64,
        b2_ptr: u64,
        w3_ptr: u64,
        b3_ptr: u64,
        h_aux1_out_ptr: u64,
        h_aux2_out_ptr: u64,
        h_s2_aux_out_ptr: u64,
        b: usize,
        encoder_out_dim: usize,
        h1: usize,
        h2: usize,
        aux_hidden_dim: usize,
    ) -> Result<(), MLError> {
        let b_i32 = b as i32;
        let enc_i32 = encoder_out_dim as i32;
        let h1_i32 = h1 as i32;
        let h2_i32 = h2 as i32;
        let aux_i32 = aux_hidden_dim as i32;
        // Shared memory: H1 + H2 floats — Layer-1 cache + Layer-2 cache.
        let smem_bytes = (h1 as u32 + h2 as u32) * std::mem::size_of::<f32>() as u32;
        unsafe {
            stream
                .launch_builder(&self.forward_kernel)
                .arg(&x_in_ptr)
                .arg(&w1_ptr)
                .arg(&b1_ptr)
                .arg(&w2_ptr)
                .arg(&b2_ptr)
                .arg(&w3_ptr)
                .arg(&b3_ptr)
                .arg(&h_aux1_out_ptr)
                .arg(&h_aux2_out_ptr)
                .arg(&h_s2_aux_out_ptr)
                .arg(&b_i32)
                .arg(&enc_i32)
                .arg(&h1_i32)
                .arg(&h2_i32)
                .arg(&aux_i32)
                .launch(LaunchConfig {
                    grid_dim: (b as u32, 1, 1),
                    block_dim: (AUX_TRUNK_BLOCK, 1, 1),
                    shared_mem_bytes: smem_bytes,
                })
                .map_err(|e| MLError::ModelError(format!("aux_trunk_forward: {e}")))?;
        }
        Ok(())
    }
}

/// Block dim for the backward kernels — power-of-two so the shmem-tree
/// reduce in `aux_trunk_bwd_dW_reduce` / `aux_trunk_bwd_db_reduce` halves
/// cleanly without odd-stride bookkeeping. Mirrors `AUX_TRUNK_BWD_BLOCK`
/// in `aux_trunk_backward_kernel.cu`.
const AUX_TRUNK_BWD_BLOCK: u32 = 256;

/// Backward orchestrator — owns three pre-loaded `CudaFunction` handles
/// for `aux_trunk_bwd_dh_pre`, `aux_trunk_bwd_dW_reduce`, and
/// `aux_trunk_bwd_db_reduce`. A single `launch()` call orchestrates the
/// full backward pass over all three trunk layers without allocating
/// per-sample partial buffers (per-element block-tree-reduce instead).
///
/// CRITICAL: this op set does NOT compute or write `dx_in`. The encoder
/// boundary is the stop-gradient per the locked design decision in
/// `2026-05-07-sp14-layer-c-separate-aux-trunk.md` §C.4. The structural
/// enforcement is in the kernel signatures — none of the three kernels
/// accept a `dx_in_out` pointer, so the kernel set literally cannot
/// touch encoder gradient memory. This is verified by the
/// `aux_trunk_backward_does_not_write_dx` oracle test which inspects the
/// kernel source for any `dx_in` write pattern.
///
/// # Pearls applied
///
/// - `pearl_no_host_branches_in_captured_graph.md` — all three
///   `CudaFunction`s are pre-loaded at construction. The backward
///   `launch()` issues a fixed sequence of seven kernel launches with
///   no host-side branching.
/// - `feedback_no_atomicadd.md` — every dW/db output cell is owned by
///   exactly one block; block-tree-reduce over the batch dim within
///   shared memory.
/// - `feedback_no_stubs.md` — full backward, no zero-fill placeholders,
///   no skipped layers.
#[allow(missing_debug_implementations)]
pub(crate) struct AuxTrunkBackwardOps {
    /// Pre-pass: per-sample kernel that emits scratch buffers
    /// `dh_aux1_pre [B, H1]` and `dh_aux2_pre [B, H2]`.
    dh_pre_kernel: CudaFunction,
    /// Generic outer-product reduce for any dW matrix:
    /// `dW[k, j] = sum_b A[b, k] * B[b, j]`. Launched 3× per backward.
    dW_reduce_kernel: CudaFunction,
    /// Generic batch reduce for any db vector:
    /// `db[j] = sum_b B[b, j]`. Launched 3× per backward.
    db_reduce_kernel: CudaFunction,
}

impl AuxTrunkBackwardOps {
    /// Load the three backward kernel handles from the precompiled cubin.
    /// Mirrors `AuxTrunkForwardOps::new`'s loader pattern. All three
    /// kernels live in a single cubin so one `load_cubin` call is
    /// sufficient.
    pub(crate) fn new(stream: &Arc<CudaStream>) -> Result<Self, MLError> {
        let context = stream.context();
        let module = context
            .load_cubin(AUX_TRUNK_BACKWARD_CUBIN.to_vec())
            .map_err(|e| MLError::ModelError(format!("aux_trunk_backward cubin load: {e}")))?;
        let dh_pre_kernel = module
            .load_function("aux_trunk_bwd_dh_pre")
            .map_err(|e| MLError::ModelError(format!("aux_trunk_bwd_dh_pre load: {e}")))?;
        let dW_reduce_kernel = module
            .load_function("aux_trunk_bwd_dW_reduce")
            .map_err(|e| MLError::ModelError(format!("aux_trunk_bwd_dW_reduce load: {e}")))?;
        let db_reduce_kernel = module
            .load_function("aux_trunk_bwd_db_reduce")
            .map_err(|e| MLError::ModelError(format!("aux_trunk_bwd_db_reduce load: {e}")))?;
        Ok(Self {
            dh_pre_kernel,
            dW_reduce_kernel,
            db_reduce_kernel,
        })
    }

    /// Launch the full backward pass for the aux trunk.
    ///
    /// Orchestrates seven kernel launches in fixed sequence (no host-
    /// side branching, capture-friendly):
    ///   1. `aux_trunk_bwd_dh_pre`     — writes `dh_aux2_pre`, `dh_aux1_pre`.
    ///   2. `aux_trunk_bwd_dW_reduce`  — writes `dW3 = h_aux2.T @ d_logits`.
    ///   3. `aux_trunk_bwd_db_reduce`  — writes `db3 = sum_b d_logits[b, :]`.
    ///   4. `aux_trunk_bwd_dW_reduce`  — writes `dW2 = h_aux1.T @ dh_aux2_pre`.
    ///   5. `aux_trunk_bwd_db_reduce`  — writes `db2 = sum_b dh_aux2_pre[b, :]`.
    ///   6. `aux_trunk_bwd_dW_reduce`  — writes `dW1 = x_in.T @ dh_aux1_pre`.
    ///   7. `aux_trunk_bwd_db_reduce`  — writes `db1 = sum_b dh_aux1_pre[b, :]`.
    ///
    /// Caller-owned buffers (raw `u64` device pointers; mirrors
    /// `AuxTrunkForwardOps::launch` and `gpu_aux_heads.rs`):
    ///   - `dh_s2_aux_in_ptr`     `[B, AUX_HIDDEN_DIM]`  upstream gradient.
    ///   - `x_in_ptr`             `[B, ENCODER_OUT_DIM]` saved-fwd input.
    ///   - `h_aux1_ptr`           `[B, H1]`              saved-fwd post-ELU.
    ///   - `h_aux2_ptr`           `[B, H2]`              saved-fwd post-ELU.
    ///   - `w2_ptr` / `w3_ptr`    weight matrices for backward matmuls.
    ///   - `dh_aux1_pre_scratch`  `[B, H1]`              caller-allocated scratch.
    ///   - `dh_aux2_pre_scratch`  `[B, H2]`              caller-allocated scratch.
    ///   - `dW1_ptr` / `db1_ptr`  `[ENCODER_OUT_DIM, H1]` / `[H1]`.
    ///   - `dW2_ptr` / `db2_ptr`  `[H1, H2]`              / `[H2]`.
    ///   - `dW3_ptr` / `db3_ptr`  `[H2, AUX_HIDDEN_DIM]`  / `[AUX_HIDDEN_DIM]`.
    ///
    /// Output gradient buffers are OVERWRITTEN (not accumulated). If
    /// caller wants accumulation across multiple backward calls, they
    /// must SAXPY into a separate accumulator after this call. (Aux
    /// trunk's gradient is computed once per training step in C.5; no
    /// accumulation needed.)
    ///
    /// NO `dx_in_out` parameter — that is the structural stop-gradient
    /// at the encoder boundary. Encoder gradient stays Q-shaped.
    #[allow(clippy::too_many_arguments)]
    pub(crate) fn launch(
        &self,
        stream: &Arc<CudaStream>,
        dh_s2_aux_in_ptr: u64,
        x_in_ptr: u64,
        h_aux1_ptr: u64,
        h_aux2_ptr: u64,
        w2_ptr: u64,
        w3_ptr: u64,
        dh_aux1_pre_scratch_ptr: u64,
        dh_aux2_pre_scratch_ptr: u64,
        dW1_ptr: u64,
        db1_ptr: u64,
        dW2_ptr: u64,
        db2_ptr: u64,
        dW3_ptr: u64,
        db3_ptr: u64,
        b: usize,
        encoder_out_dim: usize,
        h1: usize,
        h2: usize,
        aux_hidden_dim: usize,
    ) -> Result<(), MLError> {
        let b_i32 = b as i32;
        // `encoder_out_dim` is consumed by `launch_dW_reduce` for dW1
        // (Krows = ENCODER_OUT_DIM); the dh_pre kernel doesn't need it as
        // a runtime arg because dh_pre only computes hidden-layer
        // gradients. `enc_i32` would be redundant here.
        let h1_i32 = h1 as i32;
        let h2_i32 = h2 as i32;
        let aux_i32 = aux_hidden_dim as i32;
        let block_smem = AUX_TRUNK_BWD_BLOCK * std::mem::size_of::<f32>() as u32;

        // ── 1. Pre-pass: dh_aux2_pre + dh_aux1_pre ────────────────────
        // Block: AUX_TRUNK_BWD_BLOCK threads. Grid: (B, 1, 1).
        // Shared memory: H2 floats (Layer-2 post-multiply cache).
        let dh_pre_smem = h2 as u32 * std::mem::size_of::<f32>() as u32;
        unsafe {
            stream
                .launch_builder(&self.dh_pre_kernel)
                .arg(&dh_s2_aux_in_ptr)
                .arg(&w3_ptr)
                .arg(&w2_ptr)
                .arg(&h_aux1_ptr)
                .arg(&h_aux2_ptr)
                .arg(&dh_aux2_pre_scratch_ptr)
                .arg(&dh_aux1_pre_scratch_ptr)
                .arg(&b_i32)
                .arg(&h1_i32)
                .arg(&h2_i32)
                .arg(&aux_i32)
                .launch(LaunchConfig {
                    grid_dim: (b as u32, 1, 1),
                    block_dim: (AUX_TRUNK_BWD_BLOCK, 1, 1),
                    shared_mem_bytes: dh_pre_smem,
                })
                .map_err(|e| MLError::ModelError(format!("aux_trunk_bwd_dh_pre: {e}")))?;
        }

        // ── 2. dW3 = h_aux2.T @ d_logits  [H2, AUX_HIDDEN_DIM] ────────
        self.launch_dW_reduce(
            stream,
            h_aux2_ptr,
            dh_s2_aux_in_ptr,
            dW3_ptr,
            b,
            h2,
            aux_hidden_dim,
            block_smem,
        )?;

        // ── 3. db3 = sum_b d_logits[b, :]  [AUX_HIDDEN_DIM] ───────────
        self.launch_db_reduce(stream, dh_s2_aux_in_ptr, db3_ptr, b, aux_hidden_dim, block_smem)?;

        // ── 4. dW2 = h_aux1.T @ dh_aux2_pre  [H1, H2] ─────────────────
        self.launch_dW_reduce(
            stream,
            h_aux1_ptr,
            dh_aux2_pre_scratch_ptr,
            dW2_ptr,
            b,
            h1,
            h2,
            block_smem,
        )?;

        // ── 5. db2 = sum_b dh_aux2_pre[b, :]  [H2] ────────────────────
        self.launch_db_reduce(stream, dh_aux2_pre_scratch_ptr, db2_ptr, b, h2, block_smem)?;

        // ── 6. dW1 = x_in.T @ dh_aux1_pre  [ENCODER_OUT_DIM, H1] ──────
        self.launch_dW_reduce(
            stream,
            x_in_ptr,
            dh_aux1_pre_scratch_ptr,
            dW1_ptr,
            b,
            encoder_out_dim,
            h1,
            block_smem,
        )?;

        // ── 7. db1 = sum_b dh_aux1_pre[b, :]  [H1] ────────────────────
        self.launch_db_reduce(stream, dh_aux1_pre_scratch_ptr, db1_ptr, b, h1, block_smem)?;

        // STOP — no Layer-1 dx kernel. Encoder boundary is the stop-grad.
        Ok(())
    }

    /// Helper: launch `aux_trunk_bwd_dW_reduce` for one weight matrix.
    /// Grid: `(Krows * Jcols, 1, 1)` — one block per output element.
    #[allow(clippy::too_many_arguments)]
    fn launch_dW_reduce(
        &self,
        stream: &Arc<CudaStream>,
        a_ptr: u64,
        b_grad_ptr: u64,
        dw_out_ptr: u64,
        b: usize,
        krows: usize,
        jcols: usize,
        smem_bytes: u32,
    ) -> Result<(), MLError> {
        let b_i32 = b as i32;
        let k_i32 = krows as i32;
        let j_i32 = jcols as i32;
        let grid = (krows * jcols) as u32;
        unsafe {
            stream
                .launch_builder(&self.dW_reduce_kernel)
                .arg(&a_ptr)
                .arg(&b_grad_ptr)
                .arg(&dw_out_ptr)
                .arg(&b_i32)
                .arg(&k_i32)
                .arg(&j_i32)
                .launch(LaunchConfig {
                    grid_dim: (grid, 1, 1),
                    block_dim: (AUX_TRUNK_BWD_BLOCK, 1, 1),
                    shared_mem_bytes: smem_bytes,
                })
                .map_err(|e| MLError::ModelError(format!("aux_trunk_bwd_dW_reduce: {e}")))?;
        }
        Ok(())
    }

    /// Helper: launch `aux_trunk_bwd_db_reduce` for one bias vector.
    /// Grid: `(Jcols, 1, 1)` — one block per output element.
    fn launch_db_reduce(
        &self,
        stream: &Arc<CudaStream>,
        b_grad_ptr: u64,
        db_out_ptr: u64,
        b: usize,
        jcols: usize,
        smem_bytes: u32,
    ) -> Result<(), MLError> {
        let b_i32 = b as i32;
        let j_i32 = jcols as i32;
        unsafe {
            stream
                .launch_builder(&self.db_reduce_kernel)
                .arg(&b_grad_ptr)
                .arg(&db_out_ptr)
                .arg(&b_i32)
                .arg(&j_i32)
                .launch(LaunchConfig {
                    grid_dim: (jcols as u32, 1, 1),
                    block_dim: (AUX_TRUNK_BWD_BLOCK, 1, 1),
                    shared_mem_bytes: smem_bytes,
                })
                .map_err(|e| MLError::ModelError(format!("aux_trunk_bwd_db_reduce: {e}")))?;
        }
        Ok(())
    }
}

/// SP14 Layer C Phase C.4b (2026-05-08): adaptive aux prediction horizon
/// producer.
///
/// Drives `ISV[AUX_PRED_HORIZON_BARS_INDEX=450]` from
/// `ISV[AVG_WIN_HOLD_TIME_BARS_INDEX=451]` via Pearl-A first-observation
/// bootstrap + slow Wiener-α EMA. Single-thread, single-block kernel —
/// horizon is slow-moving so per-epoch boundary launch is appropriate
/// (per-step would be wasteful and would track sample noise).
///
/// # Pearls applied
/// - `pearl_first_observation_bootstrap.md` — sentinel = 60.0; first
///   valid observation REPLACES (no blend) so the cold-start sentinel
///   never contaminates the EMA.
/// - `pearl_wiener_optimal_adaptive_alpha.md` — fixed α=0.01 fallback
///   when no target-variance EMA exists.
/// - `pearl_no_host_branches_in_captured_graph.md` — `CudaFunction`
///   pre-loaded once at construction.
/// - `feedback_isv_for_adaptive_bounds.md` — h_min=1, h_max=240 are
///   fundamental floors/ceilings, not tuning parameters.
#[allow(missing_debug_implementations)]
pub(crate) struct AuxHorizonUpdateOps {
    update_kernel: CudaFunction,
}

impl AuxHorizonUpdateOps {
    pub(crate) fn new(stream: &Arc<CudaStream>) -> Result<Self, MLError> {
        let context = stream.context();
        let module = context
            .load_cubin(AUX_HORIZON_UPDATE_CUBIN.to_vec())
            .map_err(|e| MLError::ModelError(format!("aux_horizon_update cubin load: {e}")))?;
        let update_kernel = module
            .load_function("aux_horizon_update")
            .map_err(|e| MLError::ModelError(format!("aux_horizon_update load: {e}")))?;
        Ok(Self { update_kernel })
    }

    /// Launch the horizon producer.
    ///
    /// Args:
    ///   - `isv_ptr`: ISV[ISV_TOTAL_DIM] device pointer (mapped-pinned).
    ///   - `isv_h_idx`: AUX_PRED_HORIZON_BARS_INDEX (450).
    ///   - `isv_h_target_idx`: AVG_WIN_HOLD_TIME_BARS_INDEX (451).
    ///   - `isv_h_target_var_idx`: variance EMA slot for target, or `-1`
    ///     if none (uses fixed α=0.01).
    ///   - `h_min`, `h_max`: fundamental bounds (1.0, 240.0).
    ///   - `sentinel_h`: SENTINEL_AUX_PRED_HORIZON_BARS (60.0).
    #[allow(clippy::too_many_arguments)]
    pub(crate) fn launch(
        &self,
        stream: &Arc<CudaStream>,
        isv_ptr: u64,
        isv_h_idx: i32,
        isv_h_target_idx: i32,
        isv_h_target_var_idx: i32,
        h_min: f32,
        h_max: f32,
        sentinel_h: f32,
    ) -> Result<(), MLError> {
        unsafe {
            stream
                .launch_builder(&self.update_kernel)
                .arg(&isv_ptr)
                .arg(&isv_h_idx)
                .arg(&isv_h_target_idx)
                .arg(&isv_h_target_var_idx)
                .arg(&h_min)
                .arg(&h_max)
                .arg(&sentinel_h)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (1, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("aux_horizon_update: {e}")))?;
        }
        Ok(())
    }
}

/// SP14 Layer C Phase C.4b (2026-05-08): avg winning hold time EMA producer.
///
/// Sweeps the per-epoch `hold_at_exit_per_sample` +
/// `trade_profitable_per_sample` buffers (populated by
/// `unified_env_step_core` in `experience_kernels.cu`) and writes the
/// EMA-blended mean to `ISV[AVG_WIN_HOLD_TIME_BARS_INDEX=451]` for
/// downstream consumption by `AuxHorizonUpdateOps`.
///
/// Single-block kernel; block-tree-reduce over the per-sample buffer.
/// Pearl-A first-observation bootstrap; α=0.05 EMA thereafter.
///
/// # Pearls applied
/// - `feedback_no_atomicadd.md` — single block, block-tree-reduce in shmem.
/// - `pearl_first_observation_bootstrap.md` — sentinel = 0.0 → REPLACE.
/// - `pearl_no_host_branches_in_captured_graph.md` — `CudaFunction`
///   pre-loaded at construction; on-device `count == 0` guard.
#[allow(missing_debug_implementations)]
pub(crate) struct AvgWinHoldTimeUpdateOps {
    update_kernel: CudaFunction,
}

impl AvgWinHoldTimeUpdateOps {
    /// Block dim used by the producer kernel — must match `BLK_DIM` in
    /// `avg_win_hold_time_update_kernel.cu`.
    const BLK_DIM: u32 = 256;
    /// Steady-state EMA blend rate. Slow enough to filter
    /// sample-distribution noise, fast enough to track policy evolution
    /// across epochs.
    pub(crate) const ALPHA: f32 = 0.05;

    pub(crate) fn new(stream: &Arc<CudaStream>) -> Result<Self, MLError> {
        let context = stream.context();
        let module = context
            .load_cubin(AVG_WIN_HOLD_TIME_UPDATE_CUBIN.to_vec())
            .map_err(|e| MLError::ModelError(format!("avg_win_hold_time_update cubin load: {e}")))?;
        let update_kernel = module
            .load_function("avg_win_hold_time_update")
            .map_err(|e| MLError::ModelError(format!("avg_win_hold_time_update load: {e}")))?;
        Ok(Self { update_kernel })
    }

    /// Launch the producer.
    ///
    /// Args:
    ///   - `hold_at_exit_ptr`: f32 device ptr `[total_samples]`.
    ///   - `trade_profitable_ptr`: i32 device ptr `[total_samples]`.
    ///   - `total_samples`: N*L (B*T total samples this epoch).
    ///   - `isv_ptr`: ISV[ISV_TOTAL_DIM] device pointer.
    ///   - `isv_target_idx`: AVG_WIN_HOLD_TIME_BARS_INDEX (451).
    ///   - `sentinel`: SENTINEL_AVG_WIN_HOLD_TIME_BARS (0.0).
    ///   - `alpha`: EMA blend rate (use `Self::ALPHA = 0.05`).
    #[allow(clippy::too_many_arguments)]
    pub(crate) fn launch(
        &self,
        stream: &Arc<CudaStream>,
        hold_at_exit_ptr: u64,
        trade_profitable_ptr: u64,
        total_samples: i32,
        isv_ptr: u64,
        isv_target_idx: i32,
        sentinel: f32,
        alpha: f32,
    ) -> Result<(), MLError> {
        let smem_bytes = 2 * Self::BLK_DIM * std::mem::size_of::<f32>() as u32;
        unsafe {
            stream
                .launch_builder(&self.update_kernel)
                .arg(&hold_at_exit_ptr)
                .arg(&trade_profitable_ptr)
                .arg(&total_samples)
                .arg(&isv_ptr)
                .arg(&isv_target_idx)
                .arg(&sentinel)
                .arg(&alpha)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (Self::BLK_DIM, 1, 1),
                    shared_mem_bytes: smem_bytes,
                })
                .map_err(|e| MLError::ModelError(format!("avg_win_hold_time_update: {e}")))?;
        }
        Ok(())
    }
}

/// Class A P0-A (2026-05-08): adaptive REWARD_POS/NEG_CAP producer.
///
/// Sweeps the per-epoch `step_ret_per_sample` + `trade_close_per_sample`
/// buffers (populated by `unified_env_step_core` in `experience_kernels.cu`),
/// computes a Welford `mean + Z_99 × sigma` p99 estimator over winning
/// realized returns plus a `max(winning_returns)` conservative takeover,
/// applies a 1.5× safety factor, clamps to dimensional bounds [1, 50],
/// and writes both `ISV[REWARD_POS_CAP_ADAPTIVE_INDEX=452]` (POS cap)
/// and `ISV[REWARD_NEG_CAP_ADAPTIVE_INDEX=453]` (NEG = −2 × POS,
/// preserving Kahneman/Tversky 2:1 loss-aversion asymmetry per
/// `pearl_audit_unboundedness_for_implicit_asymmetry` — moved from
/// hardcoded scalar to producer-time multiplier; SINGLE source of truth).
///
/// Single-block kernel; block-tree-reduce over the per-sample buffer
/// (no atomicAdd per `feedback_no_atomicadd.md`). Pearl-A
/// first-observation bootstrap (sentinel 5.0/-10.0 matches pre-fix
/// hardcoded constants for bit-identical cold-start); α=0.01 slow EMA
/// thereafter.
///
/// Per-epoch boundary launch — reward distribution is the foundation
/// of training and shouldn't move fast; per-step would track sample noise.
///
/// # Pearls applied
/// - `feedback_no_atomicadd.md` — single block, block-tree-reduce in shmem.
/// - `pearl_first_observation_bootstrap.md` — sentinel = 5.0 → REPLACE.
/// - `pearl_no_host_branches_in_captured_graph.md` — `CudaFunction`
///   pre-loaded at construction; on-device guards.
/// - `pearl_symmetric_clamp_audit.md` — bilateral `fmaxf(lo, fminf(x, hi))`
///   on POS, NEG derived clamp.
/// - `feedback_isv_for_adaptive_bounds.md` — POS bounds [1, 50] are
///   Category-1 dimensional safety, NOT tuning.
#[allow(missing_debug_implementations)]
pub(crate) struct RewardCapUpdateOps {
    update_kernel: CudaFunction,
}

impl RewardCapUpdateOps {
    /// Block dim used by the producer kernel — must match `BLK_DIM` in
    /// `reward_cap_update_kernel.cu`.
    const BLK_DIM: u32 = 256;

    pub(crate) fn new(stream: &Arc<CudaStream>) -> Result<Self, MLError> {
        let context = stream.context();
        let module = context
            .load_cubin(REWARD_CAP_UPDATE_CUBIN.to_vec())
            .map_err(|e| MLError::ModelError(format!("reward_cap_update cubin load: {e}")))?;
        let update_kernel = module
            .load_function("reward_cap_update")
            .map_err(|e| MLError::ModelError(format!("reward_cap_update load: {e}")))?;
        Ok(Self { update_kernel })
    }

    /// Launch the adaptive reward-cap producer.
    ///
    /// Args:
    ///   - `step_ret_ptr`: f32 device ptr `[total_samples]` —
    ///     `step_ret_core` raw signed per-step return.
    ///   - `trade_close_ptr`: i32 device ptr `[total_samples]` — 1 iff
    ///     exiting_trade || reversing_trade.
    ///   - `total_samples`: N*L (B*T total samples this epoch).
    ///   - `isv_ptr`: ISV[ISV_TOTAL_DIM] device pointer.
    ///   - `pos_idx`: REWARD_POS_CAP_ADAPTIVE_INDEX (452).
    ///   - `neg_idx`: REWARD_NEG_CAP_ADAPTIVE_INDEX (453).
    ///   - `sentinel_pos`: SENTINEL_REWARD_POS_CAP (5.0).
    ///   - `sentinel_neg`: SENTINEL_REWARD_NEG_CAP (-10.0).
    ///   - `safety_factor`: REWARD_CAP_SAFETY_FACTOR (1.5).
    ///   - `neg_to_pos_ratio`: REWARD_NEG_TO_POS_RATIO (2.0).
    ///   - `pos_min`, `pos_max`: REWARD_POS_CAP_MIN/MAX (1.0, 50.0).
    ///   - `alpha`: REWARD_CAP_EMA_ALPHA (0.01).
    #[allow(clippy::too_many_arguments)]
    pub(crate) fn launch(
        &self,
        stream: &Arc<CudaStream>,
        step_ret_ptr: u64,
        trade_close_ptr: u64,
        total_samples: i32,
        isv_ptr: u64,
        pos_idx: i32,
        neg_idx: i32,
        sentinel_pos: f32,
        sentinel_neg: f32,
        safety_factor: f32,
        neg_to_pos_ratio: f32,
        pos_min: f32,
        pos_max: f32,
        alpha: f32,
    ) -> Result<(), MLError> {
        // 4 arrays × BLK_DIM × sizeof(f32) — see the kernel comment block.
        let smem_bytes = 4 * Self::BLK_DIM * std::mem::size_of::<f32>() as u32;
        unsafe {
            stream
                .launch_builder(&self.update_kernel)
                .arg(&step_ret_ptr)
                .arg(&trade_close_ptr)
                .arg(&total_samples)
                .arg(&isv_ptr)
                .arg(&pos_idx)
                .arg(&neg_idx)
                .arg(&sentinel_pos)
                .arg(&sentinel_neg)
                .arg(&safety_factor)
                .arg(&neg_to_pos_ratio)
                .arg(&pos_min)
                .arg(&pos_max)
                .arg(&alpha)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (Self::BLK_DIM, 1, 1),
                    shared_mem_bytes: smem_bytes,
                })
                .map_err(|e| MLError::ModelError(format!("reward_cap_update: {e}")))?;
        }
        Ok(())
    }
}

/// SP18 v2 Phase 3 (2026-05-09): adaptive HOLD_REWARD_POS/NEG_CAP producer.
///
/// Drives `ISV[HOLD_REWARD_POS_CAP_INDEX=483]` and
/// `ISV[HOLD_REWARD_NEG_CAP_INDEX=484]` from `p99(|step_ret|)` over
/// Long/Short trade closes (per spec DD3=b). Mirrors `RewardCapUpdateOps`
/// for the position-side cap (ISV[452]/[453]) but with two structural
/// differences:
///
///   1. Filter is `(is_close && step_ret != 0)` (Long/Short closes —
///      both winners and losers) vs. `(is_close && step_ret > 0)`
///      (winners only) for the position-side cap. The Hold opp-cost
///      consumer needs the magnitude scale of *all* Long/Short
///      realised closes, not just winners.
///
///   2. EMA blend uses Wiener-optimal α derived from Welford
///      accumulators in slots [487..493) (per
///      `pearl_wiener_optimal_adaptive_alpha`; floor at
///      WELFORD_ALPHA_MIN=0.4 per
///      `pearl_wiener_alpha_floor_for_nonstationary` since the
///      policy-realised distribution is intrinsically non-stationary
///      as the policy adapts), vs. fixed α=0.01 for the position-side
///      cap (whose underlying alpha distribution is approximately
///      stationary).
///
/// Pearl-A first-observation bootstrap (sentinel POS=5.0/NEG=-10.0
/// matches the position-side cap pattern for bit-identical cold-start).
/// Block-tree-reduce (no atomicAdd per `feedback_no_atomicadd.md`) on
/// sum/sumsq/count/max in shmem; thread 0 finalises the
/// Welford+Wiener+Pearl-A+clamp pipeline and writes both POS and NEG
/// (NEG = −2 × POS, DD5=b mirrored asymmetry, single source of truth
/// at producer time per
/// `pearl_audit_unboundedness_for_implicit_asymmetry`).
///
/// `CudaFunction` pre-loaded at construction per
/// `pearl_no_host_branches_in_captured_graph.md`; per-epoch boundary
/// launch right after `RewardCapUpdateOps` so both cap producers
/// share the same `step_ret_per_sample` + `trade_close_per_sample`
/// source buffers.
///
/// # Pearls applied
/// - `feedback_no_atomicadd.md` — block-tree-reduce in shmem; one
///   global write per slot from thread 0.
/// - `pearl_first_observation_bootstrap.md` — sentinel match (within
///   1e-6 of SENTINEL_HOLD_REWARD_POS_CAP=5.0) → REPLACE; otherwise
///   Wiener-α blend.
/// - `pearl_wiener_optimal_adaptive_alpha.md` — α derived from
///   Welford variance accumulators, not hardcoded.
/// - `pearl_wiener_alpha_floor_for_nonstationary.md` — α floored at
///   0.4 to preserve catch-up bandwidth on the policy-realised
///   non-stationary signal.
/// - `pearl_no_host_branches_in_captured_graph.md` — `CudaFunction`
///   pre-loaded at construction; on-device guards.
/// - `pearl_symmetric_clamp_audit.md` — bilateral
///   `fmaxf(lo, fminf(x, hi))` on POS pre-blend, post-blend, and
///   on derived NEG.
/// - `feedback_isv_for_adaptive_bounds.md` — POS bounds [0.5, 50.0]
///   are Category-1 dimensional safety floors, NOT tuning. NEG range
///   [−100, −1] is derived as `−ratio × POS_bounds`.
/// - `pearl_fused_per_group_statistics_oracle.md` — sum/sumsq/count/max
///   accumulated in ONE fused reduction; not separate launches.
#[allow(missing_debug_implementations)]
pub(crate) struct HoldRewardCapUpdateOps {
    update_kernel: CudaFunction,
}

impl HoldRewardCapUpdateOps {
    /// Block dim used by the producer kernel — must match `HRC_BLK_DIM`
    /// in `hold_reward_cap_update_kernel.cu`.
    const BLK_DIM: u32 = 256;

    pub(crate) fn new(stream: &Arc<CudaStream>) -> Result<Self, MLError> {
        let context = stream.context();
        let module = context
            .load_cubin(HOLD_REWARD_CAP_UPDATE_CUBIN.to_vec())
            .map_err(|e| MLError::ModelError(format!("hold_reward_cap_update cubin load: {e}")))?;
        let update_kernel = module
            .load_function("hold_reward_cap_update")
            .map_err(|e| MLError::ModelError(format!("hold_reward_cap_update load: {e}")))?;
        Ok(Self { update_kernel })
    }

    /// Launch the SP18 D-leg adaptive Hold-reward-cap producer.
    ///
    /// Args:
    ///   - `step_ret_ptr`: f32 device ptr `[total_samples]` —
    ///     `step_ret_core` raw signed per-step return (same source the
    ///     position-side `RewardCapUpdateOps::launch` reads from).
    ///   - `trade_close_ptr`: i32 device ptr `[total_samples]` — 1 iff
    ///     exiting_trade || reversing_trade.
    ///   - `total_samples`: N*L (B*T total samples this epoch).
    ///   - `isv_ptr`: ISV[ISV_TOTAL_DIM] device pointer (mapped-pinned).
    ///   - `pos_idx`: HOLD_REWARD_POS_CAP_INDEX (483).
    ///   - `neg_idx`: HOLD_REWARD_NEG_CAP_INDEX (484).
    ///   - Welford accumulator slot indices: HRC_TARGET_MEAN_INDEX (487),
    ///     HRC_TARGET_M2_INDEX (488), HRC_DIFF_MEAN_INDEX (489),
    ///     HRC_DIFF_M2_INDEX (490), HRC_PREV_TARGET_INDEX (491),
    ///     HRC_SAMPLE_COUNT_INDEX (492).
    ///   - `sentinel_pos`: SENTINEL_HOLD_REWARD_POS_CAP (5.0).
    ///   - `sentinel_neg`: SENTINEL_HOLD_REWARD_NEG_CAP (-10.0; held
    ///     for symmetry with the position-side kernel signature).
    ///   - `safety_factor`: HOLD_REWARD_CAP_SAFETY_FACTOR (1.5).
    ///   - `neg_to_pos_ratio`: REWARD_NEG_TO_POS_RATIO (2.0; same
    ///     Kahneman 2:1 ratio as the position-side cap, single source
    ///     of truth at producer time).
    ///   - `pos_min`/`pos_max`: HOLD_REWARD_POS_CAP_MIN/MAX_BOUND
    ///     (0.5, 50.0).
    #[allow(clippy::too_many_arguments)]
    pub(crate) fn launch(
        &self,
        stream: &Arc<CudaStream>,
        step_ret_ptr: u64,
        trade_close_ptr: u64,
        total_samples: i32,
        isv_ptr: u64,
        pos_idx: i32,
        neg_idx: i32,
        target_mean_idx: i32,
        target_m2_idx: i32,
        diff_mean_idx: i32,
        diff_m2_idx: i32,
        prev_target_idx: i32,
        sample_count_idx: i32,
        sentinel_pos: f32,
        sentinel_neg: f32,
        safety_factor: f32,
        neg_to_pos_ratio: f32,
        pos_min: f32,
        pos_max: f32,
    ) -> Result<(), MLError> {
        // 4 arrays × BLK_DIM × sizeof(f32) — see kernel comment block.
        let smem_bytes = 4 * Self::BLK_DIM * std::mem::size_of::<f32>() as u32;
        unsafe {
            stream
                .launch_builder(&self.update_kernel)
                .arg(&step_ret_ptr)
                .arg(&trade_close_ptr)
                .arg(&total_samples)
                .arg(&isv_ptr)
                .arg(&pos_idx)
                .arg(&neg_idx)
                .arg(&target_mean_idx)
                .arg(&target_m2_idx)
                .arg(&diff_mean_idx)
                .arg(&diff_m2_idx)
                .arg(&prev_target_idx)
                .arg(&sample_count_idx)
                .arg(&sentinel_pos)
                .arg(&sentinel_neg)
                .arg(&safety_factor)
                .arg(&neg_to_pos_ratio)
                .arg(&pos_min)
                .arg(&pos_max)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (Self::BLK_DIM, 1, 1),
                    shared_mem_bytes: smem_bytes,
                })
                .map_err(|e| MLError::ModelError(format!("hold_reward_cap_update: {e}")))?;
        }
        Ok(())
    }
}

/// Class A P1-Producer (2026-05-08): adaptive Bayesian Kelly priors producer.
///
/// Replaces the previously-hardcoded `prior_wins=2.0f, prior_losses=2.0f,
/// prior_sum_wins=0.01f, prior_sum_losses=0.01f` constants used in
/// `kelly_cap_update_kernel.cu` and `trade_physics.cuh::kelly_position_cap`
/// with ISV-driven slow-EMA values fed by the
/// `kelly_bayesian_priors_update_kernel` producer at per-fold-end (per-epoch
/// boundary in the current scheduling). Aggregates the realized
/// PS_KELLY_{WIN_COUNT, LOSS_COUNT, SUM_WINS, SUM_LOSSES} fields across all
/// envs from the same `portfolio_state[n_envs, PS_STRIDE]` buffer that
/// `kelly_cap_update_kernel` reads from at the same boundary, then
/// slow-EMA-blends into ISV[KELLY_PRIOR_*_INDEX] = ISV[454..458).
///
/// Single-block 256-thread kernel; shmem block-tree-reduce
/// (no atomicAdd per `feedback_no_atomicadd.md`). `CudaFunction` pre-loaded
/// at construction per `pearl_no_host_branches_in_captured_graph.md`.
///
/// Pearls applied:
/// - `feedback_no_atomicadd.md` — block-tree-reduce in shmem, single
///   global write per slot from thread 0.
/// - `pearl_first_observation_bootstrap.md` — sentinel match (within 1e-6
///   of pre-P1-Producer hardcoded value) → REPLACE; otherwise EMA blend.
/// - `pearl_no_host_branches_in_captured_graph.md` — `CudaFunction`
///   pre-loaded at construction; on-device guard for "no realized trades".
/// - `feedback_no_stubs.md` — full body, no placeholder.
/// - `feedback_isv_for_adaptive_bounds.md` — count bounds [0.5, 100],
///   sum bounds [0.001, 1.0] are dimensional safety floors, NOT tuning.
/// - `pearl_symmetric_clamp_audit.md` — bilateral
///   `fmaxf(lo, fminf(x, hi))` clamp on each slot before writing.
/// - `pearl_controller_anchors_isv_driven.md` — every controller anchor
///   (priors are anchors for the Kelly cap formula) is ISV-driven.
#[allow(missing_debug_implementations)]
pub(crate) struct KellyBayesianPriorsUpdateOps {
    update_kernel: CudaFunction,
}

impl KellyBayesianPriorsUpdateOps {
    /// Block dim used by the producer kernel — must match `BLK_DIM` in
    /// `kelly_bayesian_priors_update_kernel.cu`.
    const BLK_DIM: u32 = 256;

    pub(crate) fn new(stream: &Arc<CudaStream>) -> Result<Self, MLError> {
        let context = stream.context();
        let module = context
            .load_cubin(KELLY_BAYESIAN_PRIORS_UPDATE_CUBIN.to_vec())
            .map_err(|e| MLError::ModelError(format!("kelly_bayesian_priors_update cubin load: {e}")))?;
        let update_kernel = module
            .load_function("kelly_bayesian_priors_update")
            .map_err(|e| MLError::ModelError(format!("kelly_bayesian_priors_update load: {e}")))?;
        Ok(Self { update_kernel })
    }

    /// Launch the adaptive Bayesian-priors producer.
    ///
    /// Args:
    ///   - `portfolio_state_ptr`: f32 device ptr `[n_envs * ps_stride]` —
    ///     the same buffer `kelly_cap_update_kernel` reads from. We use
    ///     the four PS_KELLY_* fields per env.
    ///   - `n_envs`: number of envs (alloc_episodes from the collector).
    ///   - `ps_stride`: PS_STRIDE from state_layout.cuh (43 post-Plan-3-D.4c).
    ///   - `isv_ptr`: ISV[ISV_TOTAL_DIM] device pointer (mapped-pinned).
    ///   - `wins_idx`/`losses_idx`/`sum_wins_idx`/`sum_losses_idx`:
    ///     KELLY_PRIOR_*_INDEX (454..458).
    ///   - `sentinel_*`: SENTINEL_KELLY_PRIOR_* (2.0/2.0/0.01/0.01).
    ///   - `count_min`/`count_max`: KELLY_PRIOR_COUNT_MIN/MAX (0.5, 100).
    ///   - `sum_min`/`sum_max`: KELLY_PRIOR_SUM_MIN/MAX (0.001, 1.0).
    ///   - `alpha`: KELLY_PRIOR_EMA_ALPHA (0.005, slow per-fold cadence).
    #[allow(clippy::too_many_arguments)]
    pub(crate) fn launch(
        &self,
        stream: &Arc<CudaStream>,
        portfolio_state_ptr: u64,
        n_envs: i32,
        ps_stride: i32,
        isv_ptr: u64,
        wins_idx: i32,
        losses_idx: i32,
        sum_wins_idx: i32,
        sum_losses_idx: i32,
        sentinel_wins: f32,
        sentinel_losses: f32,
        sentinel_sum_wins: f32,
        sentinel_sum_losses: f32,
        count_min: f32,
        count_max: f32,
        sum_min: f32,
        sum_max: f32,
        alpha: f32,
    ) -> Result<(), MLError> {
        // 4 arrays × BLK_DIM × sizeof(f32) — see the kernel comment block.
        let smem_bytes = 4 * Self::BLK_DIM * std::mem::size_of::<f32>() as u32;
        unsafe {
            stream
                .launch_builder(&self.update_kernel)
                .arg(&portfolio_state_ptr)
                .arg(&n_envs)
                .arg(&ps_stride)
                .arg(&isv_ptr)
                .arg(&wins_idx)
                .arg(&losses_idx)
                .arg(&sum_wins_idx)
                .arg(&sum_losses_idx)
                .arg(&sentinel_wins)
                .arg(&sentinel_losses)
                .arg(&sentinel_sum_wins)
                .arg(&sentinel_sum_losses)
                .arg(&count_min)
                .arg(&count_max)
                .arg(&sum_min)
                .arg(&sum_max)
                .arg(&alpha)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (Self::BLK_DIM, 1, 1),
                    shared_mem_bytes: smem_bytes,
                })
                .map_err(|e| MLError::ModelError(format!("kelly_bayesian_priors_update: {e}")))?;
        }
        Ok(())
    }
}

/// Class A audit-fix Batch 4-A (2026-05-08): adaptive DD saturation floor producer.
///
/// Sweeps the per-env `dd_state_per_env[n_envs * 6]` tile (populated by
/// `dd_state_kernel` at the same boundary), aggregates DD_MAX (offset 1
/// within each env's tile) across envs, computes a Welford `mean +
/// Z_75 × sigma` p75 estimator with a `max(p75, mean)` robustness guard,
/// applies a 1.5× safety factor, clamps to dimensional bounds [0.10, 0.50],
/// and writes `ISV[DD_SATURATION_FLOOR_ADAPTIVE_INDEX=458]`. Replaces the
/// hardcoded `0.25f` saturation floor in `trade_physics.cuh::apply_margin_cap`
/// (the upper end of the linear position-size scaling ramp).
///
/// Single-block kernel; block-tree-reduce over the per-env tile (no
/// atomicAdd per `feedback_no_atomicadd.md`). Pearl-A first-observation
/// bootstrap (sentinel 0.25 matches pre-fix hardcoded constant for
/// bit-identical cold-start); α=0.01 slow EMA thereafter (DD distribution
/// is a slow-moving fold-volatility property — same cadence as the P0-A
/// REWARD_POS_CAP producer).
///
/// Per-epoch boundary launch — DD distribution shouldn't move fast.
///
/// # Pearls applied
/// - `feedback_no_atomicadd.md` — single block, block-tree-reduce in shmem.
/// - `pearl_first_observation_bootstrap.md` — sentinel 0.25 → REPLACE.
/// - `pearl_no_host_branches_in_captured_graph.md` — `CudaFunction`
///   pre-loaded at construction; on-device guards.
/// - `pearl_symmetric_clamp_audit.md` — bilateral `fmaxf(lo, fminf(x, hi))`.
/// - `feedback_isv_for_adaptive_bounds.md` — bounds [0.10, 0.50] are
///   Category-1 dimensional safety, NOT tuning.
#[allow(missing_debug_implementations)]
pub(crate) struct DdSaturationFloorUpdateOps {
    update_kernel: CudaFunction,
}

impl DdSaturationFloorUpdateOps {
    /// Block dim used by the producer kernel — must match `BLK_DIM` in
    /// `dd_saturation_floor_update_kernel.cu`.
    const BLK_DIM: u32 = 256;

    pub(crate) fn new(stream: &Arc<CudaStream>) -> Result<Self, MLError> {
        let context = stream.context();
        let module = context
            .load_cubin(DD_SATURATION_FLOOR_UPDATE_CUBIN.to_vec())
            .map_err(|e| MLError::ModelError(format!("dd_saturation_floor_update cubin load: {e}")))?;
        let update_kernel = module
            .load_function("dd_saturation_floor_update")
            .map_err(|e| MLError::ModelError(format!("dd_saturation_floor_update load: {e}")))?;
        Ok(Self { update_kernel })
    }

    /// Launch the adaptive DD saturation floor producer.
    ///
    /// Args:
    ///   - `dd_state_per_env_ptr`: f32 device ptr `[n_envs * dd_state_stride]`
    ///     — the same buffer `compute_sp15_final_reward_kernel` reads from.
    ///     We use the per-env DD_MAX field at offset `dd_max_off`.
    ///   - `n_envs`: number of envs (alloc_episodes from the collector).
    ///   - `dd_state_stride`: 6 (mirrors the per-env tile layout in
    ///     `dd_state_kernel`).
    ///   - `dd_max_off`: 1 (DD_MAX offset within each env's tile).
    ///   - `isv_ptr`: ISV[ISV_TOTAL_DIM] device pointer (mapped-pinned).
    ///   - `floor_idx`: DD_SATURATION_FLOOR_ADAPTIVE_INDEX (458).
    ///   - `sentinel_floor`: SENTINEL_DD_SATURATION_FLOOR (0.25).
    ///   - `safety_factor`: DD_SATURATION_FLOOR_SAFETY_FACTOR (1.5).
    ///   - `floor_min`/`floor_max`: DD_SATURATION_FLOOR_MIN/MAX (0.10, 0.50).
    ///   - `alpha`: DD_SATURATION_FLOOR_EMA_ALPHA (0.01, slow per-epoch).
    #[allow(clippy::too_many_arguments)]
    pub(crate) fn launch(
        &self,
        stream: &Arc<CudaStream>,
        dd_state_per_env_ptr: u64,
        n_envs: i32,
        dd_state_stride: i32,
        dd_max_off: i32,
        isv_ptr: u64,
        floor_idx: i32,
        sentinel_floor: f32,
        safety_factor: f32,
        floor_min: f32,
        floor_max: f32,
        alpha: f32,
    ) -> Result<(), MLError> {
        // 4 arrays × BLK_DIM × sizeof(f32) — see the kernel comment block.
        let smem_bytes = 4 * Self::BLK_DIM * std::mem::size_of::<f32>() as u32;
        unsafe {
            stream
                .launch_builder(&self.update_kernel)
                .arg(&dd_state_per_env_ptr)
                .arg(&n_envs)
                .arg(&dd_state_stride)
                .arg(&dd_max_off)
                .arg(&isv_ptr)
                .arg(&floor_idx)
                .arg(&sentinel_floor)
                .arg(&safety_factor)
                .arg(&floor_min)
                .arg(&floor_max)
                .arg(&alpha)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (Self::BLK_DIM, 1, 1),
                    shared_mem_bytes: smem_bytes,
                })
                .map_err(|e| MLError::ModelError(format!("dd_saturation_floor_update: {e}")))?;
        }
        Ok(())
    }
}

/// SP16 Phase 1 (revised, 2026-05-08): adaptive MIN_HOLD_TEMPERATURE
/// producer driven by hold-rate overrun (signal swap from slot 373).
///
/// Single-thread cold-path kernel (per-epoch boundary cadence) that
/// reads two SP13 hold-pricing chain slots from ISV:
///
///   - `ISV[HOLD_RATE_OBSERVED_EMA_INDEX=382]` — per-fold EMA of the
///     per-step Hold-pick rate, populated by `apply_fixed_alpha_ema_kernel`
///     chained off `hold_rate_observer_kernel`.
///   - `ISV[HOLD_RATE_TARGET_INDEX=381]` — static target (default 0.20).
///
/// And maps the overrun into a [5, 50] temperature:
///
///     overrun           = max(0, observed − target)
///     overrun_norm      = clamp(overrun / max(target, 0.01), 0, 1)
///     temp              = TEMP_MIN + (TEMP_MAX − TEMP_MIN) × overrun_norm
///
/// then slow-EMA-blends into
/// `ISV[MIN_HOLD_TEMPERATURE_ADAPTIVE_INDEX=460]` with α=0.05.
///
/// Per `compute_min_hold_penalty` at trade_physics.cuh:575
/// (`soft_factor = deficit / (deficit + T)`), HIGH T = forgiving
/// (saturates → 0, no exit penalty), LOW T = sharp (saturates → 1,
/// max exit penalty). The hold-rate-overrun mapping therefore gives
/// the over-holding scenario its designed behaviour:
///   - observed = 2× target → overrun_norm = 1.0 → temp = 50
///     (permissive — exit ramp out of over-holding)
///   - observed = target    → overrun_norm = 0   → temp = 5
///     (strict — force commitment at the hold-rate target)
///   - observed < target    → overrun_norm = 0   → temp = 5
///     (under-holding — strict, no relaxation)
///
/// Signal swap rationale (post-mortem of train-multi-seed-pfh9n,
/// 2026-05-08): the original mapping read
/// `ISV[AUX_DIR_ACC_SHORT_EMA_INDEX=373]` but slot 373 stayed at
/// sentinel 0.5 in Fold 1 — the per-step aux producer either didn't
/// push it off sentinel quickly enough or the binary classifier
/// converged within ε of 0.5 in noisy environments. The kernel's
/// early-return-on-sentinel guard then fired on every launch and slot
/// 460 stayed pinned at 50 for the entire fold. Hold rates in
/// contrast are measured per-epoch from realised actions and never at
/// a "no-data" sentinel after epoch 1, so the fold-1 chain breakage
/// goes away.
///
/// Cold-start sentinel guard preserved: when `observed_hold_rate` is
/// at the SP13 fold-reset sentinel 0.0 (no per-step Hold observations
/// yet in the new fold), the kernel keeps slot 460 unchanged — the
/// consumer falls back to MIN_HOLD_TEMPERATURE_FALLBACK=50.0 via the
/// same sentinel-detect path used pre-swap.
///
/// # Pearls applied
/// - `feedback_no_atomicadd.md` — single-thread kernel; no reductions.
/// - `pearl_first_observation_bootstrap.md` — sentinel 50.0 → REPLACE.
/// - `pearl_no_host_branches_in_captured_graph.md` — `CudaFunction`
///   pre-loaded at construction; on-device guards.
/// - `pearl_symmetric_clamp_audit.md` — bilateral
///   `fmaxf(lo, fminf(x, hi))` clamp on target_temp + post-blend.
/// - `feedback_isv_for_adaptive_bounds.md` — bounds [5, 50] are
///   Category-1 dimensional safety floors (matches the original
///   schedule range MIN_HOLD_TEMPERATURE_END=5 ↔
///   MIN_HOLD_TEMPERATURE_START=50), NOT tuning.
/// - `feedback_no_partial_refactor.md` — kernel + launcher + call site
///   + tests + audit doc updated atomically in the same commit.
#[allow(missing_debug_implementations)]
pub(crate) struct MinHoldTemperatureUpdateOps {
    update_kernel: CudaFunction,
}

impl MinHoldTemperatureUpdateOps {
    pub(crate) fn new(stream: &Arc<CudaStream>) -> Result<Self, MLError> {
        let context = stream.context();
        let module = context
            .load_cubin(MIN_HOLD_TEMPERATURE_UPDATE_CUBIN.to_vec())
            .map_err(|e| MLError::ModelError(format!("min_hold_temperature_update cubin load: {e}")))?;
        let update_kernel = module
            .load_function("min_hold_temperature_update")
            .map_err(|e| MLError::ModelError(format!("min_hold_temperature_update load: {e}")))?;
        Ok(Self { update_kernel })
    }

    /// Launch the adaptive MIN_HOLD_TEMPERATURE producer (SP16 Phase 1
    /// revised — hold-rate-overrun driven; SP16 T3 — Wiener-optimal α).
    ///
    /// Args:
    ///   - `isv_ptr`: ISV[ISV_TOTAL_DIM] device pointer (mapped-pinned).
    ///   - `temp_idx`: MIN_HOLD_TEMPERATURE_ADAPTIVE_INDEX (460).
    ///   - `hold_rate_observed_idx`: HOLD_RATE_OBSERVED_EMA_INDEX (382).
    ///   - `hold_rate_target_idx`: HOLD_RATE_TARGET_INDEX (381).
    ///   - Welford accumulator slot indices (T3): MHT_TARGET_MEAN_INDEX
    ///     (468), MHT_TARGET_M2_INDEX (469), MHT_DIFF_MEAN_INDEX (470),
    ///     MHT_DIFF_M2_INDEX (471), MHT_PREV_TARGET_INDEX (472),
    ///     MHT_SAMPLE_COUNT_INDEX (473).
    ///   - `sentinel_temp`: SENTINEL_MIN_HOLD_TEMPERATURE (50.0).
    ///   - `sentinel_observed_hold_rate`: SENTINEL_HOLD_RATE_OBSERVED
    ///     (0.0; matches the SP13 fold-reset registry sentinel for
    ///     `sp13_hold_rate_observed_ema`).
    ///   - `temp_min`/`temp_max`: MIN_HOLD_TEMPERATURE_MIN/MAX (5.0, 50.0).
    ///
    /// SP16 T3: the `alpha` argument is gone — α is computed inside the
    /// kernel from the Welford accumulators (Wiener-optimal: α = diff_var
    /// / (diff_var + sample_var + ε)).
    #[allow(clippy::too_many_arguments)]
    pub(crate) fn launch(
        &self,
        stream: &Arc<CudaStream>,
        isv_ptr: u64,
        temp_idx: i32,
        hold_rate_observed_idx: i32,
        hold_rate_target_idx: i32,
        target_mean_idx: i32,
        target_m2_idx: i32,
        diff_mean_idx: i32,
        diff_m2_idx: i32,
        prev_target_idx: i32,
        sample_count_idx: i32,
        sentinel_temp: f32,
        sentinel_observed_hold_rate: f32,
        temp_min: f32,
        temp_max: f32,
    ) -> Result<(), MLError> {
        unsafe {
            stream
                .launch_builder(&self.update_kernel)
                .arg(&isv_ptr)
                .arg(&temp_idx)
                .arg(&hold_rate_observed_idx)
                .arg(&hold_rate_target_idx)
                .arg(&target_mean_idx)
                .arg(&target_m2_idx)
                .arg(&diff_mean_idx)
                .arg(&diff_m2_idx)
                .arg(&prev_target_idx)
                .arg(&sample_count_idx)
                .arg(&sentinel_temp)
                .arg(&sentinel_observed_hold_rate)
                .arg(&temp_min)
                .arg(&temp_max)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (1, 1, 1),
                    shared_mem_bytes: 0,
                })
                .map_err(|e| MLError::ModelError(format!("min_hold_temperature_update: {e}")))?;
        }
        Ok(())
    }
}

/// SP14 Layer C Phase C.6 (2026-05-08): h_s2_aux RMS EMA producer.
///
/// Computes `RMS(h_s2_aux) = sqrt(mean(h_s2_aux²))` over the
/// `[B, SH2]` aux trunk output and EMA-blends the result into
/// `ISV[H_S2_AUX_RMS_EMA_INDEX=449]`.
///
/// ISV slot 449 is in the SP14 Layer C block (outside the SP4/SP5 wiener
/// buffer linear span), so Pearl-A bootstrap logic is embedded in the
/// kernel body rather than delegated to `apply_pearls_ad_kernel`. Fixed
/// α=0.05 (per-step cadence blend rate; no Wiener-optimal adaptive α
/// available without a wiener_state_buf triple for this slot).
///
/// Single-block 256-thread kernel; shmem block-tree-reduce (no atomicAdd
/// per `feedback_no_atomicadd.md`). `CudaFunction` pre-loaded at
/// construction per `pearl_no_host_branches_in_captured_graph.md`.
///
/// # Pearls applied
/// - `feedback_no_atomicadd.md` — block-tree-reduce in shmem, one global
///   write from thread 0.
/// - `pearl_first_observation_bootstrap.md` — sentinel 0.0 → REPLACE.
/// - `pearl_no_host_branches_in_captured_graph.md` — `CudaFunction`
///   pre-loaded at construction; no per-launch cubin load.
/// - `feedback_no_stubs.md` — full body, no placeholder.
#[allow(missing_debug_implementations)]
pub(crate) struct HS2AuxRmsEmaOps {
    update_kernel: CudaFunction,
}

impl HS2AuxRmsEmaOps {
    /// Block dim — must match the kernel's `BLK_DIM` of 256.
    const BLK_DIM: u32 = 256;
    /// Per-step EMA blend rate. Matches the aux trunk forward launch
    /// cadence — one observation per collector step.
    pub(crate) const ALPHA: f32 = 0.05;

    pub(crate) fn new(stream: &Arc<CudaStream>) -> Result<Self, MLError> {
        let context = stream.context();
        let module = context
            .load_cubin(H_S2_AUX_RMS_EMA_CUBIN.to_vec())
            .map_err(|e| MLError::ModelError(format!("h_s2_aux_rms_ema cubin load: {e}")))?;
        let update_kernel = module
            .load_function("h_s2_aux_rms_ema_update")
            .map_err(|e| MLError::ModelError(format!("h_s2_aux_rms_ema_update load: {e}")))?;
        Ok(Self { update_kernel })
    }

    /// Launch the RMS EMA producer.
    ///
    /// Args:
    ///   - `h_s2_aux_ptr`: aux trunk output `[B, SH2]` device pointer.
    ///   - `b`: batch size (B).
    ///   - `sh2`: aux trunk output width (SH2 = AUX_HIDDEN_DIM = 256).
    ///   - `isv_ptr`: ISV[ISV_TOTAL_DIM] device pointer (mapped-pinned).
    ///   - `isv_target_idx`: H_S2_AUX_RMS_EMA_INDEX (449).
    ///   - `alpha`: EMA blend rate (use `Self::ALPHA = 0.05`).
    pub(crate) fn launch(
        &self,
        stream: &Arc<CudaStream>,
        h_s2_aux_ptr: u64,
        b: i32,
        sh2: i32,
        isv_ptr: u64,
        isv_target_idx: i32,
        alpha: f32,
    ) -> Result<(), MLError> {
        let smem_bytes = Self::BLK_DIM * std::mem::size_of::<f32>() as u32;
        unsafe {
            stream
                .launch_builder(&self.update_kernel)
                .arg(&h_s2_aux_ptr)
                .arg(&b)
                .arg(&sh2)
                .arg(&isv_ptr)
                .arg(&isv_target_idx)
                .arg(&alpha)
                .launch(LaunchConfig {
                    grid_dim: (1, 1, 1),
                    block_dim: (Self::BLK_DIM, 1, 1),
                    shared_mem_bytes: smem_bytes,
                })
                .map_err(|e| MLError::ModelError(format!("h_s2_aux_rms_ema_update: {e}")))?;
        }
        Ok(())
    }
}