#![allow(unsafe_code)] // Required for CUDA kernel launches. // Allow non-snake-case for `dW{1,2,3}_ptr` / `dW_reduce_kernel` / // `launch_dW_reduce` — the math notation `dW` is the universally // recognised symbol for weight gradient (matches the kernel parameter // names in `aux_trunk_backward_kernel.cu`). Snake-casing them // (`d_w1_ptr`, `d_w_reduce_kernel`) loses the math correspondence and // hurts readability when paired with the kernel source. #![allow(non_snake_case)] //! `gpu_aux_trunk` — SP14 Layer C Phase C.3 (2026-05-08). //! //! Rust wrapper for the auxiliary trunk forward kernel //! (`aux_trunk_forward_kernel.cu`). Owns the pre-loaded `CudaFunction` //! handle so launches in the captured graph never touch host-side cubin //! load paths (per `pearl_no_host_branches_in_captured_graph.md`). //! //! # Architecture //! //! 3-layer Linear→ELU→Linear→ELU→Linear MLP parallel to Q's GRN trunk. //! Both trunks read the shared encoder output (`x_in [B, ENCODER_OUT_DIM]`); //! Q trunk produces `h_s2`, aux trunk produces `h_s2_aux`. Aux backward //! (Phase C.4) terminates at the encoder boundary — Q-loss is the sole //! shaping force on the encoder per the locked design decision in //! `2026-05-07-sp14-layer-c-separate-aux-trunk.md`. //! //! Saved-for-backward outputs (consumed by aux trunk backward in C.4): //! //! - `h_aux1 [B, H1]` — Layer-1 post-ELU activation //! - `h_aux2 [B, H2]` — Layer-2 post-ELU activation //! - `h_s2_aux [B, AUX_HIDDEN_DIM]` — final linear output (no activation) //! //! Production topology (verified C.2): //! - `ENCODER_OUT_DIM = config.shared_h1 = 256` //! - `H1 = AUX_TRUNK_H1 = 256` //! - `H2 = AUX_TRUNK_H2 = 128` (mild bottleneck) //! - `AUX_HIDDEN_DIM = config.shared_h2 = 256` (matches aux head input dim; //! internal aux head hidden layer enlarged 32 → 128 on 2026-05-13; //! initial 32 → 256 attempt OOM'd L40S — dialed back to 128) //! //! # Pearls applied //! //! - `pearl_no_host_branches_in_captured_graph.md` — `CudaFunction` //! pre-loaded once at construction; no `load_cubin` / `load_function` //! on the launch path. //! - `feedback_no_atomicadd.md` — forward kernel writes only per-(b, j) //! outputs; no contention across blocks. //! - `feedback_no_stubs.md` — full launcher, no placeholder body. use std::sync::Arc; use cudarc::driver::{CudaFunction, CudaStream, LaunchConfig, PushKernelArg}; use crate::MLError; use super::gpu_dqn_trainer::{ AUX_HORIZON_UPDATE_CUBIN, AUX_TRUNK_BACKWARD_CUBIN, AUX_TRUNK_FORWARD_CUBIN, AVG_WIN_HOLD_TIME_UPDATE_CUBIN, DD_SATURATION_FLOOR_UPDATE_CUBIN, HOLD_REWARD_CAP_UPDATE_CUBIN, H_S2_AUX_RMS_EMA_CUBIN, KELLY_BAYESIAN_PRIORS_UPDATE_CUBIN, MIN_HOLD_TEMPERATURE_UPDATE_CUBIN, REWARD_CAP_UPDATE_CUBIN, }; /// Hidden width of the aux trunk's first internal layer (Linear_1 → ELU /// output). Mirrors the encoder output dimension so Layer-1 acts as a /// representation-mixing identity-shaped projection. pub(crate) const AUX_TRUNK_H1: usize = 256; /// Hidden width of the aux trunk's second internal layer (Linear_2 → ELU /// output). Smaller than H1 for a mild bottleneck — encourages the aux /// trunk to compress its representation before lifting back up to /// `AUX_HIDDEN_DIM` in Layer 3. /// /// **Kept at 128 (2026-05-13)**: initial enlargement to 256 alongside /// `AUX_HIDDEN_DIM 32 → 256` OOM'd L40S — trunk dW partials grew too. /// Kept at 128 for memory budget; head's 4× enlargement to 128 is the /// primary capacity uplift. pub(crate) const AUX_TRUNK_H2: usize = 128; /// Block dim used by the per-sample forward kernel. Mirrors /// `aux_heads_kernel.cu`'s `AUX_BLOCK = 256`. const AUX_TRUNK_BLOCK: u32 = 256; /// Forward orchestrator — owns the single `CudaFunction` for /// `aux_trunk_forward`. Backward kernel + ops land in Phase C.4. /// /// Dropping this struct releases the underlying `CudaFunction` handle /// via cudarc's RAII; no explicit teardown required. #[allow(missing_debug_implementations)] pub(crate) struct AuxTrunkForwardOps { /// Pre-loaded forward kernel handle (loaded once at construction; /// never re-loaded on the launch path). forward_kernel: CudaFunction, } impl AuxTrunkForwardOps { /// Load the forward kernel handle from the precompiled cubin. /// Mirrors `GrnBlock::new`'s loader pattern. pub(crate) fn new(stream: &Arc) -> Result { let context = stream.context(); let module = context .load_cubin(AUX_TRUNK_FORWARD_CUBIN.to_vec()) .map_err(|e| MLError::ModelError(format!("aux_trunk_forward cubin load: {e}")))?; let forward_kernel = module .load_function("aux_trunk_forward") .map_err(|e| MLError::ModelError(format!("aux_trunk_forward load: {e}")))?; Ok(Self { forward_kernel }) } /// Launch `aux_trunk_forward`: 3-layer Linear→ELU→Linear→ELU→Linear MLP. /// /// Caller-owned buffers (raw `u64` device pointers for graph-capture /// safety; mirrors `gpu_grn.rs::forward_raw` and `gpu_aux_heads.rs`): /// * `x_in_ptr` — `[B, ENCODER_OUT_DIM]` row-major (encoder output). /// * `w1_ptr`/`b1_ptr` — `[ENCODER_OUT_DIM, H1]` / `[H1]`. /// * `w2_ptr`/`b2_ptr` — `[H1, H2]` / `[H2]`. /// * `w3_ptr`/`b3_ptr` — `[H2, AUX_HIDDEN_DIM]` / `[AUX_HIDDEN_DIM]`. /// * `h_aux1_out_ptr` — `[B, H1]` SAVED post-ELU Layer-1 (consumed by C.4 backward). /// * `h_aux2_out_ptr` — `[B, H2]` SAVED post-ELU Layer-2 (consumed by C.4 backward). /// * `h_s2_aux_out_ptr` — `[B, AUX_HIDDEN_DIM]` final aux trunk output. /// /// `h1` / `h2` / `aux_hidden_dim` are passed as runtime args for /// kernel-signature stability (a future bump won't require an ABI /// change). `encoder_out_dim` is the encoder's actual output width /// (typically `config.shared_h1 = 256`). /// /// Block: `AUX_TRUNK_BLOCK = 256` threads. /// Grid: `B` blocks (one block per batch row). /// Shared mem: `(H1 + H2) * sizeof(f32)` bytes — Layer-1 cache for /// Layer-2 matmul + Layer-2 cache for Layer-3 matmul. #[allow(clippy::too_many_arguments)] pub(crate) fn launch( &self, stream: &Arc, x_in_ptr: u64, w1_ptr: u64, b1_ptr: u64, w2_ptr: u64, b2_ptr: u64, w3_ptr: u64, b3_ptr: u64, h_aux1_out_ptr: u64, h_aux2_out_ptr: u64, h_s2_aux_out_ptr: u64, b: usize, encoder_out_dim: usize, h1: usize, h2: usize, aux_hidden_dim: usize, ) -> Result<(), MLError> { let b_i32 = b as i32; let enc_i32 = encoder_out_dim as i32; let h1_i32 = h1 as i32; let h2_i32 = h2 as i32; let aux_i32 = aux_hidden_dim as i32; // Shared memory: H1 + H2 floats — Layer-1 cache + Layer-2 cache. let smem_bytes = (h1 as u32 + h2 as u32) * std::mem::size_of::() as u32; unsafe { stream .launch_builder(&self.forward_kernel) .arg(&x_in_ptr) .arg(&w1_ptr) .arg(&b1_ptr) .arg(&w2_ptr) .arg(&b2_ptr) .arg(&w3_ptr) .arg(&b3_ptr) .arg(&h_aux1_out_ptr) .arg(&h_aux2_out_ptr) .arg(&h_s2_aux_out_ptr) .arg(&b_i32) .arg(&enc_i32) .arg(&h1_i32) .arg(&h2_i32) .arg(&aux_i32) .launch(LaunchConfig { grid_dim: (b as u32, 1, 1), block_dim: (AUX_TRUNK_BLOCK, 1, 1), shared_mem_bytes: smem_bytes, }) .map_err(|e| MLError::ModelError(format!("aux_trunk_forward: {e}")))?; } Ok(()) } } /// Block dim for the backward kernels — power-of-two so the shmem-tree /// reduce in `aux_trunk_bwd_dW_reduce` / `aux_trunk_bwd_db_reduce` halves /// cleanly without odd-stride bookkeeping. Mirrors `AUX_TRUNK_BWD_BLOCK` /// in `aux_trunk_backward_kernel.cu`. const AUX_TRUNK_BWD_BLOCK: u32 = 256; /// Backward orchestrator — owns three pre-loaded `CudaFunction` handles /// for `aux_trunk_bwd_dh_pre`, `aux_trunk_bwd_dW_reduce`, and /// `aux_trunk_bwd_db_reduce`. A single `launch()` call orchestrates the /// full backward pass over all three trunk layers without allocating /// per-sample partial buffers (per-element block-tree-reduce instead). /// /// CRITICAL: this op set does NOT compute or write `dx_in`. The encoder /// boundary is the stop-gradient per the locked design decision in /// `2026-05-07-sp14-layer-c-separate-aux-trunk.md` §C.4. The structural /// enforcement is in the kernel signatures — none of the three kernels /// accept a `dx_in_out` pointer, so the kernel set literally cannot /// touch encoder gradient memory. This is verified by the /// `aux_trunk_backward_does_not_write_dx` oracle test which inspects the /// kernel source for any `dx_in` write pattern. /// /// # Pearls applied /// /// - `pearl_no_host_branches_in_captured_graph.md` — all three /// `CudaFunction`s are pre-loaded at construction. The backward /// `launch()` issues a fixed sequence of seven kernel launches with /// no host-side branching. /// - `feedback_no_atomicadd.md` — every dW/db output cell is owned by /// exactly one block; block-tree-reduce over the batch dim within /// shared memory. /// - `feedback_no_stubs.md` — full backward, no zero-fill placeholders, /// no skipped layers. #[allow(missing_debug_implementations)] pub(crate) struct AuxTrunkBackwardOps { /// Pre-pass: per-sample kernel that emits scratch buffers /// `dh_aux1_pre [B, H1]` and `dh_aux2_pre [B, H2]`. dh_pre_kernel: CudaFunction, /// Generic outer-product reduce for any dW matrix: /// `dW[k, j] = sum_b A[b, k] * B[b, j]`. Launched 3× per backward. dW_reduce_kernel: CudaFunction, /// Generic batch reduce for any db vector: /// `db[j] = sum_b B[b, j]`. Launched 3× per backward. db_reduce_kernel: CudaFunction, } impl AuxTrunkBackwardOps { /// Load the three backward kernel handles from the precompiled cubin. /// Mirrors `AuxTrunkForwardOps::new`'s loader pattern. All three /// kernels live in a single cubin so one `load_cubin` call is /// sufficient. pub(crate) fn new(stream: &Arc) -> Result { let context = stream.context(); let module = context .load_cubin(AUX_TRUNK_BACKWARD_CUBIN.to_vec()) .map_err(|e| MLError::ModelError(format!("aux_trunk_backward cubin load: {e}")))?; let dh_pre_kernel = module .load_function("aux_trunk_bwd_dh_pre") .map_err(|e| MLError::ModelError(format!("aux_trunk_bwd_dh_pre load: {e}")))?; let dW_reduce_kernel = module .load_function("aux_trunk_bwd_dW_reduce") .map_err(|e| MLError::ModelError(format!("aux_trunk_bwd_dW_reduce load: {e}")))?; let db_reduce_kernel = module .load_function("aux_trunk_bwd_db_reduce") .map_err(|e| MLError::ModelError(format!("aux_trunk_bwd_db_reduce load: {e}")))?; Ok(Self { dh_pre_kernel, dW_reduce_kernel, db_reduce_kernel, }) } /// Launch the full backward pass for the aux trunk. /// /// Orchestrates seven kernel launches in fixed sequence (no host- /// side branching, capture-friendly): /// 1. `aux_trunk_bwd_dh_pre` — writes `dh_aux2_pre`, `dh_aux1_pre`. /// 2. `aux_trunk_bwd_dW_reduce` — writes `dW3 = h_aux2.T @ d_logits`. /// 3. `aux_trunk_bwd_db_reduce` — writes `db3 = sum_b d_logits[b, :]`. /// 4. `aux_trunk_bwd_dW_reduce` — writes `dW2 = h_aux1.T @ dh_aux2_pre`. /// 5. `aux_trunk_bwd_db_reduce` — writes `db2 = sum_b dh_aux2_pre[b, :]`. /// 6. `aux_trunk_bwd_dW_reduce` — writes `dW1 = x_in.T @ dh_aux1_pre`. /// 7. `aux_trunk_bwd_db_reduce` — writes `db1 = sum_b dh_aux1_pre[b, :]`. /// /// Caller-owned buffers (raw `u64` device pointers; mirrors /// `AuxTrunkForwardOps::launch` and `gpu_aux_heads.rs`): /// - `dh_s2_aux_in_ptr` `[B, AUX_HIDDEN_DIM]` upstream gradient. /// - `x_in_ptr` `[B, ENCODER_OUT_DIM]` saved-fwd input. /// - `h_aux1_ptr` `[B, H1]` saved-fwd post-ELU. /// - `h_aux2_ptr` `[B, H2]` saved-fwd post-ELU. /// - `w2_ptr` / `w3_ptr` weight matrices for backward matmuls. /// - `dh_aux1_pre_scratch` `[B, H1]` caller-allocated scratch. /// - `dh_aux2_pre_scratch` `[B, H2]` caller-allocated scratch. /// - `dW1_ptr` / `db1_ptr` `[ENCODER_OUT_DIM, H1]` / `[H1]`. /// - `dW2_ptr` / `db2_ptr` `[H1, H2]` / `[H2]`. /// - `dW3_ptr` / `db3_ptr` `[H2, AUX_HIDDEN_DIM]` / `[AUX_HIDDEN_DIM]`. /// /// Output gradient buffers are OVERWRITTEN (not accumulated). If /// caller wants accumulation across multiple backward calls, they /// must SAXPY into a separate accumulator after this call. (Aux /// trunk's gradient is computed once per training step in C.5; no /// accumulation needed.) /// /// NO `dx_in_out` parameter — that is the structural stop-gradient /// at the encoder boundary. Encoder gradient stays Q-shaped. #[allow(clippy::too_many_arguments)] pub(crate) fn launch( &self, stream: &Arc, dh_s2_aux_in_ptr: u64, x_in_ptr: u64, h_aux1_ptr: u64, h_aux2_ptr: u64, w2_ptr: u64, w3_ptr: u64, dh_aux1_pre_scratch_ptr: u64, dh_aux2_pre_scratch_ptr: u64, dW1_ptr: u64, db1_ptr: u64, dW2_ptr: u64, db2_ptr: u64, dW3_ptr: u64, db3_ptr: u64, b: usize, encoder_out_dim: usize, h1: usize, h2: usize, aux_hidden_dim: usize, ) -> Result<(), MLError> { let b_i32 = b as i32; // `encoder_out_dim` is consumed by `launch_dW_reduce` for dW1 // (Krows = ENCODER_OUT_DIM); the dh_pre kernel doesn't need it as // a runtime arg because dh_pre only computes hidden-layer // gradients. `enc_i32` would be redundant here. let h1_i32 = h1 as i32; let h2_i32 = h2 as i32; let aux_i32 = aux_hidden_dim as i32; let block_smem = AUX_TRUNK_BWD_BLOCK * std::mem::size_of::() as u32; // ── 1. Pre-pass: dh_aux2_pre + dh_aux1_pre ──────────────────── // Block: AUX_TRUNK_BWD_BLOCK threads. Grid: (B, 1, 1). // Shared memory: H2 floats (Layer-2 post-multiply cache). let dh_pre_smem = h2 as u32 * std::mem::size_of::() as u32; unsafe { stream .launch_builder(&self.dh_pre_kernel) .arg(&dh_s2_aux_in_ptr) .arg(&w3_ptr) .arg(&w2_ptr) .arg(&h_aux1_ptr) .arg(&h_aux2_ptr) .arg(&dh_aux2_pre_scratch_ptr) .arg(&dh_aux1_pre_scratch_ptr) .arg(&b_i32) .arg(&h1_i32) .arg(&h2_i32) .arg(&aux_i32) .launch(LaunchConfig { grid_dim: (b as u32, 1, 1), block_dim: (AUX_TRUNK_BWD_BLOCK, 1, 1), shared_mem_bytes: dh_pre_smem, }) .map_err(|e| MLError::ModelError(format!("aux_trunk_bwd_dh_pre: {e}")))?; } // ── 2. dW3 = h_aux2.T @ d_logits [H2, AUX_HIDDEN_DIM] ──────── self.launch_dW_reduce( stream, h_aux2_ptr, dh_s2_aux_in_ptr, dW3_ptr, b, h2, aux_hidden_dim, block_smem, )?; // ── 3. db3 = sum_b d_logits[b, :] [AUX_HIDDEN_DIM] ─────────── self.launch_db_reduce(stream, dh_s2_aux_in_ptr, db3_ptr, b, aux_hidden_dim, block_smem)?; // ── 4. dW2 = h_aux1.T @ dh_aux2_pre [H1, H2] ───────────────── self.launch_dW_reduce( stream, h_aux1_ptr, dh_aux2_pre_scratch_ptr, dW2_ptr, b, h1, h2, block_smem, )?; // ── 5. db2 = sum_b dh_aux2_pre[b, :] [H2] ──────────────────── self.launch_db_reduce(stream, dh_aux2_pre_scratch_ptr, db2_ptr, b, h2, block_smem)?; // ── 6. dW1 = x_in.T @ dh_aux1_pre [ENCODER_OUT_DIM, H1] ────── self.launch_dW_reduce( stream, x_in_ptr, dh_aux1_pre_scratch_ptr, dW1_ptr, b, encoder_out_dim, h1, block_smem, )?; // ── 7. db1 = sum_b dh_aux1_pre[b, :] [H1] ──────────────────── self.launch_db_reduce(stream, dh_aux1_pre_scratch_ptr, db1_ptr, b, h1, block_smem)?; // STOP — no Layer-1 dx kernel. Encoder boundary is the stop-grad. Ok(()) } /// Helper: launch `aux_trunk_bwd_dW_reduce` for one weight matrix. /// Grid: `(Krows * Jcols, 1, 1)` — one block per output element. #[allow(clippy::too_many_arguments)] fn launch_dW_reduce( &self, stream: &Arc, a_ptr: u64, b_grad_ptr: u64, dw_out_ptr: u64, b: usize, krows: usize, jcols: usize, smem_bytes: u32, ) -> Result<(), MLError> { let b_i32 = b as i32; let k_i32 = krows as i32; let j_i32 = jcols as i32; let grid = (krows * jcols) as u32; unsafe { stream .launch_builder(&self.dW_reduce_kernel) .arg(&a_ptr) .arg(&b_grad_ptr) .arg(&dw_out_ptr) .arg(&b_i32) .arg(&k_i32) .arg(&j_i32) .launch(LaunchConfig { grid_dim: (grid, 1, 1), block_dim: (AUX_TRUNK_BWD_BLOCK, 1, 1), shared_mem_bytes: smem_bytes, }) .map_err(|e| MLError::ModelError(format!("aux_trunk_bwd_dW_reduce: {e}")))?; } Ok(()) } /// Helper: launch `aux_trunk_bwd_db_reduce` for one bias vector. /// Grid: `(Jcols, 1, 1)` — one block per output element. fn launch_db_reduce( &self, stream: &Arc, b_grad_ptr: u64, db_out_ptr: u64, b: usize, jcols: usize, smem_bytes: u32, ) -> Result<(), MLError> { let b_i32 = b as i32; let j_i32 = jcols as i32; unsafe { stream .launch_builder(&self.db_reduce_kernel) .arg(&b_grad_ptr) .arg(&db_out_ptr) .arg(&b_i32) .arg(&j_i32) .launch(LaunchConfig { grid_dim: (jcols as u32, 1, 1), block_dim: (AUX_TRUNK_BWD_BLOCK, 1, 1), shared_mem_bytes: smem_bytes, }) .map_err(|e| MLError::ModelError(format!("aux_trunk_bwd_db_reduce: {e}")))?; } Ok(()) } } /// SP14 Layer C Phase C.4b (2026-05-08): adaptive aux prediction horizon /// producer. /// /// Drives `ISV[AUX_PRED_HORIZON_BARS_INDEX=450]` from /// `ISV[AVG_WIN_HOLD_TIME_BARS_INDEX=451]` via Pearl-A first-observation /// bootstrap + slow Wiener-α EMA. Single-thread, single-block kernel — /// horizon is slow-moving so per-epoch boundary launch is appropriate /// (per-step would be wasteful and would track sample noise). /// /// # Pearls applied /// - `pearl_first_observation_bootstrap.md` — sentinel = 60.0; first /// valid observation REPLACES (no blend) so the cold-start sentinel /// never contaminates the EMA. /// - `pearl_wiener_optimal_adaptive_alpha.md` — fixed α=0.01 fallback /// when no target-variance EMA exists. /// - `pearl_no_host_branches_in_captured_graph.md` — `CudaFunction` /// pre-loaded once at construction. /// - `feedback_isv_for_adaptive_bounds.md` — h_min=1, h_max=240 are /// fundamental floors/ceilings, not tuning parameters. #[allow(missing_debug_implementations)] pub(crate) struct AuxHorizonUpdateOps { update_kernel: CudaFunction, } impl AuxHorizonUpdateOps { pub(crate) fn new(stream: &Arc) -> Result { let context = stream.context(); let module = context .load_cubin(AUX_HORIZON_UPDATE_CUBIN.to_vec()) .map_err(|e| MLError::ModelError(format!("aux_horizon_update cubin load: {e}")))?; let update_kernel = module .load_function("aux_horizon_update") .map_err(|e| MLError::ModelError(format!("aux_horizon_update load: {e}")))?; Ok(Self { update_kernel }) } /// Launch the horizon producer. /// /// Args: /// - `isv_ptr`: ISV[ISV_TOTAL_DIM] device pointer (mapped-pinned). /// - `isv_h_idx`: AUX_PRED_HORIZON_BARS_INDEX (450). /// - `isv_h_target_idx`: AVG_WIN_HOLD_TIME_BARS_INDEX (451). /// - `isv_h_target_var_idx`: variance EMA slot for target, or `-1` /// if none (uses fixed α=0.01). /// - `h_min`, `h_max`: fundamental bounds (1.0, 240.0). /// - `sentinel_h`: SENTINEL_AUX_PRED_HORIZON_BARS (60.0). #[allow(clippy::too_many_arguments)] pub(crate) fn launch( &self, stream: &Arc, isv_ptr: u64, isv_h_idx: i32, isv_h_target_idx: i32, isv_h_target_var_idx: i32, h_min: f32, h_max: f32, sentinel_h: f32, ) -> Result<(), MLError> { unsafe { stream .launch_builder(&self.update_kernel) .arg(&isv_ptr) .arg(&isv_h_idx) .arg(&isv_h_target_idx) .arg(&isv_h_target_var_idx) .arg(&h_min) .arg(&h_max) .arg(&sentinel_h) .launch(LaunchConfig { grid_dim: (1, 1, 1), block_dim: (1, 1, 1), shared_mem_bytes: 0, }) .map_err(|e| MLError::ModelError(format!("aux_horizon_update: {e}")))?; } Ok(()) } } /// SP14 Layer C Phase C.4b (2026-05-08): avg winning hold time EMA producer. /// /// Sweeps the per-epoch `hold_at_exit_per_sample` + /// `trade_profitable_per_sample` buffers (populated by /// `unified_env_step_core` in `experience_kernels.cu`) and writes the /// EMA-blended mean to `ISV[AVG_WIN_HOLD_TIME_BARS_INDEX=451]` for /// downstream consumption by `AuxHorizonUpdateOps`. /// /// Single-block kernel; block-tree-reduce over the per-sample buffer. /// Pearl-A first-observation bootstrap; α=0.05 EMA thereafter. /// /// # Pearls applied /// - `feedback_no_atomicadd.md` — single block, block-tree-reduce in shmem. /// - `pearl_first_observation_bootstrap.md` — sentinel = 0.0 → REPLACE. /// - `pearl_no_host_branches_in_captured_graph.md` — `CudaFunction` /// pre-loaded at construction; on-device `count == 0` guard. #[allow(missing_debug_implementations)] pub(crate) struct AvgWinHoldTimeUpdateOps { update_kernel: CudaFunction, } impl AvgWinHoldTimeUpdateOps { /// Block dim used by the producer kernel — must match `BLK_DIM` in /// `avg_win_hold_time_update_kernel.cu`. const BLK_DIM: u32 = 256; /// Steady-state EMA blend rate. Slow enough to filter /// sample-distribution noise, fast enough to track policy evolution /// across epochs. pub(crate) const ALPHA: f32 = 0.05; pub(crate) fn new(stream: &Arc) -> Result { let context = stream.context(); let module = context .load_cubin(AVG_WIN_HOLD_TIME_UPDATE_CUBIN.to_vec()) .map_err(|e| MLError::ModelError(format!("avg_win_hold_time_update cubin load: {e}")))?; let update_kernel = module .load_function("avg_win_hold_time_update") .map_err(|e| MLError::ModelError(format!("avg_win_hold_time_update load: {e}")))?; Ok(Self { update_kernel }) } /// Launch the producer. /// /// Args: /// - `hold_at_exit_ptr`: f32 device ptr `[total_samples]`. /// - `trade_profitable_ptr`: i32 device ptr `[total_samples]`. /// - `total_samples`: N*L (B*T total samples this epoch). /// - `isv_ptr`: ISV[ISV_TOTAL_DIM] device pointer. /// - `isv_target_idx`: AVG_WIN_HOLD_TIME_BARS_INDEX (451). /// - `sentinel`: SENTINEL_AVG_WIN_HOLD_TIME_BARS (0.0). /// - `alpha`: EMA blend rate (use `Self::ALPHA = 0.05`). #[allow(clippy::too_many_arguments)] pub(crate) fn launch( &self, stream: &Arc, hold_at_exit_ptr: u64, trade_profitable_ptr: u64, total_samples: i32, isv_ptr: u64, isv_target_idx: i32, sentinel: f32, alpha: f32, ) -> Result<(), MLError> { let smem_bytes = 2 * Self::BLK_DIM * std::mem::size_of::() as u32; unsafe { stream .launch_builder(&self.update_kernel) .arg(&hold_at_exit_ptr) .arg(&trade_profitable_ptr) .arg(&total_samples) .arg(&isv_ptr) .arg(&isv_target_idx) .arg(&sentinel) .arg(&alpha) .launch(LaunchConfig { grid_dim: (1, 1, 1), block_dim: (Self::BLK_DIM, 1, 1), shared_mem_bytes: smem_bytes, }) .map_err(|e| MLError::ModelError(format!("avg_win_hold_time_update: {e}")))?; } Ok(()) } } /// Class A P0-A (2026-05-08): adaptive REWARD_POS/NEG_CAP producer. /// /// Sweeps the per-epoch `step_ret_per_sample` + `trade_close_per_sample` /// buffers (populated by `unified_env_step_core` in `experience_kernels.cu`), /// computes a Welford `mean + Z_99 × sigma` p99 estimator over winning /// realized returns plus a `max(winning_returns)` conservative takeover, /// applies a 1.5× safety factor, clamps to dimensional bounds [1, 50], /// and writes both `ISV[REWARD_POS_CAP_ADAPTIVE_INDEX=452]` (POS cap) /// and `ISV[REWARD_NEG_CAP_ADAPTIVE_INDEX=453]` (NEG = −2 × POS, /// preserving Kahneman/Tversky 2:1 loss-aversion asymmetry per /// `pearl_audit_unboundedness_for_implicit_asymmetry` — moved from /// hardcoded scalar to producer-time multiplier; SINGLE source of truth). /// /// Single-block kernel; block-tree-reduce over the per-sample buffer /// (no atomicAdd per `feedback_no_atomicadd.md`). Pearl-A /// first-observation bootstrap (sentinel 5.0/-10.0 matches pre-fix /// hardcoded constants for bit-identical cold-start); α=0.01 slow EMA /// thereafter. /// /// Per-epoch boundary launch — reward distribution is the foundation /// of training and shouldn't move fast; per-step would track sample noise. /// /// # Pearls applied /// - `feedback_no_atomicadd.md` — single block, block-tree-reduce in shmem. /// - `pearl_first_observation_bootstrap.md` — sentinel = 5.0 → REPLACE. /// - `pearl_no_host_branches_in_captured_graph.md` — `CudaFunction` /// pre-loaded at construction; on-device guards. /// - `pearl_symmetric_clamp_audit.md` — bilateral `fmaxf(lo, fminf(x, hi))` /// on POS, NEG derived clamp. /// - `feedback_isv_for_adaptive_bounds.md` — POS bounds [1, 50] are /// Category-1 dimensional safety, NOT tuning. #[allow(missing_debug_implementations)] pub(crate) struct RewardCapUpdateOps { update_kernel: CudaFunction, } impl RewardCapUpdateOps { /// Block dim used by the producer kernel — must match `BLK_DIM` in /// `reward_cap_update_kernel.cu`. const BLK_DIM: u32 = 256; pub(crate) fn new(stream: &Arc) -> Result { let context = stream.context(); let module = context .load_cubin(REWARD_CAP_UPDATE_CUBIN.to_vec()) .map_err(|e| MLError::ModelError(format!("reward_cap_update cubin load: {e}")))?; let update_kernel = module .load_function("reward_cap_update") .map_err(|e| MLError::ModelError(format!("reward_cap_update load: {e}")))?; Ok(Self { update_kernel }) } /// Launch the adaptive reward-cap producer. /// /// Args: /// - `step_ret_ptr`: f32 device ptr `[total_samples]` — /// `step_ret_core` raw signed per-step return. /// - `trade_close_ptr`: i32 device ptr `[total_samples]` — 1 iff /// exiting_trade || reversing_trade. /// - `total_samples`: N*L (B*T total samples this epoch). /// - `isv_ptr`: ISV[ISV_TOTAL_DIM] device pointer. /// - `pos_idx`: REWARD_POS_CAP_ADAPTIVE_INDEX (452). /// - `neg_idx`: REWARD_NEG_CAP_ADAPTIVE_INDEX (453). /// - `sentinel_pos`: SENTINEL_REWARD_POS_CAP (5.0). /// - `sentinel_neg`: SENTINEL_REWARD_NEG_CAP (-10.0). /// - `safety_factor`: REWARD_CAP_SAFETY_FACTOR (1.5). /// - `neg_to_pos_ratio`: REWARD_NEG_TO_POS_RATIO (2.0). /// - `pos_min`, `pos_max`: REWARD_POS_CAP_MIN/MAX (1.0, 50.0). /// - `alpha`: REWARD_CAP_EMA_ALPHA (0.01). #[allow(clippy::too_many_arguments)] pub(crate) fn launch( &self, stream: &Arc, step_ret_ptr: u64, trade_close_ptr: u64, total_samples: i32, isv_ptr: u64, pos_idx: i32, neg_idx: i32, sentinel_pos: f32, sentinel_neg: f32, safety_factor: f32, neg_to_pos_ratio: f32, pos_min: f32, pos_max: f32, alpha: f32, ) -> Result<(), MLError> { // 4 arrays × BLK_DIM × sizeof(f32) — see the kernel comment block. let smem_bytes = 4 * Self::BLK_DIM * std::mem::size_of::() as u32; unsafe { stream .launch_builder(&self.update_kernel) .arg(&step_ret_ptr) .arg(&trade_close_ptr) .arg(&total_samples) .arg(&isv_ptr) .arg(&pos_idx) .arg(&neg_idx) .arg(&sentinel_pos) .arg(&sentinel_neg) .arg(&safety_factor) .arg(&neg_to_pos_ratio) .arg(&pos_min) .arg(&pos_max) .arg(&alpha) .launch(LaunchConfig { grid_dim: (1, 1, 1), block_dim: (Self::BLK_DIM, 1, 1), shared_mem_bytes: smem_bytes, }) .map_err(|e| MLError::ModelError(format!("reward_cap_update: {e}")))?; } Ok(()) } } /// SP18 v2 Phase 3 (2026-05-09): adaptive HOLD_REWARD_POS/NEG_CAP producer. /// /// Drives `ISV[HOLD_REWARD_POS_CAP_INDEX=483]` and /// `ISV[HOLD_REWARD_NEG_CAP_INDEX=484]` from `p99(|step_ret|)` over /// Long/Short trade closes (per spec DD3=b). Mirrors `RewardCapUpdateOps` /// for the position-side cap (ISV[452]/[453]) but with two structural /// differences: /// /// 1. Filter is `(is_close && step_ret != 0)` (Long/Short closes — /// both winners and losers) vs. `(is_close && step_ret > 0)` /// (winners only) for the position-side cap. The Hold opp-cost /// consumer needs the magnitude scale of *all* Long/Short /// realised closes, not just winners. /// /// 2. EMA blend uses Wiener-optimal α derived from Welford /// accumulators in slots [487..493) (per /// `pearl_wiener_optimal_adaptive_alpha`; floor at /// WELFORD_ALPHA_MIN=0.4 per /// `pearl_wiener_alpha_floor_for_nonstationary` since the /// policy-realised distribution is intrinsically non-stationary /// as the policy adapts), vs. fixed α=0.01 for the position-side /// cap (whose underlying alpha distribution is approximately /// stationary). /// /// Pearl-A first-observation bootstrap (sentinel POS=5.0/NEG=-10.0 /// matches the position-side cap pattern for bit-identical cold-start). /// Block-tree-reduce (no atomicAdd per `feedback_no_atomicadd.md`) on /// sum/sumsq/count/max in shmem; thread 0 finalises the /// Welford+Wiener+Pearl-A+clamp pipeline and writes both POS and NEG /// (NEG = −2 × POS, DD5=b mirrored asymmetry, single source of truth /// at producer time per /// `pearl_audit_unboundedness_for_implicit_asymmetry`). /// /// `CudaFunction` pre-loaded at construction per /// `pearl_no_host_branches_in_captured_graph.md`; per-epoch boundary /// launch right after `RewardCapUpdateOps` so both cap producers /// share the same `step_ret_per_sample` + `trade_close_per_sample` /// source buffers. /// /// # Pearls applied /// - `feedback_no_atomicadd.md` — block-tree-reduce in shmem; one /// global write per slot from thread 0. /// - `pearl_first_observation_bootstrap.md` — sentinel match (within /// 1e-6 of SENTINEL_HOLD_REWARD_POS_CAP=5.0) → REPLACE; otherwise /// Wiener-α blend. /// - `pearl_wiener_optimal_adaptive_alpha.md` — α derived from /// Welford variance accumulators, not hardcoded. /// - `pearl_wiener_alpha_floor_for_nonstationary.md` — α floored at /// 0.4 to preserve catch-up bandwidth on the policy-realised /// non-stationary signal. /// - `pearl_no_host_branches_in_captured_graph.md` — `CudaFunction` /// pre-loaded at construction; on-device guards. /// - `pearl_symmetric_clamp_audit.md` — bilateral /// `fmaxf(lo, fminf(x, hi))` on POS pre-blend, post-blend, and /// on derived NEG. /// - `feedback_isv_for_adaptive_bounds.md` — POS bounds [0.5, 50.0] /// are Category-1 dimensional safety floors, NOT tuning. NEG range /// [−100, −1] is derived as `−ratio × POS_bounds`. /// - `pearl_fused_per_group_statistics_oracle.md` — sum/sumsq/count/max /// accumulated in ONE fused reduction; not separate launches. #[allow(missing_debug_implementations)] pub(crate) struct HoldRewardCapUpdateOps { update_kernel: CudaFunction, } impl HoldRewardCapUpdateOps { /// Block dim used by the producer kernel — must match `HRC_BLK_DIM` /// in `hold_reward_cap_update_kernel.cu`. const BLK_DIM: u32 = 256; pub(crate) fn new(stream: &Arc) -> Result { let context = stream.context(); let module = context .load_cubin(HOLD_REWARD_CAP_UPDATE_CUBIN.to_vec()) .map_err(|e| MLError::ModelError(format!("hold_reward_cap_update cubin load: {e}")))?; let update_kernel = module .load_function("hold_reward_cap_update") .map_err(|e| MLError::ModelError(format!("hold_reward_cap_update load: {e}")))?; Ok(Self { update_kernel }) } /// Launch the SP18 D-leg adaptive Hold-reward-cap producer. /// /// Args: /// - `step_ret_ptr`: f32 device ptr `[total_samples]` — /// `step_ret_core` raw signed per-step return (same source the /// position-side `RewardCapUpdateOps::launch` reads from). /// - `trade_close_ptr`: i32 device ptr `[total_samples]` — 1 iff /// exiting_trade || reversing_trade. /// - `total_samples`: N*L (B*T total samples this epoch). /// - `isv_ptr`: ISV[ISV_TOTAL_DIM] device pointer (mapped-pinned). /// - `pos_idx`: HOLD_REWARD_POS_CAP_INDEX (483). /// - `neg_idx`: HOLD_REWARD_NEG_CAP_INDEX (484). /// - Welford accumulator slot indices: HRC_TARGET_MEAN_INDEX (487), /// HRC_TARGET_M2_INDEX (488), HRC_DIFF_MEAN_INDEX (489), /// HRC_DIFF_M2_INDEX (490), HRC_PREV_TARGET_INDEX (491), /// HRC_SAMPLE_COUNT_INDEX (492). /// - `sentinel_pos`: SENTINEL_HOLD_REWARD_POS_CAP (5.0). /// - `sentinel_neg`: SENTINEL_HOLD_REWARD_NEG_CAP (-10.0; held /// for symmetry with the position-side kernel signature). /// - `safety_factor`: HOLD_REWARD_CAP_SAFETY_FACTOR (1.5). /// - `neg_to_pos_ratio`: REWARD_NEG_TO_POS_RATIO (2.0; same /// Kahneman 2:1 ratio as the position-side cap, single source /// of truth at producer time). /// - `pos_min`/`pos_max`: HOLD_REWARD_POS_CAP_MIN/MAX_BOUND /// (0.5, 50.0). #[allow(clippy::too_many_arguments)] pub(crate) fn launch( &self, stream: &Arc, step_ret_ptr: u64, trade_close_ptr: u64, total_samples: i32, isv_ptr: u64, pos_idx: i32, neg_idx: i32, target_mean_idx: i32, target_m2_idx: i32, diff_mean_idx: i32, diff_m2_idx: i32, prev_target_idx: i32, sample_count_idx: i32, sentinel_pos: f32, sentinel_neg: f32, safety_factor: f32, neg_to_pos_ratio: f32, pos_min: f32, pos_max: f32, ) -> Result<(), MLError> { // 4 arrays × BLK_DIM × sizeof(f32) — see kernel comment block. let smem_bytes = 4 * Self::BLK_DIM * std::mem::size_of::() as u32; unsafe { stream .launch_builder(&self.update_kernel) .arg(&step_ret_ptr) .arg(&trade_close_ptr) .arg(&total_samples) .arg(&isv_ptr) .arg(&pos_idx) .arg(&neg_idx) .arg(&target_mean_idx) .arg(&target_m2_idx) .arg(&diff_mean_idx) .arg(&diff_m2_idx) .arg(&prev_target_idx) .arg(&sample_count_idx) .arg(&sentinel_pos) .arg(&sentinel_neg) .arg(&safety_factor) .arg(&neg_to_pos_ratio) .arg(&pos_min) .arg(&pos_max) .launch(LaunchConfig { grid_dim: (1, 1, 1), block_dim: (Self::BLK_DIM, 1, 1), shared_mem_bytes: smem_bytes, }) .map_err(|e| MLError::ModelError(format!("hold_reward_cap_update: {e}")))?; } Ok(()) } } /// Class A P1-Producer (2026-05-08): adaptive Bayesian Kelly priors producer. /// /// Replaces the previously-hardcoded `prior_wins=2.0f, prior_losses=2.0f, /// prior_sum_wins=0.01f, prior_sum_losses=0.01f` constants used in /// `kelly_cap_update_kernel.cu` and `trade_physics.cuh::kelly_position_cap` /// with ISV-driven slow-EMA values fed by the /// `kelly_bayesian_priors_update_kernel` producer at per-fold-end (per-epoch /// boundary in the current scheduling). Aggregates the realized /// PS_KELLY_{WIN_COUNT, LOSS_COUNT, SUM_WINS, SUM_LOSSES} fields across all /// envs from the same `portfolio_state[n_envs, PS_STRIDE]` buffer that /// `kelly_cap_update_kernel` reads from at the same boundary, then /// slow-EMA-blends into ISV[KELLY_PRIOR_*_INDEX] = ISV[454..458). /// /// Single-block 256-thread kernel; shmem block-tree-reduce /// (no atomicAdd per `feedback_no_atomicadd.md`). `CudaFunction` pre-loaded /// at construction per `pearl_no_host_branches_in_captured_graph.md`. /// /// Pearls applied: /// - `feedback_no_atomicadd.md` — block-tree-reduce in shmem, single /// global write per slot from thread 0. /// - `pearl_first_observation_bootstrap.md` — sentinel match (within 1e-6 /// of pre-P1-Producer hardcoded value) → REPLACE; otherwise EMA blend. /// - `pearl_no_host_branches_in_captured_graph.md` — `CudaFunction` /// pre-loaded at construction; on-device guard for "no realized trades". /// - `feedback_no_stubs.md` — full body, no placeholder. /// - `feedback_isv_for_adaptive_bounds.md` — count bounds [0.5, 100], /// sum bounds [0.001, 1.0] are dimensional safety floors, NOT tuning. /// - `pearl_symmetric_clamp_audit.md` — bilateral /// `fmaxf(lo, fminf(x, hi))` clamp on each slot before writing. /// - `pearl_controller_anchors_isv_driven.md` — every controller anchor /// (priors are anchors for the Kelly cap formula) is ISV-driven. #[allow(missing_debug_implementations)] pub(crate) struct KellyBayesianPriorsUpdateOps { update_kernel: CudaFunction, } impl KellyBayesianPriorsUpdateOps { /// Block dim used by the producer kernel — must match `BLK_DIM` in /// `kelly_bayesian_priors_update_kernel.cu`. const BLK_DIM: u32 = 256; pub(crate) fn new(stream: &Arc) -> Result { let context = stream.context(); let module = context .load_cubin(KELLY_BAYESIAN_PRIORS_UPDATE_CUBIN.to_vec()) .map_err(|e| MLError::ModelError(format!("kelly_bayesian_priors_update cubin load: {e}")))?; let update_kernel = module .load_function("kelly_bayesian_priors_update") .map_err(|e| MLError::ModelError(format!("kelly_bayesian_priors_update load: {e}")))?; Ok(Self { update_kernel }) } /// Launch the adaptive Bayesian-priors producer. /// /// Args: /// - `portfolio_state_ptr`: f32 device ptr `[n_envs * ps_stride]` — /// the same buffer `kelly_cap_update_kernel` reads from. We use /// the four PS_KELLY_* fields per env. /// - `n_envs`: number of envs (alloc_episodes from the collector). /// - `ps_stride`: PS_STRIDE from state_layout.cuh (43 post-Plan-3-D.4c). /// - `isv_ptr`: ISV[ISV_TOTAL_DIM] device pointer (mapped-pinned). /// - `wins_idx`/`losses_idx`/`sum_wins_idx`/`sum_losses_idx`: /// KELLY_PRIOR_*_INDEX (454..458). /// - `sentinel_*`: SENTINEL_KELLY_PRIOR_* (2.0/2.0/0.01/0.01). /// - `count_min`/`count_max`: KELLY_PRIOR_COUNT_MIN/MAX (0.5, 100). /// - `sum_min`/`sum_max`: KELLY_PRIOR_SUM_MIN/MAX (0.001, 1.0). /// - `alpha`: KELLY_PRIOR_EMA_ALPHA (0.005, slow per-fold cadence). #[allow(clippy::too_many_arguments)] pub(crate) fn launch( &self, stream: &Arc, portfolio_state_ptr: u64, n_envs: i32, ps_stride: i32, isv_ptr: u64, wins_idx: i32, losses_idx: i32, sum_wins_idx: i32, sum_losses_idx: i32, sentinel_wins: f32, sentinel_losses: f32, sentinel_sum_wins: f32, sentinel_sum_losses: f32, count_min: f32, count_max: f32, sum_min: f32, sum_max: f32, alpha: f32, ) -> Result<(), MLError> { // 4 arrays × BLK_DIM × sizeof(f32) — see the kernel comment block. let smem_bytes = 4 * Self::BLK_DIM * std::mem::size_of::() as u32; unsafe { stream .launch_builder(&self.update_kernel) .arg(&portfolio_state_ptr) .arg(&n_envs) .arg(&ps_stride) .arg(&isv_ptr) .arg(&wins_idx) .arg(&losses_idx) .arg(&sum_wins_idx) .arg(&sum_losses_idx) .arg(&sentinel_wins) .arg(&sentinel_losses) .arg(&sentinel_sum_wins) .arg(&sentinel_sum_losses) .arg(&count_min) .arg(&count_max) .arg(&sum_min) .arg(&sum_max) .arg(&alpha) .launch(LaunchConfig { grid_dim: (1, 1, 1), block_dim: (Self::BLK_DIM, 1, 1), shared_mem_bytes: smem_bytes, }) .map_err(|e| MLError::ModelError(format!("kelly_bayesian_priors_update: {e}")))?; } Ok(()) } } /// Class A audit-fix Batch 4-A (2026-05-08): adaptive DD saturation floor producer. /// /// Sweeps the per-env `dd_state_per_env[n_envs * 6]` tile (populated by /// `dd_state_kernel` at the same boundary), aggregates DD_MAX (offset 1 /// within each env's tile) across envs, computes a Welford `mean + /// Z_75 × sigma` p75 estimator with a `max(p75, mean)` robustness guard, /// applies a 1.5× safety factor, clamps to dimensional bounds [0.10, 0.50], /// and writes `ISV[DD_SATURATION_FLOOR_ADAPTIVE_INDEX=458]`. Replaces the /// hardcoded `0.25f` saturation floor in `trade_physics.cuh::apply_margin_cap` /// (the upper end of the linear position-size scaling ramp). /// /// Single-block kernel; block-tree-reduce over the per-env tile (no /// atomicAdd per `feedback_no_atomicadd.md`). Pearl-A first-observation /// bootstrap (sentinel 0.25 matches pre-fix hardcoded constant for /// bit-identical cold-start); α=0.01 slow EMA thereafter (DD distribution /// is a slow-moving fold-volatility property — same cadence as the P0-A /// REWARD_POS_CAP producer). /// /// Per-epoch boundary launch — DD distribution shouldn't move fast. /// /// # Pearls applied /// - `feedback_no_atomicadd.md` — single block, block-tree-reduce in shmem. /// - `pearl_first_observation_bootstrap.md` — sentinel 0.25 → REPLACE. /// - `pearl_no_host_branches_in_captured_graph.md` — `CudaFunction` /// pre-loaded at construction; on-device guards. /// - `pearl_symmetric_clamp_audit.md` — bilateral `fmaxf(lo, fminf(x, hi))`. /// - `feedback_isv_for_adaptive_bounds.md` — bounds [0.10, 0.50] are /// Category-1 dimensional safety, NOT tuning. #[allow(missing_debug_implementations)] pub(crate) struct DdSaturationFloorUpdateOps { update_kernel: CudaFunction, } impl DdSaturationFloorUpdateOps { /// Block dim used by the producer kernel — must match `BLK_DIM` in /// `dd_saturation_floor_update_kernel.cu`. const BLK_DIM: u32 = 256; pub(crate) fn new(stream: &Arc) -> Result { let context = stream.context(); let module = context .load_cubin(DD_SATURATION_FLOOR_UPDATE_CUBIN.to_vec()) .map_err(|e| MLError::ModelError(format!("dd_saturation_floor_update cubin load: {e}")))?; let update_kernel = module .load_function("dd_saturation_floor_update") .map_err(|e| MLError::ModelError(format!("dd_saturation_floor_update load: {e}")))?; Ok(Self { update_kernel }) } /// Launch the adaptive DD saturation floor producer. /// /// Args: /// - `dd_state_per_env_ptr`: f32 device ptr `[n_envs * dd_state_stride]` /// — the same buffer `compute_sp15_final_reward_kernel` reads from. /// We use the per-env DD_MAX field at offset `dd_max_off`. /// - `n_envs`: number of envs (alloc_episodes from the collector). /// - `dd_state_stride`: 6 (mirrors the per-env tile layout in /// `dd_state_kernel`). /// - `dd_max_off`: 1 (DD_MAX offset within each env's tile). /// - `isv_ptr`: ISV[ISV_TOTAL_DIM] device pointer (mapped-pinned). /// - `floor_idx`: DD_SATURATION_FLOOR_ADAPTIVE_INDEX (458). /// - `sentinel_floor`: SENTINEL_DD_SATURATION_FLOOR (0.25). /// - `safety_factor`: DD_SATURATION_FLOOR_SAFETY_FACTOR (1.5). /// - `floor_min`/`floor_max`: DD_SATURATION_FLOOR_MIN/MAX (0.10, 0.50). /// - `alpha`: DD_SATURATION_FLOOR_EMA_ALPHA (0.01, slow per-epoch). #[allow(clippy::too_many_arguments)] pub(crate) fn launch( &self, stream: &Arc, dd_state_per_env_ptr: u64, n_envs: i32, dd_state_stride: i32, dd_max_off: i32, isv_ptr: u64, floor_idx: i32, sentinel_floor: f32, safety_factor: f32, floor_min: f32, floor_max: f32, alpha: f32, ) -> Result<(), MLError> { // 4 arrays × BLK_DIM × sizeof(f32) — see the kernel comment block. let smem_bytes = 4 * Self::BLK_DIM * std::mem::size_of::() as u32; unsafe { stream .launch_builder(&self.update_kernel) .arg(&dd_state_per_env_ptr) .arg(&n_envs) .arg(&dd_state_stride) .arg(&dd_max_off) .arg(&isv_ptr) .arg(&floor_idx) .arg(&sentinel_floor) .arg(&safety_factor) .arg(&floor_min) .arg(&floor_max) .arg(&alpha) .launch(LaunchConfig { grid_dim: (1, 1, 1), block_dim: (Self::BLK_DIM, 1, 1), shared_mem_bytes: smem_bytes, }) .map_err(|e| MLError::ModelError(format!("dd_saturation_floor_update: {e}")))?; } Ok(()) } } /// SP16 Phase 1 (revised, 2026-05-08): adaptive MIN_HOLD_TEMPERATURE /// producer driven by hold-rate overrun (signal swap from slot 373). /// /// Single-thread cold-path kernel (per-epoch boundary cadence) that /// reads two SP13 hold-pricing chain slots from ISV: /// /// - `ISV[HOLD_RATE_OBSERVED_EMA_INDEX=382]` — per-fold EMA of the /// per-step Hold-pick rate, populated by `apply_fixed_alpha_ema_kernel` /// chained off `hold_rate_observer_kernel`. /// - `ISV[HOLD_RATE_TARGET_INDEX=381]` — static target (default 0.20). /// /// And maps the overrun into a [5, 50] temperature: /// /// overrun = max(0, observed − target) /// overrun_norm = clamp(overrun / max(target, 0.01), 0, 1) /// temp = TEMP_MIN + (TEMP_MAX − TEMP_MIN) × overrun_norm /// /// then slow-EMA-blends into /// `ISV[MIN_HOLD_TEMPERATURE_ADAPTIVE_INDEX=460]` with α=0.05. /// /// Per `compute_min_hold_penalty` at trade_physics.cuh:575 /// (`soft_factor = deficit / (deficit + T)`), HIGH T = forgiving /// (saturates → 0, no exit penalty), LOW T = sharp (saturates → 1, /// max exit penalty). The hold-rate-overrun mapping therefore gives /// the over-holding scenario its designed behaviour: /// - observed = 2× target → overrun_norm = 1.0 → temp = 50 /// (permissive — exit ramp out of over-holding) /// - observed = target → overrun_norm = 0 → temp = 5 /// (strict — force commitment at the hold-rate target) /// - observed < target → overrun_norm = 0 → temp = 5 /// (under-holding — strict, no relaxation) /// /// Signal swap rationale (post-mortem of train-multi-seed-pfh9n, /// 2026-05-08): the original mapping read /// `ISV[AUX_DIR_ACC_SHORT_EMA_INDEX=373]` but slot 373 stayed at /// sentinel 0.5 in Fold 1 — the per-step aux producer either didn't /// push it off sentinel quickly enough or the binary classifier /// converged within ε of 0.5 in noisy environments. The kernel's /// early-return-on-sentinel guard then fired on every launch and slot /// 460 stayed pinned at 50 for the entire fold. Hold rates in /// contrast are measured per-epoch from realised actions and never at /// a "no-data" sentinel after epoch 1, so the fold-1 chain breakage /// goes away. /// /// Cold-start sentinel guard preserved: when `observed_hold_rate` is /// at the SP13 fold-reset sentinel 0.0 (no per-step Hold observations /// yet in the new fold), the kernel keeps slot 460 unchanged — the /// consumer falls back to MIN_HOLD_TEMPERATURE_FALLBACK=50.0 via the /// same sentinel-detect path used pre-swap. /// /// # Pearls applied /// - `feedback_no_atomicadd.md` — single-thread kernel; no reductions. /// - `pearl_first_observation_bootstrap.md` — sentinel 50.0 → REPLACE. /// - `pearl_no_host_branches_in_captured_graph.md` — `CudaFunction` /// pre-loaded at construction; on-device guards. /// - `pearl_symmetric_clamp_audit.md` — bilateral /// `fmaxf(lo, fminf(x, hi))` clamp on target_temp + post-blend. /// - `feedback_isv_for_adaptive_bounds.md` — bounds [5, 50] are /// Category-1 dimensional safety floors (matches the original /// schedule range MIN_HOLD_TEMPERATURE_END=5 ↔ /// MIN_HOLD_TEMPERATURE_START=50), NOT tuning. /// - `feedback_no_partial_refactor.md` — kernel + launcher + call site /// + tests + audit doc updated atomically in the same commit. #[allow(missing_debug_implementations)] pub(crate) struct MinHoldTemperatureUpdateOps { update_kernel: CudaFunction, } impl MinHoldTemperatureUpdateOps { pub(crate) fn new(stream: &Arc) -> Result { let context = stream.context(); let module = context .load_cubin(MIN_HOLD_TEMPERATURE_UPDATE_CUBIN.to_vec()) .map_err(|e| MLError::ModelError(format!("min_hold_temperature_update cubin load: {e}")))?; let update_kernel = module .load_function("min_hold_temperature_update") .map_err(|e| MLError::ModelError(format!("min_hold_temperature_update load: {e}")))?; Ok(Self { update_kernel }) } /// Launch the adaptive MIN_HOLD_TEMPERATURE producer (SP16 Phase 1 /// revised — hold-rate-overrun driven; SP16 T3 — Wiener-optimal α). /// /// Args: /// - `isv_ptr`: ISV[ISV_TOTAL_DIM] device pointer (mapped-pinned). /// - `temp_idx`: MIN_HOLD_TEMPERATURE_ADAPTIVE_INDEX (460). /// - `hold_rate_observed_idx`: HOLD_RATE_OBSERVED_EMA_INDEX (382). /// - `hold_rate_target_idx`: HOLD_RATE_TARGET_INDEX (381). /// - Welford accumulator slot indices (T3): MHT_TARGET_MEAN_INDEX /// (468), MHT_TARGET_M2_INDEX (469), MHT_DIFF_MEAN_INDEX (470), /// MHT_DIFF_M2_INDEX (471), MHT_PREV_TARGET_INDEX (472), /// MHT_SAMPLE_COUNT_INDEX (473). /// - `sentinel_temp`: SENTINEL_MIN_HOLD_TEMPERATURE (50.0). /// - `sentinel_observed_hold_rate`: SENTINEL_HOLD_RATE_OBSERVED /// (0.0; matches the SP13 fold-reset registry sentinel for /// `sp13_hold_rate_observed_ema`). /// - `temp_min`/`temp_max`: MIN_HOLD_TEMPERATURE_MIN/MAX (5.0, 50.0). /// /// SP16 T3: the `alpha` argument is gone — α is computed inside the /// kernel from the Welford accumulators (Wiener-optimal: α = diff_var /// / (diff_var + sample_var + ε)). #[allow(clippy::too_many_arguments)] pub(crate) fn launch( &self, stream: &Arc, isv_ptr: u64, temp_idx: i32, hold_rate_observed_idx: i32, hold_rate_target_idx: i32, target_mean_idx: i32, target_m2_idx: i32, diff_mean_idx: i32, diff_m2_idx: i32, prev_target_idx: i32, sample_count_idx: i32, sentinel_temp: f32, sentinel_observed_hold_rate: f32, temp_min: f32, temp_max: f32, ) -> Result<(), MLError> { unsafe { stream .launch_builder(&self.update_kernel) .arg(&isv_ptr) .arg(&temp_idx) .arg(&hold_rate_observed_idx) .arg(&hold_rate_target_idx) .arg(&target_mean_idx) .arg(&target_m2_idx) .arg(&diff_mean_idx) .arg(&diff_m2_idx) .arg(&prev_target_idx) .arg(&sample_count_idx) .arg(&sentinel_temp) .arg(&sentinel_observed_hold_rate) .arg(&temp_min) .arg(&temp_max) .launch(LaunchConfig { grid_dim: (1, 1, 1), block_dim: (1, 1, 1), shared_mem_bytes: 0, }) .map_err(|e| MLError::ModelError(format!("min_hold_temperature_update: {e}")))?; } Ok(()) } } /// SP14 Layer C Phase C.6 (2026-05-08): h_s2_aux RMS EMA producer. /// /// Computes `RMS(h_s2_aux) = sqrt(mean(h_s2_aux²))` over the /// `[B, SH2]` aux trunk output and EMA-blends the result into /// `ISV[H_S2_AUX_RMS_EMA_INDEX=449]`. /// /// ISV slot 449 is in the SP14 Layer C block (outside the SP4/SP5 wiener /// buffer linear span), so Pearl-A bootstrap logic is embedded in the /// kernel body rather than delegated to `apply_pearls_ad_kernel`. Fixed /// α=0.05 (per-step cadence blend rate; no Wiener-optimal adaptive α /// available without a wiener_state_buf triple for this slot). /// /// Single-block 256-thread kernel; shmem block-tree-reduce (no atomicAdd /// per `feedback_no_atomicadd.md`). `CudaFunction` pre-loaded at /// construction per `pearl_no_host_branches_in_captured_graph.md`. /// /// # Pearls applied /// - `feedback_no_atomicadd.md` — block-tree-reduce in shmem, one global /// write from thread 0. /// - `pearl_first_observation_bootstrap.md` — sentinel 0.0 → REPLACE. /// - `pearl_no_host_branches_in_captured_graph.md` — `CudaFunction` /// pre-loaded at construction; no per-launch cubin load. /// - `feedback_no_stubs.md` — full body, no placeholder. #[allow(missing_debug_implementations)] pub(crate) struct HS2AuxRmsEmaOps { update_kernel: CudaFunction, } impl HS2AuxRmsEmaOps { /// Block dim — must match the kernel's `BLK_DIM` of 256. const BLK_DIM: u32 = 256; /// Per-step EMA blend rate. Matches the aux trunk forward launch /// cadence — one observation per collector step. pub(crate) const ALPHA: f32 = 0.05; pub(crate) fn new(stream: &Arc) -> Result { let context = stream.context(); let module = context .load_cubin(H_S2_AUX_RMS_EMA_CUBIN.to_vec()) .map_err(|e| MLError::ModelError(format!("h_s2_aux_rms_ema cubin load: {e}")))?; let update_kernel = module .load_function("h_s2_aux_rms_ema_update") .map_err(|e| MLError::ModelError(format!("h_s2_aux_rms_ema_update load: {e}")))?; Ok(Self { update_kernel }) } /// Launch the RMS EMA producer. /// /// Args: /// - `h_s2_aux_ptr`: aux trunk output `[B, SH2]` device pointer. /// - `b`: batch size (B). /// - `sh2`: aux trunk output width (SH2 = AUX_HIDDEN_DIM = 256). /// - `isv_ptr`: ISV[ISV_TOTAL_DIM] device pointer (mapped-pinned). /// - `isv_target_idx`: H_S2_AUX_RMS_EMA_INDEX (449). /// - `alpha`: EMA blend rate (use `Self::ALPHA = 0.05`). pub(crate) fn launch( &self, stream: &Arc, h_s2_aux_ptr: u64, b: i32, sh2: i32, isv_ptr: u64, isv_target_idx: i32, alpha: f32, ) -> Result<(), MLError> { let smem_bytes = Self::BLK_DIM * std::mem::size_of::() as u32; unsafe { stream .launch_builder(&self.update_kernel) .arg(&h_s2_aux_ptr) .arg(&b) .arg(&sh2) .arg(&isv_ptr) .arg(&isv_target_idx) .arg(&alpha) .launch(LaunchConfig { grid_dim: (1, 1, 1), block_dim: (Self::BLK_DIM, 1, 1), shared_mem_bytes: smem_bytes, }) .map_err(|e| MLError::ModelError(format!("h_s2_aux_rms_ema_update: {e}")))?; } Ok(()) } }