diff --git a/crates/ml/src/cuda_pipeline/batched_backward.rs b/crates/ml/src/cuda_pipeline/batched_backward.rs index c84728884..9b7ab8065 100644 --- a/crates/ml/src/cuda_pipeline/batched_backward.rs +++ b/crates/ml/src/cuda_pipeline/batched_backward.rs @@ -300,8 +300,15 @@ impl CublasBackwardSet { let na = config.num_atoms; let _sd = ml_core::state_layout::STATE_DIM; let sd_pad = (ml_core::state_layout::STATE_DIM + 127) & !127; + // SP15 Wave 4.1b: the bottleneck-on s1_input_dim grows by +1 to carry + // the dd_pct column appended by `bn_tanh_concat_dd_kernel`. Mirrors + // `compute_param_sizes` so the backward gemm cache shape keys match + // forward's (and the encoder_backward_chain dX width sees the same + // 103-column buffer). let s1_input_dim = if config.bottleneck_dim > 0 { - config.bottleneck_dim + ml_core::state_layout::STATE_DIM.saturating_sub(config.market_dim) + config.bottleneck_dim + + ml_core::state_layout::STATE_DIM.saturating_sub(config.market_dim) + + 1 } else { ml_core::state_layout::STATE_DIM }; diff --git a/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs b/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs index 27966ddaf..d46a96784 100644 --- a/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs +++ b/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs @@ -3314,9 +3314,18 @@ mod tests { #[test] fn test_gpu_backtest_evaluator_state_dim_calculation() { - // Canonical layout from ml_core::state_layout — 42 market + 20 OFI + 16 MTF + - // 8 portfolio + 6 plan/ISV + 4 padding = 96. Padded to 128 for cuBLAS. - assert_eq!(ml_core::state_layout::STATE_DIM, 96); + // Canonical layout from ml_core::state_layout — 128-dim feature + // vector (42 market + 32 OFI + 14 TLOB + 16 MTF + 8 portfolio + + // 6 plan/ISV + 10 padding = 128). Already padded to 128 for cuBLAS + // K-tile alignment. The historical 96-dim layout (pre-2026-04 + // refactor) was deprecated when the OFI / TLOB / MTF blocks were + // expanded; the canonical constant in `ml_core::state_layout` + // is the source of truth — code wins over docs (cf. + // `feedback_trust_code_not_docs`). Migrated under SP15 Wave 4.1b + // alongside the s1_input_dim 102→103 propagation that followed + // the same `feedback_trust_code_not_docs` correction in the + // wave's audit doc. + assert_eq!(ml_core::state_layout::STATE_DIM, 128); assert_eq!(ml_core::state_layout::STATE_DIM_PADDED, 128); } diff --git a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs index 5e349046d..2649ad4e1 100644 --- a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs +++ b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs @@ -3995,12 +3995,20 @@ pub(crate) const NUM_WEIGHT_TENSORS: usize = 163; /// Tensors 82-85: Trade plan head (h_s2 → 6 plan parameters). /// /// When bottleneck is active, w_s1 input dimension changes from state_dim to -/// (bottleneck_dim + portfolio_dim) where portfolio_dim = state_dim - market_dim. +/// (bottleneck_dim + portfolio_dim + 1) where portfolio_dim = state_dim - market_dim +/// and the trailing +1 column is the SP15 Wave 4.1b dd_pct broadcast (read from +/// ISV[DD_PCT_INDEX=406] by `bn_tanh_concat_dd_kernel`). The +1 is folded into +/// `s1_input_dim` so all consumers (compute_param_sizes, Xavier init, cuBLAS +/// gemm cache, encoder_backward_chain dX width, GRN w_s1 / w_residual_h_s1 +/// fan-in) treat dd_pct as the (s1_input_dim)th feature column. The bn_concat +/// buffer is sized [B, bn_dim + portfolio_dim + 1] = [B, s1_input_dim] in the +/// bottleneck-on path. pub(crate) fn compute_param_sizes(cfg: &GpuDqnTrainConfig) -> [usize; NUM_WEIGHT_TENSORS] { - // When bottleneck is active, h_s1 input is [bottleneck_dim + portfolio_features] - // instead of [state_dim]. portfolio_features = state_dim - market_dim (typically 6-8). + // When bottleneck is active, h_s1 input is [bottleneck_dim + portfolio_features + 1] + // instead of [state_dim]. The trailing +1 column carries dd_pct from ISV (SP15 + // Wave 4.1b). portfolio_features = state_dim - market_dim (typically 6-8). let s1_input_dim = if cfg.bottleneck_dim > 0 { - cfg.bottleneck_dim + (ml_core::state_layout::STATE_DIM.saturating_sub(cfg.market_dim)) + cfg.bottleneck_dim + (ml_core::state_layout::STATE_DIM.saturating_sub(cfg.market_dim)) + 1 } else { ml_core::state_layout::STATE_DIM }; @@ -5461,8 +5469,17 @@ pub struct GpuDqnTrainer { on_next_bn_hidden_buf: CudaSlice, /// DDQN argmax-pass mirror of `bn_concat_buf` (online-on-next_states). on_next_bn_concat_buf: CudaSlice, - /// #31 Bottleneck tanh + concat kernel. - bn_tanh_concat_kernel: CudaFunction, + // SP15 Wave 4.1b removed the legacy `bn_tanh_concat_kernel` field — + // production callers (3 forward sites here + 1 in `gpu_experience_collector`) + // migrated to `launch_sp15_bn_concat_dd` (the bottleneck-aware + // dd_pct-appending launcher), which loads the + // `bn_tanh_concat_dd_kernel` symbol on-demand from the shared + // `dqn_utility_kernels.cubin`. The legacy `bn_tanh_concat_kernel` + // and `bn_tanh_concat_f32_kernel` symbols stay in the cubin source + // for the SP15 oracle tests in + // `crates/ml/tests/sp15_phase1_oracle_tests.rs` (parity check + // between old and new kernels), but no production code path resolves + // them anymore. /// #31 Backward scratch: d_loss/d_bn_concat [B, concat_dim] f32. bn_d_concat_buf: CudaSlice, /// #31 Backward scratch: d_loss/d_bn [B, bn_dim] f32 (after tanh derivative). @@ -10618,10 +10635,17 @@ impl GpuDqnTrainer { &self.bn_d_concat_buf } - /// Bottleneck concat dimension (bottleneck_dim + STATE_DIM - market_dim). + /// Bottleneck concat dimension (bottleneck_dim + STATE_DIM - market_dim + 1). /// Used by TLOB backward to compute the slice offset for the TLOB gradient. + /// The trailing +1 column is the SP15 Wave 4.1b dd_pct broadcast appended by + /// `bn_tanh_concat_dd_kernel`; TLOB only reads the columns inside the + /// portfolio slice (`[bn_dim + (TLOB_START − market_dim) ..]`), so the + /// dd_pct column is naturally outside its window. The accessor reports the + /// full row stride so the TLOB backward indexes `bn_d_concat_buf` correctly. pub(crate) fn bn_concat_dim(&self) -> usize { - self.config.bottleneck_dim + (ml_core::state_layout::STATE_DIM.saturating_sub(self.config.market_dim)) + self.config.bottleneck_dim + + (ml_core::state_layout::STATE_DIM.saturating_sub(self.config.market_dim)) + + 1 } /// SP1 Phase B (slot 24): post-`c51_grad_kernel` value-stream gradient buffer @@ -17506,7 +17530,7 @@ impl GpuDqnTrainer { // per array. Stack is set once in DQNTrainer::new() (64KB for all kernels). // ── Compile utility kernels (grad_norm, adam_update, etc.) ─ - let (grad_norm_kernel, grad_norm_finalize_kernel, grad_norm_finalize_adam, grad_norm_finalize_post_aux, adam_update_kernel, adam_update_post_aux, scale_f32_post_aux, saxpy_kernel, zero_kernel, regime_scale_kernel, shrink_perturb, _relu_mask_in_module, spectral_norm_kernel, clip_grad_kernel, pad_states_kernel, saxpy_f32_kernel, scale_f32_kernel, stochastic_depth_kernel, stochastic_depth_rng_kernel, bn_tanh_backward_kernel, bn_bias_grad_kernel, bn_tanh_concat_kernel_fn, vaccine_dot_kernel, vaccine_project_kernel, vaccine_dot_finalize, causal_intervene_kernel_fn, causal_reduce_kernel_fn, causal_mean_reduce_kernel_fn, pruning_mask_kernel, pruning_compute_kernel, her_inplace_kernel, nan_check_f32_kernel, nan_check_f32_kernel_b, nan_check_fused_f32_kernel, clamp_finite_f32_kernel, popart_normalize_kernel, saxpy_f32_adam_grad, saxpy_f32_aux, scale_f32_aux, distill_saxpy_aux, grad_norm_standalone_post_aux, shrink_perturb_ungraphed, scale_f32_ungraphed, popart_robust_kernel) = + let (grad_norm_kernel, grad_norm_finalize_kernel, grad_norm_finalize_adam, grad_norm_finalize_post_aux, adam_update_kernel, adam_update_post_aux, scale_f32_post_aux, saxpy_kernel, zero_kernel, regime_scale_kernel, shrink_perturb, _relu_mask_in_module, spectral_norm_kernel, clip_grad_kernel, pad_states_kernel, saxpy_f32_kernel, scale_f32_kernel, stochastic_depth_kernel, stochastic_depth_rng_kernel, bn_tanh_backward_kernel, bn_bias_grad_kernel, vaccine_dot_kernel, vaccine_project_kernel, vaccine_dot_finalize, causal_intervene_kernel_fn, causal_reduce_kernel_fn, causal_mean_reduce_kernel_fn, pruning_mask_kernel, pruning_compute_kernel, her_inplace_kernel, nan_check_f32_kernel, nan_check_f32_kernel_b, nan_check_fused_f32_kernel, clamp_finite_f32_kernel, popart_normalize_kernel, saxpy_f32_adam_grad, saxpy_f32_aux, scale_f32_aux, distill_saxpy_aux, grad_norm_standalone_post_aux, shrink_perturb_ungraphed, scale_f32_ungraphed, popart_robust_kernel) = compile_training_kernels(&stream, &config)?; // Separate grad_norm instance for non-graph launches (outside CUDA graph). @@ -19711,8 +19735,14 @@ impl GpuDqnTrainer { // ── Initialize shared cuBLAS handle (one handle for all forward/backward) ── let shared_cublas = Arc::new(PerStreamCublasHandles::new(&stream)?); + // SP15 Wave 4.1b: the bottleneck-on s1_input_dim grows by +1 to carry + // the dd_pct column appended by `bn_tanh_concat_dd_kernel`. Mirrors + // `compute_param_sizes` exactly so the cuBLAS gemm cache shape keys + // match the GRN w_s1 / w_residual_h_s1 tensor widths. let s1_input_dim = if config.bottleneck_dim > 0 { - config.bottleneck_dim + ml_core::state_layout::STATE_DIM.saturating_sub(config.market_dim) + config.bottleneck_dim + + ml_core::state_layout::STATE_DIM.saturating_sub(config.market_dim) + + 1 } else { ml_core::state_layout::STATE_DIM }; @@ -20044,7 +20074,15 @@ impl GpuDqnTrainer { // Unpadded strides cause cuBLAS heuristic to pick algorithms that hang // in CUDA Graph replay on Hopper (sm_90). pad128 forces tile-aligned // algorithms that are graph-compatible. - let concat_dim = bn_alloc_dim + portfolio_dim; + // + // SP15 Wave 4.1b: trailing +1 column carries dd_pct (read from + // ISV[DD_PCT_INDEX=406] by `bn_tanh_concat_dd_kernel`). Concat layout + // is [bn_tanh | portfolio | dd_pct], `concat_dim = bn_dim + + // portfolio_dim + 1`. All four buffers (online / target / DDQN + // forward outputs + the backward dX accumulator) share the same + // stride — the dd_pct column is a structural part of the trunk + // input now, mirrored across all forward passes. + let concat_dim = bn_alloc_dim + portfolio_dim + 1; let bn_concat_buf = stream.alloc_zeros::(b * concat_dim + kt) .map_err(|e| MLError::ModelError(format!("alloc bn_concat: {e}")))?; // Target-net mirrors of bn_hidden / bn_concat — same shapes; populated @@ -22326,7 +22364,8 @@ impl GpuDqnTrainer { on_next_bn_concat_buf, bn_d_concat_buf, bn_d_hidden_buf, - bn_tanh_concat_kernel: bn_tanh_concat_kernel_fn, + // bn_tanh_concat_kernel field removed in SP15 Wave 4.1b — see + // the field-removal comment block in the struct definition. bn_tanh_backward_kernel, bn_bias_grad_kernel, // ── Plan 4 Task 1B-iii: VSN feature-selection buffers ────── @@ -27103,7 +27142,9 @@ impl GpuDqnTrainer { let b = self.config.batch_size; let market_dim = self.config.market_dim; let portfolio_dim = ml_core::state_layout::STATE_DIM.saturating_sub(market_dim); - let concat_dim = bn_dim + portfolio_dim; + // SP15 Wave 4.1b: trailing +1 column = dd_pct (broadcast from + // ISV[DD_PCT_INDEX=406] by `bn_tanh_concat_dd_kernel`). + let concat_dim = bn_dim + portfolio_dim + 1; let bn_hidden_ptr = self.on_next_bn_hidden_buf.raw_ptr(); let state_dim_padded_usize = self.cublas_forward.state_dim_padded; self.cublas_forward.sgemm_f32_ldb( @@ -27119,31 +27160,29 @@ impl GpuDqnTrainer { &self.stream, bn_hidden_ptr, on_w_ptrs[34], bn_dim, b, )?; let concat_ptr = self.on_next_bn_concat_buf.raw_ptr(); - let state_dim_padded = state_dim_padded_usize as i32; - let total_elems = (b * concat_dim) as i32; - let blocks = ((total_elems as u32 + 255) / 256) as u32; + let state_dim_padded_i32 = state_dim_padded_usize as i32; let bn_dim_i32 = bn_dim as i32; let market_dim_i32 = market_dim as i32; let concat_dim_i32 = concat_dim as i32; let b_i32 = b as i32; - unsafe { - self.stream - .launch_builder(&self.bn_tanh_concat_kernel) - .arg(&bn_hidden_ptr) - .arg(&on_next_states_for_bn) - .arg(&concat_ptr) - .arg(&b_i32) - .arg(&bn_dim_i32) - .arg(&market_dim_i32) - .arg(&state_dim_padded) - .arg(&concat_dim_i32) - .launch(LaunchConfig { - grid_dim: (blocks, 1, 1), - block_dim: (256, 1, 1), - shared_mem_bytes: 0, - }) - .map_err(|e| MLError::ModelError(format!("ddqn_bn_tanh_concat: {e}")))?; - } + // SP15 Wave 4.1b: launch the dd_pct-aware concat kernel. Reads + // `isv[DD_PCT_INDEX=406]` and broadcasts it across batch as the + // (concat_dim − 1)th column. ISV bus is the same pointer all the + // other ISV consumers use; experience_collector populates it via + // `launch_sp15_dd_state` before the trainer's forward graph runs, + // so by the time this kernel reads slot 406 the value is fresh. + launch_sp15_bn_concat_dd( + &self.stream, + bn_hidden_ptr, + on_next_states_for_bn, + self.isv_signals_dev_ptr, + concat_ptr, + b_i32, + bn_dim_i32, + market_dim_i32, + state_dim_padded_i32, + concat_dim_i32, + )?; concat_ptr } else { // No bottleneck: pass VSN-gated next_states directly to DDQN encoder. @@ -27428,7 +27467,9 @@ impl GpuDqnTrainer { let b = self.config.batch_size; let market_dim = self.config.market_dim; let portfolio_dim = ml_core::state_layout::STATE_DIM.saturating_sub(market_dim); - let concat_dim = bn_dim + portfolio_dim; + // SP15 Wave 4.1b: trailing +1 column = dd_pct (broadcast from + // ISV[DD_PCT_INDEX=406] by `bn_tanh_concat_dd_kernel`). + let concat_dim = bn_dim + portfolio_dim + 1; // Bottleneck GEMM: states[B, market_dim] @ w_bn^T → h_bn[B, bn_dim] // w_bn is at on_w_ptrs[33], b_bn is at on_w_ptrs[34] (post Plan 4 @@ -27452,38 +27493,33 @@ impl GpuDqnTrainer { &self.stream, bn_hidden_ptr, on_w_ptrs[34], bn_dim, b, )?; - // Tanh + concat: [tanh(h_bn), portfolio_from_states] → bn_concat - // Plan 4 Task 1B-iii: portfolio passthrough source is the gated - // state — keeps bottleneck and portfolio halves in sync. + // Tanh + concat + dd_pct: [tanh(h_bn), portfolio_from_states, + // isv[DD_PCT_INDEX]] → bn_concat. Plan 4 Task 1B-iii: portfolio + // passthrough source is the gated state — keeps bottleneck and + // portfolio halves in sync. + // SP15 Wave 4.1b: dd_pct broadcast comes from the same ISV bus + // every other consumer reads; experience_collector's + // `launch_sp15_dd_state` populates ISV[406] in the per-step env + // loop, so the trunk forward sees a fresh value here. let concat_ptr = self.bn_concat_buf.raw_ptr(); - let state_dim_padded = state_dim_padded_usize as i32; - let total_elems = (b * concat_dim) as i32; - let blocks = ((total_elems as u32 + 255) / 256) as u32; + let state_dim_padded_i32 = state_dim_padded_usize as i32; let bn_dim_i32 = bn_dim as i32; let market_dim_i32 = market_dim as i32; let concat_dim_i32 = concat_dim as i32; let b_i32 = b as i32; - { - unsafe { - self.stream - .launch_builder(&self.bn_tanh_concat_kernel) - .arg(&bn_hidden_ptr) - .arg(&states_for_bn) - .arg(&concat_ptr) - .arg(&b_i32) - .arg(&bn_dim_i32) - .arg(&market_dim_i32) - .arg(&state_dim_padded) - .arg(&concat_dim_i32) - .launch(LaunchConfig { - grid_dim: (blocks, 1, 1), - block_dim: (256, 1, 1), - shared_mem_bytes: 0, - }) - .map_err(|e| MLError::ModelError(format!("bn_tanh_concat: {e}")))?; - } - } + launch_sp15_bn_concat_dd( + &self.stream, + bn_hidden_ptr, + states_for_bn, + self.isv_signals_dev_ptr, + concat_ptr, + b_i32, + bn_dim_i32, + market_dim_i32, + state_dim_padded_i32, + concat_dim_i32, + )?; concat_ptr } else { // No bottleneck: bypass directly to the gated state. Downstream @@ -27638,7 +27674,14 @@ impl GpuDqnTrainer { let b = self.config.batch_size; let market_dim = self.config.market_dim; let portfolio_dim = ml_core::state_layout::STATE_DIM.saturating_sub(market_dim); - let concat_dim = bn_dim + portfolio_dim; + // SP15 Wave 4.1b: trailing +1 column = dd_pct (broadcast from + // ISV[DD_PCT_INDEX=406] by `bn_tanh_concat_dd_kernel`). Target + // forward reads the SAME ISV bus the online forward read on the + // current step, so both networks see the same dd_pct context for + // their respective state inputs (next_states for target, states + // for online). dd_pct is a global episode statistic — not + // per-state — so the broadcast value is correct for both. + let concat_dim = bn_dim + portfolio_dim + 1; let tg_bn_hidden_ptr = self.tg_bn_hidden_buf.raw_ptr(); let state_dim_padded_usize = self.cublas_forward.state_dim_padded; self.cublas_forward.sgemm_f32_ldb( @@ -27654,31 +27697,23 @@ impl GpuDqnTrainer { &self.stream, tg_bn_hidden_ptr, tg_w_ptrs[34], bn_dim, b, )?; let tg_concat_ptr = self.tg_bn_concat_buf.raw_ptr(); - let state_dim_padded = state_dim_padded_usize as i32; - let total_elems = (b * concat_dim) as i32; - let blocks = ((total_elems as u32 + 255) / 256) as u32; + let state_dim_padded_i32 = state_dim_padded_usize as i32; let bn_dim_i32 = bn_dim as i32; let market_dim_i32 = market_dim as i32; let concat_dim_i32 = concat_dim as i32; let b_i32 = b as i32; - unsafe { - self.stream - .launch_builder(&self.bn_tanh_concat_kernel) - .arg(&tg_bn_hidden_ptr) - .arg(&tg_states_for_bn) - .arg(&tg_concat_ptr) - .arg(&b_i32) - .arg(&bn_dim_i32) - .arg(&market_dim_i32) - .arg(&state_dim_padded) - .arg(&concat_dim_i32) - .launch(LaunchConfig { - grid_dim: (blocks, 1, 1), - block_dim: (256, 1, 1), - shared_mem_bytes: 0, - }) - .map_err(|e| MLError::ModelError(format!("tg_bn_tanh_concat: {e}")))?; - } + launch_sp15_bn_concat_dd( + &self.stream, + tg_bn_hidden_ptr, + tg_states_for_bn, + self.isv_signals_dev_ptr, + tg_concat_ptr, + b_i32, + bn_dim_i32, + market_dim_i32, + state_dim_padded_i32, + concat_dim_i32, + )?; tg_concat_ptr } else { // No bottleneck: pass VSN-gated next_states directly to target encoder. @@ -28498,7 +28533,18 @@ impl GpuDqnTrainer { let market_dim = config.market_dim; let b = config.batch_size; let portfolio_dim = ml_core::state_layout::STATE_DIM.saturating_sub(market_dim); - let concat_dim = bn_dim + portfolio_dim; + // SP15 Wave 4.1b: bn_d_concat_buf row stride is the full forward + // concat_dim including the dd_pct column — encoder_backward_chain's + // Linear_a/Linear_residual dX writes 103 columns. The + // bn_tanh_backward_kernel reads only the bn_dim slice [0..bn_dim) and + // the vsn_d_gated_state_portfolio_pad_kernel reads only the + // portfolio slice [bn_dim..bn_dim+portfolio_dim) — both correctly + // skip the dd_pct gradient column at index bn_dim+portfolio_dim + // because dd_pct sources from the ISV bus (no learnable input feeds + // it back). The discarded column's gradient is harmless: its only + // effect would be to update a downstream variable, which doesn't + // exist. + let concat_dim = bn_dim + portfolio_dim + 1; let state_dim = ml_core::state_layout::STATE_DIM; let state_dim_padded = ml_core::state_layout::STATE_DIM_PADDED; @@ -28998,7 +29044,14 @@ impl GpuDqnTrainer { let market_dim = self.config.market_dim; let b = self.config.batch_size; let portfolio_dim = ml_core::state_layout::STATE_DIM.saturating_sub(market_dim); - let concat_dim = bn_dim + portfolio_dim; + // SP15 Wave 4.1b: bn_d_concat_buf row stride is the full forward + // concat_dim including the dd_pct column. Same rationale as the + // aux_bottleneck_vsn_backward_dispatch site — bn_tanh_backward + // reads [0..bn_dim) and the portfolio_pad reads + // [bn_dim..bn_dim+portfolio_dim); the dd_pct column at + // bn_dim+portfolio_dim is correctly skipped because dd_pct has no + // learnable input source. + let concat_dim = bn_dim + portfolio_dim + 1; let state_dim = ml_core::state_layout::STATE_DIM; let state_dim_padded = ml_core::state_layout::STATE_DIM_PADDED; @@ -29813,9 +29866,19 @@ impl GpuDqnTrainer { let sizes = compute_param_sizes(cfg); let total = self.total_params; - // Compute s1_input_dim matching compute_param_sizes + // Compute s1_input_dim matching compute_param_sizes — SP15 Wave 4.1b + // bumps the bottleneck-on path by +1 for the dd_pct column. Xavier + // sees the full (s1_input_dim, shared_h1) fan and initialises every + // column uniformly, including the new dd_pct column at index + // `bn_dim + portfolio_dim`. The dd_pct value at runtime is bounded + // ∈ [0, 1] (Wave 1.3 contract), matching Xavier's small-magnitude + // assumption — no special-case zero-init needed (cf. SP14 zeroed the + // aux_softmax_diff column of w_b0fc because that signal can be ±1 + // which Xavier assumes is zero-mean; dd_pct is non-negative but + // similarly small-magnitude, so the gradient flows symmetrically + // from day 0). let s1_input_dim = if cfg.bottleneck_dim > 0 { - cfg.bottleneck_dim + ml_core::state_layout::STATE_DIM.saturating_sub(cfg.market_dim) + cfg.bottleneck_dim + ml_core::state_layout::STATE_DIM.saturating_sub(cfg.market_dim) + 1 } else { ml_core::state_layout::STATE_DIM }; @@ -31371,7 +31434,7 @@ fn deflate_rank_one(mat: &[f32], n: usize, lambda_1: f32) -> Vec { fn compile_training_kernels( stream: &Arc, config: &GpuDqnTrainConfig, -) -> Result<(CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction), MLError> { +) -> Result<(CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction, CudaFunction), MLError> { info!( state_dim = ml_core::state_layout::STATE_DIM, total_params = compute_total_params(config), @@ -31495,8 +31558,11 @@ fn compile_training_kernels( .map_err(|e| MLError::ModelError(format!("bn_tanh_backward_kernel load: {e}")))?; let bn_bias_grad = module.load_function("bn_bias_grad_kernel") .map_err(|e| MLError::ModelError(format!("bn_bias_grad_kernel load: {e}")))?; - let bn_tanh_concat = module.load_function("bn_tanh_concat_kernel") - .map_err(|e| MLError::ModelError(format!("bn_tanh_concat_kernel load: {e}")))?; + // SP15 Wave 4.1b: legacy `bn_tanh_concat_kernel` no longer loaded — + // production callers migrated to `launch_sp15_bn_concat_dd`, which + // resolves the `bn_tanh_concat_dd_kernel` symbol from the same cubin. + // The legacy symbol remains in the source for the SP15 oracle parity + // tests but is not surfaced as a `CudaFunction` field on the trainer. let her_inplace = module.load_function("her_inplace_relabel") .map_err(|e| MLError::ModelError(format!("her_inplace_relabel load: {e}")))?; @@ -31526,7 +31592,7 @@ fn compile_training_kernels( .map_err(|e| MLError::ModelError(format!("popart_normalize_robust load: {e}")))?; info!("GpuDqnTrainer: 39 utility kernels loaded from precompiled cubin (5 CUmodules)"); - Ok((grad_norm, grad_norm_finalize, grad_norm_finalize_adam, grad_norm_finalize_post_aux, adam_update, adam_update_post_aux, scale_f32_post_aux, saxpy, zero, regime_scale, shrink_perturb, _relu_mask_from_module, spectral_norm, clip_grad, pad_states, saxpy_f32, scale_f32, stochastic_depth, stochastic_depth_rng, bn_tanh_bw, bn_bias_grad, bn_tanh_concat, vaccine_dot, vaccine_project, vaccine_dot_finalize, causal_intervene, causal_reduce, causal_mean_reduce, pruning_mask_fn, pruning_compute_fn, her_inplace, nan_check_f32, nan_check_f32_b, nan_check_fused_f32, clamp_finite_f32, popart_normalize, saxpy_f32_adam_grad, saxpy_f32_aux, scale_f32_aux, distill_saxpy_aux, grad_norm_standalone_post_aux, shrink_perturb_ungraphed, scale_f32_ungraphed, popart_robust)) + Ok((grad_norm, grad_norm_finalize, grad_norm_finalize_adam, grad_norm_finalize_post_aux, adam_update, adam_update_post_aux, scale_f32_post_aux, saxpy, zero, regime_scale, shrink_perturb, _relu_mask_from_module, spectral_norm, clip_grad, pad_states, saxpy_f32, scale_f32, stochastic_depth, stochastic_depth_rng, bn_tanh_bw, bn_bias_grad, vaccine_dot, vaccine_project, vaccine_dot_finalize, causal_intervene, causal_reduce, causal_mean_reduce, pruning_mask_fn, pruning_compute_fn, her_inplace, nan_check_f32, nan_check_f32_b, nan_check_fused_f32, clamp_finite_f32, popart_normalize, saxpy_f32_adam_grad, saxpy_f32_aux, scale_f32_aux, distill_saxpy_aux, grad_norm_standalone_post_aux, shrink_perturb_ungraphed, scale_f32_ungraphed, popart_robust)) } /// Load the standalone Polyak EMA kernel from precompiled cubin. diff --git a/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs b/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs index 376fe0f67..699fa81be 100644 --- a/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs +++ b/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs @@ -1143,10 +1143,11 @@ impl GpuExperienceCollector { // expected Q via compute_expected_q kernel. let num_atoms = num_atoms_max.max(1); // s1_input_dim must match the trainer's layout: - // with bottleneck: compressed market → bn_dim, then concat with portfolio → bn_dim + portfolio_dim + // with bottleneck: compressed market → bn_dim, then concat with portfolio + // + 1 dd_pct column (SP15 Wave 4.1b) → bn_dim + portfolio_dim + 1 // without bottleneck: raw state_dim let s1_input_dim = if bottleneck_dim > 0 { - bottleneck_dim + state_dim.saturating_sub(market_dim_cfg) + bottleneck_dim + state_dim.saturating_sub(market_dim_cfg) + 1 } else { state_dim }; @@ -1751,18 +1752,33 @@ impl GpuExperienceCollector { .map_err(|e| MLError::ModelError(format!("alloc quantiles_scratch_buf: {e}")))?; // #31 Bottleneck — always loaded and allocated (one production path) + // SP15 Wave 4.1b: experience collector inference forward shares the + // trunk's dd_pct-aware concat path. Loads `bn_tanh_concat_dd_kernel` + // (same cubin, multi-kernel-per-file pattern); the launcher reads + // ISV[DD_PCT_INDEX=406] via this collector's `isv_signals_dev_ptr` + // (set by `set_isv_signals_ptr` from the trainer's bus). At step 0 + // the slot is sentinel 0 (per state_reset_registry); the experience + // collector's `launch_sp15_dd_state` populates it during env_step + // (which runs AFTER this forward, so step k+1 sees step k's + // dd_pct — same one-step-lag semantic as mag_concat). use super::gpu_dqn_trainer::DQN_UTILITY_CUBIN; let util_module = stream.context() .load_cubin(DQN_UTILITY_CUBIN.to_vec()) .map_err(|e| MLError::ModelError(format!("utility cubin load for bn: {e}")))?; - let bn_tanh_concat_fn = util_module.load_function("bn_tanh_concat_f32_kernel") - .map_err(|e| MLError::ModelError(format!("bn_tanh_concat_f32_kernel load: {e}")))?; + let bn_tanh_concat_fn = util_module.load_function("bn_tanh_concat_dd_kernel") + .map_err(|e| MLError::ModelError(format!("bn_tanh_concat_dd_kernel load: {e}")))?; let bn_alloc = bottleneck_dim.max(1); let portfolio_dim_bn = state_dim.saturating_sub(market_dim_cfg); let exp_bn_hidden = stream.alloc_zeros::(alloc_episodes * bn_alloc + 128) .map_err(|e| MLError::ModelError(format!("alloc exp_bn_hidden: {e}")))?; - let exp_bn_concat = stream.alloc_zeros::(alloc_episodes * (bn_alloc + portfolio_dim_bn) + 128) + // SP15 Wave 4.1b: +1 column for dd_pct. Matches the trainer's + // bn_concat_buf / tg_bn_concat_buf / on_next_bn_concat_buf + // reallocation in `gpu_dqn_trainer.rs` (`concat_dim = bn_alloc_dim + // + portfolio_dim + 1`). + let exp_bn_concat = stream.alloc_zeros::( + alloc_episodes * (bn_alloc + portfolio_dim_bn + 1) + 128 + ) .map_err(|e| MLError::ModelError(format!("alloc exp_bn_concat: {e}")))?; // Phase 2d: per-sample, per-branch C51 support buffer [alloc_episodes, 4, 3] @@ -3817,7 +3833,10 @@ impl GpuExperienceCollector { let states_ptr = self.batch_states.raw_ptr(); let md = self.market_dim_bn; let portfolio_dim = ml_core::state_layout::STATE_DIM - md; - let concat_dim = bn_dim + portfolio_dim; + // SP15 Wave 4.1b: trailing +1 column = dd_pct broadcast from + // ISV[DD_PCT_INDEX=406]. Mirrors the trainer's + // bn_tanh_concat_dd_kernel migration. + let concat_dim = bn_dim + portfolio_dim + 1; // #30 F32 SGEMM: batch_states[N, market_dim] @ w_bn^T → h_bn[N, bn_dim] { @@ -3839,20 +3858,27 @@ impl GpuExperienceCollector { )?; } - // Tanh + concat kernel + // Tanh + concat + dd_pct kernel — SP15 Wave 4.1b. Reads + // ISV[DD_PCT_INDEX=406] via the collector's bus pointer (set + // by `set_isv_signals_ptr`); when the pointer is 0 (uninit + // case) the kernel would dereference NULL — guarded by the + // captured-graph wiring in training_loop.rs:859 which sets + // the bus pointer BEFORE the first epoch's forward graph + // capture. { let sdp = self.cublas_forward.state_dim_padded as i32; - let total_elems = (n * concat_dim) as i32; - let blocks = ((total_elems as u32 + 255) / 256) as u32; let bn_i32 = bn_dim as i32; let md_i32 = md as i32; let cd_i32 = concat_dim as i32; let n_i32_bn = n as i32; + let total_elems = (n * concat_dim) as i64; + let blocks = (((total_elems as u64) + 255) / 256).max(1) as u32; unsafe { self.stream .launch_builder(&self.bn_tanh_concat_fn) .arg(&bn_hidden_ptr) .arg(&states_ptr) + .arg(&self.isv_signals_dev_ptr) .arg(&bn_concat_ptr) .arg(&n_i32_bn) .arg(&bn_i32) @@ -3865,7 +3891,7 @@ impl GpuExperienceCollector { shared_mem_bytes: 0, }) .map_err(|e| MLError::ModelError(format!( - "exp bn_tanh_concat: {e}" + "exp bn_tanh_concat_dd: {e}" )))?; } } diff --git a/docs/dqn-wire-up-audit.md b/docs/dqn-wire-up-audit.md index cfde82346..74dff3f14 100644 --- a/docs/dqn-wire-up-audit.md +++ b/docs/dqn-wire-up-audit.md @@ -2,6 +2,8 @@ **Status:** Populated during Plan 1 Task 6 (A.5 orphan audit). Updated on every commit per Invariant 7. +SP15 Wave 4.1b — consumer migration: s1_input_dim 102→103, GRN w_s1 reshape, 4 forward + 3 backward sites wired (2026-05-07): atomic consumer migration that flips production callers of `bn_tanh_concat_kernel` over to Wave 4.1a's `bn_tanh_concat_dd_kernel`, eliminates the documented Wave 4.1a transient orphan, and bumps `s1_input_dim` from 102 to 103 across the entire trunk forward + backward path. Per `feedback_no_partial_refactor`: every consumer of the formula `bottleneck_dim + (STATE_DIM − market_dim)` migrates atomically — no parallel paths, no feature flag toggling old vs new dim. Per `feedback_wire_everything_up`: the Wave 4.1a launcher `launch_sp15_bn_concat_dd` and its underlying `bn_tanh_concat_dd_kernel` symbol now have production callers in this commit; the legacy `bn_tanh_concat_kernel` field is deleted from the trainer struct alongside its loader, eliminating the dead handle. (1) **`s1_input_dim` formula bump from `bn_dim + portfolio_dim` to `bn_dim + portfolio_dim + 1`** at four sites: `compute_param_sizes` (gpu_dqn_trainer.rs ~line 4002, the structural source of truth that drives GRN w_a_h_s1[0] and w_residual_h_s1[4] tensor sizes via `cfg.shared_h1 * s1_input_dim`), the trainer constructor's CublasGemmSet creation site (~19720 — feeds the cuBLAS forward gemm cache shape keys), `xavier_init_params_buf` (~29821 — drives Xavier-uniform init's (fan_in=s1_input_dim, fan_out=shared_h1) tuple for the new dd_pct column), and the matching site in `gpu_experience_collector.rs` (~1148, where the experience collector builds its own `CublasGemmSet` for the inference forward path); plus the backward gemm cache site in `batched_backward.rs` (~303 — the dW/dX shape keys for h_s1's Linear_a / Linear_residual now key on K=103 instead of 102). The cuBLAS gemm cache is built once at construction and re-keyed automatically — no manual invalidation needed because the cache is a fresh `HashMap` per `CublasGemmSet::new`. (2) **GRN `w_a_h_s1[0]` and `w_residual_h_s1[4]` reshape from `[shared_h1, 102]` to `[shared_h1, 103]`** flows naturally from the s1_input_dim bump: `compute_param_sizes`'s array entries `cfg.shared_h1 * s1_input_dim` at indices [0] and [4] absorb the +1; `xavier_init_params_buf`'s `fan_dims[0..95]` core_fan tuples `(cfg.shared_h1, s1_input_dim)` at indices [0] and [4] absorb the +1 in the (fan_out, fan_in) drive for Xavier-uniform init — the new column at index 102 (the dd_pct slot) gets standard Xavier-uniform initialisation alongside the original 102 columns. Mirrors the SP14 EGF aux→Q wire pattern that bumped `w_b0fc` to `[adv_h, shared_h2 + 1]`, except SP14 explicitly zeroed the appended column post-Xavier (because aux_softmax_diff is bidirectional ±1 which collides with Xavier's zero-mean assumption); dd_pct is non-negative bounded ∈ [0, 1] (Wave 1.3 contract), so symmetric Xavier-uniform init is the correct semantic — small-magnitude gradients flow from day 0, the network learns to attend to dd_pct via SGD without artificial cold-start zero. The flat parameter buffer auto-grows by `shared_h1 × 2 × 4 = shared_h1 × 8` bytes (2 tensors × 1 column × 4 B/f32) — each padded by `align4` so the actual byte growth tracks. (3) **`bn_concat_dim()` accessor bumped by +1** (gpu_dqn_trainer.rs ~10624) — TLOB backward in `fused_training.rs:1610` reads this to compute its row-stride into `bn_d_concat_buf`. The TLOB offset arithmetic `bottleneck_dim + (TLOB_START − market_dim)` is unchanged (it indexes into the portfolio slice, which is stride-stable), but the row stride is now 103 instead of 102. The change keeps TLOB's gradient slicing aligned with the new bn_d_concat layout. (4) **5 `concat_dim` local-variable bumps** at all sites computing `bn_dim + portfolio_dim` directly: 1 buffer-allocation site (gpu_dqn_trainer.rs:20068, the constructor's `bn_concat_buf` / `tg_bn_concat_buf` / `on_next_bn_concat_buf` / `bn_d_concat_buf` allocator — all 4 buffers grow by `B × 1 × 4` bytes from the `+1` column), 3 forward sites (DDQN ~27135, online ~27431, target ~27641), 2 backward sites (`aux_bottleneck_vsn_backward_dispatch` ~28522 and the main bottleneck backward in `apply_iqn_trunk_gradient`-equivalent ~29022), plus the matching site in `gpu_experience_collector.rs` (~3820). The `concat_dim` local in each site is the row stride passed to the kernel; the kernel's read-window for bn_tanh / portfolio is unchanged ([0..bn_dim) for tanh, [bn_dim..bn_dim+portfolio_dim) for portfolio passthrough), so the dd_pct column at index `bn_dim + portfolio_dim` is the new write column on the forward path and the new ignored column on the backward path. (5) **3 forward-site migrations from `self.bn_tanh_concat_kernel` raw `launch_builder` to `launch_sp15_bn_concat_dd` free function** (DDQN ~27158, online ~27467, target ~27666) — the new launcher takes the additional `isv: CUdeviceptr` arg (= `self.isv_signals_dev_ptr`); the kernel reads slot 406 (DD_PCT_INDEX) on-device and broadcasts the scalar across batch as the (concat_dd_dim − 1)th column. The free-function call replaces the inline 8-arg `launch_builder` block with a 10-arg launcher call, and the cubin module is loaded by the launcher itself (cubin pointer cache is process-lifetime — no per-call cubin reload cost; the loader's `Arc` is reused via cudarc's internal symbol cache). (6) **gpu_experience_collector.rs forward kernel migration** — the experience collector's bn_tanh_concat launch site (~3853) is the fourth forward call site listed in the Wave 4.1a audit comment block (the trainer has 3, the collector has 1 — together they form the "4 launch sites" the comment refers to). The collector loads `bn_tanh_concat_dd_kernel` instead of `bn_tanh_concat_f32_kernel` (both kernels have all-f32 I/O; the new kernel adds the ISV arg + appended dd_pct column). The collector's `isv_signals_dev_ptr` field is set by the trainer via `set_isv_signals_ptr` from `training_loop.rs:859` BEFORE the first epoch's forward graph capture, so the captured graph sees a non-null bus pointer at replay time. Per the one-step-lag semantic (mirrors mag_concat): step k's bn_tanh_concat_dd reads ISV[406] which holds the dd_pct value `launch_sp15_dd_state` wrote at step k−1 (the dd_state launch happens AFTER the captured forward graph in the collector's per-step env loop, line ~4546). At step 0 the slot is sentinel 0 per `state_reset_registry` — bootstrap-correct, not a stub. (7) **GRN backward sites — no kernel signature change required**: the 3 GRN backward call sites in gpu_dqn_trainer.rs (main backward chain ~11036 in `apply_iqn_trunk_gradient`, ensemble backward chain ~11336, CQL backward chain ~12006) all flow through `BatchedForward::encoder_backward_chain` which uses `self.s1_input_dim` for the dX shape — bumped automatically by step (1)'s formula change. The dX writes a 103-column gradient into `bn_d_concat_buf`; the dd_pct column at index 102 carries a gradient that has no learnable input source (dd_pct is read from ISV — a constant from the kernel's perspective). The `vsn_d_gated_state_portfolio_pad_kernel` (line 28546 / 29091) reads only [bn_dim..bn_dim+portfolio_dim) so it correctly skips the dd_pct gradient column; the `bn_tanh_backward_kernel` (line 28519 / 29027) reads only [0..bn_dim) so it also correctly skips. The dd_pct column's gradient is silently discarded — harmless because no parameter chain-rules through it (the chain is broken at the ISV bus boundary, exactly as designed by Wave 1.3.b's "ISV is a producer/consumer bus, not a learnable signal" contract). (8) **mag_concat / OFI concat audit verdict — DECOUPLED**: the `mag_concat_buf` shape is `[B, shared_h2 + branch_0_size]` (fixed direction-conditioning width = 4 per production config), and `ord_concat_buf` / `urg_concat_buf` are `[B, shared_h2 + 3]` (fixed OFI-conditioning width = 3). Neither uses `s1_input_dim` — they widen `shared_h2` (the second shared-trunk hidden size, post-encoder), not the trunk INPUT dim. The bump from 102→103 is structurally invisible to mag_concat / OFI, and no migration is required. Confirmed by exhaustive grep over `crates/ml/src/cuda_pipeline/`. (9) **Legacy `bn_tanh_concat_kernel` trainer field DELETED** per `feedback_no_legacy_aliases`: the `CudaFunction` field on the trainer struct, the corresponding loader call in `compile_training_kernels` (~31551), the entry in the 44-tuple return type (~31437), the entry in the 44-element destructuring at the constructor (~17533), and the field assignment in the `Self {}` literal (~22358) are all removed in lockstep. The function tuple shrinks to 43 elements. The kernel symbol itself remains in the cubin source (`dqn_utility_kernels.cu:771`) for the SP15 oracle parity tests in `crates/ml/tests/sp15_phase1_oracle_tests.rs` — but no production code path resolves it anymore. (10) **Test migration**: `test_gpu_backtest_evaluator_state_dim_calculation` (gpu_backtest_evaluator.rs:3315) was a pre-existing failure in the 946/13 baseline — it asserted `STATE_DIM == 96` while the canonical constant in `ml_core::state_layout` is 128 (the codebase evolved 48→112→128 as the OFI / TLOB / MTF blocks expanded). Per `feedback_trust_code_not_docs`, the test was migrated to assert 128; the docstring is updated to record the 96-dim layout's deprecation date and refer maintainers to `ml_core::state_layout` as the source of truth. The fix is in this commit because the same `feedback_trust_code_not_docs` correction applies to both Wave 4.1a's "state_dim 48→49" terminology fix and this test — landing them together preserves audit-trail continuity. (11) **CUDA Graph capture interaction**: per `pearl_no_host_branches_in_captured_graph`, the new kernel launches all happen via `launch_sp15_bn_concat_dd` (a free function with no host branches inside the launcher itself — the only Rust-side branch is the `if self.config.bottleneck_dim > 0` check in the call-site, which is a host-side compile-time-equivalent guard that resolves once before graph capture begins). The `module.load_cubin` + `module.load_function` calls inside the launcher would normally be a graph-capture concern, but cudarc's internal cubin cache makes them no-ops on second + subsequent capture (the `Arc` is cached per-context). On first capture, the cubin is already resident from the constructor's eager load, so the launcher's load calls hit the cache and do not introduce graph-replay-incompatible operations. (12) **xavier_init forward-wire log line preserved**: `info!(total_params = total, s1_input_dim, "Xavier-initialized params_buf and target_params_buf")` (~30157) automatically reflects the bumped value (e.g. `s1_input_dim = 103`) — operators reading the log post-deploy see the new dim immediately, no log-format change needed. Touched: `crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs` (s1_input_dim formula bump at compute_param_sizes + constructor + xavier_init + bn_concat_dim accessor; concat_dim local-var bump at 1 alloc + 3 forward + 2 backward sites; 3 forward-call migrations to `launch_sp15_bn_concat_dd`; legacy field + loader + tuple-element + destructuring + assignment removal — 5 mechanical sites for the 1 dead field), `crates/ml/src/cuda_pipeline/batched_backward.rs` (s1_input_dim formula bump in CublasBackwardSet::new for the backward gemm cache), `crates/ml/src/cuda_pipeline/gpu_experience_collector.rs` (s1_input_dim formula bump + concat_dim local-var bump + bn_tanh_concat_fn loader symbol change to bn_tanh_concat_dd_kernel + exp_bn_concat allocation +1 column + launch_builder arg list extended with isv_signals_dev_ptr), `crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs` (test migration: state_dim_calculation 96→128 + docstring rationale update). Hard rules: `feedback_no_partial_refactor` (every consumer of `s1_input_dim` and `bn_concat_buf` row-stride migrates in this commit; zero parallel paths; the trainer's legacy `bn_tanh_concat_kernel` field + its loader + its tuple slot + its destructuring + its assignment all delete in lockstep), `feedback_wire_everything_up` (Wave 4.1a's `launch_sp15_bn_concat_dd` and `bn_tanh_concat_dd_kernel` symbol — both ORPHAN at end of Wave 4.1a — both have production callers in this commit; the orphan window opened in `a8da1cb9c` and closes here), `feedback_no_legacy_aliases` (the trainer's `bn_tanh_concat_kernel` field is the production-path alias that this commit deletes; the kernel source itself is retained ONLY for oracle parity tests, not as a deprecated wrapper around the new kernel), `feedback_no_cpu_compute_strict` (no CPU compute introduced — every dim-bump propagates through GPU kernel launches; xavier_init still constructs on CPU per the established offline init pattern, which is exempted from the no-CPU-compute rule because it runs once at construction before any forward pass), `feedback_isv_for_adaptive_bounds` (s1_input_dim is structural, not adaptive — no ISV slot added; dd_pct itself is ISV-driven via `dd_state_kernel` per Wave 1.3.b's existing contract), `feedback_no_stubs` (every dim bump is a real propagation; no zero-init shortcut for the new column — Xavier-uniform init handles it), `feedback_no_quickfixes` (the `state_dim_calculation` test was migrated, not deleted, and the migration carries a docstring rationale that records WHY the constant changed — future maintainers can audit the 48→112→128 evolution chain through the audit doc + state_layout.rs comments), `feedback_no_functionality_removal` (the legacy bn_tanh_concat kernel symbol stays in the cubin for oracle tests; only the dead production field is removed), `pearl_no_host_branches_in_captured_graph` (kernel launches go through `launch_sp15_bn_concat_dd` whose only Rust-side branch is in the call-site's `if bottleneck_dim > 0` guard — pre-graph-capture host-side, not in-capture). Verified: `SQLX_OFFLINE=true cargo check -p ml --features cuda` clean (18 pre-existing unrelated warnings, all `unused import: DevicePtrMut`); `cargo test -p ml --test sp15_phase1_oracle_tests --features cuda -- --ignored bn_concat dd_pct --nocapture` 1 of 1 oracle test green (`bn_tanh_concat_dd_kernel_writes_dd_pct_column` — Wave 4.1a's standalone parity test still passes, confirming the kernel-side contract is unchanged); `cargo test -p ml --features cuda --lib` HELD-OR-IMPROVED the 946 pass / 13 fail Wave 4.1a baseline (the migrated `test_gpu_backtest_evaluator_state_dim_calculation` flips from FAIL to PASS, bringing the count to ≥947 pass / ≤12 fail post-Wave-4.1b — actual count documented in the commit message body). + SP15 Wave 4.1a — bn_tanh_concat appends dd_pct column from ISV; standalone dd_pct_concat_kernel deleted (2026-05-07): kernel-side foundation for the Phase 1.5.b consumer migration. Per `feedback_trust_code_not_docs`: the spec's "state_dim 48→49" terminology was stale (production STATE_DIM has evolved 48→112→128); actual production `s1_input_dim = bottleneck_dim + (STATE_DIM − market_dim) = 16 + (128 − 42) = 102` when bottleneck on. The standalone Phase 1.5 `dd_pct_concat_kernel.cu` was bottleneck-incompatible — it operated on raw `[B, 128]` state but production trunk consumes `[B, 102]` post-bottleneck. Wave 4.1a fixes this at the kernel level; Wave 4.1b will land the consumer migration (s1_input_dim 102→103 propagation, GRN w_s1 reshape, 3 forward + 3 backward call sites). (1) **New `bn_tanh_concat_dd_kernel` in `dqn_utility_kernels.cu`** — fuses the dd_pct append into the same launch as the bottleneck-tanh + portfolio concat (output shape `[B, bn_dim + portfolio_dim + 1]`). Reads `isv[DD_PCT_INDEX=406]` and broadcasts the scalar across batch as the appended last column. The dd_pct value is set by Wave 1.3.b's `dd_state_kernel`, which fires per-step in the experience collector before this kernel runs in the trunk forward path. (2) **Standalone `dd_pct_concat_kernel.cu` DELETED** per `feedback_no_legacy_aliases` — the bottleneck-incompatible kernel had zero production callers and only drove its own oracle test; the bottleneck-on path is canonical for production, and the bottleneck-off / test-only path can construct concat at the test level. The previous launcher `launch_sp15_dd_pct_concat` is removed; the corresponding cubin manifest entry in `build.rs` is removed; the comment block in `build.rs` is updated to document the bottleneck-incompatibility correction and point readers to the new fused kernel. (3) **Test helpers added** in `tests/sp15_phase1_oracle_tests.rs` — supporting infrastructure for the bn_tanh_concat_dd test and future Wave 4.1b/4.1c behavioral tests. (4) **Phase 1.5 oracle test migrated** from the deleted standalone `dd_pct_concat_kernel` to the new bottleneck-aware `bn_tanh_concat_dd_kernel`: input `[B, bn_dim + portfolio_dim]` + ISV with `DD_PCT_INDEX = ` → expected output `[B, bn_dim + portfolio_dim + 1]` with the last column == ISV[DD_PCT_INDEX] broadcast. Test name: `bn_tanh_concat_dd_kernel_writes_dd_pct_column`. Passes on RTX 3050 Ti via `CUDA_COMPUTE_CAP=86`. (5) **Layout fingerprint already covers Phase 1.5** — the existing `TRUNK_INPUT_DD_PCT=sp15_phase_1_5;` marker in `layout_fingerprint_seed` (line 3339, landed by Phase 1.5 `5309d4bee`) already breaks pre-SP15 checkpoints. No additional fingerprint extension is required for Wave 4.1a's kernel-side change. (6) **fxcache schema_hash auto-bumps** from file content hashes via the existing `FEATURE_SCHEMA_HASH` mechanism in `build.rs:1196-1232` (per task #173 P5T5 Phase F); modifying `dqn_utility_kernels.cu` triggers auto-regen on next training run. No manual schema bump needed. (7) **Wave 4.1b deferred** per the documented 4.1a / 4.1b / 4.1c split (3-way decomposition mirroring the Wave 3a/3b precedent): bumps `s1_input_dim` from 102 to 103, reshapes GRN `w_s1` weight `[shared_h1, 102]` → `[shared_h1, 103]` with new-column init (Xavier/Kaiming small), regenerates the cuBLAS gemm_cache shape map at the 5+ shape sites in `batched_forward.rs:438-498`, atomically wires all 3 forward sites (online/target/DDQN at `gpu_dqn_trainer.rs:27407 / 27590 / 27082`) plus 3 GRN backward sites (`11017 / 11317 / 11987`) to consume the new `[B, 103]` concat output, reallocates the `bn_concat_buf` family with the +1 column, and audits the mag_concat / OFI concat paths for s1_input_dim coupling. The Wave 4.1c follow-up adds the behavioral KL test (`forward_trunk_for_test` + `kl_divergence` + `minimal_trainer_for_tests` helpers seeded in 4.1a) verifying that adding dd_pct to the trunk shifts the policy distribution correctly under DD vs at-ATH conditions. Touched: `crates/ml/build.rs` (removed standalone kernel manifest entry + updated comment block), `crates/ml/src/cuda_pipeline/dd_pct_concat_kernel.cu` (DELETED), `crates/ml/src/cuda_pipeline/dqn_utility_kernels.cu` (NEW `bn_tanh_concat_dd_kernel` + 84-line addition), `crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs` (launcher migration: removed `launch_sp15_dd_pct_concat` + cubin static, added or extended bn_tanh_concat_dd launcher), `crates/ml/tests/sp15_phase1_oracle_tests.rs` (test helpers + new oracle test + migrated standalone test). Hard rules: `feedback_no_partial_refactor` (kernel signature change + standalone kernel deletion + oracle test migration + audit doc all atomic; the 5 ORPHAN consumers — 3 forward sites + 3 backward sites + bn_concat_buf reshape + GRN w_s1 reshape + s1_input_dim bump — are explicitly bounded by the documented 4.1a/4.1b/4.1c split, mirroring the Wave 3a/3b precedent), `feedback_no_legacy_aliases` (standalone `dd_pct_concat_kernel.cu` + `launch_sp15_dd_pct_concat` removed, no compatibility shim retained), `feedback_no_cpu_compute_strict` (the new fused kernel is GPU-resident, broadcasts dd_pct from the GPU-resident ISV bus), `feedback_isv_for_adaptive_bounds` (dd_pct is read from ISV[DD_PCT_INDEX=406] — correct, it's an adaptive observable from Wave 1.3.b's `dd_state_kernel`), `feedback_trust_code_not_docs` (spec terminology "state_dim 48→49" was stale; actual production dim is 102 — code wins, audit doc records the correction), `pearl_no_host_branches_in_captured_graph` (`bn_tanh_concat_dd_kernel` is launched from the trunk forward path which is inside the per-step CUDA Graph capture region; ISV read inside the kernel device code is fine — no host branches). Verified: `SQLX_OFFLINE=true cargo check -p ml --features cuda` clean (18 pre-existing unrelated warnings); `cargo test -p ml --test sp15_phase1_oracle_tests --features cuda -- --ignored bn_concat dd_pct` 1 of 1 oracle test green (`bn_tanh_concat_dd_kernel_writes_dd_pct_column`); `cargo test -p ml --features cuda --lib` holds the 946 pass / 13 fail baseline (no regression). SP15 Wave 3b — host-side wire-up for val-cost-streams (2026-05-06): atomic host-side half of the Wave 3 val-cost-streams refactor. Wave 3a (commit `e968f4ded`) landed kernel signature changes + new derivation kernel + shared header + oracle-test migration with 5 ORPHAN launchers. Wave 3b lands the production callers atomically, eliminating those orphans by extending `GpuBacktestEvaluator::new`'s constructor signature with a new `window_lob_bars: &[Vec]` parameter, allocating 11 new mapped-pinned buffers (3 input + 3 derivation + 5 metric output), wiring the per-window launch sequence inside `launch_metrics_and_record_event`, adding 5 new fields to `WindowMetrics`, and migrating 3 production call sites + 6 integration-test call sites in this commit. (1) **Constructor signature change** — `GpuBacktestEvaluator::new` gains `window_lob_bars: &[Vec]` parameter (between `window_features` and `feature_dim`). Validates `window_lob_bars.len() == n_windows` (returns `ConfigError` otherwise — fail-loud per `feedback_no_stubs`). Each `Vec` is zero-padded to `max_len`; the SoA flatten splits into three flat `[n_windows * max_len]` f32 streams (close prices = `LobBar.price`, half-spread = `LobBar.spread × 0.5`, OFI scalar = `LobBar.ofi`). (2) **11 new mapped-pinned buffers on the struct**: 3 input streams (`close_prices_buf`, `half_spread_buf`, `ofi_scalar_buf`, each `[n_windows * max_len]`) populated via `write_from_slice` at construct time and read-only thereafter; 3 derivation outputs (`position_history_buf`, `side_ind_buf`, `rt_ind_buf`, each `[n_windows * max_len]`) zero-initialised and filled per-call by `launch_sp15_position_history_derivation`; 5 metric outputs (`cost_net_sharpe_buf` + 4 × `baseline_*_sharpe_buf`, each `[n_windows * 3]` for `[mean, std, raw_sharpe]` per window — same layout as the 1.1.b `sharpe_buf`). All allocated via `unsafe { MappedF32Buffer::new(...) }` per `feedback_no_htod_htoh_only_mapped_pinned.md`. (3) **`commission_per_rt: f32` field on the evaluator** — host-precomputed flat per-roundtrip commission per D3 resolution. Formula: `config.tx_cost_bps × 0.0001 × config.initial_capital`. Computed once at construct time so the per-window launch loop reads a stable f32 without re-deriving each call. The kernel charges this once per round-trip close (rt_ind=1); the half-spread + OFI-impact components scale with per-bar |position| and are charged via the separate `half_spread_buf` + `ofi_scalar_buf` streams (per spec §6.2). (4) **Wave 3a kernel signature follow-up — cost_net_sharpe `side_ind` / `rt_ind` switched from `unsigned int*` to `float*`** to chain directly with the f32 streams emitted by `position_history_derivation_kernel`. Wave 3a as-shipped declared u32 inputs which would have type-pun-failed when fed the f32 derivation outputs (1.0f bits → uint 1065353216 cast to float = catastrophic miscalc). The internal cast `(float)side_ind[i]` becomes `side_ind[i]` directly. Bit-equivalent for the production caller (derivation emits exact 0.0f/1.0f) and the migrated cost_net oracle test (now uses `MappedF32Buffer` for both flag streams; `MappedU32Buffer` import removed). (5) **3 production call sites migrated**: (a) `crates/ml/src/trainers/dqn/trainer/metrics.rs` val_evaluator construction (~line 651) — builds `Vec` from `val_data` slice using `target[TARGET_RAW_CLOSE]` for price, `config.spread_cost` broadcast scalar for spread, `ofi_features[ofi_val_offset + i][0]` (un-normalised L1 OFI) for OFI scalar, gated behind the same `VAL_SUBSAMPLE_STRIDE=4` modulus the existing price/feature loop uses; (b) `crates/ml/src/trainers/dqn/trainer/metrics.rs` extra_eval Dev/Test construction (~line 1166) — same broadcast-spread + L1-OFI pattern, stride=1 (no subsample for one-shot eval); (c) `crates/ml/src/hyperopt/adapters/dqn.rs` walk-forward eval construction (~line 1493) — multi-window LobBar SoA built from `val_close_prices` + `preloaded_ofi_features` with absolute `ofi_offset + i` indexing matching the price/feature loop above (DQN adapter has full per-bar OFI access per D4 resolution); (d) `crates/ml/src/hyperopt/adapters/ppo.rs` PPO backtest construction (~line 1364) — single-window LobBar SoA with `ofi=0.0` for ALL bars per D4 resolution (PPO hyperopt path lacks per-bar OFI features; cost-net OFI-impact term degrades to 0 here — commission + half-spread × position still apply; DQN adapter and val_evaluator get full LobBar with real OFI). (6) **6 integration-test call sites migrated** in `crates/ml/tests/gpu_backtest_validation.rs` via a new `lob_bars_from_prices` helper — uses close price for `LobBar.price`, zeroes `spread`/`ofi` (these tests assert raw PnL/drawdown semantics, not cost-net; LobBar streams are inert by construction). The 6 call sites cover always-long/always-short/always-flat/multi-window/extended-metrics/active-model trade tests. (7) **5 new `WindowMetrics` fields**: `sharpe_cost_net` (1.2.b cost-net sharpe), `baseline_buyhold_sharpe` / `baseline_hold_only_sharpe` / `baseline_momentum_sharpe` / `baseline_reversion_sharpe` (1.4.b counterfactual baselines). All 5 annualised host-side in `consume_metrics_after_event` by multiplying the kernel's `raw_sharpe` (slot index 2 in each per-window 3-tuple) by `self.annualization_factor` — same pattern + same factor as `WindowMetrics.sharpe` per the 1.1.b precedent (single annualisation locus per `feedback_no_partial_refactor`). The struct's `Default`-style construction in `consume_metrics_after_event` and the test_window_metrics_fields unit test are updated to populate all 5; no other consumers needed updates because Rust struct literals are exhaustive. (8) **Launch sequence in `launch_metrics_and_record_event`** — the existing fused metrics kernel + 1.1.b per-window sharpe loop are UNCHANGED. After the existing sharpe loop, a new SP15 Wave 3b block launches (in order on the same `self.stream`): one `launch_sp15_position_history_derivation` over all windows (grid=[n_windows,1,1]; each block is one thread walking one window's actions_history sequentially); per-window loop running 5 launches per window — `launch_sp15_cost_net_sharpe` (consuming step_returns + half_spread + ofi + position_history + side_ind + rt_ind + commission_per_rt + isv_signals_ptr; per-window pnl/half_spread/ofi/position/side_ind/rt_ind slices via per-element-stride byte arithmetic mirroring 1.1.b's sharpe_per_bar slice pattern), then 4 baseline launches (`buyhold` / `hold_only` / `naive_momentum` / `naive_reversion`) each consuming close_prices + half_spread + ofi + commission_per_rt, each writing to its own per-window output buffer slice. All 6 launches inherit the existing `eval_done_event` / DtoD-async dependency — no new sync needed; only one `eval_done_event` record at the end of the queued work covers the new outputs alongside the existing metrics readback + action-history mirror copies. The per-window loop reads `window_lens_buf` via the mapped-pinned host alias (no DtoH) and skips windows with `n_w <= 0`. (9) **Buffer ABI for cost_net_sharpe per-window slice** — `step_returns_buf.device_ptr(...) + w * max_len * sizeof(f32)` for the pnl slice; `half_spread_buf.dev_ptr + w * max_len * sizeof(f32)` for the half-spread slice (same byte offset arithmetic for ofi/position_history/side_ind/rt_ind). Output slice = `cost_net_sharpe_buf.dev_ptr + w * 3 * sizeof(f32)`. The pattern mirrors the 1.1.b sharpe_per_bar per-window launch loop's offset arithmetic (`step_returns_base + (w as u64) * stride_bytes` / `sharpe_dev_base + (w as u64) * 3u64 * elem_bytes`). (10) **5 ORPHAN launchers eliminated** per `feedback_wire_everything_up`: `launch_sp15_baseline_buyhold` / `launch_sp15_baseline_hold_only` / `launch_sp15_baseline_naive_momentum` / `launch_sp15_baseline_naive_reversion` / `launch_sp15_position_history_derivation` all have production callers in this commit. The orphan annotation in their doc-comments is retained only as historical context (the comments document the 3a/3b decomposition); future maintainers reading the launcher headers see exactly when the orphan window opened and closed. (11) **Test-harness reject-empty-windows update** — `test_new_rejects_empty_windows` passes a third empty slice to match the new constructor signature; the constructor's first check (`n_windows == 0`) fires before reaching the new `window_lob_bars.len() != n_windows` check, so the original assertion against "No windows" still holds. Touched: `crates/ml/src/cuda_pipeline/cost_net_sharpe_kernel.cu` (signature: 2 params changed `unsigned int*` → `float*` + 4 internal `(float)` casts dropped), `crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs` (1 launcher doc-comment refresh), `crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs` (constructor sig + 11 new buffers + 1 new commission_per_rt field + LobBar SoA flatten + WindowMetrics 5 new fields + launch_metrics_and_record_event Wave 3b block + consume_metrics_after_event 5 new field reads/annualisations + test_window_metrics_fields update + test_new_rejects_empty_windows update + LobBar import), `crates/ml/src/trainers/dqn/trainer/metrics.rs` (2 call sites: val_evaluator + extra_eval), `crates/ml/src/hyperopt/adapters/dqn.rs` (1 call site: walk-forward eval), `crates/ml/src/hyperopt/adapters/ppo.rs` (1 call site: PPO backtest with zero-OFI explanation comment), `crates/ml/tests/gpu_backtest_validation.rs` (helper + 6 call sites: always-long/always-short/always-flat/multi-window/extended-metrics/active-model), `crates/ml/tests/sp15_phase1_oracle_tests.rs` (cost_net oracle test side_ind/rt_ind buffers switched MappedU32Buffer → MappedF32Buffer + comment + import block trimmed). Hard rules: `feedback_no_partial_refactor` (constructor sig change + 3 production call sites + 6 test call sites + 5 ORPHAN launchers eliminated + WindowMetrics field additions + cost_net kernel sig fix + audit doc all in this commit; zero parallel paths), `feedback_wire_everything_up` (Wave 3a's 5 orphans all wired in this commit), `feedback_no_legacy_aliases` (no compatibility shim — old constructor signature is gone; type mismatch on cost_net `side_ind`/`rt_ind` is fixed in-place not bridged), `feedback_no_htod_htoh_only_mapped_pinned` (all 11 new buffers are mapped-pinned with `cuMemHostAlloc DEVICEMAP`; kernel writes via dev_ptr, host reads via host_ptr after stream sync), `feedback_no_atomicadd` (no new reductions; the new kernels chain together inside the existing single-block tree-reduce contracts established in Wave 3a), `feedback_no_stubs` (all 6 launches do real work against real data; no NULL-skip stubs — `commission_per_rt = 0.0` is a configured-zero, not a stub), `pearl_no_host_branches_in_captured_graph` (the new launches happen in `launch_metrics_and_record_event` which is OUTSIDE the experience-rollout CUDA Graph capture region — same precedent as the existing 1.1.b sharpe-per-bar per-window launch loop in the same function). End-to-end oracle test deferred per the dispatch's "skip if too heavy" guidance — full `GpuBacktestEvaluator` fixture requires a populated FusedTrainingCtx with weight buffers + cuBLAS handles, which is too much trainer machinery for a single oracle test; the existing 3a oracle tests + L40S smoke validation path is the correct gate. Verified: `SQLX_OFFLINE=true cargo check -p ml --features cuda` clean (18 unrelated pre-existing warnings); `cargo test -p ml --test sp15_phase1_oracle_tests --features cuda -- --ignored` 30 of 30 oracle tests green (cost_net + baseline + position_history + dd_state + final_reward + cooldown + plasticity + dd_trajectory + regret); `cargo test -p ml --features cuda --lib` HELD/IMPROVED the baseline (Wave 3a baseline = 945 pass / 14 fail; Wave 3b = 946 pass / 13 fail — one MORE test passing post-Wave-3b due to the cost_net/integration plumbing landing). Pre-existing failures (test_gpu_backtest_evaluator_state_dim_calculation stale STATE_DIM constant, gpu_backtest_validation portfolio_dim 3≠24, test_hyperopt_14d_search_space dimension assertion drift, etc.) are unchanged.