diff --git a/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs b/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs index 7277b9842..c91cf3fc1 100644 --- a/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs +++ b/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs @@ -306,6 +306,39 @@ pub struct GpuBacktestEvaluator { /// caller still needs a single-step output. gather_chunk_kernel: CudaFunction, + /// SP15 Wave 5 follow-up (2026-05-07): pre-loaded handle for the + /// `cost_net_sharpe_kernel` (cost-net Sharpe per-window). Originally + /// `launch_sp15_cost_net_sharpe` resolved the symbol via `load_cubin` + /// + `load_function` ON EVERY CALL inside the per-window eval hot loop + /// (`gpu_backtest_evaluator.rs:2877`), which is fragile across CUDA + /// context lifetimes — works in single-pass train-best context (smoke + /// `train-9bcwm` verified), fails in hyperopt-trial child stream + /// context (workflow `train-xggfc` trial 1 failed at `load + /// sp15_baseline_kernels cubin: ILLEGAL_ADDRESS`; after the host-side + /// load corrupted trial 1's context, trials 2-20 cascade-failed at + /// `Fork CUDA stream for trial`). Pre-loading once in `new()` mirrors + /// the `5d63762ab` precedent fix for `bn_tanh_concat_dd_kernel`. + sp15_cost_net_sharpe_kernel: CudaFunction, + /// SP15 Wave 5 follow-up (2026-05-07): pre-loaded handle for + /// `baseline_buyhold_kernel` (counterfactual buyhold baseline). See + /// `sp15_cost_net_sharpe_kernel` docstring for root-cause / precedent. + sp15_baseline_buyhold_kernel: CudaFunction, + /// SP15 Wave 5 follow-up (2026-05-07): pre-loaded handle for + /// `baseline_hold_only_kernel` (counterfactual hold-only baseline). + /// See `sp15_cost_net_sharpe_kernel` docstring for root-cause / + /// precedent. + sp15_baseline_hold_only_kernel: CudaFunction, + /// SP15 Wave 5 follow-up (2026-05-07): pre-loaded handle for + /// `baseline_naive_momentum_kernel` (counterfactual naive-momentum + /// baseline). See `sp15_cost_net_sharpe_kernel` docstring for + /// root-cause / precedent. + sp15_baseline_naive_momentum_kernel: CudaFunction, + /// SP15 Wave 5 follow-up (2026-05-07): pre-loaded handle for + /// `baseline_naive_reversion_kernel` (counterfactual naive-reversion + /// baseline). See `sp15_cost_net_sharpe_kernel` docstring for + /// root-cause / precedent. + sp15_baseline_naive_reversion_kernel: CudaFunction, + // Uploaded data (read-only; persists across the step loop). // MappedF32/I32Buffer (cuMemHostAlloc DEVICEMAP) — host writes via // `host_ptr.write_from_slice(...)`, kernels read via `dev_ptr` after a @@ -771,6 +804,54 @@ impl GpuBacktestEvaluator { .load_function("backtest_state_gather_chunk") .map_err(|e| MLError::ModelError(format!("backtest_state_gather_chunk load: {e}")))?; + // SP15 Wave 5 follow-up (2026-05-07): pre-load the SP15 baseline + + // cost-net cubins ONCE here, in the evaluator's CUDA context, so + // the per-window eval hot loop (`for w in 0..self.n_windows` at + // `~line 2860`) calls `launch_sp15_*` against pre-resolved + // `&CudaFunction` handles instead of resolving the symbols on + // every iteration. The on-demand pattern caused + // `CUDA_ERROR_ILLEGAL_ADDRESS` in hyperopt-trial child stream + // contexts (workflow `train-xggfc`, 2026-05-07): trial 1 failed + // mid-eval with `load sp15_baseline_kernels cubin: ILLEGAL_ADDRESS`, + // and the corrupted context cascaded into trials 2-20 failing at + // `Fork CUDA stream for trial`. Mirrors `5d63762ab` precedent for + // `bn_tanh_concat_dd_kernel`. + let sp15_cost_net_module = context + .load_cubin(super::gpu_dqn_trainer::SP15_COST_NET_SHARPE_CUBIN.to_vec()) + .map_err(|e| MLError::ModelError(format!( + "sp15_cost_net_sharpe module load: {e}" + )))?; + let sp15_cost_net_sharpe_kernel = sp15_cost_net_module + .load_function("cost_net_sharpe_kernel") + .map_err(|e| MLError::ModelError(format!( + "cost_net_sharpe_kernel load: {e}" + )))?; + let sp15_baseline_module = context + .load_cubin(super::gpu_dqn_trainer::SP15_BASELINE_KERNELS_CUBIN.to_vec()) + .map_err(|e| MLError::ModelError(format!( + "sp15_baseline_kernels module load: {e}" + )))?; + let sp15_baseline_buyhold_kernel = sp15_baseline_module + .load_function("baseline_buyhold_kernel") + .map_err(|e| MLError::ModelError(format!( + "baseline_buyhold_kernel load: {e}" + )))?; + let sp15_baseline_hold_only_kernel = sp15_baseline_module + .load_function("baseline_hold_only_kernel") + .map_err(|e| MLError::ModelError(format!( + "baseline_hold_only_kernel load: {e}" + )))?; + let sp15_baseline_naive_momentum_kernel = sp15_baseline_module + .load_function("baseline_naive_momentum_kernel") + .map_err(|e| MLError::ModelError(format!( + "baseline_naive_momentum_kernel load: {e}" + )))?; + let sp15_baseline_naive_reversion_kernel = sp15_baseline_module + .load_function("baseline_naive_reversion_kernel") + .map_err(|e| MLError::ModelError(format!( + "baseline_naive_reversion_kernel load: {e}" + )))?; + // ── Upload read-only data via mapped-pinned (no HtoD, no DtoD) ──── // Direct MappedF32/I32Buffer storage: host writes payload via // `host_ptr.write_from_slice`, kernels later read via `dev_ptr`. @@ -992,6 +1073,14 @@ impl GpuBacktestEvaluator { metrics_kernel, gather_kernel, gather_chunk_kernel, + // SP15 Wave 5 follow-up (2026-05-07): pre-loaded handles for + // SP15 baseline + cost-net launchers, see field-level + // docstrings in the struct definition. + sp15_cost_net_sharpe_kernel, + sp15_baseline_buyhold_kernel, + sp15_baseline_hold_only_kernel, + sp15_baseline_naive_momentum_kernel, + sp15_baseline_naive_reversion_kernel, prices_buf, features_buf, window_lens_buf, @@ -2886,6 +2975,7 @@ impl GpuBacktestEvaluator { commission, isv_ptr_local, cost_net_base + off_three, + &self.sp15_cost_net_sharpe_kernel, )?; // Buyhold baseline (1.4.b). @@ -2897,6 +2987,7 @@ impl GpuBacktestEvaluator { n_w, commission, baseline_buyhold_base + off_three, + &self.sp15_baseline_buyhold_kernel, )?; // Hold-only baseline. super::gpu_dqn_trainer::launch_sp15_baseline_hold_only( @@ -2907,6 +2998,7 @@ impl GpuBacktestEvaluator { n_w, commission, baseline_hold_only_base + off_three, + &self.sp15_baseline_hold_only_kernel, )?; // Naive-momentum baseline. super::gpu_dqn_trainer::launch_sp15_baseline_naive_momentum( @@ -2917,6 +3009,7 @@ impl GpuBacktestEvaluator { n_w, commission, baseline_momentum_base + off_three, + &self.sp15_baseline_naive_momentum_kernel, )?; // Naive-reversion baseline. super::gpu_dqn_trainer::launch_sp15_baseline_naive_reversion( @@ -2927,6 +3020,7 @@ impl GpuBacktestEvaluator { n_w, commission, baseline_reversion_base + off_three, + &self.sp15_baseline_naive_reversion_kernel, )?; } } diff --git a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs index cbefa5100..4ac6bc5c3 100644 --- a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs +++ b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs @@ -783,6 +783,14 @@ pub static SP15_COST_NET_SHARPE_CUBIN: &[u8] = /// `out` MUST point to ≥3 f32 slots: `[mean_pnl_net, std, sharpe_net]`. /// `isv` MUST be the ISV bus (≥443 f32 slots) — slot 407 is read for /// `ofi_lambda`, slot 408 is written with `mean(cost_t)`. +/// +/// SP15 Wave 5 follow-up (2026-05-07): pre-loaded `&CudaFunction` +/// parameter pattern, matches `5d63762ab` precedent for graph-safety + +/// cross-context-stable load. The previous on-demand `load_cubin` + +/// `load_function` per-call pattern caused `CUDA_ERROR_ILLEGAL_ADDRESS` +/// on hyperopt-trial child stream contexts (workflow `train-xggfc`, +/// trial 1 onwards). The kernel handle is now pre-loaded once in +/// `GpuBacktestEvaluator::new()` and passed through on every call. #[allow(clippy::too_many_arguments)] pub fn launch_sp15_cost_net_sharpe( stream: &Arc, @@ -796,21 +804,11 @@ pub fn launch_sp15_cost_net_sharpe( commission_per_rt: f32, isv: cudarc::driver::sys::CUdeviceptr, out: cudarc::driver::sys::CUdeviceptr, + kernel: &CudaFunction, ) -> Result<(), MLError> { - let module = stream - .context() - .load_cubin(SP15_COST_NET_SHARPE_CUBIN.to_vec()) - .map_err(|e| MLError::ModelError(format!( - "load sp15_cost_net_sharpe cubin: {e}" - )))?; - let kernel = module - .load_function("cost_net_sharpe_kernel") - .map_err(|e| MLError::ModelError(format!( - "load cost_net_sharpe_kernel function: {e}" - )))?; unsafe { stream - .launch_builder(&kernel) + .launch_builder(kernel) .arg(&pnl) .arg(&half_spread) .arg(&ofi) @@ -1168,6 +1166,11 @@ pub static SP15_BASELINE_KERNELS_CUBIN: &[u8] = /// of length `n_windows * 3`, with the launcher invoked sequentially per /// window with the per-window slice (mirror of 1.1.b sharpe_per_bar /// per-window launch sequence). +/// +/// SP15 Wave 5 follow-up (2026-05-07): pre-loaded `&CudaFunction` +/// parameter pattern, matches `5d63762ab` precedent for graph-safety + +/// cross-context-stable load. The kernel handle is pre-loaded once in +/// `GpuBacktestEvaluator::new()` and passed through on every call. pub fn launch_sp15_baseline_buyhold( stream: &Arc, prices: cudarc::driver::sys::CUdeviceptr, @@ -1176,21 +1179,11 @@ pub fn launch_sp15_baseline_buyhold( n: i32, commission_per_rt: f32, out: cudarc::driver::sys::CUdeviceptr, + kernel: &CudaFunction, ) -> Result<(), MLError> { - let module = stream - .context() - .load_cubin(SP15_BASELINE_KERNELS_CUBIN.to_vec()) - .map_err(|e| MLError::ModelError(format!( - "load sp15_baseline_kernels cubin: {e}" - )))?; - let kernel = module - .load_function("baseline_buyhold_kernel") - .map_err(|e| MLError::ModelError(format!( - "load baseline_buyhold_kernel function: {e}" - )))?; unsafe { stream - .launch_builder(&kernel) + .launch_builder(kernel) .arg(&prices) .arg(&half_spread) .arg(&ofi) @@ -1215,6 +1208,11 @@ pub fn launch_sp15_baseline_buyhold( /// On a strict no-trade trajectory `mean(pnl)=0` and `std(pnl)=0` so /// the kernel's `(std > 0) ? mean / std : 0` guard emits sharpe=0, which /// is the spec-mandated sentinel for an undefined sharpe. +/// +/// SP15 Wave 5 follow-up (2026-05-07): pre-loaded `&CudaFunction` +/// parameter pattern, matches `5d63762ab` precedent for graph-safety + +/// cross-context-stable load. The kernel handle is pre-loaded once in +/// `GpuBacktestEvaluator::new()` and passed through on every call. pub fn launch_sp15_baseline_hold_only( stream: &Arc, prices: cudarc::driver::sys::CUdeviceptr, @@ -1223,21 +1221,11 @@ pub fn launch_sp15_baseline_hold_only( n: i32, commission_per_rt: f32, out: cudarc::driver::sys::CUdeviceptr, + kernel: &CudaFunction, ) -> Result<(), MLError> { - let module = stream - .context() - .load_cubin(SP15_BASELINE_KERNELS_CUBIN.to_vec()) - .map_err(|e| MLError::ModelError(format!( - "load sp15_baseline_kernels cubin: {e}" - )))?; - let kernel = module - .load_function("baseline_hold_only_kernel") - .map_err(|e| MLError::ModelError(format!( - "load baseline_hold_only_kernel function: {e}" - )))?; unsafe { stream - .launch_builder(&kernel) + .launch_builder(kernel) .arg(&prices) .arg(&half_spread) .arg(&ofi) @@ -1260,6 +1248,11 @@ pub fn launch_sp15_baseline_hold_only( /// counterfactual baseline. Policy: `direction = sign(prices[i] − /// prices[i-1])` with |position|=1. Writes per-window `[mean, std, /// raw_sharpe]` to `out[0..3]`. +/// +/// SP15 Wave 5 follow-up (2026-05-07): pre-loaded `&CudaFunction` +/// parameter pattern, matches `5d63762ab` precedent for graph-safety + +/// cross-context-stable load. The kernel handle is pre-loaded once in +/// `GpuBacktestEvaluator::new()` and passed through on every call. pub fn launch_sp15_baseline_naive_momentum( stream: &Arc, prices: cudarc::driver::sys::CUdeviceptr, @@ -1268,21 +1261,11 @@ pub fn launch_sp15_baseline_naive_momentum( n: i32, commission_per_rt: f32, out: cudarc::driver::sys::CUdeviceptr, + kernel: &CudaFunction, ) -> Result<(), MLError> { - let module = stream - .context() - .load_cubin(SP15_BASELINE_KERNELS_CUBIN.to_vec()) - .map_err(|e| MLError::ModelError(format!( - "load sp15_baseline_kernels cubin: {e}" - )))?; - let kernel = module - .load_function("baseline_naive_momentum_kernel") - .map_err(|e| MLError::ModelError(format!( - "load baseline_naive_momentum_kernel function: {e}" - )))?; unsafe { stream - .launch_builder(&kernel) + .launch_builder(kernel) .arg(&prices) .arg(&half_spread) .arg(&ofi) @@ -1305,6 +1288,10 @@ pub fn launch_sp15_baseline_naive_momentum( /// counterfactual baseline. Policy: `direction = -sign(prices[i] − /// prices[i-1])` with |position|=1 (mirror of naive_momentum). Writes /// per-window `[mean, std, raw_sharpe]` to `out[0..3]`. +/// SP15 Wave 5 follow-up (2026-05-07): pre-loaded `&CudaFunction` +/// parameter pattern, matches `5d63762ab` precedent for graph-safety + +/// cross-context-stable load. The kernel handle is pre-loaded once in +/// `GpuBacktestEvaluator::new()` and passed through on every call. pub fn launch_sp15_baseline_naive_reversion( stream: &Arc, prices: cudarc::driver::sys::CUdeviceptr, @@ -1313,21 +1300,11 @@ pub fn launch_sp15_baseline_naive_reversion( n: i32, commission_per_rt: f32, out: cudarc::driver::sys::CUdeviceptr, + kernel: &CudaFunction, ) -> Result<(), MLError> { - let module = stream - .context() - .load_cubin(SP15_BASELINE_KERNELS_CUBIN.to_vec()) - .map_err(|e| MLError::ModelError(format!( - "load sp15_baseline_kernels cubin: {e}" - )))?; - let kernel = module - .load_function("baseline_naive_reversion_kernel") - .map_err(|e| MLError::ModelError(format!( - "load baseline_naive_reversion_kernel function: {e}" - )))?; unsafe { stream - .launch_builder(&kernel) + .launch_builder(kernel) .arg(&prices) .arg(&half_spread) .arg(&ofi) diff --git a/crates/ml/tests/sp15_phase1_oracle_tests.rs b/crates/ml/tests/sp15_phase1_oracle_tests.rs index 9fe0fe299..6c018589c 100644 --- a/crates/ml/tests/sp15_phase1_oracle_tests.rs +++ b/crates/ml/tests/sp15_phase1_oracle_tests.rs @@ -15,6 +15,7 @@ mod gpu { launch_sp15_baseline_naive_momentum, launch_sp15_baseline_naive_reversion, launch_sp15_cost_net_sharpe, launch_sp15_dd_state, launch_sp15_dd_state_reduce, launch_sp15_position_history_derivation, launch_sp15_sharpe_per_bar, + SP15_BASELINE_KERNELS_CUBIN, SP15_COST_NET_SHARPE_CUBIN, }; use ml::cuda_pipeline::mapped_pinned::{MappedF32Buffer, MappedI32Buffer}; use ml::cuda_pipeline::sp15_isv_slots::{ @@ -184,6 +185,18 @@ mod gpu { let out_buf = unsafe { MappedF32Buffer::new(3) } .expect("alloc MappedF32Buffer for cost-net sharpe output"); + // SP15 Wave 5 follow-up (2026-05-07): the launcher now consumes a + // pre-loaded `&CudaFunction` (graph-safe + cross-context-stable), + // so tests resolve the symbol once up-front rather than relying + // on the launcher itself. + let cost_net_module = stream + .context() + .load_cubin(SP15_COST_NET_SHARPE_CUBIN.to_vec()) + .expect("load sp15_cost_net_sharpe cubin"); + let cost_net_kernel = cost_net_module + .load_function("cost_net_sharpe_kernel") + .expect("load cost_net_sharpe_kernel symbol"); + launch_sp15_cost_net_sharpe( &stream, pnl_buf.dev_ptr, @@ -196,6 +209,7 @@ mod gpu { /* commission_per_rt = */ 1.50, isv_buf.dev_ptr, out_buf.dev_ptr, + &cost_net_kernel, ) .expect("launch cost_net_sharpe_kernel"); stream @@ -585,6 +599,15 @@ mod gpu { .expect("alloc MappedF32Buffer for baseline output [mean, std, raw_sharpe]"); out_buf.write_from_slice(&[0.0f32; 3]); + // SP15 Wave 5 follow-up (2026-05-07): pre-load kernel handle. + let baseline_module = stream + .context() + .load_cubin(SP15_BASELINE_KERNELS_CUBIN.to_vec()) + .expect("load sp15_baseline_kernels cubin"); + let buyhold_kernel = baseline_module + .load_function("baseline_buyhold_kernel") + .expect("load baseline_buyhold_kernel symbol"); + launch_sp15_baseline_buyhold( &stream, prices_buf.dev_ptr, @@ -593,6 +616,7 @@ mod gpu { n as i32, /* commission_per_rt = */ 0.0, out_buf.dev_ptr, + &buyhold_kernel, ) .expect("launch baseline_buyhold_kernel"); stream @@ -645,6 +669,15 @@ mod gpu { .expect("alloc MappedF32Buffer for baseline output"); out_buf.write_from_slice(&[0.0f32; 3]); + // SP15 Wave 5 follow-up (2026-05-07): pre-load kernel handle. + let baseline_module = stream + .context() + .load_cubin(SP15_BASELINE_KERNELS_CUBIN.to_vec()) + .expect("load sp15_baseline_kernels cubin"); + let hold_only_kernel = baseline_module + .load_function("baseline_hold_only_kernel") + .expect("load baseline_hold_only_kernel symbol"); + launch_sp15_baseline_hold_only( &stream, prices_buf.dev_ptr, @@ -653,6 +686,7 @@ mod gpu { n as i32, /* commission_per_rt = */ 0.0, out_buf.dev_ptr, + &hold_only_kernel, ) .expect("launch baseline_hold_only_kernel"); stream @@ -720,6 +754,18 @@ mod gpu { .expect("alloc MappedF32Buffer for reversion output"); reversion_out.write_from_slice(&[0.0f32; 3]); + // SP15 Wave 5 follow-up (2026-05-07): pre-load kernel handles. + let baseline_module = stream + .context() + .load_cubin(SP15_BASELINE_KERNELS_CUBIN.to_vec()) + .expect("load sp15_baseline_kernels cubin"); + let momentum_kernel = baseline_module + .load_function("baseline_naive_momentum_kernel") + .expect("load baseline_naive_momentum_kernel symbol"); + let reversion_kernel = baseline_module + .load_function("baseline_naive_reversion_kernel") + .expect("load baseline_naive_reversion_kernel symbol"); + launch_sp15_baseline_naive_momentum( &stream, prices_buf.dev_ptr, @@ -728,6 +774,7 @@ mod gpu { n as i32, 0.0, momentum_out.dev_ptr, + &momentum_kernel, ) .expect("launch baseline_naive_momentum_kernel"); launch_sp15_baseline_naive_reversion( @@ -738,6 +785,7 @@ mod gpu { n as i32, 0.0, reversion_out.dev_ptr, + &reversion_kernel, ) .expect("launch baseline_naive_reversion_kernel"); stream diff --git a/docs/dqn-wire-up-audit.md b/docs/dqn-wire-up-audit.md index 7a8571c28..0b3dc09ac 100644 --- a/docs/dqn-wire-up-audit.md +++ b/docs/dqn-wire-up-audit.md @@ -2,6 +2,8 @@ **Status:** Populated during Plan 1 Task 6 (A.5 orphan audit). Updated on every commit per Invariant 7. +SP15 Wave 5 follow-up — pre-load sp15_baseline + cost_net cubins (2026-05-07): root-caused L40S workflow `train-xggfc` (commit `5d63762ab`) hyperopt-trial CUDA failure to the same per-call `load_cubin` + `load_function` anti-pattern that Wave 4.1b's bn_tanh_concat_dd-fix (commit `5d63762ab`, see entry below) addressed for the trainer's bottleneck-concat path. The 5 SP15 evaluation launchers in `crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs` (`launch_sp15_cost_net_sharpe` at ~787, `launch_sp15_baseline_buyhold` at ~1171, `launch_sp15_baseline_hold_only` at ~1218, `launch_sp15_baseline_naive_momentum` at ~1263, `launch_sp15_baseline_naive_reversion` at ~1308) were resolving `cost_net_sharpe_kernel` / `baseline_*_kernel` symbols via `stream.context().load_cubin(...)` + `module.load_function(...)` ON EVERY CALL inside `GpuBacktestEvaluator`'s per-window eval hot loop (`gpu_backtest_evaluator.rs:2877..2922`, `for w in 0..self.n_windows`). Pattern works in single-pass train-best context (smoke `train-9bcwm` at `5d63762ab` ran the SAME launcher chain successfully through 2 folds × 5 epochs of train-best evaluation), but fails when re-entered from a hyperopt trial's child stream context: `train-xggfc` ran 8 hyperopt search epochs successfully, then started 20 hyperopt trial trainings; trial 1 failed at `load sp15_baseline_kernels cubin: DriverError(CUDA_ERROR_ILLEGAL_ADDRESS, "an illegal memory access was encountered")`; after trial 1's CUDA context was poisoned, trials 2-20 cascade-failed at `Fork CUDA stream for trial: ILLEGAL_ADDRESS`. Fix (atomic, mirrors `5d63762ab` precedent): added 5 `CudaFunction` fields on `GpuBacktestEvaluator` (`sp15_cost_net_sharpe_kernel`, `sp15_baseline_buyhold_kernel`, `sp15_baseline_hold_only_kernel`, `sp15_baseline_naive_momentum_kernel`, `sp15_baseline_naive_reversion_kernel`); pre-loaded both `SP15_COST_NET_SHARPE_CUBIN` (1 function) and `SP15_BASELINE_KERNELS_CUBIN` (4 functions) once in `GpuBacktestEvaluator::new()` alongside the existing `env_module` / `metrics_module` / `gather_module` loads; changed all 5 launcher signatures in `gpu_dqn_trainer.rs` to take `&CudaFunction` as a trailing parameter (no more per-call resolution); updated all 5 call sites in `gpu_backtest_evaluator.rs:2877..2922` to pass `&self.sp15_*_kernel`; updated 4 oracle test call sites in `crates/ml/tests/sp15_phase1_oracle_tests.rs` (cost_net + 3 baseline tests covering all 4 baseline kernels) to pre-load the cubin and pass the kernel handle. Both `SP15_BASELINE_KERNELS_CUBIN` and `SP15_COST_NET_SHARPE_CUBIN` were already `pub static` so no visibility promotion needed. Verification: `SQLX_OFFLINE=true CUDA_COMPUTE_CAP=86 cargo check -p ml --tests --all-targets` clean (dev profile, warnings only — no errors). Per `pearl_no_host_branches_in_captured_graph` (this is the eval-context analogue: even outside graph capture, host-side `cuModuleLoadData` is fragile across CUDA context lifetimes when the trial-level context fork hasn't fully promoted the parent's loaded modules), `feedback_no_partial_refactor`, `feedback_wire_everything_up`. + SP15 Wave 4.1b OOB-followup — pre-load bn_tanh_concat_dd_kernel (2026-05-07): root-caused the L40S smoke `train-vg5f7` (commit `bfc3ffa9d`) capture-mode SEGV to a hot-path `cuModuleLoadData` + `cuModuleGetFunction` inside `submit_forward_ops_main`'s captured `forward` child. Wave 4.1b (eb9515e41) deleted the legacy `bn_tanh_concat_kernel: CudaFunction` field from `GpuDqnTrainer` and migrated the 3 production trainer call sites (online forward at `launch_cublas_forward` ~27755, target forward at ~27949, DDQN argmax at `submit_forward_ops_ddqn` ~27418) to a `launch_sp15_bn_concat_dd` free function that resolved `bn_tanh_concat_dd_kernel` via `stream.context().load_cubin(...)` + `module.load_function(...)` ON EVERY CALL. The collector's matching call site (gpu_experience_collector.rs:4127) already pre-loaded into `bn_tanh_concat_fn` field at construction — Wave 4.1b's mistake was asymmetric: it kept the collector's pre-load pattern but introduced an on-demand loader for the trainer's three captured-graph call sites. Inside a `cuStreamBeginCapture` region, those CUDA driver API calls are NOT capturable (they allocate memory and mutate driver state), and the resulting host-side state corruption surfaced as a SEGV at exit 139 before `CAPTURE_PHASE_FORWARD_DONE` could print. Per `pearl_no_host_branches_in_captured_graph`. Fix: re-add `bn_tanh_concat_dd_kernel: CudaFunction` field on `GpuDqnTrainer`, pre-load it in `compile_training_kernels` from the same `module` as `bn_tanh_bw` / `bn_bias_grad` (forward-child captured replay group, same `DQN_UTILITY_CUBIN`), wire through ctor + the 43->44 utility-kernel return tuple, and change `launch_sp15_bn_concat_dd` to take `&CudaFunction` as a parameter (no more per-call resolution). All 3 trainer call sites now pass `&self.bn_tanh_concat_dd_kernel`. Promoted `DQN_UTILITY_CUBIN` from `pub(crate)` -> `pub` so the SP15 oracle tests + Wave 4.1c behavioral KL test (which used the launcher's old free-function pattern) can pre-load the symbol once in their setup blocks. Verification: `cargo check -p ml --tests` clean (dev + release); `cargo test -p ml --test sp15_phase1_oracle_tests -- --ignored` 36/36 pass including the dd_pct trunk-input KL test that exercises two end-to-end forwards through the modified launcher. SP15 Wave 5 SEGV diagnostic — checkpoints inside capture_training_graph (2026-05-07): smoke `train-fp7xx` (commit `a8d6c3304`) printed all 16 Phase-6/7/8 checkpoints cleanly through `STEP0_PHASE_PER_PRIORITY_DONE`. The `CAPTURE_DONE` checkpoint never printed, exit changed from 139 (SEGV) to 143 (SIGTERM) — process was killed externally ~40s after PER_PRIORITY_DONE, suggesting `capture_training_graph()` either hangs or runs too long. Added 12 sub-checkpoints across capture_training_graph: `CAPTURE_PHASE_BEGIN`, `CAPTURE_PHASE_PER_SAMPLE_DONE`, `CAPTURE_PHASE_COUNTERS_DONE`, `CAPTURE_PHASE_SPECTRAL_DONE`, `CAPTURE_PHASE_FORWARD_DONE`, `CAPTURE_PHASE_DDQN_DONE`, `CAPTURE_PHASE_AUX_DONE`, `CAPTURE_PHASE_POST_AUX_DONE`, `CAPTURE_PHASE_ADAM_GRAD_DONE`, `CAPTURE_PHASE_ADAM_UPDATE_DONE`, `CAPTURE_PHASE_MAINTENANCE_DONE`, `CAPTURE_PHASE_IQL_MODULATE_DONE`, `CAPTURE_PHASE_PER_PRIORITY_DONE`, `CAPTURE_PHASE_CHILDREN_STORED`, `CAPTURE_PHASE_PARENT_COMPOSED`. Will pinpoint which child capture or compose-step hangs/crashes. Diagnostic-only — no behavior change.