diff --git a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs index ec3fa945b..49cb59797 100644 --- a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs +++ b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs @@ -554,6 +554,27 @@ static UPDATE_ADAPTIVE_CLIP_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR" /// invariant `targets[0] = 0.0` exactly as the deleted host post-loop did. static CALIBRATE_HOMEOSTATIC_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/calibrate_homeostatic_kernel.cubin")); +/// One-shot diagnostic: stash mapped-pinned host pointers + lengths for the +/// raw fxcache `features_raw_cuda` and `targets_raw_cuda` buffers so the +/// `DIAG_AUX_LABEL` block in `aux_heads_forward` can read column 0 of the +/// source feature buffer and column 2 of the targets buffer without holding a +/// reference to the owning `DQNTrainer`. +/// +/// Set ONCE by `init_gpu_raw_buffers_from_slices` after the mapped-pinned +/// pages are allocated; addresses are stable for the buffer lifetime (the +/// `DQNTrainer` outlives every `aux_heads_forward` call). Read once at the +/// first ungraphed step 0 invocation of the kernel block at line ~12952. +/// +/// Tuple layout: `(features_host_ptr_as_usize, features_len_f32, +/// targets_host_ptr_as_usize, targets_len_f32)`. `*mut f32` is not `Send` so +/// we pack as `usize` and recover the pointer via cast at the read site +/// (mapped-pinned addresses are valid on any thread holding the CUDA context). +/// +/// **Removal gate**: delete this `OnceLock` together with the `DIAG_AUX_LABEL` +/// block once the `label_scale=5481` leak source is identified and fixed. +pub(crate) static DIAG_AUX_LABEL_SOURCE_PTRS: + std::sync::OnceLock<(usize, usize, usize, usize)> = std::sync::OnceLock::new(); + /// Plan 4 Task 1B-ii: per-group VSN MLP hidden dim. `Linear_1[g]` is /// `[VSN_HIDDEN_DIM, group_dim_g]`, `Linear_2[g]` is `[1, VSN_HIDDEN_DIM]`. /// 16 chosen as a tractable expansion that fits comfortably against the @@ -12956,6 +12977,176 @@ impl GpuDqnTrainer { scratch_dev, SCRATCH_IDX_LABEL_SCALE as i32, )?; + + // ── DIAG_AUX_LABEL: one-shot disambiguation of label_scale leak ── + // + // Production training run `train-multi-seed-bn42w` (commit 29b1d34c6, + // current HEAD) hit `label_scale=5481` at epoch 4 vs the local-smoke + // `multi_fold_convergence` baseline of ~25 at the SAME commit. The + // 5-layer data-loading defense (Fix 25) + DBN spread filter (Fix 26) + // + target stride/column drift cleanup (HEAD commit) are all in + // place — yet column 0 of `next_states_buf` still sees raw-price- + // magnitude values in production but not in the smoke path. + // + // This block fires ONCE per process (static `AtomicBool`) on the + // first invocation of `aux_heads_forward`. Step 0 of the fused + // training pipeline runs UNGRAPHED (see `fused_training.rs:: + // run_full_step` Phase 2 — `submit_forward_ops_main` is invoked + // ungraphed before `capture_training_graph`), so `clone_dtoh` here + // is safe and the captured replay never re-enters this branch + // (`DUMPED` is `true` on every subsequent call). + // + // Three numbers will pin the leak: + // 1. `aux_buf_mean_abs ~ 0.8` → z-norm features (correct) + // 2. `aux_buf_mean_abs ~ 5500` → raw_close leaked + // 3. `aux_buf_mean_abs ~ 25` → partial leak (mixed bars) + // 4. `features_col0_mean_abs ~ 5500` → fxcache writer didn't normalize + // 5. `targets_col2_mean_abs ~ 5500` only → only production reads col 2; + // smoke harness avoids it + // + // Removal gate: delete this block + the `DIAG_AUX_LABEL_SOURCE_PTRS` + // OnceLock once the leak source is identified and fixed. + { + use std::sync::atomic::{AtomicBool, Ordering}; + static DUMPED: AtomicBool = AtomicBool::new(false); + if !DUMPED.swap(true, Ordering::Relaxed) { + let batch = b; + + // Helper closures: stats over Vec. + let stats = |v: &[f32]| -> (f64, f32, f32) { + let mut mean_abs = 0.0_f64; + let mut min_v = f32::INFINITY; + let mut max_v = f32::NEG_INFINITY; + for &x in v { + mean_abs += x.abs() as f64; + if x < min_v { min_v = x; } + if x > max_v { max_v = x; } + } + let mean_abs = if !v.is_empty() { + mean_abs / v.len() as f64 + } else { + 0.0 + }; + (mean_abs, min_v, max_v) + }; + + // (1) aux_nb_label_buf [B] — what the label_scale kernel just consumed. + match self.stream.clone_dtoh(&self.aux_nb_label_buf) { + Ok(host_aux) => { + let (m, lo, hi) = stats(&host_aux); + tracing::warn!( + target: "DIAG_AUX_LABEL", + "(1) aux_nb_label_buf [B={batch}]: mean_abs={m:.4e}, min={lo:.4e}, max={hi:.4e}" + ); + + // (2) next_states_buf col 0 sample — what strided_gather READ + // to populate aux_nb_label_buf. Stride = STATE_DIM_PADDED. + match self.stream.clone_dtoh(&self.next_states_buf) { + Ok(host_next) => { + let n_samples = host_next.len() / state_dim_padded; + let mut col0: Vec = Vec::with_capacity(n_samples); + for i in 0..n_samples { + col0.push(host_next[i * state_dim_padded]); + } + let (mc, loc, hic) = stats(&col0); + let head: Vec = col0.iter().take(10).copied().collect(); + tracing::warn!( + target: "DIAG_AUX_LABEL", + "(2) next_states_buf col 0 [N={n_samples}]: mean_abs={mc:.4e}, min={loc:.4e}, max={hic:.4e}, first10={head:?}" + ); + } + Err(e) => tracing::warn!( + target: "DIAG_AUX_LABEL", + "(2) next_states_buf clone_dtoh failed: {e}" + ), + } + } + Err(e) => tracing::warn!( + target: "DIAG_AUX_LABEL", + "(1) aux_nb_label_buf clone_dtoh failed: {e}" + ), + } + + // (3) features_raw_cuda col 0 — the SOURCE buffer the + // `gpu_experience_collector::state_gather` kernel reads to + // populate `next_states_buf`. Mapped-pinned, so direct + // host-pointer arithmetic is safe (no DtoH copy needed). + // (4) targets_raw_cuda col 2 (TARGET_RAW_CLOSE) — the raw + // close-price column. If only this carries raw magnitude but + // (3) is z-normed, the leak path is via a target-buffer + // consumer, not via the feature pipeline. + if let Some(&(feat_h_ptr, feat_len, tgt_h_ptr, tgt_len)) = + DIAG_AUX_LABEL_SOURCE_PTRS.get() + { + const FEAT_DIM: usize = 42; // mirrors fxcache::FEAT_DIM + const TARGET_DIM: usize = 6; // mirrors fxcache::TARGET_DIM + const TARGET_RAW_CLOSE: usize = 2; // fxcache::TARGET_RAW_CLOSE + + // (3) features_raw_cuda col 0 sample. + let n_bars = feat_len / FEAT_DIM; + let mut feat_col0: Vec = Vec::with_capacity(n_bars); + let feat_ptr = feat_h_ptr as *const f32; + let mut feat_max_abs = 0.0_f32; + let mut feat_sum_abs = 0.0_f64; + for bar in 0..n_bars { + // SAFETY: feat_ptr aliases mapped-pinned pages of + // `MappedF32Buffer::host_ptr` for `feat_len` f32s. + // The buffer's lifetime is tied to `DQNTrainer` which + // outlives every `aux_heads_forward` call. + let v = unsafe { *feat_ptr.add(bar * FEAT_DIM) }; + if bar < 10 { feat_col0.push(v); } + let av = v.abs(); + if av > feat_max_abs { feat_max_abs = av; } + feat_sum_abs += av as f64; + } + let feat_mean_abs = if n_bars > 0 { + feat_sum_abs / n_bars as f64 + } else { + 0.0 + }; + tracing::warn!( + target: "DIAG_AUX_LABEL", + "(3) features_raw_cuda col 0 [N_bars={n_bars}]: mean_abs={feat_mean_abs:.4e}, max_abs={feat_max_abs:.4e}, first10={feat_col0:?}" + ); + + // (4) targets_raw_cuda col 2 (raw_close) sample. + let n_tgt_bars = tgt_len / TARGET_DIM; + let mut tgt_col2: Vec = Vec::with_capacity(n_tgt_bars.min(10)); + let tgt_ptr = tgt_h_ptr as *const f32; + let mut tgt_max_abs = 0.0_f32; + let mut tgt_sum_abs = 0.0_f64; + for bar in 0..n_tgt_bars { + // SAFETY: same lifetime + alias guarantees as feat_ptr above. + let v = unsafe { *tgt_ptr.add(bar * TARGET_DIM + TARGET_RAW_CLOSE) }; + if bar < 10 { tgt_col2.push(v); } + let av = v.abs(); + if av > tgt_max_abs { tgt_max_abs = av; } + tgt_sum_abs += av as f64; + } + let tgt_mean_abs = if n_tgt_bars > 0 { + tgt_sum_abs / n_tgt_bars as f64 + } else { + 0.0 + }; + tracing::warn!( + target: "DIAG_AUX_LABEL", + "(4) targets_raw_cuda col 2 (raw_close) [N_bars={n_tgt_bars}]: mean_abs={tgt_mean_abs:.4e}, max_abs={tgt_max_abs:.4e}, first10={tgt_col2:?}" + ); + } else { + tracing::warn!( + target: "DIAG_AUX_LABEL", + "(3)/(4) DIAG_AUX_LABEL_SOURCE_PTRS not set — init_gpu_raw_buffers_from_slices did not run before first aux_heads_forward (smoke path skipping fxcache?)" + ); + } + + // (5) Verdict line — interpretation reference for the operator. + tracing::warn!( + target: "DIAG_AUX_LABEL", + "verdict: aux_buf mean_abs ~ 0.8 → z-norm features (correct); ~5500 → raw_close leaked into next_states; ~25 → partial leak. features_col0 ~5500 → writer didn't normalize. targets_col2 ~5500 alone → only production consumer reads col 2; smoke doesn't." + ); + } + } + { use crate::cuda_pipeline::sp4_wiener_ema::launch_apply_pearls; // GPU Pearls A+D: stream-ordered with the producer kernel, diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index 0072888b4..b96ec0c04 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -1337,6 +1337,28 @@ impl DQNTrainer { features_buf.write_from_slice(&flat_features); self.features_raw_cuda = Some(features_buf); + // ── DIAG_AUX_LABEL one-shot diagnostic (gpu_dqn_trainer.rs) ── + // Stash the mapped-pinned host pointers + lengths so the first-batch + // diagnostic block in `aux_heads_forward` can sample column 0 of + // `features_raw_cuda` and column 2 of `targets_raw_cuda` without + // holding a reference to this `DQNTrainer`. Mapped-pinned host_ptrs + // are stable for the buffer lifetime; the `DQNTrainer` outlives every + // training step. `OnceLock::set` returns `Err` on second-or-later + // attempts (e.g. multi-fold reinit), which is harmless here — the + // first set carries the correct addresses for this process. Removal + // gate matches the consumer block at `gpu_dqn_trainer.rs:~13000`. + { + let feat_ref = self.features_raw_cuda.as_ref() + .expect("features_raw_cuda just assigned above"); + let tgt_ref = self.targets_raw_cuda.as_ref() + .expect("targets_raw_cuda assigned earlier in this fn"); + let _ = crate::cuda_pipeline::gpu_dqn_trainer::DIAG_AUX_LABEL_SOURCE_PTRS + .set(( + feat_ref.host_ptr as usize, feat_ref.len, + tgt_ref.host_ptr as usize, tgt_ref.len, + )); + } + tracing::info!("CUDA raw buffers uploaded from slices: {} bars x 42+6", num_bars); Ok(()) } diff --git a/docs/dqn-gpu-hot-path-audit.md b/docs/dqn-gpu-hot-path-audit.md index 6a6284337..007d429bd 100644 --- a/docs/dqn-gpu-hot-path-audit.md +++ b/docs/dqn-gpu-hot-path-audit.md @@ -384,3 +384,33 @@ Augments the one-shot DIAG_BUG2 block at `training_loop.rs:1785` with two narrow **Hot-path purity.** Diagnostic only. No kernel changes, no buffer-shape changes. Pre-graph-capture, single-shot. Safe to leave in until Bug 2 is fixed; remove with the eventual `pearl_safe_log_return_extreme_clamp` (or equivalent) commit. Per `feedback_no_hiding` no suppression markers added. **Cleanup gate.** Once Bug 2 is fixed (likely a tighter clamp inside `safe_log_return` or in `NormStats::normalize_batch` for log-return columns), delete the entire DIAG_BUG2 + DIAG_BUG2_v2 block at `training_loop.rs:1785-1908` in the same commit. Tracked under task #294. + +### Fix 28 — DIAG_AUX_LABEL one-shot instrumentation (label_scale=5481 leak) (2026-05-02) + +**Production blocker.** The cancelled 50-epoch run `train-multi-seed-bn42w` at HEAD `29b1d34c6` (target stride/column drift fix already applied) hit `label_scale=5481` at epoch 4. The local smoke `multi_fold_convergence` test on the *same commit* lands at ~25 — three orders of magnitude lower, with all five Fix-25 data-loading defenses active and the Fix-26 DBN spread filter live. Something between the smoke harness and the production training-data PVC is still leaking raw-price magnitude into the kernel that produces `label_scale`. + +The kernel in question is `aux_heads_kernel.cu::label_scale_ema` (launched at `gpu_dqn_trainer.rs::aux_heads_forward` Step 2b, ~line 12973). It computes `mean(|aux_nb_label_buf|)` over the batch and feeds that into the SP4 Pearls A+D Wiener-EMA producer at scratch slot 43 → ISV[`AUX_LABEL_SCALE_EMA_INDEX`]. `aux_nb_label_buf` is a dedicated `[B] f32` scratch populated each step by `strided_gather` from column 0 of `next_states_buf` (= `log_return` per `pipeline.rs`'s feature contract). z-normalised log-returns sit at ~0.8 mean-abs; raw close prices on the ES front month sit at ~5500. A fivefold-thousand discrepancy between smoke and production at the same compiled binary is impossible unless the SOURCE data fed to `state_gather` is different. + +**Four hypotheses to disambiguate** (the `(1)..(4)` numbers below match the diagnostic output lines): + +| H# | Path | Test | Expected | +|----|------|------|----------| +| H1 | aux_buf populated correctly, downstream kernel misreads | (1) aux_buf mean_abs | ~0.8 → kernel-side bug; ~5500 → no | +| H2 | next_states_buf col 0 was written with raw prices | (2) col0 mean_abs | ~5500 confirms | +| H3 | features_raw_cuda col 0 (the SOURCE) carries raw prices | (3) feat col0 mean_abs | ~5500 confirms; ~0.8 → no | +| H4 | targets_raw_cuda col 2 (raw_close) leaked into a consumer | (4) tgt col2 mean_abs | always ~5500 (it IS raw_close); informs whether a consumer wrongly read it | + +Direct stats comparison across the four numbers will pin the leak path in a single epoch-1 batch. + +**Implementation.** One-shot `static AtomicBool DUMPED` block at `gpu_dqn_trainer.rs::aux_heads_forward` immediately after the `launch_label_scale_ema` call (~line 12980), guarded so only the first ungraphed step-0 invocation reads the buffers. Step 0 of `fused_training.rs::run_full_step` runs `submit_forward_ops_main` UNGRAPHED before `capture_training_graph` records the operation set; the captured graph never re-enters the diagnostic branch (`DUMPED` is `true` on the first replay), so subsequent training steps pay zero cost. `clone_dtoh` is safe in step 0 because no graph capture is active. Tracing target `DIAG_AUX_LABEL`. Verdict line at the end documents the interpretation rules. + +**Buffer-access wiring.** `aux_nb_label_buf` and `next_states_buf` live on `GpuDqnTrainer` and are read directly via `self.stream.clone_dtoh(&...)`. `features_raw_cuda` and `targets_raw_cuda` live on `DQNTrainer` (mapped-pinned `MappedF32Buffer`); their addresses are passed through a process-static `OnceLock<(usize, usize, usize, usize)>` (`DIAG_AUX_LABEL_SOURCE_PTRS`) populated by `init_gpu_raw_buffers_from_slices` after the mapped-pinned pages are allocated. Mapped-pinned host_ptrs are stable for the buffer lifetime, the `DQNTrainer` outlives every `aux_heads_forward` call, and no DtoH copy is needed for these two reads — the kernel-visible bytes ARE the host bytes. + +**Hot-path purity.** Diagnostic only. No kernel changes, no buffer-shape changes, no struct field changes. Pre-graph-capture (step 0 ungraphed), single-shot. The static `OnceLock` set is a one-time write and the `AtomicBool::swap` is one relaxed-ordered RMW — both invisible to the captured graph. + +**Removal gate.** Once the leak source is identified and fixed, delete in the same commit: +- The `DIAG_AUX_LABEL` block at `gpu_dqn_trainer.rs::aux_heads_forward` (~line 12980). +- The `DIAG_AUX_LABEL_SOURCE_PTRS` `OnceLock` static at `gpu_dqn_trainer.rs:~556`. +- The `OnceLock::set` call at `training_loop.rs::init_gpu_raw_buffers_from_slices`. + +**Validation.** `SQLX_OFFLINE=true cargo check -p ml --offline` clean; `cargo build -p ml --release --offline --features cuda` clean. Diagnostic fires on the next L40S smoke run or production training step — no GPU spend in this commit.