From 129b7fd3d0cd71661a5d8a3e3ee5d6cdb0e49a63 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sat, 2 May 2026 13:48:59 +0200 Subject: [PATCH] =?UTF-8?q?diag(dqn):=20DIAG=5FBUG2=5Fv2=20=E2=80=94=20out?= =?UTF-8?q?lier=20disambiguation=20H1=20vs=20H2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Augments the existing DIAG_BUG2 one-shot block with two narrower scans that disambiguate where state[0] heavy-tail (mean=6e-3, std=570, mean_abs=20 over 3200 samples on smoke-test-xb78r) actually originates. H1 (data-source corruption): scan features_buf.host_ptr for bars where |feat[bar*42 + 0]| > 100. host_ptr aliases the same physical memory the kernel reads (mapped-pinned) — direct read of the data state_gather sees. Reports up to 10 outlier bar indices + mean/std/ max_abs over up to 2M bars. H2 (kernel/gather bug): scan gpu_batch.states/next_states (already DtoH-downloaded) for sample indices where |state[i*sd + 0]| > 100. Reports up to 10 outlier sample indices. Verdict line logged: feat-outliers nonempty → H1; state-outliers nonempty AND feat-outliers empty → H2. Context: smoke-test-xb78r at HEAD cba9f25ed went through DBN fallback (train_baseline_rl.rs:611, "Loaded N bars (ts to ts)" message) — no fxcache file exists on training-data PVC. DBN-fallback applies z-score normalization (line 633) which has small stddev for log-return columns; a single bar with bar.open ≈ 0 produces ln(ratio) = -30 → normalized = -30000. Few bars in 700K could carry this signature. Triggers once via static AtomicBool. Pre-graph-capture, no perf impact. Cleanup gate: remove DIAG_BUG2 block once Bug 2 root cause is fixed (likely a tighter clamp inside safe_log_return for log-return columns). Co-Authored-By: Claude Opus 4.7 (1M context) --- .../src/trainers/dqn/trainer/training_loop.rs | 67 +++++++++++++++++++ docs/dqn-gpu-hot-path-audit.md | 17 +++++ 2 files changed, 84 insertions(+) diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index 0adb43488..5a08a4829 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -1841,6 +1841,73 @@ impl DQNTrainer { target: "DIAG_BUG2", "interpretation: mean_abs ~0.001 = normalized log_return ✓ | mean_abs ~5000 = raw price ✗ (Bug 2 confirmed)" ); + + // ── DIAG_BUG2_v2: outlier disambiguation ────────────── + // H1: fxcache feat[0] has raw-price values for some bars + // H2: state_gather reads garbage (wrong bar_idx / kernel bug) + // Scan both to disambiguate. + const OUTLIER_THRESHOLD: f32 = 100.0; + const MAX_REPORT: usize = 10; + + // Outliers in gathered states[col0] + let mut state_outliers: Vec<(usize, f32)> = Vec::new(); + for i in 0..n_total { + let v = host_states[i * sd]; + if v.abs() > OUTLIER_THRESHOLD { + state_outliers.push((i, v)); + if state_outliers.len() >= MAX_REPORT { break; } + } + } + let mut next_outliers: Vec<(usize, f32)> = Vec::new(); + for i in 0..(host_next.len() / sd) { + let v = host_next[i * sd]; + if v.abs() > OUTLIER_THRESHOLD { + next_outliers.push((i, v)); + if next_outliers.len() >= MAX_REPORT { break; } + } + } + tracing::warn!( + target: "DIAG_BUG2", + "v2 state[0] outliers (|x|>{OUTLIER_THRESHOLD}): {state_outliers:?}" + ); + tracing::warn!( + target: "DIAG_BUG2", + "v2 next_state[0] outliers (|x|>{OUTLIER_THRESHOLD}): {next_outliers:?}" + ); + + // Outliers in source features_raw_cuda[bar*42 + 0] + // host_ptr aliases the same memory the kernel reads — no DtoH needed. + let market_dim_src: usize = 42; + let n_bars = features_buf.len / market_dim_src; + let mut feat_outliers: Vec<(usize, f32)> = Vec::new(); + let mut feat_max_abs = 0.0f32; + let mut feat_sum_sq: f64 = 0.0; + let mut feat_sum: f64 = 0.0; + let scan_n = n_bars.min(2_000_000); // 2M cap for safety + for b in 0..scan_n { + let v = unsafe { *features_buf.host_ptr.add(b * market_dim_src) }; + feat_sum += v as f64; + feat_sum_sq += (v as f64) * (v as f64); + if v.abs() > feat_max_abs { feat_max_abs = v.abs(); } + if v.abs() > OUTLIER_THRESHOLD && feat_outliers.len() < MAX_REPORT { + feat_outliers.push((b, v)); + } + } + let feat_mean = feat_sum / scan_n.max(1) as f64; + let feat_var = (feat_sum_sq / scan_n.max(1) as f64) - feat_mean * feat_mean; + tracing::warn!( + target: "DIAG_BUG2", + "v2 fxcache feat[0] over {scan_n} bars: mean={feat_mean:.4e}, std={:.4e}, max_abs={feat_max_abs:.4e}", + feat_var.max(0.0).sqrt() + ); + tracing::warn!( + target: "DIAG_BUG2", + "v2 fxcache feat[0] outliers (|x|>{OUTLIER_THRESHOLD}): {feat_outliers:?}" + ); + tracing::warn!( + target: "DIAG_BUG2", + "v2 verdict: feat-outliers nonempty → H1 (fxcache corruption); state-outliers nonempty AND feat-outliers empty → H2 (kernel bug)" + ); } } } diff --git a/docs/dqn-gpu-hot-path-audit.md b/docs/dqn-gpu-hot-path-audit.md index 2dc772429..c41ccc1e9 100644 --- a/docs/dqn-gpu-hot-path-audit.md +++ b/docs/dqn-gpu-hot-path-audit.md @@ -314,3 +314,20 @@ Final structural sweep that closes the residual `*_via_pinned` ctor / cold-path **Validation.** `cargo check -p ml --offline` clean. Pre-commit `check_no_dtod_via_pinned` passes — zero `upload_*_via_pinned` / `clone_to_device_*_via_pinned` callers in either of the two staged files. The Bug 2 instrumentation commit currently in flight (Fix 21 / Fix 22 chain) can now proceed past the pre-commit guard. **Out of scope.** `mapped_pinned.rs` retains the `upload_f32_via_pinned` / `upload_i32_via_pinned` / `clone_to_device_f32_via_pinned` / `clone_to_device_i32_via_pinned` helper definitions — `trainers/ppo.rs` and a small set of remaining cold-path callers still depend on them; the helpers are deleted under task #290 once the call count globally hits zero. Per `feedback_no_hiding` no suppression markers were added. + +### Fix 24 — DIAG_BUG2_v2 outlier disambiguation (H1 vs H2) (2026-05-02) +Augments the one-shot DIAG_BUG2 block at `training_loop.rs:1785` with two narrower scans that disambiguate where the state[0] heavy-tail originates. Smoke `smoke-test-xb78r` at HEAD `cba9f25ed` reported `mean=6.0e-3, std=5.7e2, mean_abs=2.0e1` over 3200 samples but printed first-5-sample state[0..5] values that all sit in [-1, 1] — implies ~1-2 of 3200 samples carry ±32000-magnitude values rather than uniform corruption. + +**What the v2 scan adds.** Two outlier sweeps, both pre-graph-capture, fired once per process via the same `static AtomicBool DUMPED`: +- **State-side sweep:** scans `gpu_batch.states` and `gpu_batch.next_states` (already DtoH-downloaded for the v1 block) for sample indices `i` where `|state[i*sd + 0]| > 100`. Reports up to 10 indices + values per buffer. +- **Source-side sweep:** scans `features_buf.host_ptr` directly via raw-pointer arithmetic for bar indices `b` where `|features[b*42 + 0]| > 100`. host_ptr aliases the same physical pages the kernel reads (mapped-pinned), so this is a faithful read of the data state_gather actually sees — no DtoH copy needed. Reports mean/std/max_abs over up to 2M bars + up to 10 outlier (bar_idx, value) pairs. + +**Disambiguation rule (logged as the verdict line):** +- feat-outliers nonempty → **H1** (data-source corruption — the fxcache or DBN-fallback feature data carries extreme values, e.g. corrupted DBN bars with bar.open ≈ 0 producing ln(ratio) = −30 → z-score normalize to −30000). +- state-outliers nonempty AND feat-outliers empty → **H2** (state_gather kernel reads stale GPU memory or bar_idx points out-of-range). + +**Context for H1.** `smoke-test-xb78r` ran via the **DBN fallback** path (`train_baseline_rl.rs:611` log message format `"Loaded N bars (ts to ts)"`), not the fxcache fast path (which prints `"bars + features from fxcache in Xs"`). `/data/feature-cache/` on training-data PVC is empty — the fxcache file does not exist, so every L40S smoke since the PVC was reset hits the DBN fallback. The fallback applies `NormStats::normalize_batch` (line 633) which is z-score normalization with column-wise mean/stddev computed over all bars. log-return columns have small stddev (~1e-3 for 1-min NQ); a single corrupted bar with `bar.open ≈ 0` produces `safe_log_return = ln(near_zero / 22000) ≈ -30`, which divided by std=0.001 yields normalized ≈ -30000. Few such bars in 700K survive normalization as outliers. + +**Hot-path purity.** Diagnostic only. No kernel changes, no buffer-shape changes. Pre-graph-capture, single-shot. Safe to leave in until Bug 2 is fixed; remove with the eventual `pearl_safe_log_return_extreme_clamp` (or equivalent) commit. Per `feedback_no_hiding` no suppression markers added. + +**Cleanup gate.** Once Bug 2 is fixed (likely a tighter clamp inside `safe_log_return` or in `NormStats::normalize_batch` for log-return columns), delete the entire DIAG_BUG2 + DIAG_BUG2_v2 block at `training_loop.rs:1785-1908` in the same commit. Tracked under task #294.