diff --git a/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs b/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs index 801f994d7..a022eb396 100644 --- a/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs +++ b/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs @@ -5515,62 +5515,79 @@ impl GpuExperienceCollector { &self.stream, &self.aux_conf_per_sample, total, "aux_conf" )?; - // SP22 H6 vNext Phase B4b-2 (2026-05-14): build the per-sample + // SP22 H6 vNext Phase B4b-2 (2026-05-14, REVISED for ring-tail + // mask bug — smoke train-k95mj repro 14:09): build the per-sample // trade-outcome label column for the emitted batch. The K=2 // sibling (`aux_sign_labels`) is filled by `aux_sign_label_kernel` // walking `0..total` directly — total = base_total × 2 includes - // the counterfactual expansion. The K=3 per-step producer + // the counterfactual expansion, AND the K=2 kernel writes the + // same `bar+lookahead` future-price label to both halves + // (label depends on bar, not action). The K=3 per-step producer // (`trade_outcome_label_kernel`, B4b-1) writes only the first // `base_total = n_episodes × timesteps` slots of // `exp_aux_to_label_per_sample` (sized `[alloc_episodes × - // alloc_timesteps]` — pre-cf-mult). A naive - // `dtod_clone_i32(src, total)` would `slice(..total)` and panic - // (cudarc safe/core.rs Option::unwrap on None — workflow - // train-xzv56 repro at fold 0, post-rollout). + // alloc_timesteps]` — pre-cf-mult). // - // Fix: alloc `[total]`, fill ALL slots with the mask sentinel - // -1 via `cuMemsetD32Async` (pattern 0xFFFFFFFF reinterprets as - // i32 = -1), then `memcpy_dtod` the first `base_total` real - // labels into the on-policy half. CF half stays at -1; the K=3 - // sparse-CE loss masks `-1` labels out of the mean + B_valid - // count, so CF samples contribute zero gradient — matches the - // K=2 head's semantic (CF labels are also undefined; the K=2 - // sign-label kernel writes the same `bar+lookahead` future- - // price label to both halves which collapses to a redundant - // observation, while we explicitly mask). + // History of fixes: + // 1. Initial B4b-2: `dtod_clone_i32(src, total)` → + // `src.slice(..total)` panicked at cudarc safe/core.rs:1648 + // (Option::unwrap on None) because src.len() = base_total < + // total. Workflow train-xzv56 fold-0 repro. + // 2. First fix (256a5fa5a): alloc `[total]`, memset CF half + // with -1 mask, memcpy on-policy half from src. Eliminated + // the panic — but introduced a SILENT bug: `insert_batch` + // is called with `bs = total = 4_096_000` and replay + // `cap = 300_000`, triggering the `off = bs - cap` + // tail-clip branch, which slices `aux_outcome.slice(off..)` + // = the LAST 300K elements of `aux_outcome_labels[total]`. + // Last 300K are entirely in the CF half (base_total = + // 2_048_000 < 3_796_000 = off). Ring fills with -1 only. + // K=3 loss reduce sees valid_count = 0 every batch, returns + // loss = 0, Pearl A bootstrap (`if loss > 0`) never fires, + // ISV[538] pinned at sentinel 0.000 across all epochs. + // Workflow train-k95mj epoch 0 repro (trade_outcome_ce = + // 0.000e0 in HEALTH_DIAG aux print). + // 3. THIS fix: duplicate the on-policy half into the CF half + // so both halves carry valid {0/1/2 if trade close, 0 + // default} labels. Mirrors aux_sign_labels' both-halves- + // filled pattern. CF samples now train on the SAME outcome + // label as their on-policy counterpart (semantically + // defensible: the trade-close event at bar t is a property + // of the bar, not the action, even though a different + // action might never have entered the trade — that's the + // same approximation aux_sign_labels makes). + // + // Net effect: insert_batch's tail-clip now retains 300K real + // labels (last 150K from CF half + last 150K from on-policy + // half), with the on-policy half's ~2.4% real trade-close rate + // → ~7K real {0/1/2} labels per ring fill. K=3 head trains. let aux_outcome_labels = { let mut buf = self.stream.alloc_zeros::(total) .map_err(|e| MLError::ModelError(format!( "alloc aux_outcome_labels i32[{total}]: {e}" )))?; - // Fill all `total` slots with i32 sentinel -1 via byte - // pattern 0xFFFFFFFF. Async on the collector stream — no - // stream-sync needed since the downstream consumers (replay - // insert + K=3 loss) launch on the same stream. - #[allow(unsafe_code)] - unsafe { - let rc = cudarc::driver::sys::cuMemsetD32Async( - buf.raw_ptr(), - 0xFFFF_FFFFu32, // i32::MIN pattern? no — 0xFFFFFFFF = i32(-1) - total, // word count (NOT bytes), per CUDA API - self.stream.cu_stream(), - ); - if rc != cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS { - return Err(MLError::ModelError(format!( - "cuMemsetD32Async aux_outcome_labels mask-fill: {rc:?}" - ))); - } - } - // Copy the on-policy half (first `base_total` slots) from - // the per-step scratch buffer. `base_total` exactly matches - // `exp_aux_to_label_per_sample.len() = alloc_episodes × - // alloc_timesteps`, so the slice fits. + // Copy on-policy half from the per-step scratch. let src_view = self.exp_aux_to_label_per_sample.slice(..base_total); - let mut dst_view = buf.slice_mut(..base_total); - self.stream.memcpy_dtod(&src_view, &mut dst_view) - .map_err(|e| MLError::ModelError(format!( - "DtoD aux_outcome_labels on-policy half: {e}" - )))?; + { + let mut dst_on_policy = buf.slice_mut(..base_total); + self.stream.memcpy_dtod(&src_view, &mut dst_on_policy) + .map_err(|e| MLError::ModelError(format!( + "DtoD aux_outcome_labels on-policy half: {e}" + )))?; + } + // Duplicate the on-policy half into the CF half. Same pattern + // as `aux_sign_labels` (both halves filled with the same + // bar-resolved label). The borrow-checker requires the + // on-policy `slice` to be re-bound here since `slice_mut` + // above released its lifetime. + { + let src_dup = self.exp_aux_to_label_per_sample.slice(..base_total); + let mut dst_cf = buf.slice_mut(base_total..total); + self.stream.memcpy_dtod(&src_dup, &mut dst_cf) + .map_err(|e| MLError::ModelError(format!( + "DtoD aux_outcome_labels CF half (dup of on-policy): {e}" + )))?; + } buf }; diff --git a/docs/dqn-wire-up-audit.md b/docs/dqn-wire-up-audit.md index deee89351..789768991 100644 --- a/docs/dqn-wire-up-audit.md +++ b/docs/dqn-wire-up-audit.md @@ -2,6 +2,58 @@ **Status:** Populated during Plan 1 Task 6 (A.5 orphan audit). Updated on every commit per Invariant 7. +## 2026-05-14 — fix(sp22-vnext): aux_outcome CF half mirror on-policy (NOT all -1) + +**Branch:** `sp20-aux-h-fixed`. **Trigger:** smoke `train-k95mj` epoch +0 + epoch 1 both printed `trade_outcome_ce=0.000e0` — K=3 CE EMA +stuck at Pearl A sentinel across all training steps. + +**Root cause chain:** + +1. `insert_batch` is called with `bs = total = 4_096_000` (cf-mult- + expanded), replay `cap = 300_000` (L40S GpuProfile). The + tail-clip branch triggers: `off = bs - cap = 3_796_000`, + slice `aux_outcome.slice(off..)` takes the last 300K elements + of `aux_outcome_labels[total]`. +2. The previous fix (`256a5fa5a`) filled the CF half `[base_total, + total) = [2_048_000, 4_096_000)` with `-1` mask sentinel via + `cuMemsetD32Async`. The last 300K elements at `[3_796_000, + 4_096_000)` are entirely in this CF half → 100% mask. +3. Replay ring fills with 300K mask labels. PER's gather samples + batches from the ring → every batch has `label == -1` for all + 4096 samples → `aux_trade_outcome_loss_reduce` returns `loss = + sh_loss[0] / fmaxf(valid=0, 1) = 0 / 1 = 0` every step. +4. `aux_outcome_ce_ema_update` (F-3 kernel) bootstrap clause is + `if (fabsf(ema_prev) < 1e-9 && loss > 0.0f)` — fires only on + non-zero loss. Loss is always 0, so `ema_prev` stays at 0 + forever → ISV[538] pinned at sentinel. +5. `HEALTH_DIAG[N]: aux [..trade_outcome_ce=0.000e0..]` across all + epochs. + +**Fix:** the CF half should mirror the on-policy half (the K=2 +sibling `aux_sign_labels` already does this — its kernel writes +the same bar-resolved label to both halves). Replace the +`cuMemsetD32Async(-1)` + single `memcpy_dtod` with TWO +`memcpy_dtod` calls: one for the on-policy half, one for the CF +half (both sourced from the same `exp_aux_to_label_per_sample +[base_total]`). + +**Why this is the right semantic:** the K=3 outcome label depends +on the bar's trade-close event, not the action. The CF action +might never have entered a trade in real execution, but the bar +where the close happened is the same. The B4b-1 kernel writes +`-1` to non-close slots (~97%) and `0/1/2` to close slots (~3%), +all in the on-policy half. Duplicating into CF preserves this +sparsity for both halves — ring fills with ~7-9K real labels per +300K-element ring tail. + +**Tests:** `cargo test -p ml --lib` 1016/0 green. Lib tests +exercise small batches that don't trigger the `off > 0` +tail-clip path; the bug is runtime-only and required a +production-shape smoke to surface. + +**Touched:** `crates/ml/src/cuda_pipeline/gpu_experience_collector.rs`. + ## 2026-05-14 — feat(sp22-vnext): F-3c follow-up — K=3 CE EMA in stdout HEALTH_DIAG line **Branch:** `sp20-aux-h-fixed`. **Trigger:** smoke `train-q5k5k`