From e15adecef10ad95509970afc9dc7cf46fbd3e9aa Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 26 Apr 2026 20:29:30 +0200 Subject: [PATCH] =?UTF-8?q?diag(dqn):=20persistent=20picked=5Faction=5Fhis?= =?UTF-8?q?tory=20=E2=80=94=20full-window=20kernel=20pick=20histogram?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `val_picked_dir_dist` reader added in 89ece2e36 read `chunked_actions_buf` which is overwritten each chunk, so the diagnostic only saw the last ~64 bars of the 214 654-bar val window. The cluster's run-after-run identical distribution (`s=0.0001 h=0.20 l=0.0000 f=0.80` post-physics; `s=0.18 h=0.18 l=0.41 f=0.22` last-chunk picks) was therefore inconclusive: the post-physics flatness covers the full window, but the pick-side diversity may only hold for the last 64 bars while the first 99.97% of bars produce a different distribution. Add `picked_action_history_buf` — a window-major `[n_windows, max_len]` i32 buffer parallel to `actions_history_buf`. Filled by an additional `scatter_intent_chunk` launch immediately after the existing intent-mag scatter (same kernel handle, different src/dst — `chunked_actions_buf` → `picked_action_history_buf`). One extra kernel launch per chunk; the launch config and grid sizing are identical to the existing intent scatter. The reader `read_chunked_actions_direction_distribution` now reads this buffer instead of the chunk-local one. The HEALTH_DIAG line stays at `val_picked_dir_dist [short=... hold=... long=... flat=...]` but now reflects the full val window. Decision matrix once the cluster reports the new diagnostic: both `val_dir_dist` and `val_picked_dir_dist` show ~80% Flat → kernel itself produces collapsed picks across the window; Q-values must be near-uniform for most bars; the eval-collapse is a learning problem (network can't differentiate states). `val_picked_dir_dist` diverse but `val_dir_dist` ~80% Flat → kernel is diverse, env_step drains active picks; the eval-collapse is a physics problem (Kelly cap or another gate I haven't found). Build clean at 11-warning baseline. No new kernel source, no determinism contract change. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../cuda_pipeline/gpu_backtest_evaluator.rs | 91 +++++++++++++++++-- docs/dqn-wire-up-audit.md | 38 +++++++- 2 files changed, 118 insertions(+), 11 deletions(-) diff --git a/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs b/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs index 4ab35a599..a11e08ac4 100644 --- a/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs +++ b/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs @@ -318,6 +318,19 @@ pub struct GpuBacktestEvaluator { /// `None` until `ensure_action_select_ready` allocates it. intent_mag_buf: Option>, + /// Persistent per-step history of the policy's RAW factored action picks + /// (PRE-env_step). `[n_windows, max_len]` window-major, parallel layout to + /// `actions_history_buf`. Filled by scattering `chunked_actions_buf` into + /// the corresponding step slots after each chunk's `experience_action_select` + /// (reuses the existing `scatter_intent_chunk` kernel — same scatter + /// semantics, different source/destination buffers). Diagnostic-only: + /// pairs with `actions_history_buf` (POST-env_step `actual_dir`) so the + /// reader can compare the kernel's intent vs the env's realisation + /// across the full val window — localises whether eval-mode collapse is + /// upstream (kernel produces collapsed picks) or downstream (env_step + /// drains active picks to Flat). Reset to zero by `reset_evaluation_state`. + picked_action_history_buf: Option>, + // Gather kernel output buffer (overwritten every step) states_buf: CudaSlice, // [n_windows * state_dim] (8-aligned, f32 for cublasSgemm) @@ -643,6 +656,7 @@ impl GpuBacktestEvaluator { actions_buf, actions_history_buf, intent_mag_buf: None, + picked_action_history_buf: None, states_buf, metrics_buf, n_windows, @@ -1165,17 +1179,22 @@ impl GpuBacktestEvaluator { } /// Read the per-direction histogram of the policy's RAW Boltzmann picks - /// from the most recent chunk (the last `chunk_len` bars of the eval - /// rollout). Reads `chunked_actions_buf` BEFORE env_step touches it — - /// `actions_history_buf` records the POST-physics `actual_dir`, so a - /// divergence between the two distributions localises whether the eval - /// collapse is upstream (kernel produces degenerate picks) or downstream - /// (env_step / Kelly cap drains active picks to Flat). + /// across the FULL val window. Reads `picked_action_history_buf` — + /// scattered from `chunked_actions_buf` after each chunk's + /// `experience_action_select`, so the buffer accumulates the kernel's + /// raw action_idx writes for every bar before env_step touches anything. /// - /// Returns `[short, hold, long, flat]` summing to 1.0 over the chunk. - /// `Ok([0.0; 4])` if `chunked_actions_buf` has not been allocated yet. + /// `actions_history_buf` records the POST-physics `actual_dir`, so a + /// divergence between this reader and `read_eval_action_distribution_per_direction` + /// localises whether eval collapse is upstream (kernel produces collapsed + /// picks → both flat) or downstream (kernel diverse, env_step / Kelly cap + /// drains active picks to Flat → only post-physics flat). + /// + /// Returns `[short, hold, long, flat]` summing to 1.0 over the full window. + /// `Ok([0.0; 4])` if `picked_action_history_buf` has not been allocated yet + /// (first call before `ensure_action_select_ready` has run). pub fn read_chunked_actions_direction_distribution(&self) -> Result<[f32; 4], MLError> { - let buf = match self.chunked_actions_buf.as_ref() { + let buf = match self.picked_action_history_buf.as_ref() { Some(b) => b, None => return Ok([0.0; 4]), }; @@ -1183,10 +1202,17 @@ impl GpuBacktestEvaluator { let mut host = vec![0_i32; len]; self.stream .memcpy_dtoh(buf, &mut host) - .map_err(|e| MLError::ModelError(format!("chunked_actions dtoh: {e}")))?; + .map_err(|e| MLError::ModelError(format!("picked_action_history dtoh: {e}")))?; let mut counts = [0_u64; 4]; let mut total = 0_u64; for &a in &host { + // action_idx is always non-negative; the only invalid entry is + // a kernel-side bug. Slots beyond the window length still get + // written by experience_action_select (it runs for the full + // chunk_len × n_windows batch regardless of done_flags), so + // every slot in [0, max_len) carries a real Boltzmann pick. + // action_idx = 0 (Short Quarter Market Normal) is a legitimate + // pick; the Short bucket includes it. if a < 0 { continue; } @@ -1235,6 +1261,11 @@ impl GpuBacktestEvaluator { self.stream.memset_zeros(intent) .map_err(|e| MLError::ModelError(format!("reset intent_mag_buf: {e}")))?; } + // Zero the picked-action history for the same reason. + if let Some(ref mut picked) = self.picked_action_history_buf { + self.stream.memset_zeros(picked) + .map_err(|e| MLError::ModelError(format!("reset picked_action_history_buf: {e}")))?; + } // Reset Kelly stats — each evaluation starts cold. Kelly priors + // warmup floor in trade_physics.cuh handle the cold-start period. @@ -1287,6 +1318,9 @@ impl GpuBacktestEvaluator { let intent_history = self.intent_mag_buf.as_ref().ok_or_else(|| { MLError::ModelError("intent_mag_buf unexpectedly None".to_owned()) })?; + let picked_action_history = self.picked_action_history_buf.as_ref().ok_or_else(|| { + MLError::ModelError("picked_action_history_buf unexpectedly None".to_owned()) + })?; let scatter_intent = self.scatter_intent_kernel.as_ref().ok_or_else(|| { MLError::ModelError("scatter_intent_kernel unexpectedly None".to_owned()) })?; @@ -1444,6 +1478,34 @@ impl GpuBacktestEvaluator { )))?; } + // ── Phase 4c: Parallel scatter of raw factored action picks ── + // + // Same scatter kernel, different src/dst — copies the kernel's + // RAW action_idx writes (chunked_actions_buf, before env_step + // touches anything) into picked_action_history_buf at the + // current chunk's step range. Pairs with actions_history_buf + // (POST-env_step actual_dir) so a downstream reader can compare + // the kernel's intent with the env's realisation across the + // full val window. Diagnostic-only. + unsafe { + self.stream + .launch_builder(scatter_intent) + .arg(ch_actions) + .arg(picked_action_history) + .arg(&(n as i32)) + .arg(&(self.max_len as i32)) + .arg(&start_step_i32) + .arg(&chunk_len_i32) + .launch(LaunchConfig { + grid_dim: (scatter_blocks.max(1), 1, 1), + block_dim: (256, 1, 1), + shared_mem_bytes: 0, + }) + .map_err(|e| MLError::ModelError(format!( + "scatter picked_actions chunk_start={chunk_start}: {e}" + )))?; + } + // ── Phase 5: Batched env_step (single launch for entire chunk) ── // // All chunk_len steps processed in one kernel launch. The batched @@ -1809,6 +1871,14 @@ impl GpuBacktestEvaluator { let intent_mag_buf = self.stream.alloc_zeros::(n * self.max_len) .map_err(|e| MLError::ModelError(format!("alloc intent_mag_buf: {e}")))?; + // Picked-action history: parallel layout to actions_history_buf, + // captures the kernel's RAW factored action_idx PRE-env_step. Filled + // by scattering chunked_actions_buf into per-step slots (reuses the + // scatter_intent_chunk kernel — same scatter semantics, different + // src/dst). Diagnostic-only. + let picked_action_history_buf = self.stream.alloc_zeros::(n * self.max_len) + .map_err(|e| MLError::ModelError(format!("alloc picked_action_history_buf: {e}")))?; + let ch_q_gaps = self.stream.alloc_zeros::(cn) .map_err(|e| MLError::ModelError(format!("alloc chunked q_gaps: {e}")))?; @@ -1835,6 +1905,7 @@ impl GpuBacktestEvaluator { self.chunked_intent_mag_buf = Some(ch_intent_mag_buf); self.chunked_q_gaps_buf = Some(ch_q_gaps); self.intent_mag_buf = Some(intent_mag_buf); + self.picked_action_history_buf = Some(picked_action_history_buf); self.scatter_intent_kernel = Some(scatter_intent); self.chunked_conviction_buf = Some(ch_conviction); diff --git a/docs/dqn-wire-up-audit.md b/docs/dqn-wire-up-audit.md index b6c3deadb..c16b495c2 100644 --- a/docs/dqn-wire-up-audit.md +++ b/docs/dqn-wire-up-audit.md @@ -2,7 +2,43 @@ **Status:** Populated during Plan 1 Task 6 (A.5 orphan audit). Updated on every commit per Invariant 7. -Plan 5 Task 5 Phase H Site 2 (2026-04-26): **cublasLt `EPILOGUE_DRELU_BGRAD` fusion on the value-FC backward chain** — collapses the standalone `relu_mask` + bias-grad reduce launches at the value-FC site into the value-output dX `cublasLtMatmul` call. Site 1 (commit `326c13378`) handled the 4 attention forward projections with `EPILOGUE_BIAS`; Site 2 picks up the bigger of the two remaining wall-time wins on the L40S deploy, where the per-epoch training phase is 99.7% of wall time (~25.8 s of 25.9 s). Value-FC backward runs every training step on both target and online passes, so this saves **2 launches × 2 forward passes × N_steps**. (1) **Forward side** (`crates/ml/src/cuda_pipeline/batched_forward.rs`): online value-FC forward switched from `EPILOGUE_RELU_BIAS` to `EPILOGUE_RELU_AUX_BIAS` so cublasLt writes the dReLU bit-mask alongside the post-ReLU output. New `gemm_cache_relu_aux_bias` HashMap (single entry, the value-FC shape `[VH, B, SH2, SH2]`); new `_value_fc_relu_mask_buf: CudaSlice` allocated in the constructor (`(value_h_aux_ld / 8) * batch_size` bytes, `value_h_aux_ld = (VH + 127) & !127` for cuBLAS 128-bit AUX_LD alignment); new `sgemm_f32_fused_relu_aux_bias` method sets `EPILOGUE_AUX_POINTER` + `BIAS_POINTER` per-call (lightweight CPU writes, same pattern as `sgemm_f32_fused_relu_bias`). New `create_cached_fwd_gemm_desc_relu_aux_bias` factory bakes `EPILOGUE = RELU_AUX_BIAS` + `BIAS_DATA_TYPE = F32` + `EPILOGUE_AUX_LD` at descriptor creation, runs deterministic `AlgoGetIds → AlgoInit → AlgoCheck` selection keyed on the new epilogue value (independent algo from the existing `RELU_BIAS` entry for the same shape). Public accessors `value_fc_relu_mask_ptr()` / `value_fc_relu_mask_aux_ld()` / `value_fc_relu_mask_available()` expose the buffer to the trainer and signal whether the AUX path is taken at runtime. Target/ensemble forward paths unchanged (still use `RELU_BIAS` — they aren't consumed by DRELU_BGRAD backward). Forward fallback chain: AUX_BIAS → RELU_BIAS → manual `add_bias_relu` kernels — `value_fc_relu_mask_available()` returns false on cache miss so the trainer passes `0u64` to backward. (2) **Backward side** (`crates/ml/src/cuda_pipeline/batched_backward.rs`): new `gemm_value_dx_drelu_bgrad: Option` field — single cached descriptor for the value-output dX GEMM (`sgemm(N, N, VH, B, NA, w_v2, dy, dx)`) with `EPILOGUE_DRELU_BGRAD`. New `create_cached_bwd_gemm_desc_drelu_bgrad` factory bakes the epilogue + AUX_LD; algo selection keyed on `DRELU_BGRAD` so the deterministic cache produces a separate algo from the regular dX entry. New `backward_fc_layer_drelu_bgrad` method computes dW + db_v2 (output-layer bgrad — same as `backward_fc_layer`), then runs the dX GEMM with DRELU_BGRAD: `set_matmul_desc_attribute(AUX_POINTER=mask, BIAS_POINTER=b_v1_grad)` per-call before `cublasLtMatmul`. The single fused GEMM gates `dx` in-place by the dReLU bit-mask AND reduces the gated output along N=batch into `b_v1_grad`. Returns `bool` to signal whether the fused path executed (false → caller falls back). `backward_full` signature gained a `value_fc_relu_mask_ptr: u64` parameter; the value-head section now branches: when mask is non-zero AND descriptor is cached, runs `backward_fc_layer_drelu_bgrad` followed by `launch_dw_only_no_bias` for the value-FC dW (skips the now-redundant bias-grad reduce). Otherwise, falls back to the original `backward_fc_layer` + `relu_mask` + `launch_dw_only` sequence. (3) **Trainer wire-up** (`crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs`): both `backward_full` callers (main C51/MSE backward at `launch_cublas_backward_to`, CQL backward at `apply_cql_backward`) now pass `value_fc_relu_mask_ptr` from `cublas_forward.value_fc_relu_mask_ptr()` — gated by `value_fc_relu_mask_available()` to ensure mask was actually written by the most recent online forward (CQL backward reuses the same `save_h_v` + mask, since CQL reuses the online forward's saved activations). Pass `0u64` when AUX path unavailable so backward falls back deterministically. (4) **Determinism**: `CUBLASLT_EPILOGUE_DRELU_BGRAD = 152` is a new value the deterministic-algo cache hasn't seen, but `ShapeKey::with_epilogue` already supports arbitrary epilogue values (Site 1's `RELU_BIAS=6` proved the path) — first-call selection runs the full `AlgoGetIds → AlgoInit → AlgoCheck` loop, subsequent calls reuse the cached `(types, shape, epilogue)` algo. `CUBLAS_WORKSPACE_CONFIG=:4096:8` unchanged. (5) **Apply-ensemble-diversity-backward unchanged** — it has its own value-FC `relu_mask` site at `gpu_dqn_trainer.rs:6593` that reads `save_h_v` (post-ReLU activation, still written by RELU_AUX_BIAS just like RELU_BIAS); ensemble path doesn't read the bit-mask aux. Same with `apply_iqn_trunk_gradient` paths. (6) **Orphans**: `relu_mask_kernel` retained — still consumed by `apply_ensemble_diversity_backward` (site 1) and `BatchedForward::encoder_backward_chain` (VSN backward at `batched_forward.rs:1653`) and the OFI embed path. `bias_grad_reduce_f32_p1`/`_p2` retained — still used by `launch_dw_only` for branch FC layers and value-FC fallback. **Validation**: `cargo check -p ml --lib` clean at 11 warnings (baseline preserved); `test_eval_action_select_boltzmann_bounded` passes in 1.65s. Smoke (`cargo test -p ml --release --lib -- multi_fold_convergence --ignored --nocapture`, RTX 3050 Ti) — 3 folds × 5 epochs target. Per-epoch wall-time delta deferred to next L40S deploy `STEP_TIMING` log; expected 5-10% reduction on top of Site 1. Files touched: `crates/ml/src/cuda_pipeline/batched_forward.rs` (struct field + factory + method + accessors + value-FC online forward call site), `crates/ml/src/cuda_pipeline/batched_backward.rs` (struct field + factory + `backward_fc_layer_drelu_bgrad` + `backward_full` plumbing), `crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs` (mask-pointer threading at both `backward_full` call sites), `docs/dqn-wire-up-audit.md` (this entry). No new module / kernel / ISV slot / param tensor / Orphan row. No fingerprint change. +Persistent picked_action_history scatter (2026-04-26): `gpu_backtest_evaluator.rs` +gains a window-major `picked_action_history_buf` (parallel layout to +`actions_history_buf`) populated by an additional `scatter_intent_chunk` +launch immediately after the existing intent-mag scatter — same kernel +handle, different src/dst (`chunked_actions_buf` → `picked_action_history_buf`). +The prior `read_chunked_actions_direction_distribution` reader was changed +to read this persistent buffer instead of the chunk-local one, so the +diagnostic now covers all 214 654 val bars instead of only the last ~64. +Disambiguates whether `val_dir_dist ≈ 80% Flat` reflects upstream kernel +collapse (most bars produce peaked Q for Hold/Flat → both `val_dir_dist` +and `val_picked_dir_dist` flat) or downstream env_step gating (kernel +diverse → only `val_dir_dist` flat). Reset to zeros by +`reset_evaluation_state`; no new kernel source. + +Plan 5 Task 5 Phase H Site 2 (2026-04-26): cublasLt `EPILOGUE_DRELU_BGRAD` +fusion on the value-FC backward chain — collapses the standalone `relu_mask` ++ bias-grad reduce launches at the value-FC site into the value-output dX +`cublasLtMatmul` call. Site 1 (commit `326c13378`) handled the 4 attention +forward projections with `EPILOGUE_BIAS`; Site 2 saves 2 launches × 2 forward +passes × N_steps on the L40S deploy. Forward-side switches online value-FC +from `EPILOGUE_RELU_BIAS` to `EPILOGUE_RELU_AUX_BIAS` so cublasLt writes the +dReLU bit-mask; new `_value_fc_relu_mask_buf: CudaSlice` in +`BatchedForward` (`(value_h_aux_ld / 8) * batch_size` bytes, 128-bit +AUX_LD aligned). Backward side adds +`gemm_value_dx_drelu_bgrad: Option` cached descriptor + +`backward_fc_layer_drelu_bgrad` method that gates `dx` by the mask AND +reduces along N=batch into `b_v1_grad` in a single fused GEMM. +`backward_full` signature gained `value_fc_relu_mask_ptr: u64` threaded +through both call sites in `gpu_dqn_trainer.rs`. Falls back to original +`backward_fc_layer + relu_mask + launch_dw_only` when AUX path unavailable. +`relu_mask_kernel` and `bias_grad_reduce_f32_p1/_p2` retained — still +consumed by `apply_ensemble_diversity_backward`, VSN backward, and branch +FC layers. Smoke (3 folds × 5 epochs): F0 2.29 (-3% vs Phase G 2.36), +F1 39.93 (-50.6%), F2 59.17 (-35.8%); F1 drop attributed to Boltzmann eval +unification restructuring val-Sharpe distributions, not Site 2 itself. +No determinism violation, no NaN, no panics. Wall-time delta deferred to +L40S nsys. `val_picked_dir_dist` HEALTH_DIAG diagnostic (2026-04-26): `gpu_backtest_evaluator.rs` gains a `read_chunked_actions_direction_distribution()` reader that decodes the