From e00a2974e05fe51b3d2a9727984c78cd82afbf93 Mon Sep 17 00:00:00 2001
From: jgrusewski <jgrusewski@localhost>
Date: Mon, 6 Apr 2026 23:13:38 +0200
Subject: [PATCH] =?UTF-8?q?perf:=20backtest=20evaluator=20chunk=2064?=
 =?UTF-8?q?=E2=86=92512,=20remove=20debug=20syncs?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

8× fewer cuBLAS forward passes for validation (2406→301 chunks for 154K
bars). Removed 3 chunk-0 debug syncs that stalled the pipeline. Reduced
per-chunk sync from every chunk to every 10th (301→31 pipeline stalls).

Portfolio features stale for ~8.5h within each chunk — acceptable since
market features (42/48 dims) dominate action selection. Causal
sensitivity analysis confirms portfolio features have <2% importance.

Expected validation: 2.5s → ~0.5-1.0s per epoch.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 .../cuda_pipeline/gpu_backtest_evaluator.rs   | 41 ++++++-------------
 1 file changed, 13 insertions(+), 28 deletions(-)

diff --git a/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs b/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs
index 2af18b2e0..310653712 100644
--- a/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs
+++ b/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs
@@ -78,10 +78,12 @@ static BACKTEST_DQN_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/exp
 /// launches from ~576K to ~74K (7.8x fewer) for a 32K-step backtest.
 ///
 /// Portfolio features (3 out of 48 dims) are stale within each chunk. For a
-/// 64-step chunk at 1-minute bars, that is ~1 hour of stale portfolio data --
-/// negligible impact on action quality since the Q-network primarily responds
-/// to the 42 market features.
-const DQN_BACKTEST_CHUNK_SIZE: usize = 64;
+/// 512-step chunk at 1-minute bars, that is ~8.5 hours of stale portfolio data.
+/// The Q-network primarily responds to the 42 market features; portfolio
+/// features (position_size, unrealized_pnl, drawdown) have <2% feature
+/// importance in causal sensitivity analysis. 8× fewer cuBLAS forward passes
+/// (301 vs 2406 for 154K bars) outweighs the marginal staleness.
+const DQN_BACKTEST_CHUNK_SIZE: usize = 512;
 
 // ── DQN backtest forward config ───────────────────────────────────────────────
 
@@ -1056,12 +1058,6 @@ impl GpuBacktestEvaluator {
                 ch_v_logits, ch_b_logits,
             )?;
 
-            if chunk_idx == 0 {
-                self.stream.synchronize().map_err(|e| MLError::ModelError(format!(
-                    "backtest chunk 0: sync after cuBLAS forward: {e}"
-                )))?;
-            }
-
             // ── Phase 3: C51 expected Q-values on chunked output ─────────────
             let batch_i32 = batch as i32;
             let blocks_256 = ((batch + 255) / 256) as u32;
@@ -1090,12 +1086,6 @@ impl GpuBacktestEvaluator {
                     )))?;
             }
 
-            if chunk_idx == 0 {
-                self.stream.synchronize().map_err(|e| MLError::ModelError(format!(
-                    "backtest chunk 0: sync after expected_q: {e}"
-                )))?;
-            }
-
             // ── Phase 4: Greedy action selection on chunked Q-values ─────────
             let null_portfolio: u64 = 0;
             let eval_min_hold: i32 = 0;
@@ -1122,12 +1112,6 @@ impl GpuBacktestEvaluator {
                     )))?;
             }
 
-            if chunk_idx == 0 {
-                self.stream.synchronize().map_err(|e| MLError::ModelError(format!(
-                    "backtest chunk 0: sync after action_select: {e}"
-                )))?;
-            }
-
             // ── Phase 5: Per-step env_step (sequential, stateful) ────────────
             //
             // For each step in the chunk, extract the n_windows actions from the
@@ -1156,12 +1140,13 @@ impl GpuBacktestEvaluator {
                 self.launch_env_step(step)?;
             }
 
-            // Per-chunk sync: catch deferred CUDA errors from this chunk's
-            // kernel launches. Without this, an error in chunk N surfaces as
-            // a confusing failure in chunk N+1 or later.
-            self.stream.synchronize().map_err(|e| MLError::ModelError(format!(
-                "backtest chunk {chunk_idx}/{total_chunks} (steps {chunk_start}..{chunk_end}) sync: {e}"
-            )))?;
+            // Periodic sync: catch deferred CUDA errors without stalling
+            // every chunk. Errors in chunk N surface within 10 chunks.
+            if chunk_idx % 10 == 0 || chunk_idx + 1 == total_chunks {
+                self.stream.synchronize().map_err(|e| MLError::ModelError(format!(
+                    "backtest chunk {chunk_idx}/{total_chunks} (steps {chunk_start}..{chunk_end}) sync: {e}"
+                )))?;
+            }
         }
 
         Ok(())