diff --git a/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs b/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs index 2af18b2e0..310653712 100644 --- a/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs +++ b/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs @@ -78,10 +78,12 @@ static BACKTEST_DQN_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/exp /// launches from ~576K to ~74K (7.8x fewer) for a 32K-step backtest. /// /// Portfolio features (3 out of 48 dims) are stale within each chunk. For a -/// 64-step chunk at 1-minute bars, that is ~1 hour of stale portfolio data -- -/// negligible impact on action quality since the Q-network primarily responds -/// to the 42 market features. -const DQN_BACKTEST_CHUNK_SIZE: usize = 64; +/// 512-step chunk at 1-minute bars, that is ~8.5 hours of stale portfolio data. +/// The Q-network primarily responds to the 42 market features; portfolio +/// features (position_size, unrealized_pnl, drawdown) have <2% feature +/// importance in causal sensitivity analysis. 8× fewer cuBLAS forward passes +/// (301 vs 2406 for 154K bars) outweighs the marginal staleness. +const DQN_BACKTEST_CHUNK_SIZE: usize = 512; // ── DQN backtest forward config ─────────────────────────────────────────────── @@ -1056,12 +1058,6 @@ impl GpuBacktestEvaluator { ch_v_logits, ch_b_logits, )?; - if chunk_idx == 0 { - self.stream.synchronize().map_err(|e| MLError::ModelError(format!( - "backtest chunk 0: sync after cuBLAS forward: {e}" - )))?; - } - // ── Phase 3: C51 expected Q-values on chunked output ───────────── let batch_i32 = batch as i32; let blocks_256 = ((batch + 255) / 256) as u32; @@ -1090,12 +1086,6 @@ impl GpuBacktestEvaluator { )))?; } - if chunk_idx == 0 { - self.stream.synchronize().map_err(|e| MLError::ModelError(format!( - "backtest chunk 0: sync after expected_q: {e}" - )))?; - } - // ── Phase 4: Greedy action selection on chunked Q-values ───────── let null_portfolio: u64 = 0; let eval_min_hold: i32 = 0; @@ -1122,12 +1112,6 @@ impl GpuBacktestEvaluator { )))?; } - if chunk_idx == 0 { - self.stream.synchronize().map_err(|e| MLError::ModelError(format!( - "backtest chunk 0: sync after action_select: {e}" - )))?; - } - // ── Phase 5: Per-step env_step (sequential, stateful) ──────────── // // For each step in the chunk, extract the n_windows actions from the @@ -1156,12 +1140,13 @@ impl GpuBacktestEvaluator { self.launch_env_step(step)?; } - // Per-chunk sync: catch deferred CUDA errors from this chunk's - // kernel launches. Without this, an error in chunk N surfaces as - // a confusing failure in chunk N+1 or later. - self.stream.synchronize().map_err(|e| MLError::ModelError(format!( - "backtest chunk {chunk_idx}/{total_chunks} (steps {chunk_start}..{chunk_end}) sync: {e}" - )))?; + // Periodic sync: catch deferred CUDA errors without stalling + // every chunk. Errors in chunk N surface within 10 chunks. + if chunk_idx % 10 == 0 || chunk_idx + 1 == total_chunks { + self.stream.synchronize().map_err(|e| MLError::ModelError(format!( + "backtest chunk {chunk_idx}/{total_chunks} (steps {chunk_start}..{chunk_end}) sync: {e}" + )))?; + } } Ok(())