From 83546b5c37739418500fc9f2b708aa11a7e5029e Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Mon, 18 May 2026 11:32:27 +0200 Subject: [PATCH] fix(ml-alpha): drop synchronize() in per-horizon pool + head bindings After adaf275af removed the synchronizes in PerHorizonTrainState's forward_with_blend / backward_through_blend, smoke alpha-perception-9l6hw still failed with CUDA_ERROR_STREAM_CAPTURE_UNSUPPORTED. Root cause: PerHorizonAttentionPool::{forward,backward} and PerHorizonResidualHead:: {forward,backward} each end with their own self.stream.synchronize(), which is illegal during CUDA Graph capture. Same fix: drop the four synchronizes. Same-stream issue order ensures the next kernel sees the previous one's output. Capture invariant preserved. Verified locally: per_horizon_full_pipeline_smoke (2/2), attention pool numgrad (1/1), residual head numgrad (1/1) all pass on RTX 3050. Co-Authored-By: Claude Opus 4.7 --- crates/ml-alpha/src/per_horizon_attention_pool.rs | 6 ++++-- crates/ml-alpha/src/per_horizon_residual_head.rs | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/crates/ml-alpha/src/per_horizon_attention_pool.rs b/crates/ml-alpha/src/per_horizon_attention_pool.rs index 44f740cd1..e1b3345a3 100644 --- a/crates/ml-alpha/src/per_horizon_attention_pool.rs +++ b/crates/ml-alpha/src/per_horizon_attention_pool.rs @@ -82,7 +82,8 @@ impl PerHorizonAttentionPool { .launch(cfg) .context("per_horizon_attention_pool_fwd")?; } - self.stream.synchronize()?; + // No synchronize: same-stream issue order is sufficient and + // synchronize is illegal during CUDA Graph capture. Ok(()) } @@ -122,7 +123,8 @@ impl PerHorizonAttentionPool { .launch(cfg) .context("per_horizon_attention_pool_bwd")?; } - self.stream.synchronize()?; + // No synchronize: same-stream issue order is sufficient and + // synchronize is illegal during CUDA Graph capture. Ok(()) } } diff --git a/crates/ml-alpha/src/per_horizon_residual_head.rs b/crates/ml-alpha/src/per_horizon_residual_head.rs index df8ef9ce2..c78bd7273 100644 --- a/crates/ml-alpha/src/per_horizon_residual_head.rs +++ b/crates/ml-alpha/src/per_horizon_residual_head.rs @@ -71,7 +71,8 @@ impl PerHorizonResidualHead { .launch(cfg) .context("per_horizon_residual_head_fwd")?; } - self.stream.synchronize()?; + // No synchronize: same-stream issue order is sufficient and + // synchronize is illegal during CUDA Graph capture. Ok(()) } @@ -107,7 +108,8 @@ impl PerHorizonResidualHead { .launch(cfg) .context("per_horizon_residual_head_bwd")?; } - self.stream.synchronize()?; + // No synchronize: same-stream issue order is sufficient and + // synchronize is illegal during CUDA Graph capture. Ok(()) } }