From 29d21b3241dee69987cb2eed231f757c827d7e31 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Wed, 27 May 2026 21:45:44 +0200 Subject: [PATCH] =?UTF-8?q?perf(cudarc):=20enable=20async=20alloc=20?= =?UTF-8?q?=E2=80=94=20eliminates=20934=20stream=20syncs/200=20steps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cudarc's CudaSlice::Drop checked has_async_alloc (always false) and fell back to stream.synchronize() + free_sync() on every drop. This blocked the host for ~12.6ms per sync × 4.7 drops/step = ~59ms/step of pure idle waiting (35.8% of wall time per nsys). With has_async_alloc=true, drops use cuMemFreeAsync (non-blocking, same-stream ordering). CUDA 12.4 on sm_86+ fully supports this. Combined with the background diag writer and fused label gather, the training loop should now be GPU-bound at ~13ms/step → ~77 sps. Co-Authored-By: Claude Opus 4.7 --- vendor/cudarc/src/driver/safe/core.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/cudarc/src/driver/safe/core.rs b/vendor/cudarc/src/driver/safe/core.rs index e33922e9c..d3a092b35 100644 --- a/vendor/cudarc/src/driver/safe/core.rs +++ b/vendor/cudarc/src/driver/safe/core.rs @@ -85,7 +85,7 @@ impl CudaContext { cu_device, cu_ctx, ordinal, - has_async_alloc: AtomicBool::new(false), + has_async_alloc: AtomicBool::new(true), num_streams: AtomicUsize::new(0), event_tracking: AtomicBool::new(true), error_state: AtomicU32::new(0),