From 365a7a17b46e2ffcdf10e8bafdef1b5394869bd4 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Mon, 6 Apr 2026 13:35:49 +0200 Subject: [PATCH] fix: pre-initialize cast kernels at buffer construction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cuModuleLoadData hangs on H100 CUDA 13 when called after CUDA graph capture — the context state appears incompatible with module loading. Pre-initialize the OnceLock during GpuReplayBuffer::new() when the context is clean and no graphs have been captured. Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/ml-dqn/src/gpu_replay_buffer.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/ml-dqn/src/gpu_replay_buffer.rs b/crates/ml-dqn/src/gpu_replay_buffer.rs index 607caf604..c6c1caf9c 100644 --- a/crates/ml-dqn/src/gpu_replay_buffer.rs +++ b/crates/ml-dqn/src/gpu_replay_buffer.rs @@ -256,6 +256,12 @@ impl GpuReplayBuffer { let update_batch_max = a32f(stream, 1, "update_batch_max")?; let update_max_merge = a32f(stream, 1, "update_max_merge")?; + // Pre-initialize cast kernels while CUDA context is clean. + // OnceLock + cuModuleLoadData requires a current CUDA context. + // If deferred to first PER update (after graph capture), the context + // may be in a state that blocks cuModuleLoadData on H100 CUDA 13. + get_cast_kernels(stream)?; + Ok(Self { config, stream: Arc::clone(stream), kernels: k, states: s, next_states: ns, actions: a, rewards: r, dones: d, priorities: p,