From 65604db516e5bfcf0085079be2ce9bf8f3709c06 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 19 Apr 2026 16:51:46 +0200 Subject: [PATCH] fix: add cuStreamSynchronize before pinned atom_stats read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit populate_q_out + q_stats_reduce launch async kernels that write to pinned device-mapped memory. The CPU was reading immediately — before the GPU finished writing. Added cuStreamSynchronize to ensure the kernel results are visible before the pinned memory read. Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs index 55a713ac3..44aa51971 100644 --- a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs +++ b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs @@ -8219,9 +8219,13 @@ impl GpuDqnTrainer { 12 * std::mem::size_of::(), self.stream.cu_stream(), ).map_err(|e| MLError::ModelError(format!("q_out sample0 DtoD: {e}")))?; } - // No sync — CPU reads previous step's values from pinned host pointer. + // Sync: populate_q_out + q_stats_reduce just launched — must wait for + // pinned memory writes to complete before CPU reads. + unsafe { + let rc = cudarc::driver::sys::cuStreamSynchronize(self.stream.cu_stream()); + assert_eq!(rc, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "q_stats stream sync"); + } - // Q-mean EMA update — AFTER stream sync so pinned memory has the GPU-written value. self.update_q_mean_ema(); // Read from pinned buffer — host[0..7] = q_stats, host[7..19] = q_out sample 0