From ff7bbc7dd94ff8b93d7e955ad35ff406a81a2692 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 19 Apr 2026 16:53:48 +0200 Subject: [PATCH] =?UTF-8?q?fix:=20remove=20cuStreamSynchronize=20=E2=80=94?= =?UTF-8?q?=20one-step=20lag=20is=20fine=20for=20atom=5Fstats?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The async pinned readback design intentionally reads previous epoch's values. Epoch 1 shows zeros (lag), epoch 2+ shows real atom stats. cuStreamSynchronize kills GPU pipeline — removed. Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs index 44aa51971..fdc543189 100644 --- a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs +++ b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs @@ -8219,12 +8219,9 @@ impl GpuDqnTrainer { 12 * std::mem::size_of::(), self.stream.cu_stream(), ).map_err(|e| MLError::ModelError(format!("q_out sample0 DtoD: {e}")))?; } - // Sync: populate_q_out + q_stats_reduce just launched — must wait for - // pinned memory writes to complete before CPU reads. - unsafe { - let rc = cudarc::driver::sys::cuStreamSynchronize(self.stream.cu_stream()); - assert_eq!(rc, cudarc::driver::sys::cudaError_enum::CUDA_SUCCESS, "q_stats stream sync"); - } + // No sync — one-step lag is fine for monitoring. populate_q_out + q_stats_reduce + // launched async; pinned memory becomes visible by the NEXT epoch's call. + // Epoch 1 reads zeros (expected), epoch 2+ reads real atom stats. self.update_q_mean_ema();