diff --git a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs index fb9eae8cf..60fcde24a 100644 --- a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs +++ b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs @@ -864,8 +864,15 @@ impl GpuDqnTrainer { self.submit_training_ops(online_dueling, online_branching)?; } + // ── Synchronize stream before readback ────────────────────── + // CUDA Graph replay and direct kernel launches are async. The stream + // must be synchronized before DtoH to ensure kernel writes are visible. + // Also required because graph capture disables event tracking, so + // cudarc's device_ptr() can't rely on write events for ordering. + self.stream.synchronize() + .map_err(|e| MLError::ModelError(format!("stream sync before readback: {e}")))?; + // ── Gather 2 scalars to host: total_loss + grad_norm ──────── - // Read each 1-element GPU buffer directly to host (2 × 4B DtoH). let mut loss_host = [0.0_f32; 1]; self.stream.memcpy_dtoh(&self.total_loss_buf, &mut loss_host) .map_err(|e| MLError::ModelError(format!("DtoH total_loss: {e}")))?;