diff --git a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs index 6da91486c..3ad4d299a 100644 --- a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs +++ b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs @@ -2143,11 +2143,30 @@ impl GpuDqnTrainer { }; let grad_norm = self.debug_buffer_norm_f32(self.ptrs.grad_buf, self.total_params).unwrap_or(f32::NAN); + // Raw DtoH probe: read first 4 floats directly from grad_buf and grad_norm_buf. + // Bypasses all kernel infrastructure to verify the buffer actually has data. + let mut raw_grad = [0.0_f32; 4]; + let mut raw_norm = [0.0_f32; 1]; + unsafe { + cudarc::driver::sys::cuStreamSynchronize(self.stream.cu_stream()); + cudarc::driver::sys::cuMemcpyDtoH_v2( + raw_grad.as_mut_ptr().cast(), self.ptrs.grad_buf, 16, + ); + cudarc::driver::sys::cuMemcpyDtoH_v2( + raw_norm.as_mut_ptr().cast(), self.ptrs.grad_norm_buf, 4, + ); + } + tracing::warn!( step, d_val_norm, d_adv_norm, grad_norm, + raw_grad_0 = raw_grad[0], + raw_grad_1 = raw_grad[1], + raw_grad_2 = raw_grad[2], + raw_grad_3 = raw_grad[3], + raw_norm = raw_norm[0], c51_alpha = self.c51_alpha, "BUFFER_DIAG: per-step buffer norms after forward+backward" );