From 9dce707348b82f02a6dd8d3cdce31c8a4f5b6516 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Mon, 13 Apr 2026 23:30:17 +0200 Subject: [PATCH] =?UTF-8?q?diag:=20raw=20DtoH=20probe=20on=20grad=5Fbuf=20?= =?UTF-8?q?=E2=80=94=20bypass=20kernel=20to=20confirm=20H100=20data?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.6 (1M context) --- .../ml/src/cuda_pipeline/gpu_dqn_trainer.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs index 6da91486c..3ad4d299a 100644 --- a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs +++ b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs @@ -2143,11 +2143,30 @@ impl GpuDqnTrainer { }; let grad_norm = self.debug_buffer_norm_f32(self.ptrs.grad_buf, self.total_params).unwrap_or(f32::NAN); + // Raw DtoH probe: read first 4 floats directly from grad_buf and grad_norm_buf. + // Bypasses all kernel infrastructure to verify the buffer actually has data. + let mut raw_grad = [0.0_f32; 4]; + let mut raw_norm = [0.0_f32; 1]; + unsafe { + cudarc::driver::sys::cuStreamSynchronize(self.stream.cu_stream()); + cudarc::driver::sys::cuMemcpyDtoH_v2( + raw_grad.as_mut_ptr().cast(), self.ptrs.grad_buf, 16, + ); + cudarc::driver::sys::cuMemcpyDtoH_v2( + raw_norm.as_mut_ptr().cast(), self.ptrs.grad_norm_buf, 4, + ); + } + tracing::warn!( step, d_val_norm, d_adv_norm, grad_norm, + raw_grad_0 = raw_grad[0], + raw_grad_1 = raw_grad[1], + raw_grad_2 = raw_grad[2], + raw_grad_3 = raw_grad[3], + raw_norm = raw_norm[0], c51_alpha = self.c51_alpha, "BUFFER_DIAG: per-step buffer norms after forward+backward" );