gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

diag: raw DtoH probe on grad_buf — bypass kernel to confirm H100 data

Browse Source 
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-04-13 23:30:17 +02:00
parent fee5a7cb3a
commit 9dce707348
1 changed files with 19 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs
View File

@@ -2143,11 +2143,30 @@ impl GpuDqnTrainer {
};
let grad_norm = self.debug_buffer_norm_f32(self.ptrs.grad_buf, self.total_params).unwrap_or(f32::NAN);
// Raw DtoH probe: read first 4 floats directly from grad_buf and grad_norm_buf.
// Bypasses all kernel infrastructure to verify the buffer actually has data.
let mut raw_grad = [0.0_f32; 4];
let mut raw_norm = [0.0_f32; 1];
unsafe {
cudarc::driver::sys::cuStreamSynchronize(self.stream.cu_stream());
cudarc::driver::sys::cuMemcpyDtoH_v2(
raw_grad.as_mut_ptr().cast(), self.ptrs.grad_buf, 16,
);
cudarc::driver::sys::cuMemcpyDtoH_v2(
raw_norm.as_mut_ptr().cast(), self.ptrs.grad_norm_buf, 4,
);
}
tracing::warn!(
step,
d_val_norm,
d_adv_norm,
grad_norm,
raw_grad_0 = raw_grad[0],
raw_grad_1 = raw_grad[1],
raw_grad_2 = raw_grad[2],
raw_grad_3 = raw_grad[3],
raw_norm = raw_norm[0],
c51_alpha = self.c51_alpha,
"BUFFER_DIAG: per-step buffer norms after forward+backward"
);