gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

diag: GPU readback after OFI upload + state_gather to trace H100 zero OFI

Browse Source 
Temporary diagnostic: readback ofi_gpu[0..20] after upload and
batch_states[66..84] after first state_gather. OFI works locally on
RTX 3050 but shows zeros on H100 with identical v4 cache.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-04-20 13:10:16 +02:00
parent 602628f698
commit 2f7828aefb
1 changed files with 30 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
crates/ml/src/cuda_pipeline/gpu_experience_collector.rs
View File

@@ -2051,6 +2051,22 @@ impl GpuExperienceCollector {
)))?;
}
// One-shot diagnostic: after first state_gather, readback batch_states[0] to verify OFI
if t == 0 && self.last_experience_count == 0 {
let sd = self.state_dim;
let mut state0 = vec![0.0_f32; sd];
if let Ok(()) = self.stream.memcpy_dtoh(&self.batch_states, &mut state0) {
let ofi_slice = &state0[66..84.min(sd)];
let ofi_nonzero = ofi_slice.iter().filter(|&&v| v != 0.0).count();
info!(
"STATE_GATHER_DIAG: state[66..84]={:?}, nonzero={}/{}, ofi_ptr=0x{:x}, ofi_dim={}, total_bars={}",
ofi_slice, ofi_nonzero, ofi_slice.len(),
self.ofi_gpu.device_ptr(&self.stream).0,
self.ofi_dim, config.total_bars,
);
}
}
// ── 2. cuBLAS Q-forward: batch_states → logits ──────────────
// Graph the ENTIRE forward (bottleneck + Q-network) for deterministic cuBLAS.
// Ungraphed cublasLtMatmul contaminates cuBLAS internal state, breaking
@@ -2923,11 +2939,24 @@ impl GpuExperienceCollector {
return Err(MLError::ModelError("OFI data empty — cannot upload empty features to GPU".into()));
}
let gpu_buf = super::clone_htod_f32(&self.stream, ofi_flat)?;
// Readback verification: confirm GPU buffer contains non-zero OFI after upload
{
let verify_len = 20.min(ofi_flat.len());
let mut readback = vec![0.0_f32; verify_len];
if let Ok(()) = self.stream.memcpy_dtoh(&gpu_buf, &mut readback) {
let nonzero = readback.iter().filter(|&&v| v != 0.0).count();
info!(
"OFI GPU readback (bar 0): {:?}, nonzero={}/{}",
&readback[..8.min(verify_len)], nonzero, verify_len
);
}
}
info!(
"OFI features uploaded to GPU: {} bars x {} dims ({:.1} KB)",
"OFI features uploaded to GPU: {} bars x {} dims ({:.1} KB), ptr=0x{:x}",
ofi_flat.len() / self.ofi_dim,
self.ofi_dim,
(ofi_flat.len() * 4) as f64 / 1024.0,
gpu_buf.device_ptr(&self.stream).0,
);
self.ofi_gpu = gpu_buf;
Ok(())