diff --git a/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs b/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs index cb5f76901..b8e93d58c 100644 --- a/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs +++ b/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs @@ -2051,6 +2051,22 @@ impl GpuExperienceCollector { )))?; } + // One-shot diagnostic: after first state_gather, readback batch_states[0] to verify OFI + if t == 0 && self.last_experience_count == 0 { + let sd = self.state_dim; + let mut state0 = vec![0.0_f32; sd]; + if let Ok(()) = self.stream.memcpy_dtoh(&self.batch_states, &mut state0) { + let ofi_slice = &state0[66..84.min(sd)]; + let ofi_nonzero = ofi_slice.iter().filter(|&&v| v != 0.0).count(); + info!( + "STATE_GATHER_DIAG: state[66..84]={:?}, nonzero={}/{}, ofi_ptr=0x{:x}, ofi_dim={}, total_bars={}", + ofi_slice, ofi_nonzero, ofi_slice.len(), + self.ofi_gpu.device_ptr(&self.stream).0, + self.ofi_dim, config.total_bars, + ); + } + } + // ── 2. cuBLAS Q-forward: batch_states → logits ────────────── // Graph the ENTIRE forward (bottleneck + Q-network) for deterministic cuBLAS. // Ungraphed cublasLtMatmul contaminates cuBLAS internal state, breaking @@ -2923,11 +2939,24 @@ impl GpuExperienceCollector { return Err(MLError::ModelError("OFI data empty — cannot upload empty features to GPU".into())); } let gpu_buf = super::clone_htod_f32(&self.stream, ofi_flat)?; + // Readback verification: confirm GPU buffer contains non-zero OFI after upload + { + let verify_len = 20.min(ofi_flat.len()); + let mut readback = vec![0.0_f32; verify_len]; + if let Ok(()) = self.stream.memcpy_dtoh(&gpu_buf, &mut readback) { + let nonzero = readback.iter().filter(|&&v| v != 0.0).count(); + info!( + "OFI GPU readback (bar 0): {:?}, nonzero={}/{}", + &readback[..8.min(verify_len)], nonzero, verify_len + ); + } + } info!( - "OFI features uploaded to GPU: {} bars x {} dims ({:.1} KB)", + "OFI features uploaded to GPU: {} bars x {} dims ({:.1} KB), ptr=0x{:x}", ofi_flat.len() / self.ofi_dim, self.ofi_dim, (ofi_flat.len() * 4) as f64 / 1024.0, + gpu_buf.device_ptr(&self.stream).0, ); self.ofi_gpu = gpu_buf; Ok(())