From 2f7828aefb3ce4ae362ad5e80f3fd425f92fc7d7 Mon Sep 17 00:00:00 2001
From: jgrusewski <jgrusewski@localhost>
Date: Mon, 20 Apr 2026 13:10:16 +0200
Subject: [PATCH] diag: GPU readback after OFI upload + state_gather to trace
 H100 zero OFI

Temporary diagnostic: readback ofi_gpu[0..20] after upload and
batch_states[66..84] after first state_gather. OFI works locally on
RTX 3050 but shows zeros on H100 with identical v4 cache.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 .../cuda_pipeline/gpu_experience_collector.rs | 31 ++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs b/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs
index cb5f76901..b8e93d58c 100644
--- a/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs
+++ b/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs
@@ -2051,6 +2051,22 @@ impl GpuExperienceCollector {
                     )))?;
             }
 
+            // One-shot diagnostic: after first state_gather, readback batch_states[0] to verify OFI
+            if t == 0 && self.last_experience_count == 0 {
+                let sd = self.state_dim;
+                let mut state0 = vec![0.0_f32; sd];
+                if let Ok(()) = self.stream.memcpy_dtoh(&self.batch_states, &mut state0) {
+                    let ofi_slice = &state0[66..84.min(sd)];
+                    let ofi_nonzero = ofi_slice.iter().filter(|&&v| v != 0.0).count();
+                    info!(
+                        "STATE_GATHER_DIAG: state[66..84]={:?}, nonzero={}/{}, ofi_ptr=0x{:x}, ofi_dim={}, total_bars={}",
+                        ofi_slice, ofi_nonzero, ofi_slice.len(),
+                        self.ofi_gpu.device_ptr(&self.stream).0,
+                        self.ofi_dim, config.total_bars,
+                    );
+                }
+            }
+
             // ── 2. cuBLAS Q-forward: batch_states → logits ──────────────
             // Graph the ENTIRE forward (bottleneck + Q-network) for deterministic cuBLAS.
             // Ungraphed cublasLtMatmul contaminates cuBLAS internal state, breaking
@@ -2923,11 +2939,24 @@ impl GpuExperienceCollector {
             return Err(MLError::ModelError("OFI data empty — cannot upload empty features to GPU".into()));
         }
         let gpu_buf = super::clone_htod_f32(&self.stream, ofi_flat)?;
+        // Readback verification: confirm GPU buffer contains non-zero OFI after upload
+        {
+            let verify_len = 20.min(ofi_flat.len());
+            let mut readback = vec![0.0_f32; verify_len];
+            if let Ok(()) = self.stream.memcpy_dtoh(&gpu_buf, &mut readback) {
+                let nonzero = readback.iter().filter(|&&v| v != 0.0).count();
+                info!(
+                    "OFI GPU readback (bar 0): {:?}, nonzero={}/{}",
+                    &readback[..8.min(verify_len)], nonzero, verify_len
+                );
+            }
+        }
         info!(
-            "OFI features uploaded to GPU: {} bars x {} dims ({:.1} KB)",
+            "OFI features uploaded to GPU: {} bars x {} dims ({:.1} KB), ptr=0x{:x}",
             ofi_flat.len() / self.ofi_dim,
             self.ofi_dim,
             (ofi_flat.len() * 4) as f64 / 1024.0,
+            gpu_buf.device_ptr(&self.stream).0,
         );
         self.ofi_gpu = gpu_buf;
         Ok(())