From 00fc16251868c7062f3e868e2f2bf41f406516ba Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Mon, 6 Apr 2026 20:22:36 +0200 Subject: [PATCH] debug: sync after each spectral norm to find hanging step Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/ml/src/trainers/dqn/fused_training.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/ml/src/trainers/dqn/fused_training.rs b/crates/ml/src/trainers/dqn/fused_training.rs index b48d1a614..bf7018dd7 100644 --- a/crates/ml/src/trainers/dqn/fused_training.rs +++ b/crates/ml/src/trainers/dqn/fused_training.rs @@ -685,7 +685,11 @@ impl FusedTrainingCtx { } else { self.trainer.apply_spectral_norm(&self.online_dueling, &self.online_branching) .map_err(|e| anyhow::anyhow!("Spectral norm (pre-forward): {e}"))?; - eprintln!("FUSED_DIAG: step {} — spectral norm applied, checking capture", self.steps_since_varmap_sync); + // Temporary: sync after spectral norm to detect if THIS step's kernel hangs + eprintln!("FUSED_DIAG: step {} — spectral norm submitted, syncing to verify", self.steps_since_varmap_sync); + self.trainer.stream().synchronize() + .map_err(|e| anyhow::anyhow!("spectral norm sync: {e}"))?; + eprintln!("FUSED_DIAG: step {} — spectral norm verified complete", self.steps_since_varmap_sync); if self.steps_since_varmap_sync == 1 { eprintln!("FUSED_DIAG: step {} — capturing graph_spectral", self.steps_since_varmap_sync); if let Err(e) = self.capture_graph_spectral() {