gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

debug: sync after each spectral norm to find hanging step

Browse Source 
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-04-06 20:22:36 +02:00
parent 1b5276a821
commit 00fc162518
1 changed files with 5 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
crates/ml/src/trainers/dqn/fused_training.rs
View File

@@ -685,7 +685,11 @@ impl FusedTrainingCtx {
} else {
self.trainer.apply_spectral_norm(&self.online_dueling, &self.online_branching)
.map_err(|e| anyhow::anyhow!("Spectral norm (pre-forward): {e}"))?;
eprintln!("FUSED_DIAG: step {} — spectral norm applied, checking capture", self.steps_since_varmap_sync);
// Temporary: sync after spectral norm to detect if THIS step's kernel hangs
eprintln!("FUSED_DIAG: step {} — spectral norm submitted, syncing to verify", self.steps_since_varmap_sync);
self.trainer.stream().synchronize()
.map_err(|e| anyhow::anyhow!("spectral norm sync: {e}"))?;
eprintln!("FUSED_DIAG: step {} — spectral norm verified complete", self.steps_since_varmap_sync);
if self.steps_since_varmap_sync == 1 {
eprintln!("FUSED_DIAG: step {} — capturing graph_spectral", self.steps_since_varmap_sync);
if let Err(e) = self.capture_graph_spectral() {