From 1ebb2ac5b0b0c1f60554bf46e3c59d4f54c9c0a9 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Mon, 6 Apr 2026 20:29:29 +0200 Subject: [PATCH] debug: add full-step sync to isolate hanging GPU operation Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/ml/src/trainers/dqn/fused_training.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/ml/src/trainers/dqn/fused_training.rs b/crates/ml/src/trainers/dqn/fused_training.rs index bf7018dd7..8d06c0a19 100644 --- a/crates/ml/src/trainers/dqn/fused_training.rs +++ b/crates/ml/src/trainers/dqn/fused_training.rs @@ -792,7 +792,10 @@ impl FusedTrainingCtx { &self.stream, ).map_err(|e| anyhow::anyhow!("PER seg_tree_update: {e}"))?; - eprintln!("FUSED_DIAG: step {} — PER update done", self.steps_since_varmap_sync); + eprintln!("FUSED_DIAG: step {} — PER update done, syncing full step", self.steps_since_varmap_sync); + self.trainer.stream().synchronize() + .map_err(|e| anyhow::anyhow!("post-step sync: {e}"))?; + eprintln!("FUSED_DIAG: step {} — full step GPU sync complete", self.steps_since_varmap_sync); // Bookkeeping. agent.fused_post_step_gpu_bookkeeping() .map_err(|e| anyhow::anyhow!("Fused GPU bookkeeping: {e}"))?;