gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

debug: add full-step sync to isolate hanging GPU operation

Browse Source 
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-04-06 20:29:29 +02:00
parent 00fc162518
commit 1ebb2ac5b0
1 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
crates/ml/src/trainers/dqn/fused_training.rs
View File

@@ -792,7 +792,10 @@ impl FusedTrainingCtx {
&self.stream,
).map_err(|e| anyhow::anyhow!("PER seg_tree_update: {e}"))?;
eprintln!("FUSED_DIAG: step {} — PER update done", self.steps_since_varmap_sync);
eprintln!("FUSED_DIAG: step {} — PER update done, syncing full step", self.steps_since_varmap_sync);
self.trainer.stream().synchronize()
.map_err(|e| anyhow::anyhow!("post-step sync: {e}"))?;
eprintln!("FUSED_DIAG: step {} — full step GPU sync complete", self.steps_since_varmap_sync);
// Bookkeeping.
agent.fused_post_step_gpu_bookkeeping()
.map_err(|e| anyhow::anyhow!("Fused GPU bookkeeping: {e}"))?;