debug: add per-step timing logs to identify hang vs slow compute

Logs step_ms for first 5 steps and every 50th step thereafter.
If steps aren't completing, the graph is hanging.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-04-18 12:56:25 +02:00
parent 93ce3e118b
commit 8d88c63434

View File

@@ -1505,7 +1505,15 @@ impl DQNTrainer {
let _fused_result = fused.run_full_step(&batch, &mut *agent, &self.device)
.context("Fused CUDA training step failed")?;
fused_total_us += fused_start.elapsed().as_micros() as u64;
let step_us = fused_start.elapsed().as_micros() as u64;
fused_total_us += step_us;
if train_step_count < 5 || train_step_count % 50 == 0 {
tracing::info!(
step = train_step_count,
step_ms = step_us / 1000,
"STEP_TIMING"
);
}
let guard_start = std::time::Instant::now();
// Run guard every 5th step — NaN/loss checks add ~12ms/step overhead.