gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cleanup: remove all H100 diagnostic eprintln and step-0 syncs

Browse Source 
Removed 19 diagnostic eprintln + 7 stream.synchronize() calls that were
added during H100 hang debugging. All hang root causes are now fixed:
- grad_norm atomicAdd → two-phase reduction
- vaccine dot+norm atomicAdd → two-phase reduction
- C51 mixup spin-wait barrier → split into two kernels

Zero diagnostic overhead in the training hot loop.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-04-06 22:30:08 +02:00
parent c0a3464017
commit e32afd62fb
2 changed files with 0 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
crates/ml/src/trainers/dqn/fused_training.rs
View File

@@ -701,14 +701,6 @@ impl FusedTrainingCtx {
self.trainer.run_nan_checks_post_forward(self.batch_size)?;
// Per-phase sync on step 0 only — catch the hang immediately
if step == 0 {
eprintln!("H100_STEP0: forward done, syncing");
self.trainer.stream().synchronize()
.map_err(|e| anyhow::anyhow!("step0 post-forward sync: {e}"))?;
eprintln!("H100_STEP0: forward sync OK");
}
// ── Step 2b: HER donor computation (outside graph_aux) ───────────
// Donor indices vary per step (random/future/final). The computation
// fills her.donor_indices GPU buffer. graph_aux captures the relabel
@@ -733,8 +725,6 @@ impl FusedTrainingCtx {
}
}
if step == 0 { eprintln!("H100_STEP0: HER done"); }
// ── Step 3: Auxiliary ops (graph_aux captured or replayed) ────────
if self.graph_aux.is_some() {
self.pre_replay_state_update(agent);
@@ -744,13 +734,6 @@ impl FusedTrainingCtx {
self.submit_aux_ops(agent, gpu_batch)?;
}
if step == 0 {
eprintln!("H100_STEP0: aux done, syncing");
self.trainer.stream().synchronize()
.map_err(|e| anyhow::anyhow!("step0 post-aux sync: {e}"))?;
eprintln!("H100_STEP0: aux sync OK");
}
// ── Step 4: Conditional ops (outside graph) ──────────────────────
// Ensemble diversity (variable topology, complex forward passes).
@@ -759,13 +742,6 @@ impl FusedTrainingCtx {
tracing::warn!("Ensemble diversity step failed (non-fatal): {e}");
}
}
if step == 0 {
eprintln!("H100_STEP0: ensemble done, syncing");
self.trainer.stream().synchronize()
.map_err(|e| anyhow::anyhow!("step0 post-ensemble sync: {e}"))?;
eprintln!("H100_STEP0: ensemble sync OK");
}
// Causal intervention.
{
let step = self.steps_since_varmap_sync;
@@ -773,13 +749,6 @@ impl FusedTrainingCtx {
tracing::warn!("Causal intervention failed (non-fatal): {e}");
}
}
if step == 0 {
eprintln!("H100_STEP0: causal done, syncing");
self.trainer.stream().synchronize()
.map_err(|e| anyhow::anyhow!("step0 post-causal sync: {e}"))?;
eprintln!("H100_STEP0: causal sync OK");
}
// Gradient vaccine (1/10 steps).
if self.steps_since_varmap_sync % 10 == 0 {
if let Some(ref vaccine_batch) = self.pending_vaccine_batch {
@@ -792,13 +761,6 @@ impl FusedTrainingCtx {
}
self.pending_vaccine_batch = None;
if step == 0 {
eprintln!("H100_STEP0: vaccine done (conditional ops complete), syncing");
self.trainer.stream().synchronize()
.map_err(|e| anyhow::anyhow!("step0 post-conditional sync: {e}"))?;
eprintln!("H100_STEP0: conditional sync OK");
}
// ── Step 5: Pruning + Adam ───────────────────────────────────────
self.trainer.apply_pruning_mask()
.map_err(|e| anyhow::anyhow!("Pruning mask apply: {e}"))?;
@@ -806,13 +768,6 @@ impl FusedTrainingCtx {
let fused_result = self.trainer.replay_adam_and_readback()
.map_err(|e| anyhow::anyhow!("graph_adam replay: {e}"))?;
if step == 0 {
eprintln!("H100_STEP0: adam done, syncing");
self.trainer.stream().synchronize()
.map_err(|e| anyhow::anyhow!("step0 post-adam sync: {e}"))?;
eprintln!("H100_STEP0: adam sync OK");
}
// ── Step 6: PER priority update ─────────────────────────────────
agent.update_priorities_from_td(
self.trainer.td_errors_buf(),
@@ -820,13 +775,6 @@ impl FusedTrainingCtx {
&self.stream,
).map_err(|e| anyhow::anyhow!("PER update: {e}"))?;
if step == 0 {
eprintln!("H100_STEP0: PER done, syncing");
self.trainer.stream().synchronize()
.map_err(|e| anyhow::anyhow!("step0 post-PER sync: {e}"))?;
eprintln!("H100_STEP0: ALL STEP 0 SYNCS PASSED");
}
// Bookkeeping.
agent.fused_post_step_gpu_bookkeeping()
.map_err(|e| anyhow::anyhow!("Fused GPU bookkeeping: {e}"))?;

2
crates/ml/src/trainers/dqn/trainer/training_loop.rs
View File

@@ -1174,7 +1174,6 @@ impl DQNTrainer {
}
}
eprintln!("H100_LOOP: starting {num_training_steps} training steps");
for _step in 0..num_training_steps {
let sample_start = std::time::Instant::now();
let (batch, vaccine_batch) = {
@@ -1258,7 +1257,6 @@ impl DQNTrainer {
}
}
eprintln!("H100_LOOP: {} steps done, flushing readback", train_step_count);
if let Some(ref mut fused) = self.fused_ctx {
let _ = fused.flush_readback();
if let Ok(stats) = fused.flush_q_stats_readback() {