diff --git a/crates/ml/src/trainers/dqn/fused_training.rs b/crates/ml/src/trainers/dqn/fused_training.rs index f45d62503..22ad1a259 100644 --- a/crates/ml/src/trainers/dqn/fused_training.rs +++ b/crates/ml/src/trainers/dqn/fused_training.rs @@ -701,14 +701,6 @@ impl FusedTrainingCtx { self.trainer.run_nan_checks_post_forward(self.batch_size)?; - // Per-phase sync on step 0 only — catch the hang immediately - if step == 0 { - eprintln!("H100_STEP0: forward done, syncing"); - self.trainer.stream().synchronize() - .map_err(|e| anyhow::anyhow!("step0 post-forward sync: {e}"))?; - eprintln!("H100_STEP0: forward sync OK"); - } - // ── Step 2b: HER donor computation (outside graph_aux) ─────────── // Donor indices vary per step (random/future/final). The computation // fills her.donor_indices GPU buffer. graph_aux captures the relabel @@ -733,8 +725,6 @@ impl FusedTrainingCtx { } } - if step == 0 { eprintln!("H100_STEP0: HER done"); } - // ── Step 3: Auxiliary ops (graph_aux captured or replayed) ──────── if self.graph_aux.is_some() { self.pre_replay_state_update(agent); @@ -744,13 +734,6 @@ impl FusedTrainingCtx { self.submit_aux_ops(agent, gpu_batch)?; } - if step == 0 { - eprintln!("H100_STEP0: aux done, syncing"); - self.trainer.stream().synchronize() - .map_err(|e| anyhow::anyhow!("step0 post-aux sync: {e}"))?; - eprintln!("H100_STEP0: aux sync OK"); - } - // ── Step 4: Conditional ops (outside graph) ────────────────────── // Ensemble diversity (variable topology, complex forward passes). @@ -759,13 +742,6 @@ impl FusedTrainingCtx { tracing::warn!("Ensemble diversity step failed (non-fatal): {e}"); } } - if step == 0 { - eprintln!("H100_STEP0: ensemble done, syncing"); - self.trainer.stream().synchronize() - .map_err(|e| anyhow::anyhow!("step0 post-ensemble sync: {e}"))?; - eprintln!("H100_STEP0: ensemble sync OK"); - } - // Causal intervention. { let step = self.steps_since_varmap_sync; @@ -773,13 +749,6 @@ impl FusedTrainingCtx { tracing::warn!("Causal intervention failed (non-fatal): {e}"); } } - if step == 0 { - eprintln!("H100_STEP0: causal done, syncing"); - self.trainer.stream().synchronize() - .map_err(|e| anyhow::anyhow!("step0 post-causal sync: {e}"))?; - eprintln!("H100_STEP0: causal sync OK"); - } - // Gradient vaccine (1/10 steps). if self.steps_since_varmap_sync % 10 == 0 { if let Some(ref vaccine_batch) = self.pending_vaccine_batch { @@ -792,13 +761,6 @@ impl FusedTrainingCtx { } self.pending_vaccine_batch = None; - if step == 0 { - eprintln!("H100_STEP0: vaccine done (conditional ops complete), syncing"); - self.trainer.stream().synchronize() - .map_err(|e| anyhow::anyhow!("step0 post-conditional sync: {e}"))?; - eprintln!("H100_STEP0: conditional sync OK"); - } - // ── Step 5: Pruning + Adam ─────────────────────────────────────── self.trainer.apply_pruning_mask() .map_err(|e| anyhow::anyhow!("Pruning mask apply: {e}"))?; @@ -806,13 +768,6 @@ impl FusedTrainingCtx { let fused_result = self.trainer.replay_adam_and_readback() .map_err(|e| anyhow::anyhow!("graph_adam replay: {e}"))?; - if step == 0 { - eprintln!("H100_STEP0: adam done, syncing"); - self.trainer.stream().synchronize() - .map_err(|e| anyhow::anyhow!("step0 post-adam sync: {e}"))?; - eprintln!("H100_STEP0: adam sync OK"); - } - // ── Step 6: PER priority update ───────────────────────────────── agent.update_priorities_from_td( self.trainer.td_errors_buf(), @@ -820,13 +775,6 @@ impl FusedTrainingCtx { &self.stream, ).map_err(|e| anyhow::anyhow!("PER update: {e}"))?; - if step == 0 { - eprintln!("H100_STEP0: PER done, syncing"); - self.trainer.stream().synchronize() - .map_err(|e| anyhow::anyhow!("step0 post-PER sync: {e}"))?; - eprintln!("H100_STEP0: ALL STEP 0 SYNCS PASSED"); - } - // Bookkeeping. agent.fused_post_step_gpu_bookkeeping() .map_err(|e| anyhow::anyhow!("Fused GPU bookkeeping: {e}"))?; diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index 184220894..fbecdab13 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -1174,7 +1174,6 @@ impl DQNTrainer { } } - eprintln!("H100_LOOP: starting {num_training_steps} training steps"); for _step in 0..num_training_steps { let sample_start = std::time::Instant::now(); let (batch, vaccine_batch) = { @@ -1258,7 +1257,6 @@ impl DQNTrainer { } } - eprintln!("H100_LOOP: {} steps done, flushing readback", train_step_count); if let Some(ref mut fused) = self.fused_ctx { let _ = fused.flush_readback(); if let Ok(stats) = fused.flush_q_stats_readback() {