diff --git a/crates/ml/src/trainers/dqn/fused_training.rs b/crates/ml/src/trainers/dqn/fused_training.rs index 87e7ac8e3..e9a8dd081 100644 --- a/crates/ml/src/trainers/dqn/fused_training.rs +++ b/crates/ml/src/trainers/dqn/fused_training.rs @@ -2856,40 +2856,47 @@ impl FusedTrainingCtx { agent: &mut DQNAgentType, gpu_batch: &crate::dqn::replay_buffer_type::GpuBatch, ) -> Result<()> { + eprintln!("CAPTURE_PHASE_BEGIN"); // Capture PER sampling child (prefix scan + binary search + gather). let per_sample = self.capture_child_graph("per_sample", |s| { agent.primary_dqn_mut().memory.gpu.sample_proportional(s.batch_size) .map_err(|e| anyhow::anyhow!("per_sample capture: {e}")) .map(|_| ()) })?; + eprintln!("CAPTURE_PHASE_PER_SAMPLE_DONE"); // Capture counters child (GPU-side step counters + stochastic depth mask). let counters = self.capture_child_graph("counters", |s| { s.submit_counters_ops() })?; + eprintln!("CAPTURE_PHASE_COUNTERS_DONE"); // Capture spectral child. let spectral = self.capture_child_graph("spectral", |s| { s.trainer.apply_spectral_norm(&s.online_dueling, &s.online_branching) .map_err(|e| anyhow::anyhow!("{e}")) })?; + eprintln!("CAPTURE_PHASE_SPECTRAL_DONE"); // Capture forward child (cuBLAS trunk + ISV + loss + backward). let forward = self.capture_child_graph("forward", |s| { s.trainer.submit_forward_ops_main() .map_err(|e| anyhow::anyhow!("{e}")) })?; + eprintln!("CAPTURE_PHASE_FORWARD_DONE"); // Capture DDQN child (Pass 3). let ddqn = self.capture_child_graph("ddqn", |s| { s.trainer.submit_forward_ops_ddqn() .map_err(|e| anyhow::anyhow!("{e}")) })?; + eprintln!("CAPTURE_PHASE_DDQN_DONE"); // Capture aux child (HER + EMA + IQL + IQN + attention + CQL). let aux = self.capture_child_graph("aux", |s| { s.submit_aux_ops(agent, gpu_batch) })?; + eprintln!("CAPTURE_PHASE_AUX_DONE"); // Capture post_aux child (selectivity + denoise + Q-stats + risk_sgd // + SP1 Phase B post-backward NaN checks). The post-backward NaN @@ -2908,6 +2915,7 @@ impl FusedTrainingCtx { s.trainer.run_nan_checks_post_backward() .map_err(|e| anyhow::anyhow!("{e}")) })?; + eprintln!("CAPTURE_PHASE_POST_AUX_DONE"); // adam_grad: mamba2 bwd + ofi_embed bwd + pruning + grad_norm // adam_update: Adam + unflatten + ISV @@ -2926,6 +2934,7 @@ impl FusedTrainingCtx { s.trainer.compute_grad_norm_for_adam() .map_err(|e| anyhow::anyhow!("{e}")) })?; + eprintln!("CAPTURE_PHASE_ADAM_GRAD_DONE"); let adam = self.capture_child_graph("adam_update", |s| { s.trainer.submit_adam_ops(&s.online_dueling, &s.online_branching) .map_err(|e| anyhow::anyhow!("{e}"))?; @@ -2946,22 +2955,26 @@ impl FusedTrainingCtx { s.trainer.update_isv_signals() .map_err(|e| anyhow::anyhow!("{e}")) })?; + eprintln!("CAPTURE_PHASE_ADAM_UPDATE_DONE"); // Capture maintenance child (causal intervention + vaccine). let maintenance = self.capture_child_graph("maintenance", |s| { s.trainer.submit_maintenance_ops() .map_err(|e| anyhow::anyhow!("{e}")) })?; + eprintln!("CAPTURE_PHASE_MAINTENANCE_DONE"); // Capture IQL modulate child (advantage-weighted PER modulation). let iql_modulate = self.capture_child_graph("iql_modulate", |s| { s.submit_iql_modulate_ops(agent, gpu_batch) })?; + eprintln!("CAPTURE_PHASE_IQL_MODULATE_DONE"); // Capture PER priority child (seg_tree update). let per_priority = self.capture_child_graph("per_priority", |s| { s.submit_per_priority_ops(agent) })?; + eprintln!("CAPTURE_PHASE_PER_PRIORITY_DONE"); // Store children. self.per_sample_child = Some(per_sample); @@ -2977,8 +2990,10 @@ impl FusedTrainingCtx { self.iql_modulate_child = Some(iql_modulate); self.per_priority_child = Some(per_priority); + eprintln!("CAPTURE_PHASE_CHILDREN_STORED"); // Compose parent graph from all children. self.compose_parent_graph()?; + eprintln!("CAPTURE_PHASE_PARENT_COMPOSED"); // Create eval-only forward exec for deterministic Q-value replay. self.create_eval_forward_exec()?; diff --git a/docs/dqn-wire-up-audit.md b/docs/dqn-wire-up-audit.md index 6f5d41349..9c52e51bd 100644 --- a/docs/dqn-wire-up-audit.md +++ b/docs/dqn-wire-up-audit.md @@ -2,6 +2,8 @@ **Status:** Populated during Plan 1 Task 6 (A.5 orphan audit). Updated on every commit per Invariant 7. +SP15 Wave 5 SEGV diagnostic — checkpoints inside capture_training_graph (2026-05-07): smoke `train-fp7xx` (commit `a8d6c3304`) printed all 16 Phase-6/7/8 checkpoints cleanly through `STEP0_PHASE_PER_PRIORITY_DONE`. The `CAPTURE_DONE` checkpoint never printed, exit changed from 139 (SEGV) to 143 (SIGTERM) — process was killed externally ~40s after PER_PRIORITY_DONE, suggesting `capture_training_graph()` either hangs or runs too long. Added 12 sub-checkpoints across capture_training_graph: `CAPTURE_PHASE_BEGIN`, `CAPTURE_PHASE_PER_SAMPLE_DONE`, `CAPTURE_PHASE_COUNTERS_DONE`, `CAPTURE_PHASE_SPECTRAL_DONE`, `CAPTURE_PHASE_FORWARD_DONE`, `CAPTURE_PHASE_DDQN_DONE`, `CAPTURE_PHASE_AUX_DONE`, `CAPTURE_PHASE_POST_AUX_DONE`, `CAPTURE_PHASE_ADAM_GRAD_DONE`, `CAPTURE_PHASE_ADAM_UPDATE_DONE`, `CAPTURE_PHASE_MAINTENANCE_DONE`, `CAPTURE_PHASE_IQL_MODULATE_DONE`, `CAPTURE_PHASE_PER_PRIORITY_DONE`, `CAPTURE_PHASE_CHILDREN_STORED`, `CAPTURE_PHASE_PARENT_COMPOSED`. Will pinpoint which child capture or compose-step hangs/crashes. Diagnostic-only — no behavior change. + SP15 Wave 5 SEGV diagnostic — extend checkpoints into Phase 6/7/8 (2026-05-07): smoke `train-xq9hg` (commit `e9d9afbd6`) got past all 13 original checkpoints (`BEGIN` → `NAN_CHECKS_DONE`) cleanly, confirming the SEGV is downstream in Phase 6 (TLOB bwd + Adam, Mamba2 bwd + Adam, OFI embed bwd + Adam, pruning_mask, branch_grad_balance, grad_norm, submit_adam_ops, q_mag/dir_bin_means_reduce, update_isv_signals), Phase 7 (submit_maintenance_ops), or Phase 8 (submit_iql_modulate_ops, submit_per_priority_ops, capture_training_graph). 14 more `STEP0_PHASE_*` checkpoints added across these phases — the next L40S smoke will localize the SEGV to a single operation. Same diagnostic-only contract; will be reverted with the root-cause fix. SP15 Wave 5 SEGV diagnostic (2026-05-07): L40S smoke `train-jfbzr` on commit `23e9a1f78` segfaulted at exit 139 in the first `FusedTrainer::run_full_step` invocation after `"GPU training guard initialized (epoch loop)"` and before any HEALTH_DIAG output. Static analysis at commit `23e9a1f78` found no defects in the suspect SP15 changes (`2e37af29d` pointer wiring, `23e9a1f78` denoise stride, Wave 4.1b bn_concat_dim, Phase 1.3.b-followup per-env DD); local repro impossible because `test_data/futures-baseline/` lacks the `.fxcache` the production path requires. This commit adds `eprintln!` checkpoints at each phase boundary of `run_full_step`'s ungraphed step-0 path (between PER sample / PopArt / counters / spectral norm / TLOB forward / forward_main / DDQN / aux_ops / post_aux / NaN checks). stderr is line-flushed (vs the tracing JSON stdout formatter which buffers), so the last printed checkpoint pinpoints the SEGV site. Diagnostic-only — no behavior change. Will be reverted after the next L40S smoke localizes the SEGV and the root-cause fix lands.