gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

diag(sp15-wave5): in-capture CAPTURE_PHASE_* checkpoints

Browse Source 
Smoke train-fp7xx printed all 16 Phase-6/7/8 checkpoints clean through
PER_PRIORITY_DONE. CAPTURE_DONE missing, exit changed 139→143 (SIGTERM)
— capture_training_graph hangs or takes ~40s past PER_PRIORITY_DONE
before Argo terminates the pod.

Adds 14 CAPTURE_PHASE_* checkpoints across the 12 child captures +
parent compose:
  BEGIN / PER_SAMPLE / COUNTERS / SPECTRAL / FORWARD / DDQN / AUX
  POST_AUX / ADAM_GRAD / ADAM_UPDATE / MAINTENANCE / IQL_MODULATE
  PER_PRIORITY / CHILDREN_STORED / PARENT_COMPOSED

Diagnostic-only.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-05-07 12:47:23 +02:00
parent a8d6c33040
commit bfc3ffa9dc
2 changed files with 17 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
crates/ml/src/trainers/dqn/fused_training.rs
View File

@@ -2856,40 +2856,47 @@ impl FusedTrainingCtx {
agent: &mut DQNAgentType, agent: &mut DQNAgentType,
gpu_batch: &crate::dqn::replay_buffer_type::GpuBatch, gpu_batch: &crate::dqn::replay_buffer_type::GpuBatch,
) -> Result<()> { ) -> Result<()> {
eprintln!("CAPTURE_PHASE_BEGIN");
// Capture PER sampling child (prefix scan + binary search + gather). // Capture PER sampling child (prefix scan + binary search + gather).
let per_sample = self.capture_child_graph("per_sample", |s| { let per_sample = self.capture_child_graph("per_sample", |s| {
agent.primary_dqn_mut().memory.gpu.sample_proportional(s.batch_size) agent.primary_dqn_mut().memory.gpu.sample_proportional(s.batch_size)
.map_err(|e| anyhow::anyhow!("per_sample capture: {e}")) .map_err(|e| anyhow::anyhow!("per_sample capture: {e}"))
.map(|_| ()) .map(|_| ())
})?; })?;
eprintln!("CAPTURE_PHASE_PER_SAMPLE_DONE");
// Capture counters child (GPU-side step counters + stochastic depth mask). // Capture counters child (GPU-side step counters + stochastic depth mask).
let counters = self.capture_child_graph("counters", |s| { let counters = self.capture_child_graph("counters", |s| {
s.submit_counters_ops() s.submit_counters_ops()
})?; })?;
eprintln!("CAPTURE_PHASE_COUNTERS_DONE");
// Capture spectral child. // Capture spectral child.
let spectral = self.capture_child_graph("spectral", |s| { let spectral = self.capture_child_graph("spectral", |s| {
s.trainer.apply_spectral_norm(&s.online_dueling, &s.online_branching) s.trainer.apply_spectral_norm(&s.online_dueling, &s.online_branching)
.map_err(|e| anyhow::anyhow!("{e}")) .map_err(|e| anyhow::anyhow!("{e}"))
})?; })?;
eprintln!("CAPTURE_PHASE_SPECTRAL_DONE");
// Capture forward child (cuBLAS trunk + ISV + loss + backward). // Capture forward child (cuBLAS trunk + ISV + loss + backward).
let forward = self.capture_child_graph("forward", |s| { let forward = self.capture_child_graph("forward", |s| {
s.trainer.submit_forward_ops_main() s.trainer.submit_forward_ops_main()
.map_err(|e| anyhow::anyhow!("{e}")) .map_err(|e| anyhow::anyhow!("{e}"))
})?; })?;
eprintln!("CAPTURE_PHASE_FORWARD_DONE");
// Capture DDQN child (Pass 3). // Capture DDQN child (Pass 3).
let ddqn = self.capture_child_graph("ddqn", |s| { let ddqn = self.capture_child_graph("ddqn", |s| {
s.trainer.submit_forward_ops_ddqn() s.trainer.submit_forward_ops_ddqn()
.map_err(|e| anyhow::anyhow!("{e}")) .map_err(|e| anyhow::anyhow!("{e}"))
})?; })?;
eprintln!("CAPTURE_PHASE_DDQN_DONE");
// Capture aux child (HER + EMA + IQL + IQN + attention + CQL). // Capture aux child (HER + EMA + IQL + IQN + attention + CQL).
let aux = self.capture_child_graph("aux", |s| { let aux = self.capture_child_graph("aux", |s| {
s.submit_aux_ops(agent, gpu_batch) s.submit_aux_ops(agent, gpu_batch)
})?; })?;
eprintln!("CAPTURE_PHASE_AUX_DONE");
// Capture post_aux child (selectivity + denoise + Q-stats + risk_sgd // Capture post_aux child (selectivity + denoise + Q-stats + risk_sgd
// + SP1 Phase B post-backward NaN checks). The post-backward NaN // + SP1 Phase B post-backward NaN checks). The post-backward NaN
@@ -2908,6 +2915,7 @@ impl FusedTrainingCtx {
s.trainer.run_nan_checks_post_backward() s.trainer.run_nan_checks_post_backward()
.map_err(|e| anyhow::anyhow!("{e}")) .map_err(|e| anyhow::anyhow!("{e}"))
})?; })?;
eprintln!("CAPTURE_PHASE_POST_AUX_DONE");
// adam_grad: mamba2 bwd + ofi_embed bwd + pruning + grad_norm // adam_grad: mamba2 bwd + ofi_embed bwd + pruning + grad_norm
// adam_update: Adam + unflatten + ISV // adam_update: Adam + unflatten + ISV
@@ -2926,6 +2934,7 @@ impl FusedTrainingCtx {
s.trainer.compute_grad_norm_for_adam() s.trainer.compute_grad_norm_for_adam()
.map_err(|e| anyhow::anyhow!("{e}")) .map_err(|e| anyhow::anyhow!("{e}"))
})?; })?;
eprintln!("CAPTURE_PHASE_ADAM_GRAD_DONE");
let adam = self.capture_child_graph("adam_update", |s| { let adam = self.capture_child_graph("adam_update", |s| {
s.trainer.submit_adam_ops(&s.online_dueling, &s.online_branching) s.trainer.submit_adam_ops(&s.online_dueling, &s.online_branching)
.map_err(|e| anyhow::anyhow!("{e}"))?; .map_err(|e| anyhow::anyhow!("{e}"))?;
@@ -2946,22 +2955,26 @@ impl FusedTrainingCtx {
s.trainer.update_isv_signals() s.trainer.update_isv_signals()
.map_err(|e| anyhow::anyhow!("{e}")) .map_err(|e| anyhow::anyhow!("{e}"))
})?; })?;
eprintln!("CAPTURE_PHASE_ADAM_UPDATE_DONE");
// Capture maintenance child (causal intervention + vaccine). // Capture maintenance child (causal intervention + vaccine).
let maintenance = self.capture_child_graph("maintenance", |s| { let maintenance = self.capture_child_graph("maintenance", |s| {
s.trainer.submit_maintenance_ops() s.trainer.submit_maintenance_ops()
.map_err(|e| anyhow::anyhow!("{e}")) .map_err(|e| anyhow::anyhow!("{e}"))
})?; })?;
eprintln!("CAPTURE_PHASE_MAINTENANCE_DONE");
// Capture IQL modulate child (advantage-weighted PER modulation). // Capture IQL modulate child (advantage-weighted PER modulation).
let iql_modulate = self.capture_child_graph("iql_modulate", |s| { let iql_modulate = self.capture_child_graph("iql_modulate", |s| {
s.submit_iql_modulate_ops(agent, gpu_batch) s.submit_iql_modulate_ops(agent, gpu_batch)
})?; })?;
eprintln!("CAPTURE_PHASE_IQL_MODULATE_DONE");
// Capture PER priority child (seg_tree update). // Capture PER priority child (seg_tree update).
let per_priority = self.capture_child_graph("per_priority", |s| { let per_priority = self.capture_child_graph("per_priority", |s| {
s.submit_per_priority_ops(agent) s.submit_per_priority_ops(agent)
})?; })?;
eprintln!("CAPTURE_PHASE_PER_PRIORITY_DONE");
// Store children. // Store children.
self.per_sample_child = Some(per_sample); self.per_sample_child = Some(per_sample);
@@ -2977,8 +2990,10 @@ impl FusedTrainingCtx {
self.iql_modulate_child = Some(iql_modulate); self.iql_modulate_child = Some(iql_modulate);
self.per_priority_child = Some(per_priority); self.per_priority_child = Some(per_priority);
eprintln!("CAPTURE_PHASE_CHILDREN_STORED");
// Compose parent graph from all children. // Compose parent graph from all children.
self.compose_parent_graph()?; self.compose_parent_graph()?;
eprintln!("CAPTURE_PHASE_PARENT_COMPOSED");
// Create eval-only forward exec for deterministic Q-value replay. // Create eval-only forward exec for deterministic Q-value replay.
self.create_eval_forward_exec()?; self.create_eval_forward_exec()?;

2
docs/dqn-wire-up-audit.md
View File

@@ -2,6 +2,8 @@
**Status:** Populated during Plan 1 Task 6 (A.5 orphan audit). Updated on every commit per Invariant 7. **Status:** Populated during Plan 1 Task 6 (A.5 orphan audit). Updated on every commit per Invariant 7.
SP15 Wave 5 SEGV diagnostic — checkpoints inside capture_training_graph (2026-05-07): smoke `train-fp7xx` (commit `a8d6c3304`) printed all 16 Phase-6/7/8 checkpoints cleanly through `STEP0_PHASE_PER_PRIORITY_DONE`. The `CAPTURE_DONE` checkpoint never printed, exit changed from 139 (SEGV) to 143 (SIGTERM) — process was killed externally ~40s after PER_PRIORITY_DONE, suggesting `capture_training_graph()` either hangs or runs too long. Added 12 sub-checkpoints across capture_training_graph: `CAPTURE_PHASE_BEGIN`, `CAPTURE_PHASE_PER_SAMPLE_DONE`, `CAPTURE_PHASE_COUNTERS_DONE`, `CAPTURE_PHASE_SPECTRAL_DONE`, `CAPTURE_PHASE_FORWARD_DONE`, `CAPTURE_PHASE_DDQN_DONE`, `CAPTURE_PHASE_AUX_DONE`, `CAPTURE_PHASE_POST_AUX_DONE`, `CAPTURE_PHASE_ADAM_GRAD_DONE`, `CAPTURE_PHASE_ADAM_UPDATE_DONE`, `CAPTURE_PHASE_MAINTENANCE_DONE`, `CAPTURE_PHASE_IQL_MODULATE_DONE`, `CAPTURE_PHASE_PER_PRIORITY_DONE`, `CAPTURE_PHASE_CHILDREN_STORED`, `CAPTURE_PHASE_PARENT_COMPOSED`. Will pinpoint which child capture or compose-step hangs/crashes. Diagnostic-only — no behavior change.
SP15 Wave 5 SEGV diagnostic — extend checkpoints into Phase 6/7/8 (2026-05-07): smoke `train-xq9hg` (commit `e9d9afbd6`) got past all 13 original checkpoints (`BEGIN` → `NAN_CHECKS_DONE`) cleanly, confirming the SEGV is downstream in Phase 6 (TLOB bwd + Adam, Mamba2 bwd + Adam, OFI embed bwd + Adam, pruning_mask, branch_grad_balance, grad_norm, submit_adam_ops, q_mag/dir_bin_means_reduce, update_isv_signals), Phase 7 (submit_maintenance_ops), or Phase 8 (submit_iql_modulate_ops, submit_per_priority_ops, capture_training_graph). 14 more `STEP0_PHASE_*` checkpoints added across these phases — the next L40S smoke will localize the SEGV to a single operation. Same diagnostic-only contract; will be reverted with the root-cause fix. SP15 Wave 5 SEGV diagnostic — extend checkpoints into Phase 6/7/8 (2026-05-07): smoke `train-xq9hg` (commit `e9d9afbd6`) got past all 13 original checkpoints (`BEGIN` → `NAN_CHECKS_DONE`) cleanly, confirming the SEGV is downstream in Phase 6 (TLOB bwd + Adam, Mamba2 bwd + Adam, OFI embed bwd + Adam, pruning_mask, branch_grad_balance, grad_norm, submit_adam_ops, q_mag/dir_bin_means_reduce, update_isv_signals), Phase 7 (submit_maintenance_ops), or Phase 8 (submit_iql_modulate_ops, submit_per_priority_ops, capture_training_graph). 14 more `STEP0_PHASE_*` checkpoints added across these phases — the next L40S smoke will localize the SEGV to a single operation. Same diagnostic-only contract; will be reverted with the root-cause fix.
SP15 Wave 5 SEGV diagnostic (2026-05-07): L40S smoke `train-jfbzr` on commit `23e9a1f78` segfaulted at exit 139 in the first `FusedTrainer::run_full_step` invocation after `"GPU training guard initialized (epoch loop)"` and before any HEALTH_DIAG output. Static analysis at commit `23e9a1f78` found no defects in the suspect SP15 changes (`2e37af29d` pointer wiring, `23e9a1f78` denoise stride, Wave 4.1b bn_concat_dim, Phase 1.3.b-followup per-env DD); local repro impossible because `test_data/futures-baseline/` lacks the `.fxcache` the production path requires. This commit adds `eprintln!` checkpoints at each phase boundary of `run_full_step`'s ungraphed step-0 path (between PER sample / PopArt / counters / spectral norm / TLOB forward / forward_main / DDQN / aux_ops / post_aux / NaN checks). stderr is line-flushed (vs the tracing JSON stdout formatter which buffers), so the last printed checkpoint pinpoints the SEGV site. Diagnostic-only — no behavior change. Will be reverted after the next L40S smoke localizes the SEGV and the root-cause fix lands. SP15 Wave 5 SEGV diagnostic (2026-05-07): L40S smoke `train-jfbzr` on commit `23e9a1f78` segfaulted at exit 139 in the first `FusedTrainer::run_full_step` invocation after `"GPU training guard initialized (epoch loop)"` and before any HEALTH_DIAG output. Static analysis at commit `23e9a1f78` found no defects in the suspect SP15 changes (`2e37af29d` pointer wiring, `23e9a1f78` denoise stride, Wave 4.1b bn_concat_dim, Phase 1.3.b-followup per-env DD); local repro impossible because `test_data/futures-baseline/` lacks the `.fxcache` the production path requires. This commit adds `eprintln!` checkpoints at each phase boundary of `run_full_step`'s ungraphed step-0 path (between PER sample / PopArt / counters / spectral norm / TLOB forward / forward_main / DDQN / aux_ops / post_aux / NaN checks). stderr is line-flushed (vs the tracing JSON stdout formatter which buffers), so the last printed checkpoint pinpoints the SEGV site. Diagnostic-only — no behavior change. Will be reverted after the next L40S smoke localizes the SEGV and the root-cause fix lands.