diag(sp15-wave5): in-capture CAPTURE_PHASE_* checkpoints

Smoke train-fp7xx printed all 16 Phase-6/7/8 checkpoints clean through
PER_PRIORITY_DONE. CAPTURE_DONE missing, exit changed 139→143 (SIGTERM)
— capture_training_graph hangs or takes ~40s past PER_PRIORITY_DONE
before Argo terminates the pod.

Adds 14 CAPTURE_PHASE_* checkpoints across the 12 child captures +
parent compose:
  BEGIN / PER_SAMPLE / COUNTERS / SPECTRAL / FORWARD / DDQN / AUX
  POST_AUX / ADAM_GRAD / ADAM_UPDATE / MAINTENANCE / IQL_MODULATE
  PER_PRIORITY / CHILDREN_STORED / PARENT_COMPOSED

Diagnostic-only.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-05-07 12:47:23 +02:00
parent a8d6c33040
commit bfc3ffa9dc
2 changed files with 17 additions and 0 deletions

View File

@@ -2856,40 +2856,47 @@ impl FusedTrainingCtx {
agent: &mut DQNAgentType,
gpu_batch: &crate::dqn::replay_buffer_type::GpuBatch,
) -> Result<()> {
eprintln!("CAPTURE_PHASE_BEGIN");
// Capture PER sampling child (prefix scan + binary search + gather).
let per_sample = self.capture_child_graph("per_sample", |s| {
agent.primary_dqn_mut().memory.gpu.sample_proportional(s.batch_size)
.map_err(|e| anyhow::anyhow!("per_sample capture: {e}"))
.map(|_| ())
})?;
eprintln!("CAPTURE_PHASE_PER_SAMPLE_DONE");
// Capture counters child (GPU-side step counters + stochastic depth mask).
let counters = self.capture_child_graph("counters", |s| {
s.submit_counters_ops()
})?;
eprintln!("CAPTURE_PHASE_COUNTERS_DONE");
// Capture spectral child.
let spectral = self.capture_child_graph("spectral", |s| {
s.trainer.apply_spectral_norm(&s.online_dueling, &s.online_branching)
.map_err(|e| anyhow::anyhow!("{e}"))
})?;
eprintln!("CAPTURE_PHASE_SPECTRAL_DONE");
// Capture forward child (cuBLAS trunk + ISV + loss + backward).
let forward = self.capture_child_graph("forward", |s| {
s.trainer.submit_forward_ops_main()
.map_err(|e| anyhow::anyhow!("{e}"))
})?;
eprintln!("CAPTURE_PHASE_FORWARD_DONE");
// Capture DDQN child (Pass 3).
let ddqn = self.capture_child_graph("ddqn", |s| {
s.trainer.submit_forward_ops_ddqn()
.map_err(|e| anyhow::anyhow!("{e}"))
})?;
eprintln!("CAPTURE_PHASE_DDQN_DONE");
// Capture aux child (HER + EMA + IQL + IQN + attention + CQL).
let aux = self.capture_child_graph("aux", |s| {
s.submit_aux_ops(agent, gpu_batch)
})?;
eprintln!("CAPTURE_PHASE_AUX_DONE");
// Capture post_aux child (selectivity + denoise + Q-stats + risk_sgd
// + SP1 Phase B post-backward NaN checks). The post-backward NaN
@@ -2908,6 +2915,7 @@ impl FusedTrainingCtx {
s.trainer.run_nan_checks_post_backward()
.map_err(|e| anyhow::anyhow!("{e}"))
})?;
eprintln!("CAPTURE_PHASE_POST_AUX_DONE");
// adam_grad: mamba2 bwd + ofi_embed bwd + pruning + grad_norm
// adam_update: Adam + unflatten + ISV
@@ -2926,6 +2934,7 @@ impl FusedTrainingCtx {
s.trainer.compute_grad_norm_for_adam()
.map_err(|e| anyhow::anyhow!("{e}"))
})?;
eprintln!("CAPTURE_PHASE_ADAM_GRAD_DONE");
let adam = self.capture_child_graph("adam_update", |s| {
s.trainer.submit_adam_ops(&s.online_dueling, &s.online_branching)
.map_err(|e| anyhow::anyhow!("{e}"))?;
@@ -2946,22 +2955,26 @@ impl FusedTrainingCtx {
s.trainer.update_isv_signals()
.map_err(|e| anyhow::anyhow!("{e}"))
})?;
eprintln!("CAPTURE_PHASE_ADAM_UPDATE_DONE");
// Capture maintenance child (causal intervention + vaccine).
let maintenance = self.capture_child_graph("maintenance", |s| {
s.trainer.submit_maintenance_ops()
.map_err(|e| anyhow::anyhow!("{e}"))
})?;
eprintln!("CAPTURE_PHASE_MAINTENANCE_DONE");
// Capture IQL modulate child (advantage-weighted PER modulation).
let iql_modulate = self.capture_child_graph("iql_modulate", |s| {
s.submit_iql_modulate_ops(agent, gpu_batch)
})?;
eprintln!("CAPTURE_PHASE_IQL_MODULATE_DONE");
// Capture PER priority child (seg_tree update).
let per_priority = self.capture_child_graph("per_priority", |s| {
s.submit_per_priority_ops(agent)
})?;
eprintln!("CAPTURE_PHASE_PER_PRIORITY_DONE");
// Store children.
self.per_sample_child = Some(per_sample);
@@ -2977,8 +2990,10 @@ impl FusedTrainingCtx {
self.iql_modulate_child = Some(iql_modulate);
self.per_priority_child = Some(per_priority);
eprintln!("CAPTURE_PHASE_CHILDREN_STORED");
// Compose parent graph from all children.
self.compose_parent_graph()?;
eprintln!("CAPTURE_PHASE_PARENT_COMPOSED");
// Create eval-only forward exec for deterministic Q-value replay.
self.create_eval_forward_exec()?;