diag(sp15-wave5): in-capture CAPTURE_PHASE_* checkpoints
Smoke train-fp7xx printed all 16 Phase-6/7/8 checkpoints clean through PER_PRIORITY_DONE. CAPTURE_DONE missing, exit changed 139→143 (SIGTERM) — capture_training_graph hangs or takes ~40s past PER_PRIORITY_DONE before Argo terminates the pod. Adds 14 CAPTURE_PHASE_* checkpoints across the 12 child captures + parent compose: BEGIN / PER_SAMPLE / COUNTERS / SPECTRAL / FORWARD / DDQN / AUX POST_AUX / ADAM_GRAD / ADAM_UPDATE / MAINTENANCE / IQL_MODULATE PER_PRIORITY / CHILDREN_STORED / PARENT_COMPOSED Diagnostic-only. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -2856,40 +2856,47 @@ impl FusedTrainingCtx {
|
||||
agent: &mut DQNAgentType,
|
||||
gpu_batch: &crate::dqn::replay_buffer_type::GpuBatch,
|
||||
) -> Result<()> {
|
||||
eprintln!("CAPTURE_PHASE_BEGIN");
|
||||
// Capture PER sampling child (prefix scan + binary search + gather).
|
||||
let per_sample = self.capture_child_graph("per_sample", |s| {
|
||||
agent.primary_dqn_mut().memory.gpu.sample_proportional(s.batch_size)
|
||||
.map_err(|e| anyhow::anyhow!("per_sample capture: {e}"))
|
||||
.map(|_| ())
|
||||
})?;
|
||||
eprintln!("CAPTURE_PHASE_PER_SAMPLE_DONE");
|
||||
|
||||
// Capture counters child (GPU-side step counters + stochastic depth mask).
|
||||
let counters = self.capture_child_graph("counters", |s| {
|
||||
s.submit_counters_ops()
|
||||
})?;
|
||||
eprintln!("CAPTURE_PHASE_COUNTERS_DONE");
|
||||
|
||||
// Capture spectral child.
|
||||
let spectral = self.capture_child_graph("spectral", |s| {
|
||||
s.trainer.apply_spectral_norm(&s.online_dueling, &s.online_branching)
|
||||
.map_err(|e| anyhow::anyhow!("{e}"))
|
||||
})?;
|
||||
eprintln!("CAPTURE_PHASE_SPECTRAL_DONE");
|
||||
|
||||
// Capture forward child (cuBLAS trunk + ISV + loss + backward).
|
||||
let forward = self.capture_child_graph("forward", |s| {
|
||||
s.trainer.submit_forward_ops_main()
|
||||
.map_err(|e| anyhow::anyhow!("{e}"))
|
||||
})?;
|
||||
eprintln!("CAPTURE_PHASE_FORWARD_DONE");
|
||||
|
||||
// Capture DDQN child (Pass 3).
|
||||
let ddqn = self.capture_child_graph("ddqn", |s| {
|
||||
s.trainer.submit_forward_ops_ddqn()
|
||||
.map_err(|e| anyhow::anyhow!("{e}"))
|
||||
})?;
|
||||
eprintln!("CAPTURE_PHASE_DDQN_DONE");
|
||||
|
||||
// Capture aux child (HER + EMA + IQL + IQN + attention + CQL).
|
||||
let aux = self.capture_child_graph("aux", |s| {
|
||||
s.submit_aux_ops(agent, gpu_batch)
|
||||
})?;
|
||||
eprintln!("CAPTURE_PHASE_AUX_DONE");
|
||||
|
||||
// Capture post_aux child (selectivity + denoise + Q-stats + risk_sgd
|
||||
// + SP1 Phase B post-backward NaN checks). The post-backward NaN
|
||||
@@ -2908,6 +2915,7 @@ impl FusedTrainingCtx {
|
||||
s.trainer.run_nan_checks_post_backward()
|
||||
.map_err(|e| anyhow::anyhow!("{e}"))
|
||||
})?;
|
||||
eprintln!("CAPTURE_PHASE_POST_AUX_DONE");
|
||||
|
||||
// adam_grad: mamba2 bwd + ofi_embed bwd + pruning + grad_norm
|
||||
// adam_update: Adam + unflatten + ISV
|
||||
@@ -2926,6 +2934,7 @@ impl FusedTrainingCtx {
|
||||
s.trainer.compute_grad_norm_for_adam()
|
||||
.map_err(|e| anyhow::anyhow!("{e}"))
|
||||
})?;
|
||||
eprintln!("CAPTURE_PHASE_ADAM_GRAD_DONE");
|
||||
let adam = self.capture_child_graph("adam_update", |s| {
|
||||
s.trainer.submit_adam_ops(&s.online_dueling, &s.online_branching)
|
||||
.map_err(|e| anyhow::anyhow!("{e}"))?;
|
||||
@@ -2946,22 +2955,26 @@ impl FusedTrainingCtx {
|
||||
s.trainer.update_isv_signals()
|
||||
.map_err(|e| anyhow::anyhow!("{e}"))
|
||||
})?;
|
||||
eprintln!("CAPTURE_PHASE_ADAM_UPDATE_DONE");
|
||||
|
||||
// Capture maintenance child (causal intervention + vaccine).
|
||||
let maintenance = self.capture_child_graph("maintenance", |s| {
|
||||
s.trainer.submit_maintenance_ops()
|
||||
.map_err(|e| anyhow::anyhow!("{e}"))
|
||||
})?;
|
||||
eprintln!("CAPTURE_PHASE_MAINTENANCE_DONE");
|
||||
|
||||
// Capture IQL modulate child (advantage-weighted PER modulation).
|
||||
let iql_modulate = self.capture_child_graph("iql_modulate", |s| {
|
||||
s.submit_iql_modulate_ops(agent, gpu_batch)
|
||||
})?;
|
||||
eprintln!("CAPTURE_PHASE_IQL_MODULATE_DONE");
|
||||
|
||||
// Capture PER priority child (seg_tree update).
|
||||
let per_priority = self.capture_child_graph("per_priority", |s| {
|
||||
s.submit_per_priority_ops(agent)
|
||||
})?;
|
||||
eprintln!("CAPTURE_PHASE_PER_PRIORITY_DONE");
|
||||
|
||||
// Store children.
|
||||
self.per_sample_child = Some(per_sample);
|
||||
@@ -2977,8 +2990,10 @@ impl FusedTrainingCtx {
|
||||
self.iql_modulate_child = Some(iql_modulate);
|
||||
self.per_priority_child = Some(per_priority);
|
||||
|
||||
eprintln!("CAPTURE_PHASE_CHILDREN_STORED");
|
||||
// Compose parent graph from all children.
|
||||
self.compose_parent_graph()?;
|
||||
eprintln!("CAPTURE_PHASE_PARENT_COMPOSED");
|
||||
|
||||
// Create eval-only forward exec for deterministic Q-value replay.
|
||||
self.create_eval_forward_exec()?;
|
||||
|
||||
Reference in New Issue
Block a user