diag(sp15-wave5): stderr checkpoints in run_full_step step-0 ungraphed path
L40S smoke train-jfbzr (commit 23e9a1f78) segfaults at exit 139 in the
first run_full_step invocation, after "GPU training guard initialized
(epoch loop)" and before any HEALTH_DIAG output. Static analysis
debugger couldn't reproduce locally (test_data/futures-baseline/ lacks
.fxcache).
This commit adds eprintln! checkpoints at each phase boundary in the
ungraphed step-0 path of FusedTrainer::run_full_step (PER sample,
PopArt, counters, spectral norm, TLOB forward, forward_main, DDQN,
aux_ops, post_aux, NaN checks).
stderr is line-flushed (vs stdout buffered through tracing JSON
formatter), so the last printed checkpoint identifies the SEGV site.
Each fires once per fold's first step (graph capture absorbs subsequent
steps), so log overhead is minimal.
Diagnostic-only commit — no behavior change. Will be reverted after the
next L40S smoke localizes the SEGV and the root-cause fix lands.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1489,6 +1489,11 @@ impl FusedTrainingCtx {
|
||||
tracing::debug!("parent graph launch DONE (async — GPU working)");
|
||||
} else {
|
||||
// Ungraphed step 0: run everything, then capture + compose.
|
||||
// SP15 Wave 5 SEGV diagnostics: stderr checkpoints survive
|
||||
// host-side crashes that don't get emitted to the tracing
|
||||
// JSON formatter (stdout is buffered; stderr is line-flushed).
|
||||
// Each checkpoint prints once per fold's first step.
|
||||
eprintln!("STEP0_PHASE_BEGIN");
|
||||
// Ensure params are initialized for the first step.
|
||||
if !self.trainer.params_initialized {
|
||||
self.trainer.flatten_online_weights(
|
||||
@@ -1496,6 +1501,7 @@ impl FusedTrainingCtx {
|
||||
).map_err(|e| anyhow::anyhow!("flatten weights: {e}"))?;
|
||||
self.trainer.params_initialized = true;
|
||||
}
|
||||
eprintln!("STEP0_PHASE_FLATTEN_ONLINE_DONE");
|
||||
if !self.trainer.target_params_initialized {
|
||||
let n_bytes = self.trainer.total_params() * std::mem::size_of::<f32>();
|
||||
let src = self.trainer.params_buf_ptr();
|
||||
@@ -1506,10 +1512,12 @@ impl FusedTrainingCtx {
|
||||
}
|
||||
self.trainer.target_params_initialized = true;
|
||||
}
|
||||
eprintln!("STEP0_PHASE_TARGET_INIT_DONE");
|
||||
|
||||
// Phase -1: PER sampling (writes directly to trainer buffers via gather_padded)
|
||||
let ptrs = agent.primary_dqn_mut().memory.gpu.sample_proportional(self.batch_size)
|
||||
.map_err(|e| anyhow::anyhow!("PER sample step 0: {e}"))?;
|
||||
eprintln!("STEP0_PHASE_PER_SAMPLE_DONE");
|
||||
let gpu_batch = crate::dqn::replay_buffer_type::GpuBatch {
|
||||
states_ptr: ptrs.states_ptr,
|
||||
actions_ptr: ptrs.actions_ptr,
|
||||
@@ -1535,9 +1543,11 @@ impl FusedTrainingCtx {
|
||||
self.trainer.normalize_rewards_popart_inplace(self.batch_size)
|
||||
.map_err(|e| anyhow::anyhow!("PopArt normalize: {e}"))?;
|
||||
}
|
||||
eprintln!("STEP0_PHASE_POPART_DONE");
|
||||
|
||||
// Phase 0b: Counter increments + stochastic depth mask (ungraphed)
|
||||
self.submit_counters_ops()?;
|
||||
eprintln!("STEP0_PHASE_COUNTERS_DONE");
|
||||
|
||||
// HER donor computation (outside graph -- indices vary per step)
|
||||
if let Some(ref mut her) = self.gpu_her {
|
||||
@@ -1563,6 +1573,7 @@ impl FusedTrainingCtx {
|
||||
// Phase 1: Spectral norm
|
||||
self.trainer.apply_spectral_norm(&self.online_dueling, &self.online_branching)
|
||||
.map_err(|e| anyhow::anyhow!("spectral norm: {e}"))?;
|
||||
eprintln!("STEP0_PHASE_SPECTRAL_NORM_DONE");
|
||||
|
||||
// D.8 TLOB forward: OFI[32]→TLOB[16] written into states_buf before trunk forward.
|
||||
{
|
||||
@@ -1571,21 +1582,26 @@ impl FusedTrainingCtx {
|
||||
.forward(states, self.batch_size)
|
||||
.map_err(|e| anyhow::anyhow!("TLOB forward: {e}"))?;
|
||||
}
|
||||
eprintln!("STEP0_PHASE_TLOB_FORWARD_DONE");
|
||||
|
||||
// Phase 2: Forward (cuBLAS trunk + ISV + loss + backward)
|
||||
self.trainer.submit_forward_ops_main()
|
||||
.map_err(|e| anyhow::anyhow!("forward ops: {e}"))?;
|
||||
eprintln!("STEP0_PHASE_FORWARD_MAIN_DONE");
|
||||
|
||||
// Phase 3: DDQN Pass 3
|
||||
self.trainer.submit_forward_ops_ddqn()
|
||||
.map_err(|e| anyhow::anyhow!("ddqn ops: {e}"))?;
|
||||
eprintln!("STEP0_PHASE_FORWARD_DDQN_DONE");
|
||||
|
||||
// Phase 4: Aux ops (HER + EMA + IQL + IQN + attention + CQL)
|
||||
self.submit_aux_ops(agent, &gpu_batch)?;
|
||||
eprintln!("STEP0_PHASE_AUX_OPS_DONE");
|
||||
|
||||
// Phase 5b: Post-aux ops (selectivity + denoise + Q-stats + risk_sgd)
|
||||
self.trainer.submit_post_aux_ops(self.batch_size)
|
||||
.map_err(|e| anyhow::anyhow!("post_aux ops: {e}"))?;
|
||||
eprintln!("STEP0_PHASE_POST_AUX_DONE");
|
||||
|
||||
// SP1 Phase B (Task 4) + SP2 (Task A4): single fused NaN-check
|
||||
// launch covers all 12 backward-path slots (24-35) in 12 blocks.
|
||||
@@ -1603,6 +1619,7 @@ impl FusedTrainingCtx {
|
||||
// are skipped by the kernel's null-pointer guard.
|
||||
self.trainer.run_nan_checks_post_backward()
|
||||
.map_err(|e| anyhow::anyhow!("post_backward nan checks: {e}"))?;
|
||||
eprintln!("STEP0_PHASE_NAN_CHECKS_DONE");
|
||||
|
||||
// Phase 6: TLOB bwd + Mamba2 bwd + OFI embed bwd + pruning + grad_norm + Adam + ISV update
|
||||
// TLOB backward: reads TLOB-slice gradient from bn_d_concat_buf, updates W_Q/K/V/O.
|
||||
|
||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user