From 004b662c803b28402c62edc796d935b6b5df94c9 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Mon, 18 May 2026 22:33:44 +0200 Subject: [PATCH] obs(ml-alpha): per-N-step train_loss log line for liveness + intra-epoch monitor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a single tracing::info!("step") emitted every 500 training steps: if epoch_train_steps % 500 == 0 { tracing::info!(epoch, step = epoch_train_steps, loss = loss, "step"); } Consumed by /tmp/alpha_monitor.py (v2: per-epoch trajectories + ISV + liveness) to render intra-epoch loss trajectory and detect stalls faster than the once-per-epoch granularity allowed. At 8000 steps/epoch and ~36s/epoch on L40S → ~16 step lines per epoch per fold, well below the cluster log volume budget. Co-Authored-By: Claude Opus 4.7 --- crates/ml-alpha/examples/alpha_train.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/crates/ml-alpha/examples/alpha_train.rs b/crates/ml-alpha/examples/alpha_train.rs index 79a299ac9..ea594aa1e 100644 --- a/crates/ml-alpha/examples/alpha_train.rs +++ b/crates/ml-alpha/examples/alpha_train.rs @@ -455,6 +455,19 @@ fn main() -> Result<()> { epoch_train_steps += 1; train_loss_running += loss; train_steps += 1; + // Periodic per-step log (every 500 steps) for liveness + + // intra-epoch trajectory monitoring. Cheap (~1 line/min on + // L40S at 8000 steps/epoch / 36s = 220 steps/s ⇒ ~16 lines + // per 500-step interval per epoch). Consumed by + // /tmp/alpha_monitor.py. + if epoch_train_steps % 500 == 0 { + tracing::info!( + epoch, + step = epoch_train_steps, + loss = loss, + "step" + ); + } snap_batch.clear(); label_batch.clear(); }