diff --git a/crates/ml/src/trainers/dqn/smoke_tests/training_stability.rs b/crates/ml/src/trainers/dqn/smoke_tests/training_stability.rs index c220bfb4f..fee6945b2 100644 --- a/crates/ml/src/trainers/dqn/smoke_tests/training_stability.rs +++ b/crates/ml/src/trainers/dqn/smoke_tests/training_stability.rs @@ -547,7 +547,14 @@ fn test_fxcache_zero_copy_training() -> anyhow::Result<()> { eprintln!("[FXCACHE] GPU upload complete"); // 5. Train each fold using the production zero-copy path - let mut fold_losses = Vec::new(); + // + // Previously this loop swallowed every training error into f64::NAN and asserted + // only !fold_losses.is_empty() after unconditionally pushing one entry at the + // top — a tautology that could never fail, masking CUDA errors, NaN explosions, + // and any regression of the zero-copy path. Errors now propagate via `?` so + // the test actually validates the path's behaviour, not just that it runs. + let mut fold_losses: Vec = Vec::new(); + let mut fold_epochs: Vec = Vec::new(); for range in &fold_ranges { let train_feat = &fxcache_data.features[range.train_start..range.train_end]; let val_feat = &fxcache_data.features[range.val_start..range.val_end]; @@ -563,27 +570,41 @@ fn test_fxcache_zero_copy_training() -> anyhow::Result<()> { rt.block_on(trainer.reset_for_fold())?; let fold = range.fold; - let result = rt.block_on(trainer.train_fold_from_slices( + let metrics = rt.block_on(trainer.train_fold_from_slices( train_feat, train_targets, |_epoch, _bytes, _is_best| Ok(String::new()), - )); - - match result { - Ok(metrics) => { - eprintln!("[FXCACHE] Fold {} complete: loss={:.4}, epochs={}", fold, metrics.loss, metrics.epochs_trained); - fold_losses.push(metrics.loss); - } - Err(e) => { - // NaN at early steps can happen with raw features + small smoketest network. - // The important thing is the code PATH works (no hang, no CUDA error). - eprintln!("[FXCACHE] Fold {} error (non-fatal for path validation): {:#}", fold, e); - fold_losses.push(f64::NAN); - } - } + ))?; + eprintln!( + "[FXCACHE] Fold {} complete: loss={:.4}, epochs={}", + fold, metrics.loss, metrics.epochs_trained, + ); + fold_losses.push(metrics.loss); + fold_epochs.push(metrics.epochs_trained); } - // 6. Verify training ran (code path validation, not training quality) - assert!(!fold_losses.is_empty(), "No folds completed"); + // 6. Real assertions on the zero-copy path. + // NOTE: directly asserting "no htod/dtoh copy happened" would require + // instrumenting the fused-training GPU data path with a copy-counter and + // exposing it through TrainingMetrics, which is out-of-scope here. What we + // can assert end-to-end is that the path produced a *valid training run*: + // any NaN/Inf/negative loss or zero-epoch fold indicates that the zero-copy + // wiring regressed (buffer not populated, kernel skipped, etc.). + assert!(!fold_losses.is_empty(), "fxcache zero-copy path produced zero folds"); + assert!( + fold_losses.iter().all(|l| l.is_finite()), + "fxcache zero-copy training produced NaN/Inf loss: {:?}", + fold_losses, + ); + assert!( + fold_losses.iter().all(|l| *l >= 0.0), + "fxcache zero-copy training produced negative loss (sign bug?): {:?}", + fold_losses, + ); + assert!( + fold_epochs.iter().all(|&e| e >= 1), + "fxcache zero-copy training completed 0 epochs in some fold: {:?}", + fold_epochs, + ); eprintln!("[FXCACHE] All {} folds passed. Losses: {:?}", fold_losses.len(), fold_losses); Ok(())