fix(smoke): test_fxcache_zero_copy_training — real failure detection
Previously the fold loop swallowed every training error into f64::NAN,
unconditionally pushed a value at the top of the loop, and then asserted
only !fold_losses.is_empty() — a tautology that could never fail. Any
CUDA error, NaN explosion, or regression of the zero-copy path passed
silently.
Now:
- Error propagation via `?` (no swallow). A zero-copy test can't
tolerate training failures — if training blew up, the zero-copy
wiring is broken and must surface.
- All fold losses must be finite and non-negative.
- Every fold must report epochs_trained >= 1 (zero-epoch fold =
kernel skipped or buffer not populated).
A direct "no htod/dtoh copy happened" assertion would need a copy-counter
instrumented into the fused-training GPU path plus a field on
TrainingMetrics. That is out of scope for this test; documented inline.
Verified PASS on laptop (RTX 3050 Ti):
loss=0.0018366, epochs=1
This commit is contained in:
@@ -547,7 +547,14 @@ fn test_fxcache_zero_copy_training() -> anyhow::Result<()> {
|
||||
eprintln!("[FXCACHE] GPU upload complete");
|
||||
|
||||
// 5. Train each fold using the production zero-copy path
|
||||
let mut fold_losses = Vec::new();
|
||||
//
|
||||
// Previously this loop swallowed every training error into f64::NAN and asserted
|
||||
// only !fold_losses.is_empty() after unconditionally pushing one entry at the
|
||||
// top — a tautology that could never fail, masking CUDA errors, NaN explosions,
|
||||
// and any regression of the zero-copy path. Errors now propagate via `?` so
|
||||
// the test actually validates the path's behaviour, not just that it runs.
|
||||
let mut fold_losses: Vec<f64> = Vec::new();
|
||||
let mut fold_epochs: Vec<u32> = Vec::new();
|
||||
for range in &fold_ranges {
|
||||
let train_feat = &fxcache_data.features[range.train_start..range.train_end];
|
||||
let val_feat = &fxcache_data.features[range.val_start..range.val_end];
|
||||
@@ -563,27 +570,41 @@ fn test_fxcache_zero_copy_training() -> anyhow::Result<()> {
|
||||
rt.block_on(trainer.reset_for_fold())?;
|
||||
|
||||
let fold = range.fold;
|
||||
let result = rt.block_on(trainer.train_fold_from_slices(
|
||||
let metrics = rt.block_on(trainer.train_fold_from_slices(
|
||||
train_feat, train_targets,
|
||||
|_epoch, _bytes, _is_best| Ok(String::new()),
|
||||
));
|
||||
|
||||
match result {
|
||||
Ok(metrics) => {
|
||||
eprintln!("[FXCACHE] Fold {} complete: loss={:.4}, epochs={}", fold, metrics.loss, metrics.epochs_trained);
|
||||
fold_losses.push(metrics.loss);
|
||||
}
|
||||
Err(e) => {
|
||||
// NaN at early steps can happen with raw features + small smoketest network.
|
||||
// The important thing is the code PATH works (no hang, no CUDA error).
|
||||
eprintln!("[FXCACHE] Fold {} error (non-fatal for path validation): {:#}", fold, e);
|
||||
fold_losses.push(f64::NAN);
|
||||
}
|
||||
}
|
||||
))?;
|
||||
eprintln!(
|
||||
"[FXCACHE] Fold {} complete: loss={:.4}, epochs={}",
|
||||
fold, metrics.loss, metrics.epochs_trained,
|
||||
);
|
||||
fold_losses.push(metrics.loss);
|
||||
fold_epochs.push(metrics.epochs_trained);
|
||||
}
|
||||
|
||||
// 6. Verify training ran (code path validation, not training quality)
|
||||
assert!(!fold_losses.is_empty(), "No folds completed");
|
||||
// 6. Real assertions on the zero-copy path.
|
||||
// NOTE: directly asserting "no htod/dtoh copy happened" would require
|
||||
// instrumenting the fused-training GPU data path with a copy-counter and
|
||||
// exposing it through TrainingMetrics, which is out-of-scope here. What we
|
||||
// can assert end-to-end is that the path produced a *valid training run*:
|
||||
// any NaN/Inf/negative loss or zero-epoch fold indicates that the zero-copy
|
||||
// wiring regressed (buffer not populated, kernel skipped, etc.).
|
||||
assert!(!fold_losses.is_empty(), "fxcache zero-copy path produced zero folds");
|
||||
assert!(
|
||||
fold_losses.iter().all(|l| l.is_finite()),
|
||||
"fxcache zero-copy training produced NaN/Inf loss: {:?}",
|
||||
fold_losses,
|
||||
);
|
||||
assert!(
|
||||
fold_losses.iter().all(|l| *l >= 0.0),
|
||||
"fxcache zero-copy training produced negative loss (sign bug?): {:?}",
|
||||
fold_losses,
|
||||
);
|
||||
assert!(
|
||||
fold_epochs.iter().all(|&e| e >= 1),
|
||||
"fxcache zero-copy training completed 0 epochs in some fold: {:?}",
|
||||
fold_epochs,
|
||||
);
|
||||
|
||||
eprintln!("[FXCACHE] All {} folds passed. Losses: {:?}", fold_losses.len(), fold_losses);
|
||||
Ok(())
|
||||
|
||||
Reference in New Issue
Block a user