fix(smoke): test_fxcache_zero_copy_training — real failure detection

Previously the fold loop swallowed every training error into f64::NAN,
unconditionally pushed a value at the top of the loop, and then asserted
only !fold_losses.is_empty() — a tautology that could never fail. Any
CUDA error, NaN explosion, or regression of the zero-copy path passed
silently.

Now:
  - Error propagation via `?` (no swallow). A zero-copy test can't
    tolerate training failures — if training blew up, the zero-copy
    wiring is broken and must surface.
  - All fold losses must be finite and non-negative.
  - Every fold must report epochs_trained >= 1 (zero-epoch fold =
    kernel skipped or buffer not populated).

A direct "no htod/dtoh copy happened" assertion would need a copy-counter
instrumented into the fused-training GPU path plus a field on
TrainingMetrics. That is out of scope for this test; documented inline.

Verified PASS on laptop (RTX 3050 Ti):
  loss=0.0018366, epochs=1
This commit is contained in:
jgrusewski
2026-04-22 08:22:00 +02:00
parent e8ecb2f626
commit 2570fe0130

View File

@@ -547,7 +547,14 @@ fn test_fxcache_zero_copy_training() -> anyhow::Result<()> {
eprintln!("[FXCACHE] GPU upload complete");
// 5. Train each fold using the production zero-copy path
let mut fold_losses = Vec::new();
//
// Previously this loop swallowed every training error into f64::NAN and asserted
// only !fold_losses.is_empty() after unconditionally pushing one entry at the
// top — a tautology that could never fail, masking CUDA errors, NaN explosions,
// and any regression of the zero-copy path. Errors now propagate via `?` so
// the test actually validates the path's behaviour, not just that it runs.
let mut fold_losses: Vec<f64> = Vec::new();
let mut fold_epochs: Vec<u32> = Vec::new();
for range in &fold_ranges {
let train_feat = &fxcache_data.features[range.train_start..range.train_end];
let val_feat = &fxcache_data.features[range.val_start..range.val_end];
@@ -563,27 +570,41 @@ fn test_fxcache_zero_copy_training() -> anyhow::Result<()> {
rt.block_on(trainer.reset_for_fold())?;
let fold = range.fold;
let result = rt.block_on(trainer.train_fold_from_slices(
let metrics = rt.block_on(trainer.train_fold_from_slices(
train_feat, train_targets,
|_epoch, _bytes, _is_best| Ok(String::new()),
));
match result {
Ok(metrics) => {
eprintln!("[FXCACHE] Fold {} complete: loss={:.4}, epochs={}", fold, metrics.loss, metrics.epochs_trained);
fold_losses.push(metrics.loss);
}
Err(e) => {
// NaN at early steps can happen with raw features + small smoketest network.
// The important thing is the code PATH works (no hang, no CUDA error).
eprintln!("[FXCACHE] Fold {} error (non-fatal for path validation): {:#}", fold, e);
fold_losses.push(f64::NAN);
}
}
))?;
eprintln!(
"[FXCACHE] Fold {} complete: loss={:.4}, epochs={}",
fold, metrics.loss, metrics.epochs_trained,
);
fold_losses.push(metrics.loss);
fold_epochs.push(metrics.epochs_trained);
}
// 6. Verify training ran (code path validation, not training quality)
assert!(!fold_losses.is_empty(), "No folds completed");
// 6. Real assertions on the zero-copy path.
// NOTE: directly asserting "no htod/dtoh copy happened" would require
// instrumenting the fused-training GPU data path with a copy-counter and
// exposing it through TrainingMetrics, which is out-of-scope here. What we
// can assert end-to-end is that the path produced a *valid training run*:
// any NaN/Inf/negative loss or zero-epoch fold indicates that the zero-copy
// wiring regressed (buffer not populated, kernel skipped, etc.).
assert!(!fold_losses.is_empty(), "fxcache zero-copy path produced zero folds");
assert!(
fold_losses.iter().all(|l| l.is_finite()),
"fxcache zero-copy training produced NaN/Inf loss: {:?}",
fold_losses,
);
assert!(
fold_losses.iter().all(|l| *l >= 0.0),
"fxcache zero-copy training produced negative loss (sign bug?): {:?}",
fold_losses,
);
assert!(
fold_epochs.iter().all(|&e| e >= 1),
"fxcache zero-copy training completed 0 epochs in some fold: {:?}",
fold_epochs,
);
eprintln!("[FXCACHE] All {} folds passed. Losses: {:?}", fold_losses.len(), fold_losses);
Ok(())