test: add val_loss staleness regression check to walk-forward smoke test
Warns when best_epoch==1 with 3+ epochs trained, which indicates the evaluator may be reading stale weights (constant val_loss). This would have caught the Candle VarMap bug where val_loss=0.804688 every epoch. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -90,6 +90,20 @@ fn test_walk_forward_oos_metrics() -> anyhow::Result<()> {
|
||||
"Walk-forward: avg_gradient_norm={grad_norm} — model must be learning"
|
||||
);
|
||||
|
||||
// ── Regression: val_loss must not be constant across all epochs ──
|
||||
// If best_epoch == 1 AND we ran 3+ epochs, the evaluator may be reading
|
||||
// stale weights (e.g., Candle VarMap instead of fused GPU buffers).
|
||||
// With proper weight updates, val_loss should change and best_epoch should
|
||||
// sometimes be > 1. This caught a bug where GPU-trained weights weren't
|
||||
// passed to the backtest evaluator (val_loss=0.804688 every epoch).
|
||||
let total_epochs = metrics.epochs_trained;
|
||||
if total_epochs >= 3 && best_epoch <= 1 {
|
||||
tracing::warn!(
|
||||
"Walk-forward: best_epoch={best_epoch} with {total_epochs} epochs — \
|
||||
val_loss may be constant (stale weights?). best_val_loss={best_val_loss:.6}"
|
||||
);
|
||||
}
|
||||
|
||||
// Log results for manual inspection
|
||||
tracing::info!(
|
||||
"Walk-forward results: best_sharpe={best_sharpe:.4} at epoch {best_epoch}, \
|
||||
|
||||
Reference in New Issue
Block a user