diff --git a/crates/ml/src/trainers/dqn/smoke_tests/helpers.rs b/crates/ml/src/trainers/dqn/smoke_tests/helpers.rs index d29bfcc62..58f75df53 100644 --- a/crates/ml/src/trainers/dqn/smoke_tests/helpers.rs +++ b/crates/ml/src/trainers/dqn/smoke_tests/helpers.rs @@ -69,6 +69,24 @@ pub(super) fn assert_finite_f32(val: f32, name: &str) { assert!(val.is_finite(), "{name} is not finite: {val}"); } +/// Load smoke test data via the trainer's data loading pipeline. +/// +/// Returns (training_data, val_data) pre-split. Both hyperopt paths +/// (preloaded + shared) need this to bypass walk-forward. +pub(super) fn load_smoke_data() -> anyhow::Result<( + Vec<(crate::features::extraction::FeatureVector, Vec)>, + Vec<(crate::features::extraction::FeatureVector, Vec)>, +)> { + let data_dir = test_data_dir() + .expect("FOXHUNT_TEST_DATA or test_data/ must exist"); + let mut trainer = smoke_trainer()?; + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + let (train, val) = rt.block_on(trainer.load_training_data(&data_dir))?; + Ok((train, val)) +} + /// Resolve single-symbol test data directory. /// /// Uses `FOXHUNT_TEST_DATA` env var if set, otherwise auto-detects diff --git a/crates/ml/src/trainers/dqn/smoke_tests/hyperopt.rs b/crates/ml/src/trainers/dqn/smoke_tests/hyperopt.rs new file mode 100644 index 000000000..33afe5d59 --- /dev/null +++ b/crates/ml/src/trainers/dqn/smoke_tests/hyperopt.rs @@ -0,0 +1,140 @@ +//! Hyperopt training path smoke tests. +//! +//! These tests exercise the `train_with_preloaded_data` and `train_with_shared_data` +//! entry points which are used by the hyperopt adapter. Unlike `trainer.train()`, +//! these paths bypass GPU walk-forward and call `train_with_data_full_loop` directly. +//! +//! Key assertions: +//! - Training completes without hang or NaN +//! - Loss and gradient norms are finite +//! - Both preloaded (owned Vec) and shared (&slice) paths produce equivalent results + +use super::helpers::*; + +/// Hyperopt preloaded path: load data once, pass owned Vecs to trainer. +/// +/// This is the primary hyperopt path — data is loaded once at trial start +/// and passed to each trial's trainer. Exercises train_with_data_full_loop +/// directly, bypassing walk-forward. +#[test] +#[ignore] // Loads real training data + GPU +fn test_hyperopt_preloaded_data() -> anyhow::Result<()> { + let (train, val) = load_smoke_data()?; + assert!(!train.is_empty(), "Training data must be non-empty"); + assert!(!val.is_empty(), "Validation data must be non-empty"); + + let p = smoke_params(); + let mut trainer = smoke_trainer_with(p)?; + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + + let metrics = rt.block_on(trainer.train_with_preloaded_data( + train, val, |_epoch, _bytes, _best| Ok("skip".to_owned()), + ))?; + + assert!( + metrics.epochs_trained >= 2, + "Hyperopt preloaded: only trained {} epochs (expected >=2)", + metrics.epochs_trained + ); + assert_finite(metrics.loss, "hyperopt_preloaded loss"); + + let grad_norm = metrics.additional_metrics.get("avg_gradient_norm") + .copied().unwrap_or(0.0); + assert!(grad_norm > 0.0, "Hyperopt preloaded: grad_norm={grad_norm} — model must be learning"); + + tracing::info!( + "Hyperopt preloaded: loss={:.6}, grad_norm={:.4}, epochs={}", + metrics.loss, grad_norm, metrics.epochs_trained + ); + Ok(()) +} + +/// Hyperopt shared path: load data once, pass &slice to trainer (zero-copy). +/// +/// Production hyperopt uses Arc-wrapped data to avoid ~150 MB deep clone +/// per trial. This test validates the zero-copy borrow path. +#[test] +#[ignore] // Loads real training data + GPU +fn test_hyperopt_shared_data() -> anyhow::Result<()> { + let (train, val) = load_smoke_data()?; + assert!(!train.is_empty(), "Training data must be non-empty"); + assert!(!val.is_empty(), "Validation data must be non-empty"); + + let p = smoke_params(); + let mut trainer = smoke_trainer_with(p)?; + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + + let metrics = rt.block_on(trainer.train_with_shared_data( + &train, val, |_epoch, _bytes, _best| Ok("skip".to_owned()), + ))?; + + assert!( + metrics.epochs_trained >= 2, + "Hyperopt shared: only trained {} epochs (expected >=2)", + metrics.epochs_trained + ); + assert_finite(metrics.loss, "hyperopt_shared loss"); + + let grad_norm = metrics.additional_metrics.get("avg_gradient_norm") + .copied().unwrap_or(0.0); + assert!(grad_norm > 0.0, "Hyperopt shared: grad_norm={grad_norm} — model must be learning"); + + tracing::info!( + "Hyperopt shared: loss={:.6}, grad_norm={:.4}, epochs={}", + metrics.loss, grad_norm, metrics.epochs_trained + ); + Ok(()) +} + +/// Hyperopt consistency: preloaded and shared paths produce similar loss. +/// +/// Both paths should call train_with_data_full_loop with the same data. +/// Due to GPU RNG (epsilon-greedy, noisy nets, domain rand), results won't +/// be identical, but loss magnitude should be in the same ballpark. +#[test] +#[ignore] // Loads real training data + GPU — runs 2 training loops +fn test_hyperopt_paths_consistent() -> anyhow::Result<()> { + let (train, val) = load_smoke_data()?; + + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + + // Run preloaded path + let mut trainer1 = smoke_trainer_with(smoke_params())?; + let m1 = rt.block_on(trainer1.train_with_preloaded_data( + train.clone(), val.clone(), |_e, _b, _best| Ok("skip".to_owned()), + ))?; + + // Run shared path + let mut trainer2 = smoke_trainer_with(smoke_params())?; + let m2 = rt.block_on(trainer2.train_with_shared_data( + &train, val, |_e, _b, _best| Ok("skip".to_owned()), + ))?; + + // Both must complete + assert!(m1.epochs_trained >= 2, "Preloaded path failed: {} epochs", m1.epochs_trained); + assert!(m2.epochs_trained >= 2, "Shared path failed: {} epochs", m2.epochs_trained); + + // Losses should be same order of magnitude (within 10x) + let ratio = if m1.loss.abs() > 1e-10 && m2.loss.abs() > 1e-10 { + (m1.loss / m2.loss).abs() + } else { + 1.0 // both near-zero is fine + }; + assert!( + ratio > 0.1 && ratio < 10.0, + "Hyperopt paths diverged: preloaded={:.6}, shared={:.6}, ratio={:.2}", + m1.loss, m2.loss, ratio + ); + + tracing::info!( + "Hyperopt consistency: preloaded={:.6}, shared={:.6}, ratio={:.2}", + m1.loss, m2.loss, ratio + ); + Ok(()) +} diff --git a/crates/ml/src/trainers/dqn/smoke_tests/mod.rs b/crates/ml/src/trainers/dqn/smoke_tests/mod.rs index 7318243bd..4effd1599 100644 --- a/crates/ml/src/trainers/dqn/smoke_tests/mod.rs +++ b/crates/ml/src/trainers/dqn/smoke_tests/mod.rs @@ -12,3 +12,7 @@ mod performance; mod gradient_budget; #[cfg(test)] mod walk_forward; +#[cfg(test)] +mod hyperopt; +#[cfg(test)] +mod regression; diff --git a/crates/ml/src/trainers/dqn/smoke_tests/regression.rs b/crates/ml/src/trainers/dqn/smoke_tests/regression.rs new file mode 100644 index 000000000..b179dabba --- /dev/null +++ b/crates/ml/src/trainers/dqn/smoke_tests/regression.rs @@ -0,0 +1,161 @@ +//! Regression tests for specific GPU training bugs. +//! +//! Each test targets a bug that was found and fixed, serving as a guard +//! against reintroduction. Tests are named after the symptom, not the fix. + +use super::helpers::*; + +/// Regression: GPU training must not hang on first epoch. +/// +/// Root cause was VRAM oversubscription from detect_gpu_hardware auto-scaling +/// n_episodes without accounting for 2x counterfactual doubling (commit 4c5f404). +/// Fixed by replacing auto-scaling with configurable gpu_n_episodes. +/// +/// This test runs a single epoch with production-like features (all on) to +/// verify the full GPU pipeline completes without deadlock. +#[test] +#[ignore] // GPU + real data +fn test_no_hang_single_epoch() -> anyhow::Result<()> { + let data_dir = test_data_dir() + .expect("FOXHUNT_TEST_DATA or test_data/ must exist"); + + let mut p = smoke_params(); + p.epochs = 1; + p.early_stopping_enabled = false; + p.min_epochs_before_stopping = 1; + + let mut trainer = smoke_trainer_with(p)?; + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + + // Must complete 1 epoch — a hang here means VRAM oversubscription returned. + let metrics = rt.block_on(trainer.train(&data_dir, |_epoch, _bytes, _best| { + Ok("skip".to_owned()) + }))?; + + assert!( + metrics.epochs_trained >= 1, + "Single-epoch test failed: {} epochs trained (expected 1). \ + Possible VRAM oversubscription hang regression.", + metrics.epochs_trained + ); + assert!( + metrics.loss.is_finite(), + "Single-epoch loss must be finite, got {}", + metrics.loss + ); + + tracing::info!( + "No-hang regression: loss={:.6}, epochs={}", + metrics.loss, metrics.epochs_trained + ); + Ok(()) +} + +/// Regression: counterfactual experiences must flow into PER buffer. +/// +/// Root cause was build_next_states_f32 receiving n_episodes instead of +/// n_episodes*2, and PER insert total missing the 2x multiplier (commit 4c5f404). +/// ~50% of augmented training data was silently dropped. +/// +/// After one epoch with buffer_size=10000, gpu_n_episodes=32, timesteps=100: +/// base = 32 * 100 = 3200 +/// counterfactual = 3200 * 2 = 6400 +/// PER buffer should have >= 6400 entries (6400 < 10000 cap) +/// +/// If counterfactual is broken again, buffer would have ~3200 entries. +#[test] +#[ignore] // GPU + real data +fn test_counterfactual_experiences_in_buffer() -> anyhow::Result<()> { + let data_dir = test_data_dir() + .expect("FOXHUNT_TEST_DATA or test_data/ must exist"); + + let mut p = smoke_params(); + p.epochs = 1; + p.early_stopping_enabled = false; + // Ensure we can distinguish 1x from 2x: buffer must be large enough to hold 2x + // smoketest defaults: gpu_n_episodes=32, gpu_timesteps=100, buffer_size=10000 + // base = 32 * 100 = 3200, counterfactual = 6400, fits in 10000 + + let mut trainer = smoke_trainer_with(p)?; + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + let _metrics = rt.block_on(trainer.train(&data_dir, |_epoch, _bytes, _best| { + Ok("skip".to_owned()) + }))?; + + // Check replay buffer size via the agent + let buffer_size = rt.block_on(async { + let agent = trainer.get_agent().read().await; + agent.get_replay_buffer_size().unwrap_or(0) + }); + + // With counterfactual: 32 * 100 * 2 = 6400 + // Without counterfactual (the old bug): 32 * 100 = 3200 + // Threshold at 4000 cleanly separates the two cases. + let min_expected = 4000; + assert!( + buffer_size >= min_expected, + "PER buffer has {} entries (expected >= {min_expected}). \ + Counterfactual experiences may be silently dropped again. \ + With gpu_n_episodes=32, timesteps=100, expected 6400 (32*100*2).", + buffer_size + ); + + tracing::info!( + "Counterfactual regression: PER buffer has {} entries (expected ~6400)", + buffer_size + ); + Ok(()) +} + +/// Regression: gpu_n_episodes config must be honored (not auto-scaled). +/// +/// Previously detect_gpu_hardware().optimal_n_episodes() overrode the config, +/// causing VRAM oversubscription on H100. Now gpu_n_episodes from TOML is used +/// directly. This test verifies a custom value propagates through. +#[test] +#[ignore] // GPU + real data +fn test_gpu_n_episodes_config_honored() -> anyhow::Result<()> { + let data_dir = test_data_dir() + .expect("FOXHUNT_TEST_DATA or test_data/ must exist"); + + let mut p = smoke_params(); + p.epochs = 1; + p.early_stopping_enabled = false; + p.gpu_n_episodes = 64; // non-default to prove config is read + + let mut trainer = smoke_trainer_with(p)?; + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + let _metrics = rt.block_on(trainer.train(&data_dir, |_epoch, _bytes, _best| { + Ok("skip".to_owned()) + }))?; + + // If auto-scaling returned, n_episodes would be >64 on any modern GPU. + // Verify training completed with the configured value by checking the + // replay buffer: entries = gpu_n_episodes * timesteps * 2 (counterfactual) + let buffer_size = rt.block_on(async { + let agent = trainer.get_agent().read().await; + agent.get_replay_buffer_size().unwrap_or(0) + }); + + // 64 episodes * 100 timesteps * 2 = 12800 + // With auto-scaling on H100 it would be ~800K+ (4096 * 100 * 2) + let max_expected = 20000; // generous upper bound for 64 episodes + assert!( + buffer_size <= max_expected, + "PER buffer has {} entries (expected <= {max_expected} for 64 episodes). \ + gpu_n_episodes config may not be honored — auto-scaling regression.", + buffer_size + ); + + tracing::info!( + "Config regression: PER buffer has {} entries (expected ~12800 for gpu_n_episodes=64)", + buffer_size + ); + Ok(()) +} diff --git a/crates/ml/src/trainers/dqn/smoke_tests/walk_forward.rs b/crates/ml/src/trainers/dqn/smoke_tests/walk_forward.rs index cb5f7954f..154c5c130 100644 --- a/crates/ml/src/trainers/dqn/smoke_tests/walk_forward.rs +++ b/crates/ml/src/trainers/dqn/smoke_tests/walk_forward.rs @@ -154,3 +154,74 @@ fn test_walk_forward_no_overfitting_50_epochs() -> anyhow::Result<()> { Ok(()) } + +/// Walk-forward multi-fold: verify fold transitions work on GPU. +/// +/// Uses tight fractions (0.3/0.1/0.1/0.1) to guarantee 2+ folds even +/// with small test data (~500 bars). This is the critical test for: +/// - reset_for_fold() clears replay buffer + epoch state +/// - graph_aux invalidation between folds +/// - gpu_data re-init with new fold range +/// - CUDA Graph forward/adam survive across fold boundaries +/// - Regime-stratified fold generation +#[test] +#[ignore] // GPU + real data — ~5s on RTX 3050 +fn test_walk_forward_multi_fold() -> anyhow::Result<()> { + let data_dir = test_data_dir() + .expect("FOXHUNT_TEST_DATA or test_data/ must exist"); + + let mut p = smoke_params(); + p.epochs = 3; + p.early_stopping_enabled = false; + p.gradient_collapse_patience = 50000; + // Tight fractions to guarantee 2+ folds on small datasets + p.wf_initial_train_fraction = 0.3; + p.wf_val_fraction = 0.1; + p.wf_test_fraction = 0.1; + p.wf_step_fraction = 0.1; + + let mut trainer = smoke_trainer_with(p)?; + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + let metrics = rt.block_on(trainer.train(&data_dir, |_epoch, _bytes, _best| { + Ok("skip".to_owned()) + }))?; + + // Must complete (train_walk_forward returns last fold's metrics) + assert!( + metrics.epochs_trained >= 2, + "Multi-fold: only trained {} epochs (expected >=2)", + metrics.epochs_trained + ); + + let best_sharpe = metrics.additional_metrics.get("best_sharpe") + .copied().unwrap_or(f64::NAN); + let best_val_loss = metrics.additional_metrics.get("best_val_loss") + .copied().unwrap_or(f64::NAN); + + assert!(best_sharpe.is_finite(), "Multi-fold: best_sharpe must be finite, got {best_sharpe}"); + assert!(best_val_loss.is_finite(), "Multi-fold: best_val_loss must be finite, got {best_val_loss}"); + + // Loss must be finite and non-NaN (no CUDA graph corruption from fold transitions) + assert!( + metrics.loss.is_finite(), + "Multi-fold: training loss must be finite (got {}), graph may be corrupted across folds", + metrics.loss + ); + + let grad_norm = metrics.additional_metrics.get("avg_gradient_norm") + .copied().unwrap_or(0.0); + assert!( + grad_norm > 0.0, + "Multi-fold: grad_norm={grad_norm} — model must be learning after fold transition" + ); + + tracing::info!( + "Walk-forward multi-fold: best_sharpe={best_sharpe:.4}, best_val_loss={best_val_loss:.6}, \ + loss={:.6}, grad_norm={grad_norm:.4}, epochs={}", + metrics.loss, metrics.epochs_trained + ); + + Ok(()) +} diff --git a/crates/ml/tests/gpu_per_integration_test.rs b/crates/ml/tests/gpu_per_integration_test.rs index e687bfbe4..bbe40df43 100644 --- a/crates/ml/tests/gpu_per_integration_test.rs +++ b/crates/ml/tests/gpu_per_integration_test.rs @@ -160,6 +160,8 @@ fn test_gpu_per_training_loop_cycle() { buf.update_priorities_gpu_raw(idx_ptr, &td_errors, batch_size).unwrap(); buf.step(); + } + // After updates, priorities should be non-uniform assert!(buf.len() == 200); }