test: smoke tests for hyperopt paths, multi-fold walk-forward, and regression guards

New test coverage for 3 previously untested GPU training paths:

- hyperopt.rs: test_hyperopt_preloaded_data, test_hyperopt_shared_data,
  test_hyperopt_paths_consistent — exercises train_with_preloaded_data and
  train_with_shared_data which bypass walk-forward (direct train_with_data_full_loop)

- walk_forward.rs: test_walk_forward_multi_fold — tight fractions (0.3/0.1/0.1/0.1)
  to guarantee 2+ folds, validating reset_for_fold, graph_aux invalidation, and
  CUDA graph survival across fold boundaries

- regression.rs: test_no_hang_single_epoch (VRAM oversubscription guard),
  test_counterfactual_experiences_in_buffer (silent data loss guard),
  test_gpu_n_episodes_config_honored (auto-scaling removal guard)

Also fixes: unclosed for-loop brace in gpu_per_integration_test.rs (pre-existing)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

crates/ml/src/trainers/dqn/smoke_tests/helpers.rs

@@ -69,6 +69,24 @@ pub(super) fn assert_finite_f32(val: f32, name: &str) {
     assert!(val.is_finite(), "{name} is not finite: {val}");
 }
 
+/// Load smoke test data via the trainer's data loading pipeline.
+///
+/// Returns (training_data, val_data) pre-split. Both hyperopt paths
+/// (preloaded + shared) need this to bypass walk-forward.
+pub(super) fn load_smoke_data() -> anyhow::Result<(
+    Vec<(crate::features::extraction::FeatureVector, Vec<f64>)>,
+    Vec<(crate::features::extraction::FeatureVector, Vec<f64>)>,
+)> {
+    let data_dir = test_data_dir()
+        .expect("FOXHUNT_TEST_DATA or test_data/ must exist");
+    let mut trainer = smoke_trainer()?;
+    let rt = tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()?;
+    let (train, val) = rt.block_on(trainer.load_training_data(&data_dir))?;
+    Ok((train, val))
+}
+
 /// Resolve single-symbol test data directory.
 ///
 /// Uses `FOXHUNT_TEST_DATA` env var if set, otherwise auto-detects

crates/ml/src/trainers/dqn/smoke_tests/hyperopt.rs

@@ -0,0 +1,140 @@
+//! Hyperopt training path smoke tests.
+//!
+//! These tests exercise the `train_with_preloaded_data` and `train_with_shared_data`
+//! entry points which are used by the hyperopt adapter. Unlike `trainer.train()`,
+//! these paths bypass GPU walk-forward and call `train_with_data_full_loop` directly.
+//!
+//! Key assertions:
+//! - Training completes without hang or NaN
+//! - Loss and gradient norms are finite
+//! - Both preloaded (owned Vec) and shared (&slice) paths produce equivalent results
+
+use super::helpers::*;
+
+/// Hyperopt preloaded path: load data once, pass owned Vecs to trainer.
+///
+/// This is the primary hyperopt path — data is loaded once at trial start
+/// and passed to each trial's trainer. Exercises train_with_data_full_loop
+/// directly, bypassing walk-forward.
+#[test]
+#[ignore] // Loads real training data + GPU
+fn test_hyperopt_preloaded_data() -> anyhow::Result<()> {
+    let (train, val) = load_smoke_data()?;
+    assert!(!train.is_empty(), "Training data must be non-empty");
+    assert!(!val.is_empty(), "Validation data must be non-empty");
+
+    let p = smoke_params();
+    let mut trainer = smoke_trainer_with(p)?;
+    let rt = tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()?;
+
+    let metrics = rt.block_on(trainer.train_with_preloaded_data(
+        train, val, |_epoch, _bytes, _best| Ok("skip".to_owned()),
+    ))?;
+
+    assert!(
+        metrics.epochs_trained >= 2,
+        "Hyperopt preloaded: only trained {} epochs (expected >=2)",
+        metrics.epochs_trained
+    );
+    assert_finite(metrics.loss, "hyperopt_preloaded loss");
+
+    let grad_norm = metrics.additional_metrics.get("avg_gradient_norm")
+        .copied().unwrap_or(0.0);
+    assert!(grad_norm > 0.0, "Hyperopt preloaded: grad_norm={grad_norm} — model must be learning");
+
+    tracing::info!(
+        "Hyperopt preloaded: loss={:.6}, grad_norm={:.4}, epochs={}",
+        metrics.loss, grad_norm, metrics.epochs_trained
+    );
+    Ok(())
+}
+
+/// Hyperopt shared path: load data once, pass &slice to trainer (zero-copy).
+///
+/// Production hyperopt uses Arc-wrapped data to avoid ~150 MB deep clone
+/// per trial. This test validates the zero-copy borrow path.
+#[test]
+#[ignore] // Loads real training data + GPU
+fn test_hyperopt_shared_data() -> anyhow::Result<()> {
+    let (train, val) = load_smoke_data()?;
+    assert!(!train.is_empty(), "Training data must be non-empty");
+    assert!(!val.is_empty(), "Validation data must be non-empty");
+
+    let p = smoke_params();
+    let mut trainer = smoke_trainer_with(p)?;
+    let rt = tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()?;
+
+    let metrics = rt.block_on(trainer.train_with_shared_data(
+        &train, val, |_epoch, _bytes, _best| Ok("skip".to_owned()),
+    ))?;
+
+    assert!(
+        metrics.epochs_trained >= 2,
+        "Hyperopt shared: only trained {} epochs (expected >=2)",
+        metrics.epochs_trained
+    );
+    assert_finite(metrics.loss, "hyperopt_shared loss");
+
+    let grad_norm = metrics.additional_metrics.get("avg_gradient_norm")
+        .copied().unwrap_or(0.0);
+    assert!(grad_norm > 0.0, "Hyperopt shared: grad_norm={grad_norm} — model must be learning");
+
+    tracing::info!(
+        "Hyperopt shared: loss={:.6}, grad_norm={:.4}, epochs={}",
+        metrics.loss, grad_norm, metrics.epochs_trained
+    );
+    Ok(())
+}
+
+/// Hyperopt consistency: preloaded and shared paths produce similar loss.
+///
+/// Both paths should call train_with_data_full_loop with the same data.
+/// Due to GPU RNG (epsilon-greedy, noisy nets, domain rand), results won't
+/// be identical, but loss magnitude should be in the same ballpark.
+#[test]
+#[ignore] // Loads real training data + GPU — runs 2 training loops
+fn test_hyperopt_paths_consistent() -> anyhow::Result<()> {
+    let (train, val) = load_smoke_data()?;
+
+    let rt = tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()?;
+
+    // Run preloaded path
+    let mut trainer1 = smoke_trainer_with(smoke_params())?;
+    let m1 = rt.block_on(trainer1.train_with_preloaded_data(
+        train.clone(), val.clone(), |_e, _b, _best| Ok("skip".to_owned()),
+    ))?;
+
+    // Run shared path
+    let mut trainer2 = smoke_trainer_with(smoke_params())?;
+    let m2 = rt.block_on(trainer2.train_with_shared_data(
+        &train, val, |_e, _b, _best| Ok("skip".to_owned()),
+    ))?;
+
+    // Both must complete
+    assert!(m1.epochs_trained >= 2, "Preloaded path failed: {} epochs", m1.epochs_trained);
+    assert!(m2.epochs_trained >= 2, "Shared path failed: {} epochs", m2.epochs_trained);
+
+    // Losses should be same order of magnitude (within 10x)
+    let ratio = if m1.loss.abs() > 1e-10 && m2.loss.abs() > 1e-10 {
+        (m1.loss / m2.loss).abs()
+    } else {
+        1.0 // both near-zero is fine
+    };
+    assert!(
+        ratio > 0.1 && ratio < 10.0,
+        "Hyperopt paths diverged: preloaded={:.6}, shared={:.6}, ratio={:.2}",
+        m1.loss, m2.loss, ratio
+    );
+
+    tracing::info!(
+        "Hyperopt consistency: preloaded={:.6}, shared={:.6}, ratio={:.2}",
+        m1.loss, m2.loss, ratio
+    );
+    Ok(())
+}

crates/ml/src/trainers/dqn/smoke_tests/mod.rs

@@ -12,3 +12,7 @@ mod performance;
 mod gradient_budget;
 #[cfg(test)]
 mod walk_forward;
+#[cfg(test)]
+mod hyperopt;
+#[cfg(test)]
+mod regression;

crates/ml/src/trainers/dqn/smoke_tests/regression.rs

@@ -0,0 +1,161 @@
+//! Regression tests for specific GPU training bugs.
+//!
+//! Each test targets a bug that was found and fixed, serving as a guard
+//! against reintroduction. Tests are named after the symptom, not the fix.
+
+use super::helpers::*;
+
+/// Regression: GPU training must not hang on first epoch.
+///
+/// Root cause was VRAM oversubscription from detect_gpu_hardware auto-scaling
+/// n_episodes without accounting for 2x counterfactual doubling (commit 4c5f404).
+/// Fixed by replacing auto-scaling with configurable gpu_n_episodes.
+///
+/// This test runs a single epoch with production-like features (all on) to
+/// verify the full GPU pipeline completes without deadlock.
+#[test]
+#[ignore] // GPU + real data
+fn test_no_hang_single_epoch() -> anyhow::Result<()> {
+    let data_dir = test_data_dir()
+        .expect("FOXHUNT_TEST_DATA or test_data/ must exist");
+
+    let mut p = smoke_params();
+    p.epochs = 1;
+    p.early_stopping_enabled = false;
+    p.min_epochs_before_stopping = 1;
+
+    let mut trainer = smoke_trainer_with(p)?;
+    let rt = tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()?;
+
+    // Must complete 1 epoch — a hang here means VRAM oversubscription returned.
+    let metrics = rt.block_on(trainer.train(&data_dir, |_epoch, _bytes, _best| {
+        Ok("skip".to_owned())
+    }))?;
+
+    assert!(
+        metrics.epochs_trained >= 1,
+        "Single-epoch test failed: {} epochs trained (expected 1). \
+         Possible VRAM oversubscription hang regression.",
+        metrics.epochs_trained
+    );
+    assert!(
+        metrics.loss.is_finite(),
+        "Single-epoch loss must be finite, got {}",
+        metrics.loss
+    );
+
+    tracing::info!(
+        "No-hang regression: loss={:.6}, epochs={}",
+        metrics.loss, metrics.epochs_trained
+    );
+    Ok(())
+}
+
+/// Regression: counterfactual experiences must flow into PER buffer.
+///
+/// Root cause was build_next_states_f32 receiving n_episodes instead of
+/// n_episodes*2, and PER insert total missing the 2x multiplier (commit 4c5f404).
+/// ~50% of augmented training data was silently dropped.
+///
+/// After one epoch with buffer_size=10000, gpu_n_episodes=32, timesteps=100:
+///   base = 32 * 100 = 3200
+///   counterfactual = 3200 * 2 = 6400
+///   PER buffer should have >= 6400 entries (6400 < 10000 cap)
+///
+/// If counterfactual is broken again, buffer would have ~3200 entries.
+#[test]
+#[ignore] // GPU + real data
+fn test_counterfactual_experiences_in_buffer() -> anyhow::Result<()> {
+    let data_dir = test_data_dir()
+        .expect("FOXHUNT_TEST_DATA or test_data/ must exist");
+
+    let mut p = smoke_params();
+    p.epochs = 1;
+    p.early_stopping_enabled = false;
+    // Ensure we can distinguish 1x from 2x: buffer must be large enough to hold 2x
+    // smoketest defaults: gpu_n_episodes=32, gpu_timesteps=100, buffer_size=10000
+    // base = 32 * 100 = 3200, counterfactual = 6400, fits in 10000
+
+    let mut trainer = smoke_trainer_with(p)?;
+    let rt = tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()?;
+    let _metrics = rt.block_on(trainer.train(&data_dir, |_epoch, _bytes, _best| {
+        Ok("skip".to_owned())
+    }))?;
+
+    // Check replay buffer size via the agent
+    let buffer_size = rt.block_on(async {
+        let agent = trainer.get_agent().read().await;
+        agent.get_replay_buffer_size().unwrap_or(0)
+    });
+
+    // With counterfactual: 32 * 100 * 2 = 6400
+    // Without counterfactual (the old bug): 32 * 100 = 3200
+    // Threshold at 4000 cleanly separates the two cases.
+    let min_expected = 4000;
+    assert!(
+        buffer_size >= min_expected,
+        "PER buffer has {} entries (expected >= {min_expected}). \
+         Counterfactual experiences may be silently dropped again. \
+         With gpu_n_episodes=32, timesteps=100, expected 6400 (32*100*2).",
+        buffer_size
+    );
+
+    tracing::info!(
+        "Counterfactual regression: PER buffer has {} entries (expected ~6400)",
+        buffer_size
+    );
+    Ok(())
+}
+
+/// Regression: gpu_n_episodes config must be honored (not auto-scaled).
+///
+/// Previously detect_gpu_hardware().optimal_n_episodes() overrode the config,
+/// causing VRAM oversubscription on H100. Now gpu_n_episodes from TOML is used
+/// directly. This test verifies a custom value propagates through.
+#[test]
+#[ignore] // GPU + real data
+fn test_gpu_n_episodes_config_honored() -> anyhow::Result<()> {
+    let data_dir = test_data_dir()
+        .expect("FOXHUNT_TEST_DATA or test_data/ must exist");
+
+    let mut p = smoke_params();
+    p.epochs = 1;
+    p.early_stopping_enabled = false;
+    p.gpu_n_episodes = 64; // non-default to prove config is read
+
+    let mut trainer = smoke_trainer_with(p)?;
+    let rt = tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()?;
+    let _metrics = rt.block_on(trainer.train(&data_dir, |_epoch, _bytes, _best| {
+        Ok("skip".to_owned())
+    }))?;
+
+    // If auto-scaling returned, n_episodes would be >64 on any modern GPU.
+    // Verify training completed with the configured value by checking the
+    // replay buffer: entries = gpu_n_episodes * timesteps * 2 (counterfactual)
+    let buffer_size = rt.block_on(async {
+        let agent = trainer.get_agent().read().await;
+        agent.get_replay_buffer_size().unwrap_or(0)
+    });
+
+    // 64 episodes * 100 timesteps * 2 = 12800
+    // With auto-scaling on H100 it would be ~800K+ (4096 * 100 * 2)
+    let max_expected = 20000; // generous upper bound for 64 episodes
+    assert!(
+        buffer_size <= max_expected,
+        "PER buffer has {} entries (expected <= {max_expected} for 64 episodes). \
+         gpu_n_episodes config may not be honored — auto-scaling regression.",
+        buffer_size
+    );
+
+    tracing::info!(
+        "Config regression: PER buffer has {} entries (expected ~12800 for gpu_n_episodes=64)",
+        buffer_size
+    );
+    Ok(())
+}

crates/ml/src/trainers/dqn/smoke_tests/walk_forward.rs

@@ -154,3 +154,74 @@ fn test_walk_forward_no_overfitting_50_epochs() -> anyhow::Result<()> {
 
     Ok(())
 }
+
+/// Walk-forward multi-fold: verify fold transitions work on GPU.
+///
+/// Uses tight fractions (0.3/0.1/0.1/0.1) to guarantee 2+ folds even
+/// with small test data (~500 bars). This is the critical test for:
+/// - reset_for_fold() clears replay buffer + epoch state
+/// - graph_aux invalidation between folds
+/// - gpu_data re-init with new fold range
+/// - CUDA Graph forward/adam survive across fold boundaries
+/// - Regime-stratified fold generation
+#[test]
+#[ignore] // GPU + real data — ~5s on RTX 3050
+fn test_walk_forward_multi_fold() -> anyhow::Result<()> {
+    let data_dir = test_data_dir()
+        .expect("FOXHUNT_TEST_DATA or test_data/ must exist");
+
+    let mut p = smoke_params();
+    p.epochs = 3;
+    p.early_stopping_enabled = false;
+    p.gradient_collapse_patience = 50000;
+    // Tight fractions to guarantee 2+ folds on small datasets
+    p.wf_initial_train_fraction = 0.3;
+    p.wf_val_fraction = 0.1;
+    p.wf_test_fraction = 0.1;
+    p.wf_step_fraction = 0.1;
+
+    let mut trainer = smoke_trainer_with(p)?;
+    let rt = tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()?;
+    let metrics = rt.block_on(trainer.train(&data_dir, |_epoch, _bytes, _best| {
+        Ok("skip".to_owned())
+    }))?;
+
+    // Must complete (train_walk_forward returns last fold's metrics)
+    assert!(
+        metrics.epochs_trained >= 2,
+        "Multi-fold: only trained {} epochs (expected >=2)",
+        metrics.epochs_trained
+    );
+
+    let best_sharpe = metrics.additional_metrics.get("best_sharpe")
+        .copied().unwrap_or(f64::NAN);
+    let best_val_loss = metrics.additional_metrics.get("best_val_loss")
+        .copied().unwrap_or(f64::NAN);
+
+    assert!(best_sharpe.is_finite(), "Multi-fold: best_sharpe must be finite, got {best_sharpe}");
+    assert!(best_val_loss.is_finite(), "Multi-fold: best_val_loss must be finite, got {best_val_loss}");
+
+    // Loss must be finite and non-NaN (no CUDA graph corruption from fold transitions)
+    assert!(
+        metrics.loss.is_finite(),
+        "Multi-fold: training loss must be finite (got {}), graph may be corrupted across folds",
+        metrics.loss
+    );
+
+    let grad_norm = metrics.additional_metrics.get("avg_gradient_norm")
+        .copied().unwrap_or(0.0);
+    assert!(
+        grad_norm > 0.0,
+        "Multi-fold: grad_norm={grad_norm} — model must be learning after fold transition"
+    );
+
+    tracing::info!(
+        "Walk-forward multi-fold: best_sharpe={best_sharpe:.4}, best_val_loss={best_val_loss:.6}, \
+         loss={:.6}, grad_norm={grad_norm:.4}, epochs={}",
+        metrics.loss, metrics.epochs_trained
+    );
+
+    Ok(())
+}

crates/ml/tests/gpu_per_integration_test.rs

@@ -160,6 +160,8 @@ fn test_gpu_per_training_loop_cycle() {
         buf.update_priorities_gpu_raw(idx_ptr, &td_errors, batch_size).unwrap();
         buf.step();
 
+    }
+
     // After updates, priorities should be non-uniform
     assert!(buf.len() == 200);
 }