diff --git a/crates/ml/tests/gpu_backtest_validation.rs b/crates/ml/tests/gpu_backtest_validation.rs new file mode 100644 index 000000000..1607f91a0 --- /dev/null +++ b/crates/ml/tests/gpu_backtest_validation.rs @@ -0,0 +1,396 @@ +//! Validates GPU backtest evaluator produces reasonable metrics +//! using synthetic data and deterministic action models. +//! +//! These tests require a CUDA GPU and are skipped gracefully when none is available. +//! Run with: `cargo test -p ml --test gpu_backtest_validation -- --ignored` + +/// Generate deterministic synthetic price data (random walk with drift) using LCG. +fn generate_prices(n_bars: usize, seed: u64, drift: f32) -> Vec<[f32; 4]> { + let mut rng_state = seed; + let mut prices = Vec::with_capacity(n_bars); + let mut price = 100.0_f32; + + for _ in 0..n_bars { + // Simple LCG for determinism + rng_state = rng_state + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + let rand_f = ((rng_state >> 33) as f32) / (u32::MAX as f32) - 0.5; + let ret = drift + rand_f * 0.02; + price *= 1.0 + ret; + let ohlc = [price * 0.999, price * 1.001, price * 0.998, price]; + prices.push(ohlc); + } + prices +} + +/// Generate minimal synthetic features (just enough for the evaluator). +fn generate_features(n_bars: usize, feature_dim: usize) -> Vec> { + (0..n_bars) + .map(|i| { + let mut fv = vec![0.0_f32; feature_dim]; + // Put some variation in features so they are not all zero + if let Some(f) = fv.get_mut(0) { + *f = (i as f32) * 0.001; + } + fv + }) + .collect() +} + +#[cfg(feature = "cuda")] +mod gpu_tests { + use super::*; + use candle_core::{Device, Tensor}; + use ml::cuda_pipeline::gpu_backtest_evaluator::{ + GpuBacktestConfig, GpuBacktestEvaluator, + }; + use ml::MLError; + + /// Skip test gracefully if no CUDA device is available. + fn try_cuda_device() -> Option { + match Device::cuda_if_available(0) { + Ok(dev) if dev.is_cuda() => Some(dev), + _ => { + eprintln!("CUDA not available, skipping GPU backtest test"); + None + } + } + } + + /// Build a closure that always returns Q-values favouring `action`. + /// + /// Returns `[batch_size, num_actions]` with 1.0 at `action` and 0.0 elsewhere. + fn constant_action_model( + action: usize, + num_actions: usize, + ) -> impl Fn(&Tensor) -> Result { + move |states: &Tensor| { + let batch_size = states + .dim(0) + .map_err(|e| MLError::ModelError(format!("{e}")))?; + let mut q_data = vec![0.0_f32; batch_size * num_actions]; + for b in 0..batch_size { + let base = b * num_actions; + if let Some(q) = q_data.get_mut(base + action) { + *q = 1.0; + } + } + Tensor::from_vec(q_data, (batch_size, num_actions), states.device()) + .map_err(|e| MLError::ModelError(format!("{e}"))) + } + } + + // ── Individual test cases ───────────────────────────────────────────────── + + /// Always-long model on upward-trending data should produce positive PnL. + #[test] + #[ignore] + fn test_always_long_on_uptrend() { + let device = match try_cuda_device() { + Some(d) => d, + None => return, + }; + + const FEATURE_DIM: usize = 10; + const N_BARS: usize = 500; + + let prices = generate_prices(N_BARS, 42, 0.001); // positive drift + let features = generate_features(N_BARS, FEATURE_DIM); + + let config = GpuBacktestConfig { + max_position: 1.0, + tx_cost_bps: 0.0, // zero costs for a clean signal + spread_cost: 0.0, + initial_capital: 100_000.0, + }; + + let mut evaluator = GpuBacktestEvaluator::new( + &[prices], + &[features], + FEATURE_DIM, + config, + &device, + ) + .expect("evaluator creation should succeed"); + + // Action 4 = Long100 + let model = constant_action_model(4, 5); + let metrics = evaluator + .evaluate(&model, 3, &device) + .expect("evaluation should succeed"); + + assert_eq!(metrics.len(), 1, "expected exactly one window result"); + let m = metrics + .first() + .expect("metrics vec must have at least one element"); + + // With positive drift and always-long, should be profitable + assert!( + m.total_pnl > 0.0, + "expected positive PnL for long on uptrend, got {}", + m.total_pnl + ); + assert!( + m.max_drawdown >= 0.0, + "drawdown should be non-negative, got {}", + m.max_drawdown + ); + assert!( + (0.0..=1.0).contains(&m.win_rate), + "win_rate {} is out of [0, 1] range", + m.win_rate + ); + } + + /// Always-long model on downward-trending data should produce negative PnL. + #[test] + #[ignore] + fn test_always_long_on_downtrend() { + let device = match try_cuda_device() { + Some(d) => d, + None => return, + }; + + const FEATURE_DIM: usize = 10; + const N_BARS: usize = 500; + + let prices = generate_prices(N_BARS, 77, -0.001); // negative drift + let features = generate_features(N_BARS, FEATURE_DIM); + + let config = GpuBacktestConfig { + max_position: 1.0, + tx_cost_bps: 0.0, + spread_cost: 0.0, + initial_capital: 100_000.0, + }; + + let mut evaluator = GpuBacktestEvaluator::new( + &[prices], + &[features], + FEATURE_DIM, + config, + &device, + ) + .expect("evaluator creation should succeed"); + + let model = constant_action_model(4, 5); // Always Long100 + let metrics = evaluator + .evaluate(&model, 3, &device) + .expect("evaluation should succeed"); + + assert_eq!(metrics.len(), 1); + let m = metrics + .first() + .expect("metrics vec must have at least one element"); + + assert!( + m.total_pnl < 0.0, + "expected negative PnL for long on downtrend, got {}", + m.total_pnl + ); + assert!( + m.max_drawdown >= 0.0, + "drawdown should be non-negative, got {}", + m.max_drawdown + ); + } + + /// Always-flat model should produce ~zero PnL and minimal trades. + #[test] + #[ignore] + fn test_always_flat_produces_no_pnl() { + let device = match try_cuda_device() { + Some(d) => d, + None => return, + }; + + const FEATURE_DIM: usize = 10; + const N_BARS: usize = 200; + + let prices = generate_prices(N_BARS, 99, 0.0); + let features = generate_features(N_BARS, FEATURE_DIM); + + let config = GpuBacktestConfig::default(); + + let mut evaluator = GpuBacktestEvaluator::new( + &[prices], + &[features], + FEATURE_DIM, + config, + &device, + ) + .expect("evaluator creation should succeed"); + + // Action 2 = Flat — never enters a position + let model = constant_action_model(2, 5); + let metrics = evaluator + .evaluate(&model, 3, &device) + .expect("evaluation should succeed"); + + assert_eq!(metrics.len(), 1); + let m = metrics + .first() + .expect("metrics vec must have at least one element"); + + // Flat action means no position changes, so PnL should be approximately zero + assert!( + m.total_pnl.abs() < 0.01, + "expected ~zero PnL for flat model, got {}", + m.total_pnl + ); + } + + /// Multiple windows must produce one result per window with sensible ordering. + #[test] + #[ignore] + fn test_multiple_windows_produce_results() { + let device = match try_cuda_device() { + Some(d) => d, + None => return, + }; + + const FEATURE_DIM: usize = 10; + const N_BARS: usize = 300; + + let prices_up = generate_prices(N_BARS, 42, 0.001); + let prices_down = generate_prices(N_BARS, 123, -0.001); + let features1 = generate_features(N_BARS, FEATURE_DIM); + let features2 = generate_features(N_BARS, FEATURE_DIM); + + let config = GpuBacktestConfig { + max_position: 1.0, + tx_cost_bps: 0.0, + spread_cost: 0.0, + initial_capital: 100_000.0, + }; + + let mut evaluator = GpuBacktestEvaluator::new( + &[prices_up, prices_down], + &[features1, features2], + FEATURE_DIM, + config, + &device, + ) + .expect("evaluator creation should succeed"); + + let model = constant_action_model(4, 5); // Always Long100 + let metrics = evaluator + .evaluate(&model, 3, &device) + .expect("evaluation should succeed"); + + assert_eq!(metrics.len(), 2, "expected exactly 2 window results"); + + let m0 = metrics.first().expect("window 0 result must exist"); + let m1 = metrics.get(1).expect("window 1 result must exist"); + + // Uptrend window (0) should be more profitable than downtrend window (1) + assert!( + m0.total_pnl > m1.total_pnl, + "expected uptrend window more profitable: {} vs {}", + m0.total_pnl, + m1.total_pnl + ); + + // Both drawdowns must be non-negative + assert!(m0.max_drawdown >= 0.0, "window 0 drawdown negative"); + assert!(m1.max_drawdown >= 0.0, "window 1 drawdown negative"); + } + + /// Extended metrics (VaR, CVaR, Calmar, Omega) must be finite and self-consistent. + #[test] + #[ignore] + fn test_extended_metrics_populated() { + let device = match try_cuda_device() { + Some(d) => d, + None => return, + }; + + const FEATURE_DIM: usize = 10; + const N_BARS: usize = 500; + + let prices = generate_prices(N_BARS, 42, 0.001); + let features = generate_features(N_BARS, FEATURE_DIM); + let config = GpuBacktestConfig::default(); + + let mut evaluator = GpuBacktestEvaluator::new( + &[prices], + &[features], + FEATURE_DIM, + config, + &device, + ) + .expect("evaluator creation should succeed"); + + let model = constant_action_model(4, 5); + let metrics = evaluator + .evaluate(&model, 3, &device) + .expect("evaluation should succeed"); + + let m = metrics + .first() + .expect("metrics vec must have at least one element"); + + assert!(!m.var_95.is_nan(), "VaR should not be NaN"); + assert!(!m.cvar_95.is_nan(), "CVaR should not be NaN"); + assert!(!m.calmar.is_nan(), "Calmar should not be NaN"); + assert!(!m.omega_ratio.is_nan(), "Omega ratio should not be NaN"); + + // CVaR (conditional VaR / expected shortfall) must be <= VaR because CVaR + // averages the worst returns that are already worse than the VaR threshold. + // Add a small tolerance for floating-point rounding. + assert!( + m.cvar_95 <= m.var_95 + 1e-3, + "CVaR {} should be <= VaR {} (mean of tail should not exceed threshold)", + m.cvar_95, + m.var_95 + ); + } + + /// total_trades must be positive when the model takes an active position. + #[test] + #[ignore] + fn test_active_model_records_trades() { + let device = match try_cuda_device() { + Some(d) => d, + None => return, + }; + + const FEATURE_DIM: usize = 10; + const N_BARS: usize = 300; + + let prices = generate_prices(N_BARS, 55, 0.001); + let features = generate_features(N_BARS, FEATURE_DIM); + let config = GpuBacktestConfig::default(); + + let mut evaluator = GpuBacktestEvaluator::new( + &[prices], + &[features], + FEATURE_DIM, + config, + &device, + ) + .expect("evaluator creation should succeed"); + + let model = constant_action_model(4, 5); // Always Long100 + let metrics = evaluator + .evaluate(&model, 3, &device) + .expect("evaluation should succeed"); + + let m = metrics + .first() + .expect("metrics vec must have at least one element"); + + // An always-long model must execute at least the initial entry trade + assert!( + m.total_trades > 0.0, + "expected at least one trade for active model, got {}", + m.total_trades + ); + assert!( + (0.0..=1.0).contains(&m.win_rate), + "win_rate {} out of [0, 1]", + m.win_rate + ); + } +}