//! Validates GPU backtest evaluator produces reasonable metrics //! using synthetic data and deterministic action models. //! //! These tests require a CUDA GPU and are skipped gracefully when none is available. //! Run with: `cargo test -p ml --test gpu_backtest_validation -- --ignored` /// Generate deterministic synthetic price data (random walk with drift) using LCG. fn generate_prices(n_bars: usize, seed: u64, drift: f32) -> Vec<[f32; 4]> { let mut rng_state = seed; let mut prices = Vec::with_capacity(n_bars); let mut price = 100.0_f32; for _ in 0..n_bars { // Simple LCG for determinism rng_state = rng_state .wrapping_mul(6_364_136_223_846_793_005) .wrapping_add(1_442_695_040_888_963_407); let rand_f = ((rng_state >> 33) as f32) / (u32::MAX as f32) - 0.5; let ret = drift + rand_f * 0.02; price *= 1.0 + ret; let ohlc = [price * 0.999, price * 1.001, price * 0.998, price]; prices.push(ohlc); } prices } /// Generate minimal synthetic features (just enough for the evaluator). fn generate_features(n_bars: usize, feature_dim: usize) -> Vec> { (0..n_bars) .map(|i| { let mut fv = vec![0.0_f32; feature_dim]; // Put some variation in features so they are not all zero if let Some(f) = fv.get_mut(0) { *f = (i as f32) * 0.001; } fv }) .collect() } #[cfg(feature = "cuda")] mod gpu_tests { use super::*; use candle_core::{Device, Tensor}; use ml::cuda_pipeline::gpu_backtest_evaluator::{ GpuBacktestConfig, GpuBacktestEvaluator, }; use ml::MLError; /// Skip test gracefully if no CUDA device is available. fn try_cuda_device() -> Option { match Device::cuda_if_available(0) { Ok(dev) if dev.is_cuda() => Some(dev), _ => { eprintln!("CUDA not available, skipping GPU backtest test"); None } } } /// Build a closure that always returns Q-values favouring `action`. /// /// Returns `[batch_size, num_actions]` with 1.0 at `action` and 0.0 elsewhere. fn constant_action_model( action: usize, num_actions: usize, ) -> impl Fn(&Tensor) -> Result { move |states: &Tensor| { let batch_size = states .dim(0) .map_err(|e| MLError::ModelError(format!("{e}")))?; let mut q_data = vec![0.0_f32; batch_size * num_actions]; for b in 0..batch_size { let base = b * num_actions; if let Some(q) = q_data.get_mut(base + action) { *q = 1.0; } } Tensor::from_vec(q_data, (batch_size, num_actions), states.device()) .map_err(|e| MLError::ModelError(format!("{e}"))) } } // ── Individual test cases ───────────────────────────────────────────────── /// Always-long model on upward-trending data should produce positive PnL. #[test] #[ignore] fn test_always_long_on_uptrend() { let device = match try_cuda_device() { Some(d) => d, None => return, }; const FEATURE_DIM: usize = 10; const N_BARS: usize = 500; let prices = generate_prices(N_BARS, 42, 0.001); // positive drift let features = generate_features(N_BARS, FEATURE_DIM); let config = GpuBacktestConfig { max_position: 1.0, tx_cost_bps: 0.0, // zero costs for a clean signal spread_cost: 0.0, initial_capital: 100_000.0, ..Default::default() }; let mut evaluator = GpuBacktestEvaluator::new( &[prices], &[features], FEATURE_DIM, config, &device, ) .expect("evaluator creation should succeed"); // Action 4 = Long100 let model = constant_action_model(4, 5); let metrics = evaluator .evaluate(&model, 3, &device) .expect("evaluation should succeed"); assert_eq!(metrics.len(), 1, "expected exactly one window result"); let m = metrics .first() .expect("metrics vec must have at least one element"); // With positive drift and always-long, should be profitable assert!( m.total_pnl > 0.0, "expected positive PnL for long on uptrend, got {}", m.total_pnl ); assert!( m.max_drawdown >= 0.0, "drawdown should be non-negative, got {}", m.max_drawdown ); assert!( (0.0..=1.0).contains(&m.win_rate), "win_rate {} is out of [0, 1] range", m.win_rate ); } /// Always-long model on downward-trending data should produce negative PnL. #[test] #[ignore] fn test_always_long_on_downtrend() { let device = match try_cuda_device() { Some(d) => d, None => return, }; const FEATURE_DIM: usize = 10; const N_BARS: usize = 500; let prices = generate_prices(N_BARS, 77, -0.001); // negative drift let features = generate_features(N_BARS, FEATURE_DIM); let config = GpuBacktestConfig { max_position: 1.0, tx_cost_bps: 0.0, spread_cost: 0.0, initial_capital: 100_000.0, ..Default::default() }; let mut evaluator = GpuBacktestEvaluator::new( &[prices], &[features], FEATURE_DIM, config, &device, ) .expect("evaluator creation should succeed"); let model = constant_action_model(4, 5); // Always Long100 let metrics = evaluator .evaluate(&model, 3, &device) .expect("evaluation should succeed"); assert_eq!(metrics.len(), 1); let m = metrics .first() .expect("metrics vec must have at least one element"); assert!( m.total_pnl < 0.0, "expected negative PnL for long on downtrend, got {}", m.total_pnl ); assert!( m.max_drawdown >= 0.0, "drawdown should be non-negative, got {}", m.max_drawdown ); } /// Always-flat model should produce ~zero PnL and minimal trades. #[test] #[ignore] fn test_always_flat_produces_no_pnl() { let device = match try_cuda_device() { Some(d) => d, None => return, }; const FEATURE_DIM: usize = 10; const N_BARS: usize = 200; let prices = generate_prices(N_BARS, 99, 0.0); let features = generate_features(N_BARS, FEATURE_DIM); let config = GpuBacktestConfig::default(); let mut evaluator = GpuBacktestEvaluator::new( &[prices], &[features], FEATURE_DIM, config, &device, ) .expect("evaluator creation should succeed"); // Action 2 = Flat — never enters a position let model = constant_action_model(2, 5); let metrics = evaluator .evaluate(&model, 3, &device) .expect("evaluation should succeed"); assert_eq!(metrics.len(), 1); let m = metrics .first() .expect("metrics vec must have at least one element"); // Flat action means no position changes, so PnL should be approximately zero assert!( m.total_pnl.abs() < 0.01, "expected ~zero PnL for flat model, got {}", m.total_pnl ); } /// Multiple windows must produce one result per window with sensible ordering. #[test] #[ignore] fn test_multiple_windows_produce_results() { let device = match try_cuda_device() { Some(d) => d, None => return, }; const FEATURE_DIM: usize = 10; const N_BARS: usize = 300; let prices_up = generate_prices(N_BARS, 42, 0.001); let prices_down = generate_prices(N_BARS, 123, -0.001); let features1 = generate_features(N_BARS, FEATURE_DIM); let features2 = generate_features(N_BARS, FEATURE_DIM); let config = GpuBacktestConfig { max_position: 1.0, tx_cost_bps: 0.0, spread_cost: 0.0, initial_capital: 100_000.0, ..Default::default() }; let mut evaluator = GpuBacktestEvaluator::new( &[prices_up, prices_down], &[features1, features2], FEATURE_DIM, config, &device, ) .expect("evaluator creation should succeed"); let model = constant_action_model(4, 5); // Always Long100 let metrics = evaluator .evaluate(&model, 3, &device) .expect("evaluation should succeed"); assert_eq!(metrics.len(), 2, "expected exactly 2 window results"); let m0 = metrics.first().expect("window 0 result must exist"); let m1 = metrics.get(1).expect("window 1 result must exist"); // Uptrend window (0) should be more profitable than downtrend window (1) assert!( m0.total_pnl > m1.total_pnl, "expected uptrend window more profitable: {} vs {}", m0.total_pnl, m1.total_pnl ); // Both drawdowns must be non-negative assert!(m0.max_drawdown >= 0.0, "window 0 drawdown negative"); assert!(m1.max_drawdown >= 0.0, "window 1 drawdown negative"); } /// Extended metrics (VaR, CVaR, Calmar, Omega) must be finite and self-consistent. #[test] #[ignore] fn test_extended_metrics_populated() { let device = match try_cuda_device() { Some(d) => d, None => return, }; const FEATURE_DIM: usize = 10; const N_BARS: usize = 500; let prices = generate_prices(N_BARS, 42, 0.001); let features = generate_features(N_BARS, FEATURE_DIM); let config = GpuBacktestConfig::default(); let mut evaluator = GpuBacktestEvaluator::new( &[prices], &[features], FEATURE_DIM, config, &device, ) .expect("evaluator creation should succeed"); let model = constant_action_model(4, 5); let metrics = evaluator .evaluate(&model, 3, &device) .expect("evaluation should succeed"); let m = metrics .first() .expect("metrics vec must have at least one element"); assert!(!m.var_95.is_nan(), "VaR should not be NaN"); assert!(!m.cvar_95.is_nan(), "CVaR should not be NaN"); assert!(!m.calmar.is_nan(), "Calmar should not be NaN"); assert!(!m.omega_ratio.is_nan(), "Omega ratio should not be NaN"); // CVaR (conditional VaR / expected shortfall) must be <= VaR because CVaR // averages the worst returns that are already worse than the VaR threshold. // Add a small tolerance for floating-point rounding. assert!( m.cvar_95 <= m.var_95 + 1e-3, "CVaR {} should be <= VaR {} (mean of tail should not exceed threshold)", m.cvar_95, m.var_95 ); } /// total_trades must be positive when the model takes an active position. #[test] #[ignore] fn test_active_model_records_trades() { let device = match try_cuda_device() { Some(d) => d, None => return, }; const FEATURE_DIM: usize = 10; const N_BARS: usize = 300; let prices = generate_prices(N_BARS, 55, 0.001); let features = generate_features(N_BARS, FEATURE_DIM); let config = GpuBacktestConfig::default(); let mut evaluator = GpuBacktestEvaluator::new( &[prices], &[features], FEATURE_DIM, config, &device, ) .expect("evaluator creation should succeed"); let model = constant_action_model(4, 5); // Always Long100 let metrics = evaluator .evaluate(&model, 3, &device) .expect("evaluation should succeed"); let m = metrics .first() .expect("metrics vec must have at least one element"); // An always-long model must execute at least the initial entry trade assert!( m.total_trades > 0.0, "expected at least one trade for active model, got {}", m.total_trades ); assert!( (0.0..=1.0).contains(&m.win_rate), "win_rate {} out of [0, 1]", m.win_rate ); } }