//! GPU Benchmark Integration Tests //! //! Comprehensive end-to-end tests for the GPU training benchmark system. //! Tests full workflow from data loading through model benchmarking to result generation. //! //! ## Test Categories //! //! 1. **Setup & Fixtures**: Test helpers and data loading //! 2. **Module Integration**: Cross-module functionality //! 3. **Model Benchmarks**: Full model training runs (GPU-only) //! 4. **End-to-End**: Complete benchmark coordinator //! 5. **Error Handling**: Failure modes and recovery //! 6. **Performance**: Overhead and efficiency validation //! //! ## GPU Test Convention //! //! Tests marked with `#[ignore]` require GPU and are slow (5-60 minutes). //! Run with: `cargo test -- --ignored --nocapture` use ml::benchmark::*; use std::path::PathBuf; use std::sync::Arc; use tokio; // ============================================================================ // Section 1: Setup and Fixtures (Test Helpers) // ============================================================================ /// Test helper: Create GPU manager with graceful CPU fallback fn setup_gpu_manager() -> Arc { Arc::new( GpuHardwareManager::new().expect("Failed to create GPU manager (check CUDA installation)"), ) } /// Test helper: Load small subset of DBN data for testing /// Uses 6E.FUT (Euro FX futures) with 1000 bars maximum fn load_test_data() -> Vec { let data_dir = PathBuf::from("test_data/real/databento/ml_training"); if !data_dir.exists() { eprintln!("WARNING: Test data directory not found: {:?}", data_dir); return vec![]; } let mut loader = DbnDataLoader::new(data_dir.clone()); // Try to load 6E.FUT (Euro FX) - one of the smaller datasets match loader.load_symbol_data("6E.FUT") { Ok(data) => { let subset: Vec<_> = data.into_iter().take(1000).collect(); println!( "[Test Helper] Loaded {} market data points from 6E.FUT", subset.len() ); subset }, Err(e) => { eprintln!("WARNING: Failed to load test data: {}", e); vec![] }, } } /// Test helper: Create minimal batch size config for testing fn create_test_batch_config() -> BatchSizeConfig { BatchSizeConfig::new(32, 1) // batch_size=32, gradient_accumulation_steps=1 } // ============================================================================ // Section 2: Module Integration Tests (6 tests) // ============================================================================ #[tokio::test] #[ignore = "GPU-only test (requires CUDA)"] async fn test_gpu_warmup_reduces_variance() { // Test that warmup protocol reduces first-epoch timing variance // Expected: >50% reduction in standard deviation let gpu_manager = setup_gpu_manager(); // Measure cold start variance (no warmup) let mut cold_times = Vec::new(); for _ in 0..5 { let start = std::time::Instant::now(); gpu_manager .device() .synchronize() .expect("Device sync failed"); cold_times.push(start.elapsed().as_micros() as f64); } let cold_mean = cold_times.iter().sum::() / cold_times.len() as f64; let cold_variance = cold_times .iter() .map(|x| (x - cold_mean).powi(2)) .sum::() / cold_times.len() as f64; let cold_std_dev = cold_variance.sqrt(); // Warmup GPU gpu_manager.warmup().expect("GPU warmup failed"); // Measure warm start variance (after warmup) let mut warm_times = Vec::new(); for _ in 0..5 { let start = std::time::Instant::now(); gpu_manager .device() .synchronize() .expect("Device sync failed"); warm_times.push(start.elapsed().as_micros() as f64); } let warm_mean = warm_times.iter().sum::() / warm_times.len() as f64; let warm_variance = warm_times .iter() .map(|x| (x - warm_mean).powi(2)) .sum::() / warm_times.len() as f64; let warm_std_dev = warm_variance.sqrt(); println!("[GPU Warmup Test]"); println!(" Cold std dev: {:.2}μs", cold_std_dev); println!(" Warm std dev: {:.2}μs", warm_std_dev); println!( " Reduction: {:.1}%", (1.0 - warm_std_dev / cold_std_dev) * 100.0 ); // Assert warmup reduces variance by >50% assert!( warm_std_dev < cold_std_dev * 0.5, "Warmup should reduce timing variance by >50% (cold: {:.2}, warm: {:.2})", cold_std_dev, warm_std_dev ); } #[tokio::test] async fn test_statistical_sampler_with_real_timings() { // Test statistical sampler with realistic epoch timings // Verify 95% CI calculation, outlier removal let mut sampler = StatisticalSampler::new(2); // 2 warmup epochs // Simulate 10 epoch timings with some variance (in seconds, not milliseconds) let epoch_times_s = vec![ 0.150, 0.145, // Warmup (ignored) 0.100, 0.102, 0.098, 0.101, 0.099, // Normal epochs 0.097, 0.103, 0.1005, 0.1015, // More normal epochs 0.200, // Outlier (spike) ]; for &time_s in epoch_times_s.iter() { sampler.add_sample(time_s); } let stats = sampler .compute_statistics() .expect("Failed to compute statistics"); println!("[Statistical Sampler Test]"); println!(" Mean: {:.2}ms", stats.mean_seconds * 1000.0); println!(" Median (P50): {:.2}ms", stats.p50_median * 1000.0); println!(" Std Dev: {:.2}ms", stats.std_dev * 1000.0); println!( " 95% CI: [{:.2}, {:.2}]ms", stats.confidence_interval_95.0 * 1000.0, stats.confidence_interval_95.1 * 1000.0 ); println!(" Number of samples: {}", stats.num_samples); println!(" Outliers removed: {}", stats.outliers_removed); // Verify samples were recorded (after warmup) assert!(stats.num_samples > 0, "Should have recorded samples"); // Verify mean is reasonable (should be ~100ms, excluding outlier) let mean_ms = stats.mean_seconds * 1000.0; assert!( mean_ms > 90.0 && mean_ms < 150.0, "Mean should be ~100ms, got {:.2}ms", mean_ms ); // Verify median is close to mean (data should be roughly normal) let median_ms = stats.p50_median * 1000.0; assert!( (median_ms - mean_ms).abs() < 50.0, "Median should be reasonably close to mean" ); // Verify 95% CI includes the mean assert!( stats.confidence_interval_95.0 < stats.mean_seconds && stats.mean_seconds < stats.confidence_interval_95.1, "Mean should be within 95% CI" ); } #[tokio::test] #[ignore = "GPU-only test"] async fn test_batch_size_finder_gpu() { // Test batch size finder on real GPU // Verify OOM detection, binary search convergence let gpu_manager = setup_gpu_manager(); let device = gpu_manager.device().clone(); let finder = BatchSizeFinder::new(device); // Simple test function that fails for very large batch sizes let test_fn = |batch_size: usize| -> Result { if batch_size > 8192 { Err(ml::MLError::ModelError("Out of memory".to_string())) // Simulate OOM } else { Ok(true) } }; let config = finder .find_optimal_batch_size(test_fn) .expect("Batch size finder failed"); println!("[Batch Size Finder Test]"); println!(" Found batch size: {}", config.batch_size); println!( " Gradient accum steps: {}", config.gradient_accumulation_steps ); println!(" Effective batch size: {}", config.effective_batch_size); // Verify batch size is reasonable assert!( config.batch_size >= 16 && config.batch_size <= 8192, "Batch size should be in range [16, 8192], got {}", config.batch_size ); // Verify effective batch size >= batch size assert!( config.effective_batch_size >= config.batch_size, "Effective batch size should be >= batch size" ); } #[tokio::test] #[ignore = "GPU-only test"] async fn test_memory_profiler_accuracy() { // Test memory profiler against real GPU allocations // Verify peak/avg calculations accurate use candle_core::{Device, Tensor}; let gpu_manager = setup_gpu_manager(); // Check if CUDA is available let device = gpu_manager.device(); if !matches!(device, Device::Cuda(_)) { println!("[Memory Profiler Test] Skipping - CUDA not available"); return; } let mut profiler = MemoryProfiler::new(0); // GPU 0 // Take baseline snapshot profiler.take_snapshot().expect("Failed to take snapshot"); // Allocate some memory (simulate model tensors) let _tensor1 = Tensor::zeros(&[1000, 1000], candle_core::DType::F32, device) .expect("Failed to allocate tensor1"); profiler.take_snapshot().expect("Failed to take snapshot"); let _tensor2 = Tensor::zeros(&[2000, 2000], candle_core::DType::F32, device) .expect("Failed to allocate tensor2"); profiler.take_snapshot().expect("Failed to take snapshot"); // Drop first tensor (memory should decrease) drop(_tensor1); profiler.take_snapshot().expect("Failed to take snapshot"); let peak_mb = profiler.peak_usage_mb(); let avg_mb = profiler.avg_usage_mb(); let snapshot_count = profiler.snapshot_count(); println!("[Memory Profiler Test]"); println!(" Peak memory: {:.2}MB", peak_mb); println!(" Average memory: {:.2}MB", avg_mb); println!(" Total snapshots: {}", snapshot_count); // Verify peak >= average (peak should be at least as much as average) assert!(peak_mb >= avg_mb, "Peak memory should be >= average memory"); // Verify we took 4 snapshots assert_eq!(snapshot_count, 4); // Verify peak is reasonable (should be >0 since we allocated tensors) assert!(peak_mb > 0.0, "Peak memory should be >0 after allocations"); } #[tokio::test] async fn test_stability_validator_detects_divergence() { // Test stability validator with diverging loss curve // Verify early detection (<5 epochs) let mut validator = StabilityValidator::new(); // Simulate diverging training (exponentially increasing loss) let losses = vec![1.0, 1.5, 2.5, 5.0, 10.0, 25.0]; let gradient_norms = vec![0.1, 0.5, 1.0, 5.0, 20.0, 100.0]; for (epoch, (&loss, &grad_norm)) in losses.iter().zip(gradient_norms.iter()).enumerate() { validator.record_loss(loss); validator.record_gradient_norm(grad_norm); let metrics = validator.validate(); println!( "[Stability Validator Test] Epoch {}: loss={:.2}, grad_norm={:.2}, trend={:?}", epoch, loss, grad_norm, metrics.loss_trend ); // Check if divergence detected early if epoch >= 4 { // Should detect divergence by epoch 4 assert!( matches!(metrics.loss_trend, LossTrend::Diverging), "Should detect diverging loss trend by epoch 4" ); assert!( matches!(metrics.gradient_health, GradientHealth::Exploding), "Should detect exploding gradients by epoch 4" ); break; } } } #[tokio::test] async fn test_data_loader_loads_all_symbols() { // Test DBN data loader with all available symbols // Verify data validation catches errors let data_dir = PathBuf::from("test_data/real/databento/ml_training"); if !data_dir.exists() { println!("[Data Loader Test] Skipping - test data not found"); return; } let mut loader = DbnDataLoader::new(data_dir); // Load all data let data = match loader.load_all_data() { Ok(d) => d, Err(e) => { println!("[Data Loader Test] Failed to load data: {}", e); return; }, }; let stats = loader.data_statistics(); println!("[Data Loader Test]"); println!(" Total files: {}", stats.total_files); println!(" Total bars: {}", stats.total_bars); println!(" Symbols: {:?}", stats.symbols); println!(" Date range: {:?}", stats.date_range); println!(" Invalid bars: {}", stats.invalid_bars); // Verify we found some data assert!(stats.total_files > 0, "Should find DBN files"); assert!(stats.total_bars > 0, "Should load some data bars"); assert!(!stats.symbols.is_empty(), "Should have symbols"); // Verify data validation (use data we already loaded) if data.is_empty() { println!("[Data Loader Test] Warning: No data loaded for validation"); return; } // Check first few bars for validity for (i, bar) in data.iter().take(10).enumerate() { assert!(bar.is_valid(), "Bar {} should be valid", i); } } // ============================================================================ // Section 3: Model Benchmark Tests (4 tests, GPU-only) // ============================================================================ #[tokio::test] #[ignore = "GPU-only, slow (5-10 min)"] async fn test_dqn_benchmark_full_run() { // Full DQN benchmark: 10 epochs, real data // Verify: Memory <150MB, training stable, statistics valid let test_data = load_test_data(); if test_data.is_empty() { println!("[DQN Benchmark Test] Skipping - no test data available"); return; } let gpu_manager = setup_gpu_manager(); let mut runner = DqnBenchmarkRunner::new(gpu_manager); let result = runner .run_benchmark(10) .await .expect("DQN benchmark failed"); println!("[DQN Benchmark Test]"); println!(" Model: {}", result.model_name); println!(" Total epochs: {}", result.total_epochs); println!( " Mean epoch time: {:.2}ms", result.statistics.mean_seconds * 1000.0 ); println!(" Peak memory: {:.2}MB", result.memory_peak_mb); println!(" Average loss: {:.4}", result.avg_loss); println!(" Stability: {:?}", result.stability); // Verify memory usage is reasonable (<150MB target) assert!( result.memory_peak_mb < 150.0, "DQN should use <150MB VRAM, got {:.2}MB", result.memory_peak_mb ); // Verify training completed all epochs assert_eq!(result.total_epochs, 10); // Verify statistics are valid assert!(result.statistics.mean_seconds > 0.0); assert!(result.statistics.std_dev >= 0.0); // Verify losses recorded assert_eq!(result.training_losses.len(), 10); assert!(result.avg_loss > 0.0); // Verify stability metrics computed // LossTrend should be one of: Converging, Diverging, or Stagnant // GradientHealth should be one of: Healthy, Exploding, or Vanishing assert!( matches!( result.stability.loss_trend, LossTrend::Converging | LossTrend::Diverging | LossTrend::Stagnant ), "Loss trend should be computed" ); assert!( matches!( result.stability.gradient_health, GradientHealth::Healthy | GradientHealth::Exploding | GradientHealth::Vanishing ), "Gradient health should be computed" ); } #[tokio::test] #[ignore = "GPU-only, slow (5-10 min)"] async fn test_ppo_benchmark_full_run() { // Full PPO benchmark: 10 epochs, real data // Verify: Memory <200MB, training stable, statistics valid let test_data = load_test_data(); if test_data.is_empty() { println!("[PPO Benchmark Test] Skipping - no test data available"); return; } let gpu_manager = setup_gpu_manager(); let mut runner = PpoBenchmarkRunner::new(gpu_manager); let result = runner .run_benchmark(10) .await .expect("PPO benchmark failed"); println!("[PPO Benchmark Test]"); println!(" Model: {}", result.model_name); println!(" Total epochs: {}", result.total_epochs); println!( " Mean epoch time: {:.2}ms", result.statistics.mean_seconds * 1000.0 ); println!(" Peak memory: {:.2}MB", result.memory_peak_mb); println!(" Avg policy loss: {:.4}", result.avg_policy_loss); println!(" Avg value loss: {:.4}", result.avg_value_loss); // Verify memory usage is reasonable (<200MB target) assert!( result.memory_peak_mb < 200.0, "PPO should use <200MB VRAM, got {:.2}MB", result.memory_peak_mb ); // Verify training completed all epochs assert_eq!(result.total_epochs, 10); // Verify statistics are valid assert!(result.statistics.mean_seconds > 0.0); assert!(result.statistics.std_dev >= 0.0); // Verify epoch times recorded assert_eq!(result.epoch_times_ms.len(), 10); // Verify losses are reasonable assert!(result.avg_policy_loss.is_finite()); assert!(result.avg_value_loss.is_finite()); } #[tokio::test] #[ignore = "GPU-only, slow (10-20 min) - PLACEHOLDER"] async fn test_mamba2_benchmark_full_run() { // Full MAMBA-2 benchmark: 10 epochs, real data // Verify: Memory 150-500MB, gradient accumulation working // TODO: Implement when MAMBA-2 benchmark module is ready // Expected to be implemented by Module 7a println!("[MAMBA-2 Benchmark Test] PLACEHOLDER - waiting for Module 7a"); println!(" Expected memory: 150-500MB VRAM"); println!(" Expected features: Gradient accumulation for large model"); // This test will be implemented after Module 7a is complete } #[tokio::test] #[ignore = "GPU-only, slow (10-20 min) - PLACEHOLDER"] async fn test_tft_benchmark_full_run() { // Full TFT benchmark: 10 epochs, real data (batch_size ≤4) // Verify: Memory 1.5-2.5GB, doesn't OOM // TODO: Implement when TFT benchmark module is ready // Expected to be implemented by Module 7b println!("[TFT Benchmark Test] PLACEHOLDER - waiting for Module 7b"); println!(" Expected memory: 1.5-2.5GB VRAM"); println!(" Expected constraint: batch_size ≤ 4 (very memory intensive)"); // This test will be implemented after Module 7b is complete } // ============================================================================ // Section 4: End-to-End Coordinator Test (1 test) // ============================================================================ #[tokio::test] #[ignore = "GPU-only, very slow (30-60 min) - PLACEHOLDER"] async fn test_full_benchmark_coordinator() { // Run complete benchmark: DQN + PPO + MAMBA-2 + TFT // Verify: // - JSON output generated // - Decision framework applied correctly // - All statistics valid // - No crashes or panics // TODO: Implement when coordinator module (Module 10) is ready println!("[Full Coordinator Test] PLACEHOLDER - waiting for Module 10"); println!(" Expected duration: 30-60 minutes"); println!(" Expected output: JSON report with all model benchmarks"); println!(" Expected decision: local_gpu or cloud_gpu recommendation"); // This test will run: // 1. Load CLI args with reduced epochs (5 instead of 30) // 2. Run all benchmarks sequentially // 3. Apply decision framework // 4. Generate JSON report // 5. Verify all outputs valid } // ============================================================================ // Section 5: Error Handling Tests (4 tests) // ============================================================================ #[tokio::test] async fn test_graceful_cpu_fallback() { // Test CPU fallback when GPU unavailable let gpu_manager = setup_gpu_manager(); let device = gpu_manager.device(); println!("[CPU Fallback Test]"); println!(" Device: {:?}", device); // GPU manager should always succeed (falls back to CPU if no CUDA) match device { candle_core::Device::Cpu => { println!(" Status: Running on CPU (CUDA not available)"); }, candle_core::Device::Cuda(_) => { println!(" Status: Running on GPU (CUDA available)"); }, _ => { panic!("Unexpected device type"); }, } // Verify device is usable let result = gpu_manager.warmup(); assert!(result.is_ok(), "Warmup should succeed on any device"); } #[tokio::test] #[ignore = "GPU-only test"] async fn test_thermal_throttling_detection() { // Test thermal warning/error detection (mock high temp) let gpu_manager = setup_gpu_manager(); // Check thermal throttling match gpu_manager.check_thermal_throttling() { Ok(is_throttling) => { println!("[Thermal Test]"); println!(" Thermal throttling: {}", is_throttling); // Verify not throttling under normal conditions assert!( !is_throttling, "GPU should not be thermal throttling under normal test conditions" ); }, Err(e) => { println!("[Thermal Test] Could not check thermal throttling: {}", e); // Not a failure - some systems don't expose thermal info }, } } #[tokio::test] #[ignore = "GPU-only test"] async fn test_oom_handling() { // Test OOM detection and recovery use candle_core::{Device, Tensor}; let gpu_manager = setup_gpu_manager(); let device = gpu_manager.device(); if !matches!(device, Device::Cuda(_)) { println!("[OOM Test] Skipping - CUDA not available"); return; } // Try to allocate unreasonably large tensor (should fail) let result = Tensor::zeros( &[100_000, 100_000], // 10 billion floats = 40GB candle_core::DType::F32, device, ); println!("[OOM Test]"); match result { Ok(_) => { println!(" WARNING: Large allocation succeeded (unexpected)"); }, Err(e) => { println!(" Successfully caught OOM error: {}", e); // This is the expected path }, } // Verify GPU is still functional after OOM let recovery_result = Tensor::zeros(&[100, 100], candle_core::DType::F32, device); assert!( recovery_result.is_ok(), "GPU should recover after OOM error" ); } #[tokio::test] async fn test_invalid_data_handling() { // Test data loader with corrupted/invalid data // Create invalid market data point let invalid_bar = MarketDataPoint { timestamp: 0, symbol: "TEST".to_string(), open: 100.0, high: 50.0, // Invalid: high < low low: 120.0, // Invalid: low > high close: 110.0, volume: -10.0, // Invalid: negative volume }; // Verify validation catches errors assert!( !invalid_bar.is_valid(), "Invalid data should be detected by validation" ); // Test with NaN values let nan_bar = MarketDataPoint { timestamp: 0, symbol: "TEST".to_string(), open: f64::NAN, high: 100.0, low: 90.0, close: 95.0, volume: 1000.0, }; assert!( !nan_bar.is_valid(), "NaN values should be detected by validation" ); // Test with inconsistent OHLC let inconsistent_bar = MarketDataPoint { timestamp: 0, symbol: "TEST".to_string(), open: 100.0, high: 105.0, low: 95.0, close: 110.0, // Invalid: close > high volume: 1000.0, }; assert!( !inconsistent_bar.is_valid(), "Inconsistent OHLC should be detected" ); println!("[Invalid Data Test]"); println!(" ✓ High < Low detected"); println!(" ✓ Negative volume detected"); println!(" ✓ NaN values detected"); println!(" ✓ Close > High detected"); } // ============================================================================ // Section 6: Performance Tests (2 tests) // ============================================================================ #[tokio::test] #[ignore = "Performance validation"] async fn test_memory_profiler_overhead() { // Verify memory profiler adds <10ms overhead per snapshot use std::time::Instant; let mut profiler = MemoryProfiler::new(0); // Measure snapshot time let iterations = 100; let start = Instant::now(); for _ in 0..iterations { profiler.take_snapshot().ok(); // Ignore errors for benchmark } let elapsed = start.elapsed(); let avg_per_snapshot = elapsed.as_micros() as f64 / iterations as f64; println!("[Memory Profiler Overhead Test]"); println!(" Total snapshots: {}", iterations); println!(" Total time: {:.2}ms", elapsed.as_millis()); println!(" Average per snapshot: {:.2}μs", avg_per_snapshot); // Verify overhead is <10ms (10,000μs) per snapshot assert!( avg_per_snapshot < 10_000.0, "Memory profiler overhead should be <10ms per snapshot, got {:.2}μs", avg_per_snapshot ); } #[tokio::test] #[ignore = "Performance validation"] async fn test_statistical_sampler_performance() { // Verify statistics computation <1ms for 20 samples use std::time::Instant; let mut sampler = StatisticalSampler::new(0); // No warmup for this test // Record 20 samples (in seconds) for i in 0..20 { sampler.add_sample(0.100 + (i as f64 * 0.0005)); // ~100ms with small increments } // Measure statistics computation time let start = Instant::now(); let _stats = sampler.compute_statistics(); let elapsed = start.elapsed(); println!("[Statistical Sampler Performance Test]"); println!(" Computation time: {:.2}μs", elapsed.as_micros()); // Verify computation is <1ms (1000μs) assert!( elapsed.as_micros() < 1000, "Statistics computation should be <1ms, got {}μs", elapsed.as_micros() ); }