//! Performance and Load Testing //! //! Comprehensive performance and load tests for the system: //! - High-frequency order submission //! - Market data processing throughput //! - ML inference performance under load //! - Concurrent user simulation //! - Latency measurements use anyhow::Result; use common::types::{Exchange, HftTimestamp, MarketTick, Price, Quantity, Symbol, TickType}; use foxhunt_e2e::e2e_test; use foxhunt_e2e::proto::trading::{ GetOrderStatusRequest, GetPortfolioSummaryRequest, OrderSide, OrderType, SubmitOrderRequest, }; use std::collections::HashMap; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tracing::info; e2e_test!( test_order_submission_throughput, |mut framework: E2ETestFramework| async { info!("โšก Starting order submission throughput test"); let trading_client = framework.get_trading_client().await?; // Measure order submission throughput let num_orders = 100; let start = Instant::now(); let mut successful_orders = 0; let mut failed_orders = 0; for i in 0..num_orders { let order = SubmitOrderRequest { symbol: "AAPL".to_string(), side: if i % 2 == 0 { OrderSide::Buy } else { OrderSide::Sell } as i32, order_type: OrderType::Limit as i32, quantity: 100.0, price: Some(150.0 + (i as f64 * 0.1)), stop_price: None, account_id: "test_account".to_string(), metadata: HashMap::new(), }; let result = trading_client.submit_order(order).await; match result { Ok(response) => { let response = response.into_inner(); // Check if we got an order_id (successful submission) if !response.order_id.is_empty() { successful_orders += 1; } else { failed_orders += 1; } }, Err(_) => { failed_orders += 1; }, } } let elapsed = start.elapsed(); let throughput = num_orders as f64 / elapsed.as_secs_f64(); info!("๐Ÿ“Š Order Submission Throughput Results:"); info!(" Total orders: {}", num_orders); info!(" Successful: {}", successful_orders); info!(" Failed: {}", failed_orders); info!(" Time elapsed: {:?}", elapsed); info!(" Throughput: {:.2} orders/sec", throughput); // Record metrics framework .performance_tracker .record_metric("order_submission_throughput", throughput)?; framework.performance_tracker.record_metric( "order_submission_success_rate", successful_orders as f64 / num_orders as f64, )?; // Assert minimum throughput (adjust based on requirements) assert!( throughput > 10.0, "Order submission throughput should be at least 10 orders/sec, got {:.2}", throughput ); info!("โœ… Order submission throughput test completed"); Ok(()) } ); e2e_test!( test_concurrent_order_processing, |mut framework: E2ETestFramework| async { info!("๐Ÿ”„ Starting concurrent order processing test"); let trading_client = framework.get_trading_client().await?; // Simulate concurrent users submitting orders let num_concurrent_users = 10; let orders_per_user = 10; let start = Instant::now(); let success_counter = Arc::new(AtomicU64::new(0)); let failure_counter = Arc::new(AtomicU64::new(0)); let mut handles = vec![]; for user_id in 0..num_concurrent_users { let mut client = trading_client.clone(); let success_counter = success_counter.clone(); let failure_counter = failure_counter.clone(); let handle = tokio::spawn(async move { for order_id in 0..orders_per_user { let order = SubmitOrderRequest { symbol: "MSFT".to_string(), side: OrderSide::Buy as i32, order_type: OrderType::Market as i32, quantity: 50.0 + (order_id as f64 * 10.0), price: None, stop_price: None, account_id: format!("test_account_{}", user_id), metadata: HashMap::new(), }; match client.submit_order(order).await { Ok(response) => { if !response.into_inner().order_id.is_empty() { success_counter.fetch_add(1, Ordering::SeqCst); } else { failure_counter.fetch_add(1, Ordering::SeqCst); } }, Err(_) => { failure_counter.fetch_add(1, Ordering::SeqCst); }, } // Small delay to simulate realistic user behavior tokio::time::sleep(Duration::from_millis(10)).await; } }); handles.push(handle); } // Wait for all users to complete for handle in handles { handle.await?; } let elapsed = start.elapsed(); let total_orders = num_concurrent_users * orders_per_user; let successful = success_counter.load(Ordering::SeqCst); let failed = failure_counter.load(Ordering::SeqCst); let throughput = total_orders as f64 / elapsed.as_secs_f64(); info!("๐Ÿ“Š Concurrent Order Processing Results:"); info!(" Concurrent users: {}", num_concurrent_users); info!(" Orders per user: {}", orders_per_user); info!(" Total orders: {}", total_orders); info!(" Successful: {}", successful); info!(" Failed: {}", failed); info!(" Time elapsed: {:?}", elapsed); info!(" Throughput: {:.2} orders/sec", throughput); framework .performance_tracker .record_metric("concurrent_order_throughput", throughput)?; framework.performance_tracker.record_metric( "concurrent_success_rate", successful as f64 / total_orders as f64, )?; info!("โœ… Concurrent order processing test completed"); Ok(()) } ); e2e_test!( test_market_data_processing_throughput, |mut framework: E2ETestFramework| async { info!("๐Ÿ“Š Starting market data processing throughput test"); // Generate large volume of market data let num_ticks = 10000; let symbols = vec!["AAPL", "MSFT", "GOOGL", "TSLA", "AMZN"]; info!("Generating {} market data ticks", num_ticks); let start = Instant::now(); let market_data = generate_high_volume_market_data(&symbols, num_ticks)?; let generation_time = start.elapsed(); info!("Market data generated in {:?}", generation_time); // Process market data through ML pipeline info!("Processing market data through ML pipeline"); let processing_start = Instant::now(); let features = framework.ml_pipeline.extract_features(&market_data).await?; let processing_time = processing_start.elapsed(); let throughput = market_data.len() as f64 / processing_time.as_secs_f64(); info!("๐Ÿ“Š Market Data Processing Results:"); info!(" Total ticks: {}", market_data.len()); info!(" Features extracted: {}", features.len()); info!(" Processing time: {:?}", processing_time); info!(" Throughput: {:.2} ticks/sec", throughput); info!( " Average latency: {:.2} ยตs/tick", processing_time.as_micros() as f64 / market_data.len() as f64 ); framework .performance_tracker .record_metric("market_data_throughput", throughput)?; framework.performance_tracker.record_metric( "feature_extraction_latency_us", processing_time.as_micros() as f64 / market_data.len() as f64, )?; // Assert minimum throughput assert!( throughput > 1000.0, "Market data processing should handle at least 1000 ticks/sec, got {:.2}", throughput ); info!("โœ… Market data processing throughput test completed"); Ok(()) } ); e2e_test!( test_ml_inference_performance, |mut framework: E2ETestFramework| async { info!("๐Ÿง  Starting ML inference performance test"); let ml_status = framework.ml_pipeline.check_models_health().await?; if !ml_status.any_available() { info!("โš ๏ธ No ML models available - using mock predictions"); } // Test inference performance with different batch sizes let batch_sizes = vec![1, 10, 50, 100]; let symbols = vec!["AAPL", "MSFT", "GOOGL"]; for batch_size in batch_sizes { info!("Testing inference with batch size: {}", batch_size); let market_data = generate_high_volume_market_data(&symbols, batch_size)?; let features = framework.ml_pipeline.extract_features(&market_data).await?; let start = Instant::now(); let _prediction = if ml_status.any_available() { framework.ml_pipeline.predict_ensemble(&features).await? } else { // Mock prediction use foxhunt_e2e::ml_pipeline::{EnsemblePrediction, PredictionType}; EnsemblePrediction { signal: 0.5, confidence: 0.8, individual_predictions: vec![], ensemble_method: "mock".to_string(), total_inference_time: Duration::from_millis(10), prediction: PredictionType::Buy, signal_strength: 0.5, } }; let inference_time = start.elapsed(); let latency_per_sample = inference_time.as_micros() as f64 / batch_size as f64; info!( " Batch size {}: inference_time={:?}, latency={:.2} ยตs/sample", batch_size, inference_time, latency_per_sample ); framework.performance_tracker.record_metric( &format!("ml_inference_latency_batch_{}", batch_size), inference_time.as_micros() as f64, )?; // Assert maximum latency for real-time trading assert!( inference_time < Duration::from_millis(100), "Inference should complete under 100ms for batch size {}, got {:?}", batch_size, inference_time ); } info!("โœ… ML inference performance test completed"); Ok(()) } ); e2e_test!( test_latency_percentiles, |mut framework: E2ETestFramework| async { info!("๐Ÿ“ˆ Starting latency percentiles test"); let trading_client = framework.get_trading_client().await?; // Measure latency distribution for order status queries let num_samples = 100; let mut latencies = Vec::with_capacity(num_samples); info!("Collecting {} latency samples", num_samples); for i in 0..num_samples { let start = Instant::now(); // Use get_order_status as a latency test (simpler than order submission) let _result = trading_client .get_order_status(GetOrderStatusRequest { order_id: format!("test_order_{}", i), }) .await; let latency = start.elapsed(); latencies.push(latency); if i % 20 == 0 { info!("Sample {}: {:?}", i, latency); } } // Calculate percentiles latencies.sort(); let p50 = latencies[num_samples * 50 / 100]; let p95 = latencies[num_samples * 95 / 100]; let p99 = latencies[num_samples * 99 / 100]; let max = latencies[num_samples - 1]; let min = latencies[0]; let avg: Duration = latencies.iter().sum::() / num_samples as u32; info!("๐Ÿ“Š Latency Percentiles (Order Status Query):"); info!(" Min: {:?}", min); info!(" p50 (median): {:?}", p50); info!(" p95: {:?}", p95); info!(" p99: {:?}", p99); info!(" Max: {:?}", max); info!(" Average: {:?}", avg); // Record metrics framework .performance_tracker .record_metric("latency_p50_us", p50.as_micros() as f64)?; framework .performance_tracker .record_metric("latency_p95_us", p95.as_micros() as f64)?; framework .performance_tracker .record_metric("latency_p99_us", p99.as_micros() as f64)?; // Assert SLA targets assert!( p95 < Duration::from_millis(100), "p95 latency should be under 100ms, got {:?}", p95 ); assert!( p99 < Duration::from_millis(200), "p99 latency should be under 200ms, got {:?}", p99 ); info!("โœ… Latency percentiles test completed"); Ok(()) } ); e2e_test!( test_sustained_load, |mut framework: E2ETestFramework| async { info!("โฑ๏ธ Starting sustained load test"); let trading_client = framework.get_trading_client().await?; // Run sustained load for 30 seconds let test_duration = Duration::from_secs(30); let request_rate = 10; // requests per second let interval = Duration::from_millis(1000 / request_rate); let start = Instant::now(); let mut request_count = 0; let mut success_count = 0; let mut error_count = 0; info!( "Running sustained load: {} req/sec for {:?}", request_rate, test_duration ); while start.elapsed() < test_duration { let _result = trading_client .get_portfolio_summary(GetPortfolioSummaryRequest { account_id: "test_account".to_string(), }) .await; match _result { Ok(_) => success_count += 1, Err(_) => error_count += 1, } request_count += 1; if request_count % 50 == 0 { let elapsed = start.elapsed(); let current_rate = request_count as f64 / elapsed.as_secs_f64(); info!( "Progress: {} requests in {:?} ({:.2} req/sec)", request_count, elapsed, current_rate ); } tokio::time::sleep(interval).await; } let total_time = start.elapsed(); let actual_rate = request_count as f64 / total_time.as_secs_f64(); let success_rate = success_count as f64 / request_count as f64; info!("๐Ÿ“Š Sustained Load Test Results:"); info!(" Duration: {:?}", total_time); info!(" Total requests: {}", request_count); info!(" Successful: {}", success_count); info!(" Errors: {}", error_count); info!(" Target rate: {} req/sec", request_rate); info!(" Actual rate: {:.2} req/sec", actual_rate); info!(" Success rate: {:.2}%", success_rate * 100.0); framework .performance_tracker .record_metric("sustained_load_actual_rate", actual_rate)?; framework .performance_tracker .record_metric("sustained_load_success_rate", success_rate)?; // Assert system handled sustained load assert!( success_rate > 0.95, "Success rate should be above 95%, got {:.2}%", success_rate * 100.0 ); info!("โœ… Sustained load test completed"); Ok(()) } ); /// Generate high volume market data for performance testing fn generate_high_volume_market_data( symbols: &[&str], total_ticks: usize, ) -> Result> { use rand::Rng; let mut rng = rand::thread_rng(); let mut ticks = Vec::with_capacity(total_ticks); let base_time = SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos() as u64; let ticks_per_symbol = total_ticks / symbols.len(); for (symbol_idx, &symbol) in symbols.into_iter().enumerate() { let base_price = 100.0 + (symbol_idx as f64 * 50.0); let mut current_price = base_price; for i in 0..ticks_per_symbol { current_price += rng.gen_range(-0.5..0.5); current_price = current_price.max(base_price * 0.9).min(base_price * 1.1); ticks.push(MarketTick::with_timestamp( Symbol::new(symbol.to_string()), Price::from_f64(current_price)?, Quantity::from_u64(rng.gen_range(100..2000))?, HftTimestamp::from_nanos( base_time + (symbol_idx * ticks_per_symbol + i) as u64 * 100, ), TickType::Trade, Exchange::NASDAQ, (symbol_idx * ticks_per_symbol + i) as u64, )); } } Ok(ticks) } #[cfg(test)] mod tests { use super::*; #[test] fn test_high_volume_data_generation() { let symbols = vec!["AAPL", "MSFT", "GOOGL"]; let data = generate_high_volume_market_data(&symbols, 1000).unwrap(); assert_eq!(data.len(), 999); // 1000 / 3 = 333 per symbol, 333 * 3 = 999 for symbol in symbols { let count = data.iter().filter(|t| t.symbol.as_str() == symbol).count(); assert!(count > 300 && count < 350); } } }