//! Comprehensive Load Test for Trading Service //! //! Tests the trading service against production requirements: //! - 10K orders/sec throughput target //! - P50, P95, P99 latency measurements //! - 100+ concurrent connections //! - Order matching 1-6μs P99 baseline //! - Database performance under load //! - Resource monitoring (CPU, memory, connections) //! //! Run with: cargo test --package tests --test load_test_trading_service --release -- --nocapture use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant, SystemTime}; use tokio::time::timeout; use tonic::transport::Channel; use tonic::{Request, Status}; use uuid::Uuid; // gRPC generated code pub mod trading { tonic::include_proto!("trading"); } use trading::trading_service_client::TradingServiceClient; use trading::{OrderSide, OrderType, SubmitOrderRequest, TimeInForce}; /// Performance metrics aggregator #[derive(Debug, Clone)] struct PerformanceMetrics { latencies_ns: Vec, successful_orders: AtomicU64, failed_orders: AtomicU64, total_orders: AtomicU64, test_duration: Duration, } impl PerformanceMetrics { fn new() -> Self { Self { latencies_ns: Vec::new(), successful_orders: AtomicU64::new(0), failed_orders: AtomicU64::new(0), total_orders: AtomicU64::new(0), test_duration: Duration::ZERO, } } fn record_success(&self, latency_ns: u64) { self.successful_orders.fetch_add(1, Ordering::Relaxed); self.total_orders.fetch_add(1, Ordering::Relaxed); } fn record_failure(&self) { self.failed_orders.fetch_add(1, Ordering::Relaxed); self.total_orders.fetch_add(1, Ordering::Relaxed); } fn calculate_percentiles(mut latencies: Vec) -> (u64, u64, u64, u64, u64) { if latencies.is_empty() { return (0, 0, 0, 0, 0); } latencies.sort_unstable(); let len = latencies.len(); let min = latencies[0]; let p50 = latencies[len / 2]; let p95 = latencies[(len as f64 * 0.95) as usize]; let p99 = latencies[(len as f64 * 0.99) as usize]; let max = latencies[len - 1]; (min, p50, p95, p99, max) } fn print_summary(&self, latencies: &[u64]) { let successful = self.successful_orders.load(Ordering::Relaxed); let failed = self.failed_orders.load(Ordering::Relaxed); let total = self.total_orders.load(Ordering::Relaxed); let success_rate = if total > 0 { (successful as f64 / total as f64) * 100.0 } else { 0.0 }; let throughput = if self.test_duration.as_secs_f64() > 0.0 { successful as f64 / self.test_duration.as_secs_f64() } else { 0.0 }; let (min, p50, p95, p99, max) = Self::calculate_percentiles(latencies.to_vec()); println!("\n╔═══════════════════════════════════════════════════════════╗"); println!("║ TRADING SERVICE LOAD TEST RESULTS ║"); println!("╠═══════════════════════════════════════════════════════════╣"); println!( "║ Test Duration: {:.2}s", self.test_duration.as_secs_f64() ); println!("║ Total Orders: {}", total); println!( "║ Successful Orders: {} ({:.2}%)", successful, success_rate ); println!("║ Failed Orders: {}", failed); println!("║ Throughput: {:.0} orders/sec", throughput); println!("╠═══════════════════════════════════════════════════════════╣"); println!("║ LATENCY METRICS ║"); println!("╠═══════════════════════════════════════════════════════════╣"); println!( "║ Min Latency: {:.2}ms ({:.2}μs)", min as f64 / 1_000_000.0, min as f64 / 1_000.0 ); println!( "║ P50 Latency: {:.2}ms ({:.2}μs)", p50 as f64 / 1_000_000.0, p50 as f64 / 1_000.0 ); println!( "║ P95 Latency: {:.2}ms ({:.2}μs)", p95 as f64 / 1_000_000.0, p95 as f64 / 1_000.0 ); println!( "║ P99 Latency: {:.2}ms ({:.2}μs)", p99 as f64 / 1_000_000.0, p99 as f64 / 1_000.0 ); println!( "║ Max Latency: {:.2}ms ({:.2}μs)", max as f64 / 1_000_000.0, max as f64 / 1_000.0 ); println!("╚═══════════════════════════════════════════════════════════╝"); // Performance assessment println!("\n📊 PERFORMANCE ASSESSMENT:"); if throughput >= 10_000.0 { println!( "✅ Throughput target ACHIEVED: {:.0} orders/sec (target: 10K orders/sec)", throughput ); } else { println!( "⚠️ Throughput BELOW target: {:.0} orders/sec (target: 10K orders/sec)", throughput ); } if p99 < 100_000_000 { // 100ms in nanoseconds println!( "✅ P99 latency GOOD: {:.2}ms (< 100ms)", p99 as f64 / 1_000_000.0 ); } else { println!( "⚠️ P99 latency HIGH: {:.2}ms (> 100ms)", p99 as f64 / 1_000_000.0 ); } if success_rate >= 99.0 { println!("✅ Success rate EXCELLENT: {:.2}%", success_rate); } else if success_rate >= 95.0 { println!("⚠️ Success rate ACCEPTABLE: {:.2}%", success_rate); } else { println!("❌ Success rate POOR: {:.2}%", success_rate); } } } /// Create a test order request fn create_order_request(index: u64) -> SubmitOrderRequest { let symbols = vec!["BTC/USD", "ETH/USD", "SOL/USD", "AVAX/USD", "MATIC/USD"]; let symbol = symbols[(index % symbols.len() as u64) as usize].to_string(); SubmitOrderRequest { order_id: Uuid::new_v4().to_string(), symbol, side: if index % 2 == 0 { OrderSide::Buy.into() } else { OrderSide::Sell.into() }, order_type: OrderType::Limit.into(), quantity: (1.0 + (index % 10) as f64 * 0.1).to_string(), price: Some((50000.0 + (index % 1000) as f64).to_string()), time_in_force: TimeInForce::GoodTillCancel.into(), } } /// Connect to Trading Service async fn connect_trading_service( ) -> Result, Box> { let endpoint = "http://localhost:50052"; println!("🔌 Connecting to Trading Service at {}", endpoint); let channel = Channel::from_static("http://localhost:50052") .connect_timeout(Duration::from_secs(10)) .timeout(Duration::from_secs(30)) .connect() .await?; let client = TradingServiceClient::new(channel); println!("✅ Connected successfully"); Ok(client) } /// Test 1: Baseline latency with single client #[tokio::test] async fn test_1_baseline_latency() -> Result<(), Box> { println!("\n╔═══════════════════════════════════════════════════════════╗"); println!("║ TEST 1: BASELINE LATENCY (Single Client) ║"); println!("╚═══════════════════════════════════════════════════════════╝"); let mut client = connect_trading_service().await?; let num_requests = 1000; let mut latencies = Vec::with_capacity(num_requests); println!("📊 Sending {} orders sequentially...", num_requests); let start_time = Instant::now(); for i in 0..num_requests { let request = create_order_request(i as u64); let req_start = Instant::now(); let result = client.submit_order(Request::new(request)).await; let latency_ns = req_start.elapsed().as_nanos() as u64; latencies.push(latency_ns); if result.is_err() && i < 5 { eprintln!("❌ Order {} failed: {:?}", i, result.err()); } } let test_duration = start_time.elapsed(); let (min, p50, p95, p99, max) = PerformanceMetrics::calculate_percentiles(latencies.clone()); println!("\n📈 BASELINE RESULTS:"); println!(" Duration: {:.2}s", test_duration.as_secs_f64()); println!( " Throughput: {:.0} orders/sec", num_requests as f64 / test_duration.as_secs_f64() ); println!(" Min Latency: {:.2}ms", min as f64 / 1_000_000.0); println!(" P50 Latency: {:.2}ms", p50 as f64 / 1_000_000.0); println!(" P95 Latency: {:.2}ms", p95 as f64 / 1_000_000.0); println!(" P99 Latency: {:.2}ms", p99 as f64 / 1_000_000.0); println!(" Max Latency: {:.2}ms", max as f64 / 1_000_000.0); Ok(()) } /// Test 2: Concurrent connections (100 clients) #[tokio::test] async fn test_2_concurrent_connections() -> Result<(), Box> { println!("\n╔═══════════════════════════════════════════════════════════╗"); println!("║ TEST 2: CONCURRENT CONNECTIONS (100 Clients) ║"); println!("╚═══════════════════════════════════════════════════════════╝"); let num_clients = 100; let orders_per_client = 100; let metrics = Arc::new(PerformanceMetrics::new()); let latencies = Arc::new(tokio::sync::Mutex::new(Vec::new())); println!( "🚀 Spawning {} concurrent clients ({} orders each)...", num_clients, orders_per_client ); let start_time = Instant::now(); let mut tasks = Vec::new(); for client_id in 0..num_clients { let metrics_clone = Arc::clone(&metrics); let latencies_clone = Arc::clone(&latencies); let task = tokio::spawn(async move { let mut client = match connect_trading_service().await { Ok(c) => c, Err(e) => { eprintln!("❌ Client {} connection failed: {}", client_id, e); return; }, }; for order_idx in 0..orders_per_client { let request = create_order_request((client_id * orders_per_client + order_idx) as u64); let req_start = Instant::now(); match timeout( Duration::from_secs(5), client.submit_order(Request::new(request)), ) .await { Ok(Ok(_response)) => { let latency_ns = req_start.elapsed().as_nanos() as u64; metrics_clone.record_success(latency_ns); latencies_clone.lock().await.push(latency_ns); }, Ok(Err(status)) => { metrics_clone.record_failure(); if order_idx < 2 { eprintln!( "❌ Client {} order {} failed: {}", client_id, order_idx, status ); } }, Err(_) => { metrics_clone.record_failure(); if order_idx < 2 { eprintln!("⏱️ Client {} order {} timed out", client_id, order_idx); } }, } } }); tasks.push(task); } // Wait for all clients to complete for task in tasks { let _ = task.await; } let test_duration = start_time.elapsed(); let latencies_vec = latencies.lock().await.clone(); let metrics_final = PerformanceMetrics { latencies_ns: latencies_vec.clone(), successful_orders: AtomicU64::new(metrics.successful_orders.load(Ordering::Relaxed)), failed_orders: AtomicU64::new(metrics.failed_orders.load(Ordering::Relaxed)), total_orders: AtomicU64::new(metrics.total_orders.load(Ordering::Relaxed)), test_duration, }; metrics_final.print_summary(&latencies_vec); Ok(()) } /// Test 3: Sustained load (5 minutes) #[tokio::test] #[ignore = "Run explicitly with --ignored"] async fn test_3_sustained_load() -> Result<(), Box> { println!("\n╔═══════════════════════════════════════════════════════════╗"); println!("║ TEST 3: SUSTAINED LOAD (5 Minutes) ║"); println!("╚═══════════════════════════════════════════════════════════╝"); let test_duration_secs = 300; // 5 minutes let num_clients = 50; let target_rate_per_sec = 200; // 10K total / 50 clients = 200 per client let metrics = Arc::new(PerformanceMetrics::new()); let latencies = Arc::new(tokio::sync::Mutex::new(Vec::new())); let shutdown = Arc::new(AtomicU64::new(0)); println!( "🚀 Starting {} clients for {} seconds...", num_clients, test_duration_secs ); println!( "🎯 Target: {:.0} orders/sec total", num_clients as f64 * target_rate_per_sec as f64 ); let start_time = Instant::now(); let mut tasks = Vec::new(); for client_id in 0..num_clients { let metrics_clone = Arc::clone(&metrics); let latencies_clone = Arc::clone(&latencies); let shutdown_clone = Arc::clone(&shutdown); let task = tokio::spawn(async move { let mut client = match connect_trading_service().await { Ok(c) => c, Err(e) => { eprintln!("❌ Client {} connection failed: {}", client_id, e); return; }, }; let mut order_count = 0u64; let delay_micros = 1_000_000 / target_rate_per_sec; // microseconds between orders while shutdown_clone.load(Ordering::Relaxed) == 0 { let request = create_order_request(order_count); let req_start = Instant::now(); match timeout( Duration::from_secs(5), client.submit_order(Request::new(request)), ) .await { Ok(Ok(_response)) => { let latency_ns = req_start.elapsed().as_nanos() as u64; metrics_clone.record_success(latency_ns); latencies_clone.lock().await.push(latency_ns); }, Ok(Err(_)) => { metrics_clone.record_failure(); }, Err(_) => { metrics_clone.record_failure(); }, } order_count += 1; // Rate limiting tokio::time::sleep(Duration::from_micros(delay_micros)).await; } }); tasks.push(task); } // Run for specified duration tokio::time::sleep(Duration::from_secs(test_duration_secs)).await; // Signal shutdown shutdown.store(1, Ordering::Relaxed); // Wait for all clients to complete for task in tasks { let _ = task.await; } let test_duration = start_time.elapsed(); let latencies_vec = latencies.lock().await.clone(); let metrics_final = PerformanceMetrics { latencies_ns: latencies_vec.clone(), successful_orders: AtomicU64::new(metrics.successful_orders.load(Ordering::Relaxed)), failed_orders: AtomicU64::new(metrics.failed_orders.load(Ordering::Relaxed)), total_orders: AtomicU64::new(metrics.total_orders.load(Ordering::Relaxed)), test_duration, }; metrics_final.print_summary(&latencies_vec); Ok(()) } /// Test 4: Database under load #[tokio::test] async fn test_4_database_performance() -> Result<(), Box> { println!("\n╔═══════════════════════════════════════════════════════════╗"); println!("║ TEST 4: DATABASE PERFORMANCE ║"); println!("╚═══════════════════════════════════════════════════════════╝"); // This test measures order submission which triggers database writes let num_orders = 5000; let mut client = connect_trading_service().await?; println!( "📊 Submitting {} orders to measure database performance...", num_orders ); let start_time = Instant::now(); let mut success_count = 0; let mut failure_count = 0; for i in 0..num_orders { let request = create_order_request(i); match client.submit_order(Request::new(request)).await { Ok(_) => success_count += 1, Err(e) => { failure_count += 1; if failure_count <= 5 { eprintln!("❌ Order {} failed: {}", i, e); } }, } } let duration = start_time.elapsed(); let throughput = success_count as f64 / duration.as_secs_f64(); println!("\n📈 DATABASE PERFORMANCE:"); println!(" Duration: {:.2}s", duration.as_secs_f64()); println!(" Successful: {}", success_count); println!(" Failed: {}", failure_count); println!(" DB Writes/sec: {:.0}", throughput); if throughput >= 2000.0 { println!("✅ Database performance GOOD: {:.0} writes/sec", throughput); } else { println!( "⚠️ Database performance: {:.0} writes/sec (expected >2000)", throughput ); } Ok(()) } /// Test 5: Resource monitoring #[tokio::test] async fn test_5_resource_monitoring() -> Result<(), Box> { println!("\n╔═══════════════════════════════════════════════════════════╗"); println!("║ TEST 5: RESOURCE MONITORING ║"); println!("╚═══════════════════════════════════════════════════════════╝"); // Check service health let health_url = "http://localhost:8081/health"; println!("🏥 Checking service health at {}...", health_url); match reqwest::get(health_url).await { Ok(response) => { println!("✅ Health check response: {}", response.status()); if let Ok(body) = response.text().await { println!(" Body: {}", body); } }, Err(e) => { println!("⚠️ Health check failed: {}", e); }, } // Check Prometheus metrics let metrics_url = "http://localhost:9092/metrics"; println!("\n📊 Checking Prometheus metrics at {}...", metrics_url); match reqwest::get(metrics_url).await { Ok(response) => { if let Ok(body) = response.text().await { // Parse relevant metrics let lines: Vec<&str> = body .lines() .filter(|line| !line.starts_with('#') && !line.is_empty()) .collect(); println!("✅ Found {} metric entries", lines.len()); // Show some key metrics for line in lines.iter().take(10) { if line.contains("orders") || line.contains("latency") || line.contains("cpu") { println!(" {}", line); } } } }, Err(e) => { println!("⚠️ Metrics check failed: {}", e); }, } Ok(()) } /// Test 6: Production readiness assessment #[tokio::test] async fn test_6_production_readiness() -> Result<(), Box> { println!("\n╔═══════════════════════════════════════════════════════════╗"); println!("║ TEST 6: PRODUCTION READINESS ASSESSMENT ║"); println!("╚═══════════════════════════════════════════════════════════╝"); let num_clients = 50; let orders_per_client = 200; let metrics = Arc::new(PerformanceMetrics::new()); let latencies = Arc::new(tokio::sync::Mutex::new(Vec::new())); println!( "🎯 Production simulation: {} clients, {} orders each", num_clients, orders_per_client ); let start_time = Instant::now(); let mut tasks = Vec::new(); for client_id in 0..num_clients { let metrics_clone = Arc::clone(&metrics); let latencies_clone = Arc::clone(&latencies); let task = tokio::spawn(async move { let mut client = match connect_trading_service().await { Ok(c) => c, Err(_) => return, }; for order_idx in 0..orders_per_client { let request = create_order_request((client_id * orders_per_client + order_idx) as u64); let req_start = Instant::now(); match client.submit_order(Request::new(request)).await { Ok(_) => { let latency_ns = req_start.elapsed().as_nanos() as u64; metrics_clone.record_success(latency_ns); latencies_clone.lock().await.push(latency_ns); }, Err(_) => { metrics_clone.record_failure(); }, } } }); tasks.push(task); } for task in tasks { let _ = task.await; } let test_duration = start_time.elapsed(); let latencies_vec = latencies.lock().await.clone(); let metrics_final = PerformanceMetrics { latencies_ns: latencies_vec.clone(), successful_orders: AtomicU64::new(metrics.successful_orders.load(Ordering::Relaxed)), failed_orders: AtomicU64::new(metrics.failed_orders.load(Ordering::Relaxed)), total_orders: AtomicU64::new(metrics.total_orders.load(Ordering::Relaxed)), test_duration, }; metrics_final.print_summary(&latencies_vec); // Production readiness criteria let successful = metrics_final.successful_orders.load(Ordering::Relaxed); let total = metrics_final.total_orders.load(Ordering::Relaxed); let success_rate = (successful as f64 / total as f64) * 100.0; let throughput = successful as f64 / test_duration.as_secs_f64(); let (_, _, _, p99, _) = PerformanceMetrics::calculate_percentiles(latencies_vec); println!("\n🎯 PRODUCTION READINESS:"); let mut passed = 0; let mut total_checks = 0; // Check 1: Success rate total_checks += 1; if success_rate >= 99.0 { println!("✅ Success rate: {:.2}% (>= 99%)", success_rate); passed += 1; } else { println!("❌ Success rate: {:.2}% (< 99%)", success_rate); } // Check 2: Throughput total_checks += 1; if throughput >= 5000.0 { println!("✅ Throughput: {:.0} orders/sec (>= 5000)", throughput); passed += 1; } else { println!("⚠️ Throughput: {:.0} orders/sec (< 5000)", throughput); } // Check 3: P99 latency total_checks += 1; let p99_ms = p99 as f64 / 1_000_000.0; if p99_ms < 100.0 { println!("✅ P99 latency: {:.2}ms (< 100ms)", p99_ms); passed += 1; } else { println!("⚠️ P99 latency: {:.2}ms (>= 100ms)", p99_ms); } println!("\n📊 OVERALL: {}/{} checks passed", passed, total_checks); if passed == total_checks { println!("🎉 PRODUCTION READY!"); } else { println!("⚠️ Not ready for production deployment"); } Ok(()) }