#![allow( clippy::unwrap_used, clippy::expect_used, clippy::manual_clamp, unused_variables, dead_code, )] //! Saturation Point Tests //! //! This module finds system capacity limits by gradually increasing load //! until saturation points are reached: //! - Throughput saturation: Max orders/sec before errors spike //! - Latency degradation: Load point where P99 latency exceeds SLA //! - Connection saturation: Max concurrent connections //! - CPU saturation: Load point where CPU hits 90% //! - Memory saturation: Load point where memory hits 90% //! - Database saturation: Max queries/sec before latency spikes //! - Network saturation: Bandwidth limit //! - Queue saturation: Point where event queues back up //! //! Run with: cargo test -p load_tests --test saturation_point_tests --release -- --nocapture --ignored use anyhow::Result; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; use sysinfo::{CpuRefreshKind, RefreshKind, System}; use tokio::task::JoinSet; use tokio::time::sleep; // Internal dependencies use trading_service_load_tests::clients::TradingClient; use trading_service_load_tests::metrics::{LoadTestMetrics, LoadTestReport}; const TRADING_SERVICE_URL: &str = "http://localhost:50051"; /// Result of a saturation test iteration #[derive(Debug, Clone)] struct SaturationResult { rps: usize, error_rate: f64, p99_latency_us: u64, throughput: f64, avg_cpu_percent: f64, avg_memory_mb: f64, } impl SaturationResult { fn from_report(report: &LoadTestReport, rps: usize, cpu: f64) -> Self { Self { rps, error_rate: report.error_rate_percent, p99_latency_us: report.latency_p99_us, throughput: report.throughput_per_sec, avg_cpu_percent: cpu, avg_memory_mb: report.avg_memory_mb, } } fn print(&self, label: &str) { println!( " {}: {} rps, {:.2}% errors, {} μs P99, {:.2}% CPU, {:.2} MB mem", label, self.rps, self.error_rate, self.p99_latency_us, self.avg_cpu_percent, self.avg_memory_mb ); } } /// Run load test at specific RPS for duration async fn run_load_at_rps(rps: usize, duration: Duration) -> Result<(LoadTestReport, f64)> { let metrics = Arc::new(LoadTestMetrics::new()); let cpu_samples = Arc::new(parking_lot::Mutex::new(Vec::new())); // Calculate concurrent clients needed (each client sends ~100 rps) let concurrent_clients = (rps / 100).max(1).min(1000); let mut join_set = JoinSet::new(); // Spawn clients for client_id in 0..concurrent_clients { let metrics = Arc::clone(&metrics); join_set.spawn(async move { let mut client = TradingClient::connect(TRADING_SERVICE_URL).await?; let start = Instant::now(); while start.elapsed() < duration { match client.submit_test_order(client_id, 0).await { Ok(latency) => metrics.record_request(latency, true), Err(_) => metrics.record_request(Duration::from_micros(0), false), } // Rate limiting per client sleep(Duration::from_micros(10_000)).await; } Ok::<_, anyhow::Error>(()) }); } // CPU monitoring let cpu_samples_clone = Arc::clone(&cpu_samples); let monitor = tokio::spawn(async move { let mut sys = System::new_with_specifics( RefreshKind::nothing().with_cpu(CpuRefreshKind::everything()), ); let sample_count = (duration.as_secs() / 2).max(1); for _ in 0..sample_count { sleep(Duration::from_secs(2)).await; sys.refresh_cpu_all(); let global_cpu = sys.global_cpu_usage(); cpu_samples_clone.lock().push(global_cpu); } }); // Wait for completion while let Some(result) = join_set.join_next().await { if let Err(e) = result { tracing::error!("Client task failed: {:?}", e); } } monitor.abort(); let report = metrics.to_report("Saturation Test"); let cpu_samples_locked = cpu_samples.lock(); let avg_cpu = if !cpu_samples_locked.is_empty() { cpu_samples_locked.iter().sum::() / cpu_samples_locked.len() as f32 } else { 0.0 }; Ok((report, avg_cpu as f64)) } // ============================================================================= // THROUGHPUT SATURATION TESTS // ============================================================================= #[tokio::test] #[ignore = "Run explicitly with --ignored"] async fn test_find_throughput_saturation_point() -> Result<()> { println!("\n🚀 Finding Throughput Saturation Point"); println!(" Testing: 100 rps → 10,000 rps in 100 rps increments"); let mut saturation_point = 0; let mut results = Vec::new(); // Ramp up from 100 to 10,000 rps for rps in (100..=10_000).step_by(100) { println!("\n Testing {} rps...", rps); let (report, cpu) = run_load_at_rps(rps, Duration::from_secs(30)).await?; let result = SaturationResult::from_report(&report, rps, cpu); result.print("Result"); results.push(result.clone()); // Check saturation conditions if result.error_rate > 1.0 { println!("\n ❌ Error rate exceeded 1% threshold"); saturation_point = rps - 100; break; } if result.p99_latency_us > 100_000 { println!("\n ❌ P99 latency exceeded 100ms SLA"); saturation_point = rps; break; } // Cool down between tests sleep(Duration::from_secs(2)).await; } // Print summary println!("\n📊 Throughput Saturation Summary:"); println!(" Saturation point: {} rps", saturation_point); println!(" Tests conducted: {}", results.len()); if !results.is_empty() { let peak = results.last().expect("INVARIANT: Collection should be non-empty"); println!("\n Peak performance:"); peak.print(" "); } // Assertions assert!( saturation_point > 1000, "System should handle at least 1,000 rps (got {})", saturation_point ); Ok(()) } #[tokio::test] #[ignore = "Long-running test - run with --ignored"] async fn test_throughput_ramp_up_curve() -> Result<()> { println!("\n🚀 Generating Throughput Ramp-Up Curve"); println!(" Testing: 100, 500, 1000, 2500, 5000, 7500, 10000 rps"); let test_points = vec![100, 500, 1000, 2500, 5000, 7500, 10_000]; let mut results = Vec::new(); for rps in test_points { println!("\n Testing {} rps...", rps); let (report, cpu) = run_load_at_rps(rps, Duration::from_secs(30)).await?; let result = SaturationResult::from_report(&report, rps, cpu); result.print("Result"); results.push(result); // Cool down sleep(Duration::from_secs(2)).await; } // Analyze curve println!("\n📊 Throughput Ramp-Up Analysis:"); println!( " {:<10} {:<12} {:<15} {:<15}", "RPS", "Error Rate", "P99 Latency", "CPU Usage" ); println!(" {}", "=".repeat(60)); for result in &results { println!( " {:<10} {:<12.2}% {:<15} μs {:<15.2}%", result.rps, result.error_rate, result.p99_latency_us, result.avg_cpu_percent ); } // Verify throughput increases monotonically (until saturation) for i in 1..results.len() { if results[i].error_rate < 1.0 && results[i - 1].error_rate < 1.0 { assert!( results[i].throughput >= results[i - 1].throughput * 0.8, "Throughput should increase with load (got {:.2} -> {:.2})", results[i - 1].throughput, results[i].throughput ); } } Ok(()) } // ============================================================================= // LATENCY DEGRADATION TESTS // ============================================================================= #[tokio::test] #[ignore = "Long-running test - run with --ignored"] async fn test_latency_degradation_curve() -> Result<()> { println!("\n🚀 Finding Latency Degradation Point"); println!(" Testing: Finding where P99 latency exceeds 100ms"); let mut results = Vec::new(); // Ramp up until latency degrades for rps in (100..=5000).step_by(100) { println!("\n Testing {} rps...", rps); let (report, cpu) = run_load_at_rps(rps, Duration::from_secs(20)).await?; let result = SaturationResult::from_report(&report, rps, cpu); result.print("Result"); results.push(result.clone()); // Stop if latency exceeds 500ms (well beyond SLA) if result.p99_latency_us > 500_000 { println!("\n ❌ P99 latency exceeded 500ms - stopping test"); break; } sleep(Duration::from_secs(2)).await; } // Analyze degradation println!("\n📊 Latency Degradation Analysis:"); println!( " {:<10} {:<15} {:<15} {:<15}", "RPS", "P50 (μs)", "P95 (μs)", "P99 (μs)" ); println!(" {}", "=".repeat(60)); let mut sla_violation_rps = None; for result in &results { println!( " {:<10} {:<15} {:<15} {:<15}", result.rps, "-", // P50 not in SaturationResult "-", // P95 not in SaturationResult result.p99_latency_us ); if sla_violation_rps.is_none() && result.p99_latency_us > 100_000 { sla_violation_rps = Some(result.rps); } } if let Some(violation_rps) = sla_violation_rps { println!( "\n ⚠️ P99 latency exceeds 100ms SLA at {} rps", violation_rps ); } else { println!("\n ✅ P99 latency within SLA for all tested loads"); } // Verify latency doesn't decrease significantly with load for i in 1..results.len() { let prev_latency = results[i - 1].p99_latency_us as f64; let curr_latency = results[i].p99_latency_us as f64; assert!( curr_latency >= prev_latency * 0.5, "Latency should not decrease significantly with load" ); } Ok(()) } #[tokio::test] #[ignore = "Long-running test - run with --ignored"] async fn test_p50_p95_p99_spread_under_load() -> Result<()> { println!("\n🚀 Testing Latency Distribution Spread Under Load"); println!(" Testing: Latency spread at 1000, 5000, 10000 rps"); let test_loads = vec![1000, 5000, 10_000]; for rps in test_loads { println!("\n Testing {} rps...", rps); let metrics = Arc::new(LoadTestMetrics::new()); let concurrent_clients = (rps / 100).max(1); let mut join_set = JoinSet::new(); for client_id in 0..concurrent_clients { let metrics = Arc::clone(&metrics); join_set.spawn(async move { let mut client = TradingClient::connect(TRADING_SERVICE_URL).await?; let start = Instant::now(); while start.elapsed() < Duration::from_secs(30) { match client.submit_test_order(client_id, 0).await { Ok(latency) => metrics.record_request(latency, true), Err(_) => metrics.record_request(Duration::from_micros(0), false), } sleep(Duration::from_micros(10_000)).await; } Ok::<_, anyhow::Error>(()) }); } while let Some(result) = join_set.join_next().await { if let Err(e) = result { tracing::error!("Client task failed: {:?}", e); } } let report = metrics.to_report("Latency Spread Test"); println!(" Results at {} rps:", rps); println!(" P50: {} μs", report.latency_p50_us); println!(" P95: {} μs", report.latency_p95_us); println!(" P99: {} μs", report.latency_p99_us); println!(" Max: {} μs", report.latency_max_us); let p99_p50_ratio = report.latency_p99_us as f64 / report.latency_p50_us.max(1) as f64; println!(" P99/P50 ratio: {:.2}x", p99_p50_ratio); // Under healthy load, P99 shouldn't be more than 10x P50 assert!( p99_p50_ratio < 10.0, "P99/P50 ratio too high: {:.2}x (suggests tail latency issues)", p99_p50_ratio ); sleep(Duration::from_secs(2)).await; } Ok(()) } // ============================================================================= // CONNECTION SATURATION TESTS // ============================================================================= #[tokio::test] #[ignore = "Long-running test - run with --ignored"] async fn test_find_connection_saturation_point() -> Result<()> { println!("\n🚀 Finding Connection Saturation Point"); println!(" Testing: 50 → 1000 concurrent connections"); let mut max_connections = 0; for num_connections in (50..=1000).step_by(50) { println!("\n Testing {} concurrent connections...", num_connections); let metrics = Arc::new(LoadTestMetrics::new()); let connection_count = Arc::new(AtomicUsize::new(0)); let failure_count = Arc::new(AtomicUsize::new(0)); let mut join_set = JoinSet::new(); for client_id in 0..num_connections { let metrics = Arc::clone(&metrics); let connection_count = Arc::clone(&connection_count); let failure_count = Arc::clone(&failure_count); join_set.spawn(async move { let connect_start = Instant::now(); match TradingClient::connect(TRADING_SERVICE_URL).await { Ok(mut client) => { connection_count.fetch_add(1, Ordering::Relaxed); metrics.record_request(connect_start.elapsed(), true); // Submit a few orders for i in 0..10 { match client.submit_test_order(client_id, i).await { Ok(latency) => metrics.record_request(latency, true), Err(_) => { failure_count.fetch_add(1, Ordering::Relaxed); metrics.record_request(Duration::from_micros(0), false); }, } sleep(Duration::from_millis(10)).await; } }, Err(_) => { failure_count.fetch_add(1, Ordering::Relaxed); metrics.record_request(connect_start.elapsed(), false); }, } Ok::<_, anyhow::Error>(()) }); } while let Some(result) = join_set.join_next().await { if let Err(e) = result { tracing::error!("Connection task failed: {:?}", e); } } let successful_connections = connection_count.load(Ordering::Relaxed); let failures = failure_count.load(Ordering::Relaxed); let failure_rate = (failures as f64 / (num_connections * 10) as f64) * 100.0; println!( " Result: {}/{} connections successful, {:.2}% failure rate", successful_connections, num_connections, failure_rate ); if failure_rate > 5.0 { println!("\n ❌ Failure rate exceeded 5% - connection saturation reached"); max_connections = num_connections - 50; break; } max_connections = num_connections; sleep(Duration::from_secs(2)).await; } println!("\n📊 Connection Saturation Summary:"); println!(" Max sustainable connections: {}", max_connections); assert!( max_connections >= 200, "Should handle at least 200 concurrent connections (got {})", max_connections ); Ok(()) } #[tokio::test] #[ignore = "Long-running test - run with --ignored"] async fn test_connection_timeout_under_saturation() -> Result<()> { println!("\n🚀 Testing Connection Timeouts Under Saturation"); println!(" Testing: 500 connections with aggressive timeout"); const NUM_CONNECTIONS: usize = 500; let timeout_count = Arc::new(AtomicUsize::new(0)); let success_count = Arc::new(AtomicUsize::new(0)); let mut join_set = JoinSet::new(); for _client_id in 0..NUM_CONNECTIONS { let timeout_count = Arc::clone(&timeout_count); let success_count = Arc::clone(&success_count); join_set.spawn(async move { let result = tokio::time::timeout( Duration::from_secs(5), TradingClient::connect(TRADING_SERVICE_URL), ) .await; match result { Ok(Ok(_client)) => { success_count.fetch_add(1, Ordering::Relaxed); }, Ok(Err(_)) | Err(_) => { timeout_count.fetch_add(1, Ordering::Relaxed); }, } Ok::<_, anyhow::Error>(()) }); } while let Some(result) = join_set.join_next().await { if let Err(e) = result { tracing::error!("Timeout test task failed: {:?}", e); } } let timeouts = timeout_count.load(Ordering::Relaxed); let successes = success_count.load(Ordering::Relaxed); let timeout_rate = (timeouts as f64 / NUM_CONNECTIONS as f64) * 100.0; println!("\n📊 Connection Timeout Results:"); println!(" Successful: {}/{}", successes, NUM_CONNECTIONS); println!( " Timeouts: {}/{} ({:.2}%)", timeouts, NUM_CONNECTIONS, timeout_rate ); assert!( timeout_rate < 10.0, "Timeout rate too high: {:.2}% (expected < 10%)", timeout_rate ); Ok(()) } // ============================================================================= // CPU SATURATION TESTS // ============================================================================= #[tokio::test] #[ignore = "Long-running test - run with --ignored"] async fn test_find_cpu_saturation_point() -> Result<()> { println!("\n🚀 Finding CPU Saturation Point"); println!(" Testing: Finding load where CPU hits 90%"); let mut cpu_saturation_rps = 0; for rps in (100..=10_000).step_by(200) { println!("\n Testing {} rps...", rps); let (report, avg_cpu) = run_load_at_rps(rps, Duration::from_secs(30)).await?; let result = SaturationResult::from_report(&report, rps, avg_cpu); result.print("Result"); if avg_cpu > 90.0 { println!("\n ❌ CPU usage exceeded 90% threshold"); cpu_saturation_rps = rps; break; } sleep(Duration::from_secs(2)).await; } println!("\n📊 CPU Saturation Summary:"); if cpu_saturation_rps > 0 { println!(" CPU saturation at: {} rps", cpu_saturation_rps); } else { println!(" CPU saturation not reached (< 90% at all tested loads)"); } Ok(()) } // ============================================================================= // MEMORY SATURATION TESTS // ============================================================================= #[tokio::test] #[ignore = "Long-running test - run with --ignored"] async fn test_memory_growth_under_load() -> Result<()> { println!("\n🚀 Testing Memory Growth Under Load"); println!(" Testing: Memory usage at 1000, 5000, 10000 rps"); let test_loads = vec![1000, 5000, 10_000]; for rps in test_loads { println!("\n Testing {} rps...", rps); let (report, cpu) = run_load_at_rps(rps, Duration::from_secs(30)).await?; let result = SaturationResult::from_report(&report, rps, cpu); result.print("Result"); // Memory should stay reasonable (< 2GB for load tests) assert!( result.avg_memory_mb < 2048.0, "Memory usage too high: {:.2} MB at {} rps", result.avg_memory_mb, rps ); sleep(Duration::from_secs(2)).await; } Ok(()) } // ============================================================================= // QUEUE SATURATION TESTS // ============================================================================= #[tokio::test] #[ignore = "Long-running test - run with --ignored"] async fn test_queue_backpressure_detection() -> Result<()> { println!("\n🚀 Testing Queue Backpressure Detection"); println!(" Testing: Sudden burst to detect queue backup"); // Send burst of orders let metrics = Arc::new(LoadTestMetrics::new()); let latency_samples = Arc::new(parking_lot::Mutex::new(Vec::new())); let mut join_set = JoinSet::new(); // Send burst from 100 clients simultaneously for client_id in 0..100 { let metrics = Arc::clone(&metrics); let latency_samples = Arc::clone(&latency_samples); join_set.spawn(async move { let mut client = TradingClient::connect(TRADING_SERVICE_URL).await?; for order_id in 0..100 { match client.submit_test_order(client_id, order_id).await { Ok(latency) => { metrics.record_request(latency, true); latency_samples .lock() .push((order_id, latency.as_micros() as u64)); }, Err(_) => { metrics.record_request(Duration::from_micros(0), false); }, } // Minimal delay for burst sleep(Duration::from_micros(100)).await; } Ok::<_, anyhow::Error>(()) }); } while let Some(result) = join_set.join_next().await { if let Err(e) = result { tracing::error!("Burst task failed: {:?}", e); } } let report = metrics.to_report("Queue Backpressure Test"); println!("\n📊 Queue Backpressure Results:"); println!(" Total requests: {}", report.total_requests); println!(" Error rate: {:.2}%", report.error_rate_percent); println!(" P99 latency: {} μs", report.latency_p99_us); println!(" Max latency: {} μs", report.latency_max_us); // Analyze latency progression (queue backup shows increasing latency) let samples = latency_samples.lock(); if samples.len() > 10 { let first_10_avg = samples[..10].iter().map(|(_, lat)| lat).sum::() / 10; let last_10_avg = samples[samples.len() - 10..] .iter() .map(|(_, lat)| lat) .sum::() / 10; println!("\n First 10 orders avg latency: {} μs", first_10_avg); println!(" Last 10 orders avg latency: {} μs", last_10_avg); if last_10_avg > first_10_avg * 2 { println!(" ⚠️ Queue backpressure detected (latency doubled)"); } } Ok(()) }