//! Sustained Load Stress Tests //! //! Tests system behavior under sustained high load over extended periods. //! - 1 hour sustained load at 50K orders/sec //! - 24 hour soak test at 10K orders/sec //! - Memory leak detection //! - Connection pool stability //! - Database performance degradation monitoring use anyhow::Result; use hdrhistogram::Histogram; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::task::JoinSet; use tokio::time::interval; use tracing::{error, info}; /// Metrics for sustained load testing #[derive(Debug, Clone)] pub struct SustainedLoadMetrics { /// Total requests sent pub total_requests: u64, /// Successful requests pub successful_requests: u64, /// Failed requests pub failed_requests: u64, /// Throughput samples (requests/sec per interval) pub throughput_samples: Vec, /// Memory usage samples (bytes) pub memory_samples: Vec, /// Latency histogram pub latency_histogram: Histogram, /// Connection pool size samples pub connection_pool_samples: Vec, /// Database query time samples (microseconds) pub db_query_times: Vec, /// Test duration pub duration: Duration, } impl Default for SustainedLoadMetrics { fn default() -> Self { Self::new() } } impl SustainedLoadMetrics { pub fn new() -> Self { Self { total_requests: 0, successful_requests: 0, failed_requests: 0, throughput_samples: Vec::new(), memory_samples: Vec::new(), latency_histogram: Histogram::::new_with_bounds(1, 60_000_000, 3).unwrap(), connection_pool_samples: Vec::new(), db_query_times: Vec::new(), duration: Duration::ZERO, } } /// Check for performance degradation over time pub fn detect_degradation(&self, threshold_percent: f64) -> bool { if self.throughput_samples.len() < 10 { return false; } // Compare first 10% vs last 10% of samples let sample_count = self.throughput_samples.len(); let first_10_pct = &self.throughput_samples[..sample_count / 10]; let last_10_pct = &self.throughput_samples[sample_count * 9 / 10..]; let avg_first: f64 = first_10_pct.iter().sum::() / first_10_pct.len() as f64; let avg_last: f64 = last_10_pct.iter().sum::() / last_10_pct.len() as f64; let degradation = (avg_first - avg_last) / avg_first * 100.0; degradation > threshold_percent } /// Check for memory leaks pub fn detect_memory_leak(&self, growth_threshold_mb: f64) -> bool { if self.memory_samples.len() < 10 { return false; } let first_mb = self.memory_samples[0] as f64 / 1_048_576.0; let last_mb = *self.memory_samples.last().expect("INVARIANT: Collection should be non-empty") as f64 / 1_048_576.0; let growth = last_mb - first_mb; growth > growth_threshold_mb } /// Calculate average throughput pub fn avg_throughput(&self) -> f64 { if self.throughput_samples.is_empty() { return 0.0; } self.throughput_samples.iter().sum::() / self.throughput_samples.len() as f64 } /// Calculate p99 latency pub fn p99_latency_us(&self) -> u64 { self.latency_histogram.value_at_quantile(0.99) } /// Calculate success rate pub fn success_rate(&self) -> f64 { if self.total_requests == 0 { return 0.0; } (self.successful_requests as f64 / self.total_requests as f64) * 100.0 } } /// Sustained load test runner pub struct SustainedLoadTest { /// Target throughput (requests per second) target_rps: usize, /// Test duration duration: Duration, /// Number of concurrent clients concurrent_clients: usize, /// Metrics collection metrics: Arc>, /// Total requests counter request_counter: Arc, /// Success counter success_counter: Arc, } impl SustainedLoadTest { /// Create a new sustained load test pub fn new(target_rps: usize, duration: Duration, concurrent_clients: usize) -> Self { Self { target_rps, duration, concurrent_clients, metrics: Arc::new(parking_lot::Mutex::new(SustainedLoadMetrics::new())), request_counter: Arc::new(AtomicU64::new(0)), success_counter: Arc::new(AtomicU64::new(0)), } } /// Run the sustained load test /// /// # Errors /// Returns error if the operation fails pub async fn run(&self) -> Result { info!( "Starting sustained load test: {} req/sec for {:?}", self.target_rps, self.duration ); let start = Instant::now(); let mut join_set = JoinSet::new(); // Spawn client tasks let requests_per_client = self.target_rps / self.concurrent_clients; let delay_between_requests = Duration::from_millis(1000 / requests_per_client as u64); for client_id in 0..self.concurrent_clients { let duration = self.duration; let delay = delay_between_requests; let request_counter = Arc::clone(&self.request_counter); let success_counter = Arc::clone(&self.success_counter); let metrics = Arc::clone(&self.metrics); join_set.spawn(async move { Self::client_workload( client_id, duration, delay, request_counter, success_counter, metrics, ) .await }); } // Spawn monitoring task let monitoring_handle = self.spawn_monitoring_task(start); // Wait for all clients to complete while let Some(result) = join_set.join_next().await { if let Err(e) = result { error!("Client task failed: {:?}", e); } } // Stop monitoring monitoring_handle.abort(); // Finalize metrics let mut metrics = self.metrics.lock(); metrics.duration = start.elapsed(); metrics.total_requests = self.request_counter.load(Ordering::Relaxed); metrics.successful_requests = self.success_counter.load(Ordering::Relaxed); metrics.failed_requests = metrics.total_requests - metrics.successful_requests; info!( "Sustained load test complete: {} requests in {:?}", metrics.total_requests, metrics.duration ); Ok(metrics.clone()) } /// Client workload: send requests at target rate async fn client_workload( client_id: usize, duration: Duration, delay: Duration, request_counter: Arc, success_counter: Arc, metrics: Arc>, ) -> Result<()> { let start = Instant::now(); while start.elapsed() < duration { let req_start = Instant::now(); // Simulate order submission let success = Self::simulate_order_submission(client_id).await; let latency = req_start.elapsed(); // Update counters request_counter.fetch_add(1, Ordering::Relaxed); if success { success_counter.fetch_add(1, Ordering::Relaxed); } // Record latency { let mut m = metrics.lock(); let _ = m.latency_histogram.record(latency.as_micros() as u64); } // Rate limiting - subtract processing time from delay to maintain target rate if latency < delay { tokio::time::sleep(delay - latency).await; } else { // Processing took longer than delay interval, no sleep needed // This will naturally reduce throughput but is realistic tokio::task::yield_now().await; } } Ok(()) } /// Simulate order submission (replace with actual gRPC call in integration tests) async fn simulate_order_submission(_client_id: usize) -> bool { // Simulate processing time (50-500μs) tokio::time::sleep(Duration::from_micros(50 + rand::random::() % 450)).await; // 99.9% success rate rand::random::() < 0.999 } /// Spawn monitoring task to collect periodic metrics fn spawn_monitoring_task(&self, start: Instant) -> tokio::task::JoinHandle<()> { let metrics = Arc::clone(&self.metrics); let request_counter = Arc::clone(&self.request_counter); let duration = self.duration; tokio::spawn(async move { let mut interval = interval(Duration::from_secs(1)); let mut last_count = 0u64; let mut sys = sysinfo::System::new_all(); while start.elapsed() < duration { interval.tick().await; // Calculate throughput for this interval let current_count = request_counter.load(Ordering::Relaxed); let throughput = (current_count - last_count) as f64; last_count = current_count; // Collect memory usage sys.refresh_all(); let memory_bytes = sys.used_memory(); // Record samples let mut m = metrics.lock(); m.throughput_samples.push(throughput); m.memory_samples.push(memory_bytes); // Simulate connection pool size (replace with actual monitoring) let pool_size = 10 + (rand::random::() % 5); m.connection_pool_samples.push(pool_size); // Simulate database query times (replace with actual monitoring) let db_query_us = 100 + (rand::random::() % 900); m.db_query_times.push(db_query_us); drop(m); } }) } /// Get current metrics pub fn get_metrics(&self) -> SustainedLoadMetrics { self.metrics.lock().clone() } } #[cfg(test)] mod tests { use super::*; #[tokio::test] async fn test_sustained_load_short_duration() { // 10 second test at 1000 req/sec let test = SustainedLoadTest::new(1000, Duration::from_secs(10), 10); let metrics = test.run().await.expect("Test failed"); assert!(metrics.total_requests > 0, "Should have sent requests"); assert!( metrics.success_rate() > 99.0, "Success rate should be > 99%" ); assert!( metrics.avg_throughput() > 700.0, "Average throughput should be reasonable given overhead (target: 1000, got: {}). \ Lower than target due to tokio scheduling overhead, lock contention, and simulated processing time.", metrics.avg_throughput() ); } #[tokio::test] async fn test_degradation_detection() { let mut metrics = SustainedLoadMetrics::new(); // Simulate stable throughput for _ in 0..100 { metrics.throughput_samples.push(1000.0); } assert!( !metrics.detect_degradation(5.0), "Should not detect degradation with stable throughput" ); // Simulate degradation for _ in 0..10 { metrics.throughput_samples.push(800.0); } assert!( metrics.detect_degradation(5.0), "Should detect 20% degradation" ); } #[tokio::test] async fn test_memory_leak_detection() { let mut metrics = SustainedLoadMetrics::new(); // Simulate stable memory for _ in 0..100 { metrics.memory_samples.push(100 * 1_048_576); // 100 MB } assert!( !metrics.detect_memory_leak(10.0), "Should not detect leak with stable memory" ); // Simulate memory growth for i in 0..10 { metrics.memory_samples.push((120 + i) * 1_048_576); // Growing } assert!( metrics.detect_memory_leak(10.0), "Should detect memory leak" ); } #[tokio::test] #[ignore = "Long running test - run manually"] async fn test_one_hour_sustained_load() { let _ = tracing_subscriber::fmt::try_init(); info!("Starting 1-hour sustained load test at 50K req/sec"); let test = SustainedLoadTest::new(50_000, Duration::from_secs(3600), 500); let metrics = test.run().await.expect("Test failed"); // Assertions assert!( metrics.total_requests > 50_000 * 3600 * 95 / 100, "Should complete > 95% of expected requests" ); assert!( metrics.success_rate() > 99.0, "Success rate should be > 99%" ); assert!( !metrics.detect_degradation(5.0), "Should not degrade > 5% over 1 hour" ); assert!( !metrics.detect_memory_leak(50.0), "Should not leak > 50MB over 1 hour" ); info!("1-hour test results: {:?}", metrics); } #[tokio::test] #[ignore = "Very long running test - run manually"] async fn test_24_hour_soak_test() { let _ = tracing_subscriber::fmt::try_init(); info!("Starting 24-hour soak test at 10K req/sec"); let test = SustainedLoadTest::new(10_000, Duration::from_secs(86400), 100); let metrics = test.run().await.expect("Test failed"); // Assertions for soak test assert!( metrics.total_requests > 10_000 * 86400 * 95 / 100, "Should complete > 95% of expected requests" ); assert!( metrics.success_rate() > 99.0, "Success rate should be > 99%" ); assert!( !metrics.detect_degradation(3.0), "Should not degrade > 3% over 24 hours" ); assert!( !metrics.detect_memory_leak(100.0), "Should not leak > 100MB over 24 hours" ); info!("24-hour soak test results: {:?}", metrics); } }