//! Concurrent Clients Stress Tests //! //! Tests system behavior with many simultaneous clients: //! - 1,000 concurrent TLI clients //! - 10,000 concurrent WebSocket connections //! - Thundering herd scenario (all clients submit at once) //! - Performance under degraded conditions use anyhow::Result; use hdrhistogram::Histogram; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::sync::Barrier; use tokio::task::JoinSet; use tracing::{error, info}; /// Concurrent client test metrics #[derive(Debug, Clone)] pub struct ConcurrentClientMetrics { /// Number of concurrent clients pub num_clients: usize, /// Total requests sent pub total_requests: u64, /// Successful requests pub successful_requests: u64, /// Failed requests pub failed_requests: u64, /// Latency histogram pub latency_histogram: Histogram, /// Fairness score (0-100, 100 = perfectly fair) pub fairness_score: f64, /// Starvation detected pub starvation_detected: bool, /// Test duration pub duration: Duration, /// Test type pub test_type: String, } impl ConcurrentClientMetrics { pub fn new(num_clients: usize, test_type: &str) -> Self { Self { num_clients, total_requests: 0, successful_requests: 0, failed_requests: 0, latency_histogram: Histogram::::new_with_bounds(1, 60_000_000, 3).unwrap(), fairness_score: 0.0, starvation_detected: false, duration: Duration::ZERO, test_type: test_type.to_string(), } } pub fn success_rate(&self) -> f64 { if self.total_requests == 0 { return 0.0; } (self.successful_requests as f64 / self.total_requests as f64) * 100.0 } pub fn p99_latency_us(&self) -> u64 { self.latency_histogram.value_at_quantile(0.99) } pub fn avg_latency_us(&self) -> f64 { self.latency_histogram.mean() } } /// Concurrent client test runner pub struct ConcurrentClientTest { /// Number of concurrent clients num_clients: usize, /// Requests per client requests_per_client: usize, /// Test type test_type: TestType, /// Metrics metrics: Arc>, /// Request counter request_counter: Arc, /// Success counter success_counter: Arc, /// Per-client request counts (for fairness) client_counts: Arc>, } #[derive(Debug, Clone, Copy)] pub enum TestType { /// Regular concurrent clients Standard, /// Thundering herd (synchronized start) ThunderingHerd, /// WebSocket-style persistent connections WebSocket, } impl ConcurrentClientTest { pub fn new(num_clients: usize, requests_per_client: usize, test_type: TestType) -> Self { let test_name = match test_type { TestType::Standard => "Standard Concurrent", TestType::ThunderingHerd => "Thundering Herd", TestType::WebSocket => "WebSocket Persistent", }; Self { num_clients, requests_per_client, test_type, metrics: Arc::new(parking_lot::Mutex::new(ConcurrentClientMetrics::new( num_clients, test_name, ))), request_counter: Arc::new(AtomicU64::new(0)), success_counter: Arc::new(AtomicU64::new(0)), client_counts: Arc::new(dashmap::DashMap::new()), } } /// /// # Errors /// Returns error if the operation fails pub async fn run(&self) -> Result { info!( "Running concurrent client test: {} clients, {:?}", self.num_clients, self.test_type ); let start = Instant::now(); match self.test_type { TestType::Standard => self.run_standard().await?, TestType::ThunderingHerd => self.run_thundering_herd().await?, TestType::WebSocket => self.run_websocket().await?, } // Calculate fairness let fairness = self.calculate_fairness(); // Finalize metrics let mut metrics = self.metrics.lock(); metrics.duration = start.elapsed(); metrics.total_requests = self.request_counter.load(Ordering::Relaxed); metrics.successful_requests = self.success_counter.load(Ordering::Relaxed); metrics.failed_requests = metrics.total_requests - metrics.successful_requests; metrics.fairness_score = fairness; metrics.starvation_detected = fairness < 50.0; info!("Concurrent client test complete: {:?}", metrics); Ok(metrics.clone()) } /// Standard concurrent clients async fn run_standard(&self) -> Result<()> { let mut join_set = JoinSet::new(); for client_id in 0..self.num_clients { let requests = self.requests_per_client; let request_counter = Arc::clone(&self.request_counter); let success_counter = Arc::clone(&self.success_counter); let metrics = Arc::clone(&self.metrics); let client_counts = Arc::clone(&self.client_counts); join_set.spawn(async move { Self::client_workload( client_id, requests, request_counter, success_counter, metrics, client_counts, ) .await }); } while let Some(result) = join_set.join_next().await { if let Err(e) = result { error!("Client failed: {:?}", e); } } Ok(()) } /// Thundering herd: all clients start simultaneously async fn run_thundering_herd(&self) -> Result<()> { let barrier = Arc::new(Barrier::new(self.num_clients)); let mut join_set = JoinSet::new(); for client_id in 0..self.num_clients { let barrier = Arc::clone(&barrier); let requests = self.requests_per_client; let request_counter = Arc::clone(&self.request_counter); let success_counter = Arc::clone(&self.success_counter); let metrics = Arc::clone(&self.metrics); let client_counts = Arc::clone(&self.client_counts); join_set.spawn(async move { // Wait for all clients to be ready barrier.wait().await; // All clients start at the same instant Self::client_workload( client_id, requests, request_counter, success_counter, metrics, client_counts, ) .await }); } while let Some(result) = join_set.join_next().await { if let Err(e) = result { error!("Thundering herd client failed: {:?}", e); } } Ok(()) } /// WebSocket-style persistent connections async fn run_websocket(&self) -> Result<()> { let mut join_set = JoinSet::new(); for client_id in 0..self.num_clients { let requests = self.requests_per_client; let request_counter = Arc::clone(&self.request_counter); let success_counter = Arc::clone(&self.success_counter); let metrics = Arc::clone(&self.metrics); let client_counts = Arc::clone(&self.client_counts); join_set.spawn(async move { // Simulate WebSocket connection lifecycle Self::websocket_client( client_id, requests, request_counter, success_counter, metrics, client_counts, ) .await }); } while let Some(result) = join_set.join_next().await { if let Err(e) = result { error!("WebSocket client failed: {:?}", e); } } Ok(()) } /// Standard client workload async fn client_workload( client_id: usize, num_requests: usize, request_counter: Arc, success_counter: Arc, metrics: Arc>, client_counts: Arc>, ) -> Result<()> { for _ in 0..num_requests { let req_start = Instant::now(); // Simulate request let success = Self::simulate_request(client_id).await; let latency = req_start.elapsed(); // Update counters request_counter.fetch_add(1, Ordering::Relaxed); if success { success_counter.fetch_add(1, Ordering::Relaxed); } // Record latency { let mut m = metrics.lock(); let _ = m.latency_histogram.record(latency.as_micros() as u64); } // Track per-client requests *client_counts.entry(client_id).or_insert(0) += 1; // Small delay between requests tokio::time::sleep(Duration::from_millis(1)).await; } Ok(()) } /// WebSocket client simulation async fn websocket_client( client_id: usize, num_messages: usize, request_counter: Arc, success_counter: Arc, metrics: Arc>, client_counts: Arc>, ) -> Result<()> { // Simulate WebSocket handshake tokio::time::sleep(Duration::from_millis(10)).await; // Send messages over persistent connection for _ in 0..num_messages { let msg_start = Instant::now(); let success = Self::simulate_websocket_message(client_id).await; let latency = msg_start.elapsed(); request_counter.fetch_add(1, Ordering::Relaxed); if success { success_counter.fetch_add(1, Ordering::Relaxed); } { let mut m = metrics.lock(); let _ = m.latency_histogram.record(latency.as_micros() as u64); } *client_counts.entry(client_id).or_insert(0) += 1; // Simulate message rate (100 msg/sec) tokio::time::sleep(Duration::from_millis(10)).await; } Ok(()) } /// Simulate request async fn simulate_request(_client_id: usize) -> bool { tokio::time::sleep(Duration::from_micros(50 + rand::random::() % 450)).await; rand::random::() < 0.999 } /// Simulate WebSocket message async fn simulate_websocket_message(_client_id: usize) -> bool { tokio::time::sleep(Duration::from_micros(20 + rand::random::() % 180)).await; rand::random::() < 0.9995 } /// Calculate fairness score (coefficient of variation) fn calculate_fairness(&self) -> f64 { let counts: Vec = self .client_counts .iter() .map(|entry| *entry.value()) .collect(); if counts.is_empty() { return 0.0; } let mean = counts.iter().sum::() as f64 / counts.len() as f64; let variance = counts .iter() .map(|&x| { let diff = x as f64 - mean; diff * diff }) .sum::() / counts.len() as f64; let std_dev = variance.sqrt(); let cv = if mean > 0.0 { std_dev / mean } else { 0.0 }; // Convert to fairness score (0-100, lower CV = higher fairness) // CV of 0 = 100% fair, CV of 1.0 = 0% fair ((1.0 - cv.min(1.0)) * 100.0).max(0.0) } } /// Performance under failure test pub struct PerformanceUnderFailureTest { /// Number of clients num_clients: usize, /// Failure type failure_type: FailureType, /// Test duration duration: Duration, /// Metrics metrics: Arc>, } #[derive(Debug, Clone, Copy)] pub enum FailureType { /// Degraded database (50% slower) DegradedDatabase, /// High network latency (100ms added) HighNetworkLatency, /// Intermittent Redis failures IntermittentRedis, } impl PerformanceUnderFailureTest { pub fn new(num_clients: usize, failure_type: FailureType, duration: Duration) -> Self { let test_name = match failure_type { FailureType::DegradedDatabase => "Degraded Database", FailureType::HighNetworkLatency => "High Network Latency", FailureType::IntermittentRedis => "Intermittent Redis", }; Self { num_clients, failure_type, duration, metrics: Arc::new(parking_lot::Mutex::new(ConcurrentClientMetrics::new( num_clients, test_name, ))), } } /// /// # Errors /// Returns error if the operation fails pub async fn run(&self) -> Result { info!( "Running performance under failure: {} clients, {:?}", self.num_clients, self.failure_type ); let start = Instant::now(); let mut join_set = JoinSet::new(); let request_counter = Arc::new(AtomicU64::new(0)); let success_counter = Arc::new(AtomicU64::new(0)); for client_id in 0..self.num_clients { let duration = self.duration; let failure_type = self.failure_type; let request_counter = Arc::clone(&request_counter); let success_counter = Arc::clone(&success_counter); let metrics = Arc::clone(&self.metrics); join_set.spawn(async move { Self::degraded_client_workload( client_id, duration, failure_type, request_counter, success_counter, metrics, ) .await }); } while let Some(result) = join_set.join_next().await { if let Err(e) = result { error!("Degraded client failed: {:?}", e); } } let mut metrics = self.metrics.lock(); metrics.duration = start.elapsed(); metrics.total_requests = request_counter.load(Ordering::Relaxed); metrics.successful_requests = success_counter.load(Ordering::Relaxed); metrics.failed_requests = metrics.total_requests - metrics.successful_requests; info!("Performance under failure test complete"); Ok(metrics.clone()) } async fn degraded_client_workload( client_id: usize, duration: Duration, failure_type: FailureType, request_counter: Arc, success_counter: Arc, metrics: Arc>, ) -> Result<()> { let start = Instant::now(); while start.elapsed() < duration { let req_start = Instant::now(); // Simulate request under degraded conditions let success = Self::simulate_degraded_request(client_id, failure_type).await; let latency = req_start.elapsed(); request_counter.fetch_add(1, Ordering::Relaxed); if success { success_counter.fetch_add(1, Ordering::Relaxed); } { let mut m = metrics.lock(); let _ = m.latency_histogram.record(latency.as_micros() as u64); } tokio::time::sleep(Duration::from_millis(10)).await; } Ok(()) } async fn simulate_degraded_request(_client_id: usize, failure_type: FailureType) -> bool { match failure_type { FailureType::DegradedDatabase => { // 50% slower queries tokio::time::sleep(Duration::from_micros(150 + rand::random::() % 450)).await; rand::random::() < 0.99 }, FailureType::HighNetworkLatency => { // Additional 100ms network latency tokio::time::sleep(Duration::from_millis(100)).await; rand::random::() < 0.999 }, FailureType::IntermittentRedis => { // 20% Redis failure rate tokio::time::sleep(Duration::from_micros(50 + rand::random::() % 450)).await; rand::random::() < 0.8 }, } } } #[cfg(test)] mod tests { use super::*; #[tokio::test] async fn test_standard_concurrent_clients() { let test = ConcurrentClientTest::new(100, 100, TestType::Standard); let metrics = test.run().await.expect("Test failed"); assert_eq!(metrics.num_clients, 100); assert!(metrics.total_requests > 0); assert!(metrics.success_rate() > 99.0); assert!( metrics.fairness_score > 50.0, "Should have reasonable fairness" ); } #[tokio::test] async fn test_thundering_herd() { let test = ConcurrentClientTest::new(500, 50, TestType::ThunderingHerd); let metrics = test.run().await.expect("Test failed"); assert!(metrics.total_requests > 0); assert!( metrics.success_rate() > 95.0, "Should handle thundering herd gracefully" ); } #[tokio::test] async fn test_websocket_connections() { let test = ConcurrentClientTest::new(100, 100, TestType::WebSocket); let metrics = test.run().await.expect("Test failed"); assert!(metrics.total_requests > 0); assert!(metrics.success_rate() > 99.0); } #[tokio::test] async fn test_performance_under_degraded_db() { let test = PerformanceUnderFailureTest::new( 50, FailureType::DegradedDatabase, Duration::from_secs(5), ); let metrics = test.run().await.expect("Test failed"); assert!(metrics.total_requests > 0); assert!( metrics.success_rate() > 95.0, "Should maintain acceptable performance under degradation" ); // Latency should be higher but not catastrophic assert!(metrics.avg_latency_us() < 1_000_000.0); // < 1 second } #[tokio::test] #[ignore = "Resource intensive - run manually"] async fn test_1000_concurrent_clients() { let _ = tracing_subscriber::fmt::try_init(); info!("Running 1,000 concurrent clients test"); let test = ConcurrentClientTest::new(1_000, 1_000, TestType::Standard); let metrics = test.run().await.expect("Test failed"); assert_eq!(metrics.num_clients, 1_000); assert!(metrics.success_rate() > 99.0); assert!(!metrics.starvation_detected, "No client should be starved"); assert!( metrics.fairness_score > 70.0, "Should maintain fairness with 1K clients" ); info!("1,000 client test results: {:?}", metrics); } #[tokio::test] #[ignore = "Resource intensive - run manually"] async fn test_10000_websocket_connections() { let _ = tracing_subscriber::fmt::try_init(); info!("Running 10,000 WebSocket connections test"); let test = ConcurrentClientTest::new(10_000, 100, TestType::WebSocket); let metrics = test.run().await.expect("Test failed"); assert_eq!(metrics.num_clients, 10_000); assert!(metrics.success_rate() > 99.0); assert!( metrics.p99_latency_us() < 10_000, "P99 latency should be < 10ms" ); info!("10,000 WebSocket test results: {:?}", metrics); } }