//! Burst Load Stress Tests //! //! Tests system behavior under sudden load spikes and gradual ramps. //! - Spike testing: 0 → 100K orders/sec in 1 second //! - Gradual ramp: 0 → 50K over 10 minutes //! - Sustained plateau: 50K for 1 hour //! - Gradual ramp down: 50K → 0 over 10 minutes use anyhow::Result; use hdrhistogram::Histogram; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::sync::Barrier; use tokio::task::JoinSet; use tracing::{error, info}; /// Load profile for burst testing #[derive(Debug, Clone)] pub enum LoadProfile { /// Immediate spike to target RPS Spike { target_rps: usize, duration: Duration, }, /// Gradual ramp up to target RPS RampUp { target_rps: usize, ramp_duration: Duration, }, /// Sustained load at target RPS Plateau { target_rps: usize, duration: Duration, }, /// Gradual ramp down from target RPS to zero RampDown { start_rps: usize, ramp_duration: Duration, }, } /// Burst load test metrics #[derive(Debug, Clone)] pub struct BurstLoadMetrics { /// Total requests sent pub total_requests: u64, /// Successful requests pub successful_requests: u64, /// Failed requests pub failed_requests: u64, /// Peak throughput achieved (req/sec) pub peak_throughput: f64, /// Throughput samples over time pub throughput_samples: Vec<(Duration, f64)>, /// Latency histogram pub latency_histogram: Histogram, /// Test duration pub duration: Duration, /// Load profile description pub profile_name: String, } impl BurstLoadMetrics { pub fn new(profile_name: String) -> Self { Self { total_requests: 0, successful_requests: 0, failed_requests: 0, peak_throughput: 0.0, throughput_samples: Vec::new(), latency_histogram: Histogram::::new_with_bounds(1, 60_000_000, 3).unwrap(), duration: Duration::ZERO, profile_name, } } /// Calculate success rate pub fn success_rate(&self) -> f64 { if self.total_requests == 0 { return 0.0; } (self.successful_requests as f64 / self.total_requests as f64) * 100.0 } /// Get p99 latency pub fn p99_latency_us(&self) -> u64 { self.latency_histogram.value_at_quantile(0.99) } /// Get average throughput pub fn avg_throughput(&self) -> f64 { if self.throughput_samples.is_empty() { return 0.0; } self.throughput_samples .iter() .map(|(_, tps)| tps) .sum::() / self.throughput_samples.len() as f64 } } /// Burst load test runner pub struct BurstLoadTest { /// Load profile to execute profile: LoadProfile, /// Maximum concurrent clients max_clients: usize, /// Metrics collection metrics: Arc>, /// Request counter request_counter: Arc, /// Success counter success_counter: Arc, } impl BurstLoadTest { /// Create new burst load test pub fn new(profile: LoadProfile, max_clients: usize) -> Self { let profile_name = match &profile { LoadProfile::Spike { target_rps, .. } => { format!("Spike to {} req/sec", target_rps) }, LoadProfile::RampUp { target_rps, .. } => { format!("Ramp up to {} req/sec", target_rps) }, LoadProfile::Plateau { target_rps, .. } => { format!("Plateau at {} req/sec", target_rps) }, LoadProfile::RampDown { start_rps, .. } => { format!("Ramp down from {} req/sec", start_rps) }, }; Self { profile, max_clients, metrics: Arc::new(parking_lot::Mutex::new(BurstLoadMetrics::new(profile_name))), request_counter: Arc::new(AtomicU64::new(0)), success_counter: Arc::new(AtomicU64::new(0)), } } /// Run the burst load test /// /// # Errors /// Returns error if the operation fails pub async fn run(&self) -> Result { let start = Instant::now(); match &self.profile { LoadProfile::Spike { target_rps, duration, } => self.run_spike(*target_rps, *duration).await?, LoadProfile::RampUp { target_rps, ramp_duration, } => self.run_ramp_up(*target_rps, *ramp_duration).await?, LoadProfile::Plateau { target_rps, duration, } => self.run_plateau(*target_rps, *duration).await?, LoadProfile::RampDown { start_rps, ramp_duration, } => self.run_ramp_down(*start_rps, *ramp_duration).await?, } // Finalize metrics let mut metrics = self.metrics.lock(); metrics.duration = start.elapsed(); metrics.total_requests = self.request_counter.load(Ordering::Relaxed); metrics.successful_requests = self.success_counter.load(Ordering::Relaxed); metrics.failed_requests = metrics.total_requests - metrics.successful_requests; Ok(metrics.clone()) } /// Run spike test: immediate load spike async fn run_spike(&self, target_rps: usize, duration: Duration) -> Result<()> { info!("Running spike test: 0 → {} req/sec", target_rps); let barrier = Arc::new(Barrier::new(self.max_clients)); let mut join_set = JoinSet::new(); // Spawn monitoring let monitoring_handle = self.spawn_monitoring_task(); // Spawn all clients simultaneously let requests_per_client = target_rps / self.max_clients; let delay = if requests_per_client > 0 { Duration::from_millis(1000 / (requests_per_client as u64)) } else { Duration::from_millis(1000) }; for client_id in 0..self.max_clients { let barrier = Arc::clone(&barrier); let request_counter = Arc::clone(&self.request_counter); let success_counter = Arc::clone(&self.success_counter); let metrics = Arc::clone(&self.metrics); join_set.spawn(async move { // Wait for all clients to be ready barrier.wait().await; // Start sending requests Self::client_workload( client_id, duration, delay, request_counter, success_counter, metrics, ) .await }); } // Wait for completion while let Some(result) = join_set.join_next().await { if let Err(e) = result { error!("Spike client failed: {:?}", e); } } monitoring_handle.abort(); info!("Spike test complete"); Ok(()) } /// Run ramp up test: gradual increase in load async fn run_ramp_up(&self, target_rps: usize, ramp_duration: Duration) -> Result<()> { info!( "Running ramp up test: 0 → {} req/sec over {:?}", target_rps, ramp_duration ); let mut join_set = JoinSet::new(); let monitoring_handle = self.spawn_monitoring_task(); let ramp_steps: u64 = 100; // 100 steps in the ramp let step_duration = ramp_duration.as_millis() as u64 / ramp_steps; for step in 0..ramp_steps { let current_rps = (target_rps * (step + 1) as usize) / ramp_steps as usize; let clients_for_step = (self.max_clients * (step + 1) as usize) / ramp_steps as usize; // Spawn additional clients for this step // Calculate how many NEW clients to spawn for this step let previous_clients = if step > 0 { (self.max_clients * step as usize) / ramp_steps as usize } else { 0 }; let new_clients = clients_for_step.saturating_sub(previous_clients); for client_id in 0..new_clients { let delay = if current_rps > 0 { Duration::from_millis(1000 / (current_rps as u64)) } else { Duration::from_millis(1000) }; let request_counter = Arc::clone(&self.request_counter); let success_counter = Arc::clone(&self.success_counter); let metrics = Arc::clone(&self.metrics); join_set.spawn(async move { Self::client_workload( client_id, Duration::from_millis(step_duration), delay, request_counter, success_counter, metrics, ) .await }); } tokio::time::sleep(Duration::from_millis(step_duration)).await; } // Wait for completion while let Some(result) = join_set.join_next().await { if let Err(e) = result { error!("Ramp up client failed: {:?}", e); } } monitoring_handle.abort(); info!("Ramp up test complete"); Ok(()) } /// Run plateau test: sustained load at target RPS async fn run_plateau(&self, target_rps: usize, duration: Duration) -> Result<()> { info!( "Running plateau test: {} req/sec for {:?}", target_rps, duration ); let mut join_set = JoinSet::new(); let monitoring_handle = self.spawn_monitoring_task(); let requests_per_client = target_rps / self.max_clients; let delay = if requests_per_client > 0 { Duration::from_millis(1000 / (requests_per_client as u64)) } else { Duration::from_millis(1000) }; for client_id in 0..self.max_clients { let request_counter = Arc::clone(&self.request_counter); let success_counter = Arc::clone(&self.success_counter); let metrics = Arc::clone(&self.metrics); join_set.spawn(async move { Self::client_workload( client_id, duration, delay, request_counter, success_counter, metrics, ) .await }); } // Wait for completion while let Some(result) = join_set.join_next().await { if let Err(e) = result { error!("Plateau client failed: {:?}", e); } } monitoring_handle.abort(); info!("Plateau test complete"); Ok(()) } /// Run ramp down test: gradual decrease in load async fn run_ramp_down(&self, start_rps: usize, ramp_duration: Duration) -> Result<()> { info!( "Running ramp down test: {} req/sec → 0 over {:?}", start_rps, ramp_duration ); let mut join_set = JoinSet::new(); let monitoring_handle = self.spawn_monitoring_task(); let ramp_steps: u64 = 100; let step_duration = ramp_duration.as_millis() as u64 / ramp_steps; // Start with all clients active for step in 0..ramp_steps { let current_rps = start_rps * (ramp_steps - step) as usize / ramp_steps as usize; if current_rps == 0 { break; } let active_clients = self.max_clients * (ramp_steps - step) as usize / ramp_steps as usize; for client_id in 0..active_clients.max(1) / ramp_steps as usize { let delay = if current_rps > 0 { Duration::from_millis(1000 / (current_rps as u64)) } else { Duration::from_millis(1000) }; let request_counter = Arc::clone(&self.request_counter); let success_counter = Arc::clone(&self.success_counter); let metrics = Arc::clone(&self.metrics); join_set.spawn(async move { Self::client_workload( client_id, Duration::from_millis(step_duration), delay, request_counter, success_counter, metrics, ) .await }); } tokio::time::sleep(Duration::from_millis(step_duration)).await; } // Wait for completion while let Some(result) = join_set.join_next().await { if let Err(e) = result { error!("Ramp down client failed: {:?}", e); } } monitoring_handle.abort(); info!("Ramp down test complete"); Ok(()) } /// Client workload async fn client_workload( client_id: usize, duration: Duration, delay: Duration, request_counter: Arc, success_counter: Arc, metrics: Arc>, ) -> Result<()> { let start = Instant::now(); while start.elapsed() < duration { let req_start = Instant::now(); // Simulate request let success = Self::simulate_request(client_id).await; let latency = req_start.elapsed(); // Update counters request_counter.fetch_add(1, Ordering::Relaxed); if success { success_counter.fetch_add(1, Ordering::Relaxed); } // Record latency { let mut m = metrics.lock(); let _ = m.latency_histogram.record(latency.as_micros() as u64); } tokio::time::sleep(delay).await; } Ok(()) } /// Simulate request (replace with actual gRPC call) async fn simulate_request(_client_id: usize) -> bool { tokio::time::sleep(Duration::from_micros(50 + rand::random::() % 450)).await; rand::random::() < 0.999 } /// Spawn monitoring task fn spawn_monitoring_task(&self) -> tokio::task::JoinHandle<()> { let metrics = Arc::clone(&self.metrics); let request_counter = Arc::clone(&self.request_counter); tokio::spawn(async move { let start = Instant::now(); let mut interval = tokio::time::interval(Duration::from_secs(1)); let mut last_count = 0u64; loop { interval.tick().await; let current_count = request_counter.load(Ordering::Relaxed); let throughput = (current_count - last_count) as f64; last_count = current_count; let elapsed = start.elapsed(); { let mut m = metrics.lock(); m.throughput_samples.push((elapsed, throughput)); if throughput > m.peak_throughput { m.peak_throughput = throughput; } } } }) } /// Get current metrics pub fn get_metrics(&self) -> BurstLoadMetrics { self.metrics.lock().clone() } } #[cfg(test)] mod tests { use super::*; #[tokio::test] async fn test_spike_load() { let profile = LoadProfile::Spike { target_rps: 10_000, duration: Duration::from_secs(5), }; let test = BurstLoadTest::new(profile, 100); let metrics = test.run().await.expect("Test failed"); assert!(metrics.total_requests > 0); assert!(metrics.success_rate() > 99.0); // Peak throughput should be reasonable for simulated workload assert!( metrics.peak_throughput > 5000.0, "Peak throughput should be > 5000/sec (got: {})", metrics.peak_throughput ); } #[tokio::test] async fn test_ramp_up() { let profile = LoadProfile::RampUp { target_rps: 5_000, ramp_duration: Duration::from_secs(10), }; let test = BurstLoadTest::new(profile, 50); let metrics = test.run().await.expect("Test failed"); assert!(metrics.total_requests > 0); assert!(metrics.success_rate() > 99.0); } #[tokio::test] #[ignore = "Long running - run manually"] async fn test_full_burst_scenario() { let _ = tracing_subscriber::fmt::try_init(); info!("Running full burst scenario"); // Phase 1: Spike to 100K req/sec for 1 second let spike = BurstLoadTest::new( LoadProfile::Spike { target_rps: 100_000, duration: Duration::from_secs(1), }, 1000, ); let spike_metrics = spike.run().await.expect("Spike failed"); info!("Spike phase: {:?}", spike_metrics); // Phase 2: Ramp up 0 → 50K over 10 minutes let ramp_up = BurstLoadTest::new( LoadProfile::RampUp { target_rps: 50_000, ramp_duration: Duration::from_secs(600), }, 500, ); let ramp_up_metrics = ramp_up.run().await.expect("Ramp up failed"); info!("Ramp up phase: {:?}", ramp_up_metrics); // Phase 3: Plateau at 50K for 1 hour let plateau = BurstLoadTest::new( LoadProfile::Plateau { target_rps: 50_000, duration: Duration::from_secs(3600), }, 500, ); let plateau_metrics = plateau.run().await.expect("Plateau failed"); info!("Plateau phase: {:?}", plateau_metrics); // Phase 4: Ramp down 50K → 0 over 10 minutes let ramp_down = BurstLoadTest::new( LoadProfile::RampDown { start_rps: 50_000, ramp_duration: Duration::from_secs(600), }, 500, ); let ramp_down_metrics = ramp_down.run().await.expect("Ramp down failed"); info!("Ramp down phase: {:?}", ramp_down_metrics); // All phases should have high success rates assert!(spike_metrics.success_rate() > 95.0); assert!(ramp_up_metrics.success_rate() > 99.0); assert!(plateau_metrics.success_rate() > 99.0); assert!(ramp_down_metrics.success_rate() > 99.0); } }