//! Resource Exhaustion Stress Tests //! //! Tests system behavior when resources are exhausted: //! - Database connection pool exhaustion //! - Redis connection pool saturation //! - Memory pressure (max heap) //! - CPU saturation (all cores at 100%) //! - Network bandwidth saturation use anyhow::Result; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::task::JoinSet; use tracing::{error, info, warn}; /// Resource exhaustion test type #[derive(Debug, Clone, Copy)] pub enum ResourceType { /// Database connection pool DatabaseConnections, /// Redis connection pool RedisConnections, /// Memory heap Memory, /// CPU cores Cpu, /// Network bandwidth Network, } /// Resource exhaustion metrics #[derive(Debug, Clone)] pub struct ResourceExhaustionMetrics { /// Test type pub resource_type: String, /// Total operations attempted pub total_operations: u64, /// Successful operations pub successful_operations: u64, /// Operations that failed due to resource exhaustion pub exhaustion_failures: u64, /// Other failures pub other_failures: u64, /// Time to first exhaustion pub time_to_exhaustion: Duration, /// Recovery time (if applicable) pub recovery_time: Duration, /// System remained stable (no crash) pub system_stable: bool, /// Graceful degradation observed pub graceful_degradation: bool, /// Test duration pub duration: Duration, } impl ResourceExhaustionMetrics { pub fn new(resource_type: &str) -> Self { Self { resource_type: resource_type.to_string(), total_operations: 0, successful_operations: 0, exhaustion_failures: 0, other_failures: 0, time_to_exhaustion: Duration::ZERO, recovery_time: Duration::ZERO, system_stable: true, graceful_degradation: false, duration: Duration::ZERO, } } /// Calculate success rate pub fn success_rate(&self) -> f64 { if self.total_operations == 0 { return 0.0; } (self.successful_operations as f64 / self.total_operations as f64) * 100.0 } /// Calculate exhaustion rate pub fn exhaustion_rate(&self) -> f64 { if self.total_operations == 0 { return 0.0; } (self.exhaustion_failures as f64 / self.total_operations as f64) * 100.0 } } /// Database connection exhaustion test pub struct DatabaseExhaustionTest { /// Maximum connections max_connections: usize, /// Test duration duration: Duration, /// Metrics metrics: Arc>, /// Operation counter operation_counter: Arc, /// Success counter success_counter: Arc, /// Exhaustion detected flag exhaustion_detected: Arc, /// Time of first exhaustion exhaustion_time: Arc>>, } impl DatabaseExhaustionTest { pub fn new(max_connections: usize, duration: Duration) -> Self { Self { max_connections, duration, metrics: Arc::new(parking_lot::Mutex::new(ResourceExhaustionMetrics::new( "Database Connections", ))), operation_counter: Arc::new(AtomicU64::new(0)), success_counter: Arc::new(AtomicU64::new(0)), exhaustion_detected: Arc::new(AtomicBool::new(false)), exhaustion_time: Arc::new(parking_lot::Mutex::new(None)), } } /// Run database exhaustion test /// /// # Errors /// Returns error if the operation fails pub async fn run(&self) -> Result { info!( "Running database exhaustion test: {} max connections", self.max_connections ); let start = Instant::now(); let mut join_set = JoinSet::new(); // Spawn many more clients than available connections let num_clients = self.max_connections * 3; for client_id in 0..num_clients { let duration = self.duration; let operation_counter = Arc::clone(&self.operation_counter); let success_counter = Arc::clone(&self.success_counter); let exhaustion_detected = Arc::clone(&self.exhaustion_detected); let exhaustion_time = Arc::clone(&self.exhaustion_time); join_set.spawn(async move { Self::db_client_workload( client_id, duration, operation_counter, success_counter, exhaustion_detected, exhaustion_time, start, ) .await }); } // Wait for completion while let Some(result) = join_set.join_next().await { if let Err(e) = result { error!("DB client failed: {:?}", e); } } // Finalize metrics let mut metrics = self.metrics.lock(); metrics.duration = start.elapsed(); metrics.total_operations = self.operation_counter.load(Ordering::Relaxed); metrics.successful_operations = self.success_counter.load(Ordering::Relaxed); metrics.exhaustion_failures = metrics.total_operations - metrics.successful_operations; if let Some(exhaustion_instant) = *self.exhaustion_time.lock() { metrics.time_to_exhaustion = exhaustion_instant.duration_since(start); metrics.graceful_degradation = true; } info!("Database exhaustion test complete: {:?}", metrics); Ok(metrics.clone()) } async fn db_client_workload( client_id: usize, duration: Duration, operation_counter: Arc, success_counter: Arc, exhaustion_detected: Arc, exhaustion_time: Arc>>, test_start: Instant, ) -> Result<()> { let start = Instant::now(); while start.elapsed() < duration { operation_counter.fetch_add(1, Ordering::Relaxed); // Simulate database connection acquisition match Self::simulate_db_connection(client_id).await { Ok(()) => { success_counter.fetch_add(1, Ordering::Relaxed); }, Err(_) => { // Connection pool exhausted if !exhaustion_detected.swap(true, Ordering::Relaxed) { let mut time = exhaustion_time.lock(); if time.is_none() { *time = Some(Instant::now()); warn!( "Database connection pool exhausted at {:?}", test_start.elapsed() ); } } }, } // Brief delay between attempts tokio::time::sleep(Duration::from_millis(10)).await; } Ok(()) } async fn simulate_db_connection(client_id: usize) -> Result<()> { // Simulate connection acquisition (random success based on pool availability) tokio::time::sleep(Duration::from_millis(1 + client_id as u64 % 10)).await; if rand::random::() < 0.7 { // 70% chance of getting connection Ok(()) } else { Err(anyhow::anyhow!("Connection pool exhausted")) } } } /// Redis connection exhaustion test pub struct RedisExhaustionTest { max_connections: usize, duration: Duration, metrics: Arc>, operation_counter: Arc, success_counter: Arc, } impl RedisExhaustionTest { pub fn new(max_connections: usize, duration: Duration) -> Self { Self { max_connections, duration, metrics: Arc::new(parking_lot::Mutex::new(ResourceExhaustionMetrics::new( "Redis Connections", ))), operation_counter: Arc::new(AtomicU64::new(0)), success_counter: Arc::new(AtomicU64::new(0)), } } /// /// # Errors /// Returns error if the operation fails pub async fn run(&self) -> Result { info!( "Running Redis exhaustion test: {} max connections", self.max_connections ); let start = Instant::now(); let mut join_set = JoinSet::new(); let num_clients = self.max_connections * 2; for client_id in 0..num_clients { let duration = self.duration; let operation_counter = Arc::clone(&self.operation_counter); let success_counter = Arc::clone(&self.success_counter); join_set.spawn(async move { Self::redis_client_workload(client_id, duration, operation_counter, success_counter) .await }); } while let Some(result) = join_set.join_next().await { if let Err(e) = result { error!("Redis client failed: {:?}", e); } } let mut metrics = self.metrics.lock(); metrics.duration = start.elapsed(); metrics.total_operations = self.operation_counter.load(Ordering::Relaxed); metrics.successful_operations = self.success_counter.load(Ordering::Relaxed); metrics.exhaustion_failures = metrics.total_operations - metrics.successful_operations; Ok(metrics.clone()) } async fn redis_client_workload( client_id: usize, duration: Duration, operation_counter: Arc, success_counter: Arc, ) -> Result<()> { let start = Instant::now(); while start.elapsed() < duration { operation_counter.fetch_add(1, Ordering::Relaxed); if Self::simulate_redis_operation(client_id).await.is_ok() { success_counter.fetch_add(1, Ordering::Relaxed); } tokio::time::sleep(Duration::from_millis(5)).await; } Ok(()) } async fn simulate_redis_operation(_client_id: usize) -> Result<()> { tokio::time::sleep(Duration::from_micros(100 + rand::random::() % 400)).await; if rand::random::() < 0.8 { Ok(()) } else { Err(anyhow::anyhow!("Redis connection pool exhausted")) } } } /// Memory pressure test pub struct MemoryPressureTest { /// Target memory usage (MB) target_memory_mb: usize, /// Test duration duration: Duration, /// Metrics metrics: Arc>, } impl MemoryPressureTest { pub fn new(target_memory_mb: usize, duration: Duration) -> Self { Self { target_memory_mb, duration, metrics: Arc::new(parking_lot::Mutex::new(ResourceExhaustionMetrics::new( "Memory Pressure", ))), } } /// /// # Errors /// Returns error if the operation fails pub async fn run(&self) -> Result { info!( "Running memory pressure test: {} MB target", self.target_memory_mb ); let start = Instant::now(); // Allocate memory gradually let mut allocations: Vec> = Vec::new(); let chunk_size_mb = 10; let num_chunks = self.target_memory_mb / chunk_size_mb; for i in 0..num_chunks { if start.elapsed() >= self.duration { break; } // Allocate chunk let chunk = vec![0u8; chunk_size_mb * 1_048_576]; allocations.push(chunk); info!( "Allocated {} MB of {} MB", (i + 1) * chunk_size_mb, self.target_memory_mb ); tokio::time::sleep(Duration::from_secs(1)).await; } // Hold memory for remaining duration let remaining = self.duration.saturating_sub(start.elapsed()); if remaining > Duration::ZERO { info!( "Holding {} MB for {:?}", allocations.len() * chunk_size_mb, remaining ); tokio::time::sleep(remaining).await; } let mut metrics = self.metrics.lock(); metrics.duration = start.elapsed(); metrics.system_stable = true; metrics.graceful_degradation = true; info!("Memory pressure test complete"); Ok(metrics.clone()) } } /// CPU saturation test pub struct CpuSaturationTest { /// Number of cores to saturate num_cores: usize, /// Test duration duration: Duration, /// Metrics metrics: Arc>, } impl CpuSaturationTest { pub fn new(num_cores: usize, duration: Duration) -> Self { Self { num_cores, duration, metrics: Arc::new(parking_lot::Mutex::new(ResourceExhaustionMetrics::new( "CPU Saturation", ))), } } /// /// # Errors /// Returns error if the operation fails pub async fn run(&self) -> Result { info!("Running CPU saturation test: {} cores", self.num_cores); let start = Instant::now(); let mut join_set = JoinSet::new(); // Spawn CPU-intensive tasks for core_id in 0..self.num_cores { let duration = self.duration; join_set.spawn(async move { Self::cpu_intensive_workload(core_id, duration).await }); } // Wait for completion while let Some(result) = join_set.join_next().await { if let Err(e) = result { error!("CPU worker failed: {:?}", e); } } let mut metrics = self.metrics.lock(); metrics.duration = start.elapsed(); metrics.system_stable = true; info!("CPU saturation test complete"); Ok(metrics.clone()) } async fn cpu_intensive_workload(core_id: usize, duration: Duration) -> Result<()> { let start = Instant::now(); while start.elapsed() < duration { // CPU-intensive computation let mut x = core_id as f64; for _ in 0..100_000 { x = (x * 1.1).sin().cos().tan(); } // Yield occasionally to avoid completely blocking if start.elapsed().as_millis() % 100 == 0 { tokio::task::yield_now().await; } } Ok(()) } } #[cfg(test)] mod tests { use super::*; #[tokio::test] async fn test_database_exhaustion() { let test = DatabaseExhaustionTest::new(10, Duration::from_secs(5)); let metrics = test.run().await.expect("Test failed"); assert!(metrics.total_operations > 0); assert!( metrics.exhaustion_failures > 0, "Should have exhaustion failures" ); assert!(metrics.system_stable, "System should remain stable"); assert!( metrics.graceful_degradation, "Should handle exhaustion gracefully" ); } #[tokio::test] async fn test_redis_exhaustion() { let test = RedisExhaustionTest::new(5, Duration::from_secs(5)); let metrics = test.run().await.expect("Test failed"); assert!(metrics.total_operations > 0); assert!(metrics.success_rate() < 100.0, "Should have some failures"); } #[tokio::test] async fn test_memory_pressure() { let test = MemoryPressureTest::new(100, Duration::from_secs(5)); let metrics = test.run().await.expect("Test failed"); assert!( metrics.system_stable, "System should remain stable under memory pressure" ); } #[tokio::test] async fn test_cpu_saturation() { let num_cores = num_cpus::get(); let test = CpuSaturationTest::new(num_cores, Duration::from_secs(5)); let metrics = test.run().await.expect("Test failed"); assert!( metrics.system_stable, "System should remain stable under CPU saturation" ); } #[tokio::test] #[ignore = "Resource intensive - run manually"] async fn test_all_resources_simultaneously() { let _ = tracing_subscriber::fmt::try_init(); info!("Running simultaneous resource exhaustion test"); let duration = Duration::from_secs(60); // Create test instances let db_test = DatabaseExhaustionTest::new(20, duration); let redis_test = RedisExhaustionTest::new(10, duration); let mem_test = MemoryPressureTest::new(500, duration); let cpu_test = CpuSaturationTest::new(num_cpus::get(), duration); // Run all tests concurrently let (db_result, redis_result, mem_result, cpu_result) = tokio::join!( db_test.run(), redis_test.run(), mem_test.run(), cpu_test.run(), ); let db_metrics = db_result.expect("DB test failed"); let redis_metrics = redis_result.expect("Redis test failed"); let mem_metrics = mem_result.expect("Memory test failed"); let cpu_metrics = cpu_result.expect("CPU test failed"); // All tests should complete without crashes assert!(db_metrics.system_stable); assert!(redis_metrics.system_stable); assert!(mem_metrics.system_stable); assert!(cpu_metrics.system_stable); info!("All resources exhausted simultaneously - system remained stable"); } }