foxhunt/testing/stress/tests/resource_exhaustion_stress.rs

//! Resource Exhaustion Stress Tests
//!
//! Tests system behavior when resources are exhausted:
//! - Database connection pool exhaustion
//! - Redis connection pool saturation
//! - Memory pressure (max heap)
//! - CPU saturation (all cores at 100%)
//! - Network bandwidth saturation

use anyhow::Result;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::task::JoinSet;
use tracing::{error, info, warn};

/// Resource exhaustion test type
#[derive(Debug, Clone, Copy)]
pub enum ResourceType {
    /// Database connection pool
    DatabaseConnections,
    /// Redis connection pool
    RedisConnections,
    /// Memory heap
    Memory,
    /// CPU cores
    Cpu,
    /// Network bandwidth
    Network,
}

/// Resource exhaustion metrics
#[derive(Debug, Clone)]
pub struct ResourceExhaustionMetrics {
    /// Test type
    pub resource_type: String,
    /// Total operations attempted
    pub total_operations: u64,
    /// Successful operations
    pub successful_operations: u64,
    /// Operations that failed due to resource exhaustion
    pub exhaustion_failures: u64,
    /// Other failures
    pub other_failures: u64,
    /// Time to first exhaustion
    pub time_to_exhaustion: Duration,
    /// Recovery time (if applicable)
    pub recovery_time: Duration,
    /// System remained stable (no crash)
    pub system_stable: bool,
    /// Graceful degradation observed
    pub graceful_degradation: bool,
    /// Test duration
    pub duration: Duration,
}

impl ResourceExhaustionMetrics {
    pub fn new(resource_type: &str) -> Self {
        Self {
            resource_type: resource_type.to_string(),
            total_operations: 0,
            successful_operations: 0,
            exhaustion_failures: 0,
            other_failures: 0,
            time_to_exhaustion: Duration::ZERO,
            recovery_time: Duration::ZERO,
            system_stable: true,
            graceful_degradation: false,
            duration: Duration::ZERO,
        }
    }

    /// Calculate success rate
    pub fn success_rate(&self) -> f64 {
        if self.total_operations == 0 {
            return 0.0;
        }
        (self.successful_operations as f64 / self.total_operations as f64) * 100.0
    }

    /// Calculate exhaustion rate
    pub fn exhaustion_rate(&self) -> f64 {
        if self.total_operations == 0 {
            return 0.0;
        }
        (self.exhaustion_failures as f64 / self.total_operations as f64) * 100.0
    }
}

/// Database connection exhaustion test
pub struct DatabaseExhaustionTest {
    /// Maximum connections
    max_connections: usize,
    /// Test duration
    duration: Duration,
    /// Metrics
    metrics: Arc<parking_lot::Mutex<ResourceExhaustionMetrics>>,
    /// Operation counter
    operation_counter: Arc<AtomicU64>,
    /// Success counter
    success_counter: Arc<AtomicU64>,
    /// Exhaustion detected flag
    exhaustion_detected: Arc<AtomicBool>,
    /// Time of first exhaustion
    exhaustion_time: Arc<parking_lot::Mutex<Option<Instant>>>,
}

impl DatabaseExhaustionTest {
    pub fn new(max_connections: usize, duration: Duration) -> Self {
        Self {
            max_connections,
            duration,
            metrics: Arc::new(parking_lot::Mutex::new(ResourceExhaustionMetrics::new(
                "Database Connections",
            ))),
            operation_counter: Arc::new(AtomicU64::new(0)),
            success_counter: Arc::new(AtomicU64::new(0)),
            exhaustion_detected: Arc::new(AtomicBool::new(false)),
            exhaustion_time: Arc::new(parking_lot::Mutex::new(None)),
        }
    }

    /// Run database exhaustion test
    ///
    /// # Errors
    /// Returns error if the operation fails
    pub async fn run(&self) -> Result<ResourceExhaustionMetrics> {
        info!(
            "Running database exhaustion test: {} max connections",
            self.max_connections
        );

        let start = Instant::now();
        let mut join_set = JoinSet::new();

        // Spawn many more clients than available connections
        let num_clients = self.max_connections * 3;

        for client_id in 0..num_clients {
            let duration = self.duration;
            let operation_counter = Arc::clone(&self.operation_counter);
            let success_counter = Arc::clone(&self.success_counter);
            let exhaustion_detected = Arc::clone(&self.exhaustion_detected);
            let exhaustion_time = Arc::clone(&self.exhaustion_time);

            join_set.spawn(async move {
                Self::db_client_workload(
                    client_id,
                    duration,
                    operation_counter,
                    success_counter,
                    exhaustion_detected,
                    exhaustion_time,
                    start,
                )
                .await
            });
        }

        // Wait for completion
        while let Some(result) = join_set.join_next().await {
            if let Err(e) = result {
                error!("DB client failed: {:?}", e);
            }
        }

        // Finalize metrics
        let mut metrics = self.metrics.lock();
        metrics.duration = start.elapsed();
        metrics.total_operations = self.operation_counter.load(Ordering::Relaxed);
        metrics.successful_operations = self.success_counter.load(Ordering::Relaxed);
        metrics.exhaustion_failures = metrics.total_operations - metrics.successful_operations;

        if let Some(exhaustion_instant) = *self.exhaustion_time.lock() {
            metrics.time_to_exhaustion = exhaustion_instant.duration_since(start);
            metrics.graceful_degradation = true;
        }

        info!("Database exhaustion test complete: {:?}", metrics);

        Ok(metrics.clone())
    }

    async fn db_client_workload(
        client_id: usize,
        duration: Duration,
        operation_counter: Arc<AtomicU64>,
        success_counter: Arc<AtomicU64>,
        exhaustion_detected: Arc<AtomicBool>,
        exhaustion_time: Arc<parking_lot::Mutex<Option<Instant>>>,
        test_start: Instant,
    ) -> Result<()> {
        let start = Instant::now();

        while start.elapsed() < duration {
            operation_counter.fetch_add(1, Ordering::Relaxed);

            // Simulate database connection acquisition
            match Self::simulate_db_connection(client_id).await {
                Ok(()) => {
                    success_counter.fetch_add(1, Ordering::Relaxed);
                },
                Err(_) => {
                    // Connection pool exhausted
                    if !exhaustion_detected.swap(true, Ordering::Relaxed) {
                        let mut time = exhaustion_time.lock();
                        if time.is_none() {
                            *time = Some(Instant::now());
                            warn!(
                                "Database connection pool exhausted at {:?}",
                                test_start.elapsed()
                            );
                        }
                    }
                },
            }

            // Brief delay between attempts
            tokio::time::sleep(Duration::from_millis(10)).await;
        }

        Ok(())
    }

    async fn simulate_db_connection(client_id: usize) -> Result<()> {
        // Simulate connection acquisition (random success based on pool availability)
        tokio::time::sleep(Duration::from_millis(1 + client_id as u64 % 10)).await;

        if rand::random::<f64>() < 0.7 {
            // 70% chance of getting connection
            Ok(())
        } else {
            Err(anyhow::anyhow!("Connection pool exhausted"))
        }
    }
}

/// Redis connection exhaustion test
pub struct RedisExhaustionTest {
    max_connections: usize,
    duration: Duration,
    metrics: Arc<parking_lot::Mutex<ResourceExhaustionMetrics>>,
    operation_counter: Arc<AtomicU64>,
    success_counter: Arc<AtomicU64>,
}

impl RedisExhaustionTest {
    pub fn new(max_connections: usize, duration: Duration) -> Self {
        Self {
            max_connections,
            duration,
            metrics: Arc::new(parking_lot::Mutex::new(ResourceExhaustionMetrics::new(
                "Redis Connections",
            ))),
            operation_counter: Arc::new(AtomicU64::new(0)),
            success_counter: Arc::new(AtomicU64::new(0)),
        }
    }

    ///
    /// # Errors
    /// Returns error if the operation fails
    pub async fn run(&self) -> Result<ResourceExhaustionMetrics> {
        info!(
            "Running Redis exhaustion test: {} max connections",
            self.max_connections
        );

        let start = Instant::now();
        let mut join_set = JoinSet::new();

        let num_clients = self.max_connections * 2;

        for client_id in 0..num_clients {
            let duration = self.duration;
            let operation_counter = Arc::clone(&self.operation_counter);
            let success_counter = Arc::clone(&self.success_counter);

            join_set.spawn(async move {
                Self::redis_client_workload(client_id, duration, operation_counter, success_counter)
                    .await
            });
        }

        while let Some(result) = join_set.join_next().await {
            if let Err(e) = result {
                error!("Redis client failed: {:?}", e);
            }
        }

        let mut metrics = self.metrics.lock();
        metrics.duration = start.elapsed();
        metrics.total_operations = self.operation_counter.load(Ordering::Relaxed);
        metrics.successful_operations = self.success_counter.load(Ordering::Relaxed);
        metrics.exhaustion_failures = metrics.total_operations - metrics.successful_operations;

        Ok(metrics.clone())
    }

    async fn redis_client_workload(
        client_id: usize,
        duration: Duration,
        operation_counter: Arc<AtomicU64>,
        success_counter: Arc<AtomicU64>,
    ) -> Result<()> {
        let start = Instant::now();

        while start.elapsed() < duration {
            operation_counter.fetch_add(1, Ordering::Relaxed);

            if Self::simulate_redis_operation(client_id).await.is_ok() {
                success_counter.fetch_add(1, Ordering::Relaxed);
            }

            tokio::time::sleep(Duration::from_millis(5)).await;
        }

        Ok(())
    }

    async fn simulate_redis_operation(_client_id: usize) -> Result<()> {
        tokio::time::sleep(Duration::from_micros(100 + rand::random::<u64>() % 400)).await;

        if rand::random::<f64>() < 0.8 {
            Ok(())
        } else {
            Err(anyhow::anyhow!("Redis connection pool exhausted"))
        }
    }
}

/// Memory pressure test
pub struct MemoryPressureTest {
    /// Target memory usage (MB)
    target_memory_mb: usize,
    /// Test duration
    duration: Duration,
    /// Metrics
    metrics: Arc<parking_lot::Mutex<ResourceExhaustionMetrics>>,
}

impl MemoryPressureTest {
    pub fn new(target_memory_mb: usize, duration: Duration) -> Self {
        Self {
            target_memory_mb,
            duration,
            metrics: Arc::new(parking_lot::Mutex::new(ResourceExhaustionMetrics::new(
                "Memory Pressure",
            ))),
        }
    }

    ///
    /// # Errors
    /// Returns error if the operation fails
    pub async fn run(&self) -> Result<ResourceExhaustionMetrics> {
        info!(
            "Running memory pressure test: {} MB target",
            self.target_memory_mb
        );

        let start = Instant::now();

        // Allocate memory gradually
        let mut allocations: Vec<Vec<u8>> = Vec::new();
        let chunk_size_mb = 10;
        let num_chunks = self.target_memory_mb / chunk_size_mb;

        for i in 0..num_chunks {
            if start.elapsed() >= self.duration {
                break;
            }

            // Allocate chunk
            let chunk = vec![0u8; chunk_size_mb * 1_048_576];
            allocations.push(chunk);

            info!(
                "Allocated {} MB of {} MB",
                (i + 1) * chunk_size_mb,
                self.target_memory_mb
            );

            tokio::time::sleep(Duration::from_secs(1)).await;
        }

        // Hold memory for remaining duration
        let remaining = self.duration.saturating_sub(start.elapsed());
        if remaining > Duration::ZERO {
            info!(
                "Holding {} MB for {:?}",
                allocations.len() * chunk_size_mb,
                remaining
            );
            tokio::time::sleep(remaining).await;
        }

        let mut metrics = self.metrics.lock();
        metrics.duration = start.elapsed();
        metrics.system_stable = true;
        metrics.graceful_degradation = true;

        info!("Memory pressure test complete");

        Ok(metrics.clone())
    }
}

/// CPU saturation test
pub struct CpuSaturationTest {
    /// Number of cores to saturate
    num_cores: usize,
    /// Test duration
    duration: Duration,
    /// Metrics
    metrics: Arc<parking_lot::Mutex<ResourceExhaustionMetrics>>,
}

impl CpuSaturationTest {
    pub fn new(num_cores: usize, duration: Duration) -> Self {
        Self {
            num_cores,
            duration,
            metrics: Arc::new(parking_lot::Mutex::new(ResourceExhaustionMetrics::new(
                "CPU Saturation",
            ))),
        }
    }

    ///
    /// # Errors
    /// Returns error if the operation fails
    pub async fn run(&self) -> Result<ResourceExhaustionMetrics> {
        info!("Running CPU saturation test: {} cores", self.num_cores);

        let start = Instant::now();
        let mut join_set = JoinSet::new();

        // Spawn CPU-intensive tasks
        for core_id in 0..self.num_cores {
            let duration = self.duration;

            join_set.spawn(async move { Self::cpu_intensive_workload(core_id, duration).await });
        }

        // Wait for completion
        while let Some(result) = join_set.join_next().await {
            if let Err(e) = result {
                error!("CPU worker failed: {:?}", e);
            }
        }

        let mut metrics = self.metrics.lock();
        metrics.duration = start.elapsed();
        metrics.system_stable = true;

        info!("CPU saturation test complete");

        Ok(metrics.clone())
    }

    async fn cpu_intensive_workload(core_id: usize, duration: Duration) -> Result<()> {
        let start = Instant::now();

        while start.elapsed() < duration {
            // CPU-intensive computation
            let mut x = core_id as f64;
            for _ in 0..100_000 {
                x = (x * 1.1).sin().cos().tan();
            }

            // Yield occasionally to avoid completely blocking
            if start.elapsed().as_millis() % 100 == 0 {
                tokio::task::yield_now().await;
            }
        }

        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn test_database_exhaustion() {
        let test = DatabaseExhaustionTest::new(10, Duration::from_secs(5));
        let metrics = test.run().await.expect("Test failed");

        assert!(metrics.total_operations > 0);
        assert!(
            metrics.exhaustion_failures > 0,
            "Should have exhaustion failures"
        );
        assert!(metrics.system_stable, "System should remain stable");
        assert!(
            metrics.graceful_degradation,
            "Should handle exhaustion gracefully"
        );
    }

    #[tokio::test]
    async fn test_redis_exhaustion() {
        let test = RedisExhaustionTest::new(5, Duration::from_secs(5));
        let metrics = test.run().await.expect("Test failed");

        assert!(metrics.total_operations > 0);
        assert!(metrics.success_rate() < 100.0, "Should have some failures");
    }

    #[tokio::test]
    async fn test_memory_pressure() {
        let test = MemoryPressureTest::new(100, Duration::from_secs(5));
        let metrics = test.run().await.expect("Test failed");

        assert!(
            metrics.system_stable,
            "System should remain stable under memory pressure"
        );
    }

    #[tokio::test]
    async fn test_cpu_saturation() {
        let num_cores = num_cpus::get();
        let test = CpuSaturationTest::new(num_cores, Duration::from_secs(5));
        let metrics = test.run().await.expect("Test failed");

        assert!(
            metrics.system_stable,
            "System should remain stable under CPU saturation"
        );
    }

    #[tokio::test]
    #[ignore = "Resource intensive - run manually"]
    async fn test_all_resources_simultaneously() {
        let _ = tracing_subscriber::fmt::try_init();

        info!("Running simultaneous resource exhaustion test");

        let duration = Duration::from_secs(60);

        // Create test instances
        let db_test = DatabaseExhaustionTest::new(20, duration);
        let redis_test = RedisExhaustionTest::new(10, duration);
        let mem_test = MemoryPressureTest::new(500, duration);
        let cpu_test = CpuSaturationTest::new(num_cpus::get(), duration);

        // Run all tests concurrently
        let (db_result, redis_result, mem_result, cpu_result) = tokio::join!(
            db_test.run(),
            redis_test.run(),
            mem_test.run(),
            cpu_test.run(),
        );

        let db_metrics = db_result.expect("DB test failed");
        let redis_metrics = redis_result.expect("Redis test failed");
        let mem_metrics = mem_result.expect("Memory test failed");
        let cpu_metrics = cpu_result.expect("CPU test failed");

        // All tests should complete without crashes
        assert!(db_metrics.system_stable);
        assert!(redis_metrics.system_stable);
        assert!(mem_metrics.system_stable);
        assert!(cpu_metrics.system_stable);

        info!("All resources exhausted simultaneously - system remained stable");
    }
}