foxhunt/services/stress_tests/tests/concurrent_clients_stress.rs

//! Concurrent Clients Stress Tests
//!
//! Tests system behavior with many simultaneous clients:
//! - 1,000 concurrent TLI clients
//! - 10,000 concurrent WebSocket connections
//! - Thundering herd scenario (all clients submit at once)
//! - Performance under degraded conditions

use anyhow::Result;
use hdrhistogram::Histogram;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::Barrier;
use tokio::task::JoinSet;
use tracing::{error, info};

/// Concurrent client test metrics
#[derive(Debug, Clone)]
pub struct ConcurrentClientMetrics {
    /// Number of concurrent clients
    pub num_clients: usize,
    /// Total requests sent
    pub total_requests: u64,
    /// Successful requests
    pub successful_requests: u64,
    /// Failed requests
    pub failed_requests: u64,
    /// Latency histogram
    pub latency_histogram: Histogram<u64>,
    /// Fairness score (0-100, 100 = perfectly fair)
    pub fairness_score: f64,
    /// Starvation detected
    pub starvation_detected: bool,
    /// Test duration
    pub duration: Duration,
    /// Test type
    pub test_type: String,
}

impl ConcurrentClientMetrics {
    pub fn new(num_clients: usize, test_type: &str) -> Self {
        Self {
            num_clients,
            total_requests: 0,
            successful_requests: 0,
            failed_requests: 0,
            latency_histogram: Histogram::<u64>::new_with_bounds(1, 60_000_000, 3).unwrap(),
            fairness_score: 0.0,
            starvation_detected: false,
            duration: Duration::ZERO,
            test_type: test_type.to_string(),
        }
    }

    pub fn success_rate(&self) -> f64 {
        if self.total_requests == 0 {
            return 0.0;
        }
        (self.successful_requests as f64 / self.total_requests as f64) * 100.0
    }

    pub fn p99_latency_us(&self) -> u64 {
        self.latency_histogram.value_at_quantile(0.99)
    }

    pub fn avg_latency_us(&self) -> f64 {
        self.latency_histogram.mean()
    }
}

/// Concurrent client test runner
pub struct ConcurrentClientTest {
    /// Number of concurrent clients
    num_clients: usize,
    /// Requests per client
    requests_per_client: usize,
    /// Test type
    test_type: TestType,
    /// Metrics
    metrics: Arc<parking_lot::Mutex<ConcurrentClientMetrics>>,
    /// Request counter
    request_counter: Arc<AtomicU64>,
    /// Success counter
    success_counter: Arc<AtomicU64>,
    /// Per-client request counts (for fairness)
    client_counts: Arc<dashmap::DashMap<usize, u64>>,
}

#[derive(Debug, Clone, Copy)]
pub enum TestType {
    /// Regular concurrent clients
    Standard,
    /// Thundering herd (synchronized start)
    ThunderingHerd,
    /// WebSocket-style persistent connections
    WebSocket,
}

impl ConcurrentClientTest {
    pub fn new(num_clients: usize, requests_per_client: usize, test_type: TestType) -> Self {
        let test_name = match test_type {
            TestType::Standard => "Standard Concurrent",
            TestType::ThunderingHerd => "Thundering Herd",
            TestType::WebSocket => "WebSocket Persistent",
        };

        Self {
            num_clients,
            requests_per_client,
            test_type,
            metrics: Arc::new(parking_lot::Mutex::new(ConcurrentClientMetrics::new(
                num_clients,
                test_name,
            ))),
            request_counter: Arc::new(AtomicU64::new(0)),
            success_counter: Arc::new(AtomicU64::new(0)),
            client_counts: Arc::new(dashmap::DashMap::new()),
        }
    }

    ///
    /// # Errors
    /// Returns error if the operation fails
    pub async fn run(&self) -> Result<ConcurrentClientMetrics> {
        info!(
            "Running concurrent client test: {} clients, {:?}",
            self.num_clients, self.test_type
        );

        let start = Instant::now();

        match self.test_type {
            TestType::Standard => self.run_standard().await?,
            TestType::ThunderingHerd => self.run_thundering_herd().await?,
            TestType::WebSocket => self.run_websocket().await?,
        }

        // Calculate fairness
        let fairness = self.calculate_fairness();

        // Finalize metrics
        let mut metrics = self.metrics.lock();
        metrics.duration = start.elapsed();
        metrics.total_requests = self.request_counter.load(Ordering::Relaxed);
        metrics.successful_requests = self.success_counter.load(Ordering::Relaxed);
        metrics.failed_requests = metrics.total_requests - metrics.successful_requests;
        metrics.fairness_score = fairness;
        metrics.starvation_detected = fairness < 50.0;

        info!("Concurrent client test complete: {:?}", metrics);

        Ok(metrics.clone())
    }

    /// Standard concurrent clients
    async fn run_standard(&self) -> Result<()> {
        let mut join_set = JoinSet::new();

        for client_id in 0..self.num_clients {
            let requests = self.requests_per_client;
            let request_counter = Arc::clone(&self.request_counter);
            let success_counter = Arc::clone(&self.success_counter);
            let metrics = Arc::clone(&self.metrics);
            let client_counts = Arc::clone(&self.client_counts);

            join_set.spawn(async move {
                Self::client_workload(
                    client_id,
                    requests,
                    request_counter,
                    success_counter,
                    metrics,
                    client_counts,
                )
                .await
            });
        }

        while let Some(result) = join_set.join_next().await {
            if let Err(e) = result {
                error!("Client failed: {:?}", e);
            }
        }

        Ok(())
    }

    /// Thundering herd: all clients start simultaneously
    async fn run_thundering_herd(&self) -> Result<()> {
        let barrier = Arc::new(Barrier::new(self.num_clients));
        let mut join_set = JoinSet::new();

        for client_id in 0..self.num_clients {
            let barrier = Arc::clone(&barrier);
            let requests = self.requests_per_client;
            let request_counter = Arc::clone(&self.request_counter);
            let success_counter = Arc::clone(&self.success_counter);
            let metrics = Arc::clone(&self.metrics);
            let client_counts = Arc::clone(&self.client_counts);

            join_set.spawn(async move {
                // Wait for all clients to be ready
                barrier.wait().await;

                // All clients start at the same instant
                Self::client_workload(
                    client_id,
                    requests,
                    request_counter,
                    success_counter,
                    metrics,
                    client_counts,
                )
                .await
            });
        }

        while let Some(result) = join_set.join_next().await {
            if let Err(e) = result {
                error!("Thundering herd client failed: {:?}", e);
            }
        }

        Ok(())
    }

    /// WebSocket-style persistent connections
    async fn run_websocket(&self) -> Result<()> {
        let mut join_set = JoinSet::new();

        for client_id in 0..self.num_clients {
            let requests = self.requests_per_client;
            let request_counter = Arc::clone(&self.request_counter);
            let success_counter = Arc::clone(&self.success_counter);
            let metrics = Arc::clone(&self.metrics);
            let client_counts = Arc::clone(&self.client_counts);

            join_set.spawn(async move {
                // Simulate WebSocket connection lifecycle
                Self::websocket_client(
                    client_id,
                    requests,
                    request_counter,
                    success_counter,
                    metrics,
                    client_counts,
                )
                .await
            });
        }

        while let Some(result) = join_set.join_next().await {
            if let Err(e) = result {
                error!("WebSocket client failed: {:?}", e);
            }
        }

        Ok(())
    }

    /// Standard client workload
    async fn client_workload(
        client_id: usize,
        num_requests: usize,
        request_counter: Arc<AtomicU64>,
        success_counter: Arc<AtomicU64>,
        metrics: Arc<parking_lot::Mutex<ConcurrentClientMetrics>>,
        client_counts: Arc<dashmap::DashMap<usize, u64>>,
    ) -> Result<()> {
        for _ in 0..num_requests {
            let req_start = Instant::now();

            // Simulate request
            let success = Self::simulate_request(client_id).await;
            let latency = req_start.elapsed();

            // Update counters
            request_counter.fetch_add(1, Ordering::Relaxed);
            if success {
                success_counter.fetch_add(1, Ordering::Relaxed);
            }

            // Record latency
            {
                let mut m = metrics.lock();
                let _ = m.latency_histogram.record(latency.as_micros() as u64);
            }

            // Track per-client requests
            *client_counts.entry(client_id).or_insert(0) += 1;

            // Small delay between requests
            tokio::time::sleep(Duration::from_millis(1)).await;
        }

        Ok(())
    }

    /// WebSocket client simulation
    async fn websocket_client(
        client_id: usize,
        num_messages: usize,
        request_counter: Arc<AtomicU64>,
        success_counter: Arc<AtomicU64>,
        metrics: Arc<parking_lot::Mutex<ConcurrentClientMetrics>>,
        client_counts: Arc<dashmap::DashMap<usize, u64>>,
    ) -> Result<()> {
        // Simulate WebSocket handshake
        tokio::time::sleep(Duration::from_millis(10)).await;

        // Send messages over persistent connection
        for _ in 0..num_messages {
            let msg_start = Instant::now();

            let success = Self::simulate_websocket_message(client_id).await;
            let latency = msg_start.elapsed();

            request_counter.fetch_add(1, Ordering::Relaxed);
            if success {
                success_counter.fetch_add(1, Ordering::Relaxed);
            }

            {
                let mut m = metrics.lock();
                let _ = m.latency_histogram.record(latency.as_micros() as u64);
            }

            *client_counts.entry(client_id).or_insert(0) += 1;

            // Simulate message rate (100 msg/sec)
            tokio::time::sleep(Duration::from_millis(10)).await;
        }

        Ok(())
    }

    /// Simulate request
    async fn simulate_request(_client_id: usize) -> bool {
        tokio::time::sleep(Duration::from_micros(50 + rand::random::<u64>() % 450)).await;
        rand::random::<f64>() < 0.999
    }

    /// Simulate WebSocket message
    async fn simulate_websocket_message(_client_id: usize) -> bool {
        tokio::time::sleep(Duration::from_micros(20 + rand::random::<u64>() % 180)).await;
        rand::random::<f64>() < 0.9995
    }

    /// Calculate fairness score (coefficient of variation)
    fn calculate_fairness(&self) -> f64 {
        let counts: Vec<u64> = self
            .client_counts
            .iter()
            .map(|entry| *entry.value())
            .collect();

        if counts.is_empty() {
            return 0.0;
        }

        let mean = counts.iter().sum::<u64>() as f64 / counts.len() as f64;
        let variance = counts
            .iter()
            .map(|&x| {
                let diff = x as f64 - mean;
                diff * diff
            })
            .sum::<f64>()
            / counts.len() as f64;

        let std_dev = variance.sqrt();
        let cv = if mean > 0.0 { std_dev / mean } else { 0.0 };

        // Convert to fairness score (0-100, lower CV = higher fairness)
        // CV of 0 = 100% fair, CV of 1.0 = 0% fair
        ((1.0 - cv.min(1.0)) * 100.0).max(0.0)
    }
}

/// Performance under failure test
pub struct PerformanceUnderFailureTest {
    /// Number of clients
    num_clients: usize,
    /// Failure type
    failure_type: FailureType,
    /// Test duration
    duration: Duration,
    /// Metrics
    metrics: Arc<parking_lot::Mutex<ConcurrentClientMetrics>>,
}

#[derive(Debug, Clone, Copy)]
pub enum FailureType {
    /// Degraded database (50% slower)
    DegradedDatabase,
    /// High network latency (100ms added)
    HighNetworkLatency,
    /// Intermittent Redis failures
    IntermittentRedis,
}

impl PerformanceUnderFailureTest {
    pub fn new(num_clients: usize, failure_type: FailureType, duration: Duration) -> Self {
        let test_name = match failure_type {
            FailureType::DegradedDatabase => "Degraded Database",
            FailureType::HighNetworkLatency => "High Network Latency",
            FailureType::IntermittentRedis => "Intermittent Redis",
        };

        Self {
            num_clients,
            failure_type,
            duration,
            metrics: Arc::new(parking_lot::Mutex::new(ConcurrentClientMetrics::new(
                num_clients,
                test_name,
            ))),
        }
    }

    ///
    /// # Errors
    /// Returns error if the operation fails
    pub async fn run(&self) -> Result<ConcurrentClientMetrics> {
        info!(
            "Running performance under failure: {} clients, {:?}",
            self.num_clients, self.failure_type
        );

        let start = Instant::now();
        let mut join_set = JoinSet::new();

        let request_counter = Arc::new(AtomicU64::new(0));
        let success_counter = Arc::new(AtomicU64::new(0));

        for client_id in 0..self.num_clients {
            let duration = self.duration;
            let failure_type = self.failure_type;
            let request_counter = Arc::clone(&request_counter);
            let success_counter = Arc::clone(&success_counter);
            let metrics = Arc::clone(&self.metrics);

            join_set.spawn(async move {
                Self::degraded_client_workload(
                    client_id,
                    duration,
                    failure_type,
                    request_counter,
                    success_counter,
                    metrics,
                )
                .await
            });
        }

        while let Some(result) = join_set.join_next().await {
            if let Err(e) = result {
                error!("Degraded client failed: {:?}", e);
            }
        }

        let mut metrics = self.metrics.lock();
        metrics.duration = start.elapsed();
        metrics.total_requests = request_counter.load(Ordering::Relaxed);
        metrics.successful_requests = success_counter.load(Ordering::Relaxed);
        metrics.failed_requests = metrics.total_requests - metrics.successful_requests;

        info!("Performance under failure test complete");

        Ok(metrics.clone())
    }

    async fn degraded_client_workload(
        client_id: usize,
        duration: Duration,
        failure_type: FailureType,
        request_counter: Arc<AtomicU64>,
        success_counter: Arc<AtomicU64>,
        metrics: Arc<parking_lot::Mutex<ConcurrentClientMetrics>>,
    ) -> Result<()> {
        let start = Instant::now();

        while start.elapsed() < duration {
            let req_start = Instant::now();

            // Simulate request under degraded conditions
            let success = Self::simulate_degraded_request(client_id, failure_type).await;
            let latency = req_start.elapsed();

            request_counter.fetch_add(1, Ordering::Relaxed);
            if success {
                success_counter.fetch_add(1, Ordering::Relaxed);
            }

            {
                let mut m = metrics.lock();
                let _ = m.latency_histogram.record(latency.as_micros() as u64);
            }

            tokio::time::sleep(Duration::from_millis(10)).await;
        }

        Ok(())
    }

    async fn simulate_degraded_request(_client_id: usize, failure_type: FailureType) -> bool {
        match failure_type {
            FailureType::DegradedDatabase => {
                // 50% slower queries
                tokio::time::sleep(Duration::from_micros(150 + rand::random::<u64>() % 450)).await;
                rand::random::<f64>() < 0.99
            },
            FailureType::HighNetworkLatency => {
                // Additional 100ms network latency
                tokio::time::sleep(Duration::from_millis(100)).await;
                rand::random::<f64>() < 0.999
            },
            FailureType::IntermittentRedis => {
                // 20% Redis failure rate
                tokio::time::sleep(Duration::from_micros(50 + rand::random::<u64>() % 450)).await;
                rand::random::<f64>() < 0.8
            },
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn test_standard_concurrent_clients() {
        let test = ConcurrentClientTest::new(100, 100, TestType::Standard);
        let metrics = test.run().await.expect("Test failed");

        assert_eq!(metrics.num_clients, 100);
        assert!(metrics.total_requests > 0);
        assert!(metrics.success_rate() > 99.0);
        assert!(
            metrics.fairness_score > 50.0,
            "Should have reasonable fairness"
        );
    }

    #[tokio::test]
    async fn test_thundering_herd() {
        let test = ConcurrentClientTest::new(500, 50, TestType::ThunderingHerd);
        let metrics = test.run().await.expect("Test failed");

        assert!(metrics.total_requests > 0);
        assert!(
            metrics.success_rate() > 95.0,
            "Should handle thundering herd gracefully"
        );
    }

    #[tokio::test]
    async fn test_websocket_connections() {
        let test = ConcurrentClientTest::new(100, 100, TestType::WebSocket);
        let metrics = test.run().await.expect("Test failed");

        assert!(metrics.total_requests > 0);
        assert!(metrics.success_rate() > 99.0);
    }

    #[tokio::test]
    async fn test_performance_under_degraded_db() {
        let test = PerformanceUnderFailureTest::new(
            50,
            FailureType::DegradedDatabase,
            Duration::from_secs(5),
        );
        let metrics = test.run().await.expect("Test failed");

        assert!(metrics.total_requests > 0);
        assert!(
            metrics.success_rate() > 95.0,
            "Should maintain acceptable performance under degradation"
        );
        // Latency should be higher but not catastrophic
        assert!(metrics.avg_latency_us() < 1_000_000.0); // < 1 second
    }

    #[tokio::test]
    #[ignore = "Resource intensive - run manually"]
    async fn test_1000_concurrent_clients() {
        let _ = tracing_subscriber::fmt::try_init();

        info!("Running 1,000 concurrent clients test");

        let test = ConcurrentClientTest::new(1_000, 1_000, TestType::Standard);
        let metrics = test.run().await.expect("Test failed");

        assert_eq!(metrics.num_clients, 1_000);
        assert!(metrics.success_rate() > 99.0);
        assert!(!metrics.starvation_detected, "No client should be starved");
        assert!(
            metrics.fairness_score > 70.0,
            "Should maintain fairness with 1K clients"
        );

        info!("1,000 client test results: {:?}", metrics);
    }

    #[tokio::test]
    #[ignore = "Resource intensive - run manually"]
    async fn test_10000_websocket_connections() {
        let _ = tracing_subscriber::fmt::try_init();

        info!("Running 10,000 WebSocket connections test");

        let test = ConcurrentClientTest::new(10_000, 100, TestType::WebSocket);
        let metrics = test.run().await.expect("Test failed");

        assert_eq!(metrics.num_clients, 10_000);
        assert!(metrics.success_rate() > 99.0);
        assert!(
            metrics.p99_latency_us() < 10_000,
            "P99 latency should be < 10ms"
        );

        info!("10,000 WebSocket test results: {:?}", metrics);
    }
}