foxhunt/services/stress_tests/tests/sustained_load_stress.rs

//! Sustained Load Stress Tests
//!
//! Tests system behavior under sustained high load over extended periods.
//! - 1 hour sustained load at 50K orders/sec
//! - 24 hour soak test at 10K orders/sec
//! - Memory leak detection
//! - Connection pool stability
//! - Database performance degradation monitoring

use anyhow::Result;
use hdrhistogram::Histogram;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::task::JoinSet;
use tokio::time::interval;
use tracing::{error, info};

/// Metrics for sustained load testing
#[derive(Debug, Clone)]
pub struct SustainedLoadMetrics {
    /// Total requests sent
    pub total_requests: u64,
    /// Successful requests
    pub successful_requests: u64,
    /// Failed requests
    pub failed_requests: u64,
    /// Throughput samples (requests/sec per interval)
    pub throughput_samples: Vec<f64>,
    /// Memory usage samples (bytes)
    pub memory_samples: Vec<u64>,
    /// Latency histogram
    pub latency_histogram: Histogram<u64>,
    /// Connection pool size samples
    pub connection_pool_samples: Vec<usize>,
    /// Database query time samples (microseconds)
    pub db_query_times: Vec<u64>,
    /// Test duration
    pub duration: Duration,
}

impl Default for SustainedLoadMetrics {
    fn default() -> Self {
        Self::new()
    }
}

impl SustainedLoadMetrics {
    pub fn new() -> Self {
        Self {
            total_requests: 0,
            successful_requests: 0,
            failed_requests: 0,
            throughput_samples: Vec::new(),
            memory_samples: Vec::new(),
            latency_histogram: Histogram::<u64>::new_with_bounds(1, 60_000_000, 3).unwrap(),
            connection_pool_samples: Vec::new(),
            db_query_times: Vec::new(),
            duration: Duration::ZERO,
        }
    }

    /// Check for performance degradation over time
    pub fn detect_degradation(&self, threshold_percent: f64) -> bool {
        if self.throughput_samples.len() < 10 {
            return false;
        }

        // Compare first 10% vs last 10% of samples
        let sample_count = self.throughput_samples.len();
        let first_10_pct = &self.throughput_samples[..sample_count / 10];
        let last_10_pct = &self.throughput_samples[sample_count * 9 / 10..];

        let avg_first: f64 = first_10_pct.iter().sum::<f64>() / first_10_pct.len() as f64;
        let avg_last: f64 = last_10_pct.iter().sum::<f64>() / last_10_pct.len() as f64;

        let degradation = (avg_first - avg_last) / avg_first * 100.0;

        degradation > threshold_percent
    }

    /// Check for memory leaks
    pub fn detect_memory_leak(&self, growth_threshold_mb: f64) -> bool {
        if self.memory_samples.len() < 10 {
            return false;
        }

        let first_mb = self.memory_samples[0] as f64 / 1_048_576.0;
        let last_mb = *self.memory_samples.last().expect("INVARIANT: Collection should be non-empty") as f64 / 1_048_576.0;

        let growth = last_mb - first_mb;

        growth > growth_threshold_mb
    }

    /// Calculate average throughput
    pub fn avg_throughput(&self) -> f64 {
        if self.throughput_samples.is_empty() {
            return 0.0;
        }
        self.throughput_samples.iter().sum::<f64>() / self.throughput_samples.len() as f64
    }

    /// Calculate p99 latency
    pub fn p99_latency_us(&self) -> u64 {
        self.latency_histogram.value_at_quantile(0.99)
    }

    /// Calculate success rate
    pub fn success_rate(&self) -> f64 {
        if self.total_requests == 0 {
            return 0.0;
        }
        (self.successful_requests as f64 / self.total_requests as f64) * 100.0
    }
}

/// Sustained load test runner
pub struct SustainedLoadTest {
    /// Target throughput (requests per second)
    target_rps: usize,
    /// Test duration
    duration: Duration,
    /// Number of concurrent clients
    concurrent_clients: usize,
    /// Metrics collection
    metrics: Arc<parking_lot::Mutex<SustainedLoadMetrics>>,
    /// Total requests counter
    request_counter: Arc<AtomicU64>,
    /// Success counter
    success_counter: Arc<AtomicU64>,
}

impl SustainedLoadTest {
    /// Create a new sustained load test
    pub fn new(target_rps: usize, duration: Duration, concurrent_clients: usize) -> Self {
        Self {
            target_rps,
            duration,
            concurrent_clients,
            metrics: Arc::new(parking_lot::Mutex::new(SustainedLoadMetrics::new())),
            request_counter: Arc::new(AtomicU64::new(0)),
            success_counter: Arc::new(AtomicU64::new(0)),
        }
    }

    /// Run the sustained load test
    ///
    /// # Errors
    /// Returns error if the operation fails
    pub async fn run(&self) -> Result<SustainedLoadMetrics> {
        info!(
            "Starting sustained load test: {} req/sec for {:?}",
            self.target_rps, self.duration
        );

        let start = Instant::now();
        let mut join_set = JoinSet::new();

        // Spawn client tasks
        let requests_per_client = self.target_rps / self.concurrent_clients;
        let delay_between_requests = Duration::from_millis(1000 / requests_per_client as u64);

        for client_id in 0..self.concurrent_clients {
            let duration = self.duration;
            let delay = delay_between_requests;
            let request_counter = Arc::clone(&self.request_counter);
            let success_counter = Arc::clone(&self.success_counter);
            let metrics = Arc::clone(&self.metrics);

            join_set.spawn(async move {
                Self::client_workload(
                    client_id,
                    duration,
                    delay,
                    request_counter,
                    success_counter,
                    metrics,
                )
                .await
            });
        }

        // Spawn monitoring task
        let monitoring_handle = self.spawn_monitoring_task(start);

        // Wait for all clients to complete
        while let Some(result) = join_set.join_next().await {
            if let Err(e) = result {
                error!("Client task failed: {:?}", e);
            }
        }

        // Stop monitoring
        monitoring_handle.abort();

        // Finalize metrics
        let mut metrics = self.metrics.lock();
        metrics.duration = start.elapsed();
        metrics.total_requests = self.request_counter.load(Ordering::Relaxed);
        metrics.successful_requests = self.success_counter.load(Ordering::Relaxed);
        metrics.failed_requests = metrics.total_requests - metrics.successful_requests;

        info!(
            "Sustained load test complete: {} requests in {:?}",
            metrics.total_requests, metrics.duration
        );

        Ok(metrics.clone())
    }

    /// Client workload: send requests at target rate
    async fn client_workload(
        client_id: usize,
        duration: Duration,
        delay: Duration,
        request_counter: Arc<AtomicU64>,
        success_counter: Arc<AtomicU64>,
        metrics: Arc<parking_lot::Mutex<SustainedLoadMetrics>>,
    ) -> Result<()> {
        let start = Instant::now();

        while start.elapsed() < duration {
            let req_start = Instant::now();

            // Simulate order submission
            let success = Self::simulate_order_submission(client_id).await;

            let latency = req_start.elapsed();

            // Update counters
            request_counter.fetch_add(1, Ordering::Relaxed);
            if success {
                success_counter.fetch_add(1, Ordering::Relaxed);
            }

            // Record latency
            {
                let mut m = metrics.lock();
                let _ = m.latency_histogram.record(latency.as_micros() as u64);
            }

            // Rate limiting - subtract processing time from delay to maintain target rate
            if latency < delay {
                tokio::time::sleep(delay - latency).await;
            } else {
                // Processing took longer than delay interval, no sleep needed
                // This will naturally reduce throughput but is realistic
                tokio::task::yield_now().await;
            }
        }

        Ok(())
    }

    /// Simulate order submission (replace with actual gRPC call in integration tests)
    async fn simulate_order_submission(_client_id: usize) -> bool {
        // Simulate processing time (50-500μs)
        tokio::time::sleep(Duration::from_micros(50 + rand::random::<u64>() % 450)).await;

        // 99.9% success rate
        rand::random::<f64>() < 0.999
    }

    /// Spawn monitoring task to collect periodic metrics
    fn spawn_monitoring_task(&self, start: Instant) -> tokio::task::JoinHandle<()> {
        let metrics = Arc::clone(&self.metrics);
        let request_counter = Arc::clone(&self.request_counter);
        let duration = self.duration;

        tokio::spawn(async move {
            let mut interval = interval(Duration::from_secs(1));
            let mut last_count = 0u64;
            let mut sys = sysinfo::System::new_all();

            while start.elapsed() < duration {
                interval.tick().await;

                // Calculate throughput for this interval
                let current_count = request_counter.load(Ordering::Relaxed);
                let throughput = (current_count - last_count) as f64;
                last_count = current_count;

                // Collect memory usage
                sys.refresh_all();
                let memory_bytes = sys.used_memory();

                // Record samples
                let mut m = metrics.lock();
                m.throughput_samples.push(throughput);
                m.memory_samples.push(memory_bytes);

                // Simulate connection pool size (replace with actual monitoring)
                let pool_size = 10 + (rand::random::<usize>() % 5);
                m.connection_pool_samples.push(pool_size);

                // Simulate database query times (replace with actual monitoring)
                let db_query_us = 100 + (rand::random::<u64>() % 900);
                m.db_query_times.push(db_query_us);
                drop(m);
            }
        })
    }

    /// Get current metrics
    pub fn get_metrics(&self) -> SustainedLoadMetrics {
        self.metrics.lock().clone()
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn test_sustained_load_short_duration() {
        // 10 second test at 1000 req/sec
        let test = SustainedLoadTest::new(1000, Duration::from_secs(10), 10);

        let metrics = test.run().await.expect("Test failed");

        assert!(metrics.total_requests > 0, "Should have sent requests");
        assert!(
            metrics.success_rate() > 99.0,
            "Success rate should be > 99%"
        );
        assert!(
            metrics.avg_throughput() > 700.0,
            "Average throughput should be reasonable given overhead (target: 1000, got: {}). \
             Lower than target due to tokio scheduling overhead, lock contention, and simulated processing time.",
            metrics.avg_throughput()
        );
    }

    #[tokio::test]
    async fn test_degradation_detection() {
        let mut metrics = SustainedLoadMetrics::new();

        // Simulate stable throughput
        for _ in 0..100 {
            metrics.throughput_samples.push(1000.0);
        }

        assert!(
            !metrics.detect_degradation(5.0),
            "Should not detect degradation with stable throughput"
        );

        // Simulate degradation
        for _ in 0..10 {
            metrics.throughput_samples.push(800.0);
        }

        assert!(
            metrics.detect_degradation(5.0),
            "Should detect 20% degradation"
        );
    }

    #[tokio::test]
    async fn test_memory_leak_detection() {
        let mut metrics = SustainedLoadMetrics::new();

        // Simulate stable memory
        for _ in 0..100 {
            metrics.memory_samples.push(100 * 1_048_576); // 100 MB
        }

        assert!(
            !metrics.detect_memory_leak(10.0),
            "Should not detect leak with stable memory"
        );

        // Simulate memory growth
        for i in 0..10 {
            metrics.memory_samples.push((120 + i) * 1_048_576); // Growing
        }

        assert!(
            metrics.detect_memory_leak(10.0),
            "Should detect memory leak"
        );
    }

    #[tokio::test]
    #[ignore = "Long running test - run manually"]
    async fn test_one_hour_sustained_load() {
        let _ = tracing_subscriber::fmt::try_init();

        info!("Starting 1-hour sustained load test at 50K req/sec");

        let test = SustainedLoadTest::new(50_000, Duration::from_secs(3600), 500);
        let metrics = test.run().await.expect("Test failed");

        // Assertions
        assert!(
            metrics.total_requests > 50_000 * 3600 * 95 / 100,
            "Should complete > 95% of expected requests"
        );
        assert!(
            metrics.success_rate() > 99.0,
            "Success rate should be > 99%"
        );
        assert!(
            !metrics.detect_degradation(5.0),
            "Should not degrade > 5% over 1 hour"
        );
        assert!(
            !metrics.detect_memory_leak(50.0),
            "Should not leak > 50MB over 1 hour"
        );

        info!("1-hour test results: {:?}", metrics);
    }

    #[tokio::test]
    #[ignore = "Very long running test - run manually"]
    async fn test_24_hour_soak_test() {
        let _ = tracing_subscriber::fmt::try_init();

        info!("Starting 24-hour soak test at 10K req/sec");

        let test = SustainedLoadTest::new(10_000, Duration::from_secs(86400), 100);
        let metrics = test.run().await.expect("Test failed");

        // Assertions for soak test
        assert!(
            metrics.total_requests > 10_000 * 86400 * 95 / 100,
            "Should complete > 95% of expected requests"
        );
        assert!(
            metrics.success_rate() > 99.0,
            "Success rate should be > 99%"
        );
        assert!(
            !metrics.detect_degradation(3.0),
            "Should not degrade > 3% over 24 hours"
        );
        assert!(
            !metrics.detect_memory_leak(100.0),
            "Should not leak > 100MB over 24 hours"
        );

        info!("24-hour soak test results: {:?}", metrics);
    }
}