foxhunt/testing/stress/tests/burst_load_stress.rs

//! Burst Load Stress Tests
//!
//! Tests system behavior under sudden load spikes and gradual ramps.
//! - Spike testing: 0 → 100K orders/sec in 1 second
//! - Gradual ramp: 0 → 50K over 10 minutes
//! - Sustained plateau: 50K for 1 hour
//! - Gradual ramp down: 50K → 0 over 10 minutes

use anyhow::Result;
use hdrhistogram::Histogram;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::Barrier;
use tokio::task::JoinSet;
use tracing::{error, info};

/// Load profile for burst testing
#[derive(Debug, Clone)]
pub enum LoadProfile {
    /// Immediate spike to target RPS
    Spike {
        target_rps: usize,
        duration: Duration,
    },
    /// Gradual ramp up to target RPS
    RampUp {
        target_rps: usize,
        ramp_duration: Duration,
    },
    /// Sustained load at target RPS
    Plateau {
        target_rps: usize,
        duration: Duration,
    },
    /// Gradual ramp down from target RPS to zero
    RampDown {
        start_rps: usize,
        ramp_duration: Duration,
    },
}

/// Burst load test metrics
#[derive(Debug, Clone)]
pub struct BurstLoadMetrics {
    /// Total requests sent
    pub total_requests: u64,
    /// Successful requests
    pub successful_requests: u64,
    /// Failed requests
    pub failed_requests: u64,
    /// Peak throughput achieved (req/sec)
    pub peak_throughput: f64,
    /// Throughput samples over time
    pub throughput_samples: Vec<(Duration, f64)>,
    /// Latency histogram
    pub latency_histogram: Histogram<u64>,
    /// Test duration
    pub duration: Duration,
    /// Load profile description
    pub profile_name: String,
}

impl BurstLoadMetrics {
    pub fn new(profile_name: String) -> Self {
        Self {
            total_requests: 0,
            successful_requests: 0,
            failed_requests: 0,
            peak_throughput: 0.0,
            throughput_samples: Vec::new(),
            latency_histogram: Histogram::<u64>::new_with_bounds(1, 60_000_000, 3).unwrap(),
            duration: Duration::ZERO,
            profile_name,
        }
    }

    /// Calculate success rate
    pub fn success_rate(&self) -> f64 {
        if self.total_requests == 0 {
            return 0.0;
        }
        (self.successful_requests as f64 / self.total_requests as f64) * 100.0
    }

    /// Get p99 latency
    pub fn p99_latency_us(&self) -> u64 {
        self.latency_histogram.value_at_quantile(0.99)
    }

    /// Get average throughput
    pub fn avg_throughput(&self) -> f64 {
        if self.throughput_samples.is_empty() {
            return 0.0;
        }
        self.throughput_samples
            .iter()
            .map(|(_, tps)| tps)
            .sum::<f64>()
            / self.throughput_samples.len() as f64
    }
}

/// Burst load test runner
pub struct BurstLoadTest {
    /// Load profile to execute
    profile: LoadProfile,
    /// Maximum concurrent clients
    max_clients: usize,
    /// Metrics collection
    metrics: Arc<parking_lot::Mutex<BurstLoadMetrics>>,
    /// Request counter
    request_counter: Arc<AtomicU64>,
    /// Success counter
    success_counter: Arc<AtomicU64>,
}

impl BurstLoadTest {
    /// Create new burst load test
    pub fn new(profile: LoadProfile, max_clients: usize) -> Self {
        let profile_name = match &profile {
            LoadProfile::Spike { target_rps, .. } => {
                format!("Spike to {} req/sec", target_rps)
            },
            LoadProfile::RampUp { target_rps, .. } => {
                format!("Ramp up to {} req/sec", target_rps)
            },
            LoadProfile::Plateau { target_rps, .. } => {
                format!("Plateau at {} req/sec", target_rps)
            },
            LoadProfile::RampDown { start_rps, .. } => {
                format!("Ramp down from {} req/sec", start_rps)
            },
        };

        Self {
            profile,
            max_clients,
            metrics: Arc::new(parking_lot::Mutex::new(BurstLoadMetrics::new(profile_name))),
            request_counter: Arc::new(AtomicU64::new(0)),
            success_counter: Arc::new(AtomicU64::new(0)),
        }
    }

    /// Run the burst load test
    ///
    /// # Errors
    /// Returns error if the operation fails
    pub async fn run(&self) -> Result<BurstLoadMetrics> {
        let start = Instant::now();

        match &self.profile {
            LoadProfile::Spike {
                target_rps,
                duration,
            } => self.run_spike(*target_rps, *duration).await?,
            LoadProfile::RampUp {
                target_rps,
                ramp_duration,
            } => self.run_ramp_up(*target_rps, *ramp_duration).await?,
            LoadProfile::Plateau {
                target_rps,
                duration,
            } => self.run_plateau(*target_rps, *duration).await?,
            LoadProfile::RampDown {
                start_rps,
                ramp_duration,
            } => self.run_ramp_down(*start_rps, *ramp_duration).await?,
        }

        // Finalize metrics
        let mut metrics = self.metrics.lock();
        metrics.duration = start.elapsed();
        metrics.total_requests = self.request_counter.load(Ordering::Relaxed);
        metrics.successful_requests = self.success_counter.load(Ordering::Relaxed);
        metrics.failed_requests = metrics.total_requests - metrics.successful_requests;

        Ok(metrics.clone())
    }

    /// Run spike test: immediate load spike
    async fn run_spike(&self, target_rps: usize, duration: Duration) -> Result<()> {
        info!("Running spike test: 0 → {} req/sec", target_rps);

        let barrier = Arc::new(Barrier::new(self.max_clients));
        let mut join_set = JoinSet::new();

        // Spawn monitoring
        let monitoring_handle = self.spawn_monitoring_task();

        // Spawn all clients simultaneously
        let requests_per_client = target_rps / self.max_clients;
        let delay = if requests_per_client > 0 {
            Duration::from_millis(1000 / (requests_per_client as u64))
        } else {
            Duration::from_millis(1000)
        };

        for client_id in 0..self.max_clients {
            let barrier = Arc::clone(&barrier);
            let request_counter = Arc::clone(&self.request_counter);
            let success_counter = Arc::clone(&self.success_counter);
            let metrics = Arc::clone(&self.metrics);

            join_set.spawn(async move {
                // Wait for all clients to be ready
                barrier.wait().await;

                // Start sending requests
                Self::client_workload(
                    client_id,
                    duration,
                    delay,
                    request_counter,
                    success_counter,
                    metrics,
                )
                .await
            });
        }

        // Wait for completion
        while let Some(result) = join_set.join_next().await {
            if let Err(e) = result {
                error!("Spike client failed: {:?}", e);
            }
        }

        monitoring_handle.abort();

        info!("Spike test complete");
        Ok(())
    }

    /// Run ramp up test: gradual increase in load
    async fn run_ramp_up(&self, target_rps: usize, ramp_duration: Duration) -> Result<()> {
        info!(
            "Running ramp up test: 0 → {} req/sec over {:?}",
            target_rps, ramp_duration
        );

        let mut join_set = JoinSet::new();
        let monitoring_handle = self.spawn_monitoring_task();

        let ramp_steps: u64 = 100; // 100 steps in the ramp
        let step_duration = ramp_duration.as_millis() as u64 / ramp_steps;

        for step in 0..ramp_steps {
            let current_rps = (target_rps * (step + 1) as usize) / ramp_steps as usize;
            let clients_for_step = (self.max_clients * (step + 1) as usize) / ramp_steps as usize;

            // Spawn additional clients for this step
            // Calculate how many NEW clients to spawn for this step
            let previous_clients = if step > 0 {
                (self.max_clients * step as usize) / ramp_steps as usize
            } else {
                0
            };
            let new_clients = clients_for_step.saturating_sub(previous_clients);
            for client_id in 0..new_clients {
                let delay = if current_rps > 0 {
                    Duration::from_millis(1000 / (current_rps as u64))
                } else {
                    Duration::from_millis(1000)
                };
                let request_counter = Arc::clone(&self.request_counter);
                let success_counter = Arc::clone(&self.success_counter);
                let metrics = Arc::clone(&self.metrics);

                join_set.spawn(async move {
                    Self::client_workload(
                        client_id,
                        Duration::from_millis(step_duration),
                        delay,
                        request_counter,
                        success_counter,
                        metrics,
                    )
                    .await
                });
            }

            tokio::time::sleep(Duration::from_millis(step_duration)).await;
        }

        // Wait for completion
        while let Some(result) = join_set.join_next().await {
            if let Err(e) = result {
                error!("Ramp up client failed: {:?}", e);
            }
        }

        monitoring_handle.abort();

        info!("Ramp up test complete");
        Ok(())
    }

    /// Run plateau test: sustained load at target RPS
    async fn run_plateau(&self, target_rps: usize, duration: Duration) -> Result<()> {
        info!(
            "Running plateau test: {} req/sec for {:?}",
            target_rps, duration
        );

        let mut join_set = JoinSet::new();
        let monitoring_handle = self.spawn_monitoring_task();

        let requests_per_client = target_rps / self.max_clients;
        let delay = if requests_per_client > 0 {
            Duration::from_millis(1000 / (requests_per_client as u64))
        } else {
            Duration::from_millis(1000)
        };

        for client_id in 0..self.max_clients {
            let request_counter = Arc::clone(&self.request_counter);
            let success_counter = Arc::clone(&self.success_counter);
            let metrics = Arc::clone(&self.metrics);

            join_set.spawn(async move {
                Self::client_workload(
                    client_id,
                    duration,
                    delay,
                    request_counter,
                    success_counter,
                    metrics,
                )
                .await
            });
        }

        // Wait for completion
        while let Some(result) = join_set.join_next().await {
            if let Err(e) = result {
                error!("Plateau client failed: {:?}", e);
            }
        }

        monitoring_handle.abort();

        info!("Plateau test complete");
        Ok(())
    }

    /// Run ramp down test: gradual decrease in load
    async fn run_ramp_down(&self, start_rps: usize, ramp_duration: Duration) -> Result<()> {
        info!(
            "Running ramp down test: {} req/sec → 0 over {:?}",
            start_rps, ramp_duration
        );

        let mut join_set = JoinSet::new();
        let monitoring_handle = self.spawn_monitoring_task();

        let ramp_steps: u64 = 100;
        let step_duration = ramp_duration.as_millis() as u64 / ramp_steps;

        // Start with all clients active
        for step in 0..ramp_steps {
            let current_rps = start_rps * (ramp_steps - step) as usize / ramp_steps as usize;
            if current_rps == 0 {
                break;
            }

            let active_clients =
                self.max_clients * (ramp_steps - step) as usize / ramp_steps as usize;

            for client_id in 0..active_clients.max(1) / ramp_steps as usize {
                let delay = if current_rps > 0 {
                    Duration::from_millis(1000 / (current_rps as u64))
                } else {
                    Duration::from_millis(1000)
                };
                let request_counter = Arc::clone(&self.request_counter);
                let success_counter = Arc::clone(&self.success_counter);
                let metrics = Arc::clone(&self.metrics);

                join_set.spawn(async move {
                    Self::client_workload(
                        client_id,
                        Duration::from_millis(step_duration),
                        delay,
                        request_counter,
                        success_counter,
                        metrics,
                    )
                    .await
                });
            }

            tokio::time::sleep(Duration::from_millis(step_duration)).await;
        }

        // Wait for completion
        while let Some(result) = join_set.join_next().await {
            if let Err(e) = result {
                error!("Ramp down client failed: {:?}", e);
            }
        }

        monitoring_handle.abort();

        info!("Ramp down test complete");
        Ok(())
    }

    /// Client workload
    async fn client_workload(
        client_id: usize,
        duration: Duration,
        delay: Duration,
        request_counter: Arc<AtomicU64>,
        success_counter: Arc<AtomicU64>,
        metrics: Arc<parking_lot::Mutex<BurstLoadMetrics>>,
    ) -> Result<()> {
        let start = Instant::now();

        while start.elapsed() < duration {
            let req_start = Instant::now();

            // Simulate request
            let success = Self::simulate_request(client_id).await;
            let latency = req_start.elapsed();

            // Update counters
            request_counter.fetch_add(1, Ordering::Relaxed);
            if success {
                success_counter.fetch_add(1, Ordering::Relaxed);
            }

            // Record latency
            {
                let mut m = metrics.lock();
                let _ = m.latency_histogram.record(latency.as_micros() as u64);
            }

            tokio::time::sleep(delay).await;
        }

        Ok(())
    }

    /// Simulate request (replace with actual gRPC call)
    async fn simulate_request(_client_id: usize) -> bool {
        tokio::time::sleep(Duration::from_micros(50 + rand::random::<u64>() % 450)).await;
        rand::random::<f64>() < 0.999
    }

    /// Spawn monitoring task
    fn spawn_monitoring_task(&self) -> tokio::task::JoinHandle<()> {
        let metrics = Arc::clone(&self.metrics);
        let request_counter = Arc::clone(&self.request_counter);

        tokio::spawn(async move {
            let start = Instant::now();
            let mut interval = tokio::time::interval(Duration::from_secs(1));
            let mut last_count = 0u64;

            loop {
                interval.tick().await;

                let current_count = request_counter.load(Ordering::Relaxed);
                let throughput = (current_count - last_count) as f64;
                last_count = current_count;

                let elapsed = start.elapsed();

                {
                    let mut m = metrics.lock();
                    m.throughput_samples.push((elapsed, throughput));
                    if throughput > m.peak_throughput {
                        m.peak_throughput = throughput;
                    }
                }
            }
        })
    }

    /// Get current metrics
    pub fn get_metrics(&self) -> BurstLoadMetrics {
        self.metrics.lock().clone()
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn test_spike_load() {
        let profile = LoadProfile::Spike {
            target_rps: 10_000,
            duration: Duration::from_secs(5),
        };

        let test = BurstLoadTest::new(profile, 100);
        let metrics = test.run().await.expect("Test failed");

        assert!(metrics.total_requests > 0);
        assert!(metrics.success_rate() > 99.0);
        // Peak throughput should be reasonable for simulated workload
        assert!(
            metrics.peak_throughput > 5000.0,
            "Peak throughput should be > 5000/sec (got: {})",
            metrics.peak_throughput
        );
    }

    #[tokio::test]
    async fn test_ramp_up() {
        let profile = LoadProfile::RampUp {
            target_rps: 5_000,
            ramp_duration: Duration::from_secs(10),
        };

        let test = BurstLoadTest::new(profile, 50);
        let metrics = test.run().await.expect("Test failed");

        assert!(metrics.total_requests > 0);
        assert!(metrics.success_rate() > 99.0);
    }

    #[tokio::test]
    #[ignore = "Long running - run manually"]
    async fn test_full_burst_scenario() {
        let _ = tracing_subscriber::fmt::try_init();

        info!("Running full burst scenario");

        // Phase 1: Spike to 100K req/sec for 1 second
        let spike = BurstLoadTest::new(
            LoadProfile::Spike {
                target_rps: 100_000,
                duration: Duration::from_secs(1),
            },
            1000,
        );
        let spike_metrics = spike.run().await.expect("Spike failed");
        info!("Spike phase: {:?}", spike_metrics);

        // Phase 2: Ramp up 0 → 50K over 10 minutes
        let ramp_up = BurstLoadTest::new(
            LoadProfile::RampUp {
                target_rps: 50_000,
                ramp_duration: Duration::from_secs(600),
            },
            500,
        );
        let ramp_up_metrics = ramp_up.run().await.expect("Ramp up failed");
        info!("Ramp up phase: {:?}", ramp_up_metrics);

        // Phase 3: Plateau at 50K for 1 hour
        let plateau = BurstLoadTest::new(
            LoadProfile::Plateau {
                target_rps: 50_000,
                duration: Duration::from_secs(3600),
            },
            500,
        );
        let plateau_metrics = plateau.run().await.expect("Plateau failed");
        info!("Plateau phase: {:?}", plateau_metrics);

        // Phase 4: Ramp down 50K → 0 over 10 minutes
        let ramp_down = BurstLoadTest::new(
            LoadProfile::RampDown {
                start_rps: 50_000,
                ramp_duration: Duration::from_secs(600),
            },
            500,
        );
        let ramp_down_metrics = ramp_down.run().await.expect("Ramp down failed");
        info!("Ramp down phase: {:?}", ramp_down_metrics);

        // All phases should have high success rates
        assert!(spike_metrics.success_rate() > 95.0);
        assert!(ramp_up_metrics.success_rate() > 99.0);
        assert!(plateau_metrics.success_rate() > 99.0);
        assert!(ramp_down_metrics.success_rate() > 99.0);
    }
}