foxhunt/services/load_tests/tests/saturation_point_tests.rs

//! Saturation Point Tests
//!
//! This module finds system capacity limits by gradually increasing load
//! until saturation points are reached:
//! - Throughput saturation: Max orders/sec before errors spike
//! - Latency degradation: Load point where P99 latency exceeds SLA
//! - Connection saturation: Max concurrent connections
//! - CPU saturation: Load point where CPU hits 90%
//! - Memory saturation: Load point where memory hits 90%
//! - Database saturation: Max queries/sec before latency spikes
//! - Network saturation: Bandwidth limit
//! - Queue saturation: Point where event queues back up
//!
//! Run with: cargo test -p load_tests --test saturation_point_tests --release -- --nocapture --ignored

use anyhow::Result;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};
use sysinfo::{CpuRefreshKind, RefreshKind, System};
use tokio::task::JoinSet;
use tokio::time::sleep;

// Internal dependencies
use trading_service_load_tests::clients::TradingClient;
use trading_service_load_tests::metrics::{LoadTestMetrics, LoadTestReport};

const TRADING_SERVICE_URL: &str = "http://localhost:50051";

/// Result of a saturation test iteration
#[derive(Debug, Clone)]
struct SaturationResult {
    rps: usize,
    error_rate: f64,
    p99_latency_us: u64,
    throughput: f64,
    avg_cpu_percent: f64,
    avg_memory_mb: f64,
}

impl SaturationResult {
    fn from_report(report: &LoadTestReport, rps: usize, cpu: f64) -> Self {
        Self {
            rps,
            error_rate: report.error_rate_percent,
            p99_latency_us: report.latency_p99_us,
            throughput: report.throughput_per_sec,
            avg_cpu_percent: cpu,
            avg_memory_mb: report.avg_memory_mb,
        }
    }

    fn print(&self, label: &str) {
        println!(
            "  {}: {} rps, {:.2}% errors, {} μs P99, {:.2}% CPU, {:.2} MB mem",
            label,
            self.rps,
            self.error_rate,
            self.p99_latency_us,
            self.avg_cpu_percent,
            self.avg_memory_mb
        );
    }
}

/// Run load test at specific RPS for duration
async fn run_load_at_rps(rps: usize, duration: Duration) -> Result<(LoadTestReport, f64)> {
    let metrics = Arc::new(LoadTestMetrics::new());
    let cpu_samples = Arc::new(parking_lot::Mutex::new(Vec::new()));

    // Calculate concurrent clients needed (each client sends ~100 rps)
    let concurrent_clients = (rps / 100).max(1).min(1000);

    let mut join_set = JoinSet::new();

    // Spawn clients
    for client_id in 0..concurrent_clients {
        let metrics = Arc::clone(&metrics);

        join_set.spawn(async move {
            let mut client = TradingClient::connect(TRADING_SERVICE_URL).await?;
            let start = Instant::now();

            while start.elapsed() < duration {
                match client.submit_test_order(client_id, 0).await {
                    Ok(latency) => metrics.record_request(latency, true),
                    Err(_) => metrics.record_request(Duration::from_micros(0), false),
                }

                // Rate limiting per client
                sleep(Duration::from_micros(10_000)).await;
            }

            Ok::<_, anyhow::Error>(())
        });
    }

    // CPU monitoring
    let cpu_samples_clone = Arc::clone(&cpu_samples);
    let monitor = tokio::spawn(async move {
        let mut sys = System::new_with_specifics(
            RefreshKind::nothing().with_cpu(CpuRefreshKind::everything()),
        );

        let sample_count = (duration.as_secs() / 2).max(1);
        for _ in 0..sample_count {
            sleep(Duration::from_secs(2)).await;
            sys.refresh_cpu_all();

            let global_cpu = sys.global_cpu_usage();
            cpu_samples_clone.lock().push(global_cpu);
        }
    });

    // Wait for completion
    while let Some(result) = join_set.join_next().await {
        if let Err(e) = result {
            tracing::error!("Client task failed: {:?}", e);
        }
    }

    monitor.abort();

    let report = metrics.to_report("Saturation Test");

    let cpu_samples_locked = cpu_samples.lock();
    let avg_cpu = if !cpu_samples_locked.is_empty() {
        cpu_samples_locked.iter().sum::<f32>() / cpu_samples_locked.len() as f32
    } else {
        0.0
    };

    Ok((report, avg_cpu as f64))
}

// =============================================================================
// THROUGHPUT SATURATION TESTS
// =============================================================================

#[tokio::test]
#[ignore = "Run explicitly with --ignored"]
async fn test_find_throughput_saturation_point() -> Result<()> {
    println!("\n🚀 Finding Throughput Saturation Point");
    println!("   Testing: 100 rps → 10,000 rps in 100 rps increments");

    let mut saturation_point = 0;
    let mut results = Vec::new();

    // Ramp up from 100 to 10,000 rps
    for rps in (100..=10_000).step_by(100) {
        println!("\n  Testing {} rps...", rps);

        let (report, cpu) = run_load_at_rps(rps, Duration::from_secs(30)).await?;
        let result = SaturationResult::from_report(&report, rps, cpu);
        result.print("Result");

        results.push(result.clone());

        // Check saturation conditions
        if result.error_rate > 1.0 {
            println!("\n  ❌ Error rate exceeded 1% threshold");
            saturation_point = rps - 100;
            break;
        }

        if result.p99_latency_us > 100_000 {
            println!("\n  ❌ P99 latency exceeded 100ms SLA");
            saturation_point = rps;
            break;
        }

        // Cool down between tests
        sleep(Duration::from_secs(2)).await;
    }

    // Print summary
    println!("\n📊 Throughput Saturation Summary:");
    println!("   Saturation point: {} rps", saturation_point);
    println!("   Tests conducted: {}", results.len());

    if !results.is_empty() {
        let peak = results.last().expect("INVARIANT: Collection should be non-empty");
        println!("\n   Peak performance:");
        peak.print("  ");
    }

    // Assertions
    assert!(
        saturation_point > 1000,
        "System should handle at least 1,000 rps (got {})",
        saturation_point
    );

    Ok(())
}

#[tokio::test]
#[ignore = "Long-running test - run with --ignored"]
async fn test_throughput_ramp_up_curve() -> Result<()> {
    println!("\n🚀 Generating Throughput Ramp-Up Curve");
    println!("   Testing: 100, 500, 1000, 2500, 5000, 7500, 10000 rps");

    let test_points = vec![100, 500, 1000, 2500, 5000, 7500, 10_000];
    let mut results = Vec::new();

    for rps in test_points {
        println!("\n  Testing {} rps...", rps);

        let (report, cpu) = run_load_at_rps(rps, Duration::from_secs(30)).await?;
        let result = SaturationResult::from_report(&report, rps, cpu);
        result.print("Result");

        results.push(result);

        // Cool down
        sleep(Duration::from_secs(2)).await;
    }

    // Analyze curve
    println!("\n📊 Throughput Ramp-Up Analysis:");
    println!(
        "   {:<10} {:<12} {:<15} {:<15}",
        "RPS", "Error Rate", "P99 Latency", "CPU Usage"
    );
    println!("   {}", "=".repeat(60));

    for result in &results {
        println!(
            "   {:<10} {:<12.2}% {:<15} μs {:<15.2}%",
            result.rps, result.error_rate, result.p99_latency_us, result.avg_cpu_percent
        );
    }

    // Verify throughput increases monotonically (until saturation)
    for i in 1..results.len() {
        if results[i].error_rate < 1.0 && results[i - 1].error_rate < 1.0 {
            assert!(
                results[i].throughput >= results[i - 1].throughput * 0.8,
                "Throughput should increase with load (got {:.2} -> {:.2})",
                results[i - 1].throughput,
                results[i].throughput
            );
        }
    }

    Ok(())
}

// =============================================================================
// LATENCY DEGRADATION TESTS
// =============================================================================

#[tokio::test]
#[ignore = "Long-running test - run with --ignored"]
async fn test_latency_degradation_curve() -> Result<()> {
    println!("\n🚀 Finding Latency Degradation Point");
    println!("   Testing: Finding where P99 latency exceeds 100ms");

    let mut results = Vec::new();

    // Ramp up until latency degrades
    for rps in (100..=5000).step_by(100) {
        println!("\n  Testing {} rps...", rps);

        let (report, cpu) = run_load_at_rps(rps, Duration::from_secs(20)).await?;
        let result = SaturationResult::from_report(&report, rps, cpu);
        result.print("Result");

        results.push(result.clone());

        // Stop if latency exceeds 500ms (well beyond SLA)
        if result.p99_latency_us > 500_000 {
            println!("\n  ❌ P99 latency exceeded 500ms - stopping test");
            break;
        }

        sleep(Duration::from_secs(2)).await;
    }

    // Analyze degradation
    println!("\n📊 Latency Degradation Analysis:");
    println!(
        "   {:<10} {:<15} {:<15} {:<15}",
        "RPS", "P50 (μs)", "P95 (μs)", "P99 (μs)"
    );
    println!("   {}", "=".repeat(60));

    let mut sla_violation_rps = None;
    for result in &results {
        println!(
            "   {:<10} {:<15} {:<15} {:<15}",
            result.rps,
            "-", // P50 not in SaturationResult
            "-", // P95 not in SaturationResult
            result.p99_latency_us
        );

        if sla_violation_rps.is_none() && result.p99_latency_us > 100_000 {
            sla_violation_rps = Some(result.rps);
        }
    }

    if let Some(violation_rps) = sla_violation_rps {
        println!(
            "\n   ⚠️  P99 latency exceeds 100ms SLA at {} rps",
            violation_rps
        );
    } else {
        println!("\n   ✅ P99 latency within SLA for all tested loads");
    }

    // Verify latency doesn't decrease significantly with load
    for i in 1..results.len() {
        let prev_latency = results[i - 1].p99_latency_us as f64;
        let curr_latency = results[i].p99_latency_us as f64;
        assert!(
            curr_latency >= prev_latency * 0.5,
            "Latency should not decrease significantly with load"
        );
    }

    Ok(())
}

#[tokio::test]
#[ignore = "Long-running test - run with --ignored"]
async fn test_p50_p95_p99_spread_under_load() -> Result<()> {
    println!("\n🚀 Testing Latency Distribution Spread Under Load");
    println!("   Testing: Latency spread at 1000, 5000, 10000 rps");

    let test_loads = vec![1000, 5000, 10_000];

    for rps in test_loads {
        println!("\n  Testing {} rps...", rps);

        let metrics = Arc::new(LoadTestMetrics::new());
        let concurrent_clients = (rps / 100).max(1);

        let mut join_set = JoinSet::new();

        for client_id in 0..concurrent_clients {
            let metrics = Arc::clone(&metrics);

            join_set.spawn(async move {
                let mut client = TradingClient::connect(TRADING_SERVICE_URL).await?;
                let start = Instant::now();

                while start.elapsed() < Duration::from_secs(30) {
                    match client.submit_test_order(client_id, 0).await {
                        Ok(latency) => metrics.record_request(latency, true),
                        Err(_) => metrics.record_request(Duration::from_micros(0), false),
                    }

                    sleep(Duration::from_micros(10_000)).await;
                }

                Ok::<_, anyhow::Error>(())
            });
        }

        while let Some(result) = join_set.join_next().await {
            if let Err(e) = result {
                tracing::error!("Client task failed: {:?}", e);
            }
        }

        let report = metrics.to_report("Latency Spread Test");

        println!("  Results at {} rps:", rps);
        println!("    P50: {} μs", report.latency_p50_us);
        println!("    P95: {} μs", report.latency_p95_us);
        println!("    P99: {} μs", report.latency_p99_us);
        println!("    Max: {} μs", report.latency_max_us);

        let p99_p50_ratio = report.latency_p99_us as f64 / report.latency_p50_us.max(1) as f64;
        println!("    P99/P50 ratio: {:.2}x", p99_p50_ratio);

        // Under healthy load, P99 shouldn't be more than 10x P50
        assert!(
            p99_p50_ratio < 10.0,
            "P99/P50 ratio too high: {:.2}x (suggests tail latency issues)",
            p99_p50_ratio
        );

        sleep(Duration::from_secs(2)).await;
    }

    Ok(())
}

// =============================================================================
// CONNECTION SATURATION TESTS
// =============================================================================

#[tokio::test]
#[ignore = "Long-running test - run with --ignored"]
async fn test_find_connection_saturation_point() -> Result<()> {
    println!("\n🚀 Finding Connection Saturation Point");
    println!("   Testing: 50 → 1000 concurrent connections");

    let mut max_connections = 0;

    for num_connections in (50..=1000).step_by(50) {
        println!("\n  Testing {} concurrent connections...", num_connections);

        let metrics = Arc::new(LoadTestMetrics::new());
        let connection_count = Arc::new(AtomicUsize::new(0));
        let failure_count = Arc::new(AtomicUsize::new(0));

        let mut join_set = JoinSet::new();

        for client_id in 0..num_connections {
            let metrics = Arc::clone(&metrics);
            let connection_count = Arc::clone(&connection_count);
            let failure_count = Arc::clone(&failure_count);

            join_set.spawn(async move {
                let connect_start = Instant::now();
                match TradingClient::connect(TRADING_SERVICE_URL).await {
                    Ok(mut client) => {
                        connection_count.fetch_add(1, Ordering::Relaxed);
                        metrics.record_request(connect_start.elapsed(), true);

                        // Submit a few orders
                        for i in 0..10 {
                            match client.submit_test_order(client_id, i).await {
                                Ok(latency) => metrics.record_request(latency, true),
                                Err(_) => {
                                    failure_count.fetch_add(1, Ordering::Relaxed);
                                    metrics.record_request(Duration::from_micros(0), false);
                                },
                            }
                            sleep(Duration::from_millis(10)).await;
                        }
                    },
                    Err(_) => {
                        failure_count.fetch_add(1, Ordering::Relaxed);
                        metrics.record_request(connect_start.elapsed(), false);
                    },
                }

                Ok::<_, anyhow::Error>(())
            });
        }

        while let Some(result) = join_set.join_next().await {
            if let Err(e) = result {
                tracing::error!("Connection task failed: {:?}", e);
            }
        }

        let successful_connections = connection_count.load(Ordering::Relaxed);
        let failures = failure_count.load(Ordering::Relaxed);
        let failure_rate = (failures as f64 / (num_connections * 10) as f64) * 100.0;

        println!(
            "  Result: {}/{} connections successful, {:.2}% failure rate",
            successful_connections, num_connections, failure_rate
        );

        if failure_rate > 5.0 {
            println!("\n  ❌ Failure rate exceeded 5% - connection saturation reached");
            max_connections = num_connections - 50;
            break;
        }

        max_connections = num_connections;
        sleep(Duration::from_secs(2)).await;
    }

    println!("\n📊 Connection Saturation Summary:");
    println!("   Max sustainable connections: {}", max_connections);

    assert!(
        max_connections >= 200,
        "Should handle at least 200 concurrent connections (got {})",
        max_connections
    );

    Ok(())
}

#[tokio::test]
#[ignore = "Long-running test - run with --ignored"]
async fn test_connection_timeout_under_saturation() -> Result<()> {
    println!("\n🚀 Testing Connection Timeouts Under Saturation");
    println!("   Testing: 500 connections with aggressive timeout");

    const NUM_CONNECTIONS: usize = 500;
    let timeout_count = Arc::new(AtomicUsize::new(0));
    let success_count = Arc::new(AtomicUsize::new(0));

    let mut join_set = JoinSet::new();

    for _client_id in 0..NUM_CONNECTIONS {
        let timeout_count = Arc::clone(&timeout_count);
        let success_count = Arc::clone(&success_count);

        join_set.spawn(async move {
            let result = tokio::time::timeout(
                Duration::from_secs(5),
                TradingClient::connect(TRADING_SERVICE_URL),
            )
            .await;

            match result {
                Ok(Ok(_client)) => {
                    success_count.fetch_add(1, Ordering::Relaxed);
                },
                Ok(Err(_)) | Err(_) => {
                    timeout_count.fetch_add(1, Ordering::Relaxed);
                },
            }

            Ok::<_, anyhow::Error>(())
        });
    }

    while let Some(result) = join_set.join_next().await {
        if let Err(e) = result {
            tracing::error!("Timeout test task failed: {:?}", e);
        }
    }

    let timeouts = timeout_count.load(Ordering::Relaxed);
    let successes = success_count.load(Ordering::Relaxed);
    let timeout_rate = (timeouts as f64 / NUM_CONNECTIONS as f64) * 100.0;

    println!("\n📊 Connection Timeout Results:");
    println!("   Successful: {}/{}", successes, NUM_CONNECTIONS);
    println!(
        "   Timeouts: {}/{} ({:.2}%)",
        timeouts, NUM_CONNECTIONS, timeout_rate
    );

    assert!(
        timeout_rate < 10.0,
        "Timeout rate too high: {:.2}% (expected < 10%)",
        timeout_rate
    );

    Ok(())
}

// =============================================================================
// CPU SATURATION TESTS
// =============================================================================

#[tokio::test]
#[ignore = "Long-running test - run with --ignored"]
async fn test_find_cpu_saturation_point() -> Result<()> {
    println!("\n🚀 Finding CPU Saturation Point");
    println!("   Testing: Finding load where CPU hits 90%");

    let mut cpu_saturation_rps = 0;

    for rps in (100..=10_000).step_by(200) {
        println!("\n  Testing {} rps...", rps);

        let (report, avg_cpu) = run_load_at_rps(rps, Duration::from_secs(30)).await?;
        let result = SaturationResult::from_report(&report, rps, avg_cpu);
        result.print("Result");

        if avg_cpu > 90.0 {
            println!("\n  ❌ CPU usage exceeded 90% threshold");
            cpu_saturation_rps = rps;
            break;
        }

        sleep(Duration::from_secs(2)).await;
    }

    println!("\n📊 CPU Saturation Summary:");
    if cpu_saturation_rps > 0 {
        println!("   CPU saturation at: {} rps", cpu_saturation_rps);
    } else {
        println!("   CPU saturation not reached (< 90% at all tested loads)");
    }

    Ok(())
}

// =============================================================================
// MEMORY SATURATION TESTS
// =============================================================================

#[tokio::test]
#[ignore = "Long-running test - run with --ignored"]
async fn test_memory_growth_under_load() -> Result<()> {
    println!("\n🚀 Testing Memory Growth Under Load");
    println!("   Testing: Memory usage at 1000, 5000, 10000 rps");

    let test_loads = vec![1000, 5000, 10_000];

    for rps in test_loads {
        println!("\n  Testing {} rps...", rps);

        let (report, cpu) = run_load_at_rps(rps, Duration::from_secs(30)).await?;
        let result = SaturationResult::from_report(&report, rps, cpu);
        result.print("Result");

        // Memory should stay reasonable (< 2GB for load tests)
        assert!(
            result.avg_memory_mb < 2048.0,
            "Memory usage too high: {:.2} MB at {} rps",
            result.avg_memory_mb,
            rps
        );

        sleep(Duration::from_secs(2)).await;
    }

    Ok(())
}

// =============================================================================
// QUEUE SATURATION TESTS
// =============================================================================

#[tokio::test]
#[ignore = "Long-running test - run with --ignored"]
async fn test_queue_backpressure_detection() -> Result<()> {
    println!("\n🚀 Testing Queue Backpressure Detection");
    println!("   Testing: Sudden burst to detect queue backup");

    // Send burst of orders
    let metrics = Arc::new(LoadTestMetrics::new());
    let latency_samples = Arc::new(parking_lot::Mutex::new(Vec::new()));

    let mut join_set = JoinSet::new();

    // Send burst from 100 clients simultaneously
    for client_id in 0..100 {
        let metrics = Arc::clone(&metrics);
        let latency_samples = Arc::clone(&latency_samples);

        join_set.spawn(async move {
            let mut client = TradingClient::connect(TRADING_SERVICE_URL).await?;

            for order_id in 0..100 {
                match client.submit_test_order(client_id, order_id).await {
                    Ok(latency) => {
                        metrics.record_request(latency, true);
                        latency_samples
                            .lock()
                            .push((order_id, latency.as_micros() as u64));
                    },
                    Err(_) => {
                        metrics.record_request(Duration::from_micros(0), false);
                    },
                }

                // Minimal delay for burst
                sleep(Duration::from_micros(100)).await;
            }

            Ok::<_, anyhow::Error>(())
        });
    }

    while let Some(result) = join_set.join_next().await {
        if let Err(e) = result {
            tracing::error!("Burst task failed: {:?}", e);
        }
    }

    let report = metrics.to_report("Queue Backpressure Test");

    println!("\n📊 Queue Backpressure Results:");
    println!("   Total requests: {}", report.total_requests);
    println!("   Error rate: {:.2}%", report.error_rate_percent);
    println!("   P99 latency: {} μs", report.latency_p99_us);
    println!("   Max latency: {} μs", report.latency_max_us);

    // Analyze latency progression (queue backup shows increasing latency)
    let samples = latency_samples.lock();
    if samples.len() > 10 {
        let first_10_avg = samples[..10].iter().map(|(_, lat)| lat).sum::<u64>() / 10;
        let last_10_avg = samples[samples.len() - 10..]
            .iter()
            .map(|(_, lat)| lat)
            .sum::<u64>()
            / 10;

        println!("\n   First 10 orders avg latency: {} μs", first_10_avg);
        println!("   Last 10 orders avg latency: {} μs", last_10_avg);

        if last_10_avg > first_10_avg * 2 {
            println!("   ⚠️  Queue backpressure detected (latency doubled)");
        }
    }

    Ok(())
}