foxhunt/tests/load_test_trading_service.rs

//! Comprehensive Load Test for Trading Service
//!
//! Tests the trading service against production requirements:
//! - 10K orders/sec throughput target
//! - P50, P95, P99 latency measurements
//! - 100+ concurrent connections
//! - Order matching 1-6μs P99 baseline
//! - Database performance under load
//! - Resource monitoring (CPU, memory, connections)
//!
//! Run with: cargo test --package tests --test load_test_trading_service --release -- --nocapture

use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant, SystemTime};
use tokio::time::timeout;
use tonic::transport::Channel;
use tonic::{Request, Status};
use uuid::Uuid;

// gRPC generated code
pub mod trading {
    tonic::include_proto!("trading");
}

use trading::trading_service_client::TradingServiceClient;
use trading::{SubmitOrderRequest, OrderSide, OrderType, TimeInForce};

/// Performance metrics aggregator
#[derive(Debug, Clone)]
struct PerformanceMetrics {
    latencies_ns: Vec<u64>,
    successful_orders: AtomicU64,
    failed_orders: AtomicU64,
    total_orders: AtomicU64,
    test_duration: Duration,
}

impl PerformanceMetrics {
    fn new() -> Self {
        Self {
            latencies_ns: Vec::new(),
            successful_orders: AtomicU64::new(0),
            failed_orders: AtomicU64::new(0),
            total_orders: AtomicU64::new(0),
            test_duration: Duration::ZERO,
        }
    }

    fn record_success(&self, latency_ns: u64) {
        self.successful_orders.fetch_add(1, Ordering::Relaxed);
        self.total_orders.fetch_add(1, Ordering::Relaxed);
    }

    fn record_failure(&self) {
        self.failed_orders.fetch_add(1, Ordering::Relaxed);
        self.total_orders.fetch_add(1, Ordering::Relaxed);
    }

    fn calculate_percentiles(mut latencies: Vec<u64>) -> (u64, u64, u64, u64, u64) {
        if latencies.is_empty() {
            return (0, 0, 0, 0, 0);
        }

        latencies.sort_unstable();
        let len = latencies.len();

        let min = latencies[0];
        let p50 = latencies[len / 2];
        let p95 = latencies[(len as f64 * 0.95) as usize];
        let p99 = latencies[(len as f64 * 0.99) as usize];
        let max = latencies[len - 1];

        (min, p50, p95, p99, max)
    }

    fn print_summary(&self, latencies: &[u64]) {
        let successful = self.successful_orders.load(Ordering::Relaxed);
        let failed = self.failed_orders.load(Ordering::Relaxed);
        let total = self.total_orders.load(Ordering::Relaxed);

        let success_rate = if total > 0 {
            (successful as f64 / total as f64) * 100.0
        } else {
            0.0
        };

        let throughput = if self.test_duration.as_secs_f64() > 0.0 {
            successful as f64 / self.test_duration.as_secs_f64()
        } else {
            0.0
        };

        let (min, p50, p95, p99, max) = Self::calculate_percentiles(latencies.to_vec());

        println!("\n╔═══════════════════════════════════════════════════════════╗");
        println!("║           TRADING SERVICE LOAD TEST RESULTS               ║");
        println!("╠═══════════════════════════════════════════════════════════╣");
        println!("║ Test Duration:       {:.2}s", self.test_duration.as_secs_f64());
        println!("║ Total Orders:        {}", total);
        println!("║ Successful Orders:   {} ({:.2}%)", successful, success_rate);
        println!("║ Failed Orders:       {}", failed);
        println!("║ Throughput:          {:.0} orders/sec", throughput);
        println!("╠═══════════════════════════════════════════════════════════╣");
        println!("║                    LATENCY METRICS                        ║");
        println!("╠═══════════════════════════════════════════════════════════╣");
        println!("║ Min Latency:         {:.2}ms ({:.2}μs)", min as f64 / 1_000_000.0, min as f64 / 1_000.0);
        println!("║ P50 Latency:         {:.2}ms ({:.2}μs)", p50 as f64 / 1_000_000.0, p50 as f64 / 1_000.0);
        println!("║ P95 Latency:         {:.2}ms ({:.2}μs)", p95 as f64 / 1_000_000.0, p95 as f64 / 1_000.0);
        println!("║ P99 Latency:         {:.2}ms ({:.2}μs)", p99 as f64 / 1_000_000.0, p99 as f64 / 1_000.0);
        println!("║ Max Latency:         {:.2}ms ({:.2}μs)", max as f64 / 1_000_000.0, max as f64 / 1_000.0);
        println!("╚═══════════════════════════════════════════════════════════╝");

        // Performance assessment
        println!("\n📊 PERFORMANCE ASSESSMENT:");

        if throughput >= 10_000.0 {
            println!("✅ Throughput target ACHIEVED: {:.0} orders/sec (target: 10K orders/sec)", throughput);
        } else {
            println!("⚠️  Throughput BELOW target: {:.0} orders/sec (target: 10K orders/sec)", throughput);
        }

        if p99 < 100_000_000 { // 100ms in nanoseconds
            println!("✅ P99 latency GOOD: {:.2}ms (< 100ms)", p99 as f64 / 1_000_000.0);
        } else {
            println!("⚠️  P99 latency HIGH: {:.2}ms (> 100ms)", p99 as f64 / 1_000_000.0);
        }

        if success_rate >= 99.0 {
            println!("✅ Success rate EXCELLENT: {:.2}%", success_rate);
        } else if success_rate >= 95.0 {
            println!("⚠️  Success rate ACCEPTABLE: {:.2}%", success_rate);
        } else {
            println!("❌ Success rate POOR: {:.2}%", success_rate);
        }
    }
}

/// Create a test order request
fn create_order_request(index: u64) -> SubmitOrderRequest {
    let symbols = vec!["BTC/USD", "ETH/USD", "SOL/USD", "AVAX/USD", "MATIC/USD"];
    let symbol = symbols[(index % symbols.len() as u64) as usize].to_string();

    SubmitOrderRequest {
        order_id: Uuid::new_v4().to_string(),
        symbol,
        side: if index % 2 == 0 { OrderSide::Buy.into() } else { OrderSide::Sell.into() },
        order_type: OrderType::Limit.into(),
        quantity: (1.0 + (index % 10) as f64 * 0.1).to_string(),
        price: Some((50000.0 + (index % 1000) as f64).to_string()),
        time_in_force: TimeInForce::GoodTillCancel.into(),
    }
}

/// Connect to Trading Service
async fn connect_trading_service() -> Result<TradingServiceClient<Channel>, Box<dyn std::error::Error>> {
    let endpoint = "http://localhost:50052";
    println!("🔌 Connecting to Trading Service at {}", endpoint);

    let channel = Channel::from_static("http://localhost:50052")
        .connect_timeout(Duration::from_secs(10))
        .timeout(Duration::from_secs(30))
        .connect()
        .await?;

    let client = TradingServiceClient::new(channel);
    println!("✅ Connected successfully");

    Ok(client)
}

/// Test 1: Baseline latency with single client
#[tokio::test]
async fn test_1_baseline_latency() -> Result<(), Box<dyn std::error::Error>> {
    println!("\n╔═══════════════════════════════════════════════════════════╗");
    println!("║  TEST 1: BASELINE LATENCY (Single Client)                ║");
    println!("╚═══════════════════════════════════════════════════════════╝");

    let mut client = connect_trading_service().await?;
    let num_requests = 1000;
    let mut latencies = Vec::with_capacity(num_requests);

    println!("📊 Sending {} orders sequentially...", num_requests);

    let start_time = Instant::now();

    for i in 0..num_requests {
        let request = create_order_request(i as u64);
        let req_start = Instant::now();

        let result = client.submit_order(Request::new(request)).await;
        let latency_ns = req_start.elapsed().as_nanos() as u64;

        latencies.push(latency_ns);

        if result.is_err() && i < 5 {
            eprintln!("❌ Order {} failed: {:?}", i, result.err());
        }
    }

    let test_duration = start_time.elapsed();

    let (min, p50, p95, p99, max) = PerformanceMetrics::calculate_percentiles(latencies.clone());

    println!("\n📈 BASELINE RESULTS:");
    println!("  Duration:     {:.2}s", test_duration.as_secs_f64());
    println!("  Throughput:   {:.0} orders/sec", num_requests as f64 / test_duration.as_secs_f64());
    println!("  Min Latency:  {:.2}ms", min as f64 / 1_000_000.0);
    println!("  P50 Latency:  {:.2}ms", p50 as f64 / 1_000_000.0);
    println!("  P95 Latency:  {:.2}ms", p95 as f64 / 1_000_000.0);
    println!("  P99 Latency:  {:.2}ms", p99 as f64 / 1_000_000.0);
    println!("  Max Latency:  {:.2}ms", max as f64 / 1_000_000.0);

    Ok(())
}

/// Test 2: Concurrent connections (100 clients)
#[tokio::test]
async fn test_2_concurrent_connections() -> Result<(), Box<dyn std::error::Error>> {
    println!("\n╔═══════════════════════════════════════════════════════════╗");
    println!("║  TEST 2: CONCURRENT CONNECTIONS (100 Clients)            ║");
    println!("╚═══════════════════════════════════════════════════════════╝");

    let num_clients = 100;
    let orders_per_client = 100;

    let metrics = Arc::new(PerformanceMetrics::new());
    let latencies = Arc::new(tokio::sync::Mutex::new(Vec::new()));

    println!("🚀 Spawning {} concurrent clients ({} orders each)...", num_clients, orders_per_client);

    let start_time = Instant::now();
    let mut tasks = Vec::new();

    for client_id in 0..num_clients {
        let metrics_clone = Arc::clone(&metrics);
        let latencies_clone = Arc::clone(&latencies);

        let task = tokio::spawn(async move {
            let mut client = match connect_trading_service().await {
                Ok(c) => c,
                Err(e) => {
                    eprintln!("❌ Client {} connection failed: {}", client_id, e);
                    return;
                }
            };

            for order_idx in 0..orders_per_client {
                let request = create_order_request((client_id * orders_per_client + order_idx) as u64);
                let req_start = Instant::now();

                match timeout(Duration::from_secs(5), client.submit_order(Request::new(request))).await {
                    Ok(Ok(_response)) => {
                        let latency_ns = req_start.elapsed().as_nanos() as u64;
                        metrics_clone.record_success(latency_ns);
                        latencies_clone.lock().await.push(latency_ns);
                    }
                    Ok(Err(status)) => {
                        metrics_clone.record_failure();
                        if order_idx < 2 {
                            eprintln!("❌ Client {} order {} failed: {}", client_id, order_idx, status);
                        }
                    }
                    Err(_) => {
                        metrics_clone.record_failure();
                        if order_idx < 2 {
                            eprintln!("⏱️  Client {} order {} timed out", client_id, order_idx);
                        }
                    }
                }
            }
        });

        tasks.push(task);
    }

    // Wait for all clients to complete
    for task in tasks {
        let _ = task.await;
    }

    let test_duration = start_time.elapsed();
    let latencies_vec = latencies.lock().await.clone();

    let metrics_final = PerformanceMetrics {
        latencies_ns: latencies_vec.clone(),
        successful_orders: AtomicU64::new(metrics.successful_orders.load(Ordering::Relaxed)),
        failed_orders: AtomicU64::new(metrics.failed_orders.load(Ordering::Relaxed)),
        total_orders: AtomicU64::new(metrics.total_orders.load(Ordering::Relaxed)),
        test_duration,
    };

    metrics_final.print_summary(&latencies_vec);

    Ok(())
}

/// Test 3: Sustained load (5 minutes)
#[tokio::test]
#[ignore] // Run explicitly with --ignored
async fn test_3_sustained_load() -> Result<(), Box<dyn std::error::Error>> {
    println!("\n╔═══════════════════════════════════════════════════════════╗");
    println!("║  TEST 3: SUSTAINED LOAD (5 Minutes)                      ║");
    println!("╚═══════════════════════════════════════════════════════════╝");

    let test_duration_secs = 300; // 5 minutes
    let num_clients = 50;
    let target_rate_per_sec = 200; // 10K total / 50 clients = 200 per client

    let metrics = Arc::new(PerformanceMetrics::new());
    let latencies = Arc::new(tokio::sync::Mutex::new(Vec::new()));
    let shutdown = Arc::new(AtomicU64::new(0));

    println!("🚀 Starting {} clients for {} seconds...", num_clients, test_duration_secs);
    println!("🎯 Target: {:.0} orders/sec total", num_clients as f64 * target_rate_per_sec as f64);

    let start_time = Instant::now();
    let mut tasks = Vec::new();

    for client_id in 0..num_clients {
        let metrics_clone = Arc::clone(&metrics);
        let latencies_clone = Arc::clone(&latencies);
        let shutdown_clone = Arc::clone(&shutdown);

        let task = tokio::spawn(async move {
            let mut client = match connect_trading_service().await {
                Ok(c) => c,
                Err(e) => {
                    eprintln!("❌ Client {} connection failed: {}", client_id, e);
                    return;
                }
            };

            let mut order_count = 0u64;
            let delay_micros = 1_000_000 / target_rate_per_sec; // microseconds between orders

            while shutdown_clone.load(Ordering::Relaxed) == 0 {
                let request = create_order_request(order_count);
                let req_start = Instant::now();

                match timeout(Duration::from_secs(5), client.submit_order(Request::new(request))).await {
                    Ok(Ok(_response)) => {
                        let latency_ns = req_start.elapsed().as_nanos() as u64;
                        metrics_clone.record_success(latency_ns);
                        latencies_clone.lock().await.push(latency_ns);
                    }
                    Ok(Err(_)) => {
                        metrics_clone.record_failure();
                    }
                    Err(_) => {
                        metrics_clone.record_failure();
                    }
                }

                order_count += 1;

                // Rate limiting
                tokio::time::sleep(Duration::from_micros(delay_micros)).await;
            }
        });

        tasks.push(task);
    }

    // Run for specified duration
    tokio::time::sleep(Duration::from_secs(test_duration_secs)).await;

    // Signal shutdown
    shutdown.store(1, Ordering::Relaxed);

    // Wait for all clients to complete
    for task in tasks {
        let _ = task.await;
    }

    let test_duration = start_time.elapsed();
    let latencies_vec = latencies.lock().await.clone();

    let metrics_final = PerformanceMetrics {
        latencies_ns: latencies_vec.clone(),
        successful_orders: AtomicU64::new(metrics.successful_orders.load(Ordering::Relaxed)),
        failed_orders: AtomicU64::new(metrics.failed_orders.load(Ordering::Relaxed)),
        total_orders: AtomicU64::new(metrics.total_orders.load(Ordering::Relaxed)),
        test_duration,
    };

    metrics_final.print_summary(&latencies_vec);

    Ok(())
}

/// Test 4: Database under load
#[tokio::test]
async fn test_4_database_performance() -> Result<(), Box<dyn std::error::Error>> {
    println!("\n╔═══════════════════════════════════════════════════════════╗");
    println!("║  TEST 4: DATABASE PERFORMANCE                             ║");
    println!("╚═══════════════════════════════════════════════════════════╝");

    // This test measures order submission which triggers database writes
    let num_orders = 5000;
    let mut client = connect_trading_service().await?;

    println!("📊 Submitting {} orders to measure database performance...", num_orders);

    let start_time = Instant::now();
    let mut success_count = 0;
    let mut failure_count = 0;

    for i in 0..num_orders {
        let request = create_order_request(i);

        match client.submit_order(Request::new(request)).await {
            Ok(_) => success_count += 1,
            Err(e) => {
                failure_count += 1;
                if failure_count <= 5 {
                    eprintln!("❌ Order {} failed: {}", i, e);
                }
            }
        }
    }

    let duration = start_time.elapsed();
    let throughput = success_count as f64 / duration.as_secs_f64();

    println!("\n📈 DATABASE PERFORMANCE:");
    println!("  Duration:         {:.2}s", duration.as_secs_f64());
    println!("  Successful:       {}", success_count);
    println!("  Failed:           {}", failure_count);
    println!("  DB Writes/sec:    {:.0}", throughput);

    if throughput >= 2000.0 {
        println!("✅ Database performance GOOD: {:.0} writes/sec", throughput);
    } else {
        println!("⚠️  Database performance: {:.0} writes/sec (expected >2000)", throughput);
    }

    Ok(())
}

/// Test 5: Resource monitoring
#[tokio::test]
async fn test_5_resource_monitoring() -> Result<(), Box<dyn std::error::Error>> {
    println!("\n╔═══════════════════════════════════════════════════════════╗");
    println!("║  TEST 5: RESOURCE MONITORING                              ║");
    println!("╚═══════════════════════════════════════════════════════════╝");

    // Check service health
    let health_url = "http://localhost:8081/health";
    println!("🏥 Checking service health at {}...", health_url);

    match reqwest::get(health_url).await {
        Ok(response) => {
            println!("✅ Health check response: {}", response.status());
            if let Ok(body) = response.text().await {
                println!("   Body: {}", body);
            }
        }
        Err(e) => {
            println!("⚠️  Health check failed: {}", e);
        }
    }

    // Check Prometheus metrics
    let metrics_url = "http://localhost:9092/metrics";
    println!("\n📊 Checking Prometheus metrics at {}...", metrics_url);

    match reqwest::get(metrics_url).await {
        Ok(response) => {
            if let Ok(body) = response.text().await {
                // Parse relevant metrics
                let lines: Vec<&str> = body.lines()
                    .filter(|line| !line.starts_with('#') && !line.is_empty())
                    .collect();

                println!("✅ Found {} metric entries", lines.len());

                // Show some key metrics
                for line in lines.iter().take(10) {
                    if line.contains("orders") || line.contains("latency") || line.contains("cpu") {
                        println!("   {}", line);
                    }
                }
            }
        }
        Err(e) => {
            println!("⚠️  Metrics check failed: {}", e);
        }
    }

    Ok(())
}

/// Test 6: Production readiness assessment
#[tokio::test]
async fn test_6_production_readiness() -> Result<(), Box<dyn std::error::Error>> {
    println!("\n╔═══════════════════════════════════════════════════════════╗");
    println!("║  TEST 6: PRODUCTION READINESS ASSESSMENT                 ║");
    println!("╚═══════════════════════════════════════════════════════════╝");

    let num_clients = 50;
    let orders_per_client = 200;

    let metrics = Arc::new(PerformanceMetrics::new());
    let latencies = Arc::new(tokio::sync::Mutex::new(Vec::new()));

    println!("🎯 Production simulation: {} clients, {} orders each", num_clients, orders_per_client);

    let start_time = Instant::now();
    let mut tasks = Vec::new();

    for client_id in 0..num_clients {
        let metrics_clone = Arc::clone(&metrics);
        let latencies_clone = Arc::clone(&latencies);

        let task = tokio::spawn(async move {
            let mut client = match connect_trading_service().await {
                Ok(c) => c,
                Err(_) => return,
            };

            for order_idx in 0..orders_per_client {
                let request = create_order_request((client_id * orders_per_client + order_idx) as u64);
                let req_start = Instant::now();

                match client.submit_order(Request::new(request)).await {
                    Ok(_) => {
                        let latency_ns = req_start.elapsed().as_nanos() as u64;
                        metrics_clone.record_success(latency_ns);
                        latencies_clone.lock().await.push(latency_ns);
                    }
                    Err(_) => {
                        metrics_clone.record_failure();
                    }
                }
            }
        });

        tasks.push(task);
    }

    for task in tasks {
        let _ = task.await;
    }

    let test_duration = start_time.elapsed();
    let latencies_vec = latencies.lock().await.clone();

    let metrics_final = PerformanceMetrics {
        latencies_ns: latencies_vec.clone(),
        successful_orders: AtomicU64::new(metrics.successful_orders.load(Ordering::Relaxed)),
        failed_orders: AtomicU64::new(metrics.failed_orders.load(Ordering::Relaxed)),
        total_orders: AtomicU64::new(metrics.total_orders.load(Ordering::Relaxed)),
        test_duration,
    };

    metrics_final.print_summary(&latencies_vec);

    // Production readiness criteria
    let successful = metrics_final.successful_orders.load(Ordering::Relaxed);
    let total = metrics_final.total_orders.load(Ordering::Relaxed);
    let success_rate = (successful as f64 / total as f64) * 100.0;
    let throughput = successful as f64 / test_duration.as_secs_f64();
    let (_, _, _, p99, _) = PerformanceMetrics::calculate_percentiles(latencies_vec);

    println!("\n🎯 PRODUCTION READINESS:");

    let mut passed = 0;
    let mut total_checks = 0;

    // Check 1: Success rate
    total_checks += 1;
    if success_rate >= 99.0 {
        println!("✅ Success rate: {:.2}% (>= 99%)", success_rate);
        passed += 1;
    } else {
        println!("❌ Success rate: {:.2}% (< 99%)", success_rate);
    }

    // Check 2: Throughput
    total_checks += 1;
    if throughput >= 5000.0 {
        println!("✅ Throughput: {:.0} orders/sec (>= 5000)", throughput);
        passed += 1;
    } else {
        println!("⚠️  Throughput: {:.0} orders/sec (< 5000)", throughput);
    }

    // Check 3: P99 latency
    total_checks += 1;
    let p99_ms = p99 as f64 / 1_000_000.0;
    if p99_ms < 100.0 {
        println!("✅ P99 latency: {:.2}ms (< 100ms)", p99_ms);
        passed += 1;
    } else {
        println!("⚠️  P99 latency: {:.2}ms (>= 100ms)", p99_ms);
    }

    println!("\n📊 OVERALL: {}/{} checks passed", passed, total_checks);

    if passed == total_checks {
        println!("🎉 PRODUCTION READY!");
    } else {
        println!("⚠️  Not ready for production deployment");
    }

    Ok(())
}