foxhunt/tests/e2e/tests/performance_load_tests.rs

//! Performance and Load Testing
//!
//! Comprehensive performance and load tests for the system:
//! - High-frequency order submission
//! - Market data processing throughput
//! - ML inference performance under load
//! - Concurrent user simulation
//! - Latency measurements

use anyhow::{Context, Result};
use e2e_tests::{e2e_test, E2ETestFramework};
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
use tracing::{info, warn};

e2e_test!(
    test_order_submission_throughput,
    |mut framework: E2ETestFramework| async {
        info!("⚡ Starting order submission throughput test");

        let trading_client = framework.get_trading_client().await?;

        // Measure order submission throughput
        let num_orders = 100;
        let start = Instant::now();

        let mut successful_orders = 0;
        let mut failed_orders = 0;

        for i in 0..num_orders {
            let order = tli::proto::trading::SubmitOrderRequest {
                symbol: "AAPL".to_string(),
                side: if i % 2 == 0 {
                    tli::proto::trading::OrderSide::Buy
                } else {
                    tli::proto::trading::OrderSide::Sell
                } as i32,
                order_type: tli::proto::trading::OrderType::Limit as i32,
                quantity: 100.0,
                price: Some(150.0 + (i as f64 * 0.1)),
                time_in_force: tli::proto::trading::TimeInForce::Day as i32,
                client_order_id: format!("THROUGHPUT_TEST_{}", i),
            };

            let result = trading_client.submit_order(order).await;

            match result {
                Ok(response) => {
                    let response = response.into_inner();
                    if response.success {
                        successful_orders += 1;
                    } else {
                        failed_orders += 1;
                    }
                }
                Err(_) => {
                    failed_orders += 1;
                }
            }
        }

        let elapsed = start.elapsed();
        let throughput = num_orders as f64 / elapsed.as_secs_f64();

        info!("📊 Order Submission Throughput Results:");
        info!("  Total orders: {}", num_orders);
        info!("  Successful: {}", successful_orders);
        info!("  Failed: {}", failed_orders);
        info!("  Time elapsed: {:?}", elapsed);
        info!("  Throughput: {:.2} orders/sec", throughput);

        // Record metrics
        framework
            .performance_tracker
            .record_metric("order_submission_throughput", throughput)?;

        framework
            .performance_tracker
            .record_metric("order_submission_success_rate",
                          successful_orders as f64 / num_orders as f64)?;

        // Assert minimum throughput (adjust based on requirements)
        assert!(
            throughput > 10.0,
            "Order submission throughput should be at least 10 orders/sec, got {:.2}",
            throughput
        );

        info!("✅ Order submission throughput test completed");

        Ok(())
    }
);

e2e_test!(
    test_concurrent_order_processing,
    |mut framework: E2ETestFramework| async {
        info!("🔄 Starting concurrent order processing test");

        let trading_client = framework.get_trading_client().await?;

        // Simulate concurrent users submitting orders
        let num_concurrent_users = 10;
        let orders_per_user = 10;

        let start = Instant::now();
        let success_counter = Arc::new(AtomicU64::new(0));
        let failure_counter = Arc::new(AtomicU64::new(0));

        let mut handles = vec![];

        for user_id in 0..num_concurrent_users {
            let mut client = trading_client.clone();
            let success_counter = success_counter.clone();
            let failure_counter = failure_counter.clone();

            let handle = tokio::spawn(async move {
                for order_id in 0..orders_per_user {
                    let order = tli::proto::trading::SubmitOrderRequest {
                        symbol: "MSFT".to_string(),
                        side: tli::proto::trading::OrderSide::Buy as i32,
                        order_type: tli::proto::trading::OrderType::Market as i32,
                        quantity: 50.0 + (order_id as f64 * 10.0),
                        price: None,
                        time_in_force: tli::proto::trading::TimeInForce::Day as i32,
                        client_order_id: format!("CONCURRENT_U{}_O{}", user_id, order_id),
                    };

                    match client.submit_order(order).await {
                        Ok(response) => {
                            if response.into_inner().success {
                                success_counter.fetch_add(1, Ordering::SeqCst);
                            } else {
                                failure_counter.fetch_add(1, Ordering::SeqCst);
                            }
                        }
                        Err(_) => {
                            failure_counter.fetch_add(1, Ordering::SeqCst);
                        }
                    }

                    // Small delay to simulate realistic user behavior
                    tokio::time::sleep(Duration::from_millis(10)).await;
                }
            });

            handles.push(handle);
        }

        // Wait for all users to complete
        for handle in handles {
            handle.await?;
        }

        let elapsed = start.elapsed();
        let total_orders = num_concurrent_users * orders_per_user;
        let successful = success_counter.load(Ordering::SeqCst);
        let failed = failure_counter.load(Ordering::SeqCst);
        let throughput = total_orders as f64 / elapsed.as_secs_f64();

        info!("📊 Concurrent Order Processing Results:");
        info!("  Concurrent users: {}", num_concurrent_users);
        info!("  Orders per user: {}", orders_per_user);
        info!("  Total orders: {}", total_orders);
        info!("  Successful: {}", successful);
        info!("  Failed: {}", failed);
        info!("  Time elapsed: {:?}", elapsed);
        info!("  Throughput: {:.2} orders/sec", throughput);

        framework
            .performance_tracker
            .record_metric("concurrent_order_throughput", throughput)?;

        framework
            .performance_tracker
            .record_metric("concurrent_success_rate",
                          successful as f64 / total_orders as f64)?;

        info!("✅ Concurrent order processing test completed");

        Ok(())
    }
);

e2e_test!(
    test_market_data_processing_throughput,
    |mut framework: E2ETestFramework| async {
        info!("📊 Starting market data processing throughput test");

        // Generate large volume of market data
        let num_ticks = 10000;
        let symbols = vec!["AAPL", "MSFT", "GOOGL", "TSLA", "AMZN"];

        info!("Generating {} market data ticks", num_ticks);
        let start = Instant::now();

        let market_data = generate_high_volume_market_data(&symbols, num_ticks)?;

        let generation_time = start.elapsed();
        info!("Market data generated in {:?}", generation_time);

        // Process market data through ML pipeline
        info!("Processing market data through ML pipeline");
        let processing_start = Instant::now();

        let features = framework
            .ml_pipeline
            .extract_features(&market_data)
            .await?;

        let processing_time = processing_start.elapsed();
        let throughput = market_data.len() as f64 / processing_time.as_secs_f64();

        info!("📊 Market Data Processing Results:");
        info!("  Total ticks: {}", market_data.len());
        info!("  Features extracted: {}", features.len());
        info!("  Processing time: {:?}", processing_time);
        info!("  Throughput: {:.2} ticks/sec", throughput);
        info!("  Average latency: {:.2} µs/tick",
              processing_time.as_micros() as f64 / market_data.len() as f64);

        framework
            .performance_tracker
            .record_metric("market_data_throughput", throughput)?;

        framework
            .performance_tracker
            .record_metric("feature_extraction_latency_us",
                          processing_time.as_micros() as f64 / market_data.len() as f64)?;

        // Assert minimum throughput
        assert!(
            throughput > 1000.0,
            "Market data processing should handle at least 1000 ticks/sec, got {:.2}",
            throughput
        );

        info!("✅ Market data processing throughput test completed");

        Ok(())
    }
);

e2e_test!(
    test_ml_inference_performance,
    |mut framework: E2ETestFramework| async {
        info!("🧠 Starting ML inference performance test");

        let ml_status = framework.ml_pipeline.check_models_health().await?;

        if !ml_status.any_available() {
            info!("⚠️  No ML models available - using mock predictions");
        }

        // Test inference performance with different batch sizes
        let batch_sizes = vec![1, 10, 50, 100];
        let symbols = vec!["AAPL", "MSFT", "GOOGL"];

        for batch_size in batch_sizes {
            info!("Testing inference with batch size: {}", batch_size);

            let market_data = generate_high_volume_market_data(&symbols, batch_size)?;
            let features = framework.ml_pipeline.extract_features(&market_data).await?;

            let start = Instant::now();

            let prediction = if ml_status.any_available() {
                framework.ml_pipeline.predict_ensemble(&features).await?
            } else {
                // Mock prediction
                use e2e_tests::ml_pipeline::{EnsemblePrediction, PredictionType};
                EnsemblePrediction {
                    signal: 0.5,
                    confidence: 0.8,
                    individual_predictions: vec![],
                    ensemble_method: "mock".to_string(),
                    total_inference_time: Duration::from_millis(10),
                    prediction: PredictionType::Buy,
                    signal_strength: 0.5,
                }
            };

            let inference_time = start.elapsed();
            let latency_per_sample = inference_time.as_micros() as f64 / batch_size as f64;

            info!("  Batch size {}: inference_time={:?}, latency={:.2} µs/sample",
                  batch_size, inference_time, latency_per_sample);

            framework
                .performance_tracker
                .record_metric(
                    &format!("ml_inference_latency_batch_{}", batch_size),
                    inference_time.as_micros() as f64,
                )?;

            // Assert maximum latency for real-time trading
            assert!(
                inference_time < Duration::from_millis(100),
                "Inference should complete under 100ms for batch size {}, got {:?}",
                batch_size,
                inference_time
            );
        }

        info!("✅ ML inference performance test completed");

        Ok(())
    }
);

e2e_test!(
    test_latency_percentiles,
    |mut framework: E2ETestFramework| async {
        info!("📈 Starting latency percentiles test");

        let trading_client = framework.get_trading_client().await?;

        // Measure latency distribution for order validation
        let num_samples = 100;
        let mut latencies = Vec::with_capacity(num_samples);

        info!("Collecting {} latency samples", num_samples);

        for i in 0..num_samples {
            let start = Instant::now();

            let _result = trading_client
                .validate_order(tli::proto::trading::ValidateOrderRequest {
                    symbol: "AAPL".to_string(),
                    side: tli::proto::trading::OrderSide::Buy as i32,
                    quantity: 100.0,
                    price: 150.0,
                    order_type: tli::proto::trading::OrderType::Limit as i32,
                })
                .await;

            let latency = start.elapsed();
            latencies.push(latency);

            if i % 20 == 0 {
                info!("Sample {}: {:?}", i, latency);
            }
        }

        // Calculate percentiles
        latencies.sort();

        let p50 = latencies[num_samples * 50 / 100];
        let p95 = latencies[num_samples * 95 / 100];
        let p99 = latencies[num_samples * 99 / 100];
        let max = latencies[num_samples - 1];
        let min = latencies[0];

        let avg: Duration = latencies.iter().sum::<Duration>() / num_samples as u32;

        info!("📊 Latency Percentiles (Order Validation):");
        info!("  Min: {:?}", min);
        info!("  p50 (median): {:?}", p50);
        info!("  p95: {:?}", p95);
        info!("  p99: {:?}", p99);
        info!("  Max: {:?}", max);
        info!("  Average: {:?}", avg);

        // Record metrics
        framework
            .performance_tracker
            .record_metric("latency_p50_us", p50.as_micros() as f64)?;

        framework
            .performance_tracker
            .record_metric("latency_p95_us", p95.as_micros() as f64)?;

        framework
            .performance_tracker
            .record_metric("latency_p99_us", p99.as_micros() as f64)?;

        // Assert SLA targets
        assert!(
            p95 < Duration::from_millis(100),
            "p95 latency should be under 100ms, got {:?}",
            p95
        );

        assert!(
            p99 < Duration::from_millis(200),
            "p99 latency should be under 200ms, got {:?}",
            p99
        );

        info!("✅ Latency percentiles test completed");

        Ok(())
    }
);

e2e_test!(
    test_sustained_load,
    |mut framework: E2ETestFramework| async {
        info!("⏱️  Starting sustained load test");

        let trading_client = framework.get_trading_client().await?;

        // Run sustained load for 30 seconds
        let test_duration = Duration::from_secs(30);
        let request_rate = 10; // requests per second
        let interval = Duration::from_millis(1000 / request_rate);

        let start = Instant::now();
        let mut request_count = 0;
        let mut success_count = 0;
        let mut error_count = 0;

        info!("Running sustained load: {} req/sec for {:?}", request_rate, test_duration);

        while start.elapsed() < test_duration {
            let _result = trading_client
                .get_account_info(tli::proto::trading::GetAccountInfoRequest {})
                .await;

            match _result {
                Ok(_) => success_count += 1,
                Err(_) => error_count += 1,
            }

            request_count += 1;

            if request_count % 50 == 0 {
                let elapsed = start.elapsed();
                let current_rate = request_count as f64 / elapsed.as_secs_f64();
                info!("Progress: {} requests in {:?} ({:.2} req/sec)",
                      request_count, elapsed, current_rate);
            }

            tokio::time::sleep(interval).await;
        }

        let total_time = start.elapsed();
        let actual_rate = request_count as f64 / total_time.as_secs_f64();
        let success_rate = success_count as f64 / request_count as f64;

        info!("📊 Sustained Load Test Results:");
        info!("  Duration: {:?}", total_time);
        info!("  Total requests: {}", request_count);
        info!("  Successful: {}", success_count);
        info!("  Errors: {}", error_count);
        info!("  Target rate: {} req/sec", request_rate);
        info!("  Actual rate: {:.2} req/sec", actual_rate);
        info!("  Success rate: {:.2}%", success_rate * 100.0);

        framework
            .performance_tracker
            .record_metric("sustained_load_actual_rate", actual_rate)?;

        framework
            .performance_tracker
            .record_metric("sustained_load_success_rate", success_rate)?;

        // Assert system handled sustained load
        assert!(
            success_rate > 0.95,
            "Success rate should be above 95%, got {:.2}%",
            success_rate * 100.0
        );

        info!("✅ Sustained load test completed");

        Ok(())
    }
);

/// Generate high volume market data for performance testing
fn generate_high_volume_market_data(
    symbols: &[&str],
    total_ticks: usize,
) -> Result<Vec<common::types::MarketTick>> {
    use common::types::{MarketTick, Symbol, Price, Quantity, TickType, Exchange, HftTimestamp};
    use rand::Rng;
    use std::time::{SystemTime, UNIX_EPOCH};

    let mut rng = rand::thread_rng();
    let mut ticks = Vec::with_capacity(total_ticks);
    let base_time = SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos() as u64;

    let ticks_per_symbol = total_ticks / symbols.len();

    for (symbol_idx, &symbol) in symbols.iter().enumerate() {
        let base_price = 100.0 + (symbol_idx as f64 * 50.0);
        let mut current_price = base_price;

        for i in 0..ticks_per_symbol {
            current_price += rng.gen_range(-0.5..0.5);
            current_price = current_price.max(base_price * 0.9).min(base_price * 1.1);

            ticks.push(MarketTick::with_timestamp(
                Symbol::new(symbol.to_string()),
                Price::from_f64(current_price)?,
                Quantity::from_u64(rng.gen_range(100..2000)),
                HftTimestamp::from_nanos(base_time + (symbol_idx * ticks_per_symbol + i) as u64 * 100),
                TickType::Trade,
                Exchange::NASDAQ,
                (symbol_idx * ticks_per_symbol + i) as u64,
            ));
        }
    }

    Ok(ticks)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_high_volume_data_generation() {
        let symbols = vec!["AAPL", "MSFT", "GOOGL"];
        let data = generate_high_volume_market_data(&symbols, 1000).unwrap();

        assert_eq!(data.len(), 999); // 1000 / 3 = 333 per symbol, 333 * 3 = 999

        for symbol in symbols {
            let count = data.iter().filter(|t| t.symbol.as_str() == symbol).count();
            assert!(count > 300 && count < 350);
        }
    }
}