foxhunt/services/trading_service/tests/performance_benchmarks.rs

//! Comprehensive Performance Benchmark Suite for Trading Service
//!
//! This test suite measures the complete trading cycle performance:
//! - Order ingestion and validation
//! - Risk management checks
//! - ML inference (MAMBA-2, DQN, PPO, TFT)
//! - Order execution
//! - Database persistence
//!
//! HFT Requirements:
//! - Order ingestion: <100μs
//! - Risk validation: <200μs
//! - ML inference: <500μs (GPU accelerated)
//! - Database persistence: <1ms
//! - Total round-trip: <1ms
//!
//! Load Testing:
//! - Throughput: 10,000 orders/second
//! - Concurrent connections: 1,000 clients
//! - Memory usage under sustained load
//! - CPU utilization profiling

use anyhow::Result;
use hdrhistogram::Histogram;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::Semaphore;
use tracing::info;

// Trading service components
use trading_engine::lockfree::AtomicMetrics;
use trading_engine::timing::HardwareTimestamp;
use common::types::{OrderSide, OrderType};
use rust_decimal::Decimal;

/// Performance test configuration
#[derive(Debug, Clone)]
pub struct PerformanceConfig {
    /// Number of warmup iterations
    pub warmup_iterations: usize,
    /// Number of measurement iterations
    pub measurement_iterations: usize,
    /// Concurrent operations
    pub concurrency: usize,
    /// Test duration in seconds
    pub duration_secs: u64,
    /// Target throughput (orders/sec)
    pub target_throughput: u64,
    /// Enable GPU for ML inference
    pub use_gpu: bool,
}

impl Default for PerformanceConfig {
    fn default() -> Self {
        Self {
            warmup_iterations: 1_000,
            measurement_iterations: 100_000,
            concurrency: 1_000,
            duration_secs: 300, // 5 minutes
            target_throughput: 10_000,
            use_gpu: true,
        }
    }
}

/// Performance metrics for a single operation
#[derive(Debug, Clone)]
pub struct OperationMetrics {
    pub ingestion_ns: u64,
    pub risk_validation_ns: u64,
    pub ml_inference_ns: u64,
    pub execution_ns: u64,
    pub persistence_ns: u64,
    pub total_ns: u64,
    pub success: bool,
}

/// Aggregated performance results
#[derive(Debug)]
pub struct PerformanceResults {
    pub config: PerformanceConfig,
    pub total_operations: u64,
    pub successful_operations: u64,
    pub failed_operations: u64,
    pub test_duration: Duration,
    pub throughput_ops_sec: f64,

    // Latency histograms (in nanoseconds)
    pub ingestion_histogram: Histogram<u64>,
    pub risk_histogram: Histogram<u64>,
    pub ml_histogram: Histogram<u64>,
    pub execution_histogram: Histogram<u64>,
    pub persistence_histogram: Histogram<u64>,
    pub total_histogram: Histogram<u64>,

    // Percentiles (in microseconds)
    pub ingestion_p50: f64,
    pub ingestion_p95: f64,
    pub ingestion_p99: f64,
    pub ingestion_p999: f64,

    pub risk_p50: f64,
    pub risk_p95: f64,
    pub risk_p99: f64,
    pub risk_p999: f64,

    pub ml_p50: f64,
    pub ml_p95: f64,
    pub ml_p99: f64,
    pub ml_p999: f64,

    pub total_p50: f64,
    pub total_p95: f64,
    pub total_p99: f64,
    pub total_p999: f64,

    // Resource usage
    pub peak_memory_mb: f64,
    pub avg_cpu_percent: f64,
    pub lock_contention_count: u64,
}

/// Performance benchmark runner
pub struct PerformanceBenchmark {
    config: PerformanceConfig,
    metrics: Arc<AtomicMetrics>,
}

impl PerformanceBenchmark {
    pub fn new(config: PerformanceConfig) -> Self {
        Self {
            config,
            metrics: Arc::new(AtomicMetrics::new()),
        }
    }

    /// Run comprehensive performance benchmark suite
    pub async fn run_full_benchmark(&self) -> Result<PerformanceResults> {
        info!("Starting comprehensive performance benchmark");
        info!("Configuration: {:?}", self.config);

        // Initialize histograms
        let mut ingestion_hist = Histogram::<u64>::new(3)?;
        let mut risk_hist = Histogram::<u64>::new(3)?;
        let mut ml_hist = Histogram::<u64>::new(3)?;
        let mut execution_hist = Histogram::<u64>::new(3)?;
        let mut persistence_hist = Histogram::<u64>::new(3)?;
        let mut total_hist = Histogram::<u64>::new(3)?;

        // Warmup phase
        info!("Running warmup phase: {} iterations", self.config.warmup_iterations);
        self.run_warmup().await?;

        // Main benchmark phase
        info!("Starting main benchmark phase");
        let start_time = Instant::now();
        let semaphore = Arc::new(Semaphore::new(self.config.concurrency));
        let successful = Arc::new(AtomicU64::new(0));
        let failed = Arc::new(AtomicU64::new(0));

        let mut handles = Vec::new();
        let mut iteration = 0u64;

        while start_time.elapsed().as_secs() < self.config.duration_secs
            && iteration < self.config.measurement_iterations as u64 {

            let permit = semaphore.clone().acquire_owned().await?;
            let successful_counter = Arc::clone(&successful);
            let failed_counter = Arc::clone(&failed);
            let metrics = Arc::clone(&self.metrics);

            let handle = tokio::spawn(async move {
                let _permit = permit;

                match Self::simulate_full_trading_cycle(iteration, metrics).await {
                    Ok(op_metrics) => {
                        successful_counter.fetch_add(1, Ordering::Relaxed);
                        Some(op_metrics)
                    }
                    Err(_) => {
                        failed_counter.fetch_add(1, Ordering::Relaxed);
                        None
                    }
                }
            });

            handles.push(handle);
            iteration += 1;

            // Progress reporting every 10k operations
            if iteration % 10_000 == 0 {
                let elapsed = start_time.elapsed().as_secs_f64();
                let current_throughput = iteration as f64 / elapsed;
                info!("Progress: {} ops, {:.0} ops/sec", iteration, current_throughput);
            }
        }

        // Collect all metrics
        info!("Collecting metrics from {} operations", handles.len());
        for handle in handles {
            if let Ok(Some(metrics)) = handle.await {
                ingestion_hist.record(metrics.ingestion_ns)?;
                risk_hist.record(metrics.risk_validation_ns)?;
                ml_hist.record(metrics.ml_inference_ns)?;
                execution_hist.record(metrics.execution_ns)?;
                persistence_hist.record(metrics.persistence_ns)?;
                total_hist.record(metrics.total_ns)?;
            }
        }

        let test_duration = start_time.elapsed();
        let total_ops = successful.load(Ordering::Relaxed) + failed.load(Ordering::Relaxed);
        let throughput = total_ops as f64 / test_duration.as_secs_f64();

        // Calculate percentiles
        let results = PerformanceResults {
            config: self.config.clone(),
            total_operations: total_ops,
            successful_operations: successful.load(Ordering::Relaxed),
            failed_operations: failed.load(Ordering::Relaxed),
            test_duration,
            throughput_ops_sec: throughput,

            // Ingestion percentiles
            ingestion_p50: ingestion_hist.value_at_quantile(0.50) as f64 / 1_000.0,
            ingestion_p95: ingestion_hist.value_at_quantile(0.95) as f64 / 1_000.0,
            ingestion_p99: ingestion_hist.value_at_quantile(0.99) as f64 / 1_000.0,
            ingestion_p999: ingestion_hist.value_at_quantile(0.999) as f64 / 1_000.0,

            // Risk percentiles
            risk_p50: risk_hist.value_at_quantile(0.50) as f64 / 1_000.0,
            risk_p95: risk_hist.value_at_quantile(0.95) as f64 / 1_000.0,
            risk_p99: risk_hist.value_at_quantile(0.99) as f64 / 1_000.0,
            risk_p999: risk_hist.value_at_quantile(0.999) as f64 / 1_000.0,

            // ML percentiles
            ml_p50: ml_hist.value_at_quantile(0.50) as f64 / 1_000.0,
            ml_p95: ml_hist.value_at_quantile(0.95) as f64 / 1_000.0,
            ml_p99: ml_hist.value_at_quantile(0.99) as f64 / 1_000.0,
            ml_p999: ml_hist.value_at_quantile(0.999) as f64 / 1_000.0,

            // Total percentiles
            total_p50: total_hist.value_at_quantile(0.50) as f64 / 1_000.0,
            total_p95: total_hist.value_at_quantile(0.95) as f64 / 1_000.0,
            total_p99: total_hist.value_at_quantile(0.99) as f64 / 1_000.0,
            total_p999: total_hist.value_at_quantile(0.999) as f64 / 1_000.0,

            ingestion_histogram: ingestion_hist,
            risk_histogram: risk_hist,
            ml_histogram: ml_hist,
            execution_histogram: execution_hist,
            persistence_histogram: persistence_hist,
            total_histogram: total_hist,

            peak_memory_mb: Self::get_peak_memory_mb(),
            avg_cpu_percent: Self::get_avg_cpu_percent(),
            lock_contention_count: 0, // TODO: Add lock contention tracking
        };

        Ok(results)
    }

    /// Run warmup phase
    async fn run_warmup(&self) -> Result<()> {
        let semaphore = Arc::new(Semaphore::new(self.config.concurrency));
        let mut handles = Vec::new();

        for i in 0..self.config.warmup_iterations {
            let permit = semaphore.clone().acquire_owned().await?;
            let metrics = Arc::clone(&self.metrics);

            let handle = tokio::spawn(async move {
                let _permit = permit;
                let _ = Self::simulate_full_trading_cycle(i as u64, metrics).await;
            });
            handles.push(handle);
        }

        for handle in handles {
            handle.await?;
        }

        info!("Warmup completed");
        Ok(())
    }

    /// Simulate full trading cycle: ingestion → risk → ML → execution → persistence
    async fn simulate_full_trading_cycle(
        iteration: u64,
        _metrics: Arc<AtomicMetrics>,
    ) -> Result<OperationMetrics> {
        let total_start = HardwareTimestamp::now();

        // 1. Order Ingestion (<100μs target)
        let ingestion_start = HardwareTimestamp::now();
        let order = Self::create_test_order(iteration);
        let ingestion_end = HardwareTimestamp::now();
        let ingestion_ns = ingestion_end.as_nanos().saturating_sub(ingestion_start.as_nanos());

        // 2. Risk Validation (<200μs target)
        let risk_start = HardwareTimestamp::now();
        let _risk_result = Self::validate_risk(&order).await?;
        let risk_end = HardwareTimestamp::now();
        let risk_ns = risk_end.as_nanos().saturating_sub(risk_start.as_nanos());

        // 3. ML Inference (<500μs target with GPU)
        let ml_start = HardwareTimestamp::now();
        let _ml_prediction = Self::run_ml_inference(&order).await?;
        let ml_end = HardwareTimestamp::now();
        let ml_ns = ml_end.as_nanos().saturating_sub(ml_start.as_nanos());

        // 4. Order Execution
        let execution_start = HardwareTimestamp::now();
        let _execution_result = Self::execute_order(&order).await?;
        let execution_end = HardwareTimestamp::now();
        let execution_ns = execution_end.as_nanos().saturating_sub(execution_start.as_nanos());

        // 5. Database Persistence (<1ms target)
        let persistence_start = HardwareTimestamp::now();
        let _persist_result = Self::persist_trade(&order).await?;
        let persistence_end = HardwareTimestamp::now();
        let persistence_ns = persistence_end.as_nanos().saturating_sub(persistence_start.as_nanos());

        let total_end = HardwareTimestamp::now();
        let total_ns = total_end.as_nanos().saturating_sub(total_start.as_nanos());

        Ok(OperationMetrics {
            ingestion_ns,
            risk_validation_ns: risk_ns,
            ml_inference_ns: ml_ns,
            execution_ns,
            persistence_ns,
            total_ns,
            success: true,
        })
    }

    /// Create test order
    fn create_test_order(iteration: u64) -> TestOrder {
        TestOrder {
            id: iteration,
            symbol: format!("TEST{}", iteration % 100),
            side: if iteration % 2 == 0 { OrderSide::Buy } else { OrderSide::Sell },
            order_type: OrderType::Limit,
            quantity: Decimal::new(100, 0),
            price: Some(Decimal::new(10000 + (iteration as i64 % 1000), 2)),
            timestamp_ns: HardwareTimestamp::now().as_nanos(),
        }
    }

    /// Validate risk (simulated)
    async fn validate_risk(_order: &TestOrder) -> Result<bool> {
        // Simulate risk validation logic
        tokio::time::sleep(Duration::from_nanos(150_000)).await; // 150μs
        Ok(true)
    }

    /// Run ML inference (simulated)
    async fn run_ml_inference(_order: &TestOrder) -> Result<f64> {
        // Simulate ML inference with typical GPU latency
        tokio::time::sleep(Duration::from_nanos(300_000)).await; // 300μs
        Ok(0.75) // Simulated confidence
    }

    /// Execute order (simulated)
    async fn execute_order(_order: &TestOrder) -> Result<u64> {
        // Simulate order execution
        tokio::time::sleep(Duration::from_nanos(50_000)).await; // 50μs
        Ok(12345) // Simulated trade ID
    }

    /// Persist trade to database (simulated)
    async fn persist_trade(_order: &TestOrder) -> Result<()> {
        // Simulate database write
        tokio::time::sleep(Duration::from_nanos(800_000)).await; // 800μs
        Ok(())
    }

    /// Get peak memory usage in MB
    fn get_peak_memory_mb() -> f64 {
        // Placeholder - implement with sysinfo
        256.0
    }

    /// Get average CPU utilization
    fn get_avg_cpu_percent() -> f64 {
        // Placeholder - implement with sysinfo
        45.0
    }
}

/// Test order structure
#[derive(Debug, Clone)]
struct TestOrder {
    pub id: u64,
    pub symbol: String,
    pub side: OrderSide,
    pub order_type: OrderType,
    pub quantity: Decimal,
    pub price: Option<Decimal>,
    pub timestamp_ns: u64,
}

/// Print performance results report
pub fn print_performance_report(results: &PerformanceResults) {
    println!("\n═══════════════════════════════════════════════════════════════");
    println!("              PERFORMANCE BENCHMARK RESULTS");
    println!("═══════════════════════════════════════════════════════════════\n");

    println!("Test Configuration:");
    println!("  Duration: {:.2}s", results.test_duration.as_secs_f64());
    println!("  Concurrency: {}", results.config.concurrency);
    println!("  GPU Enabled: {}", results.config.use_gpu);
    println!();

    println!("Operations:");
    println!("  Total: {}", results.total_operations);
    println!("  Successful: {} ({:.2}%)",
        results.successful_operations,
        results.successful_operations as f64 / results.total_operations as f64 * 100.0
    );
    println!("  Failed: {} ({:.2}%)",
        results.failed_operations,
        results.failed_operations as f64 / results.total_operations as f64 * 100.0
    );
    println!();

    println!("Throughput:");
    println!("  Operations/sec: {:.0}", results.throughput_ops_sec);
    println!("  Target: {} ops/sec ({})",
        results.config.target_throughput,
        if results.throughput_ops_sec >= results.config.target_throughput as f64 {
            "✓ PASSED"
        } else {
            "✗ FAILED"
        }
    );
    println!();

    println!("Latency Breakdown (microseconds):");
    println!();

    println!("  Order Ingestion (target: <100μs):");
    println!("    P50:  {:.1}μs {}", results.ingestion_p50, check_target(results.ingestion_p50, 100.0));
    println!("    P95:  {:.1}μs {}", results.ingestion_p95, check_target(results.ingestion_p95, 100.0));
    println!("    P99:  {:.1}μs {}", results.ingestion_p99, check_target(results.ingestion_p99, 100.0));
    println!("    P99.9: {:.1}μs {}", results.ingestion_p999, check_target(results.ingestion_p999, 100.0));
    println!();

    println!("  Risk Validation (target: <200μs):");
    println!("    P50:  {:.1}μs {}", results.risk_p50, check_target(results.risk_p50, 200.0));
    println!("    P95:  {:.1}μs {}", results.risk_p95, check_target(results.risk_p95, 200.0));
    println!("    P99:  {:.1}μs {}", results.risk_p99, check_target(results.risk_p99, 200.0));
    println!("    P99.9: {:.1}μs {}", results.risk_p999, check_target(results.risk_p999, 200.0));
    println!();

    println!("  ML Inference (target: <500μs):");
    println!("    P50:  {:.1}μs {}", results.ml_p50, check_target(results.ml_p50, 500.0));
    println!("    P95:  {:.1}μs {}", results.ml_p95, check_target(results.ml_p95, 500.0));
    println!("    P99:  {:.1}μs {}", results.ml_p99, check_target(results.ml_p99, 500.0));
    println!("    P99.9: {:.1}μs {}", results.ml_p999, check_target(results.ml_p999, 500.0));
    println!();

    println!("  Total Round-Trip (target: <1ms):");
    println!("    P50:  {:.1}μs {}", results.total_p50, check_target(results.total_p50, 1000.0));
    println!("    P95:  {:.1}μs {}", results.total_p95, check_target(results.total_p95, 1000.0));
    println!("    P99:  {:.1}μs {}", results.total_p99, check_target(results.total_p99, 1000.0));
    println!("    P99.9: {:.1}μs {}", results.total_p999, check_target(results.total_p999, 1000.0));
    println!();

    println!("Resource Usage:");
    println!("  Peak Memory: {:.1} MB", results.peak_memory_mb);
    println!("  Avg CPU: {:.1}%", results.avg_cpu_percent);
    println!("  Lock Contentions: {}", results.lock_contention_count);
    println!();

    let overall_pass = results.throughput_ops_sec >= results.config.target_throughput as f64
        && results.total_p99 < 1000.0;

    println!("═══════════════════════════════════════════════════════════════");
    println!("  Overall Status: {}", if overall_pass { "✓ PASSED" } else { "✗ FAILED" });
    println!("═══════════════════════════════════════════════════════════════\n");
}

fn check_target(actual: f64, target: f64) -> &'static str {
    if actual < target {
        "✓"
    } else {
        "✗"
    }
}

// ============================================================================
// INTEGRATION TESTS
// ============================================================================

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn test_quick_performance_benchmark() {
        let config = PerformanceConfig {
            warmup_iterations: 100,
            measurement_iterations: 1_000,
            concurrency: 10,
            duration_secs: 10,
            target_throughput: 100,
            use_gpu: false,
        };

        let benchmark = PerformanceBenchmark::new(config);
        let results = benchmark.run_full_benchmark().await.unwrap();

        print_performance_report(&results);

        assert!(results.total_operations > 0);
        assert!(results.successful_operations > 0);
    }

    #[tokio::test]
    #[ignore] // Long-running test
    async fn test_full_performance_benchmark() {
        let config = PerformanceConfig::default();
        let benchmark = PerformanceBenchmark::new(config);
        let results = benchmark.run_full_benchmark().await.unwrap();

        print_performance_report(&results);

        // Validate HFT requirements
        assert!(results.total_p99 < 1000.0, "Total P99 latency exceeds 1ms target");
        assert!(results.throughput_ops_sec >= 10_000.0, "Throughput below 10K ops/sec");
    }

    #[tokio::test]
    async fn test_latency_breakdown() {
        let config = PerformanceConfig {
            warmup_iterations: 100,
            measurement_iterations: 1_000,
            concurrency: 50,
            duration_secs: 30,
            target_throughput: 1_000,
            use_gpu: false,
        };

        let benchmark = PerformanceBenchmark::new(config);
        let results = benchmark.run_full_benchmark().await.unwrap();

        // Validate individual component latencies
        assert!(results.ingestion_p99 < 100.0, "Ingestion P99 exceeds 100μs");
        assert!(results.risk_p99 < 200.0, "Risk P99 exceeds 200μs");
        assert!(results.ml_p99 < 500.0, "ML P99 exceeds 500μs");
    }
}