foxhunt/testing/integration/performance/hft_benchmarks.rs

//! Performance Tests for Foxhunt HFT Trading System
//!
//! This module tests that the system meets strict High-Frequency Trading (HFT)
//! performance requirements. All tests validate sub-50μs latency targets and
//! ensure the system can handle the throughput demands of live trading.
//!
//! # Performance Test Coverage
//!
//! - **Latency Validation** (Sub-50μs end-to-end trading paths)
//! - **Throughput Testing** (Orders per second, market data processing)
//! - **Memory Performance** (Allocation patterns, cache efficiency)
//! - **CPU Utilization** (Core affinity, SIMD optimization)
//! - **Network Performance** (Market data ingestion, order routing)
//! - **Database Performance** (Position updates, trade recording)
//! - **ML Inference Speed** (Model prediction latency)
//! - **Concurrent Performance** (Multi-threaded safety and speed)
//!
//! # Test Philosophy
//!
//! Performance tests are designed to validate that the system meets production
//! HFT requirements under various load conditions. They measure actual latency
//! and throughput rather than relying on theoretical calculations.

// anyhow not available - using simple Result type
type Result<T> = std::result::Result<T, Box<dyn std::error::Error + Send + Sync>>;
use std::time::{Duration, Instant};
use std::sync::{Arc, atomic::{AtomicU64, AtomicUsize, Ordering}};
use std::collections::HashMap;
use tokio::time::timeout;

// Import unified types

// Import risk and ML systems
// use risk::prelude::*; // REMOVED - prelude does not exist
use ml::prelude::*;

// Import common test utilities
use crate::common::{*, test_config::*, test_utils::*, assertions::*};
use common::*;
use common::test_config::*;
use common::test_utils::*;
use common::assertions::*;

/// Performance test configuration
#[derive(Debug, Clone)]
struct PerformanceTestConfig {
    /// Maximum allowed latency for critical operations (microseconds)
    max_critical_latency_us: u64,
    /// Maximum allowed latency for non-critical operations (microseconds)
    max_standard_latency_us: u64,
    /// Target throughput (operations per second)
    target_throughput_ops: u64,
    /// Test duration for sustained load testing
    sustained_test_duration_ms: u64,
    /// Number of concurrent operations for load testing
    concurrent_operations: usize,
    /// Enable CPU-intensive optimizations testing
    test_simd_optimizations: bool,
    /// Enable memory performance testing
    test_memory_performance: bool,
    /// Enable network simulation testing
    test_network_performance: bool,
    /// Sample size for statistical measurements
    measurement_samples: usize,
}

impl Default for PerformanceTestConfig {
    fn default() -> Self {
        Self {
            max_critical_latency_us: 50,    // HFT requirement: sub-50μs
            max_standard_latency_us: 100,   // Non-critical: sub-100μs
            target_throughput_ops: 10000,   // 10k ops/sec target
            sustained_test_duration_ms: 5000, // 5 second sustained tests
            concurrent_operations: 100,     // 100 concurrent operations
            test_simd_optimizations: true,
            test_memory_performance: true,
            test_network_performance: false, // Disabled by default (requires network setup)
            measurement_samples: 1000,      // 1000 samples for statistics
        }
    }
}

/// Performance measurement result
#[derive(Debug, Clone)]
struct PerformanceMeasurement {
    operation_name: String,
    samples: Vec<Duration>,
    min_latency: Duration,
    max_latency: Duration,
    avg_latency: Duration,
    p50_latency: Duration,
    p95_latency: Duration,
    p99_latency: Duration,
    throughput_ops_per_sec: f64,
    success_rate: f64,
    memory_allocations: u64,
    cpu_utilization: f64,
}

impl PerformanceMeasurement {
    fn new(operation_name: String, mut samples: Vec<Duration>) -> Self {
        samples.sort();
        let len = samples.len();
        
        let min_latency = samples.first().copied().unwrap_or(Duration::ZERO);
        let max_latency = samples.last().copied().unwrap_or(Duration::ZERO);
        
        let avg_latency = if !samples.is_empty() {
            let total: Duration = samples.iter().sum();
            total / len as u32
        } else {
            Duration::ZERO
        };
        
        let p50_latency = samples.get(len / 2).copied().unwrap_or(Duration::ZERO);
        let p95_latency = samples.get((len * 95) / 100).copied().unwrap_or(Duration::ZERO);
        let p99_latency = samples.get((len * 99) / 100).copied().unwrap_or(Duration::ZERO);
        
        let throughput_ops_per_sec = if avg_latency.as_secs_f64() > 0.0 {
            1.0 / avg_latency.as_secs_f64()
        } else {
            0.0
        };
        
        Self {
            operation_name,
            samples,
            min_latency,
            max_latency,
            avg_latency,
            p50_latency,
            p95_latency,
            p99_latency,
            throughput_ops_per_sec,
            success_rate: 1.0, // Will be updated based on actual results
            memory_allocations: 0, // Placeholder
            cpu_utilization: 0.0, // Placeholder
        }
    }
    
    fn meets_latency_requirement(&self, max_latency_us: u64) -> bool {
        self.p99_latency.as_micros() <= max_latency_us as u128
    }
    
    fn meets_throughput_requirement(&self, min_throughput: f64) -> bool {
        self.throughput_ops_per_sec >= min_throughput
    }
}

/// Performance test suite
struct PerformanceTestSuite {
    config: PerformanceTestConfig,
    risk_engine: Option<RiskEngine>,
    ml_registry: Option<Arc<ModelRegistry>>,
    measurements: HashMap<String, PerformanceMeasurement>,
    total_operations: Arc<AtomicU64>,
    successful_operations: Arc<AtomicU64>,
    failed_operations: Arc<AtomicU64>,
}

impl PerformanceTestSuite {
    fn new() -> Self {
        setup_test_tracing();
        
        Self {
            config: PerformanceTestConfig::default(),
            risk_engine: None,
            ml_registry: None,
            measurements: HashMap::new(),
            total_operations: Arc::new(AtomicU64::new(0)),
            successful_operations: Arc::new(AtomicU64::new(0)),
            failed_operations: Arc::new(AtomicU64::new(0)),
        }
    }

    async fn setup(&mut self) -> Result<()> {
        // Initialize components for performance testing
        let risk_config = RiskConfig {
            max_position_size: Price::from_f64(100000.0)?,
            max_daily_loss: Price::from_f64(10000.0)?,
            var_confidence_level: 0.95,
            var_lookback_days: 252,
            enable_kill_switch: false, // Disabled for performance testing
            enable_circuit_breakers: false, // Disabled for clean measurements
            redis_url: "redis://localhost:6379".to_string(),
        };

        // Initialize risk engine (may fail if Redis not available)
        match RiskEngine::new(risk_config).await {
            Ok(engine) => self.risk_engine = Some(engine),
            Err(_) => tracing::warn!("Risk engine unavailable for performance testing"),
        }

        // Initialize ML registry
        let registry = get_global_registry();
        
        // Try to register models for performance testing
        if let Ok(tlob_model) = ml::model_factory::create_tlob_wrapper() {
            let _ = registry.register(Arc::from(tlob_model)).await;
        }
        
        if let Ok(dqn_model) = ml::model_factory::create_dqn_wrapper() {
            let _ = registry.register(Arc::from(dqn_model)).await;
        }

        self.ml_registry = Some(registry);

        Ok(())
    }

    /// Record operation metrics
    fn record_operation(&self, success: bool) {
        self.total_operations.fetch_add(1, Ordering::SeqCst);
        
        if success {
            self.successful_operations.fetch_add(1, Ordering::SeqCst);
        } else {
            self.failed_operations.fetch_add(1, Ordering::SeqCst);
        }
    }

    /// Measure operation latency
    async fn measure_operation<F, R>(&self, operation_name: &str, operation: F) -> Result<(R, Duration)>
    where
        F: std::future::Future<Output = Result<R>>,
    {
        let start_time = Instant::now();
        let result = operation.await;
        let latency = start_time.elapsed();
        
        self.record_operation(result.is_ok());
        
        Ok((result?, latency))
    }

    /// Measure multiple samples of an operation
    async fn measure_operation_samples<F, Fut>(&self, operation_name: &str, operation_factory: F, samples: usize) -> Result<PerformanceMeasurement>
    where
        F: Fn() -> Fut,
        Fut: std::future::Future<Output = Result<()>>,
    {
        let mut latencies = Vec::with_capacity(samples);
        let mut success_count = 0;

        for _ in 0..samples {
            let start_time = Instant::now();
            let result = operation_factory().await;
            let latency = start_time.elapsed();
            
            latencies.push(latency);
            
            if result.is_ok() {
                success_count += 1;
            }
            
            self.record_operation(result.is_ok());
        }

        let mut measurement = PerformanceMeasurement::new(operation_name.to_string(), latencies);
        measurement.success_rate = success_count as f64 / samples as f64;
        
        Ok(measurement)
    }

    /// Create test market data for performance testing
    fn create_performance_market_data(&self, index: usize) -> Result<TestMarketData> {
        TestMarketData::new(
            &format!("PERF_SYMBOL_{}", index % 100), // Cycle through 100 symbols
            50000.0 + (index as f64 % 1000.0),        // Varying prices
            1000.0 + (index as f64 % 500.0),          // Varying volumes
        )
    }

    /// Test order processing latency
    async fn test_order_processing_latency(&self) -> Result<PerformanceMeasurement> {
        let measurement = self.measure_operation_samples(
            "order_processing",
            || async {
                // Create test order
                let symbol = Symbol::from("PERF_BTC");
                let quantity = Quantity::from_f64(1.0)?;
                let price = Price::from_f64(50000.0)?;
                
                let order = Order::limit(symbol, Side::Buy, quantity, price);
                
                // Simulate order validation
                if order.quantity.to_f64() > 0.0 && order.symbol.as_str().len() > 0 {
                    Ok(())
                } else {
                    Err(anyhow::anyhow!("Invalid order"))
                }
            },
            self.config.measurement_samples,
        ).await?;

        Ok(measurement)
    }

    /// Test risk calculation latency
    async fn test_risk_calculation_latency(&self) -> Result<PerformanceMeasurement> {
        let measurement = self.measure_operation_samples(
            "risk_calculation",
            || async {
                // Create order info for risk calculation
                let order_info = OrderInfo {
                    symbol: Symbol::from("RISK_TEST"),
                    side: Side::Buy,
                    quantity: Quantity::from_f64(100.0)?,
                    price: Price::from_f64(50000.0)?,
                };

                // Test risk calculation with or without risk engine
                if let Some(ref risk_engine) = self.risk_engine {
                    match risk_engine.validate_order(&order_info).await {
                        Ok(_) => Ok(()),
                        Err(_) => Ok(()), // Risk rejection is valid for performance test
                    }
                } else {
                    // Fallback risk calculation
                    let order_value = order_info.quantity.to_f64() * order_info.price.to_f64();
                    if order_value < 100000.0 { // Simple limit check
                        Ok(())
                    } else {
                        Err(anyhow::anyhow!("Position limit exceeded"))
                    }
                }
            },
            self.config.measurement_samples,
        ).await?;

        Ok(measurement)
    }

    /// Test ML inference latency
    async fn test_ml_inference_latency(&self) -> Result<PerformanceMeasurement> {
        let measurement = self.measure_operation_samples(
            "ml_inference",
            || async {
                if let Some(ref registry) = self.ml_registry {
                    // Create test features
                    let features = Features::new(
                        vec![50000.0, 1000.0, 49999.0, 50001.0, 2.0, 1234567890.0],
                        vec!["price".to_string(), "volume".to_string(), "bid".to_string(), 
                             "ask".to_string(), "spread".to_string(), "timestamp".to_string()],
                    );

                    // Test prediction with all available models
                    let predictions = registry.predict_all(&features).await;
                    
                    // Consider it successful if at least one model responds
                    let success_count = predictions.iter().filter(|p| p.is_ok()).count();
                    
                    if success_count > 0 {
                        Ok(())
                    } else {
                        Err(anyhow::anyhow!("No successful ML predictions"))
                    }
                } else {
                    // Simulate ML inference without models
                    let input_sum: f64 = vec![50000.0, 1000.0, 49999.0, 50001.0, 2.0]
                        .iter().sum();
                    let _prediction = input_sum / 5.0; // Simple average
                    Ok(())
                }
            },
            self.config.measurement_samples,
        ).await?;

        Ok(measurement)
    }

    /// Test memory allocation performance
    async fn test_memory_performance(&self) -> Result<PerformanceMeasurement> {
        let measurement = self.measure_operation_samples(
            "memory_allocation",
            || async {
                // Test various memory allocation patterns
                
                // 1. Small allocations (typical for trading data)
                let mut small_vec: Vec<f64> = Vec::with_capacity(10);
                for i in 0..10 {
                    small_vec.push(i as f64);
                }
                
                // 2. Medium allocations (order book data)
                let mut medium_vec: Vec<Price> = Vec::with_capacity(100);
                for i in 0..100 {
                    medium_vec.push(Price::from_f64(50000.0 + i as f64)?);
                }
                
                // 3. HashMap operations (symbol lookups)
                let mut symbol_map: HashMap<String, f64> = HashMap::new();
                for i in 0..50 {
                    symbol_map.insert(format!("SYMBOL_{}", i), i as f64);
                }
                
                // 4. String operations (logging, serialization)
                let log_message = format!("Trade executed: {} shares at ${:.2}", 
                    small_vec.len(), medium_vec.len() as f64);
                
                // Verify allocations were successful
                if !small_vec.is_empty() && !medium_vec.is_empty() && 
                   !symbol_map.is_empty() && !log_message.is_empty() {
                    Ok(())
                } else {
                    Err(anyhow::anyhow!("Memory allocation failed"))
                }
            },
            self.config.measurement_samples,
        ).await?;

        Ok(measurement)
    }

    /// Test concurrent performance
    async fn test_concurrent_performance(&self) -> Result<PerformanceMeasurement> {
        let start_time = Instant::now();
        let mut latencies = Vec::new();
        let concurrent_ops = self.config.concurrent_operations;

        // Create concurrent tasks
        let mut tasks = Vec::with_capacity(concurrent_ops);
        let total_ops = Arc::new(AtomicU64::new(0));
        let successful_ops = Arc::new(AtomicU64::new(0));

        for i in 0..concurrent_ops {
            let total_ops_clone = Arc::clone(&total_ops);
            let successful_ops_clone = Arc::clone(&successful_ops);
            
            let task = tokio::spawn(async move {
                let task_start = Instant::now();
                
                // Simulate concurrent trading operations
                let symbol = format!("CONCURRENT_{}", i % 10);
                let quantity = 100.0 + (i as f64 % 900.0);
                let price = 50000.0 + (i as f64 % 1000.0);
                
                // Create order
                let order_result = Order::limit(
                    Symbol::from(symbol.as_str()),
                    if i % 2 == 0 { Side::Buy } else { Side::Sell },
                    Quantity::from_f64(quantity)?,
                    Price::from_f64(price)?,
                );
                
                total_ops_clone.fetch_add(1, Ordering::SeqCst);
                
                // Simulate order processing
                tokio::time::sleep(Duration::from_micros(10)).await;
                
                let task_latency = task_start.elapsed();
                successful_ops_clone.fetch_add(1, Ordering::SeqCst);
                
                Ok::<Duration, anyhow::Error>(task_latency)
            });
            
            tasks.push(task);
        }

        // Wait for all tasks and collect latencies
        let results = futures::future::join_all(tasks).await;
        
        for result in results {
            match result {
                Ok(Ok(latency)) => latencies.push(latency),
                Ok(Err(_)) => {}, // Task failed
                Err(_) => {},     // Task panicked
            }
        }

        let total_time = start_time.elapsed();
        let successful_count = successful_ops.load(Ordering::SeqCst);
        
        let mut measurement = PerformanceMeasurement::new("concurrent_operations".to_string(), latencies);
        measurement.success_rate = successful_count as f64 / concurrent_ops as f64;
        measurement.throughput_ops_per_sec = successful_count as f64 / total_time.as_secs_f64();

        Ok(measurement)
    }

    /// Test sustained throughput
    async fn test_sustained_throughput(&self) -> Result<PerformanceMeasurement> {
        let test_duration = Duration::from_millis(self.config.sustained_test_duration_ms);
        let start_time = Instant::now();
        let mut operation_count = 0;
        let mut latencies = Vec::new();

        while start_time.elapsed() < test_duration {
            let op_start = Instant::now();
            
            // Perform a representative trading operation
            let market_data = self.create_performance_market_data(operation_count)?;
            
            // Simulate signal generation
            let signal_strength = (market_data.price % 100.0) / 100.0;
            let side = if signal_strength > 0.5 { Side::Buy } else { Side::Sell };
            
            // Create order
            let _order = Order::market(
                market_data.symbol,
                side,
                Quantity::from_f64(signal_strength * 100.0)?,
            );
            
            let op_latency = op_start.elapsed();
            latencies.push(op_latency);
            operation_count += 1;
            
            // Small delay to prevent CPU saturation
            if operation_count % 100 == 0 {
                tokio::task::yield_now().await;
            }
        }

        let total_time = start_time.elapsed();
        let ops_per_second = operation_count as f64 / total_time.as_secs_f64();

        let mut measurement = PerformanceMeasurement::new("sustained_throughput".to_string(), latencies);
        measurement.throughput_ops_per_sec = ops_per_second;
        measurement.success_rate = 1.0; // All operations completed

        Ok(measurement)
    }

    /// Store measurement result
    fn store_measurement(&mut self, measurement: PerformanceMeasurement) {
        self.measurements.insert(measurement.operation_name.clone(), measurement);
    }

    /// Get performance summary
    fn get_performance_summary(&self) -> PerformanceSummary {
        let total_ops = self.total_operations.load(Ordering::SeqCst);
        let successful_ops = self.successful_operations.load(Ordering::SeqCst);
        let failed_ops = self.failed_operations.load(Ordering::SeqCst);

        PerformanceSummary {
            total_operations: total_ops,
            successful_operations: successful_ops,
            failed_operations: failed_ops,
            overall_success_rate: if total_ops > 0 { 
                successful_ops as f64 / total_ops as f64 
            } else { 
                0.0 
            },
            measurements: self.measurements.clone(),
        }
    }
}

/// Performance test summary
#[derive(Debug, Clone)]
struct PerformanceSummary {
    total_operations: u64,
    successful_operations: u64,
    failed_operations: u64,
    overall_success_rate: f64,
    measurements: HashMap<String, PerformanceMeasurement>,
}

impl PerformanceSummary {
    fn meets_hft_requirements(&self, config: &PerformanceTestConfig) -> bool {
        for measurement in self.measurements.values() {
            if !measurement.meets_latency_requirement(config.max_critical_latency_us) {
                return false;
            }
        }
        
        self.overall_success_rate >= 0.95 // 95% success rate requirement
    }

    fn log_summary(&self) {
        tracing::info!("=== PERFORMANCE TEST SUMMARY ===");
        tracing::info!("Total operations: {}", self.total_operations);
        tracing::info!("Successful: {}, Failed: {}", self.successful_operations, self.failed_operations);
        tracing::info!("Overall success rate: {:.2}%", self.overall_success_rate * 100.0);
        
        for measurement in self.measurements.values() {
            tracing::info!("--- {} ---", measurement.operation_name);
            tracing::info!("  Avg latency: {}μs", measurement.avg_latency.as_micros());
            tracing::info!("  P95 latency: {}μs", measurement.p95_latency.as_micros());
            tracing::info!("  P99 latency: {}μs", measurement.p99_latency.as_micros());
            tracing::info!("  Throughput: {:.0} ops/sec", measurement.throughput_ops_per_sec);
            tracing::info!("  Success rate: {:.2}%", measurement.success_rate * 100.0);
        }
    }
}

/// Test market data structure for performance testing
#[derive(Debug, Clone)]
struct TestMarketData {
    symbol: Symbol,
    price: f64,
    volume: f64,
    timestamp: u64,
}

impl TestMarketData {
    fn new(symbol: &str, price: f64, volume: f64) -> Result<Self> {
        Ok(Self {
            symbol: Symbol::from(symbol),
            price,
            volume,
            timestamp: std::time::SystemTime::now()
                .duration_since(std::time::UNIX_EPOCH)?
                .as_micros() as u64,
        })
    }
}

// ========== PERFORMANCE TESTS ==========

#[tokio::test]
async fn test_order_processing_performance() -> Result<()> {
    let mut test_suite = PerformanceTestSuite::new();
    test_suite.setup().await?;

    let measurement = test_suite.test_order_processing_latency().await?;
    test_suite.store_measurement(measurement.clone());

    // Validate latency requirements
    assert!(measurement.meets_latency_requirement(test_suite.config.max_critical_latency_us),
        "Order processing P99 latency {}μs exceeds requirement {}μs",
        measurement.p99_latency.as_micros(), test_suite.config.max_critical_latency_us);

    // Validate success rate
    assert!(measurement.success_rate >= 0.95,
        "Order processing success rate {:.2}% below 95% requirement",
        measurement.success_rate * 100.0);

    Ok(())
}

#[tokio::test]
async fn test_risk_calculation_performance() -> Result<()> {
    let mut test_suite = PerformanceTestSuite::new();
    test_suite.setup().await?;

    let measurement = test_suite.test_risk_calculation_latency().await?;
    test_suite.store_measurement(measurement.clone());

    // Validate latency requirements
    assert!(measurement.meets_latency_requirement(test_suite.config.max_critical_latency_us),
        "Risk calculation P99 latency {}μs exceeds requirement {}μs",
        measurement.p99_latency.as_micros(), test_suite.config.max_critical_latency_us);

    // Log performance metrics
    tracing::info!("Risk calculation performance:");
    tracing::info!("  Average latency: {}μs", measurement.avg_latency.as_micros());
    tracing::info!("  P99 latency: {}μs", measurement.p99_latency.as_micros());
    tracing::info!("  Success rate: {:.2}%", measurement.success_rate * 100.0);

    Ok(())
}

#[tokio::test]
async fn test_ml_inference_performance() -> Result<()> {
    let mut test_suite = PerformanceTestSuite::new();
    test_suite.setup().await?;

    let measurement = test_suite.test_ml_inference_latency().await?;
    test_suite.store_measurement(measurement.clone());

    // ML inference may have slightly higher latency tolerance
    assert!(measurement.meets_latency_requirement(test_suite.config.max_standard_latency_us),
        "ML inference P99 latency {}μs exceeds requirement {}μs",
        measurement.p99_latency.as_micros(), test_suite.config.max_standard_latency_us);

    // Validate that ML models are responsive
    assert!(measurement.success_rate > 0.0,
        "ML inference should have some successful predictions");

    Ok(())
}

#[tokio::test]
async fn test_memory_allocation_performance() -> Result<()> {
    if !PerformanceTestConfig::default().test_memory_performance {
        return Ok(());
    }

    let mut test_suite = PerformanceTestSuite::new();
    test_suite.setup().await?;

    let measurement = test_suite.test_memory_performance().await?;
    test_suite.store_measurement(measurement.clone());

    // Memory operations should be very fast
    assert!(measurement.meets_latency_requirement(test_suite.config.max_critical_latency_us),
        "Memory allocation P99 latency {}μs exceeds requirement {}μs",
        measurement.p99_latency.as_micros(), test_suite.config.max_critical_latency_us);

    // All memory operations should succeed
    assert_eq!(measurement.success_rate, 1.0,
        "Memory allocation success rate should be 100%");

    Ok(())
}

#[tokio::test]
async fn test_concurrent_operation_performance() -> Result<()> {
    let mut test_suite = PerformanceTestSuite::new();
    test_suite.setup().await?;

    let measurement = test_suite.test_concurrent_performance().await?;
    test_suite.store_measurement(measurement.clone());

    // Concurrent operations may have slightly higher latency
    assert!(measurement.meets_latency_requirement(test_suite.config.max_standard_latency_us),
        "Concurrent operations P99 latency {}μs exceeds requirement {}μs",
        measurement.p99_latency.as_micros(), test_suite.config.max_standard_latency_us);

    // Validate high success rate under concurrency
    assert!(measurement.success_rate >= 0.90,
        "Concurrent operations success rate {:.2}% below 90% requirement",
        measurement.success_rate * 100.0);

    // Validate throughput
    assert!(measurement.throughput_ops_per_sec >= 1000.0,
        "Concurrent throughput {:.0} ops/sec below 1000 ops/sec requirement",
        measurement.throughput_ops_per_sec);

    Ok(())
}

#[tokio::test]
async fn test_sustained_throughput_performance() -> Result<()> {
    let mut test_suite = PerformanceTestSuite::new();
    test_suite.setup().await?;

    let measurement = test_suite.test_sustained_throughput().await?;
    test_suite.store_measurement(measurement.clone());

    // Validate sustained throughput meets requirements
    assert!(measurement.throughput_ops_per_sec >= test_suite.config.target_throughput_ops as f64,
        "Sustained throughput {:.0} ops/sec below target {} ops/sec",
        measurement.throughput_ops_per_sec, test_suite.config.target_throughput_ops);

    // Validate latency remains acceptable under sustained load
    assert!(measurement.meets_latency_requirement(test_suite.config.max_standard_latency_us),
        "Sustained operations P99 latency {}μs exceeds requirement {}μs",
        measurement.p99_latency.as_micros(), test_suite.config.max_standard_latency_us);

    tracing::info!("Sustained throughput test completed: {:.0} ops/sec over {}ms",
        measurement.throughput_ops_per_sec, test_suite.config.sustained_test_duration_ms);

    Ok(())
}

#[tokio::test]
async fn test_end_to_end_trading_performance() -> Result<()> {
    let mut test_suite = PerformanceTestSuite::new();
    test_suite.setup().await?;

    // Test complete trading pipeline performance
    let measurement = test_suite.measure_operation_samples(
        "end_to_end_trading",
        || async {
            // 1. Market data processing
            let market_data = test_suite.create_performance_market_data(0)?;
            
            // 2. Signal generation (ML inference)
            let features = Features::new(
                vec![market_data.price, market_data.volume, market_data.timestamp as f64],
                vec!["price".to_string(), "volume".to_string(), "timestamp".to_string()],
            );
            
            let mut signal_strength = 0.5; // Default signal
            
            if let Some(ref registry) = test_suite.ml_registry {
                let predictions = registry.predict_all(&features).await;
                if let Some(Ok(first_prediction)) = predictions.into_iter().next() {
                    signal_strength = (first_prediction.value + 1.0) / 2.0; // Normalize to 0-1
                }
            }
            
            // 3. Risk validation
            let order_info = OrderInfo {
                symbol: market_data.symbol.clone(),
                side: if signal_strength > 0.5 { Side::Buy } else { Side::Sell },
                quantity: Quantity::from_f64(signal_strength * 100.0)?,
                price: Price::from_f64(market_data.price)?,
            };
            
            let risk_approved = if let Some(ref risk_engine) = test_suite.risk_engine {
                risk_engine.validate_order(&order_info).await.unwrap_or_else(|_| RiskCheckResult {
                    approved: false,
                    risk_score: 1.0,
                    violations: Vec::new(),
                    metadata: HashMap::new(),
                }).approved
            } else {
                // Simple risk check
                order_info.quantity.to_f64() * order_info.price.to_f64() < 10000.0
            };
            
            // 4. Order creation and submission
            if risk_approved {
                let _order = Order::limit(
                    order_info.symbol,
                    order_info.side,
                    order_info.quantity,
                    order_info.price,
                );
                Ok(())
            } else {
                // Risk rejection is a valid outcome
                Ok(())
            }
        },
        test_suite.config.measurement_samples / 2, // Fewer samples for complex operation
    ).await?;

    test_suite.store_measurement(measurement.clone());

    // End-to-end should meet critical latency requirements
    assert!(measurement.meets_latency_requirement(test_suite.config.max_critical_latency_us),
        "End-to-end trading P99 latency {}μs exceeds requirement {}μs",
        measurement.p99_latency.as_micros(), test_suite.config.max_critical_latency_us);

    Ok(())
}

#[tokio::test]
async fn test_system_resource_utilization() -> Result<()> {
    let mut test_suite = PerformanceTestSuite::new();
    test_suite.setup().await?;

    // Test resource utilization under load
    let start_time = Instant::now();
    let initial_memory = get_approximate_memory_usage();
    
    // Perform operations that stress different system resources
    let operations = vec![
        test_suite.test_order_processing_latency(),
        test_suite.test_risk_calculation_latency(),
        test_suite.test_ml_inference_latency(),
        test_suite.test_memory_performance(),
    ];

    // Run all operations concurrently
    let (order_result, risk_result, ml_result, memory_result) = 
        futures::future::try_join4(
            operations[0],
            operations[1],
            operations[2],
            operations[3],
        ).await?;

    let total_time = start_time.elapsed();
    let final_memory = get_approximate_memory_usage();

    // Store all measurements
    test_suite.store_measurement(order_result);
    test_suite.store_measurement(risk_result);
    test_suite.store_measurement(ml_result);
    test_suite.store_measurement(memory_result);

    // Validate resource usage
    let memory_growth = final_memory.saturating_sub(initial_memory);
    assert!(memory_growth < 100 * 1024 * 1024, // 100MB limit
        "Memory usage grew by {}MB, exceeding 100MB limit", memory_growth / (1024 * 1024));

    // Validate total execution time
    assert!(total_time.as_secs() < 30,
        "Resource utilization test took {}s, exceeding 30s limit", total_time.as_secs());

    tracing::info!("Resource utilization test completed in {}ms with {}MB memory growth",
        total_time.as_millis(), memory_growth / (1024 * 1024));

    Ok(())
}

#[tokio::test]
async fn test_comprehensive_performance_validation() -> Result<()> {
    let mut test_suite = PerformanceTestSuite::new();
    test_suite.setup().await?;

    // Run comprehensive performance test suite
    let tests = vec![
        ("order_processing", test_suite.test_order_processing_latency()),
        ("risk_calculation", test_suite.test_risk_calculation_latency()),
        ("ml_inference", test_suite.test_ml_inference_latency()),
        ("concurrent_ops", test_suite.test_concurrent_performance()),
        ("sustained_throughput", test_suite.test_sustained_throughput()),
    ];

    for (test_name, test_future) in tests {
        let measurement = test_future.await?;
        test_suite.store_measurement(measurement);
        tracing::info!("Completed performance test: {}", test_name);
    }

    // Generate comprehensive summary
    let summary = test_suite.get_performance_summary();
    summary.log_summary();

    // Validate overall HFT requirements
    assert!(summary.meets_hft_requirements(&test_suite.config),
        "System does not meet HFT performance requirements");

    // Validate individual critical operations
    for (operation_name, measurement) in &summary.measurements {
        if operation_name.contains("order") || operation_name.contains("risk") {
            assert!(measurement.meets_latency_requirement(test_suite.config.max_critical_latency_us),
                "Critical operation '{}' P99 latency {}μs exceeds {}μs requirement",
                operation_name, measurement.p99_latency.as_micros(), test_suite.config.max_critical_latency_us);
        }
    }

    tracing::info!("🎉 All performance tests passed! System meets HFT requirements.");

    Ok(())
}

// ========== UTILITY FUNCTIONS ==========

/// Approximate memory usage (placeholder implementation)
fn get_approximate_memory_usage() -> usize {
    // This is a placeholder - in a real implementation you'd use system APIs
    // to get actual memory usage
    std::process::id() as usize * 1024 // Rough approximation
}

/// Create performance test dataset
fn create_performance_dataset(size: usize) -> Result<Vec<TestMarketData>> {
    let mut dataset = Vec::with_capacity(size);
    
    for i in 0..size {
        dataset.push(TestMarketData::new(
            &format!("PERF_{}", i % 100),
            50000.0 + (i as f64 % 1000.0),
            1000.0 + (i as f64 % 500.0),
        )?);
    }
    
    Ok(dataset)
}

/// Validate performance requirements for production deployment
fn validate_production_readiness(summary: &PerformanceSummary) -> Result<()> {
    // Critical requirements for production deployment
    let requirements = vec![
        ("Overall success rate", summary.overall_success_rate >= 0.99),
        ("Total operations", summary.total_operations >= 1000),
    ];

    for (requirement, passes) in requirements {
        if !passes {
            return Err(anyhow::anyhow!("Production requirement failed: {}", requirement));
        }
    }

    // Validate each measurement meets production standards
    for (operation, measurement) in &summary.measurements {
        if measurement.p99_latency.as_micros() > 100 {
            tracing::warn!("Operation '{}' has high P99 latency: {}μs", 
                operation, measurement.p99_latency.as_micros());
        }
    }

    Ok(())
}

/// Performance test configuration for different environments
impl PerformanceTestConfig {
    fn for_development() -> Self {
        Self {
            max_critical_latency_us: 100,  // Relaxed for development
            max_standard_latency_us: 200,
            target_throughput_ops: 1000,   // Lower target
            sustained_test_duration_ms: 2000, // Shorter tests
            concurrent_operations: 50,      // Fewer concurrent ops
            measurement_samples: 100,       // Fewer samples
            ..Default::default()
        }
    }

    fn for_production() -> Self {
        Self {
            max_critical_latency_us: 50,   // Strict production requirement
            max_standard_latency_us: 100,
            target_throughput_ops: 10000,  // Full production target
            sustained_test_duration_ms: 10000, // Longer stress tests
            concurrent_operations: 200,     // High concurrency
            measurement_samples: 2000,      // More samples for accuracy
            ..Default::default()
        }
    }
}