foxhunt/crates/ml/src/benchmarks.rs

//! ML Model Benchmark Suite
//!
//! Comprehensive benchmarking for all ML models with sub-50μs inference targets.
//! Validates GPU acceleration and performance requirements for HFT systems.
//!
//! ## Metrics Coverage
//!
//! **Implemented:**
//! - System memory (sysinfo)
//! - Disk space for working directory (sysinfo)
//! - CPU temperature (platform-specific, via sysinfo)
//! - Latency statistics (p50/p95/p99)
//! - Throughput (predictions/second)
//! - Compilation time
//!
//! **Known Limitations:**
//! - GPU utilization: Requires vendor-specific APIs (NVIDIA NVML, AMD ROCm SMI)
//! - Network bandwidth: Requires system-level network interface monitoring
//! - Batch size correlation: Future enhancement for performance tuning analysis
//! - Ensemble size tradeoff: N/A (no ensemble models in current architecture)

// For canonical types

use std::fmt::Write as _;
use std::time::Instant;

use anyhow::Result;
use ml_core::cuda_autograd::GpuTensor;
use ml_core::device::MlDevice;
use ml_core::native_types::NativeDevice;
use tracing::{info, warn};

use crate::liquid::{LiquidNetwork, LiquidNetworkConfig, NetworkType};
use crate::mamba::{Mamba2Config, Mamba2SSM};
use crate::ppo::{ContinuousPPO, ContinuousPPOConfig};
use crate::tft::{TFTConfig, TemporalFusionTransformer};
use crate::tlob::{TLOBConfig, TLOBTransformer};
use crate::MLError;

/// Benchmark configuration
#[derive(Debug, Clone)]
pub struct BenchmarkConfig {
    pub warmup_runs: usize,
    pub test_runs: usize,
    pub batch_size: usize,
    pub target_latency_us: u64,
    pub enable_gpu: bool,
}

impl Default for BenchmarkConfig {
    fn default() -> Self {
        Self {
            warmup_runs: 100,
            test_runs: 1000,
            batch_size: 1,
            target_latency_us: 50,
            enable_gpu: true,
        }
    }
}

/// Benchmark results for a single model
#[derive(Debug, Clone)]
pub struct ModelBenchmarkResults {
    pub model_name: String,
    pub device: String,
    pub avg_latency_us: f64,
    pub p50_latency_us: f64,
    pub p95_latency_us: f64,
    pub p99_latency_us: f64,
    pub max_latency_us: f64,
    pub min_latency_us: f64,
    pub throughput_pps: f64,
    pub target_met: bool,
    pub compilation_time_ms: f64,
    pub memory_usage_mb: f64,
    pub gpu_utilization_percent: f64,
}

/// Complete benchmark suite results
#[derive(Debug, Clone)]
pub struct BenchmarkSuite {
    pub results: Vec<ModelBenchmarkResults>,
    pub system_info: SystemInfo,
    pub gpu_info: Option<GpuInfo>,
    pub total_duration_ms: f64,
}

#[derive(Debug, Clone)]
pub struct SystemInfo {
    pub cpu_count: usize,
    pub available_memory_gb: f64,
    pub os: String,
    pub architecture: String,
    pub disk_total_gb: f64,
    pub disk_available_gb: f64,
    pub cpu_temperature_celsius: Option<f32>,
}

#[derive(Debug, Clone)]
pub struct GpuInfo {
    pub name: String,
    pub memory_gb: f64,
    pub compute_capability: String,
    pub driver_version: String,
}

/// Main benchmark runner
#[derive(Debug)]
pub struct MLBenchmarkRunner {
    config: BenchmarkConfig,
    device: NativeDevice,
}

impl MLBenchmarkRunner {
    pub fn new(config: BenchmarkConfig) -> Result<Self, MLError> {
        // Validate CUDA is available
        let _ml_dev = MlDevice::cuda(0)?;
        let device = NativeDevice::Cuda(0);

        info!("Initialized ML Benchmark Runner on device: {:?}", device);

        Ok(Self { config, device })
    }

    /// Run complete benchmark suite
    pub async fn run_full_benchmark(&self) -> Result<BenchmarkSuite, MLError> {
        info!("Starting ML model benchmark suite");
        let start_time = Instant::now();

        let mut results = Vec::new();

        // Benchmark MAMBA-2 SSM
        if let Ok(mamba_results) = self.benchmark_mamba2().await {
            results.push(mamba_results);
        }

        // Benchmark PPO
        if let Ok(ppo_results) = self.benchmark_ppo().await {
            results.push(ppo_results);
        }

        // Benchmark TLOB Transformer
        if let Ok(tlob_results) = self.benchmark_tlob().await {
            results.push(tlob_results);
        }

        // Benchmark TFT
        if let Ok(tft_results) = self.benchmark_tft().await {
            results.push(tft_results);
        }

        // Benchmark Liquid Networks
        if let Ok(liquid_results) = self.benchmark_liquid().await {
            results.push(liquid_results);
        }

        let total_duration = start_time.elapsed().as_millis() as f64;

        let suite = BenchmarkSuite {
            results,
            system_info: self.get_system_info(),
            gpu_info: self.get_gpu_info(),
            total_duration_ms: total_duration,
        };

        info!(
            "Benchmark suite completed in {:.2}ms with {} models",
            total_duration,
            suite.results.len()
        );

        Ok(suite)
    }

    /// Benchmark `MAMBA-2` SSM
    pub async fn benchmark_mamba2(&self) -> Result<ModelBenchmarkResults, MLError> {
        info!("Benchmarking MAMBA-2 SSM");

        let config = Mamba2Config {
            d_model: 256,
            d_state: 32,
            num_layers: 4,
            target_latency_us: self.config.target_latency_us,
            batch_size: self.config.batch_size,
            hardware_aware: true,
            use_ssd: true,
            use_selective_state: true,
            ..Default::default()
        };

        let compilation_start = Instant::now();
        let ml_dev = MlDevice::cuda(0)?;
        let stream = ml_dev.cuda_stream()?;
        let mut model = Mamba2SSM::new(config, stream)?;
        let compilation_time = compilation_start.elapsed().as_millis() as f64;

        // Generate test data
        let input_data: Vec<f64> = (0..256).map(|i| (i as f64) / 256.0).collect();

        // Warmup
        for _ in 0..self.config.warmup_runs {
            let _ = model.predict_single_fast(&input_data)?;
        }

        // Benchmark
        let mut latencies = Vec::new();
        for _ in 0..self.config.test_runs {
            let start = Instant::now();
            let _ = model.predict_single_fast(&input_data)?;
            latencies.push(start.elapsed().as_micros() as f64);
        }

        Ok(self.calculate_results("MAMBA-2 SSM", latencies, compilation_time))
    }



    /// Benchmark `PPO`
    pub async fn benchmark_ppo(&self) -> Result<ModelBenchmarkResults, MLError> {
        info!("Benchmarking PPO");

        let mut config = ContinuousPPOConfig::default();
        config.state_dim = 32;
        config.policy_learning_rate = 1e-4;
        config.value_learning_rate = 1e-4;
        config.batch_size = self.config.batch_size;

        let compilation_start = Instant::now();
        let ppo = ContinuousPPO::new(config)?;
        let compilation_time = compilation_start.elapsed().as_millis() as f64;

        // Generate test data
        let state = vec![0.1_f32; 32]; // PPO expects &[f32]

        // Warmup
        for _ in 0..self.config.warmup_runs {
            let _ = ppo.act(&state)?;
        }

        // Benchmark
        let mut latencies = Vec::new();
        for _ in 0..self.config.test_runs {
            let start = Instant::now();
            let _ = ppo.act(&state)?;
            latencies.push(start.elapsed().as_micros() as f64);
        }

        Ok(self.calculate_results("PPO", latencies, compilation_time))
    }

    /// Benchmark TLOB Transformer
    pub async fn benchmark_tlob(&self) -> Result<ModelBenchmarkResults, MLError> {
        info!("Benchmarking TLOB Transformer");

        let mut config = TLOBConfig::default();
        config.feature_dim = 20;
        config.batch_size = self.config.batch_size;

        let compilation_start = Instant::now();
        let tlob = TLOBTransformer::new(config)?;
        let compilation_time = compilation_start.elapsed().as_millis() as f64;

        // Generate test order book data - using transformer TLOBFeatures
        let tlob_features = crate::tlob::transformer::TLOBFeatures {
            timestamp: 1642531200000,
            bid_prices: [100000; 10],
            ask_prices: [100010; 10],
            bid_sizes: [1000; 10],
            ask_sizes: [1000; 10],
            trade_price: 100005,
            trade_size: 500,
            spread: 10,
            mid_price: 100005,
            microstructure_features: [1, 2, 3],
        };

        // Warmup
        for _ in 0..self.config.warmup_runs {
            let _ = tlob.predict(&tlob_features)?;
        }

        // Benchmark
        let mut latencies = Vec::new();
        for _ in 0..self.config.test_runs {
            let start = Instant::now();
            let _ = tlob.predict(&tlob_features)?;
            latencies.push(start.elapsed().as_micros() as f64);
        }

        Ok(self.calculate_results("TLOB Transformer", latencies, compilation_time))
    }

    /// Benchmark `TFT`
    pub async fn benchmark_tft(&self) -> Result<ModelBenchmarkResults, MLError> {
        info!("Benchmarking Temporal Fusion Transformer");

        let config = TFTConfig {
            input_dim: 64,
            hidden_dim: 128,
            num_heads: 8,
            num_layers: 3,
            prediction_horizon: 10,
            sequence_length: 50,
            num_static_features: 5,
            num_known_features: 10,
            num_unknown_features: 20,
            batch_size: self.config.batch_size,
            max_inference_latency_us: self.config.target_latency_us,
            ..Default::default()
        };

        let compilation_start = Instant::now();
        let mut tft = TemporalFusionTransformer::new(config)?;
        let compilation_time = compilation_start.elapsed().as_millis() as f64;

        // Generate test data
        let static_features: Vec<f32> = (0..5).map(|i| i as f32 / 5.0).collect();
        let historical_features: Vec<f32> = (0..1000).map(|i| i as f32 / 1000.0).collect(); // 50x20
        let future_features: Vec<f32> = (0..100).map(|i| i as f32 / 100.0).collect(); // 10x10

        // Warmup
        for _ in 0..self.config.warmup_runs {
            let _ = tft.predict_fast(&static_features, &historical_features, &future_features)?;
        }

        // Benchmark
        let mut latencies = Vec::new();
        for _ in 0..self.config.test_runs {
            let start = Instant::now();
            let _ = tft.predict_fast(&static_features, &historical_features, &future_features)?;
            latencies.push(start.elapsed().as_micros() as f64);
        }

        Ok(self.calculate_results("TFT", latencies, compilation_time))
    }

    /// Benchmark Liquid Networks
    pub async fn benchmark_liquid(&self) -> Result<ModelBenchmarkResults, MLError> {
        info!("Benchmarking Liquid Neural Networks");

        use crate::liquid::{ActivationType, FixedPoint, LTCConfig, SolverType, PRECISION};

        let ltc_config = LTCConfig {
            input_size: 32,
            hidden_size: 64,
            tau_min: FixedPoint(PRECISION / 100),
            tau_max: FixedPoint(PRECISION),
            use_bias: true,
            solver_type: SolverType::Euler,
            activation: ActivationType::Tanh,
        };

        let config = LiquidNetworkConfig {
            network_type: NetworkType::CfC,
            input_size: 32,
            output_size: 4,
            layer_configs: vec![crate::liquid::LayerConfig::LTC(ltc_config)],
            output_layer: crate::liquid::OutputLayerConfig {
                use_linear_output: true,
                output_activation: None,
                dropout_rate: None,
            },
            default_dt: FixedPoint(PRECISION / 100),
            market_regime_adaptation: true,
        };

        let compilation_start = Instant::now();
        let mut liquid = LiquidNetwork::new(config)?;
        let compilation_time = compilation_start.elapsed().as_millis() as f64;

        // Generate test data
        let input_data: Vec<f64> = (0..32).map(|i| (i as f64) / 32.0).collect();

        // Warmup
        for _ in 0..self.config.warmup_runs {
            let _ = liquid.predict(&input_data)?;
        }

        // Benchmark
        let mut latencies = Vec::new();
        for _ in 0..self.config.test_runs {
            let start = Instant::now();
            let _ = liquid.predict(&input_data)?;
            latencies.push(start.elapsed().as_micros() as f64);
        }

        Ok(self.calculate_results("Liquid Networks", latencies, compilation_time))
    }

    fn calculate_results(
        &self,
        model_name: &str,
        mut latencies: Vec<f64>,
        compilation_time_ms: f64,
    ) -> ModelBenchmarkResults {
        latencies.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));

        let avg_latency = latencies.iter().sum::<f64>() / latencies.len() as f64;
        let min_latency = latencies.first().copied().unwrap_or(0.0);
        let max_latency = latencies.last().copied().unwrap_or(0.0);

        let p50_idx = latencies.len() / 2;
        let p95_idx = (latencies.len() * 95) / 100;
        let p99_idx = (latencies.len() * 99) / 100;

        let p50_latency = latencies.get(p50_idx).copied().unwrap_or(0.0);
        let p95_latency = latencies
            .get(p95_idx.min(latencies.len().saturating_sub(1)))
            .copied()
            .unwrap_or(0.0);
        let p99_latency = latencies
            .get(p99_idx.min(latencies.len().saturating_sub(1)))
            .copied()
            .unwrap_or(0.0);

        let throughput = if avg_latency > 0.0 {
            1_000_000.0 / avg_latency // predictions per second
        } else {
            0.0
        };

        let target_met = avg_latency <= self.config.target_latency_us as f64;

        let device_name = if self.device.is_cuda() {
            "CUDA"
        } else {
            "CPU"
        }
        .to_string();

        if !target_met {
            warn!(
                "{} average latency {:.2}μs exceeds target {}μs",
                model_name, avg_latency, self.config.target_latency_us
            );
        } else {
            info!(
                "{} meets target: {:.2}μs average latency, {:.0} pps throughput",
                model_name, avg_latency, throughput
            );
        }

        ModelBenchmarkResults {
            model_name: model_name.to_string(),
            device: device_name,
            avg_latency_us: avg_latency,
            p50_latency_us: p50_latency,
            p95_latency_us: p95_latency,
            p99_latency_us: p99_latency,
            max_latency_us: max_latency,
            min_latency_us: min_latency,
            throughput_pps: throughput,
            target_met,
            compilation_time_ms,
            memory_usage_mb: 0.0, // Known limitation: Memory measurement requires platform-specific profiling
            gpu_utilization_percent: 0.0, // Known limitation: GPU utilization requires vendor-specific APIs (NVIDIA CUDA: nvidia-ml-py/NVML, AMD ROCm: rocm-smi)
        }
    }

    fn get_system_info(&self) -> SystemInfo {
        use sysinfo::System;
        let mut sys = System::new_all();
        sys.refresh_all();
        let total_memory_kb = sys.total_memory();
        let available_memory_gb = (total_memory_kb as f64) / (1024.0 * 1024.0);

        // Disk space measurement for working directory
        let (disk_total_gb, disk_available_gb) = Self::get_disk_space();

        // CPU temperature measurement (platform-specific)
        let cpu_temperature_celsius = Self::get_cpu_temperature();

        SystemInfo {
            cpu_count: num_cpus::get(),
            available_memory_gb,
            os: std::env::consts::OS.to_string(),
            architecture: std::env::consts::ARCH.to_string(),
            disk_total_gb,
            disk_available_gb,
            cpu_temperature_celsius,
        }
    }

    /// Get disk space for working directory
    /// Returns (total_gb, available_gb)
    fn get_disk_space() -> (f64, f64) {
        use sysinfo::Disks;

        let disks = Disks::new_with_refreshed_list();
        let current_dir = std::env::current_dir().ok();

        if let Some(dir) = current_dir {
            for disk in &disks {
                if dir.starts_with(disk.mount_point()) {
                    let total_gb = disk.total_space() as f64 / (1024.0 * 1024.0 * 1024.0);
                    let available_gb = disk.available_space() as f64 / (1024.0 * 1024.0 * 1024.0);
                    return (total_gb, available_gb);
                }
            }
        }

        // Return 0.0 if unable to determine disk space
        (0.0, 0.0)
    }

    /// Get CPU temperature (platform-specific)
    /// Returns Some(temperature_celsius) if available, None otherwise
    ///
    /// Known Limitation: CPU temperature availability is platform-specific:
    /// - Linux: Requires lm_sensors or kernel interfaces
    /// - Windows: Uses WMI if available
    /// - macOS: Uses IOKit if available
    /// - Virtual machines may not expose temperature sensors
    fn get_cpu_temperature() -> Option<f32> {
        use sysinfo::Components;

        let components = Components::new_with_refreshed_list();

        // Look for CPU or package temperature sensors
        components.iter().find_map(|component| {
            let label = component.label().to_lowercase();
            if label.contains("cpu") || label.contains("package") || label.contains("core") {
                component.temperature()
            } else {
                None
            }
        })
    }

    fn get_gpu_info(&self) -> Option<GpuInfo> {
        // Known limitation: CUDA device introspection requires nvidia-ml-py or CUDA runtime APIs
        self.device.is_cuda().then(|| GpuInfo {
            name: "CUDA Device".to_owned(),
            memory_gb: 8.0,
            compute_capability: "8.0".to_owned(),
            driver_version: "Unknown".to_owned(),
        })
    }

    /// Generate performance report
    pub fn generate_report(&self, suite: &BenchmarkSuite) -> String {
        let mut report = String::new();
        report.push_str("# ML Model Performance Benchmark Report\n\n");

        // System Information
        report.push_str("## System Information\n");
        _ = writeln!(report, "- OS: {}", suite.system_info.os);
        _ = writeln!(
            report,
            "- Architecture: {}",
            suite.system_info.architecture
        );
        _ = writeln!(report, "- CPU Cores: {}", suite.system_info.cpu_count);
        _ = writeln!(
            report,
            "- Available Memory: {:.1} GB",
            suite.system_info.available_memory_gb
        );
        _ = writeln!(
            report,
            "- Disk Space: {:.1} GB total, {:.1} GB available",
            suite.system_info.disk_total_gb, suite.system_info.disk_available_gb
        );
        if let Some(temp) = suite.system_info.cpu_temperature_celsius {
            _ = writeln!(report, "- CPU Temperature: {:.1}°C", temp);
        }

        if let Some(gpu) = &suite.gpu_info {
            _ = writeln!(report, "- GPU: {}", gpu.name);
            _ = writeln!(report, "- GPU Memory: {:.1} GB", gpu.memory_gb);
            _ = writeln!(
                report,
                "- Compute Capability: {}",
                gpu.compute_capability
            );
        }

        _ = writeln!(report, "\n## Benchmark Configuration");
        _ = writeln!(
            report,
            "- Target Latency: {}μs",
            self.config.target_latency_us
        );
        _ = writeln!(report, "- Test Runs: {}", self.config.test_runs);
        _ = writeln!(report, "- Warmup Runs: {}", self.config.warmup_runs);
        _ = writeln!(report, "- Batch Size: {}", self.config.batch_size);

        report.push_str("\n## Performance Results\n\n");
        report.push_str("| Model | Device | Avg (μs) | P95 (μs) | P99 (μs) | Max (μs) | Throughput (pps) | Target Met |\n");
        report.push_str("|-------|--------|----------|----------|----------|----------|------------------|------------|\n");

        for result in &suite.results {
            let target_met = if result.target_met { "✅" } else { "❌" };
            _ = writeln!(
                report,
                "| {} | {} | {:.1} | {:.1} | {:.1} | {:.1} | {:.0} | {} |",
                result.model_name,
                result.device,
                result.avg_latency_us,
                result.p95_latency_us,
                result.p99_latency_us,
                result.max_latency_us,
                result.throughput_pps,
                target_met
            );
        }

        let models_meeting_target = suite.results.iter().filter(|r| r.target_met).count();
        _ = writeln!(
            report,
            "\n**Summary**: {}/{} models meet the <{}μs latency target",
            models_meeting_target,
            suite.results.len(),
            self.config.target_latency_us
        );

        _ = writeln!(
            report,
            "\nTotal benchmark time: {:.2}ms",
            suite.total_duration_ms
        );

        report
    }
}

/// Quick `GPU` capability test
pub fn test_gpu_acceleration() -> Result<bool, MLError> {
    info!("Testing GPU acceleration capabilities");

    let ml_dev = MlDevice::cuda(0)?;
    let stream = ml_dev.cuda_stream()?;
    info!("CUDA GPU detected and available");

    // Test basic tensor allocation on GPU
    let data_a: Vec<f32> = (0..1_000_000).map(|_| fastrand::f32()).collect();
    let _a = GpuTensor::from_host(&data_a, vec![1000, 1000], stream).map_err(|e| {
        MLError::TensorCreationError {
            operation: "GPU test tensor A".to_owned(),
            reason: e.to_string(),
        }
    })?;

    let start = Instant::now();
    let data_b: Vec<f32> = (0..1_000_000).map(|_| fastrand::f32()).collect();
    let _b = GpuTensor::from_host(&data_b, vec![1000, 1000], stream).map_err(|e| {
        MLError::TensorCreationError {
            operation: "GPU test tensor B".to_owned(),
            reason: e.to_string(),
        }
    })?;
    let gpu_time = start.elapsed();

    info!("GPU tensor allocation test completed in {:?}", gpu_time);
    Ok(true)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn test_benchmark_runner_creation() {
        let config = BenchmarkConfig::default();
        let runner = MLBenchmarkRunner::new(config).unwrap();
        assert!(runner.config.test_runs > 0);
    }

    #[tokio::test]
    async fn test_gpu_detection() {
        let result = test_gpu_acceleration();
        assert!(result.is_ok());
    }

    #[test]
    fn test_benchmark_config_default() {
        let config = BenchmarkConfig::default();
        assert_eq!(config.target_latency_us, 50);
        assert!(config.test_runs > 0);
        assert!(config.warmup_runs > 0);
    }
}