foxhunt/tests/harness/performance.rs

//! Performance Monitoring for Integration Tests
//! 
//! Tracks latency, throughput, and resource utilization during test execution
//! to ensure HFT performance requirements are met and detect regressions.

use std::collections::HashMap;
use std::time::{Duration, Instant};
use serde::{Deserialize, Serialize};
use anyhow::Result;

/// Performance monitor for tracking test execution metrics
pub struct PerformanceMonitor {
    scenarios: HashMap<String, ScenarioTracker>,
    baseline: Option<PerformanceBaseline>,
}

impl PerformanceMonitor {
    pub fn new() -> Self {
        Self {
            scenarios: HashMap::new(),
            baseline: None,
        }
    }

    /// Load performance baseline from file for regression testing
    pub async fn load_baseline(mut self, path: &str) -> Result<Self> {
        if let Ok(content) = tokio::fs::read_to_string(path).await {
            if let Ok(baseline) = serde_json::from_str::<PerformanceBaseline>(&content) {
                self.baseline = Some(baseline);
            }
        }
        Ok(self)
    }

    /// Start monitoring a test scenario
    pub fn start_scenario(&mut self, name: &str) {
        let tracker = ScenarioTracker::new();
        self.scenarios.insert(name.to_string(), tracker);
    }

    /// End monitoring and finalize metrics
    pub fn end_scenario(&mut self, name: &str) {
        if let Some(tracker) = self.scenarios.get_mut(name) {
            tracker.finalize();
        }
    }

    /// Record a latency measurement
    pub fn record_latency(&mut self, scenario: &str, operation: &str, duration: Duration) {
        if let Some(tracker) = self.scenarios.get_mut(scenario) {
            tracker.record_latency(operation, duration);
        }
    }

    /// Record throughput measurement
    pub fn record_throughput(&mut self, scenario: &str, operation: &str, count: u64, duration: Duration) {
        if let Some(tracker) = self.scenarios.get_mut(scenario) {
            tracker.record_throughput(operation, count, duration);
        }
    }

    /// Record resource utilization
    pub fn record_resource_usage(&mut self, scenario: &str, cpu_percent: f64, memory_mb: f64, gpu_percent: Option<f64>) {
        if let Some(tracker) = self.scenarios.get_mut(scenario) {
            tracker.record_resource_usage(cpu_percent, memory_mb, gpu_percent);
        }
    }

    /// Get metrics for a scenario
    pub fn get_metrics(&self, scenario: &str) -> ScenarioMetrics {
        self.scenarios.get(scenario)
            .map(|tracker| tracker.get_metrics())
            .unwrap_or_default()
    }

    /// Check if performance meets baseline requirements
    pub fn check_regression(&self, scenario: &str) -> RegressionResult {
        let current_metrics = self.get_metrics(scenario);
        
        if let Some(baseline) = &self.baseline {
            if let Some(baseline_scenario) = baseline.scenarios.get(scenario) {
                return RegressionResult::compare(&current_metrics, baseline_scenario);
            }
        }

        RegressionResult::NoBaseline
    }

    /// Save current metrics as new baseline
    pub async fn save_baseline(&self, path: &str) -> Result<()> {
        let baseline = PerformanceBaseline {
            created_at: chrono::Utc::now(),
            scenarios: self.scenarios.iter()
                .map(|(name, tracker)| (name.clone(), tracker.get_metrics()))
                .collect(),
        };

        let content = serde_json::to_string_pretty(&baseline)?;
        tokio::fs::write(path, content).await?;
        Ok(())
    }
}

/// Tracks performance metrics for a single test scenario
struct ScenarioTracker {
    start_time: Instant,
    end_time: Option<Instant>,
    latencies: HashMap<String, Vec<Duration>>,
    throughputs: HashMap<String, Vec<ThroughputMeasurement>>,
    resource_usage: Vec<ResourceUsage>,
}

impl ScenarioTracker {
    fn new() -> Self {
        Self {
            start_time: Instant::now(),
            end_time: None,
            latencies: HashMap::new(),
            throughputs: HashMap::new(),
            resource_usage: Vec::new(),
        }
    }

    fn finalize(&mut self) {
        self.end_time = Some(Instant::now());
    }

    fn record_latency(&mut self, operation: &str, duration: Duration) {
        self.latencies.entry(operation.to_string())
            .or_insert_with(Vec::new)
            .push(duration);
    }

    fn record_throughput(&mut self, operation: &str, count: u64, duration: Duration) {
        let measurement = ThroughputMeasurement { count, duration };
        self.throughputs.entry(operation.to_string())
            .or_insert_with(Vec::new)
            .push(measurement);
    }

    fn record_resource_usage(&mut self, cpu_percent: f64, memory_mb: f64, gpu_percent: Option<f64>) {
        self.resource_usage.push(ResourceUsage {
            timestamp: Instant::now(),
            cpu_percent,
            memory_mb,
            gpu_percent,
        });
    }

    fn get_metrics(&self) -> ScenarioMetrics {
        let total_duration = self.end_time
            .unwrap_or_else(Instant::now)
            .duration_since(self.start_time);

        let latency_stats = self.latencies.iter()
            .map(|(op, durations)| (op.clone(), calculate_latency_stats(durations)))
            .collect();

        let throughput_stats = self.throughputs.iter()
            .map(|(op, measurements)| (op.clone(), calculate_throughput_stats(measurements)))
            .collect();

        let resource_stats = calculate_resource_stats(&self.resource_usage);

        ScenarioMetrics {
            total_duration,
            latency_stats,
            throughput_stats,
            resource_stats,
        }
    }
}

/// Performance metrics for a test scenario
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct ScenarioMetrics {
    pub total_duration: Duration,
    pub latency_stats: HashMap<String, LatencyStats>,
    pub throughput_stats: HashMap<String, ThroughputStats>,
    pub resource_stats: ResourceStats,
}

/// Latency statistics for an operation
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LatencyStats {
    pub min: Duration,
    pub max: Duration,
    pub mean: Duration,
    pub p50: Duration,
    pub p95: Duration,
    pub p99: Duration,
    pub p999: Duration,
    pub sample_count: usize,
}

/// Throughput statistics for an operation
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ThroughputStats {
    pub operations_per_second: f64,
    pub max_ops_per_second: f64,
    pub min_ops_per_second: f64,
    pub total_operations: u64,
    pub sample_count: usize,
}

/// Resource utilization statistics
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct ResourceStats {
    pub cpu_percent_avg: f64,
    pub cpu_percent_max: f64,
    pub memory_mb_avg: f64,
    pub memory_mb_max: f64,
    pub gpu_percent_avg: Option<f64>,
    pub gpu_percent_max: Option<f64>,
}

#[derive(Debug, Clone)]
struct ThroughputMeasurement {
    count: u64,
    duration: Duration,
}

#[derive(Debug, Clone)]
struct ResourceUsage {
    timestamp: Instant,
    cpu_percent: f64,
    memory_mb: f64,
    gpu_percent: Option<f64>,
}

/// Performance baseline for regression testing
#[derive(Debug, Serialize, Deserialize)]
pub struct PerformanceBaseline {
    pub created_at: chrono::DateTime<chrono::Utc>,
    pub scenarios: HashMap<String, ScenarioMetrics>,
}

/// Result of regression analysis
#[derive(Debug)]
pub enum RegressionResult {
    NoRegression,
    LatencyRegression { operation: String, increase_percent: f64 },
    ThroughputRegression { operation: String, decrease_percent: f64 },
    ResourceRegression { resource: String, increase_percent: f64 },
    NoBaseline,
}

impl RegressionResult {
    fn compare(current: &ScenarioMetrics, baseline: &ScenarioMetrics) -> Self {
        const REGRESSION_THRESHOLD_PERCENT: f64 = 10.0; // 10% increase is considered regression

        // Check latency regressions
        for (operation, current_stats) in &current.latency_stats {
            if let Some(baseline_stats) = baseline.latency_stats.get(operation) {
                let increase_percent = ((current_stats.p95.as_nanos() as f64 / baseline_stats.p95.as_nanos() as f64) - 1.0) * 100.0;
                if increase_percent > REGRESSION_THRESHOLD_PERCENT {
                    return RegressionResult::LatencyRegression {
                        operation: operation.clone(),
                        increase_percent,
                    };
                }
            }
        }

        // Check throughput regressions
        for (operation, current_stats) in &current.throughput_stats {
            if let Some(baseline_stats) = baseline.throughput_stats.get(operation) {
                let decrease_percent = ((baseline_stats.operations_per_second / current_stats.operations_per_second) - 1.0) * 100.0;
                if decrease_percent > REGRESSION_THRESHOLD_PERCENT {
                    return RegressionResult::ThroughputRegression {
                        operation: operation.clone(),
                        decrease_percent,
                    };
                }
            }
        }

        RegressionResult::NoRegression
    }
}

fn calculate_latency_stats(durations: &[Duration]) -> LatencyStats {
    if durations.is_empty() {
        return LatencyStats {
            min: Duration::ZERO,
            max: Duration::ZERO,
            mean: Duration::ZERO,
            p50: Duration::ZERO,
            p95: Duration::ZERO,
            p99: Duration::ZERO,
            p999: Duration::ZERO,
            sample_count: 0,
        };
    }

    let mut sorted = durations.to_vec();
    sorted.sort();

    let min = sorted[0];
    let max = sorted[sorted.len() - 1];
    
    let total_nanos: u64 = sorted.iter().map(|d| d.as_nanos() as u64).sum();
    let mean = Duration::from_nanos(total_nanos / sorted.len() as u64);

    let p50 = sorted[sorted.len() * 50 / 100];
    let p95 = sorted[sorted.len() * 95 / 100];
    let p99 = sorted[sorted.len() * 99 / 100];
    let p999 = sorted[sorted.len() * 999 / 1000];

    LatencyStats {
        min,
        max,
        mean,
        p50,
        p95,
        p99,
        p999,
        sample_count: durations.len(),
    }
}

fn calculate_throughput_stats(measurements: &[ThroughputMeasurement]) -> ThroughputStats {
    if measurements.is_empty() {
        return ThroughputStats {
            operations_per_second: 0.0,
            max_ops_per_second: 0.0,
            min_ops_per_second: 0.0,
            total_operations: 0,
            sample_count: 0,
        };
    }

    let ops_per_sec: Vec<f64> = measurements.iter()
        .map(|m| m.count as f64 / m.duration.as_secs_f64())
        .collect();

    let total_operations = measurements.iter().map(|m| m.count).sum();
    let avg_ops_per_second = ops_per_sec.iter().sum::<f64>() / ops_per_sec.len() as f64;
    let max_ops_per_second = ops_per_sec.iter().fold(0.0, |a, &b| a.max(b));
    let min_ops_per_second = ops_per_sec.iter().fold(f64::INFINITY, |a, &b| a.min(b));

    ThroughputStats {
        operations_per_second: avg_ops_per_second,
        max_ops_per_second,
        min_ops_per_second,
        total_operations,
        sample_count: measurements.len(),
    }
}

fn calculate_resource_stats(usage: &[ResourceUsage]) -> ResourceStats {
    if usage.is_empty() {
        return ResourceStats::default();
    }

    let cpu_avg = usage.iter().map(|u| u.cpu_percent).sum::<f64>() / usage.len() as f64;
    let cpu_max = usage.iter().map(|u| u.cpu_percent).fold(0.0, |a, b| a.max(b));
    
    let memory_avg = usage.iter().map(|u| u.memory_mb).sum::<f64>() / usage.len() as f64;
    let memory_max = usage.iter().map(|u| u.memory_mb).fold(0.0, |a, b| a.max(b));

    let gpu_usage: Vec<f64> = usage.iter().filter_map(|u| u.gpu_percent).collect();
    let (gpu_avg, gpu_max) = if gpu_usage.is_empty() {
        (None, None)
    } else {
        let avg = gpu_usage.iter().sum::<f64>() / gpu_usage.len() as f64;
        let max = gpu_usage.iter().fold(0.0, |a, &b| a.max(b));
        (Some(avg), Some(max))
    };

    ResourceStats {
        cpu_percent_avg: cpu_avg,
        cpu_percent_max: cpu_max,
        memory_mb_avg: memory_avg,
        memory_mb_max: memory_max,
        gpu_percent_avg: gpu_avg,
        gpu_percent_max: gpu_max,
    }
}