//! Performance Monitoring for Integration Tests //! //! Tracks latency, throughput, and resource utilization during test execution //! to ensure HFT performance requirements are met and detect regressions. use std::collections::HashMap; use std::time::{Duration, Instant}; use serde::{Deserialize, Serialize}; use anyhow::Result; /// Performance monitor for tracking test execution metrics pub struct PerformanceMonitor { scenarios: HashMap, baseline: Option, } impl PerformanceMonitor { pub fn new() -> Self { Self { scenarios: HashMap::new(), baseline: None, } } /// Load performance baseline from file for regression testing pub async fn load_baseline(mut self, path: &str) -> Result { if let Ok(content) = tokio::fs::read_to_string(path).await { if let Ok(baseline) = serde_json::from_str::(&content) { self.baseline = Some(baseline); } } Ok(self) } /// Start monitoring a test scenario pub fn start_scenario(&mut self, name: &str) { let tracker = ScenarioTracker::new(); self.scenarios.insert(name.to_string(), tracker); } /// End monitoring and finalize metrics pub fn end_scenario(&mut self, name: &str) { if let Some(tracker) = self.scenarios.get_mut(name) { tracker.finalize(); } } /// Record a latency measurement pub fn record_latency(&mut self, scenario: &str, operation: &str, duration: Duration) { if let Some(tracker) = self.scenarios.get_mut(scenario) { tracker.record_latency(operation, duration); } } /// Record throughput measurement pub fn record_throughput(&mut self, scenario: &str, operation: &str, count: u64, duration: Duration) { if let Some(tracker) = self.scenarios.get_mut(scenario) { tracker.record_throughput(operation, count, duration); } } /// Record resource utilization pub fn record_resource_usage(&mut self, scenario: &str, cpu_percent: f64, memory_mb: f64, gpu_percent: Option) { if let Some(tracker) = self.scenarios.get_mut(scenario) { tracker.record_resource_usage(cpu_percent, memory_mb, gpu_percent); } } /// Get metrics for a scenario pub fn get_metrics(&self, scenario: &str) -> ScenarioMetrics { self.scenarios.get(scenario) .map(|tracker| tracker.get_metrics()) .unwrap_or_default() } /// Check if performance meets baseline requirements pub fn check_regression(&self, scenario: &str) -> RegressionResult { let current_metrics = self.get_metrics(scenario); if let Some(baseline) = &self.baseline { if let Some(baseline_scenario) = baseline.scenarios.get(scenario) { return RegressionResult::compare(¤t_metrics, baseline_scenario); } } RegressionResult::NoBaseline } /// Save current metrics as new baseline pub async fn save_baseline(&self, path: &str) -> Result<()> { let baseline = PerformanceBaseline { created_at: chrono::Utc::now(), scenarios: self.scenarios.iter() .map(|(name, tracker)| (name.clone(), tracker.get_metrics())) .collect(), }; let content = serde_json::to_string_pretty(&baseline)?; tokio::fs::write(path, content).await?; Ok(()) } } /// Tracks performance metrics for a single test scenario struct ScenarioTracker { start_time: Instant, end_time: Option, latencies: HashMap>, throughputs: HashMap>, resource_usage: Vec, } impl ScenarioTracker { fn new() -> Self { Self { start_time: Instant::now(), end_time: None, latencies: HashMap::new(), throughputs: HashMap::new(), resource_usage: Vec::new(), } } fn finalize(&mut self) { self.end_time = Some(Instant::now()); } fn record_latency(&mut self, operation: &str, duration: Duration) { self.latencies.entry(operation.to_string()) .or_insert_with(Vec::new) .push(duration); } fn record_throughput(&mut self, operation: &str, count: u64, duration: Duration) { let measurement = ThroughputMeasurement { count, duration }; self.throughputs.entry(operation.to_string()) .or_insert_with(Vec::new) .push(measurement); } fn record_resource_usage(&mut self, cpu_percent: f64, memory_mb: f64, gpu_percent: Option) { self.resource_usage.push(ResourceUsage { timestamp: Instant::now(), cpu_percent, memory_mb, gpu_percent, }); } fn get_metrics(&self) -> ScenarioMetrics { let total_duration = self.end_time .unwrap_or_else(Instant::now) .duration_since(self.start_time); let latency_stats = self.latencies.iter() .map(|(op, durations)| (op.clone(), calculate_latency_stats(durations))) .collect(); let throughput_stats = self.throughputs.iter() .map(|(op, measurements)| (op.clone(), calculate_throughput_stats(measurements))) .collect(); let resource_stats = calculate_resource_stats(&self.resource_usage); ScenarioMetrics { total_duration, latency_stats, throughput_stats, resource_stats, } } } /// Performance metrics for a test scenario #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct ScenarioMetrics { pub total_duration: Duration, pub latency_stats: HashMap, pub throughput_stats: HashMap, pub resource_stats: ResourceStats, } /// Latency statistics for an operation #[derive(Debug, Clone, Serialize, Deserialize)] pub struct LatencyStats { pub min: Duration, pub max: Duration, pub mean: Duration, pub p50: Duration, pub p95: Duration, pub p99: Duration, pub p999: Duration, pub sample_count: usize, } /// Throughput statistics for an operation #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ThroughputStats { pub operations_per_second: f64, pub max_ops_per_second: f64, pub min_ops_per_second: f64, pub total_operations: u64, pub sample_count: usize, } /// Resource utilization statistics #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct ResourceStats { pub cpu_percent_avg: f64, pub cpu_percent_max: f64, pub memory_mb_avg: f64, pub memory_mb_max: f64, pub gpu_percent_avg: Option, pub gpu_percent_max: Option, } #[derive(Debug, Clone)] struct ThroughputMeasurement { count: u64, duration: Duration, } #[derive(Debug, Clone)] struct ResourceUsage { timestamp: Instant, cpu_percent: f64, memory_mb: f64, gpu_percent: Option, } /// Performance baseline for regression testing #[derive(Debug, Serialize, Deserialize)] pub struct PerformanceBaseline { pub created_at: chrono::DateTime, pub scenarios: HashMap, } /// Result of regression analysis #[derive(Debug)] pub enum RegressionResult { NoRegression, LatencyRegression { operation: String, increase_percent: f64 }, ThroughputRegression { operation: String, decrease_percent: f64 }, ResourceRegression { resource: String, increase_percent: f64 }, NoBaseline, } impl RegressionResult { fn compare(current: &ScenarioMetrics, baseline: &ScenarioMetrics) -> Self { const REGRESSION_THRESHOLD_PERCENT: f64 = 10.0; // 10% increase is considered regression // Check latency regressions for (operation, current_stats) in ¤t.latency_stats { if let Some(baseline_stats) = baseline.latency_stats.get(operation) { let increase_percent = ((current_stats.p95.as_nanos() as f64 / baseline_stats.p95.as_nanos() as f64) - 1.0) * 100.0; if increase_percent > REGRESSION_THRESHOLD_PERCENT { return RegressionResult::LatencyRegression { operation: operation.clone(), increase_percent, }; } } } // Check throughput regressions for (operation, current_stats) in ¤t.throughput_stats { if let Some(baseline_stats) = baseline.throughput_stats.get(operation) { let decrease_percent = ((baseline_stats.operations_per_second / current_stats.operations_per_second) - 1.0) * 100.0; if decrease_percent > REGRESSION_THRESHOLD_PERCENT { return RegressionResult::ThroughputRegression { operation: operation.clone(), decrease_percent, }; } } } RegressionResult::NoRegression } } fn calculate_latency_stats(durations: &[Duration]) -> LatencyStats { if durations.is_empty() { return LatencyStats { min: Duration::ZERO, max: Duration::ZERO, mean: Duration::ZERO, p50: Duration::ZERO, p95: Duration::ZERO, p99: Duration::ZERO, p999: Duration::ZERO, sample_count: 0, }; } let mut sorted = durations.to_vec(); sorted.sort(); let min = sorted[0]; let max = sorted[sorted.len() - 1]; let total_nanos: u64 = sorted.iter().map(|d| d.as_nanos() as u64).sum(); let mean = Duration::from_nanos(total_nanos / sorted.len() as u64); let p50 = sorted[sorted.len() * 50 / 100]; let p95 = sorted[sorted.len() * 95 / 100]; let p99 = sorted[sorted.len() * 99 / 100]; let p999 = sorted[sorted.len() * 999 / 1000]; LatencyStats { min, max, mean, p50, p95, p99, p999, sample_count: durations.len(), } } fn calculate_throughput_stats(measurements: &[ThroughputMeasurement]) -> ThroughputStats { if measurements.is_empty() { return ThroughputStats { operations_per_second: 0.0, max_ops_per_second: 0.0, min_ops_per_second: 0.0, total_operations: 0, sample_count: 0, }; } let ops_per_sec: Vec = measurements.iter() .map(|m| m.count as f64 / m.duration.as_secs_f64()) .collect(); let total_operations = measurements.iter().map(|m| m.count).sum(); let avg_ops_per_second = ops_per_sec.iter().sum::() / ops_per_sec.len() as f64; let max_ops_per_second = ops_per_sec.iter().fold(0.0, |a, &b| a.max(b)); let min_ops_per_second = ops_per_sec.iter().fold(f64::INFINITY, |a, &b| a.min(b)); ThroughputStats { operations_per_second: avg_ops_per_second, max_ops_per_second, min_ops_per_second, total_operations, sample_count: measurements.len(), } } fn calculate_resource_stats(usage: &[ResourceUsage]) -> ResourceStats { if usage.is_empty() { return ResourceStats::default(); } let cpu_avg = usage.iter().map(|u| u.cpu_percent).sum::() / usage.len() as f64; let cpu_max = usage.iter().map(|u| u.cpu_percent).fold(0.0, |a, b| a.max(b)); let memory_avg = usage.iter().map(|u| u.memory_mb).sum::() / usage.len() as f64; let memory_max = usage.iter().map(|u| u.memory_mb).fold(0.0, |a, b| a.max(b)); let gpu_usage: Vec = usage.iter().filter_map(|u| u.gpu_percent).collect(); let (gpu_avg, gpu_max) = if gpu_usage.is_empty() { (None, None) } else { let avg = gpu_usage.iter().sum::() / gpu_usage.len() as f64; let max = gpu_usage.iter().fold(0.0, |a, &b| a.max(b)); (Some(avg), Some(max)) }; ResourceStats { cpu_percent_avg: cpu_avg, cpu_percent_max: cpu_max, memory_mb_avg: memory_avg, memory_mb_max: memory_max, gpu_percent_avg: gpu_avg, gpu_percent_max: gpu_max, } }