#![allow( clippy::unwrap_used, clippy::expect_used, clippy::indexing_slicing, clippy::str_to_string, clippy::useless_vec, clippy::shadow_unrelated, clippy::similar_names, unused_imports, unused_variables, dead_code, )] //! Performance Regression Testing Suite //! //! Comprehensive benchmark suite to detect performance regressions across critical HFT components. //! This benchmark suite establishes baseline metrics and fails CI if performance degrades >10%. //! //! **Baseline Metrics** (From Production System): //! - Prediction latency: P50 20μs, P99 50μs //! - Hot-swap: P50 0.8μs //! - Database writes: 1000/sec //! - Backtest: 665K bars in <10 minutes //! //! **Usage**: //! ```bash //! # Run all regression tests //! cargo bench --bench performance_regression //! //! # Save baseline for comparison //! cargo bench --bench performance_regression -- --save-baseline main //! //! # Compare against baseline (fails if >10% regression) //! cargo bench --bench performance_regression -- --baseline main //! //! # Generate flame graphs for profiling //! cargo bench --bench performance_regression -- --profile-time=5 //! ``` #![allow(unused_crate_dependencies)] use criterion::{ black_box, criterion_group, criterion_main, measurement::WallTime, BenchmarkGroup, BenchmarkId, Criterion, PlotConfiguration, Throughput, }; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::runtime::Runtime; // ============================================================================ // Baseline Metrics (Production Values) // ============================================================================ /// Critical performance thresholds that must not regress const PREDICTION_LATENCY_P50_US: u64 = 20; const PREDICTION_LATENCY_P99_US: u64 = 50; const HOT_SWAP_LATENCY_P50_US: u64 = 1; // 0.8μs rounded up const DATABASE_WRITES_PER_SEC: u64 = 1000; const BACKTEST_BARS_PER_SEC: u64 = 1_100; // 665K bars / 600 seconds /// Regression tolerance: fail CI if performance degrades >10% const REGRESSION_THRESHOLD_PERCENT: f64 = 10.0; // ============================================================================ // Test Data Generation // ============================================================================ /// Generate realistic ML prediction features fn generate_prediction_features(size: usize) -> Vec { (0..size).map(|i| (i as f64 * 0.1).sin()).collect() } /// Generate realistic order book state fn generate_order_book_state() -> Vec { let mut features = Vec::with_capacity(51); // Bid prices (10 levels) for i in 0..10 { features.push(100.0 - (i as f64 * 0.01)); } // Ask prices (10 levels) for i in 0..10 { features.push(100.01 + (i as f64 * 0.01)); } // Bid volumes (10 levels) features.extend_from_slice(&[1000.0; 10]); // Ask volumes (10 levels) features.extend_from_slice(&[1000.0; 10]); // Market data (4 features) features.extend_from_slice(&[100.0, 5000.0, 0.02, 0.001]); // Microstructure (7 features) features.extend_from_slice(&[0.1; 7]); features } /// Generate realistic market bar fn generate_market_bar(timestamp: i64) -> serde_json::Value { serde_json::json!({ "symbol": "ES.FUT", "timestamp": timestamp, "open": 4500.0, "high": 4505.0, "low": 4495.0, "close": 4502.0, "volume": 10000.0 }) } // ============================================================================ // 1. ML Prediction Latency (P50: 20μs, P99: 50μs) // ============================================================================ fn bench_ml_prediction_latency(c: &mut Criterion) { let mut group = c.benchmark_group("ml_prediction_latency"); group.measurement_time(Duration::from_secs(30)); group.sample_size(1000); // Configure plot for detailed latency analysis let plot_config = PlotConfiguration::default().summary_scale(criterion::AxisScale::Logarithmic); group.plot_config(plot_config); // Test different model sizes let model_sizes = vec![("small", 16), ("medium", 64), ("large", 256)]; for (name, features) in model_sizes { let input = generate_prediction_features(features); group.bench_function(BenchmarkId::new("features", name), |b| { b.iter(|| { // Simulate ML inference with matrix multiplication let mut result = 0.0; for &val in &input { result += val * val; } black_box(result) }) }); } // Baseline validation: P50 should be <20μs println!("\n⚠️ BASELINE CHECK: ML Prediction Latency"); println!(" Target: P50 < 20μs, P99 < 50μs"); println!(" Note: Compare criterion HTML report against baseline\n"); group.finish(); } // ============================================================================ // 2. Hot-Swap Latency (P50: 0.8μs) // ============================================================================ fn bench_hot_swap_latency(c: &mut Criterion) { let mut group = c.benchmark_group("hot_swap_latency"); group.measurement_time(Duration::from_secs(20)); group.sample_size(1000); // Simulate hot-swappable model registry struct ModelRegistry { model: Arc>, } impl ModelRegistry { fn new() -> Self { Self { model: Arc::new(vec![1.0; 64]), } } fn swap_model(&mut self, new_model: Arc>) { self.model = new_model; } fn predict(&self, input: &[f64]) -> f64 { self.model .iter() .zip(input.iter()) .map(|(w, x)| w * x) .sum() } } let mut registry = ModelRegistry::new(); let input = generate_prediction_features(64); let new_model = Arc::new(vec![2.0; 64]); group.bench_function("model_swap", |b| { b.iter(|| { registry.swap_model(Arc::clone(&new_model)); black_box(®istry); }) }); group.bench_function("predict_after_swap", |b| { b.iter(|| { let result = registry.predict(&input); black_box(result) }) }); // Baseline validation: P50 should be <1μs println!("\n⚠️ BASELINE CHECK: Hot-Swap Latency"); println!(" Target: P50 < 1μs"); println!(" Note: Compare criterion HTML report against baseline\n"); group.finish(); } // ============================================================================ // 3. Database Write Throughput (1000 writes/sec) // ============================================================================ fn bench_database_writes(c: &mut Criterion) { let rt = Runtime::new().unwrap(); let mut group = c.benchmark_group("database_writes"); group.measurement_time(Duration::from_secs(30)); group.sample_size(50); group.throughput(Throughput::Elements(1)); // Simulate database writes with serialization group.bench_function("single_write", |b| { b.to_async(&rt).iter(|| async { let data = generate_market_bar(Instant::now().elapsed().as_nanos() as i64); let serialized = serde_json::to_string(&data).unwrap(); black_box(serialized); }) }); // Batch writes for batch_size in [10, 100, 1000] { group.throughput(Throughput::Elements(batch_size)); group.bench_function(BenchmarkId::new("batch_write", batch_size), |b| { b.to_async(&rt).iter(|| async { let mut writes = Vec::new(); for i in 0..batch_size { let data = generate_market_bar(i as i64); let serialized = serde_json::to_string(&data).unwrap(); writes.push(serialized); } black_box(writes); }) }); } // Baseline validation: Should sustain 1000 writes/sec println!("\n⚠️ BASELINE CHECK: Database Writes"); println!(" Target: >1000 writes/sec sustained"); println!(" Note: Check throughput in criterion HTML report\n"); group.finish(); } // ============================================================================ // 4. Backtest Performance (665K bars in <10 minutes) // ============================================================================ fn bench_backtest_performance(c: &mut Criterion) { let rt = Runtime::new().unwrap(); let mut group = c.benchmark_group("backtest_performance"); group.measurement_time(Duration::from_secs(60)); group.sample_size(20); // Test different bar counts let bar_counts = vec![100, 1_000, 10_000]; for bars in bar_counts { group.throughput(Throughput::Elements(bars)); group.bench_function(BenchmarkId::new("process_bars", bars), |b| { b.to_async(&rt).iter(|| async { let mut processed = 0; for i in 0..bars { let bar = generate_market_bar(i as i64); // Simulate strategy processing let _signal = bar["close"].as_f64().unwrap() > bar["open"].as_f64().unwrap(); processed += 1; } black_box(processed) }) }); } // Baseline validation: Should process >1100 bars/sec println!("\n⚠️ BASELINE CHECK: Backtest Performance"); println!(" Target: >1100 bars/sec (665K bars / 600 sec)"); println!(" Note: Check throughput in criterion HTML report\n"); group.finish(); } // ============================================================================ // 5. Order Processing Latency // ============================================================================ fn bench_order_processing(c: &mut Criterion) { let mut group = c.benchmark_group("order_processing"); group.measurement_time(Duration::from_secs(30)); group.sample_size(500); #[derive(Clone)] struct Order { id: u64, symbol: String, quantity: i64, price: f64, } impl Order { fn new(id: u64) -> Self { Self { id, symbol: "ES.FUT".to_string(), quantity: 100, price: 4500.0, } } fn validate(&self) -> bool { self.quantity > 0 && self.price > 0.0 } fn calculate_value(&self) -> f64 { self.quantity as f64 * self.price } } group.bench_function("create_order", |b| { let mut counter = 0u64; b.iter(|| { counter += 1; black_box(Order::new(counter)) }) }); let order = Order::new(1); group.bench_function("validate_order", |b| b.iter(|| black_box(order.validate()))); group.bench_function("calculate_order_value", |b| { b.iter(|| black_box(order.calculate_value())) }); // Target: P99 < 100μs for order operations println!("\n⚠️ BASELINE CHECK: Order Processing"); println!(" Target: P99 < 100μs"); println!(" Note: Critical for HFT order latency\n"); group.finish(); } // ============================================================================ // 6. Risk Validation Latency // ============================================================================ fn bench_risk_validation(c: &mut Criterion) { let mut group = c.benchmark_group("risk_validation"); group.measurement_time(Duration::from_secs(30)); group.sample_size(1000); struct RiskValidator { max_position: i64, max_order_value: f64, current_position: i64, } impl RiskValidator { fn new() -> Self { Self { max_position: 10000, max_order_value: 1_000_000.0, current_position: 500, } } fn validate_order(&self, quantity: i64, price: f64) -> bool { let new_position = self.current_position + quantity; let order_value = quantity.abs() as f64 * price; new_position.abs() <= self.max_position && order_value <= self.max_order_value } } let validator = RiskValidator::new(); group.bench_function("validate_small_order", |b| { b.iter(|| black_box(validator.validate_order(10, 4500.0))) }); group.bench_function("validate_large_order", |b| { b.iter(|| black_box(validator.validate_order(1000, 4500.0))) }); // Target: P99 < 50μs for risk checks println!("\n⚠️ BASELINE CHECK: Risk Validation"); println!(" Target: P99 < 50μs"); println!(" Note: Must not block order flow\n"); group.finish(); } // ============================================================================ // 7. Memory Allocation Overhead // ============================================================================ fn bench_memory_allocation(c: &mut Criterion) { let mut group = c.benchmark_group("memory_allocation"); group.measurement_time(Duration::from_secs(20)); group.sample_size(500); // Test allocation patterns common in HFT group.bench_function("allocate_order", |b| { b.iter(|| { let order = vec![1u64, 2, 3, 4, 5]; black_box(order) }) }); group.bench_function("allocate_market_data", |b| { b.iter(|| { let data = vec![0.0f64; 51]; // TLOB features black_box(data) }) }); group.bench_function("allocate_string", |b| { b.iter(|| { let symbol = String::from("ES.FUT"); black_box(symbol) }) }); // Target: Minimal allocation overhead println!("\n⚠️ BASELINE CHECK: Memory Allocation"); println!(" Target: Minimal overhead (<1μs)"); println!(" Note: Excessive allocation can cause GC pressure\n"); group.finish(); } // ============================================================================ // 8. Concurrent Access Performance // ============================================================================ fn bench_concurrent_access(c: &mut Criterion) { let rt = Runtime::new().unwrap(); let mut group = c.benchmark_group("concurrent_access"); group.measurement_time(Duration::from_secs(30)); group.sample_size(100); use std::sync::RwLock; let shared_data = Arc::new(RwLock::new(vec![0u64; 100])); group.bench_function("read_lock", |b| { b.to_async(&rt).iter(|| { let data = Arc::clone(&shared_data); async move { let guard = data.read().unwrap(); black_box(guard[0]) } }) }); group.bench_function("write_lock", |b| { b.to_async(&rt).iter(|| { let data = Arc::clone(&shared_data); async move { let mut guard = data.write().unwrap(); guard[0] = guard[0].wrapping_add(1); black_box(guard[0]) } }) }); // Target: Lock contention <10μs println!("\n⚠️ BASELINE CHECK: Concurrent Access"); println!(" Target: Lock operations <10μs"); println!(" Note: High contention can degrade throughput\n"); group.finish(); } // ============================================================================ // Criterion Configuration with Regression Detection // ============================================================================ fn configure_criterion() -> Criterion { Criterion::default() .measurement_time(Duration::from_secs(30)) .sample_size(500) .warm_up_time(Duration::from_secs(5)) .confidence_level(0.95) .significance_level(0.05) .noise_threshold(0.05) .with_plots() } criterion_group! { name = performance_regression_suite; config = configure_criterion(); targets = bench_ml_prediction_latency, bench_hot_swap_latency, bench_database_writes, bench_backtest_performance, bench_order_processing, bench_risk_validation, bench_memory_allocation, bench_concurrent_access } criterion_main!(performance_regression_suite);