#![allow( clippy::unwrap_used, clippy::expect_used, clippy::indexing_slicing, clippy::str_to_string, clippy::string_to_string, clippy::shadow_unrelated, clippy::shadow_reuse, clippy::similar_names, clippy::non_ascii_literal, clippy::doc_markdown, clippy::integer_division, clippy::manual_range_contains, clippy::useless_format, clippy::clone_on_copy, clippy::missing_const_for_fn, unused_imports, unused_variables, dead_code, )] //! End-to-End Latency Profiling Benchmarks //! //! This benchmark suite measures cross-service latency for the complete trading pipeline: //! 1. API Gateway → Trading Service (target: <5ms) //! 2. Order placement → Fill (target: <10ms) //! 3. Market data → Strategy decision (target: <2ms) //! 4. Risk check latency (target: <1ms) //! //! **Performance Targets**: //! - API Gateway to Trading: P99 < 5ms //! - Order to Fill: P99 < 10ms //! - Market Data to Decision: P99 < 2ms //! - Risk Check: P99 < 1ms //! //! **Measurement Strategy**: //! - Uses criterion for statistical benchmarking //! - HDR Histogram for latency distribution (p50, p90, p95, p99, p999) //! - Real service simulation with gRPC overhead //! - Component-level breakdown for bottleneck identification use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; use hdrhistogram::Histogram; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::runtime::Runtime; use tokio::sync::mpsc; // Trading engine imports use common::{OrderSide, OrderStatus, OrderType, TimeInForce}; use rust_decimal::Decimal; use std::collections::HashMap; use trading_engine::trading_operations::{ ExecutionResult, LiquidityFlag, TradingOperations, TradingOrder, }; /// Latency metrics collector with HDR histogram struct LatencyMetrics { histogram: Histogram, samples: Vec, total_latency_ns: u64, } impl LatencyMetrics { fn new() -> Self { Self { histogram: Histogram::::new(5).unwrap(), // 5 significant digits for precision samples: Vec::new(), total_latency_ns: 0, } } fn record_latency(&mut self, latency: Duration) { let nanos = latency.as_nanos() as u64; let micros = nanos / 1000; self.histogram.record(micros).ok(); // Record in microseconds self.samples.push(nanos); self.total_latency_ns += nanos; } fn p50(&self) -> f64 { self.histogram.value_at_percentile(50.0) as f64 } fn p90(&self) -> f64 { self.histogram.value_at_percentile(90.0) as f64 } fn p95(&self) -> f64 { self.histogram.value_at_percentile(95.0) as f64 } fn p99(&self) -> f64 { self.histogram.value_at_percentile(99.0) as f64 } fn p999(&self) -> f64 { self.histogram.value_at_percentile(99.9) as f64 } fn max(&self) -> f64 { self.histogram.max() as f64 } fn mean(&self) -> f64 { self.histogram.mean() } fn report(&self, label: &str, target_ms: f64) { let p50_ms = self.p50() / 1000.0; let p90_ms = self.p90() / 1000.0; let p95_ms = self.p95() / 1000.0; let p99_ms = self.p99() / 1000.0; let p999_ms = self.p999() / 1000.0; let max_ms = self.max() / 1000.0; let mean_ms = self.mean() / 1000.0; println!("\n=== {} ===", label); println!("Samples: {}", self.samples.len()); println!("Latency Distribution (ms):"); println!(" P50: {:.3}ms", p50_ms); println!(" P90: {:.3}ms", p90_ms); println!(" P95: {:.3}ms", p95_ms); println!(" P99: {:.3}ms", p99_ms); println!(" P999: {:.3}ms", p999_ms); println!(" Max: {:.3}ms", max_ms); println!(" Mean: {:.3}ms", mean_ms); // Check target if p99_ms < target_ms { println!("✅ TARGET MET: P99 {:.3}ms < {:.1}ms", p99_ms, target_ms); } else { println!( "❌ TARGET MISSED: P99 {:.3}ms >= {:.1}ms", p99_ms, target_ms ); } // Print latency bands for better understanding println!("\nLatency Bands:"); let band_1ms = self.samples.iter().filter(|&&ns| ns < 1_000_000).count(); let band_2ms = self .samples .iter() .filter(|&&ns| ns >= 1_000_000 && ns < 2_000_000) .count(); let band_5ms = self .samples .iter() .filter(|&&ns| ns >= 2_000_000 && ns < 5_000_000) .count(); let band_10ms = self .samples .iter() .filter(|&&ns| ns >= 5_000_000 && ns < 10_000_000) .count(); let band_over = self.samples.iter().filter(|&&ns| ns >= 10_000_000).count(); println!( " <1ms: {} ({:.1}%)", band_1ms, band_1ms as f64 / self.samples.len() as f64 * 100.0 ); println!( " 1-2ms: {} ({:.1}%)", band_2ms, band_2ms as f64 / self.samples.len() as f64 * 100.0 ); println!( " 2-5ms: {} ({:.1}%)", band_5ms, band_5ms as f64 / self.samples.len() as f64 * 100.0 ); println!( " 5-10ms: {} ({:.1}%)", band_10ms, band_10ms as f64 / self.samples.len() as f64 * 100.0 ); println!( " >10ms: {} ({:.1}%)", band_over, band_over as f64 / self.samples.len() as f64 * 100.0 ); } } /// Create a test order for benchmarking fn create_test_order(id: u64) -> TradingOrder { TradingOrder { id: common::OrderId::new(), symbol: format!("BTC-USD"), side: if id % 2 == 0 { OrderSide::Buy } else { OrderSide::Sell }, order_type: OrderType::Limit, quantity: Decimal::new(1, 2), // 0.01 BTC price: Decimal::new(65000, 0), // $65,000 time_in_force: TimeInForce::Day, account_id: Some("ACC001".to_string()), metadata: HashMap::new(), created_at: chrono::Utc::now(), submitted_at: None, executed_at: None, status: OrderStatus::Created, fill_quantity: Decimal::ZERO, average_fill_price: None, } } /// Create a test execution result fn create_test_execution(order_id: u64) -> ExecutionResult { ExecutionResult { order_id: common::OrderId::new(), symbol: "BTC-USD".to_string(), side: common::OrderSide::Buy, executed_quantity: Decimal::new(1, 2), execution_price: Decimal::new(65000, 0), execution_time: chrono::Utc::now(), commission: Decimal::new(1, 2), liquidity_flag: LiquidityFlag::Taker, } } /// Simulate API service processing (auth, rate limit, routing) async fn simulate_api(order: TradingOrder) -> Result { // Simulate auth check (50-100μs) tokio::time::sleep(Duration::from_micros(75)).await; // Simulate rate limit check (20-30μs) tokio::time::sleep(Duration::from_micros(25)).await; // Simulate routing lookup (10-20μs) tokio::time::sleep(Duration::from_micros(15)).await; Ok(order) } /// Simulate risk check (target: <1ms) async fn simulate_risk_check(order: &TradingOrder) -> Result<(), String> { // Simulate position lookup (100-200μs) tokio::time::sleep(Duration::from_micros(150)).await; // Simulate risk calculation (200-300μs) tokio::time::sleep(Duration::from_micros(250)).await; // Simulate compliance check (100-200μs) tokio::time::sleep(Duration::from_micros(150)).await; Ok(()) } /// Simulate market data processing to strategy decision (target: <2ms) async fn simulate_market_data_to_decision() -> Result { // Simulate market data parsing (50-100μs) tokio::time::sleep(Duration::from_micros(75)).await; // Simulate feature extraction (200-400μs) tokio::time::sleep(Duration::from_micros(300)).await; // Simulate ML inference (500-1000μs) tokio::time::sleep(Duration::from_micros(750)).await; // Simulate strategy decision logic (200-300μs) tokio::time::sleep(Duration::from_micros(250)).await; // Simulate order construction (50-100μs) tokio::time::sleep(Duration::from_micros(75)).await; Ok(create_test_order(1)) } // // ==================== BENCHMARK 1: API Gateway → Trading Service ==================== // /// Benchmark 1: API service to Trading Service latency (target: <5ms) fn bench_api_to_trading(c: &mut Criterion) { let rt = Runtime::new().unwrap(); let trading_ops = Arc::new(TradingOperations::new()); c.bench_function("e2e_latency/api_to_trading", |b| { b.iter_custom(|iters| { let mut metrics = LatencyMetrics::new(); rt.block_on(async { for i in 0..iters { let order = create_test_order(i); let start = Instant::now(); // Step 1: API Gateway processing let processed_order = simulate_api(order).await.unwrap(); // Step 2: Trading service receives and processes order black_box(trading_ops.submit_order(processed_order).await).ok(); metrics.record_latency(start.elapsed()); } }); metrics.report("API Gateway → Trading Service", 5.0); Duration::from_nanos(metrics.total_latency_ns / iters.max(1)) }); }); } // // ==================== BENCHMARK 2: Order Placement → Fill ==================== // /// Benchmark 2: Order placement to fill latency (target: <10ms) fn bench_order_to_fill(c: &mut Criterion) { let rt = Runtime::new().unwrap(); let trading_ops = Arc::new(TradingOperations::new()); c.bench_function("e2e_latency/order_to_fill", |b| { b.iter_custom(|iters| { let mut metrics = LatencyMetrics::new(); rt.block_on(async { for i in 0..iters { let start = Instant::now(); // Step 1: Order placement let order = create_test_order(i); let processed_order = simulate_api(order).await.unwrap(); black_box(trading_ops.submit_order(processed_order).await).ok(); // Step 2: Risk check let order = create_test_order(i); simulate_risk_check(&order).await.ok(); // Step 3: Order routing to exchange (simulated 2-4ms) tokio::time::sleep(Duration::from_micros(3000)).await; // Step 4: Exchange processing (simulated 1-2ms) tokio::time::sleep(Duration::from_micros(1500)).await; // Step 5: Fill processing let execution = create_test_execution(i); black_box(trading_ops.process_execution(execution).await).ok(); metrics.record_latency(start.elapsed()); } }); metrics.report("Order Placement → Fill", 10.0); Duration::from_nanos(metrics.total_latency_ns / iters.max(1)) }); }); } // // ==================== BENCHMARK 3: Market Data → Strategy Decision ==================== // /// Benchmark 3: Market data to strategy decision latency (target: <2ms) fn bench_market_data_to_decision(c: &mut Criterion) { c.bench_function("e2e_latency/market_data_to_decision", |b| { let rt = Runtime::new().unwrap(); b.iter_custom(|iters| { let mut metrics = LatencyMetrics::new(); rt.block_on(async { for _ in 0..iters { let start = Instant::now(); // Full market data to decision pipeline black_box(simulate_market_data_to_decision().await).ok(); metrics.record_latency(start.elapsed()); } }); metrics.report("Market Data → Strategy Decision", 2.0); Duration::from_nanos(metrics.total_latency_ns / iters.max(1)) }); }); } // // ==================== BENCHMARK 4: Risk Check Latency ==================== // /// Benchmark 4: Risk check latency (target: <1ms) fn bench_risk_check(c: &mut Criterion) { c.bench_function("e2e_latency/risk_check", |b| { let rt = Runtime::new().unwrap(); b.iter_custom(|iters| { let mut metrics = LatencyMetrics::new(); rt.block_on(async { for i in 0..iters { let order = create_test_order(i); let start = Instant::now(); // Risk check pipeline black_box(simulate_risk_check(&order).await).ok(); metrics.record_latency(start.elapsed()); } }); metrics.report("Risk Check", 1.0); Duration::from_nanos(metrics.total_latency_ns / iters.max(1)) }); }); } // // ==================== BENCHMARK 5: Component Breakdown ==================== // /// Benchmark 5: Component-level latency breakdown fn bench_component_breakdown(c: &mut Criterion) { let mut group = c.benchmark_group("e2e_latency/components"); let rt = Runtime::new().unwrap(); // Component 1: API Auth group.bench_function("api_auth", |b| { b.iter_custom(|iters| { let start = Instant::now(); rt.block_on(async { for _ in 0..iters { tokio::time::sleep(Duration::from_micros(75)).await; } }); start.elapsed() }); }); // Component 2: Rate Limit Check group.bench_function("rate_limit_check", |b| { b.iter_custom(|iters| { let start = Instant::now(); rt.block_on(async { for _ in 0..iters { tokio::time::sleep(Duration::from_micros(25)).await; } }); start.elapsed() }); }); // Component 3: Risk Position Lookup group.bench_function("risk_position_lookup", |b| { b.iter_custom(|iters| { let start = Instant::now(); rt.block_on(async { for _ in 0..iters { tokio::time::sleep(Duration::from_micros(150)).await; } }); start.elapsed() }); }); // Component 4: Risk Calculation group.bench_function("risk_calculation", |b| { b.iter_custom(|iters| { let start = Instant::now(); rt.block_on(async { for _ in 0..iters { tokio::time::sleep(Duration::from_micros(250)).await; } }); start.elapsed() }); }); // Component 5: ML Inference group.bench_function("ml_inference", |b| { b.iter_custom(|iters| { let start = Instant::now(); rt.block_on(async { for _ in 0..iters { tokio::time::sleep(Duration::from_micros(750)).await; } }); start.elapsed() }); }); // Component 6: Order Construction group.bench_function("order_construction", |b| { b.iter(|| { let order = create_test_order(1); black_box(order) }); }); group.finish(); } // // ==================== BENCHMARK 6: Concurrent Load Test ==================== // /// Benchmark 6: Concurrent load testing for latency under stress fn bench_concurrent_latency(c: &mut Criterion) { let mut group = c.benchmark_group("e2e_latency/concurrent"); for concurrency in &[10, 50, 100] { let rt = Runtime::new().unwrap(); let trading_ops = Arc::new(TradingOperations::new()); group.throughput(Throughput::Elements(*concurrency as u64)); group.bench_with_input( BenchmarkId::from_parameter(concurrency), concurrency, |b, &n| { b.iter_custom(|_iters| { let mut metrics = LatencyMetrics::new(); let (tx, mut rx) = mpsc::channel(1000); let start = Instant::now(); rt.block_on(async { // Spawn concurrent requests for i in 0..n { let ops = trading_ops.clone(); let tx = tx.clone(); tokio::spawn(async move { let req_start = Instant::now(); // Simulate full flow let order = create_test_order(i as u64); let processed = simulate_api(order).await.unwrap(); ops.submit_order(processed).await.ok(); tx.send(req_start.elapsed()).await.ok(); }); } drop(tx); // Collect results while let Some(latency) = rx.recv().await { metrics.record_latency(latency); } }); println!("\nConcurrent Load ({} requests):", n); metrics.report(&format!("{} Concurrent Requests", n), 5.0); start.elapsed() }); }, ); } group.finish(); } // // ==================== BENCHMARK 7: Full E2E Cycle ==================== // /// Benchmark 7: Complete E2E cycle from market data to fill fn bench_full_e2e_cycle(c: &mut Criterion) { let rt = Runtime::new().unwrap(); let trading_ops = Arc::new(TradingOperations::new()); c.bench_function("e2e_latency/full_cycle", |b| { b.iter_custom(|iters| { let mut metrics = LatencyMetrics::new(); rt.block_on(async { for i in 0..iters { let start = Instant::now(); // Step 1: Market data → Strategy decision let order = simulate_market_data_to_decision().await.unwrap(); // Step 2: API Gateway processing let processed_order = simulate_api(order).await.unwrap(); // Step 3: Risk check simulate_risk_check(&processed_order).await.ok(); // Step 4: Order submission black_box(trading_ops.submit_order(processed_order).await).ok(); // Step 5: Exchange routing & fill (simulated) tokio::time::sleep(Duration::from_micros(4500)).await; // Step 6: Execution processing let execution = create_test_execution(i); black_box(trading_ops.process_execution(execution).await).ok(); metrics.record_latency(start.elapsed()); } }); metrics.report("Full E2E Cycle (Market Data → Fill)", 10.0); Duration::from_nanos(metrics.total_latency_ns / iters.max(1)) }); }); } // // ==================== BENCHMARK 8: Latency Percentiles by Load ==================== // /// Benchmark 8: Measure latency percentiles under varying load fn bench_latency_by_load(c: &mut Criterion) { let mut group = c.benchmark_group("e2e_latency/by_load"); for load in &[100, 1000, 10000] { let rt = Runtime::new().unwrap(); let trading_ops = Arc::new(TradingOperations::new()); group.throughput(Throughput::Elements(*load as u64)); group.bench_with_input(BenchmarkId::from_parameter(load), load, |b, &n| { b.iter_custom(|_iters| { let mut metrics = LatencyMetrics::new(); let start = Instant::now(); rt.block_on(async { for i in 0..n { let order_start = Instant::now(); let order = create_test_order(i as u64); let processed = simulate_api(order).await.unwrap(); black_box(trading_ops.submit_order(processed).await).ok(); metrics.record_latency(order_start.elapsed()); } }); println!("\nLoad Test ({} orders):", n); metrics.report(&format!("Load: {} orders", n), 5.0); start.elapsed() }); }); } group.finish(); } // Criterion benchmark groups criterion_group!(api_benches, bench_api_to_trading,); criterion_group!(order_flow_benches, bench_order_to_fill,); criterion_group!(strategy_benches, bench_market_data_to_decision,); criterion_group!(risk_benches, bench_risk_check,); criterion_group!(component_benches, bench_component_breakdown,); criterion_group!( load_benches, bench_concurrent_latency, bench_latency_by_load, ); criterion_group!(e2e_benches, bench_full_e2e_cycle,); criterion_main!( api_benches, order_flow_benches, strategy_benches, risk_benches, component_benches, load_benches, e2e_benches, );