#![allow(clippy::inconsistent_digit_grouping, dead_code, unused_variables)] //! Comprehensive Performance Benchmarks with Real DBN Data //! //! This benchmark suite validates production readiness by testing all components //! with real market data from Databento files. //! //! ## Performance Targets //! //! - DBN file loading: <10ms (target) / 0.70ms (baseline from Wave 18) //! - Market data query: <1ms //! - Full backtest (1 day): <5s //! - Memory usage: <100MB for typical backtest //! //! ## Real Data Files //! //! - ES.FUT (E-mini S&P 500): 390 bars, 1-minute OHLCV, 2024-01-02 //! - NQ.FUT (E-mini NASDAQ): 390 bars, 1-minute OHLCV, 2024-01-02 //! - CL.FUT (Crude Oil): 390 bars, 1-minute OHLCV, 2024-01-02 //! - ESH4 (ES March 2024): 77 bars, 1-minute OHLCV, 2024-01-03 use backtesting_service::dbn_repository::DbnMarketDataRepository; use backtesting_service::repositories::MarketDataRepository; use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; use std::collections::HashMap; use tokio::runtime::Runtime; /// Get absolute path to test data directory fn get_workspace_root() -> std::path::PathBuf { let current_dir = std::env::current_dir().expect("INVARIANT: Current directory should be accessible"); current_dir .ancestors() .find(|p| p.join("Cargo.toml").exists() && p.join("test_data").exists()) .expect("Could not find workspace root") .to_path_buf() } /// Get test file paths for all available symbols fn get_test_files() -> HashMap { let workspace_root = get_workspace_root(); let data_dir = workspace_root.join("test_data/real/databento"); let mut files = HashMap::new(); // ES.FUT - E-mini S&P 500 (390 bars) let es_file = data_dir.join("ES.FUT_ohlcv-1m_2024-01-02.dbn"); if es_file.exists() { files.insert("ES.FUT".to_string(), es_file.to_string_lossy().to_string()); } // NQ.FUT - E-mini NASDAQ (390 bars) let nq_file = data_dir.join("NQ.FUT_ohlcv-1m_2024-01-02.dbn"); if nq_file.exists() { files.insert("NQ.FUT".to_string(), nq_file.to_string_lossy().to_string()); } // CL.FUT - Crude Oil (390 bars) let cl_file = data_dir.join("CL.FUT_ohlcv-1m_2024-01-02.dbn"); if cl_file.exists() { files.insert("CL.FUT".to_string(), cl_file.to_string_lossy().to_string()); } // ESH4 - ES March 2024 contract (77 bars) // NOTE: Skipping ESH4 files - they may be in different format or compressed // let esh4_file = data_dir.join("ESH4_ohlcv-1m_2024-01-03.dbn"); // if esh4_file.exists() { // files.insert("ESH4".to_string(), esh4_file.to_string_lossy().to_string()); // } if files.is_empty() { panic!( "No valid DBN test files found in {}. Please run data validation first.", data_dir.display() ); } files } /// Benchmark 1: Single-file DBN loading (baseline) fn bench_single_file_loading(c: &mut Criterion) { let rt = Runtime::new().unwrap(); let files = get_test_files(); if !files.contains_key("ES.FUT") { eprintln!("Warning: ES.FUT test file not found, skipping single-file benchmark"); return; } let mut file_mapping = HashMap::new(); file_mapping.insert("ES.FUT".to_string(), files["ES.FUT"].clone()); let repo = rt.block_on(async { DbnMarketDataRepository::new(file_mapping).await.unwrap() }); let mut group = c.benchmark_group("single_file_loading"); group.throughput(Throughput::Elements(390)); // 390 bars expected group.bench_function("es_fut_full_day", |b| { b.to_async(&rt).iter(|| async { let symbols = vec!["ES.FUT".to_string()]; let start_time = 1704153600_000_000_000i64; // 2024-01-02 00:00:00 let end_time = 1704240000_000_000_000i64; // 2024-01-03 00:00:00 let data = repo .load_historical_data(&symbols, start_time, end_time) .await .unwrap(); black_box(data); }); }); group.finish(); } /// Benchmark 2: Multi-file loading (different symbols) fn bench_multi_file_loading(c: &mut Criterion) { let rt = Runtime::new().unwrap(); let files = get_test_files(); if files.len() < 2 { eprintln!("Warning: Need at least 2 test files, skipping multi-file benchmark"); return; } let repo = rt.block_on(async { DbnMarketDataRepository::new(files.clone()).await.unwrap() }); let mut group = c.benchmark_group("multi_file_loading"); // Test with 2, 3, and all available symbols let symbols: Vec = files.keys().cloned().collect(); let test_configs: Vec<(usize, Vec)> = if symbols.len() >= 3 { vec![ (2, vec![symbols[0].clone(), symbols[1].clone()]), ( 3, vec![symbols[0].clone(), symbols[1].clone(), symbols[2].clone()], ), ] } else if symbols.len() == 2 { vec![(2, vec![symbols[0].clone(), symbols[1].clone()])] } else { vec![] }; for (count, symbols) in test_configs { if symbols.iter().all(|s| files.contains_key(s)) { group.throughput(Throughput::Elements((count * 390) as u64)); group.bench_with_input( BenchmarkId::new("symbols", count), &symbols, |b, symbols| { b.to_async(&rt).iter(|| async { let start_time = 1704153600_000_000_000i64; let end_time = 1704240000_000_000_000i64; let data = repo .load_historical_data(symbols, start_time, end_time) .await .unwrap(); black_box(data); }); }, ); } } group.finish(); } /// Benchmark 3: Time range queries (partial day) fn bench_time_range_queries(c: &mut Criterion) { let rt = Runtime::new().unwrap(); let files = get_test_files(); if !files.contains_key("ES.FUT") { eprintln!("Warning: ES.FUT test file not found, skipping time range benchmark"); return; } let mut file_mapping = HashMap::new(); file_mapping.insert("ES.FUT".to_string(), files["ES.FUT"].clone()); let repo = rt.block_on(async { DbnMarketDataRepository::new(file_mapping).await.unwrap() }); let mut group = c.benchmark_group("time_range_queries"); // Test different time ranges let test_ranges = vec![ ( "1_hour", 1704203400_000_000_000i64, 1704207000_000_000_000i64, ), // 1 hour ( "4_hours", 1704203400_000_000_000i64, 1704217800_000_000_000i64, ), // 4 hours ( "full_day", 1704153600_000_000_000i64, 1704240000_000_000_000i64, ), // 24 hours ]; for (name, start, end) in test_ranges { group.bench_with_input( BenchmarkId::new("range", name), &(start, end), |b, &(start, end)| { b.to_async(&rt).iter(|| async { let symbols = vec!["ES.FUT".to_string()]; let data = repo .load_historical_data(&symbols, start, end) .await .unwrap(); black_box(data); }); }, ); } group.finish(); } /// Benchmark 4: Repeated queries (caching performance) fn bench_repeated_queries(c: &mut Criterion) { let rt = Runtime::new().unwrap(); let files = get_test_files(); if !files.contains_key("ES.FUT") { eprintln!("Warning: ES.FUT test file not found, skipping repeated queries benchmark"); return; } let mut file_mapping = HashMap::new(); file_mapping.insert("ES.FUT".to_string(), files["ES.FUT"].clone()); let repo = rt.block_on(async { DbnMarketDataRepository::new(file_mapping).await.unwrap() }); let mut group = c.benchmark_group("repeated_queries"); for iterations in [5, 10, 20].iter() { group.bench_with_input( BenchmarkId::from_parameter(iterations), iterations, |b, &iterations| { b.to_async(&rt).iter(|| async { for _ in 0..iterations { let symbols = vec!["ES.FUT".to_string()]; let start_time = 1704153600_000_000_000i64; let end_time = 1704240000_000_000_000i64; let data = repo .load_historical_data(&symbols, start_time, end_time) .await .unwrap(); black_box(data); } }); }, ); } group.finish(); } /// Benchmark 5: Query latency percentiles fn bench_query_latency(c: &mut Criterion) { let rt = Runtime::new().unwrap(); let files = get_test_files(); if !files.contains_key("ES.FUT") { eprintln!("Warning: ES.FUT test file not found, skipping latency benchmark"); return; } let mut file_mapping = HashMap::new(); file_mapping.insert("ES.FUT".to_string(), files["ES.FUT"].clone()); let repo = rt.block_on(async { DbnMarketDataRepository::new(file_mapping).await.unwrap() }); c.bench_function("query_latency_p50_p95_p99", |b| { b.to_async(&rt).iter(|| async { let symbols = vec!["ES.FUT".to_string()]; let start_time = 1704153600_000_000_000i64; let end_time = 1704240000_000_000_000i64; let data = repo .load_historical_data(&symbols, start_time, end_time) .await .unwrap(); black_box(data); }); }); } /// Benchmark 6: Memory usage estimation fn bench_memory_usage(c: &mut Criterion) { let rt = Runtime::new().unwrap(); let files = get_test_files(); if files.is_empty() { eprintln!("Warning: No test files found, skipping memory benchmark"); return; } let repo = rt.block_on(async { DbnMarketDataRepository::new(files.clone()).await.unwrap() }); let mut group = c.benchmark_group("memory_usage"); // Load all symbols and measure memory let all_symbols: Vec = files.keys().cloned().collect(); group.bench_function("all_symbols_full_day", |b| { b.to_async(&rt).iter(|| async { let start_time = 1704153600_000_000_000i64; let end_time = 1704240000_000_000_000i64; let data = repo .load_historical_data(&all_symbols, start_time, end_time) .await .unwrap(); // Calculate approximate memory usage let bar_size = std::mem::size_of::(); let total_bytes = data.len() * bar_size; let total_mb = total_bytes as f64 / 1_048_576.0; println!("\nMemory usage estimate:"); println!(" Bars loaded: {}", data.len()); println!(" Approx memory: {:.2} MB", total_mb); black_box(data); }); }); group.finish(); } /// Benchmark 7: Cold start performance fn bench_cold_start(c: &mut Criterion) { let rt = Runtime::new().unwrap(); let files = get_test_files(); if !files.contains_key("ES.FUT") { eprintln!("Warning: ES.FUT test file not found, skipping cold start benchmark"); return; } let mut group = c.benchmark_group("cold_start"); // Measure repository creation + first query group.bench_function("repo_creation_and_first_query", |b| { b.to_async(&rt).iter(|| async { let mut file_mapping = HashMap::new(); file_mapping.insert("ES.FUT".to_string(), files["ES.FUT"].clone()); let repo = DbnMarketDataRepository::new(file_mapping).await.unwrap(); let symbols = vec!["ES.FUT".to_string()]; let start_time = 1704153600_000_000_000i64; let end_time = 1704240000_000_000_000i64; let data = repo .load_historical_data(&symbols, start_time, end_time) .await .unwrap(); black_box(data); }); }); group.finish(); } /// Benchmark 8: Concurrent queries fn bench_concurrent_queries(c: &mut Criterion) { let rt = Runtime::new().unwrap(); let files = get_test_files(); if files.len() < 2 { eprintln!("Warning: Need at least 2 test files, skipping concurrent benchmark"); return; } let repo = rt.block_on(async { DbnMarketDataRepository::new(files.clone()).await.unwrap() }); let mut group = c.benchmark_group("concurrent_queries"); // Test concurrent queries for different symbols group.bench_function("concurrent_2_symbols", |b| { b.to_async(&rt).iter(|| async { let start_time = 1704153600_000_000_000i64; let end_time = 1704240000_000_000_000i64; // Spawn concurrent queries let handles: Vec<_> = vec!["ES.FUT", "NQ.FUT"] .into_iter() .filter(|s| files.contains_key(*s)) .map(|symbol| { let repo_clone = &repo; let symbols = vec![symbol.to_string()]; async move { repo_clone .load_historical_data(&symbols, start_time, end_time) .await .unwrap() } }) .collect(); // Wait for all to complete let results = futures::future::join_all(handles).await; black_box(results); }); }); group.finish(); } criterion_group!( benches, bench_single_file_loading, bench_multi_file_loading, bench_time_range_queries, bench_repeated_queries, bench_query_latency, bench_memory_usage, bench_cold_start, bench_concurrent_queries, ); criterion_main!(benches);