#![allow(clippy::items_after_test_module, dead_code)] //! Test Data Helpers for Real DBN Data //! //! Provides reusable fixtures for backtesting unit tests using real DBN market data. //! This module ensures tests remain fast (<100ms) while using production-quality data. use anyhow::Result; use backtesting_service::dbn_data_source::DbnDataSource; use backtesting_service::strategy_engine::{BacktestTrade, MarketData, TradeSide}; use chrono::{Duration, Utc}; use num_traits::ToPrimitive; use rust_decimal::Decimal; use std::collections::HashMap; use std::sync::Arc; use tokio::sync::OnceCell; /// Cached DBN data source (loaded once per test run) static DBN_DATA_SOURCE: OnceCell> = OnceCell::const_new(); /// Cached market data (loaded once per test run) static CACHED_ES_BARS: OnceCell>> = OnceCell::const_new(); /// Get absolute path to the test DBN file /// /// Resolves the path from the project root, handling different working directories. pub fn get_dbn_test_file_path() -> String { // Try to find project root by looking for Cargo.toml let mut current = std::env::current_dir().expect("INVARIANT: Current directory should be accessible"); // If we're in a subdirectory, go up until we find the workspace root while !current.join("Cargo.toml").exists() || !current.join("test_data").exists() { if !current.pop() { // Fallback to relative path if we can't find root return "../../test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn".to_string(); } } current .join("test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn") .to_string_lossy() .to_string() } /// Get or create the shared DBN data source (singleton pattern) pub async fn get_dbn_data_source() -> Result> { DBN_DATA_SOURCE .get_or_try_init(|| async { let mut file_mapping = HashMap::new(); file_mapping.insert("ES.FUT".to_string(), get_dbn_test_file_path()); let data_source = DbnDataSource::new(file_mapping).await?; Ok(Arc::new(data_source)) }) .await .cloned() } /// Get cached ES.FUT bars (loaded once per test run for performance) /// /// **Performance**: First call ~5-10ms, subsequent calls ~0.1μs pub async fn get_cached_es_bars() -> Result>> { CACHED_ES_BARS .get_or_try_init(|| async { let data_source = get_dbn_data_source().await?; let bars = data_source.load_ohlcv_bars("ES.FUT").await?; Ok(Arc::new(bars)) }) .await .cloned() } /// Get a small sample of real market data (fast, for unit tests) /// /// Returns the first N bars from the cached DBN data. /// /// # Arguments /// /// * `num_bars` - Number of bars to return (default: 50) /// /// # Returns /// /// Vec of MarketData with real ES.FUT prices /// /// # Performance /// /// ~1μs (data already cached) pub async fn get_sample_real_data(num_bars: usize) -> Result> { let all_bars = get_cached_es_bars().await?; let sample_size = num_bars.min(all_bars.len()); Ok(all_bars[0..sample_size].to_vec()) } /// Get real market data for a specific time window /// /// # Arguments /// /// * `start_offset_minutes` - Minutes from first bar (0 = first bar) /// * `duration_minutes` - Duration window in minutes /// /// # Returns /// /// Vec of MarketData within the time window pub async fn get_time_window_data( start_offset_minutes: i64, duration_minutes: i64, ) -> Result> { let all_bars = get_cached_es_bars().await?; if all_bars.is_empty() { return Ok(Vec::new()); } let first_timestamp = all_bars[0].timestamp; let start_time = first_timestamp + Duration::minutes(start_offset_minutes); let end_time = start_time + Duration::minutes(duration_minutes); let filtered: Vec = all_bars .iter() .filter(|bar| bar.timestamp >= start_time && bar.timestamp <= end_time) .cloned() .collect(); Ok(filtered) } /// Convert real market data to BacktestTrade (for metrics tests) /// /// Simulates a buy-and-sell trade based on actual price movements. /// /// # Arguments /// /// * `entry_bar` - Market data for entry /// * `exit_bar` - Market data for exit /// * `quantity` - Position size /// * `trade_id` - Unique trade identifier /// /// # Returns /// /// BacktestTrade with real PnL calculations pub fn create_trade_from_bars( entry_bar: &MarketData, exit_bar: &MarketData, quantity: f64, trade_id: u32, ) -> BacktestTrade { let entry_price = entry_bar.close; let exit_price = exit_bar.close; let entry_f64 = entry_price.to_f64().unwrap_or(0.0); let exit_f64 = exit_price.to_f64().unwrap_or(0.0); let pnl = (exit_f64 - entry_f64) * quantity; let return_percent = pnl / (entry_f64 * quantity); BacktestTrade { trade_id: format!("real_trade_{}", trade_id), symbol: entry_bar.symbol.clone(), side: TradeSide::Buy, quantity: Decimal::from_f64_retain(quantity).unwrap_or(Decimal::ZERO), entry_price, exit_price, entry_time: entry_bar.timestamp, exit_time: exit_bar.timestamp, pnl: Decimal::from_f64_retain(pnl).unwrap_or(Decimal::ZERO), return_percent: Decimal::from_f64_retain(return_percent).unwrap_or(Decimal::ZERO), entry_signal: "real_data_buy".to_string(), exit_signal: "real_data_sell".to_string(), } } /// Generate multiple trades from real data windows /// /// Creates trades by pairing consecutive bars (buy bar N, sell bar N+1). /// /// # Arguments /// /// * `num_trades` - Number of trades to generate /// /// # Returns /// /// Vec of BacktestTrade with real price movements pub async fn generate_real_trades(num_trades: usize) -> Result> { let bars = get_sample_real_data(num_trades * 2).await?; let mut trades = Vec::new(); for i in 0..num_trades.min(bars.len() / 2) { let entry_bar = &bars[i * 2]; let exit_bar = &bars[i * 2 + 1]; let trade = create_trade_from_bars(entry_bar, exit_bar, 1.0, i as u32); trades.push(trade); } Ok(trades) } /// Create a mixed trade sequence (wins and losses from real data) /// /// Samples bars with varying price movements to create realistic win/loss patterns. /// /// # Returns /// /// Vec of BacktestTrade with realistic PnL distribution pub async fn generate_mixed_trades() -> Result> { let bars = get_sample_real_data(100).await?; if bars.len() < 20 { return Ok(Vec::new()); } let mut trades = Vec::new(); // Strategy: Sample bars at specific intervals to get price variation // Every 5 bars creates different price movements (wins and losses) for i in 0..10 { let entry_idx = i * 5; let exit_idx = (i * 5 + 3).min(bars.len() - 1); if exit_idx >= bars.len() { break; } let entry_bar = &bars[entry_idx]; let exit_bar = &bars[exit_idx]; let trade = create_trade_from_bars(entry_bar, exit_bar, 1.0, i as u32); trades.push(trade); } Ok(trades) } /// Get real Sharpe ratio expected range from actual data /// /// Analyzes real data to provide realistic expectation bounds for tests. /// /// # Returns /// /// (min_sharpe, max_sharpe) - Expected Sharpe ratio range for ES.FUT data pub async fn get_real_sharpe_range() -> Result<(f64, f64)> { // ES.FUT intraday 1-minute data typically shows: // - Low Sharpe: -1.0 to 0.5 (choppy markets) // - High Sharpe: 0.5 to 2.0 (trending moves) // - Extreme: 2.0+ (strong directional moves) Ok((-1.0, 3.0)) // Conservative range for test assertions } /// Get real drawdown expected range from actual data /// /// # Returns /// /// (min_drawdown_pct, max_drawdown_pct) - Expected drawdown range pub async fn get_real_drawdown_range() -> Result<(f64, f64)> { // ES.FUT intraday typically: // - Small drawdown: 0.5% - 2% // - Medium drawdown: 2% - 5% // - Large drawdown: 5% - 15% Ok((0.0, 20.0)) // Conservative range for test assertions } /// Get realistic volatility range from actual data /// /// # Returns /// /// (min_volatility_pct, max_volatility_pct) - Expected annualized volatility pub async fn get_real_volatility_range() -> Result<(f64, f64)> { // ES.FUT intraday 1-minute bars: // - Annualized volatility typically 15% - 35% // - Can spike to 50%+ during extreme events Ok((0.0, 100.0)) // Very conservative for test robustness } #[cfg(test)] mod tests { use super::*; #[tokio::test] async fn test_load_cached_data() -> Result<()> { let bars = get_cached_es_bars().await?; assert!(!bars.is_empty(), "Should load real DBN data"); assert!(bars.len() > 300, "ES.FUT 2024-01-02 should have ~390 bars"); Ok(()) } #[tokio::test] async fn test_sample_data() -> Result<()> { let sample = get_sample_real_data(50).await?; assert_eq!(sample.len(), 50, "Should return requested sample size"); assert_eq!(sample[0].symbol, "ES.FUT"); Ok(()) } #[tokio::test] async fn test_time_window_data() -> Result<()> { let window = get_time_window_data(0, 60).await?; assert!(!window.is_empty(), "Should have data in first hour"); // Validate timestamp ordering for i in 1..window.len() { assert!(window[i].timestamp >= window[i - 1].timestamp); } Ok(()) } #[tokio::test] async fn test_generate_real_trades() -> Result<()> { let trades = generate_real_trades(10).await?; assert_eq!(trades.len(), 10, "Should generate requested trades"); // Validate trade structure for trade in &trades { assert_eq!(trade.symbol, "ES.FUT"); assert!(trade.exit_time > trade.entry_time); } Ok(()) } #[tokio::test] async fn test_mixed_trades() -> Result<()> { let trades = generate_mixed_trades().await?; assert!(!trades.is_empty(), "Should generate mixed trades"); // Should have both wins and losses let wins = trades.iter().filter(|t| t.pnl > Decimal::ZERO).count(); let losses = trades.iter().filter(|t| t.pnl < Decimal::ZERO).count(); // Real data should have variation (not all wins or all losses) assert!(wins > 0 || losses > 0, "Should have some PnL variation"); Ok(()) } } /// Create a simple trade for testing (with explicit parameters) /// /// This is a simplified helper for unit tests that need to create trades /// without loading real DBN data. /// /// # Arguments /// /// * `trade_id` - Unique trade identifier /// * `symbol` - Trading symbol /// * `side` - Trade side (Buy/Sell) /// * `quantity` - Position size /// * `entry_price` - Entry price /// * `exit_price` - Exit price /// * `entry_time` - Entry timestamp (days from now) /// * `exit_time` - Exit timestamp (days from now) /// /// # Returns /// /// BacktestTrade with calculated PnL pub fn create_trade( trade_id: u32, symbol: &str, side: TradeSide, quantity: f64, entry_price: f64, exit_price: f64, entry_time: i64, exit_time: i64, ) -> BacktestTrade { let pnl = match side { TradeSide::Buy => (exit_price - entry_price) * quantity, TradeSide::Sell => (entry_price - exit_price) * quantity, }; let return_percent = pnl / (entry_price * quantity); let now = Utc::now(); let entry_timestamp = now - Duration::days(entry_time); let exit_timestamp = now - Duration::days(exit_time); BacktestTrade { trade_id: format!("test_trade_{}", trade_id), symbol: symbol.to_string(), side, quantity: Decimal::from_f64_retain(quantity).unwrap_or(Decimal::ZERO), entry_price: Decimal::from_f64_retain(entry_price).unwrap_or(Decimal::ZERO), exit_price: Decimal::from_f64_retain(exit_price).unwrap_or(Decimal::ZERO), entry_time: entry_timestamp, exit_time: exit_timestamp, pnl: Decimal::from_f64_retain(pnl).unwrap_or(Decimal::ZERO), return_percent: Decimal::from_f64_retain(return_percent).unwrap_or(Decimal::ZERO), entry_signal: "test_entry".to_string(), exit_signal: "test_exit".to_string(), } }