//! DBN Market Data Helpers for E2E Tests //! //! This module provides utilities to load real market data from DBN files //! and use it in E2E trading service tests. use anyhow::{Context, Result}; use backtesting_service::dbn_data_source::DbnDataSource; use backtesting_service::strategy_engine::MarketData as BacktestMarketData; use chrono::{DateTime, Utc}; use rust_decimal::Decimal; use std::collections::HashMap; use std::path::PathBuf; use std::sync::Arc; use tokio::sync::RwLock; /// DBN test data manager /// /// Provides access to real DBN market data for E2E tests pub struct DbnTestDataManager { data_source: Arc, /// Cached market data by symbol cache: Arc>>>, } impl DbnTestDataManager { /// Create a new DBN test data manager with ES.FUT data /// /// # Returns /// /// Configured manager with ES.FUT data loaded pub async fn new() -> Result { // Get workspace root let workspace_root = Self::find_workspace_root()?; // Build file mapping for available DBN files let mut file_mapping = HashMap::new(); // ES.FUT OHLCV data (1-minute bars, 2024-01-02) let es_fut_path = workspace_root .join("test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn"); if es_fut_path.exists() { file_mapping.insert( "ES.FUT".to_string(), es_fut_path.to_string_lossy().to_string(), ); } else { anyhow::bail!("DBN test data not found: {}", es_fut_path.display()); } // Create data source let data_source = Arc::new( DbnDataSource::new(file_mapping) .await .context("Failed to create DBN data source")?, ); Ok(Self { data_source, cache: Arc::new(RwLock::new(HashMap::new())), }) } /// Find the workspace root directory fn find_workspace_root() -> Result { let current_dir = std::env::current_dir()?; // Walk up from current directory to find workspace root for ancestor in current_dir.ancestors() { let cargo_toml = ancestor.join("Cargo.toml"); let test_data = ancestor.join("test_data"); if cargo_toml.exists() && test_data.exists() { return Ok(ancestor.to_path_buf()); } } anyhow::bail!("Could not find workspace root with test_data directory") } /// Get available symbols pub fn available_symbols(&self) -> Vec { self.data_source.available_symbols() } /// Load market data for symbol (with caching) /// /// # Arguments /// /// * `symbol` - Trading symbol to load data for /// /// # Returns /// /// Vector of market data bars sorted by timestamp pub async fn load_market_data(&self, symbol: &str) -> Result> { // Check cache first { let cache = self.cache.read().await; if let Some(data) = cache.get(symbol) { return Ok(data.clone()); } } // Load from data source let data = self.data_source.load_ohlcv_bars(symbol).await?; // Cache for future use { let mut cache = self.cache.write().await; cache.insert(symbol.to_string(), data.clone()); } Ok(data) } /// Get a realistic price for a symbol based on real data /// /// # Arguments /// /// * `symbol` - Trading symbol /// /// # Returns /// /// A realistic price from the loaded data pub async fn get_realistic_price(&self, symbol: &str) -> Result { let data = self.load_market_data(symbol).await?; if data.is_empty() { anyhow::bail!("No market data available for symbol: {}", symbol); } // Use the last close price as a realistic price let last_bar = &data[data.len() - 1]; Ok(last_bar.close.to_string().parse()?) } /// Get time range for available data /// /// # Arguments /// /// * `symbol` - Trading symbol /// /// # Returns /// /// Tuple of (start_time, end_time) for available data pub async fn get_time_range(&self, symbol: &str) -> Result<(DateTime, DateTime)> { let data = self.load_market_data(symbol).await?; if data.is_empty() { anyhow::bail!("No market data available for symbol: {}", symbol); } let start_time = data[0].timestamp; let end_time = data[data.len() - 1].timestamp; Ok((start_time, end_time)) } /// Get market data for a specific time window /// /// # Arguments /// /// * `symbol` - Trading symbol /// * `start_time` - Start of time window /// * `end_time` - End of time window /// /// # Returns /// /// Filtered market data within the time window pub async fn get_data_window( &self, symbol: &str, start_time: DateTime, end_time: DateTime, ) -> Result> { let data = self.load_market_data(symbol).await?; let filtered: Vec = data .into_iter() .filter(|bar| bar.timestamp >= start_time && bar.timestamp <= end_time) .collect(); Ok(filtered) } /// Create a realistic order price based on current market data /// /// # Arguments /// /// * `symbol` - Trading symbol /// * `side` - Order side (Buy/Sell) /// * `offset_bps` - Offset in basis points from current price /// /// # Returns /// /// Realistic order price pub async fn create_realistic_order_price( &self, symbol: &str, side: &str, offset_bps: i32, ) -> Result { let current_price = self.get_realistic_price(symbol).await?; // Apply offset based on side let multiplier = match side.to_lowercase().as_str() { "buy" => 1.0 - (offset_bps as f64 / 10000.0), // Bid lower "sell" => 1.0 + (offset_bps as f64 / 10000.0), // Ask higher _ => 1.0, }; Ok(current_price * multiplier) } /// Get OHLCV data for the last N bars /// /// # Arguments /// /// * `symbol` - Trading symbol /// * `num_bars` - Number of bars to retrieve /// /// # Returns /// /// Last N bars of market data pub async fn get_last_n_bars(&self, symbol: &str, num_bars: usize) -> Result> { let data = self.load_market_data(symbol).await?; if data.len() < num_bars { return Ok(data); } let start_idx = data.len() - num_bars; Ok(data[start_idx..].to_vec()) } /// Convert BacktestMarketData to proto BarData /// /// # Arguments /// /// * `bar` - Market data bar /// /// # Returns /// /// Proto BarData message pub fn to_proto_bar_data(&self, bar: &BacktestMarketData) -> Result<(String, i64, String, f64, f64, f64, f64, u64)> { let timestamp_nanos = bar.timestamp.timestamp_nanos_opt() .ok_or_else(|| anyhow::anyhow!("Invalid timestamp"))?; let open: f64 = bar.open.to_string().parse()?; let high: f64 = bar.high.to_string().parse()?; let low: f64 = bar.low.to_string().parse()?; let close: f64 = bar.close.to_string().parse()?; let volume: u64 = bar.volume.to_string().parse()?; Ok(( bar.symbol.clone(), timestamp_nanos, "1m".to_string(), open, high, low, close, volume, )) } } /// Global DBN test data manager instance static DBN_MANAGER: tokio::sync::OnceCell = tokio::sync::OnceCell::const_new(); /// Get the global DBN test data manager /// /// Lazily initializes the manager on first access pub async fn get_dbn_manager() -> Result<&'static DbnTestDataManager> { DBN_MANAGER .get_or_try_init(|| async { DbnTestDataManager::new().await }) .await .context("Failed to initialize DBN test data manager") } #[cfg(test)] mod tests { use super::*; #[tokio::test] async fn test_dbn_manager_creation() { let manager = DbnTestDataManager::new().await; assert!(manager.is_ok(), "Failed to create DBN manager: {:?}", manager.err()); let mgr = manager.unwrap(); let symbols = mgr.available_symbols(); assert!(!symbols.is_empty(), "No symbols available"); assert!(symbols.contains(&"ES.FUT".to_string())); } #[tokio::test] async fn test_load_market_data() { let manager = DbnTestDataManager::new().await.unwrap(); let data = manager.load_market_data("ES.FUT").await; assert!(data.is_ok(), "Failed to load market data: {:?}", data.err()); let bars = data.unwrap(); assert!(!bars.is_empty(), "No market data loaded"); assert!(bars.len() > 100, "Expected more bars, got: {}", bars.len()); // Verify data quality let first_bar = &bars[0]; assert_eq!(first_bar.symbol, "ES.FUT"); let close_f64: f64 = first_bar.close.to_string().parse().unwrap(); assert!(close_f64 > 4000.0 && close_f64 < 6000.0, "Unexpected ES.FUT price: {}", close_f64); } #[tokio::test] async fn test_get_realistic_price() { let manager = DbnTestDataManager::new().await.unwrap(); let price = manager.get_realistic_price("ES.FUT").await; assert!(price.is_ok(), "Failed to get price: {:?}", price.err()); let p = price.unwrap(); assert!(p > 4000.0 && p < 6000.0, "Unexpected price: {}", p); } #[tokio::test] async fn test_get_time_range() { let manager = DbnTestDataManager::new().await.unwrap(); let range = manager.get_time_range("ES.FUT").await; assert!(range.is_ok(), "Failed to get time range: {:?}", range.err()); let (start, end) = range.unwrap(); assert!(end > start, "End time should be after start time"); } }