#![allow(unexpected_cfgs)] #![cfg(feature = "__backtesting_integration")] //! Multi-day DBN data loading tests //! //! Tests for DbnDataSource with multiple files per symbol (multi-day datasets). use anyhow::Result; use backtesting_service::dbn_data_source::DbnDataSource; use chrono::{DateTime, TimeZone, Utc}; use std::collections::HashMap; /// Helper to get workspace root fn get_workspace_root() -> std::path::PathBuf { let current_dir = std::env::current_dir().expect("INVARIANT: Current directory should be accessible"); current_dir .ancestors() .find(|p| p.join("Cargo.toml").exists() && p.join("test_data").exists()) .expect("Could not find workspace root") .to_path_buf() } /// Helper to get test file path fn get_test_file(filename: &str) -> String { get_workspace_root() .join(format!("test_data/real/databento/{}", filename)) .to_string_lossy() .to_string() } #[tokio::test] async fn test_single_file_backward_compatible() -> Result<()> { let mut file_mapping = HashMap::new(); file_mapping.insert( "ES.FUT".to_string(), get_test_file("ES.FUT_ohlcv-1m_2024-01-02.dbn"), ); let data_source = DbnDataSource::new(file_mapping).await?; // Old API should still work let bars = data_source.load_ohlcv_bars("ES.FUT").await?; assert!(!bars.is_empty(), "Should load bars"); assert!( bars.len() > 350 && bars.len() < 450, "Expected ~390 bars, got {}", bars.len() ); println!("✅ Backward compatibility: loaded {} bars", bars.len()); Ok(()) } #[tokio::test] async fn test_multi_file_creation() -> Result<()> { let mut file_mapping = HashMap::new(); file_mapping.insert( "ESH4".to_string(), vec![ get_test_file("ESH4_ohlcv-1m_2024-01-03.dbn"), get_test_file("ESH4_ohlcv-1m_2024-01-04.dbn"), get_test_file("ESH4_ohlcv-1m_2024-01-05.dbn"), ], ); let data_source = DbnDataSource::new_multi_file(file_mapping).await?; assert_eq!(data_source.available_symbols().len(), 1); assert_eq!(data_source.get_file_count("ESH4"), 3); let file_paths = data_source.get_file_paths("ESH4").unwrap(); assert_eq!(file_paths.len(), 3); println!("✅ Multi-file creation: 3 files configured for ESH4"); Ok(()) } #[tokio::test] async fn test_load_all_files() -> Result<()> { let mut file_mapping = HashMap::new(); file_mapping.insert( "ESH4".to_string(), vec![ get_test_file("ESH4_ohlcv-1m_2024-01-03.dbn"), get_test_file("ESH4_ohlcv-1m_2024-01-04.dbn"), get_test_file("ESH4_ohlcv-1m_2024-01-05.dbn"), ], ); let data_source = DbnDataSource::new_multi_file(file_mapping).await?; let start = std::time::Instant::now(); let bars = data_source.load_ohlcv_bars_all("ESH4").await?; let duration = start.elapsed(); assert!(!bars.is_empty(), "Should load bars from all files"); // Verify chronological sorting across files for i in 1..bars.len() { assert!( bars[i].timestamp >= bars[i - 1].timestamp, "Bars should be sorted chronologically across files" ); } println!( "✅ Multi-file loading: {} bars from 3 files in {:?}", bars.len(), duration ); println!( " Performance: {:.2}ms per file", duration.as_secs_f64() * 1000.0 / 3.0 ); // Performance validation: linear scaling (3 files × ~0.7ms = ~2.1ms target) assert!( duration.as_millis() < 30, "Multi-file loading took {}ms (target: <30ms for 3 files)", duration.as_millis() ); Ok(()) } #[tokio::test] async fn test_load_single_file_from_multi() -> Result<()> { let mut file_mapping = HashMap::new(); file_mapping.insert( "ESH4".to_string(), vec![ get_test_file("ESH4_ohlcv-1m_2024-01-03.dbn"), get_test_file("ESH4_ohlcv-1m_2024-01-04.dbn"), get_test_file("ESH4_ohlcv-1m_2024-01-05.dbn"), ], ); let data_source = DbnDataSource::new_multi_file(file_mapping).await?; // load_ohlcv_bars should only load first file let single_file_bars = data_source.load_ohlcv_bars("ESH4").await?; // load_ohlcv_bars_all should load all files let all_files_bars = data_source.load_ohlcv_bars_all("ESH4").await?; assert!( all_files_bars.len() > single_file_bars.len(), "All files should have more bars than single file" ); println!( "✅ Single vs all: {} bars (first file) vs {} bars (all files)", single_file_bars.len(), all_files_bars.len() ); Ok(()) } #[tokio::test] async fn test_date_range_filtering() -> Result<()> { let mut file_mapping = HashMap::new(); file_mapping.insert( "ESH4".to_string(), vec![ get_test_file("ESH4_ohlcv-1m_2024-01-03.dbn"), get_test_file("ESH4_ohlcv-1m_2024-01-04.dbn"), get_test_file("ESH4_ohlcv-1m_2024-01-05.dbn"), ], ); let data_source = DbnDataSource::new_multi_file(file_mapping).await?; // Query specific date range (Jan 4 only) let start_date = Utc.with_ymd_and_hms(2024, 1, 4, 0, 0, 0).expect("INVARIANT: Valid date/time parameters"); let end_date = Utc.with_ymd_and_hms(2024, 1, 4, 23, 59, 59).expect("INVARIANT: Valid date/time parameters"); let bars = data_source .load_ohlcv_bars_range("ESH4", start_date, end_date) .await?; // Verify all bars are within date range for bar in &bars { assert!( bar.timestamp >= start_date && bar.timestamp <= end_date, "Bar timestamp should be within requested range" ); } // All bars should be from Jan 4 for bar in &bars { assert_eq!(bar.timestamp.day(), 4, "All bars should be from Jan 4"); } println!( "✅ Date range filtering: {} bars from 2024-01-04", bars.len() ); Ok(()) } #[tokio::test] async fn test_multi_symbol_multi_file() -> Result<()> { let mut file_mapping = HashMap::new(); // ES.FUT: single file file_mapping.insert( "ES.FUT".to_string(), vec![get_test_file("ES.FUT_ohlcv-1m_2024-01-02.dbn")], ); // NQ.FUT: single file file_mapping.insert( "NQ.FUT".to_string(), vec![get_test_file("NQ.FUT_ohlcv-1m_2024-01-02.dbn")], ); let data_source = DbnDataSource::new_multi_file(file_mapping).await?; let symbols = vec!["ES.FUT".to_string(), "NQ.FUT".to_string()]; let bars = data_source.load_multi_symbol_bars_all(&symbols).await?; assert!(!bars.is_empty(), "Should load bars from multiple symbols"); // Verify bars are sorted by timestamp for i in 1..bars.len() { assert!( bars[i].timestamp >= bars[i - 1].timestamp, "Multi-symbol bars should be sorted chronologically" ); } // Verify both symbols present let has_es = bars.iter().any(|b| b.symbol == "ES.FUT"); let has_nq = bars.iter().any(|b| b.symbol == "NQ.FUT"); assert!(has_es, "Should have ES.FUT bars"); assert!(has_nq, "Should have NQ.FUT bars"); println!( "✅ Multi-symbol: {} total bars (ES.FUT + NQ.FUT)", bars.len() ); Ok(()) } #[tokio::test] async fn test_add_symbol_mapping_multi() -> Result<()> { let file_mapping = HashMap::new(); let mut data_source = DbnDataSource::new(file_mapping).await?; // Add multiple files for symbol data_source.add_symbol_mapping_multi( "ESH4".to_string(), vec![ get_test_file("ESH4_ohlcv-1m_2024-01-03.dbn"), get_test_file("ESH4_ohlcv-1m_2024-01-04.dbn"), ], ); assert_eq!(data_source.get_file_count("ESH4"), 2); let bars = data_source.load_ohlcv_bars_all("ESH4").await?; assert!(!bars.is_empty(), "Should load from dynamically added files"); println!("✅ Dynamic mapping: {} bars from 2 files", bars.len()); Ok(()) } #[tokio::test] async fn test_performance_linear_scaling() -> Result<()> { // Test 1 file let mut file_mapping_1 = HashMap::new(); file_mapping_1.insert( "ESH4".to_string(), vec![get_test_file("ESH4_ohlcv-1m_2024-01-03.dbn")], ); let data_source_1 = DbnDataSource::new_multi_file(file_mapping_1).await?; let start_1 = std::time::Instant::now(); let bars_1 = data_source_1.load_ohlcv_bars_all("ESH4").await?; let duration_1 = start_1.elapsed(); // Test 3 files let mut file_mapping_3 = HashMap::new(); file_mapping_3.insert( "ESH4".to_string(), vec![ get_test_file("ESH4_ohlcv-1m_2024-01-03.dbn"), get_test_file("ESH4_ohlcv-1m_2024-01-04.dbn"), get_test_file("ESH4_ohlcv-1m_2024-01-05.dbn"), ], ); let data_source_3 = DbnDataSource::new_multi_file(file_mapping_3).await?; let start_3 = std::time::Instant::now(); let bars_3 = data_source_3.load_ohlcv_bars_all("ESH4").await?; let duration_3 = start_3.elapsed(); let avg_per_file_1 = duration_1.as_secs_f64() * 1000.0; let avg_per_file_3 = duration_3.as_secs_f64() * 1000.0 / 3.0; println!("✅ Performance scaling:"); println!( " 1 file: {} bars in {:.2}ms", bars_1.len(), avg_per_file_1 ); println!( " 3 files: {} bars in {:.2}ms total ({:.2}ms/file)", bars_3.len(), duration_3.as_secs_f64() * 1000.0, avg_per_file_3 ); // Linear scaling validation: 3-file average should be within 2x of 1-file time // (allows for slight overhead from sorting/merging) assert!( avg_per_file_3 < avg_per_file_1 * 2.0, "Performance should scale linearly (got {:.2}ms/file for 3 files vs {:.2}ms for 1 file)", avg_per_file_3, avg_per_file_1 ); // Absolute performance: should maintain <10ms per file target assert!( avg_per_file_3 < 10.0, "Should maintain <10ms per file (got {:.2}ms)", avg_per_file_3 ); Ok(()) } #[tokio::test] async fn test_data_availability_multi_file() -> Result<()> { let mut file_mapping = HashMap::new(); file_mapping.insert( "ESH4".to_string(), vec![ get_test_file("ESH4_ohlcv-1m_2024-01-03.dbn"), get_test_file("ESH4_ohlcv-1m_2024-01-04.dbn"), ], ); let data_source = DbnDataSource::new_multi_file(file_mapping).await?; let start = Utc.with_ymd_and_hms(2024, 1, 3, 0, 0, 0).expect("INVARIANT: Valid date/time parameters"); let end = Utc.with_ymd_and_hms(2024, 1, 5, 0, 0, 0).expect("INVARIANT: Valid date/time parameters"); let available = data_source .check_data_availability("ESH4", start, end) .await?; assert!(available, "Data should be available"); // Nonexistent symbol let not_available = data_source .check_data_availability("NONEXISTENT", start, end) .await?; assert!(!not_available, "Nonexistent symbol should not be available"); println!("✅ Data availability check passed"); Ok(()) } #[tokio::test] async fn test_partial_day_range() -> Result<()> { let mut file_mapping = HashMap::new(); file_mapping.insert( "ES.FUT".to_string(), vec![get_test_file("ES.FUT_ohlcv-1m_2024-01-02.dbn")], ); let data_source = DbnDataSource::new_multi_file(file_mapping).await?; // Query just first hour let start = Utc.with_ymd_and_hms(2024, 1, 2, 14, 30, 0).expect("INVARIANT: Valid date/time parameters"); // Market open (ET) let end = Utc.with_ymd_and_hms(2024, 1, 2, 15, 30, 0).expect("INVARIANT: Valid date/time parameters"); let bars = data_source .load_ohlcv_bars_range("ES.FUT", start, end) .await?; assert!(!bars.is_empty(), "Should load partial day data"); // All bars should be within 1-hour window for bar in &bars { assert!(bar.timestamp >= start && bar.timestamp <= end); } // Should be ~60 bars (1-minute data for 1 hour) assert!( bars.len() >= 50 && bars.len() <= 70, "Expected ~60 bars for 1 hour, got {}", bars.len() ); println!("✅ Partial day: {} bars in 1-hour window", bars.len()); Ok(()) }