e8a68ee39f
- Created data/examples/download_ml_training_data.rs using reqwest + Databento HTTP API - Downloaded 90 days × 4 symbols (ES.FUT, NQ.FUT, ZN.FUT, 6E.FUT) - Files saved to test_data/real/databento/ml_training/ - Total: 360 files, 15 MB compressed DBN format - Used existing Rust pattern from download_nq_fut.rs - API key loaded from .env file - 100% success rate (360/360 files) - Ready for ML training benchmarks Next: Create simplified training benchmark for RTX 3050 Ti GPU measurements
444 lines
14 KiB
Rust
444 lines
14 KiB
Rust
//! DBN Integration Tests
|
|
//!
|
|
//! Tests for DBN file loading and integration with backtesting service.
|
|
//! Uses real market data from test_data/real/databento/.
|
|
|
|
use anyhow::Result;
|
|
|
|
mod mock_repositories;
|
|
|
|
use backtesting_service::dbn_data_source::DbnDataSource;
|
|
use backtesting_service::dbn_repository::DbnMarketDataRepository;
|
|
use backtesting_service::repositories::MarketDataRepository;
|
|
use std::collections::HashMap;
|
|
|
|
#[tokio::test]
|
|
async fn test_load_real_dbn_file() -> Result<()> {
|
|
// Create file mapping using helper to get correct path
|
|
let mut file_mapping = HashMap::new();
|
|
file_mapping.insert(
|
|
"ES.FUT".to_string(),
|
|
mock_repositories::get_dbn_test_file_path(),
|
|
);
|
|
|
|
// Create data source
|
|
let data_source = DbnDataSource::new(file_mapping).await?;
|
|
|
|
// Load bars
|
|
let bars = data_source.load_ohlcv_bars("ES.FUT").await?;
|
|
|
|
// Validate bar count (ES.FUT 2024-01-02 has ~1674 one-minute bars from real DBN data)
|
|
assert!(
|
|
!bars.is_empty(),
|
|
"Should load bars from DBN file"
|
|
);
|
|
assert!(
|
|
bars.len() > 1500 && bars.len() < 1800,
|
|
"Expected ~1674 bars (1500-1800 range) from real DBN data, got {}",
|
|
bars.len()
|
|
);
|
|
println!("✅ Loaded {} bars from real DBN file", bars.len());
|
|
|
|
// Validate first bar structure
|
|
let first_bar = &bars[0];
|
|
assert_eq!(first_bar.symbol, "ES.FUT", "Symbol should be ES.FUT");
|
|
assert!(first_bar.open > rust_decimal::Decimal::ZERO, "Open price should be positive");
|
|
assert!(first_bar.high >= first_bar.open, "High should be >= open");
|
|
assert!(first_bar.low <= first_bar.open, "Low should be <= open");
|
|
assert!(first_bar.close > rust_decimal::Decimal::ZERO, "Close price should be positive");
|
|
assert!(first_bar.volume >= rust_decimal::Decimal::ZERO, "Volume should be non-negative");
|
|
|
|
// Validate price ranges (ES.FUT typical range for 2024)
|
|
let open_f64 = first_bar.open.to_string().parse::<f64>().unwrap_or(0.0);
|
|
assert!(
|
|
open_f64 > 3500.0 && open_f64 < 5500.0,
|
|
"ES.FUT price should be in realistic range (3500-5500), got {}",
|
|
open_f64
|
|
);
|
|
|
|
// Validate OHLCV relationships
|
|
assert!(first_bar.high >= first_bar.low, "High should be >= low");
|
|
assert!(first_bar.high >= first_bar.open, "High should be >= open");
|
|
assert!(first_bar.high >= first_bar.close, "High should be >= close");
|
|
assert!(first_bar.low <= first_bar.open, "Low should be <= open");
|
|
assert!(first_bar.low <= first_bar.close, "Low should be <= close");
|
|
|
|
// Check sorting
|
|
for i in 1..bars.len() {
|
|
assert!(
|
|
bars[i].timestamp >= bars[i - 1].timestamp,
|
|
"Bars should be sorted by timestamp"
|
|
);
|
|
}
|
|
|
|
println!("✅ Data quality validation passed");
|
|
println!(
|
|
" First bar: {} @ {} (open={}, high={}, low={}, close={}, volume={})",
|
|
first_bar.symbol,
|
|
first_bar.timestamp,
|
|
first_bar.open,
|
|
first_bar.high,
|
|
first_bar.low,
|
|
first_bar.close,
|
|
first_bar.volume
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_dbn_repository_integration() -> Result<()> {
|
|
// Create repository using helper path
|
|
let mut file_mapping = HashMap::new();
|
|
file_mapping.insert(
|
|
"ES.FUT".to_string(),
|
|
mock_repositories::get_dbn_test_file_path(),
|
|
);
|
|
|
|
let repo = DbnMarketDataRepository::new(file_mapping).await?;
|
|
|
|
// Load data via repository interface
|
|
let symbols = vec!["ES.FUT".to_string()];
|
|
|
|
// 2024-01-02 00:00:00 to 2024-01-03 00:00:00 (full day)
|
|
let start_time = 1704153600_000_000_000i64;
|
|
let end_time = 1704240000_000_000_000i64;
|
|
|
|
let data = repo.load_historical_data(&symbols, start_time, end_time).await?;
|
|
|
|
assert!(!data.is_empty(), "Repository should load data");
|
|
println!("✅ Repository loaded {} bars", data.len());
|
|
|
|
// Validate time range
|
|
for bar in &data {
|
|
let ts_nanos = bar.timestamp.timestamp_nanos_opt().unwrap_or(0);
|
|
assert!(
|
|
ts_nanos >= start_time && ts_nanos <= end_time,
|
|
"Bar timestamp should be within requested range"
|
|
);
|
|
}
|
|
|
|
println!("✅ Time range filtering validated");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_dbn_data_availability() -> Result<()> {
|
|
let mut file_mapping = HashMap::new();
|
|
file_mapping.insert(
|
|
"ES.FUT".to_string(),
|
|
mock_repositories::get_dbn_test_file_path().to_string(),
|
|
);
|
|
|
|
let repo = DbnMarketDataRepository::new(file_mapping).await?;
|
|
|
|
// Check availability for both existing and non-existing symbols
|
|
let symbols = vec!["ES.FUT".to_string(), "NONEXISTENT.SYM".to_string()];
|
|
let start_time = 1704153600_000_000_000i64;
|
|
let end_time = 1704240000_000_000_000i64;
|
|
|
|
let availability = repo
|
|
.check_data_availability(&symbols, start_time, end_time)
|
|
.await?;
|
|
|
|
// ES.FUT should be available (file exists)
|
|
assert_eq!(
|
|
availability.get("ES.FUT"),
|
|
Some(&true),
|
|
"ES.FUT should be available (file exists)"
|
|
);
|
|
|
|
// NONEXISTENT should not be available
|
|
assert_eq!(
|
|
availability.get("NONEXISTENT.SYM"),
|
|
Some(&false),
|
|
"NONEXISTENT.SYM should not be available"
|
|
);
|
|
|
|
println!("✅ Data availability check passed");
|
|
println!(" ES.FUT: available");
|
|
println!(" NONEXISTENT.SYM: not available");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_timestamp_format() -> Result<()> {
|
|
let repo = mock_repositories::create_dbn_repository().await?;
|
|
|
|
let symbols = vec!["ES.FUT".to_string()];
|
|
let start_time = 1704153600_000_000_000i64; // 2024-01-02 00:00:00 UTC
|
|
let end_time = 1704240000_000_000_000i64; // 2024-01-03 00:00:00 UTC
|
|
|
|
let data = repo.load_historical_data(&symbols, start_time, end_time).await?;
|
|
|
|
assert!(!data.is_empty(), "Should have data for timestamp validation");
|
|
|
|
// Validate timestamps are in correct range (nanoseconds, Unix epoch)
|
|
for (i, bar) in data.iter().enumerate() {
|
|
let ts_nanos = bar.timestamp.timestamp_nanos_opt().unwrap_or(0);
|
|
|
|
assert!(
|
|
ts_nanos > 1700000000_000_000_000i64,
|
|
"Bar {}: Timestamp should be in nanoseconds (after 2023), got {}",
|
|
i, ts_nanos
|
|
);
|
|
assert!(
|
|
ts_nanos < 1750000000_000_000_000i64,
|
|
"Bar {}: Timestamp should be reasonable (before 2026), got {}",
|
|
i, ts_nanos
|
|
);
|
|
assert!(
|
|
ts_nanos >= start_time && ts_nanos <= end_time,
|
|
"Bar {}: Timestamp should be within requested range [{}, {}], got {}",
|
|
i, start_time, end_time, ts_nanos
|
|
);
|
|
}
|
|
|
|
// Validate timestamps are sorted
|
|
for i in 1..data.len() {
|
|
let prev_ts = data[i-1].timestamp.timestamp_nanos_opt().unwrap_or(0);
|
|
let curr_ts = data[i].timestamp.timestamp_nanos_opt().unwrap_or(0);
|
|
assert!(
|
|
curr_ts >= prev_ts,
|
|
"Bar {}: Timestamps should be sorted (prev: {}, curr: {})",
|
|
i, prev_ts, curr_ts
|
|
);
|
|
}
|
|
|
|
println!("✅ All {} timestamps valid and sorted", data.len());
|
|
println!(" First timestamp: {}", data[0].timestamp);
|
|
println!(" Last timestamp: {}", data[data.len()-1].timestamp);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_dbn_performance() -> Result<()> {
|
|
use std::time::Instant;
|
|
|
|
let mut file_mapping = HashMap::new();
|
|
file_mapping.insert(
|
|
"ES.FUT".to_string(),
|
|
mock_repositories::get_dbn_test_file_path().to_string(),
|
|
);
|
|
|
|
let data_source = DbnDataSource::new(file_mapping).await?;
|
|
|
|
// Warm-up run (file system cache)
|
|
let _ = data_source.load_ohlcv_bars("ES.FUT").await?;
|
|
|
|
// Timed run
|
|
let start = Instant::now();
|
|
let bars = data_source.load_ohlcv_bars("ES.FUT").await?;
|
|
let duration = start.elapsed();
|
|
|
|
println!("✅ Loaded {} bars in {:?}", bars.len(), duration);
|
|
|
|
// Performance validation: should be <100ms for ~400 bars (very conservative)
|
|
if bars.len() > 100 {
|
|
assert!(
|
|
duration.as_millis() < 100,
|
|
"Loading should be fast (<100ms for {} bars, got {}ms)",
|
|
bars.len(),
|
|
duration.as_millis()
|
|
);
|
|
|
|
// Calculate bars per second
|
|
let bars_per_sec = (bars.len() as f64 / duration.as_secs_f64()) as u64;
|
|
println!("✅ Performance target met: {}ms for {} bars",
|
|
duration.as_millis(),
|
|
bars.len()
|
|
);
|
|
println!(" Throughput: {} bars/sec", bars_per_sec);
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_dbn_multi_symbol_loading() -> Result<()> {
|
|
let mut file_mapping = HashMap::new();
|
|
file_mapping.insert(
|
|
"ES.FUT".to_string(),
|
|
mock_repositories::get_dbn_test_file_path().to_string(),
|
|
);
|
|
|
|
let data_source = DbnDataSource::new(file_mapping).await?;
|
|
|
|
// Load multiple symbols (only ES.FUT exists in this test)
|
|
let symbols = vec!["ES.FUT".to_string()];
|
|
let bars = data_source.load_multi_symbol_bars(&symbols).await?;
|
|
|
|
assert!(!bars.is_empty(), "Should load multi-symbol data");
|
|
println!("✅ Multi-symbol loading: {} bars", bars.len());
|
|
|
|
// All bars should be from ES.FUT
|
|
for bar in &bars {
|
|
assert_eq!(bar.symbol, "ES.FUT");
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_ohlcv_data_quality() -> Result<()> {
|
|
let repo = mock_repositories::create_dbn_repository().await?;
|
|
|
|
let symbols = vec!["ES.FUT".to_string()];
|
|
let start_time = 1704153600_000_000_000i64; // 2024-01-02 00:00:00 UTC
|
|
let end_time = 1704240000_000_000_000i64; // 2024-01-03 00:00:00 UTC
|
|
|
|
let data = repo.load_historical_data(&symbols, start_time, end_time).await?;
|
|
|
|
assert!(!data.is_empty(), "Should have data for quality validation");
|
|
|
|
let mut quality_issues = 0;
|
|
|
|
for (i, bar) in data.iter().enumerate() {
|
|
// Check OHLCV relationships
|
|
let high_gte_low = bar.high >= bar.low;
|
|
let high_gte_open = bar.high >= bar.open;
|
|
let high_gte_close = bar.high >= bar.close;
|
|
let low_lte_open = bar.low <= bar.open;
|
|
let low_lte_close = bar.low <= bar.close;
|
|
|
|
let valid = high_gte_low && high_gte_open && high_gte_close && low_lte_open && low_lte_close;
|
|
|
|
if !valid {
|
|
quality_issues += 1;
|
|
if quality_issues <= 5 {
|
|
eprintln!(
|
|
"Quality issue at bar {}: open={}, high={}, low={}, close={}",
|
|
i, bar.open, bar.high, bar.low, bar.close
|
|
);
|
|
eprintln!(" high >= low: {}", high_gte_low);
|
|
eprintln!(" high >= open: {}", high_gte_open);
|
|
eprintln!(" high >= close: {}", high_gte_close);
|
|
eprintln!(" low <= open: {}", low_lte_open);
|
|
eprintln!(" low <= close: {}", low_lte_close);
|
|
}
|
|
}
|
|
|
|
// Check positive values
|
|
assert!(
|
|
bar.open > rust_decimal::Decimal::ZERO,
|
|
"Bar {}: Open should be positive, got {}",
|
|
i, bar.open
|
|
);
|
|
assert!(
|
|
bar.high > rust_decimal::Decimal::ZERO,
|
|
"Bar {}: High should be positive, got {}",
|
|
i, bar.high
|
|
);
|
|
assert!(
|
|
bar.low > rust_decimal::Decimal::ZERO,
|
|
"Bar {}: Low should be positive, got {}",
|
|
i, bar.low
|
|
);
|
|
assert!(
|
|
bar.close > rust_decimal::Decimal::ZERO,
|
|
"Bar {}: Close should be positive, got {}",
|
|
i, bar.close
|
|
);
|
|
assert!(
|
|
bar.volume >= rust_decimal::Decimal::ZERO,
|
|
"Bar {}: Volume should be non-negative, got {}",
|
|
i, bar.volume
|
|
);
|
|
|
|
// Check realistic price ranges for ES.FUT (3500-5500 for 2024)
|
|
let close_f64 = bar.close.to_string().parse::<f64>().unwrap_or(0.0);
|
|
assert!(
|
|
close_f64 > 3000.0 && close_f64 < 6000.0,
|
|
"Bar {}: ES.FUT price {} outside realistic range (3000-6000)",
|
|
i, close_f64
|
|
);
|
|
}
|
|
|
|
assert_eq!(
|
|
quality_issues, 0,
|
|
"Found {} OHLCV data quality issues",
|
|
quality_issues
|
|
);
|
|
|
|
println!("✅ All {} bars passed OHLCV quality checks", data.len());
|
|
println!(" Zero quality issues detected");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_dbn_data_quality_validation() -> Result<()> {
|
|
let mut file_mapping = HashMap::new();
|
|
file_mapping.insert(
|
|
"ES.FUT".to_string(),
|
|
mock_repositories::get_dbn_test_file_path().to_string(),
|
|
);
|
|
|
|
let data_source = DbnDataSource::new(file_mapping).await?;
|
|
let bars = data_source.load_ohlcv_bars("ES.FUT").await?;
|
|
|
|
assert!(!bars.is_empty(), "Should have data");
|
|
|
|
// Validate OHLCV relationships
|
|
for (i, bar) in bars.iter().enumerate() {
|
|
// High >= Low
|
|
assert!(
|
|
bar.high >= bar.low,
|
|
"Bar {}: high ({}) should be >= low ({})",
|
|
i,
|
|
bar.high,
|
|
bar.low
|
|
);
|
|
|
|
// Open/Close within High/Low range
|
|
assert!(
|
|
bar.open >= bar.low && bar.open <= bar.high,
|
|
"Bar {}: open should be within [low, high]",
|
|
i
|
|
);
|
|
assert!(
|
|
bar.close >= bar.low && bar.close <= bar.high,
|
|
"Bar {}: close should be within [low, high]",
|
|
i
|
|
);
|
|
|
|
// Positive values
|
|
assert!(bar.open > rust_decimal::Decimal::ZERO, "Bar {}: open should be positive", i);
|
|
assert!(bar.volume >= rust_decimal::Decimal::ZERO, "Bar {}: volume should be non-negative", i);
|
|
|
|
// Realistic ES.FUT prices (roughly 4000-5000 range for 2024)
|
|
let close_f64 = bar.close.to_string().parse::<f64>().unwrap_or(0.0);
|
|
assert!(
|
|
close_f64 > 3000.0 && close_f64 < 6000.0,
|
|
"Bar {}: ES.FUT price {} seems unrealistic",
|
|
i,
|
|
close_f64
|
|
);
|
|
}
|
|
|
|
println!("✅ Data quality validation passed for {} bars", bars.len());
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_helper_create_dbn_repository() -> Result<()> {
|
|
// Test the helper function from mock_repositories
|
|
let repo = mock_repositories::create_dbn_repository().await?;
|
|
|
|
// Load some data
|
|
let symbols = vec!["ES.FUT".to_string()];
|
|
let start_time = 1704153600_000_000_000i64;
|
|
let end_time = 1704240000_000_000_000i64;
|
|
|
|
let data = repo.load_historical_data(&symbols, start_time, end_time).await?;
|
|
|
|
assert!(!data.is_empty(), "Helper function should create working repository");
|
|
println!("✅ Helper function test: loaded {} bars", data.len());
|
|
|
|
Ok(())
|
|
}
|