//! Multi-Symbol DBN Data Validation Example //! //! Comprehensive validation of GC (Gold), ZN (Treasury), and 6E (Euro FX) DBN data quality. //! Checks data statistics, quality, and production readiness for all three symbols. use anyhow::Result; use backtesting_service::dbn_repository::DbnMarketDataRepository; use backtesting_service::repositories::MarketDataRepository; use chrono::{DateTime, Utc}; use rust_decimal::prelude::ToPrimitive; use rust_decimal::Decimal; use std::collections::HashMap; struct SymbolConfig { symbol: String, file_path: String, expected_price_min: f64, expected_price_max: f64, description: String, } struct ValidationResult { symbol: String, total_bars: usize, ohlcv_violations: usize, zero_volumes: usize, gaps: usize, price_spikes: usize, min_price: Decimal, max_price: Decimal, avg_price: Decimal, quality_score: String, production_ready: bool, } #[tokio::main] async fn main() -> Result<()> { println!("═══════════════════════════════════════════════════════════════"); println!(" Multi-Symbol DBN Data Validation Report"); println!(" GC (Gold) | ZN (Treasury) | 6E (Euro FX)"); println!("═══════════════════════════════════════════════════════════════\n"); // Configure symbols for validation // NOTE: Using decompressed files because dbn 0.42.0 doesn't support Zstandard compression let symbols = vec![ SymbolConfig { symbol: "GC".to_string(), file_path: "test_data/real/databento/GC_continuous_ohlcv-1m_2024-01-02_to_2024-01-31.uncompressed.dbn".to_string(), expected_price_min: 2000.0, expected_price_max: 2100.0, description: "Gold Futures (Continuous)".to_string(), }, SymbolConfig { symbol: "ZN.FUT".to_string(), file_path: "test_data/real/databento/ZN.FUT_ohlcv-1m_2024-01-02_to_2024-01-31.uncompressed.dbn".to_string(), expected_price_min: 110.0, expected_price_max: 113.0, description: "10-Year Treasury Note Futures".to_string(), }, SymbolConfig { symbol: "6E.FUT".to_string(), file_path: "test_data/real/databento/6E.FUT_ohlcv-1m_2024-01-02_to_2024-01-31.uncompressed.dbn".to_string(), expected_price_min: 1.08, expected_price_max: 1.11, description: "Euro FX Futures (EUR/USD)".to_string(), }, ]; let mut results = Vec::new(); // Validate each symbol for config in &symbols { match validate_symbol(config).await { Ok(result) => { print_validation_result(&result); results.push(result); } Err(e) => { println!("❌ ERROR validating {}: {}\n", config.symbol, e); } } } // Print overall summary print_overall_summary(&results); Ok(()) } async fn validate_symbol(config: &SymbolConfig) -> Result { println!("┌────────────────────────────────────────────────────────────┐"); println!("│ Symbol: {} - {}", config.symbol, config.description); println!("└────────────────────────────────────────────────────────────┘\n"); // Load data let mut file_mapping = HashMap::new(); file_mapping.insert(config.symbol.clone(), config.file_path.clone()); let repo = DbnMarketDataRepository::new(file_mapping).await?; let symbols = vec![config.symbol.clone()]; let start_time = 1704153600_000_000_000i64; // 2024-01-02 00:00:00 UTC let end_time = 1706745600_000_000_000i64; // 2024-01-31 23:59:59 UTC let data = repo .load_historical_data(&symbols, start_time, end_time) .await?; if data.is_empty() { println!("❌ ERROR: No data loaded!\n"); return Ok(ValidationResult { symbol: config.symbol.clone(), total_bars: 0, ohlcv_violations: 0, zero_volumes: 0, gaps: 0, price_spikes: 0, min_price: Decimal::ZERO, max_price: Decimal::ZERO, avg_price: Decimal::ZERO, quality_score: "POOR".to_string(), production_ready: false, }); } // Basic statistics println!("📊 Basic Statistics:"); println!(" Total bars: {}", data.len()); println!(" Symbol: {}", data[0].symbol); println!(); // Price statistics let mut prices: Vec = data.iter().map(|b| b.close).collect(); prices.sort(); let min_price = *prices.first().unwrap(); let max_price = *prices.last().unwrap(); let median_price = prices[prices.len() / 2]; let sum_prices: Decimal = prices.iter().sum(); let avg_price = sum_prices / Decimal::from(prices.len()); println!("💰 Price Analysis:"); println!(" Min close: ${:.2}", min_price); println!(" Max close: ${:.2}", max_price); println!(" Median close: ${:.2}", median_price); println!(" Avg close: ${:.2}", avg_price); println!(" Range: ${:.2}", max_price - min_price); println!(" Expected: ${:.2} - ${:.2}", config.expected_price_min, config.expected_price_max); // Check if prices are in expected range let min_price_f64 = min_price.to_f64().unwrap_or(0.0); let max_price_f64 = max_price.to_f64().unwrap_or(0.0); let price_range_ok = min_price_f64 >= config.expected_price_min * 0.9 && max_price_f64 <= config.expected_price_max * 1.1; if price_range_ok { println!(" ✅ Price range within expected bounds"); } else { println!(" ⚠️ Price range outside expected bounds"); } println!(); // Volume statistics let total_volume: Decimal = data.iter().map(|b| b.volume).sum(); let avg_volume = total_volume / Decimal::from(data.len()); let max_volume = data.iter().map(|b| b.volume).max().unwrap_or(Decimal::ZERO); let min_volume = data.iter().map(|b| b.volume).min().unwrap_or(Decimal::ZERO); println!("📈 Volume Analysis:"); println!(" Total volume: {:.0}", total_volume); println!(" Avg volume: {:.0}", avg_volume); println!(" Max volume: {:.0}", max_volume); println!(" Min volume: {:.0}", min_volume); println!(); // Timestamp analysis let first_ts = data.first().unwrap().timestamp; let last_ts = data.last().unwrap().timestamp; let duration_seconds = (last_ts - first_ts).num_seconds(); let duration_days = duration_seconds / 86400; let duration_hours = (duration_seconds % 86400) / 3600; println!("⏰ Timestamp Analysis:"); println!(" First bar: {}", format_timestamp(&first_ts)); println!(" Last bar: {}", format_timestamp(&last_ts)); println!( " Duration: {} days, {} hours", duration_days, duration_hours ); println!(" Expected: ~29-30 days (January 2024)"); println!(); // Data quality checks println!("✅ Data Quality Checks:"); let mut gaps = 0; let mut ohlcv_violations = 0; let mut zero_volumes = 0; let mut price_spikes = 0; for i in 0..data.len() { let bar = &data[i]; // Check OHLCV relationships if !(bar.high >= bar.low && bar.high >= bar.open && bar.high >= bar.close && bar.low <= bar.open && bar.low <= bar.close) { ohlcv_violations += 1; if ohlcv_violations <= 5 { // Only print first 5 println!( " OHLCV violation at bar {}: O={} H={} L={} C={}", i, bar.open, bar.high, bar.low, bar.close ); } } // Check for zero volume if bar.volume == Decimal::ZERO { zero_volumes += 1; } // Check for timestamp gaps (should be ~60 seconds for 1-minute bars) if i > 0 { let gap = (bar.timestamp - data[i - 1].timestamp).num_seconds(); if gap > 120 { // More than 2 minutes gaps += 1; if gaps <= 5 { // Only print first 5 println!( " Large gap at bar {}: {} seconds ({} minutes)", i, gap, gap / 60 ); } } } // Check for abnormal price spikes (>20% move for commodities/FX) if i > 0 { let prev_close = data[i - 1].close; if prev_close > Decimal::ZERO { let price_change_pct = ((bar.close - prev_close) / prev_close).abs() * Decimal::from(100); if price_change_pct > Decimal::from(20) { price_spikes += 1; if price_spikes <= 5 { // Only print first 5 println!( " Price spike at bar {}: {:.2}% change (${:.2} -> ${:.2})", i, price_change_pct, prev_close, bar.close ); } } } } } // Print summary counts if ohlcv_violations > 5 { println!(" ... and {} more OHLCV violations", ohlcv_violations - 5); } if gaps > 5 { println!(" ... and {} more large gaps", gaps - 5); } if price_spikes > 5 { println!(" ... and {} more price spikes", price_spikes - 5); } println!(); println!(" OHLCV violations: {}", ohlcv_violations); println!( " Zero volumes: {} ({:.1}%)", zero_volumes, (zero_volumes as f64 / data.len() as f64) * 100.0 ); println!( " Large gaps (>2m): {} ({:.1}%)", gaps, (gaps as f64 / data.len() as f64) * 100.0 ); println!(" Price spikes (>20%): {}", price_spikes); println!(); // Overall quality assessment let zero_volume_pct = (zero_volumes as f64 / data.len() as f64) * 100.0; let gap_pct = (gaps as f64 / data.len() as f64) * 100.0; let quality_score = if ohlcv_violations == 0 && zero_volume_pct < 10.0 && gap_pct < 5.0 && price_spikes == 0 { "EXCELLENT" } else if ohlcv_violations < 5 && zero_volume_pct < 20.0 && gap_pct < 10.0 && price_spikes == 0 { "GOOD" } else if ohlcv_violations < 10 { "ACCEPTABLE" } else { "POOR" }; let production_ready = quality_score == "EXCELLENT" || quality_score == "GOOD"; println!("📋 Overall Quality Assessment: {}", quality_score); println!(); // Production readiness println!("🚀 Production Readiness:"); if production_ready { println!(" ✅ Data is suitable for backtesting"); println!(" ✅ No critical quality issues detected"); } else { println!(" ⚠️ Data quality issues detected"); println!(" ⚠️ Review violations before production use"); } println!(); Ok(ValidationResult { symbol: config.symbol.clone(), total_bars: data.len(), ohlcv_violations, zero_volumes, gaps, price_spikes, min_price, max_price, avg_price, quality_score: quality_score.to_string(), production_ready, }) } fn print_validation_result(result: &ValidationResult) { println!("═══════════════════════════════════════════════════════════════\n"); } fn print_overall_summary(results: &[ValidationResult]) { println!("┌────────────────────────────────────────────────────────────┐"); println!("│ OVERALL SUMMARY - Multi-Symbol Validation"); println!("└────────────────────────────────────────────────────────────┘\n"); println!("╔════════════╦══════════╦════════════╦═════════════╦═════════════════════╗"); println!("║ Symbol ║ Bars ║ Quality ║ OHLCV Viol. ║ Production Ready ║"); println!("╠════════════╬══════════╬════════════╬═════════════╬═════════════════════╣"); for result in results { println!( "║ {:<10} ║ {:>8} ║ {:<10} ║ {:>11} ║ {:<19} ║", result.symbol, result.total_bars, result.quality_score, result.ohlcv_violations, if result.production_ready { "✅ YES" } else { "❌ NO" } ); } println!("╚════════════╩══════════╩════════════╩═════════════╩═════════════════════╝\n"); // Detailed statistics println!("📊 Detailed Statistics:\n"); for result in results { println!(" {} ({}):", result.symbol, result.quality_score); println!(" Total bars: {}", result.total_bars); println!(" Price range: ${:.2} - ${:.2}", result.min_price, result.max_price); println!(" Avg price: ${:.2}", result.avg_price); println!(" OHLCV violations: {}", result.ohlcv_violations); println!( " Zero volumes: {} ({:.1}%)", result.zero_volumes, (result.zero_volumes as f64 / result.total_bars as f64) * 100.0 ); println!( " Large gaps: {} ({:.1}%)", result.gaps, (result.gaps as f64 / result.total_bars as f64) * 100.0 ); println!(" Price spikes: {}", result.price_spikes); println!(); } // Final recommendations println!("💡 Final Assessment:\n"); let all_ready = results.iter().all(|r| r.production_ready); let total_symbols = results.len(); let ready_count = results.iter().filter(|r| r.production_ready).count(); if all_ready { println!(" ✅ ALL {} SYMBOLS ARE PRODUCTION READY", total_symbols); println!(" ✅ Data quality meets requirements for backtesting"); println!(" ✅ No critical quality issues detected across all symbols"); println!(" ✅ Recommend proceeding with backtesting strategies"); } else { println!( " ⚠️ {}/{} symbols are production ready", ready_count, total_symbols ); println!(" ⚠️ Review symbols with quality issues:"); for result in results.iter().filter(|r| !r.production_ready) { println!(" • {} - {} quality", result.symbol, result.quality_score); } } println!(); println!("═══════════════════════════════════════════════════════════════"); } fn format_timestamp(ts: &DateTime) -> String { ts.format("%Y-%m-%d %H:%M:%S UTC").to_string() }