//! DBN Data Validation Example //! //! Comprehensive validation of ES.FUT DBN data quality. //! Checks data statistics, quality, and production readiness. use anyhow::Result; use backtesting_service::dbn_repository::DbnMarketDataRepository; use backtesting_service::repositories::MarketDataRepository; use chrono::{DateTime, Utc}; use rust_decimal::prelude::ToPrimitive; use rust_decimal::Decimal; use std::collections::HashMap; #[tokio::main] async fn main() -> Result<()> { println!("ES.FUT DBN Data Validation Report"); println!("==================================\n"); // Load data let mut file_mapping = HashMap::new(); file_mapping.insert( "ES.FUT".to_string(), "test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn".to_string(), ); let repo = DbnMarketDataRepository::new(file_mapping).await?; let symbols = vec!["ES.FUT".to_string()]; let start_time = 1704153600_000_000_000i64; // 2024-01-02 00:00:00 UTC let end_time = 1704240000_000_000_000i64; // 2024-01-03 00:00:00 UTC let data = repo .load_historical_data(&symbols, start_time, end_time) .await?; if data.is_empty() { println!("❌ ERROR: No data loaded!"); return Ok(()); } // Basic statistics println!("📊 Basic Statistics:"); println!(" Total bars: {}", data.len()); println!(" Symbol: {}", data[0].symbol); println!(); // Price statistics let mut prices: Vec = data.iter().map(|b| b.close).collect(); prices.sort(); let min_price = prices.first().expect("INVARIANT: Collection should be non-empty"); let max_price = prices.last().expect("INVARIANT: Collection should be non-empty"); let median_price = prices[prices.len() / 2]; let sum_prices: Decimal = prices.iter().sum(); let avg_price = sum_prices / Decimal::from(prices.len()); println!("💰 Price Analysis:"); println!(" Min close: ${:.2}", min_price); println!(" Max close: ${:.2}", max_price); println!(" Median close: ${:.2}", median_price); println!(" Avg close: ${:.2}", avg_price); println!(" Range: ${:.2}", max_price - min_price); println!(); // Volume statistics let total_volume: Decimal = data.iter().map(|b| b.volume).sum(); let avg_volume = total_volume / Decimal::from(data.len()); let max_volume = data.iter().map(|b| b.volume).max().unwrap_or(Decimal::ZERO); let min_volume = data.iter().map(|b| b.volume).min().unwrap_or(Decimal::ZERO); println!("📈 Volume Analysis:"); println!(" Total volume: {:.0}", total_volume); println!(" Avg volume: {:.0}", avg_volume); println!(" Max volume: {:.0}", max_volume); println!(" Min volume: {:.0}", min_volume); println!(); // Timestamp analysis let first_ts = data.first().expect("INVARIANT: Collection should be non-empty").timestamp; let last_ts = data.last().expect("INVARIANT: Collection should be non-empty").timestamp; let duration_seconds = (last_ts - first_ts).num_seconds(); let duration_hours = duration_seconds / 3600; let duration_minutes = duration_seconds / 60; println!("⏰ Timestamp Analysis:"); println!(" First bar: {}", format_timestamp(&first_ts)); println!(" Last bar: {}", format_timestamp(&last_ts)); println!( " Duration: {} hours ({} minutes)", duration_hours, duration_minutes ); println!(" Expected: ~6.5 hours (trading day)"); println!(); // Data quality checks println!("✅ Data Quality Checks:"); let mut gaps = 0; let mut ohlcv_violations = 0; let mut zero_volumes = 0; let mut price_spikes = 0; for i in 0..data.len() { let bar = &data[i]; // Check OHLCV relationships if !(bar.high >= bar.low && bar.high >= bar.open && bar.high >= bar.close && bar.low <= bar.open && bar.low <= bar.close) { ohlcv_violations += 1; println!( " OHLCV violation at bar {}: O={} H={} L={} C={}", i, bar.open, bar.high, bar.low, bar.close ); } // Check for zero volume if bar.volume == Decimal::ZERO { zero_volumes += 1; } // Check for timestamp gaps (should be ~60 seconds for 1-minute bars) if i > 0 { let gap = (bar.timestamp - data[i - 1].timestamp).num_seconds(); if gap > 120 { // More than 2 minutes gaps += 1; println!( " Large gap at bar {}: {} seconds ({} minutes)", i, gap, gap / 60 ); } } // Check for abnormal price spikes (>10% move) if i > 0 { let prev_close = data[i - 1].close; let price_change_pct = ((bar.close - prev_close) / prev_close).abs() * Decimal::from(100); if price_change_pct > Decimal::from(10) { price_spikes += 1; println!( " Price spike at bar {}: {:.2}% change (${:.2} -> ${:.2})", i, price_change_pct, prev_close, bar.close ); } } } println!(); println!(" OHLCV violations: {}", ohlcv_violations); println!(" Zero volumes: {}", zero_volumes); println!(" Large gaps (>2m): {}", gaps); println!(" Price spikes (>10%): {}", price_spikes); println!(); // Overall quality assessment let quality_score = if ohlcv_violations == 0 && zero_volumes < data.len() / 10 && gaps < data.len() / 20 && price_spikes == 0 { "EXCELLENT" } else if ohlcv_violations < 5 && zero_volumes < data.len() / 5 && gaps < data.len() / 10 { "GOOD" } else if ohlcv_violations < 10 { "ACCEPTABLE" } else { "POOR" }; println!("📋 Overall Quality Assessment: {}", quality_score); println!(); // Production readiness println!("🚀 Production Readiness:"); if quality_score == "EXCELLENT" || quality_score == "GOOD" { println!(" ✅ Data is suitable for backtesting"); println!(" ✅ No critical quality issues detected"); if data.len() >= 350 { println!(" ✅ Sufficient data coverage (~6.5 trading hours)"); } else { println!( " ⚠️ Limited data coverage ({} bars, expected ~390)", data.len() ); } } else { println!(" ⚠️ Data quality issues detected"); println!(" ⚠️ Review violations before production use"); } println!(); println!("💡 Recommendations:"); if zero_volumes > 0 { println!( " • {} bars with zero volume - may indicate low liquidity periods", zero_volumes ); } if gaps > 0 { println!( " • {} timestamp gaps detected - expected during market close/open", gaps ); } if data.len() < 350 { println!(" • Consider acquiring full trading day data (390+ bars)"); } println!(" • Data appears to be from regular trading session"); println!(" • E-mini S&P 500 futures typically have high liquidity"); Ok(()) } fn format_timestamp(ts: &DateTime) -> String { ts.format("%Y-%m-%d %H:%M:%S UTC").to_string() }