foxhunt/services/backtesting_service/examples/validate_dbn_data.rs

//! DBN Data Validation Example
//!
//! Comprehensive validation of ES.FUT DBN data quality.
//! Checks data statistics, quality, and production readiness.

use anyhow::Result;
use backtesting_service::dbn_repository::DbnMarketDataRepository;
use backtesting_service::repositories::MarketDataRepository;
use chrono::{DateTime, Utc};
use rust_decimal::prelude::ToPrimitive;
use rust_decimal::Decimal;
use std::collections::HashMap;

#[tokio::main]
async fn main() -> Result<()> {
    println!("ES.FUT DBN Data Validation Report");
    println!("==================================\n");

    // Load data
    let mut file_mapping = HashMap::new();
    file_mapping.insert(
        "ES.FUT".to_string(),
        "test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn".to_string(),
    );

    let repo = DbnMarketDataRepository::new(file_mapping).await?;

    let symbols = vec!["ES.FUT".to_string()];
    let start_time = 1704153600_000_000_000i64; // 2024-01-02 00:00:00 UTC
    let end_time = 1704240000_000_000_000i64; // 2024-01-03 00:00:00 UTC

    let data = repo
        .load_historical_data(&symbols, start_time, end_time)
        .await?;

    if data.is_empty() {
        println!("❌ ERROR: No data loaded!");
        return Ok(());
    }

    // Basic statistics
    println!("📊 Basic Statistics:");
    println!("  Total bars: {}", data.len());
    println!("  Symbol: {}", data[0].symbol);
    println!();

    // Price statistics
    let mut prices: Vec<Decimal> = data.iter().map(|b| b.close).collect();
    prices.sort();

    let min_price = prices.first().expect("INVARIANT: Collection should be non-empty");
    let max_price = prices.last().expect("INVARIANT: Collection should be non-empty");
    let median_price = prices[prices.len() / 2];
    let sum_prices: Decimal = prices.iter().sum();
    let avg_price = sum_prices / Decimal::from(prices.len());

    println!("💰 Price Analysis:");
    println!("  Min close:    ${:.2}", min_price);
    println!("  Max close:    ${:.2}", max_price);
    println!("  Median close: ${:.2}", median_price);
    println!("  Avg close:    ${:.2}", avg_price);
    println!("  Range:        ${:.2}", max_price - min_price);
    println!();

    // Volume statistics
    let total_volume: Decimal = data.iter().map(|b| b.volume).sum();
    let avg_volume = total_volume / Decimal::from(data.len());
    let max_volume = data.iter().map(|b| b.volume).max().unwrap_or(Decimal::ZERO);
    let min_volume = data.iter().map(|b| b.volume).min().unwrap_or(Decimal::ZERO);

    println!("📈 Volume Analysis:");
    println!("  Total volume: {:.0}", total_volume);
    println!("  Avg volume:   {:.0}", avg_volume);
    println!("  Max volume:   {:.0}", max_volume);
    println!("  Min volume:   {:.0}", min_volume);
    println!();

    // Timestamp analysis
    let first_ts = data.first().expect("INVARIANT: Collection should be non-empty").timestamp;
    let last_ts = data.last().expect("INVARIANT: Collection should be non-empty").timestamp;
    let duration_seconds = (last_ts - first_ts).num_seconds();
    let duration_hours = duration_seconds / 3600;
    let duration_minutes = duration_seconds / 60;

    println!("⏰ Timestamp Analysis:");
    println!("  First bar: {}", format_timestamp(&first_ts));
    println!("  Last bar:  {}", format_timestamp(&last_ts));
    println!(
        "  Duration:  {} hours ({} minutes)",
        duration_hours, duration_minutes
    );
    println!("  Expected:  ~6.5 hours (trading day)");
    println!();

    // Data quality checks
    println!("✅ Data Quality Checks:");

    let mut gaps = 0;
    let mut ohlcv_violations = 0;
    let mut zero_volumes = 0;
    let mut price_spikes = 0;

    for i in 0..data.len() {
        let bar = &data[i];

        // Check OHLCV relationships
        if !(bar.high >= bar.low
            && bar.high >= bar.open
            && bar.high >= bar.close
            && bar.low <= bar.open
            && bar.low <= bar.close)
        {
            ohlcv_violations += 1;
            println!(
                "    OHLCV violation at bar {}: O={} H={} L={} C={}",
                i, bar.open, bar.high, bar.low, bar.close
            );
        }

        // Check for zero volume
        if bar.volume == Decimal::ZERO {
            zero_volumes += 1;
        }

        // Check for timestamp gaps (should be ~60 seconds for 1-minute bars)
        if i > 0 {
            let gap = (bar.timestamp - data[i - 1].timestamp).num_seconds();
            if gap > 120 {
                // More than 2 minutes
                gaps += 1;
                println!(
                    "    Large gap at bar {}: {} seconds ({} minutes)",
                    i,
                    gap,
                    gap / 60
                );
            }
        }

        // Check for abnormal price spikes (>10% move)
        if i > 0 {
            let prev_close = data[i - 1].close;
            let price_change_pct =
                ((bar.close - prev_close) / prev_close).abs() * Decimal::from(100);
            if price_change_pct > Decimal::from(10) {
                price_spikes += 1;
                println!(
                    "    Price spike at bar {}: {:.2}% change (${:.2} -> ${:.2})",
                    i, price_change_pct, prev_close, bar.close
                );
            }
        }
    }

    println!();
    println!("  OHLCV violations: {}", ohlcv_violations);
    println!("  Zero volumes:     {}", zero_volumes);
    println!("  Large gaps (>2m): {}", gaps);
    println!("  Price spikes (>10%): {}", price_spikes);
    println!();

    // Overall quality assessment
    let quality_score = if ohlcv_violations == 0
        && zero_volumes < data.len() / 10
        && gaps < data.len() / 20
        && price_spikes == 0
    {
        "EXCELLENT"
    } else if ohlcv_violations < 5 && zero_volumes < data.len() / 5 && gaps < data.len() / 10 {
        "GOOD"
    } else if ohlcv_violations < 10 {
        "ACCEPTABLE"
    } else {
        "POOR"
    };

    println!("📋 Overall Quality Assessment: {}", quality_score);
    println!();

    // Production readiness
    println!("🚀 Production Readiness:");
    if quality_score == "EXCELLENT" || quality_score == "GOOD" {
        println!("  ✅ Data is suitable for backtesting");
        println!("  ✅ No critical quality issues detected");
        if data.len() >= 350 {
            println!("  ✅ Sufficient data coverage (~6.5 trading hours)");
        } else {
            println!(
                "  ⚠️  Limited data coverage ({} bars, expected ~390)",
                data.len()
            );
        }
    } else {
        println!("  ⚠️  Data quality issues detected");
        println!("  ⚠️  Review violations before production use");
    }

    println!();
    println!("💡 Recommendations:");
    if zero_volumes > 0 {
        println!(
            "  • {} bars with zero volume - may indicate low liquidity periods",
            zero_volumes
        );
    }
    if gaps > 0 {
        println!(
            "  • {} timestamp gaps detected - expected during market close/open",
            gaps
        );
    }
    if data.len() < 350 {
        println!("  • Consider acquiring full trading day data (390+ bars)");
    }
    println!("  • Data appears to be from regular trading session");
    println!("  • E-mini S&P 500 futures typically have high liquidity");

    Ok(())
}

fn format_timestamp(ts: &DateTime<Utc>) -> String {
    ts.format("%Y-%m-%d %H:%M:%S UTC").to_string()
}