foxhunt/services/backtesting_service/examples/validate_multi_symbol.rs

//! Multi-Symbol DBN Data Validation Example
//!
//! Comprehensive validation of GC (Gold), ZN (Treasury), and 6E (Euro FX) DBN data quality.
//! Checks data statistics, quality, and production readiness for all three symbols.

use anyhow::Result;
use backtesting_service::dbn_repository::DbnMarketDataRepository;
use backtesting_service::repositories::MarketDataRepository;
use chrono::{DateTime, Utc};
use rust_decimal::prelude::ToPrimitive;
use rust_decimal::Decimal;
use std::collections::HashMap;

struct SymbolConfig {
    symbol: String,
    file_path: String,
    expected_price_min: f64,
    expected_price_max: f64,
    description: String,
}

struct ValidationResult {
    symbol: String,
    total_bars: usize,
    ohlcv_violations: usize,
    zero_volumes: usize,
    gaps: usize,
    price_spikes: usize,
    min_price: Decimal,
    max_price: Decimal,
    avg_price: Decimal,
    quality_score: String,
    production_ready: bool,
}

#[tokio::main]
async fn main() -> Result<()> {
    println!("═══════════════════════════════════════════════════════════════");
    println!("   Multi-Symbol DBN Data Validation Report");
    println!("   GC (Gold) | ZN (Treasury) | 6E (Euro FX)");
    println!("═══════════════════════════════════════════════════════════════\n");

    // Configure symbols for validation
    // NOTE: Using decompressed files because dbn 0.42.0 doesn't support Zstandard compression
    let symbols = vec![
        SymbolConfig {
            symbol: "GC".to_string(),
            file_path: "test_data/real/databento/GC_continuous_ohlcv-1m_2024-01-02_to_2024-01-31.uncompressed.dbn".to_string(),
            expected_price_min: 2000.0,
            expected_price_max: 2100.0,
            description: "Gold Futures (Continuous)".to_string(),
        },
        SymbolConfig {
            symbol: "ZN.FUT".to_string(),
            file_path: "test_data/real/databento/ZN.FUT_ohlcv-1m_2024-01-02_to_2024-01-31.uncompressed.dbn".to_string(),
            expected_price_min: 110.0,
            expected_price_max: 113.0,
            description: "10-Year Treasury Note Futures".to_string(),
        },
        SymbolConfig {
            symbol: "6E.FUT".to_string(),
            file_path: "test_data/real/databento/6E.FUT_ohlcv-1m_2024-01-02_to_2024-01-31.uncompressed.dbn".to_string(),
            expected_price_min: 1.08,
            expected_price_max: 1.11,
            description: "Euro FX Futures (EUR/USD)".to_string(),
        },
    ];

    let mut results = Vec::new();

    // Validate each symbol
    for config in &symbols {
        match validate_symbol(config).await {
            Ok(result) => {
                print_validation_result(&result);
                results.push(result);
            }
            Err(e) => {
                println!("❌ ERROR validating {}: {}\n", config.symbol, e);
            }
        }
    }

    // Print overall summary
    print_overall_summary(&results);

    Ok(())
}

async fn validate_symbol(config: &SymbolConfig) -> Result<ValidationResult> {
    println!("┌────────────────────────────────────────────────────────────┐");
    println!("│ Symbol: {} - {}", config.symbol, config.description);
    println!("└────────────────────────────────────────────────────────────┘\n");

    // Load data
    let mut file_mapping = HashMap::new();
    file_mapping.insert(config.symbol.clone(), config.file_path.clone());

    let repo = DbnMarketDataRepository::new(file_mapping).await?;

    let symbols = vec![config.symbol.clone()];
    let start_time = 1704153600_000_000_000i64; // 2024-01-02 00:00:00 UTC
    let end_time = 1706745600_000_000_000i64; // 2024-01-31 23:59:59 UTC

    let data = repo
        .load_historical_data(&symbols, start_time, end_time)
        .await?;

    if data.is_empty() {
        println!("❌ ERROR: No data loaded!\n");
        return Ok(ValidationResult {
            symbol: config.symbol.clone(),
            total_bars: 0,
            ohlcv_violations: 0,
            zero_volumes: 0,
            gaps: 0,
            price_spikes: 0,
            min_price: Decimal::ZERO,
            max_price: Decimal::ZERO,
            avg_price: Decimal::ZERO,
            quality_score: "POOR".to_string(),
            production_ready: false,
        });
    }

    // Basic statistics
    println!("📊 Basic Statistics:");
    println!("  Total bars: {}", data.len());
    println!("  Symbol: {}", data[0].symbol);
    println!();

    // Price statistics
    let mut prices: Vec<Decimal> = data.iter().map(|b| b.close).collect();
    prices.sort();

    let min_price = *prices.first().unwrap();
    let max_price = *prices.last().unwrap();
    let median_price = prices[prices.len() / 2];
    let sum_prices: Decimal = prices.iter().sum();
    let avg_price = sum_prices / Decimal::from(prices.len());

    println!("💰 Price Analysis:");
    println!("  Min close:    ${:.2}", min_price);
    println!("  Max close:    ${:.2}", max_price);
    println!("  Median close: ${:.2}", median_price);
    println!("  Avg close:    ${:.2}", avg_price);
    println!("  Range:        ${:.2}", max_price - min_price);
    println!("  Expected:     ${:.2} - ${:.2}", config.expected_price_min, config.expected_price_max);

    // Check if prices are in expected range
    let min_price_f64 = min_price.to_f64().unwrap_or(0.0);
    let max_price_f64 = max_price.to_f64().unwrap_or(0.0);
    let price_range_ok = min_price_f64 >= config.expected_price_min * 0.9
        && max_price_f64 <= config.expected_price_max * 1.1;
    if price_range_ok {
        println!("  ✅ Price range within expected bounds");
    } else {
        println!("  ⚠️  Price range outside expected bounds");
    }
    println!();

    // Volume statistics
    let total_volume: Decimal = data.iter().map(|b| b.volume).sum();
    let avg_volume = total_volume / Decimal::from(data.len());
    let max_volume = data.iter().map(|b| b.volume).max().unwrap_or(Decimal::ZERO);
    let min_volume = data.iter().map(|b| b.volume).min().unwrap_or(Decimal::ZERO);

    println!("📈 Volume Analysis:");
    println!("  Total volume: {:.0}", total_volume);
    println!("  Avg volume:   {:.0}", avg_volume);
    println!("  Max volume:   {:.0}", max_volume);
    println!("  Min volume:   {:.0}", min_volume);
    println!();

    // Timestamp analysis
    let first_ts = data.first().unwrap().timestamp;
    let last_ts = data.last().unwrap().timestamp;
    let duration_seconds = (last_ts - first_ts).num_seconds();
    let duration_days = duration_seconds / 86400;
    let duration_hours = (duration_seconds % 86400) / 3600;

    println!("⏰ Timestamp Analysis:");
    println!("  First bar: {}", format_timestamp(&first_ts));
    println!("  Last bar:  {}", format_timestamp(&last_ts));
    println!(
        "  Duration:  {} days, {} hours",
        duration_days, duration_hours
    );
    println!("  Expected:  ~29-30 days (January 2024)");
    println!();

    // Data quality checks
    println!("✅ Data Quality Checks:");

    let mut gaps = 0;
    let mut ohlcv_violations = 0;
    let mut zero_volumes = 0;
    let mut price_spikes = 0;

    for i in 0..data.len() {
        let bar = &data[i];

        // Check OHLCV relationships
        if !(bar.high >= bar.low
            && bar.high >= bar.open
            && bar.high >= bar.close
            && bar.low <= bar.open
            && bar.low <= bar.close)
        {
            ohlcv_violations += 1;
            if ohlcv_violations <= 5 {
                // Only print first 5
                println!(
                    "    OHLCV violation at bar {}: O={} H={} L={} C={}",
                    i, bar.open, bar.high, bar.low, bar.close
                );
            }
        }

        // Check for zero volume
        if bar.volume == Decimal::ZERO {
            zero_volumes += 1;
        }

        // Check for timestamp gaps (should be ~60 seconds for 1-minute bars)
        if i > 0 {
            let gap = (bar.timestamp - data[i - 1].timestamp).num_seconds();
            if gap > 120 {
                // More than 2 minutes
                gaps += 1;
                if gaps <= 5 {
                    // Only print first 5
                    println!(
                        "    Large gap at bar {}: {} seconds ({} minutes)",
                        i,
                        gap,
                        gap / 60
                    );
                }
            }
        }

        // Check for abnormal price spikes (>20% move for commodities/FX)
        if i > 0 {
            let prev_close = data[i - 1].close;
            if prev_close > Decimal::ZERO {
                let price_change_pct =
                    ((bar.close - prev_close) / prev_close).abs() * Decimal::from(100);
                if price_change_pct > Decimal::from(20) {
                    price_spikes += 1;
                    if price_spikes <= 5 {
                        // Only print first 5
                        println!(
                            "    Price spike at bar {}: {:.2}% change (${:.2} -> ${:.2})",
                            i, price_change_pct, prev_close, bar.close
                        );
                    }
                }
            }
        }
    }

    // Print summary counts
    if ohlcv_violations > 5 {
        println!("    ... and {} more OHLCV violations", ohlcv_violations - 5);
    }
    if gaps > 5 {
        println!("    ... and {} more large gaps", gaps - 5);
    }
    if price_spikes > 5 {
        println!("    ... and {} more price spikes", price_spikes - 5);
    }

    println!();
    println!("  OHLCV violations:    {}", ohlcv_violations);
    println!(
        "  Zero volumes:        {} ({:.1}%)",
        zero_volumes,
        (zero_volumes as f64 / data.len() as f64) * 100.0
    );
    println!(
        "  Large gaps (>2m):    {} ({:.1}%)",
        gaps,
        (gaps as f64 / data.len() as f64) * 100.0
    );
    println!("  Price spikes (>20%): {}", price_spikes);
    println!();

    // Overall quality assessment
    let zero_volume_pct = (zero_volumes as f64 / data.len() as f64) * 100.0;
    let gap_pct = (gaps as f64 / data.len() as f64) * 100.0;

    let quality_score = if ohlcv_violations == 0
        && zero_volume_pct < 10.0
        && gap_pct < 5.0
        && price_spikes == 0
    {
        "EXCELLENT"
    } else if ohlcv_violations < 5 && zero_volume_pct < 20.0 && gap_pct < 10.0 && price_spikes == 0
    {
        "GOOD"
    } else if ohlcv_violations < 10 {
        "ACCEPTABLE"
    } else {
        "POOR"
    };

    let production_ready = quality_score == "EXCELLENT" || quality_score == "GOOD";

    println!("📋 Overall Quality Assessment: {}", quality_score);
    println!();

    // Production readiness
    println!("🚀 Production Readiness:");
    if production_ready {
        println!("  ✅ Data is suitable for backtesting");
        println!("  ✅ No critical quality issues detected");
    } else {
        println!("  ⚠️  Data quality issues detected");
        println!("  ⚠️  Review violations before production use");
    }

    println!();

    Ok(ValidationResult {
        symbol: config.symbol.clone(),
        total_bars: data.len(),
        ohlcv_violations,
        zero_volumes,
        gaps,
        price_spikes,
        min_price,
        max_price,
        avg_price,
        quality_score: quality_score.to_string(),
        production_ready,
    })
}

fn print_validation_result(result: &ValidationResult) {
    println!("═══════════════════════════════════════════════════════════════\n");
}

fn print_overall_summary(results: &[ValidationResult]) {
    println!("┌────────────────────────────────────────────────────────────┐");
    println!("│ OVERALL SUMMARY - Multi-Symbol Validation");
    println!("└────────────────────────────────────────────────────────────┘\n");

    println!("╔════════════╦══════════╦════════════╦═════════════╦═════════════════════╗");
    println!("║ Symbol     ║ Bars     ║ Quality    ║ OHLCV Viol. ║ Production Ready    ║");
    println!("╠════════════╬══════════╬════════════╬═════════════╬═════════════════════╣");

    for result in results {
        println!(
            "║ {:<10} ║ {:>8} ║ {:<10} ║ {:>11} ║ {:<19} ║",
            result.symbol,
            result.total_bars,
            result.quality_score,
            result.ohlcv_violations,
            if result.production_ready {
                "✅ YES"
            } else {
                "❌ NO"
            }
        );
    }

    println!("╚════════════╩══════════╩════════════╩═════════════╩═════════════════════╝\n");

    // Detailed statistics
    println!("📊 Detailed Statistics:\n");

    for result in results {
        println!("  {} ({}):", result.symbol, result.quality_score);
        println!("    Total bars:       {}", result.total_bars);
        println!("    Price range:      ${:.2} - ${:.2}", result.min_price, result.max_price);
        println!("    Avg price:        ${:.2}", result.avg_price);
        println!("    OHLCV violations: {}", result.ohlcv_violations);
        println!(
            "    Zero volumes:     {} ({:.1}%)",
            result.zero_volumes,
            (result.zero_volumes as f64 / result.total_bars as f64) * 100.0
        );
        println!(
            "    Large gaps:       {} ({:.1}%)",
            result.gaps,
            (result.gaps as f64 / result.total_bars as f64) * 100.0
        );
        println!("    Price spikes:     {}", result.price_spikes);
        println!();
    }

    // Final recommendations
    println!("💡 Final Assessment:\n");

    let all_ready = results.iter().all(|r| r.production_ready);
    let total_symbols = results.len();
    let ready_count = results.iter().filter(|r| r.production_ready).count();

    if all_ready {
        println!("  ✅ ALL {} SYMBOLS ARE PRODUCTION READY", total_symbols);
        println!("  ✅ Data quality meets requirements for backtesting");
        println!("  ✅ No critical quality issues detected across all symbols");
        println!("  ✅ Recommend proceeding with backtesting strategies");
    } else {
        println!(
            "  ⚠️  {}/{} symbols are production ready",
            ready_count, total_symbols
        );
        println!("  ⚠️  Review symbols with quality issues:");
        for result in results.iter().filter(|r| !r.production_ready) {
            println!("      • {} - {} quality", result.symbol, result.quality_score);
        }
    }

    println!();
    println!("═══════════════════════════════════════════════════════════════");
}

fn format_timestamp(ts: &DateTime<Utc>) -> String {
    ts.format("%Y-%m-%d %H:%M:%S UTC").to_string()
}