foxhunt/services/backtesting_service/tests/dbn_integration_tests.rs

#![allow(clippy::inconsistent_digit_grouping, dead_code, unused_variables, unexpected_cfgs)]
//! DBN Integration Tests
//!
//! Tests for DBN file loading and integration with backtesting service.
//! Uses real market data from test_data/real/databento/.

use anyhow::Result;
use num_traits::ToPrimitive;

mod mock_repositories;

use backtesting_service::dbn_data_source::DbnDataSource;
use backtesting_service::dbn_repository::DbnMarketDataRepository;
use backtesting_service::repositories::MarketDataRepository;
use std::collections::HashMap;

#[tokio::test]
async fn test_load_real_dbn_file() -> Result<()> {
    // Create file mapping using helper to get correct path
    let mut file_mapping = HashMap::new();
    file_mapping.insert(
        "ES.FUT".to_string(),
        mock_repositories::get_dbn_test_file_path(),
    );

    // Create data source
    let data_source = DbnDataSource::new(file_mapping).await?;

    // Load bars
    let bars = data_source.load_ohlcv_bars("ES.FUT").await?;

    // Validate bar count (ES.FUT 2024-01-02 has ~1674 one-minute bars from real DBN data)
    assert!(!bars.is_empty(), "Should load bars from DBN file");
    assert!(
        bars.len() > 1500 && bars.len() < 1800,
        "Expected ~1674 bars (1500-1800 range) from real DBN data, got {}",
        bars.len()
    );
    println!("✅ Loaded {} bars from real DBN file", bars.len());

    // Validate first bar structure
    let first_bar = &bars[0];
    assert_eq!(first_bar.symbol, "ES.FUT", "Symbol should be ES.FUT");
    assert!(
        first_bar.open > rust_decimal::Decimal::ZERO,
        "Open price should be positive"
    );
    assert!(first_bar.high >= first_bar.open, "High should be >= open");
    assert!(first_bar.low <= first_bar.open, "Low should be <= open");
    assert!(
        first_bar.close > rust_decimal::Decimal::ZERO,
        "Close price should be positive"
    );
    assert!(
        first_bar.volume >= rust_decimal::Decimal::ZERO,
        "Volume should be non-negative"
    );

    // Validate price ranges (ES.FUT typical range for 2024)
    let open_f64 = first_bar.open.to_f64().unwrap_or(0.0);
    assert!(
        open_f64 > 3500.0 && open_f64 < 5500.0,
        "ES.FUT price should be in realistic range (3500-5500), got {}",
        open_f64
    );

    // Validate OHLCV relationships
    assert!(first_bar.high >= first_bar.low, "High should be >= low");
    assert!(first_bar.high >= first_bar.open, "High should be >= open");
    assert!(first_bar.high >= first_bar.close, "High should be >= close");
    assert!(first_bar.low <= first_bar.open, "Low should be <= open");
    assert!(first_bar.low <= first_bar.close, "Low should be <= close");

    // Check sorting
    for i in 1..bars.len() {
        assert!(
            bars[i].timestamp >= bars[i - 1].timestamp,
            "Bars should be sorted by timestamp"
        );
    }

    println!("✅ Data quality validation passed");
    println!(
        "   First bar: {} @ {} (open={}, high={}, low={}, close={}, volume={})",
        first_bar.symbol,
        first_bar.timestamp,
        first_bar.open,
        first_bar.high,
        first_bar.low,
        first_bar.close,
        first_bar.volume
    );

    Ok(())
}

#[tokio::test]
async fn test_dbn_repository_integration() -> Result<()> {
    // Create repository using helper path
    let mut file_mapping = HashMap::new();
    file_mapping.insert(
        "ES.FUT".to_string(),
        mock_repositories::get_dbn_test_file_path(),
    );

    let repo = DbnMarketDataRepository::new(file_mapping).await?;

    // Load data via repository interface
    let symbols = vec!["ES.FUT".to_string()];

    // 2024-01-02 00:00:00 to 2024-01-03 00:00:00 (full day)
    let start_time = 1704153600_000_000_000i64;
    let end_time = 1704240000_000_000_000i64;

    let data = repo
        .load_historical_data(&symbols, start_time, end_time)
        .await?;

    assert!(!data.is_empty(), "Repository should load data");
    println!("✅ Repository loaded {} bars", data.len());

    // Validate time range
    for bar in &data {
        let ts_nanos = bar.timestamp.timestamp_nanos_opt().unwrap_or(0);
        assert!(
            ts_nanos >= start_time && ts_nanos <= end_time,
            "Bar timestamp should be within requested range"
        );
    }

    println!("✅ Time range filtering validated");

    Ok(())
}

// test_dbn_data_availability gated: check_data_availability method was removed from DbnMarketDataRepository
#[cfg(feature = "__backtesting_integration")]
#[tokio::test]
async fn test_dbn_data_availability() -> Result<()> {
    let mut file_mapping = HashMap::new();
    file_mapping.insert(
        "ES.FUT".to_string(),
        mock_repositories::get_dbn_test_file_path().to_string(),
    );

    let repo = DbnMarketDataRepository::new(file_mapping).await?;

    // Check availability for both existing and non-existing symbols
    let symbols = vec!["ES.FUT".to_string(), "NONEXISTENT.SYM".to_string()];
    let start_time = 1704153600_000_000_000i64;
    let end_time = 1704240000_000_000_000i64;

    let availability = repo
        .check_data_availability(&symbols, start_time, end_time)
        .await?;

    // ES.FUT should be available (file exists)
    assert_eq!(
        availability.get("ES.FUT"),
        Some(&true),
        "ES.FUT should be available (file exists)"
    );

    // NONEXISTENT should not be available
    assert_eq!(
        availability.get("NONEXISTENT.SYM"),
        Some(&false),
        "NONEXISTENT.SYM should not be available"
    );

    println!("✅ Data availability check passed");
    println!("   ES.FUT: available");
    println!("   NONEXISTENT.SYM: not available");

    Ok(())
}

#[tokio::test]
async fn test_timestamp_format() -> Result<()> {
    let repo = mock_repositories::create_dbn_repository().await?;

    let symbols = vec!["ES.FUT".to_string()];
    let start_time = 1704153600_000_000_000i64; // 2024-01-02 00:00:00 UTC
    let end_time = 1704240000_000_000_000i64; // 2024-01-03 00:00:00 UTC

    let data = repo
        .load_historical_data(&symbols, start_time, end_time)
        .await?;

    assert!(
        !data.is_empty(),
        "Should have data for timestamp validation"
    );

    // Validate timestamps are in correct range (nanoseconds, Unix epoch)
    for (i, bar) in data.iter().enumerate() {
        let ts_nanos = bar.timestamp.timestamp_nanos_opt().unwrap_or(0);

        assert!(
            ts_nanos > 1700000000_000_000_000i64,
            "Bar {}: Timestamp should be in nanoseconds (after 2023), got {}",
            i,
            ts_nanos
        );
        assert!(
            ts_nanos < 1750000000_000_000_000i64,
            "Bar {}: Timestamp should be reasonable (before 2026), got {}",
            i,
            ts_nanos
        );
        assert!(
            ts_nanos >= start_time && ts_nanos <= end_time,
            "Bar {}: Timestamp should be within requested range [{}, {}], got {}",
            i,
            start_time,
            end_time,
            ts_nanos
        );
    }

    // Validate timestamps are sorted
    for i in 1..data.len() {
        let prev_ts = data[i - 1].timestamp.timestamp_nanos_opt().unwrap_or(0);
        let curr_ts = data[i].timestamp.timestamp_nanos_opt().unwrap_or(0);
        assert!(
            curr_ts >= prev_ts,
            "Bar {}: Timestamps should be sorted (prev: {}, curr: {})",
            i,
            prev_ts,
            curr_ts
        );
    }

    println!("✅ All {} timestamps valid and sorted", data.len());
    println!("   First timestamp: {}", data[0].timestamp);
    println!("   Last timestamp: {}", data[data.len() - 1].timestamp);

    Ok(())
}

#[tokio::test]
async fn test_dbn_performance() -> Result<()> {
    use std::time::Instant;

    let mut file_mapping = HashMap::new();
    file_mapping.insert(
        "ES.FUT".to_string(),
        mock_repositories::get_dbn_test_file_path().to_string(),
    );

    let data_source = DbnDataSource::new(file_mapping).await?;

    // Warm-up run (file system cache)
    let _ = data_source.load_ohlcv_bars("ES.FUT").await?;

    // Timed run
    let start = Instant::now();
    let bars = data_source.load_ohlcv_bars("ES.FUT").await?;
    let duration = start.elapsed();

    println!("✅ Loaded {} bars in {:?}", bars.len(), duration);

    // Performance validation: should be <100ms for ~400 bars (very conservative)
    if bars.len() > 100 {
        assert!(
            duration.as_millis() < 100,
            "Loading should be fast (<100ms for {} bars, got {}ms)",
            bars.len(),
            duration.as_millis()
        );

        // Calculate bars per second
        let bars_per_sec = (bars.len() as f64 / duration.as_secs_f64()) as u64;
        println!(
            "✅ Performance target met: {}ms for {} bars",
            duration.as_millis(),
            bars.len()
        );
        println!("   Throughput: {} bars/sec", bars_per_sec);
    }

    Ok(())
}

#[tokio::test]
async fn test_dbn_multi_symbol_loading() -> Result<()> {
    let mut file_mapping = HashMap::new();
    file_mapping.insert(
        "ES.FUT".to_string(),
        mock_repositories::get_dbn_test_file_path().to_string(),
    );

    let data_source = DbnDataSource::new(file_mapping).await?;

    // Load multiple symbols (only ES.FUT exists in this test)
    let symbols = vec!["ES.FUT".to_string()];
    let bars = data_source.load_multi_symbol_bars(&symbols).await?;

    assert!(!bars.is_empty(), "Should load multi-symbol data");
    println!("✅ Multi-symbol loading: {} bars", bars.len());

    // All bars should be from ES.FUT
    for bar in &bars {
        assert_eq!(bar.symbol, "ES.FUT");
    }

    Ok(())
}

#[tokio::test]
async fn test_ohlcv_data_quality() -> Result<()> {
    let repo = mock_repositories::create_dbn_repository().await?;

    let symbols = vec!["ES.FUT".to_string()];
    let start_time = 1704153600_000_000_000i64; // 2024-01-02 00:00:00 UTC
    let end_time = 1704240000_000_000_000i64; // 2024-01-03 00:00:00 UTC

    let data = repo
        .load_historical_data(&symbols, start_time, end_time)
        .await?;

    assert!(!data.is_empty(), "Should have data for quality validation");

    let mut quality_issues = 0;

    for (i, bar) in data.iter().enumerate() {
        // Check OHLCV relationships
        let high_gte_low = bar.high >= bar.low;
        let high_gte_open = bar.high >= bar.open;
        let high_gte_close = bar.high >= bar.close;
        let low_lte_open = bar.low <= bar.open;
        let low_lte_close = bar.low <= bar.close;

        let valid =
            high_gte_low && high_gte_open && high_gte_close && low_lte_open && low_lte_close;

        if !valid {
            quality_issues += 1;
            if quality_issues <= 5 {
                eprintln!(
                    "Quality issue at bar {}: open={}, high={}, low={}, close={}",
                    i, bar.open, bar.high, bar.low, bar.close
                );
                eprintln!("  high >= low: {}", high_gte_low);
                eprintln!("  high >= open: {}", high_gte_open);
                eprintln!("  high >= close: {}", high_gte_close);
                eprintln!("  low <= open: {}", low_lte_open);
                eprintln!("  low <= close: {}", low_lte_close);
            }
        }

        // Check positive values
        assert!(
            bar.open > rust_decimal::Decimal::ZERO,
            "Bar {}: Open should be positive, got {}",
            i,
            bar.open
        );
        assert!(
            bar.high > rust_decimal::Decimal::ZERO,
            "Bar {}: High should be positive, got {}",
            i,
            bar.high
        );
        assert!(
            bar.low > rust_decimal::Decimal::ZERO,
            "Bar {}: Low should be positive, got {}",
            i,
            bar.low
        );
        assert!(
            bar.close > rust_decimal::Decimal::ZERO,
            "Bar {}: Close should be positive, got {}",
            i,
            bar.close
        );
        assert!(
            bar.volume >= rust_decimal::Decimal::ZERO,
            "Bar {}: Volume should be non-negative, got {}",
            i,
            bar.volume
        );

        // Check realistic price ranges for ES.FUT (3500-5500 for 2024)
        let close_f64 = bar.close.to_f64().unwrap_or(0.0);
        assert!(
            close_f64 > 3000.0 && close_f64 < 6000.0,
            "Bar {}: ES.FUT price {} outside realistic range (3000-6000)",
            i,
            close_f64
        );
    }

    assert_eq!(
        quality_issues, 0,
        "Found {} OHLCV data quality issues",
        quality_issues
    );

    println!("✅ All {} bars passed OHLCV quality checks", data.len());
    println!("   Zero quality issues detected");

    Ok(())
}

#[tokio::test]
async fn test_dbn_data_quality_validation() -> Result<()> {
    let mut file_mapping = HashMap::new();
    file_mapping.insert(
        "ES.FUT".to_string(),
        mock_repositories::get_dbn_test_file_path().to_string(),
    );

    let data_source = DbnDataSource::new(file_mapping).await?;
    let bars = data_source.load_ohlcv_bars("ES.FUT").await?;

    assert!(!bars.is_empty(), "Should have data");

    // Validate OHLCV relationships
    for (i, bar) in bars.iter().enumerate() {
        // High >= Low
        assert!(
            bar.high >= bar.low,
            "Bar {}: high ({}) should be >= low ({})",
            i,
            bar.high,
            bar.low
        );

        // Open/Close within High/Low range
        assert!(
            bar.open >= bar.low && bar.open <= bar.high,
            "Bar {}: open should be within [low, high]",
            i
        );
        assert!(
            bar.close >= bar.low && bar.close <= bar.high,
            "Bar {}: close should be within [low, high]",
            i
        );

        // Positive values
        assert!(
            bar.open > rust_decimal::Decimal::ZERO,
            "Bar {}: open should be positive",
            i
        );
        assert!(
            bar.volume >= rust_decimal::Decimal::ZERO,
            "Bar {}: volume should be non-negative",
            i
        );

        // Realistic ES.FUT prices (roughly 4000-5000 range for 2024)
        let close_f64 = bar.close.to_f64().unwrap_or(0.0);
        assert!(
            close_f64 > 3000.0 && close_f64 < 6000.0,
            "Bar {}: ES.FUT price {} seems unrealistic",
            i,
            close_f64
        );
    }

    println!("✅ Data quality validation passed for {} bars", bars.len());

    Ok(())
}

#[tokio::test]
async fn test_helper_create_dbn_repository() -> Result<()> {
    // Test the helper function from mock_repositories
    let repo = mock_repositories::create_dbn_repository().await?;

    // Load some data
    let symbols = vec!["ES.FUT".to_string()];
    let start_time = 1704153600_000_000_000i64;
    let end_time = 1704240000_000_000_000i64;

    let data = repo
        .load_historical_data(&symbols, start_time, end_time)
        .await?;

    assert!(
        !data.is_empty(),
        "Helper function should create working repository"
    );
    println!("✅ Helper function test: loaded {} bars", data.len());

    Ok(())
}