foxhunt/services/backtesting_service/tests/fixtures/mod.rs

//! Test Fixtures for Real DBN Market Data
//!
//! This module provides efficient, cached access to real DBN market data for tests.
//! All data is loaded once and cached in static memory, shared across test threads.
//!
//! # Performance
//!
//! - **First access**: ~5-10ms (DBN file I/O)
//! - **Subsequent access**: ~0.1μs (static memory read)
//! - **Thread-safe**: Uses `once_cell::sync::Lazy` for safe concurrent access
//!
//! # Supported Symbols
//!
//! - **ES.FUT**: E-mini S&P 500 futures (2024-01-02, ~390 bars)
//! - **NQ.FUT**: E-mini NASDAQ-100 futures (2024-01-02, ~390 bars)
//! - **CL.FUT**: WTI Crude Oil futures (2024-01-02, ~1440 bars)
//!
//! # Usage Examples
//!
//! ```rust
//! use fixtures::{get_es_fut_bars, get_nq_fut_bars, get_regime_sample};
//!
//! #[tokio::test]
//! async fn test_strategy_with_real_data() {
//!     // Fast: data cached after first call
//!     let bars = get_es_fut_bars().await.unwrap();
//!     assert!(!bars.is_empty());
//! }
//!
//! #[tokio::test]
//! async fn test_trending_regime() {
//!     let bars = get_regime_sample(RegimeType::Trending).await.unwrap();
//!     // bars filtered for trending market conditions
//! }
//! ```

use anyhow::Result;
use backtesting_service::dbn_data_source::DbnDataSource;
use num_traits::ToPrimitive;
use backtesting_service::strategy_engine::MarketData;
use chrono::{DateTime, Utc};
use once_cell::sync::Lazy;
use std::collections::HashMap;
use std::sync::Arc;
use tokio::sync::RwLock;

// ============================================================================
// Static Cached Data (Singleton Pattern)
// ============================================================================

/// Cached ES.FUT bars (loaded once per test run)
static ES_FUT_CACHE: Lazy<Arc<RwLock<Option<Vec<MarketData>>>>> =
    Lazy::new(|| Arc::new(RwLock::new(None)));

/// Cached NQ.FUT bars (loaded once per test run)
static NQ_FUT_CACHE: Lazy<Arc<RwLock<Option<Vec<MarketData>>>>> =
    Lazy::new(|| Arc::new(RwLock::new(None)));

/// Cached CL.FUT bars (loaded once per test run)
static CL_FUT_CACHE: Lazy<Arc<RwLock<Option<Vec<MarketData>>>>> =
    Lazy::new(|| Arc::new(RwLock::new(None)));

// ============================================================================
// Path Resolution
// ============================================================================

/// Get absolute path to project root
fn get_project_root() -> std::path::PathBuf {
    let mut current = std::env::current_dir().expect("INVARIANT: Current directory should be accessible");

    // Navigate up until we find the workspace root
    while !current.join("Cargo.toml").exists() || !current.join("test_data").exists() {
        if !current.pop() {
            panic!("Could not find project root");
        }
    }

    current
}

/// Get absolute path to a DBN test file
fn get_dbn_file_path(filename: &str) -> String {
    get_project_root()
        .join("test_data/real/databento")
        .join(filename)
        .to_string_lossy()
        .to_string()
}

// ============================================================================
// Core Data Loading Functions
// ============================================================================

/// Get cached ES.FUT bars (E-mini S&P 500 futures)
///
/// # Data Details
///
/// - **Symbol**: ES.FUT
/// - **Date**: 2024-01-02
/// - **Timeframe**: 1-minute OHLCV
/// - **Bar count**: ~390 bars (typical trading day)
/// - **Price range**: ~4700-4750 (typical for 2024)
///
/// # Performance
///
/// - First call: ~5-10ms (DBN file load)
/// - Subsequent: ~0.1μs (cache hit)
///
/// # Returns
///
/// `Vec<MarketData>` with chronologically sorted bars
///
/// # Example
///
/// ```rust
/// let bars = get_es_fut_bars().await?;
/// assert_eq!(bars[0].symbol, "ES.FUT");
/// assert!(bars.len() > 350);
/// ```
pub async fn get_es_fut_bars() -> Result<Vec<MarketData>> {
    // Check cache first
    {
        let cache = ES_FUT_CACHE.read().await;
        if let Some(ref bars) = *cache {
            return Ok(bars.clone());
        }
    }

    // Cache miss: load from DBN file
    let mut file_mapping = HashMap::new();
    file_mapping.insert(
        "ES.FUT".to_string(),
        get_dbn_file_path("ES.FUT_ohlcv-1m_2024-01-02.dbn"),
    );

    let data_source = DbnDataSource::new(file_mapping).await?;
    let bars = data_source.load_ohlcv_bars("ES.FUT").await?;

    // Cache for future calls
    {
        let mut cache = ES_FUT_CACHE.write().await;
        *cache = Some(bars.clone());
    }

    Ok(bars)
}

/// Get cached NQ.FUT bars (E-mini NASDAQ-100 futures)
///
/// # Data Details
///
/// - **Symbol**: NQ.FUT
/// - **Date**: 2024-01-02
/// - **Timeframe**: 1-minute OHLCV
/// - **Bar count**: ~390 bars
/// - **Price range**: ~16500-16700 (typical for 2024)
///
/// # Performance
///
/// - First call: ~5-10ms
/// - Subsequent: ~0.1μs
///
/// # Example
///
/// ```rust
/// let bars = get_nq_fut_bars().await?;
/// assert_eq!(bars[0].symbol, "NQ.FUT");
/// ```
pub async fn get_nq_fut_bars() -> Result<Vec<MarketData>> {
    // Check cache first
    {
        let cache = NQ_FUT_CACHE.read().await;
        if let Some(ref bars) = *cache {
            return Ok(bars.clone());
        }
    }

    // Cache miss: load from DBN file
    let mut file_mapping = HashMap::new();
    file_mapping.insert(
        "NQ.FUT".to_string(),
        get_dbn_file_path("NQ.FUT_ohlcv-1m_2024-01-02.dbn"),
    );

    let data_source = DbnDataSource::new(file_mapping).await?;
    let bars = data_source.load_ohlcv_bars("NQ.FUT").await?;

    // Cache for future calls
    {
        let mut cache = NQ_FUT_CACHE.write().await;
        *cache = Some(bars.clone());
    }

    Ok(bars)
}

/// Get cached CL.FUT bars (WTI Crude Oil futures)
///
/// # Data Details
///
/// - **Symbol**: CL.FUT
/// - **Date**: 2024-01-02
/// - **Timeframe**: 1-minute OHLCV
/// - **Bar count**: ~1440 bars (24-hour trading)
/// - **Price range**: ~71-73 USD/barrel (typical for 2024)
///
/// # Performance
///
/// - First call: ~10-15ms (larger file)
/// - Subsequent: ~0.2μs
///
/// # Example
///
/// ```rust
/// let bars = get_cl_fut_bars().await?;
/// assert!(bars.len() > 1400);
/// ```
pub async fn get_cl_fut_bars() -> Result<Vec<MarketData>> {
    // Check cache first
    {
        let cache = CL_FUT_CACHE.read().await;
        if let Some(ref bars) = *cache {
            return Ok(bars.clone());
        }
    }

    // Cache miss: load from DBN file
    let mut file_mapping = HashMap::new();
    file_mapping.insert(
        "CL.FUT".to_string(),
        get_dbn_file_path("CL.FUT_ohlcv-1m_2024-01-02.dbn"),
    );

    let data_source = DbnDataSource::new(file_mapping).await?;
    let bars = data_source.load_ohlcv_bars("CL.FUT").await?;

    // Cache for future calls
    {
        let mut cache = CL_FUT_CACHE.write().await;
        *cache = Some(bars.clone());
    }

    Ok(bars)
}

// ============================================================================
// Filtered Data Access
// ============================================================================

/// Get bars for a specific date and symbol
///
/// # Arguments
///
/// * `symbol` - Symbol name (e.g., "ES.FUT", "NQ.FUT", "CL.FUT")
/// * `date` - UTC date (only date component used, time ignored)
///
/// # Returns
///
/// Bars matching the specified date, or empty vec if no data available
///
/// # Example
///
/// ```rust
/// use chrono::NaiveDate;
/// let date = NaiveDate::from_ymd_opt(2024, 1, 2).unwrap().and_hms_opt(0, 0, 0).expect("INVARIANT: Valid time parameters");
/// let bars = get_bars_for_date("ES.FUT", date).await?;
/// ```
pub async fn get_bars_for_date(symbol: &str, date: DateTime<Utc>) -> Result<Vec<MarketData>> {
    let all_bars = match symbol {
        "ES.FUT" => get_es_fut_bars().await?,
        "NQ.FUT" => get_nq_fut_bars().await?,
        "CL.FUT" => get_cl_fut_bars().await?,
        _ => return Ok(Vec::new()),
    };

    // Filter by date (compare only the date component)
    let target_date = date.date_naive();
    let filtered: Vec<MarketData> = all_bars
        .into_iter()
        .filter(|bar| bar.timestamp.date_naive() == target_date)
        .collect();

    Ok(filtered)
}

/// Market regime types for filtered data access
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RegimeType {
    /// Trending market (strong directional movement)
    Trending,
    /// Ranging/sideways market (bounded oscillation)
    Ranging,
    /// High volatility period
    Volatile,
    /// Low volatility period
    Stable,
}

/// Get a sample of bars matching a specific market regime
///
/// This function analyzes cached data and returns a subset that exhibits
/// the requested market characteristics.
///
/// # Arguments
///
/// * `regime_type` - The type of market regime to sample
///
/// # Returns
///
/// Vec of 50-100 bars exhibiting the requested regime characteristics
///
/// # Regime Detection Logic
///
/// - **Trending**: Price movement > 1.5% over 60 bars
/// - **Ranging**: Price range < 0.8% over 60 bars
/// - **Volatile**: Price standard deviation > 0.5%
/// - **Stable**: Price standard deviation < 0.3%
///
/// # Example
///
/// ```rust
/// let trending_bars = get_regime_sample(RegimeType::Trending).await?;
/// // Use these bars to test trend-following strategies
/// ```
pub async fn get_regime_sample(regime_type: RegimeType) -> Result<Vec<MarketData>> {
    // Use ES.FUT as default data source (most liquid)
    let all_bars = get_es_fut_bars().await?;

    if all_bars.len() < 100 {
        return Ok(all_bars);
    }

    // Analyze bars to find regime match
    let window_size = 60; // 1 hour window for regime detection
    let mut best_match_idx = 0;
    let mut best_score = 0.0;

    for i in 0..(all_bars.len() - window_size) {
        let window = &all_bars[i..i + window_size];
        let score = calculate_regime_score(window, regime_type);

        if score > best_score {
            best_score = score;
            best_match_idx = i;
        }
    }

    // Return 100 bars starting from best match
    let end_idx = (best_match_idx + 100).min(all_bars.len());
    Ok(all_bars[best_match_idx..end_idx].to_vec())
}

/// Calculate how well a window matches a regime type
fn calculate_regime_score(window: &[MarketData], regime_type: RegimeType) -> f64 {
    if window.is_empty() {
        return 0.0;
    }

    let prices: Vec<f64> = window
        .iter()
        .map(|bar| bar.close.to_f64().unwrap_or(0.0))
        .collect();

    let first_price = prices[0];
    let last_price = *prices.last().expect("INVARIANT: Collection should be non-empty");
    let mean = prices.iter().sum::<f64>() / prices.len() as f64;

    // Calculate standard deviation
    let variance: f64 =
        prices.iter().map(|p| (p - mean).powi(2)).sum::<f64>() / prices.len() as f64;
    let std_dev = variance.sqrt();
    let std_dev_pct = (std_dev / mean) * 100.0;

    // Calculate price change percentage
    let price_change_pct = ((last_price - first_price) / first_price).abs() * 100.0;

    // Calculate price range
    let max_price = prices.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
    let min_price = prices.iter().cloned().fold(f64::INFINITY, f64::min);
    let range_pct = ((max_price - min_price) / mean) * 100.0;

    match regime_type {
        RegimeType::Trending => {
            // High price change, moderate to high volatility
            price_change_pct * 2.0 + std_dev_pct
        },
        RegimeType::Ranging => {
            // Low price change, low to moderate range
            let range_score = if range_pct < 1.0 {
                100.0 - range_pct * 20.0
            } else {
                0.0
            };
            let change_score = if price_change_pct < 0.5 { 50.0 } else { 0.0 };
            range_score + change_score
        },
        RegimeType::Volatile => {
            // High standard deviation
            std_dev_pct * 10.0
        },
        RegimeType::Stable => {
            // Low standard deviation
            if std_dev_pct < 0.5 {
                100.0 - std_dev_pct * 50.0
            } else {
                0.0
            }
        },
    }
}

// ============================================================================
// Multi-Symbol Access
// ============================================================================

/// Get bars for multiple symbols
///
/// Loads data for multiple symbols in parallel for efficiency.
///
/// # Arguments
///
/// * `symbols` - Slice of symbol names
///
/// # Returns
///
/// HashMap mapping symbol name to bars
///
/// # Example
///
/// ```rust
/// let symbols = vec!["ES.FUT", "NQ.FUT"];
/// let data = get_multi_symbol_bars(&symbols).await?;
/// assert!(data.contains_key("ES.FUT"));
/// ```
pub async fn get_multi_symbol_bars(symbols: &[&str]) -> Result<HashMap<String, Vec<MarketData>>> {
    let mut result = HashMap::new();

    // Load all symbols in parallel
    let mut handles = vec![];

    for &symbol in symbols {
        let symbol = symbol.to_string();
        handles.push(tokio::spawn(async move {
            let bars = match symbol.as_str() {
                "ES.FUT" => get_es_fut_bars().await,
                "NQ.FUT" => get_nq_fut_bars().await,
                "CL.FUT" => get_cl_fut_bars().await,
                _ => Ok(Vec::new()),
            };
            (symbol, bars)
        }));
    }

    // Collect results
    for handle in handles {
        let (symbol, bars) = handle.await?;
        if let Ok(bars) = bars {
            result.insert(symbol, bars);
        }
    }

    Ok(result)
}

// ============================================================================
// Unit Tests
// ============================================================================

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn test_es_fut_cache() -> Result<()> {
        use std::time::Instant;

        // First call (cold cache)
        let start = Instant::now();
        let bars1 = get_es_fut_bars().await?;
        let cold_duration = start.elapsed();

        assert!(!bars1.is_empty(), "Should load ES.FUT bars");
        assert!(bars1.len() > 350 && bars1.len() < 450, "Expected ~390 bars");

        // Second call (warm cache)
        let start = Instant::now();
        let bars2 = get_es_fut_bars().await?;
        let warm_duration = start.elapsed();

        assert_eq!(bars1.len(), bars2.len(), "Cache should return same data");

        println!("Cache performance:");
        println!("  Cold: {:?}", cold_duration);
        println!("  Warm: {:?}", warm_duration);

        // Warm should be significantly faster
        assert!(
            warm_duration < cold_duration / 10,
            "Cached access should be >10x faster"
        );

        Ok(())
    }

    #[tokio::test]
    async fn test_nq_fut_cache() -> Result<()> {
        let bars = get_nq_fut_bars().await?;
        assert!(!bars.is_empty(), "Should load NQ.FUT bars");
        assert_eq!(bars[0].symbol, "NQ.FUT");
        Ok(())
    }

    #[tokio::test]
    async fn test_cl_fut_cache() -> Result<()> {
        let bars = get_cl_fut_bars().await?;
        assert!(!bars.is_empty(), "Should load CL.FUT bars");
        assert!(bars.len() > 1400, "CL.FUT has 24-hour trading");
        assert_eq!(bars[0].symbol, "CL.FUT");
        Ok(())
    }

    #[tokio::test]
    async fn test_bars_for_date() -> Result<()> {
        use chrono::NaiveDate;

        let date = NaiveDate::from_ymd_opt(2024, 1, 2)
            .unwrap()
            .and_hms_opt(0, 0, 0)
            .unwrap()
            .and_utc();

        let bars = get_bars_for_date("ES.FUT", date).await?;
        assert!(!bars.is_empty(), "Should find bars for 2024-01-02");

        // All bars should be from requested date
        for bar in &bars {
            assert_eq!(bar.timestamp.date_naive(), date.date_naive());
        }

        Ok(())
    }

    #[tokio::test]
    async fn test_regime_samples() -> Result<()> {
        // Test trending regime
        let trending = get_regime_sample(RegimeType::Trending).await?;
        assert!(!trending.is_empty(), "Should find trending sample");

        // Test ranging regime
        let ranging = get_regime_sample(RegimeType::Ranging).await?;
        assert!(!ranging.is_empty(), "Should find ranging sample");

        // Test volatile regime
        let volatile = get_regime_sample(RegimeType::Volatile).await?;
        assert!(!volatile.is_empty(), "Should find volatile sample");

        // Test stable regime
        let stable = get_regime_sample(RegimeType::Stable).await?;
        assert!(!stable.is_empty(), "Should find stable sample");

        Ok(())
    }

    #[tokio::test]
    async fn test_multi_symbol_bars() -> Result<()> {
        let symbols = vec!["ES.FUT", "NQ.FUT", "CL.FUT"];
        let data = get_multi_symbol_bars(&symbols).await?;

        assert_eq!(data.len(), 3, "Should load all 3 symbols");
        assert!(data.contains_key("ES.FUT"));
        assert!(data.contains_key("NQ.FUT"));
        assert!(data.contains_key("CL.FUT"));

        // Verify data quality
        for (symbol, bars) in &data {
            assert!(!bars.is_empty(), "Symbol {} should have bars", symbol);
            assert_eq!(bars[0].symbol, *symbol);
        }

        Ok(())
    }

    #[tokio::test]
    async fn test_cache_thread_safety() -> Result<()> {
        // Spawn multiple concurrent reads
        let mut handles = vec![];

        for _ in 0..10 {
            handles.push(tokio::spawn(async { get_es_fut_bars().await }));
        }

        // All should succeed
        for handle in handles {
            let bars = handle.await??;
            assert!(!bars.is_empty());
        }

        Ok(())
    }
}