foxhunt/services/integration_tests/tests/common/dbn_helpers.rs

//! DBN Market Data Helpers for E2E Tests
//!
//! This module provides utilities to load real market data from DBN files
//! and use it in E2E trading service tests.

use anyhow::{Context, Result};
use backtesting_service::dbn_data_source::DbnDataSource;
use backtesting_service::strategy_engine::MarketData as BacktestMarketData;
use chrono::{DateTime, Utc};
use rust_decimal::Decimal;
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::Arc;
use tokio::sync::RwLock;

/// DBN test data manager
///
/// Provides access to real DBN market data for E2E tests
pub struct DbnTestDataManager {
    data_source: Arc<DbnDataSource>,
    /// Cached market data by symbol
    cache: Arc<RwLock<HashMap<String, Vec<BacktestMarketData>>>>,
}

impl DbnTestDataManager {
    /// Create a new DBN test data manager with ES.FUT data
    ///
    /// # Returns
    ///
    /// Configured manager with ES.FUT data loaded
    pub async fn new() -> Result<Self> {
        // Get workspace root
        let workspace_root = Self::find_workspace_root()?;

        // Build file mapping for available DBN files
        let mut file_mapping = HashMap::new();

        // ES.FUT OHLCV data (1-minute bars, 2024-01-02)
        let es_fut_path = workspace_root
            .join("test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn");

        if es_fut_path.exists() {
            file_mapping.insert(
                "ES.FUT".to_string(),
                es_fut_path.to_string_lossy().to_string(),
            );
        } else {
            anyhow::bail!("DBN test data not found: {}", es_fut_path.display());
        }

        // Create data source
        let data_source = Arc::new(
            DbnDataSource::new(file_mapping)
                .await
                .context("Failed to create DBN data source")?,
        );

        Ok(Self {
            data_source,
            cache: Arc::new(RwLock::new(HashMap::new())),
        })
    }

    /// Find the workspace root directory
    fn find_workspace_root() -> Result<PathBuf> {
        let current_dir = std::env::current_dir()?;

        // Walk up from current directory to find workspace root
        for ancestor in current_dir.ancestors() {
            let cargo_toml = ancestor.join("Cargo.toml");
            let test_data = ancestor.join("test_data");

            if cargo_toml.exists() && test_data.exists() {
                return Ok(ancestor.to_path_buf());
            }
        }

        anyhow::bail!("Could not find workspace root with test_data directory")
    }

    /// Get available symbols
    pub fn available_symbols(&self) -> Vec<String> {
        self.data_source.available_symbols()
    }

    /// Load market data for symbol (with caching)
    ///
    /// # Arguments
    ///
    /// * `symbol` - Trading symbol to load data for
    ///
    /// # Returns
    ///
    /// Vector of market data bars sorted by timestamp
    pub async fn load_market_data(&self, symbol: &str) -> Result<Vec<BacktestMarketData>> {
        // Check cache first
        {
            let cache = self.cache.read().await;
            if let Some(data) = cache.get(symbol) {
                return Ok(data.clone());
            }
        }

        // Load from data source
        let data = self.data_source.load_ohlcv_bars(symbol).await?;

        // Cache for future use
        {
            let mut cache = self.cache.write().await;
            cache.insert(symbol.to_string(), data.clone());
        }

        Ok(data)
    }

    /// Get a realistic price for a symbol based on real data
    ///
    /// # Arguments
    ///
    /// * `symbol` - Trading symbol
    ///
    /// # Returns
    ///
    /// A realistic price from the loaded data
    pub async fn get_realistic_price(&self, symbol: &str) -> Result<f64> {
        let data = self.load_market_data(symbol).await?;

        if data.is_empty() {
            anyhow::bail!("No market data available for symbol: {}", symbol);
        }

        // Use the last close price as a realistic price
        let last_bar = &data[data.len() - 1];
        Ok(last_bar.close.to_string().parse()?)
    }

    /// Get time range for available data
    ///
    /// # Arguments
    ///
    /// * `symbol` - Trading symbol
    ///
    /// # Returns
    ///
    /// Tuple of (start_time, end_time) for available data
    pub async fn get_time_range(&self, symbol: &str) -> Result<(DateTime<Utc>, DateTime<Utc>)> {
        let data = self.load_market_data(symbol).await?;

        if data.is_empty() {
            anyhow::bail!("No market data available for symbol: {}", symbol);
        }

        let start_time = data[0].timestamp;
        let end_time = data[data.len() - 1].timestamp;

        Ok((start_time, end_time))
    }

    /// Get market data for a specific time window
    ///
    /// # Arguments
    ///
    /// * `symbol` - Trading symbol
    /// * `start_time` - Start of time window
    /// * `end_time` - End of time window
    ///
    /// # Returns
    ///
    /// Filtered market data within the time window
    pub async fn get_data_window(
        &self,
        symbol: &str,
        start_time: DateTime<Utc>,
        end_time: DateTime<Utc>,
    ) -> Result<Vec<BacktestMarketData>> {
        let data = self.load_market_data(symbol).await?;

        let filtered: Vec<BacktestMarketData> = data
            .into_iter()
            .filter(|bar| bar.timestamp >= start_time && bar.timestamp <= end_time)
            .collect();

        Ok(filtered)
    }

    /// Create a realistic order price based on current market data
    ///
    /// # Arguments
    ///
    /// * `symbol` - Trading symbol
    /// * `side` - Order side (Buy/Sell)
    /// * `offset_bps` - Offset in basis points from current price
    ///
    /// # Returns
    ///
    /// Realistic order price
    pub async fn create_realistic_order_price(
        &self,
        symbol: &str,
        side: &str,
        offset_bps: i32,
    ) -> Result<f64> {
        let current_price = self.get_realistic_price(symbol).await?;

        // Apply offset based on side
        let multiplier = match side.to_lowercase().as_str() {
            "buy" => 1.0 - (offset_bps as f64 / 10000.0), // Bid lower
            "sell" => 1.0 + (offset_bps as f64 / 10000.0), // Ask higher
            _ => 1.0,
        };

        Ok(current_price * multiplier)
    }

    /// Get OHLCV data for the last N bars
    ///
    /// # Arguments
    ///
    /// * `symbol` - Trading symbol
    /// * `num_bars` - Number of bars to retrieve
    ///
    /// # Returns
    ///
    /// Last N bars of market data
    pub async fn get_last_n_bars(&self, symbol: &str, num_bars: usize) -> Result<Vec<BacktestMarketData>> {
        let data = self.load_market_data(symbol).await?;

        if data.len() < num_bars {
            return Ok(data);
        }

        let start_idx = data.len() - num_bars;
        Ok(data[start_idx..].to_vec())
    }

    /// Convert BacktestMarketData to proto BarData
    ///
    /// # Arguments
    ///
    /// * `bar` - Market data bar
    ///
    /// # Returns
    ///
    /// Proto BarData message
    pub fn to_proto_bar_data(&self, bar: &BacktestMarketData) -> Result<(String, i64, String, f64, f64, f64, f64, u64)> {
        let timestamp_nanos = bar.timestamp.timestamp_nanos_opt()
            .ok_or_else(|| anyhow::anyhow!("Invalid timestamp"))?;

        let open: f64 = bar.open.to_string().parse()?;
        let high: f64 = bar.high.to_string().parse()?;
        let low: f64 = bar.low.to_string().parse()?;
        let close: f64 = bar.close.to_string().parse()?;
        let volume: u64 = bar.volume.to_string().parse()?;

        Ok((
            bar.symbol.clone(),
            timestamp_nanos,
            "1m".to_string(),
            open,
            high,
            low,
            close,
            volume,
        ))
    }
}

/// Global DBN test data manager instance
static DBN_MANAGER: tokio::sync::OnceCell<DbnTestDataManager> = tokio::sync::OnceCell::const_new();

/// Get the global DBN test data manager
///
/// Lazily initializes the manager on first access
pub async fn get_dbn_manager() -> Result<&'static DbnTestDataManager> {
    DBN_MANAGER
        .get_or_try_init(|| async {
            DbnTestDataManager::new().await
        })
        .await
        .context("Failed to initialize DBN test data manager")
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn test_dbn_manager_creation() {
        let manager = DbnTestDataManager::new().await;
        assert!(manager.is_ok(), "Failed to create DBN manager: {:?}", manager.err());

        let mgr = manager.unwrap();
        let symbols = mgr.available_symbols();
        assert!(!symbols.is_empty(), "No symbols available");
        assert!(symbols.contains(&"ES.FUT".to_string()));
    }

    #[tokio::test]
    async fn test_load_market_data() {
        let manager = DbnTestDataManager::new().await.unwrap();
        let data = manager.load_market_data("ES.FUT").await;

        assert!(data.is_ok(), "Failed to load market data: {:?}", data.err());

        let bars = data.unwrap();
        assert!(!bars.is_empty(), "No market data loaded");
        assert!(bars.len() > 100, "Expected more bars, got: {}", bars.len());

        // Verify data quality
        let first_bar = &bars[0];
        assert_eq!(first_bar.symbol, "ES.FUT");

        let close_f64: f64 = first_bar.close.to_string().parse().unwrap();
        assert!(close_f64 > 4000.0 && close_f64 < 6000.0,
            "Unexpected ES.FUT price: {}", close_f64);
    }

    #[tokio::test]
    async fn test_get_realistic_price() {
        let manager = DbnTestDataManager::new().await.unwrap();
        let price = manager.get_realistic_price("ES.FUT").await;

        assert!(price.is_ok(), "Failed to get price: {:?}", price.err());

        let p = price.unwrap();
        assert!(p > 4000.0 && p < 6000.0, "Unexpected price: {}", p);
    }

    #[tokio::test]
    async fn test_get_time_range() {
        let manager = DbnTestDataManager::new().await.unwrap();
        let range = manager.get_time_range("ES.FUT").await;

        assert!(range.is_ok(), "Failed to get time range: {:?}", range.err());

        let (start, end) = range.unwrap();
        assert!(end > start, "End time should be after start time");
    }
}