foxhunt/data/examples/databento_demo.rs

//! # Databento Provider Demo
//!
//! This example demonstrates how to use the DatabentoProvider for high-performance
//! market data ingestion in the Foxhunt HFT system.
//!
//! ## Usage
//!
//! ```bash
//! # Set your Databento API key
//! export DATABENTO_API_KEY="your_api_key_here"
//!
//! # Run the demo
//! cargo run --example databento_demo --features databento
//! ```

use anyhow::Result;
use data::providers::databento::{DatabentoConfig, DatabentoProvider};
use data::providers::{MarketDataProvider, ProviderConfig};
use std::time::Duration;
use tokio::time;
use tracing::{error, info, warn};
use tracing_subscriber;
use trading_engine::trading::data_interface::{DataProvider, DataType, Subscription};
use trading_engine::types::prelude::*;

#[tokio::main]
async fn main() -> Result<()> {
    // Initialize logging
    tracing_subscriber::fmt()
        .with_max_level(tracing::Level::INFO)
        .init();

    info!("Starting Databento Provider Demo");

    // Check for API key
    let api_key = std::env::var("DATABENTO_API_KEY")
        .map_err(|_| anyhow::anyhow!("DATABENTO_API_KEY environment variable is required"))?;

    if api_key == "DATABENTO_API_KEY_REQUIRED" {
        error!("Please set your actual Databento API key in the DATABENTO_API_KEY environment variable");
        return Err(anyhow::anyhow!("API key not configured"));
    }

    // Demo 1: Basic Provider Creation and Connection
    info!("=== Demo 1: Basic Provider Setup ===");
    demo_basic_setup(&api_key).await?;

    // Demo 2: Market Data Subscription
    info!("=== Demo 2: Market Data Subscription ===");
    demo_market_data_subscription(&api_key).await?;

    // Demo 3: Health Monitoring
    info!("=== Demo 3: Health Monitoring ===");
    demo_health_monitoring(&api_key).await?;

    // Demo 4: Rate Limiting Demonstration
    info!("=== Demo 4: Rate Limiting ===");
    demo_rate_limiting(&api_key).await?;

    // Demo 5: Configuration Options
    info!("=== Demo 5: Configuration Options ===");
    demo_configuration_options(&api_key).await?;

    info!("Databento Provider Demo completed successfully!");
    Ok(())
}

/// Demonstrates basic provider setup and connection
async fn demo_basic_setup(api_key: &str) -> Result<()> {
    info!("Creating Databento provider with default configuration...");

    let config = DatabentoConfig {
        api_key: api_key.to_string(),
        datasets: vec!["XNAS.ITCH".to_string()], // NASDAQ dataset
        max_connections: 3,                      // Conservative limit
        ..Default::default()
    };

    let mut provider = DatabentoProvider::new(config)?;
    info!("Provider created successfully");

    // Test connection
    info!("Attempting to connect to Databento API...");
    match provider.connect().await {
        Ok(_) => {
            info!("Successfully connected to Databento!");

            // Check connection status
            let health = provider.get_health_status();
            info!(
                "Provider health: connected={}, subscriptions={}",
                health.connected, health.active_subscriptions
            );

            // Disconnect
            provider.disconnect().await?;
            info!("Disconnected from Databento");
        }
        Err(e) => {
            warn!("Connection failed (expected in demo): {}", e);
        }
    }

    Ok(())
}

/// Demonstrates market data subscription patterns
async fn demo_market_data_subscription(api_key: &str) -> Result<()> {
    let config = DatabentoConfig {
        api_key: api_key.to_string(),
        datasets: vec!["XNAS.ITCH".to_string()],
        ..Default::default()
    };

    let mut provider = DatabentoProvider::new(config)?;

    // Connect first
    if let Err(e) = provider.connect().await {
        warn!(
            "Skipping subscription demo due to connection failure: {}",
            e
        );
        return Ok(());
    }

    // Define symbols to subscribe to
    let symbols = vec![
        "SPY".to_string(),  // SPDR S&P 500 ETF
        "QQQ".to_string(),  // Invesco QQQ ETF
        "IWM".to_string(),  // iShares Russell 2000 ETF
        "AAPL".to_string(), // Apple Inc.
        "MSFT".to_string(), // Microsoft Corp.
    ];

    info!("Subscribing to {} symbols: {:?}", symbols.len(), symbols);

    // Subscribe using the MarketDataProvider trait
    let symbol_objects: Vec<Symbol> = symbols.iter().map(|s| Symbol::from(s.as_str())).collect();

    match provider.subscribe(symbol_objects).await {
        Ok(_) => {
            info!("Successfully subscribed to market data");

            // Get subscription status
            let subscriptions = provider.get_active_subscriptions().await;
            info!("Active subscriptions: {:?}", subscriptions);

            // Simulate receiving data for a few seconds
            info!("Simulating data reception...");
            time::sleep(Duration::from_secs(3)).await;
        }
        Err(e) => {
            warn!("Subscription failed (expected in demo): {}", e);
        }
    }

    provider.disconnect().await?;
    Ok(())
}

/// Demonstrates health monitoring capabilities
async fn demo_health_monitoring(api_key: &str) -> Result<()> {
    let config = DatabentoConfig {
        api_key: api_key.to_string(),
        ..Default::default()
    };

    let provider = DatabentoProvider::new(config)?;

    info!("Starting health monitoring...");
    provider.start_health_monitoring().await;

    // Monitor health status over time
    for i in 0..5 {
        let health = provider.get_health_status();
        info!(
            "Health check {}: connected={}, msgs/sec={:.2}, latency={}μs",
            i + 1,
            health.connected,
            health.messages_per_second,
            health.latency_micros.unwrap_or(0)
        );

        time::sleep(Duration::from_secs(2)).await;
    }

    info!("Health monitoring demo completed");
    Ok(())
}

/// Demonstrates rate limiting compliance
async fn demo_rate_limiting(api_key: &str) -> Result<()> {
    info!("Testing rate limiting compliance...");

    // Create multiple subscription requests to test rate limiting
    let symbols_batch1 = vec!["SPY", "QQQ", "IWM"];
    let symbols_batch2 = vec!["AAPL", "MSFT", "GOOGL"];
    let symbols_batch3 = vec!["TSLA", "NVDA", "AMD"];

    let config = DatabentoConfig {
        api_key: api_key.to_string(),
        ..Default::default()
    };

    let mut provider = DatabentoProvider::new(config)?;

    if let Err(e) = provider.connect().await {
        warn!("Skipping rate limit demo due to connection failure: {}", e);
        return Ok(());
    }

    let start_time = std::time::Instant::now();

    // Submit multiple batches - should be rate limited
    info!("Submitting batch 1: {:?}", symbols_batch1);
    let _ = provider
        .subscribe_symbols(
            symbols_batch1.iter().map(|s| s.to_string()).collect(),
            "XNAS.ITCH",
        )
        .await;

    info!("Submitting batch 2: {:?}", symbols_batch2);
    let _ = provider
        .subscribe_symbols(
            symbols_batch2.iter().map(|s| s.to_string()).collect(),
            "XNAS.ITCH",
        )
        .await;

    info!("Submitting batch 3: {:?}", symbols_batch3);
    let _ = provider
        .subscribe_symbols(
            symbols_batch3.iter().map(|s| s.to_string()).collect(),
            "XNAS.ITCH",
        )
        .await;

    let elapsed = start_time.elapsed();
    info!(
        "Rate limited subscriptions took: {:.2}s (should be ~1s for compliance)",
        elapsed.as_secs_f64()
    );

    provider.disconnect().await?;
    Ok(())
}

/// Demonstrates various configuration options
async fn demo_configuration_options(api_key: &str) -> Result<()> {
    info!("Testing different configuration options...");

    // Configuration 1: High-performance setup
    let high_perf_config = DatabentoConfig {
        api_key: api_key.to_string(),
        datasets: vec!["XNAS.ITCH".to_string(), "GLBX.MDP3".to_string()],
        max_connections: 8, // Near the 10 connection limit
        compression: true,
        buffer_size: 2 * 1024 * 1024, // 2MB buffer
        schemas: vec![
            "mbo".to_string(),    // Full order book
            "mbp-1".to_string(),  // Top of book
            "trades".to_string(), // All trades
        ],
        ..Default::default()
    };

    info!(
        "High-performance config: {} datasets, {} max connections",
        high_perf_config.datasets.len(),
        high_perf_config.max_connections
    );

    // Configuration 2: Conservative setup
    let conservative_config = DatabentoConfig {
        api_key: api_key.to_string(),
        datasets: vec!["XNAS.ITCH".to_string()],
        max_connections: 2,
        compression: false,
        buffer_size: 256 * 1024, // 256KB buffer
        schemas: vec![
            "mbp-1".to_string(),  // Top of book only
            "trades".to_string(), // Trades only
        ],
        reconnect_attempts: 10,
        reconnect_backoff_ms: 2000,
        ..Default::default()
    };

    info!(
        "Conservative config: {} datasets, {} max connections",
        conservative_config.datasets.len(),
        conservative_config.max_connections
    );

    // Configuration 3: Multi-asset setup
    let multi_asset_config = DatabentoConfig {
        api_key: api_key.to_string(),
        datasets: vec![
            "XNAS.ITCH".to_string(), // NASDAQ Equities
            "XNYS.ITCH".to_string(), // NYSE Equities
            "OPRA.ITCH".to_string(), // Options
        ],
        max_connections: 5,
        schemas: vec![
            "mbp-1".to_string(),
            "mbp-10".to_string(),
            "trades".to_string(),
            "ohlcv-1s".to_string(),
        ],
        ..Default::default()
    };

    info!(
        "Multi-asset config: {} datasets covering equities and options",
        multi_asset_config.datasets.len()
    );

    // Test serialization/deserialization
    let json = serde_json::to_string_pretty(&high_perf_config)?;
    info!("Configuration serialization example:\n{}", json);

    let deserialized: DatabentoConfig = serde_json::from_str(&json)?;
    info!(
        "Successfully deserialized configuration with {} datasets",
        deserialized.datasets.len()
    );

    Ok(())
}

/// Helper function to create test symbols
fn create_test_symbols() -> Vec<Symbol> {
    vec![
        Symbol::from("SPY"),
        Symbol::from("QQQ"),
        Symbol::from("IWM"),
        Symbol::from("AAPL"),
        Symbol::from("MSFT"),
    ]
}

/// Helper function to create subscription request
fn create_test_subscription() -> Subscription {
    Subscription {
        symbols: vec!["SPY".to_string(), "QQQ".to_string(), "IWM".to_string()],
        data_types: vec![DataType::Trades, DataType::Quotes, DataType::OrderBook],
        exchanges: Some(vec!["XNAS".to_string()]),
        extended_hours: false,
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_create_test_symbols() {
        let symbols = create_test_symbols();
        assert_eq!(symbols.len(), 5);
        assert_eq!(symbols[0].to_string(), "SPY");
    }

    #[test]
    fn test_create_test_subscription() {
        let subscription = create_test_subscription();
        assert_eq!(subscription.symbols.len(), 3);
        assert_eq!(subscription.data_types.len(), 3);
        assert!(subscription.exchanges.is_some());
    }
}