foxhunt/data/src/providers/databento_old.rs

//! Databento Historical Data Provider
//!
//! High-performance historical market data provider for backtesting and training.
//! Provides access to normalized, exchange-quality market data with nanosecond timestamps.

use crate::error::{DataError, Result};
use chrono::{DateTime, Utc};
use common::MarketDataEvent;
use common::{BarEvent, OrderSide};
use common::{QuoteEvent, TradeEvent};
use reqwest::Client;
use rust_decimal::Decimal;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::time::Duration;
use tokio::time::sleep;
use tracing::{debug, warn};

/// Databento API configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(super) struct DatabentoConfig {
    /// API key
    pub api_key: String,
    /// API base URL
    pub base_url: String,
    /// Request timeout in seconds
    pub timeout_seconds: u64,
    /// Rate limit (requests per second)
    pub rate_limit: u32,
    /// Maximum retries for failed requests
    pub max_retries: u32,
    /// Retry delay in milliseconds
    pub retry_delay_ms: u64,
}

impl Default for DatabentoConfig {
    fn default() -> Self {
        Self {
            api_key: std::env::var("DATABENTO_API_KEY").unwrap_or_default(),
            base_url: "https://hist.databento.com".to_string(),
            timeout_seconds: 30,
            rate_limit: 10, // 10 requests per second
            max_retries: 3,
            retry_delay_ms: 1000,
        }
    }
}

/// Databento historical data provider
pub(super) struct DatabentoHistoricalProvider {
    config: DatabentoConfig,
    client: Client,
    last_request_time: std::sync::Arc<std::sync::Mutex<std::time::Instant>>,
}

/// Databento data schema types
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(super) enum DatabentoSchema {
    /// Trade data
    #[serde(rename = "trades")]
    Trades,
    /// Market by order data (Level 3)
    #[serde(rename = "mbo")]
    MBO,
    /// Market by price data (Level 2)
    #[serde(rename = "mbp-1")]
    MBP1,
    /// Top of book quotes
    #[serde(rename = "tbbo")]
    TBBO,
    /// OHLCV bars
    #[serde(rename = "ohlcv-1s")]
    OHLCV1s,
    #[serde(rename = "ohlcv-1m")]
    OHLCV1m,
    #[serde(rename = "ohlcv-1h")]
    OHLCV1h,
    #[serde(rename = "ohlcv-1d")]
    OHLCV1d,
}

/// Databento dataset identifier
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(super) enum DatabentoDataset {
    /// NASDAQ Basic
    #[serde(rename = "XNAS.ITCH")]
    NasdaqBasic,
    /// NYSE Trades and Quotes
    #[serde(rename = "XNYS.ITCH")]
    NYSEBasic,
    /// IEX DEEP
    #[serde(rename = "XIEX.TOPS")]
    IEXDeep,
    /// CBOE BZX
    #[serde(rename = "BATS.PITCH")]
    CBOEBZX,
}

/// Databento historical request parameters
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(super) struct DatabentoRequest {
    /// Dataset to query
    pub dataset: DatabentoDataset,
    /// Data schema
    pub schema: DatabentoSchema,
    /// Start timestamp (inclusive)
    pub start: DateTime<Utc>,
    /// End timestamp (exclusive)
    pub end: DateTime<Utc>,
    /// Symbols to include (empty = all)
    pub symbols: Vec<String>,
    /// Additional filters
    pub stype_in: Option<Vec<String>>,
    /// Delivery format
    pub encoding: String,
    /// Compression type
    pub compression: String,
    /// Pretty print (for JSON)
    pub pretty_px: bool,
    /// Map symbols to human-readable names
    pub map_symbols: bool,
}

/// Databento API response
#[derive(Debug, Clone, Deserialize)]
pub(super) struct DatabentoResponse {
    /// Request ID
    pub id: Option<String>,
    /// Response data
    pub data: Vec<DatabentoRecord>,
    /// Metadata
    pub metadata: Option<DatabentoMetadata>,
    /// Error information
    pub error_message: Option<String>,
}

/// Databento metadata
#[derive(Debug, Clone, Deserialize)]
pub(super) struct DatabentoMetadata {
    /// Dataset
    pub dataset: String,
    /// Schema
    pub schema: String,
    /// Start timestamp
    pub start: DateTime<Utc>,
    /// End timestamp  
    pub end: DateTime<Utc>,
    /// Record count
    pub count: u64,
    /// Size in bytes
    pub size: u64,
}

/// Databento data record
#[derive(Debug, Clone, Deserialize)]
#[serde(untagged)]
pub(super) enum DatabentoRecord {
    /// Trade record
    Trade(DatabentoTrade),
    /// Quote record
    Quote(DatabentoQuote),
    /// OHLCV bar record
    Bar(DatabentoBar),
}

/// Databento trade record
#[derive(Debug, Clone, Deserialize)]
pub(super) struct DatabentoTrade {
    /// Timestamp (nanoseconds since Unix epoch)
    pub ts_event: i64,
    /// Timestamp when received (nanoseconds since Unix epoch)
    pub ts_recv: i64,
    /// Symbol ID
    pub instrument_id: u32,
    /// Publisher ID
    pub publisher_id: u16,
    /// Trade price (fixed-point representation)
    pub price: i64,
    /// Trade size
    pub size: u32,
    /// Trade action
    pub action: char,
    /// Trade side (if available)
    pub side: Option<char>,
    /// Trade flags
    pub flags: Option<u8>,
    /// Depth of trade
    pub depth: Option<u8>,
    /// Trade sequence number
    pub sequence: Option<u64>,
}

/// Databento quote record
#[derive(Debug, Clone, Deserialize)]
pub(super) struct DatabentoQuote {
    /// Timestamp (nanoseconds since Unix epoch)
    pub ts_event: i64,
    /// Timestamp when received (nanoseconds since Unix epoch)
    pub ts_recv: i64,
    /// Symbol ID
    pub instrument_id: u32,
    /// Publisher ID
    pub publisher_id: u16,
    /// Bid price (fixed-point representation)
    pub bid_px: i64,
    /// Ask price (fixed-point representation)
    pub ask_px: i64,
    /// Bid size
    pub bid_sz: u32,
    /// Ask size
    pub ask_sz: u32,
    /// Quote condition
    pub bid_ct: Option<u8>,
    /// Quote condition
    pub ask_ct: Option<u8>,
    /// Sequence number
    pub sequence: Option<u64>,
}

/// Databento OHLCV bar record
#[derive(Debug, Clone, Deserialize)]
pub(super) struct DatabentoBar {
    /// Timestamp (nanoseconds since Unix epoch)
    pub ts_event: i64,
    /// Symbol ID
    pub instrument_id: u32,
    /// Open price (fixed-point representation)
    pub open: i64,
    /// High price (fixed-point representation)
    pub high: i64,
    /// Low price (fixed-point representation)
    pub low: i64,
    /// Close price (fixed-point representation)
    pub close: i64,
    /// Volume
    pub volume: u64,
}

impl DatabentoHistoricalProvider {
    /// Create a new Databento historical provider
    pub(super) fn new(config: DatabentoConfig) -> Result<Self> {
        let client = Client::builder()
            .timeout(Duration::from_secs(config.timeout_seconds))
            .build()
            .map_err(|e| DataError::Network {
                message: format!("Failed to create HTTP client: {}", e),
            })?;

        Ok(Self {
            config,
            client,
            last_request_time: std::sync::Arc::new(std::sync::Mutex::new(
                std::time::Instant::now() - Duration::from_secs(1),
            )),
        })
    }

    /// Get historical trade data
    pub(super) async fn get_trades(
        &self,
        symbols: &[String],
        start: DateTime<Utc>,
        end: DateTime<Utc>,
        dataset: Option<DatabentoDataset>,
    ) -> Result<Vec<TradeEvent>> {
        let request = DatabentoRequest {
            dataset: dataset.unwrap_or(DatabentoDataset::NasdaqBasic),
            schema: DatabentoSchema::Trades,
            start,
            end,
            symbols: symbols.to_vec(),
            stype_in: None,
            encoding: "json".to_string(),
            compression: "none".to_string(),
            pretty_px: true,
            map_symbols: true,
        };

        let records = self.make_request(&request).await?;
        self.convert_to_trades(records, symbols)
    }

    /// Get historical quote data
    pub(super) async fn get_quotes(
        &self,
        symbols: &[String],
        start: DateTime<Utc>,
        end: DateTime<Utc>,
        dataset: Option<DatabentoDataset>,
    ) -> Result<Vec<QuoteEvent>> {
        let request = DatabentoRequest {
            dataset: dataset.unwrap_or(DatabentoDataset::NasdaqBasic),
            schema: DatabentoSchema::TBBO,
            start,
            end,
            symbols: symbols.to_vec(),
            stype_in: None,
            encoding: "json".to_string(),
            compression: "none".to_string(),
            pretty_px: true,
            map_symbols: true,
        };

        let records = self.make_request(&request).await?;
        self.convert_to_quotes(records, symbols)
    }

    /// Get historical OHLCV bars
    pub(super) async fn get_bars(
        &self,
        symbols: &[String],
        start: DateTime<Utc>,
        end: DateTime<Utc>,
        timeframe: &str,
        dataset: Option<DatabentoDataset>,
    ) -> Result<Vec<MarketDataEvent>> {
        let schema = match timeframe {
            "1s" => DatabentoSchema::OHLCV1s,
            "1m" | "1min" => DatabentoSchema::OHLCV1m,
            "1h" | "1hour" => DatabentoSchema::OHLCV1h,
            "1d" | "1day" => DatabentoSchema::OHLCV1d,
            _ => {
                return Err(DataError::InvalidParameter {
                    field: "timeframe".to_string(),
                    message: format!(
                        "Invalid timeframe '{}', expected: 1s, 1m, 1h, or 1d",
                        timeframe
                    ),
                });
            },
        };

        let request = DatabentoRequest {
            dataset: dataset.unwrap_or(DatabentoDataset::NasdaqBasic),
            schema,
            start,
            end,
            symbols: symbols.to_vec(),
            stype_in: None,
            encoding: "json".to_string(),
            compression: "none".to_string(),
            pretty_px: true,
            map_symbols: true,
        };

        let records = self.make_request(&request).await?;
        self.convert_to_bars(records, symbols)
    }

    /// Make API request with rate limiting and retry logic
    async fn make_request(&self, request: &DatabentoRequest) -> Result<Vec<DatabentoRecord>> {
        let mut attempt = 0;
        loop {
            // Rate limiting
            self.enforce_rate_limit().await;

            // Build request URL
            let url = format!("{}/v0/timeseries.get", self.config.base_url);

            // Serialize request parameters
            let params = serde_json::to_value(request).map_err(|e| {
                DataError::serialization(format!("Failed to serialize request: {}", e))
            })?;

            debug!("Making Databento API request: {}", url);

            // Execute request
            let response = self
                .client
                .get(&url)
                .header("Authorization", format!("Bearer {}", self.config.api_key))
                .json(&params)
                .send()
                .await
                .map_err(|e| DataError::Network {
                    message: format!("HTTP request failed: {}", e),
                })?;

            if response.status().is_success() {
                let databento_response: DatabentoResponse = response.json().await.map_err(|e| {
                    DataError::serialization(format!("Failed to parse response: {}", e))
                })?;

                if let Some(error) = databento_response.error_message {
                    return Err(DataError::Api {
                        message: error,
                        status: None,
                    });
                }

                return Ok(databento_response.data);
            }

            // Handle errors and retries
            attempt += 1;
            if attempt >= self.config.max_retries {
                return Err(DataError::Api {
                    message: format!(
                        "Request failed after {} attempts: {}",
                        attempt,
                        response.status()
                    ),
                    status: Some(response.status().to_string()),
                });
            }

            warn!(
                "Request failed (attempt {}/{}): {}. Retrying in {}ms",
                attempt,
                self.config.max_retries,
                response.status(),
                self.config.retry_delay_ms
            );

            sleep(Duration::from_millis(self.config.retry_delay_ms)).await;
        }
    }

    /// Enforce rate limiting
    async fn enforce_rate_limit(&self) {
        let min_interval = Duration::from_secs(1) / self.config.rate_limit;

        let last_request = {
            let guard = self.last_request_time.lock().unwrap();
            *guard
        };

        let elapsed = last_request.elapsed();
        if elapsed < min_interval {
            let sleep_duration = min_interval - elapsed;
            sleep(sleep_duration).await;
        }

        {
            let mut guard = self.last_request_time.lock().unwrap();
            *guard = std::time::Instant::now();
        }
    }

    /// Convert Databento records to trade events
    fn convert_to_trades(
        &self,
        records: Vec<DatabentoRecord>,
        symbols: &[String],
    ) -> Result<Vec<TradeEvent>> {
        let mut trades = Vec::new();
        let symbol_map = self.create_symbol_map(symbols);

        for record in records {
            if let DatabentoRecord::Trade(trade) = record {
                let symbol = symbol_map
                    .get(&trade.instrument_id)
                    .cloned()
                    .unwrap_or_else(|| format!("UNKNOWN_{}", trade.instrument_id));

                let price = Decimal::from(trade.price) / Decimal::from(10_000); // Assuming 4 decimal places
                let size = Decimal::from(trade.size);

                let _side = match trade.side {
                    Some('B') => OrderSide::Buy,
                    Some('S') => OrderSide::Sell,
                    _ => OrderSide::Buy, // Default to buy if unknown
                };

                let event = TradeEvent {
                    symbol,
                    timestamp: DateTime::from_timestamp_nanos(trade.ts_event),
                    price,
                    size,
                    trade_id: trade.sequence.map(|s| s.to_string()),
                    exchange: None,
                    conditions: Vec::new(),
                    sequence: trade.sequence.unwrap_or(0),
                };

                trades.push(event);
            }
        }

        trades.sort_by(|a, b| a.timestamp.cmp(&b.timestamp));
        Ok(trades)
    }

    /// Convert Databento records to quote events
    fn convert_to_quotes(
        &self,
        records: Vec<DatabentoRecord>,
        symbols: &[String],
    ) -> Result<Vec<QuoteEvent>> {
        let mut quotes = Vec::new();
        let symbol_map = self.create_symbol_map(symbols);

        for record in records {
            if let DatabentoRecord::Quote(quote) = record {
                let symbol = symbol_map
                    .get(&quote.instrument_id)
                    .cloned()
                    .unwrap_or_else(|| format!("UNKNOWN_{}", quote.instrument_id));

                let bid_price = Decimal::from(quote.bid_px) / Decimal::from(10_000);
                let ask_price = Decimal::from(quote.ask_px) / Decimal::from(10_000);
                let bid_size = Decimal::from(quote.bid_sz);
                let ask_size = Decimal::from(quote.ask_sz);

                let event = QuoteEvent {
                    symbol,
                    timestamp: DateTime::from_timestamp_nanos(quote.ts_event),
                    bid: Some(bid_price),
                    ask: Some(ask_price),
                    bid_size: Some(bid_size),
                    ask_size: Some(ask_size),
                    exchange: Some(format!("pub_{}", quote.publisher_id)),
                    bid_exchange: Some(format!("pub_{}", quote.publisher_id)),
                    ask_exchange: Some(format!("pub_{}", quote.publisher_id)),
                    conditions: Vec::new(),
                    sequence: quote.sequence.unwrap_or(0),
                };

                quotes.push(event);
            }
        }

        quotes.sort_by(|a, b| a.timestamp.cmp(&b.timestamp));
        Ok(quotes)
    }

    /// Convert Databento records to market data events (bars)
    fn convert_to_bars(
        &self,
        records: Vec<DatabentoRecord>,
        symbols: &[String],
    ) -> Result<Vec<MarketDataEvent>> {
        let mut bars = Vec::new();
        let symbol_map = self.create_symbol_map(symbols);

        for record in records {
            if let DatabentoRecord::Bar(bar) = record {
                let symbol = symbol_map
                    .get(&bar.instrument_id)
                    .cloned()
                    .unwrap_or_else(|| format!("UNKNOWN_{}", bar.instrument_id));

                let open = Decimal::from(bar.open) / Decimal::from(10_000);
                let high = Decimal::from(bar.high) / Decimal::from(10_000);
                let low = Decimal::from(bar.low) / Decimal::from(10_000);
                let close = Decimal::from(bar.close) / Decimal::from(10_000);
                let volume = Decimal::from(bar.volume);

                let timestamp = DateTime::from_timestamp_nanos(bar.ts_event);
                let bar_event = BarEvent {
                    symbol: symbol.into(),
                    open,
                    high,
                    low,
                    close,
                    volume,
                    vwap: None,
                    start_timestamp: timestamp,
                    end_timestamp: timestamp,
                    timeframe: "1m".to_string(),
                };
                let event = MarketDataEvent::Bar(bar_event);

                bars.push(event);
            }
        }

        bars.sort_by(|a, b| {
            let ts_a = match a {
                MarketDataEvent::Bar(bar_event) => bar_event.end_timestamp,
                _ => DateTime::from_timestamp(0, 0).unwrap(),
            };
            let ts_b = match b {
                MarketDataEvent::Bar(bar_event) => bar_event.end_timestamp,
                _ => DateTime::from_timestamp(0, 0).unwrap(),
            };
            ts_a.cmp(&ts_b)
        });

        Ok(bars)
    }

    /// Create a map from instrument IDs to symbol names
    fn create_symbol_map(&self, symbols: &[String]) -> HashMap<u32, String> {
        // In a real implementation, this would map Databento instrument IDs to symbols
        // For now, create a simple mapping using index
        symbols
            .iter()
            .enumerate()
            .map(|(i, symbol)| (i as u32 + 1, symbol.clone()))
            .collect()
    }

    /// Get available datasets
    pub(super) fn get_available_datasets() -> Vec<DatabentoDataset> {
        vec![
            DatabentoDataset::NasdaqBasic,
            DatabentoDataset::NYSEBasic,
            DatabentoDataset::IEXDeep,
            DatabentoDataset::CBOEBZX,
        ]
    }

    /// Get available schemas
    pub(super) fn get_available_schemas() -> Vec<DatabentoSchema> {
        vec![
            DatabentoSchema::Trades,
            DatabentoSchema::MBO,
            DatabentoSchema::MBP1,
            DatabentoSchema::TBBO,
            DatabentoSchema::OHLCV1s,
            DatabentoSchema::OHLCV1m,
            DatabentoSchema::OHLCV1h,
            DatabentoSchema::OHLCV1d,
        ]
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_config_creation() {
        let config = DatabentoConfig::default();
        assert!(!config.base_url.is_empty());
        assert!(config.timeout_seconds > 0);
    }

    #[test]
    fn test_provider_creation() {
        let config = DatabentoConfig::default();
        let provider = DatabentoHistoricalProvider::new(config);
        assert!(provider.is_ok());
    }

    #[tokio::test]
    async fn test_rate_limiting() {
        let config = DatabentoConfig {
            rate_limit: 2, // 2 requests per second
            ..Default::default()
        };
        let provider = DatabentoHistoricalProvider::new(config).unwrap();

        let start = std::time::Instant::now();
        provider.enforce_rate_limit().await;
        provider.enforce_rate_limit().await;
        provider.enforce_rate_limit().await;
        let elapsed = start.elapsed();

        // Should take at least 1 second for 3 requests with 2 req/sec limit
        assert!(elapsed >= Duration::from_millis(900));
    }
}