foxhunt/tests/integration/dual_provider_test.rs

//! Comprehensive Integration Tests for Dual-Provider System
//!
//! This module tests the integration between multiple data providers (Databento for market data
//! and broker clients for order/position data) coordinated by the DataManager, ensuring:
//! 1. Correct data streaming from both providers
//! 2. Unified feature extraction without training/serving skew
//! 3. Symbol mapping consistency between providers
//! 4. Timestamp synchronization across data sources
//! 5. Graceful error handling and reconnection
#![allow(unused_crate_dependencies)]

use chrono::{DateTime, Utc, Timelike};
use data::{
    features::{FeatureVector, TechnicalIndicators, MicrostructureAnalyzer, TemporalFeatures, PricePoint, QuoteData, TradeData, TradeDirection},
    providers::databento::{DatabentoHistoricalProvider, DatabentoConfig},
    providers::benzinga::{BenzingaHistoricalProvider, BenzingaConfig, NewsEvent},
    training_pipeline::{TechnicalIndicatorsConfig, MicrostructureConfig, MACDConfig},
    types::{MarketDataEvent, QuoteEvent, TradeEvent, Subscription, DataType, ConnectionEvent, ConnectionStatus},
    DataManager, DataConfig, DataSettings,
};
use common::prelude::*;
use common::events::OrderEvent;
use rust_decimal::Decimal;
use num_traits::FromPrimitive; // For Decimal::from_f64
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::{broadcast, mpsc, Mutex, RwLock};
use tokio::time::timeout;
use tracing::{debug, info, warn, error};

/// Mock Databento client for testing
pub struct MockDatabentoClient {
    config: DatabentoConfig,
    market_data_tx: Option<mpsc::UnboundedSender<MarketDataEvent>>,
    subscriptions: Arc<Mutex<Vec<Subscription>>>,
    connection_status: Arc<RwLock<ConnectionStatus>>,
    should_fail: Arc<RwLock<bool>>,
}

impl MockDatabentoClient {
    pub fn new(config: DatabentoConfig) -> Self {
        Self {
            config,
            market_data_tx: None,
            subscriptions: Arc::new(Mutex::new(Vec::new())),
            connection_status: Arc::new(RwLock::new(ConnectionStatus::Disconnected)),
            should_fail: Arc::new(RwLock::new(false)),
        }
    }

    pub async fn start_websocket(&mut self) -> anyhow::Result<mpsc::UnboundedReceiver<MarketDataEvent>> {
        let should_fail = *self.should_fail.read().await;
        if should_fail {
            return Err(anyhow::anyhow!("Mock connection failure"));
        }

        let (tx, rx) = mpsc::unbounded_channel();
        self.market_data_tx = Some(tx);
        *self.connection_status.write().await = ConnectionStatus::Connected;

        info!("Mock Databento WebSocket connection started");
        Ok(rx)
    }

    pub async fn subscribe(&self, subscription: &Subscription) -> anyhow::Result<()> {
        let mut subs = self.subscriptions.lock().await;
        subs.push(subscription.clone());
        info!("Mock Databento subscription added: {:?}", subscription.symbols);
        Ok(())
    }

    pub async fn emit_market_data(&self, event: MarketDataEvent) -> anyhow::Result<()> {
        if let Some(tx) = &self.market_data_tx {
            tx.send(event)
                .map_err(|e| anyhow::anyhow!("Failed to emit market data: {}", e))?;
        }
        Ok(())
    }

    pub async fn set_should_fail(&self, should_fail: bool) {
        *self.should_fail.write().await = should_fail;
    }

    pub async fn get_connection_status(&self) -> ConnectionStatus {
        *self.connection_status.read().await
    }
}

/// Mock broker client for testing order/position updates
pub struct MockBrokerClient {
    order_event_tx: Option<mpsc::UnboundedSender<OrderEvent>>,
    connection_status: Arc<RwLock<ConnectionStatus>>,
    positions: Arc<RwLock<HashMap<String, Position>>>,
    should_fail: Arc<RwLock<bool>>,
}

impl MockBrokerClient {
    pub fn new() -> Self {
        Self {
            order_event_tx: None,
            connection_status: Arc::new(RwLock::new(ConnectionStatus::Disconnected)),
            positions: Arc::new(RwLock::new(HashMap::new())),
            should_fail: Arc::new(RwLock::new(false)),
        }
    }

    pub async fn connect(&mut self) -> anyhow::Result<mpsc::UnboundedReceiver<OrderEvent>> {
        let should_fail = *self.should_fail.read().await;
        if should_fail {
            return Err(anyhow::anyhow!("Mock broker connection failure"));
        }

        let (tx, rx) = mpsc::unbounded_channel();
        self.order_event_tx = Some(tx);
        *self.connection_status.write().await = ConnectionStatus::Connected;

        info!("Mock broker client connected");
        Ok(rx)
    }

    pub async fn emit_order_event(&self, event: OrderEvent) -> anyhow::Result<()> {
        if let Some(tx) = &self.order_event_tx {
            tx.send(event)
                .map_err(|e| anyhow::anyhow!("Failed to emit order event: {}", e))?;
        }
        Ok(())
    }

    pub async fn update_position(&self, symbol: String, position: Position) {
        let mut positions = self.positions.write().await;
        positions.insert(symbol, position);
    }

    pub async fn set_should_fail(&self, should_fail: bool) {
        *self.should_fail.write().await = should_fail;
    }

    pub async fn get_connection_status(&self) -> ConnectionStatus {
        *self.connection_status.read().await
    }
}

/// Unified feature extractor that processes both market data and broker events
pub struct UnifiedFeatureExtractor {
    technical_indicators: TechnicalIndicators,
    microstructure_analyzer: MicrostructureAnalyzer,
    feature_cache: Arc<RwLock<HashMap<String, FeatureVector>>>,
    symbol_mapping: HashMap<String, String>, // provider_symbol -> normalized_symbol
}

impl UnifiedFeatureExtractor {
    pub fn new() -> Self {
        let technical_config = TechnicalIndicatorsConfig {
            ma_periods: vec![5, 10, 20],
            rsi_periods: vec![14],
            bollinger_periods: vec![20],
            macd: MACDConfig {
                fast_period: 12,
                slow_period: 26,
                signal_period: 9,
            },
            volume_indicators: true,
        };

        let microstructure_config = MicrostructureConfig {
            bid_ask_spread: true,
            volume_imbalance: true,
            price_impact: true,
            kyle_lambda: false,
            amihud_ratio: true,
            roll_spread: true,
        };

        let mut symbol_mapping = HashMap::new();
        // Example symbol mappings between providers
        symbol_mapping.insert("AAPL".to_string(), "AAPL".to_string());
        symbol_mapping.insert("GOOGL".to_string(), "GOOGL".to_string());
        symbol_mapping.insert("MSFT".to_string(), "MSFT".to_string());
        // Databento uses standard formats
        symbol_mapping.insert("BTC-USD".to_string(), "BTC-USD".to_string());

        Self {
            technical_indicators: TechnicalIndicators::new(technical_config),
            microstructure_analyzer: MicrostructureAnalyzer::new(microstructure_config),
            feature_cache: Arc::new(RwLock::new(HashMap::new())),
            symbol_mapping,
        }
    }

    /// Process market data event and extract features
    pub async fn process_market_data(&mut self, event: &MarketDataEvent) -> anyhow::Result<Option<FeatureVector>> {
        let symbol = self.normalize_symbol(event.symbol());
        let timestamp = event.timestamp().unwrap_or_else(Utc::now);

        match event {
            MarketDataEvent::Trade(trade) => {
                self.process_trade_event(&symbol, trade).await?;
            }
            MarketDataEvent::Quote(quote) => {
                self.process_quote_event(&symbol, quote).await?;
            }
            _ => {
                // Handle other event types as needed
            }
        }

        // Generate feature vector
        let feature_vector = self.generate_feature_vector(&symbol, timestamp).await?;
        
        // Cache the feature vector
        let mut cache = self.feature_cache.write().await;
        cache.insert(format!("{}_{}", symbol, timestamp.timestamp_millis()), feature_vector.clone());

        Ok(Some(feature_vector))
    }

    /// Process broker event (positions, orders, etc.)
    pub async fn process_broker_event(&mut self, event: &OrderEvent) -> anyhow::Result<()> {
        // Process broker events to update position context for features
        info!("Processing broker event: {:?}", event);
        // Implementation would update position state that affects feature calculation
        Ok(())
    }

    async fn process_trade_event(&mut self, symbol: &str, trade: &TradeEvent) -> anyhow::Result<()> {
        // Update technical indicators with trade data
        let price_point = PricePoint {
            timestamp: trade.timestamp,
            open: trade.price.to_f64(),
            high: trade.price.to_f64(),
            low: trade.price.to_f64(),
            close: trade.price.to_f64(),
        };
        self.technical_indicators.update_price(symbol, price_point);

        // Update microstructure analyzer with trade data
        let trade_data = TradeData {
            timestamp: trade.timestamp,
            price: trade.price.to_f64(),
            size: trade.size.to_f64(),
            direction: TradeDirection::Unknown, // Would need to determine from market data
        };
        self.microstructure_analyzer.update_trade(symbol, trade_data);

        Ok(())
    }

    async fn process_quote_event(&mut self, symbol: &str, quote: &QuoteEvent) -> anyhow::Result<()> {
        // Update microstructure analyzer with quote data
        if let (Some(bid), Some(ask), Some(bid_size), Some(ask_size)) = 
            (quote.bid, quote.ask, quote.bid_size, quote.ask_size) {
            let quote_data = QuoteData {
                timestamp: quote.timestamp,
                bid: bid.to_f64(),
                ask: ask.to_f64(),
                bid_size: bid_size.to_f64(),
                ask_size: ask_size.to_f64(),
            };
            self.microstructure_analyzer.update_quote(symbol, quote_data);
        }

        Ok(())
    }

    async fn generate_feature_vector(&self, symbol: &str, timestamp: DateTime<Utc>) -> anyhow::Result<FeatureVector> {
        let mut features = HashMap::new();

        // Extract technical indicator features
        let tech_features = self.technical_indicators.calculate_features(symbol);
        for (key, value) in tech_features {
            features.insert(format!("tech_{}", key), value);
        }

        // Extract microstructure features
        let micro_features = self.microstructure_analyzer.calculate_features(symbol);
        for (key, value) in micro_features {
            features.insert(format!("micro_{}", key), value);
        }

        // Extract temporal features
        let temporal_features = TemporalFeatures::extract_features(timestamp);
        for (key, value) in temporal_features {
            features.insert(format!("temporal_{}", key), value);
        }

        Ok(FeatureVector {
            timestamp,
            symbol: symbol.to_string(),
            features,
            metadata: data::features::FeatureMetadata {
                feature_descriptions: HashMap::new(),
                feature_categories: HashMap::new(),
                quality_indicators: HashMap::new(),
            },
        })
    }

    fn normalize_symbol(&self, symbol: &str) -> String {
        self.symbol_mapping
            .get(symbol)
            .cloned()
            .unwrap_or_else(|| symbol.to_string())
    }

    /// Get cached feature vector for testing consistency
    pub async fn get_cached_features(&self, symbol: &str, timestamp_millis: i64) -> Option<FeatureVector> {
        let cache = self.feature_cache.read().await;
        cache.get(&format!("{}_{}", symbol, timestamp_millis)).cloned()
    }
}

/// Test data generator for consistent testing
pub struct TestDataGenerator {
    base_timestamp: DateTime<Utc>,
    sequence: u64,
}

impl TestDataGenerator {
    pub fn new() -> Self {
        Self {
            base_timestamp: Utc::now(),
            sequence: 0,
        }
    }

    pub fn generate_trade_event(&mut self, symbol: &str, price: f64, size: f64) -> MarketDataEvent {
        let timestamp = self.base_timestamp + chrono::Duration::milliseconds(self.sequence as i64 * 100);
        self.sequence += 1;

        MarketDataEvent::Trade(TradeEvent {
            symbol: symbol.to_string(),
            price: Decimal::from_f64_retain(price).unwrap(),
            size: Decimal::from_f64_retain(size).unwrap(),
            trade_id: Some(format!("trade_{}", self.sequence)),
            exchange: Some("NASDAQ".to_string()),
            conditions: vec![],
            timestamp,
        })
    }

    pub fn generate_quote_event(&mut self, symbol: &str, bid: f64, ask: f64, bid_size: f64, ask_size: f64) -> MarketDataEvent {
        let timestamp = self.base_timestamp + chrono::Duration::milliseconds(self.sequence as i64 * 100);
        self.sequence += 1;

        MarketDataEvent::Quote(QuoteEvent {
            symbol: symbol.to_string(),
            bid: Some(Decimal::from_f64_retain(bid).unwrap()),
            ask: Some(Decimal::from_f64_retain(ask).unwrap()),
            bid_size: Some(Decimal::from_f64_retain(bid_size).unwrap()),
            ask_size: Some(Decimal::from_f64_retain(ask_size).unwrap()),
            exchange: Some("NASDAQ".to_string()),
            timestamp,
        })
    }

    pub fn generate_order_event(&mut self, symbol: &str, order_type: OrderType, side: OrderSide, quantity: f64, price: f64) -> OrderEvent {
        let timestamp = self.base_timestamp + chrono::Duration::milliseconds(self.sequence as i64 * 100);
        self.sequence += 1;

        OrderEvent {
            event_id: format!("order_{}", self.sequence),
            timestamp,
            order_id: format!("ORD_{}", self.sequence),
            symbol: symbol.to_string(),
            side,
            order_type,
            quantity: Decimal::from_f64_retain(quantity).unwrap(),
            price: Some(Decimal::from_f64_retain(price).unwrap()),
            status: OrderStatus::New,
            filled_quantity: Some(Decimal::ZERO),
            remaining_quantity: Some(Decimal::from_f64_retain(quantity).unwrap()),
            avg_fill_price: None,
            commission: Some(Decimal::ZERO),
            account_id: "TEST_ACCOUNT".to_string(),
            strategy_id: Some("TEST_STRATEGY".to_string()),
            metadata: HashMap::new(),
        }
    }

    pub fn get_current_timestamp(&self) -> DateTime<Utc> {
        self.base_timestamp + chrono::Duration::milliseconds(self.sequence as i64 * 100)
    }
}

// Test modules

#[cfg(test)]
mod tests {
    use super::*;
    use tokio::time::{sleep, Duration};

    /// Test 1: End-to-End Data Flow Test
    ///
    /// Verifies that market data flows from Databento through DataManager to feature extraction
    #[tokio::test]
    async fn test_end_to_end_data_flow() {
        tracing_subscriber::fmt::init();
        
        info!("Starting end-to-end data flow test");

        // Setup mock providers
        let databento_config = DatabentoConfig::default();
        let mut mock_databento = MockDatabentoClient::new(databento_config.clone());
        
        // Setup DataManager with mocked providers
        let data_config = DataConfig {
            interactive_brokers: None,
            settings: DataSettings::default(),
        };
        
        let mut data_manager = DataManager::new(data_config).await.expect("Failed to create DataManager");
        
        // Setup feature extractor
        let mut feature_extractor = UnifiedFeatureExtractor::new();
        
        // Setup test data generator
        let mut test_data = TestDataGenerator::new();
        
        // Subscribe to market data events from DataManager
        let mut market_data_rx = data_manager.subscribe_market_data_events();
        
        // Start the data flow simulation
        let _receiver = mock_databento.start_websocket().await.expect("Failed to start mock WebSocket");
        
        // Subscribe to AAPL data
        let subscription = Subscription::trades(vec!["AAPL".to_string()]);
        mock_databento.subscribe(&subscription).await.expect("Failed to subscribe");
        
        // Generate and emit test events
        let trade_event = test_data.generate_trade_event("AAPL", 150.0, 100.0);
        mock_databento.emit_market_data(trade_event.clone()).await.expect("Failed to emit trade data");
        
        // Process the event through feature extraction
        let feature_result = feature_extractor.process_market_data(&trade_event).await;
        assert!(feature_result.is_ok(), "Feature extraction should succeed");
        
        let features = feature_result.unwrap();
        assert!(features.is_some(), "Should generate features");
        
        let feature_vector = features.unwrap();
        assert_eq!(feature_vector.symbol, "AAPL");
        assert!(!feature_vector.features.is_empty(), "Should have extracted features");
        
        // Verify temporal features are present
        assert!(feature_vector.features.contains_key("temporal_hour"));
        assert!(feature_vector.features.contains_key("temporal_weekday"));
        
        info!("End-to-end data flow test completed successfully");
    }

    /// Test 2: Multi-Provider Synchronization Test
    ///
    /// Tests synchronization between market data and broker events
    #[tokio::test]
    async fn test_multi_provider_synchronization() {
        tracing_subscriber::fmt::init();
        
        info!("Starting multi-provider synchronization test");

        // Setup mock providers
        let mut mock_databento = MockDatabentoClient::new(DatabentoConfig::default());
        let mut mock_broker = MockBrokerClient::new();
        
        // Setup feature extractor
        let mut feature_extractor = UnifiedFeatureExtractor::new();
        let mut test_data = TestDataGenerator::new();
        
        // Start connections
        let _market_rx = mock_databento.start_websocket().await.expect("Failed to start databento");
        let _order_rx = mock_broker.connect().await.expect("Failed to connect broker");
        
        // Generate synchronized events
        let base_timestamp = test_data.get_current_timestamp();
        
        // Market data event
        let trade_event = test_data.generate_trade_event("AAPL", 150.0, 100.0);
        
        // Corresponding broker event (order fill at same price)
        let order_event = test_data.generate_order_event("AAPL", OrderType::Market, OrderSide::Buy, 100.0, 150.0);
        
        // Process both events
        let market_result = feature_extractor.process_market_data(&trade_event).await;
        assert!(market_result.is_ok(), "Market data processing should succeed");
        
        let broker_result = feature_extractor.process_broker_event(&order_event).await;
        assert!(broker_result.is_ok(), "Broker event processing should succeed");
        
        // Verify timestamp synchronization
        let market_timestamp = trade_event.timestamp().unwrap();
        let order_timestamp = order_event.timestamp;
        
        let time_diff = (market_timestamp - order_timestamp).num_milliseconds().abs();
        assert!(time_diff <= 1000, "Events should be synchronized within 1 second, got {} ms", time_diff);
        
        info!("Multi-provider synchronization test completed successfully");
    }

    /// Test 3: Feature Extraction Consistency (No Training/Serving Skew)
    ///
    /// Ensures identical features are generated for the same input data
    #[tokio::test]
    async fn test_feature_extraction_consistency() {
        tracing_subscriber::fmt::init();
        
        info!("Starting feature extraction consistency test");

        let mut feature_extractor1 = UnifiedFeatureExtractor::new();
        let mut feature_extractor2 = UnifiedFeatureExtractor::new();
        let mut test_data = TestDataGenerator::new();
        
        // Generate identical test data
        let trade_event = test_data.generate_trade_event("AAPL", 150.0, 100.0);
        let quote_event = test_data.generate_quote_event("AAPL", 149.99, 150.01, 500.0, 600.0);
        
        // Process same events through both extractors
        let result1a = feature_extractor1.process_market_data(&trade_event).await.expect("Extractor 1 trade failed");
        let result1b = feature_extractor1.process_market_data(&quote_event).await.expect("Extractor 1 quote failed");
        
        let result2a = feature_extractor2.process_market_data(&trade_event).await.expect("Extractor 2 trade failed");
        let result2b = feature_extractor2.process_market_data(&quote_event).await.expect("Extractor 2 quote failed");
        
        // Compare feature vectors
        if let (Some(features1), Some(features2)) = (result1a, result2a) {
            assert_eq!(features1.symbol, features2.symbol, "Symbols should match");
            
            // Compare feature values (allowing for small floating point differences)
            for (key, &value1) in &features1.features {
                if let Some(&value2) = features2.features.get(key) {
                    let diff = (value1 - value2).abs();
                    assert!(diff < 1e-10, "Feature {} should be identical: {} vs {}", key, value1, value2);
                } else {
                    panic!("Feature {} missing in second extractor", key);
                }
            }
            
            assert_eq!(features1.features.len(), features2.features.len(), "Feature counts should match");
        }
        
        // Test with multiple data points to verify consistency over time
        for i in 0..5 {
            let price = 150.0 + i as f64 * 0.1;
            let event = test_data.generate_trade_event("AAPL", price, 100.0);
            
            let f1 = feature_extractor1.process_market_data(&event).await.expect("Failed to process");
            let f2 = feature_extractor2.process_market_data(&event).await.expect("Failed to process");
            
            if let (Some(fv1), Some(fv2)) = (f1, f2) {
                // Check temporal features are identical
                let temp1 = fv1.features.get("temporal_hour").unwrap();
                let temp2 = fv2.features.get("temporal_hour").unwrap();
                assert_eq!(temp1, temp2, "Temporal features should be identical");
            }
        }
        
        info!("Feature extraction consistency test completed successfully");
    }

    /// Test 4: Symbol Mapping Between Providers
    ///
    /// Tests that symbols are correctly normalized between different provider formats
    #[tokio::test]
    async fn test_symbol_mapping() {
        tracing_subscriber::fmt::init();
        
        info!("Starting symbol mapping test");

        let mut feature_extractor = UnifiedFeatureExtractor::new();
        let mut test_data = TestDataGenerator::new();
        
        // Test different symbol formats
        let test_cases = vec![
            ("AAPL", "AAPL"),           // Standard equity
            ("BTC-USD", "BTC-USD"),    // Crypto with standard format
            ("GOOGL", "GOOGL"),         // Another equity
        ];
        
        for (provider_symbol, expected_normalized) in test_cases {
            let trade_event = test_data.generate_trade_event(provider_symbol, 100.0, 50.0);
            
            let result = feature_extractor.process_market_data(&trade_event).await.expect("Failed to process");
            
            if let Some(feature_vector) = result {
                assert_eq!(feature_vector.symbol, expected_normalized, 
                    "Symbol {} should be normalized to {}", provider_symbol, expected_normalized);
            }
        }
        
        // Test symbol mapping consistency
        let btc_event1 = test_data.generate_trade_event("BTC-USD", 50000.0, 0.1);
        let btc_event2 = test_data.generate_quote_event("BTC-USD", 49999.0, 50001.0, 0.5, 0.6);
        
        let result1 = feature_extractor.process_market_data(&btc_event1).await.expect("Failed to process BTC trade");
        let result2 = feature_extractor.process_market_data(&btc_event2).await.expect("Failed to process BTC quote");
        
        if let (Some(fv1), Some(fv2)) = (result1, result2) {
            assert_eq!(fv1.symbol, "BTC-USD");
            assert_eq!(fv2.symbol, "BTC-USD");
            assert_eq!(fv1.symbol, fv2.symbol, "Symbol normalization should be consistent");
        }
        
        info!("Symbol mapping test completed successfully");
    }

    /// Test 5: Timestamp Synchronization
    ///
    /// Tests proper handling of events with different timestamps
    #[tokio::test]
    async fn test_timestamp_synchronization() {
        tracing_subscriber::fmt::init();
        
        info!("Starting timestamp synchronization test");

        let mut feature_extractor = UnifiedFeatureExtractor::new();
        let mut test_data = TestDataGenerator::new();
        
        // Generate events with specific timestamp ordering
        let base_time = Utc::now();
        let events = vec![
            (base_time, "AAPL", 150.0),
            (base_time + chrono::Duration::milliseconds(100), "AAPL", 150.1),
            (base_time + chrono::Duration::milliseconds(50), "AAPL", 149.9), // Out of order
            (base_time + chrono::Duration::milliseconds(200), "AAPL", 150.2),
        ];
        
        let mut processed_timestamps = Vec::new();
        
        for (timestamp, symbol, price) in events {
            // Manually create event with specific timestamp
            let trade_event = MarketDataEvent::Trade(TradeEvent {
                symbol: symbol.to_string(),
                price: Decimal::from_f64_retain(price).unwrap(),
                size: Decimal::from_f64_retain(100.0).unwrap(),
                trade_id: Some(format!("trade_{}", timestamp.timestamp_millis())),
                exchange: Some("NASDAQ".to_string()),
                conditions: vec![],
                timestamp,
            });
            
            let result = feature_extractor.process_market_data(&trade_event).await.expect("Failed to process");
            
            if let Some(feature_vector) = result {
                processed_timestamps.push(feature_vector.timestamp);
                
                // Verify timestamp is preserved in feature vector
                assert_eq!(feature_vector.timestamp, timestamp, "Timestamp should be preserved");
                
                // Verify temporal features reflect correct timestamp
                let hour_feature = feature_vector.features.get("temporal_hour").unwrap();
                let expected_hour = timestamp.hour() as f64;
                assert_eq!(*hour_feature, expected_hour, "Temporal hour should match timestamp");
            }
        }
        
        // Verify all timestamps were processed
        assert_eq!(processed_timestamps.len(), 4, "Should process all events");
        
        info!("Timestamp synchronization test completed successfully");
    }

    /// Test 6: Error Handling and Reconnection
    ///
    /// Tests graceful handling of provider disconnections and reconnections
    #[tokio::test]
    async fn test_error_handling_and_reconnection() {
        tracing_subscriber::fmt::init();
        
        info!("Starting error handling and reconnection test");

        let mut mock_databento = MockDatabentoClient::new(DatabentoConfig::default());
        let mut mock_broker = MockBrokerClient::new();
        let mut test_data = TestDataGenerator::new();
        
        // Test initial connection
        assert_eq!(mock_databento.get_connection_status().await, ConnectionStatus::Disconnected);
        assert_eq!(mock_broker.get_connection_status().await, ConnectionStatus::Disconnected);
        
        // Successful connection
        let _market_rx = mock_databento.start_websocket().await.expect("Should connect initially");
        let _order_rx = mock_broker.connect().await.expect("Should connect initially");
        
        assert_eq!(mock_databento.get_connection_status().await, ConnectionStatus::Connected);
        assert_eq!(mock_broker.get_connection_status().await, ConnectionStatus::Connected);
        
        // Test successful data processing
        let trade_event = test_data.generate_trade_event("AAPL", 150.0, 100.0);
        mock_databento.emit_market_data(trade_event).await.expect("Should emit successfully");
        
        // Simulate connection failure
        mock_databento.set_should_fail(true).await;
        mock_broker.set_should_fail(true).await;
        
        // Test that reconnection attempts fail appropriately
        let databento_reconnect_result = mock_databento.start_websocket().await;
        let broker_reconnect_result = mock_broker.connect().await;
        
        assert!(databento_reconnect_result.is_err(), "Should fail to reconnect Databento");
        assert!(broker_reconnect_result.is_err(), "Should fail to reconnect broker");
        
        // Restore connection capability
        mock_databento.set_should_fail(false).await;
        mock_broker.set_should_fail(false).await;
        
        // Test successful reconnection
        let _market_rx_new = mock_databento.start_websocket().await.expect("Should reconnect Databento");
        let _order_rx_new = mock_broker.connect().await.expect("Should reconnect broker");
        
        assert_eq!(mock_databento.get_connection_status().await, ConnectionStatus::Connected);
        assert_eq!(mock_broker.get_connection_status().await, ConnectionStatus::Connected);
        
        // Verify data processing works after reconnection
        let trade_event_after = test_data.generate_trade_event("AAPL", 151.0, 200.0);
        mock_databento.emit_market_data(trade_event_after).await.expect("Should emit after reconnection");
        
        info!("Error handling and reconnection test completed successfully");
    }

    /// Test 7: Performance and Latency Test
    ///
    /// Tests that the system can handle high-frequency data without significant delays
    #[tokio::test]
    async fn test_performance_and_latency() {
        tracing_subscriber::fmt::init();
        
        info!("Starting performance and latency test");

        let mut feature_extractor = UnifiedFeatureExtractor::new();
        let mut test_data = TestDataGenerator::new();
        let mut mock_databento = MockDatabentoClient::new(DatabentoConfig::default());
        
        let _receiver = mock_databento.start_websocket().await.expect("Failed to start WebSocket");
        
        let num_events = 1000;
        let symbols = vec!["AAPL", "GOOGL", "MSFT", "TSLA"];
        
        let start_time = std::time::Instant::now();
        
        for i in 0..num_events {
            let symbol = symbols[i % symbols.len()];
            let price = 100.0 + (i as f64 * 0.01);
            
            let trade_event = test_data.generate_trade_event(symbol, price, 100.0);
            
            // Measure processing latency
            let process_start = std::time::Instant::now();
            let result = feature_extractor.process_market_data(&trade_event).await;
            let process_duration = process_start.elapsed();
            
            assert!(result.is_ok(), "Event {} should process successfully", i);
            assert!(process_duration.as_millis() < 10, "Processing should be fast, took {} ms", process_duration.as_millis());
            
            // Emit through mock provider for throughput test
            mock_databento.emit_market_data(trade_event).await.expect("Failed to emit");
        }
        
        let total_duration = start_time.elapsed();
        let throughput = num_events as f64 / total_duration.as_secs_f64();
        
        info!("Processed {} events in {} ms", num_events, total_duration.as_millis());
        info!("Throughput: {:.2} events/second", throughput);
        
        // Assert minimum performance requirements
        assert!(throughput > 1000.0, "Should handle at least 1000 events/second, got {:.2}", throughput);
        
        info!("Performance and latency test completed successfully");
    }

    /// Test 8: Historical vs Real-time Consistency
    ///
    /// Ensures features generated from historical data match real-time processing
    #[tokio::test]
    async fn test_historical_vs_realtime_consistency() {
        tracing_subscriber::fmt::init();
        
        info!("Starting historical vs real-time consistency test");

        // Create two identical extractors
        let mut realtime_extractor = UnifiedFeatureExtractor::new();
        let mut historical_extractor = UnifiedFeatureExtractor::new();
        let mut test_data = TestDataGenerator::new();
        
        // Generate a sequence of market events
        let events = vec![
            test_data.generate_trade_event("AAPL", 150.0, 100.0),
            test_data.generate_quote_event("AAPL", 149.98, 150.02, 500.0, 600.0),
            test_data.generate_trade_event("AAPL", 150.1, 150.0),
            test_data.generate_quote_event("AAPL", 150.08, 150.12, 400.0, 550.0),
            test_data.generate_trade_event("AAPL", 150.05, 200.0),
        ];
        
        // Process events in real-time simulation (with delays)
        let mut realtime_features = Vec::new();
        for event in &events {
            let result = realtime_extractor.process_market_data(event).await.expect("Real-time processing failed");
            if let Some(fv) = result {
                realtime_features.push(fv);
            }
            // Simulate real-time delay
            sleep(Duration::from_millis(10)).await;
        }
        
        // Process same events as historical batch (no delays)
        let mut historical_features = Vec::new();
        for event in &events {
            let result = historical_extractor.process_market_data(event).await.expect("Historical processing failed");
            if let Some(fv) = result {
                historical_features.push(fv);
            }
        }
        
        // Compare results
        assert_eq!(realtime_features.len(), historical_features.len(), "Should generate same number of feature vectors");
        
        for (rt_fv, hist_fv) in realtime_features.into_iter().zip(historical_features.into_iter()) {
            assert_eq!(rt_fv.symbol, hist_fv.symbol, "Symbols should match");
            assert_eq!(rt_fv.timestamp, hist_fv.timestamp, "Timestamps should match");
            
            // Compare all features
            for (key, &rt_value) in &rt_fv.features {
                if let Some(&hist_value) = hist_fv.features.get(key) {
                    let diff = (rt_value - hist_value).abs();
                    assert!(diff < 1e-10, "Feature {} should be identical: real-time={}, historical={}", key, rt_value, hist_value);
                } else {
                    panic!("Feature {} missing in historical processing", key);
                }
            }
            
            assert_eq!(rt_fv.features.len(), hist_fv.features.len(), "Feature counts should match");
        }
        
        info!("Historical vs real-time consistency test completed successfully");
    }
}