//! ML Trading Pipeline Integration Tests
//! 
//! Tests comprehensive integration of ML models with trading pipeline.
//! Validates end-to-end flow from market data to ML inference to trading decisions.
//! 
//! Coverage Areas:
//! - Market data preprocessing for ML features
//! - Real-time ML inference pipeline 
//! - ML model predictions to trading signals
//! - Feature engineering and data pipeline
//! - Model performance under market stress
//! - Latency optimization for HFT requirements

use std::sync::Arc;
use std::time::Duration;
use tokio::time::timeout;
use std::collections::HashMap;

// Import core types and modules
use trading_engine::{
    timing::HardwareTimestamp,
    types::prelude::*,
    simd::SimdPriceOps,
};

/// Test result type for safe error handling (no panics)
type TestResult<T> = Result<T, Box<dyn std::error::Error + Send + Sync>>;

/// ML pipeline configuration for testing
#[derive(Debug, Clone)]
pub struct MLPipelineConfig {
    pub feature_window_size: usize,
    pub prediction_horizon_ms: u64,
    pub confidence_threshold: f64,
    pub model_inference_timeout_ms: u64,
    pub max_prediction_latency_ns: u64,
}

impl Default for MLPipelineConfig {
    fn default() -> Self {
        Self {
            feature_window_size: 100,
            prediction_horizon_ms: 1000, // 1 second ahead
            confidence_threshold: 0.7,
            model_inference_timeout_ms: 10, // 10ms max
            max_prediction_latency_ns: 50_000, // 50μs for HFT
        }
    }
}

/// Market data tick for ML processing
#[derive(Debug, Clone)]
pub struct MarketTick {
    pub symbol: String,
    pub price: Decimal,
    pub volume: u64,
    pub bid: Decimal,
    pub ask: Decimal,
    pub bid_size: u64,
    pub ask_size: u64,
    pub timestamp: HardwareTimestamp,
}

impl MarketTick {
    pub fn new(symbol: String, price: Decimal, volume: u64) -> Self {
        let spread = Decimal::new(5, 2); // $0.05 spread
        Self {
            symbol,
            price,
            volume,
            bid: price - spread,
            ask: price + spread,
            bid_size: volume / 2,
            ask_size: volume / 2,
            timestamp: HardwareTimestamp::now(),
        }
    }
}

/// Feature vector for ML models
#[derive(Debug, Clone)]
pub struct FeatureVector {
    pub features: Vec<f32>,
    pub feature_names: Vec<String>,
    pub extraction_latency_ns: u64,
    pub timestamp: HardwareTimestamp,
}

impl FeatureVector {
    pub fn new(features: Vec<f32>, feature_names: Vec<String>, extraction_latency_ns: u64) -> Self {
        Self {
            features,
            feature_names,
            extraction_latency_ns,
            timestamp: HardwareTimestamp::now(),
        }
    }
}

/// ML model prediction result
#[derive(Debug, Clone)]
pub struct MLPrediction {
    pub signal: TradingSignal,
    pub confidence: f64,
    pub probability_distribution: Vec<f64>,
    pub model_name: String,
    pub inference_latency_ns: u64,
    pub timestamp: HardwareTimestamp,
}

#[derive(Debug, Clone)]
pub enum TradingSignal {
    StrongBuy(f64),    // confidence score
    Buy(f64),
    Hold(f64),
    Sell(f64),
    StrongSell(f64),
}

impl TradingSignal {
    pub fn confidence(&self) -> f64 {
        match self {
            TradingSignal::StrongBuy(conf) => *conf,
            TradingSignal::Buy(conf) => *conf,
            TradingSignal::Hold(conf) => *conf,
            TradingSignal::Sell(conf) => *conf,
            TradingSignal::StrongSell(conf) => *conf,
        }
    }
    
    pub fn is_actionable(&self, threshold: f64) -> bool {
        self.confidence() >= threshold
    }
}

/// Feature engineering pipeline
#[derive(Debug)]
pub struct FeatureEngineer {
    pub config: MLPipelineConfig,
    pub price_history: Arc<std::sync::Mutex<Vec<Decimal>>>,
    pub volume_history: Arc<std::sync::Mutex<Vec<u64>>>,
    pub spread_history: Arc<std::sync::Mutex<Vec<Decimal>>>,
}

impl FeatureEngineer {
    pub fn new(config: MLPipelineConfig) -> Self {
        Self {
            config,
            price_history: Arc::new(std::sync::Mutex::new(Vec::new())),
            volume_history: Arc::new(std::sync::Mutex::new(Vec::new())),
            spread_history: Arc::new(std::sync::Mutex::new(Vec::new())),
        }
    }
    
    /// Extract features from market tick with SIMD optimization
    pub async fn extract_features(&self, tick: &MarketTick) -> TestResult<FeatureVector> {
        let start_time = HardwareTimestamp::now();
        
        // Update price history
        if let Ok(mut price_hist) = self.price_history.lock() {
            price_hist.push(tick.price);
            if price_hist.len() > 1000 {
                price_hist.remove(0);
            }
        }
        
        // Update volume history
        if let Ok(mut volume_hist) = self.volume_history.lock() {
            volume_hist.push(tick.volume);
            if volume_hist.len() > 1000 {
                volume_hist.remove(0);
            }
        }
        
        // Calculate spread
        let spread = tick.ask - tick.bid;
        if let Ok(mut spread_hist) = self.spread_history.lock() {
            spread_hist.push(spread);
            if spread_hist.len() > 1000 {
                spread_hist.remove(0);
            }
        }
        
        // Extract technical indicators using SIMD
        let simd_start = HardwareTimestamp::now();
        let features = self.calculate_technical_features(tick).await?;
        let simd_latency = HardwareTimestamp::now().latency_ns(&simd_start);
        
        // SIMD feature calculation should be sub-microsecond
        if simd_latency > 1_000 {
            eprintln!("WARNING: SIMD feature calculation took {}ns, expected <1000ns", simd_latency);
        }
        
        let total_latency = HardwareTimestamp::now().latency_ns(&start_time);
        
        let feature_names = vec![
            "price".to_string(),
            "volume".to_string(),
            "spread".to_string(),
            "price_sma_10".to_string(),
            "price_sma_20".to_string(),
            "volume_sma_10".to_string(),
            "price_momentum".to_string(),
            "volume_momentum".to_string(),
            "spread_normalized".to_string(),
            "volatility_1min".to_string(),
        ];
        
        Ok(FeatureVector::new(features, feature_names, total_latency))
    }
    
    /// Calculate technical features using SIMD operations
    async fn calculate_technical_features(&self, tick: &MarketTick) -> TestResult<Vec<f32>> {
        let mut features = Vec::with_capacity(10);
        
        // Basic features
        features.push(tick.price.to_f32().unwrap_or(0.0));
        features.push(tick.volume as f32);
        features.push((tick.ask - tick.bid).to_f32().unwrap_or(0.0));
        
        // Moving averages using SIMD (simulated)
        let recent_prices = self.get_recent_prices(20).await?;
        if recent_prices.len() >= 10 {
            let simd_ops = if std::arch::is_x86_feature_detected!("avx2") {
                // SAFETY: AVX2 feature detection verified before SIMD operations
                unsafe { Some(SimdPriceOps::new()) }
            } else {
                None
            };

            let sma_10 = if let Some(ref ops) = simd_ops {
                ops.calculate_vwap(&recent_prices[..10], &vec![1.0; 10])
            } else {
                recent_prices[..10].iter().sum::<Decimal>() / Decimal::new(10, 0)
            };
            features.push(sma_10.to_f32().unwrap_or(0.0));

            if recent_prices.len() >= 20 {
                let sma_20 = if let Some(ref ops) = simd_ops {
                    ops.calculate_vwap(&recent_prices, &vec![1.0; recent_prices.len()])
                } else {
                    recent_prices.iter().sum::<Decimal>() / Decimal::new(recent_prices.len() as i64, 0)
                };
                features.push(sma_20.to_f32().unwrap_or(0.0));
            } else {
                features.push(0.0);
            }
        } else {
            features.push(0.0);
            features.push(0.0);
        }
        
        // Volume SMA
        let recent_volumes = self.get_recent_volumes(10).await?;
        if !recent_volumes.is_empty() {
            let volume_avg = recent_volumes.iter().sum::<u64>() as f32 / recent_volumes.len() as f32;
            features.push(volume_avg);
        } else {
            features.push(0.0);
        }
        
        // Momentum indicators
        if recent_prices.len() >= 5 {
            let price_momentum = (recent_prices[0] - recent_prices[4]).to_f32().unwrap_or(0.0);
            features.push(price_momentum);
        } else {
            features.push(0.0);
        }
        
        if recent_volumes.len() >= 5 {
            let volume_momentum = recent_volumes[0] as f32 - recent_volumes[4] as f32;
            features.push(volume_momentum);
        } else {
            features.push(0.0);
        }
        
        // Normalized spread
        let spread_normalized = if tick.price > Decimal::ZERO {
            ((tick.ask - tick.bid) / tick.price).to_f32().unwrap_or(0.0)
        } else {
            0.0
        };
        features.push(spread_normalized);
        
        // Volatility calculation using SIMD
        if recent_prices.len() >= 10 {
            let volatility = self.calculate_volatility(&recent_prices[..10])?;
            features.push(volatility);
        } else {
            features.push(0.0);
        }
        
        Ok(features)
    }
    
    async fn get_recent_prices(&self, count: usize) -> TestResult<Vec<Decimal>> {
        let mut prices = Vec::new();
        // Simulate reading from ring buffer
        // In real implementation, would use actual ring buffer data
        for i in 0..count.min(20) {
            let price = Decimal::new(150_00 + i as i64, 2); // Simulated price data
            prices.push(price);
        }
        Ok(prices)
    }
    
    async fn get_recent_volumes(&self, count: usize) -> TestResult<Vec<u64>> {
        let mut volumes = Vec::new();
        // Simulate reading from ring buffer
        for i in 0..count.min(10) {
            volumes.push(1000 + (i * 100) as u64); // Simulated volume data
        }
        Ok(volumes)
    }
    
    fn calculate_volatility(&self, prices: &[Decimal]) -> TestResult<f32> {
        if prices.is_empty() {
            return Ok(0.0);
        }
        
        let mean = prices.iter().sum::<Decimal>() / Decimal::new(prices.len() as i64, 0);
        let variance = prices.iter()
            .map(|&p| {
                let diff = p - mean;
                (diff * diff).to_f32().unwrap_or(0.0)
            })
            .sum::<f32>() / prices.len() as f32;
        
        Ok(variance.sqrt())
    }
}

/// Mock ML model for testing
#[derive(Debug, Clone)]
pub struct MockMLModel {
    pub model_name: String,
    pub config: MLPipelineConfig,
    pub inference_stats: Arc<std::sync::Mutex<Vec<u64>>>,
}

impl MockMLModel {
    pub fn new(model_name: String, config: MLPipelineConfig) -> Self {
        Self {
            model_name,
            config,
            inference_stats: Arc::new(std::sync::Mutex::new(Vec::new())),
        }
    }
    
    /// Run ML inference on feature vector
    pub async fn predict(&self, features: &FeatureVector) -> TestResult<MLPrediction> {
        let start_time = HardwareTimestamp::now();
        
        // Validate feature vector
        if features.features.len() < 5 {
            return Err(format!(
                "Insufficient features: got {}, expected at least 5", 
                features.features.len()
            ).into());
        }
        
        // Simulate model inference latency (should be <50μs for HFT)
        tokio::time::sleep(Duration::from_nanos(30_000)).await; // 30μs
        
        // Generate prediction based on features
        let prediction = self.generate_prediction(&features.features)?;
        
        let inference_latency = HardwareTimestamp::now().latency_ns(&start_time);
        
        // Record latency statistics
        if let Ok(mut stats) = self.inference_stats.lock() {
            stats.push(inference_latency);
        }
        
        // Validate HFT latency requirement
        if inference_latency > self.config.max_prediction_latency_ns {
            eprintln!("WARNING: ML inference took {}ns, exceeds limit {}ns", 
                inference_latency, self.config.max_prediction_latency_ns);
        }
        
        Ok(MLPrediction {
            signal: prediction.0,
            confidence: prediction.1,
            probability_distribution: prediction.2,
            model_name: self.model_name.clone(),
            inference_latency_ns: inference_latency,
            timestamp: HardwareTimestamp::now(),
        })
    }
    
    fn generate_prediction(&self, features: &[f32]) -> TestResult<(TradingSignal, f64, Vec<f64>)> {
        // Simple heuristic model for testing
        let price_feature = features[0];
        let volume_feature = features[1];
        let momentum_feature = features.get(6).copied().unwrap_or(0.0);
        
        // Calculate signal strength
        let signal_strength = (momentum_feature / price_feature) + (volume_feature / 10000.0);
        let confidence = (signal_strength.abs() * 0.8).min(0.95).max(0.1);
        
        let signal = if signal_strength > 0.05 {
            TradingSignal::Buy(confidence as f64)
        } else if signal_strength < -0.05 {
            TradingSignal::Sell(confidence as f64)
        } else {
            TradingSignal::Hold(confidence as f64)
        };
        
        // Generate probability distribution
        let prob_dist = match signal {
            TradingSignal::Buy(_) => vec![0.1, 0.7, 0.2, 0.0, 0.0],
            TradingSignal::Sell(_) => vec![0.0, 0.0, 0.2, 0.7, 0.1],
            _ => vec![0.0, 0.2, 0.6, 0.2, 0.0],
        };
        
        Ok((signal, confidence as f64, prob_dist))
    }
    
    pub fn get_average_latency(&self) -> TestResult<u64> {
        let stats = self.inference_stats.lock()
            .map_err(|e| format!("Failed to acquire stats lock: {}", e))?;
        
        if stats.is_empty() {
            return Ok(0);
        }
        
        let sum: u64 = stats.iter().sum();
        Ok(sum / stats.len() as u64)
    }
}

/// Trading signal processor
#[derive(Debug)]
pub struct TradingSignalProcessor {
    pub config: MLPipelineConfig,
    pub signal_history: Arc<std::sync::Mutex<Vec<MLPrediction>>>,
}

impl TradingSignalProcessor {
    pub fn new(config: MLPipelineConfig) -> Self {
        Self {
            config,
            signal_history: Arc::new(std::sync::Mutex::new(Vec::new())),
        }
    }
    
    /// Process ML prediction into trading decision
    pub async fn process_signal(&self, prediction: MLPrediction) -> TestResult<TradingDecision> {
        let start_time = HardwareTimestamp::now();
        
        // Store prediction in history
        if let Ok(mut history) = self.signal_history.lock() {
            history.push(prediction.clone());
            if history.len() > 1000 {
                history.remove(0);
            }
        }
        
        // Check if signal meets confidence threshold
        if prediction.confidence < self.config.confidence_threshold {
            return Ok(TradingDecision {
                action: TradingAction::NoAction,
                reason: format!("Confidence {} below threshold {}", 
                    prediction.confidence, self.config.confidence_threshold),
                quantity: Decimal::ZERO,
                confidence: prediction.confidence,
                processing_latency_ns: HardwareTimestamp::now().latency_ns(&start_time),
            });
        }
        
        // Generate trading action based on signal
        let (action, quantity) = match &prediction.signal {
            TradingSignal::StrongBuy(conf) => {
                let qty = Decimal::new((conf * 1000.0) as i64, 0); // Scale by confidence
                (TradingAction::Buy, qty)
            },
            TradingSignal::Buy(conf) => {
                let qty = Decimal::new((conf * 500.0) as i64, 0);
                (TradingAction::Buy, qty)
            },
            TradingSignal::StrongSell(conf) => {
                let qty = Decimal::new((conf * 1000.0) as i64, 0);
                (TradingAction::Sell, qty)
            },
            TradingSignal::Sell(conf) => {
                let qty = Decimal::new((conf * 500.0) as i64, 0);
                (TradingAction::Sell, qty)
            },
            TradingSignal::Hold(_) => {
                (TradingAction::NoAction, Decimal::ZERO)
            },
        };
        
        let processing_latency = HardwareTimestamp::now().latency_ns(&start_time);
        
        Ok(TradingDecision {
            action,
            reason: format!("ML signal: {} with confidence {}", 
                prediction.model_name, prediction.confidence),
            quantity,
            confidence: prediction.confidence,
            processing_latency_ns: processing_latency,
        })
    }
}

#[derive(Debug, Clone)]
pub struct TradingDecision {
    pub action: TradingAction,
    pub reason: String,
    pub quantity: Decimal,
    pub confidence: f64,
    pub processing_latency_ns: u64,
}

#[derive(Debug, Clone)]
pub enum TradingAction {
    Buy,
    Sell,
    NoAction,
}

// =============================================================================
// INTEGRATION TESTS
// =============================================================================

#[tokio::test]
async fn test_market_data_to_features_pipeline() -> TestResult<()> {
    let config = MLPipelineConfig::default();
    let feature_engineer = FeatureEngineer::new(config.clone());
    
    // Test 1: Single tick feature extraction
    let tick = MarketTick::new(
        "AAPL".to_string(),
        Decimal::new(150_75, 2), // $150.75
        2500
    );
    
    let features = feature_engineer.extract_features(&tick).await?;
    
    assert_eq!(features.features.len(), 10, "Should extract 10 features");
    assert!(features.extraction_latency_ns < 10_000, 
        "Feature extraction should be <10μs, got {}ns", features.extraction_latency_ns);
    
    // Validate feature values
    assert!((features.features[0] - 150.75).abs() < 0.01, "Price feature should match tick price");
    assert!((features.features[1] - 2500.0).abs() < 1.0, "Volume feature should match tick volume");
    
    // Test 2: Multiple ticks for moving averages
    let ticks = vec![
        MarketTick::new("AAPL".to_string(), Decimal::new(150_00, 2), 1000),
        MarketTick::new("AAPL".to_string(), Decimal::new(151_00, 2), 1100),
        MarketTick::new("AAPL".to_string(), Decimal::new(152_00, 2), 1200),
        MarketTick::new("AAPL".to_string(), Decimal::new(151_50, 2), 1300),
        MarketTick::new("AAPL".to_string(), Decimal::new(150_75, 2), 1400),
    ];
    
    let mut total_latency = 0u64;
    for tick in ticks {
        let features = feature_engineer.extract_features(&tick).await?;
        total_latency += features.extraction_latency_ns;
        
        assert!(features.extraction_latency_ns < 10_000, 
            "Each feature extraction should be <10μs");
    }
    
    let avg_latency = total_latency / 5;
    assert!(avg_latency < 10_000, 
        "Average feature extraction latency should be <10μs, got {}ns", avg_latency);
    
    println!("✓ Market data to features pipeline test passed (avg latency: {}ns)", avg_latency);
    Ok(())
}

#[tokio::test]
async fn test_ml_inference_pipeline() -> TestResult<()> {
    let config = MLPipelineConfig::default();
    let model = MockMLModel::new("TLOB_Transformer".to_string(), config.clone());
    
    // Test 1: Basic inference
    let features = FeatureVector::new(
        vec![150.75, 2500.0, 0.05, 150.5, 150.2, 2400.0, 0.25, 100.0, 0.0003, 0.15],
        vec!["price".to_string(), "volume".to_string()], // Simplified for test
        5_000 // 5μs feature extraction
    );
    
    let prediction = model.predict(&features).await?;
    
    assert!(prediction.inference_latency_ns < 50_000, 
        "ML inference should be <50μs, got {}ns", prediction.inference_latency_ns);
    assert!(prediction.confidence >= 0.0 && prediction.confidence <= 1.0, 
        "Confidence should be between 0 and 1, got {}", prediction.confidence);
    assert_eq!(prediction.probability_distribution.len(), 5, 
        "Should return 5 probability values");
    
    // Test 2: High-frequency inference
    let num_predictions = 100;
    let mut latencies = Vec::new();
    let start_time = HardwareTimestamp::now();
    
    for i in 0..num_predictions {
        let test_features = FeatureVector::new(
            vec![150.0 + (i as f32 * 0.1), 2500.0, 0.05, 150.5, 150.2, 2400.0, 0.25, 100.0, 0.0003, 0.15],
            vec!["price".to_string()],
            1_000 // Fast feature extraction
        );
        
        let prediction = model.predict(&test_features).await?;
        latencies.push(prediction.inference_latency_ns);
    }
    
    let total_time = HardwareTimestamp::now().latency_ns(&start_time);
    let throughput = (num_predictions as f64 / (total_time as f64 / 1_000_000_000.0)) as u64;
    
    latencies.sort_unstable();
    let p95_latency = latencies[latencies.len() * 95 / 100];
    let avg_latency = model.get_average_latency()?;
    
    assert!(p95_latency < 50_000, 
        "P95 inference latency should be <50μs, got {}ns", p95_latency);
    assert!(throughput > 1_000, 
        "ML inference throughput should be >1000/sec, got {}/sec", throughput);
    
    println!("✓ ML inference pipeline test passed: {} predictions/sec, P95: {}ns, avg: {}ns", 
        throughput, p95_latency, avg_latency);
    Ok(())
}

#[tokio::test]
async fn test_signal_processing_pipeline() -> TestResult<()> {
    let config = MLPipelineConfig::default();
    let signal_processor = TradingSignalProcessor::new(config.clone());
    
    // Test 1: High confidence buy signal
    let buy_prediction = MLPrediction {
        signal: TradingSignal::Buy(0.85),
        confidence: 0.85,
        probability_distribution: vec![0.0, 0.8, 0.2, 0.0, 0.0],
        model_name: "Test_Model".to_string(),
        inference_latency_ns: 30_000,
        timestamp: HardwareTimestamp::now(),
    };
    
    let decision = signal_processor.process_signal(buy_prediction).await?;
    
    assert!(matches!(decision.action, TradingAction::Buy), "Should generate buy action");
    assert!(decision.quantity > Decimal::ZERO, "Should have positive quantity");
    assert!(decision.processing_latency_ns < 5_000, 
        "Signal processing should be <5μs, got {}ns", decision.processing_latency_ns);
    
    // Test 2: Low confidence signal (should be filtered)
    let low_conf_prediction = MLPrediction {
        signal: TradingSignal::Buy(0.5),
        confidence: 0.5, // Below 0.7 threshold
        probability_distribution: vec![0.1, 0.5, 0.4, 0.0, 0.0],
        model_name: "Test_Model".to_string(),
        inference_latency_ns: 25_000,
        timestamp: HardwareTimestamp::now(),
    };
    
    let decision = signal_processor.process_signal(low_conf_prediction).await?;
    
    assert!(matches!(decision.action, TradingAction::NoAction), 
        "Low confidence signal should result in no action");
    assert_eq!(decision.quantity, Decimal::ZERO, "Should have zero quantity");
    
    // Test 3: Sell signal
    let sell_prediction = MLPrediction {
        signal: TradingSignal::Sell(0.9),
        confidence: 0.9,
        probability_distribution: vec![0.0, 0.0, 0.1, 0.9, 0.0],
        model_name: "Test_Model".to_string(),
        inference_latency_ns: 35_000,
        timestamp: HardwareTimestamp::now(),
    };
    
    let decision = signal_processor.process_signal(sell_prediction).await?;
    
    assert!(matches!(decision.action, TradingAction::Sell), "Should generate sell action");
    assert!(decision.quantity > Decimal::ZERO, "Should have positive quantity");
    
    println!("✓ Signal processing pipeline test passed");
    Ok(())
}

#[tokio::test]
async fn test_end_to_end_ml_trading_flow() -> TestResult<()> {
    let config = MLPipelineConfig::default();
    let feature_engineer = FeatureEngineer::new(config.clone());
    let ml_model = MockMLModel::new("End2End_Model".to_string(), config.clone());
    let signal_processor = TradingSignalProcessor::new(config.clone());
    
    // Simulate realistic market data stream
    let market_ticks = vec![
        MarketTick::new("AAPL".to_string(), Decimal::new(150_00, 2), 1000),
        MarketTick::new("AAPL".to_string(), Decimal::new(150_25, 2), 1200),
        MarketTick::new("AAPL".to_string(), Decimal::new(150_50, 2), 1400),
        MarketTick::new("AAPL".to_string(), Decimal::new(150_75, 2), 1600),
        MarketTick::new("AAPL".to_string(), Decimal::new(151_00, 2), 1800),
    ];
    
    let mut end_to_end_latencies = Vec::new();
    let mut trading_decisions = Vec::new();
    
    for tick in market_ticks {
        let pipeline_start = HardwareTimestamp::now();
        
        // Step 1: Feature extraction
        let features = feature_engineer.extract_features(&tick).await?;
        
        // Step 2: ML inference
        let prediction = ml_model.predict(&features).await?;
        
        // Step 3: Signal processing
        let decision = signal_processor.process_signal(prediction).await?;
        
        let end_to_end_latency = HardwareTimestamp::now().latency_ns(&pipeline_start);
        
        end_to_end_latencies.push(end_to_end_latency);
        trading_decisions.push(decision);
        
        // Validate end-to-end latency for HFT requirements
        assert!(end_to_end_latency < 100_000, 
            "End-to-end ML pipeline should be <100μs, got {}ns", end_to_end_latency);
    }
    
    // Analyze results
    let avg_latency = end_to_end_latencies.iter().sum::<u64>() / end_to_end_latencies.len() as u64;
    let max_latency = *end_to_end_latencies.iter().max().unwrap_or(&0);
    
    let actionable_decisions = trading_decisions.iter()
        .filter(|d| !matches!(d.action, TradingAction::NoAction))
        .count();
    
    assert!(avg_latency < 80_000, 
        "Average end-to-end latency should be <80μs, got {}ns", avg_latency);
    assert!(max_latency < 100_000, 
        "Max end-to-end latency should be <100μs, got {}ns", max_latency);
    assert!(actionable_decisions > 0, 
        "Should generate at least one actionable trading decision");
    
    println!("✓ End-to-end ML trading flow test passed: avg {}ns, max {}ns, {} actionable decisions", 
        avg_latency, max_latency, actionable_decisions);
    Ok(())
}

#[tokio::test]
async fn test_ml_pipeline_under_stress() -> TestResult<()> {
    let config = MLPipelineConfig::default();
    let feature_engineer = Arc::new(FeatureEngineer::new(config.clone()));
    let ml_model = Arc::new(MockMLModel::new("Stress_Test_Model".to_string(), config.clone()));
    let signal_processor = Arc::new(TradingSignalProcessor::new(config.clone()));
    
    // Generate high-frequency market data
    let num_ticks = 1000;
    let mut handles = Vec::new();
    let start_time = HardwareTimestamp::now();
    
    for i in 0..num_ticks {
        let feature_engineer = feature_engineer.clone();
        let ml_model = ml_model.clone();
        let signal_processor = signal_processor.clone();
        
        let handle = tokio::spawn(async move {
            let tick = MarketTick::new(
                format!("STOCK_{}", i % 10), // 10 different symbols
                Decimal::new(150_00 + (i % 100) as i64, 2),
                1000 + (i % 500) as u64
            );
            
            let pipeline_start = HardwareTimestamp::now();
            
            // Full ML pipeline
            let features = feature_engineer.extract_features(&tick).await?;
            let prediction = ml_model.predict(&features).await?;
            let decision = signal_processor.process_signal(prediction).await?;
            
            let pipeline_latency = HardwareTimestamp::now().latency_ns(&pipeline_start);
            
            Ok::<_, Box<dyn std::error::Error + Send + Sync>>((
                pipeline_latency,
                decision.action,
                prediction.confidence
            ))
        });
        
        handles.push(handle);
    }
    
    // Process all concurrent ML pipelines
    let mut results = Vec::new();
    for handle in handles {
        results.push(handle.await);
    }
    let total_time = HardwareTimestamp::now().latency_ns(&start_time);
    
    let mut successful_pipelines = 0;
    let mut pipeline_latencies = Vec::new();
    let mut actionable_count = 0;
    
    for result in results {
        match result {
            Ok(Ok((latency, action, confidence))) => {
                successful_pipelines += 1;
                pipeline_latencies.push(latency);
                
                if !matches!(action, TradingAction::NoAction) {
                    actionable_count += 1;
                }
                
                if confidence < 0.0 || confidence > 1.0 {
                    eprintln!("WARNING: Invalid confidence value: {}", confidence);
                }
            }
            Ok(Err(e)) => eprintln!("Pipeline failed: {}", e),
            Err(e) => eprintln!("Task failed: {}", e),
        }
    }
    
    // Calculate performance metrics
    let throughput = (successful_pipelines as f64 / (total_time as f64 / 1_000_000_000.0)) as u64;
    
    pipeline_latencies.sort_unstable();
    let p95_latency = pipeline_latencies.get(pipeline_latencies.len() * 95 / 100).copied().unwrap_or(0);
    let avg_latency = pipeline_latencies.iter().sum::<u64>() / pipeline_latencies.len().max(1) as u64;
    
    // Validate HFT performance under stress
    assert!(p95_latency < 100_000, 
        "P95 ML pipeline latency should be <100μs under stress, got {}ns", p95_latency);
    assert!(throughput > 500, 
        "ML pipeline throughput should be >500/sec under stress, got {}/sec", throughput);
    assert!(successful_pipelines >= num_ticks * 90 / 100, 
        "At least 90% of pipelines should succeed under stress, got {}%", 
        successful_pipelines * 100 / num_ticks);
    
    let actionable_rate = actionable_count as f64 / successful_pipelines as f64;
    assert!(actionable_rate > 0.1, 
        "At least 10% of signals should be actionable, got {:.1}%", actionable_rate * 100.0);
    
    println!("✓ ML pipeline stress test passed: {} pipelines/sec, P95: {}ns, {:.1}% actionable", 
        throughput, p95_latency, actionable_rate * 100.0);
    Ok(())
}

// =============================================================================
// INTEGRATION TEST RUNNER
// =============================================================================

#[tokio::test]
async fn run_all_ml_trading_pipeline_tests() -> TestResult<()> {
    println!("=== ML TRADING PIPELINE INTEGRATION TEST SUITE ===");
    
    let test_timeout = Duration::from_secs(120);
    
    // Run all integration tests with timeout protection
    timeout(test_timeout, async { test_market_data_to_features_pipeline().await }).await??;
    timeout(test_timeout, async { test_ml_inference_pipeline().await }).await??;
    timeout(test_timeout, async { test_signal_processing_pipeline().await }).await??;
    timeout(test_timeout, async { test_end_to_end_ml_trading_flow().await }).await??;
    timeout(test_timeout, async { test_ml_pipeline_under_stress().await }).await??;
    
    println!("=== ALL ML TRADING PIPELINE INTEGRATION TESTS PASSED ===");
    println!("✓ Market data to features pipeline with SIMD optimization");
    println!("✓ ML inference pipeline with <50μs latency");
    println!("✓ Signal processing and decision generation");
    println!("✓ End-to-end ML trading flow <100μs");
    println!("✓ High-frequency stress testing >500 pipelines/sec");
    println!("✓ Feature extraction <10μs with SIMD");
    println!("✓ ML model inference <50μs");
    println!("✓ Signal processing <5μs");
    println!("✓ 90%+ success rate under stress");
    println!("✓ 10%+ actionable trading signals");
    
    Ok(())
}