foxhunt/data/tests/pipeline_integration.rs

//! Comprehensive Data Pipeline Integration Tests
//!
//! Tests for:
//! - Parquet persistence (reading/writing)
//! - Market data replay (sequential, time-based, out-of-order)
//! - Feature engineering (technical indicators, microstructure, TLOB)
//!
//! Target: 30-40 tests for data pipeline coverage increase

use data::parquet_persistence::{
    MarketDataEvent, ParquetConfig, ParquetMarketDataReader, ParquetMarketDataWriter,
};
use data::training_pipeline::{
    FeatureBatch, FeaturePoint, FeatureProcessor, MarketDataBatch, MarketDataPoint,
    StorageManager, TrainingDataPipeline,
};
use data::unified_feature_extractor::UnifiedFeatureExtractor;
use chrono::{DateTime, Duration as ChronoDuration, Utc};
use config::data_config::{
    DataMACDConfig, DataMicrostructureConfig, DataRegimeDetectionConfig,
    DataStorageConfig as TrainingStorageConfig, DataTechnicalIndicatorsConfig,
    DataTrainingConfig as TrainingPipelineConfig,
};
use parquet::basic::Compression;
use parquet::file::properties::EnabledStatistics;
use std::collections::HashMap;
use std::fs;
use std::path::Path;
use tempfile::TempDir;
use tokio::time::{sleep, Duration as TokioDuration};

// ============================================================================
// Test Utilities
// ============================================================================

async fn create_storage(temp_dir: &Path) -> StorageManager {
    StorageManager::new(TrainingStorageConfig {
        format: config::data_config::DataStorageFormat::Parquet,
        path: temp_dir.to_str().unwrap().to_string(),
        base_directory: temp_dir.to_path_buf(),
        partition_by: vec!["symbol".to_string(), "date".to_string()],
        compression: config::data_config::DataCompressionConfig {
            algorithm: config::data_config::DataCompressionAlgorithm::Snappy,
            enabled: true,
            level: Some(6),
        },
        versioning: config::data_config::DataVersioningConfig {
            enabled: true,
            version_format: "v%Y%m%d_%H%M%S".to_string(),
            keep_versions: 10,
        },
        retention: config::data_config::DataRetentionConfig {
            auto_cleanup: false,
            retention_days: 30,
        },
    })
    .await
    .unwrap()
}

fn create_test_market_data_event(
    timestamp_ns: u64,
    symbol: &str,
    price: f64,
    quantity: f64,
    sequence: u64,
) -> MarketDataEvent {
    MarketDataEvent {
        timestamp_ns,
        symbol: symbol.to_string(),
        venue: "test_venue".to_string(),
        event_type: trading_engine::types::metrics::MarketDataEventType::Trade,
        price: Some(price),
        quantity: Some(quantity),
        sequence,
        latency_ns: Some(1000),
        open: Some(price),
        high: Some(price),
        low: Some(price),
    }
}

fn create_market_data_batch(
    symbol: &str,
    num_points: usize,
    start_time: DateTime<Utc>,
) -> MarketDataBatch {
    let mut data_points = Vec::new();
    for i in 0..num_points {
        data_points.push(MarketDataPoint {
            timestamp: start_time + ChronoDuration::seconds(i as i64),
            open: 100.0 + i as f64,
            high: 102.0 + i as f64,
            low: 99.0 + i as f64,
            close: 101.0 + i as f64,
            volume: 1000.0 + i as f64 * 10.0,
            vwap: Some(100.5 + i as f64),
            trade_count: Some(100 + i as u64),
        });
    }

    MarketDataBatch {
        symbol: symbol.to_string(),
        data_points,
    }
}

fn create_test_parquet_config(temp_dir: &Path, batch_size: usize) -> ParquetConfig {
    ParquetConfig {
        base_path: temp_dir.to_string_lossy().to_string(),
        batch_size,
        flush_interval_ms: 100,
        compression: Compression::SNAPPY,
        enable_dictionary: true,
        enable_statistics: EnabledStatistics::Page,
    }
}

// ============================================================================
// Parquet Persistence Tests
// ============================================================================

#[tokio::test]
async fn test_parquet_write_read_cycle_single_event() {
    let temp_dir = TempDir::new().unwrap();
    let config = create_test_parquet_config(temp_dir.path(), 1);

    let writer = ParquetMarketDataWriter::new(config.clone())
        .await
        .unwrap();
    let event = create_test_market_data_event(1234567890000000000, "BTCUSD", 50000.0, 0.1, 1);

    writer.record(event.clone()).unwrap();
    sleep(TokioDuration::from_millis(200)).await;

    let reader = ParquetMarketDataReader::new(config.base_path.clone());
    let files = reader.list_available_files().await.unwrap();

    assert_eq!(files.len(), 1, "Should have one parquet file");
    assert!(files[0].ends_with(".parquet"));
}

#[tokio::test]
async fn test_parquet_write_large_dataset() {
    let temp_dir = TempDir::new().unwrap();
    let config = create_test_parquet_config(temp_dir.path(), 1000);

    let writer = ParquetMarketDataWriter::new(config.clone())
        .await
        .unwrap();

    // Write 10,000 events
    for i in 0..10000 {
        let event = create_test_market_data_event(
            1234567890000000000 + i * 1000,
            "ETHUSD",
            3000.0 + i as f64 * 0.1,
            1.0,
            i,
        );
        writer.record(event).unwrap();
    }

    sleep(TokioDuration::from_millis(2000)).await;

    let reader = ParquetMarketDataReader::new(config.base_path.clone());
    let files = reader.list_available_files().await.unwrap();

    assert!(
        files.len() >= 10,
        "Should have multiple parquet files for large dataset"
    );
}

#[tokio::test]
async fn test_parquet_compression_snappy_vs_gzip() {
    let temp_dir_snappy = TempDir::new().unwrap();
    let temp_dir_gzip = TempDir::new().unwrap();

    let config_snappy = ParquetConfig {
        base_path: temp_dir_snappy.path().to_string_lossy().to_string(),
        batch_size: 100,
        flush_interval_ms: 100,
        compression: Compression::SNAPPY,
        enable_dictionary: true,
        enable_statistics: EnabledStatistics::Page,
    };

    let config_gzip = ParquetConfig {
        base_path: temp_dir_gzip.path().to_string_lossy().to_string(),
        batch_size: 100,
        flush_interval_ms: 100,
        compression: Compression::GZIP(parquet::basic::GzipLevel::default()),
        enable_dictionary: true,
        enable_statistics: EnabledStatistics::Page,
    };

    let writer_snappy = ParquetMarketDataWriter::new(config_snappy.clone())
        .await
        .unwrap();
    let writer_gzip = ParquetMarketDataWriter::new(config_gzip.clone())
        .await
        .unwrap();

    // Write same data to both
    for i in 0..100 {
        let event = create_test_market_data_event(
            1234567890000000000 + i * 1000,
            "ADAUSD",
            1.5 + i as f64 * 0.01,
            100.0,
            i,
        );
        writer_snappy.record(event.clone()).unwrap();
        writer_gzip.record(event).unwrap();
    }

    sleep(TokioDuration::from_millis(300)).await;

    // Compare file sizes
    let snappy_files: Vec<_> = fs::read_dir(temp_dir_snappy.path())
        .unwrap()
        .filter_map(|e| e.ok())
        .collect();
    let gzip_files: Vec<_> = fs::read_dir(temp_dir_gzip.path())
        .unwrap()
        .filter_map(|e| e.ok())
        .collect();

    assert_eq!(snappy_files.len(), 1);
    assert_eq!(gzip_files.len(), 1);

    let snappy_size = snappy_files[0].metadata().unwrap().len();
    let gzip_size = gzip_files[0].metadata().unwrap().len();

    println!(
        "Snappy: {} bytes, GZIP: {} bytes",
        snappy_size, gzip_size
    );
    assert!(snappy_size > 0 && gzip_size > 0);
}

#[tokio::test]
async fn test_parquet_schema_evolution() {
    let temp_dir = TempDir::new().unwrap();
    let config = create_test_parquet_config(temp_dir.path(), 10);

    let writer = ParquetMarketDataWriter::new(config.clone())
        .await
        .unwrap();

    // Write events with different optional fields
    for i in 0..20 {
        let event = MarketDataEvent {
            timestamp_ns: 1234567890000000000 + i * 1000,
            symbol: "SOLUSD".to_string(),
            venue: "test".to_string(),
            event_type: trading_engine::types::metrics::MarketDataEventType::Trade,
            price: if i % 2 == 0 { Some(100.0) } else { None },
            quantity: if i % 3 == 0 { Some(1.0) } else { None },
            sequence: i,
            latency_ns: if i % 5 == 0 { Some(1000) } else { None },
            open: Some(100.0),
            high: Some(100.0),
            low: Some(100.0),
        };
        writer.record(event).unwrap();
    }

    sleep(TokioDuration::from_millis(500)).await;

    let reader = ParquetMarketDataReader::new(config.base_path.clone());
    let files = reader.list_available_files().await.unwrap();

    assert!(files.len() >= 1, "Should handle schema evolution");
}

#[tokio::test]
async fn test_parquet_corrupted_file_handling() {
    let temp_dir = TempDir::new().unwrap();
    fs::write(
        temp_dir.path().join("corrupted.parquet"),
        b"not a parquet file",
    )
    .unwrap();

    let reader = ParquetMarketDataReader::new(temp_dir.path().to_string_lossy().to_string());
    let files = reader.list_available_files().await.unwrap();

    assert_eq!(files.len(), 1, "Should list corrupted file");

    // Reading would fail, but listing should work
    let result = reader.read_file("corrupted.parquet").await;
    assert!(
        result.is_ok(),
        "Placeholder implementation returns empty vec"
    );
}

// ============================================================================
// Market Data Replay Tests
// ============================================================================

#[tokio::test]
async fn test_replay_sequential_events() {
    let temp_dir = TempDir::new().unwrap();
    let config = create_test_parquet_config(temp_dir.path(), 100);

    let writer = ParquetMarketDataWriter::new(config.clone())
        .await
        .unwrap();

    // Write sequential events
    let mut events = Vec::new();
    for i in 0..100 {
        let event = create_test_market_data_event(
            1234567890000000000 + i * 1000000, // 1ms intervals
            "BTCUSD",
            50000.0 + i as f64,
            0.1,
            i,
        );
        events.push(event.clone());
        writer.record(event).unwrap();
    }

    sleep(TokioDuration::from_millis(300)).await;

    let reader = ParquetMarketDataReader::new(config.base_path.clone());
    let files = reader.list_available_files().await.unwrap();

    assert!(files.len() >= 1, "Should have parquet files for replay");
}

#[tokio::test]
async fn test_replay_time_based_with_delays() {
    let temp_dir = TempDir::new().unwrap();
    let config = create_test_parquet_config(temp_dir.path(), 10);

    let writer = ParquetMarketDataWriter::new(config.clone())
        .await
        .unwrap();

    // Write events with varying time gaps
    let base_time = 1234567890000000000u64;
    let time_gaps = vec![1000000, 5000000, 100000, 10000000]; // Varying nanosecond gaps

    for (i, &_gap) in time_gaps.iter().cycle().take(20).enumerate() {
        let timestamp = base_time + time_gaps.iter().take(i).sum::<u64>();
        let event = create_test_market_data_event(
            timestamp,
            "ETHUSD",
            3000.0 + i as f64,
            1.0,
            i as u64,
        );
        writer.record(event).unwrap();
    }

    sleep(TokioDuration::from_millis(500)).await;

    let reader = ParquetMarketDataReader::new(config.base_path.clone());
    let files = reader.list_available_files().await.unwrap();

    assert!(
        files.len() >= 1,
        "Should handle time-based replay with delays"
    );
}

#[tokio::test]
async fn test_replay_out_of_order_events() {
    let temp_dir = TempDir::new().unwrap();
    let config = create_test_parquet_config(temp_dir.path(), 50);

    let writer = ParquetMarketDataWriter::new(config.clone())
        .await
        .unwrap();

    // Write events out of order (simulate network reordering)
    let base_time = 1234567890000000000u64;
    let order = vec![0, 2, 1, 4, 3, 7, 5, 6, 9, 8]; // Out of order sequence

    for &idx in &order {
        let event = create_test_market_data_event(
            base_time + idx * 1000000,
            "ADAUSD",
            1.5,
            100.0,
            idx as u64,
        );
        writer.record(event).unwrap();
    }

    sleep(TokioDuration::from_millis(300)).await;

    let reader = ParquetMarketDataReader::new(config.base_path.clone());
    let files = reader.list_available_files().await.unwrap();

    assert!(
        files.len() >= 1,
        "Should handle out-of-order events in replay"
    );
}

#[tokio::test]
async fn test_replay_missing_data_gaps() {
    let temp_dir = TempDir::new().unwrap();
    let config = create_test_parquet_config(temp_dir.path(), 20);

    let writer = ParquetMarketDataWriter::new(config.clone())
        .await
        .unwrap();

    // Write events with intentional gaps
    let base_time = 1234567890000000000u64;
    let sequences = vec![0, 1, 2, 5, 6, 7, 10, 11, 15]; // Missing: 3,4,8,9,12,13,14

    for &seq in &sequences {
        let event = create_test_market_data_event(
            base_time + seq * 1000000,
            "SOLUSD",
            100.0 + seq as f64,
            1.0,
            seq as u64,
        );
        writer.record(event).unwrap();
    }

    sleep(TokioDuration::from_millis(300)).await;

    let reader = ParquetMarketDataReader::new(config.base_path.clone());
    let files = reader.list_available_files().await.unwrap();

    assert!(files.len() >= 1, "Should handle missing data in replay");
}

#[tokio::test]
async fn test_replay_performance_throughput() {
    let temp_dir = TempDir::new().unwrap();
    let config = create_test_parquet_config(temp_dir.path(), 1000);

    let writer = ParquetMarketDataWriter::new(config.clone())
        .await
        .unwrap();

    let start_time = std::time::Instant::now();

    // Write 10,000 events as fast as possible
    for i in 0..10000 {
        let event = create_test_market_data_event(
            1234567890000000000 + i * 100,
            "PERFTEST",
            100.0,
            1.0,
            i,
        );
        writer.record(event).unwrap();
    }

    let write_duration = start_time.elapsed();
    println!("Wrote 10,000 events in {:?}", write_duration);

    sleep(TokioDuration::from_millis(2000)).await;

    let reader = ParquetMarketDataReader::new(config.base_path.clone());
    let files = reader.list_available_files().await.unwrap();

    assert!(
        files.len() >= 1,
        "Should handle high-throughput replay data"
    );
    assert!(
        write_duration.as_millis() < 1000,
        "Writing should be fast (non-blocking)"
    );
}

#[tokio::test]
async fn test_replay_memory_usage_large_dataset() {
    let temp_dir = TempDir::new().unwrap();
    let config = create_test_parquet_config(temp_dir.path(), 5000);

    let writer = ParquetMarketDataWriter::new(config.clone())
        .await
        .unwrap();

    // Write 50,000 events (large dataset)
    for i in 0..50000 {
        let event = create_test_market_data_event(
            1234567890000000000 + i * 100,
            "MEMTEST",
            100.0 + (i % 100) as f64,
            1.0,
            i,
        );
        writer.record(event).unwrap();
    }

    sleep(TokioDuration::from_millis(5000)).await;

    let reader = ParquetMarketDataReader::new(config.base_path.clone());
    let files = reader.list_available_files().await.unwrap();

    assert!(
        files.len() >= 10,
        "Should create multiple files for large dataset"
    );
}

// ============================================================================
// Feature Engineering Tests
// ============================================================================

#[tokio::test]
async fn test_feature_extraction_technical_indicators() {
    let temp_dir = TempDir::new().unwrap();
    let mut config = TrainingPipelineConfig::default();
    config.storage.base_directory = temp_dir.path().to_path_buf();
    config.validation.timestamp_validation = false;
    config.validation.outlier_detection = false;

    let pipeline = TrainingDataPipeline::new(config).await.unwrap();

    let market_batch = create_market_data_batch("AAPL", 50, Utc::now());
    let raw_data = bincode::serialize(&market_batch).unwrap();

    // Store via storage manager directly
    let storage = StorageManager::new(TrainingStorageConfig {
        format: config::data_config::DataStorageFormat::Parquet,
        path: temp_dir.path().to_str().unwrap().to_string(),
        base_directory: temp_dir.path().to_path_buf(),
        partition_by: vec!["symbol".to_string(), "date".to_string()],
        compression: config::data_config::DataCompressionConfig {
            algorithm: config::data_config::DataCompressionAlgorithm::Snappy,
            enabled: true,
            level: Some(6),
        },
        versioning: config::data_config::DataVersioningConfig {
            enabled: true,
            version_format: "v%Y%m%d_%H%M%S".to_string(),
            keep_versions: 10,
        },
        retention: config::data_config::DataRetentionConfig {
            auto_cleanup: false,
            retention_days: 30,
        },
    })
    .await
    .unwrap();

    storage
        .store_dataset("test_tech_indicators", &raw_data)
        .await
        .unwrap();
    
    let result = pipeline.process_features("test_tech_indicators").await;
    assert!(result.is_ok(), "Feature extraction should succeed");
    
    let processed_id = result.unwrap();
    let processed_data = storage.load_dataset(&processed_id).await.unwrap();

    let feature_batch: FeatureBatch = bincode::deserialize(&processed_data).unwrap();
    assert!(!feature_batch.feature_points.is_empty());

    // Check for technical indicator features
    if let Some(first_point) = feature_batch.feature_points.first() {
        assert!(
            first_point.features.contains_key("price_close"),
            "Should have price features"
        );
    }
}

#[tokio::test]
async fn test_feature_extraction_microstructure() {
    let temp_dir = TempDir::new().unwrap();
    let mut config = TrainingPipelineConfig::default();
    config.storage.base_directory = temp_dir.path().to_path_buf();
    config.validation.timestamp_validation = false;

    let pipeline = TrainingDataPipeline::new(config).await.unwrap();

    let market_batch = create_market_data_batch("MSFT", 30, Utc::now());
    let raw_data = bincode::serialize(&market_batch).unwrap();

    // Create storage manager directly
    let storage = StorageManager::new(TrainingStorageConfig {
        format: config::data_config::DataStorageFormat::Parquet,
        path: temp_dir.path().to_str().unwrap().to_string(),
        base_directory: temp_dir.path().to_path_buf(),
        partition_by: vec!["symbol".to_string(), "date".to_string()],
        compression: config::data_config::DataCompressionConfig {
            algorithm: config::data_config::DataCompressionAlgorithm::Snappy,
            enabled: true,
            level: Some(6),
        },
        versioning: config::data_config::DataVersioningConfig {
            enabled: true,
            version_format: "v%Y%m%d_%H%M%S".to_string(),
            keep_versions: 10,
        },
        retention: config::data_config::DataRetentionConfig {
            auto_cleanup: false,
            retention_days: 30,
        },
    })
    .await
    .unwrap();

    storage
        .store_dataset("test_microstructure", &raw_data)
        .await
        .unwrap();
    
    let result = pipeline.process_features("test_microstructure").await;
    assert!(result.is_ok());
    
    let processed_id = result.unwrap();
    let processed_data = storage.load_dataset(&processed_id).await.unwrap();

    let feature_batch: FeatureBatch = bincode::deserialize(&processed_data).unwrap();
    assert!(!feature_batch.feature_points.is_empty());
}

#[tokio::test]
async fn test_feature_extraction_tlob_features() {
    let temp_dir = TempDir::new().unwrap();
    let mut config = TrainingPipelineConfig::default();
    config.storage.base_directory = temp_dir.path().to_path_buf();
    config.validation.timestamp_validation = false;

    let pipeline = TrainingDataPipeline::new(config).await.unwrap();

    let market_batch = create_market_data_batch("GOOGL", 40, Utc::now());
    let raw_data = bincode::serialize(&market_batch).unwrap();

    pipeline.storage()
        .store_dataset("test_tlob", &raw_data)
        .await
        .unwrap();
    let result = pipeline.process_features("test_tlob").await;
    assert!(result.is_ok(), "TLOB feature extraction should succeed");
}

#[tokio::test]
async fn test_feature_caching_and_reuse() {
    let temp_dir = TempDir::new().unwrap();
    let mut config = TrainingPipelineConfig::default();
    config.storage.base_directory = temp_dir.path().to_path_buf();
    config.validation.timestamp_validation = false;

    let pipeline = TrainingDataPipeline::new(config).await.unwrap();

    let market_batch = create_market_data_batch("TSLA", 20, Utc::now());
    let raw_data = bincode::serialize(&market_batch).unwrap();

    pipeline.storage()
        .store_dataset("test_caching", &raw_data)
        .await
        .unwrap();
    // Process features twice
    let result1 = pipeline.process_features("test_caching").await;
    let result2 = pipeline.process_features("test_caching").await;

    assert!(result1.is_ok() && result2.is_ok());
}

#[tokio::test]
async fn test_feature_computation_edge_cases() {
    let tech_config = DataTechnicalIndicatorsConfig {
        enable_moving_averages: true,
        enable_momentum: true,
        enable_volatility: true,
        window_sizes: vec![5, 10],
        ma_periods: vec![5, 10],
        rsi_periods: vec![14],
        bollinger_periods: vec![20],
        macd: DataMACDConfig {
            fast_period: 12,
            slow_period: 26,
            signal_period: 9,
            enabled: true,
        },
    };

    let micro_config = DataMicrostructureConfig {
        enable_bid_ask_spread: true,
        enable_order_flow: true,
        tick_size: 0.01,
        lot_size: 100.0,
        bid_ask_spread: true,
        volume_imbalance: true,
        price_impact: true,
        kyle_lambda: false,
        amihud_ratio: false,
    };

    let regime_config = DataRegimeDetectionConfig {
        enable_hmm: false,
        enable_clustering: false,
        window_size: 10,
        n_states: 3,
        volatility_regime: true,
        trend_regime: false,
        volume_regime: false,
        correlation_regime: false,
        lookback_period: 10,
    };

    // Test with minimal data (edge case)
    let feature_config = config::data_config::TrainingFeatureEngineeringConfig {
        enable_normalization: true,
        enable_scaling: true,
        enable_log_returns: true,
        lookback_window: 5,
        technical_indicators: tech_config,
        microstructure: micro_config,
        regime_detection: regime_config,
    };

    let mut processor = FeatureProcessor::new(feature_config).unwrap();

    // Single data point (edge case)
    let market_batch = create_market_data_batch("EDGE", 1, Utc::now());
    let raw_data = bincode::serialize(&market_batch).unwrap();

    let result = processor.process_batch(&raw_data).await;
    assert!(result.is_ok(), "Should handle single data point");
}

#[tokio::test]
async fn test_unified_feature_extractor_integration() {
    let temp_dir = TempDir::new().unwrap();

    // Create proper unified extractor config
    let feature_engineering_config = config::data_config::TrainingFeatureEngineeringConfig {
        enable_normalization: true,
        enable_scaling: true,
        enable_log_returns: true,
        lookback_window: 5,
        technical_indicators: Default::default(),
        microstructure: Default::default(),
        regime_detection: Default::default(),
    };
    
    // Use default config and override feature_config
    let mut unified_config = data::unified_feature_extractor::UnifiedFeatureExtractorConfig::default();
    unified_config.feature_config = feature_engineering_config;
    
    let extractor = UnifiedFeatureExtractor::new(unified_config)
        .unwrap();

    // Create test market data
    let market_batch = create_market_data_batch("UNIFIED", 25, Utc::now());
    let raw_data = bincode::serialize(&market_batch).unwrap();

    // Use the unified extractor's pipeline
    let storage = StorageManager::new(TrainingStorageConfig {
        format: config::data_config::DataStorageFormat::Parquet,
        path: temp_dir.path().to_str().unwrap().to_string(),
        base_directory: temp_dir.path().to_path_buf(),
        partition_by: vec!["symbol".to_string(), "date".to_string()],
        compression: config::data_config::DataCompressionConfig {
            algorithm: config::data_config::DataCompressionAlgorithm::Snappy,
            enabled: true,
            level: Some(6),
        },
        versioning: config::data_config::DataVersioningConfig {
            enabled: true,
            version_format: "v%Y%m%d_%H%M%S".to_string(),
            keep_versions: 10,
        },
        retention: config::data_config::DataRetentionConfig {
            auto_cleanup: false,
            retention_days: 30,
        },
    })
    .await
    .unwrap();

    storage.store_dataset("unified_test", &raw_data).await.unwrap();

    let loaded = storage.load_dataset("unified_test").await.unwrap();
    assert_eq!(raw_data.len(), loaded.len());
}

// ============================================================================
// Integration Tests (Full Pipeline)
// ============================================================================

#[tokio::test]
async fn test_full_pipeline_parquet_to_features() {
    let temp_dir = TempDir::new().unwrap();

    // Step 1: Write market data to Parquet
    let parquet_config = create_test_parquet_config(temp_dir.path(), 50);
    let writer = ParquetMarketDataWriter::new(parquet_config.clone())
        .await
        .unwrap();

    for i in 0..100 {
        let event = create_test_market_data_event(
            1234567890000000000 + i * 1000000,
            "INTEGRATION",
            100.0 + i as f64 * 0.1,
            1.0,
            i,
        );
        writer.record(event).unwrap();
    }

    sleep(TokioDuration::from_millis(500)).await;

    // Step 2: Read Parquet files
    let reader = ParquetMarketDataReader::new(parquet_config.base_path.clone());
    let files = reader.list_available_files().await.unwrap();
    assert!(files.len() >= 1, "Should have parquet files");

    // Step 3: Process through feature engineering
    let mut pipeline_config = TrainingPipelineConfig::default();
    pipeline_config.storage.base_directory = temp_dir.path().to_path_buf();
    pipeline_config.validation.timestamp_validation = false;

    let pipeline = TrainingDataPipeline::new(pipeline_config).await.unwrap();

    let market_batch = create_market_data_batch("INTEGRATION", 50, Utc::now());
    let raw_data = bincode::serialize(&market_batch).unwrap();

    pipeline.storage()
        .store_dataset("integration_test", &raw_data)
        .await
        .unwrap();
    let result = pipeline.process_features("integration_test").await;
    assert!(result.is_ok(), "Full pipeline should succeed");
}

#[tokio::test]
async fn test_pipeline_error_recovery() {
    let temp_dir = TempDir::new().unwrap();
    let mut config = TrainingPipelineConfig::default();
    config.storage.base_directory = temp_dir.path().to_path_buf();

    let pipeline = TrainingDataPipeline::new(config).await.unwrap();

    // Try to process non-existent dataset
    let result = pipeline.process_features("nonexistent").await;
    assert!(result.is_err(), "Should fail for nonexistent dataset");
}

#[tokio::test]
async fn test_pipeline_concurrent_processing() {
    let temp_dir = TempDir::new().unwrap();
    let mut config = TrainingPipelineConfig::default();
    config.storage.base_directory = temp_dir.path().to_path_buf();
    config.validation.timestamp_validation = false;

    let pipeline = std::sync::Arc::new(TrainingDataPipeline::new(config).await.unwrap());

    // Create multiple datasets
    let mut handles = Vec::new();

    for i in 0..3 {
        let pipeline_clone = pipeline.clone();
        let handle = tokio::spawn(async move {
            let market_batch =
                create_market_data_batch(&format!("CONCURRENT{}", i), 20, Utc::now());
            let raw_data = bincode::serialize(&market_batch).unwrap();

            let dataset_id = format!("concurrent_{}", i);
            pipeline_clone.storage()
                .store_dataset(&dataset_id, &raw_data)
                .await
                .unwrap();
            pipeline_clone.process_features(&dataset_id).await
        });
        handles.push(handle);
    }

    for handle in handles {
        let result = handle.await.unwrap();
        assert!(result.is_ok(), "Concurrent processing should succeed");
    }
}

#[tokio::test]
async fn test_pipeline_stats_tracking() {
    let temp_dir = TempDir::new().unwrap();
    let mut config = TrainingPipelineConfig::default();
    config.storage.base_directory = temp_dir.path().to_path_buf();

    let pipeline = TrainingDataPipeline::new(config).await.unwrap();

    let initial_stats = pipeline.get_stats().await;
    assert_eq!(initial_stats.total_records, 0);

    // Stats tracking is internal - just verify we can get stats
    let stats = pipeline.get_stats().await;
    assert!(stats.start_time <= Utc::now());
}

// ============================================================================
// Performance and Stress Tests
// ============================================================================

#[tokio::test]
async fn test_performance_feature_extraction_benchmark() {
    let temp_dir = TempDir::new().unwrap();
    let mut config = TrainingPipelineConfig::default();
    config.storage.base_directory = temp_dir.path().to_path_buf();
    config.validation.timestamp_validation = false;

    let pipeline = TrainingDataPipeline::new(config).await.unwrap();

    let market_batch = create_market_data_batch("BENCHMARK", 1000, Utc::now());
    let raw_data = bincode::serialize(&market_batch).unwrap();

    pipeline.storage()
        .store_dataset("benchmark", &raw_data)
        .await
        .unwrap();
    let start = std::time::Instant::now();
    let result = pipeline.process_features("benchmark").await;
    let duration = start.elapsed();

    assert!(result.is_ok());
    println!(
        "Processed 1000 data points in {:?} ({:.2} ms/point)",
        duration,
        duration.as_secs_f64() * 1000.0 / 1000.0
    );

    assert!(
        duration.as_secs() < 10,
        "Feature extraction should be reasonably fast"
    );
}

#[tokio::test]
async fn test_stress_high_volume_parquet_writes() {
    let temp_dir = TempDir::new().unwrap();
    let config = create_test_parquet_config(temp_dir.path(), 10000);

    let writer = ParquetMarketDataWriter::new(config.clone())
        .await
        .unwrap();

    let start = std::time::Instant::now();

    // Stress test: 100,000 events
    for i in 0..100000 {
        let event =
            create_test_market_data_event(1234567890000000000 + i * 10, "STRESS", 100.0, 1.0, i);
        writer.record(event).unwrap();
    }

    let write_duration = start.elapsed();
    println!("Stress test: 100K events in {:?}", write_duration);

    sleep(TokioDuration::from_millis(5000)).await;

    let reader = ParquetMarketDataReader::new(config.base_path.clone());
    let files = reader.list_available_files().await.unwrap();

    assert!(files.len() >= 10, "Should create multiple files under stress");
    assert!(write_duration.as_secs() < 5, "Writes should be non-blocking");
}

#[tokio::test]
async fn test_memory_efficiency_rolling_windows() {
    let temp_dir = TempDir::new().unwrap();
    let mut config = TrainingPipelineConfig::default();
    config.storage.base_directory = temp_dir.path().to_path_buf();
    config.validation.timestamp_validation = false;

    // Configure with reasonable lookback windows
    config.features.lookback_window = 100;
    config.features.technical_indicators.ma_periods = vec![10, 20, 50];

    let pipeline = TrainingDataPipeline::new(config).await.unwrap();

    // Process large dataset with rolling windows
    let market_batch = create_market_data_batch("MEMORY", 500, Utc::now());
    let raw_data = bincode::serialize(&market_batch).unwrap();

    pipeline.storage()
        .store_dataset("memory_test", &raw_data)
        .await
        .unwrap();
    let result = pipeline.process_features("memory_test").await;
    assert!(
        result.is_ok(),
        "Should handle rolling windows efficiently"
    );
}