foxhunt/data/tests/training_pipeline_tests.rs

//! Comprehensive Integration Tests for Training Data Pipeline
//!
//! This test suite provides extensive coverage for the training pipeline with 25+ tests
//! covering complete pipeline workflows, data source integrations, feature engineering,
//! validation, and storage operations.

use data::error::{DataError, Result};
use data::training_pipeline::*;
use std::fs::File;
use std::io::Write as IoWrite;
use std::sync::Arc;
use tempfile::{tempdir, TempDir};
use tokio::time::{timeout, Duration};

// ============================================================================
// Test Fixtures and Helpers
// ============================================================================

/// Creates a temporary directory and returns both the TempDir and config
async fn setup_test_environment() -> (TempDir, TrainingPipelineConfig) {
    let temp_dir = tempdir().expect("Failed to create temp directory");
    let mut config = TrainingPipelineConfig::default();
    config.storage.base_directory = temp_dir.path().to_path_buf();
    (temp_dir, config)
}

/// Creates sample market data for testing
fn create_sample_market_data() -> Vec<u8> {
    let sample_data = r#"
timestamp,symbol,price,volume,bid,ask
2024-01-15T09:30:00Z,SPY,450.25,1000,450.20,450.30
2024-01-15T09:30:01Z,SPY,450.30,1500,450.25,450.35
2024-01-15T09:30:02Z,SPY,450.28,800,450.23,450.33
2024-01-15T09:30:03Z,SPY,450.35,2000,450.30,450.40
2024-01-15T09:30:04Z,SPY,450.32,1200,450.27,450.37
"#;
    sample_data.as_bytes().to_vec()
}

/// Creates a dataset file in the temp directory
async fn create_test_dataset(dir: &TempDir, dataset_id: &str, data: &[u8]) {
    let file_path = dir.path().join(dataset_id);
    tokio::fs::write(file_path, data)
        .await
        .expect("Failed to create test dataset");
}

// ============================================================================
// 1. Complete Pipeline Workflow Tests (8 tests)
// ============================================================================

#[tokio::test]
async fn test_full_pipeline_workflow_end_to_end() {
    let (_temp_dir, config) = setup_test_environment().await;
    let pipeline = TrainingDataPipeline::new(config)
        .await
        .expect("Failed to create pipeline");

    // Create initial raw dataset
    let raw_data = create_sample_market_data();
    let raw_dataset_id = "raw_market_data_20240115";
    create_test_dataset(&_temp_dir, raw_dataset_id, &raw_data).await;

    // Test complete workflow: raw data -> features -> validation -> storage
    let processed_id = pipeline
        .process_features(raw_dataset_id)
        .await
        .expect("Feature processing should succeed");

    // Verify processed dataset exists
    let processed_path = _temp_dir.path().join(&processed_id);
    assert!(processed_path.exists(), "Processed dataset should exist");

    // Verify content was processed
    let processed_data = tokio::fs::read(processed_path)
        .await
        .expect("Should read processed data");
    assert_eq!(processed_data, raw_data, "Data should match (passthrough)");

    // Verify stats were updated
    let stats = pipeline.get_stats().await;
    assert!(
        stats.start_time <= stats.last_update,
        "Stats should be updated"
    );
}

#[tokio::test]
async fn test_multi_source_integration_pipeline() {
    let (_temp_dir, mut config) = setup_test_environment().await;

    // Enable databento and benzinga sources
    config.sources.databento = Some(DatabentConfig {
        api_key: "test_key".to_string(),
        symbols: vec!["SPY".to_string(), "QQQ".to_string()],
        data_types: vec!["trades".to_string(), "quotes".to_string()],
        rate_limit: 100,
        timeout: 30,
    });

    let pipeline = TrainingDataPipeline::new(config)
        .await
        .expect("Multi-source pipeline should initialize");

    // Verify clients were initialized
    assert!(
        pipeline.databento_client.is_some(),
        "Databento client should be initialized"
    );

    // Test historical data collection
    let dataset_id = pipeline
        .collect_historical_data()
        .await
        .expect("Historical data collection should succeed");

    assert!(
        dataset_id.starts_with("historical_"),
        "Dataset ID should have correct prefix"
    );
}

#[tokio::test]
async fn test_realtime_vs_batch_processing_modes() {
    let (_temp_dir, mut config) = setup_test_environment().await;

    // Test realtime disabled
    config.sources.enable_realtime = false;
    let mut pipeline = TrainingDataPipeline::new(config.clone())
        .await
        .expect("Pipeline should initialize");

    let result = pipeline.start_realtime_collection().await;
    assert!(
        result.is_ok(),
        "Realtime collection should succeed when disabled"
    );

    // Test realtime enabled
    config.sources.enable_realtime = true;
    let mut pipeline = TrainingDataPipeline::new(config)
        .await
        .expect("Pipeline should initialize");

    let result = pipeline.start_realtime_collection().await;
    assert!(
        result.is_ok(),
        "Realtime collection should succeed when enabled"
    );
}

#[tokio::test]
async fn test_pipeline_failure_recovery() {
    let (_temp_dir, config) = setup_test_environment().await;
    let pipeline = TrainingDataPipeline::new(config)
        .await
        .expect("Pipeline should initialize");

    // Test processing non-existent dataset
    let result = pipeline.process_features("non_existent_dataset").await;
    assert!(
        result.is_err(),
        "Processing non-existent dataset should fail"
    );

    match result.unwrap_err() {
        DataError::Io(_) => {} // Expected - file not found
        other => panic!("Expected IO error, got: {:?}", other),
    }
}

#[tokio::test]
async fn test_configuration_validation() {
    let (_temp_dir, mut config) = setup_test_environment().await;

    // Test with minimal configuration
    config.sources.databento = None;
    config.sources.benzinga = None;
    config.sources.interactive_brokers = None;
    config.sources.icmarkets = None;

    let pipeline = TrainingDataPipeline::new(config)
        .await
        .expect("Minimal config pipeline should initialize");

    assert!(
        pipeline.databento_client.is_none(),
        "No Databento client expected"
    );
    assert!(
        pipeline.benzinga_client.is_none(),
        "No Benzinga client expected"
    );
}

#[tokio::test]
async fn test_dataset_versioning_workflows() {
    let (_temp_dir, mut config) = setup_test_environment().await;

    // Enable versioning
    config.storage.versioning.enabled = true;
    config.storage.versioning.keep_versions = 3;

    let pipeline = TrainingDataPipeline::new(config)
        .await
        .expect("Versioned pipeline should initialize");

    // Create and process multiple datasets
    let raw_data = create_sample_market_data();

    for i in 1..=3 {
        let dataset_id = format!("version_test_{}", i);
        create_test_dataset(&_temp_dir, &dataset_id, &raw_data).await;

        let processed_id = pipeline
            .process_features(&dataset_id)
            .await
            .expect("Processing should succeed");

        assert!(
            processed_id.contains("features"),
            "Processed ID should contain 'features'"
        );
    }
}

#[tokio::test]
async fn test_parallel_processing_coordination() {
    let (_temp_dir, mut config) = setup_test_environment().await;

    // Enable parallel processing
    config.processing.parallel_processing = true;
    config.processing.worker_threads = 4;

    let pipeline = Arc::new(
        TrainingDataPipeline::new(config)
            .await
            .expect("Parallel pipeline should initialize"),
    );

    // Create test datasets
    let raw_data = create_sample_market_data();
    let mut tasks = Vec::new();

    for i in 1..=3 {
        let dataset_id = format!("parallel_test_{}", i);
        create_test_dataset(&_temp_dir, &dataset_id, &raw_data).await;

        let pipeline_clone = pipeline.clone();
        let dataset_id_clone = dataset_id.clone();

        let task =
            tokio::spawn(async move { pipeline_clone.process_features(&dataset_id_clone).await });
        tasks.push(task);
    }

    // Wait for all tasks to complete
    for task in tasks {
        let result = task.await.expect("Task should complete");
        assert!(result.is_ok(), "Parallel processing should succeed");
    }
}

#[tokio::test]
async fn test_pipeline_statistics_monitoring() {
    let (_temp_dir, config) = setup_test_environment().await;
    let pipeline = TrainingDataPipeline::new(config)
        .await
        .expect("Pipeline should initialize");

    // Get initial stats
    let initial_stats = pipeline.get_stats().await;
    assert_eq!(
        initial_stats.total_records, 0,
        "Initial stats should be zero"
    );
    assert_eq!(initial_stats.errors, 0, "Initial errors should be zero");

    // Process a dataset
    let raw_data = create_sample_market_data();
    let dataset_id = "stats_test";
    create_test_dataset(&_temp_dir, dataset_id, &raw_data).await;

    let _processed_id = pipeline
        .process_features(dataset_id)
        .await
        .expect("Processing should succeed");

    // Verify stats structure
    let final_stats = pipeline.get_stats().await;
    assert!(
        final_stats.start_time <= final_stats.last_update,
        "Timestamps should be valid"
    );
}

// ============================================================================
// 2. Data Source Integration Tests (6 tests)
// ============================================================================

#[tokio::test]
async fn test_databento_client_initialization() {
    let (_temp_dir, mut config) = setup_test_environment().await;

    config.sources.databento = Some(DatabentConfig {
        api_key: "test_databento_key".to_string(),
        symbols: vec!["SPY".to_string()],
        data_types: vec!["trades".to_string()],
        rate_limit: 100,
        timeout: 30,
    });

    let pipeline = TrainingDataPipeline::new(config)
        .await
        .expect("Databento pipeline should initialize");

    assert!(
        pipeline.databento_client.is_some(),
        "Databento client should exist"
    );
}

#[tokio::test]
async fn test_benzinga_client_initialization() {
    let (_temp_dir, mut config) = setup_test_environment().await;

    // Use the actual BenzingaConfig structure from the providers module
    config.sources.benzinga = Some(BenzingaConfig {
        api_key: "test_benzinga_key".to_string(),
        symbols: vec!["AAPL".to_string()],
        data_types: vec!["news".to_string()],
        rate_limit: 50,
        timeout: 60,
    });

    let pipeline = TrainingDataPipeline::new(config)
        .await
        .expect("Benzinga pipeline should initialize");

    assert!(
        pipeline.benzinga_client.is_some(),
        "Benzinga client should exist"
    );
}

#[tokio::test]
async fn test_interactive_brokers_config() {
    let (_temp_dir, mut config) = setup_test_environment().await;

    config.sources.interactive_brokers = Some(IBDataConfig {
        host: "127.0.0.1".to_string(),
        port: 7497,
        client_id: 1,
        symbols: vec!["SPY".to_string()],
        enable_level2: true,
    });

    let mut pipeline = TrainingDataPipeline::new(config)
        .await
        .expect("IB pipeline should initialize");

    // Test IB realtime collection
    let result = pipeline.start_realtime_collection().await;
    assert!(
        result.is_ok(),
        "IB realtime should initialize without error"
    );
}

#[tokio::test]
async fn test_icmarkets_config() {
    let (_temp_dir, mut config) = setup_test_environment().await;

    config.sources.icmarkets = Some(ICMarketsDataConfig {
        host: "fix.icmarkets.com".to_string(),
        port: 9880,
        username: "test_user".to_string(),
        password: "test_pass".to_string(),
        symbols: vec!["EURUSD".to_string()],
    });

    let mut pipeline = TrainingDataPipeline::new(config)
        .await
        .expect("ICMarkets pipeline should initialize");

    let result = pipeline.start_realtime_collection().await;
    assert!(
        result.is_ok(),
        "ICMarkets realtime should initialize without error"
    );
}

#[tokio::test]
async fn test_rate_limiting_configuration() {
    let (_temp_dir, mut config) = setup_test_environment().await;

    // Configure with low rate limits
    config.sources.databento = Some(DatabentConfig {
        api_key: "test_key".to_string(),
        symbols: vec!["SPY".to_string()],
        data_types: vec!["trades".to_string()],
        rate_limit: 1, // Very low rate limit
        timeout: 1,    // Very short timeout
    });

    let pipeline = TrainingDataPipeline::new(config)
        .await
        .expect("Rate limited pipeline should initialize");

    // Test collection with timeout
    let result = timeout(Duration::from_secs(3), pipeline.collect_historical_data()).await;
    match result {
        Ok(dataset_result) => {
            assert!(
                dataset_result.is_ok(),
                "Should handle rate limiting gracefully"
            );
        }
        Err(_) => {
            // Timeout is acceptable for rate limiting tests
        }
    }
}

#[tokio::test]
async fn test_source_configuration_validation() {
    let (_temp_dir, mut config) = setup_test_environment().await;

    // Test with invalid or empty configurations
    config.sources.databento = Some(DatabentConfig {
        api_key: "".to_string(), // Empty API key
        symbols: vec![],         // No symbols
        data_types: vec![],      // No data types
        rate_limit: 0,           // Invalid rate limit
        timeout: 0,              // Invalid timeout
    });

    // Pipeline should still initialize (validation happens at runtime)
    let pipeline = TrainingDataPipeline::new(config)
        .await
        .expect("Pipeline with empty config should still initialize");

    assert!(
        pipeline.databento_client.is_some(),
        "Client should still be created"
    );
}

// ============================================================================
// 3. Feature Engineering Tests (6 tests)
// ============================================================================

#[tokio::test]
async fn test_technical_indicators_configuration() {
    let config = TechnicalIndicatorsConfig {
        ma_periods: vec![10, 20, 50],
        rsi_periods: vec![14, 21],
        bollinger_periods: vec![20],
        macd: MACDConfig {
            fast_period: 12,
            slow_period: 26,
            signal_period: 9,
        },
        volume_indicators: true,
    };

    let calculator = TechnicalIndicatorsCalculator::new(config.clone());

    assert_eq!(calculator.config.ma_periods, vec![10, 20, 50]);
    assert_eq!(calculator.config.rsi_periods, vec![14, 21]);
    assert!(calculator.config.volume_indicators);
}

#[tokio::test]
async fn test_microstructure_analysis_configuration() {
    let config = MicrostructureConfig {
        bid_ask_spread: true,
        volume_imbalance: true,
        price_impact: true,
        kyle_lambda: true,
        amihud_ratio: true,
        roll_spread: true,
    };

    let analyzer = MicrostructureAnalyzer::new(config.clone());

    assert!(analyzer.config.bid_ask_spread);
    assert!(analyzer.config.volume_imbalance);
    assert!(analyzer.config.price_impact);
}

#[tokio::test]
async fn test_tlob_processing_configuration() {
    let config = TLOBConfig {
        book_depth: 10,
        time_window: 300,
        volume_buckets: vec![100.0, 500.0, 1000.0],
        order_flow_analytics: true,
        imbalance_calculations: true,
    };

    let processor = TLOBProcessor::new(config.clone());

    assert_eq!(processor.config.book_depth, 10);
    assert_eq!(processor.config.time_window, 300);
    assert_eq!(processor.config.volume_buckets.len(), 3);
}

#[tokio::test]
async fn test_temporal_feature_configuration() {
    let config = TemporalConfig {
        time_of_day: true,
        day_of_week: true,
        market_session: true,
        holiday_effects: true,
        expiration_effects: true,
    };

    assert!(config.time_of_day);
    assert!(config.day_of_week);
    assert!(config.market_session);
    assert!(config.holiday_effects);
    assert!(config.expiration_effects);
}

#[tokio::test]
async fn test_regime_detection_configuration() {
    let config = RegimeDetectionConfig {
        volatility_regime: true,
        trend_regime: true,
        volume_regime: true,
        correlation_regime: true,
        lookback_period: 100,
    };

    let detector = RegimeDetector::new(config.clone());

    assert!(detector.config.volatility_regime);
    assert!(detector.config.trend_regime);
    assert_eq!(detector.config.lookback_period, 100);
}

#[tokio::test]
async fn test_feature_processor_workflow() {
    let config = FeatureEngineeringConfig {
        technical_indicators: TechnicalIndicatorsConfig {
            ma_periods: vec![10, 20],
            rsi_periods: vec![14],
            bollinger_periods: vec![20],
            macd: MACDConfig {
                fast_period: 12,
                slow_period: 26,
                signal_period: 9,
            },
            volume_indicators: true,
        },
        microstructure: MicrostructureConfig {
            bid_ask_spread: true,
            volume_imbalance: true,
            price_impact: false,
            kyle_lambda: false,
            amihud_ratio: false,
            roll_spread: false,
        },
        tlob: TLOBConfig {
            book_depth: 5,
            time_window: 60,
            volume_buckets: vec![100.0, 500.0],
            order_flow_analytics: true,
            imbalance_calculations: true,
        },
        temporal: TemporalConfig {
            time_of_day: true,
            day_of_week: true,
            market_session: false,
            holiday_effects: false,
            expiration_effects: false,
        },
        regime_detection: RegimeDetectionConfig {
            volatility_regime: true,
            trend_regime: false,
            volume_regime: false,
            correlation_regime: false,
            lookback_period: 50,
        },
    };

    let processor = FeatureProcessor::new(config).expect("Feature processor should initialize");

    // Test processing (currently passthrough)
    let sample_data = create_sample_market_data();
    let result = processor.process_batch(&sample_data).await;
    assert!(result.is_ok(), "Feature processing should succeed");
}

// ============================================================================
// 4. Data Validation Tests (4 tests)
// ============================================================================

#[tokio::test]
async fn test_validation_configuration() {
    let config = DataValidationConfig {
        enable_price_validation: true,
        enable_volume_validation: true,
        price_threshold: 0.01,
        volume_threshold: 100.0,
        price_validation: true,
        max_price_change: 5.0,
        volume_validation: true,
        max_volume_change: 500.0,
        timestamp_validation: true,
        max_timestamp_drift: 1000,
        outlier_detection: true,
        outlier_method: OutlierDetectionMethod::ZScore,
        missing_data_handling: MissingDataHandling::Skip,
    };

    let validator = DataValidator::new(config.clone()).expect("Validator should initialize");

    assert!(validator.config.price_validation);
    assert_eq!(validator.config.max_price_change, 5.0);
    assert!(validator.config.volume_validation);
}

#[tokio::test]
async fn test_outlier_detection_methods() {
    let methods = vec![
        OutlierDetectionMethod::ZScore,
        OutlierDetectionMethod::IQR,
        OutlierDetectionMethod::IsolationForest,
        OutlierDetectionMethod::LocalOutlierFactor,
    ];

    for method in methods {
        let config = DataValidationConfig {
            enable_price_validation: false,
            enable_volume_validation: false,
            price_threshold: 0.01,
            volume_threshold: 100.0,
            price_validation: false,
            max_price_change: 0.0,
            volume_validation: false,
            max_volume_change: 0.0,
            timestamp_validation: false,
            max_timestamp_drift: 0,
            outlier_detection: true,
            outlier_method: method,
            missing_data_handling: MissingDataHandling::Skip,
        };

        let validator =
            DataValidator::new(config).expect("Validator should initialize with any method");

        let sample_data = create_sample_market_data();
        let result = validator.validate_batch(&sample_data).await;
        assert!(result.is_ok(), "Validation should succeed");
    }
}

#[tokio::test]
async fn test_missing_data_handling_strategies() {
    let strategies = vec![
        MissingDataHandling::Drop,
        MissingDataHandling::ForwardFill,
        MissingDataHandling::BackwardFill,
        MissingDataHandling::Interpolate,
        MissingDataHandling::Mean,
        MissingDataHandling::Median,
    ];

    for strategy in strategies {
        let config = DataValidationConfig {
            enable_price_validation: false,
            enable_volume_validation: false,
            price_threshold: 0.01,
            volume_threshold: 100.0,
            price_validation: false,
            max_price_change: 0.0,
            volume_validation: false,
            max_volume_change: 0.0,
            timestamp_validation: false,
            max_timestamp_drift: 0,
            outlier_detection: false,
            outlier_method: OutlierDetectionMethod::ZScore,
            missing_data_handling: strategy,
        };

        let validator =
            DataValidator::new(config).expect("Validator should initialize with any strategy");

        let sample_data = create_sample_market_data();
        let result = validator.validate_batch(&sample_data).await;
        assert!(result.is_ok(), "Validation should succeed");
    }
}

#[tokio::test]
async fn test_validation_workflow() {
    let (_temp_dir, config) = setup_test_environment().await;
    let pipeline = TrainingDataPipeline::new(config)
        .await
        .expect("Pipeline should initialize");

    let raw_data = create_sample_market_data();
    let dataset_id = "validation_test";
    create_test_dataset(&_temp_dir, dataset_id, &raw_data).await;

    let processed_id = pipeline
        .process_features(dataset_id)
        .await
        .expect("Processing with validation should succeed");

    let processed_path = _temp_dir.path().join(&processed_id);
    assert!(processed_path.exists(), "Validated dataset should exist");
}

// ============================================================================
// 5. Storage and Versioning Tests (3 tests)
// ============================================================================

#[tokio::test]
async fn test_storage_formats() {
    let formats = vec![
        StorageFormat::Parquet,
        StorageFormat::Arrow,
        StorageFormat::CSV,
        StorageFormat::HDF5,
    ];

    for format in formats {
        let temp_dir = tempdir().expect("Should create temp dir");
        let config = TrainingStorageConfig {
            base_directory: temp_dir.path().to_path_buf(),
            format: format.clone(),
            compression: CompressionConfig {
                algorithm: CompressionAlgorithm::ZSTD,
                level: 3,
                enabled: true,
            },
            versioning: VersioningConfig {
                enabled: false,
                version_format: "v%Y%m%d_%H%M%S".to_string(),
                keep_versions: 5,
            },
            retention: RetentionConfig {
                retention_days: 30,
                auto_cleanup: false,
                cleanup_schedule: "0 2 * * *".to_string(),
            },
        };

        let storage_manager = StorageManager::new(config)
            .await
            .expect("Storage manager should initialize");

        let test_data = create_sample_market_data();
        let dataset_id = format!("test_dataset_{:?}", format);

        let store_result = storage_manager.store_dataset(&dataset_id, &test_data).await;
        assert!(
            store_result.is_ok(),
            "Should store data in {:?} format",
            format
        );

        let load_result = storage_manager.load_dataset(&dataset_id).await;
        assert!(
            load_result.is_ok(),
            "Should load data from {:?} format",
            format
        );
    }
}

#[tokio::test]
async fn test_compression_algorithms() {
    let algorithms = vec![
        (CompressionAlgorithm::LZ4, 1),
        (CompressionAlgorithm::Snappy, 1),
        (CompressionAlgorithm::ZSTD, 3),
        (CompressionAlgorithm::GZIP, 6),
    ];

    for (algorithm, level) in algorithms {
        let temp_dir = tempdir().expect("Should create temp dir");
        let config = TrainingStorageConfig {
            base_directory: temp_dir.path().to_path_buf(),
            format: StorageFormat::Parquet,
            compression: CompressionConfig {
                algorithm: algorithm.clone(),
                level,
                enabled: true,
            },
            versioning: VersioningConfig {
                enabled: false,
                version_format: "v%Y%m%d_%H%M%S".to_string(),
                keep_versions: 5,
            },
            retention: RetentionConfig {
                retention_days: 30,
                auto_cleanup: false,
                cleanup_schedule: "0 2 * * *".to_string(),
            },
        };

        let storage_manager = StorageManager::new(config)
            .await
            .expect("Storage manager should initialize");

        assert_eq!(storage_manager.config.compression.algorithm, algorithm);
        assert_eq!(storage_manager.config.compression.level, level);
        assert!(storage_manager.config.compression.enabled);
    }
}

#[tokio::test]
async fn test_versioning_and_retention() {
    let temp_dir = tempdir().expect("Should create temp dir");
    let config = TrainingStorageConfig {
        base_directory: temp_dir.path().to_path_buf(),
        format: StorageFormat::Parquet,
        compression: CompressionConfig {
            algorithm: CompressionAlgorithm::ZSTD,
            level: 3,
            enabled: true,
        },
        versioning: VersioningConfig {
            enabled: true,
            version_format: "v%Y%m%d_%H%M%S".to_string(),
            keep_versions: 3,
        },
        retention: RetentionConfig {
            retention_days: 7,
            auto_cleanup: true,
            cleanup_schedule: "0 2 * * *".to_string(),
        },
    };

    let storage_manager = StorageManager::new(config.clone())
        .await
        .expect("Versioned storage manager should initialize");

    assert!(storage_manager.config.versioning.enabled);
    assert_eq!(storage_manager.config.versioning.keep_versions, 3);
    assert_eq!(storage_manager.config.retention.retention_days, 7);
    assert!(storage_manager.config.retention.auto_cleanup);

    // Test storing multiple versions
    let test_data = create_sample_market_data();
    for version in 1..=3 {
        let dataset_id = format!("versioned_dataset_{}", version);
        let store_result = storage_manager.store_dataset(&dataset_id, &test_data).await;
        assert!(store_result.is_ok(), "Should store version {}", version);
    }
}

// ============================================================================
// 6. Error Handling and Edge Cases (Additional tests)
// ============================================================================

#[test]
fn test_config_with_missing_env_vars() {
    std::env::remove_var("DATABENTO_API_KEY");
    std::env::remove_var("BENZINGA_API_KEY");

    let config = TrainingPipelineConfig::default();

    assert_eq!(config.sources.databento.as_ref().unwrap().api_key, "");
    assert_eq!(config.sources.benzinga.as_ref().unwrap().api_key, "");
    assert!(config.features.technical_indicators.ma_periods.len() > 0);
}

#[tokio::test]
async fn test_invalid_storage_path() {
    let temp_dir = tempdir().expect("Should create temp dir");
    let file_path = temp_dir.path().join("i_am_a_file");
    File::create(&file_path).expect("Should create file");

    let mut config = TrainingPipelineConfig::default();
    config.storage.base_directory = file_path;

    let result = TrainingDataPipeline::new(config).await;
    assert!(result.is_err(), "Pipeline creation should fail");

    match result.unwrap_err() {
        DataError::Io(_) => {} // Expected
        other => panic!("Expected IO error, got: {:?}", other),
    }
}

#[tokio::test]
async fn test_concurrent_pipeline_operations() {
    let (_temp_dir, config) = setup_test_environment().await;
    let pipeline = Arc::new(
        TrainingDataPipeline::new(config)
            .await
            .expect("Pipeline should initialize"),
    );

    let raw_data = create_sample_market_data();
    let mut handles = Vec::new();

    for i in 1..=3 {
        let dataset_id = format!("concurrent_test_{}", i);
        create_test_dataset(&_temp_dir, &dataset_id, &raw_data).await;

        let pipeline_clone = pipeline.clone();
        let dataset_id_clone = dataset_id.clone();

        let handle =
            tokio::spawn(async move { pipeline_clone.process_features(&dataset_id_clone).await });
        handles.push(handle);
    }

    for handle in handles {
        let result = handle.await.expect("Task should complete");
        assert!(result.is_ok(), "Concurrent operations should succeed");
    }
}